@swarmclawai/swarmclaw 1.2.1 → 1.2.3

package/src/lib/server/runtime/session-run-manager/enqueue.ts

@@ -0,0 +1,287 @@
+import { genId } from '@/lib/id'
+import type { SessionRunRecord } from '@/types'
+import { getSession } from '@/lib/server/sessions/session-repository'
+import { loadRuntimeSettings } from '@/lib/server/runtime/runtime-settings'
+import { log } from '@/lib/server/logger'
+import { isInternalHeartbeatRun } from '@/lib/server/runtime/heartbeat-source'
+import { getEnabledToolIds } from '@/lib/capability-selection'
+import { isAllEstopEngaged, isAutonomyEstopEngaged } from '@/lib/server/runtime/estop'
+import { getActiveSessionProcess } from '@/lib/server/runtime/runtime-state'
+
+import { cancelPendingForSession } from './cancellation'
+import {
+  abortSessionRuntime,
+  chainCallerSignal,
+  COLLECT_COALESCE_WINDOW_MS,
+  emitRunMeta,
+  executionKeyForSession,
+  incrementNonHeartbeatWork,
+  messagePreview,
+  nextQueuedAt,
+  normalizeMode,
+  queueForExecution,
+  reconcileSessionActivityLease,
+  registerRun,
+  state,
+  syncRunRecord,
+} from './state'
+import type {
+  EnqueueSessionRunInput,
+  EnqueueSessionRunResult,
+  SessionQueueMode,
+  SessionRunQueueEntry,
+} from './types'
+
+type RepairSessionRunQueueFn = (
+  sessionId: string,
+  opts?: {
+    executionKey?: string
+    maxQueuedAgeMs?: number
+    reason?: string
+  },
+) => { kickedExecutionKeys: number; recoveredQueuedRuns: number }
+
+type DrainExecutionFn = (executionKey: string) => Promise<void>
+
+const LONG_TOOL_NAMES: ReadonlySet<string> = new Set(['claude_code', 'codex_cli', 'opencode_cli'])
+
+type SessionToolConfig = {
+  tools?: string[] | null
+  extensions?: string[] | null
+}
+
+function computeEffectiveRunTimeoutMs(
+  baseTimeoutMs: number,
+  sessionTools: string[],
+  runtime: { claudeCodeTimeoutMs: number },
+): number {
+  const hasLongTool = sessionTools.some((tool) => LONG_TOOL_NAMES.has(tool))
+  if (!hasLongTool) return baseTimeoutMs
+  const toolTimeout = runtime.claudeCodeTimeoutMs + 120_000
+  return Math.max(baseTimeoutMs, toolTimeout)
+}
+
+function isAutonomyManagedEnqueue(source: string, internal: boolean): boolean {
+  return !(source === 'chat' && !internal)
+}
+
+function buildRecoveryPayload(
+  input: EnqueueSessionRunInput,
+  source: string,
+  mode: SessionQueueMode,
+  maxRuntimeMs: number | undefined,
+  executionKey: string,
+) {
+  return {
+    message: input.message,
+    imagePath: input.imagePath,
+    imageUrl: input.imageUrl,
+    attachedFiles: input.attachedFiles,
+    internal: input.internal === true,
+    source,
+    mode,
+    maxRuntimeMs,
+    modelOverride: input.modelOverride,
+    heartbeatConfig: input.heartbeatConfig,
+    replyToId: input.replyToId,
+    executionGroupKey: executionKey.startsWith('session:') ? undefined : executionKey,
+  }
+}
+
+function findDedupeMatch(sessionId: string, dedupeKey?: string) {
+  if (!dedupeKey) return null
+  const executionKey = executionKeyForSession(sessionId)
+  const running = state.runningByExecution.get(executionKey)
+  if (running?.run.sessionId === sessionId && running.run.dedupeKey === dedupeKey) return running
+  const queue = queueForExecution(executionKey)
+  return queue.find((entry) => entry.run.sessionId === sessionId && entry.run.dedupeKey === dedupeKey) || null
+}
+
+export function enqueueSessionRun(
+  input: EnqueueSessionRunInput,
+  deps: {
+    repairSessionRunQueue: RepairSessionRunQueueFn
+    drainExecution: DrainExecutionFn
+  },
+): EnqueueSessionRunResult {
+  const internal = input.internal === true
+  const mode = normalizeMode(input.mode, internal)
+  const source = input.source || 'chat'
+  if (isAllEstopEngaged()) {
+    throw new Error('Execution is blocked because all estop is engaged.')
+  }
+  if (isAutonomyEstopEngaged() && isAutonomyManagedEnqueue(source, internal)) {
+    throw new Error(`Autonomy estop is engaged. New ${source} runs are paused.`)
+  }
+  const executionKey = typeof input.executionGroupKey === 'string' && input.executionGroupKey.trim()
+    ? input.executionGroupKey.trim()
+    : executionKeyForSession(input.sessionId)
+  deps.repairSessionRunQueue(input.sessionId, {
+    executionKey,
+    reason: 'Recovered stale queued run before enqueue',
+  })
+  const runtime = loadRuntimeSettings()
+  const defaultMaxRuntimeMs = runtime.ongoingLoopMaxRuntimeMs ?? (10 * 60_000)
+  const sessionData = getSession(input.sessionId) as SessionToolConfig | null
+  const sessionTools = getEnabledToolIds(sessionData)
+  const adjustedDefaultMs = computeEffectiveRunTimeoutMs(defaultMaxRuntimeMs, sessionTools, runtime)
+  const effectiveMaxRuntimeMs = typeof input.maxRuntimeMs === 'number'
+    ? input.maxRuntimeMs
+    : adjustedDefaultMs
+
+  const dedupe = findDedupeMatch(input.sessionId, input.dedupeKey)
+  if (dedupe) {
+    const cb = input.onEvent
+    if (cb) dedupe.onEvents.push(cb)
+    if (input.callerSignal) chainCallerSignal(input.callerSignal, dedupe.signalController)
+    return {
+      runId: dedupe.run.id,
+      position: 0,
+      deduped: true,
+      promise: dedupe.promise,
+      abort: () => dedupe.signalController.abort(),
+      unsubscribe: () => {
+        if (!cb) return
+        const idx = dedupe.onEvents.indexOf(cb)
+        if (idx >= 0) dedupe.onEvents.splice(idx, 1)
+      },
+    }
+  }
+
+  if (mode === 'steer') {
+    const running = state.runningByExecution.get(executionKey)
+    if (running && running.run.sessionId === input.sessionId) {
+      running.signalController.abort()
+      try { getActiveSessionProcess(input.sessionId)?.kill?.() } catch { /* noop */ }
+    }
+    cancelPendingForSession(input.sessionId, 'Cancelled by steer mode')
+  }
+
+  if (!internal && source === 'chat') {
+    const running = state.runningByExecution.get(executionKey)
+    if (running && isInternalHeartbeatRun(running.run.internal, running.run.source)) {
+      log.info('session-run', `Preempting heartbeat ${running.run.id} for user chat on ${input.sessionId}`)
+      abortSessionRuntime(running, 'Preempted by user chat')
+      state.runningByExecution.delete(executionKey)
+    }
+  }
+
+  const running = state.runningByExecution.get(executionKey)
+  const queue = queueForExecution(executionKey)
+  if (mode === 'collect' && !input.imagePath && !input.imageUrl && !input.attachedFiles?.length) {
+    const nowMs = nextQueuedAt()
+    const candidate = queue.at(-1)
+    const canCoalesce = !!candidate
+      && candidate.run.mode === 'collect'
+      && candidate.run.internal === internal
+      && candidate.run.source === source
+      && !candidate.imagePath
+      && !candidate.imageUrl
+      && !candidate.attachedFiles?.length
+      && (nowMs - candidate.run.queuedAt) <= COLLECT_COALESCE_WINDOW_MS
+
+    if (candidate && canCoalesce) {
+      const nextChunk = input.message.trim()
+      if (nextChunk) {
+        const current = candidate.message.trim()
+        candidate.message = current
+          ? `${current}\n\n[Collected follow-up]\n${nextChunk}`
+          : nextChunk
+        candidate.run.messagePreview = messagePreview(candidate.message)
+        candidate.run.queuedAt = nowMs
+        syncRunRecord(candidate.run)
+      }
+      const coalesceCb = input.onEvent
+      if (coalesceCb) candidate.onEvents.push(coalesceCb)
+      if (input.callerSignal) chainCallerSignal(input.callerSignal, candidate.signalController)
+      emitRunMeta(candidate, 'queued', { position: 0, coalesced: true, mergedIntoRunId: candidate.run.id })
+      return {
+        runId: candidate.run.id,
+        position: 0,
+        coalesced: true,
+        promise: candidate.promise,
+        abort: () => candidate.signalController.abort(),
+        unsubscribe: () => {
+          if (!coalesceCb) return
+          const idx = candidate.onEvents.indexOf(coalesceCb)
+          if (idx >= 0) candidate.onEvents.splice(idx, 1)
+        },
+      }
+    }
+  }
+
+  const runId = genId(8)
+  const run: SessionRunRecord = {
+    id: runId,
+    sessionId: input.sessionId,
+    missionId: input.missionId ?? getSession(input.sessionId)?.missionId ?? null,
+    source,
+    internal,
+    mode,
+    status: 'queued',
+    messagePreview: messagePreview(input.message),
+    dedupeKey: input.dedupeKey,
+    queuedAt: nextQueuedAt(),
+    recoveredFromRestart: input.recoveredFromRestart === true,
+    recoveredFromRunId: input.recoveredFromRunId,
+    recoveryPayload: buildRecoveryPayload(
+      input,
+      source,
+      mode,
+      effectiveMaxRuntimeMs > 0 ? effectiveMaxRuntimeMs : undefined,
+      executionKey,
+    ),
+  }
+  registerRun(run)
+
+  let resolve!: EnqueueSessionRunResult['promise'] extends Promise<infer T> ? (value: T) => void : never
+  let reject!: (error: Error) => void
+  const promise = new Promise<import('@/lib/server/chat-execution/chat-execution-types').ExecuteChatTurnResult>((res, rej) => {
+    resolve = res
+    reject = rej
+  })
+  promise.catch(() => {})
+  state.promises.set(runId, promise)
+
+  const entry: SessionRunQueueEntry = {
+    executionKey,
+    run,
+    message: input.message,
+    imagePath: input.imagePath,
+    imageUrl: input.imageUrl,
+    attachedFiles: input.attachedFiles,
+    onEvents: input.onEvent ? [input.onEvent] : [],
+    signalController: new AbortController(),
+    maxRuntimeMs: effectiveMaxRuntimeMs > 0 ? effectiveMaxRuntimeMs : undefined,
+    modelOverride: input.modelOverride,
+    heartbeatConfig: input.heartbeatConfig,
+    replyToId: input.replyToId,
+    resolve,
+    reject,
+    promise,
+  }
+
+  if (input.callerSignal) chainCallerSignal(input.callerSignal, entry.signalController)
+
+  queue.push(entry)
+  incrementNonHeartbeatWork(entry)
+  if (entry.nonHeartbeatCounted) {
+    reconcileSessionActivityLease(input.sessionId)
+  }
+  const position = (running ? 1 : 0) + queue.length - 1
+  emitRunMeta(entry, 'queued', { position })
+  void deps.drainExecution(executionKey)
+
+  const entryCb = input.onEvent
+  return {
+    runId,
+    position,
+    promise,
+    abort: () => entry.signalController.abort(),
+    unsubscribe: () => {
+      if (!entryCb) return
+      const idx = entry.onEvents.indexOf(entryCb)
+      if (idx >= 0) entry.onEvents.splice(idx, 1)
+    },
+  }
+}

package/src/lib/server/runtime/session-run-manager/queries.ts

@@ -0,0 +1,117 @@
+import type {
+  RunEventRecord,
+  SessionQueueSnapshot,
+  SessionQueuedTurn,
+  SessionRunRecord,
+  SessionRunStatus,
+} from '@/types'
+import {
+  listPersistedRunEvents,
+  listPersistedRuns,
+  loadPersistedRun,
+} from '@/lib/server/runtime/run-ledger'
+import { isInternalHeartbeatRun } from '@/lib/server/runtime/heartbeat-source'
+
+import { state } from './state'
+import type { SessionRunQueueEntry } from './types'
+
+export function getSessionRunState(sessionId: string): {
+  runningRunId?: string
+  queueLength: number
+} {
+  const summary = getSessionExecutionState(sessionId)
+  return {
+    runningRunId: summary.runningRunId,
+    queueLength: summary.queueLength,
+  }
+}
+
+function visibleQueuedEntriesForSession(sessionId: string): SessionRunQueueEntry[] {
+  return Array.from(state.queueByExecution.values())
+    .flatMap((queue) => queue)
+    .filter((entry) => entry.run.sessionId === sessionId && entry.run.internal !== true)
+    .sort((left, right) => left.run.queuedAt - right.run.queuedAt)
+}
+
+function toQueuedTurn(entry: SessionRunQueueEntry, index: number): SessionQueuedTurn {
+  return {
+    runId: entry.run.id,
+    sessionId: entry.run.sessionId,
+    missionId: entry.run.missionId || null,
+    text: entry.message,
+    queuedAt: entry.run.queuedAt,
+    position: index + 1,
+    imagePath: entry.imagePath,
+    imageUrl: entry.imageUrl,
+    attachedFiles: entry.attachedFiles,
+    replyToId: entry.replyToId,
+    source: entry.run.source,
+  }
+}
+
+export function getSessionQueueSnapshot(sessionId: string): SessionQueueSnapshot {
+  const execution = getSessionExecutionState(sessionId)
+  const visibleQueued = visibleQueuedEntriesForSession(sessionId)
+  return {
+    sessionId,
+    activeRunId: execution.runningRunId || null,
+    queueLength: visibleQueued.length,
+    items: visibleQueued.map((entry, index) => toQueuedTurn(entry, index)),
+  }
+}
+
+export function getSessionExecutionState(sessionId: string): {
+  runningRunId?: string
+  queueLength: number
+  hasRunning: boolean
+  hasQueued: boolean
+  hasRunningHeartbeat: boolean
+  hasQueuedHeartbeat: boolean
+  hasRunningNonHeartbeat: boolean
+  hasQueuedNonHeartbeat: boolean
+} {
+  const running = Array.from(state.runningByExecution.values())
+    .find((entry) => entry.run.sessionId === sessionId)
+  const runningMatchesSession = Boolean(running)
+  const runningHeartbeat = Boolean(
+    runningMatchesSession
+    && running
+    && isInternalHeartbeatRun(running.run.internal, running.run.source),
+  )
+  const runningNonHeartbeat = Boolean(runningMatchesSession && !runningHeartbeat)
+  const queuedEntries = Array.from(state.queueByExecution.values())
+    .flatMap((queue) => queue)
+    .filter((entry) => entry.run.sessionId === sessionId)
+  const queuedHeartbeat = queuedEntries.filter((entry) =>
+    isInternalHeartbeatRun(entry.run.internal, entry.run.source),
+  ).length
+  const queuedNonHeartbeat = queuedEntries.length - queuedHeartbeat
+  return {
+    runningRunId: (runningMatchesSession && running?.run.status === 'running')
+      ? running.run.id
+      : undefined,
+    queueLength: queuedEntries.length,
+    hasRunning: Boolean(runningMatchesSession),
+    hasQueued: queuedEntries.length > 0,
+    hasRunningHeartbeat: runningHeartbeat,
+    hasQueuedHeartbeat: queuedHeartbeat > 0,
+    hasRunningNonHeartbeat: runningNonHeartbeat,
+    hasQueuedNonHeartbeat: queuedNonHeartbeat > 0,
+  }
+}
+
+export function getRunById(runId: string): SessionRunRecord | null {
+  return state.runs.get(runId) || loadPersistedRun(runId)
+}
+
+export function listRuns(params?: {
+  sessionId?: string
+  status?: SessionRunStatus
+  limit?: number
+}): SessionRunRecord[] {
+  return listPersistedRuns(params)
+}
+
+export function listRunEvents(runId: string, limit?: number): RunEventRecord[] {
+  return listPersistedRunEvents(runId, limit)
+}

package/src/lib/server/runtime/session-run-manager/recovery.ts

@@ -0,0 +1,238 @@
+import { log } from '@/lib/server/logger'
+import { errorMessage } from '@/lib/shared-utils'
+import { isAllEstopEngaged, isAutonomyEstopEngaged } from '@/lib/server/runtime/estop'
+import {
+  isRestartRecoverableSource,
+  listPersistedRuns,
+  loadRecoverableStaleRuns,
+} from '@/lib/server/runtime/run-ledger'
+
+import {
+  abortSessionRuntime,
+  clearDeferredDrain,
+  decrementNonHeartbeatWork,
+  deleteQueueEntry,
+  executionKeyForSession,
+  markPersistedRunInterrupted,
+  normalizeMode,
+  now,
+  reconcileSessionActivityLease,
+  recoveryState,
+  STALE_QUEUED_RUN_MS,
+  state,
+  syncRunRecord,
+  STUCK_RUN_THRESHOLD_MS,
+} from './state'
+import type { EnqueueSessionRunInput, SessionRunQueueEntry } from './types'
+
+type EnqueueSessionRunFn = (input: EnqueueSessionRunInput) => unknown
+type DrainExecutionFn = (executionKey: string) => Promise<void>
+
+function resolveRecoveredQueuedEntry(entry: SessionRunQueueEntry, reason: string): void {
+  decrementNonHeartbeatWork(entry)
+  if (entry.run.status === 'completed' || entry.run.status === 'failed' || entry.run.status === 'cancelled') {
+    entry.run.endedAt = entry.run.endedAt || now()
+  } else {
+    entry.run.status = 'failed'
+    entry.run.endedAt = now()
+  }
+  entry.run.error = reason
+  syncRunRecord(entry.run)
+  entry.onEvents.forEach((send) => {
+    try {
+      send({ t: 'err', text: reason })
+    } catch {
+      // Subscriber stream can be closed by the client.
+    }
+  })
+  entry.resolve({
+    runId: entry.run.id,
+    sessionId: entry.run.sessionId,
+    ...(entry.run.missionId ? { missionId: entry.run.missionId } : {}),
+    text: '',
+    persisted: false,
+    toolEvents: [],
+    error: reason,
+  })
+}
+
+export function ensureRecoveredPersistedRuns(enqueueSessionRun: EnqueueSessionRunFn): void {
+  if (recoveryState.completed) return
+  recoveryState.completed = true
+  const staleRuns = loadRecoverableStaleRuns()
+  if (!staleRuns.length) return
+  const recoveryBlocked = isAutonomyEstopEngaged() || isAllEstopEngaged()
+
+  for (const run of staleRuns) {
+    const interrupted = markPersistedRunInterrupted(run, 'Interrupted by server restart before the run completed.')
+    const payload = interrupted.recoveryPayload
+    if (
+      recoveryBlocked
+      || interrupted.recoveredFromRestart
+      || !payload
+      || !isRestartRecoverableSource(interrupted.source)
+    ) {
+      continue
+    }
+
+    try {
+      enqueueSessionRun({
+        sessionId: interrupted.sessionId,
+        message: payload.message,
+        imagePath: payload.imagePath,
+        imageUrl: payload.imageUrl,
+        attachedFiles: payload.attachedFiles,
+        internal: payload.internal,
+        source: payload.source,
+        mode: normalizeMode(payload.mode, payload.internal),
+        dedupeKey: interrupted.dedupeKey,
+        maxRuntimeMs: payload.maxRuntimeMs,
+        modelOverride: payload.modelOverride,
+        heartbeatConfig: payload.heartbeatConfig,
+        replyToId: payload.replyToId,
+        executionGroupKey: payload.executionGroupKey,
+        recoveredFromRestart: true,
+        recoveredFromRunId: interrupted.id,
+      })
+    } catch (err: unknown) {
+      log.warn('session-run', `Failed to requeue interrupted run ${interrupted.id}`, {
+        sessionId: interrupted.sessionId,
+        error: errorMessage(err),
+      })
+    }
+  }
+}
+
+export function repairSessionRunQueue(
+  sessionId: string,
+  drainExecution: DrainExecutionFn,
+  opts?: {
+    executionKey?: string
+    maxQueuedAgeMs?: number
+    reason?: string
+  },
+): {
+  kickedExecutionKeys: number
+  recoveredQueuedRuns: number
+} {
+  const maxQueuedAgeMs = Math.max(1_000, opts?.maxQueuedAgeMs ?? STALE_QUEUED_RUN_MS)
+  const reason = opts?.reason || 'Recovered stale queued run'
+  const targetExecutionKey = typeof opts?.executionKey === 'string' && opts.executionKey.trim()
+    ? opts.executionKey.trim()
+    : null
+  const queuedNow = now()
+  let kickedExecutionKeys = 0
+  let recoveredQueuedRuns = 0
+
+  for (const [executionKey, queue] of state.queueByExecution.entries()) {
+    if (targetExecutionKey && executionKey !== targetExecutionKey) continue
+    if (!queue.length) {
+      clearDeferredDrain(executionKey)
+      state.queueByExecution.delete(executionKey)
+      continue
+    }
+    if (state.runningByExecution.has(executionKey)) continue
+
+    const matching = queue.filter((entry) => entry.run.sessionId === sessionId)
+    if (!matching.length) continue
+
+    for (const entry of [...matching]) {
+      const missingPromise = !state.promises.has(entry.run.id)
+      const previousStatus = entry.run.status
+      const nonQueued = previousStatus !== 'queued'
+      const ageMs = Math.max(0, queuedNow - (entry.run.queuedAt || 0))
+      const stale = nonQueued || missingPromise || ageMs >= maxQueuedAgeMs
+      if (!stale) continue
+      if (!deleteQueueEntry(queue, entry)) continue
+      clearDeferredDrain(executionKey)
+      resolveRecoveredQueuedEntry(entry, reason)
+      recoveredQueuedRuns += 1
+      log.warn('session-run', `Recovered stale queued run ${entry.run.id}`, {
+        sessionId: entry.run.sessionId,
+        executionKey,
+        source: entry.run.source,
+        ageMs,
+        missingPromise,
+        previousStatus,
+      })
+    }
+
+    if (!queue.length) {
+      clearDeferredDrain(executionKey)
+      state.queueByExecution.delete(executionKey)
+      continue
+    }
+
+    if (queue.some((entry) => entry.run.sessionId === sessionId)) {
+      clearDeferredDrain(executionKey)
+      kickedExecutionKeys += 1
+      void drainExecution(executionKey)
+    }
+  }
+
+  if (recoveredQueuedRuns > 0) reconcileSessionActivityLease(sessionId)
+  return { kickedExecutionKeys, recoveredQueuedRuns }
+}
+
+export function sweepStuckRuns(enqueueSessionRun: EnqueueSessionRunFn): { aborted: number } {
+  const deadline = now()
+  let aborted = 0
+
+  for (const [execKey, entry] of state.runningByExecution.entries()) {
+    const age = deadline - (entry.run.startedAt || entry.run.queuedAt)
+    if (entry.maxRuntimeMs && age < entry.maxRuntimeMs * 1.5) continue
+    if (age < STUCK_RUN_THRESHOLD_MS) continue
+
+    abortSessionRuntime(entry, 'Watchdog: run exceeded maximum allowed duration')
+    state.runningByExecution.delete(execKey)
+    decrementNonHeartbeatWork(entry)
+    reconcileSessionActivityLease(entry.run.sessionId)
+    aborted++
+  }
+
+  const persistedRunning = listPersistedRuns({ status: 'running' })
+  for (const run of persistedRunning) {
+    const execKey = run.recoveryPayload?.executionGroupKey || executionKeyForSession(run.sessionId)
+    const inMemory = state.runningByExecution.get(execKey)
+    if (inMemory && inMemory.run.id === run.id) continue
+
+    const age = deadline - (run.startedAt || run.queuedAt)
+    if (age < STUCK_RUN_THRESHOLD_MS) continue
+
+    markPersistedRunInterrupted(run, 'Watchdog: orphaned run detected after server restart or HMR')
+    aborted++
+
+    const alreadyRunning = state.runningByExecution.has(execKey)
+    const alreadyQueued = (state.queueByExecution.get(execKey) || []).some((entry) => entry.run.sessionId === run.sessionId)
+    if (run.recoveryPayload && isRestartRecoverableSource(run.source) && !alreadyRunning && !alreadyQueued) {
+      try {
+        const payload = run.recoveryPayload
+        enqueueSessionRun({
+          sessionId: run.sessionId,
+          message: payload.message,
+          imagePath: payload.imagePath,
+          imageUrl: payload.imageUrl,
+          attachedFiles: payload.attachedFiles,
+          internal: payload.internal,
+          source: payload.source,
+          mode: normalizeMode(payload.mode, payload.internal),
+          dedupeKey: run.dedupeKey,
+          maxRuntimeMs: payload.maxRuntimeMs,
+          modelOverride: payload.modelOverride,
+          heartbeatConfig: payload.heartbeatConfig,
+          replyToId: payload.replyToId,
+          executionGroupKey: payload.executionGroupKey,
+          recoveredFromRestart: true,
+          recoveredFromRunId: run.id,
+        })
+      } catch (err: unknown) {
+        log.warn('session-run', `Watchdog: failed to re-enqueue orphaned run ${run.id}`, {
+          sessionId: run.sessionId,
+          error: errorMessage(err),
+        })
+      }
+    }
+  }
+
+  return { aborted }
+}