@swarmclawai/swarmclaw 1.2.1 → 1.2.3

package/src/lib/server/runtime/run-ledger.ts

@@ -1,7 +1,8 @@
 import { genId } from '@/lib/id'
 import type { RunEventRecord, SessionRunRecord, SessionRunStatus, SSEEvent } from '@/types'
 import {
-  deleteStoredItem,
+  deleteRuntimeRun,
+  deleteRuntimeRunEvent,
   loadRuntimeRun,
   loadRuntimeRunEvents,
   loadRuntimeRunEventsByRunId,
@@ -9,7 +10,7 @@ import {
   patchRuntimeRun,
   upsertRuntimeRun,
   upsertRuntimeRunEvent,
-} from '@/lib/server/storage'
+} from '@/lib/server/runtime/run-repository'
 
 const MAX_SUMMARY_CHARS = 240
 const RESTART_RECOVERABLE_SOURCES = new Set([
@@ -137,7 +138,7 @@ export function pruneOldRuns(): { prunedRuns: number; prunedEvents: number } {
       // Non-terminal (running/queued) — only prune if stuck for much longer
       if (deadline - endTs < ORPHANED_RUN_RETENTION_MS) continue
     }
-    deleteStoredItem('runtime_runs', id)
+    deleteRuntimeRun(id)
     prunedRunIds.add(id)
     prunedRuns++
   }
@@ -146,7 +147,7 @@ export function pruneOldRuns(): { prunedRuns: number; prunedEvents: number } {
   const events = loadRuntimeRunEvents()
   for (const [id, event] of Object.entries(events)) {
     if (prunedRunIds.has(event.runId)) {
-      deleteStoredItem('runtime_run_events', id)
+      deleteRuntimeRunEvent(id)
       prunedEvents++
       continue
     }
@@ -154,7 +155,7 @@ export function pruneOldRuns(): { prunedRuns: number; prunedEvents: number } {
     const parentRun = runs[event.runId]
     if (!parentRun || !TERMINAL_STATUSES.has(parentRun.status)) continue
     if (deadline - event.timestamp < RUN_EVENT_RETENTION_MS) continue
-    deleteStoredItem('runtime_run_events', id)
+    deleteRuntimeRunEvent(id)
     prunedEvents++
   }
 

package/src/lib/server/runtime/run-repository.ts

@@ -0,0 +1,73 @@
+import type { RunEventRecord, SessionRunRecord } from '@/types'
+
+import {
+  deleteStoredItem,
+  loadRuntimeRun as loadStoredRuntimeRun,
+  loadRuntimeRunEvents as loadStoredRuntimeRunEvents,
+  loadRuntimeRunEventsByRunId as loadStoredRuntimeRunEventsByRunId,
+  loadRuntimeRuns as loadStoredRuntimeRuns,
+  patchRuntimeRun as patchStoredRuntimeRun,
+  saveRuntimeRunEvents as saveStoredRuntimeRunEvents,
+  saveRuntimeRuns as saveStoredRuntimeRuns,
+  upsertRuntimeRun as upsertStoredRuntimeRun,
+  upsertRuntimeRunEvent as upsertStoredRuntimeRunEvent,
+} from '@/lib/server/storage'
+import { createRecordRepository } from '@/lib/server/persistence/repository-utils'
+
+export const runRepository = createRecordRepository<SessionRunRecord>(
+  'runtime-runs',
+  {
+    get(id) {
+      return loadStoredRuntimeRun(id) as SessionRunRecord | null
+    },
+    list() {
+      return loadStoredRuntimeRuns() as Record<string, SessionRunRecord>
+    },
+    upsert(id, value) {
+      upsertStoredRuntimeRun(id, value as SessionRunRecord)
+    },
+    replace(data) {
+      saveStoredRuntimeRuns(data as Record<string, SessionRunRecord>)
+    },
+    patch(id, updater) {
+      return patchStoredRuntimeRun(id, updater as (current: SessionRunRecord | null) => SessionRunRecord | null) as SessionRunRecord | null
+    },
+    delete(id) {
+      deleteStoredItem('runtime_runs', id)
+    },
+  },
+)
+
+export const runEventRepository = createRecordRepository<RunEventRecord>(
+  'runtime-run-events',
+  {
+    get(id) {
+      return (loadStoredRuntimeRunEvents() as Record<string, RunEventRecord>)[id] || null
+    },
+    list() {
+      return loadStoredRuntimeRunEvents() as Record<string, RunEventRecord>
+    },
+    upsert(id, value) {
+      upsertStoredRuntimeRunEvent(id, value as RunEventRecord)
+    },
+    replace(data) {
+      saveStoredRuntimeRunEvents(data as Record<string, RunEventRecord>)
+    },
+    delete(id) {
+      deleteStoredItem('runtime_run_events', id)
+    },
+  },
+)
+
+export const loadRuntimeRuns = () => runRepository.list()
+export const saveRuntimeRuns = (items: Record<string, SessionRunRecord | Record<string, unknown>>) => runRepository.replace(items as Record<string, SessionRunRecord>)
+export const loadRuntimeRun = (id: string) => runRepository.get(id)
+export const upsertRuntimeRun = (id: string, value: SessionRunRecord | Record<string, unknown>) => runRepository.upsert(id, value as SessionRunRecord)
+export const patchRuntimeRun = (id: string, updater: (current: SessionRunRecord | null) => SessionRunRecord | null) => runRepository.patch(id, updater)
+
+export const loadRuntimeRunEvents = () => runEventRepository.list()
+export const saveRuntimeRunEvents = (items: Record<string, RunEventRecord | Record<string, unknown>>) => runEventRepository.replace(items as Record<string, RunEventRecord>)
+export const upsertRuntimeRunEvent = (id: string, value: RunEventRecord | Record<string, unknown>) => runEventRepository.upsert(id, value as RunEventRecord)
+export const loadRuntimeRunEventsByRunId = (runId: string) => loadStoredRuntimeRunEventsByRunId(runId)
+export const deleteRuntimeRun = (id: string) => runRepository.delete(id)
+export const deleteRuntimeRunEvent = (id: string) => runEventRepository.delete(id)

package/src/lib/server/runtime/runtime-lock-repository.ts

@@ -0,0 +1,8 @@
+export {
+  isRuntimeLockActive,
+  pruneExpiredLocks,
+  readRuntimeLock,
+  releaseRuntimeLock,
+  renewRuntimeLock,
+  tryAcquireRuntimeLock,
+} from '@/lib/server/storage'

package/src/lib/server/runtime/runtime-settings.ts

@@ -2,7 +2,7 @@ import type { LoopMode } from '@/types'
 import {
   normalizeRuntimeSettingFields,
 } from '@/lib/runtime/runtime-loop'
-import { loadSettings } from '@/lib/server/storage'
+import { loadSettings } from '@/lib/server/settings/settings-repository'
 
 export interface RuntimeSettings {
   loopMode: LoopMode

package/src/lib/server/runtime/runtime-state.ts

@@ -0,0 +1,99 @@
+import type { ChildProcess } from 'node:child_process'
+
+import { hmrSingleton } from '@/lib/shared-utils'
+
+export type ActiveSessionProcess = {
+  runId?: string | null
+  source?: string
+  kill: (signal?: NodeJS.Signals | number) => boolean | void
+}
+
+export interface DevServerRuntime {
+  proc: ChildProcess
+  url: string
+}
+
+interface RuntimeStateRegistry {
+  activeSessionProcesses: Map<string, ActiveSessionProcess>
+  devServers: Map<string, DevServerRuntime>
+}
+
+const state = hmrSingleton<RuntimeStateRegistry>('__swarmclaw_runtime_state__', () => ({
+  activeSessionProcesses: new Map<string, ActiveSessionProcess>(),
+  devServers: new Map<string, DevServerRuntime>(),
+}))
+
+if (!state.activeSessionProcesses) state.activeSessionProcesses = new Map<string, ActiveSessionProcess>()
+if (!state.devServers) state.devServers = new Map<string, DevServerRuntime>()
+
+export const activeSessionProcesses = state.activeSessionProcesses
+export const devServers = state.devServers
+
+export function getActiveSessionProcess(sessionId: string): ActiveSessionProcess | undefined {
+  return state.activeSessionProcesses.get(sessionId)
+}
+
+export function hasActiveSessionProcess(sessionId: string): boolean {
+  return state.activeSessionProcesses.has(sessionId)
+}
+
+export function registerActiveSessionProcess(sessionId: string, process: ActiveSessionProcess): void {
+  state.activeSessionProcesses.set(sessionId, process)
+}
+
+export function stopActiveSessionProcess(sessionId: string, signal?: NodeJS.Signals | number): boolean {
+  const process = state.activeSessionProcesses.get(sessionId)
+  if (!process) return false
+  try {
+    process.kill(signal)
+  } catch {
+    // Ignore process teardown errors during cleanup.
+  }
+  state.activeSessionProcesses.delete(sessionId)
+  return true
+}
+
+export function clearActiveSessionProcess(sessionId: string): void {
+  state.activeSessionProcesses.delete(sessionId)
+}
+
+export function getDevServer(sessionId: string): DevServerRuntime | undefined {
+  return state.devServers.get(sessionId)
+}
+
+export function hasDevServer(sessionId: string): boolean {
+  return state.devServers.has(sessionId)
+}
+
+export function registerDevServer(sessionId: string, runtime: DevServerRuntime): void {
+  state.devServers.set(sessionId, runtime)
+}
+
+export function updateDevServerUrl(sessionId: string, url: string): void {
+  const runtime = state.devServers.get(sessionId)
+  if (!runtime) return
+  runtime.url = url
+}
+
+export function stopDevServer(sessionId: string): boolean {
+  const runtime = state.devServers.get(sessionId)
+  if (!runtime) return false
+  try {
+    runtime.proc.kill('SIGTERM')
+  } catch {
+    // Ignore process teardown errors during cleanup.
+  }
+  if (typeof runtime.proc.pid === 'number') {
+    try {
+      process.kill(-runtime.proc.pid, 'SIGTERM')
+    } catch {
+      // Ignore process-group teardown errors when the child is already gone.
+    }
+  }
+  state.devServers.delete(sessionId)
+  return true
+}
+
+export function clearDevServer(sessionId: string): void {
+  state.devServers.delete(sessionId)
+}

package/src/lib/server/runtime/scheduler.ts

@@ -1,4 +1,6 @@
-import { loadSchedules, loadAgents, loadTasks, upsertSchedule, upsertSchedules, upsertTask } from '@/lib/server/storage'
+import { listAgents } from '@/lib/server/agents/agent-repository'
+import { loadSchedules, upsertSchedule, upsertSchedules } from '@/lib/server/schedules/schedule-repository'
+import { loadTasks, upsertTask } from '@/lib/server/tasks/task-repository'
 import { enqueueTask } from '@/lib/server/runtime/queue'
 import { CronExpressionParser } from 'cron-parser'
 import { pushMainLoopEventToMainSessions } from '@/lib/server/agents/main-agent-loop'
@@ -90,7 +92,7 @@ function computeNextRuns() {
 async function tick(now = Date.now()) {
   await processDueWatchJobs(now)
   const schedules = loadSchedules()
-  const agents = loadAgents()
+  const agents = listAgents()
   const tasks = loadTasks()
   const inFlightScheduleKeys = new Set<string>(
     Object.values(tasks as Record<string, ScheduleTaskLike>)

package/src/lib/server/runtime/session-run-manager/cancellation.ts

@@ -0,0 +1,157 @@
+import { isInternalHeartbeatRun } from '@/lib/server/runtime/heartbeat-source'
+
+import {
+  abortSessionRuntime,
+  decrementNonHeartbeatWork,
+  emitRunMeta,
+  now,
+  reconcileSessionActivityLease,
+  state,
+  syncRunRecord,
+} from './state'
+import type { SessionRunQueueEntry } from './types'
+
+export function cancelPendingForSession(sessionId: string, reason: string): number {
+  let cancelled = 0
+  for (const [key, queue] of state.queueByExecution.entries()) {
+    if (!queue.length) continue
+    const keep: SessionRunQueueEntry[] = []
+    for (const entry of queue) {
+      if (entry.run.sessionId !== sessionId) {
+        keep.push(entry)
+        continue
+      }
+      entry.run.status = 'cancelled'
+      entry.run.endedAt = now()
+      entry.run.error = reason
+      syncRunRecord(entry.run)
+      emitRunMeta(entry, 'cancelled', { reason })
+      entry.reject(new Error(reason))
+      decrementNonHeartbeatWork(entry)
+      cancelled += 1
+    }
+    if (keep.length > 0) state.queueByExecution.set(key, keep)
+    else state.queueByExecution.delete(key)
+  }
+  reconcileSessionActivityLease(sessionId)
+  return cancelled
+}
+
+function cancelQueuedEntries(
+  matcher: (entry: SessionRunQueueEntry) => boolean,
+  reason: string,
+): { cancelled: number; sessionIds: Set<string> } {
+  let cancelled = 0
+  const sessionIds = new Set<string>()
+  for (const [key, queue] of state.queueByExecution.entries()) {
+    if (!queue.length) continue
+    const keep: SessionRunQueueEntry[] = []
+    for (const entry of queue) {
+      if (!matcher(entry)) {
+        keep.push(entry)
+        continue
+      }
+      entry.run.status = 'cancelled'
+      entry.run.endedAt = now()
+      entry.run.error = reason
+      syncRunRecord(entry.run)
+      emitRunMeta(entry, 'cancelled', { reason })
+      entry.reject(new Error(reason))
+      decrementNonHeartbeatWork(entry)
+      sessionIds.add(entry.run.sessionId)
+      cancelled += 1
+    }
+    if (keep.length > 0) state.queueByExecution.set(key, keep)
+    else state.queueByExecution.delete(key)
+  }
+  for (const sessionId of sessionIds) reconcileSessionActivityLease(sessionId)
+  return { cancelled, sessionIds }
+}
+
+export function cancelAllHeartbeatRuns(reason = 'Heartbeat disabled globally'): { cancelledQueued: number; abortedRunning: number } {
+  let cancelledQueued = 0
+  let abortedRunning = 0
+
+  for (const [key, queue] of state.queueByExecution.entries()) {
+    if (!queue.length) continue
+    const keep: SessionRunQueueEntry[] = []
+    for (const entry of queue) {
+      const isHeartbeat = isInternalHeartbeatRun(entry.run.internal, entry.run.source)
+      if (!isHeartbeat) {
+        keep.push(entry)
+        continue
+      }
+      entry.run.status = 'cancelled'
+      entry.run.endedAt = now()
+      entry.run.error = reason
+      syncRunRecord(entry.run)
+      emitRunMeta(entry, 'cancelled', { reason })
+      entry.reject(new Error(reason))
+      cancelledQueued += 1
+    }
+    if (keep.length > 0) state.queueByExecution.set(key, keep)
+    else state.queueByExecution.delete(key)
+  }
+
+  for (const entry of state.runningByExecution.values()) {
+    const isHeartbeat = isInternalHeartbeatRun(entry.run.internal, entry.run.source)
+    if (!isHeartbeat) continue
+    abortedRunning += 1
+    abortSessionRuntime(entry, reason)
+  }
+
+  return { cancelledQueued, abortedRunning }
+}
+
+export function cancelAllRuns(reason = 'Cancelled'): { cancelledQueued: number; abortedRunning: number } {
+  let cancelledQueued = 0
+  let abortedRunning = 0
+
+  for (const [key, queue] of state.queueByExecution.entries()) {
+    if (!queue.length) continue
+    for (const entry of queue) {
+      entry.run.status = 'cancelled'
+      entry.run.endedAt = now()
+      entry.run.error = reason
+      syncRunRecord(entry.run)
+      emitRunMeta(entry, 'cancelled', { reason })
+      entry.reject(new Error(reason))
+      cancelledQueued += 1
+    }
+    state.queueByExecution.delete(key)
+  }
+
+  for (const entry of state.runningByExecution.values()) {
+    abortedRunning += 1
+    abortSessionRuntime(entry, reason)
+  }
+  state.runningByExecution.clear()
+  state.nonHeartbeatWorkCount.clear()
+
+  return { cancelledQueued, abortedRunning }
+}
+
+export function cancelQueuedRunById(runId: string, reason = 'Removed from queue'): boolean {
+  const result = cancelQueuedEntries((entry) => entry.run.id === runId, reason)
+  return result.cancelled > 0
+}
+
+export function cancelQueuedRunsForSession(sessionId: string, reason = 'Cleared queued messages'): number {
+  const result = cancelQueuedEntries((entry) => entry.run.sessionId === sessionId, reason)
+  return result.cancelled
+}
+
+export function cancelSessionRuns(sessionId: string, reason = 'Cancelled'): { cancelledQueued: number; cancelledRunning: boolean } {
+  const running = Array.from(state.runningByExecution.values())
+    .find((entry) => entry.run.sessionId === sessionId)
+  let cancelledRunning = false
+  if (running) {
+    cancelledRunning = true
+    abortSessionRuntime(running, reason)
+    state.runningByExecution.delete(running.executionKey)
+    decrementNonHeartbeatWork(running)
+  }
+  const cancelledQueued = cancelPendingForSession(sessionId, reason)
+  reconcileSessionActivityLease(sessionId)
+  return { cancelledQueued, cancelledRunning }
+}

package/src/lib/server/runtime/session-run-manager/drain.ts

@@ -0,0 +1,246 @@
+import { executeSessionChatTurn } from '@/lib/server/chat-execution/chat-execution'
+import { log } from '@/lib/server/logger'
+import { isInternalHeartbeatRun } from '@/lib/server/runtime/heartbeat-source'
+import { notify } from '@/lib/server/ws-hub'
+import { errorMessage } from '@/lib/shared-utils'
+import { handleMainLoopRunResult } from '@/lib/server/agents/main-agent-loop'
+
+import {
+  clearDeferredDrain,
+  decrementNonHeartbeatWork,
+  emitRunMeta,
+  emitToSubscribers,
+  hasActiveNonHeartbeatSessionLease,
+  hasExternalSessionExecutionHold,
+  HEARTBEAT_BUSY_RETRY_MS,
+  MAX_DRAIN_DEPTH,
+  now,
+  queueAutonomyObservation,
+  queueForExecution,
+  reconcileSessionActivityLease,
+  scheduleDeferredDrain,
+  state,
+  syncRunRecord,
+} from './state'
+import type { EnqueueSessionRunInput } from './types'
+
+type EnqueueSessionRunFn = (input: EnqueueSessionRunInput) => unknown
+
+export async function drainExecution(
+  executionKey: string,
+  deps: { enqueueSessionRun: EnqueueSessionRunFn },
+): Promise<void> {
+  const depth = (state.drainDepth.get(executionKey) || 0) + 1
+  state.drainDepth.set(executionKey, depth)
+  if (depth > MAX_DRAIN_DEPTH) {
+    log.error('session-run', 'Drain recursion depth exceeded, deferring', { executionKey, depth, max: MAX_DRAIN_DEPTH })
+    state.drainDepth.delete(executionKey)
+    scheduleDeferredDrain(executionKey, (nextExecutionKey) => { void drainExecution(nextExecutionKey, deps) }, 500)
+    return
+  }
+  try {
+    if (state.runningByExecution.has(executionKey)) return
+    const queue = queueForExecution(executionKey)
+    const userIdx = queue.findIndex((entry) => !entry.run.internal)
+    let next
+    if (userIdx >= 0) {
+      next = queue.splice(userIdx, 1)[0]
+    } else {
+      const internalIdx = queue.findIndex((entry) => !isInternalHeartbeatRun(entry.run.internal, entry.run.source))
+      next = internalIdx >= 0 ? queue.splice(internalIdx, 1)[0] : queue.shift()
+    }
+    if (!next) {
+      clearDeferredDrain(executionKey)
+      return
+    }
+
+    if (isInternalHeartbeatRun(next.run.internal, next.run.source) && hasActiveNonHeartbeatSessionLease(next.run.sessionId)) {
+      queue.unshift(next)
+      scheduleDeferredDrain(executionKey, (nextExecutionKey) => { void drainExecution(nextExecutionKey, deps) }, HEARTBEAT_BUSY_RETRY_MS)
+      log.info('session-run', `Deferred heartbeat run ${next.run.id} for shared busy session`, {
+        sessionId: next.run.sessionId,
+        source: next.run.source,
+      })
+      return
+    }
+
+    if (hasExternalSessionExecutionHold(next.run.sessionId)) {
+      queue.unshift(next)
+      scheduleDeferredDrain(executionKey, (nextExecutionKey) => { void drainExecution(nextExecutionKey, deps) }, HEARTBEAT_BUSY_RETRY_MS)
+      log.info('session-run', `Deferred run ${next.run.id} for external session hold`, {
+        sessionId: next.run.sessionId,
+        source: next.run.source,
+        mode: next.run.mode,
+      })
+      return
+    }
+
+    clearDeferredDrain(executionKey)
+    state.runningByExecution.set(executionKey, next)
+    next.run.status = 'running'
+    next.run.startedAt = now()
+    syncRunRecord(next.run)
+    emitRunMeta(next, 'running')
+    log.info('session-run', `Run started ${next.run.id}`, {
+      sessionId: next.run.sessionId,
+      source: next.run.source,
+      internal: next.run.internal,
+      mode: next.run.mode,
+      timeoutMs: next.maxRuntimeMs || null,
+    })
+
+    let runtimeTimer: ReturnType<typeof setTimeout> | null = null
+    let finishedMissionId: string | null = null
+    if (next.maxRuntimeMs && next.maxRuntimeMs > 0) {
+      runtimeTimer = setTimeout(() => {
+        next.signalController.abort()
+      }, next.maxRuntimeMs)
+    }
+
+    try {
+      const result = await executeSessionChatTurn({
+        sessionId: next.run.sessionId,
+        message: next.message,
+        imagePath: next.imagePath,
+        imageUrl: next.imageUrl,
+        attachedFiles: next.attachedFiles,
+        internal: next.run.internal,
+        source: next.run.source,
+        runId: next.run.id,
+        signal: next.signalController.signal,
+        onEvent: (event) => emitToSubscribers(next, event),
+        modelOverride: next.modelOverride,
+        heartbeatConfig: next.heartbeatConfig,
+        replyToId: next.replyToId,
+      })
+
+      const failed = !!result.error
+      const aborted = next.signalController.signal.aborted
+      next.run.status = aborted ? 'cancelled' : (failed ? 'failed' : 'completed')
+      next.run.endedAt = next.run.endedAt || now()
+      next.run.error = aborted ? (next.run.error || 'Cancelled') : result.error
+      next.run.missionId = result.missionId || next.run.missionId || null
+      finishedMissionId = next.run.missionId || null
+      next.run.resultPreview = result.text?.slice(0, 280)
+      if (typeof result.inputTokens === 'number') next.run.totalInputTokens = result.inputTokens
+      if (typeof result.outputTokens === 'number') next.run.totalOutputTokens = result.outputTokens
+      if (typeof result.estimatedCost === 'number') next.run.estimatedCost = result.estimatedCost
+      syncRunRecord(next.run)
+      emitRunMeta(next, next.run.status, {
+        persisted: result.persisted,
+        hasText: !!result.text,
+        error: next.run.error || null,
+      })
+      log.info('session-run', `Run finished ${next.run.id}`, {
+        sessionId: next.run.sessionId,
+        status: next.run.status,
+        persisted: result.persisted,
+        hasText: !!result.text,
+        error: next.run.error || null,
+        durationMs: (next.run.endedAt || now()) - (next.run.startedAt || now()),
+      })
+      const followup = handleMainLoopRunResult({
+        runId: next.run.id,
+        sessionId: next.run.sessionId,
+        message: next.message,
+        internal: next.run.internal,
+        source: next.run.source,
+        resultText: result.text,
+        error: next.run.error,
+        toolEvents: result.toolEvents,
+        inputTokens: result.inputTokens,
+        outputTokens: result.outputTokens,
+        estimatedCost: result.estimatedCost,
+      })
+      queueAutonomyObservation({
+        runId: next.run.id,
+        sessionId: next.run.sessionId,
+        source: next.run.source,
+        status: next.run.status,
+        resultText: result.text,
+        error: next.run.error || null,
+        toolEvents: result.toolEvents,
+        sourceMessage: next.message,
+      })
+      if (followup) {
+        setTimeout(() => {
+          try {
+            deps.enqueueSessionRun({
+              sessionId: next.run.sessionId,
+              message: followup.message,
+              internal: true,
+              source: 'main-loop-followup',
+              mode: 'followup',
+              dedupeKey: followup.dedupeKey,
+            })
+          } catch (err: unknown) {
+            log.warn('session-run', `Main loop follow-up enqueue failed for ${next.run.sessionId}`, {
+              error: errorMessage(err),
+            })
+          }
+        }, Math.max(0, followup.delayMs || 0))
+      }
+      next.resolve(result)
+    } catch (err: unknown) {
+      const aborted = next.signalController.signal.aborted
+      next.run.status = aborted ? 'cancelled' : 'failed'
+      next.run.endedAt = now()
+      next.run.error = errorMessage(err)
+      finishedMissionId = next.run.missionId || null
+      syncRunRecord(next.run)
+      emitRunMeta(next, next.run.status, { error: next.run.error })
+      log.error('session-run', `Run failed ${next.run.id}`, {
+        sessionId: next.run.sessionId,
+        status: next.run.status,
+        error: next.run.error,
+        durationMs: (next.run.endedAt || now()) - (next.run.startedAt || now()),
+      })
+      if (err instanceof Error && err.stack) {
+        log.error('session-run', `Run failed stack trace ${next.run.id}`, {
+          sessionId: next.run.sessionId,
+          stack: err.stack,
+        })
+      }
+      queueAutonomyObservation({
+        runId: next.run.id,
+        sessionId: next.run.sessionId,
+        source: next.run.source,
+        status: next.run.status,
+        error: next.run.error || null,
+        sourceMessage: next.message,
+      })
+      next.reject(err instanceof Error ? err : new Error(next.run.error))
+    } finally {
+      if (runtimeTimer) clearTimeout(runtimeTimer)
+      state.runningByExecution.delete(executionKey)
+      decrementNonHeartbeatWork(next)
+      reconcileSessionActivityLease(next.run.sessionId)
+      notify(`stream-end:${next.run.sessionId}`)
+      if (finishedMissionId && next.run.source !== 'chat') {
+        const missionId = finishedMissionId
+        queueMicrotask(() => {
+          import('@/lib/server/missions/mission-service')
+            .then(({ loadMissionById, requestMissionTick }) => {
+              const mission = loadMissionById(missionId)
+              if (!mission) return
+              if (mission.status !== 'active') return
+              if (mission.phase === 'dispatching' || mission.phase === 'executing') return
+              requestMissionTick(missionId, 'run_drained', {
+                runId: next.run.id,
+                source: next.run.source,
+                status: next.run.status,
+              })
+            })
+            .catch((err: unknown) => {
+              log.warn('session-run', 'Mission tick failed', { missionId, runId: next.run.id, error: errorMessage(err) })
+            })
+        })
+      }
+      void drainExecution(executionKey, deps)
+    }
+  } finally {
+    const currentDepth = state.drainDepth.get(executionKey)
+    if (currentDepth && currentDepth > 1) state.drainDepth.set(executionKey, currentDepth - 1)
+    else state.drainDepth.delete(executionKey)
+  }
+}