neoagent 3.2.0 → 3.2.1-beta.1

package/server/routes/agents.js

@@ -362,8 +362,16 @@ router.get('/:id/steps', (req, res) => {
     || run.final_response
     || null;
   const usage = buildRunUsageSummary(run.id);
-
-  res.json({ run, steps, events: listRunEvents(run.id), response, usage });
+  const lifecycle = db.prepare(
+    `SELECT c.action, c.reason, c.requested_at, c.consumed_at,
+            p.phase AS checkpoint_phase, p.updated_at AS checkpoint_updated_at
+     FROM agent_runs r
+     LEFT JOIN agent_run_controls c ON c.run_id = r.id
+     LEFT JOIN agent_run_checkpoints p ON p.run_id = r.id
+     WHERE r.id = ?`,
+  ).get(run.id) || null;
+
+  res.json({ run, steps, events: listRunEvents(run.id), response, usage, lifecycle });
 });
 
 // Abort a run
@@ -379,6 +387,31 @@ router.post('/:id/abort', (req, res) => {
   }
 });
 
+router.post('/:id/pause', (req, res) => {
+  try {
+    const engine = req.app.locals.agentEngine;
+    const accepted = engine.pauseRun(req.params.id, {
+      userId: req.session.userId,
+      reason: req.body?.reason || '',
+    });
+    if (!accepted) return res.status(409).json({ error: 'Run is not active or cannot be paused.' });
+    res.json({ success: true, status: 'pausing' });
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
+router.post('/:id/resume', (req, res) => {
+  try {
+    const engine = req.app.locals.agentEngine;
+    const accepted = engine.resumeRun(req.params.id, { userId: req.session.userId });
+    if (!accepted) return res.status(409).json({ error: 'Run is not paused or cannot be resumed.' });
+    res.json({ success: true, status: 'running' });
+  } catch (err) {
+    res.status(500).json({ error: sanitizeError(err) });
+  }
+});
+
 // Delete a run
 router.delete('/:id', (req, res) => {
   const run = db.prepare('SELECT id FROM agent_runs WHERE id = ? AND user_id = ?').get(req.params.id, req.session.userId);

package/server/services/ai/loop/agent_engine_core.js

@@ -20,6 +20,13 @@ const { summarizeCapabilityHealth } = require('../capabilityHealth');
 const { shouldAcceptTaskComplete } = require('../completion');
 const { shortenRunId, summarizeForLog } = require('../logFormat');
 const { runConversation } = require('./conversation_loop');
+const {
+  checkpointRun,
+  closeRun,
+  getRunControl,
+  requestRunControl,
+  transitionRun,
+} = require('./lifecycle');
 const {
   buildChurnAssessmentPrompt,
   buildCompletionDecisionPrompt,
@@ -1073,6 +1080,59 @@ class AgentEngine {
     return isRunStoppedImpl(this, runId);
   }
 
+  async checkpointLifecycle(runId, phase, state = {}) {
+    const runMeta = this.activeRuns.get(runId);
+    if (!runMeta) return { action: 'stop' };
+    const control = getRunControl(runId);
+    if (!control) return null;
+    if (control.action !== 'pause') return control;
+
+    checkpointRun(runId, phase, {
+      iteration: Number(state.iteration) || 0,
+      stepIndex: Number(state.stepIndex) || 0,
+      currentPhase: runMeta.progressLedger?.currentPhase || phase,
+      activeTools: (runMeta.activeTools || []).map((tool) => tool.name),
+      goalContract: runMeta.goalContract || null,
+      progressLedger: this.buildProgressLedgerSnapshot(runMeta),
+    });
+    transitionRun(runId, 'paused', {}, ['running', 'pausing']);
+    db.transaction(() => {
+      db.prepare(
+        `UPDATE agent_steps
+         SET status = 'paused', error = COALESCE(NULLIF(error, ''), 'Paused by request.'), completed_at = COALESCE(completed_at, datetime('now'))
+         WHERE run_id = ? AND status = 'running'`,
+      ).run(runId);
+      db.prepare(
+        `UPDATE pending_approvals
+         SET status = 'expired', decided_at = COALESCE(decided_at, datetime('now')), updated_at = datetime('now')
+         WHERE run_id = ? AND status = 'pending'`,
+      ).run(runId);
+    })();
+    runMeta.status = 'paused';
+    runMeta.abortController = null;
+    this.stopMessagingProgressSupervisor(runId);
+    this.emit(runMeta.userId, 'run:paused', { runId, reason: control.reason || null });
+    this.recordRunEvent(runMeta.userId, runId, 'run_paused', {
+      phase,
+      reason: control.reason || null,
+    }, { agentId: runMeta.agentId });
+
+    await new Promise((resolve) => { runMeta.resumeRun = resolve; });
+    runMeta.resumeRun = null;
+    runMeta.abortController = new AbortController();
+    runMeta.status = 'running';
+    this.startMessagingProgressSupervisor(runId);
+    return null;
+  }
+
+  completeRun(runId, fields = {}) {
+    return closeRun(runId, 'completed', fields, ['running']);
+  }
+
+  failRun(runId, fields = {}) {
+    return closeRun(runId, 'failed', fields, ['running']);
+  }
+
   attachProcessToRun(runId, pid) {
     return attachProcessToRunImpl(this, runId, pid);
   }
@@ -1593,6 +1653,10 @@ class AgentEngine {
 
   interruptRun(runId, reason = 'Server shutting down while run was in progress.') {
     const runMeta = this.activeRuns.get(runId);
+    const persistedRun = runMeta || db.prepare('SELECT user_id AS userId FROM agent_runs WHERE id = ?').get(runId);
+    if (persistedRun?.userId != null) {
+      requestRunControl(runId, persistedRun.userId, 'interrupt', reason);
+    }
     const delegatedChildren = db.prepare(
       "SELECT child_run_id FROM agent_delegations WHERE parent_run_id = ? AND status = 'running'"
     ).all(runId);
@@ -1600,6 +1664,7 @@ class AgentEngine {
       runMeta.status = 'interrupted';
       runMeta.stopReason = reason;
       runMeta.aborted = true;
+      runMeta.abortController?.abort(reason);
       this.emit(runMeta.userId, 'run:stopping', { runId });
       for (const pid of runMeta.toolPids) {
         if (this.runtimeManager && typeof this.runtimeManager.killCommand === 'function') {
@@ -1621,14 +1686,7 @@ class AgentEngine {
            completed_at = datetime('now')
        WHERE parent_run_id = ? AND status = 'running'`
     ).run(reason, runId);
-    db.prepare(
-      `UPDATE agent_runs
-       SET status = 'interrupted',
-           error = COALESCE(NULLIF(error, ''), ?),
-           updated_at = datetime('now'),
-           completed_at = datetime('now')
-       WHERE id = ?`
-    ).run(reason, runId);
+    closeRun(runId, 'interrupted', { error: reason }, ['running', 'pausing', 'paused', 'resuming']);
   }
 
   interruptAllActiveRuns(reason = 'Server shutting down while run was in progress.') {
@@ -1639,12 +1697,17 @@ class AgentEngine {
 
   stopRun(runId) {
     const runMeta = this.activeRuns.get(runId);
+    const persistedRun = runMeta || db.prepare('SELECT user_id AS userId FROM agent_runs WHERE id = ?').get(runId);
+    if (persistedRun?.userId != null) {
+      requestRunControl(runId, persistedRun.userId, 'stop', 'Stopped by request.');
+    }
     const delegatedChildren = db.prepare(
       "SELECT child_run_id FROM agent_delegations WHERE parent_run_id = ? AND status = 'running'"
     ).all(runId);
     if (runMeta) {
       runMeta.status = 'stopped';
       runMeta.aborted = true;
+      runMeta.abortController?.abort('Run stopped.');
       this.emit(runMeta.userId, 'run:stopping', { runId });
       for (const pid of runMeta.toolPids) {
         if (this.runtimeManager && typeof this.runtimeManager.killCommand === 'function') {
@@ -1661,7 +1724,46 @@ class AgentEngine {
     db.prepare(
       "UPDATE agent_delegations SET status = 'stopped', updated_at = datetime('now'), completed_at = datetime('now') WHERE parent_run_id = ? AND status = 'running'"
     ).run(runId);
-    db.prepare("UPDATE agent_runs SET status = 'stopped', updated_at = datetime('now') WHERE id = ?").run(runId);
+    closeRun(runId, 'stopped', {}, ['running', 'pausing', 'paused', 'resuming']);
+  }
+
+  pauseRun(runId, { userId, reason = '' } = {}) {
+    if (!runId || userId == null) return false;
+    const runMeta = this.activeRuns.get(runId);
+    if (
+      !runMeta
+      || Number(runMeta.userId) !== Number(userId)
+      || runMeta.status !== 'running'
+      || runMeta.pauseAvailable !== true
+    ) return false;
+    const result = requestRunControl(runId, userId, 'pause', reason);
+    if (!result.accepted) return false;
+    transitionRun(runId, 'pausing', {}, ['running']);
+    runMeta.status = 'pausing';
+    runMeta.abortController?.abort('Run paused.');
+    for (const pid of runMeta.toolPids) {
+      if (this.runtimeManager && typeof this.runtimeManager.killCommand === 'function') {
+        void this.runtimeManager.killCommand(runMeta.userId, pid, 'paused');
+      }
+    }
+    this.emit(runMeta.userId, 'run:pausing', { runId, reason: reason || null });
+    return true;
+  }
+
+  resumeRun(runId, { userId } = {}) {
+    if (!runId || userId == null) return false;
+    const runMeta = this.activeRuns.get(runId);
+    if (!runMeta || Number(runMeta.userId) !== Number(userId) || runMeta.status !== 'paused') return false;
+    if (!transitionRun(runId, 'resuming', {}, ['paused'])) return false;
+    db.prepare(
+      `UPDATE agent_run_controls SET consumed_at = datetime('now')
+       WHERE run_id = ? AND action = 'pause' AND consumed_at IS NULL`,
+    ).run(runId);
+    transitionRun(runId, 'running', {}, ['resuming']);
+    this.emit(runMeta.userId, 'run:resumed', { runId });
+    this.recordRunEvent(runMeta.userId, runId, 'run_resumed', {}, { agentId: runMeta.agentId });
+    runMeta.resumeRun?.();
+    return true;
   }
 
   abort(runId, { userId } = {}) {

package/server/services/ai/loop/conversation_loop.js

@@ -709,6 +709,8 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
     voiceSessionId: options.voiceSessionId || null,
     steeringQueue: [],
     systemSteeringQueue: [],
+    abortController: new AbortController(),
+    pauseAvailable: false,
     toolPids: new Set(),
     subagentDepth: Math.max(0, Number(options.subagentDepth) || 0),
     repetitionGuard: new ToolRepetitionGuard(),
@@ -1207,8 +1209,15 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
     // regardless of which iteration they fall in.
     let consecutiveToolFailures = 0;
     const iterationBudget = new IterationBudget(maxIterations);
+    const activeRunMeta = engine.getRunMeta(runId);
+    if (activeRunMeta) activeRunMeta.pauseAvailable = !directAnswerEligible;
 
     while (!directAnswerEligible && iterationBudget.consume()) {
+      const lifecycleAtStart = await engine.checkpointLifecycle(runId, 'iteration_boundary', {
+        iteration: iterationBudget.used,
+        stepIndex,
+      });
+      if (lifecycleAtStart?.action === 'stop' || lifecycleAtStart?.action === 'interrupt') break;
       if (engine.isRunStopped(runId)) break;
       iteration = iterationBudget.used;
 
@@ -1452,7 +1461,14 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
             model,
             messages,
             tools,
-            options: { ...options, userId, agentId, runId, phase: 'model_turn' },
+            options: {
+              ...options,
+              userId,
+              agentId,
+              runId,
+              phase: 'model_turn',
+              signal: engine.getRunMeta(runId)?.abortController?.signal,
+            },
             runId,
             iteration,
           });
@@ -1512,6 +1528,21 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
       try {
         await tryModelCall();
       } catch (err) {
+        const lifecycleAbort = err?.name === 'AbortError'
+          || err?.code === 'ABORT_ERR'
+          || /abort/i.test(String(err?.name || ''))
+          || engine.getRunMeta(runId)?.abortController?.signal?.aborted === true;
+        const lifecycleControl = await engine.checkpointLifecycle(runId, 'model_boundary', {
+          iteration,
+          stepIndex,
+        });
+        if (engine.isRunStopped(runId)) break;
+        if (lifecycleAbort && !lifecycleControl && engine.getRunMeta(runId)?.status === 'running') {
+          iterationBudget.refund();
+          iteration = iterationBudget.used;
+          continue;
+        }
+        if (lifecycleControl?.action === 'stop' || lifecycleControl?.action === 'interrupt') break;
         const modelError = String(err?.message || 'Model call failed');
 
         if (modelFailureRecoveries < loopPolicy.maxModelFailureRecoveries) {
@@ -1543,6 +1574,12 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
         response = { content: streamContent, toolCalls: [], finishReason: 'stop', usage: null };
       }
 
+      const lifecycleAfterModel = await engine.checkpointLifecycle(runId, 'model_boundary', {
+        iteration,
+        stepIndex,
+      });
+      if (lifecycleAfterModel?.action === 'stop' || lifecycleAfterModel?.action === 'interrupt') break;
+
       if (response.usage) {
         totalTokens += response.usage.totalTokens || 0;
       }
@@ -1718,6 +1755,11 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
           iteration,
           options,
         });
+        const lifecycleAfterBatch = await engine.checkpointLifecycle(runId, 'tool_boundary', {
+          iteration,
+          stepIndex: batch.endingStepIndex,
+        });
+        if (lifecycleAfterBatch?.action === 'stop' || lifecycleAfterBatch?.action === 'interrupt') break;
         stepIndex = batch.endingStepIndex;
         let batchGatheredNewEvidence = false;
         for (const item of batch.results) {
@@ -1922,6 +1964,7 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
 
         let toolResult;
         let toolErrorMessage = '';
+        let toolInterruptedForPause = false;
         try {
           toolResult = await engine.executeTool(toolName, toolArgs, {
             userId,
@@ -1938,6 +1981,7 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
             deliveryState: options.deliveryState || null,
             allowMultipleProactiveMessages: options.allowMultipleProactiveMessages === true,
             allowExternalSideEffects: options.allowExternalSideEffects === true,
+            signal: engine.getRunMeta(runId)?.abortController?.signal,
           });
           engine.detachProcessFromRun(runId, toolResult?.pid);
           toolErrorMessage = inferToolFailureMessage(toolName, toolResult);
@@ -1973,17 +2017,30 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
             );
           }
         } catch (err) {
-          toolResult = { error: err.message };
-          toolErrorMessage = String(err.message || 'Tool execution failed');
-          failedStepCount++;
+          const currentRunMeta = engine.getRunMeta(runId);
+          toolInterruptedForPause = currentRunMeta?.status === 'pausing'
+            && currentRunMeta.abortController?.signal?.aborted === true;
+          toolErrorMessage = toolInterruptedForPause
+            ? 'The tool was interrupted for pause after dispatch; its external outcome is unknown.'
+            : String(err.message || 'Tool execution failed');
+          toolResult = toolInterruptedForPause
+            ? { status: 'outcome_unknown', error: toolErrorMessage }
+            : { error: err.message };
+          if (!toolInterruptedForPause) failedStepCount++;
           engine.detachProcessFromRun(runId, toolResult?.pid);
           db.prepare('UPDATE agent_steps SET status = ?, error = ?, completed_at = datetime(\'now\') WHERE id = ?')
-            .run('failed', err.message, stepId);
-          engine.emit(userId, 'run:tool_end', { runId, stepId, toolName, error: err.message, status: 'failed' });
-          engine.recordRunEvent(userId, runId, 'tool_failed', {
+            .run(toolInterruptedForPause ? 'paused' : 'failed', toolErrorMessage, stepId);
+          engine.emit(userId, 'run:tool_end', {
+            runId,
+            stepId,
             toolName,
-            status: 'failed',
-            error: err.message,
+            error: toolErrorMessage,
+            status: toolInterruptedForPause ? 'paused' : 'failed',
+          });
+          engine.recordRunEvent(userId, runId, toolInterruptedForPause ? 'tool_paused' : 'tool_failed', {
+            toolName,
+            status: toolInterruptedForPause ? 'paused' : 'failed',
+            error: toolErrorMessage,
             durationMs: Date.now() - stepStartedAt,
           }, { agentId, stepId });
           console.warn(
@@ -1991,6 +2048,12 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
           );
         }
 
+        const lifecycleAfterTool = await engine.checkpointLifecycle(runId, 'tool_boundary', {
+          iteration,
+          stepIndex,
+        });
+        if (lifecycleAfterTool?.action === 'stop' || lifecycleAfterTool?.action === 'interrupt') break;
+
         const execution = classifyToolExecution(toolName, toolArgs, toolResult, toolErrorMessage);
         execution.input = toolArgs;
         const repetitionObservation = repetitionGuard?.observe(toolName, toolArgs, toolResult);
@@ -2054,7 +2117,13 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
           tools = engine.getActiveTools(runId);
         }
 
-        if (toolErrorMessage) {
+        if (toolInterruptedForPause) {
+          consecutiveToolFailures = 0;
+          messages.push({
+            role: 'system',
+            content: `The outcome of "${toolName}" is unknown because pause interrupted it after dispatch. Do not repeat the call. First inspect or query the affected state with a safe read-only tool, then continue based on verified evidence.`,
+          });
+        } else if (toolErrorMessage) {
           consecutiveToolFailures += 1;
           const currentRunMeta = engine.getRunMeta(runId);
           trackErrorPattern(toolErrorMessage, currentRunMeta);
@@ -2182,6 +2251,9 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
       if (!engine.activeRuns.has(runId)) break;
     }
 
+    const finalizingRunMeta = engine.getRunMeta(runId);
+    if (finalizingRunMeta) finalizingRunMeta.pauseAvailable = false;
+
     if (engine.isRunStopped(runId)) {
       const stoppedRunMeta = engine.getRunMeta(runId);
       const terminalStatus = stoppedRunMeta?.status === 'interrupted' ? 'interrupted' : 'stopped';
@@ -2336,6 +2408,22 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
       }
     }
 
+    const lifecycleBeforeFinalize = await engine.checkpointLifecycle(runId, 'finalization_boundary', {
+      iteration,
+      stepIndex,
+    });
+    if (
+      engine.isRunStopped(runId)
+      || lifecycleBeforeFinalize?.action === 'stop'
+      || lifecycleBeforeFinalize?.action === 'interrupt'
+    ) {
+      const terminal = db.prepare('SELECT status FROM agent_runs WHERE id = ?').get(runId)?.status || 'stopped';
+      engine.cleanupSubagentsForRun(runId, { cancelRunning: true });
+      engine.stopMessagingProgressSupervisor(runId);
+      engine.activeRuns.delete(runId);
+      return { runId, content: '', totalTokens, iterations: iteration, status: terminal };
+    }
+
     if (
       !normalizeOutgoingMessage(lastContent, options?.source || null)
       && !messagingSent
@@ -2528,8 +2616,17 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
       }
     }
 
-    db.prepare('UPDATE agent_runs SET status = ?, total_tokens = ?, final_response = ?, updated_at = datetime(\'now\'), completed_at = datetime(\'now\') WHERE id = ?')
-      .run('completed', totalTokens, finalResponseText || null, runId);
+    const completionWon = engine.completeRun(runId, {
+      totalTokens,
+      finalResponse: finalResponseText || null,
+    });
+    if (!completionWon) {
+      const terminal = db.prepare('SELECT status FROM agent_runs WHERE id = ?').get(runId)?.status || 'stopped';
+      engine.cleanupSubagentsForRun(runId, { cancelRunning: true });
+      engine.stopMessagingProgressSupervisor(runId);
+      engine.activeRuns.delete(runId);
+      return { runId, content: '', totalTokens, iterations: iteration, status: terminal };
+    }
 
     if (conversationId && options.skipConversationMaintenance !== true) {
       await engine.refreshConversationState({
@@ -2705,15 +2802,20 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
       }
     }
 
-    db.prepare('UPDATE agent_runs SET status = ?, error = ?, final_response = ?, updated_at = datetime(\'now\') WHERE id = ?')
-      .run(
-        'failed',
-        err.message,
-        sendSucceeded
-          ? (messagingFailureContent || null)
-          : (deliverableFailureResponse || null),
-        runId,
-      );
+    const failureWon = engine.failRun(runId, {
+      error: err.message,
+      finalResponse: sendSucceeded
+        ? (messagingFailureContent || null)
+        : (deliverableFailureResponse || null),
+      totalTokens,
+    });
+    if (!failureWon) {
+      engine.cleanupSubagentsForRun(runId, { cancelRunning: true });
+      engine.stopMessagingProgressSupervisor(runId);
+      engine.activeRuns.delete(runId);
+      const terminal = db.prepare('SELECT status FROM agent_runs WHERE id = ?').get(runId)?.status || 'stopped';
+      return { runId, content: '', totalTokens, iterations: iteration, status: terminal };
+    }
     console.error(
       `[Run ${shortenRunId(runId)}] failed trigger=${triggerSource} steps=${stepIndex} tokens=${totalTokens} error=${summarizeForLog(err.message, 180)}`
     );

package/server/services/ai/loop/lifecycle.js

@@ -0,0 +1,108 @@
+'use strict';
+
+const db = require('../../../db/database');
+
+const TERMINAL_STATUSES = new Set(['completed', 'failed', 'stopped', 'interrupted']);
+const CONTROL_PRIORITY = Object.freeze({ pause: 1, stop: 2, interrupt: 3 });
+
+function getRunControl(runId) {
+  return db.prepare(
+    `SELECT action, reason, requested_at
+     FROM agent_run_controls
+     WHERE run_id = ? AND consumed_at IS NULL`,
+  ).get(runId) || null;
+}
+
+function requestRunControl(runId, userId, action, reason = '') {
+  if (!Object.hasOwn(CONTROL_PRIORITY, action)) {
+    throw new Error(`Unsupported run control action: ${action}`);
+  }
+  const run = db.prepare(
+    'SELECT status FROM agent_runs WHERE id = ? AND user_id = ?',
+  ).get(runId, userId);
+  if (!run) return { accepted: false, reason: 'not_found' };
+  if (TERMINAL_STATUSES.has(run.status)) return { accepted: false, reason: 'terminal', status: run.status };
+
+  const existing = getRunControl(runId);
+  if (existing && CONTROL_PRIORITY[existing.action] > CONTROL_PRIORITY[action]) {
+    return { accepted: false, reason: 'stronger_signal_pending', action: existing.action };
+  }
+  db.prepare(
+    `INSERT INTO agent_run_controls (run_id, user_id, action, reason)
+     VALUES (?, ?, ?, ?)
+     ON CONFLICT(run_id) DO UPDATE SET
+       action = excluded.action,
+       reason = excluded.reason,
+       requested_at = datetime('now'),
+       consumed_at = NULL`,
+  ).run(runId, userId, action, String(reason || '').slice(0, 1000));
+  return { accepted: true, action };
+}
+
+function checkpointRun(runId, phase, state = {}) {
+  db.prepare(
+    `INSERT INTO agent_run_checkpoints (run_id, version, phase, state_json)
+     VALUES (?, 1, ?, ?)
+     ON CONFLICT(run_id) DO UPDATE SET
+       version = 1,
+       phase = excluded.phase,
+       state_json = excluded.state_json,
+       updated_at = datetime('now')`,
+  ).run(runId, phase, JSON.stringify(state));
+}
+
+function transitionRun(runId, status, fields = {}, allowed = ['running']) {
+  const assignments = ['status = ?', "updated_at = datetime('now')"];
+  const values = [status];
+  if (Object.hasOwn(fields, 'error')) {
+    assignments.push('error = ?');
+    values.push(fields.error || null);
+  }
+  if (Object.hasOwn(fields, 'finalResponse')) {
+    assignments.push('final_response = ?');
+    values.push(fields.finalResponse || null);
+  }
+  if (Object.hasOwn(fields, 'totalTokens')) {
+    assignments.push('total_tokens = ?');
+    values.push(Number(fields.totalTokens) || 0);
+  }
+  if (TERMINAL_STATUSES.has(status)) assignments.push("completed_at = COALESCE(completed_at, datetime('now'))");
+  const placeholders = allowed.map(() => '?').join(', ');
+  values.push(runId, ...allowed);
+  const result = db.prepare(
+    `UPDATE agent_runs SET ${assignments.join(', ')}
+     WHERE id = ? AND status IN (${placeholders})`,
+  ).run(...values);
+  return result.changes > 0;
+}
+
+function closeRun(runId, status, fields = {}, allowed = ['running']) {
+  const transaction = db.transaction(() => {
+    if (!transitionRun(runId, status, fields, allowed)) return false;
+    db.prepare(
+      `UPDATE agent_steps
+       SET status = ?, error = COALESCE(NULLIF(error, ''), ?), completed_at = COALESCE(completed_at, datetime('now'))
+       WHERE run_id = ? AND status = 'running'`,
+    ).run(status, fields.error || null, runId);
+    db.prepare(
+      `UPDATE agent_delegations
+       SET status = ?, error = COALESCE(NULLIF(error, ''), ?), updated_at = datetime('now'), completed_at = COALESCE(completed_at, datetime('now'))
+       WHERE parent_run_id = ? AND status = 'running'`,
+    ).run(status, fields.error || null, runId);
+    db.prepare(
+      `UPDATE agent_run_controls SET consumed_at = datetime('now')
+       WHERE run_id = ? AND consumed_at IS NULL`,
+    ).run(runId);
+    return true;
+  });
+  return transaction();
+}
+
+module.exports = {
+  TERMINAL_STATUSES,
+  checkpointRun,
+  closeRun,
+  getRunControl,
+  requestRunControl,
+  transitionRun,
+};

package/server/services/ai/loop/model_io.js

@@ -34,6 +34,7 @@ async function withModelCallTimeout(promise, options = {}, label = 'Model call')
   let timer = null;
   const timeout = new Promise((_, reject) => {
     timer = setTimeout(() => {
+      options?.modelAbortController?.abort(`${label} timed out.`);
       const error = new Error(`${label} timed out after ${formatElapsedDuration(timeoutMs)}.`);
       error.code = 'MODEL_CALL_TIMEOUT';
       reject(error);
@@ -61,6 +62,7 @@ async function requestStructuredJson(engine, {
 }) {
   const startedAt = Date.now();
   const structuredStep = `model:${phase}`;
+  const signal = telemetry?.signal;
   if (telemetry?.runId) {
     engine.updateRunProgress(telemetry.runId, {
       currentPhase: 'model',
@@ -84,6 +86,7 @@ async function requestStructuredJson(engine, {
             model,
             maxTokens,
             reasoningEffort: reasoningEffort || engine.getReasoningEffort(providerName, {}),
+            signal,
           }
         ),
         telemetry || {},
@@ -140,9 +143,15 @@ async function requestModelResponse(engine, {
 }) {
   const startedAt = Date.now();
   const requestMessages = sanitizeConversationMessages(messages);
+  const modelAbortController = new AbortController();
+  const parentSignal = options.signal;
+  const abortFromParent = () => modelAbortController.abort(parentSignal?.reason);
+  if (parentSignal?.aborted) abortFromParent();
+  else parentSignal?.addEventListener('abort', abortFromParent, { once: true });
   const callOptions = {
     model,
     reasoningEffort: engine.getReasoningEffort(providerName, options),
+    signal: modelAbortController.signal,
   };
 
   const attemptModelCall = async () => {
@@ -157,7 +166,7 @@ async function requestModelResponse(engine, {
         while (true) {
           const next = await withModelCallTimeout(
             iterator.next(),
-            options,
+            { ...options, modelAbortController },
             `Model stream iteration ${iteration}`,
           );
           if (next.done) break;
@@ -194,7 +203,7 @@ async function requestModelResponse(engine, {
     } else {
       response = await withModelCallTimeout(
         provider.chat(requestMessages, tools, callOptions),
-        options,
+        { ...options, modelAbortController },
         `Model iteration ${iteration}`,
       );
     }
@@ -202,18 +211,26 @@ async function requestModelResponse(engine, {
     return { response, streamContent };
   };
 
-  const { response, streamContent } = await withProviderRetry(attemptModelCall, {
-    ...(options.retry || {}),
-    label: `Engine ${model}`,
-    isRetryable: (err) => !err?.__providerRetryUnsafe && isTransientError(err),
-    onRetry: ({ attempt, delayMs }) => {
-      engine.emit(options.userId, 'run:interim', {
-        runId,
-        message: `Model service busy; retrying (attempt ${attempt}) in ${Math.max(1, Math.round(delayMs / 1000))}s.`,
-        phase: 'recovering',
-      });
-    },
-  });
+  let response;
+  let streamContent;
+  try {
+    ({ response, streamContent } = await withProviderRetry(attemptModelCall, {
+      ...(options.retry || {}),
+      label: `Engine ${model}`,
+      isRetryable: (err) => !modelAbortController.signal.aborted
+        && !err?.__providerRetryUnsafe
+        && isTransientError(err),
+      onRetry: ({ attempt, delayMs }) => {
+        engine.emit(options.userId, 'run:interim', {
+          runId,
+          message: `Model service busy; retrying (attempt ${attempt}) in ${Math.max(1, Math.round(delayMs / 1000))}s.`,
+          phase: 'recovering',
+        });
+      },
+    }));
+  } finally {
+    parentSignal?.removeEventListener('abort', abortFromParent);
+  }
 
   const resolvedResponse = response || {
     content: streamContent,