@prohost/cli 0.2.2 → 0.3.0

package/dist/agent/run.js

@@ -16,24 +16,43 @@
  * * **Runs are deduped on ``run_id``.** Webhook deliveries retry, and each
  *   attempt is mirrored to this socket. Without dedupe a retry would execute
  *   the command a second time and post a second reply.
+ *
+ * Runs can be long — hours — so every run this process has accepted and not yet
+ * reported is heartbeated (see `heartbeat.ts`), including the ones still queued
+ * behind the executing one. That keeps the server's lease alive and is the only
+ * way a stop pressed in the UI reaches a machine that accepts no connections.
  */
 import os from 'node:os';
+import { spawnSync } from 'node:child_process';
 import WebSocket from 'ws';
 import { completeRun, postReply } from './api.js';
-import { claudeCommand, isClaudeExec, looksLikeResumeFailure, parseClaudeOutput, writeMcpConfig, } from './claude.js';
-import { MCP_API_KEY_ENV_VAR, codexCommand, isCodexExec, looksLikeCodexResumeFailure, parseCodexOutput, } from './codex.js';
+import { claudeCommand, isClaudeExec, looksLikeResumeFailure, readClaudeStream, writeMcpConfig, } from './claude.js';
+import { MCP_API_KEY_ENV_VAR, codexBinary, codexCommand, codexMcpInventoryFlag, codexProfileName, codexShellSyntax, disableEntriesThatLoad, extraMcpEntries, isCodexExec, listMcpServers, mcpServersOverride, prohostMcpEntry, looksLikeCodexResumeFailure, parseCodexOutput, } from './codex.js';
+import { EXTRA_MCP_FILENAME, loadExtraMcpServers } from './extras.js';
 import { MCP_SERVER_NAME, MCP_TOOL_PATTERN } from './mcp.js';
-import { EVENT_AGENT_RUN_REQUESTED, EVENT_MENTION_CREATED, EVENT_MESSAGE_TEAM_CHAT, eventData, parseRunRequest, } from './contract.js';
+import { EVENT_AGENT_RUN_REQUESTED, EVENT_MENTION_CREATED, EVENT_MESSAGE_TEAM_CHAT, RUN_ERROR_CANCELLED_BY_USER, eventData, heartbeatPathFor, parseRunRequest, } from './contract.js';
 import { ensureWorkspace, streamUrlFor, workspacePath } from './credentials.js';
 import { execAgent } from './exec.js';
+import { HeartbeatLoop } from './heartbeat.js';
 import { buildPrompt, replyFromStdout } from './prompt.js';
 import { RunStore } from './runstore.js';
 import { SessionStore, sessionKeyFor } from './sessionstore.js';
 import { verifySignature } from './signature.js';
 import { isWebhookEvent } from '../forward.js';
 import { DEFAULT_WS_URL } from '../listen.js';
-/** Default ceiling on one agent invocation. */
-export const DEFAULT_EXEC_TIMEOUT_MS = 300_000;
+/**
+ * Default ceiling on one agent invocation.
+ *
+ * Two hours, up from five minutes. Five was a guess made when a run meant
+ * "answer a mention", and it was wrong in the expensive direction: a paired
+ * brain given a real coding task in production was killed at 300s, mid-edit,
+ * and reported `timed out after 300000ms` to the person waiting. A coding
+ * agent's unit of work is not a chat reply, and no local clock can tell a
+ * wedged run from a productive one — the server's lease can, which is what
+ * heartbeats are for. This is now a backstop against a genuinely stuck process
+ * rather than a shape imposed on the work; `--timeout 0` removes it entirely.
+ */
+export const DEFAULT_EXEC_TIMEOUT_MS = 7_200_000;
 /** Reconnect backoff — full jitter between ``BASE`` and the current cap. */
 const RECONNECT_BASE_DELAY_MS = 1_000;
 const RECONNECT_MAX_DELAY_MS = 30_000;
@@ -64,8 +83,18 @@ function backoffDelay(attempt) {
     const cap = Math.min(RECONNECT_MAX_DELAY_MS, RECONNECT_BASE_DELAY_MS * 2 ** Math.max(0, attempt - 1));
     return Math.floor(Math.random() * cap);
 }
-/** Backoff schedule for retrying a completion the server didn't accept. */
-const COMPLETION_RETRY_DELAYS_MS = [1_000, 3_000, 9_000];
+/**
+ * Backoff schedule for retrying a completion the server didn't accept.
+ *
+ * Roughly ten minutes in total, up from thirteen seconds. The old schedule was
+ * sized for a five-minute run: losing the completion meant losing five minutes
+ * of work, and a laptop that has been offline for thirteen seconds is usually
+ * offline for longer. Now a run can be four hours of an agent's work, which
+ * makes waiting out a wifi drop overwhelmingly the better bet — and the run
+ * keeps being heartbeated for the whole retry window, so the row it is trying
+ * to close stays alive rather than being reaped underneath the retries.
+ */
+const COMPLETION_RETRY_DELAYS_MS = [1_000, 3_000, 9_000, 30_000, 60_000, 120_000, 180_000, 240_000];
 /**
  * Complete a run, retrying transient failures.
  *
@@ -116,13 +145,12 @@ function resolveWorkdir(options, log) {
     }
 }
 /**
- * Assemble the per-process runtime: workspace, session ledger, and — for Claude
- * Code only — the MCP config that gives the agent tools.
+ * Assemble the per-process runtime: workspace, session ledger, and the
+ * brain-specific MCP preparation used immediately before each run.
  *
- * Every step here is allowed to fail without stopping the harness. An agent
- * that answers from the prompt alone is the product as it shipped; refusing to
- * start because a config file couldn't be written would be a regression
- * dressed up as a safety check.
+ * Brain-specific MCP preparation happens per run. A daemon is long-lived, so
+ * an extras-file deletion or newly installed Codex server must take effect on
+ * the next invocation rather than after a restart or cache timeout.
  */
 function buildRuntime(options, log) {
     const workdir = resolveWorkdir(options, log);
@@ -134,53 +162,51 @@ function buildRuntime(options, log) {
             : 'plain';
     const skipPermissions = options.safeTools !== true;
     if (brain === 'plain') {
-        return { workdir, sessions, brain, skipPermissions, mcpWired: false };
+        return { workdir, sessions, brain, skipPermissions, prepareMcp: () => ({ mcpWired: false }) };
     }
     const mcpUrl = options.credentials.mcp_url;
-    let mcpConfigPath;
-    let codexMcpUrl;
-    let execEnv;
+    let announced = '';
+    const announce = (lines) => {
+        const message = lines.join('\n');
+        if (message === announced)
+            return;
+        for (const line of lines)
+            log(line);
+        announced = message;
+    };
     if (!mcpUrl) {
         log(`${timestamp()} ! this pairing carries no MCP endpoint; the agent will run without ProhostAI tools`);
     }
-    else if (brain === 'claude') {
-        try {
-            mcpConfigPath = writeMcpConfig({
-                mcpUrl,
-                apiKey: options.credentials.api_key,
-                env: options.env,
-            });
-        }
-        catch (err) {
-            // Said out loud rather than swallowed: the visible symptom would
-            // otherwise be an agent that answers "I can't look that up", which reads
-            // like a model problem and sends someone to the wrong file.
-            log(`${timestamp()} ! could not write the MCP config ` +
-                `(${err instanceof Error ? err.message : String(err)}); the agent will run without ` +
-                'ProhostAI tools');
+    const prepareMcp = brain === 'claude'
+        ? () => {
+            const extras = loadExtraMcpServers({ env: options.env });
+            announce(extras.warnings.map((warning) => `${timestamp()} ! ${warning}`));
+            // Always pass an explicit strict config, including when it is empty.
+            // Omitting the path would make `--strict-mcp-config` ineffective and
+            // expose Claude's user/project MCP servers under skipped permissions.
+            try {
+                const mcpConfigPath = writeMcpConfig({
+                    mcpUrl,
+                    apiKey: options.credentials.api_key,
+                    extraServers: extras.servers,
+                    env: options.env,
+                });
+                return { mcpConfigPath, mcpWired: Boolean(mcpUrl) };
+            }
+            catch (err) {
+                const detail = err instanceof Error ? err.message : String(err);
+                return {
+                    mcpWired: false,
+                    error: `could not write the strict Claude MCP config (${detail})`,
+                };
+            }
         }
-    }
-    else {
-        // Codex needs no file: the server is a `-c` override on each invocation and
-        // the key is read from the child's environment. Nothing to fail, so nothing
-        // to fall back from.
-        codexMcpUrl = mcpUrl;
-        execEnv = { [MCP_API_KEY_ENV_VAR]: options.credentials.api_key };
-    }
-    const mcpWired = Boolean(mcpConfigPath ?? codexMcpUrl);
-    const tools = mcpWired ? `, ProhostAI tools wired in as ${MCP_SERVER_NAME}` : '';
+        : codexMcpEntryProvider(options, log, workdir);
+    const tools = mcpUrl ? `, ProhostAI tools prepared per run as ${MCP_SERVER_NAME}` : '';
     const where = workdir ? ` (workspace: ${workdir})` : '';
     log(brain === 'claude'
         ? `${timestamp()} ⚙ Claude Code detected — resuming a session per conversation${tools}${where}`
         : `${timestamp()} ⚙ Codex detected — resuming a thread per conversation${tools}${where}`);
-    if (brain === 'codex' && mcpWired) {
-        // The one asymmetry worth an operator knowing about, because it is a
-        // security property they may be assuming they have. Claude Code is told to
-        // load our config and nothing else; Codex has no equivalent switch, so its
-        // own `~/.codex/config.toml` servers come along.
-        log(`${timestamp()} ! Codex has no --strict-mcp-config equivalent: this run also holds ` +
-            'whatever MCP servers your ~/.codex/config.toml defines.');
-    }
     // Said out loud every start-up, not buried in --help. This is the line that
     // tells an operator their machine will run tool calls, unprompted, on text
     // written by someone else in their workspace.
@@ -191,7 +217,153 @@ function buildRuntime(options, log) {
             'access, on instructions from your workspace. Use --safe-tools to gate them.'
         : `${timestamp()} ⚙ --safe-tools: tools are permission-gated; anything needing ` +
             'approval will not run (nobody is at a terminal to approve it).');
-    return { workdir, sessions, brain, mcpConfigPath, codexMcpUrl, execEnv, mcpWired, skipPermissions };
+    return { workdir, sessions, brain, prepareMcp, skipPermissions };
+}
+/**
+ * Prepare a fresh Codex MCP override immediately before each invocation.
+ *
+ * No cache is safe here: Codex rereads its config per invocation and the
+ * extras file is the operator's live grant list. Fresh reads make both a new
+ * configured server and a revoked extra visible to the very next run.
+ */
+export function codexMcpEntryProvider(options, log, workdir) {
+    let announced = '';
+    return () => {
+        const extras = loadExtraMcpServers({ env: options.env });
+        const lines = extras.warnings.map((warning) => `${timestamp()} ! ${warning}`);
+        const prepared = codexMcpEntries(options, extras.servers, (line) => lines.push(line), workdir);
+        const said = lines.join('\n');
+        if (said !== announced) {
+            for (const line of lines)
+                log(line);
+            announced = said;
+        }
+        return prepared;
+    };
+}
+/*
+ * Every `mcp_servers` entry a Codex run gets: ours, the operator's extras, and
+ * verified switches that keep the operator's own servers out. Preparation is
+ * fail-closed: an unprovable boundary never reaches an unsandboxed agent.
+ */
+function codexMcpEntries(options, extraServers, log, workdir) {
+    const fail = (detail) => {
+        log(`${timestamp()} ! ${detail}`);
+        return { mcpWired: false, error: detail };
+    };
+    const probe = options.probeImpl ?? defaultProbe;
+    if (codexShellSyntax(options.exec)) {
+        return fail('--exec uses shell quoting, expansion, glob, comment, escape, or control syntax that the Codex ' +
+            'isolation verifier cannot tokenize safely; use a plain argv-shaped Codex command');
+    }
+    const binary = codexBinary(options.exec);
+    const entries = [];
+    const execEnv = {};
+    if (options.credentials.mcp_url) {
+        entries.push(prohostMcpEntry(options.credentials.mcp_url));
+        execEnv[MCP_API_KEY_ENV_VAR] = options.credentials.api_key;
+    }
+    const profile = codexProfileName(options.exec);
+    if (profile) {
+        return fail(`--exec runs Codex under profile "${profile}", whose MCP servers cannot be enumerated; ` +
+            'remove the profile flag before running this harness');
+    }
+    const inventoryFlag = codexMcpInventoryFlag(options.exec);
+    if (inventoryFlag) {
+        return fail(`--exec includes ${inventoryFlag}, which can change Codex's MCP or plugin inventory, or its ` +
+            `working directory, outside the configuration the harness can verify; remove cwd, config, feature, ` +
+            'and --ignore-user-config flags');
+    }
+    const configured = listMcpServers(binary, probe, undefined, workdir);
+    if (configured === null) {
+        return fail('could not ask Codex which MCP servers it has configured');
+    }
+    const configuredNames = new Set(configured.map((server) => server.name));
+    if (configuredNames.has(MCP_SERVER_NAME)) {
+        return fail(`your Codex config already defines the reserved MCP server name "${MCP_SERVER_NAME}"; ` +
+            'rename or remove it so credentials cannot be merged into the paired server');
+    }
+    const collisions = Object.keys(extraServers).filter((name) => configuredNames.has(name));
+    if (collisions.length > 0) {
+        log(`${timestamp()} ! ${EXTRA_MCP_FILENAME} reuses names your Codex config already defines, ` +
+            `so those entries were ignored (Codex would merge their credentials): ${collisions.join(', ')}. Rename them to use them.`);
+    }
+    const usableExtras = Object.fromEntries(Object.entries(extraServers).filter(([name]) => !collisions.includes(name)));
+    const extra = extraMcpEntries(usableExtras);
+    if (extra.skipped.length > 0) {
+        log(`${timestamp()} ! could not pass these extra MCP servers to Codex ` +
+            `(value not expressible as a safe config override): ${extra.skipped.join(', ')}`);
+    }
+    let acceptedExtraEntries = [];
+    if (extra.entries.length > 0) {
+        if (listMcpServers(binary, probe, mcpServersOverride(extra.entries), workdir) === null) {
+            log(`${timestamp()} ! Codex rejected the server definitions in ${EXTRA_MCP_FILENAME}; ` +
+                'the run will continue without those optional extras');
+        }
+        else {
+            acceptedExtraEntries = extra.entries;
+            entries.push(...acceptedExtraEntries);
+            Object.assign(execEnv, extra.env);
+        }
+    }
+    // A configured collision was dropped above, so it is deliberately disabled.
+    const acceptedExtraNames = new Set(Object.keys(usableExtras).filter((name) => !extra.skipped.includes(name)));
+    if (acceptedExtraEntries.length === 0)
+        acceptedExtraNames.clear();
+    const targets = [...configuredNames].filter((name) => !acceptedExtraNames.has(name));
+    const { entries: disabled, failed } = disableEntriesThatLoad(binary, probe, targets, workdir);
+    if (failed.length > 0) {
+        return fail(`these Codex MCP servers could not be disabled: ${failed.join(', ')}`);
+    }
+    entries.push(...disabled);
+    if (disabled.length > 0) {
+        log(`${timestamp()} ⚙ your own Codex MCP servers are disabled for agent runs: ${targets.join(', ')}`);
+    }
+    // Verify the exact combined override, not its pieces. This catches a config
+    // merge that re-enables a server and a server installed between the initial
+    // survey and this final probe.
+    const override = mcpServersOverride(entries);
+    const finalListing = listMcpServers(binary, probe, override, workdir);
+    if (finalListing === null)
+        return fail('Codex rejected the final isolated MCP configuration');
+    const allowed = new Set([
+        ...(options.credentials.mcp_url ? [MCP_SERVER_NAME] : []),
+        ...acceptedExtraNames,
+    ]);
+    const enabled = new Set(finalListing.filter((server) => server.enabled).map((server) => server.name));
+    const missing = [...allowed].filter((name) => !enabled.has(name));
+    if (missing.length > 0) {
+        return fail(`expected MCP servers are missing or disabled after isolation: ${missing.join(', ')}`);
+    }
+    const unexpected = finalListing.filter((server) => server.enabled && !allowed.has(server.name));
+    if (unexpected.length > 0) {
+        return fail(`unexpected MCP servers remain enabled after isolation: ${unexpected.map((server) => server.name).join(', ')}`);
+    }
+    return {
+        codexMcpEntries: entries,
+        execEnv: Object.keys(execEnv).length > 0 ? execEnv : undefined,
+        mcpWired: Boolean(options.credentials.mcp_url),
+    };
+}
+/**
+ * Run one short command and capture stdout.
+ *
+ * `shell: true` for the same reason `exec.ts` uses it — the binary name comes
+ * out of the operator's own `--exec` string and may be a quoted path — and for
+ * the same reason it is safe: nothing event-derived reaches this command line.
+ * `cwd` must match the eventual run exactly, because Codex loads trusted
+ * project `.codex/config.toml` layers from there.
+ */
+function defaultProbe(command, cwd) {
+    const result = spawnSync(command, {
+        shell: true,
+        cwd,
+        encoding: 'utf8',
+        // Generous for a config read, short enough that a wedged binary delays
+        // a run rather than wedging the daemon.
+        timeout: 15_000,
+    });
+    return { status: result.status, stdout: result.stdout ?? '' };
 }
 /**
  * Run the agent command once and turn its output into a reply.
@@ -202,32 +374,113 @@ function buildRuntime(options, log) {
  * tell a failed turn from a successful one. A plain command's behaviour here is
  * byte-for-byte what it was before this function existed.
  */
-async function runTurn(run, prompt, options, runtime, log) {
+async function runTurn(run, prompt, options, runtime, mcp, control, log) {
     // `--timeout` is a ceiling on the run, not on each attempt: the retry below
     // must not let one run take twice what the operator allowed. Both attempts
     // share one deadline, which costs the retry nothing in practice — a rejected
     // `--resume` fails in milliseconds, long before any of the budget is spent.
     const budgetMs = options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS;
+    // `--timeout 0`: no local ceiling at all. The run then ends when the agent
+    // ends it, when somebody cancels it, or when the server stops renewing its
+    // lease — all of which are better judges of "stuck" than a fixed clock.
+    const capped = budgetMs > 0;
     const startedAt = Date.now();
     const remainingMs = () => budgetMs - (Date.now() - startedAt);
-    const exec = (command) => execAgent({
-        command,
-        prompt,
-        timeoutMs: remainingMs(),
-        cwd: runtime.workdir,
-        env: runtime.execEnv,
-        spawnImpl: options.spawnImpl,
-    });
+    const timeLeft = () => !capped || remainingMs() > 0;
+    // Which surface's session this turn belongs to, and `undefined` on a dry run
+    // — which is what makes every `remember`/`forget` below a no-op there.
+    // `--dry-run` promises to make no writes, and a resumed turn is a write: both
+    // brains record it in their transcript whether or not we post the reply, so
+    // remembering the session would leave the next *real* run continuing from an
+    // answer nobody saw. Resolved once, up here, because the streaming reader
+    // records it mid-run rather than at the end of the turn.
+    const sessionKey = options.dryRun ? undefined : sessionKeyFor(run);
+    let recordedSession;
+    /** The reader for the attempt currently running, if this brain streams. */
+    let stream;
+    /**
+     * Act on what the stream has said so far.
+     *
+     * The session id is written to the ledger **as soon as it is announced**,
+     * which is a deliberate departure from "record it once the reply is durable".
+     * That rule protects against continuing a session whose answer nobody saw;
+     * it cannot protect a run that is killed at hour two, because that run never
+     * reaches the end where the recording used to happen — and losing the id
+     * loses the only handle on two hours of the agent's context. The rejected-
+     * reply case is handled where it happens instead, by forgetting again.
+     */
+    const observe = (reader) => {
+        const id = reader.sessionId();
+        if (id && id !== recordedSession) {
+            recordedSession = id;
+            runtime.sessions.remember(sessionKey, id);
+        }
+        const activity = reader.activity();
+        if (activity)
+            control.report(activity);
+    };
+    /**
+     * Undo an eagerly recorded Claude session after an ordinary failed turn.
+     *
+     * Timeouts and cancellations deliberately keep their session so the next
+     * delivery can continue a long-running turn whose context would otherwise be
+     * lost. A command or parser failure has no such continuity guarantee: its
+     * answer was never made durable, so resuming it would skip an unseen turn.
+     */
+    const forgetRecordedSession = () => {
+        if (!recordedSession)
+            return;
+        runtime.sessions.forget(sessionKey);
+        recordedSession = undefined;
+    };
+    const exec = (command) => {
+        // A fresh reader per attempt: a retry without `--resume` is a different
+        // session, and inheriting the failed attempt's id would resume nothing.
+        const reader = runtime.brain === 'claude' ? readClaudeStream() : undefined;
+        stream = reader;
+        return execAgent({
+            command,
+            prompt,
+            timeoutMs: capped ? Math.max(1, remainingMs()) : 0,
+            cwd: runtime.workdir,
+            env: mcp.execEnv,
+            spawnImpl: options.spawnImpl,
+            cancelSignal: control.signal,
+            onStdout: reader
+                ? (chunk) => {
+                    reader.push(chunk);
+                    observe(reader);
+                }
+                : undefined,
+        });
+    };
     // Wall clock for the whole turn, retry included — what the operator waited,
     // rather than what the last attempt took.
     const elapsed = () => Date.now() - startedAt;
-    const failure = (result) => ({
-        ok: false,
-        reason: result.error ??
-            (result.exitCode === null ? 'agent command did not run' : `agent command exited ${result.exitCode}`),
-        detail: result.stderr.trim().slice(-500),
-        durationMs: elapsed(),
-    });
+    if (mcp.error) {
+        return { ok: false, reason: 'MCP isolation could not be established', detail: mcp.error, durationMs: 0 };
+    }
+    const failure = (result) => {
+        // A killed run has no reply, but it usually has most of one. Reporting the
+        // tail of what the agent was writing turns "timed out after 7200000ms" into
+        // something the person waiting can act on — and it is the only trace left
+        // of however many hours of work.
+        const salvaged = result.timedOut || result.cancelled ? stream?.salvage() : undefined;
+        // The stderr tail yields room when there is a salvaged answer to carry: the
+        // completion `error` is capped server-side, and of the two, what the agent
+        // was saying beats the last 500 characters of its logging.
+        const stderrTail = result.stderr.trim().slice(salvaged ? -200 : -500);
+        const detail = [stderrTail, salvaged ? `unfinished answer: ${salvaged}` : '']
+            .filter((part) => part.length > 0)
+            .join('\n');
+        return {
+            ok: false,
+            reason: result.error ??
+                (result.exitCode === null ? 'agent command did not run' : `agent command exited ${result.exitCode}`),
+            detail,
+            durationMs: elapsed(),
+        };
+    };
     if (runtime.brain === 'plain') {
         const result = await exec(options.exec);
         if (!result.ok)
@@ -237,24 +490,19 @@ async function runTurn(run, prompt, options, runtime, log) {
             ? { ok: true, reply, durationMs: elapsed() }
             : { ok: false, reason: 'agent produced no output', durationMs: elapsed() };
     }
-    // No session bookkeeping on a dry run. `--dry-run` promises to make no
-    // writes, and a resumed turn is a write: both brains record it in their
-    // transcript whether or not we post the reply, so remembering the session
-    // would leave the next *real* run continuing from an answer nobody saw.
-    const key = options.dryRun ? undefined : sessionKeyFor(run);
-    const sessionId = runtime.sessions.get(key);
+    const sessionId = runtime.sessions.get(sessionKey);
     const codex = runtime.brain === 'codex';
     // Same three questions for both brains — resume this thread, wire these
     // tools, gate or don't — asked with each one's own flags.
     const build = (resume) => codex
         ? codexCommand(options.exec, {
             sessionId: resume,
-            mcpUrl: runtime.codexMcpUrl,
+            mcpEntries: mcp.codexMcpEntries,
             skipPermissions: runtime.skipPermissions,
         })
         : claudeCommand(options.exec, {
             sessionId: resume,
-            mcpConfigPath: runtime.mcpConfigPath,
+            mcpConfigPath: mcp.mcpConfigPath,
             skipPermissions: runtime.skipPermissions,
         });
     let result = await exec(build(sessionId));
@@ -268,15 +516,31 @@ async function runTurn(run, prompt, options, runtime, log) {
         // Skipped rather than squeezed into a token budget: a retry with no time
         // left would be killed on the spot and reported as a timeout, which is a
         // worse error message than the resume failure it replaced.
-        remainingMs() > 0) {
+        timeLeft() &&
+        // A run somebody just stopped does not get started again on its way out.
+        !control.signal.aborted) {
         log(`    … session ${sessionId.slice(0, 12)} could not be resumed; starting a fresh one`);
-        runtime.sessions.forget(key);
+        runtime.sessions.forget(sessionKey);
+        recordedSession = undefined;
         result = await exec(build());
     }
-    if (!result.ok)
+    if (!result.ok) {
+        if (!result.timedOut && !result.cancelled)
+            forgetRecordedSession();
         return failure(result);
+    }
+    // A denied tool is not an error either brain reports — the turn succeeds and
+    // the reply just quietly lacks whatever the tool would have found. Printing
+    // it is the difference between "the agent seems dumb" and "the agent asked
+    // for WebSearch and there was nobody to approve it".
+    const reportDenials = (denials) => {
+        if (denials.length > 0) {
+            log(`    ! tools the agent asked for and was refused: ${denials.join(', ')}`);
+        }
+    };
     if (codex) {
         const output = parseCodexOutput(result.stdout);
+        reportDenials(output.denials);
         if (!output.ok) {
             return {
                 ok: false,
@@ -287,15 +551,15 @@ async function runTurn(run, prompt, options, runtime, log) {
         }
         return { ok: true, reply: output.text, durationMs: elapsed(), sessionId: output.sessionId };
     }
-    const parsed = parseClaudeOutput(result.stdout);
-    // A denied tool is not an error the agent reports — the turn succeeds and the
-    // reply just quietly lacks whatever the tool would have found. Printing it is
-    // the difference between "the agent seems dumb" and "the agent asked for
-    // WebSearch and there was nobody to approve it".
-    if (parsed.denials.length > 0) {
-        log(`    ! tools the agent asked for and was refused: ${parsed.denials.join(', ')}`);
-    }
+    // The reply comes out of the stream's own `result` event, so a run whose
+    // stdout we watched arrives already parsed. `finish` still gets the captured
+    // stdout: a command that emitted no result event at all — a login prompt, a
+    // crash banner, a `claude` that ignored `--output-format` — is a failure, and
+    // the whole-output parser is what says so in words.
+    const parsed = (stream ?? readClaudeStream()).finish(result.stdout);
+    reportDenials(parsed.denials);
     if (!parsed.ok) {
+        forgetRecordedSession();
         return {
             ok: false,
             reason: 'the agent did not produce a usable reply',
@@ -303,7 +567,13 @@ async function runTurn(run, prompt, options, runtime, log) {
             durationMs: elapsed(),
         };
     }
-    return { ok: true, reply: parsed.text, durationMs: elapsed(), sessionId: parsed.sessionId };
+    return {
+        ok: true,
+        reply: parsed.text,
+        durationMs: elapsed(),
+        sessionId: parsed.sessionId,
+        sessionRecorded: recordedSession !== undefined,
+    };
 }
 /**
  * Handle one verified run end-to-end.
@@ -311,23 +581,54 @@ async function runTurn(run, prompt, options, runtime, log) {
  * Every path except a dry run — including every failure — calls the completion
  * endpoint before returning.
  */
-async function handleRun(run, options, runtime, api, log, sleep) {
+async function handleRun(run, options, runtime, api, log, sleep, control) {
     const label = `run ${run.run_id.slice(0, 12)}`;
     const surface = run.conversation_id ?? run.kind ?? 'no conversation';
     log(`${timestamp()} ▶ ${label} started (${surface})`);
+    const mcp = runtime.prepareMcp();
     const prompt = buildPrompt(run, {
         agentName: options.credentials.agent?.name ?? 'your agent',
         // Named only when the tools are actually reachable from this invocation.
         // Telling an agent about an endpoint it has no client for is what produced
         // replies describing lookups that never happened.
-        mcpServerName: runtime.mcpWired ? MCP_SERVER_NAME : undefined,
-        // Only Claude Code's spelling has been verified against the binary, so only
-        // Claude Code gets told a pattern. See PromptContext.mcpToolPattern.
-        mcpToolPattern: runtime.brain === 'claude' && runtime.mcpWired ? MCP_TOOL_PATTERN : undefined,
+        mcpServerName: mcp.mcpWired ? MCP_SERVER_NAME : undefined,
+        // Both shaped brains spell MCP tools `mcp__<server>__<tool>` — Codex's was
+        // an open question until it was measured against the binary in 0.2.3. A
+        // `plain` command is still told the server name only: we have no idea what
+        // it is. See PromptContext.mcpToolPattern.
+        mcpToolPattern: runtime.brain !== 'plain' && mcp.mcpWired ? MCP_TOOL_PATTERN : undefined,
         toolsGated: !runtime.skipPermissions,
+        // The real number, not a vague "there is a limit" — see PromptContext.
+        timeoutMs: options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS,
     });
-    const result = await runTurn(run, prompt, options, runtime, log);
+    const result = await runTurn(run, prompt, options, runtime, mcp, control, log);
     const seconds = (result.durationMs / 1000).toFixed(1);
+    // Stopped from outside. Checked before the failure branch below, because a
+    // stopped run *is* a failed command — reporting it as one would put "timed
+    // out"-shaped noise in a thread where somebody deliberately pressed stop.
+    const stopped = control.disposition();
+    if (stopped === 'resolved') {
+        // The server already closed this run: reaped after its lease lapsed, or
+        // resolved by something else. There is nothing left to post a reply to and
+        // nothing left to complete — a completion would just 409.
+        log(`${timestamp()} ⏹ ${label} was already closed server-side (${seconds}s) — nothing was posted`);
+        return 'resolved';
+    }
+    if (stopped === 'cancelled') {
+        log(`${timestamp()} ⏹ ${label} cancelled after ${seconds}s`);
+        if (options.dryRun) {
+            log('    [dry-run] would report the run as cancelled');
+            return 'skipped';
+        }
+        // The exact marker string, alone: the server words the thread notice from
+        // it, and anything wrapped around it reads as a crash instead of a stop.
+        const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
+        if (!closed.ok) {
+            log(`    ! could not report the cancellation (HTTP ${closed.status}) ${closed.error ?? ''}`);
+            return 'unreported';
+        }
+        return 'cancelled';
+    }
     if (!result.ok) {
         const { reason } = result;
         const detail = result.detail ?? '';
@@ -378,6 +679,12 @@ async function handleRun(run, options, runtime, api, log, sleep) {
     const posted = await postReply(api, run.reply_path, run.reply_body_key, reply, run.run_id, run.reply_body_extra);
     if (!posted.ok) {
         log(`${timestamp()} ✗ ${label} reply rejected (HTTP ${posted.status}) ${posted.error ?? ''}`);
+        // Undo the mid-run recording. The session exists and the agent's transcript
+        // holds this turn, but the conversation never received its answer, so
+        // continuing it would skip reasoning nobody can see — the same rule that
+        // used to be expressed by simply not recording until the reply landed.
+        if (result.sessionRecorded)
+            runtime.sessions.forget(sessionKeyFor(run));
         const reported = await completeWithRetry(api, run.completion_path, { ok: false, error: `posting the reply failed with HTTP ${posted.status}` }, log, sleep);
         return reported.ok ? 'failed' : 'unreported';
     }
@@ -433,9 +740,79 @@ export async function runAgentHarness(options) {
     // duplicate — the run is in flight right here. Only a run in the ledger that
     // this process never handled is a candidate for restart recovery.
     const handledThisSession = new Set();
+    /** Accepted, not yet reported — the executing run and everything queued. */
+    const accepted = new Map();
+    const heartbeats = new HeartbeatLoop({ api, log, intervalMs: options.heartbeatIntervalMs });
     let queue = Promise.resolve();
     let fatal;
     let attempt = 0;
+    /** Stop tracking a run — it has reached a terminal state, one way or another. */
+    const release = (runId) => {
+        accepted.delete(runId);
+        heartbeats.untrack(runId);
+    };
+    /**
+     * Work started from a heartbeat that the harness must still wait for on the
+     * way out, but that a heartbeat tick must not wait for.
+     *
+     * Closing out a cancelled queued run can take the whole completion-retry
+     * schedule — around ten minutes against an unreachable server — and beats do
+     * not overlap. Awaited inside the tick, one unreachable completion would stop
+     * every other run being heartbeated, and the four-hour job it was queued
+     * behind would be reaped for going quiet.
+     */
+    const background = [];
+    /**
+     * Somebody pressed stop in ProhostAI.
+     *
+     * The two cases are genuinely different work. An executing run reports
+     * itself: the abort reaches `execAgent`, which signals the process group the
+     * same way a timeout does, and `handleRun` completes it with the marker
+     * error. A run still in the queue has nobody to report it — it will never
+     * execute, and the queue is about to skip it — so it is closed out here.
+     */
+    const cancelAccepted = async (entry) => {
+        if (entry.disposition)
+            return;
+        entry.disposition = 'cancelled';
+        if (entry.executing) {
+            entry.controller.abort();
+            return;
+        }
+        release(entry.run.run_id);
+        log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)} was cancelled before it started`);
+        // `--dry-run` promises to make no writes, and a completion is the most
+        // terminal write there is. The executing-run path defers to the same check
+        // inside `handleRun`; this one has to make it itself, because a queued run
+        // never reaches `handleRun` at all.
+        if (options.dryRun) {
+            log('    [dry-run] would report the run as cancelled');
+            return;
+        }
+        const closed = await completeWithRetry(api, entry.run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
+        if (closed.ok)
+            runs.markCompleted(entry.run.run_id);
+        metrics.runsFailed += 1;
+    };
+    /**
+     * The server has already closed this run out — it was reaped, or resolved by
+     * something else. Either way nothing this machine does with it can land, so
+     * the only correct move is to stop: kill the command if one is running, and
+     * post nothing at all. A completion here would be a 409 by definition.
+     */
+    const resolveAccepted = (entry) => {
+        if (entry.disposition)
+            return;
+        entry.disposition = 'resolved';
+        if (entry.executing) {
+            entry.controller.abort();
+            return;
+        }
+        release(entry.run.run_id);
+        // Terminal server-side, so a redelivery must not start it either.
+        runs.markCompleted(entry.run.run_id);
+        log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)} was closed server-side before it started`);
+    };
     const exhausted = () => options.maxConnections !== undefined && metrics.connections >= options.maxConnections;
     while (!options.signal?.aborted && fatal === undefined && !exhausted()) {
         metrics.connections += 1;
@@ -449,8 +826,14 @@ export async function runAgentHarness(options) {
         log(`${timestamp()} … disconnected; reconnecting in ${(delay / 1000).toFixed(1)}s`);
         await sleep(delay);
     }
-    // Let an in-flight run finish reporting before we hand back metrics.
+    // Let an in-flight run finish reporting before we hand back metrics — and
+    // any cancellation the heartbeat loop started and deliberately did not wait
+    // for, so the harness never exits with a run half-closed.
     await queue;
+    // Stopped first, so no further tick can start work after the wait below has
+    // taken its snapshot of `background`.
+    heartbeats.stop();
+    await Promise.all(background);
     if (fatal !== undefined) {
         log(`${timestamp()} ✗ ${fatal}`);
         metrics.fatal = fatal;
@@ -598,13 +981,36 @@ export async function runAgentHarness(options) {
                 // redelivery execute the agent a second time.
                 runs.markStarted(run.run_id, run.completion_path);
                 metrics.runsStarted += 1;
+                // Heartbeat from the moment it is accepted, not from the moment it
+                // starts. The row is `running` server-side either way, and a run that
+                // waits an hour behind a long job is reaped exactly like one that has
+                // been working for an hour.
+                const entry = { run, controller: new AbortController(), executing: false };
+                accepted.set(run.run_id, entry);
+                heartbeats.track(run.run_id, {
+                    path: heartbeatPathFor(run),
+                    progress: () => entry.progress,
+                    // Deliberately not awaited here — see `background`.
+                    onCancelRequested: () => {
+                        background.push(cancelAccepted(entry).catch((err) => {
+                            log(`${timestamp()} ✗ closing cancelled run ${run.run_id} failed: ${String(err)}`);
+                        }));
+                    },
+                    onResolved: () => resolveAccepted(entry),
+                });
                 // Serialize execution but never block the socket reader: the ack and
                 // the heartbeat have to keep flowing while the agent thinks.
                 queue = queue.then(async () => {
+                    // Settled while it waited — cancelled, or closed server-side. Both
+                    // were dealt with where they were decided; there is nothing to run
+                    // and nothing left to report.
+                    if (entry.disposition)
+                        return;
                     // Shutdown was requested while this sat in the queue. Don't start a
                     // fresh command the operator is waiting to be rid of — but do close
                     // the run out, so the server isn't left waiting on it either.
                     if (options.signal?.aborted) {
+                        release(run.run_id);
                         log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)} dropped — harness is shutting down`);
                         const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: 'the harness shut down before this run started' }, log, sleep);
                         if (closed.ok)
@@ -612,20 +1018,42 @@ export async function runAgentHarness(options) {
                         metrics.runsFailed += 1;
                         return;
                     }
-                    const outcome = await handleRun(run, options, runtime, api, log, sleep).catch((err) => {
+                    // Set before the first `await`, so a cancellation arriving from the
+                    // heartbeat either sees a queued run (and closes it out itself) or an
+                    // executing one (and aborts it) — never both, and never neither.
+                    entry.executing = true;
+                    const control = {
+                        signal: entry.controller.signal,
+                        disposition: () => entry.disposition,
+                        report: (activity) => {
+                            entry.progress = activity;
+                        },
+                    };
+                    const outcome = await handleRun(run, options, runtime, api, log, sleep, control).catch((err) => {
                         log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
                         return 'failed';
                     });
+                    // Held until here on purpose: heartbeats keep the row alive for the
+                    // whole of `handleRun`, including the ten minutes of completion
+                    // retries at the end of it.
+                    release(run.run_id);
                     // Both terminal outcomes reached the completion endpoint; only then
                     // is the run settled and safe to skip on redelivery.
                     if (outcome === 'succeeded')
                         metrics.runsSucceeded += 1;
-                    if (outcome === 'failed' || outcome === 'unreported')
+                    // A cancellation is reported to the server as `failed`, and counted
+                    // the same way here — the run did not produce the answer it was for.
+                    if (outcome === 'failed' || outcome === 'unreported' || outcome === 'cancelled') {
                         metrics.runsFailed += 1;
+                    }
                     // Only a run the server acknowledged as terminal is settled; an
                     // unreported one stays open so a redelivery can report it.
-                    if (outcome === 'succeeded' || outcome === 'failed')
+                    if (outcome === 'succeeded' ||
+                        outcome === 'failed' ||
+                        outcome === 'cancelled' ||
+                        outcome === 'resolved') {
                         runs.markCompleted(run.run_id);
+                    }
                 });
             });
             ws.on('close', () => {