@prohost/cli 0.2.3 → 0.3.0

package/dist/agent/prompt.js

@@ -6,6 +6,50 @@
  * a shell script that greps it. The reply contract is stated explicitly so an
  * agent that would otherwise narrate its reasoning returns something sendable.
  */
+/**
+ * Say the run's budget in a unit a reader thinks in.
+ *
+ * Rounded on purpose — the agent is being told roughly how much room it has,
+ * and "about 120 minutes" is the honest shape of a ceiling it will never be
+ * told the exact remainder of.
+ */
+function renderBudget(timeoutMs) {
+    if (timeoutMs >= 90 * 60_000) {
+        const hours = timeoutMs / 3_600_000;
+        return `about ${Number.isInteger(hours) ? hours : hours.toFixed(1)} hours`;
+    }
+    if (timeoutMs >= 60_000)
+        return `about ${Math.round(timeoutMs / 60_000)} minutes`;
+    return `about ${Math.max(1, Math.round(timeoutMs / 1_000))} seconds`;
+}
+/**
+ * The bullet that tells the agent how much time it actually has.
+ *
+ * Both branches are load-bearing. A named ceiling stops an agent from treating
+ * a two-hour budget like a five-minute one (the failure this replaced: a
+ * multi-minute coding task killed at 300s having been told only that a limit
+ * existed). And an *uncapped* run still needs a word about pace, because
+ * nothing is posted until the turn ends — a run that explores forever reads,
+ * from the thread, exactly like one that crashed.
+ */
+function timeBudgetLines(timeoutMs) {
+    if (timeoutMs === undefined)
+        return [];
+    if (timeoutMs <= 0) {
+        return [
+            '- There is no fixed time limit on this turn, so work that genuinely takes',
+            '  a while is fine — finish it rather than cutting it short. But nothing you',
+            '  write is posted until the turn ends, and a long silence gets reported as a',
+            '  stalled run, so keep moving toward an answer rather than exploring',
+            '  indefinitely.',
+        ];
+    }
+    return [
+        `- You have ${renderBudget(timeoutMs)} for this whole turn. Past that you are killed`,
+        '  mid-thought and the person waiting gets nothing, so spend the budget on the',
+        '  work and leave yourself room to write the answer.',
+    ];
+}
 /**
  * Describe one attachment.
  *
@@ -139,6 +183,11 @@ export function buildPrompt(run, context) {
             '  as you call them. Do the work rather than proposing it — but that\n' +
             '  also means a mistaken action happens for real, so check before you\n' +
             '  change or delete anything.', '- Nothing about your access changes after this turn. Do not say a tool is', '  "connecting", "loading", or about to become available, and do not', '  promise to follow up.', 
+    // Stated up here with the other facts about the shape of the invocation,
+    // not buried under the tool advice: how long the agent has changes how it
+    // should approach the work, so it has to be read before any of the "go and
+    // look" bullets below.
+    ...timeBudgetLines(context.timeoutMs), 
     // Claude Code lazy-loads part of its catalog — WebSearch/WebFetch among
     // them — so a run that inspects its loaded tools concludes it cannot
     // browse and says so to a customer, when one ToolSearch call away it
@@ -169,8 +218,7 @@ export function buildPrompt(run, context) {
             '  tell anyone you have no access to some repository, codebase, log, or',
             '  capability, take a QUICK look for it: the usual places a checkout',
             '  lives, `gh auth status`, whether the command is on PATH. A quick',
-            '  look, not an audit — this run has a hard time limit and somebody',
-            '  waiting at the end of it.',
+            '  look, not an audit — there is somebody waiting at the end of this.',
             '- "Looking" means running the command, not inspecting your own tool',
             '  list. Say you have no shell, no file access, or no checkout only',
             '  after an attempt failed and you can quote what it said. Introspection',

package/dist/agent/run.d.ts

@@ -16,12 +16,28 @@
  * * **Runs are deduped on ``run_id``.** Webhook deliveries retry, and each
  *   attempt is mirrored to this socket. Without dedupe a retry would execute
  *   the command a second time and post a second reply.
+ *
+ * Runs can be long — hours — so every run this process has accepted and not yet
+ * reported is heartbeated (see `heartbeat.ts`), including the ones still queued
+ * behind the executing one. That keeps the server's lease alive and is the only
+ * way a stop pressed in the UI reaches a machine that accepts no connections.
  */
 import type { spawn } from 'node:child_process';
 import WebSocket from 'ws';
 import type { AgentCredentials } from './credentials.js';
-/** Default ceiling on one agent invocation. */
-export declare const DEFAULT_EXEC_TIMEOUT_MS = 300000;
+/**
+ * Default ceiling on one agent invocation.
+ *
+ * Two hours, up from five minutes. Five was a guess made when a run meant
+ * "answer a mention", and it was wrong in the expensive direction: a paired
+ * brain given a real coding task in production was killed at 300s, mid-edit,
+ * and reported `timed out after 300000ms` to the person waiting. A coding
+ * agent's unit of work is not a chat reply, and no local clock can tell a
+ * wedged run from a productive one — the server's lease can, which is what
+ * heartbeats are for. This is now a backstop against a genuinely stuck process
+ * rather than a shape imposed on the work; `--timeout 0` removes it entirely.
+ */
+export declare const DEFAULT_EXEC_TIMEOUT_MS = 7200000;
 export interface AgentRunOptions {
     /** Shell command that runs the agent, e.g. ``claude -p``. */
     exec: string;
@@ -30,7 +46,7 @@ export interface AgentRunOptions {
     url?: string;
     /** Machine name shown in the ProhostAI UI. Defaults to the OS hostname. */
     machine?: string;
-    /** Per-run execution timeout. */
+    /** Per-run execution timeout. ``0`` disables it — see `--timeout 0`. */
     timeoutMs?: number;
     /** Print the reply that would be posted and make no writes. */
     dryRun?: boolean;
@@ -73,6 +89,12 @@ export interface AgentRunOptions {
     };
     /** Environment used to locate the run ledger (tests point this at a temp dir). */
     env?: NodeJS.ProcessEnv;
+    /**
+     * How often open runs are heartbeated. Defaults to
+     * {@link HEARTBEAT_INTERVAL_MS}; tests shorten it so a cancellation arrives
+     * inside a test's lifetime.
+     */
+    heartbeatIntervalMs?: number;
 }
 export interface AgentRunMetrics {
     connections: number;

package/dist/agent/run.js

@@ -16,26 +16,43 @@
  * * **Runs are deduped on ``run_id``.** Webhook deliveries retry, and each
  *   attempt is mirrored to this socket. Without dedupe a retry would execute
  *   the command a second time and post a second reply.
+ *
+ * Runs can be long — hours — so every run this process has accepted and not yet
+ * reported is heartbeated (see `heartbeat.ts`), including the ones still queued
+ * behind the executing one. That keeps the server's lease alive and is the only
+ * way a stop pressed in the UI reaches a machine that accepts no connections.
  */
 import os from 'node:os';
 import { spawnSync } from 'node:child_process';
 import WebSocket from 'ws';
 import { completeRun, postReply } from './api.js';
-import { claudeCommand, isClaudeExec, looksLikeResumeFailure, parseClaudeOutput, writeMcpConfig, } from './claude.js';
+import { claudeCommand, isClaudeExec, looksLikeResumeFailure, readClaudeStream, writeMcpConfig, } from './claude.js';
 import { MCP_API_KEY_ENV_VAR, codexBinary, codexCommand, codexMcpInventoryFlag, codexProfileName, codexShellSyntax, disableEntriesThatLoad, extraMcpEntries, isCodexExec, listMcpServers, mcpServersOverride, prohostMcpEntry, looksLikeCodexResumeFailure, parseCodexOutput, } from './codex.js';
 import { EXTRA_MCP_FILENAME, loadExtraMcpServers } from './extras.js';
 import { MCP_SERVER_NAME, MCP_TOOL_PATTERN } from './mcp.js';
-import { EVENT_AGENT_RUN_REQUESTED, EVENT_MENTION_CREATED, EVENT_MESSAGE_TEAM_CHAT, eventData, parseRunRequest, } from './contract.js';
+import { EVENT_AGENT_RUN_REQUESTED, EVENT_MENTION_CREATED, EVENT_MESSAGE_TEAM_CHAT, RUN_ERROR_CANCELLED_BY_USER, eventData, heartbeatPathFor, parseRunRequest, } from './contract.js';
 import { ensureWorkspace, streamUrlFor, workspacePath } from './credentials.js';
 import { execAgent } from './exec.js';
+import { HeartbeatLoop } from './heartbeat.js';
 import { buildPrompt, replyFromStdout } from './prompt.js';
 import { RunStore } from './runstore.js';
 import { SessionStore, sessionKeyFor } from './sessionstore.js';
 import { verifySignature } from './signature.js';
 import { isWebhookEvent } from '../forward.js';
 import { DEFAULT_WS_URL } from '../listen.js';
-/** Default ceiling on one agent invocation. */
-export const DEFAULT_EXEC_TIMEOUT_MS = 300_000;
+/**
+ * Default ceiling on one agent invocation.
+ *
+ * Two hours, up from five minutes. Five was a guess made when a run meant
+ * "answer a mention", and it was wrong in the expensive direction: a paired
+ * brain given a real coding task in production was killed at 300s, mid-edit,
+ * and reported `timed out after 300000ms` to the person waiting. A coding
+ * agent's unit of work is not a chat reply, and no local clock can tell a
+ * wedged run from a productive one — the server's lease can, which is what
+ * heartbeats are for. This is now a backstop against a genuinely stuck process
+ * rather than a shape imposed on the work; `--timeout 0` removes it entirely.
+ */
+export const DEFAULT_EXEC_TIMEOUT_MS = 7_200_000;
 /** Reconnect backoff — full jitter between ``BASE`` and the current cap. */
 const RECONNECT_BASE_DELAY_MS = 1_000;
 const RECONNECT_MAX_DELAY_MS = 30_000;
@@ -66,8 +83,18 @@ function backoffDelay(attempt) {
     const cap = Math.min(RECONNECT_MAX_DELAY_MS, RECONNECT_BASE_DELAY_MS * 2 ** Math.max(0, attempt - 1));
     return Math.floor(Math.random() * cap);
 }
-/** Backoff schedule for retrying a completion the server didn't accept. */
-const COMPLETION_RETRY_DELAYS_MS = [1_000, 3_000, 9_000];
+/**
+ * Backoff schedule for retrying a completion the server didn't accept.
+ *
+ * Roughly ten minutes in total, up from thirteen seconds. The old schedule was
+ * sized for a five-minute run: losing the completion meant losing five minutes
+ * of work, and a laptop that has been offline for thirteen seconds is usually
+ * offline for longer. Now a run can be four hours of an agent's work, which
+ * makes waiting out a wifi drop overwhelmingly the better bet — and the run
+ * keeps being heartbeated for the whole retry window, so the row it is trying
+ * to close stays alive rather than being reaped underneath the retries.
+ */
+const COMPLETION_RETRY_DELAYS_MS = [1_000, 3_000, 9_000, 30_000, 60_000, 120_000, 180_000, 240_000];
 /**
  * Complete a run, retrying transient failures.
  *
@@ -347,35 +374,113 @@ function defaultProbe(command, cwd) {
  * tell a failed turn from a successful one. A plain command's behaviour here is
  * byte-for-byte what it was before this function existed.
  */
-async function runTurn(run, prompt, options, runtime, mcp, log) {
+async function runTurn(run, prompt, options, runtime, mcp, control, log) {
     // `--timeout` is a ceiling on the run, not on each attempt: the retry below
     // must not let one run take twice what the operator allowed. Both attempts
     // share one deadline, which costs the retry nothing in practice — a rejected
     // `--resume` fails in milliseconds, long before any of the budget is spent.
     const budgetMs = options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS;
+    // `--timeout 0`: no local ceiling at all. The run then ends when the agent
+    // ends it, when somebody cancels it, or when the server stops renewing its
+    // lease — all of which are better judges of "stuck" than a fixed clock.
+    const capped = budgetMs > 0;
     const startedAt = Date.now();
     const remainingMs = () => budgetMs - (Date.now() - startedAt);
-    const exec = (command) => execAgent({
-        command,
-        prompt,
-        timeoutMs: remainingMs(),
-        cwd: runtime.workdir,
-        env: mcp.execEnv,
-        spawnImpl: options.spawnImpl,
-    });
+    const timeLeft = () => !capped || remainingMs() > 0;
+    // Which surface's session this turn belongs to, and `undefined` on a dry run
+    // — which is what makes every `remember`/`forget` below a no-op there.
+    // `--dry-run` promises to make no writes, and a resumed turn is a write: both
+    // brains record it in their transcript whether or not we post the reply, so
+    // remembering the session would leave the next *real* run continuing from an
+    // answer nobody saw. Resolved once, up here, because the streaming reader
+    // records it mid-run rather than at the end of the turn.
+    const sessionKey = options.dryRun ? undefined : sessionKeyFor(run);
+    let recordedSession;
+    /** The reader for the attempt currently running, if this brain streams. */
+    let stream;
+    /**
+     * Act on what the stream has said so far.
+     *
+     * The session id is written to the ledger **as soon as it is announced**,
+     * which is a deliberate departure from "record it once the reply is durable".
+     * That rule protects against continuing a session whose answer nobody saw;
+     * it cannot protect a run that is killed at hour two, because that run never
+     * reaches the end where the recording used to happen — and losing the id
+     * loses the only handle on two hours of the agent's context. The rejected-
+     * reply case is handled where it happens instead, by forgetting again.
+     */
+    const observe = (reader) => {
+        const id = reader.sessionId();
+        if (id && id !== recordedSession) {
+            recordedSession = id;
+            runtime.sessions.remember(sessionKey, id);
+        }
+        const activity = reader.activity();
+        if (activity)
+            control.report(activity);
+    };
+    /**
+     * Undo an eagerly recorded Claude session after an ordinary failed turn.
+     *
+     * Timeouts and cancellations deliberately keep their session so the next
+     * delivery can continue a long-running turn whose context would otherwise be
+     * lost. A command or parser failure has no such continuity guarantee: its
+     * answer was never made durable, so resuming it would skip an unseen turn.
+     */
+    const forgetRecordedSession = () => {
+        if (!recordedSession)
+            return;
+        runtime.sessions.forget(sessionKey);
+        recordedSession = undefined;
+    };
+    const exec = (command) => {
+        // A fresh reader per attempt: a retry without `--resume` is a different
+        // session, and inheriting the failed attempt's id would resume nothing.
+        const reader = runtime.brain === 'claude' ? readClaudeStream() : undefined;
+        stream = reader;
+        return execAgent({
+            command,
+            prompt,
+            timeoutMs: capped ? Math.max(1, remainingMs()) : 0,
+            cwd: runtime.workdir,
+            env: mcp.execEnv,
+            spawnImpl: options.spawnImpl,
+            cancelSignal: control.signal,
+            onStdout: reader
+                ? (chunk) => {
+                    reader.push(chunk);
+                    observe(reader);
+                }
+                : undefined,
+        });
+    };
     // Wall clock for the whole turn, retry included — what the operator waited,
     // rather than what the last attempt took.
     const elapsed = () => Date.now() - startedAt;
     if (mcp.error) {
         return { ok: false, reason: 'MCP isolation could not be established', detail: mcp.error, durationMs: 0 };
     }
-    const failure = (result) => ({
-        ok: false,
-        reason: result.error ??
-            (result.exitCode === null ? 'agent command did not run' : `agent command exited ${result.exitCode}`),
-        detail: result.stderr.trim().slice(-500),
-        durationMs: elapsed(),
-    });
+    const failure = (result) => {
+        // A killed run has no reply, but it usually has most of one. Reporting the
+        // tail of what the agent was writing turns "timed out after 7200000ms" into
+        // something the person waiting can act on — and it is the only trace left
+        // of however many hours of work.
+        const salvaged = result.timedOut || result.cancelled ? stream?.salvage() : undefined;
+        // The stderr tail yields room when there is a salvaged answer to carry: the
+        // completion `error` is capped server-side, and of the two, what the agent
+        // was saying beats the last 500 characters of its logging.
+        const stderrTail = result.stderr.trim().slice(salvaged ? -200 : -500);
+        const detail = [stderrTail, salvaged ? `unfinished answer: ${salvaged}` : '']
+            .filter((part) => part.length > 0)
+            .join('\n');
+        return {
+            ok: false,
+            reason: result.error ??
+                (result.exitCode === null ? 'agent command did not run' : `agent command exited ${result.exitCode}`),
+            detail,
+            durationMs: elapsed(),
+        };
+    };
     if (runtime.brain === 'plain') {
         const result = await exec(options.exec);
         if (!result.ok)
@@ -385,12 +490,7 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
             ? { ok: true, reply, durationMs: elapsed() }
             : { ok: false, reason: 'agent produced no output', durationMs: elapsed() };
     }
-    // No session bookkeeping on a dry run. `--dry-run` promises to make no
-    // writes, and a resumed turn is a write: both brains record it in their
-    // transcript whether or not we post the reply, so remembering the session
-    // would leave the next *real* run continuing from an answer nobody saw.
-    const key = options.dryRun ? undefined : sessionKeyFor(run);
-    const sessionId = runtime.sessions.get(key);
+    const sessionId = runtime.sessions.get(sessionKey);
     const codex = runtime.brain === 'codex';
     // Same three questions for both brains — resume this thread, wire these
     // tools, gate or don't — asked with each one's own flags.
@@ -416,13 +516,19 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
         // Skipped rather than squeezed into a token budget: a retry with no time
         // left would be killed on the spot and reported as a timeout, which is a
         // worse error message than the resume failure it replaced.
-        remainingMs() > 0) {
+        timeLeft() &&
+        // A run somebody just stopped does not get started again on its way out.
+        !control.signal.aborted) {
         log(`    … session ${sessionId.slice(0, 12)} could not be resumed; starting a fresh one`);
-        runtime.sessions.forget(key);
+        runtime.sessions.forget(sessionKey);
+        recordedSession = undefined;
         result = await exec(build());
     }
-    if (!result.ok)
+    if (!result.ok) {
+        if (!result.timedOut && !result.cancelled)
+            forgetRecordedSession();
         return failure(result);
+    }
     // A denied tool is not an error either brain reports — the turn succeeds and
     // the reply just quietly lacks whatever the tool would have found. Printing
     // it is the difference between "the agent seems dumb" and "the agent asked
@@ -445,9 +551,15 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
         }
         return { ok: true, reply: output.text, durationMs: elapsed(), sessionId: output.sessionId };
     }
-    const parsed = parseClaudeOutput(result.stdout);
+    // The reply comes out of the stream's own `result` event, so a run whose
+    // stdout we watched arrives already parsed. `finish` still gets the captured
+    // stdout: a command that emitted no result event at all — a login prompt, a
+    // crash banner, a `claude` that ignored `--output-format` — is a failure, and
+    // the whole-output parser is what says so in words.
+    const parsed = (stream ?? readClaudeStream()).finish(result.stdout);
     reportDenials(parsed.denials);
     if (!parsed.ok) {
+        forgetRecordedSession();
         return {
             ok: false,
             reason: 'the agent did not produce a usable reply',
@@ -455,7 +567,13 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
             durationMs: elapsed(),
         };
     }
-    return { ok: true, reply: parsed.text, durationMs: elapsed(), sessionId: parsed.sessionId };
+    return {
+        ok: true,
+        reply: parsed.text,
+        durationMs: elapsed(),
+        sessionId: parsed.sessionId,
+        sessionRecorded: recordedSession !== undefined,
+    };
 }
 /**
  * Handle one verified run end-to-end.
@@ -463,7 +581,7 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
  * Every path except a dry run — including every failure — calls the completion
  * endpoint before returning.
  */
-async function handleRun(run, options, runtime, api, log, sleep) {
+async function handleRun(run, options, runtime, api, log, sleep, control) {
     const label = `run ${run.run_id.slice(0, 12)}`;
     const surface = run.conversation_id ?? run.kind ?? 'no conversation';
     log(`${timestamp()} ▶ ${label} started (${surface})`);
@@ -480,9 +598,37 @@ async function handleRun(run, options, runtime, api, log, sleep) {
         // it is. See PromptContext.mcpToolPattern.
         mcpToolPattern: runtime.brain !== 'plain' && mcp.mcpWired ? MCP_TOOL_PATTERN : undefined,
         toolsGated: !runtime.skipPermissions,
+        // The real number, not a vague "there is a limit" — see PromptContext.
+        timeoutMs: options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS,
     });
-    const result = await runTurn(run, prompt, options, runtime, mcp, log);
+    const result = await runTurn(run, prompt, options, runtime, mcp, control, log);
     const seconds = (result.durationMs / 1000).toFixed(1);
+    // Stopped from outside. Checked before the failure branch below, because a
+    // stopped run *is* a failed command — reporting it as one would put "timed
+    // out"-shaped noise in a thread where somebody deliberately pressed stop.
+    const stopped = control.disposition();
+    if (stopped === 'resolved') {
+        // The server already closed this run: reaped after its lease lapsed, or
+        // resolved by something else. There is nothing left to post a reply to and
+        // nothing left to complete — a completion would just 409.
+        log(`${timestamp()} ⏹ ${label} was already closed server-side (${seconds}s) — nothing was posted`);
+        return 'resolved';
+    }
+    if (stopped === 'cancelled') {
+        log(`${timestamp()} ⏹ ${label} cancelled after ${seconds}s`);
+        if (options.dryRun) {
+            log('    [dry-run] would report the run as cancelled');
+            return 'skipped';
+        }
+        // The exact marker string, alone: the server words the thread notice from
+        // it, and anything wrapped around it reads as a crash instead of a stop.
+        const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
+        if (!closed.ok) {
+            log(`    ! could not report the cancellation (HTTP ${closed.status}) ${closed.error ?? ''}`);
+            return 'unreported';
+        }
+        return 'cancelled';
+    }
     if (!result.ok) {
         const { reason } = result;
         const detail = result.detail ?? '';
@@ -533,6 +679,12 @@ async function handleRun(run, options, runtime, api, log, sleep) {
     const posted = await postReply(api, run.reply_path, run.reply_body_key, reply, run.run_id, run.reply_body_extra);
     if (!posted.ok) {
         log(`${timestamp()} ✗ ${label} reply rejected (HTTP ${posted.status}) ${posted.error ?? ''}`);
+        // Undo the mid-run recording. The session exists and the agent's transcript
+        // holds this turn, but the conversation never received its answer, so
+        // continuing it would skip reasoning nobody can see — the same rule that
+        // used to be expressed by simply not recording until the reply landed.
+        if (result.sessionRecorded)
+            runtime.sessions.forget(sessionKeyFor(run));
         const reported = await completeWithRetry(api, run.completion_path, { ok: false, error: `posting the reply failed with HTTP ${posted.status}` }, log, sleep);
         return reported.ok ? 'failed' : 'unreported';
     }
@@ -588,9 +740,79 @@ export async function runAgentHarness(options) {
     // duplicate — the run is in flight right here. Only a run in the ledger that
     // this process never handled is a candidate for restart recovery.
     const handledThisSession = new Set();
+    /** Accepted, not yet reported — the executing run and everything queued. */
+    const accepted = new Map();
+    const heartbeats = new HeartbeatLoop({ api, log, intervalMs: options.heartbeatIntervalMs });
     let queue = Promise.resolve();
     let fatal;
     let attempt = 0;
+    /** Stop tracking a run — it has reached a terminal state, one way or another. */
+    const release = (runId) => {
+        accepted.delete(runId);
+        heartbeats.untrack(runId);
+    };
+    /**
+     * Work started from a heartbeat that the harness must still wait for on the
+     * way out, but that a heartbeat tick must not wait for.
+     *
+     * Closing out a cancelled queued run can take the whole completion-retry
+     * schedule — around ten minutes against an unreachable server — and beats do
+     * not overlap. Awaited inside the tick, one unreachable completion would stop
+     * every other run being heartbeated, and the four-hour job it was queued
+     * behind would be reaped for going quiet.
+     */
+    const background = [];
+    /**
+     * Somebody pressed stop in ProhostAI.
+     *
+     * The two cases are genuinely different work. An executing run reports
+     * itself: the abort reaches `execAgent`, which signals the process group the
+     * same way a timeout does, and `handleRun` completes it with the marker
+     * error. A run still in the queue has nobody to report it — it will never
+     * execute, and the queue is about to skip it — so it is closed out here.
+     */
+    const cancelAccepted = async (entry) => {
+        if (entry.disposition)
+            return;
+        entry.disposition = 'cancelled';
+        if (entry.executing) {
+            entry.controller.abort();
+            return;
+        }
+        release(entry.run.run_id);
+        log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)} was cancelled before it started`);
+        // `--dry-run` promises to make no writes, and a completion is the most
+        // terminal write there is. The executing-run path defers to the same check
+        // inside `handleRun`; this one has to make it itself, because a queued run
+        // never reaches `handleRun` at all.
+        if (options.dryRun) {
+            log('    [dry-run] would report the run as cancelled');
+            return;
+        }
+        const closed = await completeWithRetry(api, entry.run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
+        if (closed.ok)
+            runs.markCompleted(entry.run.run_id);
+        metrics.runsFailed += 1;
+    };
+    /**
+     * The server has already closed this run out — it was reaped, or resolved by
+     * something else. Either way nothing this machine does with it can land, so
+     * the only correct move is to stop: kill the command if one is running, and
+     * post nothing at all. A completion here would be a 409 by definition.
+     */
+    const resolveAccepted = (entry) => {
+        if (entry.disposition)
+            return;
+        entry.disposition = 'resolved';
+        if (entry.executing) {
+            entry.controller.abort();
+            return;
+        }
+        release(entry.run.run_id);
+        // Terminal server-side, so a redelivery must not start it either.
+        runs.markCompleted(entry.run.run_id);
+        log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)} was closed server-side before it started`);
+    };
     const exhausted = () => options.maxConnections !== undefined && metrics.connections >= options.maxConnections;
     while (!options.signal?.aborted && fatal === undefined && !exhausted()) {
         metrics.connections += 1;
@@ -604,8 +826,14 @@ export async function runAgentHarness(options) {
         log(`${timestamp()} … disconnected; reconnecting in ${(delay / 1000).toFixed(1)}s`);
         await sleep(delay);
     }
-    // Let an in-flight run finish reporting before we hand back metrics.
+    // Let an in-flight run finish reporting before we hand back metrics — and
+    // any cancellation the heartbeat loop started and deliberately did not wait
+    // for, so the harness never exits with a run half-closed.
     await queue;
+    // Stopped first, so no further tick can start work after the wait below has
+    // taken its snapshot of `background`.
+    heartbeats.stop();
+    await Promise.all(background);
     if (fatal !== undefined) {
         log(`${timestamp()} ✗ ${fatal}`);
         metrics.fatal = fatal;
@@ -753,13 +981,36 @@ export async function runAgentHarness(options) {
                 // redelivery execute the agent a second time.
                 runs.markStarted(run.run_id, run.completion_path);
                 metrics.runsStarted += 1;
+                // Heartbeat from the moment it is accepted, not from the moment it
+                // starts. The row is `running` server-side either way, and a run that
+                // waits an hour behind a long job is reaped exactly like one that has
+                // been working for an hour.
+                const entry = { run, controller: new AbortController(), executing: false };
+                accepted.set(run.run_id, entry);
+                heartbeats.track(run.run_id, {
+                    path: heartbeatPathFor(run),
+                    progress: () => entry.progress,
+                    // Deliberately not awaited here — see `background`.
+                    onCancelRequested: () => {
+                        background.push(cancelAccepted(entry).catch((err) => {
+                            log(`${timestamp()} ✗ closing cancelled run ${run.run_id} failed: ${String(err)}`);
+                        }));
+                    },
+                    onResolved: () => resolveAccepted(entry),
+                });
                 // Serialize execution but never block the socket reader: the ack and
                 // the heartbeat have to keep flowing while the agent thinks.
                 queue = queue.then(async () => {
+                    // Settled while it waited — cancelled, or closed server-side. Both
+                    // were dealt with where they were decided; there is nothing to run
+                    // and nothing left to report.
+                    if (entry.disposition)
+                        return;
                     // Shutdown was requested while this sat in the queue. Don't start a
                     // fresh command the operator is waiting to be rid of — but do close
                     // the run out, so the server isn't left waiting on it either.
                     if (options.signal?.aborted) {
+                        release(run.run_id);
                         log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)} dropped — harness is shutting down`);
                         const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: 'the harness shut down before this run started' }, log, sleep);
                         if (closed.ok)
@@ -767,20 +1018,42 @@ export async function runAgentHarness(options) {
                         metrics.runsFailed += 1;
                         return;
                     }
-                    const outcome = await handleRun(run, options, runtime, api, log, sleep).catch((err) => {
+                    // Set before the first `await`, so a cancellation arriving from the
+                    // heartbeat either sees a queued run (and closes it out itself) or an
+                    // executing one (and aborts it) — never both, and never neither.
+                    entry.executing = true;
+                    const control = {
+                        signal: entry.controller.signal,
+                        disposition: () => entry.disposition,
+                        report: (activity) => {
+                            entry.progress = activity;
+                        },
+                    };
+                    const outcome = await handleRun(run, options, runtime, api, log, sleep, control).catch((err) => {
                         log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
                         return 'failed';
                     });
+                    // Held until here on purpose: heartbeats keep the row alive for the
+                    // whole of `handleRun`, including the ten minutes of completion
+                    // retries at the end of it.
+                    release(run.run_id);
                     // Both terminal outcomes reached the completion endpoint; only then
                     // is the run settled and safe to skip on redelivery.
                     if (outcome === 'succeeded')
                         metrics.runsSucceeded += 1;
-                    if (outcome === 'failed' || outcome === 'unreported')
+                    // A cancellation is reported to the server as `failed`, and counted
+                    // the same way here — the run did not produce the answer it was for.
+                    if (outcome === 'failed' || outcome === 'unreported' || outcome === 'cancelled') {
                         metrics.runsFailed += 1;
+                    }
                     // Only a run the server acknowledged as terminal is settled; an
                     // unreported one stays open so a redelivery can report it.
-                    if (outcome === 'succeeded' || outcome === 'failed')
+                    if (outcome === 'succeeded' ||
+                        outcome === 'failed' ||
+                        outcome === 'cancelled' ||
+                        outcome === 'resolved') {
                         runs.markCompleted(run.run_id);
+                    }
                 });
             });
             ws.on('close', () => {

package/dist/index.js

@@ -43,7 +43,9 @@ function printHelp() {
         '',
         'agent run flags:',
         '  --exec           Command to run for each mention; the prompt arrives on stdin (required)',
-        '  --timeout        Seconds one run may take before it is killed. Defaults to 300.',
+        '  --timeout        Seconds one run may take before it is killed. Defaults to 7200 (2 hours).',
+        '                   Pass 0 for no local limit — the run then ends when the agent finishes,',
+        '                   when someone cancels it in ProhostAI, or when the server reaps it.',
         '  --dry-run        Print the reply that would be posted; write nothing.',
         '  --workdir        Working directory for the agent command. Defaults to $PROHOST_HOME/workspace.',
         '                   Avoid pointing it at a code checkout: agents read CLAUDE.md, .claude/settings.json',

package/dist/version.d.ts

@@ -7,6 +7,6 @@
  * `package.json`, so a release bump that forgets this file fails the suite
  * instead of shipping a `User-Agent` that lies about which build is calling.
  */
-export declare const CLI_VERSION = "0.2.3";
+export declare const CLI_VERSION = "0.3.0";
 /** Sent on every HTTP request the CLI makes back into ProhostAI. */
-export declare const USER_AGENT = "prohost-cli/0.2.3";
+export declare const USER_AGENT = "prohost-cli/0.3.0";