@prohost/cli 0.2.3 → 0.4.0

package/dist/agent/run.js

@@ -16,26 +16,61 @@
  * * **Runs are deduped on ``run_id``.** Webhook deliveries retry, and each
  *   attempt is mirrored to this socket. Without dedupe a retry would execute
  *   the command a second time and post a second reply.
+ *
+ * Runs can be long — hours — so every run this process has accepted and not yet
+ * reported is heartbeated (see `heartbeat.ts`), including the ones still queued
+ * behind the executing one. That keeps the server's lease alive and is the only
+ * way a stop pressed in the UI reaches a machine that accepts no connections.
  */
 import os from 'node:os';
 import { spawnSync } from 'node:child_process';
 import WebSocket from 'ws';
 import { completeRun, postReply } from './api.js';
-import { claudeCommand, isClaudeExec, looksLikeResumeFailure, parseClaudeOutput, writeMcpConfig, } from './claude.js';
+import { MAX_SALVAGE_CHARS, claudeCommand, isClaudeExec, looksLikeResumeFailure, readClaudeStream, writeMcpConfig, } from './claude.js';
 import { MCP_API_KEY_ENV_VAR, codexBinary, codexCommand, codexMcpInventoryFlag, codexProfileName, codexShellSyntax, disableEntriesThatLoad, extraMcpEntries, isCodexExec, listMcpServers, mcpServersOverride, prohostMcpEntry, looksLikeCodexResumeFailure, parseCodexOutput, } from './codex.js';
 import { EXTRA_MCP_FILENAME, loadExtraMcpServers } from './extras.js';
 import { MCP_SERVER_NAME, MCP_TOOL_PATTERN } from './mcp.js';
-import { EVENT_AGENT_RUN_REQUESTED, EVENT_MENTION_CREATED, EVENT_MESSAGE_TEAM_CHAT, eventData, parseRunRequest, } from './contract.js';
+import { EVENT_AGENT_RUN_REQUESTED, EVENT_MENTION_CREATED, EVENT_MESSAGE_TEAM_CHAT, RUN_ERROR_CANCELLED_BY_USER, eventData, heartbeatPathFor, parseRunRequest, } from './contract.js';
 import { ensureWorkspace, streamUrlFor, workspacePath } from './credentials.js';
 import { execAgent } from './exec.js';
+import { HeartbeatLoop } from './heartbeat.js';
 import { buildPrompt, replyFromStdout } from './prompt.js';
 import { RunStore } from './runstore.js';
 import { SessionStore, sessionKeyFor } from './sessionstore.js';
 import { verifySignature } from './signature.js';
 import { isWebhookEvent } from '../forward.js';
 import { DEFAULT_WS_URL } from '../listen.js';
-/** Default ceiling on one agent invocation. */
-export const DEFAULT_EXEC_TIMEOUT_MS = 300_000;
+/**
+ * Default ceiling on one agent invocation: none.
+ *
+ * The wall clock kept being raised — five minutes, then two hours — because it
+ * was answering the wrong question. "How long has this run been going?" does
+ * not distinguish a wedged child from a productive one; it only distinguishes
+ * long work from short, and killing long work is precisely the failure the two
+ * previous defaults each produced in production. What the harness can actually
+ * observe is whether the child is *doing* anything, which is what
+ * {@link DEFAULT_IDLE_TIMEOUT_MS} watches.
+ *
+ * So a run is now bounded by three things, none of them a fixed duration:
+ * the inactivity watchdog (default 15 minutes of total silence), the server's
+ * lease on the run (heartbeats stop, the row is reaped), and the Stop button in
+ * ProhostAI (which reaches this machine on the next heartbeat). `--timeout`
+ * remains fully supported for anyone who wants a hard ceiling as well.
+ */
+export const DEFAULT_EXEC_TIMEOUT_MS = 0;
+/**
+ * Default silence window before a run is treated as stalled.
+ *
+ * Fifteen minutes is chosen against what real work looks like, not against
+ * what feels responsive: an agent mid-task emits stream events, tool calls and
+ * log lines continuously, so the gaps in a healthy run are seconds. The long
+ * tail that legitimately goes quiet is a single slow tool call — a test suite,
+ * a build, a big fetch — and fifteen minutes clears those comfortably while
+ * still catching the failure this exists for, a child wedged on a stdin prompt
+ * that heartbeats look perfectly healthy for and that burns the operator's
+ * tokens and their agent's availability indefinitely.
+ */
+export const DEFAULT_IDLE_TIMEOUT_MS = 900_000;
 /** Reconnect backoff — full jitter between ``BASE`` and the current cap. */
 const RECONNECT_BASE_DELAY_MS = 1_000;
 const RECONNECT_MAX_DELAY_MS = 30_000;
@@ -66,8 +101,18 @@ function backoffDelay(attempt) {
     const cap = Math.min(RECONNECT_MAX_DELAY_MS, RECONNECT_BASE_DELAY_MS * 2 ** Math.max(0, attempt - 1));
     return Math.floor(Math.random() * cap);
 }
-/** Backoff schedule for retrying a completion the server didn't accept. */
-const COMPLETION_RETRY_DELAYS_MS = [1_000, 3_000, 9_000];
+/**
+ * Backoff schedule for retrying a completion the server didn't accept.
+ *
+ * Roughly ten minutes in total, up from thirteen seconds. The old schedule was
+ * sized for a five-minute run: losing the completion meant losing five minutes
+ * of work, and a laptop that has been offline for thirteen seconds is usually
+ * offline for longer. Now a run can be four hours of an agent's work, which
+ * makes waiting out a wifi drop overwhelmingly the better bet — and the run
+ * keeps being heartbeated for the whole retry window, so the row it is trying
+ * to close stays alive rather than being reaped underneath the retries.
+ */
+const COMPLETION_RETRY_DELAYS_MS = [1_000, 3_000, 9_000, 30_000, 60_000, 120_000, 180_000, 240_000];
 /**
  * Complete a run, retrying transient failures.
  *
@@ -338,6 +383,25 @@ function defaultProbe(command, cwd) {
     });
     return { status: result.status, stdout: result.stdout ?? '' };
 }
+/**
+ * What a killed plain command salvages: the last thing it printed.
+ *
+ * The streaming reader salvages the last thing the *agent* said, having parsed
+ * it out of the event stream. There is no parsing to do for a plain `--exec`
+ * command — its stdout is its reply — so its rolling tail is the same evidence
+ * reached the other way: what the run had produced before it stopped producing
+ * anything.
+ *
+ * Bounded to the same {@link MAX_SALVAGE_CHARS} for the same reason: the
+ * completion `error` this ends up in is capped server-side, and it has a stderr
+ * tail to carry as well.
+ */
+function partialStdout(tail) {
+    const text = tail.trim();
+    if (!text)
+        return undefined;
+    return text.length > MAX_SALVAGE_CHARS ? text.slice(-MAX_SALVAGE_CHARS) : text;
+}
 /**
  * Run the agent command once and turn its output into a reply.
  *
@@ -347,35 +411,151 @@ function defaultProbe(command, cwd) {
  * tell a failed turn from a successful one. A plain command's behaviour here is
  * byte-for-byte what it was before this function existed.
  */
-async function runTurn(run, prompt, options, runtime, mcp, log) {
+async function runTurn(run, prompt, options, runtime, mcp, control, log) {
     // `--timeout` is a ceiling on the run, not on each attempt: the retry below
     // must not let one run take twice what the operator allowed. Both attempts
     // share one deadline, which costs the retry nothing in practice — a rejected
     // `--resume` fails in milliseconds, long before any of the budget is spent.
     const budgetMs = options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS;
+    // `--timeout 0`, and the default: no wall-clock ceiling at all. The run then
+    // ends when the agent ends it, when the watchdog below finds it silent, when
+    // somebody cancels it, or when the server stops renewing its lease — all of
+    // which are better judges of "stuck" than a fixed clock.
+    const capped = budgetMs > 0;
+    // Not shared between attempts the way the wall clock is: the watchdog bounds
+    // silence, and a fresh attempt starts a fresh silence.
+    const idleMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
     const startedAt = Date.now();
     const remainingMs = () => budgetMs - (Date.now() - startedAt);
-    const exec = (command) => execAgent({
-        command,
-        prompt,
-        timeoutMs: remainingMs(),
-        cwd: runtime.workdir,
-        env: mcp.execEnv,
-        spawnImpl: options.spawnImpl,
-    });
+    const timeLeft = () => !capped || remainingMs() > 0;
+    // Which surface's session this turn belongs to, and `undefined` on a dry run
+    // — which is what makes every `remember`/`forget` below a no-op there.
+    // `--dry-run` promises to make no writes, and a resumed turn is a write: both
+    // brains record it in their transcript whether or not we post the reply, so
+    // remembering the session would leave the next *real* run continuing from an
+    // answer nobody saw. Resolved once, up here, because the streaming reader
+    // records it mid-run rather than at the end of the turn.
+    const sessionKey = options.dryRun ? undefined : sessionKeyFor(run);
+    let recordedSession;
+    /** The reader for the attempt currently running, if this brain streams. */
+    let stream;
+    /**
+     * Rolling tail of a plain command's stdout — the last thing it printed.
+     *
+     * Kept as the output arrives rather than sliced off the captured `stdout` at
+     * the end, because capture is bounded at {@link MAX_OUTPUT_CHARS} from the
+     * *front*: a chatty command that overran that and then stalled would
+     * otherwise salvage half a megabyte of stale opening output and call it the
+     * unfinished answer.
+     */
+    let tail = '';
+    /**
+     * Act on what the stream has said so far.
+     *
+     * The session id is written to the ledger **as soon as it is announced**,
+     * which is a deliberate departure from "record it once the reply is durable".
+     * That rule protects against continuing a session whose answer nobody saw;
+     * it cannot protect a run that is killed at hour two, because that run never
+     * reaches the end where the recording used to happen — and losing the id
+     * loses the only handle on two hours of the agent's context. The rejected-
+     * reply case is handled where it happens instead, by forgetting again.
+     */
+    const observe = (reader) => {
+        const id = reader.sessionId();
+        if (id && id !== recordedSession) {
+            recordedSession = id;
+            runtime.sessions.remember(sessionKey, id);
+        }
+        const activity = reader.activity();
+        if (activity)
+            control.report(activity);
+    };
+    /**
+     * Undo an eagerly recorded Claude session after an ordinary failed turn.
+     *
+     * Timeouts and cancellations deliberately keep their session so the next
+     * delivery can continue a long-running turn whose context would otherwise be
+     * lost. A command or parser failure has no such continuity guarantee: its
+     * answer was never made durable, so resuming it would skip an unseen turn.
+     */
+    const forgetRecordedSession = () => {
+        if (!recordedSession)
+            return;
+        runtime.sessions.forget(sessionKey);
+        recordedSession = undefined;
+    };
+    const exec = (command) => {
+        // A fresh reader per attempt: a retry without `--resume` is a different
+        // session, and inheriting the failed attempt's id would resume nothing.
+        const reader = runtime.brain === 'claude' ? readClaudeStream() : undefined;
+        stream = reader;
+        tail = '';
+        return execAgent({
+            command,
+            prompt,
+            timeoutMs: capped ? Math.max(1, remainingMs()) : 0,
+            idleTimeoutMs: idleMs,
+            cwd: runtime.workdir,
+            env: mcp.execEnv,
+            spawnImpl: options.spawnImpl,
+            cancelSignal: control.signal,
+            onStdout: reader
+                ? (chunk) => {
+                    reader.push(chunk);
+                    observe(reader);
+                }
+                : runtime.brain === 'plain'
+                    ? (chunk) => {
+                        tail = (tail + chunk).slice(-MAX_SALVAGE_CHARS);
+                    }
+                    : undefined,
+        });
+    };
     // Wall clock for the whole turn, retry included — what the operator waited,
     // rather than what the last attempt took.
     const elapsed = () => Date.now() - startedAt;
     if (mcp.error) {
         return { ok: false, reason: 'MCP isolation could not be established', detail: mcp.error, durationMs: 0 };
     }
-    const failure = (result) => ({
-        ok: false,
-        reason: result.error ??
-            (result.exitCode === null ? 'agent command did not run' : `agent command exited ${result.exitCode}`),
-        detail: result.stderr.trim().slice(-500),
-        durationMs: elapsed(),
-    });
+    const failure = (result) => {
+        // A killed run has no reply, but it usually has most of one. Reporting the
+        // tail of what the agent was writing turns "no output for 15 minutes" into
+        // something the person waiting can act on — and it is the only trace left
+        // of however many hours of work. A stalled run is the case this matters
+        // most for: the last thing it said is the best clue to what it stalled on.
+        // A plain command has no stream to salvage a parsed answer from, but its
+        // stdout *is* its answer by contract — so the tail of what it managed to
+        // print is the same evidence, reached the other way. Without this a stalled
+        // plain run reports nothing but the fact that it stalled.
+        //
+        // Deliberately NOT extended to the shaped brains. Both are run with
+        // structured output (`--output-format stream-json`, `--json`), whose
+        // stream carries the agent's reasoning and the raw output of every local
+        // command and tool call it made — which `readClaudeStream`/
+        // `parseCodexOutput` exist to filter out. This string is persisted in the
+        // completion and echoed into a workspace thread, so tailing that stream
+        // raw would publish local shell output, and whatever was in it, to
+        // everyone who can read the conversation. A shaped brain whose parser
+        // salvaged nothing salvages nothing.
+        const killed = result.timedOut || result.idleTimedOut || result.cancelled;
+        const salvaged = killed
+            ? (stream?.salvage() ?? (runtime.brain === 'plain' ? partialStdout(tail) : undefined))
+            : undefined;
+        // The stderr tail yields room when there is a salvaged answer to carry: the
+        // completion `error` is capped server-side, and of the two, what the agent
+        // was saying beats the last 500 characters of its logging.
+        const stderrTail = result.stderr.trim().slice(salvaged ? -200 : -500);
+        const detail = [stderrTail, salvaged ? `unfinished answer: ${salvaged}` : '']
+            .filter((part) => part.length > 0)
+            .join('\n');
+        return {
+            ok: false,
+            reason: result.error ??
+                (result.exitCode === null ? 'agent command did not run' : `agent command exited ${result.exitCode}`),
+            detail,
+            durationMs: elapsed(),
+        };
+    };
     if (runtime.brain === 'plain') {
         const result = await exec(options.exec);
         if (!result.ok)
@@ -385,12 +565,7 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
             ? { ok: true, reply, durationMs: elapsed() }
             : { ok: false, reason: 'agent produced no output', durationMs: elapsed() };
     }
-    // No session bookkeeping on a dry run. `--dry-run` promises to make no
-    // writes, and a resumed turn is a write: both brains record it in their
-    // transcript whether or not we post the reply, so remembering the session
-    // would leave the next *real* run continuing from an answer nobody saw.
-    const key = options.dryRun ? undefined : sessionKeyFor(run);
-    const sessionId = runtime.sessions.get(key);
+    const sessionId = runtime.sessions.get(sessionKey);
     const codex = runtime.brain === 'codex';
     // Same three questions for both brains — resume this thread, wire these
     // tools, gate or don't — asked with each one's own flags.
@@ -416,13 +591,23 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
         // Skipped rather than squeezed into a token budget: a retry with no time
         // left would be killed on the spot and reported as a timeout, which is a
         // worse error message than the resume failure it replaced.
-        remainingMs() > 0) {
+        timeLeft() &&
+        // A run somebody just stopped does not get started again on its way out.
+        !control.signal.aborted) {
         log(`    … session ${sessionId.slice(0, 12)} could not be resumed; starting a fresh one`);
-        runtime.sessions.forget(key);
+        runtime.sessions.forget(sessionKey);
+        recordedSession = undefined;
         result = await exec(build());
     }
-    if (!result.ok)
+    if (!result.ok) {
+        // A run we killed keeps its session, whichever clock killed it: the
+        // transcript is real, it holds however long the agent worked, and the next
+        // delivery on that surface can resume it. Only a turn that failed on its
+        // own — a crash, an unparseable answer — has no continuity worth keeping.
+        if (!result.timedOut && !result.idleTimedOut && !result.cancelled)
+            forgetRecordedSession();
         return failure(result);
+    }
     // A denied tool is not an error either brain reports — the turn succeeds and
     // the reply just quietly lacks whatever the tool would have found. Printing
     // it is the difference between "the agent seems dumb" and "the agent asked
@@ -445,9 +630,15 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
         }
         return { ok: true, reply: output.text, durationMs: elapsed(), sessionId: output.sessionId };
     }
-    const parsed = parseClaudeOutput(result.stdout);
+    // The reply comes out of the stream's own `result` event, so a run whose
+    // stdout we watched arrives already parsed. `finish` still gets the captured
+    // stdout: a command that emitted no result event at all — a login prompt, a
+    // crash banner, a `claude` that ignored `--output-format` — is a failure, and
+    // the whole-output parser is what says so in words.
+    const parsed = (stream ?? readClaudeStream()).finish(result.stdout);
     reportDenials(parsed.denials);
     if (!parsed.ok) {
+        forgetRecordedSession();
         return {
             ok: false,
             reason: 'the agent did not produce a usable reply',
@@ -455,7 +646,13 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
             durationMs: elapsed(),
         };
     }
-    return { ok: true, reply: parsed.text, durationMs: elapsed(), sessionId: parsed.sessionId };
+    return {
+        ok: true,
+        reply: parsed.text,
+        durationMs: elapsed(),
+        sessionId: parsed.sessionId,
+        sessionRecorded: recordedSession !== undefined,
+    };
 }
 /**
  * Handle one verified run end-to-end.
@@ -463,7 +660,7 @@ async function runTurn(run, prompt, options, runtime, mcp, log) {
  * Every path except a dry run — including every failure — calls the completion
  * endpoint before returning.
  */
-async function handleRun(run, options, runtime, api, log, sleep) {
+async function handleRun(run, options, runtime, api, log, sleep, control) {
     const label = `run ${run.run_id.slice(0, 12)}`;
     const surface = run.conversation_id ?? run.kind ?? 'no conversation';
     log(`${timestamp()} ▶ ${label} started (${surface})`);
@@ -480,9 +677,38 @@ async function handleRun(run, options, runtime, api, log, sleep) {
         // it is. See PromptContext.mcpToolPattern.
         mcpToolPattern: runtime.brain !== 'plain' && mcp.mcpWired ? MCP_TOOL_PATTERN : undefined,
         toolsGated: !runtime.skipPermissions,
+        // The real numbers, not a vague "there is a limit" — see PromptContext.
+        timeoutMs: options.timeoutMs ?? DEFAULT_EXEC_TIMEOUT_MS,
+        idleTimeoutMs: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS,
     });
-    const result = await runTurn(run, prompt, options, runtime, mcp, log);
+    const result = await runTurn(run, prompt, options, runtime, mcp, control, log);
     const seconds = (result.durationMs / 1000).toFixed(1);
+    // Stopped from outside. Checked before the failure branch below, because a
+    // stopped run *is* a failed command — reporting it as one would put "timed
+    // out"-shaped noise in a thread where somebody deliberately pressed stop.
+    const stopped = control.disposition();
+    if (stopped === 'resolved') {
+        // The server already closed this run: reaped after its lease lapsed, or
+        // resolved by something else. There is nothing left to post a reply to and
+        // nothing left to complete — a completion would just 409.
+        log(`${timestamp()} ⏹ ${label} was already closed server-side (${seconds}s) — nothing was posted`);
+        return 'resolved';
+    }
+    if (stopped === 'cancelled') {
+        log(`${timestamp()} ⏹ ${label} cancelled after ${seconds}s`);
+        if (options.dryRun) {
+            log('    [dry-run] would report the run as cancelled');
+            return 'skipped';
+        }
+        // The exact marker string, alone: the server words the thread notice from
+        // it, and anything wrapped around it reads as a crash instead of a stop.
+        const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
+        if (!closed.ok) {
+            log(`    ! could not report the cancellation (HTTP ${closed.status}) ${closed.error ?? ''}`);
+            return 'unreported';
+        }
+        return 'cancelled';
+    }
     if (!result.ok) {
         const { reason } = result;
         const detail = result.detail ?? '';
@@ -533,6 +759,12 @@ async function handleRun(run, options, runtime, api, log, sleep) {
     const posted = await postReply(api, run.reply_path, run.reply_body_key, reply, run.run_id, run.reply_body_extra);
     if (!posted.ok) {
         log(`${timestamp()} ✗ ${label} reply rejected (HTTP ${posted.status}) ${posted.error ?? ''}`);
+        // Undo the mid-run recording. The session exists and the agent's transcript
+        // holds this turn, but the conversation never received its answer, so
+        // continuing it would skip reasoning nobody can see — the same rule that
+        // used to be expressed by simply not recording until the reply landed.
+        if (result.sessionRecorded)
+            runtime.sessions.forget(sessionKeyFor(run));
         const reported = await completeWithRetry(api, run.completion_path, { ok: false, error: `posting the reply failed with HTTP ${posted.status}` }, log, sleep);
         return reported.ok ? 'failed' : 'unreported';
     }
@@ -588,9 +820,79 @@ export async function runAgentHarness(options) {
     // duplicate — the run is in flight right here. Only a run in the ledger that
     // this process never handled is a candidate for restart recovery.
     const handledThisSession = new Set();
+    /** Accepted, not yet reported — the executing run and everything queued. */
+    const accepted = new Map();
+    const heartbeats = new HeartbeatLoop({ api, log, intervalMs: options.heartbeatIntervalMs });
     let queue = Promise.resolve();
     let fatal;
     let attempt = 0;
+    /** Stop tracking a run — it has reached a terminal state, one way or another. */
+    const release = (runId) => {
+        accepted.delete(runId);
+        heartbeats.untrack(runId);
+    };
+    /**
+     * Work started from a heartbeat that the harness must still wait for on the
+     * way out, but that a heartbeat tick must not wait for.
+     *
+     * Closing out a cancelled queued run can take the whole completion-retry
+     * schedule — around ten minutes against an unreachable server — and beats do
+     * not overlap. Awaited inside the tick, one unreachable completion would stop
+     * every other run being heartbeated, and the four-hour job it was queued
+     * behind would be reaped for going quiet.
+     */
+    const background = [];
+    /**
+     * Somebody pressed stop in ProhostAI.
+     *
+     * The two cases are genuinely different work. An executing run reports
+     * itself: the abort reaches `execAgent`, which signals the process group the
+     * same way a timeout does, and `handleRun` completes it with the marker
+     * error. A run still in the queue has nobody to report it — it will never
+     * execute, and the queue is about to skip it — so it is closed out here.
+     */
+    const cancelAccepted = async (entry) => {
+        if (entry.disposition)
+            return;
+        entry.disposition = 'cancelled';
+        if (entry.executing) {
+            entry.controller.abort();
+            return;
+        }
+        release(entry.run.run_id);
+        log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)} was cancelled before it started`);
+        // `--dry-run` promises to make no writes, and a completion is the most
+        // terminal write there is. The executing-run path defers to the same check
+        // inside `handleRun`; this one has to make it itself, because a queued run
+        // never reaches `handleRun` at all.
+        if (options.dryRun) {
+            log('    [dry-run] would report the run as cancelled');
+            return;
+        }
+        const closed = await completeWithRetry(api, entry.run.completion_path, { ok: false, error: RUN_ERROR_CANCELLED_BY_USER }, log, sleep);
+        if (closed.ok)
+            runs.markCompleted(entry.run.run_id);
+        metrics.runsFailed += 1;
+    };
+    /**
+     * The server has already closed this run out — it was reaped, or resolved by
+     * something else. Either way nothing this machine does with it can land, so
+     * the only correct move is to stop: kill the command if one is running, and
+     * post nothing at all. A completion here would be a 409 by definition.
+     */
+    const resolveAccepted = (entry) => {
+        if (entry.disposition)
+            return;
+        entry.disposition = 'resolved';
+        if (entry.executing) {
+            entry.controller.abort();
+            return;
+        }
+        release(entry.run.run_id);
+        // Terminal server-side, so a redelivery must not start it either.
+        runs.markCompleted(entry.run.run_id);
+        log(`${timestamp()} ⏹ run ${entry.run.run_id.slice(0, 12)} was closed server-side before it started`);
+    };
     const exhausted = () => options.maxConnections !== undefined && metrics.connections >= options.maxConnections;
     while (!options.signal?.aborted && fatal === undefined && !exhausted()) {
         metrics.connections += 1;
@@ -604,8 +906,14 @@ export async function runAgentHarness(options) {
         log(`${timestamp()} … disconnected; reconnecting in ${(delay / 1000).toFixed(1)}s`);
         await sleep(delay);
     }
-    // Let an in-flight run finish reporting before we hand back metrics.
+    // Let an in-flight run finish reporting before we hand back metrics — and
+    // any cancellation the heartbeat loop started and deliberately did not wait
+    // for, so the harness never exits with a run half-closed.
     await queue;
+    // Stopped first, so no further tick can start work after the wait below has
+    // taken its snapshot of `background`.
+    heartbeats.stop();
+    await Promise.all(background);
     if (fatal !== undefined) {
         log(`${timestamp()} ✗ ${fatal}`);
         metrics.fatal = fatal;
@@ -753,13 +1061,36 @@ export async function runAgentHarness(options) {
                 // redelivery execute the agent a second time.
                 runs.markStarted(run.run_id, run.completion_path);
                 metrics.runsStarted += 1;
+                // Heartbeat from the moment it is accepted, not from the moment it
+                // starts. The row is `running` server-side either way, and a run that
+                // waits an hour behind a long job is reaped exactly like one that has
+                // been working for an hour.
+                const entry = { run, controller: new AbortController(), executing: false };
+                accepted.set(run.run_id, entry);
+                heartbeats.track(run.run_id, {
+                    path: heartbeatPathFor(run),
+                    progress: () => entry.progress,
+                    // Deliberately not awaited here — see `background`.
+                    onCancelRequested: () => {
+                        background.push(cancelAccepted(entry).catch((err) => {
+                            log(`${timestamp()} ✗ closing cancelled run ${run.run_id} failed: ${String(err)}`);
+                        }));
+                    },
+                    onResolved: () => resolveAccepted(entry),
+                });
                 // Serialize execution but never block the socket reader: the ack and
                 // the heartbeat have to keep flowing while the agent thinks.
                 queue = queue.then(async () => {
+                    // Settled while it waited — cancelled, or closed server-side. Both
+                    // were dealt with where they were decided; there is nothing to run
+                    // and nothing left to report.
+                    if (entry.disposition)
+                        return;
                     // Shutdown was requested while this sat in the queue. Don't start a
                     // fresh command the operator is waiting to be rid of — but do close
                     // the run out, so the server isn't left waiting on it either.
                     if (options.signal?.aborted) {
+                        release(run.run_id);
                         log(`${timestamp()} ⏹ run ${run.run_id.slice(0, 12)} dropped — harness is shutting down`);
                         const closed = await completeWithRetry(api, run.completion_path, { ok: false, error: 'the harness shut down before this run started' }, log, sleep);
                         if (closed.ok)
@@ -767,20 +1098,42 @@ export async function runAgentHarness(options) {
                         metrics.runsFailed += 1;
                         return;
                     }
-                    const outcome = await handleRun(run, options, runtime, api, log, sleep).catch((err) => {
+                    // Set before the first `await`, so a cancellation arriving from the
+                    // heartbeat either sees a queued run (and closes it out itself) or an
+                    // executing one (and aborts it) — never both, and never neither.
+                    entry.executing = true;
+                    const control = {
+                        signal: entry.controller.signal,
+                        disposition: () => entry.disposition,
+                        report: (activity) => {
+                            entry.progress = activity;
+                        },
+                    };
+                    const outcome = await handleRun(run, options, runtime, api, log, sleep, control).catch((err) => {
                         log(`${timestamp()} ✗ run ${run.run_id} crashed the harness handler: ${String(err)}`);
                         return 'failed';
                     });
+                    // Held until here on purpose: heartbeats keep the row alive for the
+                    // whole of `handleRun`, including the ten minutes of completion
+                    // retries at the end of it.
+                    release(run.run_id);
                     // Both terminal outcomes reached the completion endpoint; only then
                     // is the run settled and safe to skip on redelivery.
                     if (outcome === 'succeeded')
                         metrics.runsSucceeded += 1;
-                    if (outcome === 'failed' || outcome === 'unreported')
+                    // A cancellation is reported to the server as `failed`, and counted
+                    // the same way here — the run did not produce the answer it was for.
+                    if (outcome === 'failed' || outcome === 'unreported' || outcome === 'cancelled') {
                         metrics.runsFailed += 1;
+                    }
                     // Only a run the server acknowledged as terminal is settled; an
                     // unreported one stays open so a redelivery can report it.
-                    if (outcome === 'succeeded' || outcome === 'failed')
+                    if (outcome === 'succeeded' ||
+                        outcome === 'failed' ||
+                        outcome === 'cancelled' ||
+                        outcome === 'resolved') {
                         runs.markCompleted(run.run_id);
+                    }
                 });
             });
             ws.on('close', () => {

package/dist/index.js

@@ -25,7 +25,7 @@ function printHelp() {
         'Usage:',
         '  prohost listen --forward-to <local-url> [--api-key <key>] [--url <ws-url>] [--machine <name>] [--subscription-id <id>]',
         '  prohost agent pair --code <pairing-code> [--webhook-url <https-url>] [--base-url <url>]',
-        "  prohost agent run --exec '<command>' [--timeout <seconds>] [--dry-run] [--workdir <dir>]",
+        "  prohost agent run --exec '<command>' [--idle-timeout <seconds>] [--timeout <seconds>] [--dry-run] [--workdir <dir>]",
         "  prohost agent install-daemon --exec '<command>' [--workdir <dir>]   Keep the agent running",
         '  prohost agent uninstall-daemon',
         '',
@@ -43,7 +43,12 @@ function printHelp() {
         '',
         'agent run flags:',
         '  --exec           Command to run for each mention; the prompt arrives on stdin (required)',
-        '  --timeout        Seconds one run may take before it is killed. Defaults to 300.',
+        '  --idle-timeout   Seconds a run may produce NO output at all before it is treated as',
+        '                   stalled and killed. Defaults to 900 (15 minutes). Any byte on stdout or',
+        '                   stderr resets it, so a working agent never trips it. Pass 0 to disable.',
+        '  --timeout        Seconds one run may take before it is killed, however busy it is.',
+        '                   Defaults to 0 — no wall-clock limit, because a long run is not a stuck',
+        '                   one. Set it if you want a hard ceiling as well as the stall watchdog.',
         '  --dry-run        Print the reply that would be posted; write nothing.',
         '  --workdir        Working directory for the agent command. Defaults to $PROHOST_HOME/workspace.',
         '                   Avoid pointing it at a code checkout: agents read CLAUDE.md, .claude/settings.json',
@@ -69,7 +74,8 @@ function printHelp() {
         '',
         'agent install-daemon flags:  (macOS: launchd; Linux: prints a systemd-user unit)',
         '  --exec           The same command you would pass to `agent run` (required).',
-        '  --workdir, --timeout, --machine, --url, --allow-unverified, --safe-tools — passed to `agent run`.',
+        '  --workdir, --idle-timeout, --timeout, --machine, --url, --allow-unverified, --safe-tools',
+        '                   — passed to `agent run`.',
         '',
         'Global flags:',
         '  --help, -h       Show this help and exit.',

package/dist/version.d.ts

@@ -7,6 +7,6 @@
  * `package.json`, so a release bump that forgets this file fails the suite
  * instead of shipping a `User-Agent` that lies about which build is calling.
  */
-export declare const CLI_VERSION = "0.2.3";
+export declare const CLI_VERSION = "0.4.0";
 /** Sent on every HTTP request the CLI makes back into ProhostAI. */
-export declare const USER_AGENT = "prohost-cli/0.2.3";
+export declare const USER_AGENT = "prohost-cli/0.4.0";

package/dist/version.js

@@ -7,6 +7,6 @@
  * `package.json`, so a release bump that forgets this file fails the suite
  * instead of shipping a `User-Agent` that lies about which build is calling.
  */
-export const CLI_VERSION = '0.2.3';
+export const CLI_VERSION = '0.4.0';
 /** Sent on every HTTP request the CLI makes back into ProhostAI. */
 export const USER_AGENT = `prohost-cli/${CLI_VERSION}`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prohost/cli",
-  "version": "0.2.3",
+  "version": "0.4.0",
   "description": "Run your own AI agent as a ProhostAI teammate, and stream your account's webhooks to your laptop.",
   "type": "module",
   "license": "MIT",