@app-vitals/agent-runtime 0.0.0-stage → 0.1.0

package/claude.ts

@@ -0,0 +1,1054 @@
+import { DEFAULT_ANTHROPIC_MODEL } from "@app-vitals/agent-lib/default-agent-env";
+import type { ProgressPhase } from "@app-vitals/agent-lib/progress-phases";
+import type { ErrorCapturingClient } from "@app-vitals/agent-lib/sentry";
+import {
+  createRunCgroup,
+  killTree,
+  markCgroupUnavailable,
+  removeRunCgroup,
+} from "./process-tree-kill.ts";
+import { type ContentBlock, phaseForBlock } from "./progress-milestones.ts";
+import { type RunTelemetry, RunTelemetryAccumulator } from "./run-telemetry.ts";
+
+export type { RunTelemetry } from "./run-telemetry.ts";
+
+interface LiveClaudeConfig {
+  model: string;
+  fallbackModel?: string;
+  effortLevel?: string;
+  allowedTools: string[];
+}
+
+/**
+ * Live Claude configuration — reads from process.env directly.
+ * Updated at runtime by setLiveClaudeConfig (e.g. from agent config polling).
+ */
+export const liveClaudeConfig: LiveClaudeConfig = {
+  model: process.env.ANTHROPIC_MODEL ?? DEFAULT_ANTHROPIC_MODEL,
+  fallbackModel: process.env.ANTHROPIC_FALLBACK_MODEL,
+  effortLevel: process.env.ANTHROPIC_EFFORT_LEVEL,
+  allowedTools: process.env.AGENT_ALLOWED_TOOLS
+    ? JSON.parse(process.env.AGENT_ALLOWED_TOOLS)
+    : [],
+};
+
+export function setLiveClaudeConfig(patch: Partial<LiveClaudeConfig>): void {
+  Object.assign(liveClaudeConfig, patch);
+}
+
+/**
+ * The unconditional tool floor granted to every Claude session, regardless of
+ * what's seeded in the AgentTool DB table. These are read-only or
+ * context-only tools with no meaningful security delta from Read (which is
+ * already assumed safe) — they are permanently non-revocable.
+ *
+ * Bash, WebSearch, WebFetch, and Agent are deliberately NOT included here:
+ * they now flow entirely through `liveClaudeConfig.allowedTools` /
+ * `extraAllowedTools`, sourced from the AgentTool DB table (seeded at agent
+ * creation from the resolved Specialty manifest — see
+ * admin/src/agents-api.ts — and
+ * backfilled for existing agents). This is the capability-narrowing step: an
+ * agent with no AgentTool rows (or a 404'd config bundle) no longer gets
+ * Bash/WebSearch/WebFetch/Agent for free.
+ */
+export const FLOOR_TOOLS = [
+  "Read",
+  "Write",
+  "Edit",
+  "Glob",
+  "Grep",
+  "TodoWrite",
+  "Skill",
+];
+
+export interface TokenUsage {
+  input_tokens: number;
+  output_tokens: number;
+  cache_read_input_tokens: number;
+  cache_creation_input_tokens: number;
+}
+
+/**
+ * Per-model token usage entry, as emitted by the Claude CLI's `modelUsage`
+ * map (camelCase) — distinct from the snake_case top-level `usage` shape.
+ */
+interface ModelUsageEntry {
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadInputTokens: number;
+  cacheCreationInputTokens: number;
+  costUSD: number;
+  webSearchRequests?: number;
+  contextWindow?: number;
+  maxOutputTokens?: number;
+}
+
+/** Per-model token usage map: model name → ModelUsageEntry. */
+export type ModelUsage = Record<string, ModelUsageEntry>;
+
+export interface ClaudeRunResult {
+  result: string;
+  sessionId?: string;
+  usage?: TokenUsage;
+  totalCostUsd?: number;
+  modelUsage?: ModelUsage;
+  recoveredFromError?: boolean;
+  /**
+   * True when the stream ended cleanly (no process error) but never emitted a
+   * terminal `result` event. In that case `result` is empty and `modelUsage`
+   * carries whatever per-model usage was accumulated from the `assistant`
+   * lines (costUSD unknown → 0), rather than throwing all of it away.
+   *
+   * Caveat: this accumulated `modelUsage` (and the partial usage attached to
+   * `ClaudeTimeoutError`/`ClaudeRunError`) undercounts `outputTokens`. Each
+   * `assistant` line's `usage.output_tokens` only reflects *visible* output —
+   * extended-thinking tokens are billed as output but often come back with an
+   * empty/redacted `thinking` field, so they never show up in that per-line
+   * total. Only the terminal `result` event's `modelUsage` is authoritative;
+   * treat any partial/incomplete usage as a lower bound, not a true count.
+   * Confirmed against real `claude -p --output-format stream-json --verbose`
+   * captures during CSU-1.1 validation — see anthropics/claude-code#27361 and
+   * #64153.
+   */
+  streamIncomplete?: boolean;
+  /**
+   * Per-run measurement accumulated from the `assistant` stream lines: the
+   * first-turn context baseline, turn/tool-call counts, and per-skill token
+   * attribution (see `run-telemetry.ts`). Present whenever the stream was
+   * consumed at all — including the `streamIncomplete` return shape — and
+   * carried as `partialTelemetry` on the thrown error types. Always a lower
+   * bound on output tokens for the same reason `modelUsage` is (see above).
+   */
+  telemetry?: RunTelemetry;
+}
+
+/**
+ * Callback fired as each new assistant turn/message completes (a new distinct
+ * `message.id` is observed), receiving the running accumulated per-model total.
+ * The passed map is a fresh snapshot — safe to retain without it mutating.
+ *
+ * The optional second argument is a generic progress phase (see
+ * `@app-vitals/agent-lib/progress-phases`) derived from the tool_use/thinking/text
+ * blocks in `message.content` — fired whenever the phase changes from the
+ * last one fired for this run (see `_consumeStream`'s phase-collapse logic).
+ * Purely additive: existing single-arg callbacks keep typechecking unchanged.
+ */
+export type ProgressCallback = (
+  modelUsage: ModelUsage,
+  phase?: ProgressPhase,
+) => void;
+
+/**
+ * Callback fired the instant a session id is captured off the stream's
+ * leading `system`/`init` line — the earliest point in a run a session id is
+ * ever available, strictly before any `ProgressCallback` fire (those only
+ * happen on later `assistant` lines). Exists so a caller (and `_runClaude`'s
+ * own retry gate, see below) can learn "this run now has an id worth
+ * resuming with" without waiting for a terminal result or a thrown error to
+ * carry it.
+ */
+export type EarlySessionIdCallback = (sessionId: string) => void;
+
+/**
+ * Returns the model name with the highest outputTokens from the CLI's
+ * modelUsage map. Returns undefined when the map is empty.
+ */
+export function dominantModel(modelUsage: ModelUsage): string | undefined {
+  let best: string | undefined;
+  let bestTokens = -1;
+  for (const [model, usage] of Object.entries(modelUsage)) {
+    if (usage.outputTokens > bestTokens) {
+      bestTokens = usage.outputTokens;
+      best = model;
+    }
+  }
+  return best;
+}
+
+/**
+ * The terminal `result` stream event. Its fields are byte-identical in shape to
+ * what the old `--output-format json` single-blob mode returned, plus the
+ * stream discriminator (`type`/`subtype`).
+ */
+interface ClaudeResultEvent {
+  type: "result";
+  subtype?: string;
+  result: string;
+  session_id: string;
+  is_error: boolean;
+  api_error_status?: number;
+  usage?: TokenUsage;
+  total_cost_usd?: number;
+  modelUsage?: ModelUsage;
+}
+
+/** One `assistant` stream event — carries a turn's usage keyed by message id. */
+interface ClaudeAssistantEvent {
+  type: "assistant";
+  message: {
+    id: string;
+    role?: string;
+    model?: string;
+    usage?: TokenUsage;
+    content?: ContentBlock[];
+  };
+  /** Set on sub-agent turns: the parent session's `Agent` tool_use id. */
+  parent_tool_use_id?: string | null;
+}
+
+export class ClaudeRunError extends Error {
+  constructor(
+    message: string,
+    readonly apiErrorStatus: number | undefined,
+    readonly resultMessage: string,
+    readonly sessionId: string | undefined,
+    /** Accumulated per-model usage at the point of failure, if any. */
+    readonly modelUsage?: ModelUsage,
+    /** Telemetry accumulated before the failure, if the stream was consumed. */
+    readonly partialTelemetry?: RunTelemetry,
+  ) {
+    super(message);
+    this.name = "ClaudeRunError";
+  }
+}
+
+/**
+ * Substring match against the Claude CLI's own error text for a transient
+ * network drop between the CLI process and the Anthropic API mid-stream
+ * (Sentry issue 7616867192 / SENTRY-47, e.g. "API Error: Connection lost
+ * mid-response. The response above may be incomplete.") — distinct from
+ * other `is_error` failures (auth, rate limits, etc.), which are not
+ * eligible for `_spawn`'s single connection-lost retry.
+ */
+const CONNECTION_LOST_MID_RESPONSE = "Connection lost mid-response";
+
+function _isConnectionLostMidResponse(err: unknown): err is ClaudeRunError {
+  return (
+    err instanceof ClaudeRunError &&
+    err.resultMessage.includes(CONNECTION_LOST_MID_RESPONSE)
+  );
+}
+
+/**
+ * Thrown when a run is cancelled via its AbortSignal (e.g. a chat cancel
+ * request surfaced through the heartbeat tick). Mirrors ClaudeTimeoutError /
+ * ClaudeRunError's shape — it carries whatever session id the killed run
+ * managed to capture so the session stays resumable (see
+ * _saveSessionFromError). An aborted run is NEVER retried.
+ */
+export class ClaudeAbortedError extends Error {
+  constructor(
+    /** Session id captured off the leading system/init line, if one arrived. */
+    readonly sessionId?: string,
+    /** Per-model usage accumulated before the process was killed, if any. */
+    readonly partialModelUsage?: ModelUsage,
+    /** Telemetry accumulated before the process was killed, if any. */
+    readonly partialTelemetry?: RunTelemetry,
+  ) {
+    super("Claude session was cancelled");
+    this.name = "ClaudeAbortedError";
+  }
+}
+
+/** Which timer fired: idle-reset (no stdout activity) or the hard ceiling. */
+type ClaudeTimeoutReason = "idle" | "ceiling";
+
+export class ClaudeTimeoutError extends Error {
+  constructor(
+    readonly timeoutMs: number,
+    /** Which limit fired — idle-reset timer or the hard ceiling. */
+    readonly reason: ClaudeTimeoutReason = "ceiling",
+    /**
+     * Per-model usage accumulated before the process was killed, if any.
+     * Always partial by definition — a clean finish never reaches this path.
+     */
+    readonly partialModelUsage?: ModelUsage,
+    /**
+     * The session id captured off the leading `system`/`init` stream-json
+     * line, if one arrived before the timeout fired. A timed-out run never
+     * reaches a terminal `result` event (the only other source of a session
+     * id), so this is the only way a timeout can still carry one.
+     */
+    readonly sessionId?: string,
+    /** Telemetry accumulated before the process was killed, if any. */
+    readonly partialTelemetry?: RunTelemetry,
+  ) {
+    super(`Claude session timed out after ${timeoutMs / 1000}s (${reason})`);
+    this.name = "ClaudeTimeoutError";
+  }
+}
+
+/**
+ * Reports an error to Sentry, downgrading an expected-ceiling
+ * `ClaudeTimeoutError` — the intentional 1hr hard-ceiling backstop firing on
+ * a legitimately long-running session, not a defect — to a `captureMessage`
+ * so it stays visible without creating an actionable Sentry Issue (VITALS-
+ * OS-46: this previously kept re-triggering error-patrol as "regressed" even
+ * though nothing was broken). A `reason: "idle"` timeout (a genuine hang) and
+ * any other error are reported unchanged via `captureException`.
+ *
+ * Every call site that can receive a thrown `ClaudeTimeoutError` should
+ * report through this helper instead of calling `captureException` directly.
+ */
+export function reportClaudeError(
+  sentryClient: ErrorCapturingClient | undefined,
+  err: unknown,
+): void {
+  if (err instanceof ClaudeTimeoutError && err.reason === "ceiling") {
+    sentryClient?.captureMessage?.(
+      `Claude session hit ceiling timeout after ${
+        err.timeoutMs / 1000
+      }s (expected backstop, not a bug)`,
+    );
+    return;
+  }
+  sentryClient?.captureException(err);
+}
+
+interface ClaudeSessionStore {
+  get: (key: string) => Promise<string | undefined> | string | undefined;
+  set: (key: string, id: string) => Promise<void> | void;
+  clear?: (key: string) => Promise<void> | void;
+}
+
+/**
+ * Create a claude CLI runner.
+ *
+ * Default parameter values:
+ *  - workspace: process.cwd() — caller should inject the real workspace path
+ *  - sessions: no-op store — caller should inject a real createFileSessionStore instance
+ *  - model: undefined — falls back to liveClaudeConfig.model
+ */
+export function createRunClaude(
+  spawner: typeof Bun.spawn = Bun.spawn,
+  sessions: ClaudeSessionStore = { get: () => undefined, set: () => {} },
+  model: string | undefined = undefined,
+  workspace: string = process.cwd(),
+  sentryClient?: ErrorCapturingClient,
+  extraAllowedTools: string[] | undefined = undefined,
+  fallbackModel: string | undefined = undefined,
+  effortLevel: string | undefined = undefined,
+  timeoutMs: number = 30 * 60 * 1000,
+  /**
+   * Idle-reset timeout: cleared and restarted on every stdout line. Fires
+   * when the process goes silent for this long, even if `timeoutMs` (the
+   * hard ceiling) hasn't elapsed yet. Default 25min.
+   */
+  idleTimeoutMs: number = 25 * 60 * 1000,
+  onProgress: ProgressCallback | undefined = undefined,
+): (
+  message: string,
+  sessionKey?: string,
+  onProgress?: ProgressCallback,
+  signal?: AbortSignal,
+  extraEnv?: Record<string, string>,
+  onEarlySessionId?: EarlySessionIdCallback,
+) => Promise<ClaudeRunResult> {
+  // Per-session queue: ensures messages on the same thread run serially
+  const sessionQueues = new Map<string, Promise<unknown>>();
+
+  function _enqueue<T>(key: string, fn: () => Promise<T>): Promise<T> {
+    const prev = sessionQueues.get(key) ?? Promise.resolve();
+    const next = prev
+      .then(
+        () => fn(),
+        () => fn(),
+      )
+      .finally(() => {
+        if (sessionQueues.get(key) === next) sessionQueues.delete(key);
+      });
+    sessionQueues.set(key, next);
+    return next as Promise<T>;
+  }
+
+  function _buildArgs(
+    message: string,
+    resumeSessionId: string | undefined,
+  ): string[] {
+    const resolvedModel = model ?? liveClaudeConfig.model;
+    const resolvedFallbackModel =
+      fallbackModel ?? liveClaudeConfig.fallbackModel;
+    const resolvedEffortLevel = effortLevel ?? liveClaudeConfig.effortLevel;
+    const resolvedExtraAllowedTools =
+      extraAllowedTools ?? liveClaudeConfig.allowedTools;
+
+    // Deduplicate tools: Set preserves insertion order, so first-occurrence wins
+    const deduplicatedTools = [
+      ...new Set([...FLOOR_TOOLS, ...resolvedExtraAllowedTools]),
+    ];
+
+    const args = [
+      "-p",
+      "--output-format",
+      "stream-json",
+      "--verbose",
+      "--permission-mode",
+      "acceptEdits",
+      "--allowedTools",
+      ...deduplicatedTools,
+      "--model",
+      resolvedModel,
+      ...(resolvedFallbackModel
+        ? ["--fallback-model", resolvedFallbackModel]
+        : []),
+      ...(resolvedEffortLevel ? ["--effort", resolvedEffortLevel] : []),
+    ];
+
+    if (resumeSessionId) {
+      args.push("-r", resumeSessionId);
+    }
+
+    args.push(message);
+    return args;
+  }
+
+  /** Deep-clone a ModelUsage map so callbacks get an immutable snapshot. */
+  function _snapshotUsage(usage: ModelUsage): ModelUsage {
+    const out: ModelUsage = {};
+    for (const [model, entry] of Object.entries(usage)) {
+      out[model] = { ...entry };
+    }
+    return out;
+  }
+
+  /**
+   * Consume a stream-json NDJSON stdout stream incrementally, accumulating
+   * per-model usage from `assistant` lines (deduped by message id) and
+   * capturing the terminal `result` event if one arrives. Malformed / non-JSON
+   * lines are skipped rather than aborting the whole parse.
+   */
+  async function _consumeStream(
+    stream: ReadableStream<Uint8Array>,
+    onLine?: () => void,
+    perCallOnProgress?: ProgressCallback,
+    onEarlySessionId?: EarlySessionIdCallback,
+  ): Promise<{
+    result?: ClaudeResultEvent;
+    modelUsage: ModelUsage;
+    raw: string;
+    sessionId?: string;
+    telemetry: RunTelemetry;
+  }> {
+    const reader = stream.getReader();
+    const decoder = new TextDecoder();
+    const accumulated: ModelUsage = {};
+    const seenMessageIds = new Set<string>();
+    const telemetry = new RunTelemetryAccumulator();
+    let result: ClaudeResultEvent | undefined;
+    let buffer = "";
+    let raw = "";
+    // Last phase fired for THIS _consumeStream call (i.e. this one runClaude
+    // invocation) — reset per call, never global/module state. Used to
+    // collapse repeated identical phases to a single onProgress fire.
+    let lastFiredPhase: ProgressPhase | undefined;
+    // Captured off the very first `system`/`init` line, if one arrives — the
+    // earliest point in the stream a session id is ever available. Populated
+    // once and never overwritten, so a stray later system line (there's never
+    // more than one `init` per run in practice) can't clobber it.
+    let earlySessionId: string | undefined;
+
+    const handleLine = (raw: string) => {
+      const line = raw.trim();
+      if (!line) return;
+      onLine?.();
+      let parsed: unknown;
+      try {
+        parsed = JSON.parse(line);
+      } catch {
+        return; // skip stray non-JSON lines without losing prior accumulation
+      }
+      if (!parsed || typeof parsed !== "object") return;
+      const event = parsed as { type?: string };
+
+      if (event.type === "result") {
+        result = parsed as ClaudeResultEvent;
+        return;
+      }
+      if (event.type === "system") {
+        const system = parsed as {
+          subtype?: string;
+          session_id?: string;
+        };
+        if (
+          earlySessionId === undefined &&
+          system.subtype === "init" &&
+          system.session_id
+        ) {
+          earlySessionId = system.session_id;
+          // Fired here, on the leading init line — before any assistant-turn
+          // onProgress fire below, since those only happen on later lines.
+          onEarlySessionId?.(earlySessionId);
+        }
+        return;
+      }
+      if (event.type !== "assistant") return; // ignore user/etc.
+
+      const { message, parent_tool_use_id: parentToolUseId } =
+        parsed as ClaudeAssistantEvent;
+
+      // Tool-call counting and Skill/Agent attribution switches happen on
+      // every line (deduped by tool_use id inside the accumulator), before
+      // the usage guard — a tool_use-only line carries no `usage` yet.
+      telemetry.observeContent(message?.content);
+
+      // Walk message.content for a progress phase BEFORE the usage guard
+      // below, so a tool_use-only line (no `usage` field yet) can still fire
+      // a phase update. Each block that maps to a phase can fire its own
+      // onProgress call — but repeated identical phases collapse to a single
+      // fire (only fires when the phase differs from the last one fired this
+      // run), so e.g. two consecutive Read blocks fire "reading" once.
+      if (Array.isArray(message?.content)) {
+        for (const block of message.content) {
+          const phase = phaseForBlock(block);
+          if (phase === undefined || phase === lastFiredPhase) continue;
+          lastFiredPhase = phase;
+          onProgress?.(_snapshotUsage(accumulated), phase);
+          perCallOnProgress?.(_snapshotUsage(accumulated), phase);
+        }
+      }
+
+      if (!message?.id || !message.usage) return;
+      if (seenMessageIds.has(message.id)) return; // dedupe repeated turn lines
+      seenMessageIds.add(message.id);
+
+      const model = message.model ?? "unknown";
+      telemetry.observeUsage(
+        message.id,
+        model,
+        message.usage,
+        parentToolUseId ?? undefined,
+      );
+      let entry = accumulated[model];
+      if (!entry) {
+        entry = {
+          inputTokens: 0,
+          outputTokens: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+          costUSD: 0, // authoritative cost is only known from the result event
+        };
+        accumulated[model] = entry;
+      }
+      entry.inputTokens += message.usage.input_tokens ?? 0;
+      // NOTE: message.usage.output_tokens excludes extended-thinking tokens
+      // (billed as output, but the thinking content is often redacted before
+      // it reaches this line) — this accumulation undercounts true output
+      // tokens whenever the `result` event isn't reached. See the
+      // `streamIncomplete` doc comment above for the full explanation.
+      entry.outputTokens += message.usage.output_tokens ?? 0;
+      entry.cacheReadInputTokens += message.usage.cache_read_input_tokens ?? 0;
+      entry.cacheCreationInputTokens +=
+        message.usage.cache_creation_input_tokens ?? 0;
+
+      // Fire both the construction-time-bound closure (kept for back-compat)
+      // and the per-call callback (if provided), each with its own snapshot
+      // so neither can mutate what the other observes.
+      onProgress?.(_snapshotUsage(accumulated));
+      perCallOnProgress?.(_snapshotUsage(accumulated));
+    };
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        const chunk = decoder.decode(value, { stream: true });
+        raw += chunk;
+        buffer += chunk;
+        let nl = buffer.indexOf("\n");
+        while (nl !== -1) {
+          handleLine(buffer.slice(0, nl));
+          buffer = buffer.slice(nl + 1);
+          nl = buffer.indexOf("\n");
+        }
+      }
+      const tail = decoder.decode();
+      raw += tail;
+      buffer += tail;
+      handleLine(buffer); // trailing line without a newline
+    } finally {
+      reader.releaseLock();
+    }
+
+    return {
+      result,
+      modelUsage: accumulated,
+      raw,
+      sessionId: earlySessionId,
+      telemetry: telemetry.snapshot(),
+    };
+  }
+
+  async function _spawnOnce(
+    args: string[],
+    perCallOnProgress?: ProgressCallback,
+    signal?: AbortSignal,
+    extraEnv?: Record<string, string>,
+    onEarlySessionId?: EarlySessionIdCallback,
+  ): Promise<ClaudeRunResult> {
+    // Strip SENTRY_DSN so a spawned Claude Code session (and any `bun test`
+    // it runs internally) can never construct a real Sentry client from
+    // ambient env — this is not the agent's own operational Sentry
+    // reporting (see the reportCronFailure / captureException call sites),
+    // which runs in this parent process against its own client, unaffected
+    // by the child's env.
+    const { SENTRY_DSN: _sentryDsn, ...spawnEnv } = process.env;
+    // extraEnv is layered on last (after the SENTRY_DSN strip) as a per-run
+    // override point — if a caller somehow passed SENTRY_DSN in extraEnv, it
+    // would deliberately win.
+    //
+    // Nothing else is filtered: the CLI (and every install/build it runs in
+    // turn) inherits the ambient environment wholesale, so tooling cache vars
+    // like npm_config_cache / YARN_CACHE_FOLDER / PIP_CACHE_DIR reach child
+    // install processes intact. See plugins/shipwright/references/
+    // toolchain-patterns.md ("Environment Passthrough to Install Children")
+    // for the one place that is NOT true — a target repo whose task runner
+    // enforces a strict env allowlist.
+    const spawnEnvironment = { ...spawnEnv, ...extraEnv };
+    // Two literal option objects rather than one spread: the "pipe" literals
+    // have to stay literal for Bun.spawn's overloads to type stdout/stderr as
+    // readable streams rather than the widened number|stream|undefined union.
+    const spawnClaude = (cgroup: string | undefined) =>
+      cgroup === undefined
+        ? spawner(["claude", ...args], {
+            cwd: workspace,
+            env: spawnEnvironment,
+            stdout: "pipe",
+            stderr: "pipe",
+          })
+        : spawner(["claude", ...args], {
+            cwd: workspace,
+            env: spawnEnvironment,
+            stdout: "pipe",
+            stderr: "pipe",
+            cgroup,
+          });
+
+    // Put the CLI in its own cgroup when the container delegates cgroup v2,
+    // so the timeout/abort path can kill it and every process it forked in
+    // one atomic write. Returns undefined on the (common) read-only-cgroupfs
+    // container, where killTree falls back to a procfs descendant walk
+    // instead — see agent/src/runtime/process-tree-kill.ts.
+    let cgroupPath = createRunCgroup();
+    let proc: ReturnType<typeof spawnClaude>;
+    try {
+      proc = spawnClaude(cgroupPath);
+    } catch (err) {
+      // Bun fails the spawn outright if the cgroup can't be joined (it was
+      // removed underneath us, delegation was revoked mid-run, …). A
+      // containment optimization must never cost us the run itself: drop the
+      // cgroup, latch the mode off for later invocations, and spawn plainly.
+      if (!cgroupPath) throw err;
+      removeRunCgroup(cgroupPath);
+      markCgroupUnavailable(
+        `spawn with cgroup failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+      cgroupPath = undefined;
+      proc = spawnClaude(undefined);
+    }
+
+    let timedOut = false;
+    let aborted = false;
+    let timeoutReason: ClaudeTimeoutReason = "ceiling";
+    let firedTimeoutMs = timeoutMs;
+
+    // Cancel trigger: a third way to kill(), beside the two timeout timers. If
+    // the signal is already aborted, kill immediately; otherwise arm a listener
+    // that is REMOVED in the .finally() below (beside the two clearTimeouts) so
+    // it can't leak on a long-lived, per-thread signal across a resume-retry.
+    const onAbort = () => {
+      aborted = true;
+      killTree(proc, cgroupPath);
+    };
+    if (signal) {
+      if (signal.aborted) onAbort();
+      else signal.addEventListener("abort", onAbort);
+    }
+
+    // Hard ceiling: set once, never reset. Backstops a continuously-active
+    // but never-converging session (idle timer alone would never fire).
+    const ceilingTimer = setTimeout(() => {
+      timedOut = true;
+      timeoutReason = "ceiling";
+      firedTimeoutMs = timeoutMs;
+      killTree(proc, cgroupPath);
+    }, timeoutMs);
+
+    // Idle-reset timer: cleared/restarted on every stdout line. Fires when
+    // the process goes silent, even with ceiling headroom remaining.
+    let idleTimer: ReturnType<typeof setTimeout>;
+    const resetIdleTimer = () => {
+      clearTimeout(idleTimer);
+      idleTimer = setTimeout(() => {
+        timedOut = true;
+        timeoutReason = "idle";
+        firedTimeoutMs = idleTimeoutMs;
+        killTree(proc, cgroupPath);
+      }, idleTimeoutMs);
+    };
+    resetIdleTimer();
+
+    const [
+      {
+        result,
+        modelUsage,
+        raw,
+        sessionId: earlySessionId,
+        telemetry: rawTelemetry,
+      },
+      stderr,
+      exitCode,
+    ] = await Promise.all([
+      _consumeStream(
+        proc.stdout,
+        resetIdleTimer,
+        perCallOnProgress,
+        onEarlySessionId,
+      ),
+      new Response(proc.stderr).text(),
+      proc.exited,
+    ]).finally(() => {
+      clearTimeout(ceilingTimer);
+      clearTimeout(idleTimer);
+      if (signal) signal.removeEventListener("abort", onAbort);
+      // The CLI has exited by now, so this is also the point where any
+      // descendant that outlived it becomes visible: removeRunCgroup kills
+      // stragglers still inside before removing the directory. Best-effort —
+      // it never throws.
+      if (cgroupPath) removeRunCgroup(cgroupPath);
+    });
+
+    // A resumed session (`-r`) replays prior conversation into its first
+    // turn, so that turn is not a cold always-loaded-context baseline. Keep
+    // the rest of the telemetry (turns, tool calls, attribution) but drop
+    // the baseline so the admin series only ever contains fresh-session
+    // numbers.
+    const { firstTurn: _firstTurn, ...resumedTelemetry } = rawTelemetry;
+    const telemetry: RunTelemetry = args.includes("-r")
+      ? resumedTelemetry
+      : rawTelemetry;
+
+    // Aborted must be checked BEFORE timedOut and the exitCode checks: kill()
+    // yields exit 143, which the exitCode !== 0 branch would otherwise
+    // misclassify as a normal run failure.
+    if (aborted) {
+      throw new ClaudeAbortedError(earlySessionId, modelUsage, telemetry);
+    }
+
+    if (timedOut) {
+      throw new ClaudeTimeoutError(
+        firedTimeoutMs,
+        timeoutReason,
+        modelUsage,
+        earlySessionId,
+        telemetry,
+      );
+    }
+
+    if (exitCode !== 0) {
+      if (result?.is_error) {
+        throw new ClaudeRunError(
+          `claude exited ${exitCode}: api_error_status=${result.api_error_status ?? "unknown"} ${result.result}`,
+          result.api_error_status,
+          result.result,
+          result.session_id,
+          result.modelUsage ?? modelUsage,
+          telemetry,
+        );
+      }
+      // A truncated stream that still carried some usage: surface it on the
+      // error rather than discarding it.
+      if (Object.keys(modelUsage).length > 0) {
+        const diagnostic = stderr.trim() || "stream truncated";
+        throw new ClaudeRunError(
+          `claude exited ${exitCode}: ${diagnostic}`,
+          undefined,
+          diagnostic,
+          result?.session_id ?? earlySessionId,
+          modelUsage,
+          telemetry,
+        );
+      }
+      throw new Error(
+        `claude exited ${exitCode}: ${stderr.trim() || raw.trim()}`,
+      );
+    }
+
+    if (!result) {
+      // Clean exit, but the stream ended without a terminal result event.
+      // Surface the accumulated partial usage via a distinct return shape
+      // instead of throwing everything away.
+      return {
+        result: "",
+        sessionId: earlySessionId,
+        modelUsage,
+        streamIncomplete: true,
+        telemetry,
+      };
+    }
+    if (result.is_error) {
+      throw new ClaudeRunError(
+        `claude error: ${result.result}`,
+        result.api_error_status,
+        result.result,
+        result.session_id,
+        result.modelUsage ?? modelUsage,
+        telemetry,
+      );
+    }
+
+    return {
+      result: result.result,
+      sessionId: result.session_id,
+      usage: result.usage,
+      totalCostUsd: result.total_cost_usd,
+      modelUsage: result.modelUsage,
+      telemetry,
+    };
+  }
+
+  /**
+   * Thin wrapper around `_spawnOnce` that retries exactly once on a
+   * connection-lost-mid-response failure — a transient network drop between
+   * the Claude Code CLI process and the Anthropic API, external to this
+   * codebase (Sentry issue 7616867192 / SENTRY-47). Human decision (Dave,
+   * 2026-09-08): retry ONCE, discarding the partial response and re-running
+   * the whole spawn from scratch (never resuming it) — no further retries.
+   * Accepts the small duplicate-side-effect risk; logs the retry via Sentry
+   * so it stays visible. Any other failure (including a second
+   * connection-lost error on the retry itself) propagates unchanged.
+   */
+  async function _spawn(
+    args: string[],
+    perCallOnProgress?: ProgressCallback,
+    signal?: AbortSignal,
+    extraEnv?: Record<string, string>,
+    onEarlySessionId?: EarlySessionIdCallback,
+  ): Promise<ClaudeRunResult> {
+    try {
+      return await _spawnOnce(
+        args,
+        perCallOnProgress,
+        signal,
+        extraEnv,
+        onEarlySessionId,
+      );
+    } catch (err) {
+      if (!_isConnectionLostMidResponse(err)) throw err;
+      sentryClient?.captureMessage?.(
+        `Claude spawn: Connection lost mid-response — retrying once (session ${
+          err.sessionId ?? "unknown"
+        })`,
+      );
+      return _spawnOnce(
+        args,
+        perCallOnProgress,
+        signal,
+        extraEnv,
+        onEarlySessionId,
+      );
+    }
+  }
+
+  async function _saveSession(
+    sessionKey: string | undefined,
+    output: ClaudeRunResult,
+  ) {
+    if (sessionKey && output.sessionId) {
+      await sessions.set(sessionKey, output.sessionId);
+    }
+  }
+
+  /**
+   * Persist whatever session id a FAILED attempt still managed to capture.
+   * ClaudeRunError/ClaudeTimeoutError can both carry a `sessionId` — the CLI
+   * may have already created a real session (and transcript file) before
+   * erroring or timing out. Without this, a first-message failure leaves
+   * `sessions` empty even though a resumable session exists, so the next
+   * reply on the same Slack thread starts a brand-new, context-free session
+   * instead of resuming. Purely a side effect — never affects control flow.
+   */
+  async function _saveSessionFromError(
+    sessionKey: string | undefined,
+    err: unknown,
+  ) {
+    if (!sessionKey) return;
+    if (
+      !(
+        err instanceof ClaudeRunError ||
+        err instanceof ClaudeTimeoutError ||
+        err instanceof ClaudeAbortedError
+      )
+    )
+      return;
+    if (err.sessionId) {
+      await sessions.set(sessionKey, err.sessionId);
+    }
+  }
+
+  async function _runClaude(
+    message: string,
+    sessionKey: string | undefined,
+    perCallOnProgress: ProgressCallback | undefined,
+    signal: AbortSignal | undefined,
+    extraEnv?: Record<string, string>,
+    onEarlySessionId?: EarlySessionIdCallback,
+  ): Promise<ClaudeRunResult> {
+    const existingSessionId = sessionKey
+      ? await sessions.get(sessionKey)
+      : undefined;
+
+    const args = _buildArgs(message, existingSessionId);
+
+    // A call that starts FRESH (no existingSessionId) can still capture a
+    // session id mid-attempt — off the stream's leading system/init line,
+    // well before any terminal result or thrown error. Track it here so the
+    // retry gate below can resume with it even though the run didn't begin
+    // as a resume. `??=` keeps first-write-wins, mirroring _consumeStream's
+    // own earlySessionId semantics; the caller's own callback still fires on
+    // every capture (including a retry attempt), not just the first.
+    let capturedSessionId: string | undefined;
+    const captureEarlySessionId: EarlySessionIdCallback = (sessionId) => {
+      capturedSessionId ??= sessionId;
+      onEarlySessionId?.(sessionId);
+      // Persist immediately, not just at settle time (_saveSession /
+      // _saveSessionFromError below). If the whole agent process dies before
+      // this call settles (OOM, pod restart, redeploy) — not just the
+      // spawned `claude` CLI child being killed — a settle-time-only write
+      // would leave `sessions` empty even though the CLI already created a
+      // durably resumable session. Fire-and-forget, failure swallowed —
+      // best-effort persistence, same contract as _saveSessionFromError: a
+      // failed write here must never mask or affect control flow. The
+      // redundant write this causes at settle time is expected and safe —
+      // sessions.ts's per-key enqueue() serializes same-key writes,
+      // last-write-wins.
+      if (sessionKey) {
+        try {
+          Promise.resolve(sessions.set(sessionKey, sessionId)).catch(() => {});
+        } catch {
+          // Best-effort persistence — a synchronous throw from a misbehaving
+          // store must never mask or affect control flow either, same as the
+          // async-rejection case above.
+        }
+      }
+    };
+
+    try {
+      const output = await _spawn(
+        args,
+        perCallOnProgress,
+        signal,
+        extraEnv,
+        captureEarlySessionId,
+      );
+      await _saveSession(sessionKey, output);
+      return output;
+    } catch (err) {
+      // The initial attempt failed — save whatever session id it captured
+      // before deciding whether to retry, so even a non-retried (or
+      // retry-ineligible) failure still leaves a resumable mapping behind.
+      try {
+        await _saveSessionFromError(sessionKey, err);
+      } catch {
+        // Best-effort persistence — a failed write here must never mask or
+        // replace the original Claude error being handled in this catch block.
+      }
+
+      // A session id worth resuming with, in priority order: one already
+      // known at call start (Slack-thread resume), one captured mid-attempt
+      // off the init line even though this call started fresh, or one the
+      // thrown error itself carries (ClaudeRunError/ClaudeTimeoutError/
+      // ClaudeAbortedError all expose `sessionId`). Without at least one of
+      // these there's nothing to resume, so no retry — same as today's
+      // behavior when existingSessionId is undefined.
+      const errSessionId =
+        err instanceof ClaudeRunError ||
+        err instanceof ClaudeTimeoutError ||
+        err instanceof ClaudeAbortedError
+          ? err.sessionId
+          : undefined;
+      const retrySessionId =
+        existingSessionId ?? capturedSessionId ?? errSessionId;
+
+      // Retry once: transient blips (e.g. a socket close, or the hard
+      // wall-clock ceiling firing on a session that was still legitimately
+      // working) can self-heal on a second attempt without losing
+      // conversation context. Do NOT retry ClaudeAbortedError — the user
+      // asked to cancel, so spawning a second process would defy that. Do
+      // NOT retry an idle-reason ClaudeTimeoutError — that means the session
+      // genuinely hung, and we should surface the error rather than silently
+      // spawning a second process that would also hang; a ceiling-reason
+      // timeout is retried, since it's the expected "hit the hard cap while
+      // still working" case, not a hang signal. If the retry also fails,
+      // rethrow the ORIGINAL error and leave the session mapping untouched —
+      // an error (even a burst of them) is never treated as proof the
+      // session itself is corrupt.
+      // Also exclude a connection-lost-mid-response failure: `_spawn` already
+      // gave it its own dedicated single retry (see `_spawn`'s docstring —
+      // Dave, 2026-09-08: retry ONCE, no further retries). That error's
+      // ClaudeRunError happens to carry a `sessionId` (from the CLI's
+      // `result` event) like any other, so without this exclusion it would
+      // qualify for a SECOND, stacked retry cycle here — silently doubling
+      // the documented one-retry limit.
+      const retryBlocked =
+        err instanceof ClaudeAbortedError ||
+        (err instanceof ClaudeTimeoutError && err.reason === "idle") ||
+        _isConnectionLostMidResponse(err);
+
+      if (retrySessionId && !retryBlocked) {
+        const retryArgs = _buildArgs(message, retrySessionId);
+        try {
+          const output = await _spawn(
+            retryArgs,
+            perCallOnProgress,
+            signal,
+            extraEnv,
+            captureEarlySessionId,
+          );
+          await _saveSession(sessionKey, output);
+          return { ...output, recoveredFromError: true };
+        } catch (retryErr) {
+          // The retry also failed — overwrite with its own most-recently-known
+          // session id (if any), then still rethrow the ORIGINAL error.
+          try {
+            await _saveSessionFromError(sessionKey, retryErr);
+          } catch {
+            // Best-effort persistence — a failed write here must never mask
+            // the original error or prevent the Sentry capture/rethrow below.
+          }
+          sentryClient?.captureException(err);
+          throw err;
+        }
+      }
+      throw err;
+    }
+  }
+
+  return async function runClaude(
+    message: string,
+    sessionKey?: string,
+    onProgress?: ProgressCallback,
+    signal?: AbortSignal,
+    extraEnv?: Record<string, string>,
+    onEarlySessionId?: EarlySessionIdCallback,
+  ): Promise<ClaudeRunResult> {
+    if (sessionKey)
+      return _enqueue(sessionKey, () =>
+        _runClaude(
+          message,
+          sessionKey,
+          onProgress,
+          signal,
+          extraEnv,
+          onEarlySessionId,
+        ),
+      );
+    return _runClaude(
+      message,
+      undefined,
+      onProgress,
+      signal,
+      extraEnv,
+      onEarlySessionId,
+    );
+  };
+}