omp-conductor 0.15.12 → 0.16.0

package/src/wizard-ui.ts

@@ -36,8 +36,9 @@ export interface WizardUi {
   /** Single-line text prompt. `undefined` dismisses it; an empty submit accepts
    *  the placeholder. */
   input(title: string, placeholder?: string): Promise<string | undefined>;
-  /** Numbered single-choice list. Resolves the chosen option's label, or
-   *  `undefined` when dismissed. */
+  /** Single-choice list. On an interactive TTY the current option is rendered
+   *  inline and moved with ↑/↓ or j/k; on a pipe it stays the numbered wall.
+   *  Resolves the chosen option's label, or `undefined` when dismissed. */
   select(
     title: string,
     options: { label: string; description?: string }[],
@@ -62,6 +63,17 @@ import type { Readable, Writable } from "node:stream";
  * as long as the wizard, and {@link TerminalUi.close} releases it once, from the
  * caller's `finally`.
  *
+ * The one exception is {@link TerminalUi.select} on a real TTY, which takes the
+ * terminal over in raw mode. A live readline interface echoes every typed
+ * character back to the output and folds Enter into a `line` event, so it would
+ * smear the inline option row with pressed keys and drop stray lines into the
+ * buffer underneath it. That prompt closes the interface, reads the terminal
+ * byte by byte, and then re-creates the interface so the line protocol is still
+ * alive for the next prompt. The re-opening works on a TTY (the stream itself
+ * is untouched; closing only pauses it, and it is resumed), and the closed
+ * interface's key listeners are dropped, so nothing leaks across repeated
+ * passes of a menu.
+ *
  * Dismissal is the interface's own `close`: `rl.question()` on an exhausted
  * stream never settles on its own, so EOF (a script that ran out of answers) and
  * Ctrl-C both close the interface, and the line reader turns that into the
@@ -78,18 +90,18 @@ export interface TerminalUi extends WizardUi {
 export function terminalUi(io: { input?: Readable; output?: Writable } = {}): TerminalUi {
   const input = io.input ?? stdin;
   const output = io.output ?? stdout;
-  const rl: Interface = createInterface({ input, output });
+  let rl: Interface = createInterface({ input, output });
   let closed = false;
-  const close = (): void => {
-    if (closed) return;
-    closed = true;
-    rl.close();
+  let buffered: string[] = [];
+  let waiting: ((value: string | undefined) => void) | undefined;
+  let ended = false;
+  const deliver = (value: string | undefined): boolean => {
+    const resolve = waiting;
+    if (resolve === undefined) return false;
+    waiting = undefined;
+    resolve(value);
+    return true;
   };
-  // Ctrl-C is a cancel, not a signal to the whole process: closing the interface
-  // makes every pending and subsequent prompt resolve `undefined`, which the
-  // wizard already turns into `Cancelled`.
-  rl.once("SIGINT", close);
-
   // Lines are buffered as they arrive rather than read on demand, because on a
   // non-TTY stdin readline drains the whole pipe at once and emits `line` for
   // every one of them immediately. `rl.question()` only captures the line that
@@ -97,23 +109,40 @@ export function terminalUi(io: { input?: Readable; output?: Writable } = {}): Te
   // already flown past and then hit EOF — three prompts resolving `undefined`
   // and a wizard that cancelled itself. Buffering makes a pipe and a terminal
   // behave the same: nothing is read before it is asked for, nothing is lost.
-  const buffered: string[] = [];
-  let waiting: ((value: string | undefined) => void) | undefined;
-  let ended = false;
-  const deliver = (value: string | undefined): boolean => {
-    const resolve = waiting;
-    if (resolve === undefined) return false;
+  const attach = (iface: Interface): void => {
+    // Ctrl-C on the line protocol is a cancel, not a signal to the whole
+    // process: closing the interface (on SIGINT or on EOF) makes every pending
+    // and subsequent prompt resolve `undefined`, which the wizard already turns
+    // into `Cancelled`. The raw-mode select handles Ctrl-C itself; every other
+    // prompt relies on this.
+    iface.once("SIGINT", close);
+    iface.on("line", (text: string) => {
+      if (!deliver(text)) buffered.push(text);
+    });
+    iface.once("close", () => {
+      ended = true;
+      deliver(undefined);
+    });
+  };
+  const close = (): void => {
+    if (closed) return;
+    closed = true;
+    rl.close();
+  };
+  // The line protocol runs on the *current* interface. An interactive select
+  // takes the terminal over in raw mode, so the interface that was echoing
+  // keystrokes and folding Enter into lines has to be closed out of the way and
+  // a fresh one opened for the prompts that follow (see the lifecycle comment
+  // at the top of the interface declaration).
+  const reopen = (): void => {
+    rl.close();
+    buffered = [];
     waiting = undefined;
-    resolve(value);
-    return true;
+    ended = false;
+    rl = createInterface({ input, output });
+    attach(rl);
   };
-  rl.on("line", (text: string) => {
-    if (!deliver(text)) buffered.push(text);
-  });
-  rl.once("close", () => {
-    ended = true;
-    deliver(undefined);
-  });
+  attach(rl);
 
   /** One line, or `undefined` when the operator dismissed the prompt. */
   const line = async (query: string): Promise<string | undefined> => {
@@ -127,6 +156,37 @@ export function terminalUi(io: { input?: Readable; output?: Writable } = {}): Te
     });
   };
 
+  // -------- the interactive select: raw keys, one changing row --------
+  //
+  // On a real TTY the numbered wall is replaced by a single row that shows the
+  // option under the cursor and follows ↑/↓ or j/k. The keys are read from the
+  // raw stream byte by byte rather than through readline's keypress decode:
+  // readline only emits a lone Escape after its own escape-sequence timeout,
+  // and CSI sequences split across chunks have to be reassembled anyway. The
+  // terminal is taken over only for the duration of the prompt and restored on
+  // every exit path.
+  type TtyInput = Readable & {
+    isTTY: boolean;
+    isRaw?: boolean;
+    setRawMode(mode: boolean): void;
+  };
+  const asTty = (stream: Readable): TtyInput | undefined => {
+    const candidate = stream as Readable & Partial<TtyInput>;
+    return candidate.isTTY === true && typeof candidate.setRawMode === "function"
+      ? (candidate as TtyInput)
+      : undefined;
+  };
+
+  const ESC = 27;
+  const ENTERS = new Set([13, 10]);
+  const CTRL_C = 3;
+  const UP = 65; // CSI A after ESC [
+  const DOWN = 66; // CSI B after ESC [
+  const KEY_J = 106; // lower-case j: down
+  const KEY_K = 107; // lower-case k: up
+  const KEY_J_UP = 74; // J
+  const KEY_K_UP = 75; // K
+
   return {
     close,
     notify(message) {
@@ -156,25 +216,178 @@ export function terminalUi(io: { input?: Readable; output?: Writable } = {}): Te
       return answer.trim().length === 0 ? (placeholder ?? "") : answer;
     },
     async select(title, options, dialogOptions) {
-      output.write(`${title}\n`);
-      options.forEach((o, i) => {
-        output.write(`  ${i + 1}. ${o.label}${o.description !== undefined ? ` — ${o.description}` : ""}\n`);
-      });
-      const initial = dialogOptions?.initialIndex ?? 0;
-      const defaultLabel = options[initial]?.label ?? "";
-      for (;;) {
-        const answer = await line(
-          `Select 1-${options.length} [${initial + 1} = ${defaultLabel}] (Enter keeps it, Ctrl-C cancels): `,
-        );
-        if (answer === undefined) return undefined;
-        const a = answer.trim();
-        if (a === "") return defaultLabel; // a bare Enter re-affirms the current row
-        const n = Number(a);
-        if (Number.isInteger(n)) {
-          const picked = options[n - 1];
-          if (picked !== undefined) return picked.label;
+      const tty = asTty(input);
+      if (tty === undefined) {
+        // Non-TTY stdin (a pipe, a test harness, CI): the numbered wall, kept
+        // byte-identical so the scripted protocol cannot drift.
+        output.write(`${title}\n`);
+        options.forEach((o, i) => {
+          output.write(`  ${i + 1}. ${o.label}${o.description !== undefined ? ` — ${o.description}` : ""}\n`);
+        });
+        const initial = dialogOptions?.initialIndex ?? 0;
+        const defaultLabel = options[initial]?.label ?? "";
+        for (;;) {
+          const answer = await line(
+            `Select 1-${options.length} [${initial + 1} = ${defaultLabel}] (Enter keeps it, Ctrl-C cancels): `,
+          );
+          if (answer === undefined) return undefined;
+          const a = answer.trim();
+          if (a === "") return defaultLabel; // a bare Enter re-affirms the current row
+          const n = Number(a);
+          if (Number.isInteger(n)) {
+            const picked = options[n - 1];
+            if (picked !== undefined) return picked.label;
+          }
+          output.write(`Please enter a number between 1 and ${options.length}.\n`);
+        }
+      }
+      if (options.length === 0) return undefined;
+      // Clamp rather than wrap: the row is the operator's place in the list and
+      // pressing past an edge should not fly from bottom to top (or the other
+      // way); a bare Enter is "keep the current row", so the position is a
+      // setting, not a spinner.
+      const bound = (index: number): number => Math.max(0, Math.min(index, options.length - 1));
+      let index = bound(dialogOptions?.initialIndex ?? 0);
+      const row = (at: number): string => {
+        const o = options[at];
+        return `  › ${o?.label ?? ""}${o?.description !== undefined ? ` — ${o.description}` : ""}`;
+      };
+      let prevLen = 0;
+      const paint = (): void => {
+        const text = row(index);
+        output.write(`\r${" ".repeat(prevLen)}\r${text}`);
+        prevLen = text.length;
+      };
+      let settled: ((value: string | undefined) => void) | undefined;
+      let failed: ((err: unknown) => void) | undefined;
+      let escapeState: "idle" | "esc" | "seq" = "idle";
+      let escapeTimer: ReturnType<typeof setTimeout> | undefined;
+      const clearEscape = (): void => {
+        if (escapeTimer !== undefined) {
+          clearTimeout(escapeTimer);
+          escapeTimer = undefined;
+        }
+      };
+      // Escape sequences arrive byte by byte and a lone ESC cannot be told
+      // apart from the start of `ESC [ A` until the next byte lands; reading
+      // the whole chunk at once handles the common atomic write, and a short
+      // timer turns an unfinished sequence into a plain Escape.
+      const armEscape = (): void => {
+        if (escapeTimer !== undefined) return;
+        escapeTimer = setTimeout(() => {
+          escapeTimer = undefined;
+          escapeState = "idle";
+          settled?.(undefined);
+        }, 30);
+      };
+      const onData = (chunk: Buffer): void => {
+        try {
+          for (const byte of chunk) {
+            switch (escapeState) {
+              case "esc": {
+                clearEscape();
+                if (byte === 91 || byte === 79) {
+                  // ESC [ … or ESC O … — wait for the direction letter.
+                  escapeState = "seq";
+                  armEscape();
+                  continue;
+                }
+                // ESC followed by anything else: a plain Escape press.
+                escapeState = "idle";
+                settled?.(undefined);
+                continue;
+              }
+              case "seq": {
+                clearEscape();
+                escapeState = "idle";
+                if (byte === 65) index = bound(index - 1);
+                else if (byte === 66) index = bound(index + 1);
+                else continue; // an unknown CSI sequence is not a key we know
+                paint();
+                continue;
+              }
+              case "idle": {
+                if (ENTERS.has(byte)) {
+                  settled?.(options[index]?.label);
+                  continue;
+                }
+                if (byte === CTRL_C) {
+                  settled?.(undefined);
+                  continue;
+                }
+                if (byte === ESC) {
+                  escapeState = "esc";
+                  armEscape();
+                  continue;
+                }
+                if (byte === KEY_J || byte === KEY_J_UP) {
+                  index = bound(index + 1);
+                  paint();
+                  continue;
+                }
+                if (byte === KEY_K || byte === KEY_K_UP) {
+                  index = bound(index - 1);
+                  paint();
+                  continue;
+                }
+                // Any other byte (typos, mouse reports) is ignored.
+                continue;
+              }
+            }
+          }
+        } catch (err) {
+          failed?.(err);
         }
-        output.write(`Please enter a number between 1 and ${options.length}.\n`);
+      };
+      const onSignal = (signal: NodeJS.Signals): void => {
+        // A real signal (not the Ctrl-C byte, which raw mode turns into the
+        // cancel above) still kills the process — but only after the terminal
+        // has been handed back, so the operator's shell is never left raw.
+        clearEscape();
+        input.removeListener("data", onData);
+        tty.setRawMode(wasRaw);
+        process.removeListener("SIGINT", onSignal);
+        process.removeListener("SIGTERM", onSignal);
+        process.removeListener("SIGHUP", onSignal);
+        process.kill(process.pid, signal);
+      };
+      let wasRaw = false;
+      try {
+        // Take the terminal over. The shared interface would echo every pressed
+        // key back into the row and fold Enter into a `line` the next prompt
+        // would swallow, so it is closed for the duration and re-opened below.
+        rl.close();
+        input.resume();
+        // Captured *after* the close: while the interface is alive it keeps the
+        // terminal raw (bun's readline does), and restoring to that state would
+        // leave the operator's shell raw after a signal.
+        wasRaw = tty.isRaw ?? false;
+        tty.setRawMode(true);
+        output.write(`${title}\n`);
+        output.write("  (↑/↓ or j/k to move · Enter accepts · Ctrl-C or Esc cancels)\n");
+        paint();
+        const done = new Promise<string | undefined>((resolve, reject) => {
+          settled = resolve;
+          failed = reject;
+        });
+        input.on("data", onData);
+        process.once("SIGINT", onSignal);
+        process.once("SIGTERM", onSignal);
+        process.once("SIGHUP", onSignal);
+        const picked = await done;
+        output.write("\n");
+        return picked;
+      } finally {
+        // Every exit path — accept, cancel, an exception in rendering, a signal
+        // that did not kill us — restores the terminal exactly as it was found
+        // and re-opens the line interface for the prompts that follow.
+        clearEscape();
+        input.removeListener("data", onData);
+        tty.setRawMode(wasRaw);
+        process.removeListener("SIGINT", onSignal);
+        process.removeListener("SIGTERM", onSignal);
+        process.removeListener("SIGHUP", onSignal);
+        reopen();
       }
     },
   };
@@ -209,7 +422,9 @@ export function terminalUi(io: { input?: Readable; output?: Writable } = {}): Te
 export interface ScriptedAnswers {
   input?: Record<string, string>;
   confirm?: Record<string, boolean | boolean[] | null>;
-  select?: Record<string, string | number | null>;
+  /** The array variant is what lets one script drive a repeated menu (#417):
+   *  the review loop's consent menu can be answered "decline, then apply". */
+  select?: Record<string, string | number | null | (string | number | null)[]>;
 }
 
 export function scriptedUi(script: ScriptedAnswers = {}): WizardUi {

package/src/worker.ts

@@ -61,6 +61,17 @@ export const RESUME_PROMPT =
   "re-check the outcome of your last action before repeating it, then keep working your original " +
   "brief to the same report contract.";
 
+/**
+ * What an orphan-resumed worker is told instead of re-sending its brief. One
+ * literal so tests can pin it (#536): the original brief is already in the
+ * resumed transcript, and re-sending it is how a resumed worker ends up
+ * re-doing the work it just did.
+ */
+export const ORPHAN_RESUME_PROMPT =
+  "Your previous process was interrupted by a daemon restart. Review the transcript's final state before acting, " +
+  "then continue exactly where you left off: re-check the outcome of your last action before repeating it, and keep " +
+  "working your original brief to the same report contract.";
+
 export interface WorkerOpts {
   brief: string;
   cwd: string;
@@ -74,11 +85,29 @@ export interface WorkerOpts {
    * location; either way the real path comes back on {@link WorkerResult}.
    */
   sessionDir?: string;
+  /**
+   * Continue the most recent transcript in `sessionDir` instead of opening a
+   * blank session (#536). The daemon sets it only for an orphan-clean
+   * continuation, after verifying the prior transcript and worktree still
+   * exist — the harness's own `continueRecent` is the backstop, and a silent
+   * fallback to a fresh session is surfaced by the `sessionFile` lineage
+   * compare at the dispatch site, never left quiet.
+   */
+  resume?: boolean;
   /**
    * Model pattern for this session, in omp's model/role syntax. Omitted leaves
    * the harness to pick, which is what an unconfigured project wants.
    */
   model?: string;
+  /**
+   * Absolute path to the fleet-owned omp settings overlay (#537): the YAML the
+   * daemon materialised from the project's `ompSettings` map (plus the retry
+   * keys derived from `modelFallbacks`, which is where #539's staging lives)
+   * under the run's session directory. Forwarded to `createSession`, which
+   * loads it through `Settings.init({ configFiles: [<path>] })`. Absent, no
+   * settings are staged and dispatch is byte-for-byte what it is today.
+   */
+  ompSettingsFile?: string;
   /**
    * Effective per-shape release grants for this session. A worker is refused
    * every shape whatever they say — see {@link SessionRole} — so this is passed
@@ -149,6 +178,37 @@ export interface WorkerResult {
   turns: number;
   spendUsd: number;
   report: string;
+  /** In-session HTTP 429 responses the session recorded (stopReason "error",
+   *  errorStatus 429), counted as the messages streamed in. A healthy run
+   *  reports 0; a run the harness retried through a barrel of rate limits
+   *  carries the number, which is what distinguishes provider-capacity (the
+   *  provider was throttling all along) from an ordinary failure (#573). */
+  provider429Count: number;
+  /**
+   * The model that actually wrote this run's messages, read from
+   * `AssistantMessage.model` on the newest assistant `message_end`. Present
+   * even for a run that never failed over — it is the durable answer to "which
+   * model wrote this" (#535 slice 1, from the message field rather than the
+   * payload-free `model_changed` event). Absent only when no assistant message
+   * carried a model.
+   */
+  model?: string;
+  /** The provider that wrote them, read from the same `AssistantMessage.provider`. */
+  provider?: string;
+  /** Every within-run model fallback the harness applied
+   *  (`retry_fallback_applied`), newest first. The `to` target is what a
+   *  settlement report names when a run swapped providers mid-run. */
+  retryFallbacks: { from: string; to: string }[];
+  /** `retry_fallback_succeeded` events: within-run fallbacks the harness
+   *  confirmed recovered on. */
+  retryFallbackSucceeded: number;
+  /** Assistant messages whose `retryRecovery.recovery === "model"` — the durable
+   *  transcript record of a within-run model swap. */
+  modelRecoveries: number;
+  /** `auto_retry_start` events: in-session provider retries the harness ran. */
+  autoRetryCount: number;
+  /** `auto_compaction_start` events: in-session context compactions. */
+  autoCompactionCount: number;
   killedBy?: KilledBy;
   /** Present only when an operator terminally stopped this run. */
   stoppedReason?: string;
@@ -266,6 +326,11 @@ export async function runWorker(
       cwd: o.cwd,
       ...(o.sessionDir === undefined ? {} : { sessionDir: o.sessionDir }),
       ...(o.model === undefined ? {} : { model: o.model }),
+      ...(o.resume === undefined ? {} : { resume: o.resume }),
+      // The fleet-owned omp settings overlay (#537): carry the staged overlay
+      // path to the session so it loads the project's omp settings. Absent,
+      // nothing is staged and the harness discovers settings as it does today.
+      ...(o.ompSettingsFile === undefined ? {} : { ompSettingsFile: o.ompSettingsFile }),
       // Prevention half of #24: as a worker, structured file tools cannot leave
       // this worktree, and no release grant can ever reach this session (#122).
       role: "worker",
@@ -287,6 +352,12 @@ export async function runWorker(
         state: "stopped",
         turns: 0,
         spendUsd: 0,
+        provider429Count: 0,
+        retryFallbacks: [],
+        retryFallbackSucceeded: 0,
+        modelRecoveries: 0,
+        autoRetryCount: 0,
+        autoCompactionCount: 0,
         report: "",
         stoppedReason: "daemon shutdown began before the worker session started",
       };
@@ -303,7 +374,19 @@ export async function runWorker(
   // transcript it actually opened, and any model downgrade it announced. Read at
   // return time so a session that materialises either late is still reported
   // honestly.
-  const withSessionFacts = (result: WorkerResult): WorkerResult => {
+  // The within-run reliability surface this worker now records (#539): the
+  // resolved model/provider plus the fallback/retry/compaction events. Layered
+  // last, at return time, so every exit path reports the same shape without
+  // each spelling the metrics out by hand.
+  type ReliabilityKeys =
+    | "retryFallbacks"
+    | "retryFallbackSucceeded"
+    | "modelRecoveries"
+    | "autoRetryCount"
+    | "autoCompactionCount";
+  const withSessionFacts = (
+    result: Omit<WorkerResult, ReliabilityKeys>,
+  ): Omit<WorkerResult, ReliabilityKeys> => {
     const { sessionFile, modelFallbackMessage } = session;
     return {
       ...result,
@@ -312,9 +395,38 @@ export async function runWorker(
     };
   };
 
+  // The count fields always travel (0 for a clean run, so an absent field can
+  // never be misread); the resolved model/provider only when some assistant
+  // message actually carried them.
+  const withMetrics = (result: Omit<WorkerResult, ReliabilityKeys>): WorkerResult => ({
+    ...result,
+    ...(resolvedModel === undefined ? {} : { model: resolvedModel }),
+    ...(resolvedProvider === undefined ? {} : { provider: resolvedProvider }),
+    retryFallbacks,
+    retryFallbackSucceeded,
+    modelRecoveries,
+    autoRetryCount,
+    autoCompactionCount,
+  });
+
   let turns = 0;
   let spendUsd = 0;
+  let provider429Count = 0;
   let report = "";
+  // Which model/provider actually wrote the newest assistant message. Last
+  // assistant message wins: that is the durable answer even for a run that
+  // never failed over (#535 slice 1, read off the message field which is where
+  // the resolved model actually lives).
+  let resolvedModel: string | undefined;
+  let resolvedProvider: string | undefined;
+  // Harness reliability surface (#539): within-run provider failover and the
+  // retry/compaction activity that surrounds it, all of it events this worker
+  // does not yet subscribe to but that the run row and settlement report want.
+  let retryFallbacks: { from: string; to: string }[] = [];
+  let retryFallbackSucceeded = 0;
+  let modelRecoveries = 0;
+  let autoRetryCount = 0;
+  let autoCompactionCount = 0;
   // The newest COMPLETE `pushed-green` verdict this session emitted. Tracked
   // apart from `report` because `report` is deliberately the newest non-empty
   // text — a run cut off mid-sentence must still report what it said last —
@@ -476,6 +588,48 @@ export async function runWorker(
       spendUsd += cost;
       o.onSpend?.(spendUsd);
     }
+
+    // Count the provider rate limits the harness retried in-session (#573). A
+    // run that drowns in 429s records `stopReason:"error", errorStatus:429`
+    // dozens of times and never surfaces one as `lastError` — omp swallowed
+    // every retry — so `unknown` and the provider failover chain (#286) never
+    // see them. Counted here, alongside spend, so the classifier can tell
+    // "the provider was throttling the whole run" from an ordinary failure.
+    if (provider429FromMessage(message)) provider429Count += 1;
+
+    // The resolved model and provider live on the message, not on any event
+    // (#539). Last assistant message wins, which is the run's durable answer
+    // even when it never failed over.
+    const model = field(message, "model");
+    if (typeof model === "string" && model !== "") resolvedModel = model;
+    const provider = field(message, "provider");
+    if (typeof provider === "string" && provider !== "") resolvedProvider = provider;
+    // A within-run model swap the harness persisted into the transcript rather
+    // than only emitting as a transient event. Recovery kind "model" is the
+    // durable spelling of the same thing `retry_fallback_applied` says.
+    if (field(field(message, "retryRecovery"), "recovery") === "model") modelRecoveries += 1;
+  });
+
+  // The harness reliability events the run row and settlement report now want
+  // (#539): within-run model fallback, and the retry/compaction activity that
+  // surrounds a throttled provider. Each is a session event (AgentSessionEvent)
+  // carrying the fields this worker reads — a fallback's from→to pair, the
+  // confirmed recoveries, and the auto-retry/compaction attempt counts.
+  session.on("retry_fallback_applied", (event) => {
+    const from = field(event, "from");
+    const to = field(event, "to");
+    if (typeof from === "string" && typeof to === "string") {
+      retryFallbacks = [{ from, to }, ...retryFallbacks];
+    }
+  });
+  session.on("retry_fallback_succeeded", () => {
+    retryFallbackSucceeded += 1;
+  });
+  session.on("auto_retry_start", () => {
+    autoRetryCount += 1;
+  });
+  session.on("auto_compaction_start", () => {
+    autoCompactionCount += 1;
   });
 
   session.on("agent_end", (event) => {
@@ -540,12 +694,13 @@ export async function runWorker(
     // Our own abort surfaces here on some paths; that is a kill, not a crash.
     if (killedBy === undefined && stoppedReason === undefined) {
       const detail = cause instanceof Error ? cause.message : String(cause);
-      return withSessionFacts({
+      return withMetrics(withSessionFacts({
         state: "failed",
         turns,
         spendUsd,
+        provider429Count,
         report: report === "" ? detail : report,
-      });
+      }));
     }
   } finally {
     done = true;
@@ -560,14 +715,15 @@ export async function runWorker(
   }
 
   if (stoppedReason !== undefined) {
-    return withSessionFacts({
+    return withMetrics(withSessionFacts({
       state: "stopped",
       turns,
       spendUsd,
+      provider429Count,
       report,
       stoppedReason,
       ...(claim === undefined ? {} : { prUrl: claim.prUrl, headSha: claim.headSha }),
-    });
+    }));
   }
 
   if (killedBy !== undefined) {
@@ -575,28 +731,30 @@ export async function runWorker(
     // survive the kill. Without them `shouldContinueAfterTurnsCap` sees no
     // artifacts and charges an implementation attempt for a cap kill that had
     // real work to continue from.
-    return withSessionFacts({
+    return withMetrics(withSessionFacts({
       state: "killed",
       turns,
       spendUsd,
+      provider429Count,
       report,
       killedBy,
       ...(claim === undefined ? {} : { prUrl: claim.prUrl, headSha: claim.headSha }),
-    });
+    }));
   }
 
   // An explicit later verdict always wins: a worker that pushed green and then
   // stopped to ask a question means the question. The earlier claim is only
   // restored when the last thing said was not a verdict at all.
   if (claim !== undefined && !hasVerdictLine(report)) {
-    return withSessionFacts({ state: "pushed-green", ...claim, turns, spendUsd, report });
+    return withMetrics(withSessionFacts({ state: "pushed-green", ...claim, turns, spendUsd, provider429Count, report }));
   }
-  return withSessionFacts({
+  return withMetrics(withSessionFacts({
     ...deriveResult(report, o.repoSlug),
     turns,
     spendUsd,
+    provider429Count,
     report,
-  });
+  }));
 }
 
 /**
@@ -624,6 +782,21 @@ export function costUsdFromMessage(message: unknown): number | undefined {
   return any ? sum : undefined;
 }
 
+/**
+ * Is this assistant message a provider HTTP 429 the harness recorded mid-run?
+ *
+ * Live transcripts mark a rate-limited turn with `stopReason:"error"`,
+ * `errorStatus:429` and a message like `429 Provider returned error`, and the
+ * harness's in-session auto-retry usually swallows it — the run carries on and
+ * the 429 never surfaces as `lastError` (#573). Exported so a unit test can pin
+ * the signature without standing up a session; the count itself distinguishes a
+ * run that drowned in them from a run that hit one and recovered.
+ */
+export function provider429FromMessage(message: unknown): boolean {
+  if (field(message, "stopReason") !== "error") return false;
+  return field(message, "errorStatus") === 429;
+}
+
 /**
  * Read one property off an unvalidated harness event. The event union lives in
  * the peer dependency, so the worker narrows the handful of fields it reads