omp-conductor 0.15.13 → 0.16.0

package/src/types.ts

@@ -657,6 +657,26 @@ export interface ProjectConfig {
    * when absent or unusable.
    */
   modelFallbackThreshold?: number;
+  /**
+   * An opaque omp settings map layered into every worker session this project
+   * dispatches (#537). At dispatch it is materialised verbatim to a fleet-owned
+   * YAML overlay under the run's session directory — never inside the worktree,
+   * whose diff is a PR — and passed to the session as an omp settings overlay
+   * (`Settings.init({ configFiles: [<path>] })`), so it layers on top of both
+   * the daemon account's global config and any project `<cwd>/.omp/config.yml`
+   * without erasing either.
+   *
+   * Conductor validates YAML shape only: a non-mapping is dropped at load, and
+   * everything inside the map is omp's schema to own — an unknown key is omp's
+   * to reject, never conductor's to understand. A config edit takes effect on
+   * the next dispatch (the overlay is rewritten on every attempt), and a
+   * project without the field dispatches byte-for-byte as it always has.
+   *
+   * The retry keys derived from {@link modelFallbacks} are merged into the
+   * effective overlay (unless the map already names a `retry` mapping), which
+   * is where #539's staging lives.
+   */
+  ompSettings?: Record<string, unknown>;
   /**
    * How a stuck run reaches a human, what to do when it cannot, and who runs
    * the session that triages it. See {@link ORCHESTRATOR_MODES}.
@@ -751,6 +771,10 @@ export interface ConductorConfig {
   version: typeof CONFIG_VERSION;
   defaults: Caps;
   projects: ProjectConfig[];
+  /** Absolute directory for restorable `conductor.db` snapshots; defaults to
+   *  `<stateDir()>/backups/db` when omitted. A snapshot that cannot land there
+   *  is `doctor`'s `db-backup` failure. */
+  dbBackupDir?: string;
 }
 
 /**
@@ -1102,6 +1126,11 @@ export const FAILURE_CLASSES = [
   "settlement-stuck",
   "provider-credit",
   "provider-transient",
+  /** A run drowned in in-session provider rate limits (repeated HTTP 429s) and
+   *  never got clear of the throttle before the cap swallowed it. Distinct from
+   *  `provider-credit` (402) and `provider-transient` (a single stream fault)
+   *  because its remedy is different (#573). */
+  "provider-capacity",
   /** A reviewer closed pushed-green or pushed-pending work without merging it:
    *  a review decision, not a worker failure. */
   "returned-for-revision",
@@ -1151,6 +1180,30 @@ export interface BaseHealth {
   checkedAt: number;
 }
 
+/**
+ * A per-repo merge freeze, set when a watched merge (or live base observation)
+ * turns the base red and lifted mechanically on green or by the operator's
+ * `unfreeze` verb. While `clearedAt` is absent the repo is frozen: `prMergeVerb`
+ * refuses further merges to it with the `base-red-freeze` refusal naming the
+ * suspected culprit. It is repo-scoped, never global, so sibling repos keep
+ * merging while one base is broken.
+ */
+export interface BaseFreeze {
+  repo: string;
+  /** The merged SHA suspected of breaking the base. */
+  culpritSha: string;
+  /** Failing workflow/run evidence naming the break. */
+  detail?: string;
+  /** When the freeze was set (or last refreshed). */
+  setAt: number;
+  /** When the freeze was lifted — absent means the repo is actively frozen. */
+  clearedAt?: number;
+  /** Who lifted it: `daemon` for mechanical recovery, else the operator. */
+  clearedBy?: string;
+  /** Why it was lifted: `base-green` or the operator's override reason. */
+  clearedReason?: string;
+}
+
 /**
  * Execution state is separate from the tracker's own labels on purpose: labels
  * are coarse and human-editable, while the loop needs to distinguish "pushed
@@ -1194,6 +1247,39 @@ export interface RunRecord {
   /** Effective turn ceiling for this run; operators may only raise it. */
   maxTurns: number;
   spendUsd: number;
+  /** How many in-session HTTP 429 responses the harness retried before the run
+   *  ended, counted by the worker from the session's `message_end` events while
+   *  the run was live (#573). Absent means the column predates the count or no
+   *  worker reported one — never "no rate limit happened": a healthy run
+   *  records 0. The classifier only reads it as "sustained" above a threshold,
+   *  so a single retried 429 is noise. */
+  provider429Count?: number;
+  /**
+   * The model that actually wrote this run's messages, read from the newest
+   * assistant `message_end`'s `AssistantMessage.model` (#539, #535 slice 1).
+   * Distinct from {@link RunRecord.model} (what the daemon dispatched on):
+   * this is what the harness resolved and actually used, present even for a run
+   * that never failed over, so "which model wrote this" is answerable without
+   * reading a transcript. Absent means the run predates the column or recorded
+   * no message carrying a model.
+   */
+  resolvedModel?: string;
+  /** The provider that wrote them, from the same message. */
+  resolvedProvider?: string;
+  /**
+   * Every within-run model fallback the harness applied (`retry_fallback_applied`),
+   * newest first. The `to` target is what a settlement report names when a run
+   * swapped providers mid-run instead of dying on a throttled primary (#539).
+   */
+  retryFallbacks?: { from: string; to: string }[];
+  /** `retry_fallback_succeeded` events: within-run fallbacks the harness recovered on. */
+  retryFallbackSucceeded?: number;
+  /** Assistant messages whose `retryRecovery.recovery === "model"` (#539). */
+  modelRecoveries?: number;
+  /** `auto_retry_start` events: in-session provider retries the harness ran. */
+  autoRetryCount?: number;
+  /** `auto_compaction_start` events: in-session context compactions. */
+  autoCompactionCount?: number;
   /** omp session transcript, so a human can read what the worker actually did. */
   sessionFile?: string;
   prUrl?: string;
@@ -1288,6 +1374,8 @@ export type AdmissionHoldReason =
   | "plan-usage-cap"
   | "shutting-down"
   | "stale-base"
+  | "file-lane"
+  | "depends-on"
   | "unroutable:no-repo-label"
   | "unroutable:multiple-repo-labels"
   | "unroutable:unknown-repo";
@@ -1298,6 +1386,14 @@ export interface AdmissionHoldSummary {
   count: number;
   /** Queue-order sample, capped before persistence and rendering. */
   issues: number[];
+  /**
+   * Per-issue detail, aligned by index with {@link AdmissionHoldSummary.issues}
+   * and capped the same way. The file-lane interlock (#555) fills it so an
+   * operator reading status or the digest can see *which file* blocked the
+   * held issue and *which running issue* holds it — the same question the
+   * plain `reason` group answers only with a count.
+   */
+  details?: string[];
 }
 
 /** Persisted outcome of the latest completed dispatch tick. */
@@ -1437,9 +1533,29 @@ export interface ReportRecord {
   updatedAt: number;
   /** Backoff gate: a `pending` row is due only once this has passed. */
   nextAttemptAt: number;
-  /** Telegram's own id for the delivered message, read out of the response
-   *  body rather than inferred from the HTTP status. */
+/** Telegram's own id for the delivered message, read out of the response
+   *  body rather than inferred from the HTTP status. A report split into
+   *  several messages records the first part's id here; {@link messageIds}
+   *  carries every part. */
   messageId?: number;
+  /** Every Telegram message id a split delivery accepted, in order. Absent
+   *  for rows delivered before multi-part sends existed (#566), and whenever
+   *  Telegram answered without an id — `ok: true` is the verdict, the id is a
+   *  courtesy. */
+  messageIds?: number[];
+  /**
+   * Parts of the current delivery contract already accepted, persisted as the
+   * retry's resume point. The split is a pure function of the message text, so
+   * a count is the whole state: a retry whose message is byte-identical to the
+   * attempt that wrote this watermark (see {@link sentPartsHash}) starts at
+   * part `sentParts` instead of re-sending confirmed parts (#566).
+   */
+  sentParts?: number;
+  /** SHA-256 of the message text the `sentParts` watermark was written under.
+   * Progress is only valid while the text is unchanged — a report flagged
+   * ambiguous re-renders with a possible-repeat banner, which reshapes the
+   * split, so the watermark must not carry across it. */
+  sentPartsHash?: string;
   deliveredAt?: number;
   /** Bounded text of the last known failure, surfaced verbatim by `status`. */
   lastError?: string;
@@ -1555,10 +1671,22 @@ export type DecisionState = (typeof DECISION_STATES)[number];
  */
 export const DECISION_TTL_MS = 7 * 24 * 60 * 60_000;
 
+/** What a pending decision row is owed — answered by a human, or a condition
+ *  the orchestrator set for itself with no human in the loop. The split is a
+ *  stored column, never inferred from the wording or from whether a condition
+ *  is attached: a real question may legitimately carry a condition ("ask me
+ *  once this PR merges"), and a watch is defined by having no human, not by
+ *  having one. */
+export type DecisionKind = "question" | "watch";
+
 /** One question put to the operator, and its answer if it has one. */
 export interface DecisionRecord {
   id: string;
   project: string;
+  /** Whether this row is a question a human must answer or a watch the
+   *  orchestrator set for itself. A watch turns up under its own heading and
+   *  is never offered to the operator to resolve. */
+  kind: DecisionKind;
   /** The question verbatim, as it was sent. Re-asking must not reword it. */
   question: string;
   /** What is waiting on the answer — an issue, a release, a PR. Free text,
@@ -1583,6 +1711,11 @@ export interface DecisionRecord {
 export interface DecisionDraft {
   project: string;
   question: string;
+  /** `"watch"` marks a row the orchestrator set for itself. Absent means a
+   *  question a human must answer — the default, so every existing caller
+   *  (`message` with a non-material category, `decision open`) stays a
+   *  question without remembering to say so. */
+  kind?: DecisionKind;
   blocks?: string;
   condition?: string;
   at: number;
@@ -1642,6 +1775,35 @@ export interface Store {
     project: string,
     sinceEpochMs: number,
   ): { repo: string; baseRef?: string }[];
+  /** The per-repo base-red freeze row, active or cleared, for one routed repo. */
+  baseFreeze(project: string, repo: string): BaseFreeze | undefined;
+  /** Every base-red freeze row for a project, active first, for `status`. */
+  freezes(project: string): BaseFreeze[];
+  /**
+   * Set or refresh an *active* base-red freeze for one repo, naming the
+   * suspected culprit merge and the failing evidence. Refreshing an
+   * already-active freeze keeps the repo frozen while the same head is still
+   * red. Returns true only when it *newly* froze the repo — a transition worth
+   * surfacing as a material event — and false for a pure refresh so the queue
+   * digest is not flooded by per-tick re-observation of a still-red base.
+   */
+  setBaseFreeze(
+    project: string,
+    freeze: { repo: string; culpritSha: string; detail?: string; setAt?: number },
+  ): boolean;
+  /**
+   * Lift an active freeze — the mechanical green recovery or the operator's
+   * `unfreeze`. Records who lifted it and the ledger reason. Returns true only
+   * when an active freeze was actually lifted (so a recovery/override event is
+   * emitted once, never on an already-clear row).
+   */
+  clearBaseFreeze(
+    project: string,
+    repo: string,
+    by: string,
+    reason: string,
+    at?: number,
+  ): boolean;
   /** Newest attempt per issue that preserved work or failed to, so `status`
    *  can name every WIP tip a re-claim would build on and every tree that is
    *  still the only copy. */
@@ -1808,8 +1970,21 @@ export interface Store {
    *  means somebody else claimed it first — the guard against the same report
    *  being concurrently in flight twice. */
   claimReport(id: string, attemptId: string, at: number): ReportRecord | undefined;
-  /** Record the message id Telegram returned, and the moment it did. */
-  markReportDelivered(id: string, attemptId: string, messageId: number | undefined, at: number): boolean;
+  /** Record the message ids Telegram returned for every accepted part, and
+   *  the moment the last one landed. `messageId` (the first part's id) is
+   *  derived here, so the two can never disagree. */
+  markReportDelivered(id: string, attemptId: string, messageIds: readonly number[], at: number): boolean;
+  /** Record progress mid-split: the first `sentParts` parts of the delivery
+   *  contract fingerprinted by `sentPartsHash` are confirmed accepted. Written
+   *  before the next part ships, so a crash in the window can only ever
+   *  re-send the part that was in flight, never confirmed ones. */
+  markReportPartsSent(
+    id: string,
+    attemptId: string,
+    sentParts: number,
+    sentPartsHash: string,
+    at: number,
+  ): boolean;
   /** The attempt ended without an answer: Telegram may hold the message. The
    *  row *stays* `sending`, because that state means exactly "outcome unknown",
    *  and is flagged {@link ReportRecord.ambiguous} so the retry that stale
@@ -2182,6 +2357,14 @@ export const VERB_REFUSALS = [
   "checks-not-green",
   /** Another merge is in flight for this project. */
   "merge-in-flight",
+  /**
+   * The routed repository's base branch is frozen because a watched merge (or
+   * live base observation) turned it red — a base-red-freeze. Further merges to
+   * that repo stay refused until the base is observed green again or the
+   * operator lifts the freeze with `omp-conductor unfreeze <repo>`; sibling
+   * repos are unaffected.
+   */
+  "base-red-freeze",
   /** The label is not in this project's own vocabulary. */
   "label-not-in-vocabulary",
   /** The label is a lifecycle label; those transitions stay the daemon's (#26). */

package/src/unblock.ts

@@ -35,7 +35,7 @@
  * failed-attempt budget.
  */
 
-import { hasContinuationBudget, hasFailedAttemptBudget } from "./daemon.ts";
+import { hasContinuationBudget, hasFailedAttemptBudget } from "./admission.ts";
 import { projectLabels } from "./label-projection.ts";
 import { LIVE_STATES } from "./store.ts";
 import type { Caps, ProjectConfig, RunRecord, Store, Tracker } from "./types.ts";

package/src/upgrade-verify.ts

@@ -26,7 +26,7 @@ import { DEFAULT_PORT } from "./lifecycle.ts";
 import { dbPath, openStore } from "./store.ts";
 import { appendJournal, readUpgradeJournal, upgradeJournalPath, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
 import type { DoctorReport } from "./doctor.ts";
-import type { FleetLayers } from "./fleet.ts";
+import type { FleetLayers } from "./status-render.ts";
 import type { ReportDraft, ReportKind } from "./types.ts";
 
 /** One command run to completion with captured output. The transient unit, the

package/src/upgrade.ts

@@ -8,9 +8,8 @@ import {
   LEGACY_HERDR_SESSION_HINT,
   resolveHerdrSession,
   telegramStateDir,
-  type DispatchLayer,
-  type FleetLayers,
 } from "./fleet.ts";
+import type { DispatchLayer, FleetLayers } from "./status-render.ts";
 import { livingDaemon, restartDaemon } from "./lifecycle.ts";
 import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir, writeConfigRaw } from "./config.ts";
 import { renderBriefForProject } from "./setup.ts";

package/src/verbs/server.ts

@@ -884,6 +884,31 @@ async function prMergeVerb(
     if (paused !== undefined) return paused;
   }
 
+  // Base-red-freeze gate (#283): a repo whose base is frozen must not accept
+  // another merge, no matter how green this PR's own checks look. Deliberately
+  // placed here — after target-repo identity resolution, before the merge lock
+  // and every `gh` call — so the refusal is a verb-level assertion, not a
+  // daemon-side flag the merge path could miss. Only the frozen repo is gated:
+  // sibling repos keep merging. The freeze lifts mechanically on a green base
+  // re-observation or via the operator's sanctioned `omp-conductor unfreeze
+  // <repo>` — never by label surgery or a DB edit.
+  const freezeRepo = target === undefined ? routedRepo?.name : target.repo;
+  if (freezeRepo !== undefined) {
+    const freeze = deps.store.baseFreeze(project.name, freezeRepo);
+    if (freeze !== undefined && freeze.clearedAt === undefined) {
+      return refuse(
+        "base-red-freeze",
+        `refused: merges to ${freezeRepo} are frozen because its base is red at ` +
+          `${freeze.culpritSha.slice(0, 8)} (base-red-freeze). ${freeze.detail ?? ""} ` +
+          "This is not a merge-lock or check verdict: the base that merges land on is broken, so " +
+          "`conductor_pr_merge` for this repo is refused until the base is observed green again or the " +
+          `operator lifts the freeze with \`omp-conductor unfreeze ${freezeRepo}\`. The override is the sanctioned ` +
+          "path — never work around this refusal with label surgery or a DB edit.",
+        issue,
+      );
+    }
+  }
+
   // Taken before any network call, so two concurrent callers contend here
   // rather than both spending a `gh` round trip and racing at the merge.
   const holderId = randomUUID();

package/src/worker.ts

@@ -99,6 +99,15 @@ export interface WorkerOpts {
    * the harness to pick, which is what an unconfigured project wants.
    */
   model?: string;
+  /**
+   * Absolute path to the fleet-owned omp settings overlay (#537): the YAML the
+   * daemon materialised from the project's `ompSettings` map (plus the retry
+   * keys derived from `modelFallbacks`, which is where #539's staging lives)
+   * under the run's session directory. Forwarded to `createSession`, which
+   * loads it through `Settings.init({ configFiles: [<path>] })`. Absent, no
+   * settings are staged and dispatch is byte-for-byte what it is today.
+   */
+  ompSettingsFile?: string;
   /**
    * Effective per-shape release grants for this session. A worker is refused
    * every shape whatever they say — see {@link SessionRole} — so this is passed
@@ -169,6 +178,37 @@ export interface WorkerResult {
   turns: number;
   spendUsd: number;
   report: string;
+  /** In-session HTTP 429 responses the session recorded (stopReason "error",
+   *  errorStatus 429), counted as the messages streamed in. A healthy run
+   *  reports 0; a run the harness retried through a barrel of rate limits
+   *  carries the number, which is what distinguishes provider-capacity (the
+   *  provider was throttling all along) from an ordinary failure (#573). */
+  provider429Count: number;
+  /**
+   * The model that actually wrote this run's messages, read from
+   * `AssistantMessage.model` on the newest assistant `message_end`. Present
+   * even for a run that never failed over — it is the durable answer to "which
+   * model wrote this" (#535 slice 1, from the message field rather than the
+   * payload-free `model_changed` event). Absent only when no assistant message
+   * carried a model.
+   */
+  model?: string;
+  /** The provider that wrote them, read from the same `AssistantMessage.provider`. */
+  provider?: string;
+  /** Every within-run model fallback the harness applied
+   *  (`retry_fallback_applied`), newest first. The `to` target is what a
+   *  settlement report names when a run swapped providers mid-run. */
+  retryFallbacks: { from: string; to: string }[];
+  /** `retry_fallback_succeeded` events: within-run fallbacks the harness
+   *  confirmed recovered on. */
+  retryFallbackSucceeded: number;
+  /** Assistant messages whose `retryRecovery.recovery === "model"` — the durable
+   *  transcript record of a within-run model swap. */
+  modelRecoveries: number;
+  /** `auto_retry_start` events: in-session provider retries the harness ran. */
+  autoRetryCount: number;
+  /** `auto_compaction_start` events: in-session context compactions. */
+  autoCompactionCount: number;
   killedBy?: KilledBy;
   /** Present only when an operator terminally stopped this run. */
   stoppedReason?: string;
@@ -287,6 +327,10 @@ export async function runWorker(
       ...(o.sessionDir === undefined ? {} : { sessionDir: o.sessionDir }),
       ...(o.model === undefined ? {} : { model: o.model }),
       ...(o.resume === undefined ? {} : { resume: o.resume }),
+      // The fleet-owned omp settings overlay (#537): carry the staged overlay
+      // path to the session so it loads the project's omp settings. Absent,
+      // nothing is staged and the harness discovers settings as it does today.
+      ...(o.ompSettingsFile === undefined ? {} : { ompSettingsFile: o.ompSettingsFile }),
       // Prevention half of #24: as a worker, structured file tools cannot leave
       // this worktree, and no release grant can ever reach this session (#122).
       role: "worker",
@@ -308,6 +352,12 @@ export async function runWorker(
         state: "stopped",
         turns: 0,
         spendUsd: 0,
+        provider429Count: 0,
+        retryFallbacks: [],
+        retryFallbackSucceeded: 0,
+        modelRecoveries: 0,
+        autoRetryCount: 0,
+        autoCompactionCount: 0,
         report: "",
         stoppedReason: "daemon shutdown began before the worker session started",
       };
@@ -324,7 +374,19 @@ export async function runWorker(
   // transcript it actually opened, and any model downgrade it announced. Read at
   // return time so a session that materialises either late is still reported
   // honestly.
-  const withSessionFacts = (result: WorkerResult): WorkerResult => {
+  // The within-run reliability surface this worker now records (#539): the
+  // resolved model/provider plus the fallback/retry/compaction events. Layered
+  // last, at return time, so every exit path reports the same shape without
+  // each spelling the metrics out by hand.
+  type ReliabilityKeys =
+    | "retryFallbacks"
+    | "retryFallbackSucceeded"
+    | "modelRecoveries"
+    | "autoRetryCount"
+    | "autoCompactionCount";
+  const withSessionFacts = (
+    result: Omit<WorkerResult, ReliabilityKeys>,
+  ): Omit<WorkerResult, ReliabilityKeys> => {
     const { sessionFile, modelFallbackMessage } = session;
     return {
       ...result,
@@ -333,9 +395,38 @@ export async function runWorker(
     };
   };
 
+  // The count fields always travel (0 for a clean run, so an absent field can
+  // never be misread); the resolved model/provider only when some assistant
+  // message actually carried them.
+  const withMetrics = (result: Omit<WorkerResult, ReliabilityKeys>): WorkerResult => ({
+    ...result,
+    ...(resolvedModel === undefined ? {} : { model: resolvedModel }),
+    ...(resolvedProvider === undefined ? {} : { provider: resolvedProvider }),
+    retryFallbacks,
+    retryFallbackSucceeded,
+    modelRecoveries,
+    autoRetryCount,
+    autoCompactionCount,
+  });
+
   let turns = 0;
   let spendUsd = 0;
+  let provider429Count = 0;
   let report = "";
+  // Which model/provider actually wrote the newest assistant message. Last
+  // assistant message wins: that is the durable answer even for a run that
+  // never failed over (#535 slice 1, read off the message field which is where
+  // the resolved model actually lives).
+  let resolvedModel: string | undefined;
+  let resolvedProvider: string | undefined;
+  // Harness reliability surface (#539): within-run provider failover and the
+  // retry/compaction activity that surrounds it, all of it events this worker
+  // does not yet subscribe to but that the run row and settlement report want.
+  let retryFallbacks: { from: string; to: string }[] = [];
+  let retryFallbackSucceeded = 0;
+  let modelRecoveries = 0;
+  let autoRetryCount = 0;
+  let autoCompactionCount = 0;
   // The newest COMPLETE `pushed-green` verdict this session emitted. Tracked
   // apart from `report` because `report` is deliberately the newest non-empty
   // text — a run cut off mid-sentence must still report what it said last —
@@ -497,6 +588,48 @@ export async function runWorker(
       spendUsd += cost;
       o.onSpend?.(spendUsd);
     }
+
+    // Count the provider rate limits the harness retried in-session (#573). A
+    // run that drowns in 429s records `stopReason:"error", errorStatus:429`
+    // dozens of times and never surfaces one as `lastError` — omp swallowed
+    // every retry — so `unknown` and the provider failover chain (#286) never
+    // see them. Counted here, alongside spend, so the classifier can tell
+    // "the provider was throttling the whole run" from an ordinary failure.
+    if (provider429FromMessage(message)) provider429Count += 1;
+
+    // The resolved model and provider live on the message, not on any event
+    // (#539). Last assistant message wins, which is the run's durable answer
+    // even when it never failed over.
+    const model = field(message, "model");
+    if (typeof model === "string" && model !== "") resolvedModel = model;
+    const provider = field(message, "provider");
+    if (typeof provider === "string" && provider !== "") resolvedProvider = provider;
+    // A within-run model swap the harness persisted into the transcript rather
+    // than only emitting as a transient event. Recovery kind "model" is the
+    // durable spelling of the same thing `retry_fallback_applied` says.
+    if (field(field(message, "retryRecovery"), "recovery") === "model") modelRecoveries += 1;
+  });
+
+  // The harness reliability events the run row and settlement report now want
+  // (#539): within-run model fallback, and the retry/compaction activity that
+  // surrounds a throttled provider. Each is a session event (AgentSessionEvent)
+  // carrying the fields this worker reads — a fallback's from→to pair, the
+  // confirmed recoveries, and the auto-retry/compaction attempt counts.
+  session.on("retry_fallback_applied", (event) => {
+    const from = field(event, "from");
+    const to = field(event, "to");
+    if (typeof from === "string" && typeof to === "string") {
+      retryFallbacks = [{ from, to }, ...retryFallbacks];
+    }
+  });
+  session.on("retry_fallback_succeeded", () => {
+    retryFallbackSucceeded += 1;
+  });
+  session.on("auto_retry_start", () => {
+    autoRetryCount += 1;
+  });
+  session.on("auto_compaction_start", () => {
+    autoCompactionCount += 1;
   });
 
   session.on("agent_end", (event) => {
@@ -561,12 +694,13 @@ export async function runWorker(
     // Our own abort surfaces here on some paths; that is a kill, not a crash.
     if (killedBy === undefined && stoppedReason === undefined) {
       const detail = cause instanceof Error ? cause.message : String(cause);
-      return withSessionFacts({
+      return withMetrics(withSessionFacts({
         state: "failed",
         turns,
         spendUsd,
+        provider429Count,
         report: report === "" ? detail : report,
-      });
+      }));
     }
   } finally {
     done = true;
@@ -581,14 +715,15 @@ export async function runWorker(
   }
 
   if (stoppedReason !== undefined) {
-    return withSessionFacts({
+    return withMetrics(withSessionFacts({
       state: "stopped",
       turns,
       spendUsd,
+      provider429Count,
       report,
       stoppedReason,
       ...(claim === undefined ? {} : { prUrl: claim.prUrl, headSha: claim.headSha }),
-    });
+    }));
   }
 
   if (killedBy !== undefined) {
@@ -596,28 +731,30 @@ export async function runWorker(
     // survive the kill. Without them `shouldContinueAfterTurnsCap` sees no
     // artifacts and charges an implementation attempt for a cap kill that had
     // real work to continue from.
-    return withSessionFacts({
+    return withMetrics(withSessionFacts({
       state: "killed",
       turns,
       spendUsd,
+      provider429Count,
       report,
       killedBy,
       ...(claim === undefined ? {} : { prUrl: claim.prUrl, headSha: claim.headSha }),
-    });
+    }));
   }
 
   // An explicit later verdict always wins: a worker that pushed green and then
   // stopped to ask a question means the question. The earlier claim is only
   // restored when the last thing said was not a verdict at all.
   if (claim !== undefined && !hasVerdictLine(report)) {
-    return withSessionFacts({ state: "pushed-green", ...claim, turns, spendUsd, report });
+    return withMetrics(withSessionFacts({ state: "pushed-green", ...claim, turns, spendUsd, provider429Count, report }));
   }
-  return withSessionFacts({
+  return withMetrics(withSessionFacts({
     ...deriveResult(report, o.repoSlug),
     turns,
     spendUsd,
+    provider429Count,
     report,
-  });
+  }));
 }
 
 /**
@@ -645,6 +782,21 @@ export function costUsdFromMessage(message: unknown): number | undefined {
   return any ? sum : undefined;
 }
 
+/**
+ * Is this assistant message a provider HTTP 429 the harness recorded mid-run?
+ *
+ * Live transcripts mark a rate-limited turn with `stopReason:"error"`,
+ * `errorStatus:429` and a message like `429 Provider returned error`, and the
+ * harness's in-session auto-retry usually swallows it — the run carries on and
+ * the 429 never surfaces as `lastError` (#573). Exported so a unit test can pin
+ * the signature without standing up a session; the count itself distinguishes a
+ * run that drowned in them from a run that hit one and recovered.
+ */
+export function provider429FromMessage(message: unknown): boolean {
+  if (field(message, "stopReason") !== "error") return false;
+  return field(message, "errorStatus") === 429;
+}
+
 /**
  * Read one property off an unvalidated harness event. The event union lives in
  * the peer dependency, so the worker narrows the handful of fields it reads