omp-conductor 0.15.13 → 0.16.1

package/src/worker.ts

@@ -10,6 +10,8 @@
  * sliding into a merge queue.
  */
 
+import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
 import { createSession, disposeSession, SessionAdmissionClosedError, type AgentSessionLike } from "./omp.ts";
 import type { GateShape, ReleaseBlockContext } from "./release-policy.ts";
 import type { Caps, ResolvedGrants, RunState } from "./types.ts";
@@ -99,6 +101,15 @@ export interface WorkerOpts {
    * the harness to pick, which is what an unconfigured project wants.
    */
   model?: string;
+  /**
+   * Absolute path to the fleet-owned omp settings overlay (#537): the YAML the
+   * daemon materialised from the project's `ompSettings` map (plus the retry
+   * keys derived from `modelFallbacks`, which is where #539's staging lives)
+   * under the run's session directory. Forwarded to `createSession`, which
+   * loads it through `Settings.init({ configFiles: [<path>] })`. Absent, no
+   * settings are staged and dispatch is byte-for-byte what it is today.
+   */
+  ompSettingsFile?: string;
   /**
    * Effective per-shape release grants for this session. A worker is refused
    * every shape whatever they say — see {@link SessionRole} — so this is passed
@@ -169,6 +180,37 @@ export interface WorkerResult {
   turns: number;
   spendUsd: number;
   report: string;
+  /** In-session HTTP 429 responses the session recorded (stopReason "error",
+   *  errorStatus 429), counted as the messages streamed in. A healthy run
+   *  reports 0; a run the harness retried through a barrel of rate limits
+   *  carries the number, which is what distinguishes provider-capacity (the
+   *  provider was throttling all along) from an ordinary failure (#573). */
+  provider429Count: number;
+  /**
+   * The model that actually wrote this run's messages, read from
+   * `AssistantMessage.model` on the newest assistant `message_end`. Present
+   * even for a run that never failed over — it is the durable answer to "which
+   * model wrote this" (#535 slice 1, from the message field rather than the
+   * payload-free `model_changed` event). Absent only when no assistant message
+   * carried a model.
+   */
+  model?: string;
+  /** The provider that wrote them, read from the same `AssistantMessage.provider`. */
+  provider?: string;
+  /** Every within-run model fallback the harness applied
+   *  (`retry_fallback_applied`), newest first. The `to` target is what a
+   *  settlement report names when a run swapped providers mid-run. */
+  retryFallbacks: { from: string; to: string }[];
+  /** `retry_fallback_succeeded` events: within-run fallbacks the harness
+   *  confirmed recovered on. */
+  retryFallbackSucceeded: number;
+  /** Assistant messages whose `retryRecovery.recovery === "model"` — the durable
+   *  transcript record of a within-run model swap. */
+  modelRecoveries: number;
+  /** `auto_retry_start` events: in-session provider retries the harness ran. */
+  autoRetryCount: number;
+  /** `auto_compaction_start` events: in-session context compactions. */
+  autoCompactionCount: number;
   killedBy?: KilledBy;
   /** Present only when an operator terminally stopped this run. */
   stoppedReason?: string;
@@ -281,12 +323,27 @@ export async function runWorker(
   const schedule = deps.schedule ?? scheduleWallClock;
 
   let session: AgentSessionLike;
+  // The per-issue reviewer brief (#542): when the staged omp settings turn the
+  // advisor on, drop a WATCHDOG.md rendered from this brief's own acceptance
+  // criteria into the worktree before the session exists — the advisor's
+  // watchdog discovery runs at session start, rooted at cwd, and a file that
+  // appears afterwards is a reviewer flying blind. The managed exclude block
+  // ignores that exact path (never `.omp/`), so it cannot reach the PR diff or
+  // a salvage commit. Best-effort: a reviewer brief is advisory, and a staging
+  // failure must not cost the run.
+  if (o.resume === undefined && o.ompSettingsFile !== undefined) {
+    stageIssueWatchdog(o.cwd, o.brief, o.ompSettingsFile);
+  }
   try {
     session = await deps.createSession({
       cwd: o.cwd,
       ...(o.sessionDir === undefined ? {} : { sessionDir: o.sessionDir }),
       ...(o.model === undefined ? {} : { model: o.model }),
       ...(o.resume === undefined ? {} : { resume: o.resume }),
+      // The fleet-owned omp settings overlay (#537): carry the staged overlay
+      // path to the session so it loads the project's omp settings. Absent,
+      // nothing is staged and the harness discovers settings as it does today.
+      ...(o.ompSettingsFile === undefined ? {} : { ompSettingsFile: o.ompSettingsFile }),
       // Prevention half of #24: as a worker, structured file tools cannot leave
       // this worktree, and no release grant can ever reach this session (#122).
       role: "worker",
@@ -308,6 +365,12 @@ export async function runWorker(
         state: "stopped",
         turns: 0,
         spendUsd: 0,
+        provider429Count: 0,
+        retryFallbacks: [],
+        retryFallbackSucceeded: 0,
+        modelRecoveries: 0,
+        autoRetryCount: 0,
+        autoCompactionCount: 0,
         report: "",
         stoppedReason: "daemon shutdown began before the worker session started",
       };
@@ -324,18 +387,80 @@ export async function runWorker(
   // transcript it actually opened, and any model downgrade it announced. Read at
   // return time so a session that materialises either late is still reported
   // honestly.
-  const withSessionFacts = (result: WorkerResult): WorkerResult => {
+  // The within-run reliability surface this worker now records (#539): the
+  // resolved model/provider plus the fallback/retry/compaction events. Layered
+  // last, at return time, so every exit path reports the same shape without
+  // each spelling the metrics out by hand.
+  type ReliabilityKeys =
+    | "retryFallbacks"
+    | "retryFallbackSucceeded"
+    | "modelRecoveries"
+    | "autoRetryCount"
+    | "autoCompactionCount";
+  // Folded once, on whichever exit path actually runs: every return below
+  // passes through `withSessionFacts`, so the advisor's separately-recorded
+  // spend (#542) is added to exactly one result and reported to the daemon
+  // exactly once.
+  let advisorSpendFolded = false;
+  const withSessionFacts = (
+    result: Omit<WorkerResult, ReliabilityKeys>,
+  ): Omit<WorkerResult, ReliabilityKeys> => {
     const { sessionFile, modelFallbackMessage } = session;
+    if (!advisorSpendFolded) {
+      advisorSpendFolded = true;
+      // Advisor turns live in their own `__advisor*.jsonl` and never surface
+      // as primary `message_end`s, so without this read the run's spend — and
+      // therefore `caps.dailySpendUsd` — silently under-counts real provider
+      // consumption (the #46 shape). Read at the very end: the advisor's
+      // review of the final turn may still be landing.
+      const extra = advisorSpendUsd(sessionFile);
+      if (extra > 0) {
+        spendUsd += extra;
+        o.onSpend?.(spendUsd);
+      }
+    }
     return {
       ...result,
+      // Override whatever the exit path spelled: its literal captured the
+      // pre-fold `spendUsd`, and the advisor fold happened after.
+      spendUsd,
       ...(sessionFile === undefined ? {} : { sessionFile }),
       ...(modelFallbackMessage === undefined ? {} : { modelFallbackMessage }),
     };
   };
 
+  // The count fields always travel (0 for a clean run, so an absent field can
+  // never be misread); the resolved model/provider only when some assistant
+  // message actually carried them.
+  const withMetrics = (result: Omit<WorkerResult, ReliabilityKeys>): WorkerResult => ({
+    ...result,
+    ...(resolvedModel === undefined ? {} : { model: resolvedModel }),
+    ...(resolvedProvider === undefined ? {} : { provider: resolvedProvider }),
+    retryFallbacks,
+    retryFallbackSucceeded,
+    modelRecoveries,
+    autoRetryCount,
+    autoCompactionCount,
+  });
+
   let turns = 0;
   let spendUsd = 0;
+  let provider429Count = 0;
   let report = "";
+  // Which model/provider actually wrote the newest assistant message. Last
+  // assistant message wins: that is the durable answer even for a run that
+  // never failed over (#535 slice 1, read off the message field which is where
+  // the resolved model actually lives).
+  let resolvedModel: string | undefined;
+  let resolvedProvider: string | undefined;
+  // Harness reliability surface (#539): within-run provider failover and the
+  // retry/compaction activity that surrounds it, all of it events this worker
+  // does not yet subscribe to but that the run row and settlement report want.
+  let retryFallbacks: { from: string; to: string }[] = [];
+  let retryFallbackSucceeded = 0;
+  let modelRecoveries = 0;
+  let autoRetryCount = 0;
+  let autoCompactionCount = 0;
   // The newest COMPLETE `pushed-green` verdict this session emitted. Tracked
   // apart from `report` because `report` is deliberately the newest non-empty
   // text — a run cut off mid-sentence must still report what it said last —
@@ -497,6 +622,48 @@ export async function runWorker(
       spendUsd += cost;
       o.onSpend?.(spendUsd);
     }
+
+    // Count the provider rate limits the harness retried in-session (#573). A
+    // run that drowns in 429s records `stopReason:"error", errorStatus:429`
+    // dozens of times and never surfaces one as `lastError` — omp swallowed
+    // every retry — so `unknown` and the provider failover chain (#286) never
+    // see them. Counted here, alongside spend, so the classifier can tell
+    // "the provider was throttling the whole run" from an ordinary failure.
+    if (provider429FromMessage(message)) provider429Count += 1;
+
+    // The resolved model and provider live on the message, not on any event
+    // (#539). Last assistant message wins, which is the run's durable answer
+    // even when it never failed over.
+    const model = field(message, "model");
+    if (typeof model === "string" && model !== "") resolvedModel = model;
+    const provider = field(message, "provider");
+    if (typeof provider === "string" && provider !== "") resolvedProvider = provider;
+    // A within-run model swap the harness persisted into the transcript rather
+    // than only emitting as a transient event. Recovery kind "model" is the
+    // durable spelling of the same thing `retry_fallback_applied` says.
+    if (field(field(message, "retryRecovery"), "recovery") === "model") modelRecoveries += 1;
+  });
+
+  // The harness reliability events the run row and settlement report now want
+  // (#539): within-run model fallback, and the retry/compaction activity that
+  // surrounds a throttled provider. Each is a session event (AgentSessionEvent)
+  // carrying the fields this worker reads — a fallback's from→to pair, the
+  // confirmed recoveries, and the auto-retry/compaction attempt counts.
+  session.on("retry_fallback_applied", (event) => {
+    const from = field(event, "from");
+    const to = field(event, "to");
+    if (typeof from === "string" && typeof to === "string") {
+      retryFallbacks = [{ from, to }, ...retryFallbacks];
+    }
+  });
+  session.on("retry_fallback_succeeded", () => {
+    retryFallbackSucceeded += 1;
+  });
+  session.on("auto_retry_start", () => {
+    autoRetryCount += 1;
+  });
+  session.on("auto_compaction_start", () => {
+    autoCompactionCount += 1;
   });
 
   session.on("agent_end", (event) => {
@@ -561,12 +728,13 @@ export async function runWorker(
     // Our own abort surfaces here on some paths; that is a kill, not a crash.
     if (killedBy === undefined && stoppedReason === undefined) {
       const detail = cause instanceof Error ? cause.message : String(cause);
-      return withSessionFacts({
+      return withMetrics(withSessionFacts({
         state: "failed",
         turns,
         spendUsd,
+        provider429Count,
         report: report === "" ? detail : report,
-      });
+      }));
     }
   } finally {
     done = true;
@@ -581,14 +749,15 @@ export async function runWorker(
   }
 
   if (stoppedReason !== undefined) {
-    return withSessionFacts({
+    return withMetrics(withSessionFacts({
       state: "stopped",
       turns,
       spendUsd,
+      provider429Count,
       report,
       stoppedReason,
       ...(claim === undefined ? {} : { prUrl: claim.prUrl, headSha: claim.headSha }),
-    });
+    }));
   }
 
   if (killedBy !== undefined) {
@@ -596,28 +765,30 @@ export async function runWorker(
     // survive the kill. Without them `shouldContinueAfterTurnsCap` sees no
     // artifacts and charges an implementation attempt for a cap kill that had
     // real work to continue from.
-    return withSessionFacts({
+    return withMetrics(withSessionFacts({
       state: "killed",
       turns,
       spendUsd,
+      provider429Count,
       report,
       killedBy,
       ...(claim === undefined ? {} : { prUrl: claim.prUrl, headSha: claim.headSha }),
-    });
+    }));
   }
 
   // An explicit later verdict always wins: a worker that pushed green and then
   // stopped to ask a question means the question. The earlier claim is only
   // restored when the last thing said was not a verdict at all.
   if (claim !== undefined && !hasVerdictLine(report)) {
-    return withSessionFacts({ state: "pushed-green", ...claim, turns, spendUsd, report });
+    return withMetrics(withSessionFacts({ state: "pushed-green", ...claim, turns, spendUsd, provider429Count, report }));
   }
-  return withSessionFacts({
+  return withMetrics(withSessionFacts({
     ...deriveResult(report, o.repoSlug),
     turns,
     spendUsd,
+    provider429Count,
     report,
-  });
+  }));
 }
 
 /**
@@ -645,6 +816,21 @@ export function costUsdFromMessage(message: unknown): number | undefined {
   return any ? sum : undefined;
 }
 
+/**
+ * Is this assistant message a provider HTTP 429 the harness recorded mid-run?
+ *
+ * Live transcripts mark a rate-limited turn with `stopReason:"error"`,
+ * `errorStatus:429` and a message like `429 Provider returned error`, and the
+ * harness's in-session auto-retry usually swallows it — the run carries on and
+ * the 429 never surfaces as `lastError` (#573). Exported so a unit test can pin
+ * the signature without standing up a session; the count itself distinguishes a
+ * run that drowned in them from a run that hit one and recovered.
+ */
+export function provider429FromMessage(message: unknown): boolean {
+  if (field(message, "stopReason") !== "error") return false;
+  return field(message, "errorStatus") === 429;
+}
+
 /**
  * Read one property off an unvalidated harness event. The event union lives in
  * the peer dependency, so the worker narrows the handful of fields it reads
@@ -655,6 +841,168 @@ function field(source: unknown, key: string): unknown {
   return Reflect.get(source, key);
 }
 
+/** Advisor transcripts are recorded under this reserved stem beside the session's. */
+const ADVISOR_TRANSCRIPT_PREFIX = "__advisor";
+
+/**
+ * Whether a staged omp-settings overlay (the file `materializeOmpSettings`
+ * writes under the run's session directory) turns the omp advisor on.
+ *
+ * The overlay is conductor's own YAML — written by `yaml.stringify` over a map
+ * this package controls — so this is deliberately not a general YAML parser:
+ * it scans for an `advisor:` mapping and asks whether one of its direct
+ * `enabled:` keys reads the literal `true`. Handles both the block form the
+ * overlay is written in (`advisor:\n  enabled: true`) and an inline flow map
+ * (`advisor: { enabled: true }`); everything else — `enabled: false`, no
+ * `advisor` key, an absent overlay — answers false. The staged settings are
+ * what the harness itself resolves, so this is the same truth the session
+ * acts on, not a parallel decode. Exported so the staging decision in
+ * {@link runWorker} is pinned by a unit test.
+ */
+export function overlayEnablesAdvisor(overlayText: string): boolean {
+  const lines = overlayText.split(/\r?\n/);
+  for (let i = 0; i < lines.length; i++) {
+    const match = /^(\s*)advisor\s*:\s*(.*)$/.exec(lines[i] ?? "");
+    if (match === null) continue;
+    if (/\benabled\s*:\s*true\b/.test(match[2] ?? "")) return true;
+    const indent = match[1] ?? "";
+    for (let j = i + 1; j < lines.length; j++) {
+      const line = lines[j];
+      if (line === undefined || line.trim() === "") continue;
+      // A line at or shallower than `advisor`'s own indent ends its block.
+      if (!line.startsWith(`${indent} `) && !line.startsWith(`${indent}\t`)) break;
+      if (/^\s*enabled\s*:\s*true\s*$/.test(line)) return true;
+    }
+  }
+  return false;
+}
+
+/** A heading that begins a brief's acceptance section, whatever the brief's casing. */
+const ACCEPTANCE_HEADING = /^#{1,4}\s+acceptance\s+criteri(?:a|on)\s*$/im;
+
+/**
+ * Render the per-issue reviewer brief (#542): the acceptance section of one
+ * worker brief, wrapped so the omp advisor checks the primary against *this
+ * run's own* criteria rather than generic taste. Returns `undefined` when the
+ * brief carries no acceptance section — nothing useful to review against.
+ *
+ * The section runs from the first heading mentioning acceptance criteria to
+ * the next heading of any level (or the end of the brief). Exported so a unit
+ * test can pin the rendering without standing up a session.
+ */
+export function renderIssueWatchdog(brief: string): string | undefined {
+  const lines = brief.split(/\r?\n/);
+  let start = -1;
+  for (let i = 0; i < lines.length; i++) {
+    if (ACCEPTANCE_HEADING.test(lines[i] ?? "")) {
+      start = i + 1;
+      break;
+    }
+  }
+  if (start === -1) return undefined;
+  const body: string[] = [];
+  for (let i = start; i < lines.length; i++) {
+    const line = lines[i];
+    if (line === undefined || /^#{1,4}\s/.test(line)) break;
+    body.push(line);
+  }
+  const trimmed = body.join("\n").replace(/\n{3,}/g, "\n\n").trim();
+  if (trimmed === "") return undefined;
+  return (
+    "# Issue watchdog (rendered from this run's brief)\n\n" +
+    "This run's own acceptance criteria. Review the worker against these — not generic taste: " +
+    "a criterion is met only when the transcript or workspace shows it actually verified, and a " +
+    "worker claiming completion without evidence is a concern at least.\n\n" +
+    "## Acceptance criteria\n\n" +
+    `${trimmed}\n`
+  );
+}
+
+/**
+ * Stage the per-issue `WATCHDOG.md` into the worktree when the staged omp
+ * settings turn the advisor on (#542).
+ *
+ * The omp advisor's watchdog discovery is cwd-rooted, so per-issue guidance
+ * cannot ride the out-of-tree settings overlay: it must live inside the
+ * worktree, at `<cwd>/.omp/WATCHDOG.md`, which the managed exclude block
+ * ignores by that exact path (never the `.omp/` directory) so it can never
+ * reach the diff a worker ships or a salvage commit. Best-effort on purpose:
+ * a reviewer brief is advisory, and a worktree too broken to take it will fail
+ * the run on its own terms. Exported so the runWorker staging decision is
+ * testable without a session.
+ */
+export function stageIssueWatchdog(cwd: string, brief: string, overlayFile: string): boolean {
+  let overlayText: string;
+  try {
+    overlayText = readFileSync(overlayFile, "utf8");
+  } catch {
+    return false;
+  }
+  if (!overlayEnablesAdvisor(overlayText)) return false;
+  const watchdog = renderIssueWatchdog(brief);
+  if (watchdog === undefined) return false;
+  try {
+    const dir = join(cwd, ".omp");
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, "WATCHDOG.md"), watchdog);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Total USD cost of one run's advisor turns (#542).
+ *
+ * Advisor turns are recorded to a separate `__advisor*.jsonl` beside the
+ * session transcript — the primary session never sees them as `message_end`s —
+ * so without this a run with an advisor under-counts `spendUsd` and the daily
+ * cap is theater, the #46 failure mode. Reads every advisor transcript in the
+ * session's advisor directory and sums the same `usage.cost` blocks
+ * {@link costUsdFromMessage} reads off primary messages: one spelling, so the
+ * two cannot drift. Unreadable or absent transcripts bill nothing — the run
+ * with no advisor stays a zero, and a corrupt advisor log must not crash the
+ * settlement.
+ */
+export function advisorSpendUsd(sessionFile: string | undefined): number {
+  if (sessionFile === undefined || !sessionFile.endsWith(".jsonl")) return 0;
+  // The harness records advisor transcripts in the directory named after the
+  // primary transcript stem (`<dir>/<stem>/__advisor*.jsonl`); slicing the
+  // suffix is exactly what the harness's own cost loader does.
+  const dir = sessionFile.slice(0, -".jsonl".length);
+  let names: string[];
+  try {
+    names = readdirSync(dir);
+  } catch {
+    return 0;
+  }
+  let total = 0;
+  for (const name of names) {
+    if (!name.startsWith(ADVISOR_TRANSCRIPT_PREFIX) || !name.endsWith(".jsonl")) continue;
+    let text: string;
+    try {
+      text = readFileSync(join(dir, name), "utf8");
+    } catch {
+      continue;
+    }
+    for (const line of text.split("\n")) {
+      if (line.trim() === "") continue;
+      let entry: unknown;
+      try {
+        entry = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      if (field(entry, "type") !== "message") continue;
+      const message = field(entry, "message");
+      if (field(message, "role") !== "assistant") continue;
+      const cost = costUsdFromMessage(message);
+      if (cost !== undefined) total += cost;
+    }
+  }
+  return total;
+}
+
 /** Flatten an assistant message's content blocks to their plain text. */
 /**
  * The newest assistant text, flattened out of whatever block shape the harness

package/src/worktree.ts

@@ -156,8 +156,20 @@ const EXCLUDE_END = "# <<< omp-conductor";
  * the 2026-08-07 incident's exact shape (`.scratch82/env.sh`). Broader
  * conventions belong in a repo's own `.gitignore`, where its operator chooses
  * them, rather than being imposed by whatever dispatcher happens to be driving.
+ *
+ * The advisor watchdog (`#542`) is the deliberate exception, and the exact-path
+ * spelling is the whole point: with the advisor on, the dispatcher writes a
+ * per-issue `<worktree>/.omp/WATCHDOG.md` from the brief's acceptance criteria
+ * so the mid-run reviewer checks the worker against its own issue — and that
+ * file must never reach the diff a worker ships or the salvage commit. The
+ * standing rule says an ignored *new* file is invisible to salvage, so a name
+ * that could plausibly be a deliverable must never appear; `.omp/WATCHDOG.md`
+ * is the narrowest name that can. Only this exact file is excluded, never the
+ * `.omp/` directory — a directory-wide ignore would hide any future
+ * deliverable an operator legitimately places under `.omp/` (the very trap the
+ * `.scratch*` history describes), and the blind spot stays one path wide.
  */
-const LOCAL_EXCLUDE = [".scratch*/"];
+const LOCAL_EXCLUDE = [".scratch*/", ".omp/WATCHDOG.md"];
 
 /**
  * Adds the managed block to an `info/exclude`, preserving everything else.