omp-conductor 0.18.2 → 0.19.0

package/src/doctor.ts

@@ -61,6 +61,7 @@ import {
   sessionDirForCwd,
   telegramStateDir,
 } from "./fleet.ts";
+import { pauseInstance } from "./pause.ts";
 import type { TelegramHealth } from "./status-render.ts";
 import {
   claimedTelegramTopics,
@@ -101,6 +102,27 @@ import {
 } from "./orchestrator-tick.ts";
 import type { ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
 import { DEFAULT_ARM_PROOF, type ArmProof } from "./types.ts";
+import {
+  DEFAULT_DEPS as UPGRADE_DEPS,
+  expectedHerdrSource,
+  inspectSurfaces,
+  releaseIdentity,
+  type InstalledSurfaces,
+} from "./upgrade.ts";
+import {
+  TELEGRAM_PACKAGE,
+  checkTelegramFreshness,
+  type TelegramFreshness,
+} from "./telegram-freshness.ts";
+import { judgeSpendTelemetry, spendTelemetryDetail } from "./spend-telemetry.ts";
+
+/** One read of this host's installed surfaces: the three identities, or why
+ *  they could not be read. A failed read is a finding (`warn`, "unverified"),
+ *  never a thrown doctor run and never a silent pass (#904). */
+export type SurfaceRead =
+  | { ok: true; surfaces: InstalledSurfaces }
+  | { ok: false; detail: string };
+
 /**
  * `doctor`'s finding vocabulary. `pass`/`warn`/`fail` — a warning renders the
  * same but never flips the exit code; exit is nonzero iff any finding fails.
@@ -131,8 +153,18 @@ export interface DoctorReport {
   findings: Finding[];
 }
 
-/** How many of the most recent completed runs are sampled for spend telemetry. */
+/** How many *working* runs the spend-telemetry judgement wants. */
 export const SPEND_SAMPLE_RUNS = 5;
+/**
+ * How many rows to read to find them (#970).
+ *
+ * Over-sampled deliberately: the judgement discards 0-turn runs, and on this
+ * project 26 of 48 zero-spend rows are exactly those. Reading only
+ * `SPEND_SAMPLE_RUNS` rows would let a burst of administrative kills push the
+ * real evidence out of the window and report "nothing to judge yet" on a fleet
+ * that had plenty to judge.
+ */
+export const SPEND_SAMPLE_ROWS = SPEND_SAMPLE_RUNS * 4;
 
 /** The past incidents this doctor flags, quoted in the finding so an operator
  *  who never lived them knows which failure mode the check exists to prevent. */
@@ -144,10 +176,6 @@ const INCIDENTS = {
   spend:
     "spend telemetry was once absent, so the USD cap never fired — $0.00 spend is not proof of no spend",
   ghauth: "gh auth expired under a live daemon and every tracker call failed silently",
-  workerAcl:
-    "the #835 incident: a setup host granted the worker's path ACLs before restarting the fleet, an OMP startup chmod'd " +
-    "the agent config dir back to 0700 and rewrote the ACL mask, and the next two admitted workers died on EACCES " +
-    "before connecting — the named ACL entry was still there, only its effective permissions were gone",
 } as const;
 
 // ------------------------------------------------------------------ dependencies
@@ -172,10 +200,12 @@ export interface CanonicalUnits {
 }
 
 /** One sampled run row: `state` keeps spend from in-flight rows, which are not
- *  a telemetry statement yet. */
+ *  a telemetry statement yet, and `turns` separates a run that reported nothing
+ *  from one that genuinely did nothing (#970). */
 export interface RunSpendRow {
   state: RunState;
   spendUsd: number;
+  turns: number;
 }
 
 /** Injectable seams. Every field defaults to the production wiring, which
@@ -222,6 +252,21 @@ export interface DoctorDeps {
   canonicalUnits?: (project: ProjectConfig, cfg: ConductorConfig) => CanonicalUnits;
   /** Whether herdr is installed on this host (a `herdr` on PATH). */
   herdrInstalled?: () => boolean;
+  /**
+   * Whether a worker session could load its harness at all (#910), exercised
+   * the way a worker does: the installed peer resolved from this package's own
+   * directory under `bun --no-install`, so neither import can fall through to
+   * Bun's ambient cache. `ok: false` means no worker on this host can start.
+   */
+  workerHarness?: () => Promise<{ ok: true; version: string } | { ok: false; detail: string }>;
+  /**
+   * The pause sentinel for one project as an instance (#938): who set it, why,
+   * when, and — for a fence whose lifetime is one process's — the owning pid.
+   */
+  pauseFence?: (project?: string) => { source: string; reason?: string; since: number; owner?: number } | undefined;
+  /** Whether one pid is a live process. Injected so the abandoned-fence
+   *  finding is testable without spawning anything. */
+  pidLive?: (pid: number) => boolean;
   /** Live `herdr --session <s> agent list`, parsed through the tick's own
    *  parser (the same "one JSON line on stdout" contract recover.sh reads). */
   herdrAgents?: (session: string) => HerdrAgentList;
@@ -274,6 +319,14 @@ export interface DoctorDeps {
   /** The fleet agent name the tick config of one project names, or undefined
    *  when there is no (readable) tick — the expected live herdr pane identity. */
   tickAgentName?: (project: ProjectConfig) => string | undefined;
+  /** The three installed identities this host carries, read through
+   *  `upgrade`'s own seam; a failed read is a named finding, never a throw. */
+  installedSurfaces?: () => Promise<SurfaceRead>;
+  /** The `omp-telegram` install/daemon/published triple (#961). */
+  telegramFreshness?: () => Promise<TelegramFreshness>;
+  /** The commit one published version was cut from, for judging the herdr
+   *  plugin's pin; `undefined` when the registry cannot be read. */
+  releaseGitHead?: (version: string) => Promise<string | undefined>;
   /** Clock, so a run is deterministic in tests. */
   now?: () => number;
   /** The one opt-in side effect: send one self-identified Telegram probe. */
@@ -446,8 +499,8 @@ function defaultRecentRuns(project: string, limit: number): RunSpendRow[] {
   try {
     const placeholders = [...LIVE_STATES].map(() => "?").join(", ");
     const rows = database
-      .query<{ state: string; spendUsd: number }, [string, ...string[], number]>(
-        `SELECT state, spendUsd FROM runs
+      .query<{ state: string; spendUsd: number; turns: number }, [string, ...string[], number]>(
+        `SELECT state, spendUsd, turns FROM runs
           WHERE project = ? AND state NOT IN (${placeholders})
           ORDER BY startedAt DESC, rowid DESC
           LIMIT ?`,
@@ -457,7 +510,9 @@ function defaultRecentRuns(project: string, limit: number): RunSpendRow[] {
     for (const row of rows) {
       const state = row.state as RunState;
       // A state vocabulary the store never had is not "completed".
-      if (!LIVE_STATES.includes(state)) out.push({ state, spendUsd: row.spendUsd });
+      if (!LIVE_STATES.includes(state)) {
+        out.push({ state, spendUsd: row.spendUsd, turns: row.turns });
+      }
     }
     return out;
   } catch {
@@ -1041,22 +1096,40 @@ async function telegramProbe(probes: Probes, project: ProjectConfig | undefined,
   );
 }
 
-/** Spend telemetry: warn when the last K completed runs all recorded $0.00 —
- *  the USD cap cannot fire on zeros ($0.00 is not proof of no spend). */
+/**
+ * Spend telemetry, through the shared judgement (#970).
+ *
+ * The predicate this replaced fired only when *every* sampled run reported
+ * $0.00, and counted 0-turn kills as evidence. Replayed over this fleet's whole
+ * history it would have fired 14 times while staying silent through 37 windows
+ * that had lost a majority of their telemetry — so partial loss, the common
+ * case, was the invisible one. See `spend-telemetry.ts` for the measurements.
+ */
 function spendProbe(rows: RunSpendRow[], limit: number): Finding {
-  if (rows.length < limit) {
-    return passFinding("spend-telemetry", `${rows.length} completed run(s) observed — fewer than ${limit}, nothing to judge yet`);
-  }
-  const window = rows.slice(0, limit);
-  if (window.every((r) => r.spendUsd === 0)) {
+  const verdict = judgeSpendTelemetry(rows, limit);
+  const detail = spendTelemetryDetail(verdict);
+  if (detail !== undefined) {
     return warnFinding(
       "spend-telemetry",
-      `the last ${limit} completed runs all recorded $0.00 spend — ${INCIDENTS.spend}`,
+      `${detail} — ${INCIDENTS.spend}`,
       "verify harness spend reporting (the per-run spendUsd column / `omp usage --json`); a USD cap on top of zeros never fires",
     );
   }
-  const total = window.reduce((sum, r) => sum + r.spendUsd, 0);
-  return passFinding("spend-telemetry", `spend observed on the last ${window.length} completed runs ($${total.toFixed(2)} total)`);
+  if (verdict.kind === "insufficient") {
+    return passFinding(
+      "spend-telemetry",
+      `${verdict.worked} completed run(s) did any work — fewer than ${verdict.needed}, nothing to judge yet`,
+    );
+  }
+  // Healthy. Naming the metered share rather than only the total: a window with
+  // one missing row is fine and saying so is how a reader learns the baseline
+  // is not zero.
+  const healthy = verdict as Extract<typeof verdict, { kind: "healthy" }>;
+  return passFinding(
+    "spend-telemetry",
+    `spend observed on ${healthy.worked - healthy.missing} of the last ${healthy.worked} working runs ` +
+      `($${healthy.totalUsd.toFixed(2)} total)`,
+  );
 }
 
 /** Live `herdr --session <session> agent list` through the project's own
@@ -1411,6 +1484,45 @@ function telegramPlumbingProbe(probes: Probes, p: ProjectConfig): Finding {
  * pages instead of recovering. Absent ~ the unrecoverable default; an explicit
  * `true` is a deliberate (desktop) choice and only warned.
  */
+/**
+ * A dispatch fence whose owning process is gone (#938).
+ *
+ * Only a fence that *declared* an owner can be judged: `setPaused` records
+ * `owner=` for a setup transaction, whose lifetime is that process's, and
+ * deliberately not for an operator `hold` or the fail-closed hold a failed
+ * apply leaves behind — those are meant to outlive every process, and calling
+ * one "abandoned" would be worse than saying nothing about it.
+ *
+ * Measured 2026-08-21T13:24Z: an operator abandoned a setup that looked hung;
+ * the policy change had applied, and dispatch sat under
+ * `source=setup reason="setup apply fence"` until somebody thought to run
+ * `resume`. Nothing anywhere said the process that wrote it was gone.
+ *
+ * This never clears anything, and that is deliberate. A pid is reusable, a
+ * paused fleet is the safe state, and resuming dispatch nobody authorised is a
+ * worse outcome than a fence that needs one command. So it names the command.
+ */
+function fenceProbe(probes: Probes, project: string | undefined): Finding {
+  const id = project === undefined ? "dispatch-fence" : `dispatch-fence:${project}`;
+  const fence = probes.pauseFence(project);
+  if (fence === undefined) return passFinding(id, "dispatch is not held by a fence");
+  const why = fence.reason === undefined ? "" : ` — "${fence.reason}"`;
+  if (fence.owner === undefined) {
+    // An operator hold, or the durable hold a failed apply leaves on purpose.
+    // Both are somebody's decision rather than a leak, whatever their age.
+    return passFinding(id, `dispatch held by ${fence.source}${why} (no owning process; nothing to expire)`);
+  }
+  if (probes.pidLive(fence.owner)) {
+    return passFinding(id, `dispatch held by a live ${fence.source} (pid ${fence.owner})`);
+  }
+  return warnFinding(
+    id,
+    `dispatch is held by ${fence.source}${why}, whose process (pid ${fence.owner}) is gone: the fence outlived ` +
+      `the transaction that set it, and has stood since ${new Date(fence.since).toISOString()}`,
+    `nothing here clears it — confirm no setup is running, then \`omp-conductor resume${project === undefined ? "" : ` --project ${project}`}\``,
+  );
+}
+
 function herdrResumeProbe(probes: Probes): Finding {
   if (!probes.herdrInstalled()) {
     return passFinding("herdr-resume", "herdr not installed — nothing to check");
@@ -1505,6 +1617,169 @@ function parseEnvKey(text: string, key: string): string | undefined {
   return undefined;
 }
 
+/**
+ * Install-surface parity (#904). omp-conductor installs onto three surfaces —
+ * the Bun-global CLI/daemon tree, the omp plugin, and the herdr recovery
+ * plugin — and only a live `upgrade` invocation ever compared them. On a
+ * fleet whose installs are manual they diverge silently: measured on this
+ * host on 2026-08-22, the omp plugin sat on the withdrawn 0.18.1 release for
+ * about a day beside a 0.18.0 daemon, with no finding, no status line and no
+ * escalation.
+ *
+ * The question is inter-surface agreement on the host, never staleness
+ * against the registry: a fleet may deliberately sit on an older release,
+ * but never on two at once. `expectedGitHead` is the commit the CLI's own
+ * version was published from, so the herdr pin can be judged against the
+ * same release; `undefined` means the registry could not be read and the pin
+ * is reported as unverified rather than as a mismatch.
+ */
+/**
+ * The bootstrap deadlock (#910), which is a conjunction rather than a fault:
+ *
+ *   - no worker on this host can start, because the *installed* code cannot
+ *     load its harness; and
+ *   - the fix for that ships through a release, and a release needs a merged
+ *     worker pull request — so the broken install is what blocks its own
+ *     replacement.
+ *
+ * Kept as its own finding, never folded into the stale-setup ones, because the
+ * two states call for opposite actions. "Host runtime differs — run setup host"
+ * is *correct advice* for a stale install and *useless* here: no amount of
+ * re-running setup replaces the installed package, and diagnosing that cost
+ * real time on 2026-08-20..22. So this names the one command that breaks the
+ * cycle (#908's bootstrap identity), and an ordinary stale host still reads
+ * exactly as it did.
+ */
+function bootstrapDeadlockProbe(
+  harness: { ok: true; version: string } | { ok: false; detail: string },
+): Finding {
+  if (harness.ok) {
+    return passFinding("bootstrap-deadlock", `workers can load the installed harness (${harness.version})`);
+  }
+  return failFinding(
+    "bootstrap-deadlock",
+    `no worker can start: the installed conductor cannot load its harness — ${harness.detail}. ` +
+      "A release cannot publish the fix either, because publishing needs a merged worker pull request, " +
+      "so the installed package blocks its own replacement",
+    "install the fix by exact commit instead of by version: `omp-conductor upgrade --bootstrap <sha> " +
+      "--source <checkout at that sha>` (its checks run against the source first, and it refuses rather " +
+      "than installing a partly verified tree)",
+  );
+}
+
+/**
+ * Is the plugin a mandated conductor contract runs on actually current? (#961)
+ *
+ * Sibling to {@link surfaceParityProbe}, and deliberately a separate finding:
+ * that one is about the three surfaces of *this* package agreeing with each
+ * other, this one is about the one peer the floor's reply-in-topic instruction
+ * depends on being new enough to honour it.
+ *
+ * A `warn`, never a `fail`. Nothing here is broken on this host's own terms —
+ * the fleet dispatches, merges and reports fine on a stale plugin. What breaks
+ * is one documented instruction, and the remedy is an install the operator owns,
+ * so failing the run would make `doctor` red for something conductor must not
+ * fix itself.
+ */
+export function telegramFreshnessProbe(freshness: TelegramFreshness): Finding {
+  const id = "telegram-plugin-freshness";
+  const { installed, daemon, published } = freshness.surfaces;
+  const named =
+    `installed=${installed.kind === "version" ? installed.version : installed.kind}, ` +
+    `daemon=${daemon.kind === "version" ? daemon.version : daemon.kind}, ` +
+    `published=${published.kind === "version" ? published.version : published.kind}`;
+  switch (freshness.state) {
+    case "current":
+      return passFinding(id, `${TELEGRAM_PACKAGE} is current (${named})`);
+    case "installed-behind":
+      return warnFinding(
+        id,
+        `${freshness.detail} (${named})`,
+        `install ${TELEGRAM_PACKAGE}@${published.kind === "version" ? published.version : "latest"} and restart its daemon; a targetless telegram_send needs the newer target ladder to reply in the topic a message arrived in (#882)`,
+      );
+    case "daemon-stale":
+      return warnFinding(
+        id,
+        `${freshness.detail} (${named})`,
+        `restart the ${TELEGRAM_PACKAGE} daemon so it serves the installed version`,
+      );
+    case "not-installed":
+      return warnFinding(
+        id,
+        `${freshness.detail} (${named})`,
+        `install ${TELEGRAM_PACKAGE}, or stop relying on Telegram delivery for this fleet`,
+      );
+    case "unknown":
+      // Not a finding about the plugin: a finding about this check. An
+      // unreachable registry must never read as "behind" *or* as "current".
+      return warnFinding(
+        id,
+        `${TELEGRAM_PACKAGE} freshness unverified: ${freshness.detail ?? "no surface answered"} (${named})`,
+        "no action needed if this host has no outbound npm access; otherwise re-run doctor when it does",
+      );
+  }
+}
+
+export function surfaceParityProbe(
+  read: SurfaceRead,
+  expectedGitHead: string | undefined,
+  herdrExpected: boolean,
+): Finding {
+  const id = "install-surfaces";
+  if (!read.ok) {
+    return warnFinding(
+      id,
+      `installed surfaces unreadable: ${read.detail} — parity between the CLI, the omp plugin and the herdr plugin is unverified`,
+      "check that `omp-conductor`, `omp` and (when this host runs herdr) `herdr` are on PATH for this user, then re-run doctor",
+    );
+  }
+  const { cliVersion, ompVersion, herdrSource } = read.surfaces;
+  const named = `cli=${cliVersion}, omp=${ompVersion ?? "absent"}, herdr=${herdrSource ?? "absent"}`;
+  // A proven disagreement first: two different releases live on one host is
+  // the fault this probe exists for, and it outranks anything absent.
+  if (ompVersion !== undefined && ompVersion !== cliVersion) {
+    return failFinding(
+      id,
+      `installed surfaces disagree: the omp plugin is ${ompVersion} while the CLI/daemon tree is ${cliVersion} (${named})`,
+      `run \`omp-conductor upgrade --to ${cliVersion}\` (or to the release this fleet should be on) so all three surfaces carry one identity`,
+    );
+  }
+  if (herdrExpected && herdrSource !== undefined && expectedGitHead !== undefined) {
+    if (herdrSource.startsWith("local:")) {
+      return warnFinding(
+        id,
+        `the herdr plugin is linked to a local checkout, not a released pin (${named})`,
+        `run \`omp-conductor upgrade --to ${cliVersion}\` to replace the link with the released pin, or keep the link deliberately while developing`,
+      );
+    }
+    const expected = expectedHerdrSource(expectedGitHead);
+    if (herdrSource !== expected) {
+      return failFinding(
+        id,
+        `installed surfaces disagree: the herdr plugin is pinned to ${herdrSource} while ${cliVersion} was published from ${expectedGitHead} (${named})`,
+        `run \`omp-conductor upgrade --to ${cliVersion}\` so the herdr recovery plugin matches the release the rest of the host runs`,
+      );
+    }
+  }
+  // Nothing disagrees. Anything absent is named as absent — a surface this
+  // host does not carry is not a mismatch, and a pin that could not be
+  // verified is not a pass in disguise.
+  const absent: string[] = [];
+  if (ompVersion === undefined) absent.push("omp plugin not installed — a tick cannot arm without it");
+  if (herdrExpected && herdrSource === undefined) absent.push("herdr plugin not installed — pane recovery cannot run");
+  if (herdrExpected && herdrSource !== undefined && expectedGitHead === undefined) {
+    absent.push(`herdr pin ${herdrSource} unverified — the registry did not answer what ${cliVersion} was published from`);
+  }
+  if (absent.length > 0) {
+    return warnFinding(
+      id,
+      `${absent.join("; ")} (${named})`,
+      `run \`omp-conductor upgrade --to ${cliVersion}\` to install every surface this host needs from one release`,
+    );
+  }
+  return passFinding(id, `one identity across every installed surface (${named})`);
+}
+
 /**
  * Run every probe and assemble the stable report.
  *
@@ -1574,6 +1849,17 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   findings.push(dbProbe(probes));
   findings.push(dbBackupProbe(probes, cfg));
   findings.push(await ghAuthProbe(probes, configuredRepos(cfg)));
+  // Install-surface parity is host-wide, like the unit and the store: one
+  // read of the three identities, judged once (#904).
+  const surfaces = await probes.installedSurfaces();
+  const cliVersion = surfaces.ok ? surfaces.surfaces.cliVersion : undefined;
+  const expectedGitHead = cliVersion === undefined ? undefined : await probes.releaseGitHead(cliVersion);
+  findings.push(surfaceParityProbe(surfaces, expectedGitHead, probes.herdrInstalled()));
+  // Host-wide for the same reason: one plugin install serves every project, so
+  // it is judged once rather than repeated per project (#961).
+  findings.push(telegramFreshnessProbe(await probes.telegramFreshness()));
+  // Host-wide like the surfaces: one install, one harness, one answer (#910).
+  findings.push(bootstrapDeadlockProbe(await probes.workerHarness()));
   // The run-session root is one host-wide fact — where every run's session dir
   // (and the omp settings overlay inside it) lands — probed once and shared
   // across the per-project omp-settings findings.
@@ -1582,7 +1868,7 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   // The collected run sample across the resolved project set, newest-first;
   // spend telemetry is a single host-wide finding, not one per project.
   const spendRows: RunSpendRow[] = [];
-  for (const p of projects) spendRows.push(...probes.recentRuns(p.name, SPEND_SAMPLE_RUNS));
+  for (const p of projects) spendRows.push(...probes.recentRuns(p.name, SPEND_SAMPLE_ROWS));
 
   // Per-project probes run for every resolved project, each named in its
   // finding so a multi-project run stays legible. When nothing resolved
@@ -1630,9 +1916,13 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   if (projects.length === 0) {
     findings.push(timezoneProbe(undefined));
     findings.push(await telegramProbe(probes, undefined, checkedAt));
+    findings.push(fenceProbe(probes, undefined));
   } else {
     for (const p of projects) findings.push(timezoneProbe(p));
     for (const p of projects) findings.push(await telegramProbe(probes, p, checkedAt));
+    // Per project, because the sentinel is: one fleet's fence must never be
+    // reported against another's name.
+    for (const p of projects) findings.push(fenceProbe(probes, p.name));
   }
   findings.push(spendProbe(spendRows, SPEND_SAMPLE_RUNS));
 
@@ -1654,6 +1944,41 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   };
 }
 
+/**
+ * Exercise the worker's real import contract out of process (#910).
+ *
+ * `harness-loader.ts` is the module a worker session loads its harness through,
+ * and it is runnable — so this runs exactly that, with `--no-install`, and reads
+ * its verdict rather than re-deriving one. Anything else would be a second
+ * opinion about the launch path, which is how a doctor comes to disagree with
+ * the thing it is checking.
+ */
+async function defaultWorkerHarness(): Promise<{ ok: true; version: string } | { ok: false; detail: string }> {
+  const loader = join(import.meta.dir, "harness-loader.ts");
+  try {
+    const child = Bun.spawn(["bun", "--no-install", loader], {
+      stdin: "ignore",
+      stdout: "pipe",
+      stderr: "pipe",
+      env: process.env,
+    });
+    const stdout = new Response(child.stdout).text();
+    const stderr = new Response(child.stderr).text();
+    const code = await child.exited;
+    if (code !== 0) {
+      const detail = ((await stderr).trim() || (await stdout).trim()).split("\n")[0] ?? `exit ${code}`;
+      return { ok: false, detail };
+    }
+    const parsed: unknown = JSON.parse((await stdout).trim());
+    const version = parsed !== null && typeof parsed === "object" ? Reflect.get(parsed, "version") : undefined;
+    return typeof version === "string" && version.length > 0
+      ? { ok: true, version }
+      : { ok: false, detail: "the harness loaded but reported no version" };
+  } catch (err) {
+    return { ok: false, detail: err instanceof Error ? err.message : String(err) };
+  }
+}
+
 /** The default wiring — every production transport the rest of the package uses. */
 export function defaultProbes(): Probes {
   return {
@@ -1674,6 +1999,9 @@ export function defaultProbes(): Probes {
     telegramSend: telegramReportSend,
     canonicalUnits: defaultCanonicalUnits,
     herdrInstalled: () => Bun.which("herdr") !== null,
+    pauseFence: (project) => pauseInstance(project),
+    workerHarness: defaultWorkerHarness,
+    pidLive: (pid) => pidAlive(pid),
     herdrAgents: defaultHerdrAgents,
     herdrSession: () => resolveHerdrSessionWithBridge(),
     herdrConfig: defaultHerdrConfig,
@@ -1721,6 +2049,35 @@ export function defaultProbes(): Probes {
       if (tick.kind !== "ok") return undefined;
       return tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME;
     },
+    installedSurfaces: async () => {
+      // `upgrade`'s own reader, so doctor cannot disagree with the transaction
+      // about what is installed. A host with no herdr is asked only about the
+      // two surfaces it has, rather than failing the whole read (#904).
+      try {
+        const surfaces = await inspectSurfaces(UPGRADE_DEPS, { readHerdr: Bun.which("herdr") !== null });
+        return { ok: true, surfaces };
+      } catch (err) {
+        return { ok: false, detail: messageOf(err) };
+      }
+    },
+    telegramFreshness: () =>
+      // The module's own reader, so `doctor` and `status` cannot disagree about
+      // what is installed — the same one-seam rule `installedSurfaces` follows.
+      checkTelegramFreshness({
+        run: async (cmd, args) => {
+          const r = await UPGRADE_DEPS.run(cmd, args);
+          return { code: r.code, stdout: r.stdout };
+        },
+      }),
+    releaseGitHead: async (version) => {
+      try {
+        return (await releaseIdentity(UPGRADE_DEPS, version)).gitHead;
+      } catch {
+        // An unreadable registry makes the herdr pin unverifiable, never
+        // wrong: the probe says so instead of inventing a mismatch.
+        return undefined;
+      }
+    },
     now: Date.now,
     probeTelegram: false,
   };

package/src/failure-class.ts

@@ -103,6 +103,23 @@ export function infraLogSignature(log: string): string | undefined {
   return INFRA_LOG_SIGNATURES.find((signature) => lower.includes(signature));
 }
 
+/**
+ * The cap kills that produced nothing: the worker reached a ceiling with no PR,
+ * no observed head and no salvage commit.
+ *
+ * Named here — beside the table that assigns them — because two callers need
+ * exactly the same set and neither may guess at it: the escalate recovery
+ * quotes the transcript's last tool calls for these classes, and the one-shot
+ * model escalation (#807) fires only for these. Their `-progress` siblings are
+ * deliberately absent: a cap kill *with* work to continue from is a
+ * decomposition verdict, and retrying it on a stronger tier would hide an
+ * oversized slice behind a capability problem.
+ */
+export const SPINNING_CAP_CLASSES: readonly FailureClass[] = [
+  "turn-cap-spinning",
+  "wall-clock-cap-spinning",
+];
+
 /**
  * A stable fingerprint of the infrastructure signature list (#638). The
  * historical reconciliation persists a per-project review cursor stamped with
@@ -315,6 +332,48 @@ export function dispatchInfra(
 // front of the thrown message, so a bare /^git / would never match a real row.
 const DISPATCH_GIT_ERROR = /^(?:Error: )?git .+ exited \d+/s;
 
+/**
+ * Evidence that one review round's resumed session never got as far as a turn,
+ * or `undefined` when the round actually ran (#903).
+ *
+ * The measurement is the round's OWN session, never the run row's cumulative
+ * counters: a revision reuses its run, so `run.turns` already carries every
+ * turn the original attempt spent and would never read zero.
+ *
+ * Deliberately wider than {@link neverStarted}: any zero-turn round that
+ * produced nothing counts, whatever the error text says. The reference
+ * incident is why. During the 2026-08-21/22 worker-UID outage every revision
+ * dispatch died at turn 0 on `EACCES .../config.yml` — an error string no
+ * signature list contained — and each death still consumed one of the PR's
+ * three review rounds, until the ceiling refused any further round and the PR
+ * could not be finished at all. The discriminator that matters is "did the
+ * resumed session take a turn", not which errno the host produced; matching on
+ * errno text is how the next outage spells its failure differently and lands
+ * back in the same deadlock.
+ *
+ * Two things keep the widening honest, and both are the caller's:
+ *  - it decides only whether a round is CHARGED, never whether a run is
+ *    classified — run-level classification is untouched, so an unrecognised
+ *    turn-0 run still reads `unknown` and still spends its attempt;
+ *  - a retried round is bounded by {@link REVIEW_ROUND_INFRA_MAX_RETRIES}, so
+ *    a permanently broken host escalates instead of retrying forever.
+ *
+ * A round that produced a PR, a head or a salvage did work no matter what its
+ * turn counter says, and is never this.
+ */
+export function reviewRoundNeverWorked(round: {
+  turns: number;
+  prUrl?: string;
+  headSha?: string;
+  salvageSha?: string;
+}): string | undefined {
+  if (round.turns !== 0) return undefined;
+  if (round.prUrl !== undefined || round.headSha !== undefined || round.salvageSha !== undefined) {
+    return undefined;
+  }
+  return "the resumed session never took a turn: no turn, no commit and no pushed head";
+}
+
 /** The prefix the draining/restarting process writes to a run it killed. */
 const ADMIN_RESTART_MARKER = "admin restart:";