omp-conductor 0.18.1 → 0.19.0

package/src/doctor.ts

@@ -46,9 +46,11 @@ import {
   dbBackupDirFor,
   findProject,
   loadConfig,
+  resolveArmProof,
   resolveCaps,
   stateDir,
 } from "./config.ts";
+import { observeArmChallenge } from "./arm-challenge.ts";
 import { ompSettingsOverlay, sessionRootDir } from "./omp-settings.ts";
 import {
   DEFAULT_HERDR_UNIT,
@@ -59,6 +61,7 @@ import {
   sessionDirForCwd,
   telegramStateDir,
 } from "./fleet.ts";
+import { pauseInstance } from "./pause.ts";
 import type { TelegramHealth } from "./status-render.ts";
 import {
   claimedTelegramTopics,
@@ -85,10 +88,7 @@ import {
   SYSTEMD_UNIT_DIR,
   tickCwdForProject,
   totalConfiguredWorkers,
-  workerAclHealth,
-  type WorkerAclHealth,
 } from "./setup-host.ts";
-import { WORKER_ACCOUNT } from "./host.ts";
 import { checkTokenScopes, type ScopeCheck } from "./setup.ts";
 import { DB_SNAPSHOT_STEM, dbPath, LIVE_STATES } from "./store.ts";
 import { telegramReportSend, type ReportSend } from "./reports.ts";
@@ -101,6 +101,27 @@ import {
   type HerdrAgentList,
 } from "./orchestrator-tick.ts";
 import type { ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
+import { DEFAULT_ARM_PROOF, type ArmProof } from "./types.ts";
+import {
+  DEFAULT_DEPS as UPGRADE_DEPS,
+  expectedHerdrSource,
+  inspectSurfaces,
+  releaseIdentity,
+  type InstalledSurfaces,
+} from "./upgrade.ts";
+import {
+  TELEGRAM_PACKAGE,
+  checkTelegramFreshness,
+  type TelegramFreshness,
+} from "./telegram-freshness.ts";
+import { judgeSpendTelemetry, spendTelemetryDetail } from "./spend-telemetry.ts";
+
+/** One read of this host's installed surfaces: the three identities, or why
+ *  they could not be read. A failed read is a finding (`warn`, "unverified"),
+ *  never a thrown doctor run and never a silent pass (#904). */
+export type SurfaceRead =
+  | { ok: true; surfaces: InstalledSurfaces }
+  | { ok: false; detail: string };
 
 /**
  * `doctor`'s finding vocabulary. `pass`/`warn`/`fail` — a warning renders the
@@ -132,8 +153,18 @@ export interface DoctorReport {
   findings: Finding[];
 }
 
-/** How many of the most recent completed runs are sampled for spend telemetry. */
+/** How many *working* runs the spend-telemetry judgement wants. */
 export const SPEND_SAMPLE_RUNS = 5;
+/**
+ * How many rows to read to find them (#970).
+ *
+ * Over-sampled deliberately: the judgement discards 0-turn runs, and on this
+ * project 26 of 48 zero-spend rows are exactly those. Reading only
+ * `SPEND_SAMPLE_RUNS` rows would let a burst of administrative kills push the
+ * real evidence out of the window and report "nothing to judge yet" on a fleet
+ * that had plenty to judge.
+ */
+export const SPEND_SAMPLE_ROWS = SPEND_SAMPLE_RUNS * 4;
 
 /** The past incidents this doctor flags, quoted in the finding so an operator
  *  who never lived them knows which failure mode the check exists to prevent. */
@@ -145,10 +176,6 @@ const INCIDENTS = {
   spend:
     "spend telemetry was once absent, so the USD cap never fired — $0.00 spend is not proof of no spend",
   ghauth: "gh auth expired under a live daemon and every tracker call failed silently",
-  workerAcl:
-    "the #835 incident: a setup host granted the worker's path ACLs before restarting the fleet, an OMP startup chmod'd " +
-    "the agent config dir back to 0700 and rewrote the ACL mask, and the next two admitted workers died on EACCES " +
-    "before connecting — the named ACL entry was still there, only its effective permissions were gone",
 } as const;
 
 // ------------------------------------------------------------------ dependencies
@@ -173,10 +200,12 @@ export interface CanonicalUnits {
 }
 
 /** One sampled run row: `state` keeps spend from in-flight rows, which are not
- *  a telemetry statement yet. */
+ *  a telemetry statement yet, and `turns` separates a run that reported nothing
+ *  from one that genuinely did nothing (#970). */
 export interface RunSpendRow {
   state: RunState;
   spendUsd: number;
+  turns: number;
 }
 
 /** Injectable seams. Every field defaults to the production wiring, which
@@ -223,6 +252,21 @@ export interface DoctorDeps {
   canonicalUnits?: (project: ProjectConfig, cfg: ConductorConfig) => CanonicalUnits;
   /** Whether herdr is installed on this host (a `herdr` on PATH). */
   herdrInstalled?: () => boolean;
+  /**
+   * Whether a worker session could load its harness at all (#910), exercised
+   * the way a worker does: the installed peer resolved from this package's own
+   * directory under `bun --no-install`, so neither import can fall through to
+   * Bun's ambient cache. `ok: false` means no worker on this host can start.
+   */
+  workerHarness?: () => Promise<{ ok: true; version: string } | { ok: false; detail: string }>;
+  /**
+   * The pause sentinel for one project as an instance (#938): who set it, why,
+   * when, and — for a fence whose lifetime is one process's — the owning pid.
+   */
+  pauseFence?: (project?: string) => { source: string; reason?: string; since: number; owner?: number } | undefined;
+  /** Whether one pid is a live process. Injected so the abandoned-fence
+   *  finding is testable without spawning anything. */
+  pidLive?: (pid: number) => boolean;
   /** Live `herdr --session <s> agent list`, parsed through the tick's own
    *  parser (the same "one JSON line on stdout" contract recover.sh reads). */
   herdrAgents?: (session: string) => HerdrAgentList;
@@ -256,8 +300,10 @@ export interface DoctorDeps {
   /** The topic id an arm challenge would send into for this project —
    *  resolveProjectTopicId's pin-or-live-claim; undefined means flat chat. */
   armSendTopic?: (project: ProjectConfig) => number | undefined;
-  /** The session directories arm scans for the challenge reply on this
-   *  project — tick-cwd-derived plus the live claim's own directory. */
+  /** The session surface the claim-only plumbing verdict judges for this
+   *  project — tick-cwd-derived plus the live claim's own directory. The
+   *  challenge proof reads conductor state instead (#614), so this feeds
+   *  claim-only verdict checks only. */
   armScanDirs?: (project: ProjectConfig) => readonly string[];
   /** Whether a recorded claim or dm-owner pid is live, with omp-telegram's
    *  topics.ts semantics (EPERM is dead). */
@@ -273,15 +319,16 @@ export interface DoctorDeps {
   /** The fleet agent name the tick config of one project names, or undefined
    *  when there is no (readable) tick — the expected live herdr pane identity. */
   tickAgentName?: (project: ProjectConfig) => string | undefined;
+  /** The three installed identities this host carries, read through
+   *  `upgrade`'s own seam; a failed read is a named finding, never a throw. */
+  installedSurfaces?: () => Promise<SurfaceRead>;
+  /** The `omp-telegram` install/daemon/published triple (#961). */
+  telegramFreshness?: () => Promise<TelegramFreshness>;
+  /** The commit one published version was cut from, for judging the herdr
+   *  plugin's pin; `undefined` when the registry cannot be read. */
+  releaseGitHead?: (version: string) => Promise<string | undefined>;
   /** Clock, so a run is deterministic in tests. */
   now?: () => number;
-  /** The linked worker config paths' effective-ACL verdict for the dedicated
-   *  worker account (#835): `checkable` paths judged, `missing` the ones that
-   *  do not currently grant the worker's needed effective access. Read-only
-   *  and injectable so no test ever touches the host's ACLs; the production
-   *  wiring is the identity plan's own getfacl probe, so `doctor` and
-   *  `setup host` cannot disagree about an ACL. */
-  workerAclHealth?: () => WorkerAclHealth;
   /** The one opt-in side effect: send one self-identified Telegram probe. */
   probeTelegram?: boolean;
 }
@@ -452,8 +499,8 @@ function defaultRecentRuns(project: string, limit: number): RunSpendRow[] {
   try {
     const placeholders = [...LIVE_STATES].map(() => "?").join(", ");
     const rows = database
-      .query<{ state: string; spendUsd: number }, [string, ...string[], number]>(
-        `SELECT state, spendUsd FROM runs
+      .query<{ state: string; spendUsd: number; turns: number }, [string, ...string[], number]>(
+        `SELECT state, spendUsd, turns FROM runs
           WHERE project = ? AND state NOT IN (${placeholders})
           ORDER BY startedAt DESC, rowid DESC
           LIMIT ?`,
@@ -463,7 +510,9 @@ function defaultRecentRuns(project: string, limit: number): RunSpendRow[] {
     for (const row of rows) {
       const state = row.state as RunState;
       // A state vocabulary the store never had is not "completed".
-      if (!LIVE_STATES.includes(state)) out.push({ state, spendUsd: row.spendUsd });
+      if (!LIVE_STATES.includes(state)) {
+        out.push({ state, spendUsd: row.spendUsd, turns: row.turns });
+      }
     }
     return out;
   } catch {
@@ -762,6 +811,12 @@ function unresolvableShell(unit: string): boolean {
  * state resolved at staging time, so a rendered value doctor cannot reproduce
  * is excluded from both sides rather than reported as a difference — an
  * unresolvable canonical is "cannot compare", never drift (#511).
+ *
+ * The service `PATH=` line has no such exemption: since #879 it is canonical
+ * on both sides (`defaultServiceRuntime` composes it from the resolved fleet
+ * binaries plus the standard system directories, never from the rendering
+ * process's environment), so a differing installed value is genuine drift and
+ * must fail here with the value named.
  */
 export function unitDrift(installed: string, canonical: string): { differences: string[] } {
   const want = canonical
@@ -916,36 +971,6 @@ function ownershipProbe(probes: Probes): Finding {
   return failFinding("systemd-ownership", summary, fix);
 }
 
-/**
- * The dedicated worker account's path ACLs (#835): every linked worker config
- * path must grant the worker its *effective* search/read permissions. The
- * installed OMP harness chmods its agent config dir back to 0700 on every
- * open, and a chmod rewrites the ACL mask — the named entry survives as
- * `user:omp-worker:--x  #effective:---` while the worker's access silently
- * vanishes, which is exactly the incident this finding exists to catch
- * (`${INCIDENTS.workerAcl}`). The verdict comes from the identity plan's own
- * probe through the read-only {@link DoctorDeps.workerAclHealth} seam, so a
- * pass here is a pass the plan agrees with and a fail names the paths whose
- * mask is stripping the grant.
- */
-function workerAclProbeFinding(probes: Probes): Finding {
-  const health = probes.workerAclHealth();
-  if (health.checkable === 0) {
-    return passFinding("worker-acl", "no linked worker config paths on this host yet — nothing to check");
-  }
-  if (health.missing.length === 0) {
-    return passFinding(
-      "worker-acl",
-      `${health.checkable} linked worker config path(s) grant ${WORKER_ACCOUNT} effective search/read access`,
-    );
-  }
-  return failFinding(
-    "worker-acl",
-    `the ${WORKER_ACCOUNT} account's ACL is not effective on ${health.missing.join(", ")} — a named entry the ACL mask strips is ` +
-      "unreachable by the worker, and the next admitted worker dies on EACCES before connecting",
-    "re-run `omp-conductor setup host`: the transaction re-applies the ACL grants (mask included) after the final fleet restart",
-  );
-}
 
 /** IANA timezone in reporting config — an invalid zone silently mis-schedules
  *  the availability window and the daily digest (#273). */
@@ -1071,22 +1096,40 @@ async function telegramProbe(probes: Probes, project: ProjectConfig | undefined,
   );
 }
 
-/** Spend telemetry: warn when the last K completed runs all recorded $0.00 —
- *  the USD cap cannot fire on zeros ($0.00 is not proof of no spend). */
+/**
+ * Spend telemetry, through the shared judgement (#970).
+ *
+ * The predicate this replaced fired only when *every* sampled run reported
+ * $0.00, and counted 0-turn kills as evidence. Replayed over this fleet's whole
+ * history it would have fired 14 times while staying silent through 37 windows
+ * that had lost a majority of their telemetry — so partial loss, the common
+ * case, was the invisible one. See `spend-telemetry.ts` for the measurements.
+ */
 function spendProbe(rows: RunSpendRow[], limit: number): Finding {
-  if (rows.length < limit) {
-    return passFinding("spend-telemetry", `${rows.length} completed run(s) observed — fewer than ${limit}, nothing to judge yet`);
-  }
-  const window = rows.slice(0, limit);
-  if (window.every((r) => r.spendUsd === 0)) {
+  const verdict = judgeSpendTelemetry(rows, limit);
+  const detail = spendTelemetryDetail(verdict);
+  if (detail !== undefined) {
     return warnFinding(
       "spend-telemetry",
-      `the last ${limit} completed runs all recorded $0.00 spend — ${INCIDENTS.spend}`,
+      `${detail} — ${INCIDENTS.spend}`,
       "verify harness spend reporting (the per-run spendUsd column / `omp usage --json`); a USD cap on top of zeros never fires",
     );
   }
-  const total = window.reduce((sum, r) => sum + r.spendUsd, 0);
-  return passFinding("spend-telemetry", `spend observed on the last ${window.length} completed runs ($${total.toFixed(2)} total)`);
+  if (verdict.kind === "insufficient") {
+    return passFinding(
+      "spend-telemetry",
+      `${verdict.worked} completed run(s) did any work — fewer than ${verdict.needed}, nothing to judge yet`,
+    );
+  }
+  // Healthy. Naming the metered share rather than only the total: a window with
+  // one missing row is fine and saying so is how a reader learns the baseline
+  // is not zero.
+  const healthy = verdict as Extract<typeof verdict, { kind: "healthy" }>;
+  return passFinding(
+    "spend-telemetry",
+    `spend observed on ${healthy.worked - healthy.missing} of the last ${healthy.worked} working runs ` +
+      `($${healthy.totalUsd.toFixed(2)} total)`,
+  );
 }
 
 /** Live `herdr --session <session> agent list` through the project's own
@@ -1173,91 +1216,43 @@ function herdrAgentNameProbe(probes: Probes, p: ProjectConfig): Finding {
 }
 
 /**
- * #600 check — the orchestrator's claimed session file vs the directory arm
- * scans.
+ * #614 check — the state of this project's arming acknowledgement transaction.
  *
- * A pane resumed from a session created elsewhere keeps the original transcript
- * path (herdr pins the pane to it), so the claimed file can live outside the
- * session directory the tick cwd implies. arm follows the claim, so the
- * mismatch alone is handled — but it is exactly the shape of host that once
- * made arming silently impossible, and an operator reading transcript paths
- * should not have to rediscover it. A claimed file *outside the session tree*
- * is worse: arm refuses to arm there (the reply could never be seen), so
- * doctor fails rather than letting the fleet sit disarmed.
+ * The challenge proof is conductor-owned state now: the host records a pending
+ * challenge and waits for the acknowledgement the orchestrator's inbound
+ * adapter writes. No transcript location is diagnosed any more — where (or
+ * whether) a session file lives is not part of arming — so what is worth
+ * reporting is a transaction left open: a challenge still pending and whether
+ * it has been acknowledged, or an expired record nothing settled. Every shape
+ * is inert by construction (a reply past expiry, or for a replaced id, is
+ * refused), so this informs; only the stale leftover warns.
  */
-function armSessionDirProbe(probes: Probes, p: ProjectConfig): Finding {
-  const result = probes.claimedTopics();
-  if (result.kind !== "ok") {
-    // The registry could not be read, so the claim's session file is unknown,
-    // not known-absent. Arm scans the tick-cwd directory either way, but the
-    // finding must stay diagnostic — this is exactly the shape that once made
-    // arming silently impossible, and "no claim" would certify it as fine. A
-    // missing registry is the same unknown to this probe: the bridge may never
-    // have claimed, but nothing proves the claim names no session file.
+function armAckProbe(probes: Probes, p: ProjectConfig): Finding {
+  const now = probes.now();
+  const sighting = observeArmChallenge(p.name);
+  if (sighting === undefined) {
+    return passFinding("arm-ack", `[${p.name}] no pending arm challenge — the acknowledgement handshake is idle`);
+  }
+  const window =
+    sighting.sentAt === undefined || sighting.expiresAt === undefined
+      ? "an unreadable record window"
+      : `sent ${new Date(sighting.sentAt).toISOString()}, expires ${new Date(sighting.expiresAt).toISOString()}`;
+  if (sighting.expiresAt !== undefined && now >= sighting.expiresAt) {
     return warnFinding(
-      "arm-session-dir",
-      `[${p.name}] ` +
-        (result.kind === "missing"
-          ? `omp-telegram has no claim registry (threads.json in its state dir) — arm scans the tick-cwd session directory, but doctor cannot tell whether the claim names a session file elsewhere`
-          : `omp-telegram's claim registry is unreadable (${result.problem}) — arm scans the tick-cwd session directory, but doctor cannot tell whether the claim names a session file elsewhere`),
-      `check omp-telegram's claim registry (threads.json in its state dir) is readable and the bridge is running, then re-run doctor`,
+      "arm-ack",
+      `[${p.name}] an arming challenge from ${window} expired without being settled — it is inert (replies past expiry are refused), but it lingers until this project's next arm replaces it`,
+      `no action needed; the next \`arm\` for this project settles it — re-run doctor afterwards to confirm`,
     );
   }
-  if (result.claims.length === 0) {
+  if (sighting.acknowledgedAt !== undefined) {
     return passFinding(
-      "arm-session-dir",
-      `[${p.name}] no live omp-telegram claim — arm scans the tick-cwd session directory`,
-    );
-  }
-  const claims = result.claims;
-  const match = resolveProjectClaim(claims, p.name);
-  const scanDir = sessionDirForCwd(tickCwdForProject(p));
-  if (match.kind === "ambiguous") {
-    // Several live claims answer to the project, so the claim's session file
-    // cannot be told from a sibling pane's. Arm scans the tick-cwd directory
-    // either way, but certifying that fallback as the scan is the miss this
-    // probe exists to surface: the resumed transcript may live under another
-    // claim while doctor reads the fallback as the whole story (#626).
-    return warnFinding(
-      "arm-session-dir",
-      `[${p.name}] several live claims answer to this project (topics ${match.claimants.map((c) => c.threadId).join(", ")}), so the claim's session file is a coin toss — arm scans the tick-cwd session directory, but the resumed transcript may live elsewhere`,
-      `make exactly one live claim answer to this project: rename the other panes' herdr spaces or re-claim them one at a time, then re-run doctor — or start the orchestrator pane's session from the tick cwd ${scanDir}`,
-    );
-  }
-  const claim = match.kind === "match" ? match.claim : undefined;
-  if (claim === undefined) {
-    return passFinding(
-      "arm-session-dir",
-      `[${p.name}] no live omp-telegram claim answers to this project — arm scans the tick-cwd session directory`,
-    );
-  }
-  if (claim.sessionFile === undefined) {
-    return passFinding(
-      "arm-session-dir",
-      `[${p.name}] the live claim names no session file — arm scans the tick-cwd session directory`,
-    );
-  }
-  const root = sessionsRoot();
-  const claimDir = dirname(claim.sessionFile);
-  if (claimDir !== root && !claimDir.startsWith(join(root, sep))) {
-    return failFinding(
-      "arm-session-dir",
-      `[${p.name}] the orchestrator's claimed session file ${claim.sessionFile} is outside the session tree arm scans (${root}) — ` +
-        `arm cannot arm while the reply can only land there`,
-      `start the orchestrator pane's session under ${root}, or from the tick cwd ${scanDir}`,
-    );
-  }
-  if (claimDir !== scanDir) {
-    return warnFinding(
-      "arm-session-dir",
-      `[${p.name}] the orchestrator's claimed session file ${claim.sessionFile} lives in ${claimDir}, not the tick-cwd session directory ${scanDir} — ` +
-        `arm follows the claim, so arming works, but this is the resumed-pane shape #600 names`,
-      `optional tidy-up: start the pane's session from the tick cwd so transcript and cwd agree again`,
+      "arm-ack",
+      `[${p.name}] an arm challenge is pending (${window}) and was acknowledged ${new Date(sighting.acknowledgedAt).toISOString()} — the host consumes it on settle`,
     );
   }
   return passFinding(
-    "arm-session-dir",
-    `[${p.name}] the orchestrator's claimed session file is inside the session directory arm scans`,
+    "arm-ack",
+    `[${p.name}] an arm challenge is pending (${window}), not yet acknowledged — live if \`arm\` is running right now, otherwise its window simply runs out`,
   );
 }
 
@@ -1379,9 +1374,20 @@ function telegramPlumbingProbe(probes: Probes, p: ProjectConfig): Finding {
       `[${p.name}] outbound-only paging — no tick accessFile to arm, so no live claim, DM owner or poll ownership is required`,
     );
   }
+  // Transcript-surface identity checks belong to the claim-only proof alone:
+  // since #614 the challenge proof reads conductor acknowledgement state, so
+  // judging it by where a transcript lives could report arming impossible
+  // while `arm` succeeds. An unreadable config keeps the default proof —
+  // fail-safe to the stricter verdict, exactly like armTicks.
+  let proof: ArmProof = DEFAULT_ARM_PROOF;
+  try {
+    proof = resolveArmProof(p);
+  } catch {
+    /* unreadable config — keep today's stricter default */
+  }
   const verdict = telegramPlumbingVerdict(
     probes.armSendTopic(p),
-    { dirs: probes.armScanDirs(p) },
+    proof === "claim-only" ? { dirs: probes.armScanDirs(p) } : undefined,
     {
       channel: probes.channelState(p),
       registry: probes.claimedTopics(),
@@ -1478,6 +1484,45 @@ function telegramPlumbingProbe(probes: Probes, p: ProjectConfig): Finding {
  * pages instead of recovering. Absent ~ the unrecoverable default; an explicit
  * `true` is a deliberate (desktop) choice and only warned.
  */
+/**
+ * A dispatch fence whose owning process is gone (#938).
+ *
+ * Only a fence that *declared* an owner can be judged: `setPaused` records
+ * `owner=` for a setup transaction, whose lifetime is that process's, and
+ * deliberately not for an operator `hold` or the fail-closed hold a failed
+ * apply leaves behind — those are meant to outlive every process, and calling
+ * one "abandoned" would be worse than saying nothing about it.
+ *
+ * Measured 2026-08-21T13:24Z: an operator abandoned a setup that looked hung;
+ * the policy change had applied, and dispatch sat under
+ * `source=setup reason="setup apply fence"` until somebody thought to run
+ * `resume`. Nothing anywhere said the process that wrote it was gone.
+ *
+ * This never clears anything, and that is deliberate. A pid is reusable, a
+ * paused fleet is the safe state, and resuming dispatch nobody authorised is a
+ * worse outcome than a fence that needs one command. So it names the command.
+ */
+function fenceProbe(probes: Probes, project: string | undefined): Finding {
+  const id = project === undefined ? "dispatch-fence" : `dispatch-fence:${project}`;
+  const fence = probes.pauseFence(project);
+  if (fence === undefined) return passFinding(id, "dispatch is not held by a fence");
+  const why = fence.reason === undefined ? "" : ` — "${fence.reason}"`;
+  if (fence.owner === undefined) {
+    // An operator hold, or the durable hold a failed apply leaves on purpose.
+    // Both are somebody's decision rather than a leak, whatever their age.
+    return passFinding(id, `dispatch held by ${fence.source}${why} (no owning process; nothing to expire)`);
+  }
+  if (probes.pidLive(fence.owner)) {
+    return passFinding(id, `dispatch held by a live ${fence.source} (pid ${fence.owner})`);
+  }
+  return warnFinding(
+    id,
+    `dispatch is held by ${fence.source}${why}, whose process (pid ${fence.owner}) is gone: the fence outlived ` +
+      `the transaction that set it, and has stood since ${new Date(fence.since).toISOString()}`,
+    `nothing here clears it — confirm no setup is running, then \`omp-conductor resume${project === undefined ? "" : ` --project ${project}`}\``,
+  );
+}
+
 function herdrResumeProbe(probes: Probes): Finding {
   if (!probes.herdrInstalled()) {
     return passFinding("herdr-resume", "herdr not installed — nothing to check");
@@ -1572,6 +1617,169 @@ function parseEnvKey(text: string, key: string): string | undefined {
   return undefined;
 }
 
+/**
+ * Install-surface parity (#904). omp-conductor installs onto three surfaces —
+ * the Bun-global CLI/daemon tree, the omp plugin, and the herdr recovery
+ * plugin — and only a live `upgrade` invocation ever compared them. On a
+ * fleet whose installs are manual they diverge silently: measured on this
+ * host on 2026-08-22, the omp plugin sat on the withdrawn 0.18.1 release for
+ * about a day beside a 0.18.0 daemon, with no finding, no status line and no
+ * escalation.
+ *
+ * The question is inter-surface agreement on the host, never staleness
+ * against the registry: a fleet may deliberately sit on an older release,
+ * but never on two at once. `expectedGitHead` is the commit the CLI's own
+ * version was published from, so the herdr pin can be judged against the
+ * same release; `undefined` means the registry could not be read and the pin
+ * is reported as unverified rather than as a mismatch.
+ */
+/**
+ * The bootstrap deadlock (#910), which is a conjunction rather than a fault:
+ *
+ *   - no worker on this host can start, because the *installed* code cannot
+ *     load its harness; and
+ *   - the fix for that ships through a release, and a release needs a merged
+ *     worker pull request — so the broken install is what blocks its own
+ *     replacement.
+ *
+ * Kept as its own finding, never folded into the stale-setup ones, because the
+ * two states call for opposite actions. "Host runtime differs — run setup host"
+ * is *correct advice* for a stale install and *useless* here: no amount of
+ * re-running setup replaces the installed package, and diagnosing that cost
+ * real time on 2026-08-20..22. So this names the one command that breaks the
+ * cycle (#908's bootstrap identity), and an ordinary stale host still reads
+ * exactly as it did.
+ */
+function bootstrapDeadlockProbe(
+  harness: { ok: true; version: string } | { ok: false; detail: string },
+): Finding {
+  if (harness.ok) {
+    return passFinding("bootstrap-deadlock", `workers can load the installed harness (${harness.version})`);
+  }
+  return failFinding(
+    "bootstrap-deadlock",
+    `no worker can start: the installed conductor cannot load its harness — ${harness.detail}. ` +
+      "A release cannot publish the fix either, because publishing needs a merged worker pull request, " +
+      "so the installed package blocks its own replacement",
+    "install the fix by exact commit instead of by version: `omp-conductor upgrade --bootstrap <sha> " +
+      "--source <checkout at that sha>` (its checks run against the source first, and it refuses rather " +
+      "than installing a partly verified tree)",
+  );
+}
+
+/**
+ * Is the plugin a mandated conductor contract runs on actually current? (#961)
+ *
+ * Sibling to {@link surfaceParityProbe}, and deliberately a separate finding:
+ * that one is about the three surfaces of *this* package agreeing with each
+ * other, this one is about the one peer the floor's reply-in-topic instruction
+ * depends on being new enough to honour it.
+ *
+ * A `warn`, never a `fail`. Nothing here is broken on this host's own terms —
+ * the fleet dispatches, merges and reports fine on a stale plugin. What breaks
+ * is one documented instruction, and the remedy is an install the operator owns,
+ * so failing the run would make `doctor` red for something conductor must not
+ * fix itself.
+ */
+export function telegramFreshnessProbe(freshness: TelegramFreshness): Finding {
+  const id = "telegram-plugin-freshness";
+  const { installed, daemon, published } = freshness.surfaces;
+  const named =
+    `installed=${installed.kind === "version" ? installed.version : installed.kind}, ` +
+    `daemon=${daemon.kind === "version" ? daemon.version : daemon.kind}, ` +
+    `published=${published.kind === "version" ? published.version : published.kind}`;
+  switch (freshness.state) {
+    case "current":
+      return passFinding(id, `${TELEGRAM_PACKAGE} is current (${named})`);
+    case "installed-behind":
+      return warnFinding(
+        id,
+        `${freshness.detail} (${named})`,
+        `install ${TELEGRAM_PACKAGE}@${published.kind === "version" ? published.version : "latest"} and restart its daemon; a targetless telegram_send needs the newer target ladder to reply in the topic a message arrived in (#882)`,
+      );
+    case "daemon-stale":
+      return warnFinding(
+        id,
+        `${freshness.detail} (${named})`,
+        `restart the ${TELEGRAM_PACKAGE} daemon so it serves the installed version`,
+      );
+    case "not-installed":
+      return warnFinding(
+        id,
+        `${freshness.detail} (${named})`,
+        `install ${TELEGRAM_PACKAGE}, or stop relying on Telegram delivery for this fleet`,
+      );
+    case "unknown":
+      // Not a finding about the plugin: a finding about this check. An
+      // unreachable registry must never read as "behind" *or* as "current".
+      return warnFinding(
+        id,
+        `${TELEGRAM_PACKAGE} freshness unverified: ${freshness.detail ?? "no surface answered"} (${named})`,
+        "no action needed if this host has no outbound npm access; otherwise re-run doctor when it does",
+      );
+  }
+}
+
+export function surfaceParityProbe(
+  read: SurfaceRead,
+  expectedGitHead: string | undefined,
+  herdrExpected: boolean,
+): Finding {
+  const id = "install-surfaces";
+  if (!read.ok) {
+    return warnFinding(
+      id,
+      `installed surfaces unreadable: ${read.detail} — parity between the CLI, the omp plugin and the herdr plugin is unverified`,
+      "check that `omp-conductor`, `omp` and (when this host runs herdr) `herdr` are on PATH for this user, then re-run doctor",
+    );
+  }
+  const { cliVersion, ompVersion, herdrSource } = read.surfaces;
+  const named = `cli=${cliVersion}, omp=${ompVersion ?? "absent"}, herdr=${herdrSource ?? "absent"}`;
+  // A proven disagreement first: two different releases live on one host is
+  // the fault this probe exists for, and it outranks anything absent.
+  if (ompVersion !== undefined && ompVersion !== cliVersion) {
+    return failFinding(
+      id,
+      `installed surfaces disagree: the omp plugin is ${ompVersion} while the CLI/daemon tree is ${cliVersion} (${named})`,
+      `run \`omp-conductor upgrade --to ${cliVersion}\` (or to the release this fleet should be on) so all three surfaces carry one identity`,
+    );
+  }
+  if (herdrExpected && herdrSource !== undefined && expectedGitHead !== undefined) {
+    if (herdrSource.startsWith("local:")) {
+      return warnFinding(
+        id,
+        `the herdr plugin is linked to a local checkout, not a released pin (${named})`,
+        `run \`omp-conductor upgrade --to ${cliVersion}\` to replace the link with the released pin, or keep the link deliberately while developing`,
+      );
+    }
+    const expected = expectedHerdrSource(expectedGitHead);
+    if (herdrSource !== expected) {
+      return failFinding(
+        id,
+        `installed surfaces disagree: the herdr plugin is pinned to ${herdrSource} while ${cliVersion} was published from ${expectedGitHead} (${named})`,
+        `run \`omp-conductor upgrade --to ${cliVersion}\` so the herdr recovery plugin matches the release the rest of the host runs`,
+      );
+    }
+  }
+  // Nothing disagrees. Anything absent is named as absent — a surface this
+  // host does not carry is not a mismatch, and a pin that could not be
+  // verified is not a pass in disguise.
+  const absent: string[] = [];
+  if (ompVersion === undefined) absent.push("omp plugin not installed — a tick cannot arm without it");
+  if (herdrExpected && herdrSource === undefined) absent.push("herdr plugin not installed — pane recovery cannot run");
+  if (herdrExpected && herdrSource !== undefined && expectedGitHead === undefined) {
+    absent.push(`herdr pin ${herdrSource} unverified — the registry did not answer what ${cliVersion} was published from`);
+  }
+  if (absent.length > 0) {
+    return warnFinding(
+      id,
+      `${absent.join("; ")} (${named})`,
+      `run \`omp-conductor upgrade --to ${cliVersion}\` to install every surface this host needs from one release`,
+    );
+  }
+  return passFinding(id, `one identity across every installed surface (${named})`);
+}
+
 /**
  * Run every probe and assemble the stable report.
  *
@@ -1641,6 +1849,17 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   findings.push(dbProbe(probes));
   findings.push(dbBackupProbe(probes, cfg));
   findings.push(await ghAuthProbe(probes, configuredRepos(cfg)));
+  // Install-surface parity is host-wide, like the unit and the store: one
+  // read of the three identities, judged once (#904).
+  const surfaces = await probes.installedSurfaces();
+  const cliVersion = surfaces.ok ? surfaces.surfaces.cliVersion : undefined;
+  const expectedGitHead = cliVersion === undefined ? undefined : await probes.releaseGitHead(cliVersion);
+  findings.push(surfaceParityProbe(surfaces, expectedGitHead, probes.herdrInstalled()));
+  // Host-wide for the same reason: one plugin install serves every project, so
+  // it is judged once rather than repeated per project (#961).
+  findings.push(telegramFreshnessProbe(await probes.telegramFreshness()));
+  // Host-wide like the surfaces: one install, one harness, one answer (#910).
+  findings.push(bootstrapDeadlockProbe(await probes.workerHarness()));
   // The run-session root is one host-wide fact — where every run's session dir
   // (and the omp settings overlay inside it) lands — probed once and shared
   // across the per-project omp-settings findings.
@@ -1649,7 +1868,7 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   // The collected run sample across the resolved project set, newest-first;
   // spend telemetry is a single host-wide finding, not one per project.
   const spendRows: RunSpendRow[] = [];
-  for (const p of projects) spendRows.push(...probes.recentRuns(p.name, SPEND_SAMPLE_RUNS));
+  for (const p of projects) spendRows.push(...probes.recentRuns(p.name, SPEND_SAMPLE_ROWS));
 
   // Per-project probes run for every resolved project, each named in its
   // finding so a multi-project run stays legible. When nothing resolved
@@ -1663,7 +1882,7 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
       passFinding("herdr-agent-name", projectProblem === undefined ? "no project resolved — nothing to check" : `agent name uncheckable: ${projectProblem}`),
     );
     findings.push(
-      passFinding("arm-session-dir", projectProblem === undefined ? "no project resolved — nothing to check" : `arm session dir uncheckable: ${projectProblem}`),
+      passFinding("arm-ack", projectProblem === undefined ? "no project resolved — nothing to check" : `arm acknowledgement uncheckable: ${projectProblem}`),
     );
     findings.push(
       passFinding("topic-pin", projectProblem === undefined ? "no project resolved — nothing to check" : `topic pin uncheckable: ${projectProblem}`),
@@ -1678,7 +1897,7 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
     for (const p of projects) {
       findings.push(await labelProbe(probes, p));
       findings.push(herdrAgentNameProbe(probes, p));
-      findings.push(armSessionDirProbe(probes, p));
+      findings.push(armAckProbe(probes, p));
       findings.push(topicPinProbe(probes, p));
       findings.push(telegramPlumbingProbe(probes, p));
       findings.push(ompSettingsProbe(p, sessionRootState));
@@ -1690,10 +1909,6 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   findings.push(unitProbe(probes, projects[0], cfg));
   findings.push(recoveryProbe(probes));
   findings.push(ownershipProbe(probes));
-  // The worker's path ACLs are a host-wide fact like the unit ownership: one
-  // shared daemon account, one shared worker identity, one shared set of
-  // linked config paths. A finding once, never once per project.
-  findings.push(workerAclProbeFinding(probes));
   // #541 seam checks, host-global: the live herdr config and the plugin's
   // config.env are single files on the host, not per-project facts.
   findings.push(herdrResumeProbe(probes));
@@ -1701,9 +1916,13 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   if (projects.length === 0) {
     findings.push(timezoneProbe(undefined));
     findings.push(await telegramProbe(probes, undefined, checkedAt));
+    findings.push(fenceProbe(probes, undefined));
   } else {
     for (const p of projects) findings.push(timezoneProbe(p));
     for (const p of projects) findings.push(await telegramProbe(probes, p, checkedAt));
+    // Per project, because the sentinel is: one fleet's fence must never be
+    // reported against another's name.
+    for (const p of projects) findings.push(fenceProbe(probes, p.name));
   }
   findings.push(spendProbe(spendRows, SPEND_SAMPLE_RUNS));
 
@@ -1725,6 +1944,41 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   };
 }
 
+/**
+ * Exercise the worker's real import contract out of process (#910).
+ *
+ * `harness-loader.ts` is the module a worker session loads its harness through,
+ * and it is runnable — so this runs exactly that, with `--no-install`, and reads
+ * its verdict rather than re-deriving one. Anything else would be a second
+ * opinion about the launch path, which is how a doctor comes to disagree with
+ * the thing it is checking.
+ */
+async function defaultWorkerHarness(): Promise<{ ok: true; version: string } | { ok: false; detail: string }> {
+  const loader = join(import.meta.dir, "harness-loader.ts");
+  try {
+    const child = Bun.spawn(["bun", "--no-install", loader], {
+      stdin: "ignore",
+      stdout: "pipe",
+      stderr: "pipe",
+      env: process.env,
+    });
+    const stdout = new Response(child.stdout).text();
+    const stderr = new Response(child.stderr).text();
+    const code = await child.exited;
+    if (code !== 0) {
+      const detail = ((await stderr).trim() || (await stdout).trim()).split("\n")[0] ?? `exit ${code}`;
+      return { ok: false, detail };
+    }
+    const parsed: unknown = JSON.parse((await stdout).trim());
+    const version = parsed !== null && typeof parsed === "object" ? Reflect.get(parsed, "version") : undefined;
+    return typeof version === "string" && version.length > 0
+      ? { ok: true, version }
+      : { ok: false, detail: "the harness loaded but reported no version" };
+  } catch (err) {
+    return { ok: false, detail: err instanceof Error ? err.message : String(err) };
+  }
+}
+
 /** The default wiring — every production transport the rest of the package uses. */
 export function defaultProbes(): Probes {
   return {
@@ -1737,9 +1991,6 @@ export function defaultProbes(): Probes {
     readUnit: defaultReadUnit,
     stat: defaultStat,
     uidOf: defaultUidOf,
-    // The same effective-ACL read the identity plan plans with — doctor and
-    // setup cannot disagree about a worker grant (#835).
-    workerAclHealth: () => workerAclHealth(join(homedir(), ".omp", "agent")),
     dbIntegrity: defaultDbIntegrity,
     snapshotDirState: defaultSnapshotDirState,
     sessionRootState: defaultSessionRootState,
@@ -1748,6 +1999,9 @@ export function defaultProbes(): Probes {
     telegramSend: telegramReportSend,
     canonicalUnits: defaultCanonicalUnits,
     herdrInstalled: () => Bun.which("herdr") !== null,
+    pauseFence: (project) => pauseInstance(project),
+    workerHarness: defaultWorkerHarness,
+    pidLive: (pid) => pidAlive(pid),
     herdrAgents: defaultHerdrAgents,
     herdrSession: () => resolveHerdrSessionWithBridge(),
     herdrConfig: defaultHerdrConfig,
@@ -1795,6 +2049,35 @@ export function defaultProbes(): Probes {
       if (tick.kind !== "ok") return undefined;
       return tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME;
     },
+    installedSurfaces: async () => {
+      // `upgrade`'s own reader, so doctor cannot disagree with the transaction
+      // about what is installed. A host with no herdr is asked only about the
+      // two surfaces it has, rather than failing the whole read (#904).
+      try {
+        const surfaces = await inspectSurfaces(UPGRADE_DEPS, { readHerdr: Bun.which("herdr") !== null });
+        return { ok: true, surfaces };
+      } catch (err) {
+        return { ok: false, detail: messageOf(err) };
+      }
+    },
+    telegramFreshness: () =>
+      // The module's own reader, so `doctor` and `status` cannot disagree about
+      // what is installed — the same one-seam rule `installedSurfaces` follows.
+      checkTelegramFreshness({
+        run: async (cmd, args) => {
+          const r = await UPGRADE_DEPS.run(cmd, args);
+          return { code: r.code, stdout: r.stdout };
+        },
+      }),
+    releaseGitHead: async (version) => {
+      try {
+        return (await releaseIdentity(UPGRADE_DEPS, version)).gitHead;
+      } catch {
+        // An unreadable registry makes the herdr pin unverifiable, never
+        // wrong: the probe says so instead of inventing a mismatch.
+        return undefined;
+      }
+    },
     now: Date.now,
     probeTelegram: false,
   };