omp-conductor 0.20.2 → 0.20.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omp-conductor",
-  "version": "0.20.2",
+  "version": "0.20.3",
   "type": "module",
   "license": "MIT",
   "description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",

package/src/commands/daemon.ts

@@ -8,15 +8,7 @@
 
 import type { CommandContext } from "./context.ts";
 import { runDaemon } from "../daemon.ts";
-import { clearRecord, DEFAULT_PORT, livingDaemon, writeRecord } from "../lifecycle.ts";
-
-/**
- * `DaemonRecord.logFile` for a daemon nobody spawned. The field is required and
- * `status` prints it, so it has to say something true: a foreground daemon
- * opened no log of its own — whoever started it owns its stdout, be that
- * systemd's journal, a terminal, or a pane.
- */
-const FOREGROUND_LOG = "<inherited stdout — started in the foreground>";
+import { clearRecord, publishForegroundRecord, startRecordRepair } from "../lifecycle.ts";
 
 export async function daemonCommand(ctx: CommandContext): Promise<void> {
 // Until now only `lifecycle.startDaemon()` — the spawn path — wrote the
@@ -42,28 +34,25 @@ if (once) {
   return;
 }
 
-const running = livingDaemon();
-if (running !== undefined && running.pid !== process.pid) {
-  process.stderr.write(`omp-conductor: another daemon is alive (pid ${running.pid}); stop it first\n`);
+// The boot half of pidfile ownership (#1114): a systemd-started or hand-run
+// foreground daemon publishes its own record in lifecycle — refusing, having
+// done nothing, while another live daemon owns the record.
+const claimed = publishForegroundRecord({ port, project });
+if (claimed.kind === "foreign-live") {
+  process.stderr.write(`omp-conductor: another daemon is alive (pid ${claimed.pid}); stop it first\n`);
   process.exit(1);
 }
 
-// A living record that already names this pid was written by the `start`
-// that spawned us, and it knows the log file our stdout is really going
-// to. Replacing it with a guess would be a downgrade.
-if (running === undefined) {
-  writeRecord({
-    pid: process.pid,
-    port: port ?? DEFAULT_PORT,
-    startedAt: Date.now(),
-    logFile: FOREGROUND_LOG,
-    ...(project === undefined ? {} : { project }),
-  });
-}
-
+// The running half (#1114): while this process is the daemon it keeps its own
+// record alive, so an external sweep of the runtime directory cannot leave
+// the single-owner guard disarmed until the next restart.
+const repair = startRecordRepair({ port, project });
 try {
   await runDaemon({ once, port, project });
 } finally {
+  // Stop repairing before clearing, so no interval tick can resurrect the
+  // file after the removal below.
+  repair.stop();
   // The record names a pid that is about to stop existing. Leaving it
   // behind makes the next reader probe a ghost before believing us.
   clearRecord();

package/src/daemon/tick.ts

@@ -22,7 +22,7 @@ import { admitCandidates, startOfToday, type AdmissionHold } from "../admission.
 import { dbBackupDirFor, findProject, loadConfig, resolveCaps, stateDir } from "../config.ts";
 import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "../decisions.ts";
 import { dbSnapshotDue, dbSnapshotMarkerKey, localDayKey } from "../digest-schedule.ts";
-import { runDoctor } from "../doctor.ts";
+import { reconcileOrphanedPause, runDoctor } from "../doctor.ts";
 import { fleetLayers, herdrPaneOmpStarts, resolveHerdrSession } from "../fleet.ts";
 import { projectLabels } from "../label-projection.ts";
 import { healthCheck } from "../lifecycle.ts";
@@ -410,6 +410,20 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
     escalate: (event) => d.escalate(event),
     log,
   });
+  // The orphaned-pause watch rides the same above-the-pause-gate band (#1113):
+  // a sentinel whose writer has exited is exactly the condition a paused
+  // fleet cannot surface through its own gates below, and an `upgrade` pause
+  // can never be cleared by any future event. The notification ledger dedupes
+  // the page to once per sentinel instance; resuming stays a human decision.
+  try {
+    await reconcileOrphanedPause({
+      project: d.project.name,
+      escalate: (event) => d.escalate(event),
+      log,
+    });
+  } catch (err) {
+    log(`orphaned-pause watch failed: ${errText(err)}`);
+  }
 
   // Beside the two orchestrator watches, and above the pause gate, for the same
   // reason they are: mining reads rows this store already holds and files intake

package/src/diff-flags.ts

@@ -898,7 +898,7 @@ function scanHunks(
  * an extractor that reads any backtick span in the narrative as a claim would
  * flag every run within a day.
  *
- * Three tolerances make the matcher honest rather than decorative:
+ * Four tolerances make the matcher honest rather than decorative:
  *
  * - Claims are extracted only from a verified section — a `## Verified`-style
  *   heading or an inline `Verified:` label — never from the narrative. This is
@@ -915,6 +915,14 @@ function scanHunks(
  *   the worker *tried* and the guard stopped it (the honest #566/#570 shape,
  *   whose own words are "CI owns them"), which is not the same finding as a
  *   claim with no attempt at all.
+ * - Claims are positions before they are texts (#1102): a unit of the region
+ *   claims what it opens with — a bullet or bare line whose lead is the
+ *   backticked command, or a `**Verified:**` label declaring its whole row —
+ *   never a span quoted mid-sentence. The soft-wrapped prose that quoted
+ *   `issue is closed` inside #1097's Verified bullet is commentary whatever
+ *   words it quotes, and the structural rule says so without a runner
+ *   vocabulary that would silently stop auditing the first PR proved by
+ *   anything else.
  */
 
 /** The verified-section markers a claim must live under: a heading whose text
@@ -975,20 +983,96 @@ function claimedCommand(span: string): boolean {
   return EXECUTABLE_TOKEN.test(words[0] ?? "");
 }
 
+/** A markdown list marker opening a line: `- item`, `* item`, `1. item`,
+ *  `2) item`. The required trailing whitespace is what keeps a bold
+ *  `**Verified:**` row or an italic `*span*` from reading as a bullet. */
+const LIST_MARKER = /^[ \t]*(?:[-*+]|\d+[.)])[ \t]+/;
+
+/** The inline verified label a claims row can open with — `**Verified:**`,
+ *  `Verification:`, `proof.` — mirroring the label branch of
+ *  {@link VERIFIED_MARKER}, so the very row that opens the region also parses
+ *  as claims-bearing. */
+const CLAIMS_ROW_LABEL = /^\s*(?:\*\*)?(?:Verif(?:ied|ication)|Proof)(?:\*\*)?\s*[:.]\s*/i;
+
+/** Every backticked span in `text`, trimmed. */
+function backtickSpans(text: string): string[] {
+  return [...text.matchAll(/`([^`\n]+)`/g)].map((match) => (match[1] ?? "").trim());
+}
+
+/** The one backticked span `text` opens with, when it opens with one. */
+function leadingBacktickSpan(text: string): string | undefined {
+  const match = /^`([^`\n]+)`/.exec(text);
+  const span = (match?.[1] ?? "").trim();
+  return span === "" ? undefined : span;
+}
+
+/**
+ * The logical line-units of the verified region. A heading or a list marker
+ * always opens a unit, a blank line ends one, and every other non-blank line
+ * is the markdown lazy continuation of the unit above it. That continuation
+ * rule is the foundation of the #1102 fix: a bullet soft-wrapped across
+ * physical lines stays ONE unit, so its wrapped second line — exactly where
+ * #1097's quoted log line sat — can never pose as a fresh claim-bearing line.
+ * The cost is deliberate and stated: bare claim lines stacked without markers
+ * or blank lines merge into one unit, and only their first extracts. This
+ * fleet writes claims as bullets, so the lost shape is the rarer one.
+ */
+function* claimUnits(region: string): Generator<string[]> {
+  let unit: string[] = [];
+  for (const line of region.split("\n")) {
+    if (/^#{1,6}\s/.test(line) || LIST_MARKER.test(line)) {
+      if (unit.length > 0) yield unit;
+      unit = [line];
+    } else if (line.trim() === "") {
+      if (unit.length > 0) yield unit;
+      unit = [];
+    } else {
+      unit.push(line);
+    }
+  }
+  if (unit.length > 0) yield unit;
+}
+
+/**
+ * The claim candidates one unit makes — a matter of position, not vocabulary
+ * (#1102). A unit claims what it OPENS with:
+ *
+ * - a verified-label row (`**Verified:** …`, the #749/#731 semicolon-joined
+ *   shape) declares every span it carries a claim — the label is what marks
+ *   the whole row as proof;
+ * - any other unit contributes only its leading span — `- \`cd omp && bun test …\`
+ *   — 42 pass, 0 fail.` or a bare `\`./scripts/deploy.sh\` — deployed.` line —
+ *   whose tail is commentary about the result.
+ *
+ * A span embedded after prose is neither: `…assert exactly one log line naming
+ * \`issue is closed\`` quotes the transcript's own output, and quotation is
+ * commentary whatever the quoted words spell.
+ */
+function unitClaimSpans(unit: string[]): string[] {
+  const lead = (unit[0] ?? "").replace(LIST_MARKER, "");
+  if (CLAIMS_ROW_LABEL.test(lead)) {
+    return backtickSpans([lead.replace(CLAIMS_ROW_LABEL, ""), ...unit.slice(1)].join("\n"));
+  }
+  const span = leadingBacktickSpan(lead);
+  return span === undefined ? [] : [span];
+}
+
 /**
  * The commands a PR body claims as proof, each deduplicated by its exact span.
- * Only spans inside the verified region count, and only command-shaped ones
- * — the two filters are what keep `## Verified`-island prose out of the audit.
+ * Extraction is structural twice over: only spans inside the verified region
+ * count ({@link claimRegion}), and only spans in a claim-bearing position
+ * within it ({@link unitClaimSpans}) — the two filters together are what keep
+ * `## Verified`-island prose out of the audit.
  */
 function claimedProofCommands(body: string): string[] {
-  const region = claimRegion(body);
   const claims: string[] = [];
   const seen = new Set<string>();
-  for (const match of region.matchAll(/`([^`\n]+)`/g)) {
-    const span = (match[1] ?? "").trim();
-    if (span === "" || seen.has(span) || !claimedCommand(span)) continue;
-    seen.add(span);
-    claims.push(span);
+  for (const unit of claimUnits(claimRegion(body))) {
+    for (const span of unitClaimSpans(unit)) {
+      if (seen.has(span) || !claimedCommand(span)) continue;
+      seen.add(span);
+      claims.push(span);
+    }
   }
   return claims;
 }

package/src/doctor.ts

@@ -61,7 +61,9 @@ import {
   sessionDirForCwd,
   telegramStateDir,
 } from "./fleet.ts";
-import { pauseInstance } from "./pause.ts";
+import { errText } from "./log.ts";
+import { formatDownDuration } from "./orchestrator-down.ts";
+import { pauseInstance, pauseInstanceAt, pausedPath } from "./pause.ts";
 import type { TelegramHealth } from "./status-render.ts";
 import {
   claimIsLive,
@@ -105,7 +107,7 @@ import {
   readTickConfig,
   type HerdrAgentList,
 } from "./orchestrator-tick.ts";
-import type { Caps, ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
+import type { Caps, ConductorConfig, Escalation, ProjectConfig, RepoTarget, RunState } from "./types.ts";
 import { DEFAULT_ARM_PROOF, type ArmProof } from "./types.ts";
 import {
   DEFAULT_DEPS as UPGRADE_DEPS,
@@ -275,6 +277,13 @@ export interface DoctorDeps {
   /** Whether one pid is a live process. Injected so the abandoned-fence
    *  finding is testable without spawning anything. */
   pidLive?: (pid: number) => boolean;
+  /**
+   * Whether any conductor process is running an upgrade transaction right
+   * now (#1113) — the liveness fact an owner-less `upgrade` sentinel needs,
+   * because it records no pid of its own. Injected so the orphaned-upgrade
+   * fault is testable without scanning /proc.
+   */
+  upgradeLive?: () => boolean;
   /** Live `herdr --session <s> agent list`, parsed through the tick's own
    *  parser (the same "one JSON line on stdout" contract recover.sh reads). */
   herdrAgents?: (session: string) => HerdrAgentList;
@@ -1671,6 +1680,286 @@ function fenceProbe(probes: Probes, project: string | undefined): Finding {
   );
 }
 
+/**
+ * Whether any conductor process is running an upgrade transaction (#1113).
+ *
+ * The upgrade sentinel records no `owner=` — unlike a setup fence, whose pid
+ * is precise, the drain spans subprocesses and the detached installer is a
+ * transient unit — so the honest question is whether ANY conductor process is
+ * mid-upgrade at all. Read from `/proc` cmdlines, never inferred from journal
+ * state: a journal outlives its crash, which is exactly the ambiguity this
+ * check exists to cut through. The whole transaction family matches
+ * (`upgrade`, the detached `upgrade-install`, `upgrade-rollback`); unreadable
+ * entries are skipped, and a zombie's empty cmdline cannot match.
+ */
+function upgradeProcessLive(): boolean {
+  try {
+    for (const entry of readdirSync("/proc")) {
+      if (!/^\d+$/.test(entry)) continue;
+      let raw: string;
+      try {
+        raw = readFileSync(join("/proc", entry, "cmdline"), "utf8");
+      } catch {
+        continue; // vanished mid-scan, or not ours to read
+      }
+      const argv = raw.split("\0").filter((arg) => arg.length > 0);
+      const conductor = argv.some((arg) => /(^|\/)(omp-conductor|cli\.ts)$/.test(arg));
+      if (conductor && argv.some((arg) => arg === "upgrade" || arg.startsWith("upgrade-"))) return true;
+    }
+  } catch {
+    return false;
+  }
+  return false;
+}
+
+/**
+ * The pause-state judgement shared by the doctor row and the daemon watch
+ * (#1113), so the two surfaces cannot drift apart about which pauses still
+ * have a plausible owner. The rules, in the order they are applied:
+ *
+ *  - a recorded `owner=` is the precise fact whenever one exists (#938);
+ *  - `hold` is supported at any age — stopping the fleet deliberately is not
+ *    a fault, and must never nag;
+ *  - `upgrade` owns its drain by existing: a live transaction is fine, an
+ *    exited one is the one fault class, because no future event can ever
+ *    clear that sentinel;
+ *  - `spend-cap` clears itself when spend falls back below the cap;
+ *  - every other durable hold is somebody's decision, reported as such.
+ */
+export type HeldPause = {
+  kind: "held";
+  verdict:
+    | "operator-hold"
+    | "self-expiring"
+    | "durable"
+    | "live-owner"
+    | "live-upgrade"
+    | "dead-owner"
+    | "orphaned-upgrade";
+  source: string;
+  reason?: string;
+  since: number;
+  owner?: number;
+};
+
+export type PauseVerdict = { kind: "unpaused" } | HeldPause;
+
+export interface PauseLiveness {
+  upgradeLive(): boolean;
+  pidLive(pid: number): boolean;
+}
+
+export function classifyPause(
+  pause: { source: string; reason?: string; since: number; owner?: number } | undefined,
+  liveness: PauseLiveness,
+): PauseVerdict {
+  if (pause === undefined) return { kind: "unpaused" };
+  const base: Omit<HeldPause, "verdict"> = {
+    kind: "held",
+    source: pause.source,
+    ...(pause.reason === undefined ? {} : { reason: pause.reason }),
+    since: pause.since,
+    ...(pause.owner === undefined ? {} : { owner: pause.owner }),
+  };
+  if (pause.owner !== undefined) {
+    return { ...base, verdict: liveness.pidLive(pause.owner) ? "live-owner" : "dead-owner" };
+  }
+  if (pause.source === "hold") return { ...base, verdict: "operator-hold" };
+  if (pause.source === "upgrade") {
+    return { ...base, verdict: liveness.upgradeLive() ? "live-upgrade" : "orphaned-upgrade" };
+  }
+  if (pause.source === "spend-cap") return { ...base, verdict: "self-expiring" };
+  return { ...base, verdict: "durable" };
+}
+
+/** Fleet-scoped pages carry issue `0` — there is no tracker issue for this. */
+const PAUSE_PAGE_ISSUE = 0;
+
+/**
+ * The orphaned-upgrade page (#1113). Anchored on the sentinel's `since`, like
+ * every once-per-incident page here: the notification ledger dedupes on the
+ * summary, so a still-orphaned sentinel pages exactly once while each newly
+ * recreated sentinel — a fresh `since` — pages again.
+ */
+export function orphanedPauseEscalation(project: string, pause: HeldPause): Escalation {
+  const since = new Date(pause.since).toISOString();
+  return {
+    tier: 2,
+    category: "fleet-stopped",
+    project,
+    issue: PAUSE_PAGE_ISSUE,
+    summary: `Orphaned upgrade pause holds dispatch since ${since} — no upgrade process is running (${project})`,
+    detail: [
+      `Dispatch has been paused by the upgrade sentinel since ${since}` +
+        (pause.reason === undefined ? "" : ` — "${pause.reason}"`) +
+        ".",
+      "The upgrade transaction that wrote it has exited, so no future event will",
+      "ever clear it: dispatch stays stopped until someone resumes.",
+      "",
+      "Remedy: `omp-conductor resume --all`. Nothing auto-resumes — reporting is",
+      "the whole contract (#1113); resuming a pause nobody authorised stays yours.",
+    ].join("\n"),
+  };
+}
+
+/** One pause sentinel read: who set it, why, when — and any recorded owner. */
+type PauseInstance = { source: string; reason?: string; since: number; owner?: number };
+
+export interface OrphanedPauseDeps {
+  /** The project this daemon serves; the effective sentinel for it reads
+   *  through the same precedence the doctor row uses (project wins over the
+   *  legacy global file). */
+  project: string;
+  escalate(e: Escalation): Promise<void>;
+  log?(msg: string): void;
+  pause?: () => PauseInstance | undefined;
+  /** This project's own `paused-<project>` sentinel, told apart from the
+   *  legacy global one so a genuinely per-project orphan pages for its own
+   *  project while a shared one pages exactly once (#1113 review round 1). */
+  projectSentinel?: () => PauseInstance | undefined;
+  globalSentinel?: () => PauseInstance | undefined;
+  upgradeLive?: () => boolean;
+  pidLive?: (pid: number) => boolean;
+  /** Every configured project name — from the shared config.json every
+   *  daemon on this host reads, which is what makes the canonical owner
+   *  below one identity everywhere. Defaults to {@link loadConfig}. */
+  configuredProjects?: () => string[];
+}
+
+/**
+ * Best-effort page delivery, mirroring `safeEscalate`: a transport failure is
+ * logged, never thrown, and cannot take the tick down.
+ */
+async function deliverOrphanPage(
+  deps: OrphanedPauseDeps,
+  write: (msg: string) => void,
+  held: HeldPause,
+): Promise<void> {
+  let delivered = true;
+  try {
+    await deps.escalate(orphanedPauseEscalation(deps.project, held));
+  } catch (err) {
+    delivered = false;
+    write(`orphaned-pause escalation could not be delivered: ${errText(err)}`);
+  }
+  write(
+    `ERROR: an orphaned upgrade pause holds dispatch (since ${new Date(held.since).toISOString()})` +
+      (delivered ? " — paged once for this sentinel" : " — page unsent"),
+  );
+}
+
+/**
+ * The daemon half of #1113: raise the orphaned-upgrade case through the
+ * ordinary escalation path instead of waiting for a tick to read `status` —
+ * a stopped fleet is precisely the condition that cannot assume anything is
+ * watching. Called above the pause gate on every pass; nothing here ever
+ * resumes.
+ *
+ * "Once" needs an ownership boundary, because the ledger that dedupes pages
+ * is per-project store while the upgrade sentinel usually is not: it writes
+ * the legacy GLOBAL file, and every project's effective read falls back to
+ * it. So the two sentinel kinds are told apart before paging:
+ *
+ *  - this project's own `paused-<project>` orphan → this project pages as
+ *    itself, whatever else is configured;
+ *  - the legacy global orphan → exactly one daemon owns the incident: the
+ *    one serving the alphabetically-first configured project name — the same
+ *    name on every daemon, because they all read the same config.json. A
+ *    daemon that cannot read that config assumes it is alone and pages
+ *    anyway; staying silent on a failed read would be the exact silence
+ *    #1113 exists to end.
+ *
+ * Within either owner, the ledger dedupes on the since-anchored summary, so
+ * a still-orphaned sentinel pages once and a recreated one (fresh `since`)
+ * pages again.
+ */
+export async function reconcileOrphanedPause(deps: OrphanedPauseDeps): Promise<void> {
+  const write =
+    deps.log ??
+    ((msg: string) => process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`));
+  const liveness: PauseLiveness = {
+    upgradeLive: deps.upgradeLive ?? upgradeProcessLive,
+    pidLive: deps.pidLive ?? pidAlive,
+  };
+  const verdict = classifyPause((deps.pause ?? (() => pauseInstance(deps.project)))(), liveness);
+  if (verdict.kind !== "held" || verdict.verdict !== "orphaned-upgrade") return;
+
+  // A per-project sentinel of its own: this project owns that page outright.
+  const ownVerdict = classifyPause(
+    (deps.projectSentinel ?? (() => pauseInstanceAt(pausedPath(deps.project))))(),
+    liveness,
+  );
+  if (ownVerdict.kind === "held" && ownVerdict.verdict === "orphaned-upgrade") {
+    await deliverOrphanPage(deps, write, ownVerdict);
+    return;
+  }
+
+  // The shared global sentinel: gate on the canonical owner before paging.
+  let configured: string[] | undefined;
+  try {
+    configured = deps.configuredProjects?.() ?? loadConfig().projects.map((p) => p.name);
+  } catch {
+    configured = undefined;
+  }
+  const names = configured ?? [];
+  const owner = names.length === 0 ? deps.project : [...names].sort()[0];
+  if (deps.project !== owner) {
+    write(`orphaned upgrade pause holds dispatch — ${owner} owns the page for the global sentinel`);
+    return;
+  }
+  const globalVerdict = classifyPause(
+    (deps.globalSentinel ?? (() => pauseInstanceAt(pausedPath())))(),
+    liveness,
+  );
+  if (globalVerdict.kind === "held" && globalVerdict.verdict === "orphaned-upgrade") {
+    await deliverOrphanPage(deps, write, globalVerdict);
+  }
+}
+
+/**
+ * The dispatch-pause row itself (#1113), beside {@link fenceProbe}: the fence
+ * row judges a declared owning process, this one judges the pause — `pass`
+ * when unpaused, and when paused naming the source, the age, and whether
+ * anything still plausibly owns it. One fault class only: the orphaned
+ * upgrade sentinel, which can never clear itself. Nothing here resumes
+ * anything; reporting is the whole contract.
+ */
+function pauseProbe(probes: Probes, project: string | undefined): Finding {
+  const id = project === undefined ? "dispatch-pause" : `dispatch-pause:${project}`;
+  const v = classifyPause(probes.pauseFence(project), {
+    upgradeLive: probes.upgradeLive,
+    pidLive: probes.pidLive,
+  });
+  if (v.kind === "unpaused") return passFinding(id, "dispatch is not paused");
+  const why = v.reason === undefined ? "" : ` — "${v.reason}"`;
+  const age = formatDownDuration(Math.max(0, probes.now() - v.since));
+  const base = `dispatch paused by ${v.source}${why}, ${age} ago`;
+  switch (v.verdict) {
+    case "orphaned-upgrade":
+      return failFinding(
+        id,
+        `${base}, and no upgrade process is running — this pause can never clear itself`,
+        "run `omp-conductor resume --all` (nothing here resumes it)",
+      );
+    case "dead-owner":
+      return warnFinding(
+        id,
+        `${base} — its owning process (pid ${v.owner}) is gone`,
+        `confirm nothing is running, then \`omp-conductor resume${project === undefined ? "" : ` --project ${project}`}\``,
+      );
+    case "operator-hold":
+      return passFinding(id, `${base} — a deliberate hold, supported at any age`);
+    case "live-upgrade":
+      return passFinding(id, `${base} — the upgrade transaction is draining, and resumes dispatch when it finishes`);
+    case "live-owner":
+      return passFinding(id, `${base} — owned by live pid ${v.owner}`);
+    case "self-expiring":
+      return passFinding(id, `${base} — clears itself when spend falls below the cap`);
+    case "durable":
+      return passFinding(id, `${base} — no owning process; held until someone resumes it`);
+  }
+}
+
 function herdrResumeProbe(probes: Probes): Finding {
   if (!probes.herdrInstalled()) {
     return passFinding("herdr-resume", "herdr not installed — nothing to check");
@@ -2066,12 +2355,15 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
     findings.push(timezoneProbe(undefined));
     findings.push(await telegramProbe(probes, undefined, checkedAt));
     findings.push(fenceProbe(probes, undefined));
+    findings.push(pauseProbe(probes, undefined));
   } else {
     for (const p of projects) findings.push(timezoneProbe(p));
     for (const p of projects) findings.push(await telegramProbe(probes, p, checkedAt));
     // Per project, because the sentinel is: one fleet's fence must never be
     // reported against another's name.
     for (const p of projects) findings.push(fenceProbe(probes, p.name));
+    // The pause state itself (#1113), per project for the same reason.
+    for (const p of projects) findings.push(pauseProbe(probes, p.name));
   }
   // The billing class the spend row needs (#984). Declared only when EVERY
   // project requires a subscription credential: on a mixed fleet the zeros of
@@ -2173,6 +2465,7 @@ export function defaultProbes(): Probes {
     canonicalUnits: defaultCanonicalUnits,
     herdrInstalled: () => Bun.which("herdr") !== null,
     pauseFence: (project) => pauseInstance(project),
+    upgradeLive: upgradeProcessLive,
     workerHarness: defaultWorkerHarness,
     pidLive: (pid) => pidAlive(pid),
     herdrAgents: defaultHerdrAgents,