omp-conductor 0.18.1 → 0.19.0

package/src/fleet.ts

@@ -15,20 +15,16 @@
 
 import { spawnSync } from "node:child_process";
 import {
-  createReadStream,
   existsSync,
   mkdirSync,
-  readdirSync,
   readFileSync,
   rmSync,
-  statSync,
   writeFileSync,
 } from "node:fs";
-import { createInterface } from "node:readline";
 import { homedir } from "node:os";
 import { dirname, join, sep } from "node:path";
 import { findProject, loadConfig, resolveArmProof, stateDir } from "./config.ts";
-import { clearArmChallenge, recordArmChallenge } from "./arm-challenge.ts";
+import { clearArmTransaction, readArmAcknowledgement, recordArmChallenge } from "./arm-challenge.ts";
 import {
   claimedTelegramTopics,
   lockPidAlive,
@@ -81,6 +77,7 @@ import {
 import type { WorkerPausePhase } from "./worker.ts";
 import {
   formatFleetStatus,
+  formatGroomingStatus,
   type DaemonProjectHealth,
   type DispatchLayer,
   type FleetDaemonProbe,
@@ -328,19 +325,10 @@ export interface ArmDeps {
    * The orchestrator session file the live omp-telegram claim names for this
    * project, or undefined when there is no (readable) claim. The default
    * resolves the claim the same way the send does; tests inject a fixture.
-   * Absent → arm watches the cwd-derived session directory, the pre-claim
-   * behaviour.
+   * Claim-only feeds it to the plumbing verdict's session-identity checks;
+   * the challenge proof never reads a transcript, so it ignores the claim.
    */
   claimedSessionFile?: () => string | undefined;
-  /**
-   * Waits for the challenge to appear as a user turn somewhere under the scan
-   * directories. The waiter owns transcript discovery — not the caller —
-   * because the reply may land in a session that starts *after* the send, so a
-   * path resolved before the challenge went out can be the wrong file by the
-   * time the operator answers (#142), and because the claimed session file can
-   * sit in a different directory than the tick cwd implies (#600).
-   */
-  waitForUserTurn?: (dirs: readonly string[], code: string, sentAt: number, timeoutMs: number) => Promise<boolean>;
   now?: () => number;
   sleep?: (ms: number) => Promise<void>;
   timeoutMs?: number;
@@ -355,6 +343,14 @@ export interface ArmDeps {
   pidAlive?: (pid: number) => boolean;
   lockPidAlive?: (pid: number) => boolean;
   lockFresh?: (mtimeMs: number) => boolean;
+  /**
+   * Where the pending-proof heartbeat is written (#861). The challenge proof
+   * waits up to five minutes on a human, inside a fence that holds dispatch:
+   * without this, that wait is silent and a healthy process is indistinguishable
+   * from a dead one. Absent means no reporting — the callers that have a surface
+   * (the CLI, the wizard) pass theirs.
+   */
+  progress?: (line: string) => void;
 }
 
 export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promise<ArmResult> {
@@ -394,6 +390,13 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
   // one pane that can ask. The challenge names which one, or the operator is
   // answering a question they cannot attribute.
   const named = tick.config.project ?? projectName;
+  // The handshake state key must be exactly what the orchestrator's inbound
+  // adapter computes: TickConfig.project, undefined for a legacy unstamped
+  // config. `named` may fall back to the CLI argument for the challenge text
+  // and config lookups; the state key must not — the adapter has no CLI
+  // argument to fall back to, and a mismatched key would make arming wait on
+  // an acknowledgement that can never be written.
+  const stateKey = tick.config.project;
 
   // The arming proof is a declared per-project policy (#613). A config that
   // cannot name the project fails safe to `challenge` — today's authenticated
@@ -406,15 +409,11 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
     /* no project config — keep today's challenge behaviour */
   }
 
-  // Resolve the orchestrator's live session file *before* the challenge goes
-  // out — the same claim the send follows (#600). A pane resumed from a
-  // session created elsewhere (herdr pins it to the original transcript)
-  // writes a session file outside the directory the tick cwd implies, and a
-  // cwd-derived scan would poll the one place the reply is guaranteed not to
-  // be. A claim outside the session tree arm scans can never be answered, so
-  // that is a stop, not five minutes of polling. (Claim-only resolves the
-  // claim too — the verdict checks the same session identity — but names the
-  // refusal itself rather than throwing the transcript wording.)
+  // The orchestrator's live session file per omp-telegram's claim (#600) —
+  // input to the claim-only verdict's session-identity checks below. The
+  // challenge proof never reads it: its acknowledgement is conductor state,
+  // so where (or whether) a transcript lives is no longer part of arming
+  // (#614).
   const claimed = deps.claimedSessionFile !== undefined ? deps.claimedSessionFile() : claimedOrchestratorSessionFile(named);
 
   // Prefer the project's live forum topic so arm challenges land where
@@ -467,17 +466,6 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
     return { path, alreadyArmed, owner: channel.owner, proof };
   }
 
-  // The transcript proof needs a session tree to poll. The claim-only verdict
-  // needs no such thing — it reads session identity from omp-telegram's own
-  // state — so this stop stays on the challenge path only.
-  const dirs = armSessionScanDirs(tick.cwd, claimed);
-  if (dirs.every((d) => !existsSync(d))) {
-    throw new Error(
-      `no orchestrator session directory under ${dirs.join(" or ")} — ` +
-        `the inbound proof is read from a user turn in a transcript there. Start the pane orchestrator, let it settle, then arm again`,
-    );
-  }
-
   const code = makeChallengeCode();
   const text =
     `Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
@@ -486,54 +474,46 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
 
   const send = deps.sendChallenge ?? sendTelegramMessage;
   const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
-  // Read before the send, not after: a transcript untouched since this instant
-  // cannot contain the reply, and that is what the waiter filters on.
   const sentAt = (deps.now ?? Date.now)();
-  // The orchestrator must be able to classify the reply as an *active* arming
-  // proof for as long as this handshake is live, so the authenticated record
-  // (hash + expiry, never the code) is written before the challenge goes out
-  // and cleared the moment this end settles (#415).
-  recordArmChallenge(named, code, sentAt + timeoutMs);
+  // The orchestrator's inbound adapter can only acknowledge an *active*
+  // challenge, so the authenticated pending record (hash + expiry, never the
+  // code) is written before the challenge goes out and settled the moment
+  // this end finishes (#415). The returned id pins the wait below: an
+  // acknowledgement can only ever name the currently-pending id, so replacing
+  // a challenge makes every prior acknowledgement inert.
+  const challengeId = recordArmChallenge(stateKey, code, sentAt, sentAt + timeoutMs);
   try {
     await send(token, channel.owner, text, sendTopic);
   } catch (err) {
-    // The challenge never went out, so it must not linger as a classifiable
-    // proof either.
-    clearArmChallenge(named);
+    // The challenge never went out, so its transaction must not linger as a
+    // classifiable proof either.
+    clearArmTransaction(stateKey, challengeId);
     throw new Error(
       `arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
     );
   }
+  // What is being waited on, before the wait starts: the window, the chat the
+  // reply has to land in, and the fact that dispatch is held until it does
+  // (#861). One line, so a five-minute wait opens with an explanation rather
+  // than with silence.
+  deps.progress?.(
+    `arm: challenge sent to ${channel.owner}${sendTopic === undefined ? "" : ` (topic ${sendTopic})`} — ` +
+      `waiting up to ${armClock(timeoutMs)} for the reply. Dispatch stays held until it arrives.`,
+  );
 
-  const injected = deps.waitForUserTurn;
-  const scan: SessionScan = injected
-    ? { seen: await injected(dirs, code, sentAt, timeoutMs), scanned: [], ignored: [] }
-    : await waitForChallengeInSessions(dirs, code, sentAt, timeoutMs, deps);
-  if (!scan.seen) {
-    // The proof missed its window: a lookalike reply must not stay classifiable
-    // after arming gave up, so the record is cleared before the failure lands.
-    clearArmChallenge(named);
-    // The claimed session file is named before the bridge/token/chat block: a
-    // rotation under the window is visible from this error alone, and on the
-    // host this guard exists for the claimed file is the one that held the
-    // answer (#600).
-    const listing = [
-      ...(claimed === undefined
-        ? []
-        : [
-            `claimed orchestrator session file: ${claimed}` +
-              (dirs.includes(dirname(claimed))
-                ? " (watched)"
-                : ` (NOT under any watched dir — a reply there can never be seen)`),
-          ]),
-      `session dir: ${dirs.join(", ")}`,
-      ...scan.scanned.map((f) => `  watched: ${f}`),
-      ...scan.ignored.map((f) => `  ignored (stale, last written before the challenge): ${f}`),
-    ].join("\n");
+  // Wait for the orchestrator's own acknowledgement — conductor state written
+  // by the inbound user-turn adapter when the real reply lands (#614). No
+  // transcript is read: the proof no longer depends on where (or whether) a
+  // session file lives, which is exactly the discovery that mis-fired on the
+  // host three times (#614). A wrong-project or lookalike reply writes no
+  // acknowledgement, so the window simply runs out fail-closed.
+  if (!(await waitForArmAcknowledgement(challengeId, timeoutMs, deps))) {
+    clearArmTransaction(stateKey, challengeId);
     throw new Error(
-      `arm: the challenge never arrived as a user turn in time — NOT armed.\n` +
-        listing +
-        `\nInbound Telegram is not reaching the omp session. Check, in order:\n` +
+      `arm: the challenge was never acknowledged in time — NOT armed.\n` +
+        `The orchestrator's inbound adapter acknowledges the reply when it lands as a user turn; ` +
+        `no acknowledgement for challenge ${challengeId} arrived.\n` +
+        `Inbound Telegram is not reaching the omp session. Check, in order:\n` +
         `  * is the bridge polling?      attach and run: /telegram status\n` +
         `  * is another process holding this bot token? Telegram allows exactly one\n` +
         `    getUpdates consumer and rejects the second with HTTP 409.\n` +
@@ -542,10 +522,11 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
   }
 
   writeArmedMarker(path, channel.owner, arm);
-  // The transcript proof landed and this project is armed: the record has done
-  // its job. A stale record would also keep a later unsolicited lookalike alive
-  // longer than the fresh challenge it was cut for.
-  clearArmChallenge(named);
+  // The acknowledgement landed and this project is armed: settling clears this
+  // transaction's pending record and acknowledgement — never a newer
+  // replacement's — so a later unsolicited lookalike stays inert past this
+  // handshake.
+  clearArmTransaction(stateKey, challengeId);
   return { path, alreadyArmed, owner: channel.owner, challenge: code, proof };
 }
 
@@ -588,10 +569,14 @@ export interface HaltWithPaneResult extends HaltResult {
 export function hold(
   projectName?: string,
   source: string = "hold",
-  opts: { keepTicks?: boolean } = {},
+  opts: { keepTicks?: boolean; reason?: string } = {},
 ): HoldResult {
   const wasPaused = isPaused(projectName);
-  setPaused(true, { source }, projectName);
+  // `source` is a space-free token by contract (`pauseSourceToken`) — the fence
+  // matches on it, and whitespace there makes a pause the fence cannot prove
+  // (#552). The human sentence belongs in `reason=`, which is why this is a
+  // separate field rather than something a caller concatenates into `source`.
+  setPaused(true, { source, ...(opts.reason === undefined ? {} : { reason: opts.reason }) }, projectName);
   if (opts.keepTicks === true) return { wasPaused };
   return { wasPaused, disarmed: disarmTicks(projectName) };
 }
@@ -752,6 +737,407 @@ export interface HerdrAgent {
   sessionPath?: string;
 }
 
+/**
+ * The Herdr representation of one live conductor worker (#840).
+ *
+ * ## Why this shape, and not `herdr agent start --kind omp`
+ *
+ * The authoritative worker is the `session-host` child conductor spawns itself:
+ * it holds the typed control socket, the mediated verb socket, the worktree, the
+ * transcript, the accounting and the output-schema settlement. Starting a second
+ * OMP process in a pane would look right in the workspace and be authoritative
+ * for nothing — the named silent fake of #840 — so nothing here starts an agent.
+ *
+ * Herdr already has the verb for exactly this case: an *external* supervisor
+ * reports lifecycle onto a pane it owns (`pane report-agent`,
+ * `pane report-agent-session`, `pane release-agent`). So conductor keeps the
+ * child it already spawned, and reports that child's identity and state onto a
+ * pane whose only job is display.
+ *
+ * ## What is displayed, and why that is not "a transcript-tail pane"
+ *
+ * The pane runs `omp-conductor tail <issue>`, which is the repository's existing
+ * read-only follower. The distinction #840 draws is about *authority*, not about
+ * pixels: nothing here parses that output, and no run state, settlement, turn,
+ * spend or model fact is derived from it. Identity comes from the exact child
+ * pid handed over by the spawn path, and state comes from the typed events the
+ * proxy already emits.
+ *
+ * ## Observation-only is structural, not a gate
+ *
+ * The authoritative child is spawned with `stdin: "ignore"` — there is no file
+ * descriptor for a keystroke to travel down, from a pane or anywhere else. The
+ * pane's own process is a follower whose stdin reaches only itself. So "input
+ * cannot reach OMP, change its prompt, invoke a verb, or settle the run" is a
+ * property of the process tree rather than a check that could be forgotten.
+ */
+export interface WorkerPaneIdentity {
+  project: string;
+  issue: number;
+  attempt: number;
+  /** The run row's id — the durable identity the pane is reported under. */
+  runId: string;
+  /** The exact `session-host` pid, from the spawn path's own `onSpawn`. */
+  pid: number;
+  /** The transcript the child opened, when it has already reported one. */
+  sessionFile?: string;
+}
+
+/** A tracked pane, or the explicit reason there is none. Never a silent claim. */
+export type WorkerPaneOutcome =
+  | { kind: "tracked"; paneId: string; label: string; pid: number }
+  | { kind: "unavailable"; reason: string };
+
+/** One `herdr` invocation, injected so every path is testable with no terminal. */
+export type HerdrRun = (args: readonly string[]) => { ok: boolean; stdout: string; stderr: string };
+
+export interface WorkerPaneDeps {
+  run?: HerdrRun;
+  session?: string;
+  /** The follower command the pane displays; the CLI by default. */
+  viewer?: (identity: WorkerPaneIdentity) => readonly string[];
+}
+
+/**
+ * The pane's unique, human-readable name (#840: "uniquely named").
+ *
+ * Keyed by run id rather than issue: an issue can have several attempts, and two
+ * attempts must never resolve to one pane. The issue and attempt ride along
+ * because the operator reads this label in a workspace, not a database.
+ */
+/**
+ * The external-supervisor source every conductor pane report carries.
+ *
+ * This, not the label, is what proves a pane is conductor's: it is written by
+ * `openWorkerPane` and read back by {@link listWorkerPanes}, so the two halves
+ * of reconciliation cannot come to disagree about what "ours" means.
+ */
+export const WORKER_PANE_SOURCE = "omp-conductor";
+
+export function workerPaneLabel(identity: WorkerPaneIdentity): string {
+  return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
+}
+
+const realHerdrRun: HerdrRun = (args) => {
+  const res = spawnSync("herdr", [...args], { encoding: "utf8", timeout: 8_000, env: process.env });
+  return {
+    ok: !res.error && res.status === 0,
+    stdout: res.stdout ?? "",
+    stderr: res.stderr ?? (res.error === undefined ? "" : String(res.error.message)),
+  };
+};
+
+/** `omp-conductor tail` — read-only by construction, and never parsed here. */
+function defaultViewer(identity: WorkerPaneIdentity): readonly string[] {
+  return ["omp-conductor", "tail", String(identity.issue), "--project", identity.project];
+}
+
+/**
+ * Create the pane, run the follower in it, and report the child's identity and
+ * initial state — or say exactly why it could not.
+ *
+ * Every step is checked, and the first failure returns `unavailable` with the
+ * reason: a partially established representation (a pane with no agent identity,
+ * say) is worse than none, because it is a pane the operator would read as a
+ * tracked worker. What this deliberately does NOT do is decide what a failure
+ * means for the launch — failing closed versus running degraded is #841's
+ * policy, and inventing it here would pre-empt it.
+ */
+export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
+  const run = deps.run ?? realHerdrRun;
+  const session = deps.session ?? resolveHerdrSession();
+  const label = workerPaneLabel(identity);
+  const base = ["--session", session, "pane"];
+
+  const split = run([...base, "split", "--direction", "down", "--ratio", "0.3"]);
+  if (!split.ok) {
+    return { kind: "unavailable", reason: `herdr pane split failed: ${firstLine(split.stderr) || "no output"}` };
+  }
+  const paneId = firstLine(split.stdout);
+  if (paneId === "") {
+    return { kind: "unavailable", reason: "herdr pane split reported no pane id" };
+  }
+
+  const named = run([...base, "rename", paneId, label]);
+  if (!named.ok) {
+    return { kind: "unavailable", reason: `herdr pane rename failed: ${firstLine(named.stderr) || "no output"}` };
+  }
+
+  // Identity before display: the pane must be attributable to this exact run
+  // before it shows anything, so a pane that appears is never a pane nobody can
+  // trace back to a worker.
+  const identified = run([
+    ...base,
+    "report-agent-session",
+    paneId,
+    "--source",
+    WORKER_PANE_SOURCE,
+    "--agent",
+    label,
+    "--agent-session-id",
+    identity.runId,
+    ...(identity.sessionFile === undefined ? [] : ["--agent-session-path", identity.sessionFile]),
+    "--session-start-source",
+    WORKER_PANE_SOURCE,
+  ]);
+  if (!identified.ok) {
+    return {
+      kind: "unavailable",
+      reason: `herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
+    };
+  }
+
+  const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
+  if (!started.ok) {
+    return { kind: "unavailable", reason: `herdr pane run failed: ${firstLine(started.stderr) || "no output"}` };
+  }
+
+  // A worker that has just spawned is working by definition. The ongoing
+  // projection of turn/pause/blocked transitions is #842's, from the same typed
+  // events — never from the pane's output.
+  const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
+  if (!reported.ok) {
+    return { kind: "unavailable", reason: reported.reason };
+  }
+  return { kind: "tracked", paneId, label, pid: identity.pid };
+}
+
+/** Report one lifecycle state for a tracked pane. Monotonic `seq` is the caller's. */
+export function reportWorkerPaneState(
+  paneId: string,
+  label: string,
+  state: "working" | "idle" | "blocked" | "unknown",
+  deps: { run?: HerdrRun; session?: string; seq?: number; message?: string } = {},
+): { ok: true } | { ok: false; reason: string } {
+  const run = deps.run ?? realHerdrRun;
+  const session = deps.session ?? resolveHerdrSession();
+  const res = run([
+    "--session",
+    session,
+    "pane",
+    "report-agent",
+    paneId,
+    "--source",
+    WORKER_PANE_SOURCE,
+    "--agent",
+    label,
+    "--state",
+    state,
+    ...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
+    ...(deps.message === undefined ? [] : ["--message", deps.message]),
+  ]);
+  return res.ok
+    ? { ok: true }
+    : { ok: false, reason: `herdr pane report-agent failed: ${firstLine(res.stderr) || "no output"}` };
+}
+
+/**
+ * Hand lifecycle authority back when the worker is gone.
+ *
+ * Release, never close: whether a settled worker's pane is closed, kept, or
+ * retained for N settlements is #841's documented policy, and a slice that
+ * closed panes here would decide it by accident. Releasing says only "conductor
+ * no longer speaks for this agent", which is exactly what is true.
+ */
+export function releaseWorkerPane(
+  paneId: string,
+  label: string,
+  deps: { run?: HerdrRun; session?: string; seq?: number } = {},
+): { ok: true } | { ok: false; reason: string } {
+  const run = deps.run ?? realHerdrRun;
+  const session = deps.session ?? resolveHerdrSession();
+  const res = run([
+    "--session",
+    session,
+    "pane",
+    "release-agent",
+    paneId,
+    "--source",
+    WORKER_PANE_SOURCE,
+    "--agent",
+    label,
+    ...(deps.seq === undefined ? [] : ["--seq", String(deps.seq)]),
+  ]);
+  return res.ok
+    ? { ok: true }
+    : { ok: false, reason: `herdr pane release-agent failed: ${firstLine(res.stderr) || "no output"}` };
+}
+
+/**
+ * Hand back the pane a dead worker left behind (#842).
+ *
+ * Called for every run a restart reaps. The recorded pid is deliberately NOT
+ * consulted for liveness: a `session-host` child dies with the daemon that owned
+ * its verb socket, so an orphaned row's worker is gone whatever pid it carries —
+ * and pids are reused, so checking one is how a stranger's process comes to read
+ * as a live worker. A run with no recorded pane is a no-op, not a failure: it
+ * never had one to release.
+ */
+export function releaseOrphanedWorkerPane(
+  run: { paneId?: string; paneLabel?: string },
+  deps: { run?: HerdrRun; session?: string; seq?: number } = {},
+): { kind: "none" } | { kind: "released"; paneId: string } | { kind: "failed"; paneId: string; reason: string } {
+  if (run.paneId === undefined || run.paneLabel === undefined) return { kind: "none" };
+  const released = releaseWorkerPane(run.paneId, run.paneLabel, deps);
+  return released.ok
+    ? { kind: "released", paneId: run.paneId }
+    : { kind: "failed", paneId: run.paneId, reason: released.reason };
+}
+
+/**
+ * Every conductor-owned pane Herdr currently has, keyed by the run id it was
+ * reported under (#841).
+ *
+ * Read from `pane list`'s own agent-session record, never from a label pattern:
+ * the label is what a human reads, and matching on it is exactly how a stale
+ * lookalike (a renamed pane, a pane from a previous fleet) gets mistaken for a
+ * live worker. `source` proves conductor reported it; `value` is the run id.
+ */
+export function listWorkerPanes(
+  deps: { run?: HerdrRun; session?: string } = {},
+): { ok: true; panes: { paneId: string; runId: string; label?: string }[] } | { ok: false; reason: string } {
+  const run = deps.run ?? realHerdrRun;
+  const session = deps.session ?? resolveHerdrSession();
+  const res = run(["--session", session, "pane", "list"]);
+  if (!res.ok) {
+    return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(res.stdout);
+  } catch (err) {
+    return { ok: false, reason: `herdr pane list was unreadable: ${err instanceof Error ? err.message : String(err)}` };
+  }
+  const panes = (parsed as { result?: { panes?: unknown[] } }).result?.panes;
+  if (!Array.isArray(panes)) return { ok: false, reason: "herdr pane list carried no pane array" };
+  const owned: { paneId: string; runId: string; label?: string }[] = [];
+  for (const pane of panes) {
+    const p = pane as {
+      pane_id?: unknown;
+      agent?: unknown;
+      agent_session?: { source?: unknown; value?: unknown };
+    };
+    if (typeof p.pane_id !== "string") continue;
+    if (p.agent_session?.source !== WORKER_PANE_SOURCE) continue;
+    const runId = p.agent_session.value;
+    // Ours by source but carrying no run id: an identity nobody can resolve is
+    // not an identity. Reported as a pane with an empty run id so the caller
+    // treats it as stale rather than silently ignoring it.
+    owned.push({
+      paneId: p.pane_id,
+      runId: typeof runId === "string" ? runId : "",
+      ...(typeof p.agent === "string" ? { label: p.agent } : {}),
+    });
+  }
+  return { ok: true, panes: owned };
+}
+
+/** One live worker, as the reconciler needs to see it. */
+export interface LiveWorkerPane {
+  runId: string;
+  issue: number;
+  attempt: number;
+  project: string;
+  /** The recorded session-host pid; absent when nothing ever reported one. */
+  pid?: number;
+  paneId?: string;
+  paneLabel?: string;
+  sessionFile?: string;
+}
+
+export type WorkerPaneReconciliation =
+  /** The recorded pane is still there and still carries this run — nothing done. */
+  | { kind: "intact"; runId: string; paneId: string }
+  /** Herdr lost the pane (a restart); a new one now represents the same child. */
+  | { kind: "reassociated"; runId: string; paneId: string; label: string }
+  /** No representation, and the reason. The run keeps working regardless. */
+  | { kind: "untracked"; runId: string; reason: string }
+  /** A conductor pane whose run is not live: handed back to Herdr. */
+  | { kind: "stale-released"; paneId: string; runId: string }
+  | { kind: "stale-release-failed"; paneId: string; runId: string; reason: string };
+
+/**
+ * Make Herdr's conductor-owned panes agree with the live run set (#841).
+ *
+ * Idempotent by construction: a second pass over an already-reconciled fleet
+ * returns `intact` for every live run and finds no stale panes, so repeated
+ * daemon or Herdr restarts converge rather than accumulate.
+ *
+ * Three rules, and each exists to refuse a specific way this goes wrong:
+ *
+ * - **A live run's pane is re-created, never duplicated.** Re-association is
+ *   keyed on the run id Herdr itself reports, so a pane that is still there is
+ *   left alone. Only a run whose pane Herdr no longer has gets a new one.
+ * - **Cleanup is by exact identity, never by name or age.** A pane is stale only
+ *   when the run id it carries is absent from the live set. A worker's own pane
+ *   can therefore never be released while its run is live, whatever it is called
+ *   and however old it is.
+ * - **Nothing here can stop a worker.** The only mutation is `release-agent`,
+ *   which hands lifecycle authority back to Herdr; the authoritative child is
+ *   never signalled, and its pane is never closed.
+ */
+export function reconcileWorkerPanes(
+  live: readonly LiveWorkerPane[],
+  deps: WorkerPaneDeps = {},
+): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
+  const listed = listWorkerPanes(deps);
+  if (!listed.ok) return { ok: false, reason: listed.reason };
+  const byRun = new Map(listed.panes.map((pane) => [pane.runId, pane]));
+  const liveIds = new Set(live.map((worker) => worker.runId));
+  const outcomes: WorkerPaneReconciliation[] = [];
+
+  for (const worker of live) {
+    const held = byRun.get(worker.runId);
+    if (held !== undefined) {
+      outcomes.push({ kind: "intact", runId: worker.runId, paneId: held.paneId });
+      continue;
+    }
+    // Herdr does not have this run's pane. Only the pid makes a replacement
+    // honest: the pane represents an exact child, so without one there is
+    // nothing to represent and inventing a pane would be the silent claim this
+    // whole surface exists to avoid.
+    if (worker.pid === undefined) {
+      outcomes.push({
+        kind: "untracked",
+        runId: worker.runId,
+        reason: "no session-host pid was ever recorded for this run",
+      });
+      continue;
+    }
+    const opened = openWorkerPane(
+      {
+        project: worker.project,
+        issue: worker.issue,
+        attempt: worker.attempt,
+        runId: worker.runId,
+        pid: worker.pid,
+        ...(worker.sessionFile === undefined ? {} : { sessionFile: worker.sessionFile }),
+      },
+      deps,
+    );
+    outcomes.push(
+      opened.kind === "tracked"
+        ? { kind: "reassociated", runId: worker.runId, paneId: opened.paneId, label: opened.label }
+        : { kind: "untracked", runId: worker.runId, reason: opened.reason },
+    );
+  }
+
+  for (const pane of listed.panes) {
+    if (liveIds.has(pane.runId)) continue;
+    const released = releaseWorkerPane(pane.paneId, pane.label ?? "", deps);
+    outcomes.push(
+      released.ok
+        ? { kind: "stale-released", paneId: pane.paneId, runId: pane.runId }
+        : { kind: "stale-release-failed", paneId: pane.paneId, runId: pane.runId, reason: released.reason },
+    );
+  }
+  return { ok: true, outcomes };
+}
+
+function firstLine(text: string): string {
+  return text.split("\n", 1)[0]?.trim() ?? "";
+}
+
 /** The kill syscall, injectable so error mapping is testable without one. */
 export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
 
@@ -1528,6 +1914,9 @@ export type FleetStatusReport = StatusSnapshot & {
   failureClasses: string | undefined;
   workerPhases: { issue: number; phase: WorkerPausePhase }[];
   intake: string | undefined;
+  /** The project-scoped durable to-spec grooming lifecycle as status lines
+   *  (#809), or nothing when there is nothing to report. */
+  grooming: string | undefined;
   lastStop: DaemonStop | undefined;
   siblings: { project: string; live: number }[];
 };
@@ -1585,6 +1974,7 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
   let codeGraph: CodeGraphHealth;
   let lastStop: DaemonStop | undefined;
   let siblings: { project: string; live: number }[] = [];
+  let grooming: string | undefined;
   try {
     codeGraph =
       cached ??
@@ -1598,6 +1988,15 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
     siblings = loadConfig()
       .projects.filter((p) => p.name !== project.name)
       .map((p) => ({ project: p.name, live: store.liveRuns(p.name).length }));
+    // The durable to-spec grooming lifecycle (#809): project-scoped verdict
+    // rows from the store, plus the dispatch snapshot's routed/parked counts —
+    // no tracker read, no second cache, no evidence parsing. Rendered only
+    // when it has something to say.
+    grooming = formatGroomingStatus({
+      records: store.groomingVerdicts(project.name),
+      routed: snapshot.dispatch?.routed ?? 0,
+      parked: snapshot.dispatch?.parked ?? 0,
+    });
   } finally {
     store.close();
   }
@@ -1615,6 +2014,7 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
     failureClasses: failureClassBlock(project.name),
     workerPhases,
     intake: intakeStatusLine(project.name),
+    grooming,
     lastStop,
     siblings,
   };
@@ -1635,6 +2035,7 @@ export function renderFleetStatusReport(report: FleetStatusReport): string {
     report.intake,
     report.lastStop,
     report.siblings,
+    report.grooming,
   );
 }
 
@@ -1662,13 +2063,22 @@ function briefStatusLine(project: ProjectConfig): string | undefined {
 }
 
 /**
- * Unrecovered runs grouped by failure class (#132), or nothing when there are
- * none.
+ * Failure classes, in two blocks that mean different things (#132).
  *
  * Reported as classes rather than as row states because a row state is not an
  * issue state: the FAILED column counted four completed issues on this fleet
- * while the genuinely stuck ones were invisible (#109). A class says which of
- * those it is, and `recoveredAt` is what keeps a recovered row out of the list.
+ * while the genuinely stuck ones were invisible (#109).
+ *
+ * That defect then reappeared one level up. A single "unrecovered" list keyed on
+ * `recoveredAt IS NULL` swept in the two recorded-only actions, which never
+ * stamp it — so this fleet's status read `returned-for-revision 44`, growing by
+ * one on every review return, while every class an operator could actually act
+ * on sat at zero and invisible beneath it. A count nobody can act on, printed
+ * where the actionable ones go, is the same lie in a new column.
+ *
+ * So: `awaiting recovery` is what the sweep will pick up, and it is omitted when
+ * empty because empty is the healthy answer. `classified, no action` is the
+ * signal — review returns and holds — named as such, never as a backlog.
  */
 function failureClassBlock(projectName: string): string | undefined {
   const path = dbPath();
@@ -1676,12 +2086,23 @@ function failureClassBlock(projectName: string): string | undefined {
   let store: Store | undefined;
   try {
     store = openStore(path);
-    const counts = store.failureClassCounts(projectName);
-    if (counts.length === 0) return undefined;
-    return [
-      "failure classes (unrecovered)",
-      ...counts.map((c) => `  ${c.cls.padEnd(18)}${c.n}`),
-    ].join("\n");
+    const awaiting = store.failureClassCounts(projectName);
+    const recorded = store.recordedOnlyClassCounts(projectName);
+    if (awaiting.length === 0 && recorded.length === 0) return undefined;
+    const lines: string[] = [];
+    if (awaiting.length > 0) {
+      lines.push(
+        "failure classes (awaiting recovery)",
+        ...awaiting.map((c) => `  ${c.cls.padEnd(24)}${c.n}`),
+      );
+    }
+    if (recorded.length > 0) {
+      lines.push(
+        "failure classes (classified, no action by design)",
+        ...recorded.map((c) => `  ${c.cls.padEnd(24)}${c.n}`),
+      );
+    }
+    return lines.join("\n");
   } catch {
     return undefined;
   } finally {
@@ -1892,7 +2313,7 @@ export function sessionDirForCwd(cwd: string): string {
   return join(home, ".omp", "agent", "sessions", slug.replaceAll("/", "-"));
 }
 
-/** The session tree arm scans: every transcript under it, whatever the cwd slug. */
+/** The omp session tree the claim-only verdict judges session identity against. */
 export function sessionsRoot(): string {
   return join(homedir(), ".omp", "agent", "sessions");
 }
@@ -1901,8 +2322,9 @@ export function sessionsRoot(): string {
  * The orchestrator session file omp-telegram's live claim names for this
  * project — the transcript a resumed/restored pane actually writes, which can
  * live in a different session directory than the tick cwd implies (#600).
- * Never throws: no claim, or an unreadable config, means undefined and arm
- * falls back to the cwd-derived directory.
+ * Never throws: no claim, or an unreadable config, means undefined. Only the
+ * claim-only verdict consumes this — the challenge proof reads conductor
+ * state, not transcripts (#614).
  */
 function claimedOrchestratorSessionFile(named: string | undefined): string | undefined {
   if (named === undefined) return undefined;
@@ -1913,35 +2335,6 @@ function claimedOrchestratorSessionFile(named: string | undefined): string | und
   }
 }
 
-/**
- * The session directories `arm` watches for the reply — the cwd-derived one
- * plus, when the live claim's file lives elsewhere, that file's directory.
- *
- * The cwd-derived directory is where a *fresh* session started from the tick
- * cwd writes; the claimed file's directory is where the *restored* pane keeps
- * writing (herdr pins it to the original transcript). Watching both covers a
- * resumed session, a rotated one, and a session that starts after the send.
- *
- * A claim outside the session tree is a stop, not a slower failure: arm polls
- * only transcripts under {@link sessionsRoot}, so a claimed file elsewhere can
- * never satisfy the challenge, and five minutes of polling cannot discover a
- * file that is not in the search set. The throw names both paths.
- */
-function armSessionScanDirs(cwd: string, claimed: string | undefined): string[] {
-  const cwdDir = sessionDirForCwd(cwd);
-  if (claimed === undefined) return [cwdDir];
-  const root = sessionsRoot();
-  const claimDir = dirname(claimed);
-  if (claimDir !== root && !claimDir.startsWith(join(root, sep))) {
-    throw new Error(
-      `arm: the orchestrator's claimed session file ${claimed} is outside the session tree arm scans (${root}) — ` +
-        `no transcript there can satisfy the challenge. Resume the pane under ${root}, or start it from ${cwd}; ` +
-        `arm would otherwise poll ${cwdDir} until the timer runs out`,
-    );
-  }
-  return claimDir === cwdDir ? [cwdDir] : [cwdDir, claimDir];
-}
-
 function makeChallengeCode(): string {
   const bytes = new Uint8Array(4);
   crypto.getRandomValues(bytes);
@@ -1949,6 +2342,72 @@ function makeChallengeCode(): string {
   return `FLEET-${hex}`;
 }
 
+/**
+ * mm:ss for a window whose whole length is five minutes. Deliberately not
+ * `formatDownDuration`, which rounds to whole minutes because it reports
+ * hours-to-days outages: rounded to the minute, the last 30 seconds of this
+ * window would read "5m of 5m left" while the wait was nearly over, which is
+ * the exact ambiguity the progress lines exist to remove (#861).
+ */
+function armClock(ms: number): string {
+  const total = Math.max(0, Math.round(ms / 1000));
+  const minutes = Math.floor(total / 60);
+  const seconds = total % 60;
+  return minutes === 0 ? `${seconds}s` : `${minutes}m${String(seconds).padStart(2, "0")}s`;
+}
+
+/** How often the wait says it is still waiting. Six lines across the window. */
+const ARM_PROGRESS_INTERVAL_MS = 30_000;
+
+/**
+ * Polls the acknowledgement record for one exact challenge id until the
+ * orchestrator's inbound adapter writes it or the deadline passes (#614).
+ *
+ * The state is a small JSON file re-read every pass, never snapshotted: the
+ * acknowledgement may land at any point in the window, written by the live
+ * orchestrator process. Only a record naming this exact id satisfies the
+ * wait — an acknowledgement cut for a replaced challenge is inert here by
+ * construction, and no transcript anywhere is opened.
+ *
+ * It also *says* it is waiting (#861). Measured 2026-08-21: an arm proof sat
+ * silent for five minutes and was reported as a hung setup — the process was
+ * healthy and the operator had no way to tell. A silent five-minute wait
+ * inside a fence that holds dispatch is indistinguishable from a dead one, so
+ * the elapsed/remaining line lands every {@link ARM_PROGRESS_INTERVAL_MS}
+ * regardless of the (much shorter) poll cadence, and the terminal outcome is
+ * always printed.
+ */
+async function waitForArmAcknowledgement(
+  challengeId: string,
+  timeoutMs: number,
+  deps: ArmDeps,
+): Promise<boolean> {
+  const now = deps.now ?? Date.now;
+  const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
+  const report = deps.progress;
+  const startedAt = now();
+  const deadline = startedAt + timeoutMs;
+  let nextReportAt = startedAt + ARM_PROGRESS_INTERVAL_MS;
+  for (;;) {
+    if (readArmAcknowledgement(challengeId) !== undefined) {
+      report?.(`arm: reply acknowledged after ${armClock(now() - startedAt)} — the fleet is armed.`);
+      return true;
+    }
+    const at = now();
+    if (at >= deadline) return false;
+    if (report !== undefined && at >= nextReportAt) {
+      report(
+        `arm: still waiting for the reply — ${armClock(at - startedAt)} elapsed, ` +
+          `${armClock(deadline - at)} left. Nothing is stuck: reply in the Telegram chat with the code.`,
+      );
+      // Anchored to the clock, not to this pass, so a slow pass cannot make the
+      // cadence drift into silence.
+      while (nextReportAt <= at) nextReportAt += ARM_PROGRESS_INTERVAL_MS;
+    }
+    await sleep(5_000);
+  }
+}
+
 /**
  * The one armed-marker write both proofs share: same content, same mode, and
  * the same restamp of the pre-per-project shared marker the heartbeat still
@@ -1962,12 +2421,12 @@ function writeArmedMarker(path: string, owner: string, arm: ArmState): void {
 }
 
 /**
- * The verdict's view of the session surface: the same dirs a challenge would
- * watch, minus the transcript throw. An outside-tree claim is not a reason to
- * stop here — the claim-only verdict names that failure itself
- * (`claim-session-outside` / `dm-owner-unrelated`) with the same session
- * identity rules, so the refusal carries the fact, not an early transcript
- * wording.
+ * The session surface the claim-only verdict judges: the tick-cwd-derived
+ * session directory plus, when the live claim's file lives elsewhere in the
+ * tree, that file's directory (#600). An outside-tree claim is not a reason
+ * to stop here — the verdict names that failure itself
+ * (`claim-session-outside` / `dm-owner-unrelated`), so the refusal carries
+ * the fact.
  */
 function armVerdictScanDirs(cwd: string, claimed: string | undefined): string[] {
   const cwdDir = sessionDirForCwd(cwd);
@@ -2021,108 +2480,6 @@ async function sendTelegramMessage(
   await sendTelegram(token, owner, text, { topicId });
 }
 
-interface SessionScan {
-  seen: boolean;
-  /** `name (mtime <iso>)` for every transcript the last pass actually parsed. */
-  scanned: string[];
-  /** Same shape, for the ones skipped as written before the challenge. */
-  ignored: string[];
-}
-
-/**
- * Polls the session directories — re-read on every pass, never snapshotted —
- * until the challenge shows up as a user turn or the deadline passes.
- *
- * Discovery lives here because the reply can land in a transcript that does not
- * exist yet when the challenge is sent: a rotated session, or the first one of
- * a pane started right after arming (#142). And the scan set can be two
- * directories when the live claim's session file lives outside the tick-cwd
- * directory: the restored pane keeps writing the claimed file, so its
- * directory is watched alongside the cwd-derived one (#600).
- *
- * Files untouched since just before the send are named, not parsed: an append
- * bumps mtime, so a transcript older than the challenge cannot hold the reply,
- * and skipping it keeps a large stale session out of every 5 s pass. A missing
- * directory is not an error — it can vanish under a rotation and reappear.
- */
-async function waitForChallengeInSessions(
-  dirs: readonly string[],
-  code: string,
-  sentAt: number,
-  timeoutMs: number,
-  deps: ArmDeps,
-): Promise<SessionScan> {
-  const now = deps.now ?? Date.now;
-  const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
-  const deadline = now() + timeoutMs;
-  for (;;) {
-    const scanned: string[] = [];
-    const ignored: string[] = [];
-    for (const dir of dirs) {
-      let names: string[] = [];
-      try {
-        names = readdirSync(dir);
-      } catch {
-        /* the directory can go away under a rotation; the next pass re-reads it */
-      }
-      for (const name of names.sort()) {
-        if (!name.endsWith(".jsonl")) continue;
-        const path = join(dir, name);
-        let mtimeMs: number;
-        try {
-          mtimeMs = statSync(path).mtimeMs;
-        } catch {
-          continue; /* race */
-        }
-        const label = `${name} (mtime ${new Date(mtimeMs).toISOString()})`;
-        if (mtimeMs < sentAt - 1_000) {
-          ignored.push(label);
-          continue;
-        }
-        scanned.push(label);
-        if (await transcriptHasUserCode(path, code)) return { seen: true, scanned, ignored };
-      }
-    }
-    if (now() >= deadline) return { seen: false, scanned, ignored };
-    await sleep(5_000);
-  }
-}
-
-export async function transcriptHasUserCode(path: string, code: string): Promise<boolean> {
-  if (!existsSync(path)) return false;
-  const rl = createInterface({ input: createReadStream(path, { encoding: "utf8" }), crlfDelay: Infinity });
-  try {
-    for await (const line of rl) {
-      if (line.length === 0) continue;
-      let row: unknown;
-      try {
-        row = JSON.parse(line);
-      } catch {
-        continue;
-      }
-      if (row === null || typeof row !== "object") continue;
-      const rec = row as { readonly [key: string]: unknown };
-      if (rec["type"] !== "message") continue;
-      const message = rec["message"];
-      if (message === null || typeof message !== "object") continue;
-      const msg = message as { readonly [key: string]: unknown };
-      if (msg["role"] !== "user") continue;
-      const content = msg["content"];
-      if (!Array.isArray(content)) continue;
-      for (const part of content) {
-        if (part === null || typeof part !== "object") continue;
-        const p = part as { readonly [key: string]: unknown };
-        if (p["type"] === "text" && typeof p["text"] === "string" && p["text"].includes(code)) {
-          return true;
-        }
-      }
-    }
-  } finally {
-    rl.close();
-  }
-  return false;
-}
-
 // ---------------------------------------------------------------------------
 // probes
 // ---------------------------------------------------------------------------