omp-conductor 0.19.1 → 0.19.3

package/src/failure-class.ts

@@ -170,6 +170,16 @@ const START_FAILURE_SIGNATURES = [
   // symptom this replaced (the child dying on the peer import) already read as
   // a start failure, so charging one here would be a regression dressed as a fix.
   "worker identity unavailable",
+  // The 2026-08-2x host faults (#986), each recorded verbatim in `lastError`
+  // by the launcher or the worker identity gate — all before the session took
+  // a turn:
+  //  - the child died before it could connect to the dispatcher;
+  //  - node could not resolve a package out of the worker harness tree;
+  //  - the worker's settings file was unreadable (the EACCES outage).
+  "exited 1 before connecting",
+  "cannot find package",
+  "enoent while resolving package",
+  "failed to read settings config",
 ] as const;
 
 /**
@@ -395,6 +405,128 @@ export function adminRestartAttribution(lastError: string | undefined): string |
   return firstLine.trim();
 }
 
+/** The launcher's own statement that the session child died mid-run, before
+ *  the session ended. Written by `omp.ts` when the child exits while the
+ *  dispatcher still owns it; the exit code is the child's own, so a signal
+ *  death reads >=128 (143 = SIGTERM, 137 = SIGKILL) — an external kill,
+ *  never anything the worker's work decided (#986). */
+const CHILD_EXIT_MARKER = /session child exited (\d+) before the session ended/;
+
+/**
+ * Evidence that this run's session child was terminated by a signal, or
+ * `undefined` when it was not.
+ *
+ * Read from the surfaces a run persists — `lastError` first, then `report`,
+ * because the prompt-throw path records the launcher's message as the report
+ * and leaves `lastError` empty. Deliberately narrowed to signal-range codes:
+ * a clean nonzero exit is some process's verdict nobody has named yet, and
+ * naming it `admin-kill` would waive an attempt that may have been spent.
+ */
+export function childSignalDeath(
+  run: Pick<RunRecord, "lastError" | "report">,
+): string | undefined {
+  const text = run.lastError ?? run.report;
+  if (text === undefined) return undefined;
+  const match = CHILD_EXIT_MARKER.exec(text);
+  if (match === null) return undefined;
+  const code = Number(match[1]);
+  if (!Number.isFinite(code) || code < 128) return undefined;
+  return match[0];
+}
+
+/** The reason the settle sweep writes beside a reviewer-closed PR
+ *  (`settlementFor`), matched as recorded evidence so rows written before
+ *  that sweep stamped a class read the same way (#986). */
+const REVIEWER_CLOSED_MARKER = "closed without merging";
+
+/**
+ * Evidence that a reviewer closed this run's pushed work without merging it,
+ * or `undefined` when nothing says so. Two independent signals, either
+ * sufficient: the tracker fact (`facts.pr === "closed"`), or the settle
+ * sweep's own recorded reason in `lastError`. A review decision is not a
+ * worker failure, which is exactly what `returned-for-revision` means; the
+ * only new reading here is applying it to rows whose PR closed while the row
+ * itself had already gone terminal-failed.
+ */
+export function reviewerClosed(
+  run: Pick<RunRecord, "lastError" | "prUrl">,
+  facts: ClassifyFacts,
+): string | undefined {
+  if (run.lastError !== undefined && run.lastError.includes(REVIEWER_CLOSED_MARKER)) {
+    return run.lastError.split("\n")[0]?.trim();
+  }
+  if (facts.pr === "closed" && run.prUrl !== undefined) {
+    return `${run.prUrl} closed without merging`;
+  }
+  return undefined;
+}
+
+/** The harness's own name for giving up on a provider that kept answering with
+ *  empty assistant turns: it retries, then ends the session on the retry cap
+ *  and says so in its maintenance routing (`route: "empty-stop-retry-cap"`,
+ *  beside `Assistant returned empty stop after retry cap; try switching
+ *  models`). A closed marker written by the harness, not vendor prose. */
+const EMPTY_STOP_MARKER = "empty-stop-retry-cap";
+
+/**
+ * Evidence that the provider answered with empty turns until the harness's own
+ * retry cap ended the session, or `undefined` when nothing recorded says so
+ * (#986).
+ *
+ * Read from the terminal evidence the settlement sweep recorded, because this
+ * marker is written by the harness's session log rather than by the transcript
+ * or the row — the run itself has no error, no verdict and no idea it died,
+ * which is precisely why two of these read as `unknown` on 2026-08-23 at
+ * 100/180 and 54/180 turns while a third run on the same model finished green
+ * in the same window.
+ *
+ * Provider flake, not worker logic: the recovery continues from whatever the
+ * attempt pushed rather than charging an implementation attempt for it.
+ */
+export function modelEmptyStop(
+  run: Pick<RunRecord, "terminalEvidence">,
+): string | undefined {
+  if (run.terminalEvidence === undefined) return undefined;
+  if (!run.terminalEvidence.includes(EMPTY_STOP_MARKER)) return undefined;
+  return run.terminalEvidence;
+}
+
+/**
+ * Evidence that this run ended without ever delivering a settlement verdict,
+ * or `undefined` when something on the row says otherwise (#986).
+ *
+ * The shape: state `failed`, turns taken, no error anywhere, and a final
+ * report whose words are mid-delivery narration — "Now I'll execute …", "Type
+ * check passes. Now the test suite:" — with no `state:`/`status:` line the
+ * result parser could read. The session reached its worker and died between
+ * turns without handing over; measured over two weeks of the reference fleet,
+ * this one shape was half of all `unknown`.
+ *
+ * Deliberately narrow at every edge: turn zero belongs to
+ * {@link neverStarted}, any error text keeps the row `unknown` (an
+ * unrecognised error says something happened), and an absent report leaves
+ * genuinely nothing to name. The class charges its attempt exactly as
+ * `unknown` did — only the name becomes specific.
+ */
+export function noVerdictExit(run: RunRecord): string | undefined {
+  if (run.state !== "failed") return undefined;
+  if (run.turns <= 0) return undefined;
+  // Blank counts as absent: a row whose `lastError` is an empty string carries
+  // no error information, and reading it as "something happened" is what kept
+  // four real rows unnamed. Any non-blank text, recognised or not, still wins —
+  // an unrecognised error says something happened.
+  if (run.lastError !== undefined && run.lastError.trim() !== "") return undefined;
+  if (run.report === undefined || run.report.trim() === "") return undefined;
+  // Mirror the two verdict spellings the result path reads (`worker.ts`
+  // STATE_LINE_PATTERN for prose settlements, `renderSettlement`'s leading
+  // `status:` line for structured yields). Either one present means a verdict
+  // was delivered and this function must stay silent.
+  if (/^state:\s*\S+\s*$/im.test(run.report)) return undefined;
+  if (/^status:\s*\S+\s*$/im.test(run.report)) return undefined;
+  const lastWords = run.report.split("\n")[0]?.trim() ?? "";
+  return `the session ended without delivering a settlement verdict; last words: "${lastWords.slice(0, 80)}"`;
+}
+
 export function classifyRun(
   run: RunRecord,
   facts: ClassifyFacts,
@@ -424,6 +556,23 @@ export function classifyRun(
     };
   }
 
+  // The launcher's own statement that the session child died on a signal
+  // mid-run: an external kill — a drain, a restart, the host — not anything
+  // the worker's work decided (#986). Ahead of the provider branches on
+  // purpose: a transcript can carry an older, long-recovered stream fault,
+  // and the launcher's terminal statement outranks whatever the session said
+  // earlier. Requeued free like every other administrative kill.
+  if (run.state === "failed") {
+    const marker = childSignalDeath(run);
+    if (marker !== undefined) {
+      return {
+        cls: "admin-kill",
+        recovery: "requeue",
+        evidence: `killed externally (${marker}) — the session child was terminated, not finished`,
+      };
+    }
+  }
+
   // A billing state, not an implementation failure. Its own class because it is
   // the one outage an operator fixes with a card rather than a diagnosis, and
   // because burying a self-describing provider error in `unknown` erodes what
@@ -507,6 +656,19 @@ export function classifyRun(
     }
   }
 
+  // A reviewer closed this run's pushed work without merging it: a review
+  // decision, not a worker failure (#986). Two rows reach here — the settle
+  // sweep's own recorded reason in `lastError` on a row that predates the
+  // class, and a live `closed` tracker fact on a row whose worker died after
+  // pushing but before claiming. Recovery `none` mirrors the settle sweep
+  // exactly: the queue label is the remedy, and no action is performed here.
+  if (run.state === "failed") {
+    const closed = reviewerClosed(run, facts);
+    if (closed !== undefined) {
+      return { cls: "returned-for-revision", recovery: "none", evidence: closed };
+    }
+  }
+
   if (run.state === "blocked") {
     return {
       cls: "question",
@@ -639,6 +801,36 @@ export function classifyRun(
     }
   }
 
+  // The provider answered with empty turns until the harness gave up (#986).
+  // Below every branch that reads a real error on purpose: a row carrying its
+  // own provider fault is that fault, and this marker only decides rows that
+  // would otherwise be `unknown`. Continues rather than requeues, because the
+  // attempt's pushed work is the honest place to resume from, and a provider
+  // flake must not charge an implementation attempt (#1001). `killed` rows
+  // never reach here — a cap kill is the operative fact about a run that hit
+  // its ceiling, whatever the provider did earlier in the session.
+  if (run.state === "failed") {
+    const emptyStop = modelEmptyStop(run);
+    if (emptyStop !== undefined) {
+      return {
+        cls: "model-empty-stop",
+        recovery: "continue",
+        evidence: `the provider returned empty turns until the harness's retry cap ended the session (${emptyStop}) — turns ${run.turns}/${run.maxTurns}`,
+      };
+    }
+  }
+
+  // The session reached its worker and died between turns without handing over
+  // a verdict (#986): the largest single shape inside `unknown`. Named rather
+  // than requeued blind — an operator reading `status` learns the run stopped
+  // mid-delivery, which is a different question from "nothing is known".
+  if (run.state === "failed") {
+    const narration = noVerdictExit(run);
+    if (narration !== undefined) {
+      return { cls: "no-verdict", recovery: "escalate", evidence: narration };
+    }
+  }
+
   // Deliberately escalate rather than retry. An unrecognised shape is a gap in
   // this table, and a silent requeue would spend a budget on a cause nobody has
   // named — the exact behaviour #132 exists to end.

package/src/fleet.ts

@@ -24,7 +24,12 @@ import {
 import { homedir } from "node:os";
 import { dirname, join, sep } from "node:path";
 import { findProject, loadConfig, resolveArmProof, stateDir } from "./config.ts";
-import { clearArmTransaction, readArmAcknowledgement, recordArmChallenge } from "./arm-challenge.ts";
+import {
+  clearArmTransaction,
+  FLEET_ARM_KEY,
+  readArmAcknowledgement,
+  recordArmChallenge,
+} from "./arm-challenge.ts";
 import {
   claimedTelegramTopics,
   lockPidAlive,
@@ -398,14 +403,18 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
     return { path, alreadyArmed, owner: channel.owner, proof };
   }
 
+  const send = deps.sendChallenge ?? sendTelegramMessage;
+  const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
   const code = makeChallengeCode();
+  // Self-describing (#991): with two live challenges in one chat the operator
+  // was working out which was which from message order, and nothing said how
+  // long a code stayed good — so the safe move was to scroll for the newest,
+  // which is the work conductor-owned verification was meant to end.
   const text =
     `Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
     `Reply to this chat with exactly:\n${code}\n` +
+    `Valid for ${armClock(timeoutMs)}. ` +
     `Nothing will be dispatched until that reply is seen in the orchestrator session.`;
-
-  const send = deps.sendChallenge ?? sendTelegramMessage;
-  const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
   const sentAt = (deps.now ?? Date.now)();
   // The orchestrator's inbound adapter can only acknowledge an *active*
   // challenge, so the authenticated pending record (hash + expiry, never the
@@ -462,6 +471,149 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
   return { path, alreadyArmed, owner: channel.owner, challenge: code, proof };
 }
 
+export interface FleetArmResult {
+  /** Every project this ceremony armed, in configuration order. */
+  armed: { project: string; path: string; alreadyArmed: boolean }[];
+  owner: string;
+  /** The transaction the single reply proved. */
+  challengeId: string;
+}
+
+/**
+ * One ceremony for the whole fleet (#991).
+ *
+ * Arming was per project: a two-project fleet meant two sequential handshakes
+ * with two codes in one chat, though nothing about the fleet's state differed
+ * between them. This sends **one** challenge and arms every configured project
+ * from the single matching reply — which is a fleet-wide pending record, not a
+ * loop that sends N challenges and waits for N replies. That loop is the
+ * current friction with one command wrapped around it.
+ *
+ * Every project is validated before anything is sent, so a fleet whose second
+ * project has no `armedFile` refuses the ceremony instead of arming the first
+ * and then failing — a half-armed fleet is worse than an unarmed one, because
+ * only one of its halves dispatches.
+ *
+ * A project whose declared proof is `claim-only` is armed by this too: an
+ * authenticated round-trip is strictly stronger than the plumbing verdict that
+ * policy would have accepted, so satisfying the weaker gate with the stronger
+ * proof cannot weaken it.
+ */
+export async function armFleet(
+  projectNames: readonly string[],
+  deps: ArmDeps = {},
+): Promise<FleetArmResult> {
+  if (projectNames.length === 0) throw new Error("arm: no projects are configured");
+
+  // Resolve and validate every project first. Nothing is sent and no marker is
+  // written until the whole fleet is known to be armable.
+  const targets: { project: string; armedFile: string; accessFile: string; cwd: string; stateKey?: string }[] = [];
+  for (const name of projectNames) {
+    const tick = resolveTickConfig(name);
+    if (tick.kind === "invalid") throw new Error(`tick config invalid at ${tick.path}: ${tick.problem}`);
+    if (tick.kind === "absent") {
+      throw new Error(
+        `no ${TICK_CONFIG_FILE} under ${tickConfigSearchRoots(name).join(" or ")} for ${name} — ` +
+          `nothing would read an arm marker; drop a tick config first`,
+      );
+    }
+    if (tick.config.armedFile === undefined) {
+      throw new Error(`${tick.path} has no armedFile — ${name}'s heartbeat is ungated; add armedFile before arming`);
+    }
+    if (tick.config.accessFile === undefined) {
+      throw new Error(`${tick.path} has no accessFile — arm cannot prove an inbound channel for ${name}`);
+    }
+    targets.push({
+      project: name,
+      armedFile: tick.config.armedFile,
+      accessFile: tick.config.accessFile,
+      cwd: tick.cwd,
+      ...(tick.config.project === undefined ? {} : { stateKey: tick.config.project }),
+    });
+  }
+
+  // One bot, one chat: the channel is a host fact, so the first target's
+  // access file speaks for the fleet. A project pointing at a different file
+  // would be a different bot, which this ceremony cannot span — and would
+  // announce itself here as a down channel rather than silently arming.
+  const first = targets[0]!;
+  const channel = readPairedChannel(first.accessFile);
+  if (channel.kind === "down") {
+    throw new Error(
+      `escalation channel is not up (${first.accessFile}): ${channel.reason} — ` +
+        `pair the bot (/telegram pair) and enable the bridge (/telegram on) before arming`,
+    );
+  }
+  const token = readBotToken();
+  if (token === undefined) {
+    throw new Error(
+      `no TELEGRAM_BOT_TOKEN in ${join(telegramStateDir(), ".env")} — the arm challenge cannot send without it`,
+    );
+  }
+
+  // The first project that resolves a live topic carries the ceremony, and the
+  // message says so: a fleet-wide question still has to land somewhere an
+  // operator is reading, and every session's adapter can acknowledge it because
+  // the pending record is fleet-wide rather than topic-scoped.
+  let sendTopic: number | undefined;
+  for (const target of targets) {
+    try {
+      sendTopic = resolveProjectTopicId(findProject(loadConfig(), target.project));
+    } catch {
+      continue; /* no project config — try the next */
+    }
+    if (sendTopic !== undefined) break;
+  }
+
+  const send = deps.sendChallenge ?? sendTelegramMessage;
+  const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
+  const code = makeChallengeCode();
+  const names = targets.map((t) => t.project).join(", ");
+  const text =
+    `Fleet arming check — ${String(targets.length)} project(s): ${names}. ` +
+    `Reply to this chat with exactly:\n${code}\n` +
+    `Valid for ${armClock(timeoutMs)}, and one reply arms all of them. ` +
+    `Nothing will be dispatched until that reply is seen in the orchestrator session.`;
+
+  const sentAt = (deps.now ?? Date.now)();
+  const challengeId = recordArmChallenge(FLEET_ARM_KEY, code, sentAt, sentAt + timeoutMs);
+  try {
+    await send(token, channel.owner, text, sendTopic);
+  } catch (err) {
+    clearArmTransaction(FLEET_ARM_KEY, challengeId);
+    throw new Error(
+      `arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+  deps.progress?.(
+    `arm: one challenge sent to ${channel.owner}${sendTopic === undefined ? "" : ` (topic ${sendTopic})`} for ${names} — ` +
+      `waiting up to ${armClock(timeoutMs)} for the reply. Dispatch stays held until it arrives.`,
+  );
+
+  if (!(await waitForArmAcknowledgement(challengeId, timeoutMs, deps))) {
+    clearArmTransaction(FLEET_ARM_KEY, challengeId);
+    throw new Error(
+      `arm: the fleet challenge was never acknowledged in time — NOTHING armed.\n` +
+        `No acknowledgement for challenge ${challengeId} arrived, so no project's marker was written.\n` +
+        `Inbound Telegram is not reaching the omp session. Check, in order:\n` +
+        `  * is the bridge polling?      attach and run: /telegram status\n` +
+        `  * is another process holding this bot token? Telegram allows exactly one\n` +
+        `    getUpdates consumer and rejects the second with HTTP 409.\n` +
+        `  * did you reply in the chat the challenge names, not another one?\n`,
+    );
+  }
+
+  // Proven once, applied to every project. Markers are written after the proof,
+  // so a refused ceremony leaves the fleet exactly as it was.
+  const armed = targets.map((target) => {
+    const state = resolveArmState(target.armedFile, target.project);
+    writeArmedMarker(target.armedFile, channel.owner, state);
+    return { project: target.project, path: target.armedFile, alreadyArmed: state.armed };
+  });
+  clearArmTransaction(FLEET_ARM_KEY, challengeId);
+  return { armed, owner: channel.owner, challengeId };
+}
+
 export interface HoldResult {
   wasPaused: boolean;
   /** Absent when the caller kept the heartbeat armed. */
@@ -746,6 +898,18 @@ export interface WorkerPaneDeps {
  */
 export const WORKER_PANE_SOURCE = "omp-conductor";
 
+/**
+ * How many times this daemon re-establishes one run's pane before it stops
+ * trying (#998).
+ *
+ * The pathology being bounded is unboundedness, not frequency: a pane that
+ * cannot be created is not going to start working on the next pass, and the
+ * retries actively make it worse — `pane_split_failed: ghostty error -2` is the
+ * terminal the loop eventually produces for itself. A run with no pane keeps
+ * working; only its representation is missing.
+ */
+export const PANE_REATTEMPT_MAX = 3;
+
 export function workerPaneLabel(identity: WorkerPaneIdentity): string {
   return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
 }
@@ -764,16 +928,46 @@ function defaultViewer(identity: WorkerPaneIdentity): readonly string[] {
   return ["omp-conductor", "tail", String(identity.issue), "--project", identity.project];
 }
 
+/**
+ * The pane id inside `herdr pane split`'s answer.
+ *
+ * herdr answers every CLI call with a one-line JSON envelope
+ * (`{"id":"cli:pane:split","result":{"pane":{"pane_id":"w2:pT",…}}}`), so raw
+ * stdout is never a pane id. Passing it through verbatim made every worker
+ * launch fail `rename` with `pane_not_found` while leaking the pane it had just
+ * created (#992) — and the test fake that answered a bare `%7` is why the suite
+ * agreed. Parse the field; a shape this does not recognise is no pane id at
+ * all, never a best guess scraped out of the payload.
+ */
+function parsePaneId(stdout: string): string | undefined {
+  const line = firstLine(stdout);
+  if (line === "") return undefined;
+  let payload: unknown;
+  try {
+    payload = JSON.parse(line);
+  } catch {
+    return undefined;
+  }
+  if (payload === null || typeof payload !== "object") return undefined;
+  const result = (payload as Record<string, unknown>)["result"];
+  if (result === null || typeof result !== "object") return undefined;
+  const pane = (result as Record<string, unknown>)["pane"];
+  if (pane === null || typeof pane !== "object") return undefined;
+  const paneId = (pane as Record<string, unknown>)["pane_id"];
+  return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
+}
+
 /**
  * Create the pane, run the follower in it, and report the child's identity and
  * initial state — or say exactly why it could not.
  *
  * Every step is checked, and the first failure returns `unavailable` with the
- * reason: a partially established representation (a pane with no agent identity,
- * say) is worse than none, because it is a pane the operator would read as a
- * tracked worker. What this deliberately does NOT do is decide what a failure
- * means for the launch — failing closed versus running degraded is #841's
- * policy, and inventing it here would pre-empt it.
+ * reason — and closes the pane the split had already created, because a
+ * partially established representation is worse than none: it is a pane the
+ * operator would read as a tracked worker, and before #992 it was left behind
+ * on every single launch. What this deliberately does NOT do is decide what a
+ * failure means for the launch — failing closed versus running degraded is
+ * #841's policy, and inventing it here would pre-empt it.
  */
 export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
   const run = deps.run ?? realHerdrRun;
@@ -785,14 +979,19 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
   if (!split.ok) {
     return { kind: "unavailable", reason: `herdr pane split failed: ${firstLine(split.stderr) || "no output"}` };
   }
-  const paneId = firstLine(split.stdout);
-  if (paneId === "") {
+  const paneId = parsePaneId(split.stdout);
+  if (paneId === undefined) {
     return { kind: "unavailable", reason: "herdr pane split reported no pane id" };
   }
+  // The split has created a pane. From here every exit must take it with it.
+  const abandon = (reason: string): WorkerPaneOutcome => {
+    run([...base, "close", paneId]);
+    return { kind: "unavailable", reason };
+  };
 
   const named = run([...base, "rename", paneId, label]);
   if (!named.ok) {
-    return { kind: "unavailable", reason: `herdr pane rename failed: ${firstLine(named.stderr) || "no output"}` };
+    return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}`);
   }
 
   // Identity before display: the pane must be attributable to this exact run
@@ -813,15 +1012,12 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
     WORKER_PANE_SOURCE,
   ]);
   if (!identified.ok) {
-    return {
-      kind: "unavailable",
-      reason: `herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
-    };
+    return abandon(`herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`);
   }
 
   const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
   if (!started.ok) {
-    return { kind: "unavailable", reason: `herdr pane run failed: ${firstLine(started.stderr) || "no output"}` };
+    return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}`);
   }
 
   // A worker that has just spawned is working by definition. The ongoing
@@ -829,7 +1025,7 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
   // events — never from the pane's output.
   const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
   if (!reported.ok) {
-    return { kind: "unavailable", reason: reported.reason };
+    return abandon(reported.reason);
   }
   return { kind: "tracked", paneId, label, pid: identity.pid };
 }
@@ -982,8 +1178,15 @@ export type WorkerPaneReconciliation =
   | { kind: "intact"; runId: string; paneId: string }
   /** Herdr lost the pane (a restart); a new one now represents the same child. */
   | { kind: "reassociated"; runId: string; paneId: string; label: string }
-  /** No representation, and the reason. The run keeps working regardless. */
-  | { kind: "untracked"; runId: string; reason: string }
+  /** No representation, and the reason. The run keeps working regardless.
+   *  `attempted` marks the ones that spent a re-establishment attempt (#998),
+   *  so the caller can bound them; the no-pid case costs nothing. */
+  | { kind: "untracked"; runId: string; reason: string; attempted?: true }
+  /** The attempt budget for this run is spent, so nothing was tried this pass
+   *  (#998). Before this existed, a run whose pane could not be created was
+   *  retried on every reconciliation pass forever — and each attempt leaked a
+   *  pane until #992, ending in `ghostty error -2` once enough had piled up. */
+  | { kind: "attempts-exhausted"; runId: string; attempts: number }
   /** A conductor pane whose run is not live: handed back to Herdr. */
   | { kind: "stale-released"; paneId: string; runId: string }
   | { kind: "stale-release-failed"; paneId: string; runId: string; reason: string };
@@ -1011,6 +1214,10 @@ export type WorkerPaneReconciliation =
 export function reconcileWorkerPanes(
   live: readonly LiveWorkerPane[],
   deps: WorkerPaneDeps = {},
+  /** Attempts already spent per run id, owned by the caller so this stays pure
+   *  and a daemon restart legitimately gets a fresh budget (#998). */
+  attempts: ReadonlyMap<string, number> = new Map(),
+  maxAttempts = PANE_REATTEMPT_MAX,
 ): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
   const listed = listWorkerPanes(deps);
   if (!listed.ok) return { ok: false, reason: listed.reason };
@@ -1036,6 +1243,13 @@ export function reconcileWorkerPanes(
       });
       continue;
     }
+    // The budget, checked before the attempt: an exhausted run is skipped
+    // silently here and reported once by the caller (#998).
+    const spent = attempts.get(worker.runId) ?? 0;
+    if (spent >= maxAttempts) {
+      outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
+      continue;
+    }
     const opened = openWorkerPane(
       {
         project: worker.project,
@@ -1050,7 +1264,7 @@ export function reconcileWorkerPanes(
     outcomes.push(
       opened.kind === "tracked"
         ? { kind: "reassociated", runId: worker.runId, paneId: opened.paneId, label: opened.label }
-        : { kind: "untracked", runId: worker.runId, reason: opened.reason },
+        : { kind: "untracked", runId: worker.runId, reason: opened.reason, attempted: true },
     );
   }
 
@@ -1717,12 +1931,20 @@ export function codeGraphFromHealthz(body: string | undefined, project: string):
 }
 
 
+/** One live pause as `/healthz` reports it: the phase, plus who asked and
+ *  when where the daemon recorded provenance (#997). */
+export interface WorkerPauseView {
+  phase: WorkerPausePhase;
+  source?: string;
+  pausedAtMs?: number;
+}
+
 /** Live worker pause phases from a daemon `/healthz` body for one project. */
 export function workerPhasesFromHealthz(
   body: string | undefined,
   project: string,
-): ReadonlyMap<number, WorkerPausePhase> {
-  const phases = new Map<number, WorkerPausePhase>();
+): ReadonlyMap<number, WorkerPauseView> {
+  const phases = new Map<number, WorkerPauseView>();
   if (body === undefined) return phases;
   try {
     const payload = JSON.parse(body) as unknown;
@@ -1757,7 +1979,15 @@ export function workerPhasesFromHealthz(
         (issue as number) > 0 &&
         (phase === "pausing" || phase === "paused")
       ) {
-        phases.set(issue as number, phase);
+        const source = Reflect.get(worker, "source");
+        const pausedAtMs = Reflect.get(worker, "pausedAtMs");
+        phases.set(issue as number, {
+          phase,
+          ...(typeof source === "string" && source !== "" ? { source } : {}),
+          ...(Number.isSafeInteger(pausedAtMs) && (pausedAtMs as number) > 0
+            ? { pausedAtMs: pausedAtMs as number }
+            : {}),
+        });
       }
     }
   } catch {
@@ -1840,7 +2070,7 @@ export type FleetStatusReport = StatusSnapshot & {
   brief: string | undefined;
   decisions: string | undefined;
   failureClasses: string | undefined;
-  workerPhases: { issue: number; phase: WorkerPausePhase }[];
+  workerPhases: { issue: number; view: WorkerPauseView }[];
   intake: string | undefined;
   /** The project-scoped durable to-spec grooming lifecycle as status lines
    *  (#809), or nothing when there is nothing to report. */
@@ -1892,7 +2122,7 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
   // throwing probe cannot leak its handle.
   const store = openStore(dbPath());
   const workerPhases = [...workerPhasesFromHealthz(healthBody, project.name)].map(
-    ([issue, phase]) => ({ issue, phase }),
+    ([issue, view]) => ({ issue, view }),
   );
   // The newest host-wide stop/restart provenance (#378). Read here — not in
   // `statusSnapshot`, which is synchronous and belongs to the daemon module —
@@ -1959,7 +2189,7 @@ export function renderFleetStatusReport(report: FleetStatusReport): string {
     report.brief,
     report.decisions,
     report.failureClasses,
-    new Map(report.workerPhases.map(({ issue, phase }) => [issue, phase])),
+    new Map(report.workerPhases.map(({ issue, view }) => [issue, view])),
     report.intake,
     report.lastStop,
     report.siblings,
@@ -2008,6 +2238,10 @@ function briefStatusLine(project: ProjectConfig): string | undefined {
  * empty because empty is the healthy answer. `classified, no action` is the
  * signal — review returns and holds — named as such, never as a backlog.
  */
+/** How many surviving `unknown` rows a status render explains (#986). A board
+ *  is read at a glance: enough to show a pattern, never a backlog dump. */
+const UNKNOWN_EVIDENCE_SHOWN = 5;
+
 function failureClassBlock(projectName: string): string | undefined {
   const path = dbPath();
   if (!existsSync(path)) return undefined;
@@ -2030,6 +2264,20 @@ function failureClassBlock(projectName: string): string | undefined {
         ...recorded.map((c) => `  ${c.cls.padEnd(24)}${c.n}`),
       );
     }
+    // A surviving `unknown` is an unnamed cause, and the only way anyone closes
+    // that gap is by seeing what the settle sweep actually knew about the row
+    // (#986). Bounded to the newest few: this is a status board, not a log, and
+    // a fleet with a long unknown history must not turn one render into a scan.
+    const unknown = store.unknownRuns(projectName, UNKNOWN_EVIDENCE_SHOWN);
+    const withEvidence = unknown.filter((run) => run.terminalEvidence !== undefined);
+    if (withEvidence.length > 0) {
+      lines.push(
+        "unknown, with terminal evidence",
+        ...withEvidence.map(
+          (run) => `  #${run.issue} ${(run.terminalEvidence ?? "").slice(0, 90)}`,
+        ),
+      );
+    }
     return lines.join("\n");
   } catch {
     return undefined;