omp-conductor 0.19.2 → 0.19.4

package/src/fleet.ts

@@ -24,7 +24,12 @@ import {
 import { homedir } from "node:os";
 import { dirname, join, sep } from "node:path";
 import { findProject, loadConfig, resolveArmProof, stateDir } from "./config.ts";
-import { clearArmTransaction, readArmAcknowledgement, recordArmChallenge } from "./arm-challenge.ts";
+import {
+  clearArmTransaction,
+  FLEET_ARM_KEY,
+  readArmAcknowledgement,
+  recordArmChallenge,
+} from "./arm-challenge.ts";
 import {
   claimedTelegramTopics,
   lockPidAlive,
@@ -398,14 +403,18 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
     return { path, alreadyArmed, owner: channel.owner, proof };
   }
 
+  const send = deps.sendChallenge ?? sendTelegramMessage;
+  const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
   const code = makeChallengeCode();
+  // Self-describing (#991): with two live challenges in one chat the operator
+  // was working out which was which from message order, and nothing said how
+  // long a code stayed good — so the safe move was to scroll for the newest,
+  // which is the work conductor-owned verification was meant to end.
   const text =
     `Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
     `Reply to this chat with exactly:\n${code}\n` +
+    `Valid for ${armClock(timeoutMs)}. ` +
     `Nothing will be dispatched until that reply is seen in the orchestrator session.`;
-
-  const send = deps.sendChallenge ?? sendTelegramMessage;
-  const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
   const sentAt = (deps.now ?? Date.now)();
   // The orchestrator's inbound adapter can only acknowledge an *active*
   // challenge, so the authenticated pending record (hash + expiry, never the
@@ -462,6 +471,149 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
   return { path, alreadyArmed, owner: channel.owner, challenge: code, proof };
 }
 
+export interface FleetArmResult {
+  /** Every project this ceremony armed, in configuration order. */
+  armed: { project: string; path: string; alreadyArmed: boolean }[];
+  owner: string;
+  /** The transaction the single reply proved. */
+  challengeId: string;
+}
+
+/**
+ * One ceremony for the whole fleet (#991).
+ *
+ * Arming was per project: a two-project fleet meant two sequential handshakes
+ * with two codes in one chat, though nothing about the fleet's state differed
+ * between them. This sends **one** challenge and arms every configured project
+ * from the single matching reply — which is a fleet-wide pending record, not a
+ * loop that sends N challenges and waits for N replies. That loop is the
+ * current friction with one command wrapped around it.
+ *
+ * Every project is validated before anything is sent, so a fleet whose second
+ * project has no `armedFile` refuses the ceremony instead of arming the first
+ * and then failing — a half-armed fleet is worse than an unarmed one, because
+ * only one of its halves dispatches.
+ *
+ * A project whose declared proof is `claim-only` is armed by this too: an
+ * authenticated round-trip is strictly stronger than the plumbing verdict that
+ * policy would have accepted, so satisfying the weaker gate with the stronger
+ * proof cannot weaken it.
+ */
+export async function armFleet(
+  projectNames: readonly string[],
+  deps: ArmDeps = {},
+): Promise<FleetArmResult> {
+  if (projectNames.length === 0) throw new Error("arm: no projects are configured");
+
+  // Resolve and validate every project first. Nothing is sent and no marker is
+  // written until the whole fleet is known to be armable.
+  const targets: { project: string; armedFile: string; accessFile: string; cwd: string; stateKey?: string }[] = [];
+  for (const name of projectNames) {
+    const tick = resolveTickConfig(name);
+    if (tick.kind === "invalid") throw new Error(`tick config invalid at ${tick.path}: ${tick.problem}`);
+    if (tick.kind === "absent") {
+      throw new Error(
+        `no ${TICK_CONFIG_FILE} under ${tickConfigSearchRoots(name).join(" or ")} for ${name} — ` +
+          `nothing would read an arm marker; drop a tick config first`,
+      );
+    }
+    if (tick.config.armedFile === undefined) {
+      throw new Error(`${tick.path} has no armedFile — ${name}'s heartbeat is ungated; add armedFile before arming`);
+    }
+    if (tick.config.accessFile === undefined) {
+      throw new Error(`${tick.path} has no accessFile — arm cannot prove an inbound channel for ${name}`);
+    }
+    targets.push({
+      project: name,
+      armedFile: tick.config.armedFile,
+      accessFile: tick.config.accessFile,
+      cwd: tick.cwd,
+      ...(tick.config.project === undefined ? {} : { stateKey: tick.config.project }),
+    });
+  }
+
+  // One bot, one chat: the channel is a host fact, so the first target's
+  // access file speaks for the fleet. A project pointing at a different file
+  // would be a different bot, which this ceremony cannot span — and would
+  // announce itself here as a down channel rather than silently arming.
+  const first = targets[0]!;
+  const channel = readPairedChannel(first.accessFile);
+  if (channel.kind === "down") {
+    throw new Error(
+      `escalation channel is not up (${first.accessFile}): ${channel.reason} — ` +
+        `pair the bot (/telegram pair) and enable the bridge (/telegram on) before arming`,
+    );
+  }
+  const token = readBotToken();
+  if (token === undefined) {
+    throw new Error(
+      `no TELEGRAM_BOT_TOKEN in ${join(telegramStateDir(), ".env")} — the arm challenge cannot send without it`,
+    );
+  }
+
+  // The first project that resolves a live topic carries the ceremony, and the
+  // message says so: a fleet-wide question still has to land somewhere an
+  // operator is reading, and every session's adapter can acknowledge it because
+  // the pending record is fleet-wide rather than topic-scoped.
+  let sendTopic: number | undefined;
+  for (const target of targets) {
+    try {
+      sendTopic = resolveProjectTopicId(findProject(loadConfig(), target.project));
+    } catch {
+      continue; /* no project config — try the next */
+    }
+    if (sendTopic !== undefined) break;
+  }
+
+  const send = deps.sendChallenge ?? sendTelegramMessage;
+  const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
+  const code = makeChallengeCode();
+  const names = targets.map((t) => t.project).join(", ");
+  const text =
+    `Fleet arming check — ${String(targets.length)} project(s): ${names}. ` +
+    `Reply to this chat with exactly:\n${code}\n` +
+    `Valid for ${armClock(timeoutMs)}, and one reply arms all of them. ` +
+    `Nothing will be dispatched until that reply is seen in the orchestrator session.`;
+
+  const sentAt = (deps.now ?? Date.now)();
+  const challengeId = recordArmChallenge(FLEET_ARM_KEY, code, sentAt, sentAt + timeoutMs);
+  try {
+    await send(token, channel.owner, text, sendTopic);
+  } catch (err) {
+    clearArmTransaction(FLEET_ARM_KEY, challengeId);
+    throw new Error(
+      `arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+  deps.progress?.(
+    `arm: one challenge sent to ${channel.owner}${sendTopic === undefined ? "" : ` (topic ${sendTopic})`} for ${names} — ` +
+      `waiting up to ${armClock(timeoutMs)} for the reply. Dispatch stays held until it arrives.`,
+  );
+
+  if (!(await waitForArmAcknowledgement(challengeId, timeoutMs, deps))) {
+    clearArmTransaction(FLEET_ARM_KEY, challengeId);
+    throw new Error(
+      `arm: the fleet challenge was never acknowledged in time — NOTHING armed.\n` +
+        `No acknowledgement for challenge ${challengeId} arrived, so no project's marker was written.\n` +
+        `Inbound Telegram is not reaching the omp session. Check, in order:\n` +
+        `  * is the bridge polling?      attach and run: /telegram status\n` +
+        `  * is another process holding this bot token? Telegram allows exactly one\n` +
+        `    getUpdates consumer and rejects the second with HTTP 409.\n` +
+        `  * did you reply in the chat the challenge names, not another one?\n`,
+    );
+  }
+
+  // Proven once, applied to every project. Markers are written after the proof,
+  // so a refused ceremony leaves the fleet exactly as it was.
+  const armed = targets.map((target) => {
+    const state = resolveArmState(target.armedFile, target.project);
+    writeArmedMarker(target.armedFile, channel.owner, state);
+    return { project: target.project, path: target.armedFile, alreadyArmed: state.armed };
+  });
+  clearArmTransaction(FLEET_ARM_KEY, challengeId);
+  return { armed, owner: channel.owner, challengeId };
+}
+
 export interface HoldResult {
   wasPaused: boolean;
   /** Absent when the caller kept the heartbeat armed. */
@@ -663,7 +815,7 @@ export function startHerdrFleet(projectName?: string, deps: HerdrStartDeps = {})
 }
 
 export interface HerdrAgent {
-  name: string;
+  name?: string;
   paneId: string;
   agent?: string;
   sessionPath?: string;
@@ -746,6 +898,18 @@ export interface WorkerPaneDeps {
  */
 export const WORKER_PANE_SOURCE = "omp-conductor";
 
+/**
+ * How many times this daemon re-establishes one run's pane before it stops
+ * trying (#998).
+ *
+ * The pathology being bounded is unboundedness, not frequency: a pane that
+ * cannot be created is not going to start working on the next pass, and the
+ * retries actively make it worse — `pane_split_failed: ghostty error -2` is the
+ * terminal the loop eventually produces for itself. A run with no pane keeps
+ * working; only its representation is missing.
+ */
+export const PANE_REATTEMPT_MAX = 3;
+
 export function workerPaneLabel(identity: WorkerPaneIdentity): string {
   return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
 }
@@ -764,16 +928,46 @@ function defaultViewer(identity: WorkerPaneIdentity): readonly string[] {
   return ["omp-conductor", "tail", String(identity.issue), "--project", identity.project];
 }
 
+/**
+ * The pane id inside `herdr pane split`'s answer.
+ *
+ * herdr answers every CLI call with a one-line JSON envelope
+ * (`{"id":"cli:pane:split","result":{"pane":{"pane_id":"w2:pT",…}}}`), so raw
+ * stdout is never a pane id. Passing it through verbatim made every worker
+ * launch fail `rename` with `pane_not_found` while leaking the pane it had just
+ * created (#992) — and the test fake that answered a bare `%7` is why the suite
+ * agreed. Parse the field; a shape this does not recognise is no pane id at
+ * all, never a best guess scraped out of the payload.
+ */
+function parsePaneId(stdout: string): string | undefined {
+  const line = firstLine(stdout);
+  if (line === "") return undefined;
+  let payload: unknown;
+  try {
+    payload = JSON.parse(line);
+  } catch {
+    return undefined;
+  }
+  if (payload === null || typeof payload !== "object") return undefined;
+  const result = (payload as Record<string, unknown>)["result"];
+  if (result === null || typeof result !== "object") return undefined;
+  const pane = (result as Record<string, unknown>)["pane"];
+  if (pane === null || typeof pane !== "object") return undefined;
+  const paneId = (pane as Record<string, unknown>)["pane_id"];
+  return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
+}
+
 /**
  * Create the pane, run the follower in it, and report the child's identity and
  * initial state — or say exactly why it could not.
  *
  * Every step is checked, and the first failure returns `unavailable` with the
- * reason: a partially established representation (a pane with no agent identity,
- * say) is worse than none, because it is a pane the operator would read as a
- * tracked worker. What this deliberately does NOT do is decide what a failure
- * means for the launch — failing closed versus running degraded is #841's
- * policy, and inventing it here would pre-empt it.
+ * reason — and closes the pane the split had already created, because a
+ * partially established representation is worse than none: it is a pane the
+ * operator would read as a tracked worker, and before #992 it was left behind
+ * on every single launch. What this deliberately does NOT do is decide what a
+ * failure means for the launch — failing closed versus running degraded is
+ * #841's policy, and inventing it here would pre-empt it.
  */
 export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
   const run = deps.run ?? realHerdrRun;
@@ -785,14 +979,19 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
   if (!split.ok) {
     return { kind: "unavailable", reason: `herdr pane split failed: ${firstLine(split.stderr) || "no output"}` };
   }
-  const paneId = firstLine(split.stdout);
-  if (paneId === "") {
+  const paneId = parsePaneId(split.stdout);
+  if (paneId === undefined) {
     return { kind: "unavailable", reason: "herdr pane split reported no pane id" };
   }
+  // The split has created a pane. From here every exit must take it with it.
+  const abandon = (reason: string): WorkerPaneOutcome => {
+    run([...base, "close", paneId]);
+    return { kind: "unavailable", reason };
+  };
 
   const named = run([...base, "rename", paneId, label]);
   if (!named.ok) {
-    return { kind: "unavailable", reason: `herdr pane rename failed: ${firstLine(named.stderr) || "no output"}` };
+    return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}`);
   }
 
   // Identity before display: the pane must be attributable to this exact run
@@ -813,15 +1012,12 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
     WORKER_PANE_SOURCE,
   ]);
   if (!identified.ok) {
-    return {
-      kind: "unavailable",
-      reason: `herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
-    };
+    return abandon(`herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`);
   }
 
   const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
   if (!started.ok) {
-    return { kind: "unavailable", reason: `herdr pane run failed: ${firstLine(started.stderr) || "no output"}` };
+    return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}`);
   }
 
   // A worker that has just spawned is working by definition. The ongoing
@@ -829,7 +1025,7 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
   // events — never from the pane's output.
   const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
   if (!reported.ok) {
-    return { kind: "unavailable", reason: reported.reason };
+    return abandon(reported.reason);
   }
   return { kind: "tracked", paneId, label, pid: identity.pid };
 }
@@ -982,8 +1178,15 @@ export type WorkerPaneReconciliation =
   | { kind: "intact"; runId: string; paneId: string }
   /** Herdr lost the pane (a restart); a new one now represents the same child. */
   | { kind: "reassociated"; runId: string; paneId: string; label: string }
-  /** No representation, and the reason. The run keeps working regardless. */
-  | { kind: "untracked"; runId: string; reason: string }
+  /** No representation, and the reason. The run keeps working regardless.
+   *  `attempted` marks the ones that spent a re-establishment attempt (#998),
+   *  so the caller can bound them; the no-pid case costs nothing. */
+  | { kind: "untracked"; runId: string; reason: string; attempted?: true }
+  /** The attempt budget for this run is spent, so nothing was tried this pass
+   *  (#998). Before this existed, a run whose pane could not be created was
+   *  retried on every reconciliation pass forever — and each attempt leaked a
+   *  pane until #992, ending in `ghostty error -2` once enough had piled up. */
+  | { kind: "attempts-exhausted"; runId: string; attempts: number }
   /** A conductor pane whose run is not live: handed back to Herdr. */
   | { kind: "stale-released"; paneId: string; runId: string }
   | { kind: "stale-release-failed"; paneId: string; runId: string; reason: string };
@@ -1011,6 +1214,10 @@ export type WorkerPaneReconciliation =
 export function reconcileWorkerPanes(
   live: readonly LiveWorkerPane[],
   deps: WorkerPaneDeps = {},
+  /** Attempts already spent per run id, owned by the caller so this stays pure
+   *  and a daemon restart legitimately gets a fresh budget (#998). */
+  attempts: ReadonlyMap<string, number> = new Map(),
+  maxAttempts = PANE_REATTEMPT_MAX,
 ): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
   const listed = listWorkerPanes(deps);
   if (!listed.ok) return { ok: false, reason: listed.reason };
@@ -1036,6 +1243,13 @@ export function reconcileWorkerPanes(
       });
       continue;
     }
+    // The budget, checked before the attempt: an exhausted run is skipped
+    // silently here and reported once by the caller (#998).
+    const spent = attempts.get(worker.runId) ?? 0;
+    if (spent >= maxAttempts) {
+      outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
+      continue;
+    }
     const opened = openWorkerPane(
       {
         project: worker.project,
@@ -1050,7 +1264,7 @@ export function reconcileWorkerPanes(
     outcomes.push(
       opened.kind === "tracked"
         ? { kind: "reassociated", runId: worker.runId, paneId: opened.paneId, label: opened.label }
-        : { kind: "untracked", runId: worker.runId, reason: opened.reason },
+        : { kind: "untracked", runId: worker.runId, reason: opened.reason, attempted: true },
     );
   }
 
@@ -1354,16 +1568,22 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
     const agent = row as { readonly [key: string]: unknown };
     const name = agent["name"];
     const paneId = agent["pane_id"];
-    if (typeof name !== "string" || typeof paneId !== "string") {
+    if (typeof paneId !== "string") {
+      throw new Error(
+        "herdr agent list row is missing a string `pane_id` — cannot identify the pane",
+      );
+    }
+    if (name !== undefined && name !== null && typeof name !== "string") {
       throw new Error(
-        "herdr agent list row is missing a string `name`/`pane_id` — " +
+        `herdr agent list row for pane ${paneId} has a non-string \`name\` (${typeof name}) — ` +
           "cannot tell whether it is the conductor pane",
       );
     }
+    const parsedName = typeof name === "string" ? name : undefined;
     const rawAgent = agent["agent"];
     if (rawAgent !== undefined && rawAgent !== null && typeof rawAgent !== "string") {
       throw new Error(
-        `herdr agent list row for ${name} has a non-string \`agent\` (${typeof rawAgent}) — ` +
+        `herdr agent list row for ${parsedName ?? `pane ${paneId}`} has a non-string \`agent\` (${typeof rawAgent}) — ` +
           `cannot tell whether an agent is live`,
       );
     }
@@ -1377,7 +1597,7 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
       }
     }
     out.push({
-      name,
+      ...(parsedName === undefined ? {} : { name: parsedName }),
       paneId,
       ...(liveAgent === undefined ? {} : { agent: liveAgent }),
       ...(sessionPath === undefined ? {} : { sessionPath }),
@@ -1717,12 +1937,20 @@ export function codeGraphFromHealthz(body: string | undefined, project: string):
 }
 
 
+/** One live pause as `/healthz` reports it: the phase, plus who asked and
+ *  when where the daemon recorded provenance (#997). */
+export interface WorkerPauseView {
+  phase: WorkerPausePhase;
+  source?: string;
+  pausedAtMs?: number;
+}
+
 /** Live worker pause phases from a daemon `/healthz` body for one project. */
 export function workerPhasesFromHealthz(
   body: string | undefined,
   project: string,
-): ReadonlyMap<number, WorkerPausePhase> {
-  const phases = new Map<number, WorkerPausePhase>();
+): ReadonlyMap<number, WorkerPauseView> {
+  const phases = new Map<number, WorkerPauseView>();
   if (body === undefined) return phases;
   try {
     const payload = JSON.parse(body) as unknown;
@@ -1757,7 +1985,15 @@ export function workerPhasesFromHealthz(
         (issue as number) > 0 &&
         (phase === "pausing" || phase === "paused")
       ) {
-        phases.set(issue as number, phase);
+        const source = Reflect.get(worker, "source");
+        const pausedAtMs = Reflect.get(worker, "pausedAtMs");
+        phases.set(issue as number, {
+          phase,
+          ...(typeof source === "string" && source !== "" ? { source } : {}),
+          ...(Number.isSafeInteger(pausedAtMs) && (pausedAtMs as number) > 0
+            ? { pausedAtMs: pausedAtMs as number }
+            : {}),
+        });
       }
     }
   } catch {
@@ -1840,7 +2076,7 @@ export type FleetStatusReport = StatusSnapshot & {
   brief: string | undefined;
   decisions: string | undefined;
   failureClasses: string | undefined;
-  workerPhases: { issue: number; phase: WorkerPausePhase }[];
+  workerPhases: { issue: number; view: WorkerPauseView }[];
   intake: string | undefined;
   /** The project-scoped durable to-spec grooming lifecycle as status lines
    *  (#809), or nothing when there is nothing to report. */
@@ -1892,7 +2128,7 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
   // throwing probe cannot leak its handle.
   const store = openStore(dbPath());
   const workerPhases = [...workerPhasesFromHealthz(healthBody, project.name)].map(
-    ([issue, phase]) => ({ issue, phase }),
+    ([issue, view]) => ({ issue, view }),
   );
   // The newest host-wide stop/restart provenance (#378). Read here — not in
   // `statusSnapshot`, which is synchronous and belongs to the daemon module —
@@ -1959,7 +2195,7 @@ export function renderFleetStatusReport(report: FleetStatusReport): string {
     report.brief,
     report.decisions,
     report.failureClasses,
-    new Map(report.workerPhases.map(({ issue, phase }) => [issue, phase])),
+    new Map(report.workerPhases.map(({ issue, view }) => [issue, view])),
     report.intake,
     report.lastStop,
     report.siblings,
@@ -2008,6 +2244,10 @@ function briefStatusLine(project: ProjectConfig): string | undefined {
  * empty because empty is the healthy answer. `classified, no action` is the
  * signal — review returns and holds — named as such, never as a backlog.
  */
+/** How many surviving `unknown` rows a status render explains (#986). A board
+ *  is read at a glance: enough to show a pattern, never a backlog dump. */
+const UNKNOWN_EVIDENCE_SHOWN = 5;
+
 function failureClassBlock(projectName: string): string | undefined {
   const path = dbPath();
   if (!existsSync(path)) return undefined;
@@ -2030,6 +2270,20 @@ function failureClassBlock(projectName: string): string | undefined {
         ...recorded.map((c) => `  ${c.cls.padEnd(24)}${c.n}`),
       );
     }
+    // A surviving `unknown` is an unnamed cause, and the only way anyone closes
+    // that gap is by seeing what the settle sweep actually knew about the row
+    // (#986). Bounded to the newest few: this is a status board, not a log, and
+    // a fleet with a long unknown history must not turn one render into a scan.
+    const unknown = store.unknownRuns(projectName, UNKNOWN_EVIDENCE_SHOWN);
+    const withEvidence = unknown.filter((run) => run.terminalEvidence !== undefined);
+    if (withEvidence.length > 0) {
+      lines.push(
+        "unknown, with terminal evidence",
+        ...withEvidence.map(
+          (run) => `  #${run.issue} ${(run.terminalEvidence ?? "").slice(0, 90)}`,
+        ),
+      );
+    }
     return lines.join("\n");
   } catch {
     return undefined;