omp-conductor 0.2.1 → 0.3.1

package/src/daemon.ts

@@ -7,10 +7,13 @@
  * of it, so concurrency, daily volume, spend and per-issue attempts are counted
  * here and enforced before anything is claimed.
  */
-import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
-import { dirname, join } from "node:path";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, relative } from "node:path";
 import { configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
 import { createEscalator } from "./escalate.ts";
+import { livingDaemon } from "./lifecycle.ts";
+import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
 import { startOrchestrator } from "./orchestrator.ts";
 import type { OrchestratorHandle } from "./orchestrator.ts";
 import { branchName, route } from "./routing.ts";
@@ -27,8 +30,15 @@ import type {
   Store,
   Tracker,
 } from "./types.ts";
-import { renderBrief, runWorker } from "./worker.ts";
-import { addWorktree, mirrorPathFor, removeWorktree, worktreePathFor } from "./worktree.ts";
+import { type KilledBy, renderBrief, runWorker } from "./worker.ts";
+import {
+  addWorktree,
+  mirrorPathFor,
+  removeWorktree,
+  salvageWip,
+  type SalvageOutcome,
+  worktreePathFor,
+} from "./worktree.ts";
 
 /** Long enough that the tracker is not polled raw, short enough that a human
  *  who labels an issue sees it picked up within a coffee break. */
@@ -60,6 +70,8 @@ interface Deps {
   tracker: Tracker;
   store: Store;
   escalate(e: Escalation): Promise<void>;
+  integrity: IntegrityGate;
+  stall: StallGate;
 }
 
 // ---------------------------------------------------------------- paths & pause
@@ -69,6 +81,97 @@ export function dbPath(): string {
   return join(stateDir(), "conductor.db");
 }
 
+// ------------------------------------------------------- orchestrator liveness
+
+/**
+ * Whether the wedged-orchestrator page has already gone out for the stall
+ * currently on disk. One page per episode: the marker persists until a tick is
+ * consumed, so paging per five minutes would be paging forever.
+ */
+export interface StallGate {
+  paged: boolean;
+}
+
+export interface StallVerdict {
+  /** The marker's own line, when one is there. */
+  since?: string;
+  page: boolean;
+}
+
+/**
+ * Reads the orchestrator's stall marker and decides whether this tick pages.
+ *
+ * The marker is written by the tick extension inside the orchestrator session
+ * ({@link STALL_MARKER_FILE}) when two of its own prompts go unconsumed — the
+ * one signal that separates "the process is alive" from "the loop is reading
+ * its queue". Every other guard in this system reads healthy through a wedge:
+ * the herdr recovery plugin tests for a live process and an agent label, both
+ * of which survive it, and `/healthz` describes this daemon, which is a
+ * different process entirely.
+ *
+ * The daemon is the natural watcher precisely because it is that different
+ * process: it already wakes every five minutes, it owns a working escalation
+ * path, and nothing about its health depends on the session that is stuck. A
+ * wedged loop cannot page for itself, and the herdr plugin only runs on session
+ * lifecycle events — a session that stays alive and stops working emits none.
+ *
+ * Resets when the marker disappears, so a second stall days later pages again.
+ */
+export function checkStall(gate: StallGate, marker: string): StallVerdict {
+  if (!existsSync(marker)) {
+    gate.paged = false;
+    return { page: false };
+  }
+  const page = !gate.paged;
+  let since: string | undefined;
+  try {
+    const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
+    if (body !== undefined && body !== "") since = body;
+  } catch {
+    // An unreadable marker still means stalled; the timestamp is a nicety.
+  }
+  return { ...(since === undefined ? {} : { since }), page };
+}
+
+/**
+ * Pages tier 2 once when the orchestrator session stops draining its queue.
+ *
+ * Deliberately does not restart anything. A wedge lands mid-turn, this process
+ * cannot tell a half-applied edit from an idle loop, and killing the session
+ * could destroy work an operator would rather read first — the same refusal to
+ * guess that the recovery plugin is built on.
+ */
+async function watchOrchestrator(d: Deps): Promise<void> {
+  const marker = join(stateDir(), STALL_MARKER_FILE);
+  const verdict = checkStall(d.stall, marker);
+  if (!verdict.page) return;
+
+  log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
+  const delivered = await safeEscalate(d, {
+    tier: 2,
+    project: d.project.name,
+    issue: NO_ISSUE,
+    // Dated like the other tier-2 summaries: the dedup key is the summary, and
+    // a second wedge next month must not read as a repeat of this one.
+    summary:
+      `Orchestrator session wedged on ${new Date().toISOString().slice(0, 10)} — ` +
+      `it has stopped reading its queue (${d.project.name})`,
+    detail: [
+      verdict.since ?? "Marker present with no readable timestamp.",
+      `Marker: ${marker}`,
+      "",
+      "Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
+      "the loop is alive and consuming nothing, so ticks and your messages queue behind it unread.",
+      "",
+      "Attach and look before you act — a wedge lands mid-turn. Then SIGTERM the omp process:",
+      "herdr-conductor resumes it by exact identity, and the first consumed tick clears this marker.",
+      "",
+      "Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
+    ].join("\n"),
+  });
+  markPaged(d.stall, delivered);
+}
+
 /**
  * Pause is a file rather than process state on purpose: `omp-conductor pause`
  * and `/conductor pause` run in a different process from the daemon, and a flag
@@ -89,6 +192,99 @@ export function setPaused(v: boolean): void {
   }
 }
 
+// ----------------------------------------------------------- package integrity
+
+/** Enough differing paths to tell a deploy from a tamper at a glance; the full
+ *  list is on the host, and the answer is always "go look at the host". */
+const INTEGRITY_SAMPLE = 5;
+
+/**
+ * What the daemon booted with, and whether it has already paged about losing
+ * it. Lives exactly as long as one `runDaemon()` call — which is the whole
+ * trick: a restart re-records both.
+ */
+export interface IntegrityGate {
+  baseline: Map<string, string>;
+  paged: boolean;
+}
+
+export interface IntegrityVerdict {
+  /** Labelled, sorted differences; empty when the package is untouched. */
+  diff: string[];
+  /** Any difference at all stops the fleet. */
+  pause: boolean;
+  /** First divergent tick only — a page every five minutes is a page nobody reads. */
+  page: boolean;
+}
+
+/**
+ * sha256 of every source file the running package is made of, keyed by path
+ * relative to `root`.
+ *
+ * `import.meta.dir` is the installed `src/` of the code executing right now, so
+ * this is a self-portrait: what was actually deployed, not what some checkout
+ * on disk happens to contain. `.ts` and `.md` because both are executable in
+ * this package — the briefs under `src/briefs/` are the sessions' instructions,
+ * and rewriting one of those buys more than rewriting the dispatcher does.
+ * (A checkout also carries `*.test.ts`, which the published package excludes, so
+ * a daemon started from one is watching its tests too. That is the honest
+ * answer — its code did change — and it costs nothing on a real install.)
+ *
+ * Walking and hashing the ~30 files of this package measures 0.6 ms warm, once
+ * per five-minute tick, so a tick does it inline. No cache and no mtime
+ * shortcut on purpose: a cache is a second thing that can be wrong, and mtime
+ * is the first field anyone covering their tracks restores.
+ */
+export function packageManifest(root: string = import.meta.dir): Map<string, string> {
+  const out = new Map<string, string>();
+  const walk = (dir: string): void => {
+    for (const e of readdirSync(dir, { withFileTypes: true })) {
+      const full = join(dir, e.name);
+      if (e.isDirectory()) walk(full);
+      else if (e.isFile() && (e.name.endsWith(".ts") || e.name.endsWith(".md")))
+        out.set(relative(root, full), createHash("sha256").update(readFileSync(full)).digest("hex"));
+    }
+  };
+  walk(root);
+  return out;
+}
+
+/**
+ * Labelled rather than three arrays because every consumer — the log line, the
+ * page, the test — wants one readable list of what moved.
+ */
+export function manifestDiff(before: Map<string, string>, after: Map<string, string>): string[] {
+  const out: string[] = [];
+  for (const [path, hash] of before) {
+    const now = after.get(path);
+    if (now === undefined) out.push(`removed ${path}`);
+    else if (now !== hash) out.push(`changed ${path}`);
+  }
+  for (const path of after.keys()) if (!before.has(path)) out.push(`added ${path}`);
+  return out.sort();
+}
+
+/**
+ * The tick's decision, split from its effects so the once-only page is a thing
+ * a test can hold.
+ *
+ * `pause` stays true on every divergent tick, deliberately: an operator who
+ * resumes without restarting gets re-paused, because the boundary is still
+ * broken. `page` asks whether this tick should *try* — the caller latches the
+ * gate with {@link markPaged} only once a page actually went out, so a Telegram
+ * outage during the one tick that noticed does not buy permanent silence.
+ */
+export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>): IntegrityVerdict {
+  const diff = manifestDiff(gate.baseline, current);
+  if (diff.length === 0) return { diff, pause: false, page: false };
+  return { diff, pause: true, page: !gate.paged };
+}
+
+/** Latch a once-only page, after delivery is confirmed and never before. */
+export function markPaged(gate: { paged: boolean }, delivered: boolean): void {
+  if (delivered) gate.paged = true;
+}
+
 // ---------------------------------------------------------------------- helpers
 
 function log(msg: string): void {
@@ -157,13 +353,76 @@ async function swapLabel(tracker: Tracker, issue: number, from: string, to: stri
  * The escalator throws when no transport is configured or Telegram rejects, and
  * only records the dedup marker on success. A page that cannot be delivered
  * must not take the tick down with it — log it and let the next tick retry.
+ *
+ * Returns whether it actually went out, because "page once" and "page once
+ * *successfully*" are different promises: a caller that latches a once-only
+ * gate on the attempt turns one failed delivery into permanent silence about a
+ * condition that is still true.
  */
-async function safeEscalate(d: Deps, e: Escalation): Promise<void> {
+async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<boolean> {
   try {
     await d.escalate(e);
+    return true;
   } catch (err) {
     log(`escalation for #${e.issue} could not be delivered: ${errText(err)}`);
+    return false;
+  }
+}
+
+/**
+ * What a salvage attempt contributes to the escalation: where the work went, or
+ * that it went nowhere. Split from the effects below for the same reason
+ * `checkIntegrity` is — this wording is the whole thing a human acts on, so it
+ * is worth a test holding it, and the sha in it is the only pointer to work
+ * that no longer has any other copy.
+ */
+export function salvageLines(outcome: SalvageOutcome, worktree: string): string[] {
+  const kept = `Worktree kept for inspection: ${worktree}`;
+
+  if (outcome.kind === "nothing") return [`${kept} — nothing uncommitted to salvage`];
+
+  if (outcome.kind === "failed") {
+    return [
+      `WIP SALVAGE FAILED: ${outcome.error}`,
+      `Uncommitted work in ${worktree} is the only copy of it, and the next attempt removes that tree.`,
+    ];
   }
+
+  return [
+    `WIP committed to ${outcome.branch} @ ${outcome.sha}` +
+      (outcome.pushed
+        ? " and pushed — the work outlives this worktree"
+        : ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`),
+    kept,
+  ];
+}
+
+/**
+ * Commits and pushes whatever a dead run left uncommitted, logs the outcome and
+ * returns the escalation lines that say where that work now lives.
+ *
+ * Only ever called on a non-graceful end — a cap kill, a crashed session, a
+ * dispatch error. A `blocked` run stopped on purpose, with turns still in hand
+ * and a brief that tells it to report rather than push, so nothing is committed
+ * behind its back. The rest never got the chance: the kill is external and
+ * lands mid-edit, in the tree the next attempt removes `--force`.
+ */
+async function salvage(
+  issue: number,
+  attempt: number,
+  reason: string,
+  worktree: string,
+): Promise<string[]> {
+  const lines = salvageLines(await salvageWip(worktree, issue, attempt, reason), worktree);
+  log(`#${issue} salvage: ${lines.join(" ")}`);
+  return lines;
+}
+
+/** How a run's end is named — in the salvage commit, and to whoever reads it. */
+function endedBy(killedBy: KilledBy | undefined): string {
+  if (killedBy === "turns") return "the turns cap";
+  if (killedBy === "wallclock") return "the wall-clock cap";
+  return "a failed run";
 }
 
 async function buildBrief(
@@ -201,6 +460,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
 
   let claimed = false;
   let run: RunRecord | undefined;
+  // Hoisted out of the try so the catch path can still name the tree: a crash
+  // mid-dispatch is one of the non-graceful ends whose uncommitted work has to
+  // be salvaged too, and it is the path least likely to have committed first.
+  let worktreePath: string | undefined;
 
   try {
     // Claim on the tracker FIRST, before any local work. The label — not the
@@ -233,7 +496,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
     const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
     await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
 
-    const worktreePath = await addWorktree(
+    worktreePath = await addWorktree(
       r.repo,
       project.mirrorRoot,
       project.workspaceRoot,
@@ -257,6 +520,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
       sessionDir,
       ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
       onTurn: (n) => store.updateRun(runId, { turns: n }),
+      // Recorded the moment the session opens its transcript, not when the run
+      // ends: `omp-conductor tail` resolves an issue to a file through this row,
+      // and a path written at completion is a path nobody can follow live. The
+      // completion-time update below writes the same value again, harmlessly.
+      onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
     });
 
     // A configured model the harness could not honour means this run was done by
@@ -287,6 +555,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
       });
     } else if (result.state === "failed" || result.state === "killed") {
       await swapLabel(tracker, issue, inProgress, project.stateLabels.failed);
+      // Before the escalation is composed, so it can say where the work went —
+      // and long before the next attempt provisions over this tree.
+      const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
       await safeEscalate(d, {
         tier: 1,
         project: project.name,
@@ -301,7 +572,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
         detail: [
           `${r.issue.title}`,
           r.issue.url,
-          `Worktree kept for inspection: ${worktreePath}`,
+          ...salvaged,
           `Session: ${result.sessionFile ?? "(no transcript)"}`,
           "",
           result.report,
@@ -335,28 +606,166 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
         log(`#${issue} could not be relabelled: ${errText(relabelErr)}`);
       }
     }
+    // A crash lands anywhere, including mid-edit in a tree holding the only
+    // copy of real work. Nothing else on this path so much as looks at it.
+    const salvaged =
+      worktreePath === undefined
+        ? []
+        : await salvage(issue, attempt, "a dispatch error", worktreePath);
+
     await safeEscalate(d, {
       tier: 1,
       project: project.name,
       issue,
       runId: run?.id,
       summary: `#${issue} could not be dispatched on attempt ${attempt}`,
-      detail,
+      detail: salvaged.length === 0 ? detail : [detail, "", ...salvaged].join("\n"),
     });
     // The worktree, if one was created, is deliberately left in place: this is
-    // a failure path.
+    // a failure path, and whatever it still held is now a commit on the branch.
   }
 }
 
+// -------------------------------------------------------------------- admission
+
+/** A candidate cleared for dispatch, with the attempt number it will run as. */
+export interface Admission {
+  r: Routed;
+  attempt: number;
+}
+
+/**
+ * Which routed candidates get a worker this tick — in queue order, never more
+ * than `slots` of them.
+ *
+ * Exported so the admission rules can be pinned without spawning a worker.
+ * Every one of them exists because of a live incident, and each guards a
+ * different way the same issue gets worked twice.
+ *
+ * Takes the slice of `Deps` it actually reads rather than the whole thing: what
+ * admission is allowed to consult is the point of the function, and a `Deps`
+ * that grows a field has no business breaking these tests.
+ */
+export async function admitCandidates(
+  d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate">,
+  routed: Routed[],
+  slots: number,
+): Promise<Admission[]> {
+  const { project, caps, tracker, store } = d;
+  const busy = new Set(store.activeRuns(project.name).map((r) => r.issue));
+
+  const admitted: Admission[] = [];
+  for (const r of routed) {
+    if (admitted.length >= slots) break;
+    if (busy.has(r.issue.number)) continue;
+
+    const prior = store.attemptsFor(project.name, r.issue.number);
+    if (prior >= caps.maxAttemptsPerIssue) {
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue: r.issue.number,
+        summary: `#${r.issue.number} has used all ${caps.maxAttemptsPerIssue} attempts`,
+        detail: [
+          r.issue.title,
+          r.issue.url,
+          "Another attempt almost always means the issue itself is underspecified.",
+          "Rewrite the acceptance criteria, or take it off the queue.",
+        ].join("\n"),
+      });
+      continue;
+    }
+
+    // The busy set is built from run rows, so it can only speak for work this
+    // database recorded. Work pushed before this store existed — a migration, a
+    // wiped or relocated state dir, a restore onto a new host — looks exactly
+    // like fresh work, and a worker sent at it re-implements a finished PR. The
+    // tracker is the only party that remembers, so it is asked. The cost is
+    // bounded by free slots, not by queue depth: the call sits behind the two
+    // cheap local filters and the loop stops once the slots are full.
+    let closer: string | undefined;
+    try {
+      closer = await tracker.openCloserFor(r.issue.number);
+    } catch (err) {
+      // Fail closed, per candidate. An API error means "unknown whether
+      // finished work exists", and admitting on unknown recreates precisely the
+      // duplicate-work failure this guard exists to kill: the worst case of
+      // holding is a five-minute delay, the worst case of admitting is a burned
+      // attempt and a second PR on the same issue. Holding one candidate rather
+      // than aborting the loop is what keeps a transient GitHub failure from
+      // deadlocking the whole dispatcher; the next tick retries by itself.
+      log(`#${r.issue.number} held: open-PR check failed (${errText(err)}) — retrying next tick`);
+      continue;
+    }
+    if (closer !== undefined) {
+      log(`#${r.issue.number} skipped: open PR ${closer} already closes it`);
+      continue;
+    }
+
+    admitted.push({ r, attempt: prior + 1 });
+  }
+
+  return admitted;
+}
+
 // ----------------------------------------------------------------------- a tick
 
 async function tick(d: Deps): Promise<void> {
+  // Before the pause check, deliberately. This one is not about dispatch: the
+  // orchestrator is a different process, and it can be wedged while this fleet
+  // is paused — which is exactly the state the reference fleet was in when the
+  // failure happened. A pause silences claiming, not the operator's right to
+  // know their supervising session stopped reading its queue.
+  await watchOrchestrator(d);
+
   // A paused fleet claims nothing. Checked first so pausing takes effect on the
   // next tick without signalling the process.
   if (isPaused()) return;
 
   const { project, caps, store } = d;
 
+  // "Nobody patches the running conductor" is a hard boundary in both briefs —
+  // which makes it prompt text, and prompt text is a request. This is the half
+  // that does not negotiate: the package that dispatched the last worker must
+  // still be the package on disk, or nothing else this tick does is
+  // attributable. A legitimate deploy never trips it, because installing a new
+  // build and restarting the unit re-records the baseline from the new files;
+  // only an edit *underneath* a live daemon diverges from it.
+  const integrity = checkIntegrity(d.integrity, packageManifest());
+  if (integrity.pause) {
+    const shown = integrity.diff.slice(0, INTEGRITY_SAMPLE);
+    log(
+      `ERROR: the installed conductor changed under this daemon — ${integrity.diff.length} file(s) differ ` +
+        `(${shown.join(", ")}${integrity.diff.length > shown.length ? ", …" : ""}) — pausing`,
+    );
+    setPaused(true);
+    if (integrity.page) {
+      const delivered = await safeEscalate(d, {
+        tier: 2,
+        project: project.name,
+        issue: NO_ISSUE,
+        // Dated for the same reason the spend cap is: the dedup key is the
+        // summary, and a second tamper months later must not be swallowed as a
+        // repeat of the first.
+        summary:
+          `Installed conductor changed under a running daemon on ${new Date().toISOString().slice(0, 10)}: ` +
+          `${integrity.diff.length} file(s) differ (first: ${integrity.diff[0]}) — ${project.name} is paused`,
+        detail: [
+          `Package root: ${import.meta.dir}`,
+          ...shown,
+          ...(integrity.diff.length > shown.length ? [`… and ${integrity.diff.length - shown.length} more`] : []),
+          "",
+          "If you deployed a new build, restart the daemon — the restart re-records the baseline.",
+          "If you did not, the host edited itself while it was dispatching work: treat every run since",
+          "the last known-good restart as unattributable before resuming.",
+          "`omp-conductor resume` alone will not hold — the next tick re-pauses while the files differ.",
+        ].join("\n"),
+      });
+      markPaged(d.integrity, delivered);
+    }
+    return;
+  }
+
   // route() filters the queue through isEligible() itself, so anything already
   // carrying a state label is gone before it gets here.
   const { routed, unroutable } = route(await d.tracker.listReady(), project);
@@ -402,41 +811,18 @@ async function tick(d: Deps): Promise<void> {
     return;
   }
 
-  const active = store.activeRuns(project.name);
-  const slots = caps.maxConcurrentWorkers - active.length;
+  // Two different questions, deliberately two queries. Capacity counts worker
+  // *processes*, so a green PR awaiting a human merge must not consume a slot —
+  // two of those would otherwise stop the fleet. That same PR's *issue* must
+  // still be occupied, which is what `admitCandidates`' busy set is for.
+  const live = store.liveRuns(project.name);
+  const slots = caps.maxConcurrentWorkers - live.length;
   if (slots <= 0) {
-    log(`at capacity: ${active.length}/${caps.maxConcurrentWorkers} workers`);
+    log(`at capacity: ${live.length}/${caps.maxConcurrentWorkers} workers`);
     return;
   }
 
-  // activeRuns includes pushed-green work that is still waiting on a human
-  // merge, so this also stops a second attempt landing on a live PR.
-  const busy = new Set(active.map((r) => r.issue));
-
-  const admitted: { r: Routed; attempt: number }[] = [];
-  for (const r of routed) {
-    if (admitted.length >= slots) break;
-    if (busy.has(r.issue.number)) continue;
-
-    const prior = store.attemptsFor(project.name, r.issue.number);
-    if (prior >= caps.maxAttemptsPerIssue) {
-      await safeEscalate(d, {
-        tier: 1,
-        project: project.name,
-        issue: r.issue.number,
-        summary: `#${r.issue.number} has used all ${caps.maxAttemptsPerIssue} attempts`,
-        detail: [
-          r.issue.title,
-          r.issue.url,
-          "Another attempt almost always means the issue itself is underspecified.",
-          "Rewrite the acceptance criteria, or take it off the queue.",
-        ].join("\n"),
-      });
-      continue;
-    }
-
-    admitted.push({ r, attempt: prior + 1 });
-  }
+  const admitted = await admitCandidates(d, routed, slots);
 
   if (admitted.length === 0) return;
 
@@ -453,7 +839,10 @@ export interface StatusSnapshot {
   stateDir: string;
   paused: boolean;
   caps: Caps;
+  /** Occupied issues: live workers plus green PRs awaiting a human merge. */
   activeRuns: RunRecord[];
+  /** Runs backed by a worker process — the number capacity compares against. */
+  liveWorkers: number;
   runsToday: number;
   spendTodayUsd: number;
 }
@@ -474,6 +863,7 @@ export function statusSnapshot(project?: string): StatusSnapshot {
       paused: isPaused(),
       caps: resolveCaps(p, cfg.defaults),
       activeRuns: store.activeRuns(p.name),
+      liveWorkers: store.liveRuns(p.name).length,
       runsToday: store.runsStartedSince(p.name, since),
       spendTodayUsd: store.spendSince(p.name, since),
     };
@@ -489,7 +879,7 @@ export function formatStatus(s: StatusSnapshot): string {
     `state     ${s.stateDir}`,
     "",
     "caps",
-    `  workers            ${s.activeRuns.length} / ${s.caps.maxConcurrentWorkers}`,
+    `  workers            ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
     `  issues today       ${s.runsToday}`,
     `  spend today        $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
     `  worker max turns   ${s.caps.workerMaxTurns}`,
@@ -564,6 +954,38 @@ export function armConductor(): void {
   setPaused(false);
 }
 
+/**
+ * Settles the runs a previous daemon process left in flight.
+ *
+ * A `claimed` or `running` row is a promise that a worker exists in *some*
+ * process. This is called from a freshly started daemon, so when no other
+ * daemon is alive every such row is a worker that died with the previous
+ * process. Left "active", those rows deadlock admission forever: the slot
+ * count reads full while nothing runs, and the fleet looks busy doing nothing
+ * (found live, after a host restart killed two workers mid-run).
+ *
+ * Only the rows change. The issue keeps its in-progress label — that label is
+ * the crash guard against double-dispatch, and deciding what a dead worker's
+ * remains are worth (an open PR? unpushed commits? a dirty tree?) is the
+ * orchestrator's drain-duty judgement, not something to automate here. The
+ * rows also keep counting toward `maxAttemptsPerIssue`, so a loop of deaths
+ * still escalates instead of retrying forever.
+ *
+ * `pushed-green` rows are deliberately left alone: they hold no process — they
+ * are finished work waiting on a human merge, and they must keep occupying the
+ * issue so a second attempt cannot land on a live PR.
+ */
+export function reconcileOrphanedRuns(store: Store, project: string): RunRecord[] {
+  // Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
+  // a process dying — it is finished work waiting on a human merge.
+  const stale = store.liveRuns(project);
+  const endedAt = Date.now();
+  for (const r of stale) {
+    store.updateRun(r.id, { state: "orphaned", endedAt });
+  }
+  return stale;
+}
+
 // ------------------------------------------------------------------- the daemon
 
 export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
@@ -573,6 +995,30 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
   const store = openStore(dbPath());
   const tracker = makeTracker(project);
 
+  // Recorded here, before a single tick runs, so that the deploy an operator
+  // *means* to do never trips the tripwire: installing a new build and
+  // restarting the unit re-records this from the new files. What it catches is
+  // the other thing — the package changing while the daemon that dispatches
+  // work is holding it open, whether that is a worker that wandered out of its
+  // worktree or a human editing the live install "just to test something".
+  const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
+  log(`package integrity baseline: ${integrity.baseline.size} files under ${import.meta.dir}`);
+
+  // Before the first tick, settle what the last process left behind — unless
+  // another daemon is alive (a foreground `daemon --once` beside a running
+  // daemon must not orphan that daemon's real, live workers).
+  const alive = livingDaemon();
+  if (alive === undefined || alive.pid === process.pid) {
+    for (const r of reconcileOrphanedRuns(store, project.name)) {
+      log(
+        `#${r.issue} orphaned by a previous daemon (attempt ${r.attempt}, was ${r.state}, worktree ${r.worktree}) — ` +
+          `slot freed; the ${project.stateLabels.inProgress} label stays until the orchestrator triages what the worker left`,
+      );
+    }
+  } else {
+    log(`skipping orphan reconciliation: daemon pid ${alive.pid} is alive and owns the active runs`);
+  }
+
   // Standing orders. The orchestrator holds none of this file's context, so
   // everything it needs to act — which tracker, which labels, what the fleet
   // does — has to be said once, in words.
@@ -587,8 +1033,16 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
     "fails twice, its gates stay red, its branch conflicts, or a tripwire fires.",
     "Your job when that happens: re-brief the issue (comment what the next worker must do",
     `differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`,
-    "tier 2 and let the human decide. You never edit product code, push a branch, or merge a PR —",
-    "a worker session does all of that. Handle each escalation below before the next one.",
+    "tier 2 and let the human decide.",
+    // Worded from `authority.merge` rather than fixed, so the standing orders
+    // and the Releases section of the rendered brief cannot disagree about who
+    // is holding the merge button. The daemon still merges nothing itself.
+    project.authority.merge === "orchestrator"
+      ? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " +
+        "a time, freshness-checked against the base branch, per the Releases section of your ORCHESTRATOR.md."
+      : "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
+        "human merges.",
+    "Handle each escalation below before the next one.",
   ].join("\n");
 
   // One orchestrator per daemon run, not per tick: it is a persistent session
@@ -597,17 +1051,25 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
   // directory, deliberately not a checkout — the orchestrator re-briefs workers
   // and talks to the tracker, it does not edit product code.
   let orchestrator: OrchestratorHandle | undefined;
-  try {
-    orchestrator = await startOrchestrator({ cwd: stateDir(), brief });
-    const transcript = orchestrator.sessionFile();
-    log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
-  } catch (err) {
-    // Loudly, but not fatally: tier-1 escalations degrade to issue comments,
-    // which a human still reads. A dispatcher that refuses to run because its
-    // re-briefing channel is down helps nobody.
-    log(
-      `WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue comments: ${errText(err)}`,
-    );
+  if (project.escalation.orchestrator === "external") {
+    // An operator already runs the brain — typically a visible TUI session that
+    // drains `blocked`/`failed` off the tracker as one of its standing duties.
+    // Starting a second one here would re-triage the same issues from a
+    // transcript nobody is watching, and the two would undo each other.
+    log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
+  } else {
+    try {
+      orchestrator = await startOrchestrator({ cwd: stateDir(), brief });
+      const transcript = orchestrator.sessionFile();
+      log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
+    } catch (err) {
+      // Loudly, but not fatally: tier-1 escalations degrade to issue comments,
+      // which a human still reads. A dispatcher that refuses to run because its
+      // re-briefing channel is down helps nobody.
+      log(
+        `WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue comments: ${errText(err)}`,
+      );
+    }
   }
 
   const escalator = createEscalator(project, tracker, store, orchestrator);
@@ -617,6 +1079,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
     tracker,
     store,
     escalate: (e) => escalator.escalate(e),
+    integrity,
+    // Fresh per daemon run, like the integrity gate: a restart is entitled to
+    // page again about a stall that is still on disk.
+    stall: { paged: false },
   };
 
   if (o.once) {