omp-conductor 0.19.7 → 0.20.0

package/src/daemon/runtime.ts

@@ -0,0 +1,736 @@
+/**
+ * The daemon process itself: what `omp-conductor daemon` actually runs.
+ *
+ * Everything in this directory above it is a pure-ish pass over a `Deps`.
+ * This is where the real ones are built — the tracker, the store, the verb
+ * socket, the orchestrator session, the pool and the registries — and where the
+ * only long-lived state in the dispatcher lives: `Bun.serve`, the timers, the
+ * integrity baseline recorded at boot, and the shutdown that has to drain them
+ * all in the right order.
+ *
+ * Keeping it last and separate is what lets every pass be tested without a
+ * process. Nothing here is called by a pass; the arrow runs one way, from the
+ * runtime into the passes, which is also why `runDaemon` is the only place a
+ * dep is wired to its production implementation.
+ */
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { findProject, loadConfig, resolveCaps, resolveReleaseGrants, resolveSharedInstallAuthority, stateDir } from "../config.ts";
+import { probeCredentialClass } from "../credential-class.ts";
+import { createEscalator, escalationIssueRef } from "../escalate.ts";
+import { releaseOrphanedWorkerPane } from "../fleet.ts";
+import { probeCriticalBase, probeRunLane, pushRunBranch } from "../gitops.ts";
+import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "../graph-health.ts";
+import { ensureHttpToken } from "../http-token.ts";
+import { acquireOnceLease, livingDaemon } from "../lifecycle.ts";
+import { errText, log } from "../log.ts";
+import { reconcileOrchestratorDown } from "../orchestrator-down.ts";
+import { startOrchestrator, type OrchestratorHandle } from "../orchestrator.ts";
+import { isPaused } from "../pause.ts";
+import { recordReleaseBlock } from "../release-policy.ts";
+import { createReportOutbox, type ReportOutbox } from "../reports.ts";
+import { reconcileOrphanedRuns } from "../settlement.ts";
+import { dbPath, openStore, utcDay } from "../store.ts";
+import { checkTelegramFreshness } from "../telegram-freshness.ts";
+import { GraphqlBreaker, makeTracker } from "../tracker/github.ts";
+import { RELEASE_SHAPES, type ProjectConfig, type ResolvedGrants, type RunRecord } from "../types.ts";
+import { runCommand } from "../upgrade-verify.ts";
+import { inspectSurfaces } from "../upgrade.ts";
+import { sharedUsageSource } from "../usage.ts";
+import { githubVerbActions } from "../verbs/actions.ts";
+import { listenVerbChannel, type VerbListener } from "../verbs/server.ts";
+import { ensureVerbSocketDir, peerCredentialReader, transportBanner, verbSocketPath } from "../verbs/socket.ts";
+import type { RunPublisher } from "../worktree.ts";
+import { createWorkerPool } from "./admission-pass.ts";
+import { DEFAULT_PORT, GRAPH_HEALTH_INTERVAL_MS, PACKAGE_SRC_DIR, REPORT_DELIVERY_INTERVAL_MS, TICK_INTERVAL_MS, verbDeps, type DaemonOpts, type Deps, type DrainSignal, type IntegrityGate } from "./deps.ts";
+import { createTurnLimitRegistry, createWorkerControlRegistry } from "./dispatch.ts";
+import { daemonHttpResponse } from "./http.ts";
+import { packageManifest } from "./integrity.ts";
+import { workspaceOwnership } from "./panes.ts";
+import { reconcileCrashedReviewRevisions } from "./review.ts";
+import { reconcileHistoricalInfra } from "./settle-pass.ts";
+import { tick } from "./tick.ts";
+import { daemonHealth, daemonHealthSnapshot } from "./views.ts";
+
+export interface ProjectRuntime {
+  d: Deps;
+  outbox: ReportOutbox;
+  orchestrator?: OrchestratorHandle;
+  orchestratorVerbs?: VerbListener;
+  codeGraph: CodeGraphHealth;
+  graphProbe?: Promise<void>;
+  reportPass?: Promise<void>;
+}
+
+export function orchestratorStandingOrders(
+  project: ProjectConfig,
+  projects: readonly ProjectConfig[],
+): {
+  brief: string;
+  releaseGrants: ResolvedGrants;
+} {
+  const installAuthority = resolveSharedInstallAuthority(projects);
+  const releaseGrants = {
+    ...resolveReleaseGrants(project),
+    install: installAuthority.holder ?? "human",
+  };
+  const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator");
+  const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator");
+  return {
+    releaseGrants,
+    brief: [
+      `You are the omp-conductor orchestrator for project "${project.name}".`,
+      `Tracker: ${project.tracker.repo}. Pass --repo ${project.tracker.repo} to every gh command:`,
+      "this working directory is the conductor's state directory, not a checkout.",
+      `Labels: queue=${project.queueLabel}, running=${project.stateLabels.inProgress}, ` +
+        `blocked=${project.stateLabels.blocked}, failed=${project.stateLabels.failed}.`,
+      "The dispatcher claims queue-labelled issues, runs one worker session per attempt in its own",
+      "worktree under hard turn/wallclock/spend caps, and escalates to you when a worker blocks or",
+      "fails twice, its gates stay red, its branch conflicts, or a tripwire fires.",
+      "Your job when that happens: re-brief the issue (comment what the next worker must do",
+      `differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`,
+      "tier 2 and let the human decide.",
+      project.authority.merge === "orchestrator"
+        ? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " +
+          "a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
+        : "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
+          "human merges.",
+      `Release tool gate: ${
+        grantedShapes.length === 0
+          ? "every release and deploy shape is mechanically blocked for you"
+          : `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
+      }.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
+      ...(installAuthority.holder === undefined
+        ? [
+            `Host-global install authority conflicts: ${installAuthority.entries
+              .map((entry) => `${entry.project}=${entry.holder}`)
+              .join(", ")}. Install is blocked.`,
+          ]
+        : []),
+      "Handle each escalation below before the next one.",
+    ].join("\n"),
+  };
+}
+
+/**
+ * Pacing for the resident dispatch loop, interruptible so an operator action —
+ * `resume` — does not wait out the full interval (#380).
+ *
+ * `requestWake` is sticky until a pass absorbs it: while the long sleep is in
+ * flight it is cut short, and when a pass is mid-flight the loop skips the
+ * *following* sleep. Either way the next pass starts without the interval
+ * wait. The single boolean coalesces any number of near-simultaneous wakes
+ * into one prompt pass, and the strictly sequential loop means a wake can
+ * never start a pass on top of one already running.
+ */
+export interface DispatchPace {
+  /** Ask for a prompt pass; cuts an in-flight sleep short. Coalesces. */
+  requestWake(): void;
+  /** Absorb a pending request — true when the pass that just finished should
+   *  skip its sleep. */
+  takeWake(): boolean;
+  /** The long interval sleep; resolves early when {@link requestWake} fires. */
+  sleep(ms: number): Promise<void>;
+}
+
+export function createDispatchPace(): DispatchPace {
+  let pendingWake = false;
+  let interruptSleep: (() => void) | undefined;
+  return {
+    requestWake() {
+      pendingWake = true;
+      interruptSleep?.();
+    },
+    takeWake() {
+      const pending = pendingWake;
+      pendingWake = false;
+      return pending;
+    },
+    async sleep(ms) {
+      // A wake that landed in the gap between the loop's takeWake check and
+      // this call is absorbed here: skip the interval, and consume the flag
+      // so the pass that follows it is the only prompt one.
+      if (pendingWake) {
+        pendingWake = false;
+        return;
+      }
+      await new Promise<void>((resolve) => {
+        const timer = setTimeout(resolve, ms);
+        interruptSleep = () => {
+          // The interruption *is* the absorption: the pass that follows the
+          // sleep is the prompt pass this wake asked for, so the loop sleeps
+          // normally after it instead of running a second immediate pass.
+          pendingWake = false;
+          clearTimeout(timer);
+          resolve();
+        };
+        // A wake can land between the loop's takeWake check and this
+        // constructor; resolve immediately instead of letting it wait out
+        // the full interval.
+        if (pendingWake) interruptSleep();
+      });
+      interruptSleep = undefined;
+    },
+  };
+}
+
+export interface DispatchLoopOptions {
+  /** One dispatch pass over every configured project runtime. */
+  tick(): Promise<void>;
+  /** True once a stop is requested; checked between passes. */
+  shouldStop(): boolean;
+  pace: DispatchPace;
+  /** The scheduled interval between passes (`TICK_INTERVAL_MS` in production). */
+  tickIntervalMs: number;
+}
+
+/**
+ * The resident daemon's dispatch loop: one pass immediately, then a
+ * `tickIntervalMs` sleep the {@link DispatchPace} can cut short. Exported so
+ * the resume-wake behaviour can be pinned deterministically without a live
+ * daemon (#380).
+ */
+export async function runDispatchLoop(o: DispatchLoopOptions): Promise<void> {
+  const { tick, shouldStop, pace, tickIntervalMs } = o;
+  while (!shouldStop()) {
+    await tick();
+    if (shouldStop()) break;
+    // A wake that landed while the pass was in flight skips the following
+    // sleep: that pass made its admission decisions against pre-resume state,
+    // and the resumed project must not wait a full interval for the next one.
+    if (pace.takeWake()) continue;
+    await pace.sleep(tickIntervalMs);
+  }
+}
+
+export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
+  // A `--once` tick is still a dispatcher: it settles rows, projects labels,
+  // admits and launches workers, so it must hold the same exclusive daemon/
+  // state ownership as the long-running dispatcher for its whole lifecycle
+  // (#431). This is the one choke point every once tick passes through (both
+  // `daemon --once` and the `setup host` smoke), so ownership is acquired here
+  // — before any settlement, label, admission, worktree or worker mutation —
+  // and refused (having done none of those) while another live daemon owns the
+  // project or another live `--once` holds the lease.
+  const onceLease = o.once ? acquireOnceLease({ port: o.port, project: o.project }) : undefined;
+  if (onceLease !== undefined && !onceLease.ok) {
+    throw new Error(
+      `another daemon is alive (pid ${onceLease.pid}); stop it first — a --once tick refuses to run beside it`,
+    );
+  }
+  const cfg = loadConfig();
+  const projects = o.project === undefined ? cfg.projects : [findProject(cfg, o.project)];
+  const store = openStore(dbPath());
+  const verbPeerReader = peerCredentialReader();
+  const verbDir = ensureVerbSocketDir(stateDir());
+  const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
+  const usage = sharedUsageSource();
+  const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
+    store.updateRun(runId, { maxTurns });
+  });
+  const workerControls = createWorkerControlRegistry();
+  // Created before the pool so a worker's completion can poke the loop it
+  // shares with the interval sleep (#878).
+  const pace = createDispatchPace();
+  const workers = createWorkerPool(() => {
+    // Sticky and coalescing by construction: several workers settling together
+    // produce one pass, and a pass already running absorbs the request rather
+    // than overlapping. No second owner is created and no gate is skipped.
+    pace.requestWake();
+  });
+  const alive = livingDaemon();
+  const runtimes: ProjectRuntime[] = [];
+
+  // The stop fence (#374): one signal shared by every project's tick.
+  // `stopping` gates the run loop between whole ticks; `drain.draining` is
+  // the same event visible *inside* a tick already in flight, so a pass that
+  // was mid-admission when the stop landed re-checks it before claiming or
+  // launching. Created here, before the loop, so `stop` below and every
+  // project's `Deps` reference the same object.
+  const drain: DrainSignal = { draining: false };
+
+  log(`verb transport: ${transportBanner(verbDir, verbPeerReader)}`);
+  log(`package integrity baseline: ${integrity.baseline.size} files under ${PACKAGE_SRC_DIR}`);
+
+  for (const project of projects) {
+    const projectLog = (message: string): void => {
+      log(projects.length === 1 ? message : `[${project.name}] ${message}`);
+    };
+    const caps = resolveCaps(project, cfg.defaults);
+    const host = cfg.host;
+    // One transient-server-error breaker per project (#642): admission's
+    // GraphQL checks and the orchestrator mutation commands share it, so a 503
+    // observed by either side gates both instead of one provider outage being
+    // re-asked by every candidate AND every mutation.
+    const graphqlBreaker = new GraphqlBreaker();
+    const tracker = makeTracker(project, undefined, {
+      onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
+      onRefusal: (at) => store.recordGhRefusal?.(at),
+      onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
+    }, { graphqlBreaker });
+    const verbActions = githubVerbActions(project, undefined, undefined, graphqlBreaker);
+
+    if (alive === undefined || alive.pid === process.pid) {
+      const orphanPublisher = (run: RunRecord): RunPublisher => {
+        const repo = project.routing.repos[run.repo];
+        return async (branch) =>
+          repo === undefined
+            ? {
+                ok: false,
+                stderr:
+                  `run ${String(run.id)} names repo "${run.repo}", ` +
+                  `which project "${project.name}" no longer routes`,
+              }
+            : pushRunBranch(project, { repo, runRepoPath: run.worktree, branch });
+      };
+      for (const run of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
+        // The workspace outlives the process, so a restart inherits panes whose
+        // workers are gone (#842). A session-host child dies with the daemon that
+        // owned its socket, so an orphaned row's pane is authoritatively dead
+        // whatever its recorded pid says. Ownership is proven before the close
+        // (#1035 review): pane ids are reused after a Herdr restart, so only a
+        // pane inside a workspace conductor owns — token-marked or store-recorded,
+        // which is what survives Herdr dropping tokens on restore — whose
+        // reported identity still agrees with this row is retired. An unowned
+        // hit closes nothing and says so.
+        const orphanPane = releaseOrphanedWorkerPane(
+          { paneId: run.paneId, paneLabel: run.paneLabel, runId: run.id, project: run.project },
+          { ownership: workspaceOwnership(store) },
+        );
+        if (orphanPane.kind === "released") {
+          projectLog(
+            `#${run.issue} herdr pane ${orphanPane.paneId} retired: its worker died with the previous daemon`,
+          );
+        } else if (orphanPane.kind === "unowned") {
+          projectLog(`#${run.issue} herdr pane ${orphanPane.paneId} left alone: ${orphanPane.reason}`);
+        } else if (orphanPane.kind === "failed") {
+          projectLog(`#${run.issue} herdr pane ${orphanPane.paneId} release failed: ${orphanPane.reason}`);
+        }
+        projectLog(
+          `#${run.issue} orphaned by a previous daemon (attempt ${run.attempt}, was ${run.state}, ` +
+            `worktree ${run.worktree}) — slot freed; the ${project.stateLabels.inProgress} label stays ` +
+            "until the orchestrator triages what the worker left",
+        );
+      }
+    } else if (runtimes.length === 0) {
+      log(`skipping orphan reconciliation: daemon pid ${alive.pid} is alive and owns the active runs`);
+    }
+
+    const { brief, releaseGrants } = orchestratorStandingOrders(project, cfg.projects);
+    let orchestrator: OrchestratorHandle | undefined;
+    let orchestratorVerbs: VerbListener | undefined;
+    /** First start-failure cause, surfaced by the orchestrator-down incident (#288). */
+    let orchestratorStartError: string | undefined;
+    if (project.escalation.orchestrator === "external") {
+      projectLog(
+        "orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty",
+      );
+    } else {
+      try {
+        const orchCwd = join(
+          stateDir(),
+          projects.length === 1 ? "orchestrator" : `orchestrator-${project.name}`,
+        );
+        mkdirSync(orchCwd, { recursive: true });
+        orchestratorVerbs = await listenVerbChannel(
+          verbDeps({ project, store, tracker, verbActions }),
+          {
+            kind: "orchestrator",
+            path: verbSocketPath(verbDir, `orchestrator-${project.name}`),
+            project: project.name,
+            role: "orchestrator",
+          },
+          { ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) },
+        );
+        orchestrator = await startOrchestrator({
+          cwd: orchCwd,
+          brief,
+          releaseGrants,
+          socketPath: join(orchCwd, "ipc.sock"),
+          verbSocketPath: orchestratorVerbs.path,
+          onSpawn: (pid) => {
+            orchestratorVerbs?.bindPid(pid);
+          },
+          onChildLog: (line) => {
+            projectLog(`orchestrator ${line}`);
+          },
+          onReleaseBlocked: (shape, context) =>
+            recordReleaseBlock(project.name, "orchestrator", shape, context),
+        });
+        const transcript = orchestrator.sessionFile();
+        const loaded = orchestrator.extensionVersion();
+        projectLog(
+          `orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}` +
+            `${loaded === undefined ? "" : ` · loaded omp-conductor ${loaded}`}`,
+        );
+      } catch (err) {
+        orchestratorStartError = errText(err);
+        projectLog(
+          "WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue " +
+            `comments: ${orchestratorStartError}`,
+        );
+        await orchestratorVerbs?.close();
+        orchestratorVerbs = undefined;
+      }
+    }
+
+    let runtimeDeps: Deps | undefined;
+    const currentProject = (): ProjectConfig => runtimeDeps?.project ?? project;
+    const deliveryPolicyValid = (): boolean => runtimeDeps?.deliveryPolicyValid === true;
+    const escalator = createEscalator(
+      currentProject,
+      tracker,
+      store,
+      orchestrator,
+      Date.now,
+      deliveryPolicyValid,
+      (e) => {
+        // Every tier-1 escalation that lands on the issue-comment fallback was
+        // diverted from the orchestrator. Count it durably on the open incident
+        // (a no-op when none is open), so the page and status name how much
+        // the outage diverted (#288).
+        store.bumpOrchestratorDiverted(project.name, 1);
+        projectLog(
+          `orchestrator: tier-1 escalation on ${escalationIssueRef(e.issue)} diverted to issue comments ` +
+            `while the orchestrator was down`,
+        );
+      },
+    );
+    const outbox = createReportOutbox({
+      project: currentProject,
+      store,
+      escalate: (event) => escalator.escalate(event),
+      log: projectLog,
+      deliveryAllowed: deliveryPolicyValid,
+    });
+    if (alive === undefined || alive.pid === process.pid) {
+      for (const report of outbox.recover(Date.now())) {
+        projectLog(
+          `report ${report.id} was left mid-send by a previous daemon (attempt ${report.attempts}) — ` +
+            "retrying; its message will say it may be a repeat",
+        );
+      }
+    }
+
+    const d: Deps = {
+      project,
+      caps,
+      host,
+      tracker,
+      store,
+      drain,
+      deliveryPolicyValid: false,
+      usage,
+      escalate: (event) => escalator.escalate(event),
+      turnLimits,
+      workerControls,
+      integrity,
+      stall: { paged: false },
+      ...(orchestrator === undefined ? {} : { orchestrator }),
+      cleanup: { next: 0 },
+      probeCriticalBase: (repo, markers, branch) =>
+        probeCriticalBase(project, repo, branch, markers),
+      // The same seam `doctor` and `upgrade` read (#904/#919), never a second
+      // implementation. `readHerdr` stays on: a host that runs herdr is the
+      // case where a stale recovery pin matters, and a host without it answers
+      // "absent", which no cheap surface treats as a fault.
+      probeInstallSurfaces: () => inspectSurfaces({ run: runCommand, log, env: process.env }),
+      probeTelegramFreshness: () =>
+        // The module's own reader, so the daemon, `doctor` and `status` cannot
+        // come to disagree about what is installed (#961).
+        checkTelegramFreshness({
+          run: async (cmd, args) => {
+            const r = await runCommand(cmd, args);
+            return { code: r.code, stdout: r.stdout };
+          },
+        }),
+      probeWorktreeLane: (input) => probeRunLane(input),
+      // The credential-class fence's reader (#852): the runnable probe module,
+      // spawned against the live credential store. Wired once here so the
+      // admission gate and the launch fence ask the same question through the
+      // same transport — two readers would eventually disagree, and the one that
+      // disagreed by passing is the expensive one.
+      probeCredentialClass: (provider) => probeCredentialClass(provider),
+      // A cross-repo Depends-on prerequisite reads through the same GitHub
+      // credential/accounting seams as the project tracker — a fresh tracker
+      // scoped to the referenced repo, reusing the daemon's gh hooks so API
+      // spend and refusals are counted exactly as the main tracker's are.
+      probeIssueIn: (ownerRepo, issue) =>
+        makeTracker({ ...project, tracker: { kind: "github", repo: ownerRepo } }, undefined, {
+          onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
+          onRefusal: (at) => store.recordGhRefusal?.(at),
+        }).issueSnapshot(issue),
+      // The body twin of `probeIssueIn`, for the dependency-graph cycle pass
+      // (#421): reads a reachable routed prerequisite's body through the same
+      // per-repo tracker/credential/accounting seams.
+      probeBodyIn: (ownerRepo, issue) =>
+        makeTracker({ ...project, tracker: { kind: "github", repo: ownerRepo } }, undefined, {
+          onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
+          onRefusal: (at) => store.recordGhRefusal?.(at),
+        }).issueBody(issue),
+      ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
+      verbActions,
+    };
+    runtimeDeps = d;
+    // Review-revision restart recovery (#692): a revision the previous daemon
+    // claimed and lost — its run is `running`/`orphaned` by the orphan sweep
+    // above — is restored to `pushed-green` and re-queued so this process's
+    // first dispatch pass resumes the exact session. Runs under the same
+    // dead-daemon guard as `reconcileOrphanedRuns`, and after it, so the
+    // revision's worktree has already been salvaged to the branch before it is
+    // cleared for a fresh reattach. A round that cannot be restored was
+    // settled and escalated by the reconcile itself.
+    if (alive === undefined || alive.pid === process.pid) {
+      try {
+        for (const revision of await reconcileCrashedReviewRevisions(d)) {
+          projectLog(
+            `#${revision.issue} review round ${revision.round} recovered across restart — ` +
+              `run ${revision.runId} is pushed-green again and awaits its next dispatch pass`,
+          );
+        }
+      } catch (err) {
+        projectLog(`review revision restart recovery failed: ${errText(err)}`);
+      }
+    }
+    // Adjudication restart recovery (#932). An adjudication is `running` only
+    // while a session it launched is alive, and that session died with the
+    // previous process — nothing resumes it, because an adjudicator has no
+    // branch, no worktree and no transcript worth continuing: its whole output
+    // is a verdict it never produced.
+    //
+    // So it settles `failed`, which is honest and terminal, rather than being
+    // left `running` forever (a row nothing would ever touch again) or silently
+    // re-queued (a second launch for one head, which the one-shot exists to
+    // prevent). The one shot for that head is spent, and #876's disposition path
+    // reads `failed` exactly as it reads any other non-clear verdict.
+    if (alive === undefined || alive.pid === process.pid) {
+      for (const adjudication of store.openReviewAdjudications(project.name)) {
+        if (adjudication.state !== "running") continue;
+        store.settleReviewAdjudication(
+          adjudication.id,
+          "failed",
+          "the daemon restarted while this adjudication was running; no verdict was produced",
+          Date.now(),
+        );
+        projectLog(
+          `#${adjudication.issue} adjudication failed across restart — its session died with the previous daemon`,
+        );
+      }
+    }
+    // Startup reconciliation: close an incident carried over from a previous
+    // process when the orchestrator is up (one recovery notice), or open one
+    // when it failed to start (one down page). A daemon restarted while still
+    // down rediscovers the open incident and does not re-page it.
+    await reconcileOrchestratorDown({
+      project,
+      store,
+      orchestrator,
+      escalate: (event) => d.escalate(event),
+      ...(orchestratorStartError === undefined ? {} : { startCause: orchestratorStartError }),
+      log: projectLog,
+    });
+    runtimes.push({
+      d,
+      outbox,
+      ...(orchestrator === undefined ? {} : { orchestrator }),
+      ...(orchestratorVerbs === undefined ? {} : { orchestratorVerbs }),
+      codeGraph: pendingCodeGraph(project),
+    });
+  }
+
+  // Historical budget repair (#638): one bounded pass per daemon process,
+  // before the first tick, so admission and `unblock` read the repaired counts
+  // from the moment the daemon starts. A classifier fix changes future
+  // verdicts but not rows already settled under the old one; re-fetching their
+  // failed check logs and reclassifying the exact closed infrastructure
+  // signature returns those attempts without re-animating the runs. Best-effort
+  // per project — one unreachable project must not stop the rest from booting.
+  for (const runtime of runtimes) {
+    try {
+      await reconcileHistoricalInfra(runtime.d);
+    } catch (err) {
+      log(`historical infra reconciliation failed: ${errText(err)}`);
+    }
+  }
+
+  if (o.once) {
+    try {
+      for (const runtime of runtimes) await tick(runtime.d, workers);
+      await workers.drain();
+      for (const runtime of runtimes) await runtime.outbox.deliverDue();
+    } finally {
+      // Drop exclusivity only while it is still ours; a manager that claimed
+      // the record mid-tick is left alone. A lease acquired but never released
+      // (config/lease failure before this branch) is stale recovery's job.
+      if (onceLease !== undefined && onceLease.ok) onceLease.lease.release();
+      for (const runtime of runtimes.toReversed()) {
+        await runtime.orchestrator?.dispose();
+        await runtime.orchestratorVerbs?.close();
+      }
+      store.close();
+    }
+    return;
+  }
+
+  const refreshCodeGraph = (runtime: ProjectRuntime): void => {
+    if (runtime.graphProbe !== undefined) return;
+    runtime.graphProbe = probeCodeGraph(runtime.d.project)
+      .then((health) => {
+        runtime.codeGraph = health;
+      })
+      .catch(() => {
+        log(
+          `[${runtime.d.project.name}] code-graph health probe failed unexpectedly; ` +
+            "retaining the previous bounded result",
+        );
+      })
+      .finally(() => {
+        runtime.graphProbe = undefined;
+      });
+  };
+  for (const runtime of runtimes) refreshCodeGraph(runtime);
+  const graphTimer = setInterval(() => {
+    for (const runtime of runtimes) refreshCodeGraph(runtime);
+  }, GRAPH_HEALTH_INTERVAL_MS);
+
+  const drainReports = (runtime: ProjectRuntime): void => {
+    if (runtime.reportPass !== undefined) return;
+    runtime.reportPass = runtime.outbox
+      .deliverDue()
+      .then(() => {})
+      .catch((err: unknown) => {
+        log(`[${runtime.d.project.name}] report delivery pass failed: ${errText(err)}`);
+      })
+      .finally(() => {
+        runtime.reportPass = undefined;
+      });
+  };
+  for (const runtime of runtimes) drainReports(runtime);
+  const reportTimer = setInterval(() => {
+    for (const runtime of runtimes) drainReports(runtime);
+  }, REPORT_DELIVERY_INTERVAL_MS);
+
+  let stopping = false;
+  // The wake surface (#380): `resume` POSTs /wake, `stop` interrupts the same
+  // sleep. A wake request is sticky and coalescing — several resumes in quick
+  // succession produce one prompt pass, never an overlapping one.
+  const stop = (): void => {
+    if (stopping) return;
+    stopping = true;
+    // Close the admission fence before the loop is told, so a tick already in
+    // flight — parked at an await when the signal landed — sees `draining`
+    // when it resumes and cannot create the work this shutdown is about to
+    // wait for (#374).
+    drain.draining = true;
+    log("shutdown requested — draining dispatch; live worker sessions are not waited for");
+    pace.requestWake();
+  };
+  process.on("SIGINT", stop);
+  process.on("SIGTERM", stop);
+
+  // This loopback surface exposes liveness (open) and bounded live-run controls
+  // (authenticated). Repository and tracker mutations stay on credentialled verb
+  // sockets, where project and role come from the channel rather than a payload.
+  //
+  // The mutating half used to be open, on the reasoning that binding 127.0.0.1
+  // is itself the boundary. It is not, on a shared host: every local process has
+  // the same loopback — including the worker sessions this daemon launches — so
+  // anything running here could pause, stop or re-ceiling any run, or wake
+  // dispatch in a loop. Phase 4 closes that with a bearer token minted here,
+  // before the server exists, so no request can ever be served against a token
+  // that had not been decided yet. `GET /healthz` deliberately stays open; the
+  // reason is at {@link daemonHttpResponse} and it is load-bearing for
+  // `requireDaemonControl`.
+  const httpToken = ensureHttpToken();
+  const server = Bun.serve({
+    hostname: "127.0.0.1",
+    port: o.port ?? DEFAULT_PORT,
+    fetch: (req) =>
+      daemonHttpResponse(req, {
+        token: httpToken,
+        projects: runtimes.map((runtime) => ({
+          project: runtime.d.project.name,
+          store,
+          caps: () => runtime.d.caps,
+          // The same project prefix the runtime's own journal lines carry, so
+          // a pause audit reads like every other daemon line (#997).
+          log: (line: string) =>
+            log(runtimes.length === 1 ? line : `[${runtime.d.project.name}] ${line}`),
+        })),
+        turnLimits,
+        workerControls,
+        wake: () => {
+          log("dispatch wake requested — an immediate pass is starting");
+          pace.requestWake();
+        },
+        health: () =>
+          daemonHealth(
+            runtimes.map((runtime) => {
+              const orch = runtime.orchestrator;
+              const mode = runtime.d.project.escalation.orchestrator;
+              return daemonHealthSnapshot(
+                store,
+                runtime.d.project.name,
+                isPaused(runtime.d.project.name),
+                runtime.codeGraph,
+                workerControls,
+                orch === undefined
+                  ? // External mode is the pane's session: the daemon hosts no
+                    // child to attest. An embedded orchestrator that failed to
+                    // start is a live outage of the surface the upgrade must
+                    // promise reloaded (#832) — say so out loud.
+                    { mode: mode === "external" ? "external" : "failed" }
+                  : {
+                      mode: "embedded",
+                      ...(orch.extensionVersion() === undefined
+                        ? {}
+                        : { loaded: orch.extensionVersion() }),
+                      ...(orch.sessionFile() === undefined ? {} : { sessionFile: orch.sessionFile() }),
+                      alive: orch.alive(),
+                    },
+              );
+            }),
+          ),
+      }),
+  });
+  log(
+    projects.length === 1
+      ? `serving /healthz on :${server.port}, project ${projects[0]!.name}`
+      : `serving /healthz on :${server.port}, projects ${projects.map(({ name }) => name).join(", ")}`,
+  );
+
+  try {
+    await runDispatchLoop({
+      tick: async () => {
+        for (const runtime of runtimes) {
+          try {
+            await tick(runtime.d, workers);
+          } catch (err) {
+            log(
+              `${projects.length === 1 ? "" : `[${runtime.d.project.name}] `}` +
+                `tick failed: ${errText(err)}`,
+            );
+          }
+        }
+      },
+      shouldStop: () => stopping,
+      pace,
+      tickIntervalMs: TICK_INTERVAL_MS,
+    });
+  } finally {
+    process.off("SIGINT", stop);
+    process.off("SIGTERM", stop);
+    clearInterval(graphTimer);
+    clearInterval(reportTimer);
+    for (const runtime of runtimes) await runtime.reportPass;
+    await workers.drain();
+    await server.stop(true);
+    for (const runtime of runtimes.toReversed()) {
+      await runtime.orchestrator?.dispose();
+      await runtime.orchestratorVerbs?.close();
+    }
+    store.close();
+    log("stopped");
+    process.exitCode = 0;
+  }
+}