omp-conductor 0.15.12 → 0.16.0

package/src/daemon.ts

@@ -8,7 +8,7 @@
  * here and enforced before anything is claimed.
  */
 import { createHash } from "node:crypto";
-import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { dirname, join, relative } from "node:path";
 import {
   configPath,
@@ -39,10 +39,15 @@ import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgr
 import { fleetLayers } from "./fleet.ts";
 import { startOrchestrator } from "./orchestrator.ts";
 import type { OrchestratorHandle } from "./orchestrator.ts";
+import {
+  formatOrchestratorDown,
+  reconcileOrchestratorDown,
+} from "./orchestrator-down.ts";
 import {
   createReportOutbox,
   enqueueAvailableHeldNotices,
   formatOpenReports,
+  reliabilitySettlementLine,
   type ReportOutbox,
 } from "./reports.ts";
 import {
@@ -52,6 +57,10 @@ import {
 } from "./release-policy.ts";
 import { branchName, effectiveLabels, route } from "./routing.ts";
 import type { Routed, UnroutableReason } from "./routing.ts";
+import { admitCandidates, hasContinuationBudget, hasFailedAttemptBudget } from "./admission.ts";
+import type { Admission, AdmissionHold } from "./admission.ts";
+import { log, errText, safeEscalate } from "./log.ts";
+import { materializeOmpSettings } from "./omp-settings.ts";
 import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
 import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
 import {
@@ -67,6 +76,7 @@ import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
 import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
 import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
 import type {
+  BaseFreeze,
   BaseHealth,
   AdmissionHoldReason,
   Caps,
@@ -78,6 +88,7 @@ import type {
   MergedPrInfo,
   OpenCloser,
   ReleaseShape,
+  OrchestratorIncident,
   PrState,
   ProjectConfig,
   ReadyIssue,
@@ -100,6 +111,7 @@ import {
   type WorkerPausePhase,
   type WorkerResult,
   type RunWorkerDeps,
+  ORPHAN_RESUME_PROMPT,
   renderBrief,
   runWorker,
 } from "./worker.ts";
@@ -134,10 +146,12 @@ import { homedir } from "node:os";
 
 import {
   probeCriticalBase,
+  probeRunLane,
   pushRunBranch,
   readBaseChain,
   type CriticalBaseProbe,
   type CriticalBaseVerdict,
+  type RunLaneProbe,
   type RunRepoRef,
 } from "./gitops.ts";
 import {
@@ -166,6 +180,13 @@ const DISPATCH_INFRA_MAX_STRIKES = 3;
  *  provider itself is degraded, not unlucky, and the sweep escalates to a
  *  human instead of requeueing into a down provider forever (#220). */
 const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
+/** A provider-capacity requeue (sustained in-session rate limiting) is retried,
+ *  but only a bounded number of times: three throttled runs for one issue mean
+ *  the provider is at capacity, not unlucky, and the sweep escalates to a human
+ *  instead of requeueing into a throttled provider forever (#573). The issue's
+ *  own chain moves onto its next model per strike (via {@link FAILOVER_CLASSES}),
+ *  so a bounded chain is exhaustible; this caps the unbounded no-chain case. */
+const PROVIDER_CAPACITY_MAX_STRIKES = 3;
 /** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
  *  are maintenance, but a backlog must not turn one tick into an API burst. */
 const SALVAGED_PR_ADOPTION_BATCH = 10;
@@ -227,6 +248,13 @@ interface Deps {
   workerDeps?: RunWorkerDeps;
   integrity: IntegrityGate;
   stall: StallGate;
+  /**
+   * The embedded orchestrator session handle when one started; absent when it
+   * failed to start or the project uses an external orchestrator. Feeds the
+   * orchestrator-down reconcile ({@link reconcileOrchestratorDown}) so a
+   * crashed session pages once per incident instead of degrading quietly.
+   */
+  orchestrator?: OrchestratorHandle;
   cleanup?: RetainedCleanupCursor;
   /**
    * Reads the connecting uid off a verb socket (#126). Resolved once at startup
@@ -258,6 +286,16 @@ interface Deps {
    * a marker (a safety interlock must not silently weaken).
    */
   probeCriticalBase?: CriticalBaseProbe;
+  /**
+   * Reads one active run's file lane for the admission file-lane interlock
+   * (#555): the union of its uncommitted worktree changes and its branch-vs-base
+   * diff. Wired by `runDaemon` to the mirror/worktree-backed
+   * {@link probeRunLane}; a test injects a fake. Absent, the interlock is inert
+   * (no lane is ever known occupied), which is the issue's "fail open": the
+   * gate adds holds, it never refuses a well-formed issue for lack of this
+   * probe the way `criticalBase` does.
+   */
+  probeWorktreeLane?: RunLaneProbe;
 }
 
 /**
@@ -531,6 +569,19 @@ export function pauseInstance(
   }
 }
 
+/**
+ * The pause sentinel's `source=` token, proven from a verb. The sentinel's
+ * source line is read back as a single `\S+` token (see {@link pauseInstance}
+ * and {@link pauseProvenance}), so a verb that contains a space (`setup host`)
+ * is unrepresentable verbatim and must be encoded before it reaches disk —
+ * otherwise the fence cannot prove its own pause and refuses forever (#552).
+ * Spaces become `-`; the human-readable verb is preserved in the sentinel's
+ * `reason=` instead.
+ */
+export function pauseSourceToken(verb: string): string {
+  return verb.trim().replace(/\s+/g, "-");
+}
+
 export function setPaused(
   v: boolean,
   why?: { source: string; reason?: string },
@@ -655,14 +706,6 @@ export function markPaged(
 
 // ---------------------------------------------------------------------- helpers
 
-function log(msg: string): void {
-  process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`);
-}
-
-function errText(e: unknown): string {
-  return e instanceof Error ? (e.stack ?? e.message) : String(e);
-}
-
 /**
  * Local midnight, matching how a human reads "today".
  *
@@ -815,15 +858,6 @@ export function recordOperatorStop(
  * gate on the attempt turns one failed delivery into permanent silence about a
  * condition that is still true.
  */
-async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<boolean> {
-  try {
-    await d.escalate(e);
-    return true;
-  } catch (err) {
-    log(`escalation for ${escalationIssueRef(e.issue)} could not be delivered: ${errText(err)}`);
-    return false;
-  }
-}
 
 async function reactToProviderCredit(
   d: Deps,
@@ -1082,18 +1116,6 @@ export async function buildBrief(
 
 // ------------------------------------------------------------------- one issue
 
-/** `stops` are the operational ends that each require one resume. */
-export function hasContinuationBudget(stops: number, maxContinuations: number): boolean {
-  return stops <= maxContinuations;
-}
-
-/** True while unspent failed-implementation attempts remain. This is the
- *  dispatcher's admission gate: once every `maxAttemptsPerIssue` slot is
- *  spent, the issue is held as `failed-attempts` forever, and the `unblock`
- *  verb withholds the queue label on the same predicate (#348). */
-export function hasFailedAttemptBudget(failures: number, maxAttempts: number): boolean {
-  return failures < maxAttempts;
-}
 
 /** The failure classes `countContinuations` deliberately does not charge — the
  *  inverted copy of its exclusions, kept beside the breakdown that consumes it
@@ -1107,6 +1129,7 @@ const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
   "dispatch-infra": true,
   "provider-credit": true,
   "provider-transient": true,
+  "provider-capacity": true,
 };
 
 /** How one issue spent its continuation budget, grouped by failure class —
@@ -1607,6 +1630,50 @@ export async function inheritedPrForContinuation(
   return { prUrl: prior.prUrl, ...(prior.headSha === undefined ? {} : { headSha: prior.headSha }) };
 }
 
+/**
+ * #536: whether an orphan-clean requeue may resume the interrupted session
+ * instead of dispatching fresh.
+ *
+ * `reconcileOrphanedRuns` keeps the worktree (salvage commit included) and the
+ * transcript is file-backed, so a daemon restart can hand the next attempt
+ * back its own memory: the same worktree, the same session directory, and
+ * `resume: true` at the harness. The old dispatch built a fresh
+ * `run-<uuid>` session and re-read the repo from zero — the exact rediscovery
+ * orphan-clean spent turns on in #535.
+ *
+ * Only `orphan-clean` resumes. A cap-killed or otherwise failed worker was
+ * killed for cause, and orphan-dirty is held precisely because the worktree is
+ * the only copy. The checks here are what the daemon can prove cheaply before
+ * the claim (the transcript is present and non-empty, the worktree is
+ * present); the harness's own `continueRecent` is the backstop, and a corrupt
+ * transcript it silently falls back from is surfaced loudly by the
+ * `sessionFile` lineage compare at the dispatch site rather than left quiet.
+ */
+function orphanResumeVerdict(
+  prior: RunRecord | undefined,
+): { kind: "resume"; prior: RunRecord } | { kind: "fresh"; reason?: string } {
+  if (prior === undefined || prior.state !== "orphaned" || prior.failureClass !== "orphan-clean") {
+    return { kind: "fresh" };
+  }
+  if (prior.sessionFile === undefined) {
+    return { kind: "fresh", reason: "the orphaned attempt recorded no transcript" };
+  }
+  try {
+    if (!existsSync(prior.sessionFile)) {
+      return { kind: "fresh", reason: `transcript ${prior.sessionFile} is gone` };
+    }
+    if (statSync(prior.sessionFile).size === 0) {
+      return { kind: "fresh", reason: `transcript ${prior.sessionFile} is empty` };
+    }
+  } catch (err) {
+    return { kind: "fresh", reason: `transcript ${prior.sessionFile} is unreadable (${errText(err)})` };
+  }
+  if (prior.worktree === "" || !existsSync(prior.worktree)) {
+    return { kind: "fresh", reason: `worktree ${prior.worktree} is gone` };
+  }
+  return { kind: "resume", prior };
+}
+
 export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
   const { project, caps, tracker, store } = d;
   const issue = r.issue.number;
@@ -1770,6 +1837,49 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     // fresh issue always starts on the primary (#286). Read before the claim
     // writes this attempt's row, which would otherwise break the streak.
     const chainFacts = providerFailureFacts(store.runsForIssue(project.name, issue));
+    // Resolved before the claim, not after provisioning: the model is also
+    // part of the #536 resume decision, which has to be made before the
+    // dispatch shape (fresh provision vs kept worktree) is chosen. With no
+    // `modelFallbacks` configured this is the primary model — or none, for an
+    // unconfigured project — and today's dispatch is byte for byte what it
+    // has always been.
+    const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
+    const choice = resolveDispatchModel({
+      workerModel: project.workerModel,
+      modelFallbacks: project.modelFallbacks,
+      threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
+      streak: chainFacts.streak,
+    });
+    const clause = fallbackClause(choice, chainFacts, project.workerModel);
+
+    // #536: an orphan-clean requeue resumes the interrupted session instead of
+    // re-reading the repo from zero. Decided here, before this attempt's row
+    // exists, because `prior` is still the attempt whose work this one
+    // inherits and the whole dispatch shape follows the verdict.
+    let resuming: RunRecord | undefined;
+    const verdict = orphanResumeVerdict(prior);
+    if (verdict.kind === "resume") {
+      const resumePrior = verdict.prior;
+      // The continuation must stay on the model the interrupted session was
+      // using; a chain that now resolves differently dispatches fresh rather
+      // than quietly continuing on another model, which would smear one
+      // attempt's work across two models (#286 attribution).
+      if (resumePrior.model !== undefined && choice.model !== undefined && resumePrior.model !== choice.model) {
+        log(
+          `#${issue} attempt ${attempt} not resumed: attempt ${resumePrior.attempt} ran on ${resumePrior.model} but dispatch ` +
+            `now resolves ${choice.model} — fresh dispatch`,
+        );
+      } else {
+        resuming = resumePrior;
+        log(
+          `#${issue} attempt ${attempt} continuing session of attempt ${resuming.attempt} → ` +
+            `transcript ${resuming.sessionFile}, worktree kept`,
+        );
+      }
+    } else if (verdict.reason !== undefined) {
+      log(`#${issue} attempt ${attempt} not resumed: ${verdict.reason} — fresh dispatch`);
+    }
+
     run = store.createRun({
       project: project.name,
       issue,
@@ -1822,22 +1932,30 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
 
     // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
     // an existing path, so a retry — or a tree kept from a failed attempt — has
-    // to be cleared first. Both helpers are pure path math and removeWorktree
-    // tolerates a mirror or tree that is not there yet, so this is safe on a
-    // first attempt. addRunRepo does its own ensureMirror; calling it here too
-    // would cost a second network fetch per attempt.
-    await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
-    if (await settleStopBeforeSession()) return;
-    if (await settleDrainBeforeSession()) return;
-    const provisioned = await addRunRepo(
-      r.repo,
-      project.mirrorRoot,
-      project.workspaceRoot,
-      issue,
-      branch,
-    );
-    worktreePath = provisioned.path;
-    runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
+    // to be cleared first. Both helpers are pure path math, and removeWorktree
+    // tolerates a mirror or tree that is not there yet. An attempted resume
+    // skips the whole dance: the orphaned run's tree is the work to continue
+    // (its salvage commit is already on the branch), and re-cloning it from
+    // the mirror would be exactly the rediscovery this feature exists to skip.
+    let provisioned: Awaited<ReturnType<typeof addRunRepo>> | undefined;
+    if (resuming !== undefined) {
+      worktreePath = resuming.worktree;
+      runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
+    } else {
+      await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
+      if (await settleStopBeforeSession()) return;
+      if (await settleDrainBeforeSession()) return;
+      const provisionedTree = await addRunRepo(
+        r.repo,
+        project.mirrorRoot,
+        project.workspaceRoot,
+        issue,
+        branch,
+      );
+      provisioned = provisionedTree;
+      worktreePath = provisionedTree.path;
+      runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
+    }
     if (await settleStopBeforeSession()) return;
     if (await settleDrainBeforeSession()) return;
 
@@ -1846,11 +1964,27 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     // here would put a file that never gets written into an escalation.
     //
     // Per run rather than one shared directory, so one run's transcript cannot
-    // be truncated or replaced by the next.
+    // be truncated or replaced by the next — except for a resumed attempt,
+    // which deliberately reuses the interrupted session's directory so the
+    // SDK's `continueRecent(cwd, dir)` picks up that transcript and keeps
+    // writing it.
     const runTreeRoot = stateDir();
-    const sessionDir = join(runTreeRoot, "sessions", `run-${String(runId)}`);
+    const sessionDir =
+      resuming === undefined
+        ? join(runTreeRoot, "sessions", `run-${String(runId)}`)
+        : dirname(resuming.sessionFile!);
     mkdirSync(sessionDir, { recursive: true });
 
+    // The fleet-owned omp settings overlay (#537): the project's `ompSettings`
+    // map (plus the retry keys derived from `modelFallbacks`, #539's staging
+    // half) materialised to YAML under the run's session directory — never
+    // inside the worktree, whose diff is the PR a worker ships. Rewritten on
+    // every attempt, so a config edit takes effect on the next dispatch and a
+    // resumed attempt reuses the kept session dir with the *current* config.
+    // Absent `ompSettings` and `modelFallbacks`, no file is written, nothing
+    // is passed, and dispatch is byte-for-byte today's.
+    const ompSettingsFile = materializeOmpSettings(project, sessionDir);
+
     // ---- the run's mutation channel (#126) -------------------------------
     // A shared, daemon-owned 0711 parent with one 0600 socket per run, never a
     // per-run *directory*: a directory owned by the run principal would hand
@@ -1884,50 +2018,51 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     store.updateRun(runId, { worktree: worktreePath, state: "running" });
 
     // Where this attempt goes, and (when the failover fired) the clause that
-    // makes it attributable: with no `modelFallbacks` configured this is the
-    // primary model — or none, for an unconfigured project — and today's
-    // dispatch is byte for byte what it has always been.
-    const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
-    const choice = resolveDispatchModel({
-      workerModel: project.workerModel,
-      modelFallbacks: project.modelFallbacks,
-      threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
-      streak: chainFacts.streak,
-    });
+    // makes it attributable. `chainConfigured`/`choice`/`clause` were resolved
+    // before the claim — the resume verdict had to be made before the dispatch
+    // shape was chosen (#536) — so only the record write lives here.
     // Recorded before the launch, so even a run killed mid-flight leaves the
     // model it chose on its row. Only a chain-configured project writes the
     // column: absent `modelFallbacks` must preserve today's rows byte for byte.
     if (chainConfigured && choice.model !== undefined) {
       store.updateRun(runId, { model: choice.model });
     }
-    const clause = fallbackClause(choice, chainFacts, project.workerModel);
 
     log(
       `#${issue} attempt ${attempt}${clause === undefined ? "" : ` ${clause}`} → ${r.repo.name} ${branch}` +
-        (provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
+        (provisioned?.reattached ? " (continuation: reattached existing branch)" : ""),
     );
 
-    // The discussion is rendered at dispatch so a worker never depends on a
-    // runtime `gh` read to see the orchestrator's grooming (#517). The read is
-    // best-effort, but its failure is not silent: an unreadable tracker names
-    // itself in the brief's Discussion section instead of reading as "no
-    // comments" — the exact confusion this fix removes.
-    let comments: IssueComment[] | "unread";
-    try {
-      comments = await tracker.listComments(issue);
-    } catch (err) {
-      log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
-      comments = "unread";
-    }
+    // The continuation notice replaces the brief for a resumed attempt (#536).
+    // The original brief is already in the resumed transcript; re-sending it is
+    // how a resumed worker ends up re-doing the work it just did. Everything
+    // below the brief is fresh-dispatch-only, exactly as today.
+    let brief: string;
+    if (resuming !== undefined) {
+      brief = ORPHAN_RESUME_PROMPT;
+    } else {
+      // The discussion is rendered at dispatch so a worker never depends on a
+      // runtime `gh` read to see the orchestrator's grooming (#517). The read is
+      // best-effort, but its failure is not silent: an unreadable tracker names
+      // itself in the brief's Discussion section instead of reading as "no
+      // comments" — the exact confusion this fix removes.
+      let comments: IssueComment[] | "unread";
+      try {
+        comments = await tracker.listComments(issue);
+      } catch (err) {
+        log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
+        comments = "unread";
+      }
 
-    const brief = await buildBrief(project, r, branch, worktreePath, {
-      continuation: provisioned.reattached,
-      defaultBranch: r.repo.defaultBranch,
-      ...(provisioned.reattached && priorSalvage !== undefined
-        ? { salvagedSha: priorSalvage }
-        : {}),
-      comments,
-    });
+      brief = await buildBrief(project, r, branch, worktreePath, {
+        continuation: provisioned?.reattached === true,
+        defaultBranch: r.repo.defaultBranch,
+        ...(provisioned?.reattached === true && priorSalvage !== undefined
+          ? { salvagedSha: priorSalvage }
+          : {}),
+        comments,
+      });
+    }
     if (await settleStopBeforeSession()) return;
     if (await settleDrainBeforeSession()) return;
 
@@ -1946,6 +2081,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
           workerControl?.install(control);
         },
         sessionDir,
+        ...(resuming === undefined ? {} : { resume: true }),
         // The session's control socket, under the daemon's own state directory —
         // a child process of the daemon reaches it directly.
         socketPath: join(sessionDir, "ipc.sock"),
@@ -1962,6 +2098,12 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
           log(`#${issue} ${line}`);
         },
         ...(choice.model === undefined ? {} : { model: choice.model }),
+        // The fleet-owned omp settings overlay (#537): the staged YAML the
+        // session loads through `Settings.init({ configFiles: [<path>] })` —
+        // the project's `ompSettings` map plus the within-run failover keys
+        // #581 staged directly (the project's own chain, not an empty default).
+        // Absent both, nothing is staged, today's dispatch byte for byte.
+        ...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
         releaseGrants: resolveReleaseGrants(project),
         onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
         onTurn: (n) => store.updateRun(runId, { turns: n }),
@@ -1974,7 +2116,21 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
         // ends: `omp-conductor tail` resolves an issue to a file through this row,
         // and a path written at completion is a path nobody can follow live. The
         // completion-time update below writes the same value again, harmlessly.
-        onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
+        onSessionFile: (f) => {
+          store.updateRun(runId, { sessionFile: f });
+          // #536: the harness's own `continueRecent` can still fall back to a
+          // blank session (corrupt transcript, nothing to continue) — and a
+          // blank session in the "resumed" worktree is indistinguishable from
+          // today's dispatch unless the downgrade is named. Same-file lineage
+          // is the proof the resume happened: the resumed run must keep
+          // writing the orphaned attempt's transcript.
+          if (resuming !== undefined && f !== resuming.sessionFile) {
+            log(
+              `#${issue} attempt ${attempt} resume fell back to a fresh session: opened ${f} ` +
+                `instead of the orphaned attempt's ${resuming.sessionFile}`,
+            );
+          }
+        },
         // The last fence (#374): every pre-launch settle check above has
         // passed, but the stop can still land while the session socket is
         // binding inside `createSession`. This gate is re-checked there,
@@ -2034,10 +2190,27 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     // worker's, the disclosure becomes the diff's (#488). A diff that could
     // not be read leaves the worker's text untouched and the audit's
     // `changed-line-missing` flag says so.
-    const settlementReport =
+    // The within-run reliability sentence, appended where a human reads it
+    // (#584): which model the run finished on and whether it swapped
+    // mid-flight. Empty (undefined) for a clean run, so a run that never
+    // retried, swapped or compacted keeps today's settlement report byte for
+    // byte — the "additive" claim asserted rather than assumed.
+    const reliabilityLine = reliabilitySettlementLine({
+      resolvedModel: result.model,
+      resolvedProvider: result.provider,
+      retryFallbacks: result.retryFallbacks,
+      retryFallbackSucceeded: result.retryFallbackSucceeded,
+      modelRecoveries: result.modelRecoveries,
+      autoRetryCount: result.autoRetryCount,
+      autoCompactionCount: result.autoCompactionCount,
+    });
+
+    const settlementReport = [
       audit?.changedLine === undefined
         ? result.report
-        : withDerivedChangedLine(result.report, audit.changedLine);
+        : withDerivedChangedLine(result.report, audit.changedLine),
+      ...(reliabilityLine === undefined ? [] : ["", reliabilityLine]),
+    ].join("\n");
 
     const finalReport = [
       ...(verified.reason === undefined ? [] : [verified.reason, ""]),
@@ -2087,6 +2260,24 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
       endedAt: Date.now(),
       turns: result.turns,
       spendUsd: result.spendUsd,
+      // The count of in-session provider 429s the worker metered live, so a
+      // run the provider throttled into the ground carries its own diagnosis
+      // instead of landing `unknown` — the classifier reads it straight off
+      // this column (#573).
+      provider429Count: result.provider429Count,
+      // The within-run harness reliability surface #581 collected, persisted
+      // now that settlement owns the row (#584). The resolved model/provider
+      // only travel when some assistant message carried them (the run recorded
+      // no model, or the worker never established one); the count fields always
+      // travel, 0 for a clean run, so an absent column can never be read as a
+      // quiet fleet. Written for every terminal state, clean or not.
+      ...(result.model === undefined ? {} : { resolvedModel: result.model }),
+      ...(result.provider === undefined ? {} : { resolvedProvider: result.provider }),
+      retryFallbacks: result.retryFallbacks,
+      retryFallbackSucceeded: result.retryFallbackSucceeded,
+      modelRecoveries: result.modelRecoveries,
+      autoRetryCount: result.autoRetryCount,
+      autoCompactionCount: result.autoCompactionCount,
       // The worker only reports these when it actually established them; a kill
       // or a settle whose report named no PR must not wipe what a verb recorded
       // earlier in the same run (#468). The sink in `updateRun` skips undefined
@@ -2129,6 +2320,22 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
 
     const salvaged = settlement?.lines ?? [];
 
+    // The run's reliability news, surfaced where the tick digest reads it
+    // (#584): the digest is model-authored but consumes the store's material
+    // ledger, so a run that swapped mid-flight or rode out a throttled
+    // provider lands one event the digest can name. Clean runs record nothing
+    // here, so a quiet fleet's digest is unchanged.
+    if (reliabilityLine !== undefined) {
+      store.recordMaterialEvent({
+        project: project.name,
+        category: "reliability",
+        summary: `#${issue} ${reliabilityLine}`,
+        evidence: `${r.issue.title}\n${r.issue.url}\n\n${reliabilityLine}`,
+        occurredAt: Date.now(),
+        recordedAt: Date.now(),
+      });
+    }
+
     if (state === "stopped") {
       log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
     } else if (state === "blocked") {
@@ -2550,6 +2757,31 @@ export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "sto
       `${failed.name} failed at ${run.mergeSha} — ${failed.url}` +
       (preexisting ? " (already red before this merge)" : "");
     const flag: SettlementFlag = { kind: "base-branch-red", file: "(base branch)", detail };
+    // Freeze merges to this repo while the base it merged into is red (#283).
+    // The freeze is repo-scoped and sets independently of escalation delivery:
+    // a merge that broke the base must not be followed by another merge onto
+    // the same red base, even if paging the operator fails.
+    if (
+      d.store.setBaseFreeze(d.project.name, {
+        repo: run.repo,
+        culpritSha: run.mergeSha,
+        detail,
+        setAt: now,
+      })
+    ) {
+      d.store.recordMaterialEvent({
+        project: d.project.name,
+        category: "base-red-freeze",
+        summary: `merges to ${run.repo} frozen — base ${run.baseRef} red at ${run.mergeSha.slice(0, 8)}`,
+        evidence:
+          `${detail} This freeze names ${run.mergeSha.slice(0, 8)} as the suspected culprit merge. ` +
+          "Merges to this repo are refused until the base is green again; reverting the culprit is the " +
+          "likely remedy. The freeze lifts automatically on a green re-observation, or the operator can " +
+          "override it with `omp-conductor unfreeze <repo>`.",
+        occurredAt: now,
+        recordedAt: now,
+      });
+    }
     const delivered = await safeEscalate(d, {
       tier: 1,
       project: d.project.name,
@@ -2608,7 +2840,16 @@ export async function watchBaseHealth(
     }
 
     const previous = previousByRepo.get(repo);
+    const freeze = d.store.baseFreeze(d.project.name, repo);
+    const frozen = freeze !== undefined && freeze.clearedAt === undefined;
+    // The same-head/age shortcut exists to avoid re-querying GitHub when a
+    // terminal verdict has not moved. It must NOT skip a frozen repo: a freeze
+    // keyed on one red observation has to keep re-evaluating the same head so a
+    // green rerun clears it without operator action (tonight's evidence — a
+    // red/unknown read two minutes after the same SHA's CI succeeded — is why
+    // it cannot be trusted as terminal).
     if (
+      !frozen &&
       previous?.branch === branch &&
       previous.headSha === head &&
       (previous.verdict === "green" || previous.verdict === "red")
@@ -2668,6 +2909,43 @@ export async function watchBaseHealth(
     };
     d.store.upsertBaseHealth(d.project.name, health);
     previousByRepo.set(repo, health);
+
+    // The freeze follows the live base verdict: red arms (or re-arms) the
+    // repo-scoped freeze, green lifts it. pending/unknown leave it untouched —
+    // a stale or in-flight reading must neither create a freeze nor clear one.
+    if (verdict === "green") {
+      if (d.store.clearBaseFreeze(d.project.name, repo, "daemon", "base-green", now)) {
+        d.store.recordMaterialEvent({
+          project: d.project.name,
+          category: "base-recovered",
+          summary: `merges to ${repo} unfrozen — base ${branch} observed green at ${head.slice(0, 8)}`,
+          evidence: `The base-red freeze on ${repo} lifted automatically: ${branch} is green at ${head.slice(0, 8)}. Merges resume.`,
+          occurredAt: now,
+          recordedAt: now,
+        });
+      }
+    } else if (verdict === "red") {
+      if (
+        d.store.setBaseFreeze(d.project.name, {
+          repo,
+          culpritSha: head,
+          detail,
+          setAt: now,
+        })
+      ) {
+        d.store.recordMaterialEvent({
+          project: d.project.name,
+          category: "base-red-freeze",
+          summary: `merges to ${repo} frozen — base ${branch} red at ${head.slice(0, 8)}`,
+          evidence:
+            `${detail ?? `workflow failed at ${head.slice(0, 8)}`} This freeze names ` +
+            `${head.slice(0, 8)} as the suspected culprit. Reverting it is the likely remedy; the freeze ` +
+            "lifts automatically on green or with `omp-conductor unfreeze <repo>`.",
+          occurredAt: now,
+          recordedAt: now,
+        });
+      }
+    }
   }
 }
 
@@ -2987,22 +3265,6 @@ export async function cleanupRetainedRuns(
 
 // -------------------------------------------------------------------- admission
 
-/** A candidate cleared for dispatch, with the attempt number it will run as. */
-export interface Admission {
-  r: Routed;
-  attempt: number;
-}
-
-export interface AdmissionHold {
-  issue: number;
-  reason: AdmissionHoldReason;
-}
-
-export interface AdmissionPass {
-  admitted: Admission[];
-  holds: AdmissionHold[];
-}
-
 const HOLD_SAMPLE_SIZE = 5;
 const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
   "parent-lookup-error",
@@ -3020,11 +3282,16 @@ export function summarizeDispatch(
   completedAt = Date.now(),
   settled = 0,
 ): DispatchSummary {
-  const groups = new Map<AdmissionHoldReason, { count: number; issues: number[] }>();
+  const groups = new Map<AdmissionHoldReason, { count: number; issues: number[]; details: string[] }>();
   for (const hold of holds) {
-    const group = groups.get(hold.reason) ?? { count: 0, issues: [] };
+    const group = groups.get(hold.reason) ?? { count: 0, issues: [], details: [] };
     group.count += 1;
-    if (group.issues.length < HOLD_SAMPLE_SIZE) group.issues.push(hold.issue);
+    if (group.issues.length < HOLD_SAMPLE_SIZE) {
+      // `issues` and `details` share the sample: the detail is only kept when
+      // the issue it explains is, so the arrays stay index-aligned.
+      group.issues.push(hold.issue);
+      if (hold.detail !== undefined) group.details.push(hold.detail);
+    }
     groups.set(hold.reason, group);
   }
   return {
@@ -3036,7 +3303,12 @@ export function summarizeDispatch(
     degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
     holds: [...groups]
       .sort(([a], [b]) => a.localeCompare(b))
-      .map(([reason, group]) => ({ reason, ...group })),
+      .map(([reason, group]) => ({
+        reason,
+        count: group.count,
+        issues: group.issues,
+        ...(group.details.length === 0 ? {} : { details: group.details }),
+      })),
     settled,
   };
 }
@@ -3061,478 +3333,6 @@ export function summarizeHeldPass(settled: number, completedAt = Date.now()): Di
   };
 }
 
-/**
- * What a held plan-usage gate says to a human, if anything.
- *
- * Three different problems hide behind one hold, and they want different
- * tiers. Reaching the threshold is the guard *working*: tier 1, because the
- * fleet resumes on its own at the provider's reset and nobody needs to get
- * out of bed. Everything else — a window nothing reports, a window that
- * resolves to two allowances, a meter that has been unreadable for half an
- * hour — is dispatch stopped with no self-recovery, which is tier 2.
- *
- * Each summary carries the fact that will change when the situation does (the
- * reset instant, the configured id, the date), because the escalation ledger
- * dedupes on the summary: a stable one pages once and then goes quiet, which
- * is right for a repeated tick and wrong for the next window.
- */
-function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation | undefined {
-  const base = { project, issue: NO_ISSUE };
-  if (plan.state === "at-cap") {
-    const window = plan.window?.id ?? plan.cap?.windowId ?? "the configured window";
-    const resets =
-      plan.resetsAt === undefined
-        ? new Date().toISOString().slice(0, 10)
-        : new Date(plan.resetsAt).toISOString();
-    return {
-      ...base,
-      tier: 1,
-      summary: `Plan allowance cap reached on ${window} — ${project} is not claiming new work (window ${resets})`,
-      detail: [
-        plan.detail,
-        "Running workers finish normally; only new claims are held.",
-        "Dispatch resumes by itself once the provider reports the window reset or usage below the threshold —",
-        "no `resume` needed. Raise `caps.planUsage.maxUsedFraction` only if you mean to spend the rest.",
-      ].join("\n"),
-    };
-  }
-  if (plan.state === "blind") {
-    return {
-      ...base,
-      tier: 2,
-      category: "fleet-stopped",
-      // Dated: a meter that breaks again next month is a new incident, not a
-      // repeat of this one.
-      summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
-      detail: [
-        plan.detail,
-        "The guard admitted work while the failure looked transient and has now stopped.",
-        "Check `omp usage --json` on the fleet host, or set `caps.planUsage` to null if this fleet is unmetered.",
-      ].join("\n"),
-    };
-  }
-  if (
-    plan.state === "window-missing" ||
-    plan.state === "window-ambiguous" ||
-    plan.state === "window-uncomparable"
-  ) {
-    return {
-      ...base,
-      tier: 2,
-      category: "fleet-stopped",
-      summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
-      detail: [
-        plan.detail,
-        "Run `omp usage --json` and copy an allowance `id` into `caps.planUsage.windowId`,",
-        "or set `caps.planUsage` to null if this fleet is unmetered.",
-      ].join("\n"),
-    };
-  }
-  return undefined;
-}
-
-/**
- * Which routed candidates get a worker this tick — in queue order, never more
- * than `slots` of them. Every non-admission receives a stable reason code.
- *
- * Exported so the admission rules can be pinned without spawning a worker.
- * Every one of them exists because of a live incident, and each guards a
- * different way the same issue gets worked twice — including epic siblings
- * racing onto the same files (#48).
- *
- * Takes the slice of `Deps` it actually reads rather than the whole thing: what
- * admission is allowed to consult is the point of the function, and a `Deps`
- * that grows a field has no business breaking these tests.
- */
-export async function admitCandidates(
-  d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage" | "probeCriticalBase">,
-  routed: Routed[],
-  slots: number,
-): Promise<AdmissionPass> {
-  const { project, caps, tracker, store } = d;
-  const activeRuns = store.activeRuns(project.name);
-  const busyIssues = activeRuns.map((r) => r.issue);
-  const busy = new Set(busyIssues);
-  // issue -> its active run rows, for the pushed-green admission bypass (#175):
-  // only a worker-free pushed-green row may be bypassed, and only when *every*
-  // active run for the issue is worker-free. A live (claimed/running) row still
-  // holds unconditionally.
-  const activeByIssue = new Map<number, RunRecord[]>();
-  for (const run of activeRuns) {
-    const list = activeByIssue.get(run.issue);
-    if (list === undefined) activeByIssue.set(run.issue, [run]);
-    else list.push(run);
-  }
-  // Live worker count per repo, seeded from live runs and incremented as this
-  // same pass admits — so two same-repo candidates can never both clear the
-  // per-repo cap in one tick (#186).
-  const liveByRepo = new Map<string, number>();
-  for (const run of store.liveRuns(project.name)) {
-    liveByRepo.set(run.repo, (liveByRepo.get(run.repo) ?? 0) + 1);
-  }
-  const holds: AdmissionHold[] = [];
-  const hold = (issue: number, reason: AdmissionHoldReason): void => {
-    holds.push({ issue, reason });
-  };
-
-  // The plan allowance is a fleet-wide question, so it is asked once per pass
-  // and answers for every candidate — unlike every gate below it, which is
-  // per-issue. It sits here rather than beside the spend cap in `tick` for one
-  // reason: the spend cap *pauses the daemon* and waits for a human, and a
-  // weekly plan window resets by itself. A guard that demanded `resume` after
-  // every rollover would cost more operator attention than the guard saves
-  // (#110). Already-running workers are untouched and settle normally.
-  //
-  // Placed after the cheap local busy-set read and before the first tracker
-  // call, so a held fleet spends no GitHub API budget discovering it is held.
-  const plan = await readPlanUsage(caps.planUsage, d.usage);
-  if (plan.blocking) {
-    for (const r of routed) hold(r.issue.number, "plan-usage-cap");
-    log(`plan usage gate holding ${String(routed.length)} candidate(s): ${plan.detail}`);
-    const escalation = planUsageEscalation(project.name, plan);
-    if (escalation !== undefined) await safeEscalate(d, escalation);
-    return { admitted: [], holds };
-  }
-
-  // parent -> repo name -> blocking issue. Seeded from active runs (including
-  // pushed-green), then extended by candidates admitted earlier in this same
-  // pass so two siblings of one epic never both clear the gate in one tick.
-  // A busy issue whose run row cannot be resolved occupies the sentinel repo
-  // "" — treated as matching every repo, failing toward holding (#197).
-  const occupiedParents = new Map<number, Map<string, number>>();
-  const parentCache = new Map<number, number | undefined>();
-
-  const resolveParent = async (issue: number): Promise<number | undefined> => {
-    if (parentCache.has(issue)) return parentCache.get(issue);
-    const parent = await tracker.parentOf(issue);
-    parentCache.set(issue, parent);
-    return parent;
-  };
-
-  // Bounded by concurrent workers, not queue depth. A failed lookup here cannot
-  // mark an epic occupied; candidates still fail closed on their own parentOf.
-  for (const issue of busyIssues) {
-    try {
-      const parent = await resolveParent(issue);
-      if (parent === undefined) continue;
-      // The runs table records which repo each attempt worked in, and sibling
-      // holds are now per-repo, so a busy child only occupies its epic under
-      // that repo's name (same spelling as `createRun` writes from
-      // `r.repo.name`). A busy issue with no resolvable run row occupies the
-      // sentinel "" instead — matching every repo, failing toward holding.
-      const repo = store.latestRun(project.name, issue)?.repo ?? "";
-      const siblings = occupiedParents.get(parent);
-      if (siblings === undefined) {
-        occupiedParents.set(parent, new Map([[repo, issue]]));
-      } else if (!siblings.has(repo) && !siblings.has("")) {
-        siblings.set(repo, issue);
-      }
-    } catch (err) {
-      log(`#${issue} parent lookup failed while seeding epic occupancy (${errText(err)})`);
-    }
-  }
-
-  const admitted: Admission[] = [];
-  for (const r of routed) {
-    const issue = r.issue.number;
-    if (admitted.length >= slots) {
-      hold(issue, "capacity");
-      continue;
-    }
-    if (busy.has(issue)) {
-      // A pushed-green row is worker-free by definition (it is not in
-      // LIVE_STATES): its PR is live but no process is writing to its branch.
-      // So an issue whose active runs are ALL pushed-green is not actually
-      // occupied — the corrective attempt the operator unblocked may be
-      // admitted as a continuation of that PR, and the open-PR gate below
-      // decides the identity. Any live row still holds (#175).
-      const allWorkerFree = (activeByIssue.get(issue) ?? []).every((r) => r.state === "pushed-green");
-      if (!allWorkerFree) {
-        hold(issue, "issue-active");
-        continue;
-      }
-    }
-
-    // Per-repo concurrency: the mirror, branch-protection staleness and shared
-    // CI egress are all per-repo collision domains, so extra slots should land
-    // on other repos rather than stacking workers into the same one (#186).
-    const liveInRepo = liveByRepo.get(r.repo.name) ?? 0;
-    if (liveInRepo >= caps.maxConcurrentWorkersPerRepo) {
-      hold(issue, "repo-active");
-      log(`#${issue} skipped: ${liveInRepo} live worker(s) already in ${r.repo.name} (cap ${caps.maxConcurrentWorkersPerRepo})`);
-      continue;
-    }
-
-    const priorRuns = store.attemptsFor(project.name, issue);
-    const failures = store.failuresFor(project.name, issue);
-    if (!hasFailedAttemptBudget(failures, caps.maxAttemptsPerIssue)) {
-      hold(issue, "failed-attempts");
-      await safeEscalate(d, {
-        tier: 1,
-        project: project.name,
-        issue,
-        summary: `#${issue} has used all ${caps.maxAttemptsPerIssue} failed attempts`,
-        detail: [
-          r.issue.title,
-          r.issue.url,
-          "Another implementation attempt almost always means the issue itself is underspecified.",
-          "Rewrite the acceptance criteria, or take it off the queue.",
-        ].join("\n"),
-      });
-      continue;
-    }
-
-    const continuations = store.continuationsFor(project.name, issue);
-    if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
-      hold(issue, "continuations");
-      await safeEscalate(d, {
-        tier: 1,
-        project: project.name,
-        issue,
-        summary: `#${issue} exceeded its ${caps.maxContinuationsPerIssue}-continuation budget`,
-        detail: [
-          r.issue.title,
-          r.issue.url,
-          "Repeated cap kills, daemon orphans, or answered blocks need an operator to inspect progress.",
-        ].join("\n"),
-      });
-      continue;
-    }
-
-    // Fail closed on work that exists only in a run repo. `addRunRepo` clears
-    // the tree at <workspaceRoot>/<issue> before it provisions, so admitting
-    // this issue is what finally destroys the copy the salvage could not save
-    // (#118). Nothing here can recover it — git already refused once — so the
-    // only safe move is to refuse the claim and keep saying why until an
-    // operator has looked and run `unblock --force`.
-    const newest = store.latestRun(project.name, issue);
-    if (newest?.salvageError !== undefined && newest.salvageAckAt === undefined) {
-      hold(issue, "unsalvaged-wip");
-      await safeEscalate(d, {
-        tier: 1,
-        project: project.name,
-        issue,
-        summary: `#${issue} is holding unsalvaged work and will not be re-claimed`,
-        detail: [
-          r.issue.title,
-          r.issue.url,
-          `Attempt ${newest.attempt} could not commit its uncommitted changes: ${newest.salvageError}`,
-          `The only copy is the worktree ${newest.worktree === "" ? "(path not recorded)" : newest.worktree}.`,
-          "Dispatch is held because claiming this issue removes that tree.",
-          "Recover it by hand, then `omp-conductor unblock <n> --force` to release the hold.",
-        ].join("\n"),
-      });
-      continue;
-    }
-
-    // #428 half (a): a preserved continuation that predates a configured
-    // critical-base/safety marker must not be reattached. A base safety fix
-    // protects only branches forked after it landed — a continuation forked
-    // before it still carries the dangerous test/runtime code, and re-running
-    // it on the shared host is what SIGTERMed the production daemon. Fail
-    // closed: only a probe that proves every marker is in the reattach
-    // source's ancestry admits, and a project that names a marker but has no
-    // probe wired (never happens outside tests) holds. Both the hold and the
-    // escalation are durable across restart and orphan recovery because this
-    // gate runs every admission pass; the branch is re-admitted automatically
-    // once the operator updates it to contain the marker, without losing work.
-    const markers = project.criticalBase ?? [];
-    if (markers.length > 0) {
-      const branch = branchName(r.issue);
-      let verdict: CriticalBaseVerdict;
-      if (d.probeCriticalBase === undefined) {
-        verdict = { state: "unknown", error: "no critical-base probe is wired in this deployment" };
-      } else {
-        try {
-          verdict = await d.probeCriticalBase(r.repo, markers, branch);
-        } catch (err) {
-          verdict = { state: "unknown", error: errText(err) };
-        }
-      }
-      if (verdict.state === "stale") {
-        hold(issue, "stale-base");
-        log(
-          `#${issue} held (stale-base): continuation branch ${branch} predates critical-base marker ${verdict.marker}`,
-        );
-        await safeEscalate(d, {
-          tier: 1,
-          project: project.name,
-          issue,
-          summary: `#${issue} continuation branch predates a critical base safety commit and is held (stale-base)`,
-          detail: [
-            r.issue.title,
-            r.issue.url,
-            `The retained branch ${branch} does not contain critical-base marker ${verdict.marker}.`,
-            ...(verdict.range.length > 0
-              ? [`Base commits the branch is missing: ${verdict.range.join(", ")}`]
-              : []),
-            "Recovery: merge current base into the branch so it contains the marker, and the next",
-            "admission pass re-admits it automatically without losing the branch's work; or review",
-            "the branch by hand and clear the hold once the fix is present.",
-          ].join("\n"),
-        });
-        continue;
-      }
-      if (verdict.state === "unknown") {
-        // Fail closed: a branch that cannot be *proven* to contain the marker
-        // is refused, and the reason names the unverifiable marker so the
-        // operator can fix the fetch or the marker rather than guess.
-        hold(issue, "stale-base");
-        log(
-          `#${issue} held (stale-base): continuation branch ${branch} could not be verified ` +
-            `against critical-base marker(s) ${markers.join(", ")} (${verdict.error})`,
-        );
-        await safeEscalate(d, {
-          tier: 1,
-          project: project.name,
-          issue,
-          summary: `#${issue} continuation branch could not be verified against a critical base safety commit and is held (stale-base)`,
-          detail: [
-            r.issue.title,
-            r.issue.url,
-            `The retained branch ${branch} could not be verified against critical-base marker(s) ${markers.join(", ")}: ${verdict.error}`,
-            "Recovery: merge current base into the branch so it contains the marker, and the next",
-            "admission pass re-admits it automatically without losing the branch's work; or review",
-            "the branch by hand and clear the hold once the fix is present.",
-          ].join("\n"),
-        });
-        continue;
-      }
-    }
-
-    // Soft concurrency per epic, per repository: at most one in-flight child of
-    // a given parent in each repo. Children of one epic in *different* repos
-    // parallelise freely — `repo-active` / `maxConcurrentWorkersPerRepo` owns
-    // the same-repo collision domain (#197). The "" sentinel matches every
-    // repo. No parent means today's concurrent admission. Cheap local filters
-    // already ran; this sits before the open-PR API call so a held sibling
-    // frees the slot for unrelated work without spending a closers query.
-    let parent: number | undefined;
-    try {
-      parent = await resolveParent(issue);
-    } catch (err) {
-      hold(issue, "parent-lookup-error");
-      log(`#${issue} held: parent check failed (${errText(err)}) — retrying next tick`);
-      continue;
-    }
-    if (parent !== undefined) {
-      const occupied = occupiedParents.get(parent);
-      const blocker = occupied?.get(r.repo.name) ?? occupied?.get("");
-      if (blocker !== undefined) {
-        hold(issue, "sibling-active");
-        log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent} in ${r.repo.name}`);
-        continue;
-      }
-    }
-
-    // The busy set is built from run rows, so it can only speak for work this
-    // database recorded. Work pushed before this store existed — a migration, a
-    // wiped or relocated state dir, a restore onto a new host — looks exactly
-    // like fresh work, and a worker sent at it re-implements a finished PR. The
-    // tracker is the only party that remembers, so it is asked. The cost is
-    // bounded by free slots, not by queue depth: the call sits behind the two
-    // cheap local filters and candidates beyond capacity skip it.
-    let closer: OpenCloser | undefined;
-    try {
-      closer = await tracker.openCloserFor(issue);
-    } catch (err) {
-      // Fail closed, per candidate. An API error means "unknown whether
-      // finished work exists", and admitting on unknown recreates precisely the
-      // duplicate-work failure this guard exists to kill: the worst case of
-      // holding is a five-minute delay, the worst case of admitting is a burned
-      // attempt and a second PR on the same issue. Holding one candidate rather
-      // than aborting the loop keeps a transient GitHub failure from deadlocking
-      // the whole dispatcher; the next tick retries by itself.
-      hold(issue, "open-pr-lookup-error");
-      log(`#${issue} held: open-PR check failed (${errText(err)}) — retrying next tick`);
-      continue;
-    }
-    if (closer !== undefined) {
-      const latest = store.latestRun(project.name, issue);
-      // Terminality is the first half of the test and is not negotiable: while a
-      // run is live its worker is still pushing to that branch, and a second
-      // worker sent at the same PR is exactly the duplicate-work failure this
-      // guard exists to kill. Only a run that has stopped can be continued.
-      const retained =
-        latest?.state === "blocked" ||
-        latest?.state === "failed" ||
-        latest?.state === "killed" ||
-        latest?.state === "orphaned" ||
-        latest?.state === "pushed-green"
-          ? latest
-          : undefined;
-      // The second half asks "is this open PR our retained work", and accepts
-      // two identities for it, because the branch is the durable artefact of a
-      // retained run and the PR is not. A cap kill can end a run before any PR
-      // exists: veltro#324 attempt 1 was killed at the turns cap on
-      // 2026-08-09T00:47Z before its worker opened one, so the row kept `branch`
-      // and `prUrl` stayed NULL. chad#438 was opened from that exact branch
-      // afterwards, and URL equality — the only test 0.3.20 had — can never match
-      // a URL the terminal run never recorded, so every tick held #324 as
-      // `open-pr` until an operator closed recoverable work to free the branch
-      // (#50). An ordinary issue whose open PR is unrelated still fails both
-      // identities and stays ineligible, and an empty `headRefName` (a reply that
-      // did not carry the field) is never a match: unknown is not identity.
-      let resume: string | undefined;
-      if (retained !== undefined) {
-        if (retained.prUrl === closer.url) {
-          resume = `from ${retained.state} run (matched recorded PR URL)`;
-        } else if (closer.headRefName !== "" && retained.branch === closer.headRefName) {
-          resume = `from ${retained.state} run (matched retained branch ${closer.headRefName})`;
-        }
-      }
-      if (resume === undefined) {
-        hold(issue, "open-pr");
-        log(`#${issue} skipped: open PR ${closer.url} already closes it`);
-        continue;
-      }
-      log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
-    }
-
-    // The queue comes from GitHub's eventually-consistent search index. Re-read
-    // state and labels directly at the last possible moment so a just-closed or
-    // explicitly dequeued issue cannot turn a stale candidate into another
-    // attempt (#247).
-    let snapshot: IssueSnapshot | undefined;
-    try {
-      snapshot = await tracker.issueSnapshot(issue);
-    } catch {
-      snapshot = undefined;
-    }
-    if (snapshot === undefined) {
-      hold(issue, "issue-state-lookup-error");
-      log(`#${issue} held: issue snapshot check failed — retrying next tick`);
-      continue;
-    }
-    if (snapshot.state === "closed") {
-      hold(issue, "issue-closed");
-      log(`#${issue} skipped: issue is closed (search index lag)`);
-      continue;
-    }
-    if (!snapshot.labels.includes(project.queueLabel)) {
-      hold(issue, "issue-dequeued");
-      log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`);
-      continue;
-    }
-
-    admitted.push({ r, attempt: priorRuns + 1 });
-    liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
-    if (parent !== undefined) {
-      // Extend the epic's occupancy under this repo (slot empty by construction
-      // here — the gate above would have held the candidate otherwise) so a
-      // same-repo sibling later in this pass does not clear the gate (#197).
-      let siblings = occupiedParents.get(parent);
-      if (siblings === undefined) {
-        siblings = new Map();
-        occupiedParents.set(parent, siblings);
-      }
-      if (!siblings.has(r.repo.name)) siblings.set(r.repo.name, issue);
-    }
-  }
-
-  return { admitted, holds };
-}
-
 export interface WorkerPool {
   launch(work: Promise<void>): void;
   activeCount(): number;
@@ -3723,6 +3523,18 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
   // failure happened. A pause silences claiming, not the operator's right to
   // know their supervising session stopped reading its queue.
   await watchOrchestrator(d);
+  // The down incident is reconciled the same place and for the same reason: a
+  // session that has actually died is as much the operator's concern as one
+  // that is wedged, and restarting it is the daemon's restart either way. This
+  // is what turns a crashed orchestrator into one page ("down since <t>") plus
+  // a diverting count, instead of a warning only in daemon.log.
+  await reconcileOrchestratorDown({
+    project: d.project,
+    store: d.store,
+    orchestrator: d.orchestrator,
+    escalate: (event) => d.escalate(event),
+    log,
+  });
 
   // Settlement is maintenance, not dispatch. Run it before every gate that can
   // stop claiming — pause, integrity, spend, and capacity — so status converges
@@ -4415,6 +4227,8 @@ export interface StatusSnapshot {
    */
   /** Current live-head push-workflow verdict per recently merged repository. */
   baseHealth: BaseHealth[];
+  /** Per-repo base-red merge freezes, active first (#283). */
+  freezes: BaseFreeze[];
   verbLedger: VerbLedgerEntry[];
   /** Runs backed by a worker process — the number capacity compares against. */
   liveWorkers: number;
@@ -4450,6 +4264,13 @@ export interface StatusSnapshot {
    * see rather than a silent gap.
    */
   labelOps?: { pending: number; oldestAgeMs: number };
+  /**
+   * The orchestrator-down incident, when the embedded orchestrator is down:
+   * mode, since-moment and the tier-1 escalations diverted to issue comments
+   * so far. Absent when the orchestrator is healthy (or external), so recovery
+   * drops the degrade row from `status` (#288).
+   */
+  orchestratorDown?: OrchestratorIncident;
 }
 
 /** Builds a status reading from an already-open store. Long-lived operator
@@ -4469,6 +4290,8 @@ export function statusSnapshotFromStore(
   const dispatch = store.latestDispatch(p.name);
   const labelOpsPending = store.countPendingLabelOps(p.name);
   const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
+  // Read once: `status` renders the degrade row off this while it is down.
+  const orchestratorDown = store.orchestratorIncident(p.name);
   // Read once: the provenance read touches the filesystem, and the renderer
   // should never pay for it twice per status.
   const reason = pauseProvenance(p.name)?.reason;
@@ -4501,6 +4324,8 @@ export function statusSnapshotFromStore(
       ? {}
       : { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
     baseHealth: store.baseHealth(p.name),
+    freezes: store.freezes(p.name),
+    ...(orchestratorDown === undefined ? {} : { orchestratorDown }),
   };
 }
 
@@ -4543,6 +4368,11 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
             ? ""
             : ` (#${hold.issues.join(", #")}${hold.count > hold.issues.length ? ", …" : ""})`;
         lines.push(`    ${hold.reason}  ${hold.count}${sample}`);
+        // A `file-lane` hold groups several issues, each blocked by a different
+        // file and holder; the grouped line says how many, this says which.
+        if ((hold.details?.length ?? 0) > 0) {
+          lines.push(`      ${hold.details!.join(" | ")}`);
+        }
       }
     }
   }
@@ -4612,6 +4442,25 @@ export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
   });
 }
 
+/**
+ * The active base-red freezes as status lines — merges refused until the base
+ * is green again or the operator overrides. Active freezes only: a cleared
+ * freeze is history the ledger and digest already told an operator about.
+ */
+export function formatFreezes(freezes: readonly BaseFreeze[]): string[] {
+  const active = freezes.filter((f) => f.clearedAt === undefined);
+  if (active.length === 0) return [];
+  return [
+    "frozen repos  (merges refused until base green)",
+    ...active.map(
+      (f) =>
+        `  ${f.repo}  base red at ${f.culpritSha.slice(0, 8)}` +
+        (f.detail === undefined ? "" : `  ${f.detail}`) +
+        `  — override: omp-conductor unfreeze ${f.repo}`,
+    ),
+  ];
+}
+
 
 export function formatStatus(s: StatusSnapshot): string {
   const lines = [
@@ -4619,6 +4468,7 @@ export function formatStatus(s: StatusSnapshot): string {
     `config    ${s.configPath}`,
     `state     ${s.stateDir}`,
     "",
+    ...(s.orchestratorDown === undefined ? [] : formatOrchestratorDown(s.orchestratorDown)),
     "caps",
     `  workers            ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
     `  issues today       ${s.runsToday}`,
@@ -4660,6 +4510,7 @@ export function formatStatus(s: StatusSnapshot): string {
     }
   }
   lines.push(...formatBaseHealth(s.baseHealth));
+  lines.push(...formatFreezes(s.freezes));
   lines.push(...formatSalvagedRuns(s.salvagedRuns));
   lines.push(...formatOpenReports(s.openReports));
   lines.push(...formatVerbLedger(s.verbLedger));
@@ -5095,6 +4946,35 @@ async function recoverRun(
       log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
       return;
     }
+    // Same bound for provider-capacity: a run the provider throttled into the
+    // ground is requeued free (no attempt charged) — but a provider that
+    // throttles the same issue three times is at capacity, and a human has to
+    // check its status before hand-requeueing (#573). On a chain-configured
+    // project each requeue already moved the next attempt to the next chain
+    // model, so this escalation is what catches the no-chain case and the
+    // exhausted chain; it names every model the chain tried.
+    if (
+      cls === "provider-capacity" &&
+      store.classCountFor(project.name, run.issue, "provider-capacity") >= PROVIDER_CAPACITY_MAX_STRIKES
+    ) {
+      const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue: run.issue,
+        summary: `[provider-capacity] #${run.issue}: the model provider is throttling this run into the ground — ${evidence}`,
+        detail: [
+          `The provider answered #${run.issue} with sustained in-session rate limits ${PROVIDER_CAPACITY_MAX_STRIKES} times in a row; the harness retried each and was exhausted.`,
+          ...(tried === ""
+            ? []
+            : [`Models tried: ${tried}.`]),
+          "Check the provider's rate-limit status (and its throughput-oriented routes) before requeueing by hand.",
+        ].join("\n"),
+      });
+      store.updateRun(run.id, { recoveredAt: Date.now() });
+      log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
+      return;
+    }
     // Only when the tracker still shows this issue as ours to hand back. An
     // issue that is closed, or has no state label, was resolved by another route
     // and requeueing it would dispatch work nobody asked for.
@@ -5597,6 +5477,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
     const { brief, releaseGrants } = orchestratorStandingOrders(project);
     let orchestrator: OrchestratorHandle | undefined;
     let orchestratorVerbs: VerbListener | undefined;
+    /** First start-failure cause, surfaced by the orchestrator-down incident (#288). */
+    let orchestratorStartError: string | undefined;
     if (project.escalation.orchestrator === "external") {
       projectLog(
         "orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty",
@@ -5636,9 +5518,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
         const transcript = orchestrator.sessionFile();
         projectLog(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
       } catch (err) {
+        orchestratorStartError = errText(err);
         projectLog(
           "WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue " +
-            `comments: ${errText(err)}`,
+            `comments: ${orchestratorStartError}`,
         );
         await orchestratorVerbs?.close();
         orchestratorVerbs = undefined;
@@ -5655,6 +5538,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
       orchestrator,
       Date.now,
       deliveryPolicyValid,
+      (e) => {
+        // Every tier-1 escalation that lands on the issue-comment fallback was
+        // diverted from the orchestrator. Count it durably on the open incident
+        // (a no-op when none is open), so the page and status name how much
+        // the outage diverted (#288).
+        store.bumpOrchestratorDiverted(project.name, 1);
+        projectLog(
+          `orchestrator: tier-1 escalation on ${escalationIssueRef(e.issue)} diverted to issue comments ` +
+            `while the orchestrator was down`,
+        );
+      },
     );
     const outbox = createReportOutbox({
       project: currentProject,
@@ -5685,13 +5579,27 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
       workerControls,
       integrity,
       stall: { paged: false },
+      ...(orchestrator === undefined ? {} : { orchestrator }),
       cleanup: { next: 0 },
       probeCriticalBase: (repo, markers, branch) =>
         probeCriticalBase(project, repo, branch, markers),
+      probeWorktreeLane: (input) => probeRunLane(input),
       ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
       verbActions,
     };
     runtimeDeps = d;
+    // Startup reconciliation: close an incident carried over from a previous
+    // process when the orchestrator is up (one recovery notice), or open one
+    // when it failed to start (one down page). A daemon restarted while still
+    // down rediscovers the open incident and does not re-page it.
+    await reconcileOrchestratorDown({
+      project,
+      store,
+      orchestrator,
+      escalate: (event) => d.escalate(event),
+      ...(orchestratorStartError === undefined ? {} : { startCause: orchestratorStartError }),
+      log: projectLog,
+    });
     runtimes.push({
       d,
       outbox,
@@ -5771,7 +5679,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
     // when it resumes and cannot create the work this shutdown is about to
     // wait for (#374).
     drain.draining = true;
-    log("shutting down after active workers finish");
+    log("shutdown requested — draining dispatch; live worker sessions are not waited for");
     pace.requestWake();
   };
   process.on("SIGINT", stop);