omp-conductor 0.15.13 → 0.16.0

package/src/daemon.ts

@@ -47,6 +47,7 @@ import {
   createReportOutbox,
   enqueueAvailableHeldNotices,
   formatOpenReports,
+  reliabilitySettlementLine,
   type ReportOutbox,
 } from "./reports.ts";
 import {
@@ -56,6 +57,10 @@ import {
 } from "./release-policy.ts";
 import { branchName, effectiveLabels, route } from "./routing.ts";
 import type { Routed, UnroutableReason } from "./routing.ts";
+import { admitCandidates, hasContinuationBudget, hasFailedAttemptBudget } from "./admission.ts";
+import type { Admission, AdmissionHold } from "./admission.ts";
+import { log, errText, safeEscalate } from "./log.ts";
+import { materializeOmpSettings } from "./omp-settings.ts";
 import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
 import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
 import {
@@ -71,6 +76,7 @@ import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
 import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
 import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
 import type {
+  BaseFreeze,
   BaseHealth,
   AdmissionHoldReason,
   Caps,
@@ -140,10 +146,12 @@ import { homedir } from "node:os";
 
 import {
   probeCriticalBase,
+  probeRunLane,
   pushRunBranch,
   readBaseChain,
   type CriticalBaseProbe,
   type CriticalBaseVerdict,
+  type RunLaneProbe,
   type RunRepoRef,
 } from "./gitops.ts";
 import {
@@ -172,6 +180,13 @@ const DISPATCH_INFRA_MAX_STRIKES = 3;
  *  provider itself is degraded, not unlucky, and the sweep escalates to a
  *  human instead of requeueing into a down provider forever (#220). */
 const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
+/** A provider-capacity requeue (sustained in-session rate limiting) is retried,
+ *  but only a bounded number of times: three throttled runs for one issue mean
+ *  the provider is at capacity, not unlucky, and the sweep escalates to a human
+ *  instead of requeueing into a throttled provider forever (#573). The issue's
+ *  own chain moves onto its next model per strike (via {@link FAILOVER_CLASSES}),
+ *  so a bounded chain is exhaustible; this caps the unbounded no-chain case. */
+const PROVIDER_CAPACITY_MAX_STRIKES = 3;
 /** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
  *  are maintenance, but a backlog must not turn one tick into an API burst. */
 const SALVAGED_PR_ADOPTION_BATCH = 10;
@@ -271,6 +286,16 @@ interface Deps {
    * a marker (a safety interlock must not silently weaken).
    */
   probeCriticalBase?: CriticalBaseProbe;
+  /**
+   * Reads one active run's file lane for the admission file-lane interlock
+   * (#555): the union of its uncommitted worktree changes and its branch-vs-base
+   * diff. Wired by `runDaemon` to the mirror/worktree-backed
+   * {@link probeRunLane}; a test injects a fake. Absent, the interlock is inert
+   * (no lane is ever known occupied), which is the issue's "fail open": the
+   * gate adds holds, it never refuses a well-formed issue for lack of this
+   * probe the way `criticalBase` does.
+   */
+  probeWorktreeLane?: RunLaneProbe;
 }
 
 /**
@@ -681,14 +706,6 @@ export function markPaged(
 
 // ---------------------------------------------------------------------- helpers
 
-function log(msg: string): void {
-  process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`);
-}
-
-function errText(e: unknown): string {
-  return e instanceof Error ? (e.stack ?? e.message) : String(e);
-}
-
 /**
  * Local midnight, matching how a human reads "today".
  *
@@ -841,15 +858,6 @@ export function recordOperatorStop(
  * gate on the attempt turns one failed delivery into permanent silence about a
  * condition that is still true.
  */
-async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<boolean> {
-  try {
-    await d.escalate(e);
-    return true;
-  } catch (err) {
-    log(`escalation for ${escalationIssueRef(e.issue)} could not be delivered: ${errText(err)}`);
-    return false;
-  }
-}
 
 async function reactToProviderCredit(
   d: Deps,
@@ -1108,18 +1116,6 @@ export async function buildBrief(
 
 // ------------------------------------------------------------------- one issue
 
-/** `stops` are the operational ends that each require one resume. */
-export function hasContinuationBudget(stops: number, maxContinuations: number): boolean {
-  return stops <= maxContinuations;
-}
-
-/** True while unspent failed-implementation attempts remain. This is the
- *  dispatcher's admission gate: once every `maxAttemptsPerIssue` slot is
- *  spent, the issue is held as `failed-attempts` forever, and the `unblock`
- *  verb withholds the queue label on the same predicate (#348). */
-export function hasFailedAttemptBudget(failures: number, maxAttempts: number): boolean {
-  return failures < maxAttempts;
-}
 
 /** The failure classes `countContinuations` deliberately does not charge — the
  *  inverted copy of its exclusions, kept beside the breakdown that consumes it
@@ -1133,6 +1129,7 @@ const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
   "dispatch-infra": true,
   "provider-credit": true,
   "provider-transient": true,
+  "provider-capacity": true,
 };
 
 /** How one issue spent its continuation budget, grouped by failure class —
@@ -1978,6 +1975,16 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
         : dirname(resuming.sessionFile!);
     mkdirSync(sessionDir, { recursive: true });
 
+    // The fleet-owned omp settings overlay (#537): the project's `ompSettings`
+    // map (plus the retry keys derived from `modelFallbacks`, #539's staging
+    // half) materialised to YAML under the run's session directory — never
+    // inside the worktree, whose diff is the PR a worker ships. Rewritten on
+    // every attempt, so a config edit takes effect on the next dispatch and a
+    // resumed attempt reuses the kept session dir with the *current* config.
+    // Absent `ompSettings` and `modelFallbacks`, no file is written, nothing
+    // is passed, and dispatch is byte-for-byte today's.
+    const ompSettingsFile = materializeOmpSettings(project, sessionDir);
+
     // ---- the run's mutation channel (#126) -------------------------------
     // A shared, daemon-owned 0711 parent with one 0600 socket per run, never a
     // per-run *directory*: a directory owned by the run principal would hand
@@ -2091,6 +2098,12 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
           log(`#${issue} ${line}`);
         },
         ...(choice.model === undefined ? {} : { model: choice.model }),
+        // The fleet-owned omp settings overlay (#537): the staged YAML the
+        // session loads through `Settings.init({ configFiles: [<path>] })` —
+        // the project's `ompSettings` map plus the within-run failover keys
+        // #581 staged directly (the project's own chain, not an empty default).
+        // Absent both, nothing is staged, today's dispatch byte for byte.
+        ...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
         releaseGrants: resolveReleaseGrants(project),
         onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
         onTurn: (n) => store.updateRun(runId, { turns: n }),
@@ -2177,10 +2190,27 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
     // worker's, the disclosure becomes the diff's (#488). A diff that could
     // not be read leaves the worker's text untouched and the audit's
     // `changed-line-missing` flag says so.
-    const settlementReport =
+    // The within-run reliability sentence, appended where a human reads it
+    // (#584): which model the run finished on and whether it swapped
+    // mid-flight. Empty (undefined) for a clean run, so a run that never
+    // retried, swapped or compacted keeps today's settlement report byte for
+    // byte — the "additive" claim asserted rather than assumed.
+    const reliabilityLine = reliabilitySettlementLine({
+      resolvedModel: result.model,
+      resolvedProvider: result.provider,
+      retryFallbacks: result.retryFallbacks,
+      retryFallbackSucceeded: result.retryFallbackSucceeded,
+      modelRecoveries: result.modelRecoveries,
+      autoRetryCount: result.autoRetryCount,
+      autoCompactionCount: result.autoCompactionCount,
+    });
+
+    const settlementReport = [
       audit?.changedLine === undefined
         ? result.report
-        : withDerivedChangedLine(result.report, audit.changedLine);
+        : withDerivedChangedLine(result.report, audit.changedLine),
+      ...(reliabilityLine === undefined ? [] : ["", reliabilityLine]),
+    ].join("\n");
 
     const finalReport = [
       ...(verified.reason === undefined ? [] : [verified.reason, ""]),
@@ -2230,6 +2260,24 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
       endedAt: Date.now(),
       turns: result.turns,
       spendUsd: result.spendUsd,
+      // The count of in-session provider 429s the worker metered live, so a
+      // run the provider throttled into the ground carries its own diagnosis
+      // instead of landing `unknown` — the classifier reads it straight off
+      // this column (#573).
+      provider429Count: result.provider429Count,
+      // The within-run harness reliability surface #581 collected, persisted
+      // now that settlement owns the row (#584). The resolved model/provider
+      // only travel when some assistant message carried them (the run recorded
+      // no model, or the worker never established one); the count fields always
+      // travel, 0 for a clean run, so an absent column can never be read as a
+      // quiet fleet. Written for every terminal state, clean or not.
+      ...(result.model === undefined ? {} : { resolvedModel: result.model }),
+      ...(result.provider === undefined ? {} : { resolvedProvider: result.provider }),
+      retryFallbacks: result.retryFallbacks,
+      retryFallbackSucceeded: result.retryFallbackSucceeded,
+      modelRecoveries: result.modelRecoveries,
+      autoRetryCount: result.autoRetryCount,
+      autoCompactionCount: result.autoCompactionCount,
       // The worker only reports these when it actually established them; a kill
       // or a settle whose report named no PR must not wipe what a verb recorded
       // earlier in the same run (#468). The sink in `updateRun` skips undefined
@@ -2272,6 +2320,22 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
 
     const salvaged = settlement?.lines ?? [];
 
+    // The run's reliability news, surfaced where the tick digest reads it
+    // (#584): the digest is model-authored but consumes the store's material
+    // ledger, so a run that swapped mid-flight or rode out a throttled
+    // provider lands one event the digest can name. Clean runs record nothing
+    // here, so a quiet fleet's digest is unchanged.
+    if (reliabilityLine !== undefined) {
+      store.recordMaterialEvent({
+        project: project.name,
+        category: "reliability",
+        summary: `#${issue} ${reliabilityLine}`,
+        evidence: `${r.issue.title}\n${r.issue.url}\n\n${reliabilityLine}`,
+        occurredAt: Date.now(),
+        recordedAt: Date.now(),
+      });
+    }
+
     if (state === "stopped") {
       log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
     } else if (state === "blocked") {
@@ -2693,6 +2757,31 @@ export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "sto
       `${failed.name} failed at ${run.mergeSha} — ${failed.url}` +
       (preexisting ? " (already red before this merge)" : "");
     const flag: SettlementFlag = { kind: "base-branch-red", file: "(base branch)", detail };
+    // Freeze merges to this repo while the base it merged into is red (#283).
+    // The freeze is repo-scoped and sets independently of escalation delivery:
+    // a merge that broke the base must not be followed by another merge onto
+    // the same red base, even if paging the operator fails.
+    if (
+      d.store.setBaseFreeze(d.project.name, {
+        repo: run.repo,
+        culpritSha: run.mergeSha,
+        detail,
+        setAt: now,
+      })
+    ) {
+      d.store.recordMaterialEvent({
+        project: d.project.name,
+        category: "base-red-freeze",
+        summary: `merges to ${run.repo} frozen — base ${run.baseRef} red at ${run.mergeSha.slice(0, 8)}`,
+        evidence:
+          `${detail} This freeze names ${run.mergeSha.slice(0, 8)} as the suspected culprit merge. ` +
+          "Merges to this repo are refused until the base is green again; reverting the culprit is the " +
+          "likely remedy. The freeze lifts automatically on a green re-observation, or the operator can " +
+          "override it with `omp-conductor unfreeze <repo>`.",
+        occurredAt: now,
+        recordedAt: now,
+      });
+    }
     const delivered = await safeEscalate(d, {
       tier: 1,
       project: d.project.name,
@@ -2751,7 +2840,16 @@ export async function watchBaseHealth(
     }
 
     const previous = previousByRepo.get(repo);
+    const freeze = d.store.baseFreeze(d.project.name, repo);
+    const frozen = freeze !== undefined && freeze.clearedAt === undefined;
+    // The same-head/age shortcut exists to avoid re-querying GitHub when a
+    // terminal verdict has not moved. It must NOT skip a frozen repo: a freeze
+    // keyed on one red observation has to keep re-evaluating the same head so a
+    // green rerun clears it without operator action (tonight's evidence — a
+    // red/unknown read two minutes after the same SHA's CI succeeded — is why
+    // it cannot be trusted as terminal).
     if (
+      !frozen &&
       previous?.branch === branch &&
       previous.headSha === head &&
       (previous.verdict === "green" || previous.verdict === "red")
@@ -2811,6 +2909,43 @@ export async function watchBaseHealth(
     };
     d.store.upsertBaseHealth(d.project.name, health);
     previousByRepo.set(repo, health);
+
+    // The freeze follows the live base verdict: red arms (or re-arms) the
+    // repo-scoped freeze, green lifts it. pending/unknown leave it untouched —
+    // a stale or in-flight reading must neither create a freeze nor clear one.
+    if (verdict === "green") {
+      if (d.store.clearBaseFreeze(d.project.name, repo, "daemon", "base-green", now)) {
+        d.store.recordMaterialEvent({
+          project: d.project.name,
+          category: "base-recovered",
+          summary: `merges to ${repo} unfrozen — base ${branch} observed green at ${head.slice(0, 8)}`,
+          evidence: `The base-red freeze on ${repo} lifted automatically: ${branch} is green at ${head.slice(0, 8)}. Merges resume.`,
+          occurredAt: now,
+          recordedAt: now,
+        });
+      }
+    } else if (verdict === "red") {
+      if (
+        d.store.setBaseFreeze(d.project.name, {
+          repo,
+          culpritSha: head,
+          detail,
+          setAt: now,
+        })
+      ) {
+        d.store.recordMaterialEvent({
+          project: d.project.name,
+          category: "base-red-freeze",
+          summary: `merges to ${repo} frozen — base ${branch} red at ${head.slice(0, 8)}`,
+          evidence:
+            `${detail ?? `workflow failed at ${head.slice(0, 8)}`} This freeze names ` +
+            `${head.slice(0, 8)} as the suspected culprit. Reverting it is the likely remedy; the freeze ` +
+            "lifts automatically on green or with `omp-conductor unfreeze <repo>`.",
+          occurredAt: now,
+          recordedAt: now,
+        });
+      }
+    }
   }
 }
 
@@ -3130,22 +3265,6 @@ export async function cleanupRetainedRuns(
 
 // -------------------------------------------------------------------- admission
 
-/** A candidate cleared for dispatch, with the attempt number it will run as. */
-export interface Admission {
-  r: Routed;
-  attempt: number;
-}
-
-export interface AdmissionHold {
-  issue: number;
-  reason: AdmissionHoldReason;
-}
-
-export interface AdmissionPass {
-  admitted: Admission[];
-  holds: AdmissionHold[];
-}
-
 const HOLD_SAMPLE_SIZE = 5;
 const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
   "parent-lookup-error",
@@ -3163,11 +3282,16 @@ export function summarizeDispatch(
   completedAt = Date.now(),
   settled = 0,
 ): DispatchSummary {
-  const groups = new Map<AdmissionHoldReason, { count: number; issues: number[] }>();
+  const groups = new Map<AdmissionHoldReason, { count: number; issues: number[]; details: string[] }>();
   for (const hold of holds) {
-    const group = groups.get(hold.reason) ?? { count: 0, issues: [] };
+    const group = groups.get(hold.reason) ?? { count: 0, issues: [], details: [] };
     group.count += 1;
-    if (group.issues.length < HOLD_SAMPLE_SIZE) group.issues.push(hold.issue);
+    if (group.issues.length < HOLD_SAMPLE_SIZE) {
+      // `issues` and `details` share the sample: the detail is only kept when
+      // the issue it explains is, so the arrays stay index-aligned.
+      group.issues.push(hold.issue);
+      if (hold.detail !== undefined) group.details.push(hold.detail);
+    }
     groups.set(hold.reason, group);
   }
   return {
@@ -3179,7 +3303,12 @@ export function summarizeDispatch(
     degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
     holds: [...groups]
       .sort(([a], [b]) => a.localeCompare(b))
-      .map(([reason, group]) => ({ reason, ...group })),
+      .map(([reason, group]) => ({
+        reason,
+        count: group.count,
+        issues: group.issues,
+        ...(group.details.length === 0 ? {} : { details: group.details }),
+      })),
     settled,
   };
 }
@@ -3204,484 +3333,6 @@ export function summarizeHeldPass(settled: number, completedAt = Date.now()): Di
   };
 }
 
-/**
- * What a held plan-usage gate says to a human, if anything.
- *
- * Three different problems hide behind one hold, and they want different
- * tiers. Reaching the threshold is the guard *working*: tier 1, because the
- * fleet resumes on its own at the provider's reset and nobody needs to get
- * out of bed. Everything else — a window nothing reports, a window that
- * resolves to two allowances, a meter that has been unreadable for half an
- * hour — is dispatch stopped with no self-recovery, which is tier 2.
- *
- * Each summary carries the fact that will change when the situation does (the
- * reset instant, the configured id, the date), because the escalation ledger
- * dedupes on the summary: a stable one pages once and then goes quiet, which
- * is right for a repeated tick and wrong for the next window.
- */
-function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation | undefined {
-  const base = { project, issue: NO_ISSUE };
-  if (plan.state === "at-cap") {
-    const window = plan.window?.id ?? plan.cap?.windowId ?? "the configured window";
-    const resets =
-      plan.resetsAt === undefined
-        ? new Date().toISOString().slice(0, 10)
-        : new Date(plan.resetsAt).toISOString();
-    return {
-      ...base,
-      tier: 1,
-      summary: `Plan allowance cap reached on ${window} — ${project} is not claiming new work (window ${resets})`,
-      detail: [
-        plan.detail,
-        "Running workers finish normally; only new claims are held.",
-        "Dispatch resumes by itself once the provider reports the window reset or usage below the threshold —",
-        "no `resume` needed. Raise `caps.planUsage.maxUsedFraction` only if you mean to spend the rest.",
-      ].join("\n"),
-    };
-  }
-  if (plan.state === "blind") {
-    return {
-      ...base,
-      tier: 2,
-      category: "fleet-stopped",
-      // Dated: a meter that breaks again next month is a new incident, not a
-      // repeat of this one.
-      summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
-      detail: [
-        plan.detail,
-        "The guard admitted work while the failure looked transient and has now stopped.",
-        "Check `omp usage --json` on the fleet host, or set `caps.planUsage` to null if this fleet is unmetered.",
-      ].join("\n"),
-    };
-  }
-  if (
-    plan.state === "window-missing" ||
-    plan.state === "window-ambiguous" ||
-    plan.state === "window-uncomparable"
-  ) {
-    return {
-      ...base,
-      tier: 2,
-      category: "fleet-stopped",
-      summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
-      detail: [
-        plan.detail,
-        "Run `omp usage --json` and copy an allowance `id` into `caps.planUsage.windowId`,",
-        "or set `caps.planUsage` to null if this fleet is unmetered.",
-      ].join("\n"),
-    };
-  }
-  return undefined;
-}
-
-/**
- * Which routed candidates get a worker this tick — in queue order, never more
- * than `slots` of them. Every non-admission receives a stable reason code.
- *
- * Exported so the admission rules can be pinned without spawning a worker.
- * Every one of them exists because of a live incident, and each guards a
- * different way the same issue gets worked twice — including epic siblings
- * racing onto the same files (#48).
- *
- * Takes the slice of `Deps` it actually reads rather than the whole thing: what
- * admission is allowed to consult is the point of the function, and a `Deps`
- * that grows a field has no business breaking these tests.
- */
-export async function admitCandidates(
-  d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage" | "probeCriticalBase">,
-  routed: Routed[],
-  slots: number,
-): Promise<AdmissionPass> {
-  const { project, caps, tracker, store } = d;
-  const activeRuns = store.activeRuns(project.name);
-  const busyIssues = activeRuns.map((r) => r.issue);
-  const busy = new Set(busyIssues);
-  // issue -> its active run rows, for the pushed-green admission bypass (#175):
-  // only a worker-free pushed-green row may be bypassed, and only when *every*
-  // active run for the issue is worker-free. A live (claimed/running) row still
-  // holds unconditionally.
-  const activeByIssue = new Map<number, RunRecord[]>();
-  for (const run of activeRuns) {
-    const list = activeByIssue.get(run.issue);
-    if (list === undefined) activeByIssue.set(run.issue, [run]);
-    else list.push(run);
-  }
-  // Live worker count per repo, seeded from live runs and incremented as this
-  // same pass admits — so two same-repo candidates can never both clear the
-  // per-repo cap in one tick (#186).
-  const liveByRepo = new Map<string, number>();
-  for (const run of store.liveRuns(project.name)) {
-    liveByRepo.set(run.repo, (liveByRepo.get(run.repo) ?? 0) + 1);
-  }
-  const holds: AdmissionHold[] = [];
-  const hold = (issue: number, reason: AdmissionHoldReason): void => {
-    holds.push({ issue, reason });
-  };
-
-  // The plan allowance is a fleet-wide question, so it is asked once per pass
-  // and answers for every candidate — unlike every gate below it, which is
-  // per-issue. It sits here rather than beside the spend cap in `tick` for one
-  // reason: the spend cap *pauses the daemon* and waits for a human, and a
-  // weekly plan window resets by itself. A guard that demanded `resume` after
-  // every rollover would cost more operator attention than the guard saves
-  // (#110). Already-running workers are untouched and settle normally.
-  //
-  // Placed after the cheap local busy-set read and before the first tracker
-  // call, so a held fleet spends no GitHub API budget discovering it is held.
-  const plan = await readPlanUsage(caps.planUsage, d.usage);
-  if (plan.blocking) {
-    for (const r of routed) hold(r.issue.number, "plan-usage-cap");
-    log(`plan usage gate holding ${String(routed.length)} candidate(s): ${plan.detail}`);
-    const escalation = planUsageEscalation(project.name, plan);
-    if (escalation !== undefined) await safeEscalate(d, escalation);
-    return { admitted: [], holds };
-  }
-
-  // parent -> repo name -> blocking issue. Seeded from active runs (including
-  // pushed-green), then extended by candidates admitted earlier in this same
-  // pass so two siblings of one epic never both clear the gate in one tick.
-  // A busy issue whose run row cannot be resolved occupies the sentinel repo
-  // "" — treated as matching every repo, failing toward holding (#197).
-  const occupiedParents = new Map<number, Map<string, number>>();
-  const parentCache = new Map<number, number | undefined>();
-
-  const resolveParent = async (issue: number): Promise<number | undefined> => {
-    if (parentCache.has(issue)) return parentCache.get(issue);
-    const parent = await tracker.parentOf(issue);
-    parentCache.set(issue, parent);
-    return parent;
-  };
-
-  // Bounded by concurrent workers, not queue depth. A failed lookup here cannot
-  // mark an epic occupied; candidates still fail closed on their own parentOf.
-  for (const issue of busyIssues) {
-    try {
-      const parent = await resolveParent(issue);
-      if (parent === undefined) continue;
-      // The runs table records which repo each attempt worked in, and sibling
-      // holds are now per-repo, so a busy child only occupies its epic under
-      // that repo's name (same spelling as `createRun` writes from
-      // `r.repo.name`). A busy issue with no resolvable run row occupies the
-      // sentinel "" instead — matching every repo, failing toward holding.
-      const repo = store.latestRun(project.name, issue)?.repo ?? "";
-      const siblings = occupiedParents.get(parent);
-      if (siblings === undefined) {
-        occupiedParents.set(parent, new Map([[repo, issue]]));
-      } else if (!siblings.has(repo) && !siblings.has("")) {
-        siblings.set(repo, issue);
-      }
-    } catch (err) {
-      log(`#${issue} parent lookup failed while seeding epic occupancy (${errText(err)})`);
-    }
-  }
-
-  const admitted: Admission[] = [];
-  for (const r of routed) {
-    const issue = r.issue.number;
-    if (admitted.length >= slots) {
-      hold(issue, "capacity");
-      continue;
-    }
-    if (busy.has(issue)) {
-      // A pushed-green row is worker-free by definition (it is not in
-      // LIVE_STATES): its PR is live but no process is writing to its branch.
-      // So an issue whose active runs are ALL pushed-green is not actually
-      // occupied — the corrective attempt the operator unblocked may be
-      // admitted as a continuation of that PR, and the open-PR gate below
-      // decides the identity. Any live row still holds (#175).
-      const allWorkerFree = (activeByIssue.get(issue) ?? []).every((r) => r.state === "pushed-green");
-      if (!allWorkerFree) {
-        hold(issue, "issue-active");
-        continue;
-      }
-    }
-
-    // Per-repo concurrency: the mirror, branch-protection staleness and shared
-    // CI egress are all per-repo collision domains, so extra slots should land
-    // on other repos rather than stacking workers into the same one (#186).
-    const liveInRepo = liveByRepo.get(r.repo.name) ?? 0;
-    if (liveInRepo >= caps.maxConcurrentWorkersPerRepo) {
-      hold(issue, "repo-active");
-      log(`#${issue} skipped: ${liveInRepo} live worker(s) already in ${r.repo.name} (cap ${caps.maxConcurrentWorkersPerRepo})`);
-      continue;
-    }
-
-    const priorRuns = store.attemptsFor(project.name, issue);
-    const failures = store.failuresFor(project.name, issue);
-    if (!hasFailedAttemptBudget(failures, caps.maxAttemptsPerIssue)) {
-      hold(issue, "failed-attempts");
-      await safeEscalate(d, {
-        tier: 1,
-        project: project.name,
-        issue,
-        summary: `#${issue} has used all ${caps.maxAttemptsPerIssue} failed attempts`,
-        detail: [
-          r.issue.title,
-          r.issue.url,
-          "Another implementation attempt almost always means the issue itself is underspecified.",
-          "Rewrite the acceptance criteria, or take it off the queue.",
-        ].join("\n"),
-      });
-      continue;
-    }
-
-    const continuations = store.continuationsFor(project.name, issue);
-    if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
-      hold(issue, "continuations");
-      await safeEscalate(d, {
-        tier: 1,
-        project: project.name,
-        issue,
-        summary: `#${issue} exceeded its ${caps.maxContinuationsPerIssue}-continuation budget`,
-        detail: [
-          r.issue.title,
-          r.issue.url,
-          "Repeated cap kills, daemon orphans, or answered blocks need an operator to inspect progress.",
-        ].join("\n"),
-      });
-      continue;
-    }
-
-    // Fail closed on work that exists only in a run repo. `addRunRepo` clears
-    // the tree at <workspaceRoot>/<issue> before it provisions, so admitting
-    // this issue is what finally destroys the copy the salvage could not save
-    // (#118). Nothing here can recover it — git already refused once — so the
-    // only safe move is to refuse the claim and keep saying why until an
-    // operator has looked and run `unblock --force`.
-    const newest = store.latestRun(project.name, issue);
-    if (newest?.salvageError !== undefined && newest.salvageAckAt === undefined) {
-      hold(issue, "unsalvaged-wip");
-      await safeEscalate(d, {
-        tier: 1,
-        project: project.name,
-        issue,
-        summary: `#${issue} is holding unsalvaged work and will not be re-claimed`,
-        detail: [
-          r.issue.title,
-          r.issue.url,
-          `Attempt ${newest.attempt} could not commit its uncommitted changes: ${newest.salvageError}`,
-          `The only copy is the worktree ${newest.worktree === "" ? "(path not recorded)" : newest.worktree}.`,
-          "Dispatch is held because claiming this issue removes that tree.",
-          "Recover it by hand, then `omp-conductor unblock <n> --force` to release the hold.",
-        ].join("\n"),
-      });
-      continue;
-    }
-
-    // #428 half (a): a preserved continuation that predates a configured
-    // critical-base/safety marker must not be reattached. A base safety fix
-    // protects only branches forked after it landed — a continuation forked
-    // before it still carries the dangerous test/runtime code, and re-running
-    // it on the shared host is what SIGTERMed the production daemon. Fail
-    // closed: only a probe that proves every marker is in the reattach
-    // source's ancestry admits, and a project that names a marker but has no
-    // probe wired (never happens outside tests) holds. Both the hold and the
-    // escalation are durable across restart and orphan recovery because this
-    // gate runs every admission pass; the branch is re-admitted automatically
-    // once the operator updates it to contain the marker, without losing work.
-    const markers = project.criticalBase ?? [];
-    if (markers.length > 0) {
-      const branch = branchName(r.issue);
-      let verdict: CriticalBaseVerdict;
-      if (d.probeCriticalBase === undefined) {
-        verdict = { state: "unknown", error: "no critical-base probe is wired in this deployment" };
-      } else {
-        try {
-          verdict = await d.probeCriticalBase(r.repo, markers, branch);
-        } catch (err) {
-          verdict = { state: "unknown", error: errText(err) };
-        }
-      }
-      if (verdict.state === "stale") {
-        hold(issue, "stale-base");
-        log(
-          `#${issue} held (stale-base): continuation branch ${branch} predates critical-base marker ${verdict.marker}`,
-        );
-        await safeEscalate(d, {
-          tier: 1,
-          project: project.name,
-          issue,
-          summary: `#${issue} continuation branch predates a critical base safety commit and is held (stale-base)`,
-          detail: [
-            r.issue.title,
-            r.issue.url,
-            `The retained branch ${branch} does not contain critical-base marker ${verdict.marker}.`,
-            ...(verdict.range.length > 0
-              ? [`Base commits the branch is missing: ${verdict.range.join(", ")}`]
-              : []),
-            "Recovery: merge current base into the branch so it contains the marker, and the next",
-            "admission pass re-admits it automatically without losing the branch's work; or review",
-            "the branch by hand and clear the hold once the fix is present.",
-          ].join("\n"),
-        });
-        continue;
-      }
-      if (verdict.state === "unknown") {
-        // Fail closed: a branch that cannot be *proven* to contain the marker
-        // is refused, and the reason names the unverifiable marker so the
-        // operator can fix the fetch or the marker rather than guess.
-        hold(issue, "stale-base");
-        log(
-          `#${issue} held (stale-base): continuation branch ${branch} could not be verified ` +
-            `against critical-base marker(s) ${markers.join(", ")} (${verdict.error})`,
-        );
-        await safeEscalate(d, {
-          tier: 1,
-          project: project.name,
-          issue,
-          summary: `#${issue} continuation branch could not be verified against a critical base safety commit and is held (stale-base)`,
-          detail: [
-            r.issue.title,
-            r.issue.url,
-            `The retained branch ${branch} could not be verified against critical-base marker(s) ${markers.join(", ")}: ${verdict.error}`,
-            "Recovery: merge current base into the branch so it contains the marker, and the next",
-            "admission pass re-admits it automatically without losing the branch's work; or review",
-            "the branch by hand and clear the hold once the fix is present.",
-          ].join("\n"),
-        });
-        continue;
-      }
-    }
-
-    // Soft concurrency per epic, per repository: at most one in-flight child of
-    // a given parent in each repo. Children of one epic in *different* repos
-    // parallelise freely — `repo-active` / `maxConcurrentWorkersPerRepo` owns
-    // the same-repo collision domain (#197). The "" sentinel matches every
-    // repo. No parent means today's concurrent admission. Cheap local filters
-    // already ran; this sits before the open-PR API call so a held sibling
-    // frees the slot for unrelated work without spending a closers query.
-    let parent: number | undefined;
-    try {
-      parent = await resolveParent(issue);
-    } catch (err) {
-      hold(issue, "parent-lookup-error");
-      log(`#${issue} held: parent check failed (${errText(err)}) — retrying next tick`);
-      continue;
-    }
-    if (parent !== undefined) {
-      const occupied = occupiedParents.get(parent);
-      const blocker = occupied?.get(r.repo.name) ?? occupied?.get("");
-      // The gate serializes siblings under one epic: a held candidate must not
-      // proceed while a *different* child of the parent is occupied. But a
-      // candidate's own worker-free pushed-green row is exactly the work it is
-      // continuing, not a rival — the unblocked continuation of that same
-      // issue must not be rejected by its own occupancy, or the retained
-      // continuation deadlocks forever with the PR open.
-      if (blocker !== undefined && blocker !== issue) {
-        hold(issue, "sibling-active");
-        log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent} in ${r.repo.name}`);
-        continue;
-      }
-    }
-
-    // The busy set is built from run rows, so it can only speak for work this
-    // database recorded. Work pushed before this store existed — a migration, a
-    // wiped or relocated state dir, a restore onto a new host — looks exactly
-    // like fresh work, and a worker sent at it re-implements a finished PR. The
-    // tracker is the only party that remembers, so it is asked. The cost is
-    // bounded by free slots, not by queue depth: the call sits behind the two
-    // cheap local filters and candidates beyond capacity skip it.
-    let closer: OpenCloser | undefined;
-    try {
-      closer = await tracker.openCloserFor(issue);
-    } catch (err) {
-      // Fail closed, per candidate. An API error means "unknown whether
-      // finished work exists", and admitting on unknown recreates precisely the
-      // duplicate-work failure this guard exists to kill: the worst case of
-      // holding is a five-minute delay, the worst case of admitting is a burned
-      // attempt and a second PR on the same issue. Holding one candidate rather
-      // than aborting the loop keeps a transient GitHub failure from deadlocking
-      // the whole dispatcher; the next tick retries by itself.
-      hold(issue, "open-pr-lookup-error");
-      log(`#${issue} held: open-PR check failed (${errText(err)}) — retrying next tick`);
-      continue;
-    }
-    if (closer !== undefined) {
-      const latest = store.latestRun(project.name, issue);
-      // Terminality is the first half of the test and is not negotiable: while a
-      // run is live its worker is still pushing to that branch, and a second
-      // worker sent at the same PR is exactly the duplicate-work failure this
-      // guard exists to kill. Only a run that has stopped can be continued.
-      const retained =
-        latest?.state === "blocked" ||
-        latest?.state === "failed" ||
-        latest?.state === "killed" ||
-        latest?.state === "orphaned" ||
-        latest?.state === "pushed-green"
-          ? latest
-          : undefined;
-      // The second half asks "is this open PR our retained work", and accepts
-      // two identities for it, because the branch is the durable artefact of a
-      // retained run and the PR is not. A cap kill can end a run before any PR
-      // exists: veltro#324 attempt 1 was killed at the turns cap on
-      // 2026-08-09T00:47Z before its worker opened one, so the row kept `branch`
-      // and `prUrl` stayed NULL. chad#438 was opened from that exact branch
-      // afterwards, and URL equality — the only test 0.3.20 had — can never match
-      // a URL the terminal run never recorded, so every tick held #324 as
-      // `open-pr` until an operator closed recoverable work to free the branch
-      // (#50). An ordinary issue whose open PR is unrelated still fails both
-      // identities and stays ineligible, and an empty `headRefName` (a reply that
-      // did not carry the field) is never a match: unknown is not identity.
-      let resume: string | undefined;
-      if (retained !== undefined) {
-        if (retained.prUrl === closer.url) {
-          resume = `from ${retained.state} run (matched recorded PR URL)`;
-        } else if (closer.headRefName !== "" && retained.branch === closer.headRefName) {
-          resume = `from ${retained.state} run (matched retained branch ${closer.headRefName})`;
-        }
-      }
-      if (resume === undefined) {
-        hold(issue, "open-pr");
-        log(`#${issue} skipped: open PR ${closer.url} already closes it`);
-        continue;
-      }
-      log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
-    }
-
-    // The queue comes from GitHub's eventually-consistent search index. Re-read
-    // state and labels directly at the last possible moment so a just-closed or
-    // explicitly dequeued issue cannot turn a stale candidate into another
-    // attempt (#247).
-    let snapshot: IssueSnapshot | undefined;
-    try {
-      snapshot = await tracker.issueSnapshot(issue);
-    } catch {
-      snapshot = undefined;
-    }
-    if (snapshot === undefined) {
-      hold(issue, "issue-state-lookup-error");
-      log(`#${issue} held: issue snapshot check failed — retrying next tick`);
-      continue;
-    }
-    if (snapshot.state === "closed") {
-      hold(issue, "issue-closed");
-      log(`#${issue} skipped: issue is closed (search index lag)`);
-      continue;
-    }
-    if (!snapshot.labels.includes(project.queueLabel)) {
-      hold(issue, "issue-dequeued");
-      log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`);
-      continue;
-    }
-
-    admitted.push({ r, attempt: priorRuns + 1 });
-    liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
-    if (parent !== undefined) {
-      // Extend the epic's occupancy under this repo (slot empty by construction
-      // here — the gate above would have held the candidate otherwise) so a
-      // same-repo sibling later in this pass does not clear the gate (#197).
-      let siblings = occupiedParents.get(parent);
-      if (siblings === undefined) {
-        siblings = new Map();
-        occupiedParents.set(parent, siblings);
-      }
-      if (!siblings.has(r.repo.name)) siblings.set(r.repo.name, issue);
-    }
-  }
-
-  return { admitted, holds };
-}
-
 export interface WorkerPool {
   launch(work: Promise<void>): void;
   activeCount(): number;
@@ -4576,6 +4227,8 @@ export interface StatusSnapshot {
    */
   /** Current live-head push-workflow verdict per recently merged repository. */
   baseHealth: BaseHealth[];
+  /** Per-repo base-red merge freezes, active first (#283). */
+  freezes: BaseFreeze[];
   verbLedger: VerbLedgerEntry[];
   /** Runs backed by a worker process — the number capacity compares against. */
   liveWorkers: number;
@@ -4671,6 +4324,7 @@ export function statusSnapshotFromStore(
       ? {}
       : { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
     baseHealth: store.baseHealth(p.name),
+    freezes: store.freezes(p.name),
     ...(orchestratorDown === undefined ? {} : { orchestratorDown }),
   };
 }
@@ -4714,6 +4368,11 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
             ? ""
             : ` (#${hold.issues.join(", #")}${hold.count > hold.issues.length ? ", …" : ""})`;
         lines.push(`    ${hold.reason}  ${hold.count}${sample}`);
+        // A `file-lane` hold groups several issues, each blocked by a different
+        // file and holder; the grouped line says how many, this says which.
+        if ((hold.details?.length ?? 0) > 0) {
+          lines.push(`      ${hold.details!.join(" | ")}`);
+        }
       }
     }
   }
@@ -4783,6 +4442,25 @@ export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
   });
 }
 
+/**
+ * The active base-red freezes as status lines — merges refused until the base
+ * is green again or the operator overrides. Active freezes only: a cleared
+ * freeze is history the ledger and digest already told an operator about.
+ */
+export function formatFreezes(freezes: readonly BaseFreeze[]): string[] {
+  const active = freezes.filter((f) => f.clearedAt === undefined);
+  if (active.length === 0) return [];
+  return [
+    "frozen repos  (merges refused until base green)",
+    ...active.map(
+      (f) =>
+        `  ${f.repo}  base red at ${f.culpritSha.slice(0, 8)}` +
+        (f.detail === undefined ? "" : `  ${f.detail}`) +
+        `  — override: omp-conductor unfreeze ${f.repo}`,
+    ),
+  ];
+}
+
 
 export function formatStatus(s: StatusSnapshot): string {
   const lines = [
@@ -4832,6 +4510,7 @@ export function formatStatus(s: StatusSnapshot): string {
     }
   }
   lines.push(...formatBaseHealth(s.baseHealth));
+  lines.push(...formatFreezes(s.freezes));
   lines.push(...formatSalvagedRuns(s.salvagedRuns));
   lines.push(...formatOpenReports(s.openReports));
   lines.push(...formatVerbLedger(s.verbLedger));
@@ -5267,6 +4946,35 @@ async function recoverRun(
       log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
       return;
     }
+    // Same bound for provider-capacity: a run the provider throttled into the
+    // ground is requeued free (no attempt charged) — but a provider that
+    // throttles the same issue three times is at capacity, and a human has to
+    // check its status before hand-requeueing (#573). On a chain-configured
+    // project each requeue already moved the next attempt to the next chain
+    // model, so this escalation is what catches the no-chain case and the
+    // exhausted chain; it names every model the chain tried.
+    if (
+      cls === "provider-capacity" &&
+      store.classCountFor(project.name, run.issue, "provider-capacity") >= PROVIDER_CAPACITY_MAX_STRIKES
+    ) {
+      const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue: run.issue,
+        summary: `[provider-capacity] #${run.issue}: the model provider is throttling this run into the ground — ${evidence}`,
+        detail: [
+          `The provider answered #${run.issue} with sustained in-session rate limits ${PROVIDER_CAPACITY_MAX_STRIKES} times in a row; the harness retried each and was exhausted.`,
+          ...(tried === ""
+            ? []
+            : [`Models tried: ${tried}.`]),
+          "Check the provider's rate-limit status (and its throughput-oriented routes) before requeueing by hand.",
+        ].join("\n"),
+      });
+      store.updateRun(run.id, { recoveredAt: Date.now() });
+      log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
+      return;
+    }
     // Only when the tracker still shows this issue as ours to hand back. An
     // issue that is closed, or has no state label, was resolved by another route
     // and requeueing it would dispatch work nobody asked for.
@@ -5875,6 +5583,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
       cleanup: { next: 0 },
       probeCriticalBase: (repo, markers, branch) =>
         probeCriticalBase(project, repo, branch, markers),
+      probeWorktreeLane: (input) => probeRunLane(input),
       ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
       verbActions,
     };
@@ -5970,7 +5679,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
     // when it resumes and cannot create the work this shutdown is about to
     // wait for (#374).
     drain.draining = true;
-    log("shutting down after active workers finish");
+    log("shutdown requested — draining dispatch; live worker sessions are not waited for");
     pace.requestWake();
   };
   process.on("SIGINT", stop);