omp-conductor 0.15.13 → 0.16.1

package/src/admission.ts

@@ -0,0 +1,849 @@
+/**
+ * The admission half of the dispatcher (`daemon.ts` composes; this owns the
+ * gates).
+ *
+ * Extracted from `daemon.ts` so the most contested region — who gets a worker
+ * this tick — is a lane of its own: #283, #419, #420, #421, #507 and #555 all
+ * live here, and a diff touching only admission must not have to touch the
+ * composition root. Behavior is deliberately untouched by the move; a later
+ * slice reworks the seams.
+ *
+ * What admission is allowed to consult is the point of the module. It reads a
+ * structural slice of the daemon's deps — never the whole `Deps` and never the
+ * module that owns them, which would be the circular import the split exists
+ * to rule out. Adding a field to the daemon's deps must not break these tests.
+ */
+
+import { join } from "node:path";
+import { log, errText, safeEscalate } from "./log.ts";
+import type {
+  Caps,
+  Escalation,
+  IssueSnapshot,
+  OpenCloser,
+  ProjectConfig,
+  RunRecord,
+  Tracker,
+  Store,
+  AdmissionHoldReason,
+} from "./types.ts";
+import { readPlanUsage, type PlanUsageStatus, type UsageSource } from "./usage.ts";
+import type { CriticalBaseProbe, CriticalBaseVerdict, RunLaneProbe } from "./gitops.ts";
+import { repoSlugFor } from "./gitops.ts";
+import { branchName, type Routed } from "./routing.ts";
+import { parseDependsOn } from "./depends-on.ts";
+
+/** Fleet-wide escalations still need an issue number in the payload; 0 is the
+ *  sentinel that reads as "no issue" in every renderer. */
+const NO_ISSUE = 0;
+
+/**
+ * The slice of the daemon's `Deps` admission reads. Defined here rather than
+ * imported from `daemon.ts` (that would be the cycle this module exists to
+ * avoid); every `daemon.test.ts` fake already satisfies it, so the seam costs
+ * nothing.
+ */
+export interface AdmissionDeps {
+  project: ProjectConfig;
+  caps: Caps;
+  tracker: Tracker;
+  store: Store;
+  usage: UsageSource;
+  escalate(e: Escalation): Promise<void>;
+  probeCriticalBase?: CriticalBaseProbe;
+  probeWorktreeLane?: RunLaneProbe;
+  /**
+   * Reads one issue's tracker state in a repository the admission tracker is
+   * NOT bound to — the cross-repo half of the Depends-on interlock (#420).
+   * The tracker bound to `project.tracker.repo` can only answer same-repo
+   * references; a `owner/repo#n` prerequisite naming a different routed repo
+   * goes through this. Undefined means "could not tell" (same posture as
+   * {@link Tracker.issueSnapshot}); a routed prerequisite it cannot answer
+   * fails that candidate closed. Unset, admission fails a routed cross-repo
+   * prerequisite closed too — the daemon always wires it.
+   */
+  probeIssueIn?: (repo: string, issue: number) => Promise<IssueSnapshot | undefined>;
+}
+/** `stops` are the operational ends that each require one resume. */
+export function hasContinuationBudget(stops: number, maxContinuations: number): boolean {
+  return stops <= maxContinuations;
+}
+
+/** True while unspent failed-implementation attempts remain. This is the
+ *  dispatcher's admission gate: once every `maxAttemptsPerIssue` slot is
+ *  spent, the issue is held as `failed-attempts` forever, and the `unblock`
+ *  verb withholds the queue label on the same predicate (#348). */
+export function hasFailedAttemptBudget(failures: number, maxAttempts: number): boolean {
+  return failures < maxAttempts;
+}
+
+/** A candidate cleared for dispatch, with the attempt number it will run as. */
+export interface Admission {
+  r: Routed;
+  attempt: number;
+}
+
+export interface AdmissionHold {
+  issue: number;
+  reason: AdmissionHoldReason;
+  /** Human-readable "what" for the surface the reason group alone cannot say —
+   *  the overlapping file and the run holding it for `file-lane`. Survives only
+   *  far enough to be rendered; it never leaves the current pass's summary. */
+  detail?: string;
+}
+
+export interface AdmissionPass {
+  admitted: Admission[];
+  holds: AdmissionHold[];
+}
+
+
+/**
+ * What a held plan-usage gate says to a human, if anything.
+ *
+ * Three different problems hide behind one hold, and they want different
+ * tiers. Reaching the threshold is the guard *working*: tier 1, because the
+ * fleet resumes on its own at the provider's reset and nobody needs to get
+ * out of bed. Everything else — a window nothing reports, a window that
+ * resolves to two allowances, a meter that has been unreadable for half an
+ * hour — is dispatch stopped with no self-recovery, which is tier 2.
+ *
+ * Each summary carries the fact that will change when the situation does (the
+ * reset instant, the configured id, the date), because the escalation ledger
+ * dedupes on the summary: a stable one pages once and then goes quiet, which
+ * is right for a repeated tick and wrong for the next window.
+ */
+function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation | undefined {
+  const base = { project, issue: NO_ISSUE };
+  if (plan.state === "at-cap") {
+    const window = plan.window?.id ?? plan.cap?.windowId ?? "the configured window";
+    const resets =
+      plan.resetsAt === undefined
+        ? new Date().toISOString().slice(0, 10)
+        : new Date(plan.resetsAt).toISOString();
+    return {
+      ...base,
+      tier: 1,
+      summary: `Plan allowance cap reached on ${window} — ${project} is not claiming new work (window ${resets})`,
+      detail: [
+        plan.detail,
+        "Running workers finish normally; only new claims are held.",
+        "Dispatch resumes by itself once the provider reports the window reset or usage below the threshold —",
+        "no `resume` needed. Raise `caps.planUsage.maxUsedFraction` only if you mean to spend the rest.",
+      ].join("\n"),
+    };
+  }
+  if (plan.state === "blind") {
+    return {
+      ...base,
+      tier: 2,
+      category: "fleet-stopped",
+      // Dated: a meter that breaks again next month is a new incident, not a
+      // repeat of this one.
+      summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
+      detail: [
+        plan.detail,
+        "The guard admitted work while the failure looked transient and has now stopped.",
+        "Check `omp usage --json` on the fleet host, or set `caps.planUsage` to null if this fleet is unmetered.",
+      ].join("\n"),
+    };
+  }
+  if (
+    plan.state === "window-missing" ||
+    plan.state === "window-ambiguous" ||
+    plan.state === "window-uncomparable"
+  ) {
+    return {
+      ...base,
+      tier: 2,
+      category: "fleet-stopped",
+      summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
+      detail: [
+        plan.detail,
+        "Run `omp usage --json` and copy an allowance `id` into `caps.planUsage.windowId`,",
+        "or set `caps.planUsage` to null if this fleet is unmetered.",
+      ].join("\n"),
+    };
+  }
+  return undefined;
+}
+
+
+/**
+ * The candidate half of the file-lane interlock: the machine-readable list of
+ * files an issue declares it will touch (#555).
+ *
+ * The orchestrator already writes exactly this list into every promotion brief
+ * in prose; this parses that same sentence out of the issue body so the
+ * interlock is load-bearing rather than advisory. A body line beginning with
+ * "file lane" (case-insensitive, optional bold/heading markers) is accepted,
+ * and paths are read as backtick-delimited spans (the brief form) with a bare
+ * comma/space-separated fallback that keeps tokens that look like relative
+ * paths. Anything else — including an absent line, which is the default — is
+ * an empty lane: the issue is admitted exactly as today (`fail open`), and the
+ * gate never refuse work for wanting a lane. Exported so the format is pinned
+ * independent of admission.
+ */
+export function declaredLane(body: string): string[] {
+  const match = body.match(
+    /^\s*(?:[#>*-]\s*)*file[- ]lane\s*[:=]\s*([^\n]*)$/im,
+  );
+  if (match === null) return [];
+  const rest = match[1] ?? "";
+  const backticked = [...rest.matchAll(/`([^`]+)`/g)]
+    .map((m) => m[1]!.trim())
+    .filter(isPathLike);
+  if (backticked.length > 0) return [...new Set(backticked)];
+  return [...new Set(rest.split(/[,\s]+/).map((s) => s.trim()).filter(isPathLike))];
+}
+
+/** A plausible relative path: it has a `.` extension or a directory separator. */
+function isPathLike(token: string): boolean {
+  return token !== "" && !/\s/.test(token) && (token.includes("/") || /\.[A-Za-z0-9]{1,10}$/.test(token));
+}
+
+/**
+ * Where a `owner/repo` cross-repo reference can be read. Only repositories
+ * routed in THIS fleet are resolvable (#420): the issue source repo
+ * (`project.tracker.repo`) is answered by the existing tracker, and any routed
+ * work repo by {@link AdmissionDeps.probeIssueIn}. Anything else is unresolved
+ * — the caller fails closed rather than guess which repo was meant, and never
+ * silently resolves the reference against the candidate's own repo.
+ */
+function crossRepoTarget(project: ProjectConfig, ownerRepo: string): "tracker" | "routed" | undefined {
+  if (ownerRepo === project.tracker.repo) return "tracker";
+  for (const routed of Object.values(project.routing.repos)) {
+    if (repoSlugFor(routed) === ownerRepo) return "routed";
+  }
+  return undefined;
+}
+
+/**
+ * Which routed candidates get a worker this tick — in queue order, never more
+ * than `slots` of them. Every non-admission receives a stable reason code.
+ *
+ * Exported so the admission rules can be pinned without spawning a worker.
+ * Every one of them exists because of a live incident, and each guards a
+ * different way the same issue gets worked twice — including epic siblings
+ * racing onto the same files (#48).
+ *
+ * Takes the slice of `Deps` it actually reads rather than the whole thing: what
+ * admission is allowed to consult is the point of the function, and a `Deps`
+ * that grows a field has no business breaking these tests.
+ */
+export async function admitCandidates(
+  d: AdmissionDeps,
+  routed: Routed[],
+  slots: number,
+): Promise<AdmissionPass> {
+  const { project, caps, tracker, store } = d;
+  const activeRuns = store.activeRuns(project.name);
+  const busyIssues = activeRuns.map((r) => r.issue);
+  const busy = new Set(busyIssues);
+  // issue -> its active run rows, for the pushed-green admission bypass (#175):
+  // only a worker-free pushed-green row may be bypassed, and only when *every*
+  // active run for the issue is worker-free. A live (claimed/running) row still
+  // holds unconditionally.
+  const activeByIssue = new Map<number, RunRecord[]>();
+  for (const run of activeRuns) {
+    const list = activeByIssue.get(run.issue);
+    if (list === undefined) activeByIssue.set(run.issue, [run]);
+    else list.push(run);
+  }
+  // Live worker count per repo, seeded from live runs and incremented as this
+  // same pass admits — so two same-repo candidates can never both clear the
+  // per-repo cap in one tick (#186).
+  const liveByRepo = new Map<string, number>();
+  for (const run of store.liveRuns(project.name)) {
+    liveByRepo.set(run.repo, (liveByRepo.get(run.repo) ?? 0) + 1);
+  }
+  const holds: AdmissionHold[] = [];
+  const hold = (issue: number, reason: AdmissionHoldReason, detail?: string): void => {
+    holds.push({ issue, reason, ...(detail === undefined ? {} : { detail }) });
+  };
+
+  // The file-lane interlock (#555): repo -> file -> the issue holding that file.
+  // Seeded from the *actual* worktrees/branches of active runs — never from
+  // their issue bodies, which is precisely how #288 collided — and extended as
+  // this pass admits candidates with declared lanes, so two overlapping
+  // candidates cannot both clear the gate in one tick. Keyed by repo because a
+  // path only collides within its own checkout: a `daemon.ts` on the api repo
+  // and one on the web repo are different files. Built lazily and once, only
+  // when the first candidate that declares a lane reaches the gate, so a queue
+  // of laneless issues pays nothing for it.
+  let laneOccupancy: Map<string, Map<string, number>> | undefined;
+  const ensureLaneOccupancy = async (): Promise<Map<string, Map<string, number>>> => {
+    if (laneOccupancy !== undefined) return laneOccupancy;
+    const occupied = new Map<string, Map<string, number>>();
+    for (const run of activeRuns) {
+      if (d.probeWorktreeLane === undefined) break;
+      const base = project.routing.repos[run.repo]?.defaultBranch ?? "main";
+      let files: string[];
+      try {
+        files = await d.probeWorktreeLane({
+          worktree: run.worktree,
+          baseRef: `refs/remotes/origin/${base}`,
+          ...(run.branch === "" ? {} : { branchRef: `refs/heads/${run.branch}` }),
+          mirror: join(project.mirrorRoot, `${run.repo}.git`),
+        });
+      } catch {
+        // Fail open, like an unreadable lane: the gate never refuses a
+        // well-formed issue because one probe could not be answered.
+        files = [];
+      }
+      for (const file of files) {
+        if (occupied.get(run.repo)?.has(file) === true) continue;
+        let perRepo = occupied.get(run.repo);
+        if (perRepo === undefined) {
+          perRepo = new Map();
+          occupied.set(run.repo, perRepo);
+        }
+        perRepo.set(file, run.issue);
+      }
+    }
+    laneOccupancy = occupied;
+    return occupied;
+  };
+
+  // The plan allowance is a fleet-wide question, so it is asked once per pass
+  // and answers for every candidate — unlike every gate below it, which is
+  // per-issue. It sits here rather than beside the spend cap in `tick` for one
+  // reason: the spend cap *pauses the daemon* and waits for a human, and a
+  // weekly plan window resets by itself. A guard that demanded `resume` after
+  // every rollover would cost more operator attention than the guard saves
+  // (#110). Already-running workers are untouched and settle normally.
+  //
+  // Placed after the cheap local busy-set read and before the first tracker
+  // call, so a held fleet spends no GitHub API budget discovering it is held.
+  const plan = await readPlanUsage(caps.planUsage, d.usage);
+  if (plan.blocking) {
+    for (const r of routed) hold(r.issue.number, "plan-usage-cap");
+    log(`plan usage gate holding ${String(routed.length)} candidate(s): ${plan.detail}`);
+    const escalation = planUsageEscalation(project.name, plan);
+    if (escalation !== undefined) await safeEscalate(d, escalation);
+    return { admitted: [], holds };
+  }
+
+  // parent -> repo name -> blocking issue. Seeded from active runs (including
+  // pushed-green), then extended by candidates admitted earlier in this same
+  // pass so two siblings of one epic never both clear the gate in one tick.
+  // A busy issue whose run row cannot be resolved occupies the sentinel repo
+  // "" — treated as matching every repo, failing toward holding (#197).
+  const occupiedParents = new Map<number, Map<string, number>>();
+  const parentCache = new Map<number, number | undefined>();
+
+  const resolveParent = async (issue: number): Promise<number | undefined> => {
+    if (parentCache.has(issue)) return parentCache.get(issue);
+    const parent = await tracker.parentOf(issue);
+    parentCache.set(issue, parent);
+    return parent;
+  };
+
+  // Bounded by concurrent workers, not queue depth. A failed lookup here cannot
+  // mark an epic occupied; candidates still fail closed on their own parentOf.
+  for (const issue of busyIssues) {
+    try {
+      const parent = await resolveParent(issue);
+      if (parent === undefined) continue;
+      // The runs table records which repo each attempt worked in, and sibling
+      // holds are now per-repo, so a busy child only occupies its epic under
+      // that repo's name (same spelling as `createRun` writes from
+      // `r.repo.name`). A busy issue with no resolvable run row occupies the
+      // sentinel "" instead — matching every repo, failing toward holding.
+      const repo = store.latestRun(project.name, issue)?.repo ?? "";
+      const siblings = occupiedParents.get(parent);
+      if (siblings === undefined) {
+        occupiedParents.set(parent, new Map([[repo, issue]]));
+      } else if (!siblings.has(repo) && !siblings.has("")) {
+        siblings.set(repo, issue);
+      }
+    } catch (err) {
+      log(`#${issue} parent lookup failed while seeding epic occupancy (${errText(err)})`);
+    }
+  }
+
+  const admitted: Admission[] = [];
+  for (const r of routed) {
+    const issue = r.issue.number;
+    if (admitted.length >= slots) {
+      hold(issue, "capacity");
+      continue;
+    }
+    if (busy.has(issue)) {
+      // A pushed-green row is worker-free by definition (it is not in
+      // LIVE_STATES): its PR is live but no process is writing to its branch.
+      // So an issue whose active runs are ALL pushed-green is not actually
+      // occupied — the corrective attempt the operator unblocked may be
+      // admitted as a continuation of that PR, and the open-PR gate below
+      // decides the identity. Any live row still holds (#175).
+      const allWorkerFree = (activeByIssue.get(issue) ?? []).every((r) => r.state === "pushed-green");
+      if (!allWorkerFree) {
+        hold(issue, "issue-active");
+        continue;
+      }
+    }
+
+    // Per-repo concurrency: the mirror, branch-protection staleness and shared
+    // CI egress are all per-repo collision domains, so extra slots should land
+    // on other repos rather than stacking workers into the same one (#186).
+    const liveInRepo = liveByRepo.get(r.repo.name) ?? 0;
+    if (liveInRepo >= caps.maxConcurrentWorkersPerRepo) {
+      hold(issue, "repo-active");
+      log(`#${issue} skipped: ${liveInRepo} live worker(s) already in ${r.repo.name} (cap ${caps.maxConcurrentWorkersPerRepo})`);
+      continue;
+    }
+
+    const priorRuns = store.attemptsFor(project.name, issue);
+    const failures = store.failuresFor(project.name, issue);
+    if (!hasFailedAttemptBudget(failures, caps.maxAttemptsPerIssue)) {
+      hold(issue, "failed-attempts");
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue,
+        summary: `#${issue} has used all ${caps.maxAttemptsPerIssue} failed attempts`,
+        detail: [
+          r.issue.title,
+          r.issue.url,
+          "Another implementation attempt almost always means the issue itself is underspecified.",
+          "Rewrite the acceptance criteria, or take it off the queue.",
+        ].join("\n"),
+      });
+      continue;
+    }
+
+    const continuations = store.continuationsFor(project.name, issue);
+    if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
+      hold(issue, "continuations");
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue,
+        summary: `#${issue} exceeded its ${caps.maxContinuationsPerIssue}-continuation budget`,
+        detail: [
+          r.issue.title,
+          r.issue.url,
+          "Repeated cap kills, daemon orphans, or answered blocks need an operator to inspect progress.",
+        ].join("\n"),
+      });
+      continue;
+    }
+
+    // Fail closed on work that exists only in a run repo. `addRunRepo` clears
+    // the tree at <workspaceRoot>/<issue> before it provisions, so admitting
+    // this issue is what finally destroys the copy the salvage could not save
+    // (#118). Nothing here can recover it — git already refused once — so the
+    // only safe move is to refuse the claim and keep saying why until an
+    // operator has looked and run `unblock --force`.
+    const newest = store.latestRun(project.name, issue);
+    if (newest?.salvageError !== undefined && newest.salvageAckAt === undefined) {
+      hold(issue, "unsalvaged-wip");
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue,
+        summary: `#${issue} is holding unsalvaged work and will not be re-claimed`,
+        detail: [
+          r.issue.title,
+          r.issue.url,
+          `Attempt ${newest.attempt} could not commit its uncommitted changes: ${newest.salvageError}`,
+          `The only copy is the worktree ${newest.worktree === "" ? "(path not recorded)" : newest.worktree}.`,
+          "Dispatch is held because claiming this issue removes that tree.",
+          "Recover it by hand, then `omp-conductor unblock <n> --force` to release the hold.",
+        ].join("\n"),
+      });
+      continue;
+    }
+
+    // #428 half (a): a preserved continuation that predates a configured
+    // critical-base/safety marker must not be reattached. A base safety fix
+    // protects only branches forked after it landed — a continuation forked
+    // before it still carries the dangerous test/runtime code, and re-running
+    // it on the shared host is what SIGTERMed the production daemon. Fail
+    // closed: only a probe that proves every marker is in the reattach
+    // source's ancestry admits, and a project that names a marker but has no
+    // probe wired (never happens outside tests) holds. Both the hold and the
+    // escalation are durable across restart and orphan recovery because this
+    // gate runs every admission pass; the branch is re-admitted automatically
+    // once the operator updates it to contain the marker, without losing work.
+    const markers = project.criticalBase ?? [];
+    if (markers.length > 0) {
+      const branch = branchName(r.issue);
+      let verdict: CriticalBaseVerdict;
+      if (d.probeCriticalBase === undefined) {
+        verdict = { state: "unknown", error: "no critical-base probe is wired in this deployment" };
+      } else {
+        try {
+          verdict = await d.probeCriticalBase(r.repo, markers, branch);
+        } catch (err) {
+          verdict = { state: "unknown", error: errText(err) };
+        }
+      }
+      if (verdict.state === "stale") {
+        hold(issue, "stale-base");
+        log(
+          `#${issue} held (stale-base): continuation branch ${branch} predates critical-base marker ${verdict.marker}`,
+        );
+        await safeEscalate(d, {
+          tier: 1,
+          project: project.name,
+          issue,
+          summary: `#${issue} continuation branch predates a critical base safety commit and is held (stale-base)`,
+          detail: [
+            r.issue.title,
+            r.issue.url,
+            `The retained branch ${branch} does not contain critical-base marker ${verdict.marker}.`,
+            ...(verdict.range.length > 0
+              ? [`Base commits the branch is missing: ${verdict.range.join(", ")}`]
+              : []),
+            "Recovery: merge current base into the branch so it contains the marker, and the next",
+            "admission pass re-admits it automatically without losing the branch's work; or review",
+            "the branch by hand and clear the hold once the fix is present.",
+          ].join("\n"),
+        });
+        continue;
+      }
+      if (verdict.state === "unknown") {
+        // Fail closed: a branch that cannot be *proven* to contain the marker
+        // is refused, and the reason names the unverifiable marker so the
+        // operator can fix the fetch or the marker rather than guess.
+        hold(issue, "stale-base");
+        log(
+          `#${issue} held (stale-base): continuation branch ${branch} could not be verified ` +
+            `against critical-base marker(s) ${markers.join(", ")} (${verdict.error})`,
+        );
+        await safeEscalate(d, {
+          tier: 1,
+          project: project.name,
+          issue,
+          summary: `#${issue} continuation branch could not be verified against a critical base safety commit and is held (stale-base)`,
+          detail: [
+            r.issue.title,
+            r.issue.url,
+            `The retained branch ${branch} could not be verified against critical-base marker(s) ${markers.join(", ")}: ${verdict.error}`,
+            "Recovery: merge current base into the branch so it contains the marker, and the next",
+            "admission pass re-admits it automatically without losing the branch's work; or review",
+            "the branch by hand and clear the hold once the fix is present.",
+          ].join("\n"),
+        });
+        continue;
+      }
+    }
+
+    // Soft concurrency per epic, per repository: at most one in-flight child of
+    // a given parent in each repo. Children of one epic in *different* repos
+    // parallelise freely — `repo-active` / `maxConcurrentWorkersPerRepo` owns
+    // the same-repo collision domain (#197). The "" sentinel matches every
+    // repo. No parent means today's concurrent admission. Cheap local filters
+    // already ran; this sits before the open-PR API call so a held sibling
+    // frees the slot for unrelated work without spending a closers query.
+    let parent: number | undefined;
+    try {
+      parent = await resolveParent(issue);
+    } catch (err) {
+      hold(issue, "parent-lookup-error");
+      log(`#${issue} held: parent check failed (${errText(err)}) — retrying next tick`);
+      continue;
+    }
+    if (parent !== undefined) {
+      const occupied = occupiedParents.get(parent);
+      const blocker = occupied?.get(r.repo.name) ?? occupied?.get("");
+      // The gate serializes siblings under one epic: a held candidate must not
+      // proceed while a *different* child of the parent is occupied. But a
+      // candidate's own worker-free pushed-green row is exactly the work it is
+      // continuing, not a rival — the unblocked continuation of that same
+      // issue must not be rejected by its own occupancy, or the retained
+      // continuation deadlocks forever with the PR open.
+      if (blocker !== undefined && blocker !== issue) {
+        hold(issue, "sibling-active");
+        log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent} in ${r.repo.name}`);
+        continue;
+      }
+    }
+
+    // The busy set is built from run rows, so it can only speak for work this
+    // database recorded. Work pushed before this store existed — a migration, a
+    // wiped or relocated state dir, a restore onto a new host — looks exactly
+    // like fresh work, and a worker sent at it re-implements a finished PR. The
+    // tracker is the only party that remembers, so it is asked. The cost is
+    // bounded by free slots, not by queue depth: the call sits behind the two
+    // cheap local filters and candidates beyond capacity skip it.
+    let closer: OpenCloser | undefined;
+    try {
+      closer = await tracker.openCloserFor(issue);
+    } catch (err) {
+      // Fail closed, per candidate. An API error means "unknown whether
+      // finished work exists", and admitting on unknown recreates precisely the
+      // duplicate-work failure this guard exists to kill: the worst case of
+      // holding is a five-minute delay, the worst case of admitting is a burned
+      // attempt and a second PR on the same issue. Holding one candidate rather
+      // than aborting the loop keeps a transient GitHub failure from deadlocking
+      // the whole dispatcher; the next tick retries by itself.
+      hold(issue, "open-pr-lookup-error");
+      log(`#${issue} held: open-PR check failed (${errText(err)}) — retrying next tick`);
+      continue;
+    }
+    if (closer !== undefined) {
+      const latest = store.latestRun(project.name, issue);
+      // Terminality is the first half of the test and is not negotiable: while a
+      // run is live its worker is still pushing to that branch, and a second
+      // worker sent at the same PR is exactly the duplicate-work failure this
+      // guard exists to kill. Only a run that has stopped can be continued.
+      const retained =
+        latest?.state === "blocked" ||
+        latest?.state === "failed" ||
+        latest?.state === "killed" ||
+        latest?.state === "orphaned" ||
+        latest?.state === "pushed-green"
+          ? latest
+          : undefined;
+      // The second half asks "is this open PR our retained work", and accepts
+      // two identities for it, because the branch is the durable artefact of a
+      // retained run and the PR is not. A cap kill can end a run before any PR
+      // exists: veltro#324 attempt 1 was killed at the turns cap on
+      // 2026-08-09T00:47Z before its worker opened one, so the row kept `branch`
+      // and `prUrl` stayed NULL. chad#438 was opened from that exact branch
+      // afterwards, and URL equality — the only test 0.3.20 had — can never match
+      // a URL the terminal run never recorded, so every tick held #324 as
+      // `open-pr` until an operator closed recoverable work to free the branch
+      // (#50). An ordinary issue whose open PR is unrelated still fails both
+      // identities and stays ineligible, and an empty `headRefName` (a reply that
+      // did not carry the field) is never a match: unknown is not identity.
+      let resume: string | undefined;
+      if (retained !== undefined) {
+        if (retained.prUrl === closer.url) {
+          resume = `from ${retained.state} run (matched recorded PR URL)`;
+        } else if (closer.headRefName !== "" && retained.branch === closer.headRefName) {
+          resume = `from ${retained.state} run (matched retained branch ${closer.headRefName})`;
+        }
+      }
+      if (resume === undefined) {
+        hold(issue, "open-pr");
+        log(`#${issue} skipped: open PR ${closer.url} already closes it`);
+        continue;
+      }
+      log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
+    }
+
+    // The queue comes from GitHub's eventually-consistent search index. Re-read
+    // state and labels directly at the last possible moment so a just-closed or
+    // explicitly dequeued issue cannot turn a stale candidate into another
+    // attempt (#247).
+    let snapshot: IssueSnapshot | undefined;
+    try {
+      snapshot = await tracker.issueSnapshot(issue);
+    } catch {
+      snapshot = undefined;
+    }
+    if (snapshot === undefined) {
+      hold(issue, "issue-state-lookup-error");
+      log(`#${issue} held: issue snapshot check failed — retrying next tick`);
+      continue;
+    }
+    if (snapshot.state === "closed") {
+      hold(issue, "issue-closed");
+      log(`#${issue} skipped: issue is closed (search index lag)`);
+      continue;
+    }
+    if (!snapshot.labels.includes(project.queueLabel)) {
+      hold(issue, "issue-dequeued");
+      log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`);
+      continue;
+    }
+
+    // The Depends-on interlock (#419/#420): a candidate declares the issues it
+    // must not be dispatched before — same-repo (`#123`) or cross-repo
+    // (`owner/repo#123`) — and any *open* prerequisite holds it until every
+    // referenced issue is closed. Prerequisite state is read fresh from the
+    // tracker at claim time, every pass — never cached across ticks — so a
+    // prerequisite that reopens re-holds on the next tick. Cross-repo
+    // references resolve only against repositories routed in this fleet;
+    // graph cycles are a later slice (#421).
+    const dependsOn = parseDependsOn(r.issue.body);
+    if (dependsOn.malformed.length > 0) {
+      // A marker line with no strict `#<n>` reference (e.g. `Depends-on:
+      // #abc`) is ignored as a gate — it never holds, and it never crashes —
+      // but it is flagged so the issue is never *silently* dispatched with a
+      // dependency nobody read. Emitted once: the escalation ledger dedupes on
+      // the stable project/issue/tier/summary key, so repeated ticks do not
+      // re-page the orchestrator for the same malformed body.
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue,
+        summary: `#${issue} has a malformed Depends-on declaration and its undeclared prerequisite was not held`,
+        detail: [
+          r.issue.title,
+          r.issue.url,
+          ...dependsOn.malformed.map((line) => `Malformed declaration: ${line}`),
+          "Depends-on references must be bare issue numbers (`#<n>`). Groom the body",
+          "so the dependency is honoured, or the issue is dispatched without it.",
+        ].join("\n"),
+      });
+      log(`#${issue} flagged: malformed depends-on declaration(s): ${dependsOn.malformed.join("; ")}`);
+    }
+    if (dependsOn.refs.length > 0) {
+      let blocking: number | undefined;
+      let unreadable: number | undefined;
+      for (const ref of dependsOn.refs) {
+        let state: IssueSnapshot | undefined;
+        try {
+          state = await tracker.issueSnapshot(ref);
+        } catch {
+          state = undefined;
+        }
+        if (state?.state === "open") {
+          blocking = ref;
+          break;
+        }
+        if (state === undefined) {
+          unreadable = ref;
+          break;
+        }
+      }
+      // Fail closed on an unreadable prerequisite: admitting on unknown would
+      // dispatch work whose prerequisite may still be open, and the next tick
+      // rereads the tracker itself, so a transient failure just defers.
+      if (blocking !== undefined) {
+        hold(issue, "depends-on", `blocked by #${blocking}`);
+        log(`#${issue} held (depends-on): prerequisite #${blocking} is open`);
+        continue;
+      }
+      if (unreadable !== undefined) {
+        hold(issue, "depends-on", `prerequisite #${unreadable} state unreadable`);
+        log(`#${issue} held (depends-on): prerequisite #${unreadable} state unreadable`);
+        continue;
+      }
+    }
+    if (dependsOn.crossRefs.length > 0) {
+      // The cross-repo half (#420): each `owner/repo#n` is read from the repo
+      // it names. The fully qualified form travels in the `detail` string, so
+      // status/digest render it as `blocked by owner/repo#123` while same-repo
+      // refs stay compact `blocked by #5`.
+      let blocking: string | undefined;
+      let unreadable: string | undefined;
+      let unresolved: string | undefined;
+      for (const ref of dependsOn.crossRefs) {
+        const where = crossRepoTarget(project, ref.repo);
+        const full = `${ref.repo}#${ref.issue}`;
+        if (where === undefined) {
+          unresolved = full;
+          break;
+        }
+        let state: IssueSnapshot | undefined;
+        try {
+          if (where === "tracker") {
+            // The qualified form of the candidate's own tracker repo: the
+            // same state read the same-repo interlock uses.
+            state = await tracker.issueSnapshot(ref.issue);
+          } else if (d.probeIssueIn === undefined) {
+            // No resolver wired — fail closed, never guess the issue's state.
+            state = undefined;
+          } else {
+            state = await d.probeIssueIn(ref.repo, ref.issue);
+          }
+        } catch {
+          state = undefined;
+        }
+        if (state?.state === "open") {
+          blocking = full;
+          break;
+        }
+        if (state === undefined) {
+          unreadable = full;
+          break;
+        }
+      }
+      // Same fail-closed posture as same-repo refs: an open prerequisite
+      // holds, an unreadable one holds (the next tick rereads it fresh), and
+      // an unresolved repo holds because the dispatcher cannot verify the
+      // dependency — it is never silently treated as closed. Only the
+      // unresolved repo also surfaces one material event for grooming, and it
+      // follows the malformed-declaration escalation's stable-summary
+      // dedupe, so repeated ticks page once, not per tick.
+      if (blocking !== undefined) {
+        hold(issue, "depends-on", `blocked by ${blocking}`);
+        log(`#${issue} held (depends-on): cross-repo prerequisite ${blocking} is open`);
+        continue;
+      }
+      if (unreadable !== undefined) {
+        hold(issue, "depends-on", `prerequisite ${unreadable} state unreadable`);
+        log(`#${issue} held (depends-on): cross-repo prerequisite ${unreadable} state unreadable`);
+        continue;
+      }
+      if (unresolved !== undefined) {
+        hold(issue, "depends-on", `blocked by ${unresolved} — repo not routed in this fleet`);
+        log(`#${issue} held (depends-on): cross-repo prerequisite ${unresolved} is not a routed repo`);
+        await safeEscalate(d, {
+          tier: 1,
+          project: project.name,
+          issue,
+          summary: `#${issue} depends on ${unresolved}, a repo not routed in this fleet — held until groomed`,
+          detail: [
+            r.issue.title,
+            r.issue.url,
+            `Unroutable prerequisite: ${unresolved}`,
+            "Depends-on cross-repo references must name a repository routed in this project",
+            "(`routing.repos` or the tracker repo). Groom the body or route the repo.",
+          ].join("\n"),
+        });
+        continue;
+      }
+    }
+
+    // The file-lane interlock (#555): a candidate whose declared lane overlaps
+    // a live run's *actual* lane is held until that run's work has merged, so
+    // no second worker is sent at files another worker is still writing. The
+    // gate is the mechanical version of the prose rule that failed three times
+    // in one day. Only the machine-readable lane participates: a candidate
+    // without one is admitted exactly as today (fail open), and a candidate's
+    // own retained run never holds it — that is the continuation it continues.
+    const lane = declaredLane(r.issue.body);
+    if (lane.length > 0) {
+      const occupied = await ensureLaneOccupancy();
+      const perRepo = occupied.get(r.repo.name);
+      let blocked = false;
+      for (const file of lane) {
+        const holder = perRepo?.get(file);
+        if (holder !== undefined && holder !== issue) {
+          const detail = `${file} held by run #${holder}`;
+          hold(issue, "file-lane", detail);
+          log(`#${issue} held (file-lane): ${detail}`);
+          blocked = true;
+          break;
+        }
+      }
+      if (blocked) continue;
+    }
+
+    admitted.push({ r, attempt: priorRuns + 1 });
+    liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
+    // Same-pass sibling occupancy for the file-lane gate: once admitted, a
+    // candidate's declared lane occupies for the rest of the pass, so a later
+    // overlapping candidate is held rather than both clearing in one tick.
+    if (lane.length > 0) {
+      const occupied = await ensureLaneOccupancy();
+      let perRepo = occupied.get(r.repo.name);
+      if (perRepo === undefined) {
+        perRepo = new Map();
+        occupied.set(r.repo.name, perRepo);
+      }
+      for (const file of lane) {
+        if (!perRepo.has(file)) perRepo.set(file, issue);
+      }
+    }
+    if (parent !== undefined) {
+      // Extend the epic's occupancy under this repo (slot empty by construction
+      // here — the gate above would have held the candidate otherwise) so a
+      // same-repo sibling later in this pass does not clear the gate (#197).
+      let siblings = occupiedParents.get(parent);
+      if (siblings === undefined) {
+        siblings = new Map();
+        occupiedParents.set(parent, siblings);
+      }
+      if (!siblings.has(r.repo.name)) siblings.set(r.repo.name, issue);
+    }
+  }
+
+  return { admitted, holds };
+}