omp-conductor 0.15.11 → 0.15.12

package/src/doctor.ts

@@ -462,7 +462,7 @@ async function labelProbe(probes: Probes, project: ProjectConfig): Promise<Findi
   if (!read.ok) {
     return failFinding(
       "labels",
-      `cannot read the labels of ${project.tracker.repo}${read.detail === undefined ? "" : ` (${read.detail})`}`,
+      `[${project.name}] cannot read the labels of ${project.tracker.repo}${read.detail === undefined ? "" : ` (${read.detail})`}`,
       "check `gh` can list that repo's labels — the daemon reads them the same way",
     );
   }
@@ -474,7 +474,7 @@ async function labelProbe(probes: Probes, project: ProjectConfig): Promise<Findi
   }
   const missing = wanted.filter((w) => !present.has(w.name));
   if (missing.length === 0) {
-    return passFinding("labels", `${wanted.length} configured label(s) on ${project.tracker.repo} match exactly`);
+    return passFinding("labels", `[${project.name}] ${wanted.length} configured label(s) on ${project.tracker.repo} match exactly`);
   }
   const detail = missing.map((m) => {
     const near = nearByName.get(norm(m.name));
@@ -486,17 +486,60 @@ async function labelProbe(probes: Probes, project: ProjectConfig): Promise<Findi
   });
   return failFinding(
     "labels",
-    `${detail.join("; ")} — ${INCIDENTS.labels}`,
+    `[${project.name}] ${detail.join("; ")} — ${INCIDENTS.labels}`,
     allNearMisses
       ? "fix the label case/format in the tracker repo (or change the config to the actual spelling)"
       : "create the missing label(s) in the tracker repo (`gh label create`, or re-run `omp-conductor setup`)",
   );
 }
 
-/** Line-level drift between an installed unit and the canonical render. */
+/**
+ * A unit line whose canonical value is a property of the *rendering process*,
+ * not the deployment. The herdr unit's `Environment="SHELL=…"` carries the
+ * account's login shell as it resolved in the process that staged it (#463 —
+ * the pane-shell pin that keeps panes off dash). Doctor cannot re-derive that
+ * value: in a non-login/tick context `userInfo().shell` resolves to nothing and
+ * renders `SHELL=unknown`, while the unit that was actually staged carries
+ * `SHELL=/bin/bash`. Comparing against a value that answers to "whoever asked"
+ * can never pass, so it is excluded from the drift comparison and reported as
+ * not-comparable, never as drift (#511).
+ */
+const PROCESS_DERIVED_UNIT_LINE = /^Environment="SHELL=/;
+
+/**
+ * Whether a canonical unit's pane-shell value could not be resolved — the
+ * `SHELL=` pin rendered when the invoking process has no login shell (an empty
+ * value, or the literal `unknown` userInfo returns in a non-login context).
+ * Excluded from the drift comparison, such a value is reported as
+ * not-comparable rather than implied to match (#511).
+ */
+function unresolvableShell(unit: string): boolean {
+  const line = unit
+    .split("\n")
+    .map((l) => l.trim())
+    .find((l) => PROCESS_DERIVED_UNIT_LINE.test(l));
+  if (line === undefined) return false;
+  const value = /^Environment="SHELL=(.*)"$/.exec(line)?.[1] ?? "";
+  return value === "" || value === "unknown";
+}
+
+/**
+ * Line-level drift between an installed unit and the canonical render, ignoring
+ * lines whose canonical value is not re-derivable by doctor (the
+ * {@link PROCESS_DERIVED_UNIT_LINE} pane-shell pin). The pane shell is host
+ * state resolved at staging time, so a rendered value doctor cannot reproduce
+ * is excluded from both sides rather than reported as a difference — an
+ * unresolvable canonical is "cannot compare", never drift (#511).
+ */
 export function unitDrift(installed: string, canonical: string): { differences: string[] } {
-  const want = canonical.split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
-  const have = installed.split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
+  const want = canonical
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l.length > 0 && !PROCESS_DERIVED_UNIT_LINE.test(l));
+  const have = installed
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l.length > 0 && !PROCESS_DERIVED_UNIT_LINE.test(l));
   const wantSet = new Set(want);
   const haveSet = new Set(have);
   const differences = [
@@ -516,8 +559,9 @@ function driftLines(name: string, installed: string, canonical: string): string[
 
 /**
  * Installed unit vs the canonical staged rendering: any drift in User=,
- * Environment, WorkingDirectory, ExecStart, Restart/SuccessExitStatus, or
- * memory lines is the unit-drift class that once killed workers at turn 0.
+ * Environment (save the process-derived pane shell, which is not comparable),
+ * WorkingDirectory, ExecStart, Restart/SuccessExitStatus, or memory lines is
+ * the unit-drift class that once killed workers at turn 0.
  */
 function unitProbe(probes: Probes, project: ProjectConfig | undefined, cfg: ConductorConfig | undefined): Finding {
   if (!probes.hasSystemd()) return passFinding("systemd-unit", "no systemd directory on this host — nothing to check");
@@ -530,16 +574,21 @@ function unitProbe(probes: Probes, project: ProjectConfig | undefined, cfg: Cond
     return warnFinding("systemd-unit", `${STAGED_SERVICE_NAME} is not installed`, "install it: run `omp-conductor setup host` from the fleet account");
   }
   const problems = driftLines(STAGED_SERVICE_NAME, installedDaemon, canonical.daemon);
+  const uncomparable: string[] = [];
   if (canonical.herdr !== undefined) {
     const installedHerdr = probes.readUnit(join(SYSTEMD_UNIT_DIR, DEFAULT_HERDR_UNIT));
     if (installedHerdr === undefined) {
       problems.push(`${DEFAULT_HERDR_UNIT} is not installed (the staged plan provisions it)`);
     } else {
+      if (unresolvableShell(canonical.herdr)) {
+        uncomparable.push("the pane-shell SHELL value is not comparable (it resolves from this process, not the deployment)");
+      }
       problems.push(...driftLines(DEFAULT_HERDR_UNIT, installedHerdr, canonical.herdr));
     }
   }
   if (problems.length === 0) {
-    return passFinding("systemd-unit", "installed units match the staged render");
+    const note = uncomparable.length === 0 ? "" : ` — ${uncomparable.join("; ")}`;
+    return passFinding("systemd-unit", `installed units match the staged render${note}`);
   }
   return failFinding(
     "systemd-unit",
@@ -548,6 +597,46 @@ function unitProbe(probes: Probes, project: ProjectConfig | undefined, cfg: Cond
   );
 }
 
+/** The `OnFailure=` targets an installed unit names, each a systemd unit file. */
+function onFailureTargets(unitText: string): string[] {
+  return (unitText.match(/^OnFailure=(.*)$/gm) ?? []).flatMap((line) =>
+    line.slice("OnFailure=".length).trim().split(/\s+/).filter(Boolean),
+  );
+}
+
+/**
+ * Every `OnFailure=` target an installed fleet unit names must itself be
+ * installed. The fleet units carry the line from day one (#485); the failure
+ * this check exists for is the line present while the recovery oneshot it
+ * names is not installed — so systemd silently refuses to enqueue the job and
+ * a failed unit never recovers (#509).
+ */
+function recoveryProbe(probes: Probes): Finding {
+  if (!probes.hasSystemd()) return passFinding("systemd-recovery", "no systemd directory on this host — nothing to check");
+  const problems: string[] = [];
+  const fleetUnits: readonly [string, string][] = [
+    [STAGED_SERVICE_NAME, join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME)],
+    [DEFAULT_HERDR_UNIT, join(SYSTEMD_UNIT_DIR, DEFAULT_HERDR_UNIT)],
+  ];
+  for (const [name, path] of fleetUnits) {
+    const text = probes.readUnit(path);
+    if (text === undefined) continue; // not installed — the unit/ownership probes name it
+    for (const target of onFailureTargets(text)) {
+      if (probes.readUnit(join(SYSTEMD_UNIT_DIR, target)) === undefined) {
+        problems.push(`${name} names OnFailure=${target}, which is not installed`);
+      }
+    }
+  }
+  if (problems.length === 0) {
+    return passFinding("systemd-recovery", "every OnFailure= target names an installed unit");
+  }
+  return failFinding(
+    "systemd-recovery",
+    `${problems.join("; ")} — systemd cannot enqueue the recovery a failed fleet unit needs (#485/#509)`,
+    "install the recovery unit and playbook: run `omp-conductor setup host` from the fleet account",
+  );
+}
+
 function installedUnitUser(probes: Probes): string | undefined {
   const unit = probes.readUnit(join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME));
   if (unit === undefined) return undefined;
@@ -609,13 +698,13 @@ function timezoneProbe(project: ProjectConfig | undefined): Finding {
     return passFinding(
       "reporting-timezone",
       configured.length === 0
-        ? "no reporting timezone configured"
-        : `reporting timezone(s) valid: ${configured.map(([, tz]) => `"${tz}"`).join(", ")}`,
+        ? `[${project.name}] no reporting timezone configured`
+        : `[${project.name}] reporting timezone(s) valid: ${configured.map(([, tz]) => `"${tz}"`).join(", ")}`,
     );
   }
   return failFinding(
     "reporting-timezone",
-    `${bad.map(([label, tz]) => `${label}: "${tz}" is not a known IANA timezone`).join("; ")} — an invalid zone silently skips the availability window and the daily digest`,
+    `[${project.name}] ${bad.map(([label, tz]) => `${label}: "${tz}" is not a known IANA timezone`).join("; ")} — an invalid zone silently skips the availability window and the daily digest`,
     "set a valid IANA timezone (e.g. Europe/London) in reporting.availability or reporting.digest",
   );
 }
@@ -651,11 +740,11 @@ async function telegramProbe(probes: Probes, project: ProjectConfig | undefined,
   if (project === undefined) return passFinding("telegram", "no project resolved — nothing to check");
   const chatId = (project.escalation.telegramChatId ?? "").trim();
   if (chatId === "") {
-    return passFinding("telegram", "no escalation.telegramChatId configured — nothing to probe");
+    return passFinding("telegram", `[${project.name}] no escalation.telegramChatId configured — nothing to probe`);
   }
   const health = await probes.telegramHealth(project.name);
   const failures: string[] = [];
-  const notes: string[] = [];
+  const notes: string[] = [`[${project.name}]`];
   if (health.kind === "down") failures.push(`bot health: down (${health.detail ?? "getMe failed"})`);
   else if (health.kind === "unconfigured") notes.push(`bot health: unconfigured (${health.detail ?? "no token"})`);
   else if (health.kind === "degraded") notes.push(`bot health: degraded (${health.detail ?? ""})`);
@@ -678,7 +767,7 @@ async function telegramProbe(probes: Probes, project: ProjectConfig | undefined,
   }
   return failFinding(
     "telegram",
-    failures.join("; "),
+    `[${project.name}] ${failures.join("; ")}`,
     "fix the bot token / omp-telegram install; `doctor --probe-telegram` proves delivery end to end",
   );
 }
@@ -706,7 +795,13 @@ function spendProbe(rows: RunSpendRow[], limit: number): Finding {
 /**
  * Run every probe and assemble the stable report.
  *
- * @param projectName the `--project NAME` value (undefined on single-project hosts)
+ * Host-wide facts (config backup, store integrity, gh auth, systemd units,
+ * ownership, spend telemetry) are probed once; the per-project facts (labels,
+ * reporting timezone, telegram) are probed for every resolved project, each
+ * named (#530).
+ *
+ * @param projectName the `--project NAME` value; undefined checks every
+ *        configured project
  * @param opts injected seams; every probe defaults to the production wiring
  */
 export async function runDoctor(projectName: string | undefined, opts: DoctorDeps = {}): Promise<DoctorReport> {
@@ -722,42 +817,79 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
     configProblem = messageOf(err);
   }
 
-  let project: ProjectConfig | undefined;
+  // Resolve the project set: the named project when `--project NAME` is given,
+  // otherwise *(every)* configured project. Doctor is fleet-scoped and covers
+  // the whole host, so a no-name run checks every project's per-project facts
+  // once each, not the first one and not nothing (#530).
+  let projects: ProjectConfig[] = [];
   let projectProblem: string | undefined;
   if (cfg !== undefined) {
-    try {
-      project = findProject(cfg, projectName);
-    } catch (err) {
-      projectProblem = messageOf(err);
+    if (projectName !== undefined) {
+      try {
+        projects = [findProject(cfg, projectName)];
+      } catch (err) {
+        projectProblem = messageOf(err);
+      }
+    } else {
+      projects = cfg.projects;
     }
   }
 
   const findings: Finding[] = [];
   findings.push(configProbe(configProblem));
-  findings.push(
-    cfg === undefined
-      ? passFinding("project", "config unreadable — no project to resolve")
-      : project === undefined
-        ? failFinding(
-            "project",
-            `no project resolved: ${projectProblem ?? "unknown error"}`,
-            "pass --project NAME (doctor checks one project per run)",
-          )
-        : passFinding("project", project.name),
-  );
+  if (cfg === undefined) {
+    findings.push(passFinding("project", "config unreadable — no project to resolve"));
+  } else if (projectProblem !== undefined) {
+    findings.push(
+      failFinding(
+        "project",
+        `no project resolved: ${projectProblem}`,
+        "pass --project NAME (doctor checks the named project; with no name it checks every configured project)",
+      ),
+    );
+  } else if (projects.length === 1) {
+    findings.push(passFinding("project", projects[0]!.name));
+  } else {
+    findings.push(passFinding("project", `${projects.map((p) => p.name).join(", ")} (${projects.length} projects)`));
+  }
+
+  // Host-wide facts, probed once per run — never once per project (the daemon
+  // unit, the store and the token are shared across projects, so re-probing
+  // them per project and deduping the output would hide a real per-project
+  // difference behind a constant count).
   findings.push(backupProbe(probes));
   findings.push(dbProbe(probes));
   findings.push(await ghAuthProbe(probes, configuredRepos(cfg)));
-  findings.push(
-    project === undefined
-      ? passFinding("labels", projectProblem === undefined ? "no project resolved — nothing to check" : `labels uncheckable: ${projectProblem}`)
-      : await labelProbe(probes, project),
-  );
-  findings.push(unitProbe(probes, project, cfg));
+  // The collected run sample across the resolved project set, newest-first;
+  // spend telemetry is a single host-wide finding, not one per project.
+  const spendRows: RunSpendRow[] = [];
+  for (const p of projects) spendRows.push(...probes.recentRuns(p.name, SPEND_SAMPLE_RUNS));
+
+  // Per-project probes run for every resolved project, each named in its
+  // finding so a multi-project run stays legible. When nothing resolved
+  // (config unreadable, or --project named an unknown project), a no-op pass
+  // keeps the stable report shape rather than silently dropping the row.
+  if (projects.length === 0) {
+    findings.push(
+      passFinding("labels", projectProblem === undefined ? "no project resolved — nothing to check" : `labels uncheckable: ${projectProblem}`),
+    );
+  } else {
+    for (const p of projects) findings.push(await labelProbe(probes, p));
+  }
+  // The installed-unit check is host-global: one shared daemon. Any project
+  // renders the same canonical units (the daemon/units carry no project), so
+  // the first one stands in for the rendering seam.
+  findings.push(unitProbe(probes, projects[0], cfg));
+  findings.push(recoveryProbe(probes));
   findings.push(ownershipProbe(probes));
-  findings.push(timezoneProbe(project));
-  findings.push(await telegramProbe(probes, project, checkedAt));
-  findings.push(spendProbe(project === undefined ? [] : probes.recentRuns(project.name, SPEND_SAMPLE_RUNS), SPEND_SAMPLE_RUNS));
+  if (projects.length === 0) {
+    findings.push(timezoneProbe(undefined));
+    findings.push(await telegramProbe(probes, undefined, checkedAt));
+  } else {
+    for (const p of projects) findings.push(timezoneProbe(p));
+    for (const p of projects) findings.push(await telegramProbe(probes, p, checkedAt));
+  }
+  findings.push(spendProbe(spendRows, SPEND_SAMPLE_RUNS));
 
   const status: ReportStatus = findings.some((f) => f.status === "fail")
     ? "fail"
@@ -765,7 +897,12 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
       ? "warn"
       : "pass";
   return {
-    project: project === undefined ? projectName ?? "(none resolved)" : project.name,
+    project:
+      projects.length === 0
+        ? projectName ?? "(none resolved)"
+        : projects.length === 1
+          ? projects[0]!.name
+          : projects.map((p) => p.name).join(", "),
     checkedAt: checkedAtIso,
     status,
     findings,

package/src/failure-class.ts

@@ -222,6 +222,27 @@ export function dispatchInfra(
 // front of the thrown message, so a bare /^git / would never match a real row.
 const DISPATCH_GIT_ERROR = /^(?:Error: )?git .+ exited \d+/s;
 
+/** The prefix the draining/restarting process writes to a run it killed. */
+const ADMIN_RESTART_MARKER = "admin restart:";
+
+/**
+ * The actor a draining restart recorded on a run it killed, or `undefined`
+ * when the row carries no such attribution. Read from `lastError` for the
+ * same reason {@link dispatchInfra} is read from it: it is the one surface a
+ * run's own row already persists, and the restarting process writes it the
+ * way `recordOperatorStop` writes "operator stopped: …". Recognised as an
+ * administrative kill, never a charging failure, because nothing the worker
+ * did ended it — #512's gap was that a killed session left no attribution.
+ */
+export function adminRestartAttribution(lastError: string | undefined): string | undefined {
+  if (lastError === undefined) return undefined;
+  const marker = lastError.indexOf(ADMIN_RESTART_MARKER);
+  if (marker === -1) return undefined;
+  const firstLine = lastError.slice(marker + ADMIN_RESTART_MARKER.length).trim().split("\n")[0];
+  if (firstLine === undefined || firstLine.trim() === "") return undefined;
+  return firstLine.trim();
+}
+
 export function classifyRun(
   run: RunRecord,
   facts: ClassifyFacts,
@@ -300,6 +321,22 @@ export function classifyRun(
     }
   }
 
+  // A run the draining/restarting process attributed before it restarted the
+  // session host: killed under its own ceilings by an administrative action,
+  // not by anything the worker did. Costs no budget and requeues, naming the
+  // actor (#512). A killed session used to leave no attribution and fell
+  // through to `unknown`, charging a failure for an operator's restart.
+  if (run.state === "failed" || run.state === "killed") {
+    const actor = adminRestartAttribution(run.lastError);
+    if (actor !== undefined) {
+      return {
+        cls: "admin-kill",
+        recovery: "requeue",
+        evidence: `killed by an administrative restart (${actor}) — an operator's restart, not the run's work`,
+      };
+    }
+  }
+
   if (run.state === "blocked") {
     return {
       cls: "question",

package/src/gitops.ts

@@ -17,6 +17,7 @@
  * branch cannot disagree.
  */
 
+import { existsSync } from "node:fs";
 import { join } from "node:path";
 
 import { parseChainSource, type ChainEntry } from "./chain-check.ts";
@@ -321,3 +322,159 @@ export async function readBaseChain(
     return { ok: false, stderr: err instanceof Error ? err.message : String(err) };
   }
 }
+
+// ------------------------------------------- the shared-host critical-base guard
+
+/**
+ * The stale-base admission verdict (#428): whether a preserved continuation
+ * branch may be reattached at all, once a project marks a base commit as
+ * critical. Fail closed — a branch that cannot be *proven* to contain every
+ * marker is refused.
+ */
+export type CriticalBaseVerdict =
+  /** No preserved branch for this issue in the mirror — a clean first attempt. */
+  | { state: "no-branch" }
+  /** The reattach source contains every configured marker. */
+  | { state: "fresh" }
+  /** The reattach source predates a marker and cannot be safely advanced to it,
+   *  so the dispatcher must not reattach it. `range` names the base commits
+   *  the branch has not absorbed, for the hold's recovery wording. */
+  | { state: "stale"; marker: string; range: string[] }
+  /** The marker could not be verified at all (unresolvable, fetch failed). */
+  | { state: "unknown"; error: string };
+
+/** The stubborn-session view of the probe, injectable for deterministic tests. */
+export type CriticalBaseProbe = (
+  repo: RepoTarget,
+  markers: readonly string[],
+  branch: string,
+) => Promise<CriticalBaseVerdict>;
+
+/** The bounded sample of base commits a held branch is missing. */
+const CRITICAL_BASE_RANGE_MAX = 8;
+
+/**
+ * Answer whether the preserved branch {@link addRunRepo} would reattach
+ * contains every configured critical-base marker (#428). A base safety fix
+ * protects only branches forked after it landed; a continuation forked before
+ * it still carries the dangerous code, and re-running the lifecycle suite it
+ * retains on a shared host is what SIGTERMed the production daemon.
+ *
+ * The reattach source is the mirror's `refs/heads/<branch>`. When it lacks a
+ * marker but the live remote branch now carries it and the reattach source is
+ * that live head's ancestor, the operator advanced the branch on GitHub (e.g.
+ * merged base into the PR branch): the fast-forward is folded into the mirror
+ * — safe and work-preserving, the same reconcile `pushRunBranch` performs —
+ * and the marker is accepted. Judged against the reattach source, never the
+ * live head alone, so a stale local copy cannot smuggle pre-fix code back in.
+ */
+export async function probeCriticalBase(
+  project: Pick<ProjectConfig, "mirrorRoot">,
+  repo: RepoTarget,
+  branch: string,
+  markers: readonly string[],
+  exec: Exec = spawnCaptured,
+): Promise<CriticalBaseVerdict> {
+  const mirror = mirrorPath(project, repo);
+  // A mirror is where preserved branches live; without one there is nothing to
+  // reattach. Judged "no branch" rather than "unknown" so a fleet that has not
+  // yet created a mirror is not deadlocked at admission.
+  if (!existsSync(mirror)) return { state: "no-branch" };
+  const branchRef = `refs/heads/${branch}`;
+  const env = credentialedEnv();
+
+  const present = await exec(
+    ["git", "--git-dir", mirror, "show-ref", "--verify", "--quiet", branchRef],
+    { env },
+  );
+  if (present.code !== 0) return { state: "no-branch" };
+
+  // The configured marker is a commit that landed on base, so base has to be
+  // resolvable to prove anything about the branch against it. Fetch today's
+  // base tip into the mirror's tracking refs first, so the judgement is against
+  // the current base rather than whatever a run last left behind.
+  const baseRef = `refs/remotes/origin/${repo.defaultBranch}`;
+  const fetched = await exec(
+    ["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+refs/heads/${repo.defaultBranch}:${baseRef}`],
+    { env },
+  );
+  if (fetched.code !== 0) {
+    return {
+      state: "unknown",
+      error: scrubUserinfo(fetched.stderr.trim() || fetched.stdout.trim() || `git fetch exited ${String(fetched.code)}`),
+    };
+  }
+
+  for (const marker of markers) {
+    const resolved = await exec(
+      ["git", "--git-dir", mirror, "rev-parse", "--verify", `${marker}^{commit}`],
+      { env },
+    );
+    if (resolved.code !== 0) {
+      return {
+        state: "unknown",
+        error: `critical-base marker "${marker}" is not resolvable in the ${repo.name} mirror`,
+      };
+    }
+    const markerSha = resolved.stdout.trim();
+    if (markerSha === "") {
+      return { state: "unknown", error: `critical-base marker "${marker}" resolved to no commit` };
+    }
+
+    const localHas = await exec(
+      ["git", "--git-dir", mirror, "merge-base", "--is-ancestor", markerSha, branchRef],
+      { env },
+    );
+    if (localHas.code === 0) continue;
+
+    // The reattach source lacks the marker. If the live branch now carries it
+    // and the reattach source is its ancestor, the operator advanced the branch
+    // on GitHub (e.g. merged base into the PR branch): fold that fast-forward
+    // into the reattach source — safe, work-preserving — and accept the marker.
+    const tracked = `refs/remotes/origin/${branch}`;
+    const liveFetched = await exec(
+      ["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+refs/heads/${branch}:${tracked}`],
+      { env },
+    );
+    if (liveFetched.code !== 0) {
+      return {
+        state: "unknown",
+        error: scrubUserinfo(liveFetched.stderr.trim() || liveFetched.stdout.trim() || `git fetch ${branch} exited ${String(liveFetched.code)}`),
+      };
+    }
+    const liveHas = await exec(
+      ["git", "--git-dir", mirror, "merge-base", "--is-ancestor", markerSha, tracked],
+      { env },
+    );
+    if (liveHas.code === 0) {
+      const localIsAncestor = await exec(
+        ["git", "--git-dir", mirror, "merge-base", "--is-ancestor", branchRef, tracked],
+        { env },
+      );
+      if (localIsAncestor.code === 0) {
+        await exec(["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+${branchRef}:${branchRef}`], { env });
+        continue;
+      }
+    }
+
+    // Predates the marker and cannot be safely advanced to it. Name the base
+    // commits the branch has not absorbed so the hold's recovery is concrete.
+    const range: string[] = [];
+    const missing = await exec(
+      ["git", "--git-dir", mirror, "log", "--oneline", "--format=%h", `${branchRef}..${baseRef}`],
+      { env },
+    );
+    if (missing.code === 0) {
+      range.push(
+        ...missing.stdout
+          .trim()
+          .split("\n")
+          .filter((line) => line.length > 0)
+          .slice(0, CRITICAL_BASE_RANGE_MAX),
+      );
+    }
+    return { state: "stale", marker: markerSha, range };
+  }
+
+  return { state: "fresh" };
+}