omp-conductor 0.7.1 → 0.9.0

package/src/types.ts

@@ -42,6 +42,10 @@ export interface Caps {
   /** Parallel omp sessions. Two by default: on a small self-hosted runner pool
    *  a third worker would starve its own PR checks. */
   maxConcurrentWorkers: number;
+  /** Max live workers per repository. 1 by default: the mirror, branch-protection
+   *  staleness and shared CI egress are all per-repo collision domains (#186), so
+   *  extra slots should land on other repos. */
+  maxConcurrentWorkersPerRepo: number;
   /**
    * Rolling-day spend ceiling. `null` means no spend gate (turns + wall-clock
    * still apply). `0` is a hard stop — deliberate, not "unset".
@@ -479,6 +483,10 @@ export interface ProjectConfig {
   tracker: { kind: "github"; repo: string };
   /** The one label that means "a human has signed this off as agent-ready". */
   queueLabel: string;
+  /** Routable-candidate count below which the tick prompt tells the orchestrator
+   *  to groom the queue. Optional; defaults to {@link DEFAULT_GROOM_BELOW} in
+   *  orchestrator-tick.ts. */
+  groomBelow?: number;
   /** Labels the dispatcher writes back so the tracker alone shows live state
    *  to a human who never opens the daemon's logs. */
   stateLabels: { inProgress: string; blocked: string; failed: string };
@@ -686,6 +694,14 @@ export interface OpenCloser {
  */
 export interface Tracker {
   listReady(): Promise<ReadyIssue[]>;
+  /**
+   * Every open issue in the tracker repo (never PRs), labels included: one
+   * read that covers all lifecycle-label queries of a pass, so consumers
+   * needing several label sets pay one fetch instead of one per label, and
+   * openness itself is answerable without a per-issue lookup (#203).
+   * Complete — follows pagination to the end.
+   */
+  listOpenIssues(): Promise<ReadyIssue[]>;
   addLabel(issue: number, label: string): Promise<void>;
   removeLabel(issue: number, label: string): Promise<void>;
   comment(issue: number, body: string): Promise<void>;
@@ -903,6 +919,11 @@ export interface RunRecord {
    *  exactly as an unflagged one does, and the flags are evidence for whoever
    *  reviews the PR. Absent means the audit found nothing, or never ran. */
   settlementFlags?: SettlementFlag[];
+  /** The worker's settlement report text for this attempt, verbatim. Persisted
+   *  on every terminal update so the next attempt can pool its disclosures
+   *  (#199). Absent means the run predates the column, or was killed before the
+   *  worker returned a report. */
+  report?: string;
   /**
    * Why this run ended badly, and what the daemon did about it (#132). Absent
    * means the sweep has not looked at the row yet — never "nothing was wrong":
@@ -923,6 +944,7 @@ export type AdmissionHoldReason =
   | "continuations"
   | "parent-lookup-error"
   | "sibling-active"
+  | "repo-active"
   | "open-pr-lookup-error"
   | "open-pr"
   | "unsalvaged-wip"
@@ -1134,10 +1156,28 @@ export interface DecisionDraft {
   at: number;
 }
 
+/** One decided GitHub label change waiting to be projected (#201). */
+export interface LabelOp {
+  id: number;
+  project: string;
+  issue: number;
+  op: "add" | "remove";
+  label: string;
+  createdAt: number;
+  attempts: number;
+  nextAttemptAt: number;
+  lastError?: string;
+}
+
 /**
- * Bookkeeping only — GitHub labels remain the source of truth. The store
- * exists to answer cap questions cheaply and to survive a restart; if it is
- * ever lost, the tracker can rebuild the world.
+ * Dispatch state is store-authoritative; GitHub labels are a retried
+ * write-behind projection of it (#201). `isEligible` still consults the label
+ * set a candidate *effectively* carries — physical labels overlaid with the
+ * outbox's pending ops — so a label GitHub refuses to move cannot strand an
+ * issue the daemon has already decided to move on. A store loss no longer
+ * rebuilds the world: it also forgets what labels the tracker was told to
+ * carry, so the operator reconciles by hand, which is the only safe move once
+ * the two disagree.
  */
 export interface Store {
   createRun(r: Omit<RunRecord, "id">): RunRecord;
@@ -1170,6 +1210,11 @@ export interface Store {
    *  tail` resolves an issue number to a transcript through this; the number is
    *  what an operator has, the run id is not. */
   latestRun(project: string, issue: number): RunRecord | undefined;
+  /** Every attempt of one issue that settled with a report, in attempt order.
+   *  The settlement audit pools these as prior disclosures when a later
+   *  attempt is reconciled (#199). Rows with no report (pre-#199, or killed
+   *  before the worker returned) are skipped. */
+  attemptReports(project: string, issue: number): { attempt: number; report: string }[];
   runsStartedSince(project: string, sinceEpochMs: number): number;
   spendSince(project: string, sinceEpochMs: number): number;
   /** Idempotence guard so a retry loop cannot page a human repeatedly for the
@@ -1189,6 +1234,15 @@ export interface Store {
   /** Start the cooldown only after a tick carrying these signals was sent. */
   markFrictionSurfaced(project: string, kinds: readonly FrictionKind[], at: number): void;
   markNotified(key: string): void;
+  /** Record one observed GitHub rate-limit refusal (the tracker's hook). Rows
+   *  older than 24h are pruned in the same write (#198). */
+  recordGhRefusal?(at: number): void;
+  /** Refusals observed at or after `sinceMs` — `status` reads the last 5m. */
+  ghRefusalsSince?(sinceMs: number): { count: number; latestAt?: number };
+  /** Count one daemon-held `gh` call for the UTC day (`YYYY-MM-DD`). */
+  bumpGhCalls?(day: string, source: string): void;
+  /** The tracked call counts for a UTC day, per source. */
+  ghCallsToday?(day: string): { source: string; calls: number }[];
   /**
    * Persist a rendered report `pending`, before anything is sent — the whole
    * point of #123 is that an undelivered report is a queryable row rather than
@@ -1283,6 +1337,33 @@ export interface Store {
   releaseMergeLock(project: string, holder: string): void;
   /** The live lock, for `status` and for tests. */
   mergeLock(project: string): MergeLock | undefined;
+  /**
+   * Append decided GitHub label changes to the projection outbox (#201). One
+   * transaction: either every op lands as a pending row or none do, so the
+   * daemon's record of its own intent cannot disagree with itself. Ops for one
+   * issue apply in enqueue (id) order, which is what makes a swap atomic — the
+   * remove lands before the add exactly as enqueued.
+   */
+  enqueueLabelOps(
+    project: string,
+    ops: readonly { issue: number; op: "add" | "remove"; label: string }[],
+  ): void;
+  /** Unapplied ops whose retry backoff has elapsed, oldest first (#201). */
+  pendingLabelOps(project: string, now: number): LabelOp[];
+  /** Every unapplied op for one issue, enqueue order — the eligibility overlay
+   *  reads this so a stale GitHub label cannot outlive the daemon's intent
+   *  (#201). */
+  pendingLabelOpsFor(project: string, issue: number): LabelOp[];
+  /** The op applied; forget it. */
+  settleLabelOp(id: number): void;
+  /** The op failed; count the attempt and park it behind `nextAttemptAt`. Pass
+   *  `countAttempt: false` for a refusal that is not the op's fault (a shared
+   *  rate limit), so it does not burn the backoff escalation (#208). */
+  deferLabelOp(id: number, error: string, nextAttemptAt: number, countAttempt?: boolean): void;
+  /** Unapplied ops still owed — `status` shows the projection lag (#201). */
+  countPendingLabelOps(project: string): number;
+  /** `createdAt` of the oldest unapplied op, or `undefined` for none. */
+  oldestPendingLabelOpAt(project: string): number | undefined;
   close(): void;
 }
 
@@ -1311,6 +1392,7 @@ export interface Escalation {
  */
 export const DEFAULT_CAPS: Caps = {
   maxConcurrentWorkers: 2,
+  maxConcurrentWorkersPerRepo: 1,
   dailySpendUsd: 25,
   // Off unless an operator names a window. A default threshold would need a
   // default window id, and guessing which allowance a fleet lives on is how a

package/src/unblock.ts

@@ -15,18 +15,27 @@
  * dispatcher writes labels with, and the brief's rule stays absolute. That
  * absoluteness is worth more than the exception it replaces: orphan detection
  * is only trustworthy while every state label on the tracker was written by
- * this package.
+ * this package. The re-queue is the same port and the same principle: the
+ * queue label goes back on by default (#184) so an answered block is
+ * dispatchable again, and it stays off under `--no-requeue` or while any run
+ * for the issue is still active.
  *
- * Nothing here writes to the store, and that is a decision rather than an
- * omission. `RunState` describes what a worker process did; an answer is the
+ * Nothing here writes to the run history, and that is a decision rather than
+ * an omission. `RunState` describes what a worker process did; an answer is the
  * one event that happens outside every run, so no member fits it — folding it
  * into `merged` or `killed` would make `status` describe a run that never
- * reached either. Eligibility is read off the tracker's labels and never off a
- * run row, so the store has nothing to say here. Leaving history alone keeps
- * both budgets honest: a block consumes an operational continuation, while a
- * real implementation failure consumes the separate failed-attempt budget.
+ * reached either. The one store write is the label projection outbox (#201):
+ * a decided label change is durable locally and projected with retry, so a 403
+ * defers it instead of throwing the verb, and the daemon's next tick applies it
+ * even if this process dies before it does. Eligibility is read off the label
+ * set an issue effectively carries — physical labels overlaid with that outbox —
+ * and never off a run row, so the store has nothing else to say here. Leaving
+ * history alone keeps both budgets honest: a block consumes an operational
+ * continuation, while a real implementation failure consumes the separate
+ * failed-attempt budget.
  */
 
+import { projectLabels } from "./label-projection.ts";
 import { LIVE_STATES } from "./store.ts";
 import type { Caps, ProjectConfig, RunRecord, Store, Tracker } from "./types.ts";
 
@@ -44,11 +53,32 @@ export interface UnblockOutcome {
    *  to "will the dispatcher hold this issue as issue-active?" that #178
    *  found this verb guessing at. */
   active: boolean;
+  /** Set when a live worker (a claimed/running run) is on the issue. The
+   *  dispatcher holds a worker-backed run as issue-active unconditionally,
+   *  while worker-free pushed occupancy can bypass it (#175) — so this is
+   *  the signal the queue label and the in-flight wording follow. */
+  live?: true;
   /** Set when nothing was cleared because the newest attempt's work exists
    *  only in its worktree. Carries the salvage failure verbatim. */
   refused?: string;
   /** Set when `--force` recorded an operator's acceptance of that loss. */
   forced?: true;
+  /** Set when the queue label was re-added (the default); absent on
+   *  --no-requeue, on the refusal path, and when a live worker (claimed or
+   *  running) is still on the issue. A pushed-green/pushed-pending occupancy
+   *  is worker-free, so the label is still restored there (#175 bypasses it;
+   *  `isEligible` needs the label once the PR resolves). */
+  requeued?: true;
+  /** Set when `--no-requeue` skipped the queue-label re-add. */
+  requeueSkipped?: true;
+  /** Set when the tracker refused one or more of this verb's label ops, so
+   *  the label sync is owed rather than done: the intended label state is
+   *  durable in the store and the daemon retries it (#201). Safety is
+   *  preserved either way, but the issue is only claimable once the queue
+   *  label itself has landed — `listReady` reads the tracker by label, so a
+   *  queue-label add that has not been applied is invisible to dispatch. The
+   *  number is what is still pending. */
+  labelSyncQueued?: number;
 }
 
 /**
@@ -90,13 +120,24 @@ export async function unblockIssue(
   tracker: Tracker,
   store: Store,
   issue: number,
-  opts: { force?: boolean } = {},
+  opts: { force?: boolean; requeue?: boolean } = {},
 ): Promise<UnblockOutcome> {
+  const requeue = opts.requeue !== false;
   // Read before any label is touched: terminality is the whole of the argument
   // for clearing in-progress, so the row that carries it decides the set.
   const latest = store.latestRun(project.name, issue);
   const terminal = latest !== undefined && !LIVE_STATES.includes(latest.state);
+  // Two different occupancies, and the label decisions each follow its own:
+  // `active` (ACTIVE_STATES) is what *occupies the issue* — a pushed-green or
+  // pushed-pending run has no worker process but its PR is live, so dispatch
+  // must not start a fresh attempt behind it. `live` (LIVE_STATES) is what
+  // *holds a worker process* (claimed/running). The queue label follows
+  // `live`: the dispatcher only holds a claim as issue-active for a
+  // worker-backed run (#175 bypasses worker-free pushed-green rows), so an
+  // issue whose occupancy is pushed-only is genuinely re-queueable and
+  // `isEligible` requires the label once that PR resolves closed-unmerged.
   const active = store.activeRuns(project.name).some((r) => r.issue === issue);
+  const live = store.liveRuns(project.name).some((r) => r.issue === issue);
   const counts = {
     attemptsUsed: store.attemptsFor(project.name, issue),
     failuresUsed: store.failuresFor(project.name, issue),
@@ -123,20 +164,66 @@ export async function unblockIssue(
   if (held !== undefined) store.updateRun(held.id, { salvageAckAt: Date.now() });
 
   const cleared: string[] = [];
+  const ops: { issue: number; op: "add" | "remove"; label: string }[] = [];
   for (const label of new Set([
     project.stateLabels.blocked,
     project.stateLabels.failed,
     ...(terminal ? [project.stateLabels.inProgress] : []),
   ])) {
-    await tracker.removeLabel(issue, label);
+    ops.push({ issue, op: "remove", label });
     cleared.push(label);
   }
 
+  // The re-queue half of the verb. Clearing the state labels makes the issue
+  // *eligible*, but the dispatcher never sees an issue that does not carry the
+  // queue label — without this the unblock ends in the same alive-but-inert
+  // state it exists to end (#184). The label goes back on by default;
+  // `--no-requeue` is the "about to close it" case and leaves dispatch alone.
+  // Never under a *live worker*: the dispatcher holds a worker-backed run as
+  // issue-active anyway, and restoring the label under one is exactly the
+  // #178 inverted state. A pushed-green/pushed-pending run is worker-free —
+  // its PR is live but no process writes to its branch — so a pushed
+  // continuation still gets the label: the dispatcher bypasses worker-free
+  // pushed-green rows (#175), and `isEligible` requires the queue label once
+  // the PR resolves closed-unmerged.
+  let requeued: true | undefined;
+  let requeueSkipped: true | undefined;
+  let labelSyncQueued: number | undefined;
+  if (!live) {
+    if (requeue) {
+      ops.push({ issue, op: "add", label: project.queueLabel });
+      requeued = true;
+    } else {
+      requeueSkipped = true;
+    }
+  }
+
+  // Apply through the same projector the daemon drains (#201): the clears and
+  // the queue restore land in enqueue order, and a tracker that refuses (403,
+  // rate limit) defers them with backoff instead of throwing the verb — which
+  // is the concrete #184/#198 kill: `unblock` stops breaking on a 403. The
+  // intended label state is durable; the daemon converges the tracker. The
+  // issue only becomes claimable once the queue label itself lands —
+  // `listReady` queries by label, so a pending queue-label add is invisible to
+  // dispatch until projection succeeds.
+  if (ops.length > 0) {
+    store.enqueueLabelOps(project.name, ops);
+    await projectLabels(store, tracker, project);
+    // How much of THIS issue's sync is still owed (not the whole fleet's outbox):
+    // a backlog on other issues is not this verb's lag to report.
+    const stillPending = store.pendingLabelOpsFor(project.name, issue).length;
+    if (stillPending > 0) labelSyncQueued = stillPending;
+  }
+
   return {
     cleared,
     ...counts,
+    ...(live ? { live: true as const } : {}),
     ...(latest === undefined ? {} : { latest }),
     ...(held === undefined ? {} : { forced: true as const }),
+    ...(requeued === undefined ? {} : { requeued }),
+    ...(requeueSkipped === undefined ? {} : { requeueSkipped }),
+    ...(labelSyncQueued === undefined ? {} : { labelSyncQueued }),
   };
 }
 
@@ -175,6 +262,15 @@ export function formatUnblock(
   }
 
   const lines = [`#${issue}: cleared ${o.cleared.join(", ")}`];
+  // When the tracker refused the sync, NO label is provably applied — the
+  // "restored / eligible again" claims below would be a lie, so they branch to
+  // the queued/retry wording instead (#201).
+  const labelsPending = o.labelSyncQueued !== undefined;
+  if (o.labelSyncQueued !== undefined) {
+    lines.push(
+      `  label sync queued (${o.labelSyncQueued} pending) — the daemon retries; the intended labels are durable, and the issue is claimable once they land`,
+    );
+  }
   if (o.forced === true) {
     lines.push(
       `  forced     the unsalvaged worktree was accepted as lost or already recovered by hand — ` +
@@ -191,9 +287,15 @@ export function formatUnblock(
   }
 
   if (latest !== undefined && LIVE_STATES.includes(latest.state)) {
+    // The newest run is live, so `active` is necessarily true and the queue
+    // label was NOT re-added (unblockIssue only re-queues when no run for the
+    // issue is active). Say that, like the sibling-active branch below does —
+    // the label outcome is part of the contract (#184), and a live newest run
+    // reaches this branch instead of `else if (o.active)`.
     lines.push(
       `  in flight  attempt ${latest.attempt} is ${latest.state}, so the issue keeps ` +
         `"${project.stateLabels.inProgress}" until it ends — nothing is re-claimed before then`,
+      `  queue      "${project.queueLabel}" not restored — a run is still active; re-run unblock once it settles`,
     );
   } else if (o.failuresUsed >= caps.maxAttemptsPerIssue) {
     lines.push(
@@ -218,25 +320,59 @@ export function formatUnblock(
     // instead of promising a re-claim the next tick withholds.
     lines.push(
       latest.prUrl === undefined
-        ? `  next tick  eligible again, as long as the issue still carries "${project.queueLabel}" ` +
-            `(newest run is ${latest.state} with no recorded PR; the dispatcher still applies its open-PR check at claim time)`
+        ? labelsPending
+          ? `  next tick  label sync queued (${o.labelSyncQueued} pending) — the daemon retries; eligibility is recorded in the store, not yet on the tracker`
+          : o.requeued === true
+            ? `  next tick  eligible again — "${project.queueLabel}" restored (no-op if it was already present; ` +
+              "the dispatcher still applies its open-PR check at claim time)"
+            : `  next tick  eligible again, as long as the issue still carries "${project.queueLabel}" ` +
+                `(newest run is ${latest.state} with no recorded PR; the dispatcher still applies its open-PR check at claim time)`
         : `  next tick  eligible as a continuation of ${latest.prUrl} — the pushed run stays active until ` +
             "that PR resolves; dispatch continues on its branch",
     );
   } else if (o.active) {
-    // A run other than the newest is still active (e.g. a live sibling under a
-    // terminal newest row), so the dispatcher will hold the issue as
-    // issue-active whatever the labels now say. #178's misleading case.
-    lines.push(
-      `  in flight  a run for this issue is still active, so the dispatcher holds the issue until it settles — ` +
-        `nothing is re-claimed before then ("${project.stateLabels.inProgress}" stays unless already released)`,
-    );
+    // A run other than the newest still occupies the issue. Two shapes hide
+    // under `active`, and they demand opposite reports: a *live worker*
+    // (claimed/running) holds the issue unconditionally — #178's misleading
+    // case — while worker-free pushed runs (pushed-green/pushed-pending) are
+    // the continuation shape, because #175 lets the dispatcher admit
+    // pushed-green rows whose workers are gone. The queue label follows the
+    // same split: off under a live worker, restored under pushed-only
+    // occupancy.
+    if (o.live === true) {
+      lines.push(
+        `  in flight  a live worker is still on this issue, so the dispatcher holds it until that run settles — ` +
+          `nothing is re-claimed before then ("${project.stateLabels.inProgress}" stays unless already released)`,
+        `  queue      "${project.queueLabel}" not restored — a live worker is still on this issue; re-run unblock once it settles`,
+      );
+    } else {
+      lines.push(
+        `  next tick  no live worker is on this issue — its active run(s) are worker-free pushes; ` +
+          "dispatch continues their branch once its holds and open-PR check clear",
+      );
+      if (labelsPending) {
+        lines.push(
+          `  queue      label sync queued (${o.labelSyncQueued} pending) — the daemon retries; the intended labels are durable, and the issue is claimable once they land`,
+        );
+      } else if (o.requeued === true) {
+        lines.push(`  queue      "${project.queueLabel}" restored (no-op if it was already present)`);
+      }
+    }
   } else {
     lines.push(
-      `  next tick  eligible again, as long as the issue still carries "${project.queueLabel}" ` +
-        "(the dispatcher still applies its open-PR check at claim time)",
+      labelsPending
+        ? `  next tick  label sync queued (${o.labelSyncQueued} pending) — the daemon retries; the intended labels are durable, and the issue is claimable once they land`
+        : o.requeued === true
+          ? `  next tick  eligible again — "${project.queueLabel}" restored (no-op if it was already present; ` +
+            "the dispatcher still applies its open-PR check at claim time)"
+          : `  next tick  eligible again, as long as the issue still carries "${project.queueLabel}" ` +
+            "(the dispatcher still applies its open-PR check at claim time)",
     );
   }
 
+  if (o.requeueSkipped === true) {
+    lines.push(`  queue      "${project.queueLabel}" left untouched (--no-requeue)`);
+  }
+
   return lines.join("\n");
 }

package/src/upgrade.ts

@@ -1,9 +1,10 @@
 import { existsSync, readFileSync } from "node:fs";
+import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
 import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
 import { setPaused, statusSnapshot } from "./daemon.ts";
 import { fleetLayers, telegramStateDir, type DispatchLayer, type FleetLayers } from "./fleet.ts";
 import { livingDaemon, restartDaemon } from "./lifecycle.ts";
-import { configPath, findProject, loadConfig, resolveCaps, writeConfigRaw } from "./config.ts";
+import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, writeConfigRaw } from "./config.ts";
 import { renderBriefForProject } from "./setup.ts";
 import {
   STAGED_SERVICE_NAME,
@@ -68,7 +69,7 @@ async function runCommand(command: string, args: readonly string[]): Promise<Upg
   }
 }
 
-const DEFAULT_DEPS: UpgradeDeps = {
+export const DEFAULT_DEPS: UpgradeDeps = {
   run: runCommand,
   snapshot: statusSnapshot,
   layers: fleetLayers,
@@ -84,7 +85,7 @@ const DEFAULT_DEPS: UpgradeDeps = {
     const daemon = livingDaemon();
     return daemon === undefined ? { running: false } : { running: true, project: daemon.project };
   },
-  setPaused,
+  setPaused: (v) => setPaused(v, { source: "upgrade", reason: "upgrade, draining" }),
   restartDaemon: async () => {
     await restartDaemon({});
   },
@@ -213,17 +214,43 @@ function previousHerdrInstall(source: string): readonly [string, readonly string
   ];
 }
 
-async function waitForDrain(deps: UpgradeDeps, project?: string): Promise<void> {
+async function waitForDrain(deps: UpgradeDeps, project?: string, deadlineAt?: number): Promise<void> {
   let last = -1;
   while (true) {
     const workers = deps.snapshot(project).liveWorkers;
     if (workers === 0) return;
+    if (deadlineAt !== undefined && Date.now() >= deadlineAt) {
+      throw new Error(
+        `drain timed out with ${workers} live worker(s) still running — nothing was restarted; dispatch remains paused (omp-conductor resume to lift it, or re-run restart to keep waiting)`,
+      );
+    }
     if (workers !== last) deps.log(`waiting for ${workers} live worker(s) to finish`);
     last = workers;
     await deps.sleep(DRAIN_POLL_MS);
   }
 }
 
+/**
+ * Pause, drain, restart, restore — the trusted restart transaction, minus the
+ * install/verify steps of {@link upgradeConductor}.
+ *
+ * Mirrors the upgrade's fail-closed posture: on ANY throw after the pause the
+ * fleet stays paused (no resume in a catch) and the error rethrows, so a
+ * systemd-owned restart that fails or a drain that outlives `timeoutMs` cannot
+ * silently resume dispatch over a wedged daemon. The caller surfaces the message
+ * and exits nonzero.
+ */
+export async function drainAndRestart(
+  deps: UpgradeDeps,
+  o: { project?: string; timeoutMs: number },
+): Promise<void> {
+  const initial = deps.layers(o.project);
+  if (!initial.paused) deps.setPaused(true);
+  await waitForDrain(deps, o.project, Date.now() + o.timeoutMs);
+  await deps.restartDaemon();
+  if (!initial.paused) deps.setPaused(false);
+}
+
 function recoveryProblem(
   layers: FleetLayers,
   initial: FleetLayers,
@@ -301,6 +328,7 @@ async function rollbackUpgrade(
   herdrReloadStarted: boolean,
   daemonReloadStarted: boolean,
   configBefore?: string,
+  preUpgradeBackup?: string,
 ): Promise<void> {
   const failures: string[] = [];
 
@@ -318,12 +346,23 @@ async function rollbackUpgrade(
       const path = configPath();
       if (readFileSync(path, "utf8") !== configBefore) {
         deps.log("rollback: conductor config.json");
+        // Name the durable snapshot: the in-memory restore only runs while
+        // this process is alive, and an upgrade that died mid-flight leaves
+        // the operator needing exactly these bytes from disk.
+        deps.log(
+          preUpgradeBackup === undefined
+            ? "rollback: config.json changed during the upgrade; no durable snapshot was persisted"
+            : `rollback: config.json changed during the upgrade; pre-upgrade snapshot at ${preUpgradeBackup}`,
+        );
         // Exact bytes. `writeConfigFile(JSON.parse(...))` returns canonically
         // formatted output, which restores the keys but not the file.
         writeConfigRaw(configBefore);
       }
     } catch (err) {
-      failures.push(`could not restore config.json: ${err instanceof Error ? err.message : String(err)}`);
+      failures.push(
+        `could not restore config.json: ${err instanceof Error ? err.message : String(err)}` +
+          (preUpgradeBackup === undefined ? "" : `; pre-upgrade snapshot at ${preUpgradeBackup}`),
+      );
     }
   }
   const restore = async (label: string, command: string, args: readonly string[]): Promise<void> => {
@@ -470,13 +509,23 @@ export async function upgradeConductor(
   // Snapshotted before anything is installed, because the new daemon starts
   // during this transaction and may migrate the file — see the restore in
   // `rollbackUpgrade`. Read as bytes, not through the loader: the point is to
-  // put back exactly what was there, dialect and all.
+  // put back exactly what was there, dialect and all. The same bytes are also
+  // persisted under state, so a process death between here and the rollback
+  // cannot take the only pre-upgrade copy with it.
   let configBefore: string | undefined;
+  let preUpgradeBackup: string | undefined;
   try {
     configBefore = readFileSync(configPath(), "utf8");
+    preUpgradeBackup = copyToUniqueBackup(
+      configPath(),
+      configBackupDir(),
+      `config.json.pre-upgrade-${backupTimestamp()}`,
+    );
+    deps.log(`config backed up to ${preUpgradeBackup}`);
   } catch {
     // No readable config is not a reason to refuse an upgrade; there is simply
-    // nothing to put back.
+    // nothing to put back. A snapshot that failed to persist is not fatal
+    // either — the in-memory restore below still covers a clean rollback.
   }
 
   if (brief.kind === "missing") throw new Error("no ORCHESTRATOR.md exists for the configured project");
@@ -582,6 +631,7 @@ export async function upgradeConductor(
         herdrReloadStarted,
         daemonReloadStarted,
         configBefore,
+        preUpgradeBackup,
       );
     } catch (rollbackErr) {
       const rollback = rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr);

package/src/worktree.ts

@@ -263,14 +263,22 @@ function refreshManagedExclude(worktree: string): void {
   }
 }
 
+/**
+ * Serializes concurrent {@link ensureMirror} calls per mirror path so two
+ * dispatches cannot collide on git's ref locks for the same repo (#186). A
+ * rejected (failed) call does not poison the chain: the next caller waits on a
+ * settled promise.
+ */
+const mirrorLocks = new Map<string, Promise<unknown>>();
+
 /**
  * Returns the path of the bare mirror for `repo`, cloning it on first use and
  * refreshing it otherwise.
  *
- * ponytail: no cross-process lock. Two dispatch loops that call this for the
- * same repo at the same instant can collide on git's ref locks and one will
- * throw; the run is retried rather than corrupted. Upgrade path is a lockfile
- * in `mirrorRoot` keyed by repo name.
+ * ponytail: the lock is per process, not a cross-process lockfile in
+ * `mirrorRoot`. Two *dispatch loops* can still collide on git's ref locks and
+ * one will throw; the run is retried rather than corrupted. Upgrade path is a
+ * lockfile keyed by repo name, for a future multi-daemon host.
  *
  * ponytail: if `repo.cloneUrl` embeds credentials, `git clone` persists them in
  * the mirror's config, exactly as it would for a hand-run clone. Prefer an SSH
@@ -280,6 +288,17 @@ export async function ensureMirror(
   repo: RepoTarget,
   mirrorRoot: string,
 ): Promise<string> {
+  const key = mirrorPathFor(repo, mirrorRoot);
+  const prev = mirrorLocks.get(key) ?? Promise.resolve();
+  const next = prev.catch(() => {}).then(() => ensureMirrorUnlocked(repo, mirrorRoot));
+  mirrorLocks.set(key, next);
+  void next.catch(() => {}).finally(() => {
+    if (mirrorLocks.get(key) === next) mirrorLocks.delete(key);
+  });
+  return next;
+}
+
+async function ensureMirrorUnlocked(repo: RepoTarget, mirrorRoot: string): Promise<string> {
   mkdirSync(mirrorRoot, { recursive: true });
   const mirrorPath = mirrorPathFor(repo, mirrorRoot);