omp-conductor 0.7.1 → 0.8.0

package/src/upgrade.ts

@@ -68,7 +68,7 @@ async function runCommand(command: string, args: readonly string[]): Promise<Upg
   }
 }
 
-const DEFAULT_DEPS: UpgradeDeps = {
+export const DEFAULT_DEPS: UpgradeDeps = {
   run: runCommand,
   snapshot: statusSnapshot,
   layers: fleetLayers,
@@ -84,7 +84,7 @@ const DEFAULT_DEPS: UpgradeDeps = {
     const daemon = livingDaemon();
     return daemon === undefined ? { running: false } : { running: true, project: daemon.project };
   },
-  setPaused,
+  setPaused: (v) => setPaused(v, { source: "upgrade", reason: "upgrade, draining" }),
   restartDaemon: async () => {
     await restartDaemon({});
   },
@@ -213,17 +213,43 @@ function previousHerdrInstall(source: string): readonly [string, readonly string
   ];
 }
 
-async function waitForDrain(deps: UpgradeDeps, project?: string): Promise<void> {
+async function waitForDrain(deps: UpgradeDeps, project?: string, deadlineAt?: number): Promise<void> {
   let last = -1;
   while (true) {
     const workers = deps.snapshot(project).liveWorkers;
     if (workers === 0) return;
+    if (deadlineAt !== undefined && Date.now() >= deadlineAt) {
+      throw new Error(
+        `drain timed out with ${workers} live worker(s) still running — nothing was restarted; dispatch remains paused (omp-conductor resume to lift it, or re-run restart to keep waiting)`,
+      );
+    }
     if (workers !== last) deps.log(`waiting for ${workers} live worker(s) to finish`);
     last = workers;
     await deps.sleep(DRAIN_POLL_MS);
   }
 }
 
+/**
+ * Pause, drain, restart, restore — the trusted restart transaction, minus the
+ * install/verify steps of {@link upgradeConductor}.
+ *
+ * Mirrors the upgrade's fail-closed posture: on ANY throw after the pause the
+ * fleet stays paused (no resume in a catch) and the error rethrows, so a
+ * systemd-owned restart that fails or a drain that outlives `timeoutMs` cannot
+ * silently resume dispatch over a wedged daemon. The caller surfaces the message
+ * and exits nonzero.
+ */
+export async function drainAndRestart(
+  deps: UpgradeDeps,
+  o: { project?: string; timeoutMs: number },
+): Promise<void> {
+  const initial = deps.layers(o.project);
+  if (!initial.paused) deps.setPaused(true);
+  await waitForDrain(deps, o.project, Date.now() + o.timeoutMs);
+  await deps.restartDaemon();
+  if (!initial.paused) deps.setPaused(false);
+}
+
 function recoveryProblem(
   layers: FleetLayers,
   initial: FleetLayers,

package/src/worktree.ts

@@ -263,14 +263,22 @@ function refreshManagedExclude(worktree: string): void {
   }
 }
 
+/**
+ * Serializes concurrent {@link ensureMirror} calls per mirror path so two
+ * dispatches cannot collide on git's ref locks for the same repo (#186). A
+ * rejected (failed) call does not poison the chain: the next caller waits on a
+ * settled promise.
+ */
+const mirrorLocks = new Map<string, Promise<unknown>>();
+
 /**
  * Returns the path of the bare mirror for `repo`, cloning it on first use and
  * refreshing it otherwise.
  *
- * ponytail: no cross-process lock. Two dispatch loops that call this for the
- * same repo at the same instant can collide on git's ref locks and one will
- * throw; the run is retried rather than corrupted. Upgrade path is a lockfile
- * in `mirrorRoot` keyed by repo name.
+ * ponytail: the lock is per process, not a cross-process lockfile in
+ * `mirrorRoot`. Two *dispatch loops* can still collide on git's ref locks and
+ * one will throw; the run is retried rather than corrupted. Upgrade path is a
+ * lockfile keyed by repo name, for a future multi-daemon host.
  *
  * ponytail: if `repo.cloneUrl` embeds credentials, `git clone` persists them in
  * the mirror's config, exactly as it would for a hand-run clone. Prefer an SSH
@@ -280,6 +288,17 @@ export async function ensureMirror(
   repo: RepoTarget,
   mirrorRoot: string,
 ): Promise<string> {
+  const key = mirrorPathFor(repo, mirrorRoot);
+  const prev = mirrorLocks.get(key) ?? Promise.resolve();
+  const next = prev.catch(() => {}).then(() => ensureMirrorUnlocked(repo, mirrorRoot));
+  mirrorLocks.set(key, next);
+  void next.catch(() => {}).finally(() => {
+    if (mirrorLocks.get(key) === next) mirrorLocks.delete(key);
+  });
+  return next;
+}
+
+async function ensureMirrorUnlocked(repo: RepoTarget, mirrorRoot: string): Promise<string> {
   mkdirSync(mirrorRoot, { recursive: true });
   const mirrorPath = mirrorPathFor(repo, mirrorRoot);