agent-relay-orchestrator 0.120.6 → 0.121.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.120.6",
+  "version": "0.121.1",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -17,7 +17,7 @@
   },
   "dependencies": {
     "agent-relay-providers": "0.104.3",
-    "agent-relay-sdk": "0.2.111",
+    "agent-relay-sdk": "0.2.113",
     "callmux": "0.23.0"
   },
   "devDependencies": {

package/src/control.ts

@@ -7,6 +7,7 @@ import { readLocalProviderConfigs } from "./provider-config-migration";
 import { findSessionRecord, readRunnerInfo, spawnAgent, stopSession, type SpawnOptions } from "./spawn";
 import { cleanupWorkspace, discardRecoveryBranch, idleRefreshWorktree, mergeWorkspace, pruneWorktrees, reconcileWorkspace, refreshWorkspaceDeps, workspacesRoot } from "./workspace-probe";
 import { withMergePhaseTimeout } from "./workspace-probe/merge-timeouts";
+import { withRepoLock } from "./repo-lock";
 import { armWorkspacePrAutoMerge, mergeWorkspacePr, refreshWorkspacePrBranch } from "./workspace-pr";
 import type { WorkspaceMergeResult } from "agent-relay-sdk";
 import { execProcess } from "./process";
@@ -21,6 +22,22 @@ export function mergeCommandStatus(result: WorkspaceMergeResult): "succeeded" |
   return result.error ? "failed" : "succeeded";
 }
 
+// #1020 (B2) — command types that mutate a physical repo's git refs / worktrees. These take the
+// per-repoRoot lock (see repo-lock.ts) so they can't interleave with the background spawn's
+// acquisition + `git worktree add` on the same checkout. Read-only or worktree-filesystem-only
+// commands (e.g. workspace.deps-refresh) are omitted so a slow dep install can't stall spawns.
+const REPO_MUTATING_COMMANDS = new Set<string>([
+  "workspace.cleanup",
+  "workspace.reconcile",
+  "workspace.merge",
+  "workspace.pr-merge",
+  "workspace.pr-refresh",
+  "workspace.pr-arm-auto-merge",
+  "workspace.recovery-branch-discard",
+  "workspace.idle-refresh",
+  "workspace.prune",
+]);
+
 interface ControlHandler {
   handleCommand(command: RelayCommand): Promise<boolean>;
   getManagedAgents(): ManagedAgentReport[];
@@ -358,6 +375,18 @@ export function createControlHandler(
   }
 
   async function handleNonSpawnCommand(command: RelayCommand): Promise<boolean> {
+    // #1020 (B2) — a repo-root-mutating command takes the physical-repo lock so it can't interleave
+    // with the concurrently-dispatched background spawn (acquisition + `git worktree add`) on the
+    // same checkout. Non-spawn commands are already mutually serialized by the single-flight poll
+    // tick; this closes the one carve-out (agent.spawn). Commands without a repoRoot run unwrapped.
+    const repoRoot = typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined;
+    if (repoRoot && REPO_MUTATING_COMMANDS.has(command.type)) {
+      return withRepoLock(repoRoot, () => runNonSpawnCommand(command));
+    }
+    return runNonSpawnCommand(command);
+  }
+
+  async function runNonSpawnCommand(command: RelayCommand): Promise<boolean> {
     await relay.updateCommand(command.id, "accepted");
     await relay.updateCommand(command.id, "running");
     try {

package/src/repo-lock.ts

@@ -0,0 +1,37 @@
+import { resolve } from "node:path";
+
+// #1020 (B2) — in-process serialization of repo-root-mutating git operations on ONE physical repo.
+//
+// The command poller serializes every command inline on the poll tick EXCEPT `agent.spawn`, which
+// is dispatched to an unawaited background task (control.ts). That carve-out lets spawn-time
+// acquisition (`git fetch` / `checkout` / `merge --ff-only`) and `git worktree add` run at the same
+// wall-clock instant as an inline land's `merge --no-ff` / `push` / `worktree remove --force` in the
+// SAME checkout — a `checkout` mid-land can yank HEAD out from under the merge, and concurrent
+// worktree-admin ops race `.git/worktrees/` metadata.
+//
+// Every mutator of a given physical repoRoot takes this lock, so the background spawn and an inline
+// land can no longer interleave on the same `.git`. A physical repo on a host is owned by exactly one
+// orchestrator process, so an in-process mutex is sufficient for this same-host race (cross-process
+// same-dir collisions are the separate #B3 concern). Distinct repoRoots use distinct keys, so
+// spawns/lands on different repos still run fully in parallel.
+const chains = new Map<string, Promise<unknown>>();
+
+/**
+ * Run `fn` while holding the exclusive lock for `repoRoot`. Calls for the SAME resolved path run
+ * strictly one-at-a-time in FIFO order; calls for DIFFERENT paths run concurrently. A failure in one
+ * holder never poisons the queue — the next waiter runs regardless of how the prior one settled.
+ */
+export function withRepoLock<T>(repoRoot: string, fn: () => Promise<T>): Promise<T> {
+  const key = resolve(repoRoot);
+  const prior = chains.get(key) ?? Promise.resolve();
+  // Chain onto the prior holder; run `fn` whether the prior resolved or rejected.
+  const run = prior.then(fn, fn);
+  // Tail the queue on an error-swallowed link so one holder's rejection can't break ordering.
+  const link = run.then(() => {}, () => {});
+  chains.set(key, link);
+  void link.then(() => {
+    // GC the map entry once this link is the tail and has settled.
+    if (chains.get(key) === link) chains.delete(key);
+  });
+  return run;
+}

package/src/spawn/acquisition.ts

@@ -4,6 +4,7 @@ import { basename, dirname, join, resolve } from "node:path";
 import { errMessage, isPathWithinBase } from "agent-relay-sdk";
 import { git, requireGit } from "../git";
 import { execProcess } from "../process";
+import { withRepoLock } from "../repo-lock";
 import type { ProjectAcquisitionManifest } from "./types";
 
 export type ProjectRootSyncAction = "synced" | "noop" | "skipped-dirty" | "skipped-diverged";
@@ -33,7 +34,11 @@ export async function applyProjectAcquisitionManifest(
   const rootPath = resolve(manifest.rootPath);
   const prior = inFlight.get(rootPath);
   if (prior) return prior;
-  const run = withAcquisitionLock(rootPath, baseDir, () => applyManifestLocked(manifest, baseDir));
+  // #1020 (B2) — take the physical-repo lock BEFORE the acquisition's git mutations so a land/cleanup
+  // on the same checkout (which also takes it) can't interleave. The file lock below still guards
+  // against a second orchestrator process; this in-process lock guards against the concurrent
+  // background spawn dispatch on THIS process.
+  const run = withRepoLock(rootPath, () => withAcquisitionLock(rootPath, baseDir, () => applyManifestLocked(manifest, baseDir)));
   inFlight.set(rootPath, run);
   try {
     return await run;

package/src/workspace-probe/merge.ts

@@ -1007,7 +1007,13 @@ async function mergeRebaseFf(
         signal,
       );
       if (!merge.ok) {
-        await mergeGit(["merge", "--abort"], baseWorktree.path, "rebase", `workspace merge abort failed no-ff ${branch}`, signal);
+        // #1020: run the abort-recovery WITHOUT the merge signal. When the no-ff merge failed
+        // *because* the total-merge timeout signal fired (killing git mid-merge), reusing that
+        // aborted signal would make `mergeGit` rethrow on its `throwIfMergeAborted` guard BEFORE
+        // the abort runs, leaving the shared base checkout stuck mid-merge (MERGE_HEAD, conflicted
+        // index). Same reasoning as restoreBaseWorktreeToSnapshot (#950) — this cleanup must run to
+        // completion even under an aborted/timed-out merge.
+        await mergeGit(["merge", "--abort"], baseWorktree.path, "cleanup", `workspace merge abort failed no-ff ${branch}`);
         return head({ conflict: true, status: "conflict", error: merge.stderr || "merge into base failed" });
       }
       baseTip = (await mergeGit(["rev-parse", "HEAD"], baseWorktree.path, "rebase", `workspace merge resolve ${base} after no-ff`, signal)).stdout;

package/src/workspace-probe/probe.ts

@@ -4,6 +4,7 @@ import { join, resolve } from "node:path";
 import type { WorkspaceMode, WorkspaceProbe } from "agent-relay-sdk";
 import { errMessage } from "agent-relay-sdk";
 import { git, requireGit } from "../git";
+import { withRepoLock } from "../repo-lock";
 import { provisionWorkspaceDeps, provisionWorkspaceSymlinks } from "./deps";
 import { availableBranch, branchName, terminalBaseRef, workspaceId, workspacesRoot, repoSlug } from "./names";
 import { parseWorktrees, shortBranch } from "./parse";
@@ -112,9 +113,16 @@ export async function resolveSpawnWorkspace(input: WorkspaceResolutionInput): Pr
       });
     }
 
-    const branch = await availableBranch(repoRoot, branchName(input, id));
-    mkdirSync(join(worktreePath, ".."), { recursive: true });
-    await requireGit(["worktree", "add", "-b", branch, worktreePath, startSha], repoRoot);
+    // #1020 (B2) — serialize the branch resolve + `git worktree add` against a concurrent
+    // land/cleanup on the same repoRoot (both take this lock), so worktree-admin ops can't race
+    // `.git/worktrees/` metadata. Deps/symlinks provisioning below touches only the new worktree
+    // dir, so it stays outside the lock to keep the hold time short.
+    const branch = await withRepoLock(repoRoot, async () => {
+      const resolved = await availableBranch(repoRoot, branchName(input, id));
+      mkdirSync(join(worktreePath, ".."), { recursive: true });
+      await requireGit(["worktree", "add", "-b", resolved, worktreePath, startSha], repoRoot);
+      return resolved;
+    });
     const deps = await provisionWorkspaceDeps(repoRoot, worktreePath);
     const symlinks = provisionWorkspaceSymlinks(repoRoot, worktreePath, input.workspaceSymlinks ?? []);
     return {
@@ -158,9 +166,15 @@ export async function resolveSpawnWorkspace(input: WorkspaceResolutionInput): Pr
     });
   }
 
-  const branch = await availableBranch(repoRoot, branchName(input, id));
-  mkdirSync(join(worktreePath, ".."), { recursive: true });
-  await requireGit(["worktree", "add", "-b", branch, worktreePath, baseSha], repoRoot);
+  // #1020 (B2) — serialize the branch resolve + `git worktree add` against a concurrent
+  // land/cleanup on the same repoRoot (both take this lock); provisioning below is worktree-local
+  // and stays outside the lock.
+  const branch = await withRepoLock(repoRoot, async () => {
+    const resolved = await availableBranch(repoRoot, branchName(input, id));
+    mkdirSync(join(worktreePath, ".."), { recursive: true });
+    await requireGit(["worktree", "add", "-b", resolved, worktreePath, baseSha], repoRoot);
+    return resolved;
+  });
 
   // A fresh worktree has no node_modules (git worktrees don't share
   // gitignored/untracked files). Provision deps before handing cwd to the