agent-relay-orchestrator 0.118.4 → 0.119.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.118.4",
+  "version": "0.119.0",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -17,7 +17,7 @@
   },
   "dependencies": {
     "agent-relay-providers": "0.104.1",
-    "agent-relay-sdk": "0.2.101",
+    "agent-relay-sdk": "0.2.102",
     "callmux": "0.23.0"
   },
   "devDependencies": {

package/src/control.ts

@@ -1,11 +1,12 @@
-import { errMessage, isRecord, normalizeAgentLifecycle, normalizeWorkspaceMode } from "agent-relay-sdk";
+import { errMessage, isRecord, normalizeAgentLifecycle, normalizeWorkspaceMode, Semaphore } from "agent-relay-sdk";
 import { getAllManifests, getManifest } from "agent-relay-providers";
 import type { OrchestratorConfig } from "./config";
 import type { ManagedAgentReport, RelayClient, RelayCommand } from "./relay";
 import { handleSelfUpgrade } from "./self-upgrade";
 import { readLocalProviderConfigs } from "./provider-config-migration";
-import { spawnAgent, stopSession, type SpawnOptions } from "./spawn";
+import { findSessionRecord, readRunnerInfo, spawnAgent, stopSession, type SpawnOptions } from "./spawn";
 import { cleanupWorkspace, discardRecoveryBranch, idleRefreshWorktree, mergeWorkspace, pruneWorktrees, reconcileWorkspace, refreshWorkspaceDeps, workspacesRoot } from "./workspace-probe";
+import { withMergePhaseTimeout } from "./workspace-probe/merge-timeouts";
 import { armWorkspacePrAutoMerge, mergeWorkspacePr, refreshWorkspacePrBranch } from "./workspace-pr";
 import type { WorkspaceMergeResult } from "agent-relay-sdk";
 
@@ -23,22 +24,167 @@ interface ControlHandler {
   handleCommand(command: RelayCommand): Promise<boolean>;
   getManagedAgents(): ManagedAgentReport[];
   setManagedAgents(agents: ManagedAgentReport[]): void;
+  // #930/#880 — spawn commands dispatch in the background (claimed synchronously,
+  // then run past registration under the concurrency cap). These expose that
+  // in-flight background work so callers/tests can observe and settle it.
+  spawnsInFlight(): number;
+  settleSpawns(): Promise<void>;
+}
+
+const DEFAULT_MAX_CONCURRENT_SPAWNS = 2;
+
+// #930/#880 — per-host spawn concurrency cap. Bursts of spawns racing worktree
+// materialization + port churn during the registration window are what surface
+// as `EADDRINUSE`/"operation was aborted" on the loser (#880). Bound how many
+// spawns materialize at once. Parsed once at handler construction; invalid or
+// out-of-range values fall back to the safe default rather than throwing.
+export function resolveMaxConcurrentSpawns(env: NodeJS.ProcessEnv = process.env): number {
+  const raw = env.AGENT_RELAY_MAX_CONCURRENT_SPAWNS;
+  if (raw === undefined || raw.trim() === "") return DEFAULT_MAX_CONCURRENT_SPAWNS;
+  const parsed = Number(raw);
+  if (!Number.isInteger(parsed) || parsed < 1) return DEFAULT_MAX_CONCURRENT_SPAWNS;
+  return parsed;
+}
+
+// #930/#880 — how long a spawn slot is held while we wait for the child to
+// register (bind its local control/ws server — the contended resource). This is
+// a fail-safe upper bound only: a child that never registers must not wedge a
+// slot forever, so we release after this deadline even if it hasn't appeared.
+// The happy path releases as soon as the child's runner-info file materializes,
+// well under this ceiling.
+const DEFAULT_SPAWN_REGISTRATION_TIMEOUT_MS = 30_000;
+const SPAWN_REGISTRATION_POLL_MS = 100;
+
+export function resolveSpawnRegistrationTimeoutMs(env: NodeJS.ProcessEnv = process.env): number {
+  const raw = env.AGENT_RELAY_SPAWN_REGISTRATION_TIMEOUT_MS;
+  if (raw === undefined || raw.trim() === "") return DEFAULT_SPAWN_REGISTRATION_TIMEOUT_MS;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_SPAWN_REGISTRATION_TIMEOUT_MS;
+  return parsed;
+}
+
+/** Awaits a spawn slot until the child registers, or resolves false at the fail-safe deadline. */
+export type WaitForRegistration = (agent: ManagedAgentReport, timeoutMs: number) => Promise<boolean>;
+
+// #930/#880 — the orchestrator observes a child registering through its runner
+// info file: the runner writes `controlUrl` immediately after binding its local
+// control/MCP server (runner-core writeRunnerInfoFile, right after startMcpProxy),
+// which is exactly #880's contended ws-bind window. So the file's presence is the
+// host-local signal that the child is past that window; holding the slot until it
+// appears is what bounds how many children materialize→bind at once.
+async function waitForManagedRegistration(agent: ManagedAgentReport, timeoutMs: number): Promise<boolean> {
+  const session = agent.sessionName ?? agent.tmuxSession;
+  if (!session) return false;
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    const record = findSessionRecord({ tmuxSession: session, agentId: agent.agentId });
+    if (record && readRunnerInfo(record)) return true;
+    if (Date.now() >= deadline) return false;
+    await Bun.sleep(SPAWN_REGISTRATION_POLL_MS);
+  }
+}
+
+interface ControlHandlerDeps {
+  // Injectable spawn dispatch — defaults to the real spawnAgent. Only tests
+  // override it, to gate dispatch and observe the concurrency bound directly.
+  spawnAgent?: typeof spawnAgent;
+  // Override the cap directly, bypassing the env parse (tests).
+  maxConcurrentSpawns?: number;
+  // Injectable registration wait — defaults to polling the child's runner info
+  // file. Tests override it to gate the registration window explicitly.
+  waitForRegistration?: WaitForRegistration;
+  // Override the fail-safe registration timeout, bypassing the env parse (tests).
+  registrationTimeoutMs?: number;
 }
 
 export function createControlHandler(
   config: OrchestratorConfig,
   relay: RelayClient,
+  deps: ControlHandlerDeps = {},
 ): ControlHandler {
   let managedAgents: ManagedAgentReport[] = [];
+  const dispatchSpawn = deps.spawnAgent ?? spawnAgent;
+  // In-memory counter is correct here: a slot only needs to live within this
+  // orchestrator process, and a crash resets in-flight spawns anyway (#881 Q1).
+  const spawnSlots = new Semaphore(deps.maxConcurrentSpawns ?? resolveMaxConcurrentSpawns());
+  const waitForRegistration = deps.waitForRegistration ?? waitForManagedRegistration;
+  const registrationTimeoutMs = deps.registrationTimeoutMs ?? resolveSpawnRegistrationTimeoutMs();
+  // Backgrounded spawn tasks: each is claimed synchronously (status → accepted)
+  // then runs to completion off the poll tick, holding a slot across the child's
+  // registration window. Tracked so tests/shutdown can observe and settle them.
+  const backgroundSpawns = new Set<Promise<void>>();
 
-  async function handleSpawn(ctrl: Record<string, any>): Promise<boolean> {
+  async function handleSpawn(ctrl: Record<string, any>): Promise<ManagedAgentReport> {
     const opts = spawnOptionsFromControl(ctrl, config);
-    const agent = await spawnAgent(opts, config);
+    const agent = await dispatchSpawn(opts, config);
     managedAgents.push(agent);
     console.error(`[orchestrator] Spawned ${opts.provider} agent: ${agent.tmuxSession}`);
+    return agent;
+  }
+
+  // #930/#880 — dispatch a spawn command concurrently, up to the per-host cap.
+  //
+  // The command is CLAIMED (status → accepted) synchronously here, BEFORE we
+  // return, so the next pollCommands() tick can never re-return this still-pending
+  // row and double-dispatch it. The slot itself is taken in the BACKGROUND via
+  // acquire() (strict FIFO — no starvation), so the poll tick is never blocked
+  // waiting for a free slot, preserving the #864 single-flight of the tick.
+  //
+  // The background task holds its slot across the child's registration window
+  // (#880's contended ws-bind resource) and frees it in `finally` on EVERY path:
+  // registration success, spawn failure, or the fail-safe registration timeout.
+  async function dispatchSpawnCommand(command: RelayCommand): Promise<boolean> {
+    await relay.updateCommand(command.id, "accepted");
+    // #930 — guard the backgrounded task against unhandled rejection. runSpawnToRegistration
+    // settles its own errors, but a relay blip in the ≤30s in-flight window can reject the
+    // status write in its catch block too; without a terminal `.catch` here that escapes as
+    // an unhandledRejection (log spam, or a crash if the runtime exits on it). Track the
+    // GUARDED promise so settleSpawns() awaits a promise that never rejects.
+    const task = runSpawnToRegistration(command).catch((error) => {
+      console.error(`[orchestrator] backgrounded spawn task error (swallowed): ${errMessage(error)}`);
+    });
+    backgroundSpawns.add(task);
+    void task.finally(() => backgroundSpawns.delete(task));
     return true;
   }
 
+  async function runSpawnToRegistration(command: RelayCommand): Promise<void> {
+    // Blocks FIFO until a slot frees — this is what bounds concurrent spawns to N.
+    await spawnSlots.acquire();
+    try {
+      await relay.updateCommand(command.id, "running");
+      const agent = await handleSpawn(command.params);
+      const registered = await waitForRegistration(agent, registrationTimeoutMs);
+      if (!registered) {
+        // Not a spawn failure — the process launched; it just didn't register in
+        // time. Release the slot (fail-safe) so a slow/never-registering child
+        // can't wedge capacity, but still settle the command as succeeded.
+        console.error(`[orchestrator] spawn ${agent.tmuxSession} did not register within ${registrationTimeoutMs}ms; releasing slot (fail-safe)`);
+      }
+      await relay.updateCommand(command.id, "succeeded", { managedAgents });
+      await relay.updateManagedAgents(managedAgents);
+    } catch (error) {
+      console.error(`[orchestrator] spawn dispatch failed: ${errMessage(error)}`);
+      // #930 — the `failed` status write can itself reject during a relay blip. Wrap it so
+      // a failed status write can't re-throw out of the backgrounded task (belt-and-suspenders
+      // with the `.catch` in dispatchSpawnCommand). The command may strand non-terminal until
+      // the TTL sweep / watchdog reconciles it, but the task settles cleanly regardless.
+      try {
+        await relay.updateCommand(command.id, "failed", undefined, errMessage(error));
+      } catch (statusError) {
+        console.error(`[orchestrator] failed to mark spawn command failed: ${errMessage(statusError)}`);
+      }
+    } finally {
+      spawnSlots.release();
+    }
+  }
+
+  async function settleSpawns(): Promise<void> {
+    while (backgroundSpawns.size > 0) {
+      await Promise.all([...backgroundSpawns]);
+    }
+  }
+
   async function handleShutdown(ctrl: Record<string, any>, restart = false): Promise<Record<string, unknown>> {
     const current = managedAgentShutdownTarget(managedAgents, ctrl);
     const session = current?.sessionName ?? current?.tmuxSession;
@@ -85,14 +231,18 @@ export function createControlHandler(
   }
 
   async function handleCommand(command: RelayCommand): Promise<boolean> {
+    // #930/#880 — spawns dispatch concurrently in the background under the
+    // per-host cap (see dispatchSpawnCommand). Every other command is handled
+    // inline, synchronously with the poll tick, exactly as before.
+    if (command.type === "agent.spawn") return dispatchSpawnCommand(command);
+    return handleNonSpawnCommand(command);
+  }
+
+  async function handleNonSpawnCommand(command: RelayCommand): Promise<boolean> {
     await relay.updateCommand(command.id, "accepted");
     await relay.updateCommand(command.id, "running");
     try {
-      if (command.type === "agent.spawn") {
-        const handled = await handleSpawn(command.params);
-        if (!handled) throw new Error("spawn failed");
-        await relay.updateCommand(command.id, "succeeded", { managedAgents });
-      } else if (command.type === "agent.shutdown" || command.type === "agent.restart") {
+      if (command.type === "agent.shutdown" || command.type === "agent.restart") {
         const result = await handleShutdown(command.params, command.type === "agent.restart");
         await relay.updateCommand(command.id, "succeeded", result);
       } else if (command.type === "workspace.cleanup") {
@@ -118,26 +268,42 @@ export function createControlHandler(
         });
         await relay.updateCommand(command.id, "succeeded", result);
       } else if (command.type === "workspace.merge") {
-        const result = await mergeWorkspace({
-          id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
-          repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
-          worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
-          branch: typeof command.params.branch === "string" ? command.params.branch : undefined,
-          baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined,
-          baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined,
-          strategy: command.params.strategy === "pr" || command.params.strategy === "rebase-ff" || command.params.strategy === "auto" ? command.params.strategy : undefined,
-          deleteBranch: command.params.deleteBranch !== false,
-          push: command.params.push !== false,
-          prTitle: typeof command.params.prTitle === "string" ? command.params.prTitle : undefined,
-          prBody: typeof command.params.prBody === "string" ? command.params.prBody : undefined,
-          autoMerge: command.params.autoMerge === "on-green" || command.params.autoMerge === "on-approval" || command.params.autoMerge === "manual" ? command.params.autoMerge : undefined,
-          prLanded: isRecord(command.params.prLanded)
-            ? {
-                sha: typeof command.params.prLanded.sha === "string" ? command.params.prLanded.sha : undefined,
-                subject: typeof command.params.prLanded.subject === "string" ? command.params.prLanded.subject : undefined,
-              }
-            : undefined,
-        });
+        let result: WorkspaceMergeResult;
+        try {
+          result = await withMergePhaseTimeout("total", () => mergeWorkspace({
+            id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
+            repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
+            worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
+            branch: typeof command.params.branch === "string" ? command.params.branch : undefined,
+            baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined,
+            baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined,
+            strategy: command.params.strategy === "pr" || command.params.strategy === "rebase-ff" || command.params.strategy === "auto" ? command.params.strategy : undefined,
+            deleteBranch: command.params.deleteBranch !== false,
+            push: command.params.push !== false,
+            prTitle: typeof command.params.prTitle === "string" ? command.params.prTitle : undefined,
+            prBody: typeof command.params.prBody === "string" ? command.params.prBody : undefined,
+            autoMerge: command.params.autoMerge === "on-green" || command.params.autoMerge === "on-approval" || command.params.autoMerge === "manual" ? command.params.autoMerge : undefined,
+            prLanded: isRecord(command.params.prLanded)
+              ? {
+                  sha: typeof command.params.prLanded.sha === "string" ? command.params.prLanded.sha : undefined,
+                  subject: typeof command.params.prLanded.subject === "string" ? command.params.prLanded.subject : undefined,
+                }
+              : undefined,
+          }));
+        } catch (err) {
+          const branch = typeof command.params.branch === "string" ? command.params.branch : undefined;
+          const baseRef = typeof command.params.baseRef === "string" ? command.params.baseRef : undefined;
+          result = {
+            workspaceId: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
+            strategy: command.params.strategy === "pr" ? "pr" : "rebase-ff",
+            merged: false,
+            status: "review_requested",
+            ...(branch ? { branch } : {}),
+            ...(baseRef ? { baseRef } : {}),
+            error: errMessage(err),
+          };
+          console.error(`[orchestrator] workspace.merge failed before completion: ${result.error}`);
+        }
         // #638 — settle `failed` (carrying the error) for a no-op merge instead of
         // `succeeded`; see mergeCommandStatus.
         await relay.updateCommand(command.id, mergeCommandStatus(result), result as unknown as Record<string, unknown>, result.error);
@@ -233,7 +399,7 @@ export function createControlHandler(
     managedAgents = agents;
   }
 
-  return { handleCommand, getManagedAgents, setManagedAgents };
+  return { handleCommand, getManagedAgents, setManagedAgents, spawnsInFlight: () => backgroundSpawns.size, settleSpawns };
 }
 
 export function managedAgentShutdownTarget(agents: ManagedAgentReport[], ctrl: Record<string, any>): ManagedAgentReport | undefined {

package/src/workspace-probe/merge-timeouts.ts

@@ -1,8 +1,12 @@
-export type MergePhase = "preview" | "fetch" | "synthesize" | "worktree-add" | "cleanup";
+export type MergePhase = "total" | "prep" | "preview" | "fetch" | "rebase" | "gates" | "synthesize" | "worktree-add" | "cleanup";
 
 const MERGE_PHASE_TIMEOUTS_MS: Record<MergePhase, number> = {
+  total: 10 * 60_000,
+  prep: 60_000,
   preview: 30_000,
   fetch: 60_000,
+  rebase: 60_000,
+  gates: 5 * 60_000,
   synthesize: 60_000,
   "worktree-add": 60_000,
   cleanup: 30_000,

package/src/workspace-probe/merge.ts

@@ -1,35 +1,50 @@
-import { existsSync, mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
+import { existsSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { errMessage, type BaseWorktreeSyncResult, type WorkspaceMergePreview, type WorkspaceMergeResult } from "agent-relay-sdk";
 import { git, gitRaw } from "../git";
 import { execProcess } from "../process";
 import { prMergedState } from "../workspace-pr";
 import { refreshWorkspaceDeps } from "./deps";
-import { type LandGatesResult, runLandGates } from "./land-gates-runner";
-import { populateMergeState, resolveBranchRef, syncBaseFromOrigin, upstreamRef, workspaceGitState } from "./git-state";
-import { mergePhaseTimeoutMs, withMergePhaseTimeout } from "./merge-timeouts";
+import { type LandGatesResult } from "./land-gates-runner";
+import { populateMergeState, resolveBranchRef, syncBaseFromOrigin, workspaceGitState } from "./git-state";
+import { type MergePhase, mergePhaseTimeoutMs, withMergePhaseTimeout } from "./merge-timeouts";
 import { nextBranchName } from "./names";
 import { parseWorktrees, shortBranch } from "./parse";
 import { json, resolveRequestedPath } from "./request";
 import type { WorkspaceMergeInput } from "./types";
 import { workspacePushEnabled } from "../config";
 
+async function mergeGit(args: string[], cwd: string, phase: MergePhase, timeoutLabel?: string): ReturnType<typeof git> {
+  return git(args, cwd, {
+    timeoutMs: mergePhaseTimeoutMs(phase),
+    timeoutLabel: timeoutLabel ?? `workspace merge ${phase} git ${args.join(" ")}`,
+  });
+}
+
+async function mergeGitRaw(args: string[], cwd: string, phase: MergePhase, timeoutLabel?: string): ReturnType<typeof gitRaw> {
+  return gitRaw(args, cwd, {
+    timeoutMs: mergePhaseTimeoutMs(phase),
+    timeoutLabel: timeoutLabel ?? `workspace merge ${phase} git ${args.join(" ")}`,
+  });
+}
+
+function gitError(result: Awaited<ReturnType<typeof git>>, fallback: string): string {
+  return result.stderr || result.stdout || fallback;
+}
+
 /** Behind-count of HEAD relative to `base`, from inside `worktreePath`. */
-async function countBehind(worktreePath: string, base: string): Promise<number> {
-  const counts = await git(["rev-list", "--left-right", "--count", `${base}...HEAD`], worktreePath);
-  if (!counts.ok || !counts.stdout) return 0;
+async function countBehind(worktreePath: string, base: string): Promise<{ behind: number } | { error: string }> {
+  const counts = await mergeGit(["rev-list", "--left-right", "--count", `${base}...HEAD`], worktreePath, "rebase", "workspace merge count behind integration base");
+  if (!counts.ok || !counts.stdout) return { error: gitError(counts, "failed to count branch behind integration base") };
   const behind = Number(counts.stdout.split(/\s+/)[0]);
-  return Number.isFinite(behind) ? behind : 0;
+  return Number.isFinite(behind) ? { behind } : { error: "git rev-list produced an invalid behind count" };
 }
 
 async function hasOriginRemote(cwd: string): Promise<boolean> {
   return (await git(["remote", "get-url", "origin"], cwd)).ok;
 }
 
-function ghAvailable(): boolean {
-  return Boolean(Bun.which("gh"));
-}
+function ghAvailable(): boolean { return Boolean(Bun.which("gh")); }
 
 /**
  * Ground-truth merge state for `branch`, via gh.
@@ -61,11 +76,11 @@ async function baseBranchName(worktreePath: string, baseRef?: string): Promise<s
 
 /** Locate the worktree (if any) that currently has `branch` checked out. */
 async function worktreeForBranch(repoRoot: string, branch: string): Promise<{ path: string; dirty: boolean } | undefined> {
-  const list = await git(["worktree", "list", "--porcelain"], repoRoot);
+  const list = await mergeGit(["worktree", "list", "--porcelain"], repoRoot, "rebase", "workspace merge list worktrees");
   if (!list.ok) return undefined;
   const match = parseWorktrees(list.stdout).find((worktree) => worktree.branch === branch);
   if (!match) return undefined;
-  const status = await git(["status", "--porcelain"], match.path);
+  const status = await mergeGit(["status", "--porcelain"], match.path, "rebase", `workspace merge status ${branch} worktree`);
   return { path: match.path, dirty: status.ok ? status.stdout.length > 0 : true };
 }
 
@@ -74,7 +89,7 @@ function splitNul(stdout: string): string[] {
 }
 
 async function dirtyPathSet(worktreePath: string): Promise<Set<string>> {
-  const status = await gitRaw(["status", "--porcelain=v1", "-z", "--untracked-files=all"], worktreePath);
+  const status = await mergeGitRaw(["status", "--porcelain=v1", "-z", "--untracked-files=all"], worktreePath, "rebase", "workspace merge dirty path scan");
   if (!status.ok) return new Set<string>();
   const paths = new Set<string>();
   const entries = splitNul(status.stdout);
@@ -90,25 +105,25 @@ async function dirtyPathSet(worktreePath: string): Promise<Set<string>> {
 }
 
 async function changedPathList(worktreePath: string, oldBaseTip: string, newBaseTip: string): Promise<string[]> {
-  const diff = await gitRaw(["diff", "--name-only", "-z", oldBaseTip, newBaseTip], worktreePath);
+  const diff = await mergeGitRaw(["diff", "--name-only", "-z", oldBaseTip, newBaseTip], worktreePath, "rebase", "workspace merge changed path scan");
   return diff.ok ? splitNul(diff.stdout) : [];
 }
 
 async function restorePathsFromHead(worktreePath: string, paths: string[]): Promise<boolean> {
   if (paths.length === 0) return true;
-  return (await git(["restore", "--staged", "--worktree", "--", ...paths], worktreePath)).ok;
+  return (await mergeGit(["restore", "--staged", "--worktree", "--", ...paths], worktreePath, "rebase", "workspace merge restore clean landed paths")).ok;
 }
 
 async function resetIndexPathsToHead(worktreePath: string, paths: string[]): Promise<boolean> {
   if (paths.length === 0) return true;
-  return (await git(["reset", "-q", "HEAD", "--", ...paths], worktreePath)).ok;
+  return (await mergeGit(["reset", "-q", "HEAD", "--", ...paths], worktreePath, "rebase", "workspace merge reset preserved paths")).ok;
 }
 
 /** Landed paths whose working copy still differs from the advanced HEAD (ground truth, not
  *  exit codes): the checkout is mixed for these — reads/builds/publishes see STALE files. */
 async function pathsDifferingFromHead(worktreePath: string, paths: string[]): Promise<string[]> {
   if (paths.length === 0) return [];
-  const diff = await gitRaw(["diff", "--name-only", "-z", "HEAD", "--", ...paths], worktreePath);
+  const diff = await mergeGitRaw(["diff", "--name-only", "-z", "HEAD", "--", ...paths], worktreePath, "rebase", "workspace merge verify base worktree sync");
   // A failed diff can't prove the checkout is clean — treat every landed path as suspect.
   return diff.ok ? splitNul(diff.stdout) : [...paths];
 }
@@ -152,7 +167,7 @@ async function syncDirtyBaseWorktreeAfterRefAdvance(
   dirtyBefore: Set<string> | undefined,
 ): Promise<BaseWorktreeSyncResult> {
   if (!baseWorktree?.dirty || !dirtyBefore) return RECONCILED;
-  const readTree = await git(["read-tree", "-m", "-u", oldBaseTip, newBaseTip], baseWorktree.path);
+  const readTree = await mergeGit(["read-tree", "-m", "-u", oldBaseTip, newBaseTip], baseWorktree.path, "rebase", "workspace merge sync dirty base worktree");
   if (readTree.ok) return RECONCILED;
 
   const changedPaths = await changedPathList(baseWorktree.path, oldBaseTip, newBaseTip);
@@ -352,11 +367,12 @@ export async function mergeWorkspace(input: WorkspaceMergeInput): Promise<Worksp
   if (!input.worktreePath) return { strategy: "rebase-ff", merged: false, status: "review_requested", error: "worktreePath required", workspaceId: input.id };
   const worktreePath = resolve(input.worktreePath);
   const repoRoot = input.repoRoot ? resolve(input.repoRoot) : worktreePath;
+  console.error(`[orchestrator] workspace.merge prep-start workspace=${input.id ?? "(unknown)"} worktree=${worktreePath} repo=${repoRoot}`);
   // Probe the live HEAD branch first — it's the authoritative source. Fall back to the
   // DB-recorded branch only when the live probe fails (detached HEAD, missing worktree, etc.).
   // This fixes #232: a stale DB branch value (non-null mismatch) would pass through the
   // `input.branch ?? ...` guard unchanged and cause git to attempt merging a non-existent ref.
-  const liveBranch = shortBranch((await git(["symbolic-ref", "--quiet", "--short", "HEAD"], worktreePath)).stdout || undefined);
+  const liveBranch = shortBranch((await mergeGit(["symbolic-ref", "--quiet", "--short", "HEAD"], worktreePath, "prep", "workspace merge resolve live branch")).stdout || undefined);
   const branch = liveBranch ?? input.branch;
   let preview: WorkspaceMergePreview;
   try {
@@ -398,6 +414,7 @@ export async function mergeWorkspace(input: WorkspaceMergeInput): Promise<Worksp
   if (preview.conflict) return head({ conflict: true, status: "conflict", error: "merge would conflict with base" });
 
   if (strategy === "pr") return await mergePr(input, worktreePath, branch, preview, head);
+  console.error(`[orchestrator] workspace.merge rebase-start workspace=${input.id ?? "(unknown)"} branch=${branch ?? "(unknown)"} base=${preview.baseRef ?? "(unknown)"}`);
   return await mergeRebaseFf(input, worktreePath, repoRoot, branch, preview, head);
 }
 
@@ -559,25 +576,23 @@ async function recordNoFfMerge(
   branchSha: string,
   message: string,
 ): Promise<{ ok: true; mergeSha: string } | { ok: false; conflict?: boolean; error: string }> {
-  const synth = await synthesizeNoFfMerge(repoRoot, baseSha, branchSha, message);
+  const synth = await synthesizeNoFfMerge(repoRoot, baseSha, branchSha, message, mergePhaseTimeoutMs("synthesize"));
   if (!synth.ok) return synth;
-  const update = await git(["update-ref", `refs/heads/${base}`, synth.mergeSha, baseSha], repoRoot);
+  const update = await mergeGit(["update-ref", `refs/heads/${base}`, synth.mergeSha, baseSha], repoRoot, "rebase", `workspace merge advance ${base} to synthesized merge`);
   if (!update.ok) return { ok: false, error: update.stderr || "failed to advance base ref" };
   return { ok: true, mergeSha: synth.mergeSha };
 }
 
+function decommissionedLandGatesResult(): LandGatesResult {
+  return { ran: 0, warnings: [] };
+}
+
 /**
- * Run the repo's configured land gates against the EXACT tree that will become base's new tip,
- * BEFORE any ref advance (#902 BLOCKING 1 / closes #903). The candidate tree depends on `behind`:
- *  - behind === 0: the landing worktree HEAD already IS that tree (a clean fast-forward), so gates
- *    run in `worktreePath` in place — byte-identical to today's straight-FF path.
- *  - behind  >  0: base has advanced, so the tree that will land is the no-ff MERGE of
- *    `integrationBaseSha` and `headSha`, NOT the worktree HEAD. Synthesize that merge commit with
- *    plumbing (no ref moved), check it out in a throwaway DETACHED worktree, and gate THAT tree.
- *    The temp worktree is torn down on EVERY exit path (including a gate that throws), so a land
- *    can never strand an orphan worktree or leave the synthesized merge half-applied.
- * Returns the gate outcome, or an `abort` describing a merge result the caller must return early
- * (a real merge conflict computing the integrated tree, or a failure materializing it).
+ * Land gates are decommissioned (#923). Release-time gates in `release.ts` are the
+ * quality boundary; workspace landing must never launch repo-configured gate subprocesses.
+ * Keep this function as the single merge-path kill switch so populated
+ * `.agent-relay/land-gates.json` files are treated as always-empty without changing the
+ * rest of the rebase/fetch/no-ff landing flow.
  */
 async function runLandGatesOnIntegratedTree(
   repoRoot: string,
@@ -587,50 +602,12 @@ async function runLandGatesOnIntegratedTree(
   headSha: string,
   mergeMessage: string,
 ): Promise<{ gates: LandGatesResult } | { abort: { conflict?: boolean; error: string } }> {
-  if (behind === 0) return { gates: await runLandGates(worktreePath) };
-
-  let synth: Awaited<ReturnType<typeof synthesizeNoFfMerge>>;
-  try {
-    synth = await withMergePhaseTimeout("synthesize", () => synthesizeNoFfMerge(repoRoot, integrationBaseSha, headSha, mergeMessage, mergePhaseTimeoutMs("synthesize")));
-  } catch (err) {
-    return { abort: { error: errMessage(err) } };
-  }
-  if (!synth.ok) return { abort: { conflict: synth.conflict, error: synth.error } };
-
-  // Detached worktree in an isolated temp dir so the gates see the integrated tree on disk
-  // without disturbing the landing worktree, the base checkout, or any ref. `git worktree add`
-  // creates the leaf, so hand it a not-yet-existing path under a freshly made parent dir.
-  const tmpParent = mkdtempSync(join(tmpdir(), "agent-relay-landgate-"));
-  const tmpWorktree = join(tmpParent, "tree");
-  let add: Awaited<ReturnType<typeof git>>;
-  try {
-    add = await withMergePhaseTimeout("worktree-add", () => git(
-      ["worktree", "add", "--detach", tmpWorktree, synth.mergeSha],
-      repoRoot,
-      { timeoutMs: mergePhaseTimeoutMs("worktree-add"), timeoutLabel: "workspace merge land-gate worktree add" },
-    ));
-  } catch (err) {
-    rmSync(tmpParent, { recursive: true, force: true });
-    return { abort: { error: errMessage(err) } };
-  }
-  if (!add.ok) {
-    rmSync(tmpParent, { recursive: true, force: true });
-    return { abort: { error: add.stderr || "failed to materialize integrated tree for land gates" } };
-  }
-  try {
-    return { gates: await runLandGates(tmpWorktree) };
-  } finally {
-    try {
-      await withMergePhaseTimeout("cleanup", () => git(
-        ["worktree", "remove", "--force", tmpWorktree],
-        repoRoot,
-        { timeoutMs: mergePhaseTimeoutMs("cleanup"), timeoutLabel: "workspace merge land-gate worktree cleanup" },
-      ));
-    } catch (err) {
-      console.error(`[orchestrator] land-gate integrated worktree cleanup timed out/failed: ${errMessage(err)}`);
-    }
-    rmSync(tmpParent, { recursive: true, force: true });
-  }
+  void repoRoot;
+  void integrationBaseSha;
+  void headSha;
+  void mergeMessage;
+  console.error(`[orchestrator] workspace.merge gate-skip worktree=${worktreePath} behind=${behind} reason=land-gates-decommissioned`);
+  return { gates: decommissionedLandGatesResult() };
 }
 
 /**
@@ -647,7 +624,7 @@ async function syncLocalBaseToUpstream(
   base: string,
   upstream: string,
 ): Promise<{ ok: true; baseSync?: BaseWorktreeSyncResult } | { ok: false; error: string }> {
-  const upstreamSha = (await git(["rev-parse", "--verify", upstream], worktreePath)).stdout;
+  const upstreamSha = (await mergeGit(["rev-parse", "--verify", upstream], worktreePath, "rebase", `workspace merge resolve ${upstream}`)).stdout;
   if (!upstreamSha) return { ok: false, error: `cannot resolve ${upstream} to sync ${base}` };
   const baseWorktree = await worktreeForBranch(repoRoot, base);
   // Sync IN the base worktree only when it's clean — that keeps its working tree consistent
@@ -656,14 +633,14 @@ async function syncLocalBaseToUpstream(
   // lands: advance the ref directly with update-ref, then best-effort sync the checked-out
   // index/worktree forward for paths that are not human-modified (#681).
   if (baseWorktree && !baseWorktree.dirty) {
-    const ff = await git(["merge", "--ff-only", upstream], baseWorktree.path);
+    const ff = await mergeGit(["merge", "--ff-only", upstream], baseWorktree.path, "rebase", `workspace merge fast-forward ${base} to ${upstream}`);
     if (!ff.ok) return { ok: false, error: ff.stderr || `failed to fast-forward ${base} to ${upstream}` };
     return { ok: true };
   }
-  const oldBaseTip = (await git(["rev-parse", base], repoRoot)).stdout;
+  const oldBaseTip = (await mergeGit(["rev-parse", base], repoRoot, "rebase", `workspace merge resolve ${base} before upstream sync`)).stdout;
   const dirtyBefore = baseWorktree?.dirty ? await dirtyPathSet(baseWorktree.path) : undefined;
   const updateArgs = oldBaseTip ? ["update-ref", `refs/heads/${base}`, upstreamSha, oldBaseTip] : ["update-ref", `refs/heads/${base}`, upstreamSha];
-  const update = await git(updateArgs, repoRoot);
+  const update = await mergeGit(updateArgs, repoRoot, "rebase", `workspace merge update ${base} to ${upstream}`);
   if (!update.ok) return { ok: false, error: update.stderr || `failed to advance ${base} to ${upstream}` };
   const baseSync = oldBaseTip ? await syncDirtyBaseWorktreeAfterRefAdvance(base, baseWorktree, oldBaseTip, upstreamSha, dirtyBefore) : undefined;
   return { ok: true, baseSync };
@@ -699,7 +676,8 @@ async function mergeRebaseFf(
   // points (the upstream sync below and the final land). A mixed state from either is surfaced
   // loudly rather than swallowed as a log warning (#824).
   let baseSync: BaseWorktreeSyncResult | undefined;
-  const upstream = await upstreamRef(worktreePath, base);
+  const upstreamResult = await mergeGit(["rev-parse", "--abbrev-ref", `${base}@{upstream}`], worktreePath, "rebase", `workspace merge resolve upstream for ${base}`);
+  const upstream = upstreamResult.ok && upstreamResult.stdout ? upstreamResult.stdout : undefined;
   const slash = upstream ? upstream.indexOf("/") : -1;
   const remote = slash > 0 ? upstream!.slice(0, slash) : undefined; // remote of a `remote/branch` upstream
   const pushEnabled = input.push !== false && workspacePushEnabled() && Boolean(remote);
@@ -708,18 +686,19 @@ async function mergeRebaseFf(
   // gives them new SHAs and breaks traceability (the branch.landed SHA must exist on
   // base verbatim). headSha is the preserved landed commit; when base has advanced we
   // tie the branch in with a no-ff merge so the agent's commits keep their identity.
-  const headSha = (await git(["rev-parse", "HEAD"], worktreePath)).stdout;
+  const headResult = await mergeGit(["rev-parse", "HEAD"], worktreePath, "rebase", "workspace merge resolve workspace HEAD before gates");
+  const headSha = headResult.stdout;
+  if (!headResult.ok || !headSha) return head({ status: "review_requested", error: gitError(headResult, "failed to resolve workspace HEAD before gates") });
   // Subject of the landed commit for the relay's branch.landed notice (#239). Best-effort:
   // an empty/failed read just omits it from the message body.
-  const landedSubject = (await git(["log", "-1", "--format=%s", headSha], worktreePath)).stdout || undefined;
-
-  // #902 BLOCKING 2 — NO mutation of `refs/heads/<base>` may happen until gates pass, so resolve
-  // the SHA the work will integrate onto WITHOUT moving the ref. Origin-ahead is the common
-  // concurrent case (#638): we still fetch fresh origin to compute the real merge, but we defer
-  // the local-base sync to AFTER the gates. On a gate failure `refs/heads/<base>` must be byte-
-  // identical to before the land attempt — so the only `refs/heads/<base>` mutation is the
-  // sync+advance below, all of it gated.
-  let integrationBaseSha = (await git(["rev-parse", base], repoRoot)).stdout;
+  const landedSubject = (await mergeGit(["log", "-1", "--format=%s", headSha], worktreePath, "rebase", "workspace merge read landed commit subject")).stdout || undefined;
+
+  // Resolve the SHA the work will integrate onto WITHOUT moving the ref. Origin-ahead is
+  // the common concurrent case (#638): we still fetch fresh origin to compute the real merge,
+  // but defer the local-base sync until the final base-ref mutation block below.
+  const integrationBaseResult = await mergeGit(["rev-parse", base], repoRoot, "rebase", `workspace merge resolve integration base ${base}`);
+  let integrationBaseSha = integrationBaseResult.stdout;
+  if (!integrationBaseResult.ok || !integrationBaseSha) return head({ status: "review_requested", error: gitError(integrationBaseResult, `failed to resolve integration base ${base}`) });
   let needSync = false;
   if (upstream && remote && pushEnabled) {
     try {
@@ -732,13 +711,13 @@ async function mergeRebaseFf(
     } catch (err) {
       return head({ status: "review_requested", error: errMessage(err) });
     }
-    if (!(await git(["merge-base", "--is-ancestor", upstream, base], worktreePath)).ok) {
+    if (!(await mergeGit(["merge-base", "--is-ancestor", upstream, base], worktreePath, "rebase", `workspace merge compare ${upstream} to ${base}`)).ok) {
       // Origin moved ahead. Sync-then-land iff local base is cleanly behind (ancestor
       // of upstream); otherwise it's genuine divergence — refuse without mutating.
-      if (!(await git(["merge-base", "--is-ancestor", base, upstream], worktreePath)).ok) {
+      if (!(await mergeGit(["merge-base", "--is-ancestor", base, upstream], worktreePath, "rebase", `workspace merge compare ${base} to ${upstream}`)).ok) {
         return head({ status: "review_requested", error: `local ${base} has diverged from ${upstream} (commits not on origin); sync before landing` });
       }
-      const upstreamSha = (await git(["rev-parse", "--verify", upstream], worktreePath)).stdout;
+      const upstreamSha = (await mergeGit(["rev-parse", "--verify", upstream], worktreePath, "rebase", `workspace merge resolve ${upstream} for integration`)).stdout;
       if (!upstreamSha) return head({ status: "review_requested", error: `cannot resolve ${upstream} to sync ${base}` });
       // Integrate onto fresh origin (the tree that will land), but DON'T advance local base yet.
       integrationBaseSha = upstreamSha;
@@ -747,18 +726,14 @@ async function mergeRebaseFf(
   }
 
   // Behind relative to the TRUE integration base (origin-ahead ⟹ behind>0 ⟹ a real no-ff merge).
-  const behind = await countBehind(worktreePath, integrationBaseSha);
-
-  // #902 BLOCKING 1 / closes #903 — run the repo's configured land gates against the EXACT tree
-  // that will become base's new tip, BEFORE advancing the ref. behind===0 → the worktree HEAD IS
-  // that tree (clean fast-forward); behind>0 → the synthesized no-ff merge of integrationBaseSha +
-  // HEAD, gated in a throwaway worktree (see runLandGatesOnIntegratedTree). A repo with no
-  // `.agent-relay/land-gates.json` runs ZERO gates (missing-file → empty list, no subprocess), so
-  // behind===0 is byte-identical to the prior land path (ironclad opt-in / zero regression). A
-  // failing REQUIRED gate aborts the land here WITHOUT advancing/syncing the ref — returned as
-  // status:"review_requested" carrying `gateFailure`, NOT "conflict": gate failures bounce back to
-  // the worker, never the merge-conflict/steward path. Optional-gate failures ride along as
-  // `gateWarnings` on the successful land below.
+  const behindResult = await countBehind(worktreePath, integrationBaseSha);
+  if ("error" in behindResult) return head({ status: "review_requested", error: behindResult.error });
+  const behind = behindResult.behind;
+
+  // #923 — land gates are decommissioned. This call is now a hard kill switch that always
+  // returns an empty gate set, even when `.agent-relay/land-gates.json` is populated. Keep it
+  // before base-ref mutation so the rest of the historical land ordering stays intact, but no
+  // gate subprocess or detached gate worktree can be launched at land time.
   const gateRun = await runLandGatesOnIntegratedTree(repoRoot, worktreePath, behind, integrationBaseSha, headSha, landMergeMessage(branch, landedSubject));
   if ("abort" in gateRun) {
     return gateRun.abort.conflict
@@ -771,9 +746,9 @@ async function mergeRebaseFf(
   }
   const gateWarningsField = gates.warnings.length ? { gateWarnings: gates.warnings } : {};
 
-  // Gates passed — only NOW is it safe to touch `refs/heads/<base>`. If origin moved ahead, sync
-  // local base up to the fetched upstream first (#638); this is the FIRST mutation of the base ref
-  // in the whole land path, so a gate failure above can never leave it advanced (#902 BLOCKING 2).
+  // Only NOW touch `refs/heads/<base>`. If origin moved ahead, sync local base up to the fetched
+  // upstream first (#638); this preserves the pre-existing land ordering while gate execution
+  // remains permanently disabled.
   if (needSync) {
     const synced = await syncLocalBaseToUpstream(repoRoot, worktreePath, base, upstream!);
     if (!synced.ok) return head({ status: "review_requested", error: synced.error });
@@ -792,28 +767,32 @@ async function mergeRebaseFf(
   const dirtyBasePathsBefore = baseWorktree?.dirty ? await dirtyPathSet(baseWorktree.path) : undefined;
   if (baseWorktree && !baseWorktree.dirty) {
     if (behind === 0) {
-      const ff = await git(["merge", "--ff-only", branch], baseWorktree.path);
+      const ff = await mergeGit(["merge", "--ff-only", branch], baseWorktree.path, "rebase", `workspace merge fast-forward ${base} to ${branch}`);
       if (!ff.ok) return head({ status: "review_requested", error: ff.stderr || "fast-forward into base failed" });
     } else {
-      const merge = await git(
+      const merge = await mergeGit(
         ["-c", `user.name=${LAND_COMMITTER.name}`, "-c", `user.email=${LAND_COMMITTER.email}`, "merge", "--no-ff", "-m", landMergeMessage(branch, landedSubject), branch],
         baseWorktree.path,
+        "rebase",
+        `workspace merge no-ff ${branch} into ${base}`,
       );
       if (!merge.ok) {
-        await git(["merge", "--abort"], baseWorktree.path);
+        await mergeGit(["merge", "--abort"], baseWorktree.path, "rebase", `workspace merge abort failed no-ff ${branch}`);
         return head({ conflict: true, status: "conflict", error: merge.stderr || "merge into base failed" });
       }
-      baseTip = (await git(["rev-parse", "HEAD"], baseWorktree.path)).stdout;
+      baseTip = (await mergeGit(["rev-parse", "HEAD"], baseWorktree.path, "rebase", `workspace merge resolve ${base} after no-ff`)).stdout;
     }
   } else if (behind === 0) {
-    const oldBaseTip = (await git(["rev-parse", base], repoRoot)).stdout;
+    const oldBaseTip = (await mergeGit(["rev-parse", base], repoRoot, "rebase", `workspace merge resolve ${base} before fast-forward ref update`)).stdout;
     const update = oldBaseTip
-      ? await git(["update-ref", `refs/heads/${base}`, headSha, oldBaseTip], repoRoot)
-      : await git(["update-ref", `refs/heads/${base}`, headSha], repoRoot);
+      ? await mergeGit(["update-ref", `refs/heads/${base}`, headSha, oldBaseTip], repoRoot, "rebase", `workspace merge update ${base} fast-forward`)
+      : await mergeGit(["update-ref", `refs/heads/${base}`, headSha], repoRoot, "rebase", `workspace merge create/update ${base} fast-forward`);
     if (!update.ok) return head({ status: "review_requested", error: update.stderr || "failed to advance base ref" });
     if (oldBaseTip) baseSync = mergeBaseSyncResults(baseSync, await syncDirtyBaseWorktreeAfterRefAdvance(base, baseWorktree, oldBaseTip, headSha, dirtyBasePathsBefore));
   } else {
-    const baseSha = (await git(["rev-parse", base], repoRoot)).stdout;
+    const baseShaResult = await mergeGit(["rev-parse", base], repoRoot, "rebase", `workspace merge resolve ${base} before no-ff ref update`);
+    const baseSha = baseShaResult.stdout;
+    if (!baseShaResult.ok || !baseSha) return head({ status: "review_requested", error: gitError(baseShaResult, `failed to resolve ${base} before no-ff ref update`) });
     const merged = await recordNoFfMerge(repoRoot, base, baseSha, headSha, landMergeMessage(branch, landedSubject));
     if (!merged.ok) return head(merged.conflict ? { conflict: true, status: "conflict", error: merged.error } : { status: "review_requested", error: merged.error });
     baseTip = merged.mergeSha;
@@ -833,7 +812,7 @@ async function mergeRebaseFf(
   // means origin raced us — surface it instead of claiming an unpublished land.
   let pushed = false;
   if (upstream && remote && pushEnabled) {
-    const push = await git(["push", remote, `${base}:${base}`], worktreePath);
+    const push = await mergeGit(["push", remote, `${base}:${base}`], worktreePath, "rebase", `workspace merge push ${base} to ${remote}`);
     if (!push.ok) return head({ status: "review_requested", mergedSha: headSha, error: push.stderr || `git push to ${remote}/${base} failed` });
     pushed = true;
   }
@@ -847,11 +826,11 @@ async function mergeRebaseFf(
   const deleteBranch = input.deleteBranch !== false;
   if (!deleteBranch) {
     const fresh = await nextBranchName(repoRoot, branch);
-    const switched = await git(["checkout", "-B", fresh, base], worktreePath);
+    const switched = await mergeGit(["checkout", "-B", fresh, base], worktreePath, "rebase", `workspace merge recycle worktree to ${fresh}`);
     if (switched.ok) {
       // The old branch is now fully contained in base (fast-forwarded, or merged in
       // as the no-ff merge's second parent) — drop the litter.
-      const oldDeleted = (await git(["branch", "-D", branch], repoRoot)).ok;
+      const oldDeleted = (await mergeGit(["branch", "-D", branch], repoRoot, "cleanup", `workspace merge delete landed branch ${branch}`)).ok;
       // The worktree just moved onto the advanced base, which may declare deps the
       // shared (symlinked) node_modules lacks. Re-provision so the recycled session
       // continues from a buildable state (issue #51). No-op when nothing is stale.
@@ -862,8 +841,8 @@ async function mergeRebaseFf(
     // Recycle failed — keep the existing branch. Still landed, still active.
     return head({ merged: true, status: "active", mergedSha: headSha, baseSha: baseTip, subject: landedSubject, worktreeRemoved: false, branchDeleted: false, pushed, ...baseWorktreeSyncField, ...gateWarningsField, error: undefined });
   }
-  const removed = await git(["worktree", "remove", "--force", worktreePath], repoRoot);
+  const removed = await mergeGit(["worktree", "remove", "--force", worktreePath], repoRoot, "cleanup", "workspace merge remove landed worktree");
   const worktreeRemoved = removed.ok;
-  const branchDeleted = worktreeRemoved ? (await git(["branch", "-D", branch], repoRoot)).ok : false;
+  const branchDeleted = worktreeRemoved ? (await mergeGit(["branch", "-D", branch], repoRoot, "cleanup", `workspace merge delete landed branch ${branch}`)).ok : false;
   return head({ merged: true, status: "merged", mergedSha: headSha, baseSha: baseTip, subject: landedSubject, worktreeRemoved, branchDeleted, pushed, ...baseWorktreeSyncField, ...gateWarningsField, error: undefined });
 }