agent-relay-orchestrator 0.118.5 → 0.119.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.118.5",
+  "version": "0.119.0",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -17,7 +17,7 @@
   },
   "dependencies": {
     "agent-relay-providers": "0.104.1",
-    "agent-relay-sdk": "0.2.101",
+    "agent-relay-sdk": "0.2.102",
     "callmux": "0.23.0"
   },
   "devDependencies": {

package/src/control.ts

@@ -1,10 +1,10 @@
-import { errMessage, isRecord, normalizeAgentLifecycle, normalizeWorkspaceMode } from "agent-relay-sdk";
+import { errMessage, isRecord, normalizeAgentLifecycle, normalizeWorkspaceMode, Semaphore } from "agent-relay-sdk";
 import { getAllManifests, getManifest } from "agent-relay-providers";
 import type { OrchestratorConfig } from "./config";
 import type { ManagedAgentReport, RelayClient, RelayCommand } from "./relay";
 import { handleSelfUpgrade } from "./self-upgrade";
 import { readLocalProviderConfigs } from "./provider-config-migration";
-import { spawnAgent, stopSession, type SpawnOptions } from "./spawn";
+import { findSessionRecord, readRunnerInfo, spawnAgent, stopSession, type SpawnOptions } from "./spawn";
 import { cleanupWorkspace, discardRecoveryBranch, idleRefreshWorktree, mergeWorkspace, pruneWorktrees, reconcileWorkspace, refreshWorkspaceDeps, workspacesRoot } from "./workspace-probe";
 import { withMergePhaseTimeout } from "./workspace-probe/merge-timeouts";
 import { armWorkspacePrAutoMerge, mergeWorkspacePr, refreshWorkspacePrBranch } from "./workspace-pr";
@@ -24,22 +24,167 @@ interface ControlHandler {
   handleCommand(command: RelayCommand): Promise<boolean>;
   getManagedAgents(): ManagedAgentReport[];
   setManagedAgents(agents: ManagedAgentReport[]): void;
+  // #930/#880 — spawn commands dispatch in the background (claimed synchronously,
+  // then run past registration under the concurrency cap). These expose that
+  // in-flight background work so callers/tests can observe and settle it.
+  spawnsInFlight(): number;
+  settleSpawns(): Promise<void>;
+}
+
+const DEFAULT_MAX_CONCURRENT_SPAWNS = 2;
+
+// #930/#880 — per-host spawn concurrency cap. Bursts of spawns racing worktree
+// materialization + port churn during the registration window are what surface
+// as `EADDRINUSE`/"operation was aborted" on the loser (#880). Bound how many
+// spawns materialize at once. Parsed once at handler construction; invalid or
+// out-of-range values fall back to the safe default rather than throwing.
+export function resolveMaxConcurrentSpawns(env: NodeJS.ProcessEnv = process.env): number {
+  const raw = env.AGENT_RELAY_MAX_CONCURRENT_SPAWNS;
+  if (raw === undefined || raw.trim() === "") return DEFAULT_MAX_CONCURRENT_SPAWNS;
+  const parsed = Number(raw);
+  if (!Number.isInteger(parsed) || parsed < 1) return DEFAULT_MAX_CONCURRENT_SPAWNS;
+  return parsed;
+}
+
+// #930/#880 — how long a spawn slot is held while we wait for the child to
+// register (bind its local control/ws server — the contended resource). This is
+// a fail-safe upper bound only: a child that never registers must not wedge a
+// slot forever, so we release after this deadline even if it hasn't appeared.
+// The happy path releases as soon as the child's runner-info file materializes,
+// well under this ceiling.
+const DEFAULT_SPAWN_REGISTRATION_TIMEOUT_MS = 30_000;
+const SPAWN_REGISTRATION_POLL_MS = 100;
+
+export function resolveSpawnRegistrationTimeoutMs(env: NodeJS.ProcessEnv = process.env): number {
+  const raw = env.AGENT_RELAY_SPAWN_REGISTRATION_TIMEOUT_MS;
+  if (raw === undefined || raw.trim() === "") return DEFAULT_SPAWN_REGISTRATION_TIMEOUT_MS;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_SPAWN_REGISTRATION_TIMEOUT_MS;
+  return parsed;
+}
+
+/** Awaits a spawn slot until the child registers, or resolves false at the fail-safe deadline. */
+export type WaitForRegistration = (agent: ManagedAgentReport, timeoutMs: number) => Promise<boolean>;
+
+// #930/#880 — the orchestrator observes a child registering through its runner
+// info file: the runner writes `controlUrl` immediately after binding its local
+// control/MCP server (runner-core writeRunnerInfoFile, right after startMcpProxy),
+// which is exactly #880's contended ws-bind window. So the file's presence is the
+// host-local signal that the child is past that window; holding the slot until it
+// appears is what bounds how many children materialize→bind at once.
+async function waitForManagedRegistration(agent: ManagedAgentReport, timeoutMs: number): Promise<boolean> {
+  const session = agent.sessionName ?? agent.tmuxSession;
+  if (!session) return false;
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    const record = findSessionRecord({ tmuxSession: session, agentId: agent.agentId });
+    if (record && readRunnerInfo(record)) return true;
+    if (Date.now() >= deadline) return false;
+    await Bun.sleep(SPAWN_REGISTRATION_POLL_MS);
+  }
+}
+
+interface ControlHandlerDeps {
+  // Injectable spawn dispatch — defaults to the real spawnAgent. Only tests
+  // override it, to gate dispatch and observe the concurrency bound directly.
+  spawnAgent?: typeof spawnAgent;
+  // Override the cap directly, bypassing the env parse (tests).
+  maxConcurrentSpawns?: number;
+  // Injectable registration wait — defaults to polling the child's runner info
+  // file. Tests override it to gate the registration window explicitly.
+  waitForRegistration?: WaitForRegistration;
+  // Override the fail-safe registration timeout, bypassing the env parse (tests).
+  registrationTimeoutMs?: number;
 }
 
 export function createControlHandler(
   config: OrchestratorConfig,
   relay: RelayClient,
+  deps: ControlHandlerDeps = {},
 ): ControlHandler {
   let managedAgents: ManagedAgentReport[] = [];
+  const dispatchSpawn = deps.spawnAgent ?? spawnAgent;
+  // In-memory counter is correct here: a slot only needs to live within this
+  // orchestrator process, and a crash resets in-flight spawns anyway (#881 Q1).
+  const spawnSlots = new Semaphore(deps.maxConcurrentSpawns ?? resolveMaxConcurrentSpawns());
+  const waitForRegistration = deps.waitForRegistration ?? waitForManagedRegistration;
+  const registrationTimeoutMs = deps.registrationTimeoutMs ?? resolveSpawnRegistrationTimeoutMs();
+  // Backgrounded spawn tasks: each is claimed synchronously (status → accepted)
+  // then runs to completion off the poll tick, holding a slot across the child's
+  // registration window. Tracked so tests/shutdown can observe and settle them.
+  const backgroundSpawns = new Set<Promise<void>>();
 
-  async function handleSpawn(ctrl: Record<string, any>): Promise<boolean> {
+  async function handleSpawn(ctrl: Record<string, any>): Promise<ManagedAgentReport> {
     const opts = spawnOptionsFromControl(ctrl, config);
-    const agent = await spawnAgent(opts, config);
+    const agent = await dispatchSpawn(opts, config);
     managedAgents.push(agent);
     console.error(`[orchestrator] Spawned ${opts.provider} agent: ${agent.tmuxSession}`);
+    return agent;
+  }
+
+  // #930/#880 — dispatch a spawn command concurrently, up to the per-host cap.
+  //
+  // The command is CLAIMED (status → accepted) synchronously here, BEFORE we
+  // return, so the next pollCommands() tick can never re-return this still-pending
+  // row and double-dispatch it. The slot itself is taken in the BACKGROUND via
+  // acquire() (strict FIFO — no starvation), so the poll tick is never blocked
+  // waiting for a free slot, preserving the #864 single-flight of the tick.
+  //
+  // The background task holds its slot across the child's registration window
+  // (#880's contended ws-bind resource) and frees it in `finally` on EVERY path:
+  // registration success, spawn failure, or the fail-safe registration timeout.
+  async function dispatchSpawnCommand(command: RelayCommand): Promise<boolean> {
+    await relay.updateCommand(command.id, "accepted");
+    // #930 — guard the backgrounded task against unhandled rejection. runSpawnToRegistration
+    // settles its own errors, but a relay blip in the ≤30s in-flight window can reject the
+    // status write in its catch block too; without a terminal `.catch` here that escapes as
+    // an unhandledRejection (log spam, or a crash if the runtime exits on it). Track the
+    // GUARDED promise so settleSpawns() awaits a promise that never rejects.
+    const task = runSpawnToRegistration(command).catch((error) => {
+      console.error(`[orchestrator] backgrounded spawn task error (swallowed): ${errMessage(error)}`);
+    });
+    backgroundSpawns.add(task);
+    void task.finally(() => backgroundSpawns.delete(task));
     return true;
   }
 
+  async function runSpawnToRegistration(command: RelayCommand): Promise<void> {
+    // Blocks FIFO until a slot frees — this is what bounds concurrent spawns to N.
+    await spawnSlots.acquire();
+    try {
+      await relay.updateCommand(command.id, "running");
+      const agent = await handleSpawn(command.params);
+      const registered = await waitForRegistration(agent, registrationTimeoutMs);
+      if (!registered) {
+        // Not a spawn failure — the process launched; it just didn't register in
+        // time. Release the slot (fail-safe) so a slow/never-registering child
+        // can't wedge capacity, but still settle the command as succeeded.
+        console.error(`[orchestrator] spawn ${agent.tmuxSession} did not register within ${registrationTimeoutMs}ms; releasing slot (fail-safe)`);
+      }
+      await relay.updateCommand(command.id, "succeeded", { managedAgents });
+      await relay.updateManagedAgents(managedAgents);
+    } catch (error) {
+      console.error(`[orchestrator] spawn dispatch failed: ${errMessage(error)}`);
+      // #930 — the `failed` status write can itself reject during a relay blip. Wrap it so
+      // a failed status write can't re-throw out of the backgrounded task (belt-and-suspenders
+      // with the `.catch` in dispatchSpawnCommand). The command may strand non-terminal until
+      // the TTL sweep / watchdog reconciles it, but the task settles cleanly regardless.
+      try {
+        await relay.updateCommand(command.id, "failed", undefined, errMessage(error));
+      } catch (statusError) {
+        console.error(`[orchestrator] failed to mark spawn command failed: ${errMessage(statusError)}`);
+      }
+    } finally {
+      spawnSlots.release();
+    }
+  }
+
+  async function settleSpawns(): Promise<void> {
+    while (backgroundSpawns.size > 0) {
+      await Promise.all([...backgroundSpawns]);
+    }
+  }
+
   async function handleShutdown(ctrl: Record<string, any>, restart = false): Promise<Record<string, unknown>> {
     const current = managedAgentShutdownTarget(managedAgents, ctrl);
     const session = current?.sessionName ?? current?.tmuxSession;
@@ -86,14 +231,18 @@ export function createControlHandler(
   }
 
   async function handleCommand(command: RelayCommand): Promise<boolean> {
+    // #930/#880 — spawns dispatch concurrently in the background under the
+    // per-host cap (see dispatchSpawnCommand). Every other command is handled
+    // inline, synchronously with the poll tick, exactly as before.
+    if (command.type === "agent.spawn") return dispatchSpawnCommand(command);
+    return handleNonSpawnCommand(command);
+  }
+
+  async function handleNonSpawnCommand(command: RelayCommand): Promise<boolean> {
     await relay.updateCommand(command.id, "accepted");
     await relay.updateCommand(command.id, "running");
     try {
-      if (command.type === "agent.spawn") {
-        const handled = await handleSpawn(command.params);
-        if (!handled) throw new Error("spawn failed");
-        await relay.updateCommand(command.id, "succeeded", { managedAgents });
-      } else if (command.type === "agent.shutdown" || command.type === "agent.restart") {
+      if (command.type === "agent.shutdown" || command.type === "agent.restart") {
         const result = await handleShutdown(command.params, command.type === "agent.restart");
         await relay.updateCommand(command.id, "succeeded", result);
       } else if (command.type === "workspace.cleanup") {
@@ -250,7 +399,7 @@ export function createControlHandler(
     managedAgents = agents;
   }
 
-  return { handleCommand, getManagedAgents, setManagedAgents };
+  return { handleCommand, getManagedAgents, setManagedAgents, spawnsInFlight: () => backgroundSpawns.size, settleSpawns };
 }
 
 export function managedAgentShutdownTarget(agents: ManagedAgentReport[], ctrl: Record<string, any>): ManagedAgentReport | undefined {

package/src/workspace-probe/merge.ts

@@ -1,12 +1,11 @@
-import { existsSync, mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
+import { existsSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { errMessage, type BaseWorktreeSyncResult, type WorkspaceMergePreview, type WorkspaceMergeResult } from "agent-relay-sdk";
 import { git, gitRaw } from "../git";
 import { execProcess } from "../process";
 import { prMergedState } from "../workspace-pr";
 import { refreshWorkspaceDeps } from "./deps";
-import { type LandGatesResult, runLandGates } from "./land-gates-runner";
+import { type LandGatesResult } from "./land-gates-runner";
 import { populateMergeState, resolveBranchRef, syncBaseFromOrigin, workspaceGitState } from "./git-state";
 import { type MergePhase, mergePhaseTimeoutMs, withMergePhaseTimeout } from "./merge-timeouts";
 import { nextBranchName } from "./names";
@@ -584,18 +583,16 @@ async function recordNoFfMerge(
   return { ok: true, mergeSha: synth.mergeSha };
 }
 
+function decommissionedLandGatesResult(): LandGatesResult {
+  return { ran: 0, warnings: [] };
+}
+
 /**
- * Run the repo's configured land gates against the EXACT tree that will become base's new tip,
- * BEFORE any ref advance (#902 BLOCKING 1 / closes #903). The candidate tree depends on `behind`:
- *  - behind === 0: the landing worktree HEAD already IS that tree (a clean fast-forward), so gates
- *    run in `worktreePath` in place — byte-identical to today's straight-FF path.
- *  - behind  >  0: base has advanced, so the tree that will land is the no-ff MERGE of
- *    `integrationBaseSha` and `headSha`, NOT the worktree HEAD. Synthesize that merge commit with
- *    plumbing (no ref moved), check it out in a throwaway DETACHED worktree, and gate THAT tree.
- *    The temp worktree is torn down on EVERY exit path (including a gate that throws), so a land
- *    can never strand an orphan worktree or leave the synthesized merge half-applied.
- * Returns the gate outcome, or an `abort` describing a merge result the caller must return early
- * (a real merge conflict computing the integrated tree, or a failure materializing it).
+ * Land gates are decommissioned (#923). Release-time gates in `release.ts` are the
+ * quality boundary; workspace landing must never launch repo-configured gate subprocesses.
+ * Keep this function as the single merge-path kill switch so populated
+ * `.agent-relay/land-gates.json` files are treated as always-empty without changing the
+ * rest of the rebase/fetch/no-ff landing flow.
  */
 async function runLandGatesOnIntegratedTree(
   repoRoot: string,
@@ -605,51 +602,12 @@ async function runLandGatesOnIntegratedTree(
   headSha: string,
   mergeMessage: string,
 ): Promise<{ gates: LandGatesResult } | { abort: { conflict?: boolean; error: string } }> {
-  console.error(`[orchestrator] workspace.merge gate-start worktree=${worktreePath} behind=${behind}`);
-  if (behind === 0) return { gates: await withMergePhaseTimeout("gates", () => runLandGates(worktreePath)) };
-
-  let synth: Awaited<ReturnType<typeof synthesizeNoFfMerge>>;
-  try {
-    synth = await withMergePhaseTimeout("synthesize", () => synthesizeNoFfMerge(repoRoot, integrationBaseSha, headSha, mergeMessage, mergePhaseTimeoutMs("synthesize")));
-  } catch (err) {
-    return { abort: { error: errMessage(err) } };
-  }
-  if (!synth.ok) return { abort: { conflict: synth.conflict, error: synth.error } };
-
-  // Detached worktree in an isolated temp dir so the gates see the integrated tree on disk
-  // without disturbing the landing worktree, the base checkout, or any ref. `git worktree add`
-  // creates the leaf, so hand it a not-yet-existing path under a freshly made parent dir.
-  const tmpParent = mkdtempSync(join(tmpdir(), "agent-relay-landgate-"));
-  const tmpWorktree = join(tmpParent, "tree");
-  let add: Awaited<ReturnType<typeof git>>;
-  try {
-    add = await withMergePhaseTimeout("worktree-add", () => git(
-      ["worktree", "add", "--detach", tmpWorktree, synth.mergeSha],
-      repoRoot,
-      { timeoutMs: mergePhaseTimeoutMs("worktree-add"), timeoutLabel: "workspace merge land-gate worktree add" },
-    ));
-  } catch (err) {
-    rmSync(tmpParent, { recursive: true, force: true });
-    return { abort: { error: errMessage(err) } };
-  }
-  if (!add.ok) {
-    rmSync(tmpParent, { recursive: true, force: true });
-    return { abort: { error: add.stderr || "failed to materialize integrated tree for land gates" } };
-  }
-  try {
-    return { gates: await withMergePhaseTimeout("gates", () => runLandGates(tmpWorktree)) };
-  } finally {
-    try {
-      await withMergePhaseTimeout("cleanup", () => git(
-        ["worktree", "remove", "--force", tmpWorktree],
-        repoRoot,
-        { timeoutMs: mergePhaseTimeoutMs("cleanup"), timeoutLabel: "workspace merge land-gate worktree cleanup" },
-      ));
-    } catch (err) {
-      console.error(`[orchestrator] land-gate integrated worktree cleanup timed out/failed: ${errMessage(err)}`);
-    }
-    rmSync(tmpParent, { recursive: true, force: true });
-  }
+  void repoRoot;
+  void integrationBaseSha;
+  void headSha;
+  void mergeMessage;
+  console.error(`[orchestrator] workspace.merge gate-skip worktree=${worktreePath} behind=${behind} reason=land-gates-decommissioned`);
+  return { gates: decommissionedLandGatesResult() };
 }
 
 /**
@@ -735,12 +693,9 @@ async function mergeRebaseFf(
   // an empty/failed read just omits it from the message body.
   const landedSubject = (await mergeGit(["log", "-1", "--format=%s", headSha], worktreePath, "rebase", "workspace merge read landed commit subject")).stdout || undefined;
 
-  // #902 BLOCKING 2 — NO mutation of `refs/heads/<base>` may happen until gates pass, so resolve
-  // the SHA the work will integrate onto WITHOUT moving the ref. Origin-ahead is the common
-  // concurrent case (#638): we still fetch fresh origin to compute the real merge, but we defer
-  // the local-base sync to AFTER the gates. On a gate failure `refs/heads/<base>` must be byte-
-  // identical to before the land attempt — so the only `refs/heads/<base>` mutation is the
-  // sync+advance below, all of it gated.
+  // Resolve the SHA the work will integrate onto WITHOUT moving the ref. Origin-ahead is
+  // the common concurrent case (#638): we still fetch fresh origin to compute the real merge,
+  // but defer the local-base sync until the final base-ref mutation block below.
   const integrationBaseResult = await mergeGit(["rev-parse", base], repoRoot, "rebase", `workspace merge resolve integration base ${base}`);
   let integrationBaseSha = integrationBaseResult.stdout;
   if (!integrationBaseResult.ok || !integrationBaseSha) return head({ status: "review_requested", error: gitError(integrationBaseResult, `failed to resolve integration base ${base}`) });
@@ -775,16 +730,10 @@ async function mergeRebaseFf(
   if ("error" in behindResult) return head({ status: "review_requested", error: behindResult.error });
   const behind = behindResult.behind;
 
-  // #902 BLOCKING 1 / closes #903 — run the repo's configured land gates against the EXACT tree
-  // that will become base's new tip, BEFORE advancing the ref. behind===0 → the worktree HEAD IS
-  // that tree (clean fast-forward); behind>0 → the synthesized no-ff merge of integrationBaseSha +
-  // HEAD, gated in a throwaway worktree (see runLandGatesOnIntegratedTree). A repo with no
-  // `.agent-relay/land-gates.json` runs ZERO gates (missing-file → empty list, no subprocess), so
-  // behind===0 is byte-identical to the prior land path (ironclad opt-in / zero regression). A
-  // failing REQUIRED gate aborts the land here WITHOUT advancing/syncing the ref — returned as
-  // status:"review_requested" carrying `gateFailure`, NOT "conflict": gate failures bounce back to
-  // the worker, never the merge-conflict/steward path. Optional-gate failures ride along as
-  // `gateWarnings` on the successful land below.
+  // #923 — land gates are decommissioned. This call is now a hard kill switch that always
+  // returns an empty gate set, even when `.agent-relay/land-gates.json` is populated. Keep it
+  // before base-ref mutation so the rest of the historical land ordering stays intact, but no
+  // gate subprocess or detached gate worktree can be launched at land time.
   const gateRun = await runLandGatesOnIntegratedTree(repoRoot, worktreePath, behind, integrationBaseSha, headSha, landMergeMessage(branch, landedSubject));
   if ("abort" in gateRun) {
     return gateRun.abort.conflict
@@ -797,9 +746,9 @@ async function mergeRebaseFf(
   }
   const gateWarningsField = gates.warnings.length ? { gateWarnings: gates.warnings } : {};
 
-  // Gates passed — only NOW is it safe to touch `refs/heads/<base>`. If origin moved ahead, sync
-  // local base up to the fetched upstream first (#638); this is the FIRST mutation of the base ref
-  // in the whole land path, so a gate failure above can never leave it advanced (#902 BLOCKING 2).
+  // Only NOW touch `refs/heads/<base>`. If origin moved ahead, sync local base up to the fetched
+  // upstream first (#638); this preserves the pre-existing land ordering while gate execution
+  // remains permanently disabled.
   if (needSync) {
     const synced = await syncLocalBaseToUpstream(repoRoot, worktreePath, base, upstream!);
     if (!synced.ok) return head({ status: "review_requested", error: synced.error });