agent-relay-orchestrator 0.118.2 → 0.118.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.118.2",
+  "version": "0.118.4",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {

package/src/git.ts

@@ -3,21 +3,23 @@
 // `git -C` invocation lives in one place.
 
 import { execProcess } from "./process";
+import type { ExecResult } from "./process";
 
-interface GitResult {
-  ok: boolean;
-  stdout: string;
-  stderr: string;
+type GitResult = Pick<ExecResult, "ok" | "stdout" | "stderr" | "exitCode" | "timedOut">;
+
+interface GitOptions {
+  timeoutMs?: number;
+  timeoutLabel?: string;
 }
 
 /** Run `git -C cwd <args>` and capture trimmed stdout/stderr; never throws. */
-export async function git(args: string[], cwd: string): Promise<GitResult> {
-  return await execProcess(["git", "-C", cwd, ...args]);
+export async function git(args: string[], cwd: string, options: GitOptions = {}): Promise<GitResult> {
+  return await execProcess(["git", "-C", cwd, ...args], options);
 }
 
 /** Run `git -C cwd <args>` and preserve stdout exactly for path-safe parsers. */
-export async function gitRaw(args: string[], cwd: string): Promise<GitResult> {
-  return await execProcess(["git", "-C", cwd, ...args], { trimStdout: false });
+export async function gitRaw(args: string[], cwd: string, options: GitOptions = {}): Promise<GitResult> {
+  return await execProcess(["git", "-C", cwd, ...args], { ...options, trimStdout: false });
 }
 
 /** Like {@link git} but throws on a non-zero exit, returning stdout on success. */

package/src/process.ts

@@ -3,6 +3,7 @@ export interface ExecResult {
   exitCode: number | null;
   stdout: string;
   stderr: string;
+  timedOut?: boolean;
 }
 
 interface ExecOptions {
@@ -12,11 +13,56 @@ interface ExecOptions {
   stderr?: "pipe" | "ignore";
   trimStdout?: boolean;
   trimStderr?: boolean;
+  timeoutMs?: number;
+  timeoutLabel?: string;
+  streamDrainGraceMs?: number;
 }
 
-async function readStream(stream: ReadableStream<Uint8Array> | undefined): Promise<string> {
-  if (!stream) return "";
-  return await new Response(stream).text();
+const DEFAULT_STREAM_DRAIN_GRACE_MS = 1_000;
+
+interface StreamCapture {
+  done: Promise<void>;
+  text(): string;
+  cancel(): void;
+}
+
+function captureStream(stream: ReadableStream<Uint8Array> | undefined): StreamCapture {
+  if (!stream) return { done: Promise.resolve(), text: () => "", cancel: () => {} };
+  const reader = stream.getReader();
+  const decoder = new TextDecoder();
+  let output = "";
+  let canceled = false;
+  const done = (async () => {
+    try {
+      while (!canceled) {
+        const chunk = await reader.read();
+        if (chunk.done) break;
+        output += decoder.decode(chunk.value, { stream: true });
+      }
+      output += decoder.decode();
+    } catch {
+      // Intentional cancellation on process timeout or a stuck post-exit pipe.
+    } finally {
+      try { reader.releaseLock(); } catch {}
+    }
+  })();
+  return {
+    done,
+    text: () => output,
+    cancel: () => {
+      canceled = true;
+      void reader.cancel().catch(() => {});
+    },
+  };
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function timeoutMessage(cmd: string[], options: ExecOptions): string {
+  const label = options.timeoutLabel ?? cmd.join(" ");
+  return `${label} timed out after ${options.timeoutMs}ms`;
 }
 
 export async function execProcess(cmd: string[], options: ExecOptions = {}): Promise<ExecResult> {
@@ -27,15 +73,50 @@ export async function execProcess(cmd: string[], options: ExecOptions = {}): Pro
     stdout: options.stdout ?? "pipe",
     stderr: options.stderr ?? "pipe",
   });
-  const [exitCode, stdout, stderr] = await Promise.all([
-    proc.exited,
-    options.stdout === "ignore" ? Promise.resolve("") : readStream(proc.stdout),
-    options.stderr === "ignore" ? Promise.resolve("") : readStream(proc.stderr),
-  ]);
+  const stdoutCapture = options.stdout === "ignore" ? captureStream(undefined) : captureStream(proc.stdout);
+  const stderrCapture = options.stderr === "ignore" ? captureStream(undefined) : captureStream(proc.stderr);
+  let timedOut = false;
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+  let killTimeout: ReturnType<typeof setTimeout> | undefined;
+  let timeoutResolve: ((value: null) => void) | undefined;
+  const timeoutPromise = new Promise<null>((resolve) => { timeoutResolve = resolve; });
+  if (options.timeoutMs && options.timeoutMs > 0) {
+    timeout = setTimeout(() => {
+      timedOut = true;
+      try { proc.kill("SIGTERM"); } catch {}
+      killTimeout = setTimeout(() => {
+        try { proc.kill("SIGKILL"); } catch {}
+      }, 1_000);
+      killTimeout.unref?.();
+      timeoutResolve?.(null);
+    }, options.timeoutMs);
+    timeout.unref?.();
+  }
+
+  const exitCode = await (timeout ? Promise.race([proc.exited, timeoutPromise]) : proc.exited);
+  if (timeout) clearTimeout(timeout);
+  if (!timedOut && killTimeout) clearTimeout(killTimeout);
+  if (timedOut) {
+    stdoutCapture.cancel();
+    stderrCapture.cancel();
+  }
+
+  const drainGraceMs = options.streamDrainGraceMs ?? DEFAULT_STREAM_DRAIN_GRACE_MS;
+  await Promise.race([Promise.allSettled([stdoutCapture.done, stderrCapture.done]), sleep(drainGraceMs)]);
+  stdoutCapture.cancel();
+  stderrCapture.cancel();
+
+  const stdout = stdoutCapture.text();
+  let stderr = stderrCapture.text();
+  if (timedOut) {
+    const msg = timeoutMessage(cmd, options);
+    stderr = stderr ? `${stderr}\n${msg}` : msg;
+  }
   return {
-    ok: exitCode === 0,
+    ok: !timedOut && exitCode === 0,
     exitCode,
     stdout: options.trimStdout === false ? stdout : stdout.trim(),
     stderr: options.trimStderr === false ? stderr : stderr.trim(),
+    ...(timedOut ? { timedOut } : {}),
   };
 }

package/src/workspace-probe/merge-timeouts.ts

@@ -0,0 +1,41 @@
+export type MergePhase = "preview" | "fetch" | "synthesize" | "worktree-add" | "cleanup";
+
+const MERGE_PHASE_TIMEOUTS_MS: Record<MergePhase, number> = {
+  preview: 30_000,
+  fetch: 60_000,
+  synthesize: 60_000,
+  "worktree-add": 60_000,
+  cleanup: 30_000,
+};
+
+function positiveEnvMs(name: string): number | undefined {
+  const raw = process.env[name];
+  if (!raw) return undefined;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
+}
+
+export function mergePhaseTimeoutMs(phase: MergePhase): number {
+  const key = `AGENT_RELAY_WORKSPACE_MERGE_${phase.toUpperCase().replace(/-/g, "_")}_TIMEOUT_MS`;
+  return positiveEnvMs(key)
+    ?? positiveEnvMs("AGENT_RELAY_WORKSPACE_MERGE_PHASE_TIMEOUT_MS")
+    ?? MERGE_PHASE_TIMEOUTS_MS[phase];
+}
+
+export async function withMergePhaseTimeout<T>(phase: MergePhase, run: () => Promise<T>): Promise<T> {
+  const timeoutMs = mergePhaseTimeoutMs(phase);
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+  const work = run();
+  work.catch(() => {});
+  try {
+    return await Promise.race([
+      work,
+      new Promise<T>((_, reject) => {
+        timeout = setTimeout(() => reject(new Error(`workspace merge phase "${phase}" timed out after ${timeoutMs}ms`)), timeoutMs);
+        timeout.unref?.();
+      }),
+    ]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}

package/src/workspace-probe/merge.ts

@@ -1,13 +1,14 @@
 import { existsSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
-import type { BaseWorktreeSyncResult, WorkspaceMergePreview, WorkspaceMergeResult } from "agent-relay-sdk";
+import { errMessage, type BaseWorktreeSyncResult, type WorkspaceMergePreview, type WorkspaceMergeResult } from "agent-relay-sdk";
 import { git, gitRaw } from "../git";
 import { execProcess } from "../process";
 import { prMergedState } from "../workspace-pr";
 import { refreshWorkspaceDeps } from "./deps";
 import { type LandGatesResult, runLandGates } from "./land-gates-runner";
 import { populateMergeState, resolveBranchRef, syncBaseFromOrigin, upstreamRef, workspaceGitState } from "./git-state";
+import { mergePhaseTimeoutMs, withMergePhaseTimeout } from "./merge-timeouts";
 import { nextBranchName } from "./names";
 import { parseWorktrees, shortBranch } from "./parse";
 import { json, resolveRequestedPath } from "./request";
@@ -357,7 +358,12 @@ export async function mergeWorkspace(input: WorkspaceMergeInput): Promise<Worksp
   // `input.branch ?? ...` guard unchanged and cause git to attempt merging a non-existent ref.
   const liveBranch = shortBranch((await git(["symbolic-ref", "--quiet", "--short", "HEAD"], worktreePath)).stdout || undefined);
   const branch = liveBranch ?? input.branch;
-  const preview = await previewWorkspaceMerge({ worktreePath, baseRef: input.baseRef, baseSha: input.baseSha, strategy: input.strategy });
+  let preview: WorkspaceMergePreview;
+  try {
+    preview = await withMergePhaseTimeout("preview", () => previewWorkspaceMerge({ worktreePath, baseRef: input.baseRef, baseSha: input.baseSha, strategy: input.strategy }));
+  } catch (err) {
+    return { workspaceId: input.id, strategy: "rebase-ff", merged: false, status: "review_requested", branch, error: errMessage(err) };
+  }
   const strategy = preview.strategy;
   const head = (field: Partial<WorkspaceMergeResult>): WorkspaceMergeResult => ({ workspaceId: input.id, strategy, merged: false, status: "review_requested", branch, baseRef: preview.baseRef, ...field });
 
@@ -531,14 +537,16 @@ async function synthesizeNoFfMerge(
   baseSha: string,
   branchSha: string,
   message: string,
+  timeoutMs?: number,
 ): Promise<{ ok: true; mergeSha: string } | { ok: false; conflict?: boolean; error: string }> {
-  const tree = await git(["merge-tree", "--write-tree", baseSha, branchSha], repoRoot);
+  const tree = await git(["merge-tree", "--write-tree", baseSha, branchSha], repoRoot, { timeoutMs, timeoutLabel: "workspace merge synthesize merge-tree" });
   if (!tree.ok) return { ok: false, conflict: true, error: tree.stdout || tree.stderr || "merge conflict computing tree" };
   const treeOid = tree.stdout.split("\n")[0]?.trim();
   if (!treeOid) return { ok: false, error: "merge-tree produced no tree oid" };
   const commit = await git(
     ["-c", `user.name=${LAND_COMMITTER.name}`, "-c", `user.email=${LAND_COMMITTER.email}`, "commit-tree", treeOid, "-p", baseSha, "-p", branchSha, "-m", message],
     repoRoot,
+    { timeoutMs, timeoutLabel: "workspace merge synthesize commit-tree" },
   );
   if (!commit.ok || !commit.stdout) return { ok: false, error: commit.stderr || "commit-tree failed" };
   return { ok: true, mergeSha: commit.stdout };
@@ -581,7 +589,12 @@ async function runLandGatesOnIntegratedTree(
 ): Promise<{ gates: LandGatesResult } | { abort: { conflict?: boolean; error: string } }> {
   if (behind === 0) return { gates: await runLandGates(worktreePath) };
 
-  const synth = await synthesizeNoFfMerge(repoRoot, integrationBaseSha, headSha, mergeMessage);
+  let synth: Awaited<ReturnType<typeof synthesizeNoFfMerge>>;
+  try {
+    synth = await withMergePhaseTimeout("synthesize", () => synthesizeNoFfMerge(repoRoot, integrationBaseSha, headSha, mergeMessage, mergePhaseTimeoutMs("synthesize")));
+  } catch (err) {
+    return { abort: { error: errMessage(err) } };
+  }
   if (!synth.ok) return { abort: { conflict: synth.conflict, error: synth.error } };
 
   // Detached worktree in an isolated temp dir so the gates see the integrated tree on disk
@@ -589,7 +602,17 @@ async function runLandGatesOnIntegratedTree(
   // creates the leaf, so hand it a not-yet-existing path under a freshly made parent dir.
   const tmpParent = mkdtempSync(join(tmpdir(), "agent-relay-landgate-"));
   const tmpWorktree = join(tmpParent, "tree");
-  const add = await git(["worktree", "add", "--detach", tmpWorktree, synth.mergeSha], repoRoot);
+  let add: Awaited<ReturnType<typeof git>>;
+  try {
+    add = await withMergePhaseTimeout("worktree-add", () => git(
+      ["worktree", "add", "--detach", tmpWorktree, synth.mergeSha],
+      repoRoot,
+      { timeoutMs: mergePhaseTimeoutMs("worktree-add"), timeoutLabel: "workspace merge land-gate worktree add" },
+    ));
+  } catch (err) {
+    rmSync(tmpParent, { recursive: true, force: true });
+    return { abort: { error: errMessage(err) } };
+  }
   if (!add.ok) {
     rmSync(tmpParent, { recursive: true, force: true });
     return { abort: { error: add.stderr || "failed to materialize integrated tree for land gates" } };
@@ -597,7 +620,15 @@ async function runLandGatesOnIntegratedTree(
   try {
     return { gates: await runLandGates(tmpWorktree) };
   } finally {
-    await git(["worktree", "remove", "--force", tmpWorktree], repoRoot);
+    try {
+      await withMergePhaseTimeout("cleanup", () => git(
+        ["worktree", "remove", "--force", tmpWorktree],
+        repoRoot,
+        { timeoutMs: mergePhaseTimeoutMs("cleanup"), timeoutLabel: "workspace merge land-gate worktree cleanup" },
+      ));
+    } catch (err) {
+      console.error(`[orchestrator] land-gate integrated worktree cleanup timed out/failed: ${errMessage(err)}`);
+    }
     rmSync(tmpParent, { recursive: true, force: true });
   }
 }
@@ -691,7 +722,16 @@ async function mergeRebaseFf(
   let integrationBaseSha = (await git(["rev-parse", base], repoRoot)).stdout;
   let needSync = false;
   if (upstream && remote && pushEnabled) {
-    await git(["fetch", remote, base], worktreePath); // best-effort freshness; a stale ref can only under-detect divergence
+    try {
+      const fetch = await withMergePhaseTimeout("fetch", () => git(
+        ["fetch", remote, base],
+        worktreePath,
+        { timeoutMs: mergePhaseTimeoutMs("fetch"), timeoutLabel: `workspace merge fetch ${remote}/${base}` },
+      ));
+      if (!fetch.ok && fetch.timedOut) return head({ status: "review_requested", error: fetch.stderr || `fetch ${remote}/${base} timed out` });
+    } catch (err) {
+      return head({ status: "review_requested", error: errMessage(err) });
+    }
     if (!(await git(["merge-base", "--is-ancestor", upstream, base], worktreePath)).ok) {
       // Origin moved ahead. Sync-then-land iff local base is cleanly behind (ancestor
       // of upstream); otherwise it's genuine divergence — refuse without mutating.