agent-relay-orchestrator 0.84.0 → 0.86.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.84.0",
+  "version": "0.86.0",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -16,7 +16,7 @@
     "test": "bun test"
   },
   "dependencies": {
-    "agent-relay-sdk": "0.2.64"
+    "agent-relay-sdk": "0.2.66"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/src/control.ts

@@ -6,6 +6,17 @@ import { readLocalProviderConfigs } from "./provider-config-migration";
 import { spawnAgent, stopSession, type SpawnOptions } from "./spawn";
 import { cleanupWorkspace, discardRecoveryBranch, mergeWorkspace, pruneWorktrees, reconcileWorkspace, refreshWorkspaceDeps, workspacesRoot } from "./workspace-probe";
 import { armWorkspacePrAutoMerge, mergeWorkspacePr, refreshWorkspacePrBranch } from "./workspace-pr";
+import type { WorkspaceMergeResult } from "agent-relay-sdk";
+
+// #638 — settle a `workspace.merge` command on whether it made progress, not on whether
+// the merge function threw. Every no-progress outcome (origin moved ahead, an unpredicted
+// replay conflict, a push race) returns merged:false with an `error` set; every genuine
+// land / recycle / noop-resolve / PR-opened leaves `error` undefined. Reporting `succeeded`
+// for an unlanded branch is what let the auto-merge job believe it was progressing and
+// busy-loop the repo's merge lease. Mirrors the pr-merge / pr-arm settle contract.
+export function mergeCommandStatus(result: WorkspaceMergeResult): "succeeded" | "failed" {
+  return result.error ? "failed" : "succeeded";
+}
 
 interface ControlHandler {
   handleCommand(command: RelayCommand): Promise<boolean>;
@@ -132,7 +143,9 @@ export function createControlHandler(
               }
             : undefined,
         });
-        await relay.updateCommand(command.id, "succeeded", result as unknown as Record<string, unknown>);
+        // #638 — settle `failed` (carrying the error) for a no-op merge instead of
+        // `succeeded`; see mergeCommandStatus.
+        await relay.updateCommand(command.id, mergeCommandStatus(result), result as unknown as Record<string, unknown>, result.error);
       } else if (command.type === "workspace.pr-arm-auto-merge") {
         const rawPrNumber = command.params.prNumber;
         const result = armWorkspacePrAutoMerge({

package/src/spawn/supervisor.ts

@@ -178,6 +178,37 @@ function logFileDiagnostics(logFile: string): Pick<ManagedSessionExitDiagnostics
   }
 }
 
+/** Map systemd ExecMain* / Result + a best-effort journal OOM probe into signal/exitCode/oom (#636). */
+function terminationDiagnostics(systemd: ManagedSessionExitDiagnostics["systemd"], pid?: number): Pick<ManagedSessionExitDiagnostics, "signal" | "exitCode" | "oom"> {
+  const out: Pick<ManagedSessionExitDiagnostics, "signal" | "exitCode" | "oom"> = {};
+  if (systemd?.result === "oom-kill") {
+    out.oom = { source: "systemd", detail: "Result=oom-kill" };
+  }
+  if (systemd?.execMainCode && /killed|dumped/i.test(systemd.execMainCode) && systemd.execMainStatus) {
+    out.signal = /^\d+$/.test(systemd.execMainStatus) ? `signal ${systemd.execMainStatus}` : `SIG${systemd.execMainStatus.replace(/^SIG/i, "")}`;
+  } else if (systemd?.execMainCode === "exited" && systemd.execMainStatus && /^\d+$/.test(systemd.execMainStatus)) {
+    out.exitCode = Number(systemd.execMainStatus);
+  }
+  // Best-effort kernel OOM-killer probe by pid — wrapped so a missing/permission-denied journal
+  // never breaks diagnosis. Only consulted when systemd didn't already flag the OOM.
+  if (!out.oom && pid && process.platform === "linux") {
+    try {
+      const result = Bun.spawnSync(["journalctl", "--user", "-k", "--no-pager", "-n", "200", "--grep", `Killed process ${pid}|Out of memory`], {
+        stdin: "ignore",
+        stdout: "pipe",
+        stderr: "ignore",
+      });
+      const text = result.exitCode === 0 ? result.stdout.toString() : "";
+      if (new RegExp(`Killed process ${pid}\\b|Out of memory: Killed process ${pid}\\b`).test(text)) {
+        out.oom = { source: "journal", detail: `OOM-killer hit pid ${pid}` };
+      }
+    } catch {
+      // journal unavailable — leave OOM unset.
+    }
+  }
+  return out;
+}
+
 function describeSessionExit(record: SessionRecord, diagnostics: Omit<ManagedSessionExitDiagnostics, "lastError">): string {
   if (record.provider === "claude") {
     const modelUnavailable = extractClaudeModelUnavailableMessage((diagnostics.logTail ?? []).join("\n"));
@@ -211,6 +242,8 @@ export function diagnoseSessionExit(input: { agentId?: string; policyName?: stri
   const terminalAvailable = tmuxHasSession(record.name, readRunnerInfo(record)?.tmuxSocket);
   const log = logFileDiagnostics(record.logFile);
   const runnerInfoPresent = record.runnerInfoFile ? existsSync(record.runnerInfoFile) : false;
+  const systemd = supervisor.type === "systemd" && supervisor.unit ? systemdUnitDiagnostics(supervisor.unit) : undefined;
+  const termination = terminationDiagnostics(systemd, record.pid ?? currentPid);
   const unavailable = [
     ...(log.logUnavailable ? [`stdout/stderr log unavailable: ${log.logUnavailable}`] : []),
     ...(log.logEmpty ? ["stdout/stderr log empty"] : []),
@@ -243,7 +276,10 @@ export function diagnoseSessionExit(input: { agentId?: string; policyName?: stri
     logTail: log.logTail,
     runnerInfoFile: record.runnerInfoFile,
     runnerInfoPresent,
-    ...(supervisor.type === "systemd" && supervisor.unit ? { systemd: systemdUnitDiagnostics(supervisor.unit) } : {}),
+    ...(systemd ? { systemd } : {}),
+    ...(termination.signal ? { signal: termination.signal } : {}),
+    ...(termination.exitCode !== undefined ? { exitCode: termination.exitCode } : {}),
+    ...(termination.oom ? { oom: termination.oom } : {}),
     ...(unavailable.length ? { unavailable } : {}),
   };
   return {

package/src/workspace-probe/merge.ts

@@ -421,6 +421,39 @@ function recordNoFfMerge(
   return { ok: true, mergeSha };
 }
 
+/**
+ * Fast-forward the local `base` ref to its fetched `upstream` tip (#638 concurrent-lane
+ * recovery). The caller has verified base is a strict ancestor of upstream — a clean ff,
+ * no divergence, nothing to lose. When base is checked out in a worktree, ff it there so
+ * that working tree stays consistent (refuse if it's dirty — can't ff cleanly); otherwise
+ * advance the ref directly. Returns an error string only when the sync genuinely can't be
+ * performed, so the merge path can surface it as a no-progress failure.
+ */
+function syncLocalBaseToUpstream(
+  repoRoot: string,
+  worktreePath: string,
+  base: string,
+  upstream: string,
+): { ok: true } | { ok: false; error: string } {
+  const upstreamSha = git(["rev-parse", "--verify", upstream], worktreePath).stdout;
+  if (!upstreamSha) return { ok: false, error: `cannot resolve ${upstream} to sync ${base}` };
+  const baseWorktree = worktreeForBranch(repoRoot, base);
+  // Sync IN the base worktree only when it's clean — that keeps its working tree consistent
+  // with the advanced ref (the pristine home-repo checkout). When the base worktree is dirty
+  // (#644: a human's WIP in the shared checkout) we must NOT refuse and stall the whole repo's
+  // lands: advance the ref directly with update-ref instead. This is immune to the checkout's
+  // state and never reads or writes the human's working tree — their uncommitted work is left
+  // exactly as-is (the ref moves underneath; their files on disk are untouched).
+  if (baseWorktree && !baseWorktree.dirty) {
+    const ff = git(["merge", "--ff-only", upstream], baseWorktree.path);
+    if (!ff.ok) return { ok: false, error: ff.stderr || `failed to fast-forward ${base} to ${upstream}` };
+    return { ok: true };
+  }
+  const update = git(["update-ref", `refs/heads/${base}`, upstreamSha], repoRoot);
+  if (!update.ok) return { ok: false, error: update.stderr || `failed to advance ${base} to ${upstream}` };
+  return { ok: true };
+}
+
 function mergeRebaseFf(
   input: WorkspaceMergeInput,
   worktreePath: string,
@@ -433,11 +466,20 @@ function mergeRebaseFf(
   if (!base) return head({ status: "review_requested", error: "no base branch to merge into" });
   if (!branch) return head({ status: "review_requested", error: "cannot determine agent branch" });
 
-  // Reconcile with origin before landing (#190/#203). When base tracks an
-  // upstream (e.g. main -> origin/main) and we'll push, fetch it and refuse if
-  // origin has moved ahead of local base: pushing would then be a non-fast-forward,
-  // and we won't rewrite published history or strand a local-only land. The
-  // refusal happens BEFORE we mutate anything, so a diverged base is a clean no-op.
+  // Reconcile with origin before landing (#190/#203/#638). When base tracks an
+  // upstream (e.g. main -> origin/main) and we'll push, fetch it and check whether
+  // origin has moved ahead of local base.
+  //
+  // Origin-ahead is the COMMON case under concurrency (#638): a sibling lane lands
+  // and advances origin/<base> while this lane still sits on a stale local base.
+  // Refusing here floods the steward on every normal multi-lane batch. Instead:
+  //   - If local base carries nothing origin lacks (it's a strict ancestor of the
+  //     fetched upstream), fast-forward local base to the upstream and land onto the
+  //     FRESH origin (the no-ff merge below ties the branch onto the advanced base).
+  //     This is a clean recovery — no steward escalation.
+  //   - Refuse ONLY on genuine divergence: local base has commits not on origin, so
+  //     a sync would rewrite/discard published-or-local history. A real failure must
+  //     still set `error` so the relay's no-progress backoff (#638) can engage.
   const upstream = upstreamRef(worktreePath, base);
   const slash = upstream ? upstream.indexOf("/") : -1;
   const remote = slash > 0 ? upstream!.slice(0, slash) : undefined; // remote of a `remote/branch` upstream
@@ -445,7 +487,13 @@ function mergeRebaseFf(
   if (upstream && remote && pushEnabled) {
     git(["fetch", remote, base], worktreePath); // best-effort freshness; a stale ref can only under-detect divergence
     if (!git(["merge-base", "--is-ancestor", upstream, base], worktreePath).ok) {
-      return head({ status: "review_requested", error: `origin moved ahead of local ${base}; sync ${base} with ${upstream} before landing` });
+      // Origin moved ahead. Sync-then-land iff local base is cleanly behind (ancestor
+      // of upstream); otherwise it's genuine divergence — refuse without mutating.
+      if (!git(["merge-base", "--is-ancestor", base, upstream], worktreePath).ok) {
+        return head({ status: "review_requested", error: `local ${base} has diverged from ${upstream} (commits not on origin); sync before landing` });
+      }
+      const synced = syncLocalBaseToUpstream(repoRoot, worktreePath, base, upstream);
+      if (!synced.ok) return head({ status: "review_requested", error: synced.error });
     }
   }
 
@@ -460,13 +508,16 @@ function mergeRebaseFf(
   const behind = countBehind(worktreePath, base);
 
   // Advance base. `baseTip` is base's new tip after the land: it equals headSha on a
-  // clean fast-forward, or the merge commit on a no-ff merge. If base is checked out
-  // somewhere, operate in that worktree so its working tree stays consistent; otherwise
-  // move/synthesize the ref directly. Refuse if the base worktree has uncommitted changes.
+  // clean fast-forward, or the merge commit on a no-ff merge. Operate IN the base worktree
+  // only when it exists AND is clean — that keeps its working tree consistent with the
+  // advanced ref (the pristine home-repo checkout). When the base worktree is dirty
+  // (#644: a human's WIP in the shared checkout) we must NOT refuse and stall every lane's
+  // land: fall through to ref-plumbing (update-ref / synthesized no-ff merge) below, which
+  // advances refs/heads/<base> without reading or touching that working tree. The human's
+  // uncommitted work is left exactly as-is — the ref moves underneath, their files untouched.
   let baseTip = headSha;
   const baseWorktree = worktreeForBranch(repoRoot, base);
-  if (baseWorktree) {
-    if (baseWorktree.dirty) return head({ status: "review_requested", error: `base branch '${base}' has uncommitted changes in ${baseWorktree.path}` });
+  if (baseWorktree && !baseWorktree.dirty) {
     if (behind === 0) {
       const ff = git(["merge", "--ff-only", branch], baseWorktree.path);
       if (!ff.ok) return head({ status: "review_requested", error: ff.stderr || "fast-forward into base failed" });