agent-relay-orchestrator 0.129.16 → 0.129.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.129.16",
+  "version": "0.129.18",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -17,7 +17,7 @@
   },
   "dependencies": {
     "agent-relay-providers": "0.104.6",
-    "agent-relay-sdk": "0.2.131",
+    "agent-relay-sdk": "0.2.132",
     "callmux": "0.24.2"
   },
   "devDependencies": {

package/src/api.ts

@@ -5,6 +5,7 @@ import type { ServerWebSocket } from "bun";
 import { proxyArtifactRequest } from "./artifact-proxy";
 import { fireAndForget } from "./async-guard";
 import type { OrchestratorConfig } from "./config";
+import type { CommandLoopHealth } from "./command-poller";
 import type { ProviderProbeCache } from "./provider-probe";
 import type { RegistrationDriver } from "./registration";
 import type { RelayClient } from "./relay";
@@ -392,7 +393,7 @@ export interface CodexResetConsumer {
   }>;
 }
 
-export function startApiServer(config: OrchestratorConfig, probeCache: ProviderProbeCache, relay?: RelayClient, codexResetConsumer?: CodexResetConsumer, registration?: Pick<RegistrationDriver, "getHealth">): { stop(): void; url: string } {
+export function startApiServer(config: OrchestratorConfig, probeCache: ProviderProbeCache, relay?: RelayClient, codexResetConsumer?: CodexResetConsumer, registration?: Pick<RegistrationDriver, "getHealth">, commandLoop?: { getHealth(): CommandLoopHealth }): { stop(): void; url: string } {
   const server = Bun.serve<TerminalSocketData>({
     port: config.apiPort,
     hostname: "0.0.0.0",
@@ -679,10 +680,16 @@ export function startApiServer(config: OrchestratorConfig, probeCache: ProviderP
         // degraded peer must not make this process look un-upgradable. Read `degraded`.
         const peer = relay?.getHealth();
         const reg = registration?.getHealth();
+        // #1760 — surface the command loop's own health so a CRAWLING loop (a stuck inline handler
+        // head-of-line-blocking the queue) is distinguishable from a healthy one. Kept OUT of the
+        // top-level `degraded` flag on purpose: `degraded`/`ok` gate the self-upgrade guard's
+        // Relay-independent readiness probe, and a transiently-stalled command loop must not make
+        // this process look un-upgradable. Consumers read `commandLoop.stalled` directly.
         return json({
           status: "ok", ok: true, id: config.id, hostname: config.hostname,
           degraded: Boolean(peer?.degraded || reg?.degraded),
           ...(peer ? { relay: peer } : {}), ...(reg ? { registration: reg } : {}),
+          ...(commandLoop ? { commandLoop: commandLoop.getHealth() } : {}),
         });
       }
 

package/src/command-poller.ts

@@ -5,6 +5,23 @@ interface CommandPollerControl {
   handleCommand(command: RelayCommand): Promise<boolean>;
 }
 
+// #1760 — a health snapshot of the command loop, surfaced on /api/health so a CRAWLING loop (a
+// healthy heartbeat while commands sit unhandled for minutes) is detectable without log archaeology.
+export interface CommandLoopHealth {
+  /** True while a poll+handle cycle is running. */
+  inFlight: boolean;
+  /** Where the in-flight cycle is: waiting on the network poll, or running a command handler. */
+  phase: "idle" | "poll" | "handler";
+  /** How long the in-flight cycle has been running (0 when idle). */
+  activeForMs: number;
+  /** The command currently being handled inline, when phase === "handler". */
+  handlingType?: string;
+  handlingId?: string;
+  /** True once the in-flight cycle has exceeded the watchdog threshold — the crawl signal. */
+  stalled: boolean;
+  stalledForMs?: number;
+}
+
 interface CommandPollerOptions {
   relay: Pick<RelayClient, "connected" | "pollCommands">;
   control: CommandPollerControl;
@@ -35,10 +52,21 @@ export function createCommandPoller({
   let activePollController: AbortController | undefined;
   let activePollStartedAt = 0;
   let activePhase: "idle" | "poll" | "handler" = "idle";
+  // #1760 — the command currently being handled inline, and the timestamp at which the loop was first
+  // observed stalled in the current cycle (0 when not stalled). Only used for detection/surfacing.
+  let activeCommandType: string | undefined;
+  let activeCommandId: string | undefined;
+  let stalledSince = 0;
   let lastTickErrored = false;
   let lastCycleCompletedAt = now();
   let generation = 0;
 
+  function clearActiveCommand(): void {
+    activeCommandType = undefined;
+    activeCommandId = undefined;
+    stalledSince = 0;
+  }
+
   async function tick(): Promise<boolean> {
     if (!relay.connected || inFlight) return false;
     const tickGeneration = generation;
@@ -59,7 +87,14 @@ export function createCommandPoller({
       for (const command of commands) {
         if (tickGeneration !== generation) return false;
         log(`[orchestrator] Handling command: ${command.type} ${command.id}`);
-        await control.handleCommand(command);
+        activeCommandType = command.type;
+        activeCommandId = command.id;
+        stalledSince = 0;
+        try {
+          await control.handleCommand(command);
+        } finally {
+          if (tickGeneration === generation) clearActiveCommand();
+        }
       }
       return true;
     } catch (err) {
@@ -72,6 +107,7 @@ export function createCommandPoller({
         inFlight = false;
         activePhase = "idle";
         activePollStartedAt = 0;
+        clearActiveCommand();
       }
     }
   }
@@ -103,18 +139,43 @@ export function createCommandPoller({
     if (watchdogTimer || watchdogMs <= 0 || watchdogCheckMs <= 0) return;
     watchdogTimer = setInterval(() => {
       if (stopped) return;
-      if (activePhase !== "poll" || !activePollController) return;
+      if (!inFlight || activePhase === "idle" || activePollStartedAt === 0) return;
       const staleForMs = now() - activePollStartedAt;
       if (staleForMs < watchdogMs) return;
-      generation += 1;
-      activePollController.abort();
-      activePollController = undefined;
-      activePollStartedAt = 0;
-      activePhase = "idle";
-      inFlight = false;
-      log(`[orchestrator] Poll loop watchdog: command poll stuck for ${staleForMs}ms; aborting stuck poll and restarting`);
-      if (!timer) schedule(0);
-      lastCycleCompletedAt = now();
+      if (activePhase === "poll" && activePollController) {
+        // A stuck network poll is safely abortable (its AbortController cancels the in-flight
+        // request); bump the generation, abort, and restart the cycle. Unchanged from pre-#1760.
+        generation += 1;
+        activePollController.abort();
+        activePollController = undefined;
+        activePollStartedAt = 0;
+        activePhase = "idle";
+        inFlight = false;
+        clearActiveCommand();
+        log(`[orchestrator] Poll loop watchdog: command poll stuck for ${staleForMs}ms; aborting stuck poll and restarting`);
+        if (!timer) schedule(0);
+        lastCycleCompletedAt = now();
+        return;
+      }
+      if (activePhase === "handler") {
+        // #1760 — a command handler (agent.restart / agent.shutdown / workspace.merge / …) is running
+        // INLINE in the poll tick and has exceeded the threshold, head-of-line-blocking every command
+        // queued behind it. We DETECT and SURFACE it (log once + a health flag) but deliberately do
+        // NOT abort: unlike a network poll, a handler is mid-mutation (a git land, a session teardown)
+        // and cancelling it by bumping the generation would free the single-flight guard while the
+        // work runs on, risking a concurrent re-dispatch of the same command. Spawn handling is
+        // already backgrounded off the tick (dispatchSpawnCommand), so it cannot land here; the
+        // inline handlers that can are bounded by their own timeouts. Surfacing turns a silent crawl
+        // into an observable one.
+        if (stalledSince === 0) {
+          stalledSince = activePollStartedAt;
+          log(
+            `[orchestrator] Command loop STALLED: handling ${activeCommandType ?? "command"} ${activeCommandId ?? ""} for ${staleForMs}ms ` +
+            `(heartbeat stays healthy; commands queued behind it are blocked). Not aborting an in-flight mutating handler.`,
+          );
+        }
+        return;
+      }
     }, watchdogCheckMs);
     watchdogTimer.unref?.();
   }
@@ -134,16 +195,33 @@ export function createCommandPoller({
     activePollController = undefined;
     activePollStartedAt = 0;
     activePhase = "idle";
+    clearActiveCommand();
     if (timer) clearTimeout(timer);
     timer = undefined;
     if (watchdogTimer) clearInterval(watchdogTimer);
     watchdogTimer = undefined;
   }
 
+  function getHealth(): CommandLoopHealth {
+    const active = inFlight && activePollStartedAt > 0;
+    const activeForMs = active ? Math.max(0, now() - activePollStartedAt) : 0;
+    const stalled = active && stalledSince > 0;
+    return {
+      inFlight,
+      phase: activePhase,
+      activeForMs,
+      stalled,
+      ...(stalled ? { stalledForMs: Math.max(0, now() - stalledSince) } : {}),
+      ...(activePhase === "handler" && activeCommandType ? { handlingType: activeCommandType } : {}),
+      ...(activePhase === "handler" && activeCommandId ? { handlingId: activeCommandId } : {}),
+    };
+  }
+
   return {
     tick,
     start,
     stop,
+    getHealth,
     get inFlight() {
       return inFlight;
     },

package/src/config.ts

@@ -118,6 +118,18 @@ export function tmuxSocketSweepIntervalMs(): number {
   return envNonNegativeMax("AGENT_RELAY_TMUX_SOCKET_SWEEP_INTERVAL_MS", 10 * 60 * 1000, 60_000);
 }
 
+// #1760 — a HARD per-socket bound on the sweep's `tmux list-sessions` liveness probe. The probe
+// runs via Bun.spawnSync ON the event loop; a WEDGED tmux server (process alive, socket
+// unresponsive) makes `list-sessions` block until the server answers — which, unbounded, freezes
+// the whole orchestrator, the command poll loop included (the wbox stall in #1760: healthy
+// heartbeat, commands crawling ~11–15 min behind). With the bound a single wedged socket costs at
+// most this long and its socket is KEPT (a wedged-but-present server must never have its socket
+// yanked). 2s is generous for a healthy `list-sessions` even under load; clamped so it can never be
+// set so low that a slow-but-live probe is misread.
+export function tmuxSocketSweepProbeTimeoutMs(): number {
+  return envNonNegativeMax("AGENT_RELAY_TMUX_SOCKET_SWEEP_PROBE_TIMEOUT_MS", 2_000, 250);
+}
+
 // #1514 — reaper for orphaned, idle managed tmux sessions (e.g. a headless Claude
 // session wedged at the "Exit anyway / Move to background / Stay" dialog). On by
 // default; set AGENT_RELAY_WEDGED_SESSION_REAP=0 to disable.

package/src/index.ts

@@ -99,6 +99,8 @@ const registration = createRegistrationDriver({ register: () => relay.register()
 const POLL_INTERVAL_MS = 3_000;
 const GUEST_REAP_INTERVAL_MS = 60_000;
 let commandPoller: ReturnType<typeof createCommandPoller> | null = null;
+// #1760 — health reported before the poll loop is up (or after shutdown): an idle, non-stalled loop.
+const nullCommandLoopHealth = { getHealth: () => ({ inFlight: false, phase: "idle" as const, activeForMs: 0, stalled: false }) };
 let healthCheckTimer: Timer | null = null;
 let guestReaperTimer: Timer | null = null;
 let apiServer: { stop(): void; url: string } | null = null;
@@ -111,7 +113,11 @@ async function startup(): Promise<void> {
   console.error(`[orchestrator]   env keys: ${Object.keys(config.env).length}`);
 
   // Start API server before registration so we can advertise the URL
-  apiServer = startApiServer(config, probeCache, relay, quotaPoller, registration);
+  // #1760 — the poller is created later in startPolling(); pass a lazy accessor so /api/health reads
+  // whichever poller is live at request time (null before startPolling, giving no commandLoop field).
+  apiServer = startApiServer(config, probeCache, relay, quotaPoller, registration, {
+    getHealth: () => (commandPoller ?? nullCommandLoopHealth).getHealth(),
+  });
   console.error(`[orchestrator]   apiUrl: ${apiServer.url}`);
 
   relay.setApiUrl(apiServer.url);

package/src/maintenance.ts

@@ -3,6 +3,7 @@ import type { OrchestratorConfig } from "./config";
 import {
   tmuxSocketSweepEnabled,
   tmuxSocketSweepIntervalMs,
+  tmuxSocketSweepProbeTimeoutMs,
   wedgedSessionIdleThresholdMs,
   wedgedSessionReapEnabled,
   wedgedSessionReapIntervalMs,
@@ -31,10 +32,18 @@ export function maintenanceJobDefinitions(config: OrchestratorConfig): Orchestra
       intervalMs: tmuxSocketSweepIntervalMs(),
       runOnStart: true,
       handler() {
-        const result = sweepStaleTmuxSockets();
+        const result = sweepStaleTmuxSockets({ probeTimeoutMs: tmuxSocketSweepProbeTimeoutMs() });
         if (result.scanned > 0 || result.failed.length > 0) {
           console.error(
-            `[orchestrator] Tmux socket sweep: removed=${result.removed} kept=${result.kept} scanned=${result.scanned} dir=${result.dir}`,
+            `[orchestrator] Tmux socket sweep: removed=${result.removed} kept=${result.kept} timedOut=${result.timedOut} scanned=${result.scanned} dir=${result.dir}`,
+          );
+        }
+        // #1760 — a socket whose liveness probe hit the timeout is a wedged/unresponsive tmux server.
+        // Surface it distinctly: this is the host-level state that stalls the command loop, and a
+        // plain restart won't clear it (the tmux server outlives the orchestrator).
+        if (result.timedOut > 0) {
+          console.error(
+            `[orchestrator] Tmux socket sweep: ${result.timedOut} socket(s) unresponsive (list-sessions timed out at ${tmuxSocketSweepProbeTimeoutMs()}ms) — kept, not removed; a wedged tmux server may need manual intervention (#1760)`,
           );
         }
         for (const failure of result.failed) {

package/src/tmux-socket-sweeper.ts

@@ -4,11 +4,29 @@ import { tmuxCommand } from "agent-relay-sdk/tmux-utils";
 
 const SOCKET_PREFIX = "agent-relay-";
 
+// #1760 — default hard bound for the per-socket liveness probe when a caller passes none. Mirrors
+// tmuxSocketSweepProbeTimeoutMs()'s default; kept as a module constant so the sweeper stays
+// dependency-free of config.ts (it is imported directly by tests and by the SDK-adjacent code).
+export const DEFAULT_TMUX_SOCKET_PROBE_TIMEOUT_MS = 2_000;
+
+// "live"    — a tmux server answered `list-sessions` (exit 0). Keep the socket.
+// "dead"    — tmux exited non-zero ("no server running …"): a stale socket. Remove it.
+// "unknown" — the probe did NOT exit on its own (killed at the timeout / by a signal): the socket is
+//             present but unresponsive — a WEDGED server. Keep it (never yank a possibly-live
+//             server's socket) and, crucially, never block the event loop past the bound. (#1760)
+export type TmuxSocketLiveness = "live" | "dead" | "unknown";
+
+export type TmuxSocketProbe = (socketName: string, timeoutMs: number, env?: NodeJS.ProcessEnv) => TmuxSocketLiveness;
+
 interface TmuxSocketSweepResult {
   dir: string;
   scanned: number;
   removed: number;
   kept: number;
+  // #1760 — sockets whose liveness probe hit the timeout (a wedged/unresponsive tmux server). These
+  // are counted within `kept` and surfaced separately so a hung socket is visible in the sweep log
+  // instead of silently inflating `kept`.
+  timedOut: number;
   failed: Array<{ socket: string; error: string }>;
 }
 
@@ -19,9 +37,13 @@ export function tmuxSocketDir(env: NodeJS.ProcessEnv = process.env): string {
   return basename(tmuxTmp) === `tmux-${uid}` ? tmuxTmp : join(tmuxTmp, `tmux-${uid}`);
 }
 
-export function sweepStaleTmuxSockets(input: { dir?: string; env?: NodeJS.ProcessEnv } = {}): TmuxSocketSweepResult {
+export function sweepStaleTmuxSockets(
+  input: { dir?: string; env?: NodeJS.ProcessEnv; probeTimeoutMs?: number; probe?: TmuxSocketProbe } = {},
+): TmuxSocketSweepResult {
   const dir = input.dir ?? tmuxSocketDir(input.env);
-  const result: TmuxSocketSweepResult = { dir, scanned: 0, removed: 0, kept: 0, failed: [] };
+  const probeTimeoutMs = input.probeTimeoutMs ?? DEFAULT_TMUX_SOCKET_PROBE_TIMEOUT_MS;
+  const probe = input.probe ?? tmuxSocketServerState;
+  const result: TmuxSocketSweepResult = { dir, scanned: 0, removed: 0, kept: 0, timedOut: 0, failed: [] };
   if (!existsSync(dir)) return result;
 
   for (const entry of readdirSync(dir)) {
@@ -38,8 +60,13 @@ export function sweepStaleTmuxSockets(input: { dir?: string; env?: NodeJS.Proces
     if (!removable) continue;
 
     result.scanned += 1;
-    if (tmuxSocketHasLiveServer(entry, input.env)) {
+    const liveness = probe(entry, probeTimeoutMs, input.env);
+    if (liveness !== "dead") {
+      // "live" (a server answered) OR "unknown" (the probe hit the timeout against a wedged socket) —
+      // either way KEEP the socket. Removing a wedged-but-present server's socket is destructive, and
+      // we must never block the event loop waiting to be certain it is dead. (#1760)
       result.kept += 1;
+      if (liveness === "unknown") result.timedOut += 1;
       continue;
     }
 
@@ -54,12 +81,25 @@ export function sweepStaleTmuxSockets(input: { dir?: string; env?: NodeJS.Proces
   return result;
 }
 
-function tmuxSocketHasLiveServer(socketName: string, env?: NodeJS.ProcessEnv): boolean {
+export function tmuxSocketServerState(socketName: string, timeoutMs: number, env?: NodeJS.ProcessEnv): TmuxSocketLiveness {
   const result = Bun.spawnSync(tmuxCommand(socketName, "list-sessions"), {
     stdin: "ignore",
     stdout: "ignore",
     stderr: "ignore",
+    // #1760 — the load-bearing bound: without it a wedged server's socket blocks this synchronous
+    // call (and thus the whole event loop) indefinitely.
+    ...(timeoutMs > 0 ? { timeout: timeoutMs } : {}),
+    // Force-kill on timeout with SIGKILL, NOT the default SIGTERM: the `tmux list-sessions` CLIENT
+    // traps SIGTERM and exits cleanly (observed exitCode 0 — or, on some builds, 1), which would
+    // misread a wedged server as "live" or, worse, "dead" (→ its socket removed). SIGKILL is
+    // uncatchable, so a timed-out probe reliably reports exitCode === null → "unknown". This only
+    // kills the short-lived probe client; the tmux SERVER is never signalled.
+    killSignal: "SIGKILL",
     ...(env ? { env } : {}),
   });
-  return result.exitCode === 0;
+  if (result.exitCode === 0) return "live";
+  // Killed at the timeout (SIGKILL) → null exit code: the socket is present but unresponsive. Treat
+  // as unknown (keep it), never as dead — a wedged-but-live server must not lose its socket.
+  if (result.exitCode === null) return "unknown";
+  return "dead";
 }