agent-relay-orchestrator 0.119.7 → 0.119.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.119.7",
+  "version": "0.119.9",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -17,7 +17,7 @@
   },
   "dependencies": {
     "agent-relay-providers": "0.104.3",
-    "agent-relay-sdk": "0.2.105",
+    "agent-relay-sdk": "0.2.106",
     "callmux": "0.23.0"
   },
   "devDependencies": {

package/src/command-poller.ts

@@ -10,24 +10,53 @@ interface CommandPollerOptions {
   log?: (message: string) => void;
   intervalMs?: number;
   errorBackoffMs?: number;
+  watchdogMs?: number;
+  watchdogCheckMs?: number;
+  now?: () => number;
 }
 
-export function createCommandPoller({ relay, control, log = console.error, intervalMs = 3_000, errorBackoffMs = 3_000 }: CommandPollerOptions) {
+const DEFAULT_WATCHDOG_MS = 5 * 60 * 1000;
+
+export function createCommandPoller({
+  relay,
+  control,
+  log = console.error,
+  intervalMs = 3_000,
+  errorBackoffMs = 3_000,
+  watchdogMs = DEFAULT_WATCHDOG_MS,
+  watchdogCheckMs = Math.min(60_000, watchdogMs),
+  now = Date.now,
+}: CommandPollerOptions) {
   let inFlight = false;
   let stopped = true;
   let timer: ReturnType<typeof setTimeout> | undefined;
+  let watchdogTimer: ReturnType<typeof setInterval> | undefined;
+  let activePollController: AbortController | undefined;
+  let activePollStartedAt = 0;
+  let activePhase: "idle" | "poll" | "handler" = "idle";
   let lastTickErrored = false;
+  let lastCycleCompletedAt = now();
+  let generation = 0;
 
   async function tick(): Promise<boolean> {
     if (!relay.connected || inFlight) return false;
+    const tickGeneration = generation;
+    const controller = new AbortController();
     inFlight = true;
+    activePollController = controller;
+    activePollStartedAt = now();
+    activePhase = "poll";
     lastTickErrored = false;
     try {
-      const commands = await relay.pollCommands();
+      const commands = await relay.pollCommands(controller.signal);
+      if (tickGeneration !== generation) return false;
+      if (activePollController === controller) activePollController = undefined;
+      activePhase = "handler";
       if (commands.length > 0) {
         log(`[orchestrator] Received ${commands.length} command(s)`);
       }
       for (const command of commands) {
+        if (tickGeneration !== generation) return false;
         log(`[orchestrator] Handling command: ${command.type} ${command.id}`);
         await control.handleCommand(command);
       }
@@ -37,7 +66,12 @@ export function createCommandPoller({ relay, control, log = console.error, inter
       log(`[orchestrator] Poll error: ${err}`);
       return false;
     } finally {
-      inFlight = false;
+      if (activePollController === controller) activePollController = undefined;
+      if (tickGeneration === generation) {
+        inFlight = false;
+        activePhase = "idle";
+        activePollStartedAt = 0;
+      }
     }
   }
 
@@ -45,12 +79,12 @@ export function createCommandPoller({ relay, control, log = console.error, inter
     if (stopped) return;
     timer = setTimeout(() => {
       timer = undefined;
-      void runCycle();
+      void runCycle(generation);
     }, delayMs);
     timer.unref?.();
   }
 
-  async function runCycle(): Promise<void> {
+  async function runCycle(runGeneration: number): Promise<void> {
     let errored = false;
     try {
       await tick();
@@ -59,20 +93,50 @@ export function createCommandPoller({ relay, control, log = console.error, inter
       errored = true;
       log(`[orchestrator] Poll loop error: ${err}`);
     } finally {
-      if (!stopped) schedule(errored ? errorBackoffMs : intervalMs);
+      if (runGeneration === generation) lastCycleCompletedAt = now();
+      if (!stopped && runGeneration === generation) schedule(errored ? errorBackoffMs : intervalMs);
     }
   }
 
+  function startWatchdog(): void {
+    if (watchdogTimer || watchdogMs <= 0 || watchdogCheckMs <= 0) return;
+    watchdogTimer = setInterval(() => {
+      if (stopped) return;
+      if (activePhase !== "poll" || !activePollController) return;
+      const staleForMs = now() - activePollStartedAt;
+      if (staleForMs < watchdogMs) return;
+      generation += 1;
+      activePollController.abort();
+      activePollController = undefined;
+      activePollStartedAt = 0;
+      activePhase = "idle";
+      inFlight = false;
+      log(`[orchestrator] Poll loop watchdog: command poll stuck for ${staleForMs}ms; aborting stuck poll and restarting`);
+      if (!timer) schedule(0);
+      lastCycleCompletedAt = now();
+    }, watchdogCheckMs);
+    watchdogTimer.unref?.();
+  }
+
   function start(): void {
     if (!stopped) return;
     stopped = false;
+    lastCycleCompletedAt = now();
+    startWatchdog();
     schedule(0);
   }
 
   function stop(): void {
     stopped = true;
+    generation += 1;
+    activePollController?.abort();
+    activePollController = undefined;
+    activePollStartedAt = 0;
+    activePhase = "idle";
     if (timer) clearTimeout(timer);
     timer = undefined;
+    if (watchdogTimer) clearInterval(watchdogTimer);
+    watchdogTimer = undefined;
   }
 
   return {

package/src/relay.ts

@@ -9,7 +9,7 @@ export interface RelayClient {
   register(): Promise<void>;
   heartbeat(): Promise<void>;
   updateManagedAgents(agents: ManagedAgentReport[], exitedAgents?: ManagedSessionExitDiagnostics[]): Promise<void>;
-  pollCommands(): Promise<RelayCommand[]>;
+  pollCommands(signal?: AbortSignal): Promise<RelayCommand[]>;
   getCommand(commandId: string): Promise<RelayCommand | null>;
   updateCommand(commandId: string, status: string, result?: Record<string, unknown>, error?: string): Promise<boolean>;
   acquireProviderQuotaLease(orchestratorId: string, input: ProviderQuotaLeaseAcquireInput): Promise<ProviderQuotaLeaseAcquireResult>;
@@ -213,9 +213,9 @@ export function createRelayClient(config: OrchestratorConfig, probeCache: Provid
     });
   }
 
-  async function pollCommands(): Promise<RelayCommand[]> {
+  async function pollCommands(signal?: AbortSignal): Promise<RelayCommand[]> {
     const url = `/commands?target=${encodeURIComponent(agentId)}&status=pending&limit=50`;
-    const res = await apiCall("GET", url);
+    const res = await http.request("GET", `/api${url}`, undefined, signal ? { signal } : undefined);
     if (!res.ok) return [];
     return await res.json() as RelayCommand[];
   }

package/src/spawn/terminal.ts

@@ -274,8 +274,7 @@ export { tmuxCommand };
 
 // Lightweight liveness for the live terminal stream's backfill metadata — avoids a full
 // capture-pane just to learn whether the pane/agent are still up.
-export function sessionLiveness(name: string): { running: boolean; agentAlive: boolean } {
-  const socketName = tmuxSocketForSession(name);
+export function sessionLiveness(name: string, socketName = tmuxSocketForSession(name)): { running: boolean; agentAlive: boolean } {
   return { running: tmuxHasSession(name, socketName), agentAlive: isSessionAlive(name) };
 }
 

package/src/terminal-stream.ts

@@ -368,7 +368,7 @@ class SessionStream {
   private socket: string | undefined;
 
   start(): void {
-    const socket = tmuxSocketForSession(this.session);
+    const socket = this.config.env.AGENT_RELAY_TMUX_SOCKET || tmuxSocketForSession(this.session);
     this.socket = socket;
     // Pin the window size before attaching so the control client can't reflow the
     // pane (default window-size would shrink it to the new client's 80x24). These are
@@ -786,7 +786,7 @@ class SessionStream {
     const dims = await this.paneDims();
     if (dims.cols) this.termCols = dims.cols;
     if (dims.rows) this.termRows = dims.rows;
-    const live = sessionLiveness(this.session);
+    const live = sessionLiveness(this.session, this.config.env.AGENT_RELAY_TMUX_SOCKET || tmuxSocketForSession(this.session));
     tdbg(`backfill ${this.session} req=${cols}x${rows} term=${this.termCols}x${this.termRows} contentLen=${content.length} viewers=${this.subscribers.size}`);
     return {
       session: this.session,

package/src/workspace-probe/git-state.ts

@@ -55,9 +55,10 @@ export async function populateMergeState(cwd: string, targetRef: string, state:
     // compare against the base branch's upstream when it has one (squash PRs
     // land on the remote), treat an identical tree as fully landed (covers a
     // multi-commit squash), else count only commits whose patch isn't already
-    // present in base (git cherry '+'). Staleness of the compare ref can only
-    // under-count landings, never invent one — so `landed` is safe to act on.
-    const cherryBase = await upstreamRef(cwd, base) ?? base;
+    // present in base (git cherry '+'). When the base tracks an upstream, fetch
+    // before comparing: a stale origin/main can both hide a real squash landing
+    // and prove a false tree match against old remote state.
+    const cherryBase = await syncBaseFromOrigin(cwd, base) ?? base;
     if ((await git(["diff", "--quiet", cherryBase, targetRef], cwd)).ok) {
       state.unmergedAhead = 0;
     } else {
@@ -86,7 +87,7 @@ export async function upstreamRef(worktreePath: string, base: string, signal?: A
  * base is never advanced, so cutting from local base would recycle onto stale main. Fetching
  * the upstream and returning `origin/<base>` (when it resolves) yields a branch off CURRENT
  * base. Falls back to the local base ref when there is no upstream (no remote) or the
- * upstream ref can't be resolved — the caller still verifies it contains the merge SHA.
+ * upstream ref can't be fetched/resolved — callers must not act on a stale remote-tracking ref.
  */
 export async function syncBaseFromOrigin(worktreePath: string, base: string | undefined, signal?: AbortSignal): Promise<string | undefined> {
   if (!base) return undefined;
@@ -95,7 +96,10 @@ export async function syncBaseFromOrigin(worktreePath: string, base: string | un
   const slash = upstream.indexOf("/");
   const remote = slash > 0 ? upstream.slice(0, slash) : undefined;
   const remoteBranch = slash > 0 ? upstream.slice(slash + 1) : base;
-  if (remote) await git(["fetch", remote, remoteBranch], worktreePath, { signal }); // best-effort freshness
+  if (remote) {
+    const fetch = await git(["fetch", remote, `${remoteBranch}:refs/remotes/${remote}/${remoteBranch}`], worktreePath, { signal });
+    if (!fetch.ok) return base;
+  }
   return (await git(["rev-parse", "--verify", "--quiet", upstream], worktreePath, { signal })).ok ? upstream : base;
 }