agent-relay-orchestrator 0.129.20 → 0.129.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.129.20",
+  "version": "0.129.22",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -17,7 +17,7 @@
   },
   "dependencies": {
     "agent-relay-providers": "0.104.6",
-    "agent-relay-sdk": "0.2.132",
+    "agent-relay-sdk": "0.2.133",
     "callmux": "0.24.2"
   },
   "devDependencies": {

package/src/control.ts

@@ -1,4 +1,5 @@
 import { errMessage, isPathWithinBase, isRecord, normalizeAgentLifecycle, normalizeWorkspaceMode, Semaphore } from "agent-relay-sdk";
+import { renderTmuxSessionTarget } from "agent-relay-sdk/tmux-utils";
 import { fireAndForget } from "./async-guard";
 import { getAllManifests, getManifest } from "agent-relay-providers";
 import { SCHEDULER_COMMAND_MAX_CONCURRENT, SCHEDULER_COMMAND_MAX_OUTPUT_BYTES, type OrchestratorConfig } from "./config";
@@ -327,7 +328,23 @@ export function createControlHandler(
 
   async function handleShutdown(ctrl: Record<string, any>, restart = false): Promise<Record<string, unknown>> {
     const current = managedAgentShutdownTarget(managedAgents, ctrl);
-    const session = current?.sessionName ?? current?.tmuxSession;
+    // #1746 RC-1(c) — when the in-memory managedAgents report has no match (an orchestrator that
+    // restarted and lost its report cache, or a report/record divergence), fall back to the durable
+    // on-disk session state before giving up. findSessionRecord (post-fix) tries every key, so a live
+    // session the report missed can still be resolved and force-killed rather than silently skipped.
+    const fallbackRecord = current
+      ? undefined
+      : findSessionRecord({
+          agentId: typeof ctrl.agentId === "string" && ctrl.agentId ? ctrl.agentId : undefined,
+          spawnRequestId: typeof ctrl.spawnRequestId === "string" && ctrl.spawnRequestId ? ctrl.spawnRequestId : undefined,
+          policyName: typeof ctrl.policyName === "string" && ctrl.policyName ? ctrl.policyName : undefined,
+          tmuxSession: typeof ctrl.sessionName === "string" && ctrl.sessionName
+            ? ctrl.sessionName
+            : typeof ctrl.tmuxSession === "string" && ctrl.tmuxSession
+              ? ctrl.tmuxSession
+              : undefined,
+        });
+    const session = current?.sessionName ?? current?.tmuxSession ?? fallbackRecord?.name;
     const restartSpawn = isRecord(ctrl.restartSpawn) ? ctrl.restartSpawn : undefined;
     if (!session) {
       let restarted: ManagedAgentReport | undefined;
@@ -339,12 +356,17 @@ export function createControlHandler(
           spawnSlots.release();
         }
       }
+      // #1746 RC-1(c) — the target resolved to NOTHING and (for a restart) no successor was spawned:
+      // dispatchStop never ran, so we CANNOT claim the process was stopped. Signal `targetNotFound` so
+      // the caller marks the command FAILED (a red terminal outcome the dashboard/watchdog can act on)
+      // instead of the old inert `succeeded`, which deregistered live agents and wrote a false
+      // `agent.exited`. A restart that DID respawn a successor is a legitimate success.
       return {
         stopped: false,
         wasRunning: false,
         restart,
         restarted: Boolean(restarted),
-        ...(restarted ? { agent: restarted } : {}),
+        ...(restarted ? { agent: restarted } : { targetNotFound: true }),
         policyName: ctrl.policyName,
         spawnRequestId: ctrl.spawnRequestId,
       };
@@ -510,7 +532,25 @@ export function createControlHandler(
     try {
       if (command.type === "agent.shutdown" || command.type === "agent.restart") {
         const result = await handleShutdown(command.params, command.type === "agent.restart");
-        await relay.updateCommand(command.id, "succeeded", result);
+        // #1746 RC-1(c) — a shutdown is a KILL-SWITCH: never report success unless the target was
+        // resolved AND is confirmed gone. Two outcomes fail the command with a red terminal reason so
+        // the dashboard/watchdog can escalate (and no false `agent.exited` is written server-side):
+        //   - target-not-found: nothing matched, dispatchStop never ran (the surviving-agent bug);
+        //   - kill-failed: the session was resolved but its process survived graceful+forceful stop.
+        // Restart keeps its existing settlement — its successor spawn + predecessor teardown are
+        // reconciled by the restart path itself, not by this kill-switch gate.
+        const shutdownFailure = command.type === "agent.shutdown"
+          ? (result.targetNotFound === true
+              ? "target-not-found: no live session, agent, spawn, or policy matched the shutdown request"
+              : result.wasRunning === true && result.stopped !== true
+                ? "kill-failed: target process survived graceful and forceful stop"
+                : undefined)
+          : undefined;
+        if (shutdownFailure) {
+          await relay.updateCommand(command.id, "failed", result, shutdownFailure);
+        } else {
+          await relay.updateCommand(command.id, "succeeded", result);
+        }
       } else if (command.type === "workspace.cleanup") {
         const result = await cleanupWorkspace({
           id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
@@ -700,21 +740,47 @@ export function createControlHandler(
 }
 
 export function managedAgentShutdownTarget(agents: ManagedAgentReport[], ctrl: Record<string, any>): ManagedAgentReport | undefined {
+  // #1746 RC-1 — resolve the shutdown target by trying EVERY key present on the command in priority
+  // order, returning the first that matches an agent. The prior code early-returned on the
+  // first-SPECIFIED key (session) even when it did NOT match, so a session-name divergence stranded
+  // the whole lookup at `undefined` — the force-kill machinery was never reached and the inert
+  // shutdown was reported as `succeeded` while the process stayed alive (the silent zombie). A MISS on
+  // one key must fall through to the next, never short-circuit the resolution.
   const requestedSession = typeof ctrl.sessionName === "string" && ctrl.sessionName
     ? ctrl.sessionName
     : typeof ctrl.tmuxSession === "string" && ctrl.tmuxSession
       ? ctrl.tmuxSession
       : undefined;
-  if (requestedSession) return agents.find((agent) => agent.sessionName === requestedSession || agent.tmuxSession === requestedSession);
+  if (requestedSession) {
+    // #1583/#1746 RC-1(b) — the runner publishes tmux's RENDERED session name (`.`/`:`→`_`) while the
+    // orchestrator record keeps the REQUESTED (dotted) name; normalize BOTH sides so a rendering
+    // divergence can't miss. Raw equality is tried first, then the rendered comparison.
+    const renderedRequest = renderTmuxSessionTarget(requestedSession);
+    const bySession = agents.find((agent) =>
+      agent.sessionName === requestedSession ||
+      agent.tmuxSession === requestedSession ||
+      (agent.sessionName != null && renderTmuxSessionTarget(agent.sessionName) === renderedRequest) ||
+      renderTmuxSessionTarget(agent.tmuxSession) === renderedRequest);
+    if (bySession) return bySession;
+  }
 
   const agentId = typeof ctrl.agentId === "string" && ctrl.agentId ? ctrl.agentId : undefined;
-  if (agentId) return agents.find((agent) => agent.agentId === agentId);
+  if (agentId) {
+    const byAgentId = agents.find((agent) => agent.agentId === agentId);
+    if (byAgentId) return byAgentId;
+  }
 
   const spawnRequestId = typeof ctrl.spawnRequestId === "string" && ctrl.spawnRequestId ? ctrl.spawnRequestId : undefined;
-  if (spawnRequestId) return agents.find((agent) => agent.spawnRequestId === spawnRequestId);
+  if (spawnRequestId) {
+    const bySpawnRequestId = agents.find((agent) => agent.spawnRequestId === spawnRequestId);
+    if (bySpawnRequestId) return bySpawnRequestId;
+  }
 
   const policyName = typeof ctrl.policyName === "string" && ctrl.policyName ? ctrl.policyName : undefined;
-  if (policyName) return agents.find((agent) => agent.policyName === policyName);
+  if (policyName) {
+    const byPolicyName = agents.find((agent) => agent.policyName === policyName);
+    if (byPolicyName) return byPolicyName;
+  }
   return undefined;
 }
 

package/src/index.ts

@@ -217,9 +217,13 @@ async function healthCheck(): Promise<void> {
       continue;
     }
     if (liveness === "dead") {
+      // #1746 fwd — resolve THIS specific dead agent by its UNIQUE keys only. Since #1746 fwd made
+      // selectSessionRecord "try every key" (a MISS falls through to the next, for the kill-switch),
+      // passing policyName here would let a full-key-set miss (this agent's own record already removed)
+      // fall through to the policyName branch and misattribute a live sibling policy session's exit —
+      // flipping the sibling offline while it's still running and dropping this agent with no exit record.
       const diagnostics = diagnoseSessionExit({
         agentId: refreshed.agentId,
-        policyName: refreshed.policyName,
         spawnRequestId: refreshed.spawnRequestId,
         tmuxSession: sessionName,
         systemdOverride: capturedSystemd,

package/src/spawn/runtime.ts

@@ -3,7 +3,7 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 import type { ManagedAgentReport } from "../relay";
 import { classifyPidLiveness, isPidAlive, isValidPid, parseProcStateIsZombie, readProcessStartMs, type PidLiveness } from "agent-relay-sdk/process-utils";
-import { tmuxCommand, tmuxHasSession } from "agent-relay-sdk/tmux-utils";
+import { renderTmuxSessionTarget, tmuxCommand, tmuxHasSession } from "agent-relay-sdk/tmux-utils";
 import { LOG_DIR, RUNNER_INFO_DIR, SESSION_DIR, STATE_FILE, TOMBSTONE_DIR } from "./constants";
 import { systemdMainPid, systemdUnitDiagnostics, systemdUnitLivenessFromDiagnostics, type SystemdUnitDiagnostics, type SystemdUnitLiveness } from "./systemd";
 import { sanitizeFsName } from "agent-relay-sdk/fs-name";
@@ -184,20 +184,31 @@ export function sessionReportFields(record: Pick<SessionRecord, "name" | "superv
 }
 
 export function selectSessionRecord(records: SessionRecord[], input: { agentId?: string; policyName?: string; spawnRequestId?: string; tmuxSession?: string }): SessionRecord | undefined {
-  if (input.tmuxSession) return records.find((record) => record.name === input.tmuxSession);
-
-  if (input.spawnRequestId) {
-    return records.find((record) =>
-      record.spawnRequestId === input.spawnRequestId &&
-      (!input.policyName || record.policyName === input.policyName)
-    );
+  // #1746 RC-1 — try EVERY provided key, falling through on a MISS rather than early-returning
+  // `undefined` on the first-specified one. A session-name lookup that misses (e.g. the runner's
+  // rendered `.`/`:`→`_` name vs the record's requested name, #1583) must still resolve by
+  // spawnRequestId/agentId/policyName so the host-side shutdown fallback can find the live session.
+  if (input.tmuxSession) {
+    const rendered = renderTmuxSessionTarget(input.tmuxSession);
+    const bySession = records.find((record) => record.name === input.tmuxSession || renderTmuxSessionTarget(record.name) === rendered);
+    if (bySession) return bySession;
   }
 
+  // #1746 fwd BLOCKER — each key falls through on a MISS instead of early-returning `undefined`. The
+  // prior code `return`ed the `.find()` result for the first-specified key even when it was undefined,
+  // so a spawnRequestId that missed (or whose policyName conjunct rejected the match) stranded the whole
+  // lookup and never tried agentId/policyName — the exact surviving-agent path the kill-switch must
+  // resolve. This mirrors control.ts `managedAgentShutdownTarget`: match each UNIQUE key on its own
+  // (agentId → spawnRequestId → policyName, per e12b4c2a's stated canonical order) with no cross-key
+  // conjunct, so a divergent policyName can never veto an otherwise-exact agentId/spawnRequestId hit.
   if (input.agentId) {
-    return records.find((record) =>
-      record.agentId === input.agentId &&
-      (!input.policyName || record.policyName === input.policyName)
-    );
+    const byAgentId = records.find((record) => record.agentId === input.agentId);
+    if (byAgentId) return byAgentId;
+  }
+
+  if (input.spawnRequestId) {
+    const bySpawnRequestId = records.find((record) => record.spawnRequestId === input.spawnRequestId);
+    if (bySpawnRequestId) return bySpawnRequestId;
   }
 
   if (input.policyName) {

package/src/spawn/sessions.ts

@@ -69,10 +69,15 @@ export function managedSessionLivenessDetailed(name: string): SessionRecordLiven
 }
 
 export function refreshManagedAgentReport(agent: ManagedAgentReport): ManagedAgentReport {
+  // #1746 fwd — refresh a SPECIFIC known agent by its UNIQUE keys only. Since #1746 fwd made
+  // selectSessionRecord "try every key" (a MISS falls through to the next, for the kill-switch),
+  // passing policyName here would let a full-key-set miss (this agent's own session gone from state)
+  // fall through to the policyName branch and refresh THIS agent's report with a live SIBLING policy
+  // session's fields. The unique keys fully identify the agent; a genuine miss must return the agent
+  // unchanged (as before), never a sibling.
   const record = findSessionRecord({
     tmuxSession: agent.sessionName ?? agent.tmuxSession,
     agentId: agent.agentId,
-    policyName: agent.policyName,
     spawnRequestId: agent.spawnRequestId,
   });
   if (!record) return agent;

package/src/spawn/spawn-agent.ts

@@ -154,7 +154,12 @@ export async function spawnAgent(
 
 function existingSpawnSession(opts: SpawnOptions, deps: Pick<SpawnAgentDeps, "findSessionRecord" | "sessionRecordLiveness" | "currentSessionPid" | "sessionReportFields">): ManagedAgentReport | undefined {
   if (!opts.spawnRequestId) return undefined;
-  const record = deps.findSessionRecord({ spawnRequestId: opts.spawnRequestId, policyName: opts.policyName });
+  // #1746 fwd — this is an EXACT-spawnRequestId idempotency probe ("is THIS request already running?"),
+  // not a target resolution. Since #1746 fwd made selectSessionRecord "try every key" (a MISS on one key
+  // falls through to the next, for the kill-switch), passing policyName here would let a FRESH
+  // spawnRequestId that misses fall through to the policyName branch and wrongly dedupe a genuinely-new
+  // spawn onto an existing policy session. spawnRequestId is unique, so match it and it ALONE.
+  const record = deps.findSessionRecord({ spawnRequestId: opts.spawnRequestId });
   if (!record) return undefined;
   const liveness = deps.sessionRecordLiveness(record);
   if (liveness === "dead") return undefined;