agent-relay-orchestrator 0.10.19 → 0.10.21

package/src/artifact-proxy.ts

@@ -0,0 +1,173 @@
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join, relative, resolve } from "node:path";
+import type { OrchestratorConfig } from "./config";
+
+const SAFE_ARTIFACT_ID = /^[a-zA-Z0-9._-]{1,160}$/;
+const CONTENT_ROUTE = /^\/api\/artifacts\/([^/]+)\/content$/;
+const CACHEABLE_RESPONSE_HEADERS = [
+  "content-type",
+  "content-disposition",
+  "x-artifact-digest",
+  "etag",
+  "last-modified",
+  "cache-control",
+];
+const FORWARDED_REQUEST_HEADERS = [
+  "content-type",
+  "x-artifact-filename",
+  "x-artifact-digest",
+  "x-artifact-kind",
+  "x-artifact-sensitivity",
+  "x-artifact-expires-at",
+];
+const RESPONSE_HEADERS = [
+  "content-type",
+  "content-length",
+  "content-disposition",
+  "x-artifact-digest",
+  "etag",
+  "last-modified",
+  "cache-control",
+];
+
+interface CacheMeta {
+  headers: Record<string, string>;
+  cachedAt: number;
+}
+
+export function artifactProxyBaseUrl(config: Pick<OrchestratorConfig, "apiPort">): string {
+  return `http://127.0.0.1:${config.apiPort}/api/artifacts`;
+}
+
+function artifactCacheRoot(): string {
+  const configured = process.env.AGENT_RELAY_ARTIFACT_CACHE_DIR || "~/.agent-relay/cache/artifacts";
+  if (configured === "~") return homedir();
+  if (configured.startsWith("~/")) return join(homedir(), configured.slice(2));
+  return configured;
+}
+
+function cachePaths(id: string, root = artifactCacheRoot()): { dir: string; content: string; meta: string } {
+  if (!SAFE_ARTIFACT_ID.test(id)) throw new Error("invalid artifact id");
+  const base = resolve(root);
+  const dir = resolve(base, id);
+  const rel = relative(base, dir);
+  if (rel.startsWith("..") || rel.startsWith("/") || rel === "") throw new Error("artifact cache path escapes root");
+  return {
+    dir,
+    content: join(dir, "content"),
+    meta: join(dir, "meta.json"),
+  };
+}
+
+function responseHeaders(source: Headers, names = RESPONSE_HEADERS): Headers {
+  const headers = new Headers();
+  for (const name of names) {
+    const value = source.get(name);
+    if (value) headers.set(name, value);
+  }
+  headers.set("Access-Control-Allow-Origin", "*");
+  return headers;
+}
+
+function forwardHeaders(req: Request, config: OrchestratorConfig): Headers {
+  const headers = new Headers();
+  for (const name of FORWARDED_REQUEST_HEADERS) {
+    const value = req.headers.get(name);
+    if (value) headers.set(name, value);
+  }
+  const token = config.token || req.headers.get("x-agent-relay-token");
+  if (token) headers.set("X-Agent-Relay-Token", token);
+  return headers;
+}
+
+function relayUrl(req: Request, config: OrchestratorConfig): URL {
+  const url = new URL(req.url);
+  return new URL(url.pathname + url.search, config.relayUrl);
+}
+
+async function relayFetch(req: Request, config: OrchestratorConfig): Promise<Response> {
+  return fetch(relayUrl(req, config), {
+    method: req.method,
+    headers: forwardHeaders(req, config),
+    body: req.method === "GET" || req.method === "HEAD" ? undefined : req.body,
+  });
+}
+
+function cachedContentResponse(id: string): Response | null {
+  let paths;
+  try {
+    paths = cachePaths(id);
+  } catch {
+    return null;
+  }
+  if (!existsSync(paths.content) || !existsSync(paths.meta)) return null;
+  try {
+    const meta = JSON.parse(readFileSync(paths.meta, "utf8")) as CacheMeta;
+    const stat = statSync(paths.content);
+    const headers = new Headers(meta.headers);
+    headers.set("Content-Length", String(stat.size));
+    return new Response(readFileSync(paths.content), { status: 200, headers });
+  } catch {
+    return null;
+  }
+}
+
+function writeCachedContent(id: string, bytes: ArrayBuffer, headers: Headers): void {
+  const paths = cachePaths(id);
+  mkdirSync(paths.dir, { recursive: true, mode: 0o700 });
+  const cachedHeaders: Record<string, string> = {};
+  for (const name of CACHEABLE_RESPONSE_HEADERS) {
+    const value = headers.get(name);
+    if (value) cachedHeaders[name] = value;
+  }
+  cachedHeaders["content-length"] = String(bytes.byteLength);
+  writeFileSync(paths.content, Buffer.from(bytes));
+  writeFileSync(paths.meta, JSON.stringify({ headers: cachedHeaders, cachedAt: Date.now() } satisfies CacheMeta, null, 2) + "\n");
+}
+
+async function proxyContent(req: Request, config: OrchestratorConfig, id: string): Promise<Response> {
+  if (req.method === "GET") {
+    const cached = cachedContentResponse(id);
+    if (cached) return cached;
+  }
+
+  const relay = await relayFetch(req, config);
+  const headers = responseHeaders(relay.headers);
+  if (req.method === "HEAD" || !relay.ok) {
+    return new Response(req.method === "HEAD" ? null : relay.body, { status: relay.status, headers });
+  }
+
+  const bytes = await relay.arrayBuffer();
+  try {
+    writeCachedContent(id, bytes, headers);
+  } catch (e) {
+    console.error(`[orchestrator] artifact cache write failed for ${id}: ${(e as Error).message}`);
+  }
+  return new Response(bytes, { status: relay.status, headers });
+}
+
+export async function proxyArtifactRequest(req: Request, config: OrchestratorConfig): Promise<Response> {
+  const url = new URL(req.url);
+  if (url.pathname === "/api/artifacts" && (req.method === "GET" || req.method === "POST")) {
+    const relay = await relayFetch(req, config);
+    return new Response(relay.body, { status: relay.status, headers: responseHeaders(relay.headers) });
+  }
+
+  const contentMatch = url.pathname.match(CONTENT_ROUTE);
+  if (contentMatch && (req.method === "GET" || req.method === "HEAD")) {
+    const id = decodeURIComponent(contentMatch[1]!);
+    if (!SAFE_ARTIFACT_ID.test(id)) return new Response(JSON.stringify({ error: "invalid artifact id" }), { status: 400, headers: { "Content-Type": "application/json" } });
+    return proxyContent(req, config, id);
+  }
+
+  const artifactMatch = url.pathname.match(/^\/api\/artifacts\/([^/]+)$/);
+  if (artifactMatch && (req.method === "GET" || req.method === "DELETE")) {
+    const id = decodeURIComponent(artifactMatch[1]!);
+    if (!SAFE_ARTIFACT_ID.test(id)) return new Response(JSON.stringify({ error: "invalid artifact id" }), { status: 400, headers: { "Content-Type": "application/json" } });
+    const relay = await relayFetch(req, config);
+    return new Response(relay.body, { status: relay.status, headers: responseHeaders(relay.headers) });
+  }
+
+  return new Response(JSON.stringify({ error: "Not found" }), { status: 404, headers: { "Content-Type": "application/json" } });
+}

package/src/control.ts

@@ -1,6 +1,8 @@
 import type { OrchestratorConfig } from "./config";
 import type { ManagedAgentReport, RelayClient, RelayCommand } from "./relay";
-import { spawnAgent, stopSession, type SpawnOptions } from "./tmux";
+import { handleSelfUpgrade } from "./self-upgrade";
+import { spawnAgent, stopSession, type SpawnOptions } from "./spawn";
+import { cleanupWorkspace, mergeWorkspace, reconcileWorkspace } from "./workspace-probe";
 
 interface ControlHandler {
   handleCommand(command: RelayCommand): Promise<boolean>;
@@ -15,18 +17,7 @@ export function createControlHandler(
   let managedAgents: ManagedAgentReport[] = [];
 
   async function handleSpawn(ctrl: Record<string, any>): Promise<boolean> {
-    const opts: SpawnOptions = {
-      provider: ctrl.provider || "claude",
-      cwd: ctrl.cwd || config.baseDir,
-      label: ctrl.label,
-      approvalMode: ctrl.approvalMode || "guarded",
-      prompt: ctrl.prompt,
-      tags: Array.isArray(ctrl.tags) ? ctrl.tags.filter((item): item is string => typeof item === "string") : undefined,
-      capabilities: Array.isArray(ctrl.capabilities) ? ctrl.capabilities.filter((item): item is string => typeof item === "string") : undefined,
-      providerArgs: Array.isArray(ctrl.providerArgs) ? ctrl.providerArgs.filter((item): item is string => typeof item === "string") : undefined,
-      policyName: typeof ctrl.policyName === "string" ? ctrl.policyName : undefined,
-      spawnRequestId: typeof ctrl.spawnRequestId === "string" ? ctrl.spawnRequestId : undefined,
-    };
+    const opts = spawnOptionsFromControl(ctrl, config);
 
     try {
       const agent = await spawnAgent(opts, config);
@@ -40,17 +31,30 @@ export function createControlHandler(
   }
 
   async function handleShutdown(ctrl: Record<string, any>, restart = false): Promise<Record<string, unknown>> {
-    const session = typeof ctrl.tmuxSession === "string"
-      ? ctrl.tmuxSession
-      : managedAgents.find((agent) => agent.agentId === ctrl.agentId || (ctrl.policyName && agent.policyName === ctrl.policyName))?.tmuxSession;
+    const current = managedAgentShutdownTarget(managedAgents, ctrl);
+    const session = current?.sessionName ?? current?.tmuxSession;
     if (!session) return { stopped: false, wasRunning: false };
-    const result = await stopSession(session, config, typeof ctrl.reason === "string" ? ctrl.reason : restart ? "restart" : "shutdown", ctrl.graceful !== false);
-    managedAgents = managedAgents.filter((agent) => agent.tmuxSession !== session);
+    const result = await stopSession(session, config, typeof ctrl.reason === "string" ? ctrl.reason : restart ? "restart" : "shutdown", ctrl.graceful !== false, shutdownTimeoutMs(ctrl));
+    managedAgents = managedAgents.filter((agent) => (agent.sessionName ?? agent.tmuxSession) !== session);
+    const restartSpawn = isRecord(ctrl.restartSpawn) ? ctrl.restartSpawn : undefined;
+    // A managed restart carries a fresh spawnRequestId in restartSpawn — keep it.
+    // Falling back to the live agent's params would reuse the stale id and break
+    // relay correlation, so drop it and let spawnAgent assign a new identity.
+    const restartSource = (restartSpawn ?? (current ? { ...current, spawnRequestId: undefined } : undefined)) as Record<string, any> | undefined;
+    let restarted: ManagedAgentReport | undefined;
+    if (restart && restartSource) {
+      restarted = await spawnAgent(spawnOptionsFromRestartSource(restartSource, config), config);
+      managedAgents.push(restarted);
+      console.error(`[orchestrator] Restarted ${restarted.provider} agent: ${restarted.tmuxSession}`);
+    }
     return {
       ...result,
       restart,
+      restarted: Boolean(restarted),
+      ...(restarted ? { agent: restarted } : {}),
       policyName: ctrl.policyName,
       spawnRequestId: ctrl.spawnRequestId,
+      sessionName: session,
       tmuxSession: session,
     };
   }
@@ -66,6 +70,44 @@ export function createControlHandler(
       } else if (command.type === "agent.shutdown" || command.type === "agent.restart") {
         const result = await handleShutdown(command.params, command.type === "agent.restart");
         await relay.updateCommand(command.id, "succeeded", result);
+      } else if (command.type === "workspace.cleanup") {
+        const result = cleanupWorkspace({
+          id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
+          repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
+          worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
+          branch: typeof command.params.branch === "string" ? command.params.branch : undefined,
+          deleteBranch: command.params.deleteBranch !== false,
+        });
+        await relay.updateCommand(command.id, "succeeded", result);
+      } else if (command.type === "workspace.reconcile") {
+        const result = reconcileWorkspace({
+          id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
+          repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
+          worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
+          branch: typeof command.params.branch === "string" ? command.params.branch : undefined,
+          baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined,
+          baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined,
+        });
+        await relay.updateCommand(command.id, "succeeded", result);
+      } else if (command.type === "workspace.merge") {
+        const result = mergeWorkspace({
+          id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
+          repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
+          worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
+          branch: typeof command.params.branch === "string" ? command.params.branch : undefined,
+          baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined,
+          baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined,
+          strategy: command.params.strategy === "pr" || command.params.strategy === "rebase-ff" || command.params.strategy === "auto" ? command.params.strategy : undefined,
+          deleteBranch: command.params.deleteBranch !== false,
+          prTitle: typeof command.params.prTitle === "string" ? command.params.prTitle : undefined,
+          prBody: typeof command.params.prBody === "string" ? command.params.prBody : undefined,
+        });
+        await relay.updateCommand(command.id, "succeeded", result as unknown as Record<string, unknown>);
+      } else if (command.type === "orchestrator.upgrade") {
+        // Install + restart ourselves. Intentionally NOT marked "succeeded": the
+        // relay settles it by reconciling the version we report after we restart,
+        // since the success ack can't survive our own process teardown.
+        await handleSelfUpgrade(command, config, relay);
       } else {
         throw new Error(`unsupported orchestrator command: ${command.type}`);
       }
@@ -87,3 +129,99 @@ export function createControlHandler(
 
   return { handleCommand, getManagedAgents, setManagedAgents };
 }
+
+export function managedAgentShutdownTarget(agents: ManagedAgentReport[], ctrl: Record<string, any>): ManagedAgentReport | undefined {
+  const requestedSession = typeof ctrl.sessionName === "string" && ctrl.sessionName
+    ? ctrl.sessionName
+    : typeof ctrl.tmuxSession === "string" && ctrl.tmuxSession
+      ? ctrl.tmuxSession
+      : undefined;
+  if (requestedSession) return agents.find((agent) => agent.sessionName === requestedSession || agent.tmuxSession === requestedSession);
+
+  const agentId = typeof ctrl.agentId === "string" && ctrl.agentId ? ctrl.agentId : undefined;
+  if (agentId) return agents.find((agent) => agent.agentId === agentId);
+
+  const spawnRequestId = typeof ctrl.spawnRequestId === "string" && ctrl.spawnRequestId ? ctrl.spawnRequestId : undefined;
+  if (spawnRequestId) return agents.find((agent) => agent.spawnRequestId === spawnRequestId);
+
+  const policyName = typeof ctrl.policyName === "string" && ctrl.policyName ? ctrl.policyName : undefined;
+  if (policyName) return agents.find((agent) => agent.policyName === policyName);
+  return undefined;
+}
+
+function shutdownTimeoutMs(ctrl: Record<string, any>): number | undefined {
+  return Number.isSafeInteger(ctrl.timeoutMs) && ctrl.timeoutMs > 0 ? Math.min(ctrl.timeoutMs, 60_000) : undefined;
+}
+
+export function spawnOptionsFromControl(ctrl: Record<string, any>, config: OrchestratorConfig): SpawnOptions {
+  return {
+    provider: ctrl.provider || "claude",
+    cwd: ctrl.cwd || config.baseDir,
+    rig: typeof ctrl.rig === "string" ? ctrl.rig : undefined,
+    model: modelFromControl(ctrl),
+    effort: typeof ctrl.effort === "string" ? ctrl.effort : undefined,
+    profile: typeof ctrl.profile === "string" ? ctrl.profile : undefined,
+    workspaceMode: workspaceMode(ctrl.workspaceMode),
+    agentProfile: isRecord(ctrl.agentProfile) ? ctrl.agentProfile : undefined,
+    label: ctrl.label,
+    agentId: typeof ctrl.agentId === "string" ? ctrl.agentId : undefined,
+    approvalMode: ctrl.approvalMode || "guarded",
+    prompt: ctrl.prompt,
+    systemPromptAppend: typeof ctrl.systemPromptAppend === "string" ? ctrl.systemPromptAppend : undefined,
+    tags: stringArray(ctrl.tags),
+    capabilities: stringArray(ctrl.capabilities),
+    providerArgs: stringArray(ctrl.providerArgs),
+    env: stringRecord(ctrl.env),
+    policyName: typeof ctrl.policyName === "string" ? ctrl.policyName : undefined,
+    spawnRequestId: typeof ctrl.spawnRequestId === "string" ? ctrl.spawnRequestId : undefined,
+    automationId: typeof ctrl.automationId === "string" ? ctrl.automationId : undefined,
+    automationRunId: typeof ctrl.automationRunId === "string" ? ctrl.automationRunId : undefined,
+  };
+}
+
+export function spawnOptionsFromRestartSource(restartSource: Record<string, any>, config: OrchestratorConfig): SpawnOptions {
+  return {
+    provider: restartSource.provider === "codex" ? "codex" : "claude",
+    cwd: typeof restartSource.cwd === "string" ? restartSource.cwd : config.baseDir,
+    model: modelFromControl(restartSource),
+    effort: typeof restartSource.effort === "string" ? restartSource.effort : undefined,
+    profile: typeof restartSource.profile === "string" ? restartSource.profile : undefined,
+    workspaceMode: workspaceMode(restartSource.workspaceMode),
+    agentProfile: isRecord(restartSource.agentProfile) ? restartSource.agentProfile : undefined,
+    label: typeof restartSource.label === "string" ? restartSource.label : undefined,
+    agentId: typeof restartSource.agentId === "string" ? restartSource.agentId : undefined,
+    approvalMode: typeof restartSource.approvalMode === "string" ? restartSource.approvalMode : "guarded",
+    prompt: typeof restartSource.prompt === "string" ? restartSource.prompt : undefined,
+    systemPromptAppend: typeof restartSource.systemPromptAppend === "string" ? restartSource.systemPromptAppend : undefined,
+    tags: stringArray(restartSource.tags),
+    capabilities: stringArray(restartSource.capabilities),
+    providerArgs: stringArray(restartSource.providerArgs),
+    env: stringRecord(restartSource.env),
+    policyName: typeof restartSource.policyName === "string" ? restartSource.policyName : undefined,
+    spawnRequestId: typeof restartSource.spawnRequestId === "string" ? restartSource.spawnRequestId : undefined,
+    automationId: typeof restartSource.automationId === "string" ? restartSource.automationId : undefined,
+    automationRunId: typeof restartSource.automationRunId === "string" ? restartSource.automationRunId : undefined,
+  };
+}
+
+function modelFromControl(ctrl: Record<string, any>): string | undefined {
+  return typeof ctrl.providerModel === "string" ? ctrl.providerModel : typeof ctrl.model === "string" ? ctrl.model : undefined;
+}
+
+function stringRecord(value: unknown): Record<string, string> | undefined {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
+  const entries = Object.entries(value).filter((entry): entry is [string, string] => typeof entry[1] === "string");
+  return entries.length ? Object.fromEntries(entries) : undefined;
+}
+
+function stringArray(value: unknown): string[] | undefined {
+  return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : undefined;
+}
+
+function workspaceMode(value: unknown): SpawnOptions["workspaceMode"] {
+  return value === "isolated" || value === "shared" || value === "inherit" ? value : undefined;
+}
+
+function isRecord(value: unknown): value is Record<string, any> {
+  return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}

package/src/index.ts

@@ -1,10 +1,12 @@
 #!/usr/bin/env bun
 import { loadConfig, initConfigFile } from "./config";
 import { createRelayClient } from "./relay";
+import type { ManagedSessionExitDiagnostics } from "./relay";
 import { createControlHandler } from "./control";
-import { hasSession } from "./tmux";
+import { diagnoseSessionExit, isSessionAlive, refreshManagedAgentReport } from "./spawn";
 import { startApiServer } from "./api";
 import { recoverManagedAgents } from "./recovery";
+import { ProviderProbeCache } from "./provider-probe";
 
 const args = process.argv.slice(2);
 
@@ -36,7 +38,8 @@ Config file: ~/.agent-relay/orchestrator.json
 }
 
 const config = loadConfig();
-const relay = createRelayClient(config);
+const probeCache = new ProviderProbeCache(config);
+const relay = createRelayClient(config, probeCache);
 const control = createControlHandler(config, relay);
 
 const POLL_INTERVAL_MS = 3_000;
@@ -53,7 +56,7 @@ async function startup(): Promise<void> {
   console.error(`[orchestrator]   env keys: ${Object.keys(config.env).length}`);
 
   // Start API server before registration so we can advertise the URL
-  apiServer = startApiServer(config);
+  apiServer = startApiServer(config, probeCache);
   console.error(`[orchestrator]   apiUrl: ${apiServer.url}`);
 
   // Register with relay. The server and orchestrator are often restarted
@@ -108,16 +111,59 @@ async function registerUntilConnected(): Promise<void> {
 async function healthCheck(): Promise<void> {
   const agents = control.getManagedAgents();
   let changed = false;
+  const exitedAgents: ManagedSessionExitDiagnostics[] = [];
+  let remainingAgents = agents;
   for (const agent of agents) {
-    const alive = await hasSession(agent.tmuxSession);
+    const refreshed = refreshManagedAgentReport(agent);
+    if (JSON.stringify(refreshed) !== JSON.stringify(agent)) {
+      remainingAgents = remainingAgents.map((item) => item.agentId === agent.agentId ? refreshed : item);
+      control.setManagedAgents(remainingAgents);
+      changed = true;
+    }
+    const sessionName = refreshed.sessionName ?? refreshed.tmuxSession;
+    const alive = isSessionAlive(sessionName);
     if (!alive) {
-      console.error(`[orchestrator] Session dead: ${agent.tmuxSession} — removing from managed list`);
-      control.setManagedAgents(agents.filter((a) => a.tmuxSession !== agent.tmuxSession));
+      const diagnostics = diagnoseSessionExit({
+        agentId: refreshed.agentId,
+        policyName: refreshed.policyName,
+        spawnRequestId: refreshed.spawnRequestId,
+        tmuxSession: sessionName,
+      }) ?? {
+        agentId: refreshed.agentId,
+        provider: refreshed.provider,
+        sessionName,
+        tmuxSession: refreshed.tmuxSession,
+        cwd: refreshed.cwd,
+        label: refreshed.label,
+        policyName: refreshed.policyName,
+        spawnRequestId: refreshed.spawnRequestId,
+        supervisor: refreshed.supervisor ?? "unknown",
+        systemdUnit: refreshed.systemdUnit,
+        terminalSession: refreshed.terminalSession,
+        terminalAvailable: refreshed.terminalAvailable,
+        pid: refreshed.pid,
+        startedAt: refreshed.startedAt,
+        detectedAt: Date.now(),
+        runtimeMs: Math.max(0, Date.now() - refreshed.startedAt),
+        runnerInfoPresent: false,
+        unavailable: ["orchestrator session record unavailable", "stdout/stderr log unavailable"],
+        lastError: `managed ${refreshed.provider} session ${sessionName} disappeared; orchestrator session record unavailable`,
+      };
+      exitedAgents.push(diagnostics);
+      console.error(`[orchestrator] Session dead: ${sessionName} — ${diagnostics.lastError}`);
+      if (diagnostics.logFile) {
+        console.error(`[orchestrator]   log: ${diagnostics.logFile} (${diagnostics.logBytes ?? "unknown"} bytes${diagnostics.logEmpty ? ", empty" : ""})`);
+      }
+      if (diagnostics.logTail?.length) {
+        console.error(`[orchestrator]   last log line: ${diagnostics.logTail[diagnostics.logTail.length - 1]}`);
+      }
+      remainingAgents = remainingAgents.filter((a) => (a.sessionName ?? a.tmuxSession) !== sessionName);
+      control.setManagedAgents(remainingAgents);
       changed = true;
     }
   }
   if (changed) {
-    await relay.updateManagedAgents(control.getManagedAgents());
+    await relay.updateManagedAgents(control.getManagedAgents(), exitedAgents);
   }
 }
 

package/src/provider-probe.ts

@@ -0,0 +1,184 @@
+import { accessSync, constants, existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { delimiter, join, resolve } from "node:path";
+import { providerCatalogList, type ProviderCatalogEntry } from "agent-relay-sdk/provider-catalog";
+import type { OrchestratorConfig } from "./config";
+import { VERSION } from "./version";
+
+interface ProviderProbeDetail {
+  command: string;
+  path?: string;
+  ok: boolean;
+  version?: string;
+  error?: string;
+}
+
+interface ProviderProbeResult {
+  name: "claude" | "codex";
+  available: boolean;
+  checkedAt: number;
+  reason?: string;
+  version?: string;
+  runnerVersion: string;
+  features?: Record<string, boolean>;
+  cli: ProviderProbeDetail;
+  runner: ProviderProbeDetail;
+}
+
+interface ProviderProbeSnapshot {
+  providers: ("claude" | "codex")[];
+  providerStatus: ProviderProbeResult[];
+  providerCatalog: ProviderCatalogEntry[];
+  checkedAt: number;
+}
+
+const DEFAULT_TTL_MS = 30_000;
+const DEFAULT_TIMEOUT_MS = 2_500;
+
+export class ProviderProbeCache {
+  private snapshot?: ProviderProbeSnapshot;
+
+  constructor(
+    private readonly config: OrchestratorConfig,
+    private readonly ttlMs = DEFAULT_TTL_MS,
+    private readonly timeoutMs = DEFAULT_TIMEOUT_MS,
+  ) {}
+
+  async getSnapshot(refresh = false): Promise<ProviderProbeSnapshot> {
+    const now = Date.now();
+    if (!refresh && this.snapshot && now - this.snapshot.checkedAt < this.ttlMs) {
+      return this.snapshot;
+    }
+    const providerStatus = await Promise.all(this.config.providers.map((provider) => probeProvider(provider, this.timeoutMs)));
+    const providers = providerStatus.filter((status) => status.available).map((status) => status.name);
+    this.snapshot = {
+      providers,
+      providerStatus,
+      providerCatalog: providerCatalogList().filter((entry) => this.config.providers.includes(entry.provider)),
+      checkedAt: Date.now(),
+    };
+    return this.snapshot;
+  }
+}
+
+export async function probeProvider(provider: "claude" | "codex", timeoutMs = DEFAULT_TIMEOUT_MS): Promise<ProviderProbeResult> {
+  const cli = await probeCommand(provider, ["--version"], timeoutMs, provider, (path) => isRelayProviderShim(path, provider));
+  const runner = await probeRunner(provider, timeoutMs);
+  const available = cli.ok && runner.ok;
+  const features = probeProviderFeatures(provider);
+  return {
+    name: provider,
+    available,
+    checkedAt: Date.now(),
+    reason: available ? undefined : unavailableReason(cli, runner),
+    version: cli.version,
+    runnerVersion: VERSION,
+    ...(Object.keys(features).length > 0 ? { features } : {}),
+    cli,
+    runner,
+  };
+}
+
+function probeProviderFeatures(provider: "claude" | "codex"): Record<string, boolean> {
+  const features: Record<string, boolean> = {};
+  if (provider === "claude") {
+    const hasRigBinary = !!resolveExecutable("claude-rig");
+    const hasRigDir = existsSync(join(homedir(), ".claude-rig"));
+    if (hasRigBinary || hasRigDir) features.rig = true;
+  }
+  return features;
+}
+
+async function probeRunner(provider: "claude" | "codex", timeoutMs: number): Promise<ProviderProbeDetail> {
+  const repoLauncher = resolve(import.meta.dir, "../../runner/src/index.ts");
+  if (existsSync(repoLauncher)) {
+    const bun = resolveExecutable("bun");
+    if (!bun) return { command: "bun", ok: false, error: "bun executable not found" };
+    return probeCommand(bun, ["run", repoLauncher, provider, "--help"], timeoutMs, `bun run ${repoLauncher}`);
+  }
+  return probeCommand(`${provider}-relay`, ["--help"], timeoutMs);
+}
+
+async function probeCommand(
+  command: string,
+  args: string[],
+  timeoutMs: number,
+  displayCommand = command,
+  skipPath?: (path: string) => boolean,
+): Promise<ProviderProbeDetail> {
+  const path = resolveExecutable(command, skipPath);
+  if (!path) return { command: displayCommand, ok: false, error: `${displayCommand} executable not found` };
+
+  let proc: Bun.Subprocess<"ignore", "pipe", "pipe"> | undefined;
+  try {
+    proc = Bun.spawn([path, ...args], { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
+    const output = await Promise.race([
+      processOutput(proc),
+      Bun.sleep(timeoutMs).then(() => "timeout" as const),
+    ]);
+    if (output === "timeout") {
+      try { proc.kill("SIGKILL"); } catch {}
+      return { command: displayCommand, path, ok: false, error: `probe timed out after ${timeoutMs}ms` };
+    }
+    const { exitCode, stdout, stderr } = output;
+    return {
+      command: displayCommand,
+      path,
+      ok: exitCode === 0,
+      version: firstLine(stdout),
+      error: exitCode === 0 ? undefined : firstLine(stderr) || `exit code ${exitCode}`,
+    };
+  } catch (error) {
+    try { proc?.kill("SIGKILL"); } catch {}
+    return { command: displayCommand, path, ok: false, error: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+async function processOutput(proc: Bun.Subprocess<"ignore", "pipe", "pipe">): Promise<{ exitCode: number; stdout: string; stderr: string }> {
+  const [stdout, stderr, exitCode] = await Promise.all([
+    new Response(proc.stdout).text(),
+    new Response(proc.stderr).text(),
+    proc.exited,
+  ]);
+  return { stdout, stderr, exitCode };
+}
+
+function unavailableReason(cli: ProviderProbeDetail, runner: ProviderProbeDetail): string {
+  if (!cli.ok) return cli.error || `${cli.command} unavailable`;
+  if (!runner.ok) return runner.error || `${runner.command} unavailable`;
+  return "provider unavailable";
+}
+
+function firstLine(value: string): string | undefined {
+  return value.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
+}
+
+export function resolveExecutable(command: string, skipPath?: (path: string) => boolean): string | undefined {
+  if (command.includes("/")) {
+    const path = resolve(command);
+    return isExecutable(path) && !skipPath?.(path) ? path : undefined;
+  }
+  for (const dir of (process.env.PATH || "").split(delimiter).filter(Boolean)) {
+    const path = resolve(dir, command);
+    if (isExecutable(path) && !skipPath?.(path)) return path;
+  }
+  return undefined;
+}
+
+function isExecutable(path: string): boolean {
+  try {
+    accessSync(path, constants.X_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function isRelayProviderShim(path: string, provider: "claude" | "codex"): boolean {
+  try {
+    const content = readFileSync(path, "utf8").slice(0, 1024);
+    return content.includes(`${provider}-relay ${provider}`);
+  } catch {
+    return false;
+  }
+}

package/src/recovery.ts

@@ -1,6 +1,6 @@
 import type { OrchestratorConfig } from "./config";
 import type { ManagedAgentReport, RelayClient } from "./relay";
-import { recoverExistingSessions } from "./tmux";
+import { recoverExistingSessions } from "./spawn";
 
 interface ManagedAgentControl {
   setManagedAgents(agents: ManagedAgentReport[]): void;