agent-relay-server 0.32.2 → 0.32.4

package/src/routes/orchestrator-proxy.ts

@@ -0,0 +1,160 @@
+// Auto-split from routes.ts (#299). Domain: orchestrator-proxy.
+import { RELAY_TOKEN_HEADER, SPAWN_PROVIDERS, isRecord } from "agent-relay-sdk";
+import { ValidationError, getOrchestrator, orchestratorHeartbeat } from "../db";
+import { authorizeRoute, cleanJsonArray, error, json, type Handler } from "./_shared";
+import { cleanStringArray } from "../validation";
+import { emitOrchestratorStatus } from "../sse";
+import { type OrchestratorRuntimeInput, type SpawnProvider } from "../types";
+
+export const getOrchestratorDirectories: Handler = async (req, params) => {
+  const orch = getOrchestrator(params.id!);
+  if (!orch) return error("orchestrator not found", 404);
+  if (!orch.apiUrl) return error("orchestrator does not expose an API", 422);
+  if (orch.status !== "online") return error("orchestrator is offline", 422);
+  const url = new URL(req.url);
+  const path = url.searchParams.get("path") || "";
+  const proxyUrl = `${orch.apiUrl}/api/directories${path ? `?path=${encodeURIComponent(path)}` : ""}`;
+  try {
+    const res = await fetch(proxyUrl, { signal: AbortSignal.timeout(5_000) });
+    const body = await res.json();
+    return json(body, res.status);
+  } catch (e) {
+    return error(`Failed to reach orchestrator API: ${(e as Error).message}`, 502);
+  }
+};
+
+export const postOrchestratorCreateDirectory: Handler = async (req, params) => {
+  const orch = getOrchestrator(params.id!);
+  if (!orch) return error("orchestrator not found", 404);
+  if (!orch.apiUrl) return error("orchestrator does not expose an API", 422);
+  if (orch.status !== "online") return error("orchestrator is offline", 422);
+  try {
+    const body = await req.json();
+    const res = await fetch(`${orch.apiUrl}/api/directories`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(5_000),
+    });
+    const result = await res.json();
+    return json(result, res.status);
+  } catch (e) {
+    return error(`Failed to reach orchestrator API: ${(e as Error).message}`, 502);
+  }
+};
+
+export const getOrchestratorFilesList: Handler = (req, params) =>
+  proxyOrchestratorGet(req, params.id!, "/api/files/list");
+
+export const getOrchestratorFileRead: Handler = (req, params) =>
+  proxyOrchestratorGet(req, params.id!, "/api/files/read");
+
+export const getOrchestratorFileStat: Handler = (req, params) =>
+  proxyOrchestratorGet(req, params.id!, "/api/files/stat");
+
+export const getOrchestratorWorkspaceProbe: Handler = (req, params) =>
+  proxyOrchestratorGet(req, params.id!, "/api/workspace/probe");
+
+async function proxyOrchestratorGet(req: Request, orchestratorId: string, path: string): Promise<Response> {
+  const orch = getOrchestrator(orchestratorId);
+  if (!orch) return error("orchestrator not found", 404);
+  if (!orch.apiUrl) return error("orchestrator does not expose an API", 422);
+  if (orch.status !== "online") return error("orchestrator is offline", 422);
+  const incoming = new URL(req.url);
+  const proxyUrl = `${orch.apiUrl}${path}${incoming.search}`;
+  const headers: Record<string, string> = {};
+  const relayToken = process.env.AGENT_RELAY_TOKEN;
+  if (relayToken) headers[RELAY_TOKEN_HEADER] = relayToken;
+  try {
+    const res = await fetch(proxyUrl, { headers, signal: AbortSignal.timeout(10_000) });
+    const contentType = res.headers.get("content-type") ?? "";
+    if (contentType.includes("text/event-stream")) {
+      return new Response(res.body, { status: res.status, headers: { "Content-Type": contentType } });
+    }
+    const body = await res.text();
+    return new Response(body, { status: res.status, headers: { "Content-Type": contentType || "application/json" } });
+  } catch (e) {
+    return error(`Failed to reach orchestrator API: ${(e as Error).message}`, 502);
+  }
+}
+
+async function proxyOrchestratorPost(req: Request, orchestratorId: string, path: string): Promise<Response> {
+  const orch = getOrchestrator(orchestratorId);
+  if (!orch) return error("orchestrator not found", 404);
+  if (!orch.apiUrl) return error("orchestrator does not expose an API", 422);
+  if (orch.status !== "online") return error("orchestrator is offline", 422);
+  const incoming = new URL(req.url);
+  const proxyUrl = `${orch.apiUrl}${path}${incoming.search}`;
+  const headers: Record<string, string> = {
+    "Content-Type": req.headers.get("content-type") ?? "application/json",
+  };
+  const relayToken = process.env.AGENT_RELAY_TOKEN;
+  if (relayToken) headers[RELAY_TOKEN_HEADER] = relayToken;
+  try {
+    const body = await req.text();
+    const res = await fetch(proxyUrl, {
+      method: "POST",
+      headers,
+      body,
+      signal: AbortSignal.timeout(10_000),
+    });
+    const contentType = res.headers.get("content-type") ?? "";
+    return new Response(await res.text(), { status: res.status, headers: { "Content-Type": contentType || "application/json" } });
+  } catch (e) {
+    return error(`Failed to reach orchestrator API: ${(e as Error).message}`, 502);
+  }
+}
+
+export const getOrchestratorProviders: Handler = async (req, params) => {
+  const res = await proxyOrchestratorGet(req, params.id!, "/api/providers");
+  if (!res.ok) return res;
+  const incoming = new URL(req.url);
+  if (incoming.searchParams.get("refresh") !== "1") return res;
+
+  const payload = await res.clone().json().catch(() => null) as unknown;
+  if (!isRecord(payload)) return res;
+  try {
+    const providers = cleanStringArray(payload.providers, "providers", { itemMax: 80, maxItems: 50 }) as SpawnProvider[] | undefined;
+    if (providers) {
+      for (const p of providers) {
+        if (!SPAWN_PROVIDERS.includes(p as any)) {
+          throw new ValidationError(`invalid provider: ${p}. Must be one of: ${SPAWN_PROVIDERS.join(", ")}`);
+        }
+      }
+    }
+    const updated = orchestratorHeartbeat(params.id!, {
+      providers,
+      providerStatus: cleanJsonArray(payload.providerStatus, "providerStatus") as OrchestratorRuntimeInput["providerStatus"],
+      providerCatalog: cleanJsonArray(payload.providerCatalog, "providerCatalog") as OrchestratorRuntimeInput["providerCatalog"],
+    });
+    if (updated) emitOrchestratorStatus(params.id!);
+  } catch (e) {
+    if (e instanceof ValidationError) return error(e.message, 400);
+    throw e;
+  }
+  return res;
+};
+
+export const getOrchestratorSessions: Handler = (req, params) => proxyOrchestratorGet(req, params.id!, "/api/sessions");
+
+export const getOrchestratorVersion: Handler = (req, params) => proxyOrchestratorGet(req, params.id!, "/api/version");
+
+export const getOrchestratorLogs: Handler = (req, params) => {
+  const denied = authorizeRoute(req, { scope: "logs:read", resource: { orchestratorId: params.id!, logsRead: true } });
+  return denied ?? proxyOrchestratorGet(req, params.id!, `/api/logs/${encodeURIComponent(params.session!)}`);
+};
+
+export const getOrchestratorTerminal: Handler = (req, params) => {
+  const denied = authorizeRoute(req, { scope: "terminal:attach", resource: { orchestratorId: params.id!, terminalAttach: true } });
+  return denied ?? proxyOrchestratorGet(req, params.id!, `/api/terminal/${encodeURIComponent(params.session!)}`);
+};
+
+export const postOrchestratorTerminalInput: Handler = (req, params) => {
+  const denied = authorizeRoute(req, { scope: "terminal:attach", resource: { orchestratorId: params.id!, terminalAttach: true } });
+  return denied ?? proxyOrchestratorPost(req, params.id!, `/api/terminal/${encodeURIComponent(params.session!)}/input`);
+};
+
+export const postOrchestratorTerminalResize: Handler = (req, params) => {
+  const denied = authorizeRoute(req, { scope: "terminal:attach", resource: { orchestratorId: params.id!, terminalAttach: true } });
+  return denied ?? proxyOrchestratorPost(req, params.id!, `/api/terminal/${encodeURIComponent(params.session!)}/resize`);
+};

package/src/routes/orchestrator.ts

@@ -0,0 +1,490 @@
+// Auto-split from routes.ts (#299). Domain: orchestrator.
+import { APPROVAL_MODES, SPAWN_PROVIDERS, VALID_WORKSPACE_MODES, isRecord } from "agent-relay-sdk";
+import { CONTRACT_VERSIONS, parseRuntimeCapabilities, parseRuntimeContracts, parseRuntimePackage, type RuntimeCapabilities, type RuntimeContracts, type RuntimePackageMetadata } from "../contracts";
+import { VERSION } from "../config";
+import { ValidationError, deleteOrchestrator, getOrchestrator, listOrchestrators, orchestratorHeartbeat, recordAgentExitDiagnostics, setOrchestratorUpgradeState, updateManagedAgents, upsertOrchestrator } from "../db";
+import { auditEvent, authAuditMetadata, authorizeRoute, cleanJsonArray, cleanSafeNumber, dashboardAttribution, emitCommand, error, isRootCredentialRequest, json, parseBody, spawnRequestId, type Handler } from "./_shared";
+import { cleanMeta, cleanString, cleanStringArray, cleanWorkspaceMetadata, optionalEnum } from "../validation";
+import { createCommand, updateCommand } from "../commands-db";
+import { emitAgentStatus, emitOrchestratorStatus } from "../sse";
+import { getLifecycleManager } from "../lifecycle-manager";
+import { issueOrchestratorRuntimeToken } from "../runtime-tokens";
+import { type ManagedAgent, type ManagedSessionExitDiagnostics, type OrchestratorRuntimeInput, type RegisterOrchestratorInput, type SpawnApprovalMode, type SpawnProvider } from "../types";
+
+function cleanProtocolVersion(value: unknown): number | undefined {
+  if (value === undefined) return undefined;
+  if (typeof value === "number" && Number.isInteger(value) && value > 0) return value;
+  throw new ValidationError("protocolVersion must be a positive integer");
+}
+
+function cleanRuntimePackageMetadata(value: unknown): RuntimePackageMetadata | undefined {
+  if (value === undefined) return undefined;
+  const metadata = parseRuntimePackage(value);
+  if (!metadata) throw new ValidationError("package metadata must include name and version");
+  return metadata;
+}
+
+function cleanRuntimeContracts(value: unknown): RuntimeContracts | undefined {
+  if (value === undefined) return undefined;
+  if (!isRecord(value)) throw new ValidationError("contracts must be an object");
+  const contracts = parseRuntimeContracts(value);
+  const allowed = new Set(Object.keys(CONTRACT_VERSIONS));
+  for (const key of Object.keys(value)) {
+    if (!allowed.has(key)) throw new ValidationError(`unsupported contract: ${key}`);
+    if (contracts[key as keyof typeof CONTRACT_VERSIONS] === undefined) {
+      throw new ValidationError(`${key} must be a positive integer`);
+    }
+  }
+  return contracts;
+}
+
+function cleanRuntimeCapabilities(value: unknown): RuntimeCapabilities | undefined {
+  if (value === undefined) return undefined;
+  if (!isRecord(value)) throw new ValidationError("capabilities must be an object");
+  const capabilities = parseRuntimeCapabilities(value);
+  if (Object.keys(capabilities).length !== Object.keys(value).length) {
+    throw new ValidationError("capabilities values must be boolean");
+  }
+  return capabilities;
+}
+
+export const postOrchestrator: Handler = async (req) => {
+  const parsed = await parseBody<unknown>(req);
+  if (!parsed.ok) return error(parsed.error, parsed.status);
+  try {
+    if (!isRecord(parsed.body)) return error("body required");
+    const id = cleanString(parsed.body.id, "id", { required: true, max: 120 })!;
+    const hostname = cleanString(parsed.body.hostname, "hostname", { required: true, max: 120 })!;
+    const baseDir = cleanString(parsed.body.baseDir, "baseDir", { required: true, max: 500 })!;
+    const providers = cleanStringArray(parsed.body.providers, "providers", { itemMax: 80, maxItems: 50 }) as SpawnProvider[] | undefined;
+    if (providers) {
+      for (const p of providers) {
+        if (!SPAWN_PROVIDERS.includes(p as any)) {
+          return error(`invalid provider: ${p}. Must be one of: ${SPAWN_PROVIDERS.join(", ")}`);
+        }
+      }
+    }
+    const envKeys = cleanStringArray(parsed.body.envKeys, "envKeys", { itemMax: 80, maxItems: 50 });
+    const apiUrl = cleanString(parsed.body.apiUrl, "apiUrl", { max: 500 });
+    const packageMetadata = cleanRuntimePackageMetadata(parsed.body.package);
+    const contracts = cleanRuntimeContracts(parsed.body.contracts);
+    const runtimeCapabilities = cleanRuntimeCapabilities(parsed.body.capabilities);
+    const version = cleanString(parsed.body.version, "version", { max: 80 });
+    const gitSha = cleanString(parsed.body.gitSha, "gitSha", { max: 80 });
+    const protocolVersion = cleanProtocolVersion(parsed.body.protocolVersion);
+    const providerStatus = cleanJsonArray(parsed.body.providerStatus, "providerStatus");
+    const providerCatalog = cleanJsonArray(parsed.body.providerCatalog, "providerCatalog");
+    const meta = cleanMeta(parsed.body.meta);
+	    const orch = upsertOrchestrator({
+      id,
+      hostname,
+      providers: providers ?? ["claude", "codex"],
+      baseDir,
+      apiUrl,
+      envKeys,
+      package: packageMetadata,
+      contracts,
+      capabilities: runtimeCapabilities,
+      version,
+      protocolVersion,
+      gitSha,
+      providerStatus: providerStatus as RegisterOrchestratorInput["providerStatus"],
+      providerCatalog: providerCatalog as RegisterOrchestratorInput["providerCatalog"],
+      meta,
+    });
+    reconcileOrchestratorUpgrade(orch);
+	    auditEvent({
+      clientId: "server-orchestrator-register-" + id + "-" + Date.now(),
+      kind: "state",
+      title: "Orchestrator registered",
+      body: hostname,
+      meta: id,
+      icon: "ti-server-2",
+      view: "orchestrators",
+	      metadata: { orchestratorId: id, providers: orch.providers, ...authAuditMetadata(req) },
+	    });
+	    const runtimeToken = isRootCredentialRequest(req)
+	      ? issueOrchestratorRuntimeToken({ orchestratorId: id, baseDir, createdBy: `orchestrator:${id}` })
+	      : undefined;
+	    return json(runtimeToken ? { ...orch, runtimeToken } : orch, 201);
+  } catch (e) {
+    if (e instanceof ValidationError) return error(e.message, 400);
+    throw e;
+  }
+};
+
+export const getOrchestrators: Handler = () => {
+  return json(listOrchestrators());
+};
+
+export const getOrchestratorById: Handler = (_req, params) => {
+  const orch = getOrchestrator(params.id!);
+  if (!orch) return error("orchestrator not found", 404);
+  return json(orch);
+};
+
+export const postOrchestratorHeartbeat: Handler = async (req, params) => {
+  const parsed = await parseBody<unknown>(req);
+  if (!parsed.ok) return error(parsed.error, parsed.status);
+  try {
+    const denied = authorizeRoute(req, { scope: "command:write", resource: { orchestratorId: params.id! } });
+    if (denied) return denied;
+    const body = isRecord(parsed.body) ? parsed.body : {};
+    const runtime: OrchestratorRuntimeInput = {
+      package: cleanRuntimePackageMetadata(body.package),
+      contracts: cleanRuntimeContracts(body.contracts),
+      capabilities: cleanRuntimeCapabilities(body.capabilities),
+      version: cleanString(body.version, "version", { max: 80 }),
+      protocolVersion: cleanProtocolVersion(body.protocolVersion),
+      gitSha: cleanString(body.gitSha, "gitSha", { max: 80 }),
+      providers: cleanStringArray(body.providers, "providers", { itemMax: 80, maxItems: 50 }) as SpawnProvider[] | undefined,
+      providerStatus: cleanJsonArray(body.providerStatus, "providerStatus") as OrchestratorRuntimeInput["providerStatus"],
+      providerCatalog: cleanJsonArray(body.providerCatalog, "providerCatalog") as OrchestratorRuntimeInput["providerCatalog"],
+    };
+    if (runtime.providers) {
+      for (const p of runtime.providers) {
+        if (!SPAWN_PROVIDERS.includes(p as any)) throw new ValidationError(`invalid provider: ${p}. Must be one of: ${SPAWN_PROVIDERS.join(", ")}`);
+      }
+    }
+    const orch = orchestratorHeartbeat(params.id!, runtime);
+    if (!orch) return error("orchestrator not found", 404);
+    reconcileOrchestratorUpgrade(orch);
+    return json({ ok: true });
+  } catch (e) {
+    if (e instanceof ValidationError) return error(e.message, 400);
+    throw e;
+  }
+};
+
+export const patchOrchestratorAgents: Handler = async (req, params) => {
+  const parsed = await parseBody<unknown>(req);
+  if (!parsed.ok) return error(parsed.error, parsed.status);
+  try {
+    const orch = getOrchestrator(params.id!);
+    if (!orch) return error("orchestrator not found", 404);
+    const denied = authorizeRoute(req, { scope: "command:write", resource: { orchestratorId: orch.id } });
+    if (denied) return denied;
+    if (!isRecord(parsed.body)) return error("body required");
+    const agents = parsed.body.agents;
+    if (!Array.isArray(agents)) return error("agents must be an array");
+    const cleaned: ManagedAgent[] = agents.map((a: any) => {
+      if (!isRecord(a)) throw new ValidationError("each agent must be an object");
+      const sessionName = cleanString(a.sessionName, "sessionName", { max: 240 })
+        ?? cleanString(a.tmuxSession, "tmuxSession", { required: true, max: 240 })!;
+      return {
+        agentId: cleanString(a.agentId, "agentId", { max: 240 }) || "",
+        provider: optionalEnum(a.provider, "provider", SPAWN_PROVIDERS)! as SpawnProvider,
+        sessionName,
+        tmuxSession: cleanString(a.tmuxSession, "tmuxSession", { max: 240 }) ?? sessionName,
+        supervisor: optionalEnum(a.supervisor, "supervisor", ["process", "systemd", "launchd", "unknown"] as const),
+        systemdUnit: cleanString(a.systemdUnit, "systemdUnit", { max: 240 }),
+        terminalSession: cleanString(a.terminalSession, "terminalSession", { max: 240 }),
+        terminalAvailable: typeof a.terminalAvailable === "boolean" ? a.terminalAvailable : undefined,
+        cwd: cleanString(a.cwd, "cwd", { required: true, max: 500 })!,
+        profile: cleanString(a.profile, "profile", { max: 120 }),
+        workspaceMode: optionalEnum(a.workspaceMode, "workspaceMode", VALID_WORKSPACE_MODES),
+        workspace: cleanWorkspaceMetadata(a.workspace, "workspace"),
+        label: cleanString(a.label, "label", { max: 120 }),
+        approvalMode: (optionalEnum(a.approvalMode, "approvalMode", APPROVAL_MODES, "guarded") ?? "guarded") as SpawnApprovalMode,
+        policyName: cleanString(a.policyName, "policyName", { max: 120 }),
+        spawnRequestId: cleanString(a.spawnRequestId, "spawnRequestId", { max: 160 }),
+        automationRunId: cleanString(a.automationRunId, "automationRunId", { max: 160 }),
+        pid: typeof a.pid === "number" && Number.isSafeInteger(a.pid) ? a.pid : undefined,
+        startedAt: typeof a.startedAt === "number" ? a.startedAt : Date.now(),
+      };
+    });
+    const exitedAgents = parsed.body.exitedAgents === undefined
+      ? []
+      : Array.isArray(parsed.body.exitedAgents)
+        ? parsed.body.exitedAgents.map(cleanManagedSessionExitDiagnostics)
+        : (() => { throw new ValidationError("exitedAgents must be an array"); })();
+    const updated = updateManagedAgents(params.id!, cleaned);
+    for (const exited of exitedAgents) {
+      const agent = recordAgentExitDiagnostics(exited.agentId, exited);
+      if (agent) emitAgentStatus(agent.id);
+      auditEvent({
+        clientId: [
+          "managed-session-exit",
+          params.id!,
+          exited.spawnRequestId ?? exited.tmuxSession,
+          String(exited.detectedAt),
+        ].join(":"),
+        kind: "state",
+        title: "Managed agent exited",
+        body: exited.lastError,
+        meta: exited.policyName ?? exited.label ?? exited.tmuxSession,
+        icon: "ti-alert-triangle",
+        view: "agents",
+        agentId: exited.agentId,
+        metadata: {
+          severity: "warning",
+          orchestratorId: params.id!,
+          provider: exited.provider,
+          sessionName: exited.sessionName,
+          tmuxSession: exited.tmuxSession,
+          policyName: exited.policyName,
+          spawnRequestId: exited.spawnRequestId,
+          cwd: exited.cwd,
+          supervisor: exited.supervisor,
+          systemdUnit: exited.systemdUnit,
+          terminalSession: exited.terminalSession,
+          terminalAvailable: exited.terminalAvailable,
+          logFile: exited.logFile,
+          logBytes: exited.logBytes,
+          logEmpty: exited.logEmpty,
+          runnerInfoPresent: exited.runnerInfoPresent,
+          systemd: exited.systemd,
+          unavailable: exited.unavailable,
+          lastExit: exited,
+        },
+      });
+    }
+    getLifecycleManager().onOrchestratorManagedAgentsReported(params.id!, cleaned, exitedAgents);
+    if (updated) emitOrchestratorStatus(params.id!);
+    return json(updated);
+  } catch (e) {
+    if (e instanceof ValidationError) return error(e.message, 400);
+    throw e;
+  }
+};
+
+function cleanManagedSessionExitDiagnostics(value: unknown, index: number): ManagedSessionExitDiagnostics {
+  if (!isRecord(value)) throw new ValidationError(`exitedAgents[${index}] must be an object`);
+  const provider = optionalEnum(value.provider, `exitedAgents[${index}].provider`, SPAWN_PROVIDERS);
+  if (!provider) throw new ValidationError(`exitedAgents[${index}].provider required`);
+  const systemd = isRecord(value.systemd)
+    ? {
+      unit: cleanString(value.systemd.unit, `exitedAgents[${index}].systemd.unit`, { required: true, max: 240 })!,
+      activeState: cleanString(value.systemd.activeState, `exitedAgents[${index}].systemd.activeState`, { max: 80 }),
+      subState: cleanString(value.systemd.subState, `exitedAgents[${index}].systemd.subState`, { max: 80 }),
+      result: cleanString(value.systemd.result, `exitedAgents[${index}].systemd.result`, { max: 80 }),
+      execMainCode: cleanString(value.systemd.execMainCode, `exitedAgents[${index}].systemd.execMainCode`, { max: 80 }),
+      execMainStatus: cleanString(value.systemd.execMainStatus, `exitedAgents[${index}].systemd.execMainStatus`, { max: 80 }),
+      mainPid: cleanSafeNumber(value.systemd.mainPid),
+      unavailable: cleanString(value.systemd.unavailable, `exitedAgents[${index}].systemd.unavailable`, { max: 500 }),
+    }
+    : undefined;
+  const logTail = Array.isArray(value.logTail)
+    ? value.logTail
+      .map((line, lineIndex) => cleanString(line, `exitedAgents[${index}].logTail[${lineIndex}]`, { max: 1000 }))
+      .filter(Boolean)
+      .slice(-20) as string[]
+    : undefined;
+  const unavailable = Array.isArray(value.unavailable)
+    ? value.unavailable
+      .map((item, itemIndex) => cleanString(item, `exitedAgents[${index}].unavailable[${itemIndex}]`, { max: 500 }))
+      .filter(Boolean)
+      .slice(0, 20) as string[]
+    : undefined;
+  const diagnostics: ManagedSessionExitDiagnostics = {
+    agentId: cleanString(value.agentId, `exitedAgents[${index}].agentId`, { required: true, max: 240 })!,
+    provider: provider as SpawnProvider,
+    workspaceMode: optionalEnum(value.workspaceMode, `exitedAgents[${index}].workspaceMode`, VALID_WORKSPACE_MODES),
+    workspace: cleanWorkspaceMetadata(value.workspace, `exitedAgents[${index}].workspace`),
+    sessionName: cleanString(value.sessionName, `exitedAgents[${index}].sessionName`, { max: 240 }),
+    tmuxSession: cleanString(value.tmuxSession, `exitedAgents[${index}].tmuxSession`, { required: true, max: 240 })!,
+    cwd: cleanString(value.cwd, `exitedAgents[${index}].cwd`, { required: true, max: 500 })!,
+    label: cleanString(value.label, `exitedAgents[${index}].label`, { max: 120 }),
+    policyName: cleanString(value.policyName, `exitedAgents[${index}].policyName`, { max: 120 }),
+    spawnRequestId: cleanString(value.spawnRequestId, `exitedAgents[${index}].spawnRequestId`, { max: 160 }),
+    automationRunId: cleanString(value.automationRunId, `exitedAgents[${index}].automationRunId`, { max: 160 }),
+    supervisor: optionalEnum(value.supervisor, `exitedAgents[${index}].supervisor`, ["process", "systemd", "launchd", "unknown"] as const, "unknown")!,
+    systemdUnit: cleanString(value.systemdUnit, `exitedAgents[${index}].systemdUnit`, { max: 240 }),
+    terminalSession: cleanString(value.terminalSession, `exitedAgents[${index}].terminalSession`, { max: 240 }),
+    terminalAvailable: typeof value.terminalAvailable === "boolean" ? value.terminalAvailable : undefined,
+    pid: cleanSafeNumber(value.pid),
+    currentPid: cleanSafeNumber(value.currentPid),
+    startedAt: cleanSafeNumber(value.startedAt) ?? Date.now(),
+    detectedAt: cleanSafeNumber(value.detectedAt) ?? Date.now(),
+    runtimeMs: cleanSafeNumber(value.runtimeMs) ?? 0,
+    logFile: cleanString(value.logFile, `exitedAgents[${index}].logFile`, { max: 500 }),
+    logBytes: cleanSafeNumber(value.logBytes),
+    logEmpty: typeof value.logEmpty === "boolean" ? value.logEmpty : undefined,
+    logTail,
+    runnerInfoFile: cleanString(value.runnerInfoFile, `exitedAgents[${index}].runnerInfoFile`, { max: 500 }),
+    runnerInfoPresent: typeof value.runnerInfoPresent === "boolean" ? value.runnerInfoPresent : undefined,
+    systemd,
+    unavailable,
+    lastError: cleanString(value.lastError, `exitedAgents[${index}].lastError`, { required: true, max: 1000 })!,
+  };
+  if (JSON.stringify(diagnostics).length > 16_384) throw new ValidationError(`exitedAgents[${index}] is too large`);
+  return diagnostics;
+}
+
+const ORCH_UPGRADE_DEADLINE_MS = 5 * 60_000;
+
+const ORCH_UPGRADE_SEMVER_RE = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/;
+
+const VALID_ORCH_UPGRADE_PROVIDERS = ["auto", "all", "codex", "claude", "orchestrator"] as const;
+
+function semverCore(v: string): [number, number, number] | null {
+  const m = /^(\d+)\.(\d+)\.(\d+)/.exec(v);
+  return m ? [Number(m[1]), Number(m[2]), Number(m[3])] : null;
+}
+
+function compareSemverCore(a: string, b: string): number {
+  const ca = semverCore(a), cb = semverCore(b);
+  if (!ca || !cb) return 0;
+  for (let i = 0; i < 3; i++) if (ca[i] !== cb[i]) return ca[i]! - cb[i]!;
+  return 0;
+}
+
+function orchestratorSelfUpgradeError(orch: ReturnType<typeof getOrchestrator>): string | undefined {
+  if (!orch) return "orchestrator not found";
+  if (orch.status !== "online") return "orchestrator is offline";
+  if (!orch.capabilities?.relayCommandBus) {
+    return `orchestrator ${orch.id} does not support relay command bus upgrades; upgrade it manually first`;
+  }
+  if (!orch.capabilities?.selfUpgrade) {
+    return `orchestrator ${orch.id}${orch.version ? ` v${orch.version}` : ""} does not advertise self-upgrade support; upgrade it manually once to ${VERSION} or newer`;
+  }
+  if ((orch.supervisor !== "systemd" && orch.supervisor !== "launchd") || !orch.selfUnit) {
+    return `orchestrator ${orch.id} cannot self-upgrade under supervisor ${orch.supervisor ?? "unknown"}; upgrade it manually`;
+  }
+  return undefined;
+}
+
+function reconcileOrchestratorUpgrade(orch: ReturnType<typeof getOrchestrator>): void {
+  if (!orch) return;
+  const up = orch.upgrade;
+  if (!up) return;
+  const reported = orch.version;
+  if (reported && reported === up.desiredVersion && up.status === "succeeded" && up.error) {
+    const { error: _previousError, ...successState } = up;
+    setOrchestratorUpgradeState(orch.id, successState);
+    emitOrchestratorStatus(orch.id);
+    return;
+  }
+  if (reported && reported === up.desiredVersion && (up.status === "pending" || up.status === "failed")) {
+    if (up.commandId) updateCommand(up.commandId, { status: "succeeded", result: { version: reported, ...(up.fromVersion ? { fromVersion: up.fromVersion } : {}) } });
+    const { error: _previousError, ...successState } = up;
+    setOrchestratorUpgradeState(orch.id, { ...successState, status: "succeeded", settledAt: Date.now() });
+    auditEvent({
+      clientId: `server-orchestrator-upgraded-${orch.id}-${Date.now()}`,
+      kind: "state",
+      title: "Orchestrator upgraded",
+      body: `${up.fromVersion ?? "?"} → ${reported}`,
+      meta: orch.id,
+      icon: "ti-arrow-up-circle",
+      view: "orchestrators",
+      metadata: { orchestratorId: orch.id, version: reported, commandId: up.commandId },
+    });
+    emitOrchestratorStatus(orch.id);
+  } else if (up.status === "pending" && Date.now() - up.requestedAt > ORCH_UPGRADE_DEADLINE_MS) {
+    const errMsg = `upgrade timed out; orchestrator still reports ${reported ?? "unknown"} (wanted ${up.desiredVersion})`;
+    if (up.commandId) updateCommand(up.commandId, { status: "failed", error: errMsg });
+    setOrchestratorUpgradeState(orch.id, { ...up, status: "failed", settledAt: Date.now(), error: errMsg });
+    auditEvent({
+      clientId: `server-orchestrator-upgrade-failed-${orch.id}-${Date.now()}`,
+      kind: "state",
+      title: "Orchestrator upgrade failed",
+      body: errMsg,
+      meta: orch.id,
+      icon: "ti-alert-triangle",
+      view: "orchestrators",
+      metadata: { orchestratorId: orch.id, commandId: up.commandId },
+    });
+    emitOrchestratorStatus(orch.id);
+  }
+}
+
+export const postOrchestratorAction: Handler = async (req, params) => {
+  const parsed = await parseBody<unknown>(req);
+  if (!parsed.ok) return error(parsed.error, parsed.status);
+  try {
+    const orch = getOrchestrator(params.id!);
+    if (!orch) return error("orchestrator not found", 404);
+
+    if (!isRecord(parsed.body)) return error("body required");
+    const action = optionalEnum(parsed.body.action, "action", ["restart", "shutdown", "upgrade"] as const);
+    if (!action) return error("action required");
+
+    if (action === "upgrade") {
+      const denied = authorizeRoute(req, { scope: "command:write", resource: { orchestratorId: orch.id } });
+      if (denied) return denied;
+      const targetVersion = cleanString(parsed.body.targetVersion, "targetVersion", { max: 80 }) ?? VERSION;
+      if (!ORCH_UPGRADE_SEMVER_RE.test(targetVersion)) return error("targetVersion must be a semver like 0.10.20");
+      const force = parsed.body.force === true;
+      const providers = (cleanStringArray(parsed.body.providers, "providers", { itemMax: 80, maxItems: 50 }) ?? ["orchestrator"])
+        .filter((p) => (VALID_ORCH_UPGRADE_PROVIDERS as readonly string[]).includes(p));
+      if (!providers.length) return error(`providers must be any of: ${VALID_ORCH_UPGRADE_PROVIDERS.join(", ")}`);
+      const selfUpgradeError = orchestratorSelfUpgradeError(orch);
+      if (selfUpgradeError) return error(selfUpgradeError, 409);
+      if (!force && orch.version && compareSemverCore(targetVersion, orch.version) < 0) {
+        return error(`refusing downgrade ${orch.version} → ${targetVersion}; pass force to override`, 409);
+      }
+      const requestedAt = Date.now();
+      const command = createCommand({
+        type: "orchestrator.upgrade",
+        source: "system",
+        target: orch.agentId,
+        ttlMs: ORCH_UPGRADE_DEADLINE_MS + 5 * 60_000,
+        params: { targetVersion, providers, force, orchestratorId: orch.id, requestedBy: "dashboard", requestedAt },
+      });
+      setOrchestratorUpgradeState(orch.id, {
+        desiredVersion: targetVersion,
+        status: "pending",
+        commandId: command.id,
+        providers,
+        ...(orch.version ? { fromVersion: orch.version } : {}),
+        requestedBy: "dashboard",
+        requestedAt,
+      });
+      emitCommand(command);
+      emitOrchestratorStatus(orch.id);
+      auditEvent({
+        clientId: `server-orchestrator-upgrade-${orch.id}-${command.id}`,
+        kind: "state",
+        title: "Orchestrator upgrade requested",
+        body: `${orch.version ?? "?"} → ${targetVersion}`,
+        meta: orch.id,
+        icon: "ti-arrow-up-circle",
+        view: "orchestrators",
+        metadata: { orchestratorId: orch.id, targetVersion, providers, commandId: command.id, ...authAuditMetadata(req), ...dashboardAttribution(req, parsed.body.surface) },
+      });
+      return json({ ok: true, action, command, targetVersion }, 202);
+    }
+    const agentId = cleanString(parsed.body.agentId, "agentId", { max: 240 });
+    const policyName = cleanString(parsed.body.policyName, "policyName", { max: 120 });
+    const spawnRequestId = cleanString(parsed.body.spawnRequestId, "spawnRequestId", { max: 160 });
+    const sessionName = cleanString(parsed.body.sessionName, "sessionName", { max: 200 });
+    const tmuxSession = cleanString(parsed.body.tmuxSession, "tmuxSession", { max: 200 });
+    const reason = cleanString(parsed.body.reason, "reason", { max: 200 }) ?? "manual";
+    const denied = authorizeRoute(req, {
+      scope: "command:write",
+      resource: { orchestratorId: orch.id, agentId, policyName, spawnRequestId },
+    });
+    if (denied) return denied;
+
+    const command = createCommand({
+      type: action === "restart" ? "agent.restart" : "agent.shutdown",
+      source: "system",
+      target: orch.agentId,
+      correlationId: spawnRequestId,
+      params: { action, agentId, policyName, spawnRequestId, sessionName, tmuxSession, reason, requestedBy: "dashboard", requestedAt: Date.now(), orchestratorId: orch.id },
+    });
+    emitCommand(command);
+    auditEvent({
+      clientId: "server-orchestrator-action-" + orch.id + "-" + action + "-" + command.id,
+      kind: "state",
+      title: `Agent ${action} requested`,
+      body: agentId || "all",
+      meta: orch.id,
+      icon: action === "restart" ? "ti-refresh" : "ti-power",
+      view: "orchestrators",
+      metadata: { orchestratorId: orch.id, action, agentId, commandId: command.id, ...authAuditMetadata(req), ...dashboardAttribution(req, parsed.body.surface) },
+    });
+    return json({ ok: true, action, command }, 202);
+  } catch (e) {
+    if (e instanceof ValidationError) return error(e.message, 400);
+    throw e;
+  }
+};
+
+export const deleteOrchestratorById: Handler = (req, params) => {
+  const denied = authorizeRoute(req, { scope: "system:admin", resource: { orchestratorId: params.id! } });
+  if (denied) return denied;
+  const deleted = deleteOrchestrator(params.id!);
+  if (!deleted) return error("orchestrator not found", 404);
+  return json({ ok: true });
+};