agent-relay-orchestrator 0.10.19 → 0.10.21

package/src/relay.ts

@@ -1,10 +1,13 @@
 import type { OrchestratorConfig } from "./config";
-import { GIT_SHA, ORCHESTRATOR_PROTOCOL_VERSION, VERSION } from "./version";
+import type { ProviderProbeCache } from "./provider-probe";
+import { detectSelfSupervision } from "./self-supervision";
+import { GIT_SHA, ORCHESTRATOR_PROTOCOL_VERSION, VERSION, runtimeMetadata } from "./version";
+import type { WorkspaceMetadata, WorkspaceMode } from "agent-relay-sdk";
 
 export interface RelayClient {
   register(): Promise<void>;
   heartbeat(): Promise<void>;
-  updateManagedAgents(agents: ManagedAgentReport[]): Promise<void>;
+  updateManagedAgents(agents: ManagedAgentReport[], exitedAgents?: ManagedSessionExitDiagnostics[]): Promise<void>;
   pollCommands(): Promise<RelayCommand[]>;
   updateCommand(commandId: string, status: string, result?: Record<string, unknown>, error?: string): Promise<boolean>;
   setApiUrl(url: string): void;
@@ -16,16 +19,68 @@ export interface RelayClient {
 export interface ManagedAgentReport {
   agentId: string;
   provider: "claude" | "codex";
+  model?: string;
+  effort?: string;
+  profile?: string;
+  workspaceMode?: WorkspaceMode;
+  workspace?: WorkspaceMetadata;
+  sessionName?: string;
+  supervisor?: "process" | "systemd" | "unknown";
+  systemdUnit?: string;
+  terminalSession?: string;
+  terminalAvailable?: boolean;
   tmuxSession: string;
   cwd: string;
   label?: string;
   approvalMode: string;
   policyName?: string;
   spawnRequestId?: string;
+  automationRunId?: string;
   pid?: number;
   startedAt: number;
 }
 
+export interface ManagedSessionExitDiagnostics {
+  agentId: string;
+  provider: "claude" | "codex";
+  workspaceMode?: WorkspaceMode;
+  workspace?: WorkspaceMetadata;
+  sessionName?: string;
+  tmuxSession: string;
+  cwd: string;
+  label?: string;
+  policyName?: string;
+  spawnRequestId?: string;
+  automationRunId?: string;
+  supervisor: "process" | "systemd" | "unknown";
+  systemdUnit?: string;
+  terminalSession?: string;
+  terminalAvailable?: boolean;
+  pid?: number;
+  currentPid?: number;
+  startedAt: number;
+  detectedAt: number;
+  runtimeMs: number;
+  logFile?: string;
+  logBytes?: number;
+  logEmpty?: boolean;
+  logTail?: string[];
+  runnerInfoFile?: string;
+  runnerInfoPresent?: boolean;
+  systemd?: {
+    unit: string;
+    activeState?: string;
+    subState?: string;
+    result?: string;
+    execMainCode?: string;
+    execMainStatus?: string;
+    mainPid?: number;
+    unavailable?: string;
+  };
+  unavailable?: string[];
+  lastError: string;
+}
+
 export interface RelayCommand {
   id: string;
   type: string;
@@ -39,17 +94,40 @@ const BACKOFF_SCHEDULE_MS = [
 ];
 const MAX_BACKOFF_MS = 3_600_000; // 1 hour
 
-export function createRelayClient(config: OrchestratorConfig): RelayClient {
+export function buildRegistrationMeta(
+  config: Pick<OrchestratorConfig, "tmuxPrefix">,
+  runtime: ReturnType<typeof runtimeMetadata>,
+  now = Date.now,
+  pid = process.pid,
+): Record<string, unknown> {
+  const supervision = detectSelfSupervision();
+  return {
+    ...runtime,
+    pid,
+    tmuxPrefix: config.tmuxPrefix,
+    startedAt: now(),
+    version: VERSION,
+    protocolVersion: ORCHESTRATOR_PROTOCOL_VERSION,
+    gitSha: GIT_SHA,
+    supervisor: supervision.supervisor,
+    ...(supervision.selfUnit ? { selfUnit: supervision.selfUnit } : {}),
+    ...(supervision.runtimePrefix ? { runtimePrefix: supervision.runtimePrefix } : {}),
+  };
+}
+
+export function createRelayClient(config: OrchestratorConfig, probeCache: ProviderProbeCache): RelayClient {
   const agentId = `orchestrator-${config.id}`;
   let heartbeatTimer: Timer | null = null;
   let connected = false;
   let backoffIndex = 0;
   let cursorFloor = 0;
   let apiUrl: string | undefined;
+  let runtimeToken: string | undefined;
 
   function headers(): Record<string, string> {
     const h: Record<string, string> = { "Content-Type": "application/json" };
-    if (config.token) h["X-Agent-Relay-Token"] = config.token;
+    const token = runtimeToken ?? config.token;
+    if (token) h["X-Agent-Relay-Token"] = token;
     return h;
   }
 
@@ -64,29 +142,31 @@ export function createRelayClient(config: OrchestratorConfig): RelayClient {
   }
 
   async function register(): Promise<void> {
+    const runtime = runtimeMetadata();
+    const providerSnapshot = await probeCache.getSnapshot(true);
     const res = await apiCall("POST", "/orchestrators", {
       id: config.id,
       hostname: config.hostname,
-      providers: config.providers,
+      providers: providerSnapshot.providers,
+      providerStatus: providerSnapshot.providerStatus,
+      providerCatalog: providerSnapshot.providerCatalog,
       baseDir: config.baseDir,
       apiUrl,
       envKeys: Object.keys(config.env),
+      package: runtime.package,
+      contracts: runtime.contracts,
+      capabilities: runtime.capabilities,
       version: VERSION,
       protocolVersion: ORCHESTRATOR_PROTOCOL_VERSION,
       gitSha: GIT_SHA,
-      meta: {
-        pid: process.pid,
-        tmuxPrefix: config.tmuxPrefix,
-        startedAt: Date.now(),
-        version: VERSION,
-        protocolVersion: ORCHESTRATOR_PROTOCOL_VERSION,
-        gitSha: GIT_SHA,
-      },
+      meta: buildRegistrationMeta(config, runtime),
     });
     if (!res.ok) {
       const err = await res.text();
       throw new Error(`Failed to register orchestrator: ${res.status} ${err}`);
     }
+    const registered = await res.json().catch(() => null) as { runtimeToken?: { token?: string } } | null;
+    if (registered?.runtimeToken?.token) runtimeToken = registered.runtimeToken.token;
     connected = true;
     backoffIndex = 0;
 
@@ -102,10 +182,18 @@ export function createRelayClient(config: OrchestratorConfig): RelayClient {
 
   async function heartbeat(): Promise<void> {
     try {
+      const runtime = runtimeMetadata();
+      const providerSnapshot = await probeCache.getSnapshot();
       const res = await apiCall("POST", `/orchestrators/${config.id}/heartbeat`, {
+        package: runtime.package,
+        contracts: runtime.contracts,
+        capabilities: runtime.capabilities,
         version: VERSION,
         protocolVersion: ORCHESTRATOR_PROTOCOL_VERSION,
         gitSha: GIT_SHA,
+        providers: providerSnapshot.providers,
+        providerStatus: providerSnapshot.providerStatus,
+        providerCatalog: providerSnapshot.providerCatalog,
       });
       if (!res.ok) throw new Error(`heartbeat failed: ${res.status}`);
       if (!connected) {
@@ -136,8 +224,11 @@ export function createRelayClient(config: OrchestratorConfig): RelayClient {
     }
   }
 
-  async function updateManagedAgents(agents: ManagedAgentReport[]): Promise<void> {
-    await apiCall("PATCH", `/orchestrators/${config.id}/agents`, { agents });
+  async function updateManagedAgents(agents: ManagedAgentReport[], exitedAgents: ManagedSessionExitDiagnostics[] = []): Promise<void> {
+    await apiCall("PATCH", `/orchestrators/${config.id}/agents`, {
+      agents,
+      ...(exitedAgents.length ? { exitedAgents } : {}),
+    });
   }
 
   async function pollCommands(): Promise<RelayCommand[]> {

package/src/self-supervision.ts

@@ -0,0 +1,82 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+export interface SelfSupervision {
+  supervisor: "process" | "systemd" | "unknown";
+  selfUnit?: string;
+  runtimePrefix?: string;
+}
+
+let cached: SelfSupervision | undefined;
+
+/**
+ * Detect how this orchestrator process is supervised so the relay can target a
+ * remote self-upgrade at the correct systemd unit and install prefix. Result is
+ * stable for the process lifetime, so it is computed once and cached.
+ */
+export function detectSelfSupervision(moduleUrl: string = import.meta.url): SelfSupervision {
+  if (cached) return cached;
+  cached = { supervisor: detectSupervisorRaw(), runtimePrefix: detectRuntimePrefix(moduleUrl) };
+  const unit = detectSystemdUnit();
+  if (unit) {
+    cached.supervisor = "systemd";
+    cached.selfUnit = unit;
+  }
+  return cached;
+}
+
+/** Reset the cache. Test-only. */
+export function resetSelfSupervisionCache(): void {
+  cached = undefined;
+}
+
+function detectSupervisorRaw(): SelfSupervision["supervisor"] {
+  // /proc only exists on Linux; on macOS (launchd) we can't introspect cheaply.
+  try {
+    readFileSync("/proc/self/cgroup", "utf8");
+    return "process";
+  } catch {
+    return "unknown";
+  }
+}
+
+/**
+ * Parse the systemd unit owning this process from /proc/self/cgroup. A
+ * --user service cgroup looks like:
+ *   0::/user.slice/user-1000.slice/user@1000.service/app.slice/agent-relay-orchestrator.service
+ * We want the LAST `*.service` segment that isn't the user manager itself.
+ */
+export function parseSystemdUnitFromCgroup(cgroup: string): string | undefined {
+  const services = cgroup
+    .split("\n")
+    .flatMap((line) => line.split("/"))
+    .filter((seg) => seg.endsWith(".service"))
+    .filter((seg) => !/^user@\d+\.service$/.test(seg) && seg !== "init.scope");
+  const last = services.at(-1);
+  return last && last.length > ".service".length ? last : undefined;
+}
+
+function detectSystemdUnit(): string | undefined {
+  try {
+    return parseSystemdUnitFromCgroup(readFileSync("/proc/self/cgroup", "utf8"));
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * The install prefix is the directory above `node_modules` when the orchestrator
+ * runs from an installed package (e.g. ~/.agent-relay/runtime). Undefined when
+ * running from a source/workspace checkout.
+ */
+export function detectRuntimePrefix(moduleUrl: string): string | undefined {
+  let path: string;
+  try {
+    path = fileURLToPath(moduleUrl);
+  } catch {
+    return undefined;
+  }
+  const marker = "/node_modules/";
+  const idx = path.indexOf(marker);
+  return idx >= 0 ? path.slice(0, idx) : undefined;
+}

package/src/self-upgrade.ts

@@ -0,0 +1,143 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import type { OrchestratorConfig } from "./config";
+import type { RelayClient, RelayCommand } from "./relay";
+import { detectSelfSupervision } from "./self-supervision";
+
+const VALID_PROVIDERS = new Set(["auto", "all", "codex", "claude", "orchestrator"]);
+const SEMVER_RE = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/;
+
+export interface SelfUpgradeRunner {
+  run(cmd: string[]): Promise<{ exitCode: number; stdout: string; stderr: string }>;
+  commandExists(name: string): boolean;
+}
+
+const defaultRunner: SelfUpgradeRunner = {
+  async run(cmd) {
+    const proc = Bun.spawn({ cmd, stdout: "pipe", stderr: "pipe" });
+    const [stdout, stderr] = await Promise.all([
+      new Response(proc.stdout).text(),
+      new Response(proc.stderr).text(),
+    ]);
+    const exitCode = await proc.exited;
+    return { exitCode, stdout, stderr };
+  },
+  commandExists(name) {
+    try {
+      return Bun.spawnSync({ cmd: ["which", name], stdout: "ignore", stderr: "ignore" }).exitCode === 0;
+    } catch {
+      return false;
+    }
+  },
+};
+
+export interface SelfUpgradePlan {
+  targetVersion: string;
+  providers: string[];
+  unit: string;
+  runtimePrefix?: string;
+  binary: string;
+  installCmd: string[];
+  restartCmd: string[];
+  /** restart runs decoupled from this process's cgroup (transient unit) */
+  restartDetached: boolean;
+}
+
+/**
+ * Build the upgrade plan from a command's params + detected self-supervision.
+ * Throws (with an operator-facing message) when the request is invalid or the
+ * host can't self-restart — caught by the caller and reported as a failed command.
+ */
+export function planSelfUpgrade(
+  params: Record<string, unknown>,
+  supervision = detectSelfSupervision(),
+  runner: SelfUpgradeRunner = defaultRunner,
+): SelfUpgradePlan {
+  const targetVersion = typeof params.targetVersion === "string" ? params.targetVersion.trim() : "";
+  if (!SEMVER_RE.test(targetVersion)) {
+    throw new Error(`invalid targetVersion "${targetVersion}" (expected x.y.z)`);
+  }
+  const providers = normalizeProviders(params.providers);
+
+  if (supervision.supervisor !== "systemd" || !supervision.selfUnit) {
+    throw new Error("orchestrator is not under systemd --user; remote self-upgrade requires a systemd unit (P1)");
+  }
+  const unit = supervision.selfUnit;
+  const binary = resolveBinary(supervision.runtimePrefix);
+
+  const installCmd = [
+    binary, "upgrade",
+    "--version", targetVersion,
+    "--providers", providers.join(","),
+    "--no-restart",
+    "--yes",
+  ];
+  if (supervision.runtimePrefix) installCmd.push("--runtime-prefix", supervision.runtimePrefix);
+
+  // Decouple the restart from this orchestrator's own cgroup: restarting our unit
+  // SIGTERMs us, and a child in our cgroup would be killed mid-restart. systemd-run
+  // schedules it as an independent transient unit that survives our teardown.
+  const restartDetached = runner.commandExists("systemd-run");
+  const restartCmd = restartDetached
+    ? ["systemd-run", "--user", "--collect", "--description", "agent-relay orchestrator self-upgrade restart", "systemctl", "--user", "restart", unit]
+    : ["setsid", "systemctl", "--user", "restart", unit];
+
+  return { targetVersion, providers, unit, runtimePrefix: supervision.runtimePrefix, binary, installCmd, restartCmd, restartDetached };
+}
+
+/**
+ * Handle an orchestrator.upgrade command: validate, install the target version
+ * WITHOUT restart (so install failures are caught while we're still alive), then
+ * launch a decoupled restart. The command is intentionally left "running" — the
+ * relay settles it by reconciling the version we report after we come back up.
+ */
+export async function handleSelfUpgrade(
+  command: RelayCommand,
+  _config: OrchestratorConfig,
+  relay: RelayClient,
+  runner: SelfUpgradeRunner = defaultRunner,
+): Promise<void> {
+  const plan = planSelfUpgrade(command.params, detectSelfSupervision(), runner);
+  await relay.updateCommand(command.id, "running", {
+    phase: "installing",
+    targetVersion: plan.targetVersion,
+    providers: plan.providers,
+    unit: plan.unit,
+  });
+
+  const install = await runner.run(plan.installCmd);
+  if (install.exitCode !== 0) {
+    throw new Error(`install failed (exit ${install.exitCode}): ${(install.stderr || install.stdout).trim().slice(-500)}`);
+  }
+
+  await relay.updateCommand(command.id, "running", {
+    phase: "restart-pending",
+    targetVersion: plan.targetVersion,
+    unit: plan.unit,
+    restartDetached: plan.restartDetached,
+  });
+
+  // Fire the restart and return. We do not await its effect — it tears us down.
+  const restart = await runner.run(plan.restartCmd);
+  if (restart.exitCode !== 0) {
+    throw new Error(`restart failed (exit ${restart.exitCode}): ${(restart.stderr || restart.stdout).trim().slice(-500)}`);
+  }
+  console.error(`[orchestrator] self-upgrade to ${plan.targetVersion} installed; restart dispatched for ${plan.unit}`);
+}
+
+function normalizeProviders(value: unknown): string[] {
+  const list = Array.isArray(value)
+    ? value.filter((v): v is string => typeof v === "string").map((v) => v.trim()).filter(Boolean)
+    : [];
+  const filtered = list.filter((p) => VALID_PROVIDERS.has(p));
+  const providers = filtered.length ? filtered : ["orchestrator"];
+  return [...new Set(providers)];
+}
+
+function resolveBinary(runtimePrefix?: string): string {
+  if (runtimePrefix) {
+    const local = join(runtimePrefix, "node_modules", ".bin", "agent-relay");
+    if (existsSync(local)) return local;
+  }
+  return "agent-relay";
+}