agent-relay-orchestrator 0.62.2 → 0.63.0

package/src/spawn/spawn-agent.ts

@@ -0,0 +1,94 @@
+import { closeSync, existsSync, openSync, rmSync } from "node:fs";
+import type { OrchestratorConfig } from "../config";
+import { resolveSpawnWorkspace, workspacesRoot } from "../workspace-probe";
+import type { ManagedAgentReport } from "../relay";
+import { buildEnv, buildRunnerCommand, defaultSpawnLabel, isWithinBaseDir, sessionName } from "./command";
+import { addSessionRecord, ensureLogDir, ensureRunnerInfoDir, logFilePath, runnerInfoPath, sessionReportFields } from "./runtime";
+import { managedAgentId } from "./sessions";
+import { spawnRunner } from "./supervisor";
+import type { SpawnOptions } from "./types";
+
+export async function spawnAgent(
+  opts: SpawnOptions,
+  config: OrchestratorConfig,
+): Promise<ManagedAgentReport> {
+  const label = opts.label || defaultSpawnLabel();
+  const agentId = opts.agentId || managedAgentId(config, opts.provider, label);
+  const name = sessionName(config, opts.provider, label, opts.spawnRequestId ?? agentId);
+
+  if (!existsSync(opts.cwd)) {
+    throw new Error(`cwd does not exist: ${opts.cwd}`);
+  }
+  if (!isWithinBaseDir(opts.cwd, config.baseDir)) {
+    throw new Error(`cwd must be within base directory: ${config.baseDir}`);
+  }
+
+  const resolvedWorkspace = await resolveSpawnWorkspace({
+    ...opts,
+    label,
+    workspaceSymlinks: opts.workspaceSymlinks,
+    workspaceRoot: workspacesRoot(config.baseDir),
+  });
+  const spawnOpts = { ...opts, label, agentId, cwd: resolvedWorkspace.cwd, workspace: resolvedWorkspace.workspace };
+
+  const command = buildRunnerCommand(spawnOpts, config);
+
+  ensureLogDir();
+  ensureRunnerInfoDir();
+  const logFile = logFilePath(name);
+  const runnerInfoFile = runnerInfoPath(name);
+  rmSync(runnerInfoFile, { force: true });
+  const env = buildEnv({ ...spawnOpts, env: { ...(spawnOpts.env ?? {}), AGENT_RELAY_RUNNER_INFO_FILE: runnerInfoFile } }, config, logFile, name);
+  const logFd = openSync(logFile, "w");
+
+  console.error(`[orchestrator] Spawning ${opts.provider} agent: ${name}`);
+  console.error(`[orchestrator]   cwd: ${opts.cwd}`);
+  console.error(`[orchestrator]   command: ${command.join(" ")}`);
+  console.error(`[orchestrator]   log: ${logFile}`);
+
+  closeSync(logFd);
+
+  const runner = spawnRunner(name, command, spawnOpts.cwd, env, logFile);
+
+  addSessionRecord({
+    name,
+    pid: runner.pid,
+    supervisor: runner.supervisor,
+    provider: spawnOpts.provider,
+    model: spawnOpts.model,
+    effort: spawnOpts.effort,
+    profile: spawnOpts.profile,
+    workspaceMode: spawnOpts.workspaceMode, lifecycle: spawnOpts.lifecycle ?? "persistent",
+    workspace: spawnOpts.workspace,
+    label,
+    cwd: spawnOpts.cwd,
+    logFile,
+    runnerInfoFile,
+    agentId,
+    approvalMode: spawnOpts.approvalMode,
+    policyName: spawnOpts.policyName,
+    spawnRequestId: spawnOpts.spawnRequestId,
+    automationId: spawnOpts.automationId,
+    automationRunId: spawnOpts.automationRunId,
+    startedAt: Date.now(),
+  });
+
+  return {
+    agentId,
+    provider: spawnOpts.provider,
+    model: spawnOpts.model,
+    effort: spawnOpts.effort,
+    profile: spawnOpts.profile,
+    workspaceMode: spawnOpts.workspaceMode, lifecycle: spawnOpts.lifecycle ?? "persistent",
+    workspace: spawnOpts.workspace,
+    ...sessionReportFields({ name, supervisor: runner.supervisor, runnerInfoFile, agentId, provider: spawnOpts.provider }),
+    cwd: spawnOpts.cwd,
+    label,
+    approvalMode: spawnOpts.approvalMode || "guarded",
+    policyName: spawnOpts.policyName,
+    spawnRequestId: spawnOpts.spawnRequestId,
+    automationRunId: spawnOpts.automationRunId,
+    pid: runner.pid,
+    startedAt: Date.now(),
+  };
+}

package/src/spawn/supervisor.ts

@@ -0,0 +1,347 @@
+import { chmodSync, closeSync, existsSync, openSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { OrchestratorConfig } from "../config";
+import type { ManagedSessionExitDiagnostics } from "../relay";
+import { errMessage, extractClaudeModelUnavailableMessage } from "agent-relay-sdk";
+import { isPidAlive } from "agent-relay-sdk/process-utils";
+import { shellEscape } from "agent-relay-sdk/shell-utils";
+import { tmuxHasSession } from "agent-relay-sdk/tmux-utils";
+import { sanitizeFsName } from "agent-relay-sdk/fs-name";
+import { SESSION_DIR } from "./constants";
+import { logLines } from "./log-utils";
+import { currentSessionPid, ensureSessionDir, findSessionRecord, isSessionRecordAlive, loadState, logFilePath, readRunnerInfo, removeSessionRecord, sessionSupervisor } from "./runtime";
+import { systemdMainPid, systemdUnitName } from "./systemd";
+import type { SessionRecord, SessionSupervisor, SpawnedRunner } from "./types";
+
+export function spawnRunner(name: string, command: string[], cwd: string, env: Record<string, string>, logFile: string): SpawnedRunner {
+  if (shouldUseSystemdSupervisor()) {
+    try {
+      return spawnSystemdRunner(name, command, cwd, env, logFile);
+    } catch (error) {
+      console.error(`[orchestrator] systemd runner supervisor unavailable for ${name}: ${errMessage(error)}`);
+      console.error("[orchestrator] Falling back to process child; this agent will not survive orchestrator service restart.");
+    }
+  }
+
+  const launchScript = launchScriptPath(name);
+  ensureSessionDir();
+  writeFileSync(launchScript, buildLaunchScript(command, cwd, env), { mode: 0o700 });
+  chmodSync(launchScript, 0o700);
+
+  const logFd = openSync(logFile, "a");
+  try {
+    const proc = Bun.spawn([launchScript], {
+      cwd,
+      env,
+      stdin: "ignore",
+      stdout: logFd,
+      stderr: logFd,
+    });
+    return { pid: proc.pid, supervisor: { type: "process", launchScript } };
+  } finally {
+    closeSync(logFd);
+  }
+}
+
+function shouldUseSystemdSupervisor(): boolean {
+  if (process.platform !== "linux") return false;
+  if (process.env.AGENT_RELAY_DISABLE_SYSTEMD_SUPERVISOR === "1") return false;
+  if (process.env.AGENT_RELAY_FORCE_SYSTEMD_SUPERVISOR === "1") return true;
+  const result = Bun.spawnSync(["systemctl", "--user", "show-environment"], {
+    stdin: "ignore",
+    stdout: "ignore",
+    stderr: "ignore",
+  });
+  return result.exitCode === 0;
+}
+
+function spawnSystemdRunner(name: string, command: string[], cwd: string, env: Record<string, string>, logFile: string): SpawnedRunner {
+  const unit = systemdUnitName(name);
+  const launchScript = launchScriptPath(name);
+  ensureSessionDir();
+  writeFileSync(launchScript, buildLaunchScript(command, cwd, env), { mode: 0o700 });
+  chmodSync(launchScript, 0o700);
+
+  Bun.spawnSync(["systemctl", "--user", "stop", `${unit}.service`], {
+    stdin: "ignore",
+    stdout: "ignore",
+    stderr: "ignore",
+  });
+
+  const result = Bun.spawnSync([
+    "systemd-run",
+    "--user",
+    `--unit=${unit}`,
+    "--collect",
+    "--property=KillMode=control-group",
+    `--property=StandardOutput=append:${logFile}`,
+    `--property=StandardError=append:${logFile}`,
+    launchScript,
+  ], {
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  if (result.exitCode !== 0) {
+    const stderr = result.stderr.toString().trim();
+    throw new Error(stderr || `systemd-run failed with exit code ${result.exitCode}`);
+  }
+
+  const pid = waitForSystemdMainPid(unit, 2_000);
+  if (!pid) throw new Error(`systemd unit ${unit}.service started without a MainPID`);
+  return { pid, supervisor: { type: "systemd", unit, launchScript } };
+}
+
+function launchScriptPath(session: string): string {
+  const safe = sanitizeFsName(session, { replacement: "-", trimEdge: true, fallback: "agent" });
+  return join(SESSION_DIR, `${safe}.sh`);
+}
+
+export function buildLaunchScript(command: string[], cwd: string, env: Record<string, string>): string {
+  const exports = Object.entries(env)
+    .filter(([key, value]) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(key) && value !== undefined)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([key, value]) => `export ${key}=${shellEscape(String(value))}`);
+  return [
+    "#!/usr/bin/env bash",
+    "set -euo pipefail",
+    ...exports,
+    `cd ${shellEscape(cwd)}`,
+    `exec ${command.map(shellEscape).join(" ")}`,
+    "",
+  ].join("\n");
+}
+
+function waitForSystemdMainPid(unit: string, timeoutMs: number): number {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const pid = systemdMainPid(unit);
+    if (pid > 0 && isPidAlive(pid)) return pid;
+    Bun.sleepSync(50);
+  }
+  return 0;
+}
+
+function systemdUnitDiagnostics(unit: string): NonNullable<ManagedSessionExitDiagnostics["systemd"]> {
+  const result = Bun.spawnSync([
+    "systemctl", "--user", "show", `${unit}.service`,
+    "-p", "ActiveState",
+    "-p", "SubState",
+    "-p", "Result",
+    "-p", "ExecMainCode",
+    "-p", "ExecMainStatus",
+    "-p", "MainPID",
+  ], {
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  if (result.exitCode !== 0) {
+    return {
+      unit,
+      unavailable: result.stderr.toString().trim() || `systemctl show exited with ${result.exitCode}`,
+    };
+  }
+  const props = new Map<string, string>();
+  for (const line of result.stdout.toString().split("\n")) {
+    const index = line.indexOf("=");
+    if (index <= 0) continue;
+    props.set(line.slice(0, index), line.slice(index + 1));
+  }
+  const mainPid = Number(props.get("MainPID"));
+  return {
+    unit,
+    activeState: props.get("ActiveState") || undefined,
+    subState: props.get("SubState") || undefined,
+    result: props.get("Result") || undefined,
+    execMainCode: props.get("ExecMainCode") || undefined,
+    execMainStatus: props.get("ExecMainStatus") || undefined,
+    mainPid: Number.isFinite(mainPid) && mainPid > 0 ? mainPid : undefined,
+  };
+}
+
+function logFileDiagnostics(logFile: string): Pick<ManagedSessionExitDiagnostics, "logBytes" | "logEmpty" | "logTail"> & { logUnavailable?: string } {
+  try {
+    const stat = statSync(logFile);
+    if (stat.size === 0) return { logBytes: 0, logEmpty: true, logTail: [] };
+    const content = readFileSync(logFile, "utf8");
+    return {
+      logBytes: stat.size,
+      logEmpty: false,
+      logTail: logLines(content).slice(-20),
+    };
+  } catch (error) {
+    return {
+      logUnavailable: errMessage(error),
+    };
+  }
+}
+
+function describeSessionExit(record: SessionRecord, diagnostics: Omit<ManagedSessionExitDiagnostics, "lastError">): string {
+  if (record.provider === "claude") {
+    const modelUnavailable = extractClaudeModelUnavailableMessage((diagnostics.logTail ?? []).join("\n"));
+    if (modelUnavailable) return modelUnavailable;
+  }
+  const seconds = Math.max(0, Math.round(diagnostics.runtimeMs / 1000));
+  const parts = [`managed ${record.provider} session ${record.name} exited after ${seconds}s`];
+  if (diagnostics.systemd?.unavailable) {
+    parts.push(`systemd status unavailable: ${diagnostics.systemd.unavailable}`);
+  } else if (diagnostics.systemd) {
+    const state = [diagnostics.systemd.activeState, diagnostics.systemd.subState].filter(Boolean).join("/") || "unknown";
+    const result = diagnostics.systemd.result || "unknown";
+    const exit = [diagnostics.systemd.execMainCode, diagnostics.systemd.execMainStatus].filter(Boolean).join("/") || "unknown";
+    parts.push(`systemd ${diagnostics.systemd.unit}.service state=${state} result=${result} exit=${exit}`);
+  }
+  if (diagnostics.logEmpty) {
+    parts.push("stdout/stderr log is empty");
+  } else if (diagnostics.logBytes === undefined) {
+    parts.push("stdout/stderr log unavailable");
+  }
+  if (!diagnostics.runnerInfoPresent) parts.push("runner info was not written");
+  return parts.join("; ");
+}
+
+export function diagnoseSessionExit(input: { agentId?: string; policyName?: string; spawnRequestId?: string; tmuxSession?: string }): ManagedSessionExitDiagnostics | null {
+  const record = findSessionRecord(input);
+  if (!record) return null;
+  const detectedAt = Date.now();
+  const supervisor = sessionSupervisor(record);
+  const currentPid = currentSessionPid(record);
+  const terminalAvailable = tmuxHasSession(record.name, readRunnerInfo(record)?.tmuxSocket);
+  const log = logFileDiagnostics(record.logFile);
+  const runnerInfoPresent = record.runnerInfoFile ? existsSync(record.runnerInfoFile) : false;
+  const unavailable = [
+    ...(log.logUnavailable ? [`stdout/stderr log unavailable: ${log.logUnavailable}`] : []),
+    ...(log.logEmpty ? ["stdout/stderr log empty"] : []),
+    ...(!runnerInfoPresent ? ["runner info unavailable"] : []),
+  ];
+  const base: Omit<ManagedSessionExitDiagnostics, "lastError"> = {
+    agentId: record.agentId,
+    provider: record.provider as "claude" | "codex",
+    workspaceMode: record.workspaceMode,
+    workspace: record.workspace ?? (record.workspaceMode ? { mode: "shared", requestedMode: record.workspaceMode } : undefined),
+    sessionName: record.name,
+    tmuxSession: record.name,
+    cwd: record.cwd,
+    label: record.label,
+    policyName: record.policyName,
+    spawnRequestId: record.spawnRequestId,
+    automationRunId: record.automationRunId,
+    supervisor: supervisor.type,
+    ...(supervisor.type === "systemd" && supervisor.unit ? { systemdUnit: supervisor.unit } : {}),
+    terminalSession: record.name,
+    terminalAvailable,
+    pid: record.pid,
+    currentPid,
+    startedAt: record.startedAt,
+    detectedAt,
+    runtimeMs: Math.max(0, detectedAt - record.startedAt),
+    logFile: record.logFile,
+    logBytes: log.logBytes,
+    logEmpty: log.logEmpty,
+    logTail: log.logTail,
+    runnerInfoFile: record.runnerInfoFile,
+    runnerInfoPresent,
+    ...(supervisor.type === "systemd" && supervisor.unit ? { systemd: systemdUnitDiagnostics(supervisor.unit) } : {}),
+    ...(unavailable.length ? { unavailable } : {}),
+  };
+  return {
+    ...base,
+    lastError: describeSessionExit(record, base),
+  };
+}
+
+function stopSystemdUnit(unit: string): void {
+  Bun.spawnSync(["systemctl", "--user", "stop", `${unit}.service`], {
+    stdin: "ignore",
+    stdout: "ignore",
+    stderr: "ignore",
+  });
+}
+
+function killSystemdUnit(unit: string): void {
+  Bun.spawnSync(["systemctl", "--user", "kill", "--kill-whom=all", "--signal=SIGKILL", `${unit}.service`], {
+    stdin: "ignore",
+    stdout: "ignore",
+    stderr: "ignore",
+  });
+}
+
+function cleanupSupervisor(supervisor: SessionSupervisor): void {
+  if (supervisor.type === "systemd" && supervisor.unit) stopSystemdUnit(supervisor.unit);
+  if (supervisor.launchScript) rmSync(supervisor.launchScript, { force: true });
+}
+
+export function cleanupSessionRecord(record: SessionRecord): void {
+  cleanupSupervisor(sessionSupervisor(record));
+  if (record.runnerInfoFile) rmSync(record.runnerInfoFile, { force: true });
+}
+
+export async function stopSession(name: string, config: OrchestratorConfig, reason: string, graceful = true, timeoutMs?: number): Promise<{ stopped: boolean; wasRunning: boolean }> {
+  if (!name.startsWith(`${config.tmuxPrefix}-`)) throw new Error("session is not managed by this orchestrator");
+
+  const records = loadState();
+  const record = records.find((r) => r.name === name);
+  if (!record || !isSessionRecordAlive(record)) {
+    if (record) cleanupSessionRecord(record);
+    removeSessionRecord(name);
+    return { stopped: false, wasRunning: false };
+  }
+
+  const pid = currentSessionPid(record);
+  console.error(`[orchestrator] Stopping session ${name} (pid ${pid}): ${reason}`);
+
+  const supervisor = sessionSupervisor(record);
+  const gracefulTimeoutMs = sessionStopTimeoutMs(graceful, timeoutMs);
+  if (supervisor.type === "systemd" && supervisor.unit) {
+    stopSystemdUnit(supervisor.unit);
+    const deadline = Date.now() + gracefulTimeoutMs;
+    while (Date.now() < deadline && isSessionRecordAlive(record)) {
+      await Bun.sleep(200);
+    }
+    if (isSessionRecordAlive(record)) {
+      killSystemdUnit(supervisor.unit);
+      const killDeadline = Date.now() + 2_000;
+      while (Date.now() < killDeadline && isSessionRecordAlive(record)) {
+        await Bun.sleep(100);
+      }
+    }
+    if (isSessionRecordAlive(record)) return { stopped: false, wasRunning: true };
+    cleanupSessionRecord(record);
+    removeSessionRecord(name);
+    return { stopped: true, wasRunning: true };
+  }
+
+  if (graceful) {
+    try { process.kill(pid, "SIGTERM"); } catch {}
+    const deadline = Date.now() + gracefulTimeoutMs;
+    while (Date.now() < deadline && isPidAlive(pid)) {
+      await Bun.sleep(200);
+    }
+  }
+
+  if (isPidAlive(pid)) {
+    try { process.kill(pid, "SIGKILL"); } catch {}
+    const deadline = Date.now() + 2_000;
+    while (Date.now() < deadline && isPidAlive(pid)) {
+      await Bun.sleep(100);
+    }
+  }
+
+  // Never report success while the process is still alive: deleting the session
+  // record here would orphan a running process with no handle to stop it again.
+  if (isPidAlive(pid)) {
+    console.error(`[orchestrator] Session ${name} (pid ${pid}) survived SIGKILL; keeping record for retry`);
+    return { stopped: false, wasRunning: true };
+  }
+
+  cleanupSessionRecord(record);
+  removeSessionRecord(name);
+  return { stopped: true, wasRunning: true };
+}
+
+function sessionStopTimeoutMs(graceful: boolean, timeoutMs?: number): number {
+  if (!graceful) return 2_000;
+  if (!Number.isSafeInteger(timeoutMs) || !timeoutMs || timeoutMs <= 0) return 10_000;
+  return Math.min(timeoutMs, 60_000);
+}

package/src/spawn/systemd.ts

@@ -0,0 +1,17 @@
+import { sanitizeFsName } from "agent-relay-sdk/fs-name";
+
+export function systemdUnitName(session: string): string {
+  const safe = sanitizeFsName(session, { replacement: "-", trimEdge: true, fallback: "agent" });
+  return `agent-relay-managed-${safe}`.slice(0, 180);
+}
+
+export function systemdMainPid(unit: string): number {
+  const result = Bun.spawnSync(["systemctl", "--user", "show", `${unit}.service`, "-p", "MainPID", "--value"], {
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "ignore",
+  });
+  if (result.exitCode !== 0) return 0;
+  const pid = Number(result.stdout.toString().trim());
+  return Number.isFinite(pid) ? pid : 0;
+}