agent-relay-orchestrator 0.129.6 → 0.129.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.129.6",
+  "version": "0.129.7",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -17,7 +17,7 @@
   },
   "dependencies": {
     "agent-relay-providers": "0.104.4",
-    "agent-relay-sdk": "0.2.125",
+    "agent-relay-sdk": "0.2.126",
     "callmux": "0.23.0"
   },
   "devDependencies": {

package/src/config.ts

@@ -53,8 +53,17 @@ export function providerHomeRootFromEnv(): string {
   return process.env.AGENT_RELAY_PROVIDER_HOME_ROOT || join(homedir(), ".agent-relay", "provider-homes");
 }
 
+// An ephemeral/test instance advertises AGENT_RELAY_EPHEMERAL=1 (see
+// src/execution-mode.ts). Mirrored here so the orchestrator package stays
+// dependency-free of the server package.
+export function isEphemeralMode(): boolean {
+  return process.env.AGENT_RELAY_EPHEMERAL === "1";
+}
+
 export function disableSystemdSupervisor(): boolean {
-  return process.env.AGENT_RELAY_DISABLE_SYSTEMD_SUPERVISOR === "1";
+  // Force process (never systemd) supervision in ephemeral mode so an ephemeral
+  // boot can never create/adopt host `agent-relay-*` systemd units (#1280).
+  return process.env.AGENT_RELAY_DISABLE_SYSTEMD_SUPERVISOR === "1" || isEphemeralMode();
 }
 
 export function forceSystemdSupervisor(): boolean {

package/src/relay.ts

@@ -1,4 +1,6 @@
 import type { OrchestratorConfig } from "./config";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
 import type { ProviderProbeCache } from "./provider-probe";
 import { detectSelfSupervision } from "./self-supervision";
 import { GIT_SHA, ORCHESTRATOR_PROTOCOL_VERSION, VERSION, runtimeMetadata } from "./version";
@@ -83,6 +85,7 @@ export function buildRegistrationMeta(
   pid = process.pid,
 ): Record<string, unknown> {
   const supervision = detectSelfSupervision();
+  const artifactSha = supervision.runtimePrefix ? readArtifactSha(supervision.runtimePrefix) : undefined;
   return {
     ...runtime,
     pid,
@@ -90,13 +93,27 @@ export function buildRegistrationMeta(
     startedAt: now(),
     version: VERSION,
     protocolVersion: ORCHESTRATOR_PROTOCOL_VERSION,
-    gitSha: GIT_SHA,
+    gitSha: artifactSha ?? GIT_SHA,
     supervisor: supervision.supervisor,
     ...(supervision.selfUnit ? { selfUnit: supervision.selfUnit } : {}),
     ...(supervision.runtimePrefix ? { runtimePrefix: supervision.runtimePrefix } : {}),
   };
 }
 
+function readArtifactSha(runtimePrefix: string): string | undefined {
+  try {
+    const sha = readFileSync(join(runtimePrefix, ".agent-relay-artifact-sha"), "utf8").trim();
+    return /^[0-9a-f]{40}$/i.test(sha) ? sha : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function reportedGitSha(): string | undefined {
+  const runtimePrefix = detectSelfSupervision().runtimePrefix;
+  return (runtimePrefix ? readArtifactSha(runtimePrefix) : undefined) ?? GIT_SHA;
+}
+
 export function createRelayClient(config: OrchestratorConfig, probeCache: ProviderProbeCache): RelayClient {
   const agentId = `orchestrator-${config.id}`;
   let heartbeatTimer: Timer | null = null;
@@ -132,7 +149,7 @@ export function createRelayClient(config: OrchestratorConfig, probeCache: Provid
       capabilities: runtime.capabilities,
       version: VERSION,
       protocolVersion: ORCHESTRATOR_PROTOCOL_VERSION,
-      gitSha: GIT_SHA,
+      gitSha: reportedGitSha(),
       meta: buildRegistrationMeta(config, runtime),
     });
     if (!res.ok) {
@@ -165,7 +182,7 @@ export function createRelayClient(config: OrchestratorConfig, probeCache: Provid
         capabilities: runtime.capabilities,
         version: VERSION,
         protocolVersion: ORCHESTRATOR_PROTOCOL_VERSION,
-        gitSha: GIT_SHA,
+        gitSha: reportedGitSha(),
         providers: providerSnapshot.providers,
         providerStatus: providerSnapshot.providerStatus,
         providerCatalog: providerSnapshot.providerCatalog,

package/src/self-upgrade.ts

@@ -2,7 +2,9 @@ import { join } from "node:path";
 import { existsSync } from "node:fs";
 import { homedir } from "node:os";
 import { getAllManifests } from "agent-relay-providers";
+import { gitCheckoutArtifactPaths, prepareGitCheckoutArtifact, rollbackRuntimeSymlink, swapRuntimeSymlink, type GitCheckoutArtifact } from "agent-relay-sdk/git-checkout-artifact";
 import { runInstallWithRetry } from "agent-relay-sdk";
+import { shellQuote } from "agent-relay-sdk/shell-utils";
 import type { OrchestratorConfig } from "./config";
 import type { RelayClient, RelayCommand } from "./relay";
 import { detectSelfSupervision, type SelfSupervision } from "./self-supervision";
@@ -22,14 +24,19 @@ interface SelfUpgradeOptions {
 }
 
 export interface SelfUpgradeRunner {
-  run(cmd: string[]): Promise<{ exitCode: number; stdout: string; stderr: string }>;
+  run(cmd: string[], cwd?: string): Promise<{ exitCode: number; stdout: string; stderr: string }>;
   commandExists(name: string): boolean;
   fileExists?(path: string): boolean;
 }
 
+function localReadinessUrl(apiPort: number | undefined = 4860): string {
+  const port = Number.isInteger(apiPort) && apiPort! > 0 && apiPort! <= 65_535 ? apiPort : 4860;
+  return `http://127.0.0.1:${port}`;
+}
+
 const defaultRunner: SelfUpgradeRunner = {
-  async run(cmd) {
-    const proc = Bun.spawn({ cmd, stdout: "pipe", stderr: "pipe" });
+  async run(cmd, cwd) {
+    const proc = Bun.spawn({ cmd, cwd, stdout: "pipe", stderr: "pipe" });
     const [stdout, stderr] = await Promise.all([
       new Response(proc.stdout).text(),
       new Response(proc.stderr).text(),
@@ -58,6 +65,7 @@ interface SelfUpgradePlan {
   restartCmd: string[];
   /** restart runs decoupled from this process's cgroup (transient unit) */
   restartDetached: boolean;
+  artifact?: GitCheckoutArtifact;
 }
 
 /**
@@ -69,18 +77,29 @@ export function planSelfUpgrade(
   params: Record<string, unknown>,
   supervision = detectSelfSupervision(),
   runner: SelfUpgradeRunner = defaultRunner,
+  readinessUrl = localReadinessUrl(),
 ): SelfUpgradePlan {
   const targetVersion = typeof params.targetVersion === "string" ? params.targetVersion.trim() : "";
   if (!SEMVER_RE.test(targetVersion)) {
     throw new Error(`invalid targetVersion "${targetVersion}" (expected x.y.z)`);
   }
   const providers = normalizeProviders(params.providers);
+  const mode = params.mode === "git-checkout" ? "git-checkout" : "npm-reuse";
 
   if ((supervision.supervisor !== "systemd" && supervision.supervisor !== "launchd") || !supervision.selfUnit) {
     throw new Error("orchestrator is not under systemd or launchd; remote self-upgrade requires a managed service");
   }
   const unit = supervision.selfUnit;
-  const installCmd = buildInstallCommand(targetVersion, providers, supervision, runner);
+  const installCmd = mode === "git-checkout"
+    ? []
+    : buildInstallCommand(targetVersion, providers, supervision, runner);
+  const artifact = mode === "git-checkout"
+    ? gitCheckoutArtifactPaths(
+      homedir(),
+      requiredCommitSha(params.commitSha),
+      requiredRepoUrl(params.repoUrl),
+    )
+    : undefined;
 
   let restartDetached: boolean;
   let restartCmd: string[];
@@ -89,19 +108,29 @@ export function planSelfUpgrade(
     const uid = process.getuid?.() ?? 501;
     // Shell-background the kickstart so it survives our process teardown when
     // launchd kills us. The shell exits immediately; launchd handles the restart.
-    restartCmd = ["/bin/sh", "-c", `launchctl kickstart -kp gui/${uid}/${unit} &`];
+    const artifactGuard = artifact
+      ? launchdArtifactRestartGuard(uid, unit, supervision.runtimePrefix ?? join(homedir(), ".agent-relay", "runtime"), readinessUrl)
+      : `launchctl kickstart -kp gui/${uid}/${unit}`;
+    restartCmd = ["/bin/sh", "-c", `${artifactGuard} &`];
     restartDetached = true;
   } else {
     // Decouple the restart from this orchestrator's own cgroup: restarting our unit
     // SIGTERMs us, and a child in our cgroup would be killed mid-restart. systemd-run
     // schedules it as an independent transient unit that survives our teardown.
     restartDetached = runner.commandExists("systemd-run");
+    const artifactGuard = artifact
+      ? artifactRestartGuard(unit, supervision.runtimePrefix ?? join(homedir(), ".agent-relay", "runtime"), readinessUrl)
+      : undefined;
     restartCmd = restartDetached
-      ? ["systemd-run", "--user", "--collect", "--description", "agent-relay orchestrator self-upgrade restart", "systemctl", "--user", "restart", unit]
-      : ["setsid", "systemctl", "--user", "restart", unit];
+      ? artifactGuard
+        ? ["systemd-run", "--user", "--collect", "--description", "agent-relay git-checkout self-upgrade restart", "/bin/sh", "-c", artifactGuard]
+        : ["systemd-run", "--user", "--collect", "--description", "agent-relay orchestrator self-upgrade restart", "systemctl", "--user", "restart", unit]
+      : artifactGuard
+        ? ["setsid", "/bin/sh", "-c", artifactGuard]
+        : ["setsid", "systemctl", "--user", "restart", unit];
   }
 
-  return { targetVersion, providers, unit, runtimePrefix: supervision.runtimePrefix, installCmd, restartCmd, restartDetached };
+  return { targetVersion, providers, unit, runtimePrefix: supervision.runtimePrefix, installCmd, restartCmd, restartDetached, artifact };
 }
 
 /**
@@ -112,19 +141,43 @@ export function planSelfUpgrade(
  */
 export async function handleSelfUpgrade(
   command: RelayCommand,
-  _config: OrchestratorConfig,
+  config: OrchestratorConfig,
   relay: RelayClient,
   runner: SelfUpgradeRunner = defaultRunner,
   opts: SelfUpgradeOptions = {},
 ): Promise<void> {
-  const plan = planSelfUpgrade(command.params, opts.supervision ?? detectSelfSupervision(), runner);
+  const plan = planSelfUpgrade(command.params, opts.supervision ?? detectSelfSupervision(), runner, localReadinessUrl(config.apiPort));
   await relay.updateCommand(command.id, "running", {
     phase: "installing",
     targetVersion: plan.targetVersion,
     providers: plan.providers,
     unit: plan.unit,
+    ...(plan.artifact ? { mode: "git-checkout", commitSha: plan.artifact.sha } : {}),
   });
 
+  if (plan.artifact) {
+    await prepareGitCheckoutArtifact(plan.artifact, runner.run.bind(runner));
+    const runtimePath = plan.runtimePrefix ?? join(homedir(), ".agent-relay", "runtime");
+    const swap = swapRuntimeSymlink(runtimePath, plan.artifact.checkoutDir);
+    try {
+      const restart = await runner.run(plan.restartCmd);
+      if (restart.exitCode !== 0) throw new Error(`restart failed (exit ${restart.exitCode}): ${(restart.stderr || restart.stdout).trim().slice(-500)}`);
+    } catch (error) {
+      rollbackRuntimeSymlink(swap);
+      throw error;
+    }
+    await relay.updateCommand(command.id, "running", {
+      phase: "restart-pending",
+      targetVersion: plan.targetVersion,
+      unit: plan.unit,
+      restartDetached: plan.restartDetached,
+      mode: "git-checkout",
+      commitSha: plan.artifact.sha,
+    });
+    console.error(`[orchestrator] git-checkout self-upgrade to ${plan.artifact.sha} prepared; restart dispatched for ${plan.unit}`);
+    return;
+  }
+
   const install = await runInstallWithRetry(plan.installCmd, runner.run.bind(runner), {
     sleep: opts.sleep,
     retries: opts.installRetries,
@@ -160,6 +213,57 @@ export async function handleSelfUpgrade(
   console.error(`[orchestrator] self-upgrade to ${plan.targetVersion} installed; restart dispatched for ${plan.unit}`);
 }
 
+function requiredCommitSha(value: unknown): string {
+  if (typeof value !== "string" || !/^[0-9a-f]{40}$/i.test(value.trim())) throw new Error("git-checkout mode requires a 40-character commitSha");
+  return value.trim();
+}
+
+function requiredRepoUrl(value: unknown): string {
+  if (typeof value !== "string" || !value.trim()) throw new Error("git-checkout mode requires repoUrl");
+  return value.trim();
+}
+
+/** Detached supervisor-side guard: active service keeps the new pointer; a failed restart restores it. */
+function readinessGuard(baseUrl: string): string {
+  const pattern = '"status"[[:space:]]*:[[:space:]]*"ok"';
+  return [
+    "ready=0",
+    "for attempt in $(seq 1 30); do",
+    `if curl -fsS --max-time 5 ${shellQuote(`${baseUrl}/api/health`)} | grep -Eq ${shellQuote(pattern)}; then ready=$((ready + 1)); else ready=0; fi`,
+    '[ "$ready" -ge 2 ] && break',
+    "sleep 2",
+    "done",
+    '[ "$ready" -ge 2 ]',
+  ].join("; ");
+}
+
+function artifactRestartGuard(unit: string, runtimePath: string, readinessUrl: string): string {
+  const previous = `${runtimePath}.previous`;
+  const legacy = `${runtimePath}.legacy`;
+  return [
+    `systemctl --user restart ${shellQuote(unit)}`,
+    // On success drop the rollback pointer AND the relocated original runtime (.legacy),
+    // which swapRuntimeSymlink parks aside only until the new runtime proves healthy.
+    // On failure mv -T is a single rename over the live symlink — .previous is always a
+    // symlink now, so a prior real DIRECTORY rolls back without ENOTDIR (#1287 Bug B) —
+    // and a rm-then-mv rollback would have a fatal no-runtime-pointer window.
+    `if ${readinessGuard(readinessUrl)}; then rm -rf ${shellQuote(previous)} ${shellQuote(legacy)}; else mv -fT ${shellQuote(previous)} ${shellQuote(runtimePath)}; systemctl --user restart ${shellQuote(unit)}; exit 1; fi`,
+  ].join("; ");
+}
+
+function launchdArtifactRestartGuard(uid: number, unit: string, runtimePath: string, readinessUrl: string): string {
+  const previous = `${runtimePath}.previous`;
+  const legacy = `${runtimePath}.legacy`;
+  const service = `gui/${uid}/${unit}`;
+  return [
+    `launchctl kickstart -kp ${shellQuote(service)}`,
+    // BSD mv follows a destination symlink to a directory unless -h is present. .previous
+    // is always a symlink, so -fh replaces runtimePath itself and preserves the atomic
+    // rollback. Do not unlink runtimePath first or a guard death can strand launchd.
+    `if ${readinessGuard(readinessUrl)}; then rm -rf ${shellQuote(previous)} ${shellQuote(legacy)}; else mv -fh ${shellQuote(previous)} ${shellQuote(runtimePath)}; launchctl kickstart -kp ${shellQuote(service)}; exit 1; fi`,
+  ].join("; ");
+}
+
 function normalizeProviders(value: unknown): string[] {
   const list = Array.isArray(value)
     ? value.filter((v): v is string => typeof v === "string").map((v) => v.trim()).filter(Boolean)