@forgezero/agent 0.1.85 → 0.1.87

package/dist/agent-update-helper.d.ts

@@ -7,15 +7,21 @@ export declare const AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.jso
 /** Bounded, group-readable evidence consumed by the unprivileged Agent heartbeat. */
 export declare const AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update-receipt.json";
 export type AgentUpdateRequest = {
-    op: 'apply';
+    op: 'prepare';
     target: 'compute' | 'metal';
     attemptId?: string;
     currentVersion: string;
     release: AgentRelease;
+} | {
+    op: 'commit' | 'abort';
+    target: 'compute' | 'metal';
+    attemptId: string;
+    currentVersion: string;
+    targetVersion: string;
 };
 export type AgentUpdateResponse = {
     ok: true;
-    status: 'staged';
+    status: 'prepared' | 'active' | 'aborted';
     version: string;
     attemptId: string;
 } | {
@@ -53,6 +59,8 @@ export declare function activateAgentRelease(staged: StagedAgentRelease, options
     journalPath?: string;
     attemptId?: string;
     now?: () => number;
+    candidatePrepared?: boolean;
+    publicSocketPath?: string;
 }): Promise<{
     ok: true;
     version: string;
@@ -70,6 +78,7 @@ export declare function recoverInterruptedAgentUpdate(options?: {
     run?: (input: UpdateCommand) => Promise<UpdateCommandResult>;
     probe?: () => Promise<boolean>;
     now?: () => number;
+    publicSocketPath?: string;
 }): Promise<AgentUpdateReceipt | undefined>;
 export declare function startAgentUpdateHelper(options?: {
     socketPath?: string;

package/dist/agent-update-helper.js

@@ -16,6 +16,7 @@ import {
 } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
+var DEFAULT_AGENT_CANDIDATE_LINK = `${DEFAULT_AGENT_RELEASE_ROOT}/candidate`;
 var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
 var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
 var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
@@ -209,25 +210,90 @@ function restoreAgentRelease(staged) {
     rmSync(next, { force: true });
   }
 }
+function selectAgentCandidate(staged) {
+  const link = join(dirname(staged.currentLink), "candidate");
+  const next = `${link}.${randomUUID()}.next`;
+  try {
+    symlinkSync(staged.nextTarget, next);
+    renameSync(next, link);
+    syncPath(dirname(link));
+  } finally {
+    rmSync(next, { force: true });
+  }
+}
+function clearAgentCandidate(root = DEFAULT_AGENT_RELEASE_ROOT) {
+  rmSync(join(resolve(root), "candidate"), { force: true });
+}
 
 // src/agent-update-helper.ts
-import { randomUUID as randomUUID2 } from "node:crypto";
+import { randomUUID as randomUUID3 } from "node:crypto";
 import {
-  chmodSync as chmodSync2,
+  chmodSync as chmodSync3,
   closeSync as closeSync2,
-  existsSync as existsSync2,
+  existsSync as existsSync3,
   fsyncSync as fsyncSync2,
   mkdirSync as mkdirSync2,
   openSync as openSync2,
   readFileSync as readFileSync2,
-  renameSync as renameSync2,
-  rmSync as rmSync2,
-  unlinkSync,
+  renameSync as renameSync3,
+  rmSync as rmSync3,
+  unlinkSync as unlinkSync2,
   writeFileSync as writeFileSync2
 } from "node:fs";
-import { connect, createServer } from "node:net";
-import { dirname as dirname2, join as join2, resolve as resolve2 } from "node:path";
+import { connect as connect2, createServer as createServer2 } from "node:net";
+import { dirname as dirname3, join as join2, resolve as resolve2 } from "node:path";
 import { DEFAULT_SOCKET } from "@forgezero/vault";
+
+// src/agent-handover.ts
+import { randomUUID as randomUUID2 } from "node:crypto";
+import { chmodSync as chmodSync2, existsSync as existsSync2, renameSync as renameSync2, rmSync as rmSync2, symlinkSync as symlinkSync2, unlinkSync } from "node:fs";
+import { basename, dirname as dirname2 } from "node:path";
+import { connect, createServer } from "node:net";
+var DEFAULT_AGENT_CANDIDATE_READY_SOCKET = "/run/forgezero/candidate-ready.sock";
+function switchAgentSocketRoute(route, backend) {
+  if (dirname2(route) !== dirname2(backend))
+    throw new Error("Agent handover sockets must share one runtime directory");
+  const target = basename(backend);
+  const next = `${route}.${randomUUID2()}.next`;
+  try {
+    symlinkSync2(target, next);
+    renameSync2(next, route);
+  } finally {
+    rmSync2(next, { force: true });
+  }
+}
+function probeAgentCandidate(expected, socketPath = DEFAULT_AGENT_CANDIDATE_READY_SOCKET, timeoutMs = 5000) {
+  return new Promise((resolve2) => {
+    const socket = connect(socketPath);
+    let buffer = "";
+    let settled = false;
+    const finish = (value) => {
+      if (settled)
+        return;
+      settled = true;
+      clearTimeout(timer);
+      socket.destroy();
+      resolve2(value);
+    };
+    const timer = setTimeout(() => finish(false), timeoutMs);
+    socket.on("data", (chunk) => {
+      buffer += chunk.toString("utf8");
+      const newline = buffer.indexOf(`
+`);
+      if (newline < 0)
+        return;
+      try {
+        const value = JSON.parse(buffer.slice(0, newline));
+        finish(value.version === expected.version && typeof value.nodeKey === "string" && typeof value.bound === "boolean" && ["ready", "partial", "unavailable", "unbound"].includes(value.vault ?? ""));
+      } catch {
+        finish(false);
+      }
+    });
+    socket.on("error", () => finish(false));
+  });
+}
+
+// src/agent-update-helper.ts
 var AGENT_UPDATE_GROUP = "forgezero-update";
 var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
 var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
@@ -322,13 +388,13 @@ function validateJournal(value, root) {
   return journal;
 }
 function readJournal(path, root) {
-  if (!existsSync2(path))
+  if (!existsSync3(path))
     return;
   return validateJournal(JSON.parse(readFileSync2(path, "utf8")), root);
 }
 function writeAtomic(path, value, mode) {
-  mkdirSync2(dirname2(path), { recursive: true, mode: 493 });
-  const next = `${path}.${randomUUID2()}.next`;
+  mkdirSync2(dirname3(path), { recursive: true, mode: 493 });
+  const next = `${path}.${randomUUID3()}.next`;
   let file;
   try {
     file = openSync2(next, "wx", mode);
@@ -337,8 +403,8 @@ function writeAtomic(path, value, mode) {
     fsyncSync2(file);
     closeSync2(file);
     file = undefined;
-    renameSync2(next, path);
-    const directory = openSync2(dirname2(path), "r");
+    renameSync3(next, path);
+    const directory = openSync2(dirname3(path), "r");
     try {
       fsyncSync2(directory);
     } finally {
@@ -347,7 +413,7 @@ function writeAtomic(path, value, mode) {
   } finally {
     if (file !== undefined)
       closeSync2(file);
-    rmSync2(next, { force: true });
+    rmSync3(next, { force: true });
   }
 }
 var publicReceipt = (journal) => {
@@ -380,7 +446,7 @@ function writeUpdateState(journalPath, receiptPath, journal) {
 }
 function readAgentUpdateReceipt(path = AGENT_UPDATE_RECEIPT) {
   try {
-    if (!existsSync2(path))
+    if (!existsSync3(path))
       return;
     return validateReceipt(JSON.parse(readFileSync2(path, "utf8")));
   } catch {
@@ -420,10 +486,32 @@ var restartAgent = async (target, run) => {
     throw new Error(`systemd could not restart ${service}`);
   }
 };
+var candidateUnit = (target) => target === "compute" ? "forgezero-agent-candidate.service" : "forgezero-metal-agent-candidate.service";
+var startCandidate = async (staged, target, run, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
+  selectAgentCandidate(staged);
+  if (!await runOk(run, "/usr/bin/systemctl", ["restart", candidateUnit(target)])) {
+    clearAgentCandidate(dirname3(staged.currentLink));
+    throw new Error(`systemd could not start ${candidateUnit(target)}`);
+  }
+  if (!await probe({ version: staged.version })) {
+    await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
+    clearAgentCandidate(dirname3(staged.currentLink));
+    throw new Error("the candidate Agent did not prove its identity and durable state");
+  }
+};
+var stopCandidate = async (staged, target, run) => {
+  await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
+  clearAgentCandidate(dirname3(staged.currentLink));
+};
+var socketPaths = (publicSocket = process.env.FZ_AGENT_PUBLIC_SOCKET ?? DEFAULT_SOCKET) => ({
+  route: `${publicSocket}.backend`,
+  active: `${publicSocket}.backend.active`,
+  candidate: `${publicSocket}.backend.candidate`
+});
 var targetProbe = (target, run) => target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]);
 function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
   return new Promise((resolve3) => {
-    const socket = connect(socketPath);
+    const socket = connect2(socketPath);
     let settled = false;
     let buffer = "";
     const finish = (value) => {
@@ -456,12 +544,13 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
 async function activateAgentRelease(staged, options = {}) {
   const run = options.run ?? runCommand;
   const target = options.target ?? "compute";
-  const probe = options.probe ?? targetProbe(target, run);
+  const paths = socketPaths(options.publicSocketPath);
+  const probe = options.probe ?? (target === "compute" ? () => probeAgentSocket(paths.active) : targetProbe(target, run));
   const now = options.now ?? Date.now;
   const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
   const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
-  const previous = readJournal(journalPath, dirname2(staged.currentLink));
-  const attemptId = options.attemptId ?? randomUUID2();
+  const previous = readJournal(journalPath, dirname3(staged.currentLink));
+  const attemptId = options.attemptId ?? randomUUID3();
   if (!ATTEMPT_ID.test(attemptId))
     throw new Error("Agent update attempt ID is invalid");
   const startedAtTs = now();
@@ -483,11 +572,18 @@ async function activateAgentRelease(staged, options = {}) {
   writeUpdateState(journalPath, receiptPath, journal);
   let selectionAttempted = false;
   try {
+    if (!options.candidatePrepared)
+      await startCandidate(staged, target, run);
+    if (target === "compute")
+      switchAgentSocketRoute(paths.route, paths.candidate);
     selectionAttempted = true;
     selectAgentRelease(staged);
     await restartAgent(target, run);
     if (!await probe())
-      throw new Error("the replacement Agent did not answer its retained Vault socket");
+      throw new Error("the replacement Agent did not answer its active Vault backend");
+    if (target === "compute")
+      switchAgentSocketRoute(paths.route, paths.active);
+    await stopCandidate(staged, target, run);
     journal = { ...journal, outcome: "active", updatedAtTs: now(), rollbackHealthy: undefined };
     writeUpdateState(journalPath, receiptPath, journal);
     run({
@@ -505,10 +601,13 @@ async function activateAgentRelease(staged, options = {}) {
         restored = true;
         await restartAgent(target, run);
         rollbackHealthy = await probe();
+        if (rollbackHealthy && target === "compute")
+          switchAgentSocketRoute(paths.route, paths.active);
       } catch {
         rollbackHealthy = false;
       }
     }
+    await stopCandidate(staged, target, run).catch(() => {});
     const failures = failureCount + 1;
     const updatedAtTs = now();
     journal = {
@@ -528,28 +627,38 @@ async function recoverInterruptedAgentUpdate(options = {}) {
   const root = resolve2(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
   const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
   const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const run = options.run ?? runCommand;
   const journal = readJournal(journalPath, root);
-  if (!journal)
+  if (!journal) {
+    for (const target of ["compute", "metal"]) {
+      await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
+    }
+    clearAgentCandidate(root);
     return;
+  }
   if (journal.outcome !== "activating") {
+    await stopCandidate(stagedFromJournal(journal, root), journal.target, run).catch(() => {});
     writeAtomic(receiptPath, publicReceipt(journal), 416);
     return publicReceipt(journal);
   }
   const staged = stagedFromJournal(journal, root);
-  if (!existsSync2(join2(root, journal.previousTarget))) {
+  if (!existsSync3(join2(root, journal.previousTarget))) {
     throw new Error("Agent update rollback release is missing");
   }
-  const run = options.run ?? runCommand;
-  const probe = options.probe ?? targetProbe(journal.target, run);
+  const paths = socketPaths(options.publicSocketPath);
+  const probe = options.probe ?? (journal.target === "compute" ? () => probeAgentSocket(paths.active) : targetProbe(journal.target, run));
   restoreAgentRelease(staged);
   let rollbackHealthy = false;
   let failureMessage = "activation was interrupted before its health verdict became durable";
   try {
     await restartAgent(journal.target, run);
     rollbackHealthy = await probe();
+    if (rollbackHealthy && journal.target === "compute")
+      switchAgentSocketRoute(paths.route, paths.active);
   } catch (cause) {
     failureMessage = `${failureMessage}; ${cause instanceof Error ? cause.message : String(cause)}`;
   }
+  await stopCandidate(staged, journal.target, run).catch(() => {});
   const failures = journal.failureCount + 1;
   const updatedAtTs = (options.now ?? Date.now)();
   const recovered = {
@@ -566,15 +675,23 @@ async function recoverInterruptedAgentUpdate(options = {}) {
 }
 function startAgentUpdateHelper(options = {}) {
   const socketPath = options.socketPath ?? DEFAULT_AGENT_UPDATE_SOCKET;
-  if (existsSync2(socketPath))
-    unlinkSync(socketPath);
-  mkdirSync2(dirname2(socketPath), { recursive: true, mode: 488 });
-  const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
+  if (existsSync3(socketPath))
+    unlinkSync2(socketPath);
+  mkdirSync2(dirname3(socketPath), { recursive: true, mode: 488 });
   const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
   const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
   const releaseRoot = options.root ?? DEFAULT_AGENT_RELEASE_ROOT;
-  const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, { target, attemptId, journalPath, receiptPath, now: options.now }));
+  const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
+  const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, {
+    target,
+    attemptId,
+    journalPath,
+    receiptPath,
+    now: options.now,
+    candidatePrepared: true
+  }));
   let busy = true;
+  let pending;
   let blocked;
   (options.recover ?? (() => recoverInterruptedAgentUpdate({
     root: releaseRoot,
@@ -586,7 +703,7 @@ function startAgentUpdateHelper(options = {}) {
   }).finally(() => {
     busy = false;
   });
-  const server = createServer((socket) => {
+  const server = createServer2((socket) => {
     let buffer = "";
     socket.on("data", (chunk) => {
       buffer += chunk.toString("utf8");
@@ -605,16 +722,49 @@ function startAgentUpdateHelper(options = {}) {
       Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
         if (blocked)
           throw new Error(`update journal needs operator recovery: ${blocked}`);
-        if (busy)
-          throw new Error("another Agent update or recovery is already active");
-        if (request.op !== "apply")
-          throw new Error("unknown update operation");
         if (request.target !== "compute" && request.target !== "metal") {
           throw new Error("agent update target is invalid");
         }
-        const attemptId = request.attemptId ?? randomUUID2();
+        const attemptId = request.attemptId ?? randomUUID3();
         if (!ATTEMPT_ID.test(attemptId))
           throw new Error("Agent update attempt ID is invalid");
+        if (request.op === "commit" || request.op === "abort") {
+          if (!pending || pending.attemptId !== attemptId || pending.target !== request.target || pending.staged.fromVersion !== request.currentVersion || pending.staged.version !== request.targetVersion) {
+            throw new Error("Agent update handover does not match the prepared candidate");
+          }
+          const selected = pending;
+          pending = undefined;
+          if (request.op === "abort") {
+            await stopCandidate(selected.staged, selected.target, runCommand);
+            busy = false;
+            const response3 = {
+              ok: true,
+              status: "aborted",
+              version: selected.staged.version,
+              attemptId
+            };
+            socket.end(`${JSON.stringify(response3)}
+`);
+            return;
+          }
+          const outcome = await activate(selected.staged, selected.target, attemptId);
+          busy = false;
+          if (outcome?.ok === false)
+            throw new Error(outcome.reason ?? "Agent handover failed");
+          const response2 = {
+            ok: true,
+            status: "active",
+            version: selected.staged.version,
+            attemptId
+          };
+          socket.end(`${JSON.stringify(response2)}
+`);
+          return;
+        }
+        if (request.op !== "prepare")
+          throw new Error("unknown update operation");
+        if (busy)
+          throw new Error("another Agent update or recovery is already active");
         const prior = readJournal(journalPath, releaseRoot);
         const now = (options.now ?? Date.now)();
         if (prior?.targetVersion === request.release.version && (prior.outcome === "rolled-back" || prior.outcome === "failed") && (prior.retryAfterTs ?? 0) > now)
@@ -625,17 +775,23 @@ function startAgentUpdateHelper(options = {}) {
           currentVersion: request.currentVersion,
           root: releaseRoot
         });
-        const response = { ok: true, status: "staged", version: staged.version, attemptId };
+        await startCandidate(staged, request.target, runCommand);
+        pending = { staged, target: request.target, attemptId };
+        ownsBusy = false;
+        setTimer(() => {
+          if (pending?.attemptId !== attemptId)
+            return;
+          const expired = pending;
+          pending = undefined;
+          stopCandidate(expired.staged, expired.target, runCommand).finally(() => {
+            busy = false;
+          });
+        }, 180000);
+        const response = { ok: true, status: "prepared", version: staged.version, attemptId };
         socket.end(`${JSON.stringify(response)}
 `);
-        setTimer(() => void activate(staged, request.target, attemptId).catch((cause) => {
-          blocked = cause instanceof Error ? cause.message : String(cause);
-        }).finally(() => {
-          busy = false;
-        }), 100);
-        ownsBusy = false;
       }).catch((cause) => {
-        if (ownsBusy)
+        if (ownsBusy && !pending)
           busy = false;
         const response = {
           ok: false,
@@ -647,12 +803,12 @@ function startAgentUpdateHelper(options = {}) {
     });
     socket.on("error", () => socket.destroy());
   });
-  server.listen(socketPath, () => chmodSync2(socketPath, 432));
+  server.listen(socketPath, () => chmodSync3(socketPath, 432));
   return server;
 }
 function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, timeoutMs = 90000) {
   return new Promise((resolve3, reject) => {
-    const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
+    const socket = connect2(socketPath, () => socket.write(`${JSON.stringify(request)}
 `));
     let buffer = "";
     socket.setTimeout(timeoutMs, () => {

package/dist/agent-update.d.ts

@@ -23,6 +23,7 @@ export interface StagedAgentRelease {
     currentLink: string;
 }
 export declare const DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
+export declare const DEFAULT_AGENT_CANDIDATE_LINK = "/opt/forgezero/agent/candidate";
 export declare const DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
 export declare const MAX_AGENT_TARBALL_BYTES: number;
 /** Root helper input is data, never a command or filesystem path. */
@@ -45,3 +46,6 @@ export declare function stageAgentRelease(releaseInput: AgentRelease, options: {
 export declare function selectAgentRelease(staged: StagedAgentRelease): void;
 /** Roll back only to the exact link target captured before activation. */
 export declare function restoreAgentRelease(staged: StagedAgentRelease): void;
+/** Select a verified candidate without changing the active generation. */
+export declare function selectAgentCandidate(staged: StagedAgentRelease): void;
+export declare function clearAgentCandidate(root?: string): void;

package/dist/agent-update.js

@@ -16,6 +16,7 @@ import {
 } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
+var DEFAULT_AGENT_CANDIDATE_LINK = `${DEFAULT_AGENT_RELEASE_ROOT}/candidate`;
 var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
 var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
 var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
@@ -209,13 +210,30 @@ function restoreAgentRelease(staged) {
     rmSync(next, { force: true });
   }
 }
+function selectAgentCandidate(staged) {
+  const link = join(dirname(staged.currentLink), "candidate");
+  const next = `${link}.${randomUUID()}.next`;
+  try {
+    symlinkSync(staged.nextTarget, next);
+    renameSync(next, link);
+    syncPath(dirname(link));
+  } finally {
+    rmSync(next, { force: true });
+  }
+}
+function clearAgentCandidate(root = DEFAULT_AGENT_RELEASE_ROOT) {
+  rmSync(join(resolve(root), "candidate"), { force: true });
+}
 export {
   validateAgentRelease,
   stageAgentRelease,
   selectAgentRelease,
+  selectAgentCandidate,
   restoreAgentRelease,
   compareVersions,
+  clearAgentCandidate,
   MAX_AGENT_TARBALL_BYTES,
   DEFAULT_AGENT_UPDATE_SOCKET,
-  DEFAULT_AGENT_RELEASE_ROOT
+  DEFAULT_AGENT_RELEASE_ROOT,
+  DEFAULT_AGENT_CANDIDATE_LINK
 };