@forgezero/agent 0.1.31 → 0.1.32

package/dist/agent-update-helper.d.ts

@@ -2,10 +2,14 @@ import { type Server } from 'node:net';
 import { type AgentRelease, type StagedAgentRelease, type UpdateCommand, type UpdateCommandResult } from './agent-update';
 export declare const AGENT_UPDATE_GROUP = "forgezero-update";
 export declare const AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
-export declare const AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update.json";
+/** Root-private crash transaction. Existing 0.1.31 hosts may contain its legacy success receipt here. */
+export declare const AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
+/** Bounded, group-readable evidence consumed by the unprivileged Agent heartbeat. */
+export declare const AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update-receipt.json";
 export type AgentUpdateRequest = {
     op: 'apply';
     target: 'compute' | 'metal';
+    attemptId?: string;
     currentVersion: string;
     release: AgentRelease;
 };
@@ -13,6 +17,7 @@ export type AgentUpdateResponse = {
     ok: true;
     status: 'staged';
     version: string;
+    attemptId: string;
 } | {
     ok: false;
     error: {
@@ -20,6 +25,24 @@ export type AgentUpdateResponse = {
         message: string;
     };
 };
+export type AgentUpdateOutcome = 'activating' | 'active' | 'rolled-back' | 'failed';
+/** Bounded, non-secret update evidence included in the next signed heartbeat. */
+export interface AgentUpdateReceipt {
+    attemptId: string;
+    fromVersion: string;
+    targetVersion: string;
+    outcome: AgentUpdateOutcome;
+    startedAtTs: number;
+    updatedAtTs: number;
+    retryAfterTs?: number;
+    rollbackHealthy?: boolean;
+    reason?: {
+        code: string;
+        message: string;
+    };
+}
+/** Read only the bounded evidence safe to send to the control plane. */
+export declare function readAgentUpdateReceipt(path?: string): AgentUpdateReceipt | undefined;
 /** Prove a newly started Agent process answers, not merely that PID 1 holds its socket. */
 export declare function probeAgentSocket(socketPath?: string, timeoutMs?: number): Promise<boolean>;
 export declare function activateAgentRelease(staged: StagedAgentRelease, options?: {
@@ -27,6 +50,8 @@ export declare function activateAgentRelease(staged: StagedAgentRelease, options
     run?: (input: UpdateCommand) => Promise<UpdateCommandResult>;
     probe?: () => Promise<boolean>;
     receiptPath?: string;
+    journalPath?: string;
+    attemptId?: string;
     now?: () => number;
 }): Promise<{
     ok: true;
@@ -34,12 +59,26 @@ export declare function activateAgentRelease(staged: StagedAgentRelease, options
 } | {
     ok: false;
     rolledBack: boolean;
+    rollbackHealthy: boolean;
     reason: string;
 }>;
+/** Restore a transaction that lost power or the helper before its health verdict became durable. */
+export declare function recoverInterruptedAgentUpdate(options?: {
+    root?: string;
+    receiptPath?: string;
+    journalPath?: string;
+    run?: (input: UpdateCommand) => Promise<UpdateCommandResult>;
+    probe?: () => Promise<boolean>;
+    now?: () => number;
+}): Promise<AgentUpdateReceipt | undefined>;
 export declare function startAgentUpdateHelper(options?: {
     socketPath?: string;
     root?: string;
-    activate?: (staged: StagedAgentRelease, target: AgentUpdateRequest['target']) => Promise<unknown>;
+    activate?: (staged: StagedAgentRelease, target: AgentUpdateRequest['target'], attemptId: string) => Promise<unknown>;
+    receiptPath?: string;
+    journalPath?: string;
+    recover?: () => Promise<unknown>;
+    now?: () => number;
     setTimer?: (callback: () => void, ms: number) => unknown;
 }): Server;
 export declare function requestAgentUpdate(request: AgentUpdateRequest, socketPath?: string, timeoutMs?: number): Promise<AgentUpdateResponse>;

package/dist/agent-update-helper.js

@@ -2,8 +2,11 @@
 import { createHash, timingSafeEqual, randomUUID } from "node:crypto";
 import {
   chmodSync,
+  closeSync,
   existsSync,
+  fsyncSync,
   mkdirSync,
+  openSync,
   readFileSync,
   readlinkSync,
   renameSync,
@@ -17,6 +20,25 @@ var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
 var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
 var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
 var REGISTRY = "registry.npmjs.org";
+var syncPath = (path) => {
+  const descriptor = openSync(path, "r");
+  try {
+    fsyncSync(descriptor);
+  } finally {
+    closeSync(descriptor);
+  }
+};
+var syncReleaseDirectory = (directory) => {
+  for (const path of [
+    join(directory, "package.json"),
+    join(directory, "dist", "fz-agent.js"),
+    join(directory, "dist", "fz.js"),
+    join(directory, "dist"),
+    directory,
+    dirname(directory)
+  ])
+    syncPath(path);
+};
 function validateAgentRelease(release) {
   if (release?.package !== "@forgezero/agent")
     throw new Error("agent update package is fixed");
@@ -135,16 +157,24 @@ async function stageAgentRelease(releaseInput, options) {
       ]
     }, "agent update extraction");
     await validateReleaseDirectory(unpacked, release, run);
-    if (!existsSync(finalDirectory))
+    if (!existsSync(finalDirectory)) {
       renameSync(unpacked, finalDirectory);
-    else
+      syncReleaseDirectory(finalDirectory);
+    } else
       await validateReleaseDirectory(finalDirectory, release, run);
     if (!existsSync(currentLink)) {
       throw new Error("agent update requires an active immutable release to roll back to");
     }
     const previousTarget = readlinkSync(currentLink);
+    if (previousTarget !== join("versions", options.currentVersion)) {
+      throw new Error("agent update current release does not match the running version");
+    }
+    if (!existsSync(join(root, previousTarget))) {
+      throw new Error("agent update rollback release is missing");
+    }
     return {
       version: release.version,
+      fromVersion: options.currentVersion,
       directory: finalDirectory,
       previousTarget,
       nextTarget: join("versions", release.version),
@@ -159,6 +189,7 @@ function selectAgentRelease(staged) {
   try {
     symlinkSync(staged.nextTarget, next);
     renameSync(next, staged.currentLink);
+    syncPath(dirname(staged.currentLink));
   } finally {
     rmSync(next, { force: true });
   }
@@ -168,25 +199,188 @@ function restoreAgentRelease(staged) {
   try {
     symlinkSync(staged.previousTarget, next);
     renameSync(next, staged.currentLink);
+    syncPath(dirname(staged.currentLink));
   } finally {
     rmSync(next, { force: true });
   }
 }
 
 // src/agent-update-helper.ts
-import { chmodSync as chmodSync2, existsSync as existsSync2, mkdirSync as mkdirSync2, renameSync as renameSync2, unlinkSync, writeFileSync as writeFileSync2 } from "node:fs";
+import { randomUUID as randomUUID2 } from "node:crypto";
+import {
+  chmodSync as chmodSync2,
+  closeSync as closeSync2,
+  existsSync as existsSync2,
+  fsyncSync as fsyncSync2,
+  mkdirSync as mkdirSync2,
+  openSync as openSync2,
+  readFileSync as readFileSync2,
+  renameSync as renameSync2,
+  rmSync as rmSync2,
+  unlinkSync,
+  writeFileSync as writeFileSync2
+} from "node:fs";
 import { connect, createServer } from "node:net";
-import { dirname as dirname2 } from "node:path";
+import { dirname as dirname2, join as join2, resolve as resolve2 } from "node:path";
 import { DEFAULT_SOCKET } from "@forgezero/vault";
 var AGENT_UPDATE_GROUP = "forgezero-update";
 var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
-var AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update.json";
+var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
+var AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update-receipt.json";
 var MAX_REQUEST_BYTES = 8 * 1024;
 var COMPUTE_HELPER_UNITS = [
   "forgezero-deploy-runner.service",
   "forgezero-lifecycle-helper.service",
   "forgezero-software-helper.service"
 ];
+var VERSION2 = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
+var ATTEMPT_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+var REASON_CODE = /^[A-Z][A-Z0-9_]{0,63}$/;
+var MAX_REASON_BYTES = 512;
+var UPDATE_RETRY_BASE_MS = 5 * 60000;
+var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
+var boundedMessage = (value) => {
+  let message = value.replace(/[\r\n]+/g, " ").trim();
+  while (Buffer.byteLength(message, "utf8") > MAX_REASON_BYTES)
+    message = message.slice(0, -1);
+  return message;
+};
+var reason = (code, message) => ({
+  code: REASON_CODE.test(code) ? code : "UPDATE_FAILED",
+  message: boundedMessage(message) || "Agent update failed"
+});
+var validTime = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+function validateReceipt(value) {
+  if (!value || typeof value !== "object")
+    throw new Error("Agent update receipt is malformed");
+  const receipt = value;
+  if (!receipt.attemptId || !ATTEMPT_ID.test(receipt.attemptId))
+    throw new Error("Agent update attempt ID is invalid");
+  if (!receipt.fromVersion || !VERSION2.test(receipt.fromVersion))
+    throw new Error("Agent update source version is invalid");
+  if (!receipt.targetVersion || !VERSION2.test(receipt.targetVersion))
+    throw new Error("Agent update target version is invalid");
+  if (!["activating", "active", "rolled-back", "failed"].includes(receipt.outcome ?? "")) {
+    throw new Error("Agent update outcome is invalid");
+  }
+  if (!validTime(receipt.startedAtTs) || !validTime(receipt.updatedAtTs)) {
+    throw new Error("Agent update timestamps are invalid");
+  }
+  if (receipt.retryAfterTs !== undefined && !validTime(receipt.retryAfterTs)) {
+    throw new Error("Agent update retry timestamp is invalid");
+  }
+  if (receipt.rollbackHealthy !== undefined && typeof receipt.rollbackHealthy !== "boolean") {
+    throw new Error("Agent update rollback health is invalid");
+  }
+  if (receipt.reason && (!REASON_CODE.test(receipt.reason.code) || typeof receipt.reason.message !== "string" || Buffer.byteLength(receipt.reason.message, "utf8") > MAX_REASON_BYTES))
+    throw new Error("Agent update failure reason is invalid");
+  return {
+    attemptId: receipt.attemptId,
+    fromVersion: receipt.fromVersion,
+    targetVersion: receipt.targetVersion,
+    outcome: receipt.outcome,
+    startedAtTs: receipt.startedAtTs,
+    updatedAtTs: receipt.updatedAtTs,
+    ...receipt.retryAfterTs === undefined ? {} : { retryAfterTs: receipt.retryAfterTs },
+    ...receipt.rollbackHealthy === undefined ? {} : { rollbackHealthy: receipt.rollbackHealthy },
+    ...receipt.reason === undefined ? {} : { reason: receipt.reason }
+  };
+}
+function validateJournal(value, root) {
+  if (!value || typeof value !== "object")
+    throw new Error("Agent update journal is malformed");
+  const legacy = value;
+  if (legacy.schemaVersion === undefined) {
+    if (typeof legacy.version === "string" && VERSION2.test(legacy.version) && legacy.outcome === "active" && validTime(legacy.updatedAtTs) && Object.keys(value).every((key) => ["version", "outcome", "updatedAtTs"].includes(key)))
+      return;
+    throw new Error("Agent update legacy receipt is malformed");
+  }
+  const journal = value;
+  const receipt = validateReceipt(journal);
+  if (journal.schemaVersion !== 1)
+    throw new Error("Agent update journal schema is unsupported");
+  if (journal.target !== "compute" && journal.target !== "metal")
+    throw new Error("Agent update target is invalid");
+  if (!Number.isSafeInteger(journal.failureCount) || (journal.failureCount ?? -1) < 0) {
+    throw new Error("Agent update failure count is invalid");
+  }
+  const releaseRoot = resolve2(root);
+  if (journal.currentLink !== join2(releaseRoot, "current"))
+    throw new Error("Agent update current link is invalid");
+  if (journal.previousTarget !== join2("versions", receipt.fromVersion)) {
+    throw new Error("Agent update rollback target is invalid");
+  }
+  if (journal.nextTarget !== join2("versions", receipt.targetVersion)) {
+    throw new Error("Agent update next target is invalid");
+  }
+  return journal;
+}
+function readJournal(path, root) {
+  if (!existsSync2(path))
+    return;
+  return validateJournal(JSON.parse(readFileSync2(path, "utf8")), root);
+}
+function writeAtomic(path, value, mode) {
+  mkdirSync2(dirname2(path), { recursive: true, mode: 493 });
+  const next = `${path}.${randomUUID2()}.next`;
+  let file;
+  try {
+    file = openSync2(next, "wx", mode);
+    writeFileSync2(file, `${JSON.stringify(value)}
+`);
+    fsyncSync2(file);
+    closeSync2(file);
+    file = undefined;
+    renameSync2(next, path);
+    const directory = openSync2(dirname2(path), "r");
+    try {
+      fsyncSync2(directory);
+    } finally {
+      closeSync2(directory);
+    }
+  } finally {
+    if (file !== undefined)
+      closeSync2(file);
+    rmSync2(next, { force: true });
+  }
+}
+var publicReceipt = (journal) => {
+  const {
+    attemptId,
+    fromVersion,
+    targetVersion,
+    outcome,
+    startedAtTs,
+    updatedAtTs,
+    retryAfterTs,
+    rollbackHealthy,
+    reason: failureReason
+  } = journal;
+  return {
+    attemptId,
+    fromVersion,
+    targetVersion,
+    outcome,
+    startedAtTs,
+    updatedAtTs,
+    ...retryAfterTs === undefined ? {} : { retryAfterTs },
+    ...rollbackHealthy === undefined ? {} : { rollbackHealthy },
+    ...failureReason === undefined ? {} : { reason: failureReason }
+  };
+};
+function writeUpdateState(journalPath, receiptPath, journal) {
+  writeAtomic(journalPath, journal, 384);
+  writeAtomic(receiptPath, publicReceipt(journal), 416);
+}
+function readAgentUpdateReceipt(path = AGENT_UPDATE_RECEIPT) {
+  try {
+    if (!existsSync2(path))
+      return;
+    return validateReceipt(JSON.parse(readFileSync2(path, "utf8")));
+  } catch {
+    return;
+  }
+}
 var runCommand = async (input) => {
   const child = Bun.spawn([input.command, ...input.args], {
     cwd: input.cwd,
@@ -202,8 +396,27 @@ var runCommand = async (input) => {
   return { exitCode, output: `${stdout}${stderr}` };
 };
 var runOk = async (run, command2, args) => (await run({ command: command2, args })).exitCode === 0;
+var retryAfter = (now, failures) => now + Math.min(UPDATE_RETRY_MAX_MS, UPDATE_RETRY_BASE_MS * 2 ** Math.min(16, Math.max(0, failures - 1)));
+var stagedFromJournal = (journal, root) => ({
+  version: journal.targetVersion,
+  fromVersion: journal.fromVersion,
+  directory: join2(resolve2(root), journal.nextTarget),
+  previousTarget: journal.previousTarget,
+  nextTarget: journal.nextTarget,
+  currentLink: journal.currentLink
+});
+var restartAgent = async (target, run) => {
+  const helpers = target === "compute" ? COMPUTE_HELPER_UNITS : ["forgezero-metal-helper.service"];
+  for (const unit of helpers)
+    await run({ command: "/usr/bin/systemctl", args: ["try-restart", unit] });
+  const service = target === "compute" ? "forgezero-agent.service" : "forgezero-metal-agent.service";
+  if (!await runOk(run, "/usr/bin/systemctl", ["restart", service])) {
+    throw new Error(`systemd could not restart ${service}`);
+  }
+};
+var targetProbe = (target, run) => target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]);
 function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
-  return new Promise((resolve2) => {
+  return new Promise((resolve3) => {
     const socket = connect(socketPath);
     let settled = false;
     let buffer = "";
@@ -213,7 +426,7 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
       settled = true;
       clearTimeout(timer);
       socket.destroy();
-      resolve2(value);
+      resolve3(value);
     };
     const timer = setTimeout(() => finish(false), timeoutMs);
     socket.on("connect", () => socket.write(`{"op":"identity"}
@@ -237,55 +450,136 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
 async function activateAgentRelease(staged, options = {}) {
   const run = options.run ?? runCommand;
   const target = options.target ?? "compute";
-  const probe = options.probe ?? (target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]));
-  const restart = async () => {
-    const helpers = target === "compute" ? COMPUTE_HELPER_UNITS : ["forgezero-metal-helper.service"];
-    for (const unit of helpers) {
-      await run({ command: "/usr/bin/systemctl", args: ["try-restart", unit] });
-    }
-    const service = target === "compute" ? "forgezero-agent.service" : "forgezero-metal-agent.service";
-    const restarted = await runOk(run, "/usr/bin/systemctl", ["restart", service]);
-    if (!restarted)
-      throw new Error(`systemd could not restart ${service}`);
+  const probe = options.probe ?? targetProbe(target, run);
+  const now = options.now ?? Date.now;
+  const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
+  const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const previous = readJournal(journalPath, dirname2(staged.currentLink));
+  const attemptId = options.attemptId ?? randomUUID2();
+  if (!ATTEMPT_ID.test(attemptId))
+    throw new Error("Agent update attempt ID is invalid");
+  const startedAtTs = now();
+  const failureCount = previous?.targetVersion === staged.version ? previous.failureCount : 0;
+  let journal = {
+    schemaVersion: 1,
+    attemptId,
+    target,
+    fromVersion: staged.fromVersion,
+    targetVersion: staged.version,
+    outcome: "activating",
+    startedAtTs,
+    updatedAtTs: startedAtTs,
+    currentLink: staged.currentLink,
+    previousTarget: staged.previousTarget,
+    nextTarget: staged.nextTarget,
+    failureCount
   };
-  let selected = false;
+  writeUpdateState(journalPath, receiptPath, journal);
+  let selectionAttempted = false;
   try {
+    selectionAttempted = true;
     selectAgentRelease(staged);
-    selected = true;
-    await restart();
+    await restartAgent(target, run);
     if (!await probe())
       throw new Error("the replacement Agent did not answer its retained Vault socket");
-    const receipt = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
-    mkdirSync2(dirname2(receipt), { recursive: true, mode: 493 });
-    const next = `${receipt}.next`;
-    writeFileSync2(next, JSON.stringify({
-      version: staged.version,
-      outcome: "active",
-      updatedAtTs: (options.now ?? Date.now)()
-    }) + `
-`, { mode: 420 });
-    renameSync2(next, receipt);
+    journal = { ...journal, outcome: "active", updatedAtTs: now(), rollbackHealthy: undefined };
+    writeUpdateState(journalPath, receiptPath, journal);
     run({
       command: "/usr/bin/systemctl",
       args: ["try-restart", "--no-block", "forgezero-agent-update-helper.service"]
     });
     return { ok: true, version: staged.version };
   } catch (cause) {
-    const reason = cause instanceof Error ? cause.message : String(cause);
-    if (selected) {
-      restoreAgentRelease(staged);
-      await restart();
+    const message = cause instanceof Error ? cause.message : String(cause);
+    let rollbackHealthy = false;
+    let restored = false;
+    if (selectionAttempted) {
+      try {
+        restoreAgentRelease(staged);
+        restored = true;
+        await restartAgent(target, run);
+        rollbackHealthy = await probe();
+      } catch {
+        rollbackHealthy = false;
+      }
     }
-    return { ok: false, rolledBack: selected, reason };
+    const failures = failureCount + 1;
+    const updatedAtTs = now();
+    journal = {
+      ...journal,
+      outcome: rollbackHealthy ? "rolled-back" : "failed",
+      updatedAtTs,
+      retryAfterTs: retryAfter(updatedAtTs, failures),
+      rollbackHealthy,
+      reason: reason(rollbackHealthy ? "REPLACEMENT_UNHEALTHY" : "ROLLBACK_UNHEALTHY", rollbackHealthy ? message : `${message}; the restored Agent did not pass its health probe`),
+      failureCount: failures
+    };
+    writeUpdateState(journalPath, receiptPath, journal);
+    return { ok: false, rolledBack: restored, rollbackHealthy, reason: message };
   }
 }
+async function recoverInterruptedAgentUpdate(options = {}) {
+  const root = resolve2(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
+  const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
+  const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const journal = readJournal(journalPath, root);
+  if (!journal)
+    return;
+  if (journal.outcome !== "activating") {
+    writeAtomic(receiptPath, publicReceipt(journal), 416);
+    return publicReceipt(journal);
+  }
+  const staged = stagedFromJournal(journal, root);
+  if (!existsSync2(join2(root, journal.previousTarget))) {
+    throw new Error("Agent update rollback release is missing");
+  }
+  const run = options.run ?? runCommand;
+  const probe = options.probe ?? targetProbe(journal.target, run);
+  restoreAgentRelease(staged);
+  let rollbackHealthy = false;
+  let failureMessage = "activation was interrupted before its health verdict became durable";
+  try {
+    await restartAgent(journal.target, run);
+    rollbackHealthy = await probe();
+  } catch (cause) {
+    failureMessage = `${failureMessage}; ${cause instanceof Error ? cause.message : String(cause)}`;
+  }
+  const failures = journal.failureCount + 1;
+  const updatedAtTs = (options.now ?? Date.now)();
+  const recovered = {
+    ...journal,
+    outcome: rollbackHealthy ? "rolled-back" : "failed",
+    updatedAtTs,
+    retryAfterTs: retryAfter(updatedAtTs, failures),
+    rollbackHealthy,
+    reason: reason(rollbackHealthy ? "ACTIVATION_INTERRUPTED" : "ROLLBACK_UNHEALTHY", failureMessage),
+    failureCount: failures
+  };
+  writeUpdateState(journalPath, receiptPath, recovered);
+  return readAgentUpdateReceipt(receiptPath);
+}
 function startAgentUpdateHelper(options = {}) {
   const socketPath = options.socketPath ?? DEFAULT_AGENT_UPDATE_SOCKET;
   if (existsSync2(socketPath))
     unlinkSync(socketPath);
   mkdirSync2(dirname2(socketPath), { recursive: true, mode: 488 });
   const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
-  const activate = options.activate ?? ((staged, target) => activateAgentRelease(staged, { target }));
+  const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
+  const releaseRoot = options.root ?? DEFAULT_AGENT_RELEASE_ROOT;
+  const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, { target, attemptId, journalPath, receiptPath, now: options.now }));
+  let busy = true;
+  let blocked;
+  (options.recover ?? (() => recoverInterruptedAgentUpdate({
+    root: releaseRoot,
+    journalPath,
+    receiptPath,
+    now: options.now
+  })))().catch((cause) => {
+    blocked = cause instanceof Error ? cause.message : String(cause);
+  }).finally(() => {
+    busy = false;
+  });
   const server = createServer((socket) => {
     let buffer = "";
     socket.on("data", (chunk) => {
@@ -301,22 +595,42 @@ function startAgentUpdateHelper(options = {}) {
         return;
       const line = buffer.slice(0, newline);
       buffer = "";
+      let ownsBusy = false;
       Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
+        if (blocked)
+          throw new Error(`update journal needs operator recovery: ${blocked}`);
+        if (busy)
+          throw new Error("another Agent update or recovery is already active");
         if (request.op !== "apply")
           throw new Error("unknown update operation");
         if (request.target !== "compute" && request.target !== "metal") {
           throw new Error("agent update target is invalid");
         }
+        const attemptId = request.attemptId ?? randomUUID2();
+        if (!ATTEMPT_ID.test(attemptId))
+          throw new Error("Agent update attempt ID is invalid");
+        const prior = readJournal(journalPath, releaseRoot);
+        const now = (options.now ?? Date.now)();
+        if (prior?.targetVersion === request.release.version && (prior.outcome === "rolled-back" || prior.outcome === "failed") && (prior.retryAfterTs ?? 0) > now)
+          throw new Error(`Agent update ${request.release.version} is quarantined until ${prior.retryAfterTs}`);
+        busy = true;
+        ownsBusy = true;
         const staged = await stageAgentRelease(request.release, {
           currentVersion: request.currentVersion,
-          root: options.root ?? DEFAULT_AGENT_RELEASE_ROOT
+          root: releaseRoot
         });
-        const response = { ok: true, status: "staged", version: staged.version };
+        const response = { ok: true, status: "staged", version: staged.version, attemptId };
         socket.end(`${JSON.stringify(response)}
-`, () => {
-          setTimer(() => void activate(staged, request.target), 100);
-        });
+`);
+        setTimer(() => void activate(staged, request.target, attemptId).catch((cause) => {
+          blocked = cause instanceof Error ? cause.message : String(cause);
+        }).finally(() => {
+          busy = false;
+        }), 100);
+        ownsBusy = false;
       }).catch((cause) => {
+        if (ownsBusy)
+          busy = false;
         const response = {
           ok: false,
           error: { code: "UPDATE_REFUSED", message: cause instanceof Error ? cause.message : String(cause) }
@@ -331,7 +645,7 @@ function startAgentUpdateHelper(options = {}) {
   return server;
 }
 function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, timeoutMs = 90000) {
-  return new Promise((resolve2, reject) => {
+  return new Promise((resolve3, reject) => {
     const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
 `));
     let buffer = "";
@@ -347,7 +661,7 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
         return;
       socket.end();
       try {
-        resolve2(JSON.parse(buffer.slice(0, newline)));
+        resolve3(JSON.parse(buffer.slice(0, newline)));
       } catch (cause) {
         reject(cause);
       }
@@ -358,9 +672,12 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
 export {
   startAgentUpdateHelper,
   requestAgentUpdate,
+  recoverInterruptedAgentUpdate,
+  readAgentUpdateReceipt,
   probeAgentSocket,
   activateAgentRelease,
   AGENT_UPDATE_RECEIPT,
+  AGENT_UPDATE_JOURNAL,
   AGENT_UPDATE_HELPER_UNIT_PATH,
   AGENT_UPDATE_GROUP
 };

package/dist/agent-update.d.ts

@@ -16,6 +16,7 @@ export interface UpdateCommandResult {
 }
 export interface StagedAgentRelease {
     version: string;
+    fromVersion: string;
     directory: string;
     previousTarget: string;
     nextTarget: string;