@forgezero/agent 0.1.31 → 0.1.32

package/dist/fz-agent.js

@@ -3,8 +3,8 @@
 
 // src/index.ts
 import { randomBytes as randomBytes5 } from "crypto";
-import { readFileSync as readFileSync8, writeFileSync as writeFileSync8, existsSync as existsSync13, mkdirSync as mkdirSync9, chmodSync as chmodSync12 } from "fs";
-import { dirname as dirname7, join as join5 } from "path";
+import { readFileSync as readFileSync9, writeFileSync as writeFileSync8, existsSync as existsSync13, mkdirSync as mkdirSync9, chmodSync as chmodSync12 } from "fs";
+import { dirname as dirname7, join as join6 } from "path";
 
 // ../access/dist/security.js
 var HEX = Array.from({ length: 256 }, (_, index) => index.toString(16).padStart(2, "0"));
@@ -7581,8 +7581,11 @@ function materializeWarpMdm(options) {
 import { createHash as createHash3, timingSafeEqual, randomUUID as randomUUID2 } from "crypto";
 import {
   chmodSync as chmodSync9,
+  closeSync,
   existsSync as existsSync10,
+  fsyncSync,
   mkdirSync as mkdirSync6,
+  openSync,
   readFileSync as readFileSync6,
   readlinkSync,
   renameSync as renameSync4,
@@ -7596,6 +7599,25 @@ var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
 var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
 var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
 var REGISTRY = "registry.npmjs.org";
+var syncPath = (path) => {
+  const descriptor = openSync(path, "r");
+  try {
+    fsyncSync(descriptor);
+  } finally {
+    closeSync(descriptor);
+  }
+};
+var syncReleaseDirectory = (directory) => {
+  for (const path of [
+    join4(directory, "package.json"),
+    join4(directory, "dist", "fz-agent.js"),
+    join4(directory, "dist", "fz.js"),
+    join4(directory, "dist"),
+    directory,
+    dirname4(directory)
+  ])
+    syncPath(path);
+};
 function validateAgentRelease(release) {
   if (release?.package !== "@forgezero/agent")
     throw new Error("agent update package is fixed");
@@ -7714,16 +7736,24 @@ async function stageAgentRelease(releaseInput, options) {
       ]
     }, "agent update extraction");
     await validateReleaseDirectory(unpacked, release, run);
-    if (!existsSync10(finalDirectory))
+    if (!existsSync10(finalDirectory)) {
       renameSync4(unpacked, finalDirectory);
-    else
+      syncReleaseDirectory(finalDirectory);
+    } else
       await validateReleaseDirectory(finalDirectory, release, run);
     if (!existsSync10(currentLink)) {
       throw new Error("agent update requires an active immutable release to roll back to");
     }
     const previousTarget = readlinkSync(currentLink);
+    if (previousTarget !== join4("versions", options.currentVersion)) {
+      throw new Error("agent update current release does not match the running version");
+    }
+    if (!existsSync10(join4(root, previousTarget))) {
+      throw new Error("agent update rollback release is missing");
+    }
     return {
       version: release.version,
+      fromVersion: options.currentVersion,
       directory: finalDirectory,
       previousTarget,
       nextTarget: join4("versions", release.version),
@@ -7738,6 +7768,7 @@ function selectAgentRelease(staged) {
   try {
     symlinkSync2(staged.nextTarget, next);
     renameSync4(next, staged.currentLink);
+    syncPath(dirname4(staged.currentLink));
   } finally {
     rmSync2(next, { force: true });
   }
@@ -7747,24 +7778,187 @@ function restoreAgentRelease(staged) {
   try {
     symlinkSync2(staged.previousTarget, next);
     renameSync4(next, staged.currentLink);
+    syncPath(dirname4(staged.currentLink));
   } finally {
     rmSync2(next, { force: true });
   }
 }
 
 // src/agent-update-helper.ts
-import { chmodSync as chmodSync10, existsSync as existsSync11, mkdirSync as mkdirSync7, renameSync as renameSync5, unlinkSync as unlinkSync9, writeFileSync as writeFileSync7 } from "fs";
+import { randomUUID as randomUUID3 } from "crypto";
+import {
+  chmodSync as chmodSync10,
+  closeSync as closeSync2,
+  existsSync as existsSync11,
+  fsyncSync as fsyncSync2,
+  mkdirSync as mkdirSync7,
+  openSync as openSync2,
+  readFileSync as readFileSync7,
+  renameSync as renameSync5,
+  rmSync as rmSync3,
+  unlinkSync as unlinkSync9,
+  writeFileSync as writeFileSync7
+} from "fs";
 import { connect as connect5, createServer as createServer6 } from "net";
-import { dirname as dirname5 } from "path";
+import { dirname as dirname5, join as join5, resolve as resolve3 } from "path";
 var AGENT_UPDATE_GROUP = "forgezero-update";
 var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
-var AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update.json";
+var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
+var AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update-receipt.json";
 var MAX_REQUEST_BYTES5 = 8 * 1024;
 var COMPUTE_HELPER_UNITS = [
   "forgezero-deploy-runner.service",
   "forgezero-lifecycle-helper.service",
   "forgezero-software-helper.service"
 ];
+var VERSION2 = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
+var ATTEMPT_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+var REASON_CODE = /^[A-Z][A-Z0-9_]{0,63}$/;
+var MAX_REASON_BYTES = 512;
+var UPDATE_RETRY_BASE_MS = 5 * 60000;
+var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
+var boundedMessage = (value) => {
+  let message = value.replace(/[\r\n]+/g, " ").trim();
+  while (Buffer.byteLength(message, "utf8") > MAX_REASON_BYTES)
+    message = message.slice(0, -1);
+  return message;
+};
+var reason = (code, message) => ({
+  code: REASON_CODE.test(code) ? code : "UPDATE_FAILED",
+  message: boundedMessage(message) || "Agent update failed"
+});
+var validTime = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+function validateReceipt(value) {
+  if (!value || typeof value !== "object")
+    throw new Error("Agent update receipt is malformed");
+  const receipt = value;
+  if (!receipt.attemptId || !ATTEMPT_ID.test(receipt.attemptId))
+    throw new Error("Agent update attempt ID is invalid");
+  if (!receipt.fromVersion || !VERSION2.test(receipt.fromVersion))
+    throw new Error("Agent update source version is invalid");
+  if (!receipt.targetVersion || !VERSION2.test(receipt.targetVersion))
+    throw new Error("Agent update target version is invalid");
+  if (!["activating", "active", "rolled-back", "failed"].includes(receipt.outcome ?? "")) {
+    throw new Error("Agent update outcome is invalid");
+  }
+  if (!validTime(receipt.startedAtTs) || !validTime(receipt.updatedAtTs)) {
+    throw new Error("Agent update timestamps are invalid");
+  }
+  if (receipt.retryAfterTs !== undefined && !validTime(receipt.retryAfterTs)) {
+    throw new Error("Agent update retry timestamp is invalid");
+  }
+  if (receipt.rollbackHealthy !== undefined && typeof receipt.rollbackHealthy !== "boolean") {
+    throw new Error("Agent update rollback health is invalid");
+  }
+  if (receipt.reason && (!REASON_CODE.test(receipt.reason.code) || typeof receipt.reason.message !== "string" || Buffer.byteLength(receipt.reason.message, "utf8") > MAX_REASON_BYTES))
+    throw new Error("Agent update failure reason is invalid");
+  return {
+    attemptId: receipt.attemptId,
+    fromVersion: receipt.fromVersion,
+    targetVersion: receipt.targetVersion,
+    outcome: receipt.outcome,
+    startedAtTs: receipt.startedAtTs,
+    updatedAtTs: receipt.updatedAtTs,
+    ...receipt.retryAfterTs === undefined ? {} : { retryAfterTs: receipt.retryAfterTs },
+    ...receipt.rollbackHealthy === undefined ? {} : { rollbackHealthy: receipt.rollbackHealthy },
+    ...receipt.reason === undefined ? {} : { reason: receipt.reason }
+  };
+}
+function validateJournal(value, root) {
+  if (!value || typeof value !== "object")
+    throw new Error("Agent update journal is malformed");
+  const legacy = value;
+  if (legacy.schemaVersion === undefined) {
+    if (typeof legacy.version === "string" && VERSION2.test(legacy.version) && legacy.outcome === "active" && validTime(legacy.updatedAtTs) && Object.keys(value).every((key) => ["version", "outcome", "updatedAtTs"].includes(key)))
+      return;
+    throw new Error("Agent update legacy receipt is malformed");
+  }
+  const journal = value;
+  const receipt = validateReceipt(journal);
+  if (journal.schemaVersion !== 1)
+    throw new Error("Agent update journal schema is unsupported");
+  if (journal.target !== "compute" && journal.target !== "metal")
+    throw new Error("Agent update target is invalid");
+  if (!Number.isSafeInteger(journal.failureCount) || (journal.failureCount ?? -1) < 0) {
+    throw new Error("Agent update failure count is invalid");
+  }
+  const releaseRoot = resolve3(root);
+  if (journal.currentLink !== join5(releaseRoot, "current"))
+    throw new Error("Agent update current link is invalid");
+  if (journal.previousTarget !== join5("versions", receipt.fromVersion)) {
+    throw new Error("Agent update rollback target is invalid");
+  }
+  if (journal.nextTarget !== join5("versions", receipt.targetVersion)) {
+    throw new Error("Agent update next target is invalid");
+  }
+  return journal;
+}
+function readJournal(path, root) {
+  if (!existsSync11(path))
+    return;
+  return validateJournal(JSON.parse(readFileSync7(path, "utf8")), root);
+}
+function writeAtomic(path, value, mode) {
+  mkdirSync7(dirname5(path), { recursive: true, mode: 493 });
+  const next = `${path}.${randomUUID3()}.next`;
+  let file;
+  try {
+    file = openSync2(next, "wx", mode);
+    writeFileSync7(file, `${JSON.stringify(value)}
+`);
+    fsyncSync2(file);
+    closeSync2(file);
+    file = undefined;
+    renameSync5(next, path);
+    const directory = openSync2(dirname5(path), "r");
+    try {
+      fsyncSync2(directory);
+    } finally {
+      closeSync2(directory);
+    }
+  } finally {
+    if (file !== undefined)
+      closeSync2(file);
+    rmSync3(next, { force: true });
+  }
+}
+var publicReceipt = (journal) => {
+  const {
+    attemptId,
+    fromVersion,
+    targetVersion,
+    outcome,
+    startedAtTs,
+    updatedAtTs,
+    retryAfterTs,
+    rollbackHealthy,
+    reason: failureReason
+  } = journal;
+  return {
+    attemptId,
+    fromVersion,
+    targetVersion,
+    outcome,
+    startedAtTs,
+    updatedAtTs,
+    ...retryAfterTs === undefined ? {} : { retryAfterTs },
+    ...rollbackHealthy === undefined ? {} : { rollbackHealthy },
+    ...failureReason === undefined ? {} : { reason: failureReason }
+  };
+};
+function writeUpdateState(journalPath, receiptPath, journal) {
+  writeAtomic(journalPath, journal, 384);
+  writeAtomic(receiptPath, publicReceipt(journal), 416);
+}
+function readAgentUpdateReceipt(path = AGENT_UPDATE_RECEIPT) {
+  try {
+    if (!existsSync11(path))
+      return;
+    return validateReceipt(JSON.parse(readFileSync7(path, "utf8")));
+  } catch {
+    return;
+  }
+}
 var runCommand = async (input) => {
   const child = Bun.spawn([input.command, ...input.args], {
     cwd: input.cwd,
@@ -7780,8 +7974,27 @@ var runCommand = async (input) => {
   return { exitCode, output: `${stdout}${stderr}` };
 };
 var runOk = async (run, command2, args) => (await run({ command: command2, args })).exitCode === 0;
+var retryAfter = (now, failures) => now + Math.min(UPDATE_RETRY_MAX_MS, UPDATE_RETRY_BASE_MS * 2 ** Math.min(16, Math.max(0, failures - 1)));
+var stagedFromJournal = (journal, root) => ({
+  version: journal.targetVersion,
+  fromVersion: journal.fromVersion,
+  directory: join5(resolve3(root), journal.nextTarget),
+  previousTarget: journal.previousTarget,
+  nextTarget: journal.nextTarget,
+  currentLink: journal.currentLink
+});
+var restartAgent = async (target, run) => {
+  const helpers = target === "compute" ? COMPUTE_HELPER_UNITS : ["forgezero-metal-helper.service"];
+  for (const unit of helpers)
+    await run({ command: "/usr/bin/systemctl", args: ["try-restart", unit] });
+  const service = target === "compute" ? "forgezero-agent.service" : "forgezero-metal-agent.service";
+  if (!await runOk(run, "/usr/bin/systemctl", ["restart", service])) {
+    throw new Error(`systemd could not restart ${service}`);
+  }
+};
+var targetProbe = (target, run) => target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]);
 function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
-  return new Promise((resolve3) => {
+  return new Promise((resolve4) => {
     const socket = connect5(socketPath);
     let settled = false;
     let buffer = "";
@@ -7791,7 +8004,7 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
       settled = true;
       clearTimeout(timer);
       socket.destroy();
-      resolve3(value);
+      resolve4(value);
     };
     const timer = setTimeout(() => finish(false), timeoutMs);
     socket.on("connect", () => socket.write(`{"op":"identity"}
@@ -7815,55 +8028,136 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
 async function activateAgentRelease(staged, options = {}) {
   const run = options.run ?? runCommand;
   const target = options.target ?? "compute";
-  const probe = options.probe ?? (target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]));
-  const restart = async () => {
-    const helpers = target === "compute" ? COMPUTE_HELPER_UNITS : ["forgezero-metal-helper.service"];
-    for (const unit of helpers) {
-      await run({ command: "/usr/bin/systemctl", args: ["try-restart", unit] });
-    }
-    const service = target === "compute" ? "forgezero-agent.service" : "forgezero-metal-agent.service";
-    const restarted = await runOk(run, "/usr/bin/systemctl", ["restart", service]);
-    if (!restarted)
-      throw new Error(`systemd could not restart ${service}`);
+  const probe = options.probe ?? targetProbe(target, run);
+  const now = options.now ?? Date.now;
+  const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
+  const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const previous = readJournal(journalPath, dirname5(staged.currentLink));
+  const attemptId = options.attemptId ?? randomUUID3();
+  if (!ATTEMPT_ID.test(attemptId))
+    throw new Error("Agent update attempt ID is invalid");
+  const startedAtTs = now();
+  const failureCount = previous?.targetVersion === staged.version ? previous.failureCount : 0;
+  let journal = {
+    schemaVersion: 1,
+    attemptId,
+    target,
+    fromVersion: staged.fromVersion,
+    targetVersion: staged.version,
+    outcome: "activating",
+    startedAtTs,
+    updatedAtTs: startedAtTs,
+    currentLink: staged.currentLink,
+    previousTarget: staged.previousTarget,
+    nextTarget: staged.nextTarget,
+    failureCount
   };
-  let selected = false;
+  writeUpdateState(journalPath, receiptPath, journal);
+  let selectionAttempted = false;
   try {
+    selectionAttempted = true;
     selectAgentRelease(staged);
-    selected = true;
-    await restart();
+    await restartAgent(target, run);
     if (!await probe())
       throw new Error("the replacement Agent did not answer its retained Vault socket");
-    const receipt = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
-    mkdirSync7(dirname5(receipt), { recursive: true, mode: 493 });
-    const next = `${receipt}.next`;
-    writeFileSync7(next, JSON.stringify({
-      version: staged.version,
-      outcome: "active",
-      updatedAtTs: (options.now ?? Date.now)()
-    }) + `
-`, { mode: 420 });
-    renameSync5(next, receipt);
+    journal = { ...journal, outcome: "active", updatedAtTs: now(), rollbackHealthy: undefined };
+    writeUpdateState(journalPath, receiptPath, journal);
     run({
       command: "/usr/bin/systemctl",
       args: ["try-restart", "--no-block", "forgezero-agent-update-helper.service"]
     });
     return { ok: true, version: staged.version };
   } catch (cause) {
-    const reason = cause instanceof Error ? cause.message : String(cause);
-    if (selected) {
-      restoreAgentRelease(staged);
-      await restart();
+    const message = cause instanceof Error ? cause.message : String(cause);
+    let rollbackHealthy = false;
+    let restored = false;
+    if (selectionAttempted) {
+      try {
+        restoreAgentRelease(staged);
+        restored = true;
+        await restartAgent(target, run);
+        rollbackHealthy = await probe();
+      } catch {
+        rollbackHealthy = false;
+      }
     }
-    return { ok: false, rolledBack: selected, reason };
+    const failures = failureCount + 1;
+    const updatedAtTs = now();
+    journal = {
+      ...journal,
+      outcome: rollbackHealthy ? "rolled-back" : "failed",
+      updatedAtTs,
+      retryAfterTs: retryAfter(updatedAtTs, failures),
+      rollbackHealthy,
+      reason: reason(rollbackHealthy ? "REPLACEMENT_UNHEALTHY" : "ROLLBACK_UNHEALTHY", rollbackHealthy ? message : `${message}; the restored Agent did not pass its health probe`),
+      failureCount: failures
+    };
+    writeUpdateState(journalPath, receiptPath, journal);
+    return { ok: false, rolledBack: restored, rollbackHealthy, reason: message };
   }
 }
+async function recoverInterruptedAgentUpdate(options = {}) {
+  const root = resolve3(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
+  const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
+  const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const journal = readJournal(journalPath, root);
+  if (!journal)
+    return;
+  if (journal.outcome !== "activating") {
+    writeAtomic(receiptPath, publicReceipt(journal), 416);
+    return publicReceipt(journal);
+  }
+  const staged = stagedFromJournal(journal, root);
+  if (!existsSync11(join5(root, journal.previousTarget))) {
+    throw new Error("Agent update rollback release is missing");
+  }
+  const run = options.run ?? runCommand;
+  const probe = options.probe ?? targetProbe(journal.target, run);
+  restoreAgentRelease(staged);
+  let rollbackHealthy = false;
+  let failureMessage = "activation was interrupted before its health verdict became durable";
+  try {
+    await restartAgent(journal.target, run);
+    rollbackHealthy = await probe();
+  } catch (cause) {
+    failureMessage = `${failureMessage}; ${cause instanceof Error ? cause.message : String(cause)}`;
+  }
+  const failures = journal.failureCount + 1;
+  const updatedAtTs = (options.now ?? Date.now)();
+  const recovered = {
+    ...journal,
+    outcome: rollbackHealthy ? "rolled-back" : "failed",
+    updatedAtTs,
+    retryAfterTs: retryAfter(updatedAtTs, failures),
+    rollbackHealthy,
+    reason: reason(rollbackHealthy ? "ACTIVATION_INTERRUPTED" : "ROLLBACK_UNHEALTHY", failureMessage),
+    failureCount: failures
+  };
+  writeUpdateState(journalPath, receiptPath, recovered);
+  return readAgentUpdateReceipt(receiptPath);
+}
 function startAgentUpdateHelper(options = {}) {
   const socketPath = options.socketPath ?? DEFAULT_AGENT_UPDATE_SOCKET;
   if (existsSync11(socketPath))
     unlinkSync9(socketPath);
   mkdirSync7(dirname5(socketPath), { recursive: true, mode: 488 });
   const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
-  const activate = options.activate ?? ((staged, target) => activateAgentRelease(staged, { target }));
+  const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
+  const releaseRoot = options.root ?? DEFAULT_AGENT_RELEASE_ROOT;
+  const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, { target, attemptId, journalPath, receiptPath, now: options.now }));
+  let busy = true;
+  let blocked;
+  (options.recover ?? (() => recoverInterruptedAgentUpdate({
+    root: releaseRoot,
+    journalPath,
+    receiptPath,
+    now: options.now
+  })))().catch((cause) => {
+    blocked = cause instanceof Error ? cause.message : String(cause);
+  }).finally(() => {
+    busy = false;
+  });
   const server = createServer6((socket) => {
     let buffer = "";
     socket.on("data", (chunk) => {
@@ -7879,22 +8173,42 @@ function startAgentUpdateHelper(options = {}) {
         return;
       const line = buffer.slice(0, newline);
       buffer = "";
+      let ownsBusy = false;
       Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
+        if (blocked)
+          throw new Error(`update journal needs operator recovery: ${blocked}`);
+        if (busy)
+          throw new Error("another Agent update or recovery is already active");
         if (request.op !== "apply")
           throw new Error("unknown update operation");
         if (request.target !== "compute" && request.target !== "metal") {
           throw new Error("agent update target is invalid");
         }
+        const attemptId = request.attemptId ?? randomUUID3();
+        if (!ATTEMPT_ID.test(attemptId))
+          throw new Error("Agent update attempt ID is invalid");
+        const prior = readJournal(journalPath, releaseRoot);
+        const now = (options.now ?? Date.now)();
+        if (prior?.targetVersion === request.release.version && (prior.outcome === "rolled-back" || prior.outcome === "failed") && (prior.retryAfterTs ?? 0) > now)
+          throw new Error(`Agent update ${request.release.version} is quarantined until ${prior.retryAfterTs}`);
+        busy = true;
+        ownsBusy = true;
         const staged = await stageAgentRelease(request.release, {
           currentVersion: request.currentVersion,
-          root: options.root ?? DEFAULT_AGENT_RELEASE_ROOT
+          root: releaseRoot
         });
-        const response = { ok: true, status: "staged", version: staged.version };
+        const response = { ok: true, status: "staged", version: staged.version, attemptId };
         socket.end(`${JSON.stringify(response)}
-`, () => {
-          setTimer(() => void activate(staged, request.target), 100);
-        });
+`);
+        setTimer(() => void activate(staged, request.target, attemptId).catch((cause) => {
+          blocked = cause instanceof Error ? cause.message : String(cause);
+        }).finally(() => {
+          busy = false;
+        }), 100);
+        ownsBusy = false;
       }).catch((cause) => {
+        if (ownsBusy)
+          busy = false;
         const response = {
           ok: false,
           error: { code: "UPDATE_REFUSED", message: cause instanceof Error ? cause.message : String(cause) }
@@ -7909,7 +8223,7 @@ function startAgentUpdateHelper(options = {}) {
   return server;
 }
 function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, timeoutMs = 90000) {
-  return new Promise((resolve3, reject) => {
+  return new Promise((resolve4, reject) => {
     const socket = connect5(socketPath, () => socket.write(`${JSON.stringify(request)}
 `));
     let buffer = "";
@@ -7925,7 +8239,7 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
         return;
       socket.end();
       try {
-        resolve3(JSON.parse(buffer.slice(0, newline)));
+        resolve4(JSON.parse(buffer.slice(0, newline)));
       } catch (cause) {
         reject(cause);
       }
@@ -7935,14 +8249,14 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
 }
 
 // src/agent-heartbeat.ts
-import { readFileSync as readFileSync7 } from "fs";
+import { readFileSync as readFileSync8 } from "fs";
 
 // src/version.ts
-var VERSION2 = "0.1.31";
+var VERSION3 = "0.1.32";
 
 // src/agent-heartbeat.ts
 var unquote = (value) => value.replace(/^['"]|['"]$/g, "");
-function observeAgentHost(version = VERSION2, mode = "enrolled", osRelease = readFileSync7("/etc/os-release", "utf8"), architecture = process.arch) {
+function observeAgentHost(version = VERSION3, mode = "enrolled", osRelease = readFileSync8("/etc/os-release", "utf8"), architecture = process.arch) {
   const values = Object.fromEntries(osRelease.split(`
 `).flatMap((line) => {
     const separator = line.indexOf("=");
@@ -7956,23 +8270,51 @@ function observeAgentHost(version = VERSION2, mode = "enrolled", osRelease = rea
   };
 }
 async function heartbeatAgentOnce(options) {
-  const observation = (options.observation ?? (() => observeAgentHost(options.version ?? VERSION2, options.mode)))();
+  let observation = (options.observation ?? (() => observeAgentHost(options.version ?? VERSION3, options.mode)))();
+  const update = observation.update ?? readAgentUpdateReceipt(options.receiptPath ?? AGENT_UPDATE_RECEIPT);
+  if (update)
+    observation = { ...observation, update };
   const response = await postSignedNode(options, "v1/node/heartbeat", observation);
-  if (response.desiredAgentRelease) {
-    const release = validateAgentRelease(response.desiredAgentRelease);
+  const desired = response.desiredAgentUpdate ?? (response.desiredAgentRelease ? {
+    attemptId: `legacy:${response.desiredAgentRelease.version}`,
+    leaseExpiresAtTs: Number.MAX_SAFE_INTEGER,
+    release: response.desiredAgentRelease
+  } : undefined);
+  if (desired) {
+    const release = validateAgentRelease(desired.release);
+    const now = (options.now ?? Date.now)();
+    if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(desired.attemptId)) {
+      throw new Error("agent update attempt ID is invalid");
+    }
+    if (!Number.isSafeInteger(desired.leaseExpiresAtTs) || desired.leaseExpiresAtTs <= now) {
+      options.onEvent?.("update-lease-expired", { attemptId: desired.attemptId, to: release.version });
+      return response;
+    }
     if (compareVersions(release.version, observation.version) > 0) {
+      if (update?.targetVersion === release.version && (update.outcome === "rolled-back" || update.outcome === "failed") && (update.retryAfterTs ?? 0) > now) {
+        options.onEvent?.("update-quarantined", {
+          attemptId: desired.attemptId,
+          to: release.version,
+          retryAfterTs: update.retryAfterTs
+        });
+        return response;
+      }
       let prepared = false;
       try {
         await options.prepareUpdate?.(release);
         prepared = true;
-        const applied = await (options.applyUpdate ?? ((next, current) => requestAgentUpdate({
+        const applied = await (options.applyUpdate ?? ((next, current, attemptId) => requestAgentUpdate({
           op: "apply",
           target: options.updateTarget ?? "compute",
           release: next,
-          currentVersion: current
-        })))(release, observation.version);
+          currentVersion: current,
+          attemptId
+        })))(release, observation.version, desired.attemptId);
         if (!applied.ok)
           throw new Error(`agent update refused: ${applied.error.message}`);
+        if (applied.attemptId !== desired.attemptId) {
+          throw new Error("agent update helper returned the wrong rollout attempt");
+        }
         options.onEvent?.("update-staged", { from: observation.version, to: release.version });
       } catch (cause) {
         if (prepared)
@@ -8092,7 +8434,7 @@ function startSoftwareHelper(options = {}) {
 }
 function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
   validateSoftwareRequirements(requirements);
-  return new Promise((resolve3, reject) => {
+  return new Promise((resolve4, reject) => {
     const socket = connect6(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
 `));
     let buffer = "";
@@ -8111,7 +8453,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
         const response = JSON.parse(buffer.slice(0, newline));
         if (!response.ok || !response.results)
           throw new Error(response.error?.message ?? "software helper refused the request");
-        resolve3(response.results);
+        resolve4(response.results);
       } catch (cause) {
         reject(cause);
       }
@@ -8123,7 +8465,7 @@ function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCK
 // src/index.ts
 function loadOrCreateSeed(path) {
   if (existsSync13(path)) {
-    const seed2 = new Uint8Array(Buffer.from(readFileSync8(path, "utf8").trim(), "base64url"));
+    const seed2 = new Uint8Array(Buffer.from(readFileSync9(path, "utf8").trim(), "base64url"));
     if (seed2.length < 32) {
       throw new Error(`agent: the seed at ${path} is too short to derive a key from.`);
     }
@@ -8156,7 +8498,7 @@ function loadSeedCredential(name = DEFAULT_SEED_CREDENTIAL, directory = process.
   const path = `${directory}/${name}`;
   if (!existsSync13(path))
     throw new Error(`agent: the systemd credential ${name} is missing at ${path}.`);
-  const seed = new Uint8Array(Buffer.from(readFileSync8(path, "utf8").trim(), "base64url"));
+  const seed = new Uint8Array(Buffer.from(readFileSync9(path, "utf8").trim(), "base64url"));
   if (seed.length < 32)
     throw new Error(`agent: the systemd credential ${name} is too short to derive a key from.`);
   return seed;
@@ -8166,7 +8508,7 @@ function loadTextCredential(name, directory = process.env.CREDENTIALS_DIRECTORY)
     throw new Error("agent: CREDENTIALS_DIRECTORY is missing; systemd did not load the credential.");
   if (!/^[A-Za-z0-9_.-]+$/.test(name))
     throw new Error("agent: invalid systemd credential name.");
-  const value = readFileSync8(`${directory}/${name}`, "utf8").trim();
+  const value = readFileSync9(`${directory}/${name}`, "utf8").trim();
   if (!value)
     throw new Error(`agent: systemd credential ${name} is empty.`);
   return value;
@@ -8216,7 +8558,7 @@ if (import.meta.main) {
   const args = process.argv.slice(2);
   if (args.includes("--help") || args.includes("-h")) {
     console.log([
-      `fz-agent ${VERSION2}`,
+      `fz-agent ${VERSION3}`,
       "",
       "Runs inside managed compute and answers secret requests over a local socket.",
       "It holds no configuration of its own \u2014 everything comes from the",
@@ -8244,7 +8586,7 @@ if (import.meta.main) {
     process.exit(0);
   }
   if (args.includes("--version") || args.includes("-v")) {
-    console.log(VERSION2);
+    console.log(VERSION3);
     process.exit(0);
   }
   const command2 = args.find((arg) => !arg.startsWith("-"));
@@ -8274,7 +8616,7 @@ if (import.meta.main) {
       keys: keys2,
       label: process.env.FZ_NODE_LABEL,
       edgeHostname: process.env.FZ_NODE_HOSTNAME,
-      gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ? readFileSync8(process.env.FZ_GIT_PUBLIC_KEY_FILE, "utf8").trim() : undefined,
+      gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ? readFileSync9(process.env.FZ_GIT_PUBLIC_KEY_FILE, "utf8").trim() : undefined,
       privateNetworkAttachment: privateNetworkAttachmentFromEnvironment()
     });
     console.log(`[agent] enrolled ${binding2.computeReference} in project ${binding2.projectKey}/${binding2.environmentKey}`);
@@ -8284,7 +8626,7 @@ if (import.meta.main) {
     const profilePath = args.find((arg) => arg.startsWith("--profile="))?.slice("--profile=".length);
     if (!profilePath)
       throw new Error("metal-helper requires --profile=/absolute/path.json");
-    const profile2 = JSON.parse(readFileSync8(profilePath, "utf8"));
+    const profile2 = JSON.parse(readFileSync9(profilePath, "utf8"));
     const helper = startMetalHelper({
       profile: profile2,
       socketPath: process.env.FZ_METAL_HELPER_SOCKET ?? DEFAULT_METAL_HELPER_SOCKET
@@ -8356,7 +8698,7 @@ if (import.meta.main) {
       if (stopping)
         return;
       stopping = true;
-      await new Promise((resolve3) => helper.close(() => resolve3()));
+      await new Promise((resolve4) => helper.close(() => resolve4()));
       process.exit(0);
     };
     process.on("SIGTERM", () => void stop());
@@ -8375,7 +8717,7 @@ if (import.meta.main) {
       if (stopping)
         return;
       stopping = true;
-      await new Promise((resolve3) => helper.close(() => resolve3()));
+      await new Promise((resolve4) => helper.close(() => resolve4()));
       process.exit(0);
     };
     process.on("SIGTERM", () => void stop());
@@ -8402,7 +8744,7 @@ if (import.meta.main) {
     const profilePath = args.find((arg) => arg.startsWith("--profile="))?.slice("--profile=".length);
     if (!profilePath)
       throw new Error("metal-isolation requires --profile=/absolute/path.json");
-    const profile2 = JSON.parse(readFileSync8(profilePath, "utf8"));
+    const profile2 = JSON.parse(readFileSync9(profilePath, "utf8"));
     await applyMetalIsolation(profile2);
     console.log("[metal-isolation] host and guest cgroup boundaries active");
     process.exit(0);
@@ -8447,7 +8789,7 @@ if (import.meta.main) {
     if (claimArg !== "-" && !claimArg.startsWith("/")) {
       throw new Error("metal-apply claim path must be absolute");
     }
-    const raw = readFileSync8(claimArg === "-" ? "/dev/stdin" : claimArg, "utf8");
+    const raw = readFileSync9(claimArg === "-" ? "/dev/stdin" : claimArg, "utf8");
     if (Buffer.byteLength(raw) > 32 * 1024)
       throw new Error("metal-apply claim exceeds 32 KiB");
     const claim = JSON.parse(raw);
@@ -8512,7 +8854,7 @@ if (import.meta.main) {
       const deadline = Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 120000));
       const drained = await Promise.race([
         Promise.all([pull.stop(), heartbeat.stop()]).then(() => true),
-        new Promise((resolve3) => setTimeout(() => resolve3(false), deadline))
+        new Promise((resolve4) => setTimeout(() => resolve4(false), deadline))
       ]);
       console.log(`[metal-agent] ${signal}: ${drained ? "drained" : "deadline reached; claim left fenced for recovery"}`);
       process.exit(drained ? 0 : 1);
@@ -8556,7 +8898,7 @@ if (import.meta.main) {
     record: (entry) => console.log(`[agent] ${entry.op} ${entry.outcome}${entry.detail ? ` ${entry.detail}` : ""}`)
   });
   const { nodeKey, keys, server } = running;
-  console.log(`[agent] ${VERSION2} signing as ${nodeKey}`);
+  console.log(`[agent] ${VERSION3} signing as ${nodeKey}`);
   const enrolmentStatePath = process.env.FZ_ENROL_STATE_FILE ?? DEFAULT_ENROLMENT_STATE_PATH;
   let binding = loadGuestBinding(enrolmentStatePath, nodeKey);
   const enrolmentCredential = process.env.FZ_ENROL_TOKEN_CREDENTIAL;
@@ -8572,7 +8914,7 @@ if (import.meta.main) {
       keys,
       label: process.env.FZ_NODE_LABEL,
       edgeHostname: process.env.FZ_NODE_HOSTNAME,
-      gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ? readFileSync8(process.env.FZ_GIT_PUBLIC_KEY_FILE, "utf8").trim() : undefined,
+      gitDeployPublicKey: process.env.FZ_GIT_PUBLIC_KEY_FILE ? readFileSync9(process.env.FZ_GIT_PUBLIC_KEY_FILE, "utf8").trim() : undefined,
       privateNetworkAttachment: privateNetworkAttachmentFromEnvironment()
     });
     console.log(`[agent] enrolled ${binding.computeReference} in project ${binding.projectKey}/${binding.environmentKey}`);
@@ -8661,7 +9003,7 @@ if (import.meta.main) {
         return [name, process.env[name]];
       })),
       gitCredentialPath: process.env.CREDENTIALS_DIRECTORY ? `${process.env.CREDENTIALS_DIRECTORY}/git-deploy-key` : undefined,
-      knownHostsPath: source.knownHosts ? join5(root, "cache", `known-hosts-${key}`) : undefined,
+      knownHostsPath: source.knownHosts ? join6(root, "cache", `known-hosts-${key}`) : undefined,
       knownHostsContent: source.knownHosts,
       cache: deploymentSecrets,
       ensureSoftware: (requirements) => requestSoftware(requirements, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
@@ -8736,7 +9078,7 @@ if (import.meta.main) {
       const deadlineMs = Math.max(1, Number(process.env.FZ_DRAIN_DEADLINE_MS ?? 30000));
       const deadline = Date.now() + deadlineMs;
       const remaining = () => Math.max(1, deadline - Date.now());
-      const controlClosed = control ? await settleWithin(new Promise((resolve3) => control.close(() => resolve3())), remaining()) : true;
+      const controlClosed = control ? await settleWithin(new Promise((resolve4) => control.close(() => resolve4())), remaining()) : true;
       const pullDrain = pull?.stop() ?? Promise.resolve();
       const vaultDrain = vaultSync?.stop() ?? Promise.resolve();
       const attestationDrain = attestationLoop?.stop() ?? Promise.resolve();
@@ -8840,6 +9182,8 @@ export {
   requestAgentUpdate,
   renderWarpMdm,
   removeMetalGuest,
+  recoverInterruptedAgentUpdate,
+  readAgentUpdateReceipt,
   pullProvisioningOnce,
   pullMigrationOnce,
   pullDeploymentOnce,
@@ -8877,7 +9221,7 @@ export {
   allocateCpuPool,
   allocateAddress,
   activateAgentRelease,
-  VERSION2 as VERSION,
+  VERSION3 as VERSION,
   SUPPORTED_GUEST_IMAGE,
   SOFTWARE_HELPER_UNIT_PATH,
   SOFTWARE_HELPER_GROUP,
@@ -8898,6 +9242,7 @@ export {
   DEFAULT_AGENT_RELEASE_ROOT,
   CacheError,
   AGENT_UPDATE_RECEIPT,
+  AGENT_UPDATE_JOURNAL,
   AGENT_UPDATE_HELPER_UNIT_PATH,
   AGENT_UPDATE_GROUP
 };