@forgezero/agent 0.1.31 → 0.1.33

package/dist/agent-heartbeat.js

@@ -2,8 +2,11 @@
 import { createHash, timingSafeEqual, randomUUID } from "node:crypto";
 import {
   chmodSync,
+  closeSync,
   existsSync,
+  fsyncSync,
   mkdirSync,
+  openSync,
   readFileSync,
   readlinkSync,
   renameSync,
@@ -17,6 +20,25 @@ var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
 var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
 var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
 var REGISTRY = "registry.npmjs.org";
+var syncPath = (path) => {
+  const descriptor = openSync(path, "r");
+  try {
+    fsyncSync(descriptor);
+  } finally {
+    closeSync(descriptor);
+  }
+};
+var syncReleaseDirectory = (directory) => {
+  for (const path of [
+    join(directory, "package.json"),
+    join(directory, "dist", "fz-agent.js"),
+    join(directory, "dist", "fz.js"),
+    join(directory, "dist"),
+    directory,
+    dirname(directory)
+  ])
+    syncPath(path);
+};
 function validateAgentRelease(release) {
   if (release?.package !== "@forgezero/agent")
     throw new Error("agent update package is fixed");
@@ -135,16 +157,24 @@ async function stageAgentRelease(releaseInput, options) {
       ]
     }, "agent update extraction");
     await validateReleaseDirectory(unpacked, release, run);
-    if (!existsSync(finalDirectory))
+    if (!existsSync(finalDirectory)) {
       renameSync(unpacked, finalDirectory);
-    else
+      syncReleaseDirectory(finalDirectory);
+    } else
       await validateReleaseDirectory(finalDirectory, release, run);
     if (!existsSync(currentLink)) {
       throw new Error("agent update requires an active immutable release to roll back to");
     }
     const previousTarget = readlinkSync(currentLink);
+    if (previousTarget !== join("versions", options.currentVersion)) {
+      throw new Error("agent update current release does not match the running version");
+    }
+    if (!existsSync(join(root, previousTarget))) {
+      throw new Error("agent update rollback release is missing");
+    }
     return {
       version: release.version,
+      fromVersion: options.currentVersion,
       directory: finalDirectory,
       previousTarget,
       nextTarget: join("versions", release.version),
@@ -159,6 +189,7 @@ function selectAgentRelease(staged) {
   try {
     symlinkSync(staged.nextTarget, next);
     renameSync(next, staged.currentLink);
+    syncPath(dirname(staged.currentLink));
   } finally {
     rmSync(next, { force: true });
   }
@@ -168,25 +199,189 @@ function restoreAgentRelease(staged) {
   try {
     symlinkSync(staged.previousTarget, next);
     renameSync(next, staged.currentLink);
+    syncPath(dirname(staged.currentLink));
   } finally {
     rmSync(next, { force: true });
   }
 }
 
 // src/agent-update-helper.ts
-import { chmodSync as chmodSync2, existsSync as existsSync2, mkdirSync as mkdirSync2, renameSync as renameSync2, unlinkSync, writeFileSync as writeFileSync2 } from "node:fs";
+import { randomUUID as randomUUID2 } from "node:crypto";
+import {
+  chmodSync as chmodSync2,
+  closeSync as closeSync2,
+  existsSync as existsSync2,
+  fsyncSync as fsyncSync2,
+  mkdirSync as mkdirSync2,
+  openSync as openSync2,
+  readFileSync as readFileSync2,
+  renameSync as renameSync2,
+  rmSync as rmSync2,
+  unlinkSync,
+  writeFileSync as writeFileSync2
+} from "node:fs";
 import { connect, createServer } from "node:net";
-import { dirname as dirname2 } from "node:path";
+import { dirname as dirname2, join as join2, resolve as resolve2 } from "node:path";
 import { DEFAULT_SOCKET } from "@forgezero/vault";
 var AGENT_UPDATE_GROUP = "forgezero-update";
 var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
-var AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update.json";
+var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
+var AGENT_UPDATE_RECEIPT = "/var/lib/forgezero/agent-update-receipt.json";
 var MAX_REQUEST_BYTES = 8 * 1024;
 var COMPUTE_HELPER_UNITS = [
+  "forgezero-agent-egress.service",
   "forgezero-deploy-runner.service",
   "forgezero-lifecycle-helper.service",
   "forgezero-software-helper.service"
 ];
+var VERSION2 = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
+var ATTEMPT_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+var REASON_CODE = /^[A-Z][A-Z0-9_]{0,63}$/;
+var MAX_REASON_BYTES = 512;
+var UPDATE_RETRY_BASE_MS = 5 * 60000;
+var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
+var boundedMessage = (value) => {
+  let message = value.replace(/[\r\n]+/g, " ").trim();
+  while (Buffer.byteLength(message, "utf8") > MAX_REASON_BYTES)
+    message = message.slice(0, -1);
+  return message;
+};
+var reason = (code, message) => ({
+  code: REASON_CODE.test(code) ? code : "UPDATE_FAILED",
+  message: boundedMessage(message) || "Agent update failed"
+});
+var validTime = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+function validateReceipt(value) {
+  if (!value || typeof value !== "object")
+    throw new Error("Agent update receipt is malformed");
+  const receipt = value;
+  if (!receipt.attemptId || !ATTEMPT_ID.test(receipt.attemptId))
+    throw new Error("Agent update attempt ID is invalid");
+  if (!receipt.fromVersion || !VERSION2.test(receipt.fromVersion))
+    throw new Error("Agent update source version is invalid");
+  if (!receipt.targetVersion || !VERSION2.test(receipt.targetVersion))
+    throw new Error("Agent update target version is invalid");
+  if (!["activating", "active", "rolled-back", "failed"].includes(receipt.outcome ?? "")) {
+    throw new Error("Agent update outcome is invalid");
+  }
+  if (!validTime(receipt.startedAtTs) || !validTime(receipt.updatedAtTs)) {
+    throw new Error("Agent update timestamps are invalid");
+  }
+  if (receipt.retryAfterTs !== undefined && !validTime(receipt.retryAfterTs)) {
+    throw new Error("Agent update retry timestamp is invalid");
+  }
+  if (receipt.rollbackHealthy !== undefined && typeof receipt.rollbackHealthy !== "boolean") {
+    throw new Error("Agent update rollback health is invalid");
+  }
+  if (receipt.reason && (!REASON_CODE.test(receipt.reason.code) || typeof receipt.reason.message !== "string" || Buffer.byteLength(receipt.reason.message, "utf8") > MAX_REASON_BYTES))
+    throw new Error("Agent update failure reason is invalid");
+  return {
+    attemptId: receipt.attemptId,
+    fromVersion: receipt.fromVersion,
+    targetVersion: receipt.targetVersion,
+    outcome: receipt.outcome,
+    startedAtTs: receipt.startedAtTs,
+    updatedAtTs: receipt.updatedAtTs,
+    ...receipt.retryAfterTs === undefined ? {} : { retryAfterTs: receipt.retryAfterTs },
+    ...receipt.rollbackHealthy === undefined ? {} : { rollbackHealthy: receipt.rollbackHealthy },
+    ...receipt.reason === undefined ? {} : { reason: receipt.reason }
+  };
+}
+function validateJournal(value, root) {
+  if (!value || typeof value !== "object")
+    throw new Error("Agent update journal is malformed");
+  const legacy = value;
+  if (legacy.schemaVersion === undefined) {
+    if (typeof legacy.version === "string" && VERSION2.test(legacy.version) && legacy.outcome === "active" && validTime(legacy.updatedAtTs) && Object.keys(value).every((key) => ["version", "outcome", "updatedAtTs"].includes(key)))
+      return;
+    throw new Error("Agent update legacy receipt is malformed");
+  }
+  const journal = value;
+  const receipt = validateReceipt(journal);
+  if (journal.schemaVersion !== 1)
+    throw new Error("Agent update journal schema is unsupported");
+  if (journal.target !== "compute" && journal.target !== "metal")
+    throw new Error("Agent update target is invalid");
+  if (!Number.isSafeInteger(journal.failureCount) || (journal.failureCount ?? -1) < 0) {
+    throw new Error("Agent update failure count is invalid");
+  }
+  const releaseRoot = resolve2(root);
+  if (journal.currentLink !== join2(releaseRoot, "current"))
+    throw new Error("Agent update current link is invalid");
+  if (journal.previousTarget !== join2("versions", receipt.fromVersion)) {
+    throw new Error("Agent update rollback target is invalid");
+  }
+  if (journal.nextTarget !== join2("versions", receipt.targetVersion)) {
+    throw new Error("Agent update next target is invalid");
+  }
+  return journal;
+}
+function readJournal(path, root) {
+  if (!existsSync2(path))
+    return;
+  return validateJournal(JSON.parse(readFileSync2(path, "utf8")), root);
+}
+function writeAtomic(path, value, mode) {
+  mkdirSync2(dirname2(path), { recursive: true, mode: 493 });
+  const next = `${path}.${randomUUID2()}.next`;
+  let file;
+  try {
+    file = openSync2(next, "wx", mode);
+    writeFileSync2(file, `${JSON.stringify(value)}
+`);
+    fsyncSync2(file);
+    closeSync2(file);
+    file = undefined;
+    renameSync2(next, path);
+    const directory = openSync2(dirname2(path), "r");
+    try {
+      fsyncSync2(directory);
+    } finally {
+      closeSync2(directory);
+    }
+  } finally {
+    if (file !== undefined)
+      closeSync2(file);
+    rmSync2(next, { force: true });
+  }
+}
+var publicReceipt = (journal) => {
+  const {
+    attemptId,
+    fromVersion,
+    targetVersion,
+    outcome,
+    startedAtTs,
+    updatedAtTs,
+    retryAfterTs,
+    rollbackHealthy,
+    reason: failureReason
+  } = journal;
+  return {
+    attemptId,
+    fromVersion,
+    targetVersion,
+    outcome,
+    startedAtTs,
+    updatedAtTs,
+    ...retryAfterTs === undefined ? {} : { retryAfterTs },
+    ...rollbackHealthy === undefined ? {} : { rollbackHealthy },
+    ...failureReason === undefined ? {} : { reason: failureReason }
+  };
+};
+function writeUpdateState(journalPath, receiptPath, journal) {
+  writeAtomic(journalPath, journal, 384);
+  writeAtomic(receiptPath, publicReceipt(journal), 416);
+}
+function readAgentUpdateReceipt(path = AGENT_UPDATE_RECEIPT) {
+  try {
+    if (!existsSync2(path))
+      return;
+    return validateReceipt(JSON.parse(readFileSync2(path, "utf8")));
+  } catch {
+    return;
+  }
+}
 var runCommand = async (input) => {
   const child = Bun.spawn([input.command, ...input.args], {
     cwd: input.cwd,
@@ -202,8 +397,27 @@ var runCommand = async (input) => {
   return { exitCode, output: `${stdout}${stderr}` };
 };
 var runOk = async (run, command2, args) => (await run({ command: command2, args })).exitCode === 0;
+var retryAfter = (now, failures) => now + Math.min(UPDATE_RETRY_MAX_MS, UPDATE_RETRY_BASE_MS * 2 ** Math.min(16, Math.max(0, failures - 1)));
+var stagedFromJournal = (journal, root) => ({
+  version: journal.targetVersion,
+  fromVersion: journal.fromVersion,
+  directory: join2(resolve2(root), journal.nextTarget),
+  previousTarget: journal.previousTarget,
+  nextTarget: journal.nextTarget,
+  currentLink: journal.currentLink
+});
+var restartAgent = async (target, run) => {
+  const helpers = target === "compute" ? COMPUTE_HELPER_UNITS : ["forgezero-metal-helper.service"];
+  for (const unit of helpers)
+    await run({ command: "/usr/bin/systemctl", args: ["try-restart", unit] });
+  const service = target === "compute" ? "forgezero-agent.service" : "forgezero-metal-agent.service";
+  if (!await runOk(run, "/usr/bin/systemctl", ["restart", service])) {
+    throw new Error(`systemd could not restart ${service}`);
+  }
+};
+var targetProbe = (target, run) => target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]);
 function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
-  return new Promise((resolve2) => {
+  return new Promise((resolve3) => {
     const socket = connect(socketPath);
     let settled = false;
     let buffer = "";
@@ -213,7 +427,7 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
       settled = true;
       clearTimeout(timer);
       socket.destroy();
-      resolve2(value);
+      resolve3(value);
     };
     const timer = setTimeout(() => finish(false), timeoutMs);
     socket.on("connect", () => socket.write(`{"op":"identity"}
@@ -237,47 +451,113 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
 async function activateAgentRelease(staged, options = {}) {
   const run = options.run ?? runCommand;
   const target = options.target ?? "compute";
-  const probe = options.probe ?? (target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]));
-  const restart = async () => {
-    const helpers = target === "compute" ? COMPUTE_HELPER_UNITS : ["forgezero-metal-helper.service"];
-    for (const unit of helpers) {
-      await run({ command: "/usr/bin/systemctl", args: ["try-restart", unit] });
-    }
-    const service = target === "compute" ? "forgezero-agent.service" : "forgezero-metal-agent.service";
-    const restarted = await runOk(run, "/usr/bin/systemctl", ["restart", service]);
-    if (!restarted)
-      throw new Error(`systemd could not restart ${service}`);
+  const probe = options.probe ?? targetProbe(target, run);
+  const now = options.now ?? Date.now;
+  const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
+  const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const previous = readJournal(journalPath, dirname2(staged.currentLink));
+  const attemptId = options.attemptId ?? randomUUID2();
+  if (!ATTEMPT_ID.test(attemptId))
+    throw new Error("Agent update attempt ID is invalid");
+  const startedAtTs = now();
+  const failureCount = previous?.targetVersion === staged.version ? previous.failureCount : 0;
+  let journal = {
+    schemaVersion: 1,
+    attemptId,
+    target,
+    fromVersion: staged.fromVersion,
+    targetVersion: staged.version,
+    outcome: "activating",
+    startedAtTs,
+    updatedAtTs: startedAtTs,
+    currentLink: staged.currentLink,
+    previousTarget: staged.previousTarget,
+    nextTarget: staged.nextTarget,
+    failureCount
   };
-  let selected = false;
+  writeUpdateState(journalPath, receiptPath, journal);
+  let selectionAttempted = false;
   try {
+    selectionAttempted = true;
     selectAgentRelease(staged);
-    selected = true;
-    await restart();
+    await restartAgent(target, run);
     if (!await probe())
       throw new Error("the replacement Agent did not answer its retained Vault socket");
-    const receipt = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
-    mkdirSync2(dirname2(receipt), { recursive: true, mode: 493 });
-    const next = `${receipt}.next`;
-    writeFileSync2(next, JSON.stringify({
-      version: staged.version,
-      outcome: "active",
-      updatedAtTs: (options.now ?? Date.now)()
-    }) + `
-`, { mode: 420 });
-    renameSync2(next, receipt);
+    journal = { ...journal, outcome: "active", updatedAtTs: now(), rollbackHealthy: undefined };
+    writeUpdateState(journalPath, receiptPath, journal);
     run({
       command: "/usr/bin/systemctl",
       args: ["try-restart", "--no-block", "forgezero-agent-update-helper.service"]
     });
     return { ok: true, version: staged.version };
   } catch (cause) {
-    const reason = cause instanceof Error ? cause.message : String(cause);
-    if (selected) {
-      restoreAgentRelease(staged);
-      await restart();
+    const message = cause instanceof Error ? cause.message : String(cause);
+    let rollbackHealthy = false;
+    let restored = false;
+    if (selectionAttempted) {
+      try {
+        restoreAgentRelease(staged);
+        restored = true;
+        await restartAgent(target, run);
+        rollbackHealthy = await probe();
+      } catch {
+        rollbackHealthy = false;
+      }
     }
-    return { ok: false, rolledBack: selected, reason };
+    const failures = failureCount + 1;
+    const updatedAtTs = now();
+    journal = {
+      ...journal,
+      outcome: rollbackHealthy ? "rolled-back" : "failed",
+      updatedAtTs,
+      retryAfterTs: retryAfter(updatedAtTs, failures),
+      rollbackHealthy,
+      reason: reason(rollbackHealthy ? "REPLACEMENT_UNHEALTHY" : "ROLLBACK_UNHEALTHY", rollbackHealthy ? message : `${message}; the restored Agent did not pass its health probe`),
+      failureCount: failures
+    };
+    writeUpdateState(journalPath, receiptPath, journal);
+    return { ok: false, rolledBack: restored, rollbackHealthy, reason: message };
+  }
+}
+async function recoverInterruptedAgentUpdate(options = {}) {
+  const root = resolve2(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
+  const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
+  const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const journal = readJournal(journalPath, root);
+  if (!journal)
+    return;
+  if (journal.outcome !== "activating") {
+    writeAtomic(receiptPath, publicReceipt(journal), 416);
+    return publicReceipt(journal);
+  }
+  const staged = stagedFromJournal(journal, root);
+  if (!existsSync2(join2(root, journal.previousTarget))) {
+    throw new Error("Agent update rollback release is missing");
   }
+  const run = options.run ?? runCommand;
+  const probe = options.probe ?? targetProbe(journal.target, run);
+  restoreAgentRelease(staged);
+  let rollbackHealthy = false;
+  let failureMessage = "activation was interrupted before its health verdict became durable";
+  try {
+    await restartAgent(journal.target, run);
+    rollbackHealthy = await probe();
+  } catch (cause) {
+    failureMessage = `${failureMessage}; ${cause instanceof Error ? cause.message : String(cause)}`;
+  }
+  const failures = journal.failureCount + 1;
+  const updatedAtTs = (options.now ?? Date.now)();
+  const recovered = {
+    ...journal,
+    outcome: rollbackHealthy ? "rolled-back" : "failed",
+    updatedAtTs,
+    retryAfterTs: retryAfter(updatedAtTs, failures),
+    rollbackHealthy,
+    reason: reason(rollbackHealthy ? "ACTIVATION_INTERRUPTED" : "ROLLBACK_UNHEALTHY", failureMessage),
+    failureCount: failures
+  };
+  writeUpdateState(journalPath, receiptPath, recovered);
+  return readAgentUpdateReceipt(receiptPath);
 }
 function startAgentUpdateHelper(options = {}) {
   const socketPath = options.socketPath ?? DEFAULT_AGENT_UPDATE_SOCKET;
@@ -285,7 +565,22 @@ function startAgentUpdateHelper(options = {}) {
     unlinkSync(socketPath);
   mkdirSync2(dirname2(socketPath), { recursive: true, mode: 488 });
   const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
-  const activate = options.activate ?? ((staged, target) => activateAgentRelease(staged, { target }));
+  const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
+  const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
+  const releaseRoot = options.root ?? DEFAULT_AGENT_RELEASE_ROOT;
+  const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, { target, attemptId, journalPath, receiptPath, now: options.now }));
+  let busy = true;
+  let blocked;
+  (options.recover ?? (() => recoverInterruptedAgentUpdate({
+    root: releaseRoot,
+    journalPath,
+    receiptPath,
+    now: options.now
+  })))().catch((cause) => {
+    blocked = cause instanceof Error ? cause.message : String(cause);
+  }).finally(() => {
+    busy = false;
+  });
   const server = createServer((socket) => {
     let buffer = "";
     socket.on("data", (chunk) => {
@@ -301,22 +596,42 @@ function startAgentUpdateHelper(options = {}) {
         return;
       const line = buffer.slice(0, newline);
       buffer = "";
+      let ownsBusy = false;
       Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
+        if (blocked)
+          throw new Error(`update journal needs operator recovery: ${blocked}`);
+        if (busy)
+          throw new Error("another Agent update or recovery is already active");
         if (request.op !== "apply")
           throw new Error("unknown update operation");
         if (request.target !== "compute" && request.target !== "metal") {
           throw new Error("agent update target is invalid");
         }
+        const attemptId = request.attemptId ?? randomUUID2();
+        if (!ATTEMPT_ID.test(attemptId))
+          throw new Error("Agent update attempt ID is invalid");
+        const prior = readJournal(journalPath, releaseRoot);
+        const now = (options.now ?? Date.now)();
+        if (prior?.targetVersion === request.release.version && (prior.outcome === "rolled-back" || prior.outcome === "failed") && (prior.retryAfterTs ?? 0) > now)
+          throw new Error(`Agent update ${request.release.version} is quarantined until ${prior.retryAfterTs}`);
+        busy = true;
+        ownsBusy = true;
         const staged = await stageAgentRelease(request.release, {
           currentVersion: request.currentVersion,
-          root: options.root ?? DEFAULT_AGENT_RELEASE_ROOT
+          root: releaseRoot
         });
-        const response = { ok: true, status: "staged", version: staged.version };
+        const response = { ok: true, status: "staged", version: staged.version, attemptId };
         socket.end(`${JSON.stringify(response)}
-`, () => {
-          setTimer(() => void activate(staged, request.target), 100);
-        });
+`);
+        setTimer(() => void activate(staged, request.target, attemptId).catch((cause) => {
+          blocked = cause instanceof Error ? cause.message : String(cause);
+        }).finally(() => {
+          busy = false;
+        }), 100);
+        ownsBusy = false;
       }).catch((cause) => {
+        if (ownsBusy)
+          busy = false;
         const response = {
           ok: false,
           error: { code: "UPDATE_REFUSED", message: cause instanceof Error ? cause.message : String(cause) }
@@ -331,7 +646,7 @@ function startAgentUpdateHelper(options = {}) {
   return server;
 }
 function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, timeoutMs = 90000) {
-  return new Promise((resolve2, reject) => {
+  return new Promise((resolve3, reject) => {
     const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
 `));
     let buffer = "";
@@ -347,7 +662,7 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
         return;
       socket.end();
       try {
-        resolve2(JSON.parse(buffer.slice(0, newline)));
+        resolve3(JSON.parse(buffer.slice(0, newline)));
       } catch (cause) {
         reject(cause);
       }
@@ -357,7 +672,7 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
 }
 
 // src/agent-heartbeat.ts
-import { readFileSync as readFileSync2 } from "node:fs";
+import { readFileSync as readFileSync3 } from "node:fs";
 
 // src/signed-node-http.ts
 import {
@@ -418,8 +733,8 @@ async function postSignedNode(options, path, body) {
   const payload = await response.json().catch(() => null);
   if (!response.ok) {
     const failure = payload;
-    const reason = failure ? failure.error?.message ?? failure.message : undefined;
-    throw new SignedNodeHttpError(response.status, reason || `signed node request returned HTTP ${response.status}`);
+    const reason2 = failure ? failure.error?.message ?? failure.message : undefined;
+    throw new SignedNodeHttpError(response.status, reason2 || `signed node request returned HTTP ${response.status}`);
   }
   try {
     return await openResponse(recipient.secretKey, signature, payload);
@@ -429,11 +744,15 @@ async function postSignedNode(options, path, body) {
 }
 
 // src/version.ts
-var VERSION2 = "0.1.31";
+var VERSION3 = "0.1.33";
 
 // src/agent-heartbeat.ts
 var unquote = (value) => value.replace(/^['"]|['"]$/g, "");
-function observeAgentHost(version = VERSION2, mode = "enrolled", osRelease = readFileSync2("/etc/os-release", "utf8"), architecture = process.arch) {
+var remoteOutcome = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : cause instanceof SignedNodeHttpError ? "retryable" : "failed";
+
+class AgentUpdateRefusedError extends Error {
+}
+function observeAgentHost(version = VERSION3, mode = "enrolled", osRelease = readFileSync3("/etc/os-release", "utf8"), architecture = process.arch) {
   const values = Object.fromEntries(osRelease.split(`
 `).flatMap((line) => {
     const separator = line.indexOf("=");
@@ -447,23 +766,59 @@ function observeAgentHost(version = VERSION2, mode = "enrolled", osRelease = rea
   };
 }
 async function heartbeatAgentOnce(options) {
-  const observation = (options.observation ?? (() => observeAgentHost(options.version ?? VERSION2, options.mode)))();
+  let observation = (options.observation ?? (() => observeAgentHost(options.version ?? VERSION3, options.mode)))();
+  const update = observation.update ?? readAgentUpdateReceipt(options.receiptPath ?? AGENT_UPDATE_RECEIPT);
+  if (update)
+    observation = { ...observation, update };
   const response = await postSignedNode(options, "v1/node/heartbeat", observation);
-  if (response.desiredAgentRelease) {
-    const release = validateAgentRelease(response.desiredAgentRelease);
+  const desired = response.desiredAgentUpdate ?? (response.desiredAgentRelease ? {
+    attemptId: `legacy:${response.desiredAgentRelease.version}`,
+    leaseExpiresAtTs: Number.MAX_SAFE_INTEGER,
+    release: response.desiredAgentRelease
+  } : undefined);
+  if (desired) {
+    const release = validateAgentRelease(desired.release);
+    const now = (options.now ?? Date.now)();
+    if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(desired.attemptId)) {
+      throw new Error("agent update attempt ID is invalid");
+    }
+    if (!Number.isSafeInteger(desired.leaseExpiresAtTs) || desired.leaseExpiresAtTs <= now) {
+      options.onEvent?.("update-lease-expired", { attemptId: desired.attemptId, to: release.version });
+      return response;
+    }
     if (compareVersions(release.version, observation.version) > 0) {
+      if (update?.targetVersion === release.version && (update.outcome === "rolled-back" || update.outcome === "failed") && (update.retryAfterTs ?? 0) > now) {
+        options.onEvent?.("update-quarantined", {
+          attemptId: desired.attemptId,
+          to: release.version,
+          retryAfterTs: update.retryAfterTs
+        });
+        return response;
+      }
       let prepared = false;
       try {
         await options.prepareUpdate?.(release);
         prepared = true;
-        const applied = await (options.applyUpdate ?? ((next, current) => requestAgentUpdate({
-          op: "apply",
-          target: options.updateTarget ?? "compute",
-          release: next,
-          currentVersion: current
-        })))(release, observation.version);
-        if (!applied.ok)
-          throw new Error(`agent update refused: ${applied.error.message}`);
+        const apply = async () => {
+          const applied = await (options.applyUpdate ?? ((next, current, attemptId) => requestAgentUpdate({
+            op: "apply",
+            target: options.updateTarget ?? "compute",
+            release: next,
+            currentVersion: current,
+            attemptId
+          })))(release, observation.version, desired.attemptId);
+          if (!applied.ok)
+            throw new AgentUpdateRefusedError(`agent update refused: ${applied.error.message}`);
+          if (applied.attemptId !== desired.attemptId) {
+            throw new Error("agent update helper returned the wrong rollout attempt");
+          }
+          return applied;
+        };
+        if (options.telemetry) {
+          await options.telemetry.observe("agent.update", apply, () => "success", (cause) => cause instanceof AgentUpdateRefusedError ? "refused" : "failed");
+        } else {
+          await apply();
+        }
         options.onEvent?.("update-staged", { from: observation.version, to: release.version });
       } catch (cause) {
         if (prepared)
@@ -484,7 +839,8 @@ function startAgentHeartbeat(options) {
     if (stopped || active)
       return;
     let nextSeconds = 30;
-    active = heartbeatAgentOnce(options).then((response) => {
+    const heartbeat = () => heartbeatAgentOnce(options);
+    active = (options.telemetry ? options.telemetry.observe("agent.heartbeat", heartbeat, () => "success", remoteOutcome) : heartbeat()).then((response) => {
       nextSeconds = Math.max(5, Math.min(response.intervalSeconds, 300));
     }).catch((cause) => options.onEvent?.("heartbeat-failed", cause)).finally(() => {
       active = null;