@hivemagnus/magnus-agent 1.1.2 → 1.1.4

package/.env.example

@@ -54,6 +54,8 @@ MAGNUS_AGENT_PRIVILEGED=true
 # Independent key for explicit command-receipt reconciliation. Leave unset unless
 # your control plane uses it. Linux /usr/bin/flock is required for commands.
 # COMMAND_RECONCILIATION_SECRET=
+# Root-owned JSON list of operator-reviewed historical mutation records (read at startup).
+# MAGNUS_AGENT_REVIEWED_HISTORICAL_RECORDS=/etc/magnus/reviewed-historical-records.json
 # Rotation windows: previous values accepted alongside the current ones.
 # NODE_MANAGER_SECRET_PREVIOUS=
 # RESULT_SIGNING_PREVIOUS_SECRET=
@@ -126,4 +128,5 @@ LOG_LEVEL=info
 # MAGNUS_AGENT_UPGRADE_REQUIRE_PROVENANCE=true
 # Offline nodes: install from hivemagnus-magnus-agent-<X>.tgz files in this directory.
 # MAGNUS_AGENT_UPGRADE_SOURCE_DIR=
+# npm for self-upgrades (default: the npm that ships with the running Node.js, then npm on PATH).
 # MAGNUS_AGENT_NPM=npm

package/README.md

@@ -138,11 +138,32 @@ After that, the server can upgrade the agent from the console's **Upgrade agents
 
 1. The agent downloads the exact version the server names and checks its npm `sha512` integrity. It
    also runs `npm audit signatures` (`MAGNUS_AGENT_UPGRADE_REQUIRE_PROVENANCE`).
-2. It installs that version beside the running one, points `./current` at it and exits.
+2. It installs that version beside the running one and checks that it runs on this Node.js
+   (`engines`), works with the installed launcher, and reports the requested version. Then it points
+   `./current` at it and exits.
 3. The launcher starts the new version on trial.
 4. If the new version does not report an accepted heartbeat within
    `MAGNUS_AGENT_UPGRADE_CONFIRM_SECONDS` (default 180), or crashes twice, the launcher switches back
-   to the previous version.
+   to the previous version. A new version that runs but cannot reach the control API gets up to three
+   windows, and the revert reason then says so.
+
+Every step has a time limit, and the whole upgrade stops after 13 minutes (`SELF_UPGRADE_TIMEOUT`).
+npm, its lifecycle scripts and the smoke test run without the agent's credentials. The agent uses
+`MAGNUS_AGENT_NPM` when set, otherwise the npm that ships with the Node.js it runs on, then npm on
+`PATH`. A retried
+request with the same `requestId` returns the current state instead of an error.
+
+The state file (`upgrade-state.json`) and the links are written durably (fsync and rename), and the
+agent and the launcher lock the state file while they change it. `./last-good` points at the last
+confirmed version. The launcher switches to it when `versions/<from>` is missing, and when the state
+file is unreadable and `./current` keeps exiting. When a write fails (full or read-only disk), the
+launcher logs it and still starts an agent instead of crash-looping.
+
+A confirmed version installs its own `launcher/launcher.mjs` when that declares a higher
+`LAUNCHER_PROTOCOL` and passes a self-check; the old file is kept as `launcher.mjs.previous`. In the
+`--unprivileged` mode, `launcher/` stays owned by root, so the launcher is updated only by
+`install-service`. A release that needs a newer launcher than the node runs is refused with
+`SELF_UPGRADE_LAUNCHER`.
 
 The agent writes only inside its install directory and never uses sudo. Set
 `MAGNUS_AGENT_SELF_UPGRADE=false` to refuse upgrade requests on a node. Offline nodes can install from
@@ -195,6 +216,30 @@ The entries can only touch what the comma-separated allowlists name:
 `_TOKEN_FILES` and `_SIGNING_ENV_FILES`. Every list is empty by default, so
 renewal does nothing until you configure it.
 
+### Reviewed historical mutation records
+
+A node with retained mutation records whose outcome is unknown can settle them only through a signed
+historical quiescence receipt. The receipt counts only for records an operator has reviewed, listed
+in a root-controlled JSON file (`MAGNUS_AGENT_REVIEWED_HISTORICAL_RECORDS`, default
+`/etc/magnus/reviewed-historical-records.json`):
+
+```json
+{
+  "version": 1,
+  "records": {
+    "0b6f1c2e-6d3a-4f6e-9c1d-2a7b8e9f0a1b": { "nodeFqdn": "node-1.example.com", "recordSha256": "<64 lowercase hex>" }
+  }
+}
+```
+
+The keys are the mutation record ids. `recordSha256` is the SHA-256 of the retained record file, as
+in the signed observation. Entries for other nodes (`nodeFqdn` other than `HOST_FQDN`) are ignored, so
+one file can serve a fleet. The file and its directory must be owned by root and not group- or
+world-writable, and the file must not be a symlink or larger than 1 MiB. `0644` works for an
+unprivileged agent too. The agent reads the file once at startup, so restart it after a change. A
+missing file means no reviewed records. A file that fails these checks or the schema is refused as a
+whole, and the agent logs the reason at startup.
+
 ### Hermes registry sync
 
 `deploy/magnus-hermes-sync.service` and its timer run `dist/hermes-sync.js`

package/deploy/magnus-agent-unprivileged.service

@@ -13,6 +13,11 @@
 #
 # Use a NODE_MANAGER_PORT above 1023, and make the TLS key readable by
 # magnus-agent. Install this file as /etc/systemd/system/magnus-agent.service.
+#
+# These example units start the agent directly and cannot self-upgrade. For
+# server-driven upgrades, run `sudo magnus-agent install-service` instead: it
+# moves the install into versions/<X>, installs the upgrade launcher and writes
+# a unit that starts it (the cron path below node_modules keeps working).
 [Unit]
 Description=Magnus Agent (unprivileged)
 After=network-online.target

package/deploy/magnus-agent.service

@@ -10,6 +10,11 @@
 # policy and the agent's own allowlists. This is privileged mode
 # (MAGNUS_AGENT_PRIVILEGED=true, the default); to run without root, use
 # magnus-agent-unprivileged.service instead.
+#
+# These example units start the agent directly and cannot self-upgrade. For
+# server-driven upgrades, run `sudo magnus-agent install-service` instead: it
+# moves the install into versions/<X>, installs the upgrade launcher and writes
+# a unit that starts it (the cron path below node_modules keeps working).
 [Unit]
 Description=Magnus Agent
 After=network-online.target

package/dist/agent.d.ts

@@ -1,3 +1,5 @@
+/** On an upgrade trial: retry fast, without backoff, so a short control API outage does not end the trial. */
+export declare const TRIAL_RETRY_MS = 5000;
 export declare function getHeartbeatStatus(): {
     running: boolean;
     lastAcceptedAt: string | null;

package/dist/agent.js

@@ -7,9 +7,12 @@ import { controlApiFetch } from './tls.js';
 import { startSpan, formatTraceparent } from './collectors/otel-collector.js';
 import { collectHermesState } from './executors/hermes-manager.js';
 import { existsSync } from 'node:fs';
-import { confirmTrial, selfUpgradeStatus } from './upgrade/self-upgrade.js';
+import { confirmTrial, controlApiUnreachable, recordTrialAlive, selfUpgradeStatus, trialOf } from './upgrade/self-upgrade.js';
+import { commandLedger } from './executors/command-executor.js';
 const HEARTBEAT_INTERVAL_MS = 30_000; // 30s
 const HEARTBEAT_BACKOFF_MAX_MS = 5 * 60_000; // 5 min max backoff
+/** On an upgrade trial: retry fast, without backoff, so a short control API outage does not end the trial. */
+export const TRIAL_RETRY_MS = 5_000;
 const HERMES_DISCOVERY_INTERVAL_MS = Number(process.env['HERMES_DISCOVERY_INTERVAL_MS'] ?? 3 * 60 * 60 * 1000);
 let heartbeatTimer = null;
 let heartbeatRunning = false;
@@ -72,6 +75,11 @@ async function collectHealthWithDeadline() {
         clearTimeout(timer);
     }
 }
+/** Self-upgrade status for the heartbeat, with the sticky local enrollment marker. */
+async function heartbeatSelfUpgrade() {
+    const enrollment = await commandLedger.managedEnrollment().catch(() => null);
+    return { ...selfUpgradeStatus(AGENT_VERSION), managedEnrollment: enrollment ? { nodeId: enrollment.nodeId, nodeFqdn: enrollment.nodeFqdn } : null };
+}
 /** Stop the heartbeat loop. */
 export function stopHeartbeatLoop() {
     heartbeatRunning = false;
@@ -89,6 +97,7 @@ export function startHeartbeatLoop() {
         if (!heartbeatRunning)
             return;
         const span = startSpan('heartbeat.send', { kind: 'client' });
+        let posting = false;
         try {
             const health = await collectHealthWithDeadline();
             // A cached Hermes snapshot is sent while a stale one refreshes in the
@@ -97,6 +106,8 @@ export function startHeartbeatLoop() {
             if (!hermesSnapshot)
                 await Promise.race([refresh, delay(HERMES_HEARTBEAT_WAIT_MS)]);
             const hermes = hermesSnapshot;
+            const selfUpgrade = await heartbeatSelfUpgrade();
+            posting = true;
             await ky.post(`${CONTROL_API_URL}/api/v1/heartbeat`, {
                 json: {
                     agentVersion: AGENT_VERSION,
@@ -114,7 +125,7 @@ export function startHeartbeatLoop() {
                     },
                     ...(hermes ? { hermes } : {}),
                     nodeVersion: process.version,
-                    selfUpgrade: selfUpgradeStatus(AGENT_VERSION),
+                    selfUpgrade,
                 },
                 headers: {
                     'x-node-token': NODE_TOKEN,
@@ -150,8 +161,12 @@ export function startHeartbeatLoop() {
             if (consecutiveFailures === 1 || Math.log2(consecutiveFailures) % 1 === 0) {
                 logger.warn(`Heartbeat failed to send`, { attempt: consecutiveFailures, error: message });
             }
+            // On trial: tell the launcher this version is alive, and retry at a fixed short interval.
+            const onTrial = trialOf(AGENT_VERSION) !== null;
+            if (onTrial && posting && controlApiUnreachable(err))
+                recordTrialAlive(AGENT_VERSION, message);
             // Back off retry interval: 30s → 60s → 120s … capped at 5min
-            const backoffMs = Math.min(HEARTBEAT_INTERVAL_MS * Math.pow(2, consecutiveFailures - 1), HEARTBEAT_BACKOFF_MAX_MS);
+            const backoffMs = onTrial ? TRIAL_RETRY_MS : Math.min(HEARTBEAT_INTERVAL_MS * Math.pow(2, consecutiveFailures - 1), HEARTBEAT_BACKOFF_MAX_MS);
             heartbeatTimer = setTimeout(() => void send(), backoffMs);
             return;
         }

package/dist/env-catalog.js

@@ -7,6 +7,7 @@ export const agentEnvCatalog = [
     { name: 'NODE_MANAGER_SECRET_PREVIOUS', required: false, secret: true, group: 'core', description: 'Previous node credential, accepted only during a rotation window.' },
     { name: 'RESULT_SIGNING_PREVIOUS_SECRET', required: false, secret: true, group: 'core', description: 'Previous result-signing key, accepted only during a rotation window.' },
     { name: 'COMMAND_RECONCILIATION_SECRET', required: false, secret: true, generate: 'hex32', group: 'core', description: 'Independent key that authorizes explicit command-receipt reconciliation. Leave unset unless your control plane uses it.' },
+    { name: 'MAGNUS_AGENT_REVIEWED_HISTORICAL_RECORDS', required: false, secret: false, default: '/etc/magnus/reviewed-historical-records.json', group: 'core', description: 'Root-owned JSON file of operator-reviewed historical mutation records ({"version":1,"records":{"<record id>":{"nodeFqdn","recordSha256"}}}). Read at startup; a missing file means none, a file that is not root-controlled or fails the schema is refused.' },
     { name: 'HOST_FQDN', required: false, secret: false, default: 'hostname --fqdn', group: 'core', description: 'Host identity reported to the Control API.' },
     { name: 'NODE_MANAGER_PORT', required: false, secret: false, default: String(DEFAULT_PORT), group: 'core', description: 'Listener port.' },
     { name: 'NODE_MANAGER_BIND_HOST', required: false, secret: false, default: '0.0.0.0 (TLS) / 127.0.0.1 (HTTP)', group: 'core', description: 'Listener address.' },
@@ -16,7 +17,7 @@ export const agentEnvCatalog = [
     { name: 'MAGNUS_AGENT_UPGRADE_CONFIRM_SECONDS', required: false, secret: false, default: '180', group: 'core', description: 'Seconds a newly installed version has to report an accepted heartbeat before the launcher switches back to the previous one (10-3600).' },
     { name: 'MAGNUS_AGENT_UPGRADE_REQUIRE_PROVENANCE', required: false, secret: false, default: 'true', group: 'core', description: 'Run `npm audit signatures` on a staged upgrade and refuse it unless every package has valid registry signatures. Set false only for locally built tarballs.' },
     { name: 'MAGNUS_AGENT_UPGRADE_SOURCE_DIR', required: false, secret: false, group: 'core', description: 'Install self-upgrades from tarballs in this directory (hivemagnus-magnus-agent-<X>.tgz) instead of the npm registry, for offline nodes. The integrity check still applies.' },
-    { name: 'MAGNUS_AGENT_NPM', required: false, secret: false, default: 'npm', group: 'core', description: 'npm executable used for self-upgrades.' },
+    { name: 'MAGNUS_AGENT_NPM', required: false, secret: false, group: 'core', description: 'npm executable used for self-upgrades. Default: the npm that ships with the running Node.js, then npm on PATH.' },
     { name: 'MAGNUS_EXECUTION_SUPERVISOR_ENABLED', required: false, secret: false, default: 'false', group: 'core', description: 'Run commands under the durable execution supervisor.' },
     { name: 'CONTROL_API_CA_PATH', required: false, secret: false, group: 'tls', description: 'PEM CA bundle trusted for a private-CA Control API. Verification is never disabled.' },
     { name: 'TLS_CERT_PATH', required: false, secret: false, group: 'tls', description: 'Listener certificate (absolute or relative to AGENT_INSTALL_DIR). Required with TLS_KEY_PATH when NODE_ENV=production.' },

package/dist/executors/historical-mutation.d.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import { type ReviewedHistoricalRecords } from './reviewed-historical-records.js';
 export declare const historicalDrainSchema: z.ZodObject<{
     nodeId: z.ZodString;
     nodeFqdn: z.ZodString;
@@ -234,12 +235,11 @@ export interface HistoricalKeys {
     result: string;
     reconciliation: string;
 }
-export type ReviewedHistoricalRecords = Readonly<Record<string, {
-    nodeFqdn: string;
-    recordSha256: string;
-}>>;
+export type { ReviewedHistoricalRecords };
 /** Records an operator has independently inspected and approved for historical
- * settlement. Empty by default: each entry needs its own reviewed source change. */
+ * settlement, from the root-controlled MAGNUS_AGENT_REVIEWED_HISTORICAL_RECORDS
+ * file, read once at startup. Empty when the file is absent or refused. */
+export declare const reviewedHistoricalRecordsLoad: import("./reviewed-historical-records.js").ReviewedRecordsLoad;
 export declare const reviewedHistoricalRecords: ReviewedHistoricalRecords;
 export declare const historicalHash: (value: unknown) => string;
 export declare function validateHistoricalKeys(keys: HistoricalKeys): void;
@@ -250,5 +250,4 @@ export declare function verifyHistoricalObservation(raw: unknown, key: string):
 export declare function verifyHistoricalAssertion(raw: unknown, keys: HistoricalKeys): HistoricalAssertion;
 export declare function verifyHistoricalReceipt(raw: unknown, keys: HistoricalKeys): HistoricalReceipt;
 export declare function readHostBoot(): Promise<HostBoot>;
-export {};
 //# sourceMappingURL=historical-mutation.d.ts.map

package/dist/executors/historical-mutation.js

@@ -2,6 +2,9 @@
 import { createHash, createHmac, timingSafeEqual } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 import { z } from 'zod';
+import { HOST_FQDN } from '../env.js';
+import { logger } from '../utils/logger.js';
+import { loadReviewedHistoricalRecords } from './reviewed-historical-records.js';
 const sha = z.string().regex(/^[a-f0-9]{64}$/);
 export const historicalDrainSchema = z.object({ nodeId: z.string().min(1).max(128),
     nodeFqdn: z.string().min(1).max(253), epoch: z.number().int().positive(), barrierId: z.string().guid() }).strict();
@@ -22,8 +25,18 @@ const receiptUnsigned = z.object({ version: z.literal(1), assertion: historicalA
     originalOutcome: z.literal('unknown') }).strict();
 export const historicalReceiptSchema = receiptUnsigned.extend({ signature: sha }).strict();
 /** Records an operator has independently inspected and approved for historical
- * settlement. Empty by default: each entry needs its own reviewed source change. */
-export const reviewedHistoricalRecords = Object.freeze({});
+ * settlement, from the root-controlled MAGNUS_AGENT_REVIEWED_HISTORICAL_RECORDS
+ * file, read once at startup. Empty when the file is absent or refused. */
+export const reviewedHistoricalRecordsLoad = loadReviewedHistoricalRecords({
+    path: process.env['MAGNUS_AGENT_REVIEWED_HISTORICAL_RECORDS']?.trim() || undefined, hostname: HOST_FQDN
+});
+if (reviewedHistoricalRecordsLoad.problem) {
+    logger.warn(`Reviewed historical records refused; historical settlements are not honoured: ${reviewedHistoricalRecordsLoad.problem}`);
+}
+else if (reviewedHistoricalRecordsLoad.count > 0) {
+    logger.info(`Loaded ${reviewedHistoricalRecordsLoad.count} reviewed historical record(s) for ${HOST_FQDN} from ${reviewedHistoricalRecordsLoad.path}`);
+}
+export const reviewedHistoricalRecords = reviewedHistoricalRecordsLoad.records;
 export const historicalHash = (value) => createHash('sha256').update(JSON.stringify(value)).digest('hex');
 export function validateHistoricalKeys(keys) {
     if (keys.result.length < 32 || keys.reconciliation.length < 32 || keys.result === keys.reconciliation)

package/dist/executors/reviewed-historical-records.d.ts

@@ -0,0 +1,22 @@
+export declare const DEFAULT_REVIEWED_HISTORICAL_RECORDS_PATH = "/etc/magnus/reviewed-historical-records.json";
+export declare const REVIEWED_RECORDS_MAX_BYTES: number;
+export type ReviewedHistoricalRecords = Readonly<Record<string, Readonly<{
+    nodeFqdn: string;
+    recordSha256: string;
+}>>>;
+export interface ReviewedRecordsLoad {
+    path: string;
+    records: ReviewedHistoricalRecords;
+    /** Entries for this node that were loaded. */
+    count: number;
+    /** Why the file was refused, or null (also when it is absent). */
+    problem: string | null;
+}
+export interface LoadOptions {
+    path?: string;
+    hostname: string;
+    /** Owner the file and its directory must have; 0 (root) except in tests. */
+    ownerUid?: number;
+}
+export declare function loadReviewedHistoricalRecords(options: LoadOptions): ReviewedRecordsLoad;
+//# sourceMappingURL=reviewed-historical-records.d.ts.map

package/dist/executors/reviewed-historical-records.js

@@ -0,0 +1,117 @@
+/**
+ * Operator-reviewed historical mutation records, from a root-controlled file
+ * (MAGNUS_AGENT_REVIEWED_HISTORICAL_RECORDS, default
+ * /etc/magnus/reviewed-historical-records.json). Only records listed there can
+ * be settled by, or count as settled through, a signed historical receipt.
+ *
+ * Read once when the agent starts, so the set cannot change under a running
+ * drain; edit the file and restart the agent to change it. A missing file is
+ * the empty set. A file that root does not control (not owned by uid 0, group-
+ * or world-writable, a symlink, in such a directory) or that fails the schema
+ * is refused as a whole: the set is empty and the reason is logged.
+ *
+ * {
+ *   "version": 1,
+ *   "records": {
+ *     "<record id (UUID)>": { "nodeFqdn": "node-1.example.com", "recordSha256": "<64 hex>" }
+ *   }
+ * }
+ */
+import { closeSync, constants, fstatSync, lstatSync, openSync, readSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { z } from 'zod';
+export const DEFAULT_REVIEWED_HISTORICAL_RECORDS_PATH = '/etc/magnus/reviewed-historical-records.json';
+export const REVIEWED_RECORDS_MAX_BYTES = 1024 * 1024;
+const reviewedFileSchema = z.object({
+    version: z.literal(1),
+    records: z.record(z.string().guid(), z.object({
+        nodeFqdn: z.string().min(1).max(253),
+        recordSha256: z.string().regex(/^[a-f0-9]{64}$/),
+    }).strict()),
+}).strict();
+function controlled(info, what, ownerUid) {
+    if (info.isSymbolicLink())
+        return `${what} is a symlink`;
+    if (info.uid !== ownerUid)
+        return `${what} is owned by uid ${info.uid}, not ${ownerUid === 0 ? 'root' : ownerUid}`;
+    if (info.mode & 0o022)
+        return `${what} is group- or world-writable`;
+    return null;
+}
+const empty = Object.freeze({});
+export function loadReviewedHistoricalRecords(options) {
+    const path = resolve(options.path || DEFAULT_REVIEWED_HISTORICAL_RECORDS_PATH);
+    const ownerUid = options.ownerUid ?? 0;
+    const refuse = (problem) => ({ path, records: empty, count: 0, problem });
+    let info;
+    try {
+        info = lstatSync(path);
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return { path, records: empty, count: 0, problem: null };
+        return refuse(`cannot inspect ${path}: ${error.message}`);
+    }
+    const problem = controlled(info, path, ownerUid);
+    if (problem)
+        return refuse(problem);
+    if (!info.isFile())
+        return refuse(`${path} is not a regular file`);
+    if (info.size > REVIEWED_RECORDS_MAX_BYTES)
+        return refuse(`${path} is larger than ${REVIEWED_RECORDS_MAX_BYTES} bytes`);
+    let parent;
+    try {
+        parent = lstatSync(dirname(path));
+    }
+    catch (error) {
+        return refuse(`cannot inspect ${dirname(path)}: ${error.message}`);
+    }
+    const parentProblem = controlled(parent, dirname(path), ownerUid);
+    if (parentProblem)
+        return refuse(parentProblem);
+    if (!parent.isDirectory())
+        return refuse(`${dirname(path)} is not a directory`);
+    let text;
+    try {
+        // O_NOFOLLOW and the same inode as inspected: the file cannot be swapped in between.
+        const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+        try {
+            const opened = fstatSync(fd);
+            if (opened.ino !== info.ino || opened.dev !== info.dev || opened.size > REVIEWED_RECORDS_MAX_BYTES)
+                return refuse(`${path} changed while it was read`);
+            const buffer = Buffer.alloc(opened.size);
+            let offset = 0;
+            while (offset < buffer.length) {
+                const read = readSync(fd, buffer, offset, buffer.length - offset, offset);
+                if (read === 0)
+                    break;
+                offset += read;
+            }
+            text = buffer.subarray(0, offset).toString('utf8');
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    catch (error) {
+        return refuse(`cannot read ${path}: ${error.message}`);
+    }
+    let parsed;
+    try {
+        const result = reviewedFileSchema.safeParse(JSON.parse(text));
+        if (!result.success)
+            return refuse(`${path} does not match the schema: ${result.error.issues.slice(0, 3).map(issue => `${issue.path.join('.') || '(root)'}: ${issue.message}`).join('; ')}`);
+        parsed = result.data;
+    }
+    catch (error) {
+        return refuse(`${path} is not valid JSON: ${error.message}`);
+    }
+    // Entries for other nodes are ignored, so one file can serve a whole fleet.
+    const records = {};
+    for (const [id, entry] of Object.entries(parsed.records)) {
+        if (entry.nodeFqdn === options.hostname)
+            records[id] = Object.freeze({ nodeFqdn: entry.nodeFqdn, recordSha256: entry.recordSha256 });
+    }
+    return { path, records: Object.freeze(records), count: Object.keys(records).length, problem: null };
+}
+//# sourceMappingURL=reviewed-historical-records.js.map

package/dist/executors/scheduler-runner.js

@@ -1,14 +1,25 @@
 #!/usr/bin/env node
 /** Fixed cron entrypoint. Load the installed private environment before runtime imports. */
+import { lstatSync } from 'node:fs';
 import { basename, dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { config } from 'dotenv';
 const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
 // The deployment helper installs the tarball below the stable host directory;
 // its private .env and .magnus ledger are outside node_modules and survive upgrades.
-const install = basename(packageRoot) === 'magnus-agent' && basename(dirname(packageRoot)) === '@hivemagnus'
+let install = basename(packageRoot) === 'magnus-agent' && basename(dirname(packageRoot)) === '@hivemagnus'
     && basename(dirname(dirname(packageRoot))) === 'node_modules'
     ? resolve(packageRoot, '../../..') : packageRoot;
+// Versioned layout: Node resolves the cron path (<dir>/node_modules or <dir>/current) to
+// <dir>/versions/<X>; the stable directory is the one that holds ./current.
+if (basename(dirname(install)) === 'versions') {
+    const stable = dirname(dirname(install));
+    try {
+        if (lstatSync(resolve(stable, 'current')).isSymbolicLink())
+            install = stable;
+    }
+    catch { /* not the versioned layout */ }
+}
 for (const key of ['CONTROL_API_URL', 'HOST_FQDN', 'NODE_MANAGER_SECRET', 'NODE_MANAGER_SECRET_PREVIOUS',
     'RESULT_SIGNING_SECRET', 'MAGNUS_EXECUTION_SUPERVISOR_ENABLED', 'CONTROL_API_CA_PATH', 'COMMAND_RECONCILIATION_SECRET'])
     delete process.env[key];

package/dist/upgrade/install-service.js

@@ -6,6 +6,7 @@
  *   /opt/magnus-agent/.env
  *   /opt/magnus-agent/versions/<X>/node_modules/@hivemagnus/magnus-agent
  *   /opt/magnus-agent/current -> versions/<X>
+ *   /opt/magnus-agent/last-good -> versions/<X>   (the last confirmed version)
  *   /opt/magnus-agent/launcher/launcher.mjs   (ExecStart)
  *
  * After this the agent upgrades itself inside that directory; it never needs
@@ -15,7 +16,7 @@ import { execFileSync } from 'node:child_process';
 import { copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, renameSync, symlinkSync, writeFileSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { AGENT_PACKAGE, pointCurrentAt } from './self-upgrade.js';
+import { AGENT_PACKAGE, ALIVE_FILE, LOCK_FILE, STATE_FILE, linkVersion, pointCurrentAt, pointLink } from './self-upgrade.js';
 export const INSTALL_SERVICE_USAGE = `Usage: sudo magnus-agent install-service [options]
 
 Moves this install into the versioned layout used by self-upgrades, installs
@@ -188,12 +189,16 @@ export async function installService(argv) {
         }
         act(`point ${current} at versions/${version}`, () => pointCurrentAt(dir, version));
     }
+    // The launcher falls back to the last confirmed version when the upgrade state is unreadable.
+    if (!isSymlink(join(dir, 'last-good'))) {
+        act(`link ${join(dir, 'last-good')} to the running version`, () => pointLink(dir, 'last-good', linkVersion(dir, 'current') ?? version));
+    }
     // Paths written before the versioned layout (cron lines, other units) keep working.
     if (!existsSync(join(dir, 'node_modules')) && !isSymlink(join(dir, 'node_modules'))) {
         act(`link ${join(dir, 'node_modules')} to current/node_modules`, () => symlinkSync(join('current', 'node_modules'), join(dir, 'node_modules'), 'dir'));
     }
-    // The launcher is copied from the release that runs install-service, not linked:
-    // self-upgrades never replace it.
+    // The launcher is copied from the release that runs install-service, not linked. A confirmed
+    // self-upgrade replaces it only with a copy that declares a higher LAUNCHER_PROTOCOL.
     act(`install ${join(dir, 'launcher', 'launcher.mjs')}`, () => {
         mkdirSync(join(dir, 'launcher'), { recursive: true, mode: 0o755 });
         copyFileSync(join(isSymlink(current) ? realpathSync(join(current, 'node_modules', ...AGENT_PACKAGE.split('/'))) : here, 'launcher', 'launcher.mjs'), join(dir, 'launcher', 'launcher.mjs'));
@@ -209,8 +214,13 @@ export async function installService(argv) {
             execFileSync('chown', [`${user}:${user}`, dir]);
             execFileSync('chown', ['-R', `${user}:${user}`, join(dir, 'versions'), join(dir, '.npm')]);
             execFileSync('chown', ['-h', `${user}:${user}`, current]);
-            if (isSymlink(join(dir, 'node_modules')))
-                execFileSync('chown', ['-h', `${user}:${user}`, join(dir, 'node_modules')]);
+            for (const link of ['node_modules', 'last-good'])
+                if (isSymlink(join(dir, link)))
+                    execFileSync('chown', ['-h', `${user}:${user}`, join(dir, link)]);
+            // Left behind by an earlier privileged run: the unprivileged agent must be able to replace them.
+            const leftovers = ['staging', '.magnus', STATE_FILE, LOCK_FILE, ALIVE_FILE, 'launcher-crashes.json'].map(name => join(dir, name)).filter(path => existsSync(path));
+            if (leftovers.length)
+                execFileSync('chown', ['-R', `${user}:${user}`, ...leftovers]);
         });
     }
     act(`write ${options.unitPath}${existsSync(options.unitPath) ? ` (previous unit kept as ${options.unitPath}.pre-launcher)` : ''}`, () => {

package/dist/upgrade/routes.js

@@ -2,7 +2,7 @@ import { Hono } from 'hono';
 import { agentUpgradeRequestSchema } from '@hivemagnus/magnus-contracts';
 import { logger } from '../utils/logger.js';
 import { AGENT_VERSION } from '../version.js';
-import { SelfUpgradeError, selfUpgradeStatus, stageUpgrade } from './self-upgrade.js';
+import { EXIT_BACKSTOP_MS, SelfUpgradeError, selfUpgradeStatus, stageUpgrade } from './self-upgrade.js';
 /** Seconds between the reply and the exit that hands over to the launcher. */
 export const RESTART_DELAY_MS = 3_000;
 export const agentUpgradeRoutes = new Hono();
@@ -14,9 +14,14 @@ agentUpgradeRoutes.post('/', async (c) => {
     }
     try {
         const result = await stageUpgrade({ request: parsed.data, version: AGENT_VERSION });
+        // A retried request: report the current state, change nothing.
+        if (result.repeated)
+            return c.json({ success: true, data: { accepted: true, ...result } });
         logger.info(`Self-upgrade staged: ${result.from} -> ${result.to}; restarting through the launcher`);
         // Reply first; the launcher starts versions/<to> on trial after this process exits.
         setTimeout(() => process.kill(process.pid, 'SIGTERM'), RESTART_DELAY_MS).unref();
+        // Backstop: a shutdown that hangs must not keep the old version running on the new ./current.
+        setTimeout(() => process.exit(0), EXIT_BACKSTOP_MS).unref();
         return c.json({ success: true, data: { accepted: true, ...result, restartInSeconds: RESTART_DELAY_MS / 1000 } });
     }
     catch (error) {