@hivemagnus/magnus-server 1.1.7 → 1.1.8

package/.env.example

@@ -197,5 +197,6 @@ CORS_ORIGIN=https://magnus.example.com:3000
 # Server-driven agent upgrades (console "Upgrade agents", `magnus-server agents upgrade`).
 # MAGNUS_AGENT_UPGRADES=true
 # MAGNUS_AGENT_UPGRADE_CONCURRENCY=5
+# MAGNUS_AGENT_UPGRADE_MAX_FAILURES=2
 # The retired git-checkout self-update endpoints (source installs only).
 # MAGNUS_LEGACY_GIT_SELF_UPDATE=false

package/agent-version.json

@@ -1,3 +1,3 @@
 {
-  "version": "1.1.7"
+  "version": "1.1.8"
 }

package/dist/cli/agents.js

@@ -1,25 +1,48 @@
-/** `magnus-server agents upgrade`: the console's "Upgrade agents" action from a shell. */
+/** `magnus-server agents upgrade`: the console's "Upgrade agents" action from a shell, and `agents doctor`, its check. */
+import { upgradeRemedy } from '@hivemagnus/magnus-contracts';
 import { UsageError } from './args.js';
 export const AGENTS_USAGE = `Usage: magnus-server agents upgrade [options] [--env-file FILE]
+       magnus-server agents doctor [--nodes a,b] [--to X.Y.Z] [--heal] [--env-file FILE]
+       magnus-server agents fence NODE VALUE [--env-file FILE]
        magnus-server agents stop RUN_ID [--env-file FILE]
 
-Upgrades agents to the version bundled with this server (or --to), one canary
-first, then MAGNUS_AGENT_UPGRADE_CONCURRENCY at a time. Each agent installs the
-exact version beside the running one and switches back by itself if the new
-version does not report a heartbeat in time. A revert, a release problem or an
-agent that goes silent stops the run; an agent that refuses before changing
-anything (busy, enrollment, unreachable) is reported as refused and the run
-goes on.
+upgrade: moves agents to the version bundled with this server (or --to). Every
+selected node is checked first and blocked nodes are left out with the reason.
+One canary goes first, then MAGNUS_AGENT_UPGRADE_CONCURRENCY at a time. Each
+agent installs the exact version beside the running one and switches back by
+itself if the new version does not report a heartbeat in time.
+
+Only evidence about the release stops a run: an integrity or source failure at
+once, a failure on the canary, ambiguous answers from canaries with nothing
+confirmed, and after that MAGNUS_AGENT_UPGRADE_MAX_FAILURES reverts or failed
+installs. A problem of one node (busy, enrollment, unreachable,
+a lost reply) is recorded and the run goes on; transient ones are tried again at
+the end. The summary lists each node needing attention, with what to do.
+
+Exit status: 0 succeeded, 2 partial (some nodes need attention), 1 otherwise.
 
 Options:
   --to X.Y.Z            Agent version (default: the one bundled with this server)
   --integrity sha512-…  npm integrity of that version (default: read from the registry)
   --nodes a,b           Only these nodes (names, host names or ids)
   --canary NODE         Upgrade this node first (default: the first eligible one)
+  --max-failures N      Release failures that stop this run (default: MAGNUS_AGENT_UPGRADE_MAX_FAILURES)
   --allow-prerelease    Required for a prerelease version such as 1.2.0-rc.1
   --retry-reverted      Also offer nodes that already reverted this version
-  --retry RUN_ID        Retry the failed, refused, skipped and interrupted nodes of a run
-  --dry-run             Show which agents would be upgraded, and why others would not
+  --retry RUN_ID        Retry the nodes of a run that were not upgraded
+  --acknowledge-suspicion  With --retry: also retry older agents after a run that stopped on their ambiguous answers
+  --dry-run             Show the full per-node report and start nothing
+
+doctor: the same read-only check for every node: what each agent reports about
+its layout, launcher, disk, npm, ledger, fences and enrollment, and what to do
+about each problem. --heal also repairs what the server owns: it moves a fence
+generation up to what the agent signs that it holds, and settles leases stuck
+unknown when the agent's ledger proves they never ran. Exit status 1 when a node
+is blocked.
+
+fence: moves the server's fence generation for a node to VALUE + 1. Only ever up,
+and audited. For agents older than 1.1.8, which do not say what fence they hold:
+read it from .magnus/command-receipts/fence-*.json on the node.
 
 Ctrl-C asks the run to stop after the nodes in progress; a second Ctrl-C leaves
 at once, and the run is then shown as interrupted.
@@ -38,9 +61,19 @@ export function parseAgentsArgs(args) {
             throw new UsageError('agents stop takes one RUN_ID');
         return { stop: id, dryRun: false };
     }
-    if (sub !== 'upgrade')
+    if (sub === 'fence') {
+        const [node, value, ...extra] = rest;
+        if (node === '--help' || node === '-h')
+            return 'help';
+        if (!node || !value || node.startsWith('--') || extra.length)
+            throw new UsageError('agents fence takes NODE and VALUE');
+        if (!/^[1-9][0-9]{0,18}$/.test(value))
+            throw new UsageError('agents fence: VALUE must be a positive decimal');
+        return { fence: { node, value }, dryRun: false };
+    }
+    if (sub !== 'upgrade' && sub !== 'doctor')
         throw new UsageError(`unknown agents subcommand "${sub}"`);
-    const options = { dryRun: false };
+    const options = { dryRun: false, ...(sub === 'doctor' ? { doctor: true } : {}) };
     for (let index = 0; index < rest.length; index++) {
         const arg = rest[index];
         const [name, inline] = arg.includes('=') ? [arg.slice(0, arg.indexOf('=')), arg.slice(arg.indexOf('=') + 1)] : [arg, undefined];
@@ -64,8 +97,18 @@ export function parseAgentsArgs(args) {
             options.allowPrerelease = true;
         else if (name === '--retry-reverted')
             options.retryReverted = true;
+        else if (name === '--acknowledge-suspicion')
+            options.acknowledgeSuspicion = true;
         else if (name === '--dry-run')
             options.dryRun = true;
+        else if (name === '--heal' && sub === 'doctor')
+            options.heal = true;
+        else if (name === '--max-failures') {
+            const budget = Number(value());
+            if (!Number.isInteger(budget) || budget < 1 || budget > 1000)
+                throw new UsageError('--max-failures must be a whole number from 1 to 1000');
+            options.maxFailures = budget;
+        }
         else if (name === '--help' || name === '-h')
             return 'help';
         else
@@ -77,6 +120,34 @@ export function parseAgentsArgs(args) {
         throw new UsageError('--retry reuses the version, integrity and nodes of that run; drop --to, --integrity, --nodes and --dry-run');
     return options;
 }
+const remedyLines = (item) => [
+    `  ${item.name}: ${item.code}${item.unknownCode ? ' [unrecognised code]' : ''}${item.detail ? ` (${item.detail})` : ''}`,
+    `    cause:  ${item.cause}`,
+    `    remedy: ${item.remedy}`,
+];
+/** What one node's check found, in the order someone would act on it. */
+export function doctorLines(report) {
+    const lines = [`Target magnus-agent ${report.version}${report.release.verified ? '' : ` (not verified on the registry: ${report.release.reason ?? 'unknown'})`}`];
+    for (const node of report.nodes) {
+        const preflight = node.preflight;
+        const verdict = node.eligible ? 'ready' : node.code && upgradeRemedy(node.code).informational ? 'not applicable' : 'blocked';
+        const facts = preflight?.report ? [preflight.report.layout, preflight.report.disk ? `${Math.floor(preflight.report.disk.freeBytes / (1024 * 1024))} MiB free` : null,
+            preflight.report.npm?.version ? `npm ${preflight.report.npm.version}` : null].filter(Boolean).join(', ') : `preflight ${preflight?.status ?? 'not run'}${preflight?.detail ? `: ${preflight.detail}` : ''}`;
+        lines.push(`${node.name}  ${node.version}  ${verdict}  (${facts})`);
+        if (!node.eligible && node.code)
+            lines.push(...remedyLines({ name: 'reason', code: node.code, detail: node.reason, cause: upgradeRemedy(node.code).cause, remedy: upgradeRemedy(node.code).remedy }));
+        for (const problem of preflight?.problems ?? []) {
+            if (problem.code === node.code)
+                continue;
+            const remedy = upgradeRemedy(problem.code);
+            lines.push(...remedyLines({ name: problem.blocking ? 'blocks' : 'note', code: problem.code, detail: problem.detail, cause: remedy.cause, remedy: remedy.remedy }));
+        }
+        for (const healed of preflight?.healed ?? [])
+            lines.push(`  repaired: ${healed}`);
+    }
+    lines.push(`${report.eligible} ready, ${report.blocked} blocked, ${report.nodes.length - report.eligible - report.blocked} not applicable.`);
+    return lines;
+}
 const nodeLine = (node) => `  ${node.name}: ${node.state}${node.detail ? ` (${node.code ? `${node.code}: ` : ''}${node.detail})` : node.code ? ` (${node.code})` : ''}`;
 export async function commandAgents(args) {
     const options = parseAgentsArgs(args);
@@ -93,6 +164,21 @@ export async function commandAgents(args) {
             console.log(`Asked run ${run.id} to stop; it stops before its next batch.`);
             return 0;
         }
+        if (options.fence) {
+            const row = await getDb().node.findFirst({ where: { OR: [{ id: options.fence.node }, { name: options.fence.node }, { hostname: options.fence.node }] }, select: { id: true, name: true } });
+            if (!row) {
+                console.error(`Unknown node: ${options.fence.node}`);
+                return 1;
+            }
+            const { advanceCommandFence } = await import('../services/command-fence-repair.js');
+            const moved = await advanceCommandFence({ nodeId: row.id, resourceKey: `node:${row.id}`, highWater: options.fence.value, source: 'operator', actorId: 'system:magnus-server-cli' });
+            if (moved.status === 'rejected') {
+                console.error(`${moved.code}: node ${row.name} ${moved.detail}`);
+                return 1;
+            }
+            console.log(moved.status === 'advanced' ? `${row.name}: fence generation moved from ${moved.from} to ${moved.to}.` : `${row.name}: the fence generation is already past ${options.fence.value}; nothing changed.`);
+            return 0;
+        }
         const resolveNodes = async (names) => {
             const rows = await getDb().node.findMany({
                 where: { OR: [{ id: { in: names } }, { name: { in: names } }, { hostname: { in: names } }] },
@@ -116,6 +202,12 @@ export async function commandAgents(args) {
                 canaryNodeId = resolved.byName(options.canary);
         }
         const deps = service.defaultDeps();
+        if (options.doctor) {
+            const report = await service.checkUpgrade({ version: options.to, integrity: options.integrity, nodeIds, heal: options.heal ?? false, deps, allowRetryReverted: options.retryReverted });
+            for (const line of doctorLines(report))
+                console.log(line);
+            return report.blocked ? 1 : 0;
+        }
         const seen = new Map();
         const save = deps.save;
         deps.save = async (run) => {
@@ -131,27 +223,47 @@ export async function commandAgents(args) {
         let started;
         if (options.retry) {
             started = await service.retryUpgradeRun(options.retry, { actorId: 'system:magnus-server-cli', deps, canaryNodeId,
-                allowRetryReverted: options.retryReverted, allowPrerelease: options.allowPrerelease });
+                allowRetryReverted: options.retryReverted, allowPrerelease: options.allowPrerelease, acknowledgeSuspicion: options.acknowledgeSuspicion, ...(options.maxFailures ? { maxFailures: options.maxFailures } : {}) });
             console.log(`Retrying run ${options.retry} as ${started.run.id}: ${started.run.nodes.length} agent(s) to ${started.run.version}`);
         }
         else {
             const release = await service.resolveAgentRelease({ version: options.to, integrity: options.integrity });
-            const candidates = await service.listUpgradeCandidates(release.version, nodeIds, { allowRetryReverted: options.retryReverted });
             console.log(`magnus-agent ${release.version} (${release.integrity.slice(0, 19)}…)`);
-            for (const candidate of candidates) {
-                console.log(`  ${candidate.eligible ? '→' : '·'} ${candidate.name.padEnd(28)} ${candidate.version.padEnd(12)} ${candidate.eligible ? 'will upgrade' : candidate.reason}`);
-            }
-            const eligible = candidates.filter(candidate => candidate.eligible);
+            const showCandidates = (candidates) => {
+                for (const candidate of candidates) {
+                    console.log(`  ${candidate.eligible ? '→' : '·'} ${candidate.name.padEnd(28)} ${candidate.version.padEnd(12)} ${candidate.eligible ? 'will upgrade' : candidate.reason}`);
+                }
+            };
             if (options.dryRun) {
-                console.log(`Dry run: ${eligible.length} of ${candidates.length} agent(s) would be upgraded.`);
+                const report = await service.checkUpgrade({ version: release.version, integrity: release.integrity, nodeIds, deps, everyOnlineNode: false, allowRetryReverted: options.retryReverted });
+                showCandidates(report.nodes);
+                for (const node of report.nodes) {
+                    if (!node.eligible && node.code && !upgradeRemedy(node.code).informational)
+                        console.log(`    ${node.name}: ${node.code}: ${node.remedy}`);
+                    if (node.preflight?.status === 'unavailable')
+                        console.log(`    ${node.name}: preflight unavailable (${node.preflight.detail ?? 'older agent'}); heartbeat data was used`);
+                }
+                console.log(`Dry run: ${report.eligible} of ${report.nodes.length} agent(s) would be upgraded.`);
                 return 0;
             }
-            if (eligible.length === 0) {
+            try {
+                started = await service.startUpgradeRun({ version: release.version, integrity: release.integrity, nodeIds, canaryNodeId, allowPrerelease: options.allowPrerelease,
+                    allowRetryReverted: options.retryReverted, actorId: 'system:magnus-server-cli', deps, ...(options.maxFailures ? { maxFailures: options.maxFailures } : {}) });
+            }
+            catch (error) {
+                if (error.code !== 'NO_ELIGIBLE_AGENTS')
+                    throw error;
+                const candidates = await service.listUpgradeCandidates(release.version, nodeIds, { allowRetryReverted: options.retryReverted });
+                showCandidates(candidates);
                 console.log('Nothing to upgrade.');
                 return candidates.some(candidate => candidate.reason !== 'already on this version') ? 1 : 0;
             }
-            started = await service.startUpgradeRun({ version: release.version, integrity: release.integrity, nodeIds: eligible.map(node => node.nodeId),
-                canaryNodeId, allowPrerelease: options.allowPrerelease, allowRetryReverted: options.retryReverted, actorId: 'system:magnus-server-cli', deps });
+            showCandidates(started.candidates);
+            for (const candidate of started.candidates)
+                for (const healed of candidate.preflight?.healed ?? [])
+                    console.log(`    ${candidate.name}: repaired: ${healed}`);
+            for (const warning of started.run.warnings ?? [])
+                console.warn(`    ${warning.name}: ${warning.code}: ${warning.detail}`);
         }
         let interrupts = 0;
         const onSigint = () => {
@@ -168,9 +280,17 @@ export async function commandAgents(args) {
             for (const node of run.excluded ?? [])
                 if (options.nodes || options.retry)
                     console.log(`  ${node.name}: not upgraded (${node.reason})`);
-            console.log(`${run.status === 'succeeded' ? 'Done' : run.status === 'stopped' ? 'Stopped' : 'Failed'}: ${run.message ?? run.status}`);
-            console.log(`Run ${run.id}${run.nodes.every(node => node.state === 'confirmed') ? '' : `; retry the rest with: magnus-server agents upgrade --retry ${run.id}`}`);
-            return run.status === 'succeeded' ? 0 : 1;
+            if (run.suspectedRelease)
+                console.log(`Suspected release problem: ${run.suspectedRelease}`);
+            if (run.attention?.length) {
+                console.log(`Needs attention (${run.attention.length}):`);
+                for (const item of run.attention)
+                    for (const line of remedyLines(item))
+                        console.log(line);
+            }
+            console.log(`${run.status === 'succeeded' ? 'Done' : run.status === 'partial' ? 'Partial' : run.status === 'stopped' ? 'Stopped' : 'Failed'}: ${run.message ?? run.status}`);
+            console.log(`Run ${run.id}${run.nodes.every(node => node.state === 'confirmed') && !run.attention?.length ? '' : `; retry the rest with: magnus-server agents upgrade --retry ${run.id}`}`);
+            return run.status === 'succeeded' ? 0 : run.status === 'partial' ? 2 : 1;
         }
         finally {
             process.off('SIGINT', onSigint);

package/dist/cli/args.js

@@ -11,7 +11,7 @@ Commands:
   restore FILE  Replace the database with a dump (stop the server first; asks unless --yes)
   upgrade       Upgrade a \`magnus init\` project (runs \`magnus upgrade\`; see --help there)
   rollback      Switch a \`magnus init\` project back to the previous version
-  agents        Agent fleet commands: \`agents upgrade [--to X.Y.Z] [--nodes a,b] [--dry-run]\`
+  agents        Agent fleet commands: \`agents upgrade [--to X.Y.Z] [--nodes a,b] [--dry-run]\`, \`agents doctor [--nodes a,b]\`
   env-schema    Print the documented environment variables
   help          Show this help
   version       Print the version

package/dist/cli/env-catalog.js

@@ -111,6 +111,7 @@ export const ENV_CATALOG = [
     { name: 'MAGNUS_UPDATE_REGISTRY', group: 'upgrades', default: 'https://registry.npmjs.org', description: 'npm registry used by the update check.' },
     { name: 'MAGNUS_AGENT_UPGRADES', group: 'upgrades', default: 'true', description: 'Allow server-driven agent upgrades (console action, API and `magnus-server agents upgrade`). Agents must also allow it (MAGNUS_AGENT_SELF_UPGRADE).' },
     { name: 'MAGNUS_AGENT_UPGRADE_CONCURRENCY', group: 'upgrades', default: '5', description: 'Agents upgraded in parallel after the first (canary) agent confirms. Maximum 50.' },
+    { name: 'MAGNUS_AGENT_UPGRADE_MAX_FAILURES', group: 'upgrades', default: '2', description: 'Failures pointing at the release (revert, failed install, trial that never confirms) that stop an agent upgrade run after its canary. Problems of one node never stop a run. A run can override it.' },
     { name: 'MAGNUS_LEGACY_GIT_SELF_UPDATE', group: 'upgrades', default: 'false', description: 'true: keep the retired git-checkout self-update endpoints (/api/v1/self-update/apply and /restart) for source installs. npm installs use `magnus upgrade`.' },
     { name: 'NODE_EXTRA_CA_CERTS', group: 'upgrades', description: 'Extra CA bundle (PEM) trusted when the server calls agents, for agent certificates from a private CA. Node reads it at process start: set it in the unit; `magnus-server agents` also honours it from --env-file.' },
     { name: 'MAGNUS_PROJECT_DIR', group: 'upgrades', description: 'Managed project created by `magnus init` (holds magnus-project.json). `magnus-server upgrade` finds it from the install path when unset.' },

package/dist/config/features.js

@@ -24,6 +24,7 @@ export function envSwitch(value, fallback) {
 }
 export function upgradeSettings(env = process.env) {
     const concurrency = Number.parseInt(env['MAGNUS_AGENT_UPGRADE_CONCURRENCY'] ?? '', 10);
+    const maxFailures = Number.parseInt(env['MAGNUS_AGENT_UPGRADE_MAX_FAILURES'] ?? '', 10);
     return {
         autoMigrate: envSwitch(env['MAGNUS_AUTO_MIGRATE'], false),
         autoMigrateBackup: envSwitch(env['MAGNUS_AUTO_MIGRATE_BACKUP'], true),
@@ -32,6 +33,7 @@ export function upgradeSettings(env = process.env) {
         updateRegistry: (env['MAGNUS_UPDATE_REGISTRY']?.trim() || 'https://registry.npmjs.org').replace(/\/+$/, ''),
         agentUpgrades: envSwitch(env['MAGNUS_AGENT_UPGRADES'], true),
         agentUpgradeConcurrency: Number.isInteger(concurrency) && concurrency > 0 ? Math.min(concurrency, 50) : 5,
+        agentUpgradeMaxFailures: Number.isInteger(maxFailures) && maxFailures > 0 ? Math.min(maxFailures, 1000) : 2,
         legacyGitSelfUpdate: envSwitch(env['MAGNUS_LEGACY_GIT_SELF_UPDATE'], false),
     };
 }

package/dist/index.js

@@ -24,6 +24,7 @@ const { startRolloutReconciler, stopRolloutReconciler } = await import('./servic
 const { startApplicationProbes, stopApplicationProbes } = await import('./services/application-probes.js');
 const { startNodeIssueMonitor } = await import('./services/node-issue-monitor.js');
 const { stopNodeIssueMonitor } = await import('./services/node-issue-monitor.js');
+const { startUpgradeReadinessMonitor, stopUpgradeReadinessMonitor } = await import('./services/agent-upgrade-readiness.js');
 const { startLogRetention } = await import('./services/log-retention.js');
 const { stopLogRetention } = await import('./services/log-retention.js');
 const { startHealthRetention } = await import('./services/health-retention.js');
@@ -58,6 +59,7 @@ const { CONTROL_API_VERSION } = await import('./config/version.js');
 function startBackgroundServices(listenPort) {
     writeRuntimeFile({ version: CONTROL_API_VERSION, port: listenPort, host: listener.hostname });
     startNodeIssueMonitor();
+    startUpgradeReadinessMonitor();
     startApplicationProbes();
     startLogRetention();
     startHealthRetention();
@@ -130,6 +132,7 @@ async function shutdown(signal) {
         await Promise.race([(async () => {
                 stopRolloutReconciler();
                 stopNodeIssueMonitor();
+                stopUpgradeReadinessMonitor();
                 stopApplicationProbes();
                 stopLogRetention();
                 stopHealthRetention();

package/dist/openapi/spec.js

@@ -655,7 +655,7 @@ const paths = {
         get: {
             tags: ['Nodes'],
             summary: 'List nodes',
-            description: 'Returns all fleet nodes with agent info and health metadata. Supports pagination and status filtering.',
+            description: 'Returns all fleet nodes with agent info and health metadata. Supports pagination and status filtering. Each node has `upgradeReadiness`: `status` (ok, not-ready, or unknown for agents that do not report it) and `problems` with a code, a detail, a cause and a remedy, such as an enrollment stuck draining, a marker the server does not share, a fence ahead of the server, an unresolved ledger, low disk or missing npm.',
             security: apiSecretAuth,
             parameters: [
                 qp('page', 'Page number (1-based)', { type: 'integer' }),
@@ -2156,7 +2156,7 @@ const paths = {
         get: {
             tags: ['Agent upgrades'],
             summary: 'Which agents can take a version',
-            description: 'For every node: its agent version and whether it can upgrade itself to `version` (default: the agent version bundled with this server) now, or why not: offline, not set up with `magnus-agent install-service`, MAGNUS_AGENT_SELF_UPGRADE=false, another major version, already current, already reverted this version, a closed release admission fence, a resource enrollment that is not complete (for example "stuck in draining since <date>"), held participant work or command leases, or an agent whose sticky managed-enrollment marker the server does not share. `release.verified` is false when the registry does not have the version.',
+            description: 'For every node: its agent version and whether it can upgrade itself to `version` (default: the agent version bundled with this server) now, or why not: offline, not set up with `magnus-agent install-service`, MAGNUS_AGENT_SELF_UPGRADE=false, another major version, already current, already reverted this version, a closed release admission fence, a resource enrollment that is not complete (for example "stuck in draining since <date>"), held participant work or command leases, or an agent whose sticky managed-enrollment marker the server does not share. Each blocked node has a `code` from the upgrade problem catalogue. `preflight` is the latest result of the agent check for this version, when one is recent, and only for ADMIN. `release.verified` is false when the registry does not have the version.',
             security: apiSecretAuth,
             parameters: [qp('version', 'Target agent version (X.Y.Z)'), qp('allowRetryReverted', '`true` to offer nodes that already reverted this version')],
             responses: { 200: success('Eligibility', { type: 'object', properties: {
@@ -2164,7 +2164,7 @@ const paths = {
                         prerelease: { type: 'boolean' },
                         release: { type: 'object', properties: { verified: { type: 'boolean' }, integrity: { type: ['string', 'null'] }, reason: { type: ['string', 'null'] } } },
                         nodes: { type: 'array', items: { type: 'object', properties: { nodeId: { type: 'string' }, name: { type: 'string' },
-                                    version: { type: 'string' }, eligible: { type: 'boolean' }, reason: { type: ['string', 'null'] } } } },
+                                    version: { type: 'string' }, eligible: { type: 'boolean' }, reason: { type: ['string', 'null'] }, code: { type: ['string', 'null'] }, preflight: { type: ['object', 'null'], description: 'status: ok, warning, blocked, unavailable (older agent) or unreachable; problems: code, detail, blocking' } } } },
                         latestRun: { type: ['object', 'null'] },
                     } }), 400: err400, 401: err401 },
         },
@@ -2173,7 +2173,7 @@ const paths = {
         post: {
             tags: ['Agent upgrades'],
             summary: 'Upgrade agents',
-            description: 'Starts an upgrade run (ADMIN, MAGNUS_AGENT_UPGRADES=true): one canary, then MAGNUS_AGENT_UPGRADE_CONCURRENCY agents at a time. Each agent installs the exact version and npm integrity and switches back by itself if the new version does not report a heartbeat in time. A revert, a release problem (integrity, install failure) or an agent that goes silent after the request stops the run. An agent that refuses before changing anything is recorded as `refused` and the run goes on; a refused canary hands the role to the next node. Only one run at a time. Selected nodes that are not eligible are listed in `excluded`; unknown node ids are a 400.',
+            description: 'Starts an upgrade run (ADMIN, MAGNUS_AGENT_UPGRADES=true): every selected node is checked first, then one canary, then MAGNUS_AGENT_UPGRADE_CONCURRENCY agents at a time. Each agent installs the exact version and npm integrity and switches back by itself if the new version does not report a heartbeat in time. With `dryRun` it returns 200 with the full per-node report and starts nothing. Only evidence about the release stops a run: an integrity or source failure at once, a release failure on the canary, `maxFailures` ambiguous canaries with nothing confirmed (at least two, reported as suspect but unproven in `suspectedRelease`; nodes that cannot report their fence go last and do not count while another may still confirm), and after the canary `maxFailures` reverts, failed installs or trials that never confirm. A problem of one node is recorded and the run goes on: a refused canary hands the role to the next node, transient refusals are tried again at the end of the wave, and a node whose reply was lost is watched by request id (`watching`, then `ambiguous`) without holding a batch slot past its window. The final status is `succeeded`, `partial` (node-level problems, nothing against the release), `failed` (release evidence), `stopped` or `interrupted`, and `attention` lists each node with a code, a cause and a remedy; a run is `succeeded` only if every selected node is confirmed or has an informational reason. `warnings` lists what the check found that does not stop a node. Only one run at a time. Selected nodes that are not eligible are listed in `excluded`; unknown node ids are a 400.',
             security: apiSecretAuth,
             requestBody: { required: false, content: { 'application/json': { schema: { type: 'object', properties: {
                                 version: { type: 'string', description: 'Agent version (default: the one bundled with this server)' },
@@ -2182,8 +2182,10 @@ const paths = {
                                 canaryNodeId: { type: 'string', description: 'Upgrade this node first' },
                                 allowPrerelease: { type: 'boolean', description: 'Required for a prerelease version' },
                                 allowRetryReverted: { type: 'boolean', description: 'Also upgrade nodes that already reverted this version' },
+                                dryRun: { type: 'boolean', description: 'Report on every selected node and start nothing' },
+                                maxFailures: { type: 'integer', minimum: 1, maximum: 1000, description: 'Release failures that stop this run (default MAGNUS_AGENT_UPGRADE_MAX_FAILURES)' },
                             } } } } },
-            responses: { 202: success('Run started'), 400: err400, 401: err401, 403: err403, 409: err409 },
+            responses: { 200: success('Dry run report'), 202: success('Run started'), 400: err400, 401: err401, 403: err403, 409: err409 },
         },
     },
     '/agents/upgrade/runs/latest': {
@@ -2218,11 +2220,11 @@ const paths = {
         post: {
             tags: ['Agent upgrades'],
             summary: 'Retry an upgrade run',
-            description: 'ADMIN. Starts a new run with the same version and integrity for the failed, refused, skipped and interrupted nodes of a finished run (reverted nodes only with allowRetryReverted).',
+            description: 'ADMIN. Starts a new run with the same version and integrity for the nodes of a finished run that were not upgraded: failed, refused, skipped, ambiguous and interrupted ones, and those the check held back (reverted nodes only with allowRetryReverted).',
             security: apiSecretAuth,
             parameters: [pp('id', 'Run ID')],
             requestBody: { required: false, content: { 'application/json': { schema: { type: 'object', properties: {
-                                canaryNodeId: { type: 'string' }, allowRetryReverted: { type: 'boolean' },
+                                canaryNodeId: { type: 'string' }, allowRetryReverted: { type: 'boolean' }, maxFailures: { type: 'integer', minimum: 1, maximum: 1000 }, acknowledgeSuspicion: { type: 'boolean', description: 'After a run that stopped on ambiguous answers from older agents, include older agents again; without this they are held back with RETRY_HELD_SUSPECTED' },
                             } } } } },
             responses: { 202: success('Run started'), 400: err400, 401: err401, 403: err403, 404: err404, 409: err409 },
         },

package/dist/routes/agent-upgrades.js

@@ -4,12 +4,13 @@ import { authMiddleware, getUser, requireRole } from '../middleware/auth.js';
 import { HttpError } from '../middleware/error-handler.js';
 import { upgradeSettings } from '../config/features.js';
 import { AGENT_VERSION } from '../config/version.js';
-import { getUpgradeRun, isPrerelease, latestUpgradeRun, listUpgradeCandidates, retryUpgradeRun, startUpgradeRun, stopUpgradeRun, verifyAgentRelease } from '../services/agent-upgrade-service.js';
+import { checkUpgrade, getUpgradeRun, isPrerelease, latestUpgradeRun, listUpgradeCandidates, retryUpgradeRun, startUpgradeRun, stopUpgradeRun, verifyAgentRelease, withCachedPreflight } from '../services/agent-upgrade-service.js';
 /** /api/v1/agents/upgrade: which agents can take a version, and fleet upgrade runs. */
 export const agentUpgradesRouter = new Hono();
 agentUpgradesRouter.use('*', authMiddleware);
 const version = z.string().regex(/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/);
 const nodeId = z.string().min(1).max(128);
+const maxFailures = z.number().int().min(1).max(1000);
 agentUpgradesRouter.get('/eligibility', async (c) => {
     const target = version.safeParse(c.req.query('version') ?? AGENT_VERSION);
     if (!target.success)
@@ -21,7 +22,9 @@ agentUpgradesRouter.get('/eligibility', async (c) => {
     return c.json({ success: true, data: {
             version: target.data, serverVersion: AGENT_VERSION, upgradesEnabled: upgradeSettings().agentUpgrades,
             disabledReason: upgradeSettings().agentUpgrades ? null : 'MAGNUS_AGENT_UPGRADES=false on the server',
-            prerelease: isPrerelease(target.data), release, nodes, latestRun,
+            prerelease: isPrerelease(target.data), release,
+            // Reports hold paths, fences and command ids: only the role that can run upgrades sees them.
+            nodes: getUser(c).role === 'ADMIN' ? withCachedPreflight(nodes, target.data) : nodes.map(({ preflight: _preflight, ...node }) => node), latestRun,
         } });
 });
 const startSchema = z.object({
@@ -31,6 +34,9 @@ const startSchema = z.object({
     canaryNodeId: nodeId.optional(),
     allowPrerelease: z.boolean().optional(),
     allowRetryReverted: z.boolean().optional(),
+    /** Report on every selected node and start nothing: the console's Check action. */
+    dryRun: z.boolean().optional(),
+    maxFailures: maxFailures.optional(),
 }).strict();
 const parseBody = async (schema, raw) => {
     const parsed = schema.safeParse(raw);
@@ -40,7 +46,10 @@ const parseBody = async (schema, raw) => {
 };
 agentUpgradesRouter.post('/', requireRole('ADMIN'), async (c) => {
     const body = await parseBody(startSchema, await c.req.json().catch(() => ({})));
-    const { run } = await startUpgradeRun({ ...body, actorId: getUser(c).userId });
+    const { dryRun, ...options } = body;
+    if (dryRun)
+        return c.json({ success: true, data: { dryRun: true, ...await checkUpgrade(options) } });
+    const { run } = await startUpgradeRun({ ...options, actorId: getUser(c).userId });
     return c.json({ success: true, data: run }, 202);
 });
 agentUpgradesRouter.get('/runs/latest', async (c) => c.json({ success: true, data: await latestUpgradeRun() }));
@@ -54,6 +63,8 @@ agentUpgradesRouter.post('/runs/:id/stop', requireRole('ADMIN'), async (c) => c.
 const retrySchema = z.object({
     canaryNodeId: nodeId.optional(),
     allowRetryReverted: z.boolean().optional(),
+    maxFailures: maxFailures.optional(),
+    acknowledgeSuspicion: z.boolean().optional(),
 }).strict();
 agentUpgradesRouter.post('/runs/:id/retry', requireRole('ADMIN'), async (c) => {
     const body = await parseBody(retrySchema, await c.req.json().catch(() => ({})));

package/dist/routes/nodes.js

@@ -17,6 +17,7 @@ import { getNodeBaseUrl } from '../utils/node-url.js';
 import { getNodeManagerSecret } from '../utils/env.js';
 import { ensureControlHostNode } from '../services/control-host-node.js';
 import { withReadRetry } from '../db/read-retry.js';
+import { readinessOfNodes } from '../services/agent-upgrade-readiness.js';
 import { parseTailscaleNodeKeyPosture, summarizeTailscaleNodeKeys, TAILSCALE_RENEWAL_WARNING_DAYS, } from '../services/tailscale-node-key-service.js';
 function parseSystemUpdatePosture(value) {
     try {
@@ -116,9 +117,10 @@ nodesRouter.get('/', async (c) => {
     ]));
     // Strip token from API response — never expose node secrets to the frontend
     const safeNodes = nodes.map(({ token: _token, ...rest }) => rest);
+    const readiness = await readinessOfNodes(safeNodes);
     return c.json({
         success: true,
-        data: safeNodes,
+        data: safeNodes.map(node => ({ ...node, upgradeReadiness: readiness.get(node.id) ?? null })),
         meta: { total, page, pageSize, hasMore: page * pageSize < total },
     });
 });
@@ -494,7 +496,8 @@ nodesRouter.get('/:nodeId', async (c) => {
     if (!node)
         throw new HttpError(404, 'NOT_FOUND', 'Node not found');
     const { token: _token, ...safeNode } = node;
-    return c.json({ success: true, data: safeNode });
+    const readiness = await readinessOfNodes([safeNode]);
+    return c.json({ success: true, data: { ...safeNode, upgradeReadiness: readiness.get(safeNode.id) ?? null } });
 });
 // ── Register a new node ──
 nodesRouter.post('/register', requireRole('ADMIN', 'OPERATOR'), async (c) => {

package/dist/sdk/node-api-client.js

@@ -5,11 +5,13 @@ import { COMMAND_COMPLETION_REGISTRY, COMMAND_COMPLETION_REGISTRY_SHA256, comman
 import { commandReconciliationSchema } from '../schemas/command-reconciliation.js';
 import { admitResourceCommand, markResourceDispatched, failResourceCommand, settleResourceResult, settleResourceReconciliation } from '../services/command-resource-leases.js';
 import { commandFenceSchema, commandResultSchema } from '../schemas/node-responses.js';
+import { advanceCommandFence } from '../services/command-fence-repair.js';
 import { commandRequestDigest } from '../security/command-request.js';
 import ky, { HTTPError } from 'ky';
 import { HttpError } from '../middleware/error-handler.js';
 import { createHmac, timingSafeEqual } from 'node:crypto';
 import { z } from 'zod';
+import { agentUpgradePreflightSchema } from '@hivemagnus/magnus-contracts';
 import { RESULT_SIGNING_PREVIOUS_SECRET, RESULT_SIGNING_SECRET } from '../utils/env.js';
 import { correlateSkillPublicationNoEffect } from '../services/skill-publication-settlement.js';
 import { commandResultSigningPayload, verifyCommandResultSignature, } from '../security/command-result-signature.js';
@@ -125,10 +127,52 @@ export function isConnectError(error) {
 function errorText(error) {
     return (error instanceof Error ? error.message : String(error)).slice(0, 1000);
 }
+const certainRefusalOf = (error) => error?.certainRefusal;
+/** The signed refusal in an agent's error reply. An older agent, or a reply without one, says nothing about execution. */
+export function parseAgentRefusal(error) {
+    if (!(error instanceof HTTPError))
+        return null;
+    const body = (error.data ?? null);
+    const refusal = body?.error?.refusal;
+    if (body?.error?.executed !== false || !refusal || typeof refusal.commandId !== 'string' || typeof refusal.requestSha256 !== 'string'
+        || typeof refusal.code !== 'string' || typeof refusal.signature !== 'string' || !(refusal.highWater === null || typeof refusal.highWater === 'string')
+        || typeof refusal.nodeFqdn !== 'string' || typeof refusal.leaseId !== 'string')
+        return null;
+    return { commandId: refusal.commandId, requestSha256: refusal.requestSha256, code: refusal.code, highWater: refusal.highWater, nodeFqdn: refusal.nodeFqdn, leaseId: refusal.leaseId, signature: refusal.signature,
+        message: typeof body.error.message === 'string' ? body.error.message.slice(0, 1000) : refusal.code };
+}
+/**
+ * Close the lease of a dispatch that threw. A verified refusal settles it as
+ * not-dispatched, and a STALE_FENCE moves the server's fence past the agent's
+ * high-water so the next request can pass. Everything else stays `unknown`.
+ */
+async function settleFailedDispatch(admission, actorId, error) {
+    const refusal = parseAgentRefusal(error);
+    const settled = await failResourceCommand(admission, refusal ?? undefined);
+    if (!refusal || !settled)
+        return;
+    let code = refusal.code;
+    let { message } = refusal;
+    const resourceKey = admission.payload.coordination?.resourceKey;
+    if (code === 'STALE_FENCE' && refusal.highWater && resourceKey) {
+        try {
+            const advanced = await advanceCommandFence({ nodeId: admission.nodeId, resourceKey, highWater: refusal.highWater, source: 'stale-fence-reply', actorId });
+            if (advanced.status === 'rejected')
+                ({ code, detail: message } = advanced);
+        }
+        catch (repairError) {
+            console.warn(`[resource-leases] fence of ${admission.nodeId} could not be advanced: ${repairError instanceof Error ? repairError.message : String(repairError)}`);
+        }
+    }
+    Object.assign(error, { certainRefusal: { code, message } });
+}
 /** Sort an upgradeAgent failure by whether the request can have reached the agent. */
 export function classifyAgentRequestError(error, dispatched) {
+    const refused = certainRefusalOf(error);
+    if (refused)
+        return new NotDispatchedError(refused.code, refused.message, error);
     if (!dispatched)
-        return error instanceof NotDispatchedError ? error : new NotDispatchedError(error instanceof HttpError ? error.code : error instanceof AgentResponseError && error.code ? error.code : 'NOT_DISPATCHED', errorText(error), error);
+        return error instanceof NotDispatchedError ? error : new NotDispatchedError(error instanceof HttpError ? error.code : error instanceof AgentResponseError && error.code ? error.code : isConnectError(error) ? 'AGENT_UNREACHABLE' : 'NOT_DISPATCHED', errorText(error), error);
     if (error instanceof AgentResponseError)
         return error;
     if (error instanceof HTTPError)
@@ -225,7 +269,7 @@ export class NodeApiClient {
                     acknowledgedAt: new Date() } };
         }
         catch (error) {
-            await failResourceCommand(admission);
+            await settleFailedDispatch(admission, this.resourceContext.actorId, error);
             throw error;
         }
     }
@@ -324,7 +368,7 @@ export class NodeApiClient {
         }
         catch (error) {
             if (admission)
-                await failResourceCommand(admission);
+                await settleFailedDispatch(admission, this.resourceContext.actorId, error);
             throw error;
         }
     }
@@ -506,6 +550,41 @@ export class NodeApiClient {
             throw classifyAgentRequestError(error, dispatched);
         }
     }
+    /**
+     * The agent's read-only upgrade report: its GET needs no lease or claim.
+     * `unavailable` is an agent older than 1.1.8 (404) or a report that does not
+     * parse; neither says anything against the node.
+     */
+    async getUpgradePreflight(query, timeoutMs = 5_000) {
+        let body;
+        try {
+            body = await this.api.get('agent-upgrade/preflight', { timeout: timeoutMs, retry: 0,
+                searchParams: { ...(query.target ? { target: query.target } : {}), ...(query.engines ? { engines: query.engines } : {}), ...(query.nonce ? { nonce: query.nonce } : {}) } }).json();
+        }
+        catch (error) {
+            if (error instanceof HTTPError && error.response.status === 404)
+                return { status: 'unavailable', detail: 'this agent has no preflight check' };
+            if (error instanceof SyntaxError)
+                return { status: 'unavailable', detail: 'the preflight answer is not JSON' };
+            return { status: 'unreachable', detail: errorText(error) };
+        }
+        const parsed = agentUpgradePreflightSchema.safeParse(body?.data);
+        return parsed.success ? { status: 'ok', report: parsed.data } : { status: 'unavailable', detail: 'the preflight answer does not parse' };
+    }
+    /** Signed account of what the agent's ledger holds for a command; null from an agent without this route. */
+    async getCommandTrace(commandId, nonce, resourceKey, timeoutMs = 5_000) {
+        if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(commandId))
+            throw new Error('Invalid command identifier');
+        try {
+            const response = await this.api.get(`commands/receipts/${encodeURIComponent(commandId)}/trace`, { timeout: timeoutMs, retry: 0, searchParams: { nonce, resourceKey } }).json();
+            return response.success ? response.data : null;
+        }
+        catch (error) {
+            if (error instanceof HTTPError && error.response.status === 404)
+                return null;
+            throw error;
+        }
+    }
     // ── Ping ──
     async ping() {
         return this.api.get('ping').json();