@hivemagnus/magnus-server 1.1.1 → 1.1.3

package/dist/cli.js

@@ -56,9 +56,11 @@ async function migrate() {
     catch {
         return fatal('Migration error: the prisma CLI package is not installed next to magnus-server.');
     }
+    // MAGNUS_MIGRATE_DATABASE_URL: a direct connection for migrations when DATABASE_URL goes through a pooler.
+    const migrateUrl = process.env['MAGNUS_MIGRATE_DATABASE_URL']?.trim();
     const code = await new Promise((resolvePromise, reject) => {
         const child = spawn(process.execPath, [prismaCli, 'migrate', 'deploy', '--config', PRISMA_CONFIG], {
-            cwd: PACKAGE_ROOT, env: process.env, stdio: 'inherit',
+            cwd: PACKAGE_ROOT, env: migrateUrl ? { ...process.env, DATABASE_URL: migrateUrl } : process.env, stdio: 'inherit',
         });
         child.on('error', reject);
         child.on('close', exitCode => resolvePromise(exitCode ?? 1));
@@ -141,13 +143,23 @@ async function commandMigrations(format) {
     try {
         const state = await readMigrationState(client, MIGRATIONS_DIR);
         if (format === 'json') {
-            console.log(JSON.stringify({ version: await version(), packaged: state.packaged, applied: [...state.applied].sort(), pending: state.pending, fresh: state.fresh }, null, 2));
+            const size = await client.query('SELECT pg_database_size(current_database())::text AS bytes, current_setting(\'server_version_num\') AS version')
+                .then(result => result.rows[0]).catch(() => undefined);
+            console.log(JSON.stringify({
+                version: await version(), packaged: state.packaged, applied: [...state.applied].sort(), pending: state.pending, fresh: state.fresh,
+                failed: state.failed, unknown: state.unknown,
+                databaseBytes: size ? Number(size.bytes) : null, postgresMajor: size ? Math.floor(Number(size.version) / 10_000) : null,
+            }, null, 2));
         }
         else {
             for (const name of state.packaged) {
                 const pending = state.pending.find(migration => migration.name === name);
                 console.log(`  ${pending ? '…' : '✓'} ${name}${pending ? ` (pending, ${pending.impact}${pending.note ? `: ${pending.note}` : ''})` : ''}`);
             }
+            for (const migration of state.unknown)
+                console.log(`  ? ${migration.name} (applied by a newer release; unknown to this version)`);
+            for (const name of state.failed)
+                console.log(`  ✗ ${name} (failed; see \`prisma migrate resolve\` or restore a dump)`);
             console.log(state.pending.length === 0 ? 'All migrations applied.' : `${state.pending.length} pending; run \`magnus-server migrate\`.`);
         }
         return 0;
@@ -159,17 +171,19 @@ async function commandMigrations(format) {
 /** `pg_dump -Fc` into MAGNUS_BACKUP_DIR; prints JSON with the file and restore command. */
 async function commandBackup(label) {
     const { resolveDatabaseUrl } = await import('./db/client.js');
-    const { backupDatabase } = await import('./cli/migrations.js');
-    const { defaultBackupDir } = await import('./cli/auto-migrate.js');
+    const { backupDatabase, backupKeep, readMigrationState } = await import('./cli/migrations.js');
+    const { defaultBackupDir, migrateDatabaseUrl } = await import('./cli/auto-migrate.js');
     const { magnusHome } = await import('./utils/magnus-home.js');
     const pg = (await import('pg')).default;
-    const databaseUrl = resolveDatabaseUrl();
+    const databaseUrl = migrateDatabaseUrl(process.env, resolveDatabaseUrl());
     const client = new pg.Client({ connectionString: databaseUrl, connectionTimeoutMillis: 10_000 });
     await client.connect();
     try {
+        const state = await readMigrationState(client, MIGRATIONS_DIR).catch(() => null);
         const result = await backupDatabase({
             databaseUrl, client, backupDir: defaultBackupDir(process.env, magnusHome()),
             label: label ?? `manual-${await version()}`, pgDump: process.env['MAGNUS_PG_DUMP'],
+            lastApplied: state?.lastApplied ?? null, keep: backupKeep(),
         });
         console.log(JSON.stringify(result));
         return 0;
@@ -178,18 +192,76 @@ async function commandBackup(label) {
         await client.end().catch(() => undefined);
     }
 }
+/** Replace the database with a dump; refuses while this host's server is running. */
+async function commandRestore(file, yes) {
+    const { resolveDatabaseUrl } = await import('./db/client.js');
+    const { pgRestoreFor, restoreDatabase } = await import('./cli/migrations.js');
+    const { migrateDatabaseUrl } = await import('./cli/auto-migrate.js');
+    const { runningServer } = await import('./utils/runtime-file.js');
+    const running = runningServer();
+    if (running)
+        fatal(`The server (pid ${running.pid}, ${running.version}) is still running; stop it before restoring (for example: systemctl stop magnus-server, or docker compose stop magnus-server).`);
+    const path = resolve(file);
+    if (!yes) {
+        if (!process.stdin.isTTY)
+            fatal('Restoring replaces the whole database; pass --yes to confirm non-interactively.');
+        const { createInterface } = await import('node:readline/promises');
+        const rl = createInterface({ input: process.stdin, output: process.stdout });
+        const answer = await rl.question(`Replace the database with ${path}? Changes made since the dump are lost [y/N] `).finally(() => rl.close());
+        if (!/^y(es)?$/i.test(answer.trim()))
+            return 1;
+    }
+    const pg = (await import('pg')).default;
+    const databaseUrl = migrateDatabaseUrl(process.env, resolveDatabaseUrl());
+    const client = new pg.Client({ connectionString: databaseUrl, connectionTimeoutMillis: 10_000 });
+    await client.connect();
+    try {
+        await restoreDatabase({ databaseUrl, file: path, client, pgRestore: pgRestoreFor(process.env['MAGNUS_PG_DUMP']), log: line => console.log(line) });
+        console.log(`Restored ${path}. Start the server version that matches the dump.`);
+        return 0;
+    }
+    catch (error) {
+        return fatal(`Restore failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    finally {
+        await client.end().catch(() => undefined);
+    }
+}
+/** Warn at startup when a newer release migrated the database (best effort). */
+async function warnSchemaAhead() {
+    try {
+        const { resolveDatabaseUrl } = await import('./db/client.js');
+        const { readMigrationState } = await import('./cli/migrations.js');
+        const pg = (await import('pg')).default;
+        const client = new pg.Client({ connectionString: resolveDatabaseUrl(), connectionTimeoutMillis: 5_000 });
+        await client.connect();
+        try {
+            const state = await readMigrationState(client, MIGRATIONS_DIR);
+            if (state.unknown.length > 0) {
+                console.warn(`[startup] warning: the database has ${state.unknown.length} migration(s) this version does not know (a newer release applied them): `
+                    + `${state.unknown.map(m => m.name).join(', ')}. If this version misbehaves, upgrade again or restore the dump taken before that upgrade.`);
+            }
+            if (state.failed.length > 0)
+                console.warn(`[startup] warning: failed migration(s) ${state.failed.join(', ')}; run \`magnus-server migrations\`.`);
+        }
+        finally {
+            await client.end().catch(() => undefined);
+        }
+    }
+    catch { /* the server reports database problems itself */ }
+}
 /** `MAGNUS_AUTO_MIGRATE=true`: back up and migrate under an advisory lock before listening. */
 async function startupAutoMigrate() {
     const { upgradeSettings } = await import('./config/features.js');
     const settings = upgradeSettings();
     if (!settings.autoMigrate)
-        return;
+        return warnSchemaAhead();
     const { resolveDatabaseUrl } = await import('./db/client.js');
-    const { autoMigrate, defaultBackupDir } = await import('./cli/auto-migrate.js');
+    const { autoMigrate, defaultBackupDir, migrateDatabaseUrl } = await import('./cli/auto-migrate.js');
     const { magnusHome } = await import('./utils/magnus-home.js');
     try {
         await autoMigrate({
-            databaseUrl: resolveDatabaseUrl(),
+            databaseUrl: migrateDatabaseUrl(process.env, resolveDatabaseUrl()),
             migrationsDir: MIGRATIONS_DIR,
             backupDir: defaultBackupDir(process.env, magnusHome()),
             version: await version(),
@@ -243,6 +315,9 @@ async function run(argv) {
         case 'backup':
             loadEnvFiles(options.envFiles);
             return commandBackup(options.label);
+        case 'restore':
+            loadEnvFiles(options.envFiles);
+            return commandRestore(options.file, options.yes);
         case 'upgrade':
         case 'rollback': {
             const { delegateToProject } = await import('./cli/upgrade.js');

package/dist/index.js

@@ -56,7 +56,7 @@ const { writeRuntimeFile } = await import('./utils/runtime-file.js');
 const { CONTROL_API_VERSION } = await import('./config/version.js');
 /** Background services shared by the TLS and development listeners. */
 function startBackgroundServices(listenPort) {
-    writeRuntimeFile({ version: CONTROL_API_VERSION, port: listenPort });
+    writeRuntimeFile({ version: CONTROL_API_VERSION, port: listenPort, host: listener.hostname });
     startNodeIssueMonitor();
     startApplicationProbes();
     startLogRetention();

package/dist/openapi/spec.js

@@ -2156,13 +2156,16 @@ const paths = {
         get: {
             tags: ['Agent upgrades'],
             summary: 'Which agents can take a version',
-            description: 'For every node: its agent version and whether it can upgrade itself to `version` (default: the agent version this server ships with) now, or why not (offline, not set up with `magnus-agent install-service`, MAGNUS_AGENT_SELF_UPGRADE=false, another major version, already current).',
+            description: 'For every node: its agent version and whether it can upgrade itself to `version` (default: the agent version bundled with this server) now, or why not: offline, not set up with `magnus-agent install-service`, MAGNUS_AGENT_SELF_UPGRADE=false, another major version, already current, already reverted this version, a closed release admission fence, a resource enrollment that is not complete (for example "stuck in draining since <date>"), held participant work or command leases, or an agent whose sticky managed-enrollment marker the server does not share. `release.verified` is false when the registry does not have the version.',
             security: apiSecretAuth,
-            parameters: [qp('version', 'Target agent version (X.Y.Z)')],
+            parameters: [qp('version', 'Target agent version (X.Y.Z)'), qp('allowRetryReverted', '`true` to offer nodes that already reverted this version')],
             responses: { 200: success('Eligibility', { type: 'object', properties: {
-                        version: { type: 'string' }, upgradesEnabled: { type: 'boolean' }, disabledReason: { type: ['string', 'null'] },
+                        version: { type: 'string' }, serverVersion: { type: 'string' }, upgradesEnabled: { type: 'boolean' }, disabledReason: { type: ['string', 'null'] },
+                        prerelease: { type: 'boolean' },
+                        release: { type: 'object', properties: { verified: { type: 'boolean' }, integrity: { type: ['string', 'null'] }, reason: { type: ['string', 'null'] } } },
                         nodes: { type: 'array', items: { type: 'object', properties: { nodeId: { type: 'string' }, name: { type: 'string' },
                                     version: { type: 'string' }, eligible: { type: 'boolean' }, reason: { type: ['string', 'null'] } } } },
+                        latestRun: { type: ['object', 'null'] },
                     } }), 400: err400, 401: err401 },
         },
     },
@@ -2170,12 +2173,15 @@ const paths = {
         post: {
             tags: ['Agent upgrades'],
             summary: 'Upgrade agents',
-            description: 'Starts an upgrade run (ADMIN, MAGNUS_AGENT_UPGRADES=true): one canary, then MAGNUS_AGENT_UPGRADE_CONCURRENCY agents at a time, stopping at the first agent that does not confirm. Each agent installs the exact version and npm integrity and switches back by itself if the new version does not report a heartbeat in time.',
+            description: 'Starts an upgrade run (ADMIN, MAGNUS_AGENT_UPGRADES=true): one canary, then MAGNUS_AGENT_UPGRADE_CONCURRENCY agents at a time. Each agent installs the exact version and npm integrity and switches back by itself if the new version does not report a heartbeat in time. A revert, a release problem (integrity, install failure) or an agent that goes silent after the request stops the run. An agent that refuses before changing anything is recorded as `refused` and the run goes on; a refused canary hands the role to the next node. Only one run at a time. Selected nodes that are not eligible are listed in `excluded`; unknown node ids are a 400.',
             security: apiSecretAuth,
             requestBody: { required: false, content: { 'application/json': { schema: { type: 'object', properties: {
-                                version: { type: 'string', description: 'Agent version (default: the one this server ships with)' },
+                                version: { type: 'string', description: 'Agent version (default: the one bundled with this server)' },
                                 integrity: { type: 'string', description: 'npm sha512 integrity (default: read from the registry)' },
-                                nodeIds: { type: 'array', items: { type: 'string' }, description: 'Only these nodes' },
+                                nodeIds: { type: 'array', minItems: 1, items: { type: 'string' }, description: 'Only these nodes (leave out for every eligible agent)' },
+                                canaryNodeId: { type: 'string', description: 'Upgrade this node first' },
+                                allowPrerelease: { type: 'boolean', description: 'Required for a prerelease version' },
+                                allowRetryReverted: { type: 'boolean', description: 'Also upgrade nodes that already reverted this version' },
                             } } } } },
             responses: { 202: success('Run started'), 400: err400, 401: err401, 403: err403, 409: err409 },
         },
@@ -2192,12 +2198,35 @@ const paths = {
         get: {
             tags: ['Agent upgrades'],
             summary: 'Upgrade run status',
-            description: 'Per-node state of a run: pending, requested, confirmed, reverted, failed or skipped.',
+            description: 'Run status (running, succeeded, failed, stopped or interrupted) and per-node state: pending, requested, confirmed, reverted, failed, refused or skipped, with the agent error code and message. A run whose server stopped renewing its lease (restart, interrupted CLI) is shown as interrupted; its requested nodes are settled from their heartbeat reports.',
             security: apiSecretAuth,
             parameters: [pp('id', 'Run ID')],
             responses: { 200: success('Run'), 401: err401, 404: err404 },
         },
     },
+    '/agents/upgrade/runs/{id}/stop': {
+        post: {
+            tags: ['Agent upgrades'],
+            summary: 'Stop an upgrade run',
+            description: 'ADMIN. The run stops before its next batch; nodes already asked to upgrade finish their trial.',
+            security: apiSecretAuth,
+            parameters: [pp('id', 'Run ID')],
+            responses: { 202: success('Stop requested'), 401: err401, 403: err403, 404: err404, 409: err409 },
+        },
+    },
+    '/agents/upgrade/runs/{id}/retry': {
+        post: {
+            tags: ['Agent upgrades'],
+            summary: 'Retry an upgrade run',
+            description: 'ADMIN. Starts a new run with the same version and integrity for the failed, refused, skipped and interrupted nodes of a finished run (reverted nodes only with allowRetryReverted).',
+            security: apiSecretAuth,
+            parameters: [pp('id', 'Run ID')],
+            requestBody: { required: false, content: { 'application/json': { schema: { type: 'object', properties: {
+                                canaryNodeId: { type: 'string' }, allowRetryReverted: { type: 'boolean' },
+                            } } } } },
+            responses: { 202: success('Run started'), 400: err400, 401: err401, 403: err403, 404: err404, 409: err409 },
+        },
+    },
     '/self-update/status': {
         get: {
             tags: ['Self-Update'],

package/dist/routes/agent-upgrades.js

@@ -4,32 +4,43 @@ import { authMiddleware, getUser, requireRole } from '../middleware/auth.js';
 import { HttpError } from '../middleware/error-handler.js';
 import { upgradeSettings } from '../config/features.js';
 import { AGENT_VERSION } from '../config/version.js';
-import { getUpgradeRun, latestUpgradeRun, listUpgradeCandidates, startUpgradeRun } from '../services/agent-upgrade-service.js';
+import { getUpgradeRun, isPrerelease, latestUpgradeRun, listUpgradeCandidates, retryUpgradeRun, startUpgradeRun, stopUpgradeRun, verifyAgentRelease } from '../services/agent-upgrade-service.js';
 /** /api/v1/agents/upgrade: which agents can take a version, and fleet upgrade runs. */
 export const agentUpgradesRouter = new Hono();
 agentUpgradesRouter.use('*', authMiddleware);
 const version = z.string().regex(/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/);
+const nodeId = z.string().min(1).max(128);
 agentUpgradesRouter.get('/eligibility', async (c) => {
     const target = version.safeParse(c.req.query('version') ?? AGENT_VERSION);
     if (!target.success)
         throw new HttpError(400, 'INVALID_VERSION', 'version must be X.Y.Z');
-    const nodes = await listUpgradeCandidates(target.data);
+    const [nodes, release, latestRun] = await Promise.all([
+        listUpgradeCandidates(target.data, undefined, { allowRetryReverted: c.req.query('allowRetryReverted') === 'true' }),
+        verifyAgentRelease(target.data), latestUpgradeRun()
+    ]);
     return c.json({ success: true, data: {
             version: target.data, serverVersion: AGENT_VERSION, upgradesEnabled: upgradeSettings().agentUpgrades,
             disabledReason: upgradeSettings().agentUpgrades ? null : 'MAGNUS_AGENT_UPGRADES=false on the server',
-            nodes, latestRun: await latestUpgradeRun(),
+            prerelease: isPrerelease(target.data), release, nodes, latestRun,
         } });
 });
 const startSchema = z.object({
     version: version.optional(),
     integrity: z.string().regex(/^sha512-[A-Za-z0-9+/]+={0,2}$/).optional(),
-    nodeIds: z.array(z.string().min(1).max(128)).max(1000).optional(),
+    nodeIds: z.array(nodeId).min(1, 'nodeIds must name at least one node; leave it out for every eligible agent').max(1000).optional(),
+    canaryNodeId: nodeId.optional(),
+    allowPrerelease: z.boolean().optional(),
+    allowRetryReverted: z.boolean().optional(),
 }).strict();
-agentUpgradesRouter.post('/', requireRole('ADMIN'), async (c) => {
-    const parsed = startSchema.safeParse(await c.req.json().catch(() => ({})));
+const parseBody = async (schema, raw) => {
+    const parsed = schema.safeParse(raw);
     if (!parsed.success)
         throw new HttpError(400, 'VALIDATION_ERROR', parsed.error.issues.map(issue => issue.message).join('; '));
-    const { run } = await startUpgradeRun({ ...parsed.data, actorId: getUser(c).userId });
+    return parsed.data;
+};
+agentUpgradesRouter.post('/', requireRole('ADMIN'), async (c) => {
+    const body = await parseBody(startSchema, await c.req.json().catch(() => ({})));
+    const { run } = await startUpgradeRun({ ...body, actorId: getUser(c).userId });
     return c.json({ success: true, data: run }, 202);
 });
 agentUpgradesRouter.get('/runs/latest', async (c) => c.json({ success: true, data: await latestUpgradeRun() }));
@@ -39,4 +50,14 @@ agentUpgradesRouter.get('/runs/:id', async (c) => {
         throw new HttpError(404, 'NOT_FOUND', 'Upgrade run not found');
     return c.json({ success: true, data: run });
 });
+agentUpgradesRouter.post('/runs/:id/stop', requireRole('ADMIN'), async (c) => c.json({ success: true, data: await stopUpgradeRun(c.req.param('id'), getUser(c).userId) }, 202));
+const retrySchema = z.object({
+    canaryNodeId: nodeId.optional(),
+    allowRetryReverted: z.boolean().optional(),
+}).strict();
+agentUpgradesRouter.post('/runs/:id/retry', requireRole('ADMIN'), async (c) => {
+    const body = await parseBody(retrySchema, await c.req.json().catch(() => ({})));
+    const { run } = await retryUpgradeRun(c.req.param('id'), { ...body, actorId: getUser(c).userId });
+    return c.json({ success: true, data: run }, 202);
+});
 //# sourceMappingURL=agent-upgrades.js.map

package/dist/routes/heartbeat.js

@@ -332,7 +332,8 @@ heartbeatRouter.post('/', async (c) => {
                 }) },
         });
     }
-    if (existing && body.selfUpgrade !== undefined) {
+    // Also without a (valid) report, so an agent that stopped reporting one does not stay UPDATING.
+    if (existing) {
         await recordSelfUpgradeReport({ id: nodeId, name: existing.name }, body).catch((err) => {
             console.error(`[agent-upgrade] could not record the self-upgrade report of ${existing.name}:`, err instanceof Error ? err.message : err);
         });

package/dist/routes/resource-leases.js

@@ -11,6 +11,7 @@ import { HttpError } from '../middleware/error-handler.js';
 import { NODE_MANAGER_SECRET } from '../utils/env.js';
 import { verifyWithRotation } from '../services/credential-rotation-service.js';
 import { resourceLeasesRequested } from '../services/command-resource-leases.js';
+import { readOwnershipHistory, stickyEnrollmentObserved } from '../services/resource-ownership.js';
 import { activateArdentScope, ardentScopeActivationSchema, assertArdentScope, inspectArdentScope } from '../services/ardent-scope-enrollment.js';
 import { legacyAdmissionRequestSchema, resourceClaimSchema } from '../security/managed-http.js';
 import { participantWorkReconciliationSchema, historicalParticipantWorkReconciliationSchema } from '../schemas/resource-participant-reconciliation.js';
@@ -144,14 +145,11 @@ resourceLeasesRouter.post('/legacy-admission', async (c) => {
         if (current?.hostname !== body.nodeFqdn)
             throw new HttpError(409, 'RESOURCE_TARGET_INVALID', 'Inventory target changed');
         await assertNodeAdmissionOpen(tx, node.id);
-        const [enrollment, leaseHistory, managedCronHistory, participantHistory] = await Promise.all([
-            tx.resourceEnrollment.findUnique({ where: { nodeId: node.id } }),
-            tx.commandResourceLease.findFirst({ where: { nodeId: node.id }, select: { id: true } }),
-            tx.managedCronDefinition.findFirst({ where: { nodeId: node.id }, select: { cronJobId: true } }),
-            tx.resourceParticipantWork.findFirst({ where: { nodeId: node.id }, select: { workId: true } }),
+        const [enrollment, history] = await Promise.all([
+            tx.resourceEnrollment.findUnique({ where: { nodeId: node.id } }), readOwnershipHistory(tx, node.id)
         ]);
-        const enrolled = Boolean(enrollment && enrollment.state !== 'legacy')
-            || Boolean(leaseHistory || managedCronHistory || participantHistory);
+        // Sticky rule (see resource-ownership.ts): the agent stores `enrolled` for good.
+        const enrolled = stickyEnrollmentObserved(enrollment, history);
         const gateEnabled = resourceLeasesRequested();
         return { ...body, nodeId: node.id, enrolled, gateEnabled, allowedLegacy: !enrolled && !gateEnabled };
     });

package/dist/sdk/node-api-client.js

@@ -6,7 +6,7 @@ import { commandReconciliationSchema } from '../schemas/command-reconciliation.j
 import { admitResourceCommand, markResourceDispatched, failResourceCommand, settleResourceResult, settleResourceReconciliation } from '../services/command-resource-leases.js';
 import { commandFenceSchema, commandResultSchema } from '../schemas/node-responses.js';
 import { commandRequestDigest } from '../security/command-request.js';
-import ky from 'ky';
+import ky, { HTTPError } from 'ky';
 import { HttpError } from '../middleware/error-handler.js';
 import { createHmac, timingSafeEqual } from 'node:crypto';
 import { z } from 'zod';
@@ -63,6 +63,82 @@ async function rejectUnprivilegedAgent({ response }) {
 export function isAgentNotPrivileged(error) {
     return error instanceof HttpError && error.code === AGENT_NOT_PRIVILEGED;
 }
+/** `{ success: false, error: { code, message } }` from an agent reply (object or JSON text). */
+export function agentErrorBody(raw) {
+    let value = raw;
+    if (typeof raw === 'string') {
+        try {
+            value = JSON.parse(raw);
+        }
+        catch {
+            return null;
+        }
+    }
+    const error = value?.error;
+    if (!error || typeof error.code !== 'string')
+        return null;
+    return { code: error.code.slice(0, 64), message: typeof error.message === 'string' ? error.message.slice(0, 1000) : error.code };
+}
+/** The agent answered a mutation with an error status. `agentStatus` is not an HTTP status for our callers. */
+export class AgentResponseError extends Error {
+    agentStatus;
+    code;
+    agentMessage;
+    constructor(agentStatus, body) {
+        super(`Native mutation completed with HTTP ${agentStatus}${body ? `: ${body.code}: ${body.message}` : ''}`);
+        this.name = 'AgentResponseError';
+        this.agentStatus = agentStatus;
+        this.code = body?.code ?? null;
+        this.agentMessage = body?.message ?? null;
+    }
+}
+/** Thrown by `upgradeAgent` for every failure before the request left: the node is untouched. */
+export class NotDispatchedError extends HttpError {
+    dispatched = false;
+    constructor(code, message, cause) {
+        super(409, code, message);
+        this.name = 'NotDispatchedError';
+        if (cause !== undefined)
+            this.cause = cause;
+    }
+}
+/** The request may have reached the agent but no answer came back. */
+export class DispatchOutcomeUnknownError extends Error {
+    code = 'MUTATION_OUTCOME_UNKNOWN';
+    constructor(message, cause) {
+        super(message);
+        this.name = 'DispatchOutcomeUnknownError';
+        if (cause !== undefined)
+            this.cause = cause;
+    }
+}
+const CONNECT_ERRORS = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EHOSTUNREACH', 'ENETUNREACH', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT']);
+/** A connection that never opened cannot have carried the request. */
+export function isConnectError(error) {
+    for (let current = error, depth = 0; current && typeof current === 'object' && depth < 5; current = current.cause, depth++) {
+        const code = current.code;
+        if (typeof code === 'string' && CONNECT_ERRORS.has(code))
+            return true;
+    }
+    return false;
+}
+function errorText(error) {
+    return (error instanceof Error ? error.message : String(error)).slice(0, 1000);
+}
+/** Sort an upgradeAgent failure by whether the request can have reached the agent. */
+export function classifyAgentRequestError(error, dispatched) {
+    if (!dispatched)
+        return error instanceof NotDispatchedError ? error : new NotDispatchedError(error instanceof HttpError ? error.code : error instanceof AgentResponseError && error.code ? error.code : 'NOT_DISPATCHED', errorText(error), error);
+    if (error instanceof AgentResponseError)
+        return error;
+    if (error instanceof HTTPError)
+        return new AgentResponseError(error.response.status, agentErrorBody(error.data));
+    if (error instanceof HttpError && error.code === AGENT_NOT_PRIVILEGED)
+        return new AgentResponseError(403, { code: error.code, message: error.message });
+    if (isConnectError(error))
+        return new NotDispatchedError('AGENT_UNREACHABLE', `agent unreachable: ${errorText(error)}`, error);
+    return new DispatchOutcomeUnknownError(`no answer from the agent after the request was sent: ${errorText(error)}`, error);
+}
 export class NodeApiClient {
     api;
     nodeToken;
@@ -85,7 +161,7 @@ export class NodeApiClient {
         });
     }
     /** All finite native mutations use the same central owner and node execution slot. */
-    async mutateWithEvidence(method, path, body, timeout = this.defaultTimeout, commandId, skillPublication, requireManaged = false) {
+    async mutateWithEvidence(method, path, body, timeout = this.defaultTimeout, commandId, skillPublication, requireManaged = false, dispatch) {
         if (!this.resourceContext)
             throw new Error('Managed mutation requires canonical resource node identity');
         const serialized = body === undefined ? '' : JSON.stringify(body);
@@ -97,6 +173,11 @@ export class NodeApiClient {
         const admission = await admitResourceCommand(this.resourceContext, identity);
         if (!admission && requireManaged)
             throw new Error('This operation requires enrolled managed admission; direct node mutation is forbidden');
+        if (!admission) {
+            if (dispatch?.pingFirst)
+                await this.api.get('ping', { timeout: 10_000, retry: 0 });
+            dispatch?.onDispatch();
+        }
         if (!admission)
             return { response: await this.api(path, { method,
                     ...(body === undefined ? {} : { body: serialized, headers: { 'content-type': 'application/json' } }), timeout, retry: 0 }).json() };
@@ -110,6 +191,7 @@ export class NodeApiClient {
             if (!capability.success || capability.data?.features?.managedMutations !== 1)
                 throw new Error('Node does not support managed mutations; operation not dispatched');
             await markResourceDispatched(admission);
+            dispatch?.onDispatch();
             const managedRequest = managedHttpRequestSchema.parse({
                 claim: { leaseId: admission.leaseId, nodeId: admission.nodeId, nodeFqdn: admission.nodeFqdn },
                 commandId: admission.commandId, method, path: '/' + path, body: serialized,
@@ -132,7 +214,7 @@ export class NodeApiClient {
                     bundleSha256: skillPublication.bundleSha256, responseBody: response.body,
                 });
             if (response.status >= 400)
-                throw new Error(`Native mutation completed with HTTP ${response.status}`);
+                throw new AgentResponseError(response.status, agentErrorBody(response.body));
             const commandSha256 = receipt.coordination?.requestSha256;
             if (!commandSha256 || commandSha256 !== commandRequestDigest(admission.payload))
                 throw new Error('Managed mutation command evidence differs');
@@ -393,7 +475,17 @@ export class NodeApiClient {
     // ── Agent self-upgrade ──
     /** Stage an exact agent version; the agent restarts into it through its launcher. */
     async upgradeAgent(request) {
-        return this.mutate('POST', 'agent-upgrade', request, 15 * 60_000);
+        // Every failure is typed: NotDispatchedError (node untouched), AgentResponseError
+        // (the agent answered with an error) or DispatchOutcomeUnknownError (sent, no answer).
+        let dispatched = false;
+        try {
+            return (await this.mutateWithEvidence('POST', 'agent-upgrade', request, 15 * 60_000, 
+            // A stable command id lets an operator find (and reconcile) the lease of a request whose reply was lost.
+            /^[a-zA-Z0-9._-]{1,100}$/.test(request.requestId) ? `agent-upgrade-${request.requestId}` : undefined, undefined, false, { pingFirst: true, onDispatch: () => { dispatched = true; } })).response;
+        }
+        catch (error) {
+            throw classifyAgentRequestError(error, dispatched);
+        }
     }
     // ── Ping ──
     async ping() {

package/dist/server.js

@@ -96,6 +96,7 @@ const v1 = new Hono();
 v1.use('/auth/*', rateLimitMiddleware({ windowMs: 60_000, max: 20, keyPrefix: 'rl:auth', failClosed: true }));
 v1.use('/deploy/*', rateLimitMiddleware({ windowMs: 60_000, max: 10, keyPrefix: 'rl:deploy', failClosed: true }));
 v1.use('/agents/upgrade', rateLimitMiddleware({ windowMs: 60_000, max: 10, keyPrefix: 'rl:agent-upgrade', failClosed: true }));
+v1.use('/agents/upgrade/runs/:id/retry', rateLimitMiddleware({ windowMs: 60_000, max: 10, keyPrefix: 'rl:agent-upgrade', failClosed: true }));
 v1.use('/self-update/*', rateLimitMiddleware({ windowMs: 60_000, max: 10, keyPrefix: 'rl:selfupdate', failClosed: true }));
 v1.use('/commands/*', rateLimitMiddleware({ windowMs: 60_000, max: 60, keyPrefix: 'rl:commands' }));
 v1.use('/quick-command/*', rateLimitMiddleware({ windowMs: 60_000, max: 30, keyPrefix: 'rl:quickcmd' }));