@hivemagnus/magnus-server 1.1.5 → 1.1.8

package/dist/sdk/node-api-client.js

@@ -1,15 +1,17 @@
 import { createHash, randomUUID } from 'node:crypto';
-import { MANAGED_CLAIM_HEADER, httpIdentity, httpReceiptBody, managedHttpRequestSchema } from '../security/managed-http.js';
+import { MANAGED_CLAIM_HEADER, credentialMutationIdentity, httpIdentity, httpReceiptBody, managedHttpRequestSchema } from '../security/managed-http.js';
 import { nativeCompletionDescriptor, nativeCompletionResourceKey } from '../security/native-completion-registry.js';
 import { COMMAND_COMPLETION_REGISTRY, COMMAND_COMPLETION_REGISTRY_SHA256, commandCompletionDescriptor } from '../security/command-completion-registry.js';
 import { commandReconciliationSchema } from '../schemas/command-reconciliation.js';
 import { admitResourceCommand, markResourceDispatched, failResourceCommand, settleResourceResult, settleResourceReconciliation } from '../services/command-resource-leases.js';
 import { commandFenceSchema, commandResultSchema } from '../schemas/node-responses.js';
+import { advanceCommandFence } from '../services/command-fence-repair.js';
 import { commandRequestDigest } from '../security/command-request.js';
 import ky, { HTTPError } from 'ky';
 import { HttpError } from '../middleware/error-handler.js';
 import { createHmac, timingSafeEqual } from 'node:crypto';
 import { z } from 'zod';
+import { agentUpgradePreflightSchema } from '@hivemagnus/magnus-contracts';
 import { RESULT_SIGNING_PREVIOUS_SECRET, RESULT_SIGNING_SECRET } from '../utils/env.js';
 import { correlateSkillPublicationNoEffect } from '../services/skill-publication-settlement.js';
 import { commandResultSigningPayload, verifyCommandResultSignature, } from '../security/command-result-signature.js';
@@ -125,10 +127,52 @@ export function isConnectError(error) {
 function errorText(error) {
     return (error instanceof Error ? error.message : String(error)).slice(0, 1000);
 }
+const certainRefusalOf = (error) => error?.certainRefusal;
+/** The signed refusal in an agent's error reply. An older agent, or a reply without one, says nothing about execution. */
+export function parseAgentRefusal(error) {
+    if (!(error instanceof HTTPError))
+        return null;
+    const body = (error.data ?? null);
+    const refusal = body?.error?.refusal;
+    if (body?.error?.executed !== false || !refusal || typeof refusal.commandId !== 'string' || typeof refusal.requestSha256 !== 'string'
+        || typeof refusal.code !== 'string' || typeof refusal.signature !== 'string' || !(refusal.highWater === null || typeof refusal.highWater === 'string')
+        || typeof refusal.nodeFqdn !== 'string' || typeof refusal.leaseId !== 'string')
+        return null;
+    return { commandId: refusal.commandId, requestSha256: refusal.requestSha256, code: refusal.code, highWater: refusal.highWater, nodeFqdn: refusal.nodeFqdn, leaseId: refusal.leaseId, signature: refusal.signature,
+        message: typeof body.error.message === 'string' ? body.error.message.slice(0, 1000) : refusal.code };
+}
+/**
+ * Close the lease of a dispatch that threw. A verified refusal settles it as
+ * not-dispatched, and a STALE_FENCE moves the server's fence past the agent's
+ * high-water so the next request can pass. Everything else stays `unknown`.
+ */
+async function settleFailedDispatch(admission, actorId, error) {
+    const refusal = parseAgentRefusal(error);
+    const settled = await failResourceCommand(admission, refusal ?? undefined);
+    if (!refusal || !settled)
+        return;
+    let code = refusal.code;
+    let { message } = refusal;
+    const resourceKey = admission.payload.coordination?.resourceKey;
+    if (code === 'STALE_FENCE' && refusal.highWater && resourceKey) {
+        try {
+            const advanced = await advanceCommandFence({ nodeId: admission.nodeId, resourceKey, highWater: refusal.highWater, source: 'stale-fence-reply', actorId });
+            if (advanced.status === 'rejected')
+                ({ code, detail: message } = advanced);
+        }
+        catch (repairError) {
+            console.warn(`[resource-leases] fence of ${admission.nodeId} could not be advanced: ${repairError instanceof Error ? repairError.message : String(repairError)}`);
+        }
+    }
+    Object.assign(error, { certainRefusal: { code, message } });
+}
 /** Sort an upgradeAgent failure by whether the request can have reached the agent. */
 export function classifyAgentRequestError(error, dispatched) {
+    const refused = certainRefusalOf(error);
+    if (refused)
+        return new NotDispatchedError(refused.code, refused.message, error);
     if (!dispatched)
-        return error instanceof NotDispatchedError ? error : new NotDispatchedError(error instanceof HttpError ? error.code : error instanceof AgentResponseError && error.code ? error.code : 'NOT_DISPATCHED', errorText(error), error);
+        return error instanceof NotDispatchedError ? error : new NotDispatchedError(error instanceof HttpError ? error.code : error instanceof AgentResponseError && error.code ? error.code : isConnectError(error) ? 'AGENT_UNREACHABLE' : 'NOT_DISPATCHED', errorText(error), error);
     if (error instanceof AgentResponseError)
         return error;
     if (error instanceof HTTPError)
@@ -225,13 +269,32 @@ export class NodeApiClient {
                     acknowledgedAt: new Date() } };
         }
         catch (error) {
-            await failResourceCommand(admission);
+            await settleFailedDispatch(admission, this.resourceContext.actorId, error);
             throw error;
         }
     }
     async mutate(method, path, body, timeout = this.defaultTimeout) {
         return (await this.mutateWithEvidence(method, path, body, timeout)).response;
     }
+    /**
+     * A body holding a credential never goes through the digest in mutateWithEvidence.
+     * Enrolled nodes refuse it exactly as they refuse any mutation without a reviewed
+     * completion boundary; other nodes receive the same direct POST as before.
+     */
+    async mutateWithCredential(path, body, timeout = this.defaultTimeout) {
+        if (!this.resourceContext)
+            throw new Error('Managed mutation requires canonical resource node identity');
+        const nativePath = '/' + path;
+        if (nativeCompletionDescriptor('POST', nativePath))
+            throw new Error('A managed native path cannot carry an unbound credential mutation');
+        const identity = credentialMutationIdentity('native-' + randomUUID(), 'POST', nativePath, Math.min(3600, Math.ceil(timeout / 1000)));
+        const admission = await admitResourceCommand(this.resourceContext, identity);
+        if (admission) {
+            await failResourceCommand(admission);
+            throw new Error('This enrolled-node mutation lacks a reviewed completion boundary; not dispatched');
+        }
+        return this.api(path, { method: 'POST', body: JSON.stringify(body), headers: { 'content-type': 'application/json' }, timeout, retry: 0 }).json();
+    }
     async evidencedMutation(method, path, body, commandId) {
         const result = await this.mutateWithEvidence(method, path, body, this.defaultTimeout, commandId);
         if (!result.evidence)
@@ -305,7 +368,7 @@ export class NodeApiClient {
         }
         catch (error) {
             if (admission)
-                await failResourceCommand(admission);
+                await settleFailedDispatch(admission, this.resourceContext.actorId, error);
             throw error;
         }
     }
@@ -487,6 +550,41 @@ export class NodeApiClient {
             throw classifyAgentRequestError(error, dispatched);
         }
     }
+    /**
+     * The agent's read-only upgrade report: its GET needs no lease or claim.
+     * `unavailable` is an agent older than 1.1.8 (404) or a report that does not
+     * parse; neither says anything against the node.
+     */
+    async getUpgradePreflight(query, timeoutMs = 5_000) {
+        let body;
+        try {
+            body = await this.api.get('agent-upgrade/preflight', { timeout: timeoutMs, retry: 0,
+                searchParams: { ...(query.target ? { target: query.target } : {}), ...(query.engines ? { engines: query.engines } : {}), ...(query.nonce ? { nonce: query.nonce } : {}) } }).json();
+        }
+        catch (error) {
+            if (error instanceof HTTPError && error.response.status === 404)
+                return { status: 'unavailable', detail: 'this agent has no preflight check' };
+            if (error instanceof SyntaxError)
+                return { status: 'unavailable', detail: 'the preflight answer is not JSON' };
+            return { status: 'unreachable', detail: errorText(error) };
+        }
+        const parsed = agentUpgradePreflightSchema.safeParse(body?.data);
+        return parsed.success ? { status: 'ok', report: parsed.data } : { status: 'unavailable', detail: 'the preflight answer does not parse' };
+    }
+    /** Signed account of what the agent's ledger holds for a command; null from an agent without this route. */
+    async getCommandTrace(commandId, nonce, resourceKey, timeoutMs = 5_000) {
+        if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(commandId))
+            throw new Error('Invalid command identifier');
+        try {
+            const response = await this.api.get(`commands/receipts/${encodeURIComponent(commandId)}/trace`, { timeout: timeoutMs, retry: 0, searchParams: { nonce, resourceKey } }).json();
+            return response.success ? response.data : null;
+        }
+        catch (error) {
+            if (error instanceof HTTPError && error.response.status === 404)
+                return null;
+            throw error;
+        }
+    }
     // ── Ping ──
     async ping() {
         return this.api.get('ping').json();
@@ -558,10 +656,10 @@ export class NodeApiClient {
         return this.api.get('users').json();
     }
     async provisionUser(payload) {
-        return this.mutate('POST', 'users/provision', payload);
+        return this.mutateWithCredential('users/provision', payload);
     }
     async changeUserPassword(payload) {
-        return this.mutate('POST', 'users/change-password', payload);
+        return this.mutateWithCredential('users/change-password', payload);
     }
 }
 //# sourceMappingURL=node-api-client.js.map

package/dist/security/command-result-signature.js

@@ -31,4 +31,26 @@ export function verifyCommandResultSignature(result, activeSecret, previousSecre
     }
     return matched;
 }
+export function commandRefusalSigningPayload(refusal) {
+    return JSON.stringify({ domain: 'magnus.command-refusal.v1', commandId: refusal.commandId, requestSha256: refusal.requestSha256,
+        code: refusal.code, highWater: refusal.highWater, nodeFqdn: refusal.nodeFqdn, leaseId: refusal.leaseId });
+}
+export function commandAbsenceSigningPayload(absence) {
+    return JSON.stringify({ domain: 'magnus.command-absence.v1', commandId: absence.commandId, nodeFqdn: absence.nodeFqdn, nonce: absence.nonce,
+        ledgerExists: absence.ledgerExists, receipt: absence.receipt, activeLock: absence.activeLock, resourceKey: absence.resourceKey,
+        fenceHighWater: absence.fenceHighWater, markerState: absence.markerState, markerNodeId: absence.markerNodeId, admissionFailedFence: absence.admissionFailedFence });
+}
+/** The fence high-water a preflight reports, bound to the node and a fresh nonce. Only this can move a server generation. */
+export function fenceProofSigningPayload(proof) {
+    return JSON.stringify({ domain: 'magnus.fence-high-water.v1', nonce: proof.nonce, nodeFqdn: proof.nodeFqdn,
+        fenceHighWater: Object.fromEntries(Object.entries(proof.fenceHighWater).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) });
+}
+/** HMAC check of a signed refusal or absence payload against the active or previous result key. */
+export function verifyCommandEvidence(payload, signature, activeSecret, previousSecret = '') {
+    if (typeof signature !== 'string' || !/^[a-f0-9]{64}$/.test(signature) || activeSecret.length < 32)
+        return false;
+    const actual = Buffer.from(signature, 'hex');
+    return [activeSecret, previousSecret].filter(secret => secret.length >= 32)
+        .some(secret => timingSafeEqual(actual, createHmac('sha256', secret).update(payload).digest()));
+}
 //# sourceMappingURL=command-result-signature.js.map

package/dist/security/managed-http.js

@@ -1,5 +1,5 @@
 /** Versioned native HTTP identity. Hashes bind request bytes; bodies stay off receipts. */
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { z } from 'zod';
 import { commandFenceSchema } from '../schemas/node-responses.js';
 export const managedClaimSchema = z.object({
@@ -23,12 +23,26 @@ export const managedHttpRequestSchema = z.object({
 export const MANAGED_HTTP_COMMAND = '@magnus/native-http/v1';
 export const MANAGED_CLAIM_HEADER = 'x-magnus-resource-claim';
 export function sha256(value) { return createHash('sha256').update(value).digest('hex'); }
-export function httpIdentity(commandId, method, path, body, timeoutSeconds) {
+function assertCanonicalMutation(method, path) {
     if (!['POST', 'PUT', 'PATCH', 'DELETE'].includes(method) || !/^\/[a-zA-Z0-9/_%.:-]+$/.test(path)
         || path.includes('..') || path.includes('//'))
         throw new Error('Canonical native mutation path required');
+}
+export function httpIdentity(commandId, method, path, body, timeoutSeconds) {
+    assertCanonicalMutation(method, path);
     return { commandId, command: MANAGED_HTTP_COMMAND, args: [method, path, sha256(body)], timeoutSeconds };
 }
+/**
+ * Identity for a mutation whose body holds a credential. A stored digest of
+ * such a body could be guessed offline, so the digest slot holds a random value
+ * instead of binding the body. That is safe only because these paths are absent
+ * from the native completion registry on every released agent: they are never
+ * sent through managed HTTP, so no agent ever recomputes this identity.
+ */
+export function credentialMutationIdentity(commandId, method, path, timeoutSeconds) {
+    assertCanonicalMutation(method, path);
+    return { commandId, command: MANAGED_HTTP_COMMAND, args: [method, path, sha256(randomUUID())], timeoutSeconds };
+}
 export function httpReceiptBody(method, path, status, body) {
     return JSON.stringify({ version: 1, method, path, status, responseSha256: sha256(body) });
 }

package/dist/services/agent-upgrade-preflight.js

@@ -0,0 +1,162 @@
+import { randomUUID } from 'node:crypto';
+import { getDb } from '../db/client.js';
+import { fenceProofSigningPayload, verifyCommandEvidence } from '../security/command-result-signature.js';
+import { NodeApiClient } from '../sdk/node-api-client.js';
+import { getNodeManagerSecret, RESULT_SIGNING_PREVIOUS_SECRET, RESULT_SIGNING_SECRET } from '../utils/env.js';
+import { getNodeBaseUrl } from '../utils/node-url.js';
+import { advanceCommandFence, checkReportedFence, fenceIsAhead, isOwnFenceKey } from './command-fence-repair.js';
+import { recoverAbsentLeases } from './command-resource-leases.js';
+/**
+ * Why self-upgrade is off, from what the agent says. Only an operator's choice
+ * and a layout that cannot do it are expected; anything else is a problem someone
+ * has to fix, so it must not read as expected and let a run look finished.
+ */
+export function selfUpgradeOffCode(report) {
+    switch (report.reasonCode) {
+        case 'SWITCHED_OFF':
+        case 'CONTAINER':
+        case 'LEGACY_LAYOUT': return 'SELF_UPGRADE_DISABLED';
+        case 'NOT_WRITABLE': return 'PREFLIGHT_NOT_WRITABLE';
+        case 'NPM': return 'PREFLIGHT_NPM';
+        case undefined:
+        case null:
+            // An agent older than 1.1.8 sends text only. Its layouts and this one exact string are all it can say safely.
+            return report.layout !== 'versioned' || report.reason === 'MAGNUS_AGENT_SELF_UPGRADE=false on this node' ? 'SELF_UPGRADE_DISABLED' : 'SELF_UPGRADE_UNAVAILABLE';
+        default: return 'SELF_UPGRADE_UNAVAILABLE';
+    }
+}
+/** Hold a report against what an upgrade needs. A field the agent left out proves nothing either way. */
+export function evaluatePreflight(report, context) {
+    const problems = [];
+    const add = (code, detail, blocking = true) => problems.push({ code, detail, blocking });
+    const versioned = report.layout === 'versioned';
+    if (report.enabled === false && !versioned)
+        add(selfUpgradeOffCode({ reasonCode: report.reasonCode, reason: report.reason ?? null, layout: report.layout ?? 'legacy' }), report.reason ?? 'self-upgrade is unavailable on this node');
+    if (versioned) {
+        if (report.writable && !report.writable.ok)
+            add('PREFLIGHT_NOT_WRITABLE', `the agent cannot write ${report.writable.blocked?.join(', ') || 'its install directory'}`);
+        if (report.npm && (!report.npm.present || report.npm.problem))
+            add('PREFLIGHT_NPM', report.npm.problem ?? 'npm is missing');
+        if (report.fromVersionComplete === false)
+            add('PREFLIGHT_FROM_INCOMPLETE', `versions/${report.version} is incomplete or ./current does not run it, so a failed upgrade could not switch back`);
+        if (report.enabled === false && !report.writable?.blocked?.length && !report.npm?.problem)
+            add(selfUpgradeOffCode({ reasonCode: report.reasonCode, reason: report.reason ?? null, layout: 'versioned' }), report.reason ?? 'self-upgrade is unavailable on this node');
+    }
+    if (report.disk?.low)
+        add('PREFLIGHT_DISK_LOW', `${mib(report.disk.freeBytes)} MiB free in the install directory; an upgrade needs ${mib(report.disk.requiredBytes)} MiB`);
+    if (report.engines?.compatible === false)
+        add('PREFLIGHT_ENGINE', report.engines.problem ?? `the target does not support Node.js ${report.engines.node}`);
+    const ledger = report.ledger;
+    if (ledger && !ledger.readable)
+        add('LEDGER_UNAVAILABLE', ledger.error ?? 'the command ledger cannot be read', context.managed);
+    // A command running right now finishes by itself: it is a busy slot, which dispatch refuses and the end of the wave retries.
+    else if (ledger?.activeLock?.receipt === 'running')
+        add('RESOURCE_BUSY', `${ledger.activeLock.commandId} is running on this node right now`, false);
+    else if (ledger?.activeLock && !['none', 'completed', 'reconciled'].includes(ledger.activeLock.receipt)) {
+        add('LEDGER_UNRESOLVED', `active.lock is held by ${ledger.activeLock.commandId}, whose receipt is ${ledger.activeLock.receipt}`, context.managed);
+    }
+    else if (ledger?.unresolvedReceipts)
+        add('LEDGER_UNRESOLVED', `${ledger.unresolvedReceipts} receipt(s) have no terminal outcome`, false);
+    const marker = report.managedEnrollment;
+    if (report.managedEnrollmentState === 'unknown')
+        add('ENROLLMENT_MARKER_UNKNOWN', 'the agent could not read its managed-enrollment marker');
+    if (marker && (marker.nodeId !== context.nodeId || marker.nodeFqdn !== context.hostname))
+        add('ENROLLMENT_MARKER_MISMATCH', `the agent's managed-enrollment marker is for ${marker.nodeFqdn} (${marker.nodeId}), not this node`);
+    for (const [key, value] of Object.entries(report.fenceHighWater ?? {})) {
+        // Keys of other domains are not this node's to report, and nothing here adopts them.
+        if (!isOwnFenceKey(context.nodeId, key))
+            continue;
+        const check = checkReportedFence(context.nodeId, key, value);
+        if (!check.ok)
+            add(check.code, `the agent ${check.detail}`);
+        else if (fenceIsAhead(check.value, context.generations[key] ?? 0n))
+            add('FENCE_AHEAD', `the agent holds fence ${value} for ${key}; the server's generation is ${(context.generations[key] ?? 0n).toString()}`, false);
+    }
+    return problems;
+}
+const mib = (bytes) => Math.floor(bytes / (1024 * 1024));
+export function preflightStatus(problems) {
+    return problems.some(problem => problem.blocking) ? 'blocked' : problems.length ? 'warning' : 'ok';
+}
+/** Bounded concurrency: one slow agent never delays the others by more than its timeout. */
+export async function mapLimit(items, limit, work) {
+    const results = new Array(items.length);
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (next < items.length) {
+            const index = next++;
+            results[index] = await work(items[index]);
+        }
+    }));
+    return results;
+}
+/** The first version that reports its fence, answers the trace and says what it refused. */
+export const CERTAIN_REFUSALS_SINCE = '1.1.8';
+export const PREFLIGHT_CONCURRENCY = 16;
+export const PREFLIGHT_TIMEOUT_MS = 4_000;
+function clientFor(node) {
+    return new NodeApiClient({ baseUrl: getNodeBaseUrl(node.ipAddress, node.port), resourceNodeId: node.id, nodeToken: getNodeManagerSecret(),
+        resourceActorId: 'system:agent-upgrade', timeout: PREFLIGHT_TIMEOUT_MS });
+}
+const generationsOf = async (nodeId) => {
+    const rows = await getDb().commandResourceFence.findMany({ where: { OR: [{ resourceKey: `node:${nodeId}` }, { resourceKey: { startsWith: `node:${nodeId}:` } }] } });
+    return Object.fromEntries(rows.map(row => [row.resourceKey, row.generation]));
+};
+/** Ask one agent. Never throws: a node's problem must not stop the check of the others. */
+export async function preflightNode(target, query, options = {}) {
+    const checkedAt = new Date(options.now ?? Date.now()).toISOString();
+    try {
+        const node = await getDb().node.findUnique({ where: { id: target.nodeId }, select: { id: true, ipAddress: true, port: true } });
+        if (!node)
+            return { status: 'unavailable', checkedAt, problems: [], detail: 'node is no longer in the inventory' };
+        const nonce = randomUUID();
+        const answer = await clientFor(node).getUpgradePreflight({ ...query, nonce }, options.timeoutMs ?? PREFLIGHT_TIMEOUT_MS);
+        if (answer.status !== 'ok')
+            return { status: answer.status, checkedAt, problems: [], detail: answer.detail };
+        const problems = evaluatePreflight(answer.report, { nodeId: target.nodeId, hostname: target.hostname, managed: target.managed, generations: await generationsOf(target.nodeId) });
+        const proof = answer.report.fenceProof;
+        const fencesVerified = Boolean(proof && proof.nonce === nonce && proof.nodeFqdn.toLowerCase() === target.hostname.toLowerCase() && verifyCommandEvidence(fenceProofSigningPayload({ nonce, nodeFqdn: proof.nodeFqdn, fenceHighWater: answer.report.fenceHighWater ?? {} }), proof.signature, RESULT_SIGNING_SECRET, RESULT_SIGNING_PREVIOUS_SECRET));
+        return { status: preflightStatus(problems), checkedAt, problems, report: answer.report, fencesVerified };
+    }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        console.warn(`[agent-upgrade] preflight of ${target.nodeId} failed: ${detail}`);
+        return { status: 'unreachable', checkedAt, problems: [], detail };
+    }
+}
+/** Ask one node about its leases stuck `unknown`, and settle what its signed account proves. Safe to repeat. */
+export async function recoverNodeLeases(nodeId, actorId, now = Date.now()) {
+    const node = await getDb().node.findUnique({ where: { id: nodeId }, select: { id: true, ipAddress: true, port: true, hostname: true } });
+    if (!node)
+        return [];
+    const client = clientFor(node);
+    return recoverAbsentLeases(node.id, node.hostname, (commandId, nonce, resourceKey) => client.getCommandTrace(commandId, nonce, resourceKey, PREFLIGHT_TIMEOUT_MS), actorId, now, (commandId, expected) => client.getCommandReceipt(commandId, expected));
+}
+/** What a check repaired, in words. Fence and lease repairs are safe to repeat; each change is audited. */
+export async function healNode(target, preflight, actorId = 'system:agent-upgrade') {
+    const healed = [];
+    const problems = [...preflight.problems];
+    const fences = Object.entries(preflight.report?.fenceHighWater ?? {});
+    if (fences.length && !preflight.fencesVerified)
+        console.warn(`[agent-upgrade] the fences reported by ${target.nodeId} carry no valid proof; generations unchanged`);
+    for (const [resourceKey, highWater] of preflight.fencesVerified ? fences : []) {
+        if (!isOwnFenceKey(target.nodeId, resourceKey))
+            continue;
+        const advanced = await advanceCommandFence({ nodeId: target.nodeId, resourceKey, highWater, source: 'preflight', actorId });
+        if (advanced.status !== 'advanced')
+            continue;
+        healed.push(`moved the fence of ${resourceKey} from ${advanced.from} to ${advanced.to}`);
+        const index = problems.findIndex(problem => problem.code === 'FENCE_AHEAD' && problem.detail.includes(` for ${resourceKey};`));
+        if (index >= 0)
+            problems.splice(index, 1);
+    }
+    const leases = await recoverNodeLeases(target.nodeId, actorId);
+    for (const lease of leases) {
+        if (lease.outcome === 'settled')
+            healed.push(`settled lease ${lease.commandId} as not dispatched: the agent holds no receipt for it`);
+        if (lease.outcome === 'unverified' && lease.code)
+            problems.push({ code: lease.code, detail: `lease ${lease.commandId}: ${lease.detail ?? 'unverified'}`, blocking: false });
+    }
+    return { healed, leases, problems };
+}
+//# sourceMappingURL=agent-upgrade-preflight.js.map

package/dist/services/agent-upgrade-readiness.js

@@ -0,0 +1,203 @@
+/**
+ * Whether a node could take an agent upgrade, all the time and not just when
+ * one starts. Pure evaluation of what the agent's heartbeat reports against the
+ * server's own state, so an operator sees drift weeks before an upgrade, and a
+ * monitor that notifies when a node has been not ready for a day.
+ */
+import { compareVersions, isVersion, upgradeRemedy } from '@hivemagnus/magnus-contracts';
+import { getDb } from '../db/index.js';
+import { SENTINEL_NODE_IDS } from '../utils/sentinel-nodes.js';
+import { upgradeSettings } from '../config/features.js';
+import { CERTAIN_REFUSALS_SINCE, recoverNodeLeases, selfUpgradeOffCode } from './agent-upgrade-preflight.js';
+import { storedSelfUpgrade } from './agent-upgrade-service.js';
+import { checkReportedFence, fenceIsAhead, isOwnFenceKey } from './command-fence-repair.js';
+import { ABSENT_RECEIPT_MARGIN_MS, HELD_RESOURCE_STATES } from './command-resource-leases.js';
+import { enrollmentIsComplete } from './resource-enrollment.js';
+import { createNotificationInTransaction } from './notification-service.js';
+import { matchingMaintenance } from './maintenance-service.js';
+const DAY_MS = 24 * 60 * 60_000;
+const problem = (code, detail) => {
+    const { cause, remedy } = upgradeRemedy(code);
+    return { code, detail, cause, remedy };
+};
+export function computeUpgradeReadiness(input) {
+    const now = input.now ?? Date.now();
+    const report = storedSelfUpgrade(input.metadata);
+    const problems = [];
+    const enrollment = input.enrollment ?? null;
+    if (enrollment?.state === 'draining') {
+        const since = enrollment.startedAt ?? enrollment.updatedAt;
+        problems.push(problem('ENROLLMENT_DRAINING', `resource enrollment has been draining for ${Math.floor((now - since.getTime()) / DAY_MS)} day(s), since ${since.toISOString().slice(0, 10)}`));
+    }
+    if (report?.managedEnrollmentState === 'unknown')
+        problems.push(problem('ENROLLMENT_MARKER_UNKNOWN', 'the agent could not read its managed-enrollment marker'));
+    const marker = report?.managedEnrollment;
+    if (marker && (!enrollmentIsComplete(enrollment, marker.nodeFqdn) || marker.nodeId !== input.id || marker.nodeFqdn !== input.hostname))
+        problems.push(problem('ENROLLMENT_MARKER_MISMATCH', `the agent holds a managed-enrollment marker for ${marker.nodeFqdn}, but the server has ${enrollment ? `a ${enrollment.state} enrollment for ${enrollment.nodeFqdn}` : 'no enrollment row'}`));
+    for (const [key, value] of Object.entries(report?.fenceHighWater ?? {})) {
+        if (!isOwnFenceKey(input.id, key))
+            continue;
+        const check = checkReportedFence(input.id, key, value);
+        if (!check.ok)
+            problems.push(problem(check.code, `the agent ${check.detail}`));
+        else if (fenceIsAhead(check.value, input.generations?.[key] ?? 0n))
+            problems.push(problem('FENCE_AHEAD', `the agent holds fence ${value} for ${key}; the server's generation is ${(input.generations?.[key] ?? 0n).toString()}`));
+    }
+    const ledger = report?.ledger;
+    if (ledger && !ledger.readable)
+        problems.push(problem('LEDGER_UNAVAILABLE', ledger.error ?? 'the agent cannot read its command ledger'));
+    else if (ledger?.unresolvedReceipts)
+        problems.push(problem('LEDGER_UNRESOLVED', `${ledger.unresolvedReceipts} receipt(s) in the agent ledger have no terminal outcome${ledger.activeLock ? `; active.lock is held by ${ledger.activeLock.commandId}` : ''}`));
+    if (input.heldLeases)
+        problems.push(problem('LEASE_HELD', `${input.heldLeases} command lease(s) are reserved, running or unknown`));
+    // Only a node that could upgrade itself needs npm and disk for it. One that is off because of a problem says which.
+    const canSelfUpgrade = report?.layout === 'versioned' && report.enabled;
+    if (report && !report.enabled && report.layout === 'versioned') {
+        const code = selfUpgradeOffCode(report);
+        if (!upgradeRemedy(code).informational)
+            problems.push(problem(code, report.reason ?? 'self-upgrade is unavailable on this node'));
+    }
+    if (canSelfUpgrade && report?.disk?.low)
+        problems.push(problem('PREFLIGHT_DISK_LOW', `${Math.floor(report.disk.freeBytes / (1024 * 1024))} MiB free in the install directory; an upgrade needs ${Math.floor(report.disk.requiredBytes / (1024 * 1024))} MiB`));
+    if (canSelfUpgrade && report?.npm && (!report.npm.present || report.npm.problem))
+        problems.push(problem('PREFLIGHT_NPM', report.npm.problem ?? 'npm is missing on this node'));
+    const reportsDrift = Boolean(report && (report.ledger || report.disk || report.npm));
+    return { status: problems.length ? 'not-ready' : reportsDrift && report?.enabled ? 'ok' : 'unknown', problems, observedAt: report?.observedAt ?? null };
+}
+/** Readiness of many nodes at once, for list pages: one query per kind of server state. Never fails a page. */
+export async function readinessOfNodes(nodes, now = Date.now()) {
+    if (!nodes.length)
+        return new Map();
+    try {
+        const db = getDb();
+        const ids = nodes.map(node => node.id);
+        const [enrollments, fences, leases] = await Promise.all([
+            db.resourceEnrollment.findMany({ where: { nodeId: { in: ids } }, select: { nodeId: true, state: true, nodeFqdn: true, startedAt: true, updatedAt: true } }),
+            db.commandResourceFence.findMany({ where: { OR: ids.flatMap(id => [{ resourceKey: `node:${id}` }, { resourceKey: `node:${id}:ardent` }]) } }),
+            db.commandResourceLease.groupBy({ by: ['nodeId'], where: { nodeId: { in: ids }, state: { in: [...HELD_RESOURCE_STATES] } }, _count: { _all: true } }),
+        ]);
+        const generations = Object.fromEntries(fences.map(row => [row.resourceKey, row.generation]));
+        return new Map(nodes.map(node => [node.id, computeUpgradeReadiness({ id: node.id, hostname: node.hostname, metadata: node.metadata, now,
+                enrollment: enrollments.find(row => row.nodeId === node.id) ?? null, generations,
+                heldLeases: leases.find(row => row.nodeId === node.id)?._count._all ?? 0 })]));
+    }
+    catch (error) {
+        console.warn(`[upgrade-readiness] could not be computed: ${error instanceof Error ? error.message : String(error)}`);
+        return new Map();
+    }
+}
+// ── Monitor: notify when a node has been not ready for a day ──
+/** A node not ready for this long gets a notification, once per episode. */
+export const NOT_READY_NOTIFY_MS = DAY_MS;
+const CHECK_INTERVAL_MS = 10 * 60_000;
+let timer = null;
+/** How often one node is asked about its stuck leases. */
+export const LEASE_RECOVERY_INTERVAL_MS = 60 * 60_000;
+const lastLeaseRecovery = new Map();
+export function resetLeaseRecoveryLimits() { lastLeaseRecovery.clear(); }
+/**
+ * Settle leases nobody will come back for: no run may ever start for a node that
+ * already runs the target. Past the deadline plus a margin, on agents that answer
+ * the trace, at most once an hour per node; only signed ledger evidence settles
+ * one, and each settlement is audited by recoverAbsentLeases.
+ */
+async function recoverStuckLeases(now) {
+    const db = getDb();
+    const stuck = await db.commandResourceLease.findMany({ where: { state: 'unknown', deadline: { lt: new Date(now - ABSENT_RECEIPT_MARGIN_MS) } }, select: { nodeId: true }, distinct: ['nodeId'] });
+    if (!stuck.length)
+        return;
+    const nodes = await db.node.findMany({ where: { id: { in: stuck.map(row => row.nodeId) }, status: { in: ['ONLINE', 'DEGRADED'] } }, select: { id: true, name: true, agent: { select: { version: true } } } });
+    for (const node of nodes) {
+        const version = node.agent?.version ?? '';
+        if (!isVersion(version) || compareVersions(version, CERTAIN_REFUSALS_SINCE) < 0)
+            continue;
+        if (now - (lastLeaseRecovery.get(node.id) ?? -Infinity) < LEASE_RECOVERY_INTERVAL_MS)
+            continue;
+        lastLeaseRecovery.set(node.id, now);
+        const results = await recoverNodeLeases(node.id, 'system:upgrade-readiness', now).catch((error) => {
+            console.warn(`[upgrade-readiness] lease recovery on ${node.name} failed: ${error instanceof Error ? error.message : String(error)}`);
+            return [];
+        });
+        for (const lease of results)
+            if (lease.outcome === 'unverified')
+                console.warn(`[upgrade-readiness] lease ${lease.commandId} on ${node.name} stays held: ${lease.code}: ${lease.detail}`);
+    }
+}
+export async function checkUpgradeReadiness(now = Date.now()) {
+    const db = getDb();
+    await recoverStuckLeases(now).catch((error) => console.warn(`[upgrade-readiness] lease recovery failed: ${error instanceof Error ? error.message : String(error)}`));
+    const nodes = await db.node.findMany({ where: { id: { notIn: SENTINEL_NODE_IDS }, status: { in: ['ONLINE', 'DEGRADED'] } }, select: { id: true, name: true, hostname: true, metadata: true } });
+    const readiness = await readinessOfNodes(nodes, now);
+    // Lease recovery above runs regardless: leases settle whether or not this server drives upgrades.
+    const notify = upgradeSettings().agentUpgrades;
+    for (const node of nodes) {
+        const state = readiness.get(node.id);
+        if (!state)
+            continue;
+        // One node's failure is that node's: the rest of the tick still runs.
+        await checkOneNode(node, state, notify, now).catch((error) => console.warn(`[upgrade-readiness] ${node.name}: ${error instanceof Error ? error.message : String(error)}`));
+    }
+}
+async function checkOneNode(node, state, notify, now) {
+    const db = getDb();
+    const metadata = parseMetadata(node.metadata);
+    const since = typeof metadata['upgradeNotReadySince'] === 'string' ? metadata['upgradeNotReadySince'] : null;
+    // Read again right before writing: heartbeats keep their report in the same column.
+    const patch = async (change) => {
+        const row = await db.node.findUnique({ where: { id: node.id }, select: { metadata: true } });
+        await db.node.update({ where: { id: node.id }, data: { metadata: JSON.stringify(change(parseMetadata(row?.metadata))) } });
+    };
+    if (state.status !== 'not-ready') {
+        if (since)
+            await patch(withoutEpisode);
+        return;
+    }
+    if (!since)
+        return patch(current => ({ ...current, upgradeNotReadySince: new Date(now).toISOString() }));
+    if (!notify || metadata['upgradeNotReadyNotifiedAt'] || now - Date.parse(since) < NOT_READY_NOTIFY_MS)
+        return;
+    const maintenance = await matchingMaintenance(node.id);
+    await db.$transaction(async (tx) => {
+        // Several servers may run this monitor: the lock and a fresh read make one of them the one that notifies.
+        await tx.$executeRaw `SELECT pg_advisory_xact_lock(hashtextextended(${`upgrade-not-ready:${node.id}`}, 0))`;
+        const row = await tx.node.findUnique({ where: { id: node.id }, select: { metadata: true } });
+        const current = parseMetadata(row?.metadata);
+        if (current['upgradeNotReadyNotifiedAt'])
+            return;
+        await createNotificationInTransaction(tx, {
+            type: 'node.upgrade_not_ready', severity: 'warning', title: `${node.name} could not take an agent upgrade`,
+            message: `Not ready since ${since}: ${state.problems.map(item => `${item.code}: ${item.detail}. ${item.remedy}`).join(' ')}`.slice(0, 2000),
+            nodeId: node.id, nodeName: node.name, metadata: { since, codes: state.problems.map(item => item.code) },
+        }, maintenance?.id ?? null);
+        await tx.node.update({ where: { id: node.id }, data: { metadata: JSON.stringify({ ...current, upgradeNotReadyNotifiedAt: new Date(now).toISOString() }) } });
+    });
+}
+function parseMetadata(raw) {
+    try {
+        const parsed = JSON.parse(raw ?? '{}');
+        return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
+    }
+    catch (error) {
+        console.warn(`[upgrade-readiness] node metadata is not JSON: ${error instanceof Error ? error.message : String(error)}`);
+        return {};
+    }
+}
+function withoutEpisode(metadata) {
+    const { upgradeNotReadySince: _since, upgradeNotReadyNotifiedAt: _notified, ...rest } = metadata;
+    return rest;
+}
+export function startUpgradeReadinessMonitor() {
+    if (timer)
+        return;
+    const run = () => void checkUpgradeReadiness().catch((error) => console.error(`[upgrade-readiness] check failed: ${error instanceof Error ? error.message : String(error)}`));
+    run();
+    timer = setInterval(run, CHECK_INTERVAL_MS);
+    timer.unref?.();
+}
+export function stopUpgradeReadinessMonitor() {
+    if (timer) {
+        clearInterval(timer);
+        timer = null;
+    }
+}
+//# sourceMappingURL=agent-upgrade-readiness.js.map