@hivemagnus/magnus-server 1.1.7 → 1.1.8

package/dist/services/command-fence-repair.js

@@ -0,0 +1,72 @@
+/**
+ * Repairing the distance between the server's command fence generation and a
+ * node ledger that has seen a higher fence (written by an older build, or
+ * surviving a database restore). A fence is an ordering token the server
+ * owns: the ledger refuses anything at or below its high-water, so the only
+ * safe repair is to move the server's generation UP.
+ *
+ * Trust, precisely. The value must be signed with the fleet's result key, under
+ * a nonce this server just chose (a preflight) or bound to this node's lease and
+ * request (a STALE_FENCE refusal), and it must arrive over a connection this server
+ * opened to the node's inventory address. Only that node's own keys are accepted, and
+ * the cap keeps a value from pushing the column near BIGINT overflow. The result
+ * key and the node credential are shared by the fleet, so these signatures tell
+ * agents apart only by what they claim: whoever holds them can sign for any node.
+ * The limit is the connection, which ends at whatever the inventory says. A
+ * heartbeat with the fleet credential can repoint a node only while its stored
+ * address is loopback or unroutable (shouldRepairNodeIp), or to the DNS name of its own
+ * agent certificate; a routable address stays. So a holder of the fleet secrets,
+ * with a certificate the server trusts for its own address, could answer for a node
+ * in that state. It can then inflate that node's counters and settle its leases that
+ * left no receipt there. The cost of the counter is only numbering: a generation
+ * moves up and nothing depends on its size.
+ * Heartbeat values are never used for this.
+ *
+ * command_resource_fences has no database trigger. The increase-only triggers
+ * guard the separate shared_resource_fences table, so "never lower" is enforced
+ * here, under the same advisory lock admission takes for this node.
+ */
+import { getDb } from '../db/index.js';
+import { logAuditEvent } from './audit-service.js';
+/** Well below BIGINT's 9223372036854775807, with room for every later increment. */
+export const FENCE_HIGH_WATER_CAP = 4000000000000000000n;
+/** A node's fences live under `node:<id>` or `node:<id>:<scope>`. */
+export const isOwnFenceKey = (nodeId, resourceKey) => resourceKey === `node:${nodeId}` || resourceKey.startsWith(`node:${nodeId}:`);
+export function checkReportedFence(nodeId, resourceKey, value) {
+    if (!isOwnFenceKey(nodeId, resourceKey))
+        return { ok: false, code: 'FENCE_INVALID', detail: `reported a fence for ${resourceKey.slice(0, 100)}, which is not its own` };
+    if (typeof value !== 'string' || !/^[1-9][0-9]{0,18}$/.test(value))
+        return { ok: false, code: 'FENCE_INVALID', detail: `reported a fence for ${resourceKey} that is not a positive decimal` };
+    const parsed = BigInt(value);
+    if (parsed > FENCE_HIGH_WATER_CAP)
+        return { ok: false, code: 'FENCE_OVERFLOW', detail: `reports fence ${value} for ${resourceKey}, above the ${FENCE_HIGH_WATER_CAP} the server adopts` };
+    return { ok: true, value: parsed };
+}
+/** The generation admission would hand out next is below or at the agent's high-water. */
+export const fenceIsAhead = (highWater, generation) => highWater >= generation + 1n;
+/** Move the generation to high-water + 1, audited. Never lowers: the check and the write hold the node's admission lock. */
+export async function advanceCommandFence(input) {
+    const check = checkReportedFence(input.nodeId, input.resourceKey, input.highWater);
+    if (!check.ok) {
+        console.warn(`[command-fence] node ${input.nodeId} ${check.detail}; generation unchanged`);
+        return { status: 'rejected', code: check.code, detail: check.detail };
+    }
+    const moved = await getDb().$transaction(async (tx) => {
+        await tx.$executeRaw `SELECT pg_advisory_xact_lock(hashtextextended(${`command-resource-node:${input.nodeId}`}, 0))`;
+        const row = await tx.commandResourceFence.findUnique({ where: { resourceKey: input.resourceKey } });
+        const from = row?.generation ?? 0n;
+        if (!fenceIsAhead(check.value, from))
+            return null;
+        const to = check.value + 1n;
+        await tx.commandResourceFence.upsert({ where: { resourceKey: input.resourceKey },
+            create: { resourceKey: input.resourceKey, generation: to }, update: { generation: to } });
+        return { from, to };
+    });
+    if (!moved)
+        return { status: 'current' };
+    await logAuditEvent({ action: 'agent.fence.advanced', actorId: input.actorId ?? 'system:agent-upgrade', actorType: 'system', targetType: 'node', targetId: input.nodeId,
+        details: { resourceKey: input.resourceKey, oldGeneration: moved.from.toString(), newGeneration: moved.to.toString(), reportedHighWater: check.value.toString(), source: input.source } })
+        .catch((error) => console.warn(`[command-fence] audit of ${input.nodeId} failed: ${error instanceof Error ? error.message : String(error)}`));
+    return { status: 'advanced', ...moved };
+}
+//# sourceMappingURL=command-fence-repair.js.map

package/dist/services/command-resource-leases.js

@@ -6,18 +6,19 @@ import { assertNodeAdmissionOpen } from './admission-fence.js';
  * A node requires the explicit durable participant drain before its first lease; switching the flag off does
  * not remove held/unknown ownership or permit a legacy bypass on that node.
  */
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { nanoid } from 'nanoid';
 import { getDb } from '../db/index.js';
 import { HttpError } from '../middleware/error-handler.js';
 import { commandRequestDigest } from '../security/command-request.js';
 import { commandFenceSchema } from '../schemas/node-responses.js';
-import { verifyCommandResultSignature } from '../security/command-result-signature.js';
+import { commandAbsenceSigningPayload, commandRefusalSigningPayload, verifyCommandEvidence, verifyCommandResultSignature } from '../security/command-result-signature.js';
 import { RESULT_SIGNING_SECRET, RESULT_SIGNING_PREVIOUS_SECRET } from '../utils/env.js';
 import { getNodeBaseUrl } from '../utils/node-url.js';
 import { COMMAND_COMPLETION_REGISTRY, COMMAND_COMPLETION_REGISTRY_SHA256, commandCompletionDescriptor } from '../security/command-completion-registry.js';
 import { MANAGED_HTTP_COMMAND } from '../security/managed-http.js';
 import { NATIVE_COMPLETION_REGISTRY, NATIVE_COMPLETION_REGISTRY_SHA256, nativeCompletionDescriptor } from '../security/native-completion-registry.js';
+import { logAuditEvent } from './audit-service.js';
 const HELD = ['reserved', 'running', 'unknown'];
 export class ResourceLeaseConflict extends HttpError {
     constructor() { super(409, 'RESOURCE_LEASE_CONFLICT', 'Node or service has an unsettled command owner; reconcile its evidence before retrying'); }
@@ -204,28 +205,154 @@ export async function markResourceDispatched(admission, verifyBeforeDispatch) {
             throw new HttpError(409, 'RESOURCE_DEADLINE_INVALID', 'Resource admission expired or changed before dispatch');
     });
 }
-export async function failResourceCommand(admission) {
+/** The agent's signed word that this exact request did not start and has no receipt. Unsigned or mismatched words prove nothing. */
+function refusalBindsLease(row, nodeFqdn, refusal) {
+    return refusal.commandId === row.commandId && refusal.requestSha256 === row.requestSha256 && refusal.leaseId === row.id && refusal.nodeFqdn === nodeFqdn
+        && verifyCommandEvidence(commandRefusalSigningPayload(refusal), refusal.signature, RESULT_SIGNING_SECRET, RESULT_SIGNING_PREVIOUS_SECRET);
+}
+/**
+ * Close a lease whose dispatch ended without a result. A lease never sent, or
+ * one the agent proved it refused, is `not-dispatched`; anything else is
+ * `unknown` and held until reconciled. Answers whether the refusal settled it.
+ */
+export async function failResourceCommand(admission, refusal) {
     const db = getDb();
-    await db.$transaction(async (tx) => {
+    return db.$transaction(async (tx) => {
         const row = await tx.commandResourceLease.findUnique({ where: { id: admission.leaseId } });
         if (!row || row.state === 'released')
-            return;
+            return false;
         const notSent = row.state === 'reserved';
+        const refused = !notSent && refusal !== undefined && refusalBindsLease(row, admission.nodeFqdn, refusal);
+        if (refusal && !notSent && !refused)
+            console.warn(`[resource-leases] refusal for ${row.commandId} does not verify; the lease stays uncertain`);
         const changed = await tx.commandResourceLease.updateMany({ where: { id: row.id, state: row.state }, data: notSent
                 ? { state: 'released', settledAt: new Date(), settlement: 'not-dispatched',
                     evidenceSha256: createHash('sha256').update(`pre-post:${row.commandId}:${row.requestSha256}`).digest('hex') }
-                : { state: 'unknown' } });
+                : refused ? { state: 'released', settledAt: new Date(), settlement: 'not-dispatched',
+                    evidenceSha256: createHash('sha256').update(`certain-refusal:${JSON.stringify(refusal)}`).digest('hex') }
+                    : { state: 'unknown' } });
         if (!changed.count)
-            return;
+            return false;
         if (row.directCommand) {
             await tx.remoteCommand.updateMany({ where: { id: row.commandId, status: 'RUNNING' }, data: { status: 'FAILED', completedAt: new Date() } });
             await tx.commandResult.upsert({ where: { commandId: row.commandId }, update: {}, create: {
                     commandId: row.commandId, exitCode: -1, stdout: '', durationMs: 0,
-                    stderr: notSent ? 'Resource admission ended before dispatch.' : 'Execution outcome unknown; resource ownership retained until verified evidence reconciliation.'
+                    stderr: notSent ? 'Resource admission ended before dispatch.' : refused ? `The node refused the request before it started (${refusal.code}); nothing ran.`
+                        : 'Execution outcome unknown; resource ownership retained until verified evidence reconciliation.'
                 } });
         }
+        return refused;
     });
 }
+/** Past the deadline the ledger refuses a late request, and /claim stops confirming it; the margin covers clock skew. */
+export const ABSENT_RECEIPT_MARGIN_MS = 60_000;
+/**
+ * What a signed trace with no receipt says about a lease held for this node, or
+ * why it says nothing. "No receipt" alone proves nothing, since a ledger that is
+ * gone holds nothing either. execute() writes in this order: claim, fence, marker,
+ * durable receipt, and only then runs the callback. So in a ledger that is there:
+ * - no fence, with this node's marker: the marker shows this is its ledger, and the
+ *   fence comes before everything that could run, so no request passed admission.
+ * - no fence and no marker: nothing shows this is the ledger the command would have
+ *   used. Stays held.
+ * - high-water below the lease's fence: the request never passed admission.
+ * - above it: the stale-refusal case, or a later fence won.
+ * - equal, with this command's claim: a crash or a failed write between the claim and
+ *   the receipt, which never started.
+ * - equal, with its durable record of a failed admission for this fence: the same.
+ * - equal with neither: the receipt was removed, or a build without such records
+ *   failed the admission. These look the same, so it stays held.
+ * Not provable from here: a ledger wiped and used again looks like a later fence
+ * winning, and one wiped and told its enrollment again looks like the first case.
+ */
+export function judgeTrace(answer, nodeId, leaseFence) {
+    const no = (code, detail) => ({ settle: false, code, detail });
+    if (!answer.ledgerExists)
+        return no('LEDGER_MISSING', 'the agent reports no ledger directory');
+    if (answer.markerState === 'unknown')
+        return no('ENROLLMENT_MARKER_UNKNOWN', 'the agent could not read its enrollment marker');
+    if (answer.markerState === 'present' && answer.markerNodeId !== nodeId)
+        return no('ENROLLMENT_MARKER_MISMATCH', `the agent's enrollment marker names ${answer.markerNodeId ?? 'nothing'}`);
+    if (answer.fenceHighWater === null || !/^[0-9]{1,20}$/.test(answer.fenceHighWater)) {
+        if (answer.markerState === 'present' && !answer.activeLock)
+            return { settle: true };
+        return no('LEDGER_NO_FENCE', `the ledger holds no fence for ${answer.resourceKey} and nothing shows it is this node's`);
+    }
+    const highWater = BigInt(answer.fenceHighWater);
+    if (highWater === leaseFence && !answer.activeLock && answer.admissionFailedFence !== String(leaseFence))
+        return no('LEDGER_EVIDENCE_INCONSISTENT', `the ledger accepted fence ${leaseFence} but holds no receipt, claim or failed admission for it`);
+    return { settle: true };
+}
+export async function recoverAbsentLeases(nodeId, nodeFqdn, trace, actorId = 'system:agent-upgrade', now = Date.now(), signedReceipt) {
+    const db = getDb();
+    const rows = await db.commandResourceLease.findMany({ where: { nodeId, state: 'unknown' } });
+    const results = [];
+    for (const row of rows) {
+        const { commandId } = row;
+        if (row.completionClass === 'managed-scheduler' || row.completionDescriptorId === 'skill-install') {
+            results.push({ commandId, outcome: 'unsupported' });
+            continue;
+        }
+        if (row.deadline.getTime() + ABSENT_RECEIPT_MARGIN_MS > now) {
+            results.push({ commandId, outcome: 'not-past-deadline' });
+            continue;
+        }
+        const coordination = commandFenceSchema.parse(JSON.parse(row.coordinationJson));
+        const nonce = randomUUID();
+        let answer;
+        try {
+            answer = await trace(commandId, nonce, coordination.resourceKey);
+        }
+        catch (error) {
+            console.warn(`[resource-leases] trace of ${commandId} failed: ${error instanceof Error ? error.message : String(error)}`);
+            results.push({ commandId, outcome: 'unavailable' });
+            continue;
+        }
+        // An agent without this route keeps the current behaviour: held until reconciled.
+        if (!answer) {
+            results.push({ commandId, outcome: 'unsupported' });
+            continue;
+        }
+        if (answer.commandId !== commandId || answer.nonce !== nonce || answer.nodeFqdn !== nodeFqdn || answer.resourceKey !== coordination.resourceKey
+            || !verifyCommandEvidence(commandAbsenceSigningPayload(answer), answer.signature, RESULT_SIGNING_SECRET, RESULT_SIGNING_PREVIOUS_SECRET)) {
+            console.warn(`[resource-leases] trace of ${commandId} does not verify; the lease stays held`);
+            results.push({ commandId, outcome: 'unverified', code: 'LEDGER_EVIDENCE_INCONSISTENT', detail: 'the signed answer does not bind this command, node and nonce' });
+            continue;
+        }
+        if (answer.receipt) {
+            // The usual path out for a receipt: a signed result or reconciliation settles it, anything else stays held.
+            await signedReceipt?.(commandId, { nodeFqdn, requestSha256: row.requestSha256 }).catch((error) => {
+                console.warn(`[resource-leases] receipt of ${commandId} settles nothing: ${error instanceof Error ? error.message : String(error)}`);
+            });
+            results.push({ commandId, outcome: 'has-trace' });
+            continue;
+        }
+        const judged = judgeTrace(answer, nodeId, BigInt(coordination.fence));
+        if (!judged.settle) {
+            console.warn(`[resource-leases] lease ${commandId} stays held: ${judged.code}: ${judged.detail}`);
+            results.push({ commandId, outcome: 'unverified', code: judged.code, detail: judged.detail });
+            continue;
+        }
+        const evidence = createHash('sha256').update(`receipt-absent:${JSON.stringify(answer)}`).digest('hex');
+        const settled = await db.$transaction(async (tx) => {
+            await tx.$executeRaw `SELECT pg_advisory_xact_lock(hashtextextended(${`command-resource-node:${nodeId}`}, 0))`;
+            const changed = await tx.commandResourceLease.updateMany({ where: { id: row.id, state: 'unknown' },
+                data: { state: 'released', settledAt: new Date(), settlement: 'not-dispatched', evidenceSha256: evidence } });
+            if (!changed.count)
+                return false;
+            if (row.directCommand)
+                await tx.remoteCommand.updateMany({ where: { id: commandId, status: 'RUNNING' }, data: { status: 'FAILED', completedAt: new Date() } });
+            return true;
+        });
+        if (!settled)
+            continue;
+        await logAuditEvent({ action: 'agent.lease.recovered', actorId, actorType: 'system', targetType: 'node', targetId: nodeId,
+            details: { commandId, leaseId: row.id, settlement: 'not-dispatched', evidenceSha256: evidence, reason: 'the agent ledger holds no receipt for it after the lease deadline', fenceHighWater: answer.fenceHighWater, leaseFence: coordination.fence, activeLock: answer.activeLock } })
+            .catch((error) => console.warn(`[resource-leases] audit of ${commandId} failed: ${error instanceof Error ? error.message : String(error)}`));
+        results.push({ commandId, outcome: 'settled' });
+    }
+    return results;
+}
 export async function retainUnknownResourceCommand(commandId) {
     await getDb().$transaction(async (tx) => {
         // Persisted 'reserved' proves the sole POST barrier was never crossed. This

package/dist/services/resource-enrollment.js

@@ -222,6 +222,8 @@ export async function readParticipantReceipt(response, limit = PARTICIPANT_RECEI
         await reader.cancel().catch(() => undefined);
     }
 }
+/** The reply is a fixed code; what was wrong with the stored evidence goes to the log. */
+const warnEvidence = (what, nodeId, error) => console.warn(`[resource-enrollment] ${what} of node ${nodeId} does not parse: ${error instanceof Error ? error.message : String(error)}`);
 export async function refreshEnrollmentParticipant(nodeId, participant) {
     // Fixed inventories and origins only: operator input cannot choose an endpoint or key.
     const prepared = await getDb().$transaction(async (tx) => {
@@ -328,7 +330,8 @@ export async function acceptSchedulerBoundaryAttestation(nodeId, raw) {
         try {
             receipt = schedulerBoundaryReceiptSchema.parse(JSON.parse(ack?.receiptJson ?? 'null'));
         }
-        catch {
+        catch (error) {
+            warnEvidence('scheduler receipt', nodeId, error);
             fail('SCHEDULER_BOUNDARY_EVIDENCE_INVALID', 'Current signed scheduler receipt is required');
         }
         const { signature: receiptSignature, ...receiptUnsigned } = receipt;
@@ -411,7 +414,8 @@ export async function completeEnrollment(nodeId) {
                     reviewed = z.object({ receipt: schedulerBoundaryReceiptSchema, assertion: schedulerBoundaryAttestationSchema }).strict()
                         .parse(JSON.parse(ack.receiptJson ?? 'null'));
                 }
-                catch {
+                catch (error) {
+                    warnEvidence('reviewed scheduler receipt', nodeId, error);
                     fail('RESOURCE_DRAIN_INCOMPLETE', 'Reviewed scheduler receipt is unavailable');
                 }
                 const { receipt, assertion } = reviewed;
@@ -440,7 +444,8 @@ export async function completeEnrollment(nodeId) {
                     ? apiDrainAttestationSchema.parse(JSON.parse(ack.receiptJson ?? 'null'))
                     : drainReceiptSchema.parse(JSON.parse(ack.receiptJson ?? 'null'));
             }
-            catch {
+            catch (error) {
+                warnEvidence(`${participant} receipt`, nodeId, error);
                 fail('RESOURCE_DRAIN_INCOMPLETE', 'Stored participant receipt is unavailable');
             }
             const { signature, ...unsigned } = receipt;
@@ -487,7 +492,8 @@ export async function dashboardParticipantReadiness() {
             await tx.resourceParticipantWork.findFirst({ select: { workId: true } });
         }, { maxWait: 2_000, timeout: 3_000 });
     }
-    catch {
+    catch (error) {
+        console.warn(`[resource-enrollment] participant schema check failed: ${error instanceof Error ? error.message : String(error)}`);
         throw new HttpError(503, 'RESOURCE_PARTICIPANT_UNAVAILABLE', 'Durable participant schema is not ready');
     }
     return { version: 1, participant: 'dashboard', protocol: 'resource-enrollment-v1', ready: true };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hivemagnus/magnus-server",
-  "version": "1.1.7",
+  "version": "1.1.8",
   "description": "Magnus server: the fleet control plane API (Hono, PostgreSQL, Redis) for nodes running magnus-agent, with optional Hive IAM, Hive Router, HiveMem, Hermes Agent and external security console integrations.",
   "keywords": [
     "magnus",
@@ -84,7 +84,7 @@
     "prepublishOnly": "npm run db:generate && npm run clean && npm run build"
   },
   "dependencies": {
-    "@hivemagnus/magnus-contracts": "1.1.7",
+    "@hivemagnus/magnus-contracts": "1.1.8",
     "@hono/node-server": "^2.1.5",
     "@hono/swagger-ui": "^0.6.2",
     "@prisma/adapter-pg": "7.10.1",
@@ -102,7 +102,7 @@
     "ws": "^8.22.0",
     "yaml": "2.9.1",
     "zod": "^4.6.7",
-    "@hivemagnus/magnus-skills": "1.1.7",
+    "@hivemagnus/magnus-skills": "1.1.8",
     "prisma": "^7.10.1"
   },
   "devDependencies": {