@hivemagnus/magnus-agent 1.1.5 → 1.1.8

package/README.md

@@ -18,6 +18,10 @@ integrations is optional and stays inert until you configure it.
 - Command execution with timeouts, signed results and a durable command ledger.
 - Cron management, including finite managed schedules (see
   [MANAGED_SCHEDULER.md](https://github.com/ChefTaco/magnus/blob/main/packages/agent/MANAGED_SCHEDULER.md)).
+  Nodes upgraded from before 1.1.6 may keep one stale remembered Hermes schedule
+  source per past Hermes run; see
+  [Hermes schedule sources remembered before 1.1.6](https://github.com/ChefTaco/magnus/blob/main/packages/agent/MANAGED_SCHEDULER.md#hermes-schedule-sources-remembered-before-116)
+  for the cleanup.
 - File read and write with symlink-resolved path checks. Agent state and
   secrets can never be read or written through the API.
 - Skill publication, listing and removal, for example from

package/dist/agent.js

@@ -7,6 +7,7 @@ import { controlApiFetch } from './tls.js';
 import { startSpan, formatTraceparent } from './collectors/otel-collector.js';
 import { collectHermesState } from './executors/hermes-manager.js';
 import { existsSync } from 'node:fs';
+import { heartbeatDrift } from './upgrade/preflight.js';
 import { confirmTrial, controlApiUnreachable, recordTrialAlive, selfUpgradeStatus, trialOf } from './upgrade/self-upgrade.js';
 import { commandLedger } from './executors/command-executor.js';
 const HEARTBEAT_INTERVAL_MS = 30_000; // 30s
@@ -77,8 +78,11 @@ async function collectHealthWithDeadline() {
 }
 /** Self-upgrade status for the heartbeat, with the sticky local enrollment marker. */
 async function heartbeatSelfUpgrade() {
-    const enrollment = await commandLedger.managedEnrollment().catch(() => null);
-    return { ...selfUpgradeStatus(AGENT_VERSION), managedEnrollment: enrollment ? { nodeId: enrollment.nodeId, nodeFqdn: enrollment.nodeFqdn } : null };
+    const marker = await commandLedger.enrollmentMarker();
+    if (marker.error)
+        logger.warn(`Local enrollment marker could not be read: ${marker.error}`);
+    return { ...selfUpgradeStatus(AGENT_VERSION), ...heartbeatDrift(commandLedger),
+        managedEnrollment: marker.marker ? { nodeId: marker.marker.nodeId, nodeFqdn: marker.marker.nodeFqdn } : null, managedEnrollmentState: marker.state };
 }
 /** Stop the heartbeat loop. */
 export function stopHeartbeatLoop() {

package/dist/executors/command-executor.js

@@ -2,14 +2,16 @@ import { managedExecution } from '../security/managed-context.js';
 import { Hono } from 'hono';
 import { spawn } from 'node:child_process';
 import { z } from 'zod';
-import { RESULT_SIGNING_SECRET, MAGNUS_EXECUTION_SUPERVISOR_ENABLED } from '../env.js';
+import { HOST_FQDN, RESULT_SIGNING_SECRET, MAGNUS_EXECUTION_SUPERVISOR_ENABLED } from '../env.js';
 import { startSpan } from '../collectors/otel-collector.js';
-import { CommandLedger, CommandAdmissionError, commandFenceSchema, commandReconciliationSchema } from './command-ledger.js';
-import { signCommandResult } from '../security/command-result-signature.js';
+import { admissionCause, CommandLedger, CommandAdmissionError, commandFenceSchema, commandReconciliationSchema, commandRequestDigest } from './command-ledger.js';
+import { commandAbsenceSigningPayload, signCommandEvidence, signCommandResult } from '../security/command-result-signature.js';
+import { refusalError, refusalStatus } from './command-refusal.js';
 import { ExecutionSupervisor } from './execution-supervisor.js';
 import { mutationBarrier } from './mutation-barrier.js';
 import { COMMAND_COMPLETION_REGISTRY, COMMAND_COMPLETION_REGISTRY_SHA256, commandCompletionDescriptor } from '../security/command-completion-registry.js';
 import { childEnvironment } from '../utils/child-env.js';
+import { logger } from '../utils/logger.js';
 export const commandRoutes = new Hono();
 export const commandLedger = new CommandLedger();
 export const executionSupervisor = new ExecutionSupervisor();
@@ -119,7 +121,8 @@ commandRoutes.post('/execute', async (c) => {
     catch (error) {
         if (!(error instanceof CommandAdmissionError))
             throw error;
-        return c.json({ success: false, error: { code: error.code, message: error.message } }, error.code === 'LEDGER_UNAVAILABLE' ? 503 : 409);
+        logger.warn(`Command ${parsed.data.commandId} refused: ${error.code}: ${error.message}${admissionCause(error)}`);
+        return c.json({ success: false, error: await refusalError({ commandId: parsed.data.commandId, requestSha256: commandRequestDigest(parsed.data), leaseId: managedExecution.getStore()?.claim?.leaseId }, error.code, error.message, { notStarted: error.notStarted, ...(error.highWater ? { highWater: error.highWater } : {}) }) }, refusalStatus(error.code));
     }
 });
 /**
@@ -138,11 +141,11 @@ export async function executeCommand(input) {
     // Parse/snapshot even internal callers before awaiting or executing effects.
     const opts = executeBodySchema.parse(input);
     if (draining)
-        throw new CommandAdmissionError('LEDGER_UNAVAILABLE', 'Node is draining; no command executed');
+        throw new CommandAdmissionError('LEDGER_UNAVAILABLE', 'Node is draining; no command executed', { notStarted: true });
     if (RESULT_SIGNING_SECRET.length < 32)
         throw new CommandAdmissionError('LEDGER_UNAVAILABLE', 'Command signing is unavailable; no command executed');
     if (opts.coordination && !MAGNUS_EXECUTION_SUPERVISOR_ENABLED)
-        throw new CommandAdmissionError('LEDGER_UNAVAILABLE', 'Managed commands require the node execution supervisor; no command executed');
+        throw new CommandAdmissionError('LEDGER_UNAVAILABLE', 'Managed commands require the node execution supervisor; no command executed', { notStarted: true });
     const pending = commandLedger.execute(opts, (digest) => MAGNUS_EXECUTION_SUPERVISOR_ENABLED
         ? executeSupervised(opts, digest) : executeProcess(opts, digest), managedExecution.getStore()?.claim);
     pendingExecutions.add(pending);
@@ -293,10 +296,33 @@ commandRoutes.get('/receipts/:commandId', async (c) => {
             return c.json({ success: false, error: { code: 'NOT_FOUND', message: 'Command receipt not found' } }, 404);
         return c.json({ success: true, data: row });
     }
-    catch {
+    catch (error) {
+        logger.warn(`Command receipt ${id} could not be read: ${error instanceof Error ? error.message : String(error)}`);
         return c.json({ success: false, error: { code: 'LEDGER_UNAVAILABLE', message: 'Command receipt unavailable; do not replay' } }, 503);
     }
 });
+// Whether anything local refers to a command, for leases the server holds as
+// `unknown`. Read only and signed with a nonce: it says what this ledger holds
+// now, and the server decides what that proves.
+commandRoutes.get('/receipts/:commandId/trace', async (c) => {
+    const id = c.req.param('commandId');
+    const nonce = c.req.query('nonce') ?? '';
+    const resourceKey = c.req.query('resourceKey') ?? '';
+    if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(id) || !/^[a-zA-Z0-9-]{16,64}$/.test(nonce) || !/^[a-zA-Z0-9][a-zA-Z0-9:._-]{0,199}$/.test(resourceKey))
+        return c.json({ success: false, error: { code: 'VALIDATION_ERROR', message: 'Invalid command identifier, nonce or resource key' } }, 400);
+    try {
+        const trace = await commandLedger.commandTrace(id, resourceKey);
+        if (trace.marker.error)
+            logger.warn(`Enrollment marker could not be read: ${trace.marker.error}`);
+        const unsigned = { commandId: id, nodeFqdn: HOST_FQDN, nonce, ledgerExists: trace.ledgerExists, receipt: trace.receipt, activeLock: trace.activeLock, resourceKey,
+            fenceHighWater: trace.fenceHighWater, markerState: trace.marker.state, markerNodeId: trace.marker.marker?.nodeId ?? null, admissionFailedFence: trace.admissionFailedFence };
+        return c.json({ success: true, data: { ...unsigned, signature: signCommandEvidence(commandAbsenceSigningPayload(unsigned), RESULT_SIGNING_SECRET) } });
+    }
+    catch (error) {
+        logger.warn(`Command trace ${id} could not be read: ${error instanceof Error ? error.message : String(error)}`);
+        return c.json({ success: false, error: { code: 'LEDGER_UNAVAILABLE', message: 'Command ledger unavailable' } }, 503);
+    }
+});
 // Existing node authentication is necessary but insufficient: only the separate
 // service reconciliation key can attest that an operator reviewed the evidence.
 // This releases a receipt claim, never executes or authorizes remediation.
@@ -311,6 +337,7 @@ commandRoutes.post('/receipts/:commandId/reconcile', async (c) => {
     catch (error) {
         if (!(error instanceof CommandAdmissionError))
             throw error;
+        logger.warn(`Reconciliation of ${parsed.data.commandId} refused: ${error.code}: ${error.message}${admissionCause(error)}`);
         return c.json({ success: false, error: { code: error.code, message: error.message } }, error.code === 'LEDGER_UNAVAILABLE' ? 503 : error.code === 'INVALID_RECONCILIATION' ? 403 : 409);
     }
 });

package/dist/executors/command-ledger.d.ts

@@ -55,9 +55,59 @@ export declare const commandReconciliationSchema: z.ZodObject<{
     expiresAt: z.ZodNumber;
     signature: z.ZodString;
 }, z.core.$strict>;
+export type CommandAdmissionCode = 'RESOURCE_BUSY' | 'OUTCOME_UNKNOWN' | 'IDEMPOTENCY_CONFLICT' | 'STALE_FENCE' | 'LEASE_EXPIRED' | 'LEDGER_UNAVAILABLE' | 'INVALID_RECONCILIATION' | 'UNMANAGED_MUTATION' | 'RESOURCE_CLAIM_INVALID';
+/** What went wrong underneath an admission error, for a WARN line. */
+export declare const admissionCause: (error: CommandAdmissionError) => string;
 export declare class CommandAdmissionError extends Error {
-    readonly code: 'RESOURCE_BUSY' | 'OUTCOME_UNKNOWN' | 'IDEMPOTENCY_CONFLICT' | 'STALE_FENCE' | 'LEASE_EXPIRED' | 'LEDGER_UNAVAILABLE' | 'INVALID_RECONCILIATION' | 'UNMANAGED_MUTATION';
-    constructor(code: 'RESOURCE_BUSY' | 'OUTCOME_UNKNOWN' | 'IDEMPOTENCY_CONFLICT' | 'STALE_FENCE' | 'LEASE_EXPIRED' | 'LEDGER_UNAVAILABLE' | 'INVALID_RECONCILIATION' | 'UNMANAGED_MUTATION', message: string);
+    readonly code: CommandAdmissionCode;
+    /** The callback demonstrably never ran and this command id has no receipt: a certain refusal. */
+    notStarted: boolean;
+    /** STALE_FENCE: the fence this ledger already accepted for the key, so the server can move past it. */
+    readonly highWater?: {
+        resourceKey: string;
+        fence: string;
+    };
+    constructor(code: CommandAdmissionCode, message: string, details?: {
+        notStarted?: boolean;
+        highWater?: {
+            resourceKey: string;
+            fence: string;
+        };
+        cause?: unknown;
+    });
+}
+declare const enrollmentMarkerSchema: z.ZodObject<{
+    version: z.ZodLiteral<1>;
+    nodeId: z.ZodString;
+    nodeFqdn: z.ZodString;
+}, z.core.$strict>;
+export type EnrollmentMarker = {
+    state: 'absent' | 'present' | 'unknown';
+    marker: z.infer<typeof enrollmentMarkerSchema> | null;
+    error?: string;
+};
+export interface CommandTrace {
+    ledgerExists: boolean;
+    receipt: boolean;
+    activeLock: boolean;
+    fenceHighWater: string | null;
+    marker: EnrollmentMarker;
+    /** This request passed the fence, then a later write failed, and the callback did not start. */
+    admissionFailed: boolean;
+    admissionFailedFence: string | null;
+}
+/** Read-only view of the ledger for preflight checks and heartbeats. */
+export interface LedgerHealth {
+    readable: boolean;
+    activeLock: {
+        commandId: string;
+        receipt: string;
+    } | null;
+    nonTerminalReceipts: number;
+    unresolvedReceipts: number;
+    truncated: boolean;
+    fenceHighWater: Record<string, string>;
+    error: string | null;
 }
 export declare function commandRequestDigest(input: CommandIdentity): string;
 export declare class CommandLedger {
@@ -82,6 +132,15 @@ export declare class CommandLedger {
     private read;
     private write;
     private receiptName;
+    private failedAdmissionName;
+    /**
+     * A write after the fence failed, so the fence is ahead with no receipt, and the
+     * callback did not start. Say so durably, or that state looks like a receipt that
+     * went missing. If not even this can be written, keep the claim instead: it is
+     * already on disk, needs no new space, and says the same (recoverClaim clears it).
+     * A receipt that did land must keep its claim too, or it can never be reconciled.
+     */
+    private keepEvidenceOfFailedAdmission;
     receipt(id: string): Promise<{
         state: string;
         version: 1;
@@ -309,8 +368,28 @@ export declare class CommandLedger {
             } | undefined;
         } | undefined;
     }>;
+    /**
+     * The sticky marker as it is, without creating anything. An unreadable marker
+     * is `unknown`, not absent: reporting nothing could hide a mismatch.
+     */
+    enrollmentMarker(): Promise<EnrollmentMarker>;
+    /**
+     * What this ledger holds about a command, read only and without the mutex. On its
+     * own "no receipt" proves nothing: a ledger that is gone holds nothing either. So
+     * this also says whether there is a ledger, what fence it last accepted for the
+     * key and what its marker says. Ordering in execute(): active.lock, fence, durable
+     * running receipt, and only then the callback, so a claim without a receipt
+     * never started.
+     */
+    commandTrace(id: string, resourceKey?: string): Promise<CommandTrace>;
+    /**
+     * Read-only health for preflight checks and heartbeats. Takes no mutex and
+     * creates nothing, so asking never competes with a running command.
+     */
+    health(): Promise<LedgerHealth>;
     /** Persist an authoritative observation; never clears an active/unknown claim. */
     observeManagedEnrollment(identity: Pick<ManagedClaim, 'nodeId' | 'nodeFqdn'>): Promise<void>;
     execute(input: CommandIdentity, run: (digest: string) => Promise<CommandReceiptResult>, enrollment?: ManagedClaim): Promise<CommandReceiptResult>;
 }
+export {};
 //# sourceMappingURL=command-ledger.d.ts.map

package/dist/executors/command-ledger.js

@@ -2,7 +2,7 @@
 import { createHash, randomUUID } from 'node:crypto';
 import { constants } from 'node:fs';
 import { spawn } from 'node:child_process';
-import { lstat, mkdir, open, rename, unlink } from 'node:fs/promises';
+import { lstat, mkdir, open, readdir, rename, unlink } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 import { z } from 'zod';
 import { HOST_FQDN, NODE_MANAGER_SECRET, RESULT_SIGNING_SECRET } from '../env.js';
@@ -48,13 +48,29 @@ const receiptSchema = z.object({
     finishedAt: z.number().int().nonnegative().optional(), coordination: commandFenceSchema.optional(),
     result: resultSchema.optional(), reconciliation: commandReconciliationSchema.optional(),
 }).strict();
+/** What went wrong underneath an admission error, for a WARN line. */
+export const admissionCause = (error) => error.cause === undefined ? '' : ` (cause: ${error.cause instanceof Error ? error.cause.message : String(error.cause)})`;
 export class CommandAdmissionError extends Error {
     code;
-    constructor(code, message) {
-        super(message);
+    /** The callback demonstrably never ran and this command id has no receipt: a certain refusal. */
+    notStarted;
+    /** STALE_FENCE: the fence this ledger already accepted for the key, so the server can move past it. */
+    highWater;
+    constructor(code, message, details = {}) {
+        // The reply keeps a fixed code and message; the cause stays here for the caller's log.
+        super(message, details.cause === undefined ? undefined : { cause: details.cause });
         this.code = code;
+        this.notStarted = details.notStarted ?? false;
+        if (details.highWater)
+            this.highWater = details.highWater;
     }
 }
+const enrollmentMarkerSchema = z.object({ version: z.literal(1), nodeId: z.string().min(1).max(128), nodeFqdn: z.string().min(1).max(253) }).strict();
+/** Bounds one scan: receipts are never pruned, and a heartbeat must stay cheap. */
+const HEALTH_SCAN_LIMIT = 5000;
+const HEALTH_FENCE_LIMIT = 64;
+/** `state` precedes the potentially large result in every receipt this ledger writes. */
+const RECEIPT_STATE = /"state":"([a-z]+)"/;
 const hash = (value) => createHash('sha256').update(value).digest('hex');
 export function commandRequestDigest(input) {
     return hash(JSON.stringify({ commandId: input.commandId, command: input.command, args: input.args,
@@ -175,6 +191,25 @@ export class CommandLedger {
         }
     }
     receiptName(id) { return 'receipt-' + hash(id) + '.json'; }
+    failedAdmissionName(id) { return 'admission-failed-' + hash(id) + '.json'; }
+    /**
+     * A write after the fence failed, so the fence is ahead with no receipt, and the
+     * callback did not start. Say so durably, or that state looks like a receipt that
+     * went missing. If not even this can be written, keep the claim instead: it is
+     * already on disk, needs no new space, and says the same (recoverClaim clears it).
+     * A receipt that did land must keep its claim too, or it can never be reconciled.
+     */
+    async keepEvidenceOfFailedAdmission(input) {
+        try {
+            if (await this.read(this.receiptName(input.commandId)) !== undefined)
+                return true;
+            await this.write(this.failedAdmissionName(input.commandId), { commandId: input.commandId, fence: input.coordination.fence });
+            return false;
+        }
+        catch { // no-reason-needed: the original failure is what is reported; keeping the claim is the fallback
+            return true;
+        }
+    }
     async receipt(id) {
         if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(id))
             throw new Error('Invalid command identifier');
@@ -250,7 +285,7 @@ export class CommandLedger {
         catch (error) {
             if (error instanceof CommandAdmissionError)
                 throw error;
-            throw new CommandAdmissionError('LEDGER_UNAVAILABLE', 'Reconciliation could not be persisted; inspect the receipt before retrying the same decision');
+            throw new CommandAdmissionError('LEDGER_UNAVAILABLE', 'Reconciliation could not be persisted; inspect the receipt before retrying the same decision', { cause: error });
         }
         finally {
             await mutex?.close();
@@ -261,7 +296,7 @@ export class CommandLedger {
         const value = await this.read('managed-enrollment.json');
         if (value === undefined)
             return null;
-        return z.object({ version: z.literal(1), nodeId: z.string().min(1).max(128), nodeFqdn: z.string().min(1).max(253) }).strict().parse(value);
+        return enrollmentMarkerSchema.parse(value);
     }
     /** A drain cannot equate an idle process or expired fence with settled effects. */
     async drainSnapshot() {
@@ -351,7 +386,7 @@ export class CommandLedger {
             if (!prior || BigInt(fence.fence) > BigInt(prior.fence))
                 await this.write(fenceName, { resourceKey: fence.resourceKey, fence: fence.fence });
             else if (prior && BigInt(fence.fence) < BigInt(prior.fence))
-                throw new CommandAdmissionError('STALE_FENCE', 'A successor fence already owns this resource');
+                throw new CommandAdmissionError('STALE_FENCE', 'A successor fence already owns this resource', { highWater: { resourceKey: fence.resourceKey, fence: prior.fence } });
             if (!enrolled)
                 await this.write('managed-enrollment.json', { version: 1, nodeId: enrollment.nodeId, nodeFqdn: enrollment.nodeFqdn });
             this.checkResult(result, input.commandId, digest, fence);
@@ -364,6 +399,113 @@ export class CommandLedger {
             await mutex.close();
         }
     }
+    /**
+     * The sticky marker as it is, without creating anything. An unreadable marker
+     * is `unknown`, not absent: reporting nothing could hide a mismatch.
+     */
+    async enrollmentMarker() {
+        try {
+            const raw = await this.read('managed-enrollment.json');
+            if (raw === undefined)
+                return { state: 'absent', marker: null };
+            return { state: 'present', marker: enrollmentMarkerSchema.parse(raw) };
+        }
+        catch (error) {
+            if (error.code === 'ENOENT' || error.code === 'ENOTDIR')
+                return { state: 'absent', marker: null };
+            return { state: 'unknown', marker: null, error: error instanceof Error ? error.message : String(error) };
+        }
+    }
+    /**
+     * What this ledger holds about a command, read only and without the mutex. On its
+     * own "no receipt" proves nothing: a ledger that is gone holds nothing either. So
+     * this also says whether there is a ledger, what fence it last accepted for the
+     * key and what its marker says. Ordering in execute(): active.lock, fence, durable
+     * running receipt, and only then the callback, so a claim without a receipt
+     * never started.
+     */
+    async commandTrace(id, resourceKey) {
+        if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(id))
+            throw new Error('Invalid command identifier');
+        let ledgerExists = true;
+        try {
+            ledgerExists = (await lstat(this.root)).isDirectory();
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT' && error.code !== 'ENOTDIR')
+                throw error;
+            ledgerExists = false;
+        }
+        const marker = await this.enrollmentMarker();
+        if (!ledgerExists)
+            return { ledgerExists, receipt: false, activeLock: false, fenceHighWater: null, marker, admissionFailed: false, admissionFailedFence: null };
+        const raw = await this.read('active.lock');
+        const receipt = await this.read(this.receiptName(id));
+        const fence = resourceKey ? await this.read('fence-' + hash(resourceKey) + '.json') : undefined;
+        const failed = await this.read(this.failedAdmissionName(id));
+        const admissionFailedFence = failed?.commandId === id && typeof failed.fence === 'string' && /^[0-9]{1,20}$/.test(failed.fence) ? failed.fence : null;
+        return { ledgerExists, receipt: receipt !== undefined, activeLock: raw !== undefined && lockSchema.parse(raw).commandId === id,
+            admissionFailed: admissionFailedFence !== null, admissionFailedFence,
+            fenceHighWater: fence?.resourceKey === resourceKey && typeof fence?.fence === 'string' && /^[0-9]{1,20}$/.test(fence.fence) ? fence.fence : null, marker };
+    }
+    /**
+     * Read-only health for preflight checks and heartbeats. Takes no mutex and
+     * creates nothing, so asking never competes with a running command.
+     */
+    async health() {
+        const health = { readable: true, activeLock: null, nonTerminalReceipts: 0, unresolvedReceipts: 0, truncated: false, fenceHighWater: {}, error: null };
+        let names;
+        try {
+            names = await readdir(this.root);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return health;
+            return { ...health, readable: false, error: 'the ledger directory cannot be read' };
+        }
+        try {
+            const receiptState = async (name) => {
+                const file = await open(join(this.root, name), constants.O_RDONLY | constants.O_NOFOLLOW);
+                try {
+                    const { buffer, bytesRead } = await file.read({ buffer: Buffer.alloc(1024), position: 0 });
+                    return RECEIPT_STATE.exec(buffer.toString('utf8', 0, bytesRead))?.[1] ?? null;
+                }
+                finally {
+                    await file.close();
+                }
+            };
+            const running = new Set([...this.active].map(id => this.receiptName(id)));
+            const receipts = names.filter(name => name.startsWith('receipt-'));
+            health.truncated = receipts.length > HEALTH_SCAN_LIMIT;
+            for (const name of receipts.slice(0, HEALTH_SCAN_LIMIT)) {
+                const state = await receiptState(name);
+                if (state === 'completed' || state === 'reconciled')
+                    continue;
+                health.nonTerminalReceipts += 1;
+                if (!running.has(name))
+                    health.unresolvedReceipts += 1;
+            }
+            for (const name of names.filter(entry => entry.startsWith('fence-')).slice(0, HEALTH_FENCE_LIMIT)) {
+                const record = await this.read(name);
+                if (typeof record?.resourceKey === 'string' && typeof record.fence === 'string' && /^[0-9]{1,20}$/.test(record.fence))
+                    health.fenceHighWater[record.resourceKey] = record.fence;
+            }
+            const raw = await this.read('active.lock');
+            if (raw !== undefined) {
+                const claim = lockSchema.parse(raw);
+                const state = await receiptState(this.receiptName(claim.commandId)).catch((error) => {
+                    if (error.code === 'ENOENT')
+                        return 'none';
+                    throw error;
+                });
+                health.activeLock = { commandId: claim.commandId, receipt: state === 'running' ? (this.active.has(claim.commandId) ? 'running' : 'unknown') : state ?? 'invalid' };
+            }
+            return health;
+        }
+        catch (error) {
+            return { ...health, readable: false, error: `the ledger holds an unreadable record (${error instanceof Error ? error.name : 'error'})` };
+        }
+    }
     /** Persist an authoritative observation; never clears an active/unknown claim. */
     async observeManagedEnrollment(identity) {
         const value = z.object({ nodeId: z.string().min(1).max(128), nodeFqdn: z.string().min(1).max(253) }).strict().parse(identity);
@@ -385,7 +527,7 @@ export class CommandLedger {
         // Snapshot before awaiting; callers cannot mutate accepted scope or identity.
         input = structuredClone(input);
         const digest = commandRequestDigest(input);
-        let held = false, started = false, admitted = false;
+        let held = false, started = false, admitted = false, noReceipt = false, fenceWritten = false, keepClaim = false;
         let mutex;
         const previous = async () => {
             const row = await this.receipt(input.commandId);
@@ -403,6 +545,7 @@ export class CommandLedger {
             const replay = await previous();
             if (replay)
                 return replay;
+            noReceipt = true;
             mutex = await this.acquire();
             await this.recoverClaim();
             const enrolled = await this.managedEnrollment();
@@ -425,8 +568,9 @@ export class CommandLedger {
                 if (prior && (prior.resourceKey !== fence.resourceKey || !/^[1-9][0-9]{0,18}$/.test(prior.fence)))
                     throw new Error('Invalid fencing record');
                 if (prior && BigInt(fence.fence) <= BigInt(prior.fence))
-                    throw new CommandAdmissionError('STALE_FENCE', 'Stale resource fence cannot execute');
+                    throw new CommandAdmissionError('STALE_FENCE', 'Stale resource fence cannot execute', { highWater: { resourceKey: fence.resourceKey, fence: prior.fence } });
                 await this.write(name, { resourceKey: fence.resourceKey, fence: fence.fence });
+                fenceWritten = true;
             }
             if (enrollment && !enrolled)
                 await this.write('managed-enrollment.json', { version: 1, nodeId: enrollment.nodeId, nodeFqdn: enrollment.nodeFqdn });
@@ -443,15 +587,22 @@ export class CommandLedger {
             return result;
         }
         catch (error) {
-            if (error instanceof CommandAdmissionError)
+            // `started` stays true from just before the callback until its terminal receipt is durable.
+            const certain = !started && noReceipt;
+            if (error instanceof CommandAdmissionError) {
+                if (certain && !['OUTCOME_UNKNOWN', 'IDEMPOTENCY_CONFLICT'].includes(error.code))
+                    error.notStarted = true;
                 throw error;
-            throw new CommandAdmissionError(started ? 'OUTCOME_UNKNOWN' : 'LEDGER_UNAVAILABLE', started ? 'Execution receipt is uncertain; retained for reconciliation and never replayed' : 'Command ledger unavailable; inspect its receipt before retrying');
+            }
+            if (certain && fenceWritten)
+                keepClaim = await this.keepEvidenceOfFailedAdmission(input);
+            throw new CommandAdmissionError(started ? 'OUTCOME_UNKNOWN' : 'LEDGER_UNAVAILABLE', started ? 'Execution receipt is uncertain; retained for reconciliation and never replayed' : 'Command ledger unavailable; inspect its receipt before retrying', { notStarted: certain, cause: error });
         }
         finally {
             if (admitted)
                 this.active.delete(input.commandId);
             try {
-                if (held && !started)
+                if (held && !started && !keepClaim)
                     await this.releaseClaim();
             }
             catch {

package/dist/executors/command-refusal.d.ts

@@ -0,0 +1,43 @@
+/**
+ * The error body of a refusal. Only with a local proof that the request did
+ * not start, nothing local for its id and a lease to name, does it carry
+ * `executed: false` and a signature bound to the request, the lease and this node;
+ * the server settles the lease on that alone. Anything less stays uncertain, the
+ * safe answer for an older server too. A claim this command still holds counts as
+ * something local: the request may be between its claim and its receipt.
+ */
+export declare function refusalError(request: {
+    commandId: string;
+    requestSha256: string;
+    leaseId?: string | undefined;
+}, code: string, message: string, details: {
+    notStarted: boolean;
+    highWater?: {
+        resourceKey: string;
+        fence: string;
+    };
+}): Promise<{
+    highWater?: string | undefined;
+    resourceKey?: string | undefined;
+    code: string;
+    message: string;
+} | {
+    dispatched: boolean;
+    executed: boolean;
+    refusal: {
+        signature: string;
+        commandId: string;
+        requestSha256: string;
+        code: string;
+        highWater: string | null;
+        nodeFqdn: string;
+        leaseId: string;
+    };
+    highWater?: string | undefined;
+    resourceKey?: string | undefined;
+    code: string;
+    message: string;
+}>;
+/** Same statuses as /commands/execute. */
+export declare const refusalStatus: (code: string) => 409 | 503;
+//# sourceMappingURL=command-refusal.d.ts.map

package/dist/executors/command-refusal.js

@@ -0,0 +1,32 @@
+import { HOST_FQDN, RESULT_SIGNING_SECRET } from '../env.js';
+import { commandRefusalSigningPayload, signCommandEvidence } from '../security/command-result-signature.js';
+import { logger } from '../utils/logger.js';
+import { commandLedger } from './command-executor.js';
+/**
+ * The error body of a refusal. Only with a local proof that the request did
+ * not start, nothing local for its id and a lease to name, does it carry
+ * `executed: false` and a signature bound to the request, the lease and this node;
+ * the server settles the lease on that alone. Anything less stays uncertain, the
+ * safe answer for an older server too. A claim this command still holds counts as
+ * something local: the request may be between its claim and its receipt.
+ */
+export async function refusalError(request, code, message, details) {
+    const base = { code, message, ...(details.highWater ? { highWater: details.highWater.fence, resourceKey: details.highWater.resourceKey } : {}) };
+    if (!details.notStarted || !request.leaseId)
+        return base;
+    try {
+        const trace = await commandLedger.commandTrace(request.commandId);
+        if (trace.receipt || trace.activeLock)
+            return base;
+        const unsigned = { commandId: request.commandId, requestSha256: request.requestSha256, code, highWater: details.highWater?.fence ?? null, nodeFqdn: HOST_FQDN, leaseId: request.leaseId };
+        return { ...base, dispatched: false, executed: false,
+            refusal: { ...unsigned, signature: signCommandEvidence(commandRefusalSigningPayload(unsigned), RESULT_SIGNING_SECRET) } };
+    }
+    catch (error) {
+        logger.warn(`Refusal of ${request.commandId} (${code}) stays uncertain: ${error instanceof Error ? error.message : String(error)}`);
+        return base;
+    }
+}
+/** Same statuses as /commands/execute. */
+export const refusalStatus = (code) => code === 'LEDGER_UNAVAILABLE' ? 503 : 409;
+//# sourceMappingURL=command-refusal.js.map