@north-light/api 0.3.580 → 0.3.587

package/dist/core/canvas/deliveries.d.ts

@@ -0,0 +1,125 @@
+import type { DatabaseSync } from 'node:sqlite';
+import { type DeliveryFailure, type DeliverySettlement as ProcessDeliverySettlement } from './delivery-contract.js';
+export type DeliveryState = 'pending' | 'handed_off' | 'acknowledged' | 'permanent_failed';
+export type DeliveryUrgency = 'critical' | 'urgent' | 'normal' | 'deferred';
+export interface DeliveryRecord {
+    deliveryId: string;
+    messageId: string;
+    destinationId: string;
+    urgency: DeliveryUrgency;
+    wake: 0 | 1;
+    state: DeliveryState;
+    attempts: number;
+    nextAttemptAt: number | null;
+    claimedAt: number | null;
+    claimOwner: string | null;
+    acknowledgedAt: number | null;
+    failedAt: number | null;
+    lastFailure: DeliveryFailure | null;
+    idempotencyKey: string | null;
+    canceledAt: number | null;
+    createdAt: number;
+    updatedAt: number;
+}
+export interface ClaimedNodeMail extends DeliveryRecord {
+    message: {
+        messageId: string;
+        seq: number;
+        operationId: string;
+        createdAt: string;
+        senderNodeId: string | null;
+        senderName: string | null;
+        kind: 'update' | 'urgent' | 'final' | 'message' | 'completed';
+        label: string;
+        bodyRef: string | null;
+        bodyJson: unknown | null;
+        disposition: 'human-answer' | 'human-canceled' | null;
+    };
+    handoff: 'fresh' | 'inherited';
+}
+type StoredDelivery = {
+    delivery_id: string;
+    message_id: string;
+    destination_id: string;
+    urgency: DeliveryUrgency;
+    wake: number;
+    state: DeliveryState;
+    attempts: number;
+    next_attempt_at: number | null;
+    claimed_at: number | null;
+    claim_owner: string | null;
+    acknowledged_at: number | null;
+    failed_at: number | null;
+    last_failure_json: string | null;
+    idempotency_key: string | null;
+    canceled_at: number | null;
+    created_at: number;
+    updated_at: number;
+};
+export declare function deliveryFrom(row: StoredDelivery): DeliveryRecord;
+/** Claim one broker channel. A refreshing or stale broker receives hold and changes nothing.
+ * Empty polls have no handoff to commit. If mail arrives after the plain read,
+ * the next tick claims it, just as if it arrived after the write transaction. */
+export declare function claimNodeMail(nodeId: string, executionId: string, channel: 'wake' | 'passive' | 'deferred_run', now?: number): Promise<{
+    action: 'claimed' | 'hold';
+    deliveries: ClaimedNodeMail[];
+}>;
+/** Pre-launch wake claim. It uses the same state transition but deliberately does not reject refresh intent. */
+export declare function claimKickoffMail(nodeId: string, executionId: string, now?: number): Promise<ClaimedNodeMail[]>;
+export type DeliverySettlement = {
+    state: 'acknowledged';
+} | {
+    state: 'released';
+} | {
+    state: 'permanent_failed';
+    failure: DeliveryFailure;
+} | {
+    state: 'pending';
+    nextAttemptAt: number;
+    failure: DeliveryFailure;
+};
+export declare function normalizedSettlement(settlement: DeliverySettlement | ProcessDeliverySettlement): DeliverySettlement;
+/** The one execution/attempt/owner fence for node mail and external actions. */
+export declare function settleDelivery(deliveryId: string, attempts: number, claimOwner: string, settlement: DeliverySettlement | ProcessDeliverySettlement, now?: number): Promise<boolean>;
+/** Deliberate operator retry of a delivery that failed for good: back to pending, due now,
+ *  with a fresh attempt budget. The id and the claim fence are unchanged, and the last
+ *  failure stays recorded until the next attempt replaces it. Returns false when the id
+ *  is not a failed, uncanceled push delivery. */
+export declare function retryFailedDelivery(deliveryId: string, now?: number): Promise<boolean>;
+/** Broker acknowledgement additionally checks that the execution is live before the per-row fence.
+ *  Refresh intent does not refuse it: a yielding turn still acknowledges the mail
+ *  its conversation consumed, and the watcher withholds the queued mail the
+ *  refresh abort discarded so the replacement execution replays only that. */
+export declare function acknowledgeNodeMail(nodeId: string, executionId: string, deliveries: Array<{
+    deliveryId: string;
+    attempts: number;
+    disposition: 'acknowledged' | 'released';
+}>, now?: number): Promise<{
+    action: 'acknowledged' | 'hold';
+    results: Array<{
+        deliveryId: string;
+        outcome: 'acknowledged' | 'released' | 'stale';
+    }>;
+}>;
+/** The immutable action document belonging to one external-action obligation. */
+export declare function messageBodyForDelivery(deliveryId: string): Promise<unknown | null>;
+export declare function duePushDeliveries(now: number): Promise<DeliveryRecord[]>;
+export declare function claimPushDelivery(deliveryId: string, now: number, claimOwner: string): Promise<DeliveryRecord | null>;
+export declare function resetHandedOffDeliveries(now: number): Promise<number>;
+/** The lane checks the claim immediately before launching the action. An action
+ * superseded after claim but before dispatch must not start with stale input. */
+export declare function pushDeliveryClaimIsCurrent(delivery: DeliveryRecord, owner: string): Promise<boolean>;
+/** Reset mail held by an execution that is no longer this node's one live execution.
+ * Push receiver deliveries have a due timestamp and are claimed by the daemon lane. */
+export declare function resetStaleExecutionMail(now: number): Promise<number>;
+/** The one definition of wake mail that can still move a node, as a condition on a `deliveries` row aliased `d`;
+ *  `states` is the delivery states that count. */
+export declare const WAKE_MAIL_SQL: (d: string, states: string) => string;
+/** Unseen wake mail: not yet acknowledged by an execution of the node. */
+export declare const UNSEEN_WAKE_MAIL_SQL: (d: string) => string;
+export declare function hasUnseenWakeMail(nodeId: string): Promise<boolean>;
+export declare function hasWakeMailSince(nodeId: string, since: number): Promise<boolean>;
+export declare function acknowledgedNodeMailCount(nodeId: string): Promise<number>;
+/** Create the messaging schema and import every legacy mailbox/action/wait row. The caller owns the encompassing migration transaction. */
+export declare function migrateMessagingState(db: DatabaseSync): void;
+export {};

package/dist/core/canvas/deliveries.js

@@ -0,0 +1,393 @@
+import { prepared as cachedStatement } from '../sqlite-statements.js';
+import { existsSync, readFileSync } from 'node:fs';
+import { randomBytes } from 'node:crypto';
+import { withCanvasWrite, openDb } from './db.js';
+import { inboxPath, passivePath } from './paths.js';
+import { withRetryBudget } from './delivery-contract.js';
+import { emitEvent } from '../events/emit.js';
+import { mapAsync } from '../async-array.js';
+export function deliveryFrom(row) {
+    return {
+        deliveryId: row.delivery_id,
+        messageId: row.message_id,
+        destinationId: row.destination_id,
+        urgency: row.urgency,
+        wake: row.wake === 1 ? 1 : 0,
+        state: row.state,
+        attempts: row.attempts,
+        nextAttemptAt: row.next_attempt_at,
+        claimedAt: row.claimed_at,
+        claimOwner: row.claim_owner,
+        acknowledgedAt: row.acknowledged_at,
+        failedAt: row.failed_at,
+        lastFailure: row.last_failure_json === null ? null : JSON.parse(row.last_failure_json),
+        idempotencyKey: row.idempotency_key,
+        canceledAt: row.canceled_at,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+    };
+}
+async function executionIsLive(db, nodeId, executionId, rejectRefresh) {
+    const row = await db.prepare(`SELECT 1 FROM nodes n JOIN executions e ON e.node_id = n.node_id
+      WHERE n.node_id = ? AND e.execution_id = ? AND e.state != 'exited'${rejectRefresh ? " AND n.intent IS DISTINCT FROM 'refresh'" : ''}`).get(nodeId, executionId);
+    return row !== undefined;
+}
+async function nodeMailRows(db, nodeId, executionId, wake, freshIds, deferredRun = false) {
+    const rows = await db.prepare(`SELECT d.*, m.message_id AS m_message_id, m.seq, m.operation_id, m.created_at AS message_created_at,
+            m.sender_node_id, m.sender_name, m.kind, m.label, m.body_ref, m.body_json, m.disposition
+       FROM deliveries d JOIN messages m ON m.message_id = d.message_id
+      WHERE d.destination_id = ? AND d.wake = ?
+        ${deferredRun ? "AND m.kind = 'message' AND m.body_json ->> 'start_turn' = 'false'" : wake === 0 ? "AND NOT (m.kind = 'message' AND coalesce(m.body_json ->> 'start_turn', 'true') = 'false')" : ''}
+        AND d.canceled_at IS NULL
+        AND d.state = 'handed_off' AND d.claim_owner = ?
+      ORDER BY m.seq`).all(nodeId, wake, executionId);
+    return rows.map((row) => ({
+        ...deliveryFrom(row),
+        message: {
+            messageId: row['m_message_id'],
+            seq: Number(row['seq']),
+            operationId: row['operation_id'],
+            createdAt: row['message_created_at'],
+            senderNodeId: row['sender_node_id'],
+            senderName: row['sender_name'],
+            kind: row['kind'],
+            label: row['label'],
+            bodyRef: row['body_ref'],
+            bodyJson: row['body_json'] === null ? null : JSON.parse(row['body_json']),
+            disposition: row['disposition'],
+        },
+        handoff: freshIds?.has(row.delivery_id) === true ? 'fresh' : 'inherited',
+    }));
+}
+const PENDING_NODE_MAIL_SQL = `SELECT d.delivery_id FROM deliveries d JOIN messages m ON m.message_id = d.message_id
+  WHERE d.destination_id = ? AND d.wake = ? AND d.state = 'pending' AND d.canceled_at IS NULL
+    AND NOT (d.wake = 0 AND m.kind = 'message' AND coalesce(m.body_json ->> 'start_turn', 'true') = 'false')`;
+const HAND_OFF_NODE_MAIL_SQL = `UPDATE deliveries SET state = 'handed_off', attempts = attempts + 1, claim_owner = ?, claimed_at = ?, updated_at = ?
+  WHERE delivery_id IN (${PENDING_NODE_MAIL_SQL}) RETURNING delivery_id`;
+/** Claim one broker channel. A refreshing or stale broker receives hold and changes nothing.
+ * Empty polls have no handoff to commit. If mail arrives after the plain read,
+ * the next tick claims it, just as if it arrived after the write transaction. */
+export async function claimNodeMail(nodeId, executionId, channel, now = Date.now()) {
+    const wake = channel === 'wake' ? 1 : 0;
+    if (channel === 'deferred_run') {
+        return await withCanvasWrite(async (db) => {
+            if (!await executionIsLive(db, nodeId, executionId, true))
+                return { action: 'hold', deliveries: [] };
+            const rows = await db.prepare(`SELECT d.delivery_id FROM deliveries d JOIN messages m ON m.message_id = d.message_id
+        WHERE d.destination_id = ? AND d.wake = 0 AND d.canceled_at IS NULL
+          AND m.kind = 'message' AND m.body_json ->> 'start_turn' = 'false'
+          AND (d.state = 'pending' OR (d.state = 'handed_off' AND d.claim_owner = ?))`).all(nodeId, executionId);
+            const freshIds = new Set(rows.length === 0 ? [] : (await db.prepare(`UPDATE deliveries SET state = 'handed_off', attempts = attempts + 1, claim_owner = ?, claimed_at = ?, updated_at = ?
+        WHERE delivery_id IN (${rows.map(() => '?').join(',')}) AND state = 'pending' RETURNING delivery_id`).all(executionId, now, now, ...rows.map((row) => row.delivery_id))).map((row) => row.delivery_id));
+            return { action: 'claimed', deliveries: await nodeMailRows(db, nodeId, executionId, 0, freshIds, true) };
+        });
+    }
+    const db = await openDb();
+    if (!db.inTransaction && await db.prepare(PENDING_NODE_MAIL_SQL).get(nodeId, wake) === undefined) {
+        if (!await executionIsLive(db, nodeId, executionId, true))
+            return { action: 'hold', deliveries: [] };
+        return { action: 'claimed', deliveries: await nodeMailRows(db, nodeId, executionId, wake) };
+    }
+    return await withCanvasWrite(async (db) => {
+        if (!await executionIsLive(db, nodeId, executionId, true))
+            return { action: 'hold', deliveries: [] };
+        const freshIds = new Set((await db.prepare(HAND_OFF_NODE_MAIL_SQL).all(executionId, now, now, nodeId, wake)).map((row) => row.delivery_id));
+        return { action: 'claimed', deliveries: await nodeMailRows(db, nodeId, executionId, wake, freshIds) };
+    });
+}
+/** Pre-launch wake claim. It uses the same state transition but deliberately does not reject refresh intent. */
+export async function claimKickoffMail(nodeId, executionId, now = Date.now()) {
+    return await withCanvasWrite(async (db) => {
+        if (!await executionIsLive(db, nodeId, executionId, false))
+            throw new Error(`execution ${executionId} is not live for node ${nodeId}`);
+        const freshIds = new Set((await db.prepare(`UPDATE deliveries SET state = 'handed_off', attempts = attempts + 1, claim_owner = ?, claimed_at = ?, updated_at = ?
+        WHERE destination_id = ? AND wake = 1 AND state = 'pending' AND canceled_at IS NULL RETURNING delivery_id`).all(executionId, now, now, nodeId)).map((row) => row.delivery_id));
+        return await nodeMailRows(db, nodeId, executionId, 1, freshIds);
+    });
+}
+// Convert the existing subprocess contract at the one shared fence. The old names do not escape this module.
+export function normalizedSettlement(settlement) {
+    if (settlement.state === 'accepted')
+        return { state: 'acknowledged' };
+    return settlement;
+}
+/** The one execution/attempt/owner fence for node mail and external actions. */
+export async function settleDelivery(deliveryId, attempts, claimOwner, settlement, now = Date.now()) {
+    const exhausted = [];
+    const settled = await withCanvasWrite(async (db) => {
+        // A canceled in-flight outcome cannot retry after its successor is queued.
+        // Keep the claim held until the process exits, then retire it atomically so
+        // the successor's claim guard sees no competing executor.
+        const canceled = await db.prepare('SELECT canceled_at FROM deliveries WHERE delivery_id = ? AND next_attempt_at IS NOT NULL')
+            .get(deliveryId);
+        const requested = canceled?.canceled_at != null ? { state: 'acknowledged' } : normalizedSettlement(settlement);
+        // A retried delivery has a finite attempt budget: its last failure is terminal, not another retry.
+        const budget = withRetryBudget(requested, attempts);
+        const next = budget.settlement;
+        const failure = next.state === 'pending' || next.state === 'permanent_failed' ? JSON.stringify(next.failure) : null;
+        const state = next.state === 'released' ? 'pending' : next.state;
+        const nextAttemptAt = next.state === 'pending' ? next.nextAttemptAt : state === 'pending' ? now : null;
+        const changed = Number((await db.prepare(`UPDATE deliveries
+          SET state = ?, next_attempt_at = ?, claimed_at = NULL, claim_owner = NULL,
+              acknowledged_at = ?, failed_at = ?, last_failure_json = ?, updated_at = ?
+        WHERE delivery_id = ? AND state = 'handed_off' AND attempts = ? AND claim_owner = ?`).run(state, nextAttemptAt, state === 'acknowledged' ? now : null, state === 'permanent_failed' ? now : null, failure, now, deliveryId, attempts, claimOwner)).changes) === 1;
+        if (changed && budget.exhausted && next.state === 'permanent_failed') {
+            const row = await db.prepare('SELECT destination_id, message_id FROM deliveries WHERE delivery_id = ?')
+                .get(deliveryId);
+            exhausted.push({ failure: next.failure, destinationId: row.destination_id, messageId: row.message_id });
+        }
+        return changed;
+    });
+    for (const spent of exhausted) {
+        emitEvent({ level: 'error', event: 'delivery.exhausted', outcome: 'failed', attempt: attempts,
+            fields: { delivery_id: deliveryId, destination_id: spent.destinationId, message_id: spent.messageId,
+                failure: spent.failure, retry: `crtr sys retry-delivery ${deliveryId}` } });
+    }
+    return settled;
+}
+/** Deliberate operator retry of a delivery that failed for good: back to pending, due now,
+ *  with a fresh attempt budget. The id and the claim fence are unchanged, and the last
+ *  failure stays recorded until the next attempt replaces it. Returns false when the id
+ *  is not a failed, uncanceled push delivery. */
+export async function retryFailedDelivery(deliveryId, now = Date.now()) {
+    return Number((await (await openDb()).prepare(`UPDATE deliveries
+        SET state = 'pending', attempts = 0, next_attempt_at = ?, failed_at = NULL, updated_at = ?
+      WHERE delivery_id = ? AND state = 'permanent_failed' AND canceled_at IS NULL AND last_failure_json IS NOT NULL`).run(now, now, deliveryId)).changes) === 1;
+}
+/** Broker acknowledgement additionally checks that the execution is live before the per-row fence.
+ *  Refresh intent does not refuse it: a yielding turn still acknowledges the mail
+ *  its conversation consumed, and the watcher withholds the queued mail the
+ *  refresh abort discarded so the replacement execution replays only that. */
+export async function acknowledgeNodeMail(nodeId, executionId, deliveries, now = Date.now()) {
+    return await withCanvasWrite(async (db) => {
+        if (!await executionIsLive(db, nodeId, executionId, false))
+            return { action: 'hold', results: [] };
+        const results = (await mapAsync(deliveries, async (delivery) => {
+            const owned = await db.prepare("SELECT 1 FROM deliveries WHERE delivery_id = ? AND destination_id = ?").get(delivery.deliveryId, nodeId) !== undefined;
+            return {
+                deliveryId: delivery.deliveryId,
+                outcome: owned && await settleDelivery(delivery.deliveryId, delivery.attempts, executionId, { state: delivery.disposition }, now)
+                    ? delivery.disposition
+                    : 'stale',
+            };
+        }));
+        return { action: 'acknowledged', results };
+    });
+}
+/** The immutable action document belonging to one external-action obligation. */
+export async function messageBodyForDelivery(deliveryId) {
+    const row = await (await openDb()).prepare(`SELECT m.body_json FROM deliveries d JOIN messages m ON m.message_id = d.message_id
+      WHERE d.delivery_id = ?`).get(deliveryId);
+    return row === undefined || row.body_json === null ? null : JSON.parse(row.body_json);
+}
+export async function duePushDeliveries(now) {
+    return (await (await openDb()).prepare(`SELECT * FROM deliveries
+      WHERE state = 'pending' AND canceled_at IS NULL AND next_attempt_at <= ?
+      ORDER BY next_attempt_at, created_at, delivery_id`).all(now)).map(deliveryFrom);
+}
+export async function claimPushDelivery(deliveryId, now, claimOwner) {
+    return await withCanvasWrite(async (db) => {
+        // A superseded outcome action may already have started. Its replacement is
+        // not executable until that attempt settles (or boot recovery retires it).
+        const result = await db.prepare(`UPDATE deliveries AS d SET state = 'handed_off', attempts = attempts + 1, claimed_at = ?, claim_owner = ?, updated_at = ?
+        WHERE d.delivery_id = ? AND d.state = 'pending' AND d.canceled_at IS NULL AND d.next_attempt_at <= ?
+          AND NOT EXISTS (SELECT 1 FROM deliveries old JOIN messages prior ON prior.message_id = old.message_id
+            JOIN messages next ON next.message_id = d.message_id
+            WHERE old.destination_id = d.destination_id AND old.delivery_id != d.delivery_id
+              AND old.state = 'handed_off' AND old.canceled_at IS NOT NULL
+              AND prior.event = 'outcome' AND next.event = 'outcome' AND prior.object_id = next.object_id)`).run(now, claimOwner, now, deliveryId, now);
+        if (Number(result.changes) === 0)
+            return null;
+        return deliveryFrom(await db.prepare('SELECT * FROM deliveries WHERE delivery_id = ?').get(deliveryId));
+    });
+}
+export async function resetHandedOffDeliveries(now) {
+    return await withCanvasWrite(async (db) => {
+        // Boot has reaped the previous daemon's detached work. Canceled attempts
+        // must retire, not return to pending and block their replacement forever.
+        const retired = await db.prepare(`UPDATE deliveries SET state = 'permanent_failed', next_attempt_at = NULL,
+      claimed_at = NULL, claim_owner = NULL, failed_at = ?, updated_at = ?
+      WHERE state = 'handed_off' AND canceled_at IS NOT NULL AND next_attempt_at IS NOT NULL`).run(now, now);
+        const released = await db.prepare(`UPDATE deliveries SET state = 'pending', next_attempt_at = CASE WHEN next_attempt_at IS NOT NULL THEN ?::bigint ELSE NULL END,
+         claimed_at = NULL, claim_owner = NULL, updated_at = ?
+         WHERE state = 'handed_off' AND (canceled_at IS NULL OR next_attempt_at IS NULL)`).run(now, now);
+        return Number(retired.changes) + Number(released.changes);
+    });
+}
+/** The lane checks the claim immediately before launching the action. An action
+ * superseded after claim but before dispatch must not start with stale input. */
+export async function pushDeliveryClaimIsCurrent(delivery, owner) {
+    return await (await openDb()).prepare(`SELECT 1 FROM deliveries WHERE delivery_id = ? AND state = 'handed_off'
+    AND attempts = ? AND claim_owner = ? AND canceled_at IS NULL`).get(delivery.deliveryId, delivery.attempts, owner) !== undefined;
+}
+/** Reset mail held by an execution that is no longer this node's one live execution.
+ * Push receiver deliveries have a due timestamp and are claimed by the daemon lane. */
+export async function resetStaleExecutionMail(now) {
+    return Number((await (await openDb()).prepare(`UPDATE deliveries AS d SET state = 'pending', claimed_at = NULL, claim_owner = NULL, updated_at = ?
+      WHERE d.state = 'handed_off' AND d.next_attempt_at IS NULL
+        AND NOT EXISTS (
+          SELECT 1 FROM executions e WHERE e.node_id = d.destination_id AND e.execution_id = d.claim_owner AND e.state != 'exited'
+        )`).run(now)).changes);
+}
+/** The one definition of wake mail that can still move a node, as a condition on a `deliveries` row aliased `d`;
+ *  `states` is the delivery states that count. */
+export const WAKE_MAIL_SQL = (d, states) => `${d}.wake = 1 AND ${d}.canceled_at IS NULL AND ${d}.urgency != 'deferred' AND ${d}.state IN (${states})`;
+/** Unseen wake mail: not yet acknowledged by an execution of the node. */
+export const UNSEEN_WAKE_MAIL_SQL = (d) => WAKE_MAIL_SQL(d, "'pending', 'handed_off'");
+export async function hasUnseenWakeMail(nodeId) {
+    return await (await openDb()).prepare(`SELECT 1 FROM deliveries d WHERE d.destination_id = ? AND ${UNSEEN_WAKE_MAIL_SQL('d')} LIMIT 1`).get(nodeId) !== undefined;
+}
+export async function hasWakeMailSince(nodeId, since) {
+    return await (await openDb()).prepare(`SELECT 1 FROM deliveries WHERE destination_id = ? AND wake = 1
+       AND urgency != 'deferred' AND created_at >= ? LIMIT 1`).get(nodeId, since) !== undefined;
+}
+export async function acknowledgedNodeMailCount(nodeId) {
+    return Number((await (await openDb()).prepare("SELECT COUNT(*) AS count FROM deliveries WHERE destination_id = ? AND acknowledged_at IS NOT NULL").get(nodeId)).count);
+}
+function newId() {
+    let id;
+    do
+        id = randomBytes(16).toString('hex');
+    while (id === '00000000000000000000000000000000');
+    return id;
+}
+function readJsonLines(path) {
+    if (!existsSync(path))
+        return [];
+    const raw = readFileSync(path, 'utf8');
+    if (raw.trim() === '')
+        return [];
+    return raw.split('\n').filter((line) => line.trim() !== '').map((line, index) => {
+        let entry;
+        try {
+            entry = JSON.parse(line);
+        }
+        catch (error) {
+            throw new Error(`invalid mailbox JSON at ${path}:${index + 1}: ${error.message}`);
+        }
+        if (!/^(?!0{32}$)[0-9a-f]{32}$/.test(entry.entry_id) || !/^(?!0{32}$)[0-9a-f]{32}$/.test(entry.operation_id)) {
+            throw new Error(`invalid mailbox identity at ${path}:${index + 1}`);
+        }
+        if (typeof entry.ts !== 'string' || typeof entry.label !== 'string')
+            throw new Error(`incomplete mailbox entry at ${path}:${index + 1}`);
+        return entry;
+    });
+}
+function readCanceled(path) {
+    if (!existsSync(path))
+        return new Set();
+    return new Set(readFileSync(path, 'utf8').split('\n').map((line) => line.trim()).filter(Boolean));
+}
+/** Create the messaging schema and import every legacy mailbox/action/wait row. The caller owns the encompassing migration transaction. */
+export function migrateMessagingState(db) {
+    db.exec(`
+CREATE TABLE IF NOT EXISTS messages (
+  message_id TEXT PRIMARY KEY,
+  seq INTEGER NOT NULL UNIQUE,
+  operation_id TEXT NOT NULL,
+  created_at TEXT NOT NULL,
+  sender_node_id TEXT,
+  sender_name TEXT,
+  kind TEXT NOT NULL CHECK (kind IN ('update', 'urgent', 'final', 'message', 'completed')),
+  label TEXT NOT NULL,
+  body_ref TEXT,
+  body_json TEXT,
+  disposition TEXT CHECK (disposition IS NULL OR disposition IN ('human-answer', 'human-canceled'))
+);
+CREATE TABLE IF NOT EXISTS deliveries (
+  delivery_id TEXT PRIMARY KEY,
+  message_id TEXT REFERENCES messages(message_id),
+  destination_kind TEXT NOT NULL CHECK (destination_kind IN ('node_inbox', 'external_action')),
+  destination_id TEXT NOT NULL,
+  urgency TEXT NOT NULL CHECK (urgency IN ('critical', 'urgent', 'normal', 'deferred')),
+  wake INTEGER NOT NULL CHECK (wake IN (0, 1)),
+  state TEXT NOT NULL CHECK (state IN ('armed', 'pending', 'handed_off', 'acknowledged', 'permanent_failed')),
+  attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0),
+  next_attempt_at INTEGER,
+  claimed_at INTEGER,
+  claim_owner TEXT,
+  acknowledged_at INTEGER,
+  failed_at INTEGER,
+  last_failure_json TEXT,
+  adapter_json TEXT,
+  effect_json TEXT,
+  idempotency_key TEXT,
+  canceled_at INTEGER,
+  created_at INTEGER NOT NULL,
+  updated_at INTEGER NOT NULL,
+  CHECK (state = 'armed' OR message_id IS NOT NULL)
+);
+CREATE UNIQUE INDEX IF NOT EXISTS messages_seq ON messages(seq);
+CREATE INDEX IF NOT EXISTS deliveries_node_inbox ON deliveries(destination_id, wake, state, created_at) WHERE destination_kind = 'node_inbox';
+CREATE INDEX IF NOT EXISTS deliveries_external_due ON deliveries(state, next_attempt_at) WHERE destination_kind = 'external_action';
+CREATE INDEX IF NOT EXISTS deliveries_handed_off ON deliveries(claim_owner) WHERE state = 'handed_off';
+CREATE INDEX IF NOT EXISTS deliveries_message ON deliveries(message_id);
+CREATE UNIQUE INDEX IF NOT EXISTS deliveries_idempotency ON deliveries(idempotency_key) WHERE idempotency_key IS NOT NULL;
+CREATE TABLE IF NOT EXISTS waits (
+  wait_id TEXT PRIMARY KEY,
+  node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE,
+  kind TEXT NOT NULL CHECK (kind = 'controller'),
+  target_id TEXT NOT NULL,
+  generation INTEGER NOT NULL,
+  created_at INTEGER NOT NULL,
+  satisfied_at INTEGER
+);
+CREATE UNIQUE INDEX IF NOT EXISTS waits_live ON waits(node_id) WHERE satisfied_at IS NULL;
+CREATE INDEX IF NOT EXISTS waits_node ON waits(node_id);
+`);
+    const mail = [];
+    for (const row of cachedStatement(db, 'SELECT node_id FROM nodes').all()) {
+        const nodeId = row.node_id;
+        const inbox = readJsonLines(inboxPath(nodeId));
+        const cursor = existsSync(`${inboxPath(nodeId)}.cursor`) ? readFileSync(`${inboxPath(nodeId)}.cursor`, 'utf8').trim() : '';
+        const cursorIndex = inbox.findIndex((entry) => entry.entry_id === cursor);
+        const canceled = readCanceled(`${inboxPath(nodeId)}.canceled`);
+        inbox.forEach((entry, physicalIndex) => mail.push({ nodeId, wake: 1, source: 'inbox', fileRank: 0, physicalIndex, entry, acknowledged: cursorIndex >= 0 && physicalIndex <= cursorIndex, canceled: canceled.has(entry.entry_id), seq: 0 }));
+        for (const [fileRank, path] of [[0, `${passivePath(nodeId)}.draining`], [1, passivePath(nodeId)]]) {
+            readJsonLines(path).forEach((entry, physicalIndex) => mail.push({ nodeId, wake: 0, source: 'passive', fileRank, physicalIndex, entry, acknowledged: false, canceled: false, seq: 0 }));
+        }
+    }
+    mail.sort((a, b) => a.entry.ts.localeCompare(b.entry.ts) || a.nodeId.localeCompare(b.nodeId) || a.fileRank - b.fileRank || a.physicalIndex - b.physicalIndex);
+    mail.forEach((entry, index) => { entry.seq = index + 1; });
+    const groups = new Map();
+    for (const entry of mail) {
+        const key = `${entry.nodeId}\0${entry.source}`;
+        const group = groups.get(key) ?? [];
+        group.push(entry);
+        groups.set(key, group);
+    }
+    for (const group of groups.values()) {
+        const available = group.map((entry) => entry.seq).sort((a, b) => a - b);
+        group.sort((a, b) => a.fileRank - b.fileRank || a.physicalIndex - b.physicalIndex).forEach((entry, index) => { entry.seq = available[index]; });
+    }
+    const insertMessage = cachedStatement(db, `INSERT INTO messages (message_id, seq, operation_id, created_at, sender_node_id, sender_name, kind, label, body_ref, body_json, disposition) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+    const insertDelivery = cachedStatement(db, `INSERT INTO deliveries (delivery_id, message_id, destination_kind, destination_id, urgency, wake, state, attempts, acknowledged_at, canceled_at, created_at, updated_at) VALUES (?, ?, 'node_inbox', ?, ?, ?, ?, 0, ?, ?, ?, ?)`);
+    for (const imported of mail) {
+        const entry = imported.entry;
+        insertMessage.run(entry.entry_id, imported.seq, entry.operation_id, entry.ts, entry.from, entry.from_name ?? null, entry.kind, entry.label, entry.ref ?? null, entry.data === undefined ? null : JSON.stringify(entry.data), entry.disposition ?? null);
+        const timestamp = Date.parse(entry.ts);
+        const now = Number.isFinite(timestamp) ? timestamp : 0;
+        insertDelivery.run(newId(), entry.entry_id, imported.nodeId, entry.tier, imported.wake, imported.acknowledged ? 'acknowledged' : 'pending', imported.acknowledged ? now : null, imported.canceled ? now : null, now, now);
+    }
+    const actionRows = cachedStatement(db, 'SELECT * FROM human_action_deliveries').all();
+    for (const row of actionRows) {
+        const messageId = newId();
+        insertMessage.run(messageId, nextSeq(db), newId(), new Date(row.created_at).toISOString(), null, null, 'completed', row.request_id, null, row.completion_json, null);
+        const state = row.state === 'running' ? 'handed_off' : row.state === 'accepted' ? 'acknowledged' : row.state;
+        cachedStatement(db, `INSERT INTO deliveries (delivery_id, message_id, destination_kind, destination_id, urgency, wake, state, attempts, next_attempt_at, claimed_at, claim_owner, acknowledged_at, failed_at, last_failure_json, adapter_json, idempotency_key, created_at, updated_at) VALUES (?, ?, 'external_action', 'human_action', 'normal', 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(newId(), messageId, state, row.attempt, row.next_attempt_at, row.claimed_at, row.claim_owner, row.accepted_at, row.permanent_failed_at, row.last_failure_json, JSON.stringify({ adapter: 'subprocess', argv: JSON.parse(row.argv_json), cwd: row.cwd }), `human-action:${row.request_id}`, row.created_at, row.updated_at);
+    }
+    const outcomeRows = cachedStatement(db, 'SELECT * FROM node_outcome_deliveries').all();
+    for (const row of outcomeRows) {
+        const messageId = row.document_json === null ? null : newId();
+        if (messageId !== null)
+            insertMessage.run(messageId, nextSeq(db), newId(), new Date(row.created_at).toISOString(), null, null, 'completed', row.node_id, null, row.document_json, null);
+        const state = row.document_json === null ? 'armed' : row.state === 'running' ? 'handed_off' : row.state === 'accepted' ? 'acknowledged' : row.state;
+        cachedStatement(db, `INSERT INTO deliveries (delivery_id, message_id, destination_kind, destination_id, urgency, wake, state, attempts, next_attempt_at, claimed_at, claim_owner, acknowledged_at, failed_at, last_failure_json, adapter_json, idempotency_key, created_at, updated_at) VALUES (?, ?, 'external_action', ?, 'normal', 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(newId(), messageId, row.action_name, state, row.attempt, row.next_attempt_at, row.claimed_at, row.claim_owner, row.accepted_at, row.permanent_failed_at, row.last_failure_json, JSON.stringify({ adapter: 'subprocess', argv: JSON.parse(row.argv_json), cwd: row.cwd, config_origin: row.config_origin, payload: row.payload_json === null ? null : JSON.parse(row.payload_json) }), `node-outcome:${row.node_id}`, row.created_at, row.updated_at);
+    }
+    const waiting = cachedStatement(db, 'SELECT node_id, waiting_for, waiting_for_generation FROM nodes WHERE waiting_for IS NOT NULL').all();
+    for (const row of waiting)
+        cachedStatement(db, "INSERT INTO waits (wait_id, node_id, kind, target_id, generation, created_at, satisfied_at) VALUES (?, ?, 'controller', ?, ?, ?, NULL)").run(newId(), row.node_id, row.waiting_for, row.waiting_for_generation, Date.now());
+}
+function nextSeq(db) {
+    return Number(cachedStatement(db, 'SELECT COALESCE(MAX(seq), 0) AS seq FROM messages').get().seq) + 1;
+}

package/dist/core/canvas/delivery-contract.d.ts

@@ -0,0 +1,37 @@
+export interface DeliveryFailure {
+    kind: 'exit' | 'signal' | 'timeout' | 'spawn_error' | 'grant_refused';
+    exitCode?: number;
+    signal?: string;
+    message?: string;
+    stderr?: string;
+}
+export type DeliverySettlement = {
+    state: 'accepted';
+} | {
+    state: 'permanent_failed';
+    failure: DeliveryFailure;
+} | {
+    state: 'pending';
+    nextAttemptAt: number;
+    failure: DeliveryFailure;
+};
+/** Durable exponential retry delay for the attempt that just failed. */
+export declare function deliveryBackoffMs(attempt: number): number;
+/** Attempts a retried delivery gets before it is failed for good. With the one-hour
+ *  delay cap this is roughly ten hours of trying: long enough to ride out a
+ *  receiver outage, short enough that undeliverable work becomes a visible failure
+ *  instead of a row that stays pending and scheduled forever. */
+export declare const DELIVERY_MAX_ATTEMPTS = 20;
+/** The retry budget applied at the one settlement fence. A retryable failure of
+ *  the last permitted attempt becomes a terminal one that keeps the original
+ *  failure's detail and says why it stopped. `exhausted` is true only for that
+ *  conversion, so the caller announces it exactly once. */
+export declare function withRetryBudget<T extends {
+    state: string;
+}>(settlement: T, attempts: number): {
+    settlement: T | {
+        state: 'permanent_failed';
+        failure: DeliveryFailure;
+    };
+    exhausted: boolean;
+};

package/dist/core/canvas/delivery-contract.js

@@ -0,0 +1,30 @@
+// The durable contract shared by every retried delivery: what a failed attempt
+// records, how an attempt settles, and when the next attempt becomes due. The
+// canvas modules own the rows; `daemon/delivery/run-delivery.ts` owns the
+// process that produces these settlements.
+/** Durable exponential retry delay for the attempt that just failed. */
+export function deliveryBackoffMs(attempt) {
+    return Math.min(5_000 * 2 ** (attempt - 1), 3_600_000);
+}
+/** Attempts a retried delivery gets before it is failed for good. With the one-hour
+ *  delay cap this is roughly ten hours of trying: long enough to ride out a
+ *  receiver outage, short enough that undeliverable work becomes a visible failure
+ *  instead of a row that stays pending and scheduled forever. */
+export const DELIVERY_MAX_ATTEMPTS = 20;
+/** The retry budget applied at the one settlement fence. A retryable failure of
+ *  the last permitted attempt becomes a terminal one that keeps the original
+ *  failure's detail and says why it stopped. `exhausted` is true only for that
+ *  conversion, so the caller announces it exactly once. */
+export function withRetryBudget(settlement, attempts) {
+    if (settlement.state !== 'pending' || attempts < DELIVERY_MAX_ATTEMPTS)
+        return { settlement, exhausted: false };
+    const failure = settlement.failure;
+    const cause = failure.message ?? failure.exitCode?.toString() ?? failure.signal ?? failure.kind;
+    return {
+        exhausted: true,
+        settlement: {
+            state: 'permanent_failed',
+            failure: { ...failure, message: `retry budget exhausted after ${attempts} attempts; last failure: ${cause}` },
+        },
+    };
+}