@north-light/api 0.3.580 → 0.3.588

package/dist/core/canvas/recovery.js

@@ -0,0 +1,713 @@
+import { prepared as cachedStatement } from '../sqlite-statements.js';
+import { randomUUID } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { withCanvasWrite, openDb } from './db.js';
+import { jobDir, nodesRoot } from './paths.js';
+import { errorClassFromFault } from '../fault-classifier.js';
+import { parseProviderRateLimit } from '../provider-failure.js';
+import { emitEvent } from '../events/emit.js';
+import { operationIdContext, validateOperationId } from '../events/operation-id.js';
+export const FAULT_QUIET_MS = 5 * 60_000;
+const MESSAGE_CAP = 500;
+const FAULT_LINKS = ['pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'daemon→node', 'crtr→pi'];
+const LEGACY_FAULT_LINKS = [...FAULT_LINKS, 'broker↔crtrd'];
+const FAULT_KINDS = ['rate-limit', 'overloaded', 'connection', 'auth', 'protocol', 'context-overflow', 'other', 'wedged', 'model-not-found'];
+const RETRY_DISPOSITIONS = ['auto', 'manual', 'fatal'];
+const RETRY_OWNERS = ['sdk', 'daemon', 'client'];
+const EPISODE_STATES = ['pending', 'admitted', 'invalidated'];
+function now() {
+    return new Date().toISOString();
+}
+function tableColumns(db, table) {
+    return new Set(db.prepare(`PRAGMA table_info(${table})`).all().map(({ name }) => name));
+}
+function addExecutionColumns(db) {
+    const columns = tableColumns(db, 'executions');
+    const additions = [
+        ['turn_open_at', 'TEXT'],
+        ['turn_progress_at', 'TEXT'],
+        ['turn_closed_at', 'TEXT'],
+        ['turn_end_reason', "TEXT CHECK (turn_end_reason IS NULL OR turn_end_reason IN ('settled', 'teardown-aborted'))"],
+        ['exit_diagnostic_json', 'TEXT'],
+    ];
+    for (const [name, definition] of additions)
+        if (!columns.has(name))
+            db.exec(`ALTER TABLE executions ADD COLUMN ${name} ${definition}`);
+}
+function createFaultTable(db) {
+    db.exec(`
+CREATE TABLE IF NOT EXISTS node_faults (
+  fault_id TEXT PRIMARY KEY,
+  node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE,
+  link TEXT NOT NULL CHECK (link IN ('pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'daemon→node', 'crtr→pi')),
+  op TEXT NOT NULL,
+  kind TEXT NOT NULL CHECK (kind IN ('rate-limit', 'overloaded', 'connection', 'auth', 'protocol', 'context-overflow', 'other', 'wedged', 'model-not-found')),
+  message TEXT NOT NULL,
+  since TEXT NOT NULL,
+  operation_id TEXT NOT NULL,
+  anchor_entry_id TEXT,
+  retry_disposition TEXT NOT NULL CHECK (retry_disposition IN ('auto', 'manual', 'fatal')),
+  retry_by TEXT CHECK (retry_by IS NULL OR retry_by IN ('sdk', 'daemon', 'client')),
+  retry_attempt INTEGER,
+  retry_max INTEGER,
+  retry_next_at TEXT,
+  provider_error_json TEXT,
+  active INTEGER NOT NULL DEFAULT 0 CHECK (active IN (0, 1)),
+  episode_state TEXT CHECK (episode_state IS NULL OR episode_state IN ('pending', 'admitted', 'invalidated')),
+  episode_binding_id TEXT REFERENCES sessions(binding_id),
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_active ON node_faults(node_id) WHERE active = 1;
+CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_episode ON node_faults(node_id) WHERE episode_state IS NOT NULL;
+`);
+}
+function contains(values, value) {
+    return typeof value === 'string' && values.includes(value);
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+export function parseProviderError(value) {
+    if (!isRecord(value) || typeof value.provider !== 'string' || value.provider === '' || typeof value.model !== 'string' || value.model === '')
+        return undefined;
+    const status = typeof value.status === 'number' && Number.isFinite(value.status) ? value.status : undefined;
+    const code = typeof value.code === 'string' || typeof value.code === 'number' ? value.code : undefined;
+    const errno = typeof value.errno === 'string' ? value.errno : undefined;
+    const body = typeof value.body === 'string' ? value.body : undefined;
+    const ratelimit = parseProviderRateLimit(value.ratelimit);
+    const causes = Array.isArray(value.causes)
+        ? value.causes.flatMap((item) => {
+            if (!isRecord(item))
+                return [];
+            const cause = {
+                ...(typeof item.message === 'string' ? { message: item.message } : {}),
+                ...(typeof item.status === 'number' && Number.isFinite(item.status) ? { status: item.status } : {}),
+                ...(typeof item.code === 'string' || typeof item.code === 'number' ? { code: item.code } : {}),
+                ...(typeof item.errno === 'string' ? { errno: item.errno } : {}),
+                ...(typeof item.body === 'string' ? { body: item.body } : {}),
+            };
+            return Object.keys(cause).length === 0 ? [] : [cause];
+        })
+        : undefined;
+    return {
+        provider: value.provider,
+        model: value.model,
+        ...(status === undefined ? {} : { status }),
+        ...(code === undefined ? {} : { code }),
+        ...(errno === undefined ? {} : { errno }),
+        ...(body === undefined ? {} : { body }),
+        ...(causes === undefined || causes.length === 0 ? {} : { causes }),
+        ...(ratelimit === undefined ? {} : { ratelimit }),
+    };
+}
+function parseLegacyFault(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        if (!contains(LEGACY_FAULT_LINKS, parsed.link) || typeof parsed.op !== 'string' || parsed.op === '' || !contains(FAULT_KINDS, parsed.kind) || typeof parsed.message !== 'string' || typeof parsed.since !== 'string')
+            return null;
+        const operationId = validateOperationId(parsed.operation_id);
+        if (!isRecord(parsed.retry) || !contains(RETRY_DISPOSITIONS, parsed.retry.disposition))
+            return null;
+        if (parsed.retry.by !== undefined && !contains(RETRY_OWNERS, parsed.retry.by))
+            return null;
+        if (parsed.retry.attempt !== undefined && typeof parsed.retry.attempt !== 'number')
+            return null;
+        if (parsed.retry.max !== undefined && typeof parsed.retry.max !== 'number')
+            return null;
+        if (parsed.retry.nextAt !== undefined && typeof parsed.retry.nextAt !== 'string')
+            return null;
+        if (parsed.anchorEntryId !== undefined && (typeof parsed.anchorEntryId !== 'string' || parsed.anchorEntryId === ''))
+            return null;
+        const providerError = parsed.providerError === undefined ? undefined : parseProviderError(parsed.providerError);
+        if (parsed.providerError !== undefined && providerError === undefined)
+            return null;
+        return {
+            link: parsed.link,
+            op: parsed.op,
+            kind: parsed.kind,
+            retry: {
+                disposition: parsed.retry.disposition,
+                ...(parsed.retry.by === undefined ? {} : { by: parsed.retry.by }),
+                ...(parsed.retry.attempt === undefined ? {} : { attempt: parsed.retry.attempt }),
+                ...(parsed.retry.max === undefined ? {} : { max: parsed.retry.max }),
+                ...(parsed.retry.nextAt === undefined ? {} : { nextAt: parsed.retry.nextAt }),
+            },
+            message: parsed.message,
+            since: parsed.since,
+            operation_id: operationId,
+            ...(parsed.anchorEntryId === undefined ? {} : { anchorEntryId: parsed.anchorEntryId }),
+            ...(providerError === undefined ? {} : { providerError }),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+function canonicalPath(path) {
+    const absolute = resolve(path);
+    try {
+        return realpathSync(absolute);
+    }
+    catch {
+        return absolute;
+    }
+}
+function parseLegacyEpisode(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        if (!contains(EPISODE_STATES, parsed.state) || typeof parsed.sessionFile !== 'string' || parsed.sessionFile === '')
+            return null;
+        const fault = JSON.stringify(parsed.fault);
+        const parsedFault = parseLegacyFault(fault);
+        if (parsedFault === null || parsedFault.link !== 'pi→provider' || parsedFault.retry.disposition !== 'auto' || parsedFault.retry.by !== 'daemon')
+            return null;
+        const anchor = parsed.anchorEntryId;
+        if (anchor !== null && typeof anchor !== 'string')
+            return null;
+        if ((parsedFault.anchorEntryId ?? null) !== anchor)
+            return null;
+        return { state: parsed.state, sessionFile: canonicalPath(parsed.sessionFile), fault: parsedFault };
+    }
+    catch {
+        return null;
+    }
+}
+function resolveOperationId(explicit) {
+    if (explicit !== undefined)
+        return validateOperationId(explicit);
+    const current = operationIdContext.current();
+    return current ?? operationIdContext.fresh((operationId) => operationId);
+}
+function clampMessage(message) {
+    return message.slice(0, MESSAGE_CAP);
+}
+function faultPriority(fault) {
+    if (fault.link === 'pi→provider' && fault.retry.disposition === 'auto' && fault.retry.by === 'daemon')
+        return 3;
+    if (fault.retry.disposition === 'fatal')
+        return 2;
+    return 1;
+}
+function encodeFault(fault) {
+    return {
+        link: fault.link,
+        op: fault.op,
+        kind: fault.kind,
+        message: fault.message,
+        since: fault.since,
+        operationId: fault.operation_id,
+        anchorEntryId: fault.anchorEntryId ?? null,
+        retryDisposition: fault.retry.disposition,
+        retryBy: fault.retry.by ?? null,
+        retryAttempt: fault.retry.attempt ?? null,
+        retryMax: fault.retry.max ?? null,
+        retryNextAt: fault.retry.nextAt ?? null,
+        providerError: fault.providerError === undefined ? null : JSON.stringify(fault.providerError),
+    };
+}
+export function hydrateFault(row) {
+    const provider = row['provider_error_json'];
+    const providerError = typeof provider === 'string' ? parseProviderError(JSON.parse(provider)) : undefined;
+    if (typeof provider === 'string' && providerError === undefined)
+        throw new Error(`invalid provider error on stored fault ${row['fault_id']}`);
+    const retry = {
+        disposition: row['retry_disposition'],
+        ...(row['retry_by'] === null ? {} : { by: row['retry_by'] }),
+        ...(row['retry_attempt'] === null ? {} : { attempt: row['retry_attempt'] }),
+        ...(row['retry_max'] === null ? {} : { max: row['retry_max'] }),
+        ...(row['retry_next_at'] === null ? {} : { nextAt: row['retry_next_at'] }),
+    };
+    return {
+        faultId: row['fault_id'],
+        link: row['link'],
+        op: row['op'],
+        kind: row['kind'],
+        message: row['message'],
+        since: row['since'],
+        operation_id: row['operation_id'],
+        retry,
+        ...(row['anchor_entry_id'] === null ? {} : { anchorEntryId: row['anchor_entry_id'] }),
+        ...(providerError === undefined ? {} : { providerError }),
+        ...(typeof row['notified_since'] === 'string' ? { notifiedSince: row['notified_since'] } : {}),
+    };
+}
+async function activeFault(db, nodeId) {
+    const row = await db.prepare('SELECT * FROM node_faults WHERE node_id = ? AND active = 1').get(nodeId);
+    return row === undefined ? null : hydrateFault(row);
+}
+function storedFault(input) {
+    if (!contains(FAULT_LINKS, input.link))
+        throw new Error(`fault link is not durable: ${input.link}`);
+    return {
+        link: input.link,
+        op: input.op,
+        kind: input.kind,
+        retry: input.retry,
+        message: clampMessage(input.message),
+        since: input.since ?? now(),
+        operation_id: resolveOperationId(input.operation_id),
+        ...(typeof input.anchorEntryId === 'string' && input.anchorEntryId !== '' ? { anchorEntryId: input.anchorEntryId } : {}),
+        ...(input.providerError === undefined ? {} : { providerError: input.providerError }),
+    };
+}
+function emitRecordedFault(nodeId, fault) {
+    emitEvent({
+        level: fault.retry.disposition === 'fatal' ? 'error' : 'warn',
+        event: 'fault.recorded',
+        operation_id: fault.operation_id,
+        node_id: nodeId,
+        error_class: errorClassFromFault(fault),
+        fields: {
+            link: fault.link, op: fault.op, kind: fault.kind, retry: fault.retry, message: fault.message,
+            ...(fault.anchorEntryId === undefined ? {} : { anchor_entry_id: fault.anchorEntryId }),
+            ...(fault.providerError === undefined ? {} : { provider_error: fault.providerError }),
+        },
+    });
+}
+const INSERT_FAULT_SQL = 'INSERT INTO node_faults (fault_id, node_id, link, op, kind, message, since, operation_id, anchor_entry_id, retry_disposition, retry_by, retry_attempt, retry_max, retry_next_at, provider_error_json, active, episode_state, episode_binding_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
+function faultInsertParams(faultId, nodeId, fault, active, episode) {
+    const encoded = encodeFault(fault);
+    const at = now();
+    return [faultId, nodeId, encoded.link, encoded.op, encoded.kind, encoded.message, encoded.since, encoded.operationId, encoded.anchorEntryId, encoded.retryDisposition, encoded.retryBy, encoded.retryAttempt, encoded.retryMax, encoded.retryNextAt, encoded.providerError, active ? 1 : 0, episode?.state ?? null, episode?.bindingId ?? null, at, at];
+}
+async function insertFault(db, nodeId, fault, active, episode) {
+    const faultId = randomUUID();
+    await db.prepare(INSERT_FAULT_SQL).run(...faultInsertParams(faultId, nodeId, fault, active, episode));
+    return { ...fault, faultId };
+}
+async function updateFault(db, faultId, fault, episode) {
+    const encoded = encodeFault(fault);
+    await db.prepare('UPDATE node_faults SET link = ?, op = ?, kind = ?, message = ?, since = ?, operation_id = ?, anchor_entry_id = ?, retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, provider_error_json = ?, episode_state = COALESCE(?, episode_state), episode_binding_id = COALESCE(?, episode_binding_id), updated_at = ? WHERE fault_id = ?')
+        .run(encoded.link, encoded.op, encoded.kind, encoded.message, encoded.since, encoded.operationId, encoded.anchorEntryId, encoded.retryDisposition, encoded.retryBy, encoded.retryAttempt, encoded.retryMax, encoded.retryNextAt, encoded.providerError, episode?.state ?? null, episode?.bindingId ?? null, now(), faultId);
+    return { ...fault, faultId };
+}
+async function deleteIfUnretained(db, faultId) {
+    await db.prepare('DELETE FROM node_faults WHERE fault_id = ? AND active = 0 AND episode_state IS NULL').run(faultId);
+}
+async function replaceActiveFault(db, nodeId, fault, episode) {
+    const current = await activeFault(db, nodeId);
+    if (current === null) {
+        // A revive clears the active marker while retaining its bound provider-retry
+        // episode. Startup re-records that same fault before arming its retry, so
+        // restore the retained row instead of creating an unpaired active fault.
+        const retained = await db.prepare('SELECT fault_id, link, kind, since, operation_id FROM node_faults WHERE node_id = ? AND active = 0 AND episode_state IS NOT NULL').get(nodeId);
+        if (retained !== undefined && retained.link === fault.link && retained.kind === fault.kind && retained.since === fault.since && retained.operation_id === fault.operation_id) {
+            await db.prepare('UPDATE node_faults SET active = 1, updated_at = ? WHERE fault_id = ?').run(now(), retained.fault_id);
+            return await updateFault(db, retained.fault_id, fault);
+        }
+        return await insertFault(db, nodeId, fault, true, episode);
+    }
+    if (current.link === fault.link && current.kind === fault.kind) {
+        // An automatic retry carries its attempt count forward; any other disposition is exactly what was recorded.
+        const retry = fault.retry.disposition === 'auto' ? { ...current.retry, ...fault.retry } : fault.retry;
+        // The episode continues while the disposition is unchanged (a fatal one only while it repeats the same message);
+        // a fault that becomes or stops being fatal starts a new episode, so each fatal episode is announced once.
+        const continues = current.retry.disposition === fault.retry.disposition && (fault.retry.disposition !== 'fatal' || current.message === fault.message);
+        const since = continues ? current.since : fault.since;
+        const updated = await updateFault(db, current.faultId, { ...fault, since, anchorEntryId: fault.anchorEntryId ?? current.anchorEntryId, retry }, episode);
+        // A new episode has not been announced, even when its start equals the old one's.
+        if (!continues)
+            await db.prepare('UPDATE node_faults SET notified_since = NULL WHERE fault_id = ?').run(current.faultId);
+        return updated;
+    }
+    if (current.link !== fault.link && faultPriority(current) > faultPriority(fault))
+        return null;
+    await db.prepare('UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?').run(now(), current.faultId);
+    await deleteIfUnretained(db, current.faultId);
+    return await insertFault(db, nodeId, fault, true, episode);
+}
+async function liveBinding(db, nodeId, sessionId) {
+    const row = await db.prepare('SELECT binding_id FROM sessions WHERE node_id = ? AND session_id = ? AND unbound_at IS NULL').get(nodeId, sessionId);
+    return row?.binding_id ?? null;
+}
+export async function openTurn(executionId, _opts) {
+    return await withCanvasWrite(async (db) => {
+        const at = now();
+        const open = await db.prepare("UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(at, executionId);
+        if (open.changes > 0)
+            return true;
+        return (await db.prepare("UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ? AND state != 'exited'").run(at, at, executionId)).changes > 0;
+    });
+}
+export async function recordTurnProgress(executionId) {
+    return (await (await openDb()).prepare("UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(now(), executionId)).changes > 0;
+}
+/** Closes the open turn. A `settled` close also stamps the node's `reply_at` (the dashboard's unread marker) in the same
+ *  write, and `read_at` too when a viewer was shown the conversation while the turn ended. A close that changes no row writes neither. */
+export async function closeTurn(executionId, opts) {
+    return await withCanvasWrite(async (db) => {
+        const at = now();
+        const applied = (await db.prepare("UPDATE executions SET turn_closed_at = ?, turn_end_reason = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(at, opts.endReason, executionId)).changes > 0;
+        if (applied && opts.endReason === 'settled') {
+            const owner = '(SELECT node_id FROM executions WHERE execution_id = ?)';
+            if (opts.viewerPresent === true)
+                await db.prepare(`UPDATE nodes SET reply_at = ?, read_at = ? WHERE node_id = ${owner}`).run(at, at, executionId);
+            else
+                await db.prepare(`UPDATE nodes SET reply_at = ? WHERE node_id = ${owner}`).run(at, executionId);
+        }
+        return applied;
+    });
+}
+/** Stamps `read_at`: a viewer has just been shown this conversation. */
+export async function markNodeRead(nodeId) {
+    return (await (await openDb()).prepare('UPDATE nodes SET read_at = ? WHERE node_id = ?').run(now(), nodeId)).changes > 0;
+}
+/** The one writer of an `executions.exit_diagnostic_json` envelope: the schema tag plus one payload the DTOs declare. */
+export function exitDiagnosticJson(payload) {
+    return JSON.stringify({ schema: 'crtr.execution-exit-diagnostic/v1', ...payload });
+}
+export async function recordTransportPoisonVerdict(executionId, verdict) {
+    if (!Number.isInteger(verdict.consecutivePasses) || verdict.consecutivePasses < 1 || verdict.routes.length === 0 || !verdict.routes.every((route) => route !== ''))
+        throw new Error('invalid transport-poison verdict');
+    return (await (await openDb()).prepare("UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ? AND state != 'exited'").run(exitDiagnosticJson({ transport_poison: { consecutive_passes: verdict.consecutivePasses, routes: verdict.routes } }), executionId)).changes > 0;
+}
+export async function readTransportPoisonVerdict(nodeId) {
+    const row = await (await openDb()).prepare("SELECT exit_diagnostic_json FROM executions WHERE node_id = ? ORDER BY seq DESC LIMIT 1").get(nodeId);
+    if (row === undefined || row.exit_diagnostic_json === null)
+        return null;
+    try {
+        const parsed = JSON.parse(row.exit_diagnostic_json);
+        const poison = parsed.transport_poison;
+        if (parsed.schema !== 'crtr.execution-exit-diagnostic/v1' || poison === undefined || !Number.isInteger(poison.consecutive_passes) || poison.consecutive_passes < 1 || !Array.isArray(poison.routes) || poison.routes.length === 0 || !poison.routes.every((route) => typeof route === 'string' && route !== ''))
+            return null;
+        return { consecutivePasses: poison.consecutive_passes, routes: poison.routes };
+    }
+    catch {
+        return null;
+    }
+}
+export async function readLiveTurn(nodeId) {
+    const row = await (await openDb()).prepare("SELECT turn_open_at, turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND state != 'exited'").get(nodeId);
+    if (row === undefined)
+        return null;
+    return { busy: row.turn_open_at !== null && row.turn_closed_at === null, progressAt: row.turn_progress_at };
+}
+export async function readInterruptedTurn(nodeId) {
+    const db = await openDb();
+    const bearing = await db.prepare("SELECT turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL ORDER BY seq DESC LIMIT 1").get(nodeId);
+    if (bearing === undefined)
+        return null;
+    const latest = await db.prepare('SELECT turn_end_reason FROM executions WHERE node_id = ? ORDER BY seq DESC LIMIT 1').get(nodeId);
+    return { interrupted: bearing.turn_closed_at === null || latest?.turn_end_reason === 'teardown-aborted', progressAt: bearing.turn_progress_at };
+}
+/** Whether any execution of this node ever admitted a turn (agent_start recorded
+ *  by the daemon). This, not the transcript, is the durable fact that a node's
+ *  birth task was delivered: a session file holding only the bearings message
+ *  exists before the kickoff is admitted. */
+export async function hasAdmittedTurn(nodeId) {
+    return await (await openDb()).prepare('SELECT 1 FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL LIMIT 1').get(nodeId) !== undefined;
+}
+export async function recordFault(nodeId, input) {
+    const fault = storedFault(input);
+    emitRecordedFault(nodeId, fault);
+    return await withCanvasWrite(async (db) => await replaceActiveFault(db, nodeId, fault));
+}
+/** Record that the fatal-fault notice for this fault row's current episode has been published. Callers run it in the write that publishes. */
+export async function markFaultNotified(faultId) {
+    return await withCanvasWrite(async (db) => (await db.prepare('UPDATE node_faults SET notified_since = since, updated_at = ? WHERE fault_id = ?').run(now(), faultId)).changes > 0);
+}
+function isUnnotifiedFatalProviderFault(fault) {
+    return fault.link === 'pi→provider' && fault.retry.disposition === 'fatal' && fault.notifiedSince !== fault.since;
+}
+/** Nodes whose active fatal provider fault has not been announced yet, read from the fault table so a node with no live broker is still found. */
+export async function listUnnotifiedFatalFaultNodes() {
+    return (await (await openDb()).prepare("SELECT f.node_id FROM node_faults f JOIN nodes n ON n.node_id = f.node_id WHERE f.active = 1 AND f.link = 'pi→provider' AND f.retry_disposition = 'fatal' AND (f.notified_since IS NULL OR f.notified_since != f.since) AND n.status IN ('active', 'idle')").all()).map((r) => r.node_id);
+}
+let fatalFaultAnnouncer = null;
+/** The daemon registers the one function that publishes a fatal provider fault's notice. Every clear of an unannounced fatal fault runs it first, so no clear can drop a notice. */
+export function setFatalFaultAnnouncer(announce) {
+    fatalFaultAnnouncer = announce;
+}
+export async function clearFault(nodeId, opts) {
+    // Most clears have no active fault. Avoid taking the canvas writer lock just
+    // to discover that fact; still recheck under the lock for a real mutation.
+    const candidate = await activeFault(await openDb(), nodeId);
+    if (candidate === null || (opts?.link !== undefined && candidate.link !== opts.link))
+        return false;
+    return await withCanvasWrite(async (db) => {
+        const current = await activeFault(db, nodeId);
+        if (current === null || (opts?.link !== undefined && current.link !== opts.link))
+            return false;
+        if (fatalFaultAnnouncer !== null && isUnnotifiedFatalProviderFault(current))
+            await fatalFaultAnnouncer(nodeId);
+        const episode = await db.prepare('SELECT episode_state FROM node_faults WHERE fault_id = ?').get(current.faultId);
+        await db.prepare('UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?').run(now(), current.faultId);
+        if (episode.episode_state === 'pending' && opts?.preserveEpisode !== true)
+            await db.prepare('UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE fault_id = ?').run(now(), current.faultId);
+        await deleteIfUnretained(db, current.faultId);
+        const operationId = resolveOperationId();
+        emitEvent({
+            level: 'info', event: 'fault.cleared', operation_id: operationId, node_id: nodeId, error_class: errorClassFromFault(current),
+            fields: { cleared_fault_operation_id: current.operation_id, link: current.link, op: current.op, kind: current.kind, retry: current.retry, message: current.message, since: current.since, ...(current.anchorEntryId === undefined ? {} : { anchor_entry_id: current.anchorEntryId }) },
+        });
+        return true;
+    });
+}
+export async function readFault(nodeId) {
+    return await activeFault(await openDb(), nodeId);
+}
+export async function recordPendingEpisode(nodeId, input) {
+    const fault = storedFault(input.input);
+    if (fault.link !== 'pi→provider' || fault.retry.disposition !== 'auto' || fault.retry.by !== 'daemon')
+        throw new Error('provider retry episode requires an automatic daemon provider fault');
+    emitRecordedFault(nodeId, fault);
+    return await withCanvasWrite(async (db) => {
+        const bindingId = await liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return null;
+        const current = await activeFault(db, nodeId);
+        if (current !== null && (current.link !== fault.link || current.kind !== fault.kind) && faultPriority(current) > faultPriority(fault))
+            return null;
+        const existingEpisodes = await db.prepare('SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_state IS NOT NULL').all(nodeId);
+        await db.prepare('UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_state IS NOT NULL').run(now(), nodeId);
+        for (const row of existingEpisodes)
+            await deleteIfUnretained(db, row.fault_id);
+        const stored = await replaceActiveFault(db, nodeId, fault, { state: 'pending', bindingId });
+        if (stored === null)
+            return null;
+        await db.prepare('UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?').run('pending', bindingId, now(), stored.faultId);
+        return stored;
+    });
+}
+export async function admitEpisode(nodeId, input) {
+    return await withCanvasWrite(async (db) => {
+        // A node with a refresh pending (`node yield`, promote) has abandoned this
+        // cycle: its fresh cycle invalidates the episode at startup. Until the old
+        // broker dies, its in-process retry timer must not spend a recovery turn
+        // here, the same rule mail claims follow while intent='refresh'.
+        if (await db.prepare("SELECT 1 FROM nodes WHERE node_id = ? AND intent = 'refresh'").get(nodeId) !== undefined)
+            return false;
+        const bindingId = await liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return false;
+        return (await db.prepare("UPDATE node_faults SET episode_state = 'admitted', retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, updated_at = ? WHERE node_id = ? AND fault_id = ? AND episode_state = 'pending' AND episode_binding_id = ?").run(input.retry.disposition, input.retry.by ?? null, input.retry.attempt ?? null, input.retry.max ?? null, input.retry.nextAt ?? null, now(), nodeId, input.faultId, bindingId)).changes > 0;
+    });
+}
+/** Atomically settle only the admitted attempt that rejected its recovery prompt. A newer turn or episode must never be overwritten by its late result. */
+export async function settleRejectedEpisode(nodeId, input) {
+    const fault = storedFault(input.fault);
+    return await withCanvasWrite(async (db) => {
+        const bindingId = await liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return false;
+        const current = await activeFault(db, nodeId);
+        if (current?.faultId !== input.faultId)
+            return false;
+        const admitted = await db.prepare("SELECT 1 FROM node_faults WHERE node_id = ? AND fault_id = ? AND episode_state = 'admitted' AND episode_binding_id = ?").get(nodeId, input.faultId, bindingId);
+        if (admitted === undefined)
+            return false;
+        emitRecordedFault(nodeId, fault);
+        if (input.reopen) {
+            await updateFault(db, input.faultId, fault, { state: 'pending', bindingId });
+        }
+        else {
+            // Update the owned row in place even when the rejection changes its kind.
+            // Clearing by fault id cannot clear a different episode on the session.
+            await updateFault(db, input.faultId, fault);
+            await db.prepare('UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE fault_id = ?').run(now(), input.faultId);
+            await db.prepare('UPDATE node_faults SET notified_since = NULL WHERE fault_id = ?').run(input.faultId);
+        }
+        return true;
+    });
+}
+export async function reopenEpisode(nodeId, input) {
+    return await withCanvasWrite(async (db) => {
+        const bindingId = await liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return false;
+        return (await db.prepare("UPDATE node_faults SET episode_state = 'pending', updated_at = ? WHERE node_id = ? AND episode_state = 'admitted' AND episode_binding_id = ?").run(now(), nodeId, bindingId)).changes > 0;
+    });
+}
+export async function invalidateEpisode(nodeId, input) {
+    return await withCanvasWrite(async (db) => {
+        const bindingId = await liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return false;
+        return (await db.prepare("UPDATE node_faults SET episode_state = 'invalidated', updated_at = ? WHERE node_id = ? AND episode_state != 'invalidated' AND episode_state IS NOT NULL AND episode_binding_id = ?").run(now(), nodeId, bindingId)).changes > 0;
+    });
+}
+export async function clearEpisode(nodeId, input) {
+    return await withCanvasWrite(async (db) => {
+        const bindingId = await liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return false;
+        const rows = await db.prepare('SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL').all(nodeId, bindingId);
+        if (rows.length === 0)
+            return false;
+        await db.prepare('UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL').run(now(), nodeId, bindingId);
+        for (const row of rows)
+            await deleteIfUnretained(db, row.fault_id);
+        return true;
+    });
+}
+export async function readEpisode(nodeId) {
+    const row = await (await openDb()).prepare('SELECT f.*, s.session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.node_id = ? AND f.episode_state IS NOT NULL').get(nodeId);
+    if (row === undefined)
+        return null;
+    return { state: row['episode_state'], sessionId: row['session_id'], fault: hydrateFault(row) };
+}
+/** Every node with an active daemon-retried provider fault (connection,
+ *  rate-limit, overloaded) and a pending episode on its live session binding.
+ *  The daemon backstop reads this to find retries whose broker timer died. */
+export async function listPendingProviderRetries() {
+    const rows = await (await openDb()).prepare("SELECT f.*, s.session_id AS bound_session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.active = 1 AND f.episode_state = 'pending' AND s.unbound_at IS NULL AND s.session_id IS NOT NULL AND f.link = 'pi→provider' AND f.retry_disposition = 'auto' AND f.retry_by = 'daemon' AND f.kind IN ('connection', 'rate-limit', 'overloaded')").all();
+    return rows.map((row) => ({ nodeId: row['node_id'], sessionId: row['bound_session_id'], fault: hydrateFault(row), updatedAt: row['updated_at'], nudgedAt: row['nudged_at'] ?? null }));
+}
+/** Claim one nudge of a fault: only the caller whose conditional UPDATE changes the row may send it, and a later claim waits out `graceMs`. */
+export async function claimFaultNudge(faultId, nowMs, graceMs) {
+    const result = await (await openDb()).prepare("UPDATE node_faults SET nudged_at = ? WHERE fault_id = ? AND active = 1 AND episode_state = 'pending' AND (nudged_at IS NULL OR nudged_at <= ?)")
+        .run(new Date(nowMs).toISOString(), faultId, new Date(nowMs - graceMs).toISOString());
+    return result.changes === 1;
+}
+export async function beginBootFaultAttempt(nodeId) {
+    const preLaunchClear = await readFault(nodeId) === null;
+    return {
+        async explanation() {
+            if (!preLaunchClear)
+                return null;
+            const fault = await readFault(nodeId);
+            if (fault === null || fault.link !== 'crtr→pi' || fault.retry.disposition !== 'fatal')
+                return null;
+            const op = fault.op.trim();
+            const message = fault.message.trim();
+            return op === '' || message === '' ? null : `${op}: ${message}`;
+        },
+    };
+}
+function markerMtime(path, fallback) {
+    try {
+        return statSync(path).mtime.toISOString();
+    }
+    catch {
+        return fallback;
+    }
+}
+function legacyFile(path, required) {
+    if (!existsSync(path))
+        return null;
+    try {
+        return readFileSync(path, 'utf8');
+    }
+    catch (error) {
+        if (!required)
+            return null;
+        throw new Error(`cannot import recovery input ${path}: ${error.message}`);
+    }
+}
+function latestExecution(db, nodeId) {
+    const row = cachedStatement(db, 'SELECT execution_id, started_at FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1').get(nodeId);
+    return row ?? null;
+}
+function syntheticExecution(db, nodeId) {
+    const columns = tableColumns(db, 'nodes');
+    const launched = columns.has('launched_at')
+        ? cachedStatement(db, 'SELECT COALESCE(launched_at, created) AS started_at FROM nodes WHERE node_id = ?').get(nodeId)
+        : cachedStatement(db, 'SELECT created AS started_at FROM nodes WHERE node_id = ?').get(nodeId);
+    if (launched === undefined)
+        throw new Error(`cannot import recovery witness for unknown node ${nodeId}`);
+    const executionId = randomUUID();
+    cachedStatement(db, "INSERT INTO executions (execution_id, node_id, daemon_epoch, pid, process_identity, state, started_at, ended_at, exit_reason) VALUES (?, ?, NULL, NULL, NULL, 'exited', ?, ?, 'pre-consolidation')").run(executionId, nodeId, launched.started_at, launched.started_at);
+    return { execution_id: executionId, started_at: launched.started_at };
+}
+/** The migration chain's synchronous insert (`migrateRecoveryWitnesses` runs on the raw connection). */
+function insertMigratedFault(db, nodeId, fault, active, episode) {
+    const faultId = randomUUID();
+    cachedStatement(db, INSERT_FAULT_SQL).run(...faultInsertParams(faultId, nodeId, fault, active, episode));
+    return { ...fault, faultId };
+}
+function migrationBinding(db, nodeId, transcriptPath, legacyPath) {
+    const current = cachedStatement(db, 'SELECT binding_id, node_id FROM sessions WHERE transcript_path = ? AND unbound_at IS NULL').get(transcriptPath);
+    if (current !== undefined) {
+        if (current.node_id !== nodeId)
+            throw new Error(`cannot import provider retry ${legacyPath}: transcript is live-owned by ${current.node_id}`);
+        return current.binding_id;
+    }
+    const known = cachedStatement(db, 'SELECT binding_id FROM sessions WHERE node_id = ? AND transcript_path = ? ORDER BY rowid DESC LIMIT 1').get(nodeId, transcriptPath);
+    if (known !== undefined)
+        return known.binding_id;
+    const created = cachedStatement(db, 'SELECT created FROM nodes WHERE node_id = ?').get(nodeId);
+    if (created === undefined)
+        throw new Error(`cannot import provider retry ${legacyPath}: unknown node ${nodeId}`);
+    const bindingId = randomUUID();
+    cachedStatement(db, 'INSERT INTO sessions (binding_id, session_id, node_id, transcript_path, source_session_id, fork_entry_id, created_at, bound_at, unbound_at) VALUES (?, NULL, ?, ?, NULL, NULL, ?, ?, ?)').run(bindingId, nodeId, transcriptPath, created.created, created.created, now());
+    return bindingId;
+}
+/** Import legacy witness and recovery files inside the parent's one shared consolidation transaction. */
+/** v98 — `notified_since` on a fault row: the episode start its fatal notice was published for.
+ *  No backfill: a fatal fault already active at upgrade reads as unannounced and is announced once more,
+ *  because the old in-memory record cannot be rebuilt and a backfill could silence a fault never announced.
+ *  Guarded like v95: a fixture that starts past v39 never created the table. */
+export function addFaultNotifiedSince(db) {
+    if (db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'node_faults'").get() === undefined)
+        return;
+    if (!tableColumns(db, 'node_faults').has('notified_since'))
+        db.exec('ALTER TABLE node_faults ADD COLUMN notified_since TEXT');
+}
+export function migrateRecoveryWitnesses(db) {
+    addExecutionColumns(db);
+    createFaultTable(db);
+    if (!existsSync(nodesRoot()))
+        return;
+    const entries = readdirSync(nodesRoot(), { withFileTypes: true }).filter((entry) => entry.isDirectory()).sort((a, b) => a.name.localeCompare(b.name));
+    const nodeExists = cachedStatement(db, 'SELECT 1 FROM nodes WHERE node_id = ?');
+    for (const entry of entries) {
+        const nodeId = entry.name;
+        const directory = jobDir(nodeId);
+        const busyPath = join(directory, 'busy');
+        const abortPath = join(directory, 'clean-abort');
+        const poisonPath = join(directory, 'transport-poison.json');
+        const faultPath = join(directory, 'fault');
+        const episodePath = join(directory, 'provider-retry');
+        const busy = existsSync(busyPath);
+        const aborted = existsSync(abortPath);
+        if (!busy && !aborted && !existsSync(poisonPath) && !existsSync(faultPath) && !existsSync(episodePath))
+            continue;
+        if (!existsSync(join(nodesRoot(), nodeId, 'meta.json')) && nodeExists.get(nodeId) === undefined)
+            continue;
+        const poison = legacyFile(poisonPath, false);
+        const legacyFault = legacyFile(faultPath, true);
+        const legacyEpisode = legacyFile(episodePath, true);
+        if (!busy && !aborted && poison === null && legacyFault === null && legacyEpisode === null)
+            continue;
+        let execution = latestExecution(db, nodeId);
+        if (execution === null)
+            execution = syntheticExecution(db, nodeId);
+        if (busy) {
+            const at = markerMtime(busyPath, execution.started_at);
+            cachedStatement(db, 'UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ?').run(at, at, execution.execution_id);
+        }
+        else if (aborted) {
+            const at = markerMtime(abortPath, execution.started_at);
+            cachedStatement(db, "UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = ?, turn_end_reason = 'teardown-aborted' WHERE execution_id = ?").run(at, at, at, execution.execution_id);
+        }
+        if (poison !== null) {
+            try {
+                const parsed = JSON.parse(poison);
+                if (Number.isInteger(parsed.consecutivePasses) && parsed.consecutivePasses >= 1 && Array.isArray(parsed.routes) && parsed.routes.length > 0 && parsed.routes.every((route) => typeof route === 'string' && route !== '')) {
+                    cachedStatement(db, 'UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ?').run(exitDiagnosticJson({ transport_poison: { consecutive_passes: parsed.consecutivePasses, routes: parsed.routes } }), execution.execution_id);
+                }
+            }
+            catch {
+                // Diagnostic-only legacy context is intentionally omitted when corrupt.
+            }
+        }
+        const active = legacyFault === null ? null : parseLegacyFault(legacyFault);
+        if (legacyFault !== null && active === null)
+            throw new Error(`cannot import recovery fault ${faultPath}`);
+        const episode = legacyEpisode === null ? null : parseLegacyEpisode(legacyEpisode);
+        if (legacyEpisode !== null && episode === null)
+            throw new Error(`cannot import provider retry ${episodePath}`);
+        let activeStored = active === null || active.link === 'broker↔crtrd' ? null : insertMigratedFault(db, nodeId, active, true);
+        if (episode !== null) {
+            const bindingId = migrationBinding(db, nodeId, episode.sessionFile, episodePath);
+            if (activeStored !== null && activeStored.since === episode.fault.since) {
+                cachedStatement(db, 'UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?').run(episode.state, bindingId, now(), activeStored.faultId);
+            }
+            else {
+                insertMigratedFault(db, nodeId, episode.fault, false, { state: episode.state, bindingId });
+            }
+        }
+    }
+}