@north-light/crouter 0.3.312 → 0.3.314

package/dist/core/canvas/recovery.js

@@ -0,0 +1,588 @@
+import { randomUUID } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { openDb, withCanvasWrite } from './db.js';
+import { jobDir, nodesRoot } from './paths.js';
+import { errorClassFromFault } from '../fault-classifier.js';
+import { emitEvent } from '../events/emit.js';
+import { operationIdContext, validateOperationId } from '../events/operation-id.js';
+export const FAULT_QUIET_MS = 5 * 60_000;
+const MESSAGE_CAP = 500;
+const FAULT_LINKS = ['pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'daemon→node', 'crtr→pi'];
+const LEGACY_FAULT_LINKS = [...FAULT_LINKS, 'broker↔crtrd'];
+const FAULT_KINDS = ['rate-limit', 'overloaded', 'connection', 'auth', 'protocol', 'context-overflow', 'other', 'wedged', 'model-not-found'];
+const RETRY_DISPOSITIONS = ['auto', 'manual', 'fatal'];
+const RETRY_OWNERS = ['sdk', 'daemon', 'client'];
+const EPISODE_STATES = ['pending', 'admitted', 'invalidated'];
+function now() {
+    return new Date().toISOString();
+}
+function tableColumns(db, table) {
+    return new Set(db.prepare(`PRAGMA table_info(${table})`).all().map(({ name }) => name));
+}
+function addExecutionColumns(db) {
+    const columns = tableColumns(db, 'executions');
+    const additions = [
+        ['turn_open_at', 'TEXT'],
+        ['turn_progress_at', 'TEXT'],
+        ['turn_closed_at', 'TEXT'],
+        ['turn_end_reason', "TEXT CHECK (turn_end_reason IS NULL OR turn_end_reason IN ('settled', 'teardown-aborted'))"],
+        ['exit_diagnostic_json', 'TEXT'],
+    ];
+    for (const [name, definition] of additions)
+        if (!columns.has(name))
+            db.exec(`ALTER TABLE executions ADD COLUMN ${name} ${definition}`);
+}
+function createFaultTable(db) {
+    db.exec(`
+CREATE TABLE IF NOT EXISTS node_faults (
+  fault_id TEXT PRIMARY KEY,
+  node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE,
+  link TEXT NOT NULL CHECK (link IN ('pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'daemon→node', 'crtr→pi')),
+  op TEXT NOT NULL,
+  kind TEXT NOT NULL CHECK (kind IN ('rate-limit', 'overloaded', 'connection', 'auth', 'protocol', 'context-overflow', 'other', 'wedged', 'model-not-found')),
+  message TEXT NOT NULL,
+  since TEXT NOT NULL,
+  operation_id TEXT NOT NULL,
+  anchor_entry_id TEXT,
+  retry_disposition TEXT NOT NULL CHECK (retry_disposition IN ('auto', 'manual', 'fatal')),
+  retry_by TEXT CHECK (retry_by IS NULL OR retry_by IN ('sdk', 'daemon', 'client')),
+  retry_attempt INTEGER,
+  retry_max INTEGER,
+  retry_next_at TEXT,
+  provider_error_json TEXT,
+  active INTEGER NOT NULL DEFAULT 0 CHECK (active IN (0, 1)),
+  episode_state TEXT CHECK (episode_state IS NULL OR episode_state IN ('pending', 'admitted', 'invalidated')),
+  episode_binding_id TEXT REFERENCES sessions(binding_id),
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_active ON node_faults(node_id) WHERE active = 1;
+CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_episode ON node_faults(node_id) WHERE episode_state IS NOT NULL;
+`);
+}
+function contains(values, value) {
+    return typeof value === 'string' && values.includes(value);
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+export function parseProviderError(value) {
+    if (!isRecord(value) || typeof value.provider !== 'string' || value.provider === '' || typeof value.model !== 'string' || value.model === '')
+        return undefined;
+    const status = typeof value.status === 'number' && Number.isFinite(value.status) ? value.status : undefined;
+    const code = typeof value.code === 'string' || typeof value.code === 'number' ? value.code : undefined;
+    const errno = typeof value.errno === 'string' ? value.errno : undefined;
+    const body = typeof value.body === 'string' ? value.body : undefined;
+    const causes = Array.isArray(value.causes)
+        ? value.causes.flatMap((item) => {
+            if (!isRecord(item))
+                return [];
+            const cause = {
+                ...(typeof item.message === 'string' ? { message: item.message } : {}),
+                ...(typeof item.status === 'number' && Number.isFinite(item.status) ? { status: item.status } : {}),
+                ...(typeof item.code === 'string' || typeof item.code === 'number' ? { code: item.code } : {}),
+                ...(typeof item.errno === 'string' ? { errno: item.errno } : {}),
+                ...(typeof item.body === 'string' ? { body: item.body } : {}),
+            };
+            return Object.keys(cause).length === 0 ? [] : [cause];
+        })
+        : undefined;
+    return {
+        provider: value.provider,
+        model: value.model,
+        ...(status === undefined ? {} : { status }),
+        ...(code === undefined ? {} : { code }),
+        ...(errno === undefined ? {} : { errno }),
+        ...(body === undefined ? {} : { body }),
+        ...(causes === undefined || causes.length === 0 ? {} : { causes }),
+    };
+}
+function parseLegacyFault(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        if (!contains(LEGACY_FAULT_LINKS, parsed.link) || typeof parsed.op !== 'string' || parsed.op === '' || !contains(FAULT_KINDS, parsed.kind) || typeof parsed.message !== 'string' || typeof parsed.since !== 'string')
+            return null;
+        const operationId = validateOperationId(parsed.operation_id);
+        if (!isRecord(parsed.retry) || !contains(RETRY_DISPOSITIONS, parsed.retry.disposition))
+            return null;
+        if (parsed.retry.by !== undefined && !contains(RETRY_OWNERS, parsed.retry.by))
+            return null;
+        if (parsed.retry.attempt !== undefined && typeof parsed.retry.attempt !== 'number')
+            return null;
+        if (parsed.retry.max !== undefined && typeof parsed.retry.max !== 'number')
+            return null;
+        if (parsed.retry.nextAt !== undefined && typeof parsed.retry.nextAt !== 'string')
+            return null;
+        if (parsed.anchorEntryId !== undefined && (typeof parsed.anchorEntryId !== 'string' || parsed.anchorEntryId === ''))
+            return null;
+        const providerError = parsed.providerError === undefined ? undefined : parseProviderError(parsed.providerError);
+        if (parsed.providerError !== undefined && providerError === undefined)
+            return null;
+        return {
+            link: parsed.link,
+            op: parsed.op,
+            kind: parsed.kind,
+            retry: {
+                disposition: parsed.retry.disposition,
+                ...(parsed.retry.by === undefined ? {} : { by: parsed.retry.by }),
+                ...(parsed.retry.attempt === undefined ? {} : { attempt: parsed.retry.attempt }),
+                ...(parsed.retry.max === undefined ? {} : { max: parsed.retry.max }),
+                ...(parsed.retry.nextAt === undefined ? {} : { nextAt: parsed.retry.nextAt }),
+            },
+            message: parsed.message,
+            since: parsed.since,
+            operation_id: operationId,
+            ...(parsed.anchorEntryId === undefined ? {} : { anchorEntryId: parsed.anchorEntryId }),
+            ...(providerError === undefined ? {} : { providerError }),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+function canonicalPath(path) {
+    const absolute = resolve(path);
+    try {
+        return realpathSync(absolute);
+    }
+    catch {
+        return absolute;
+    }
+}
+function parseLegacyEpisode(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        if (!contains(EPISODE_STATES, parsed.state) || typeof parsed.sessionFile !== 'string' || parsed.sessionFile === '')
+            return null;
+        const fault = JSON.stringify(parsed.fault);
+        const parsedFault = parseLegacyFault(fault);
+        if (parsedFault === null || parsedFault.link !== 'pi→provider' || parsedFault.retry.disposition !== 'auto' || parsedFault.retry.by !== 'daemon')
+            return null;
+        const anchor = parsed.anchorEntryId;
+        if (anchor !== null && typeof anchor !== 'string')
+            return null;
+        if ((parsedFault.anchorEntryId ?? null) !== anchor)
+            return null;
+        return { state: parsed.state, sessionFile: canonicalPath(parsed.sessionFile), fault: parsedFault };
+    }
+    catch {
+        return null;
+    }
+}
+function resolveOperationId(explicit) {
+    if (explicit !== undefined)
+        return validateOperationId(explicit);
+    const current = operationIdContext.current();
+    return current ?? operationIdContext.fresh((operationId) => operationId);
+}
+function clampMessage(message) {
+    return message.slice(0, MESSAGE_CAP);
+}
+function faultPriority(fault) {
+    if (fault.link === 'pi→provider' && fault.retry.disposition === 'auto' && fault.retry.by === 'daemon')
+        return 3;
+    if (fault.retry.disposition === 'fatal')
+        return 2;
+    return 1;
+}
+function encodeFault(fault) {
+    return {
+        link: fault.link,
+        op: fault.op,
+        kind: fault.kind,
+        message: fault.message,
+        since: fault.since,
+        operationId: fault.operation_id,
+        anchorEntryId: fault.anchorEntryId ?? null,
+        retryDisposition: fault.retry.disposition,
+        retryBy: fault.retry.by ?? null,
+        retryAttempt: fault.retry.attempt ?? null,
+        retryMax: fault.retry.max ?? null,
+        retryNextAt: fault.retry.nextAt ?? null,
+        providerError: fault.providerError === undefined ? null : JSON.stringify(fault.providerError),
+    };
+}
+function hydrateFault(row) {
+    const provider = row['provider_error_json'];
+    const providerError = typeof provider === 'string' ? parseProviderError(JSON.parse(provider)) : undefined;
+    if (typeof provider === 'string' && providerError === undefined)
+        throw new Error(`invalid provider error on stored fault ${row['fault_id']}`);
+    const retry = {
+        disposition: row['retry_disposition'],
+        ...(row['retry_by'] === null ? {} : { by: row['retry_by'] }),
+        ...(row['retry_attempt'] === null ? {} : { attempt: row['retry_attempt'] }),
+        ...(row['retry_max'] === null ? {} : { max: row['retry_max'] }),
+        ...(row['retry_next_at'] === null ? {} : { nextAt: row['retry_next_at'] }),
+    };
+    return {
+        faultId: row['fault_id'],
+        link: row['link'],
+        op: row['op'],
+        kind: row['kind'],
+        message: row['message'],
+        since: row['since'],
+        operation_id: row['operation_id'],
+        retry,
+        ...(row['anchor_entry_id'] === null ? {} : { anchorEntryId: row['anchor_entry_id'] }),
+        ...(providerError === undefined ? {} : { providerError }),
+    };
+}
+function activeFault(db, nodeId) {
+    const row = db.prepare('SELECT * FROM node_faults WHERE node_id = ? AND active = 1').get(nodeId);
+    return row === undefined ? null : hydrateFault(row);
+}
+function storedFault(input) {
+    if (!contains(FAULT_LINKS, input.link))
+        throw new Error(`fault link is not durable: ${input.link}`);
+    return {
+        link: input.link,
+        op: input.op,
+        kind: input.kind,
+        retry: input.retry,
+        message: clampMessage(input.message),
+        since: input.since ?? now(),
+        operation_id: resolveOperationId(input.operation_id),
+        ...(typeof input.anchorEntryId === 'string' && input.anchorEntryId !== '' ? { anchorEntryId: input.anchorEntryId } : {}),
+        ...(input.providerError === undefined ? {} : { providerError: input.providerError }),
+    };
+}
+function emitRecordedFault(nodeId, fault) {
+    emitEvent({
+        level: fault.retry.disposition === 'fatal' ? 'error' : 'warn',
+        event: 'fault.recorded',
+        operation_id: fault.operation_id,
+        node_id: nodeId,
+        error_class: errorClassFromFault(fault),
+        fields: {
+            link: fault.link, op: fault.op, kind: fault.kind, retry: fault.retry, message: fault.message,
+            ...(fault.anchorEntryId === undefined ? {} : { anchor_entry_id: fault.anchorEntryId }),
+            ...(fault.providerError === undefined ? {} : { provider_error: fault.providerError }),
+        },
+    });
+}
+function insertFault(db, nodeId, fault, active, episode) {
+    const faultId = randomUUID();
+    const encoded = encodeFault(fault);
+    const at = now();
+    db.prepare('INSERT INTO node_faults (fault_id, node_id, link, op, kind, message, since, operation_id, anchor_entry_id, retry_disposition, retry_by, retry_attempt, retry_max, retry_next_at, provider_error_json, active, episode_state, episode_binding_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+        .run(faultId, nodeId, encoded.link, encoded.op, encoded.kind, encoded.message, encoded.since, encoded.operationId, encoded.anchorEntryId, encoded.retryDisposition, encoded.retryBy, encoded.retryAttempt, encoded.retryMax, encoded.retryNextAt, encoded.providerError, active ? 1 : 0, episode?.state ?? null, episode?.bindingId ?? null, at, at);
+    return { ...fault, faultId };
+}
+function updateFault(db, faultId, fault, episode) {
+    const encoded = encodeFault(fault);
+    db.prepare('UPDATE node_faults SET link = ?, op = ?, kind = ?, message = ?, since = ?, operation_id = ?, anchor_entry_id = ?, retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, provider_error_json = ?, episode_state = COALESCE(?, episode_state), episode_binding_id = COALESCE(?, episode_binding_id), updated_at = ? WHERE fault_id = ?')
+        .run(encoded.link, encoded.op, encoded.kind, encoded.message, encoded.since, encoded.operationId, encoded.anchorEntryId, encoded.retryDisposition, encoded.retryBy, encoded.retryAttempt, encoded.retryMax, encoded.retryNextAt, encoded.providerError, episode?.state ?? null, episode?.bindingId ?? null, now(), faultId);
+    return { ...fault, faultId };
+}
+function deleteIfUnretained(db, faultId) {
+    db.prepare('DELETE FROM node_faults WHERE fault_id = ? AND active = 0 AND episode_state IS NULL').run(faultId);
+}
+function replaceActiveFault(db, nodeId, fault, episode) {
+    const current = activeFault(db, nodeId);
+    if (current === null) {
+        // A revive clears the active marker while retaining its bound provider-retry
+        // episode. Startup re-records that same fault before arming its retry, so
+        // restore the retained row instead of creating an unpaired active fault.
+        const retained = db.prepare('SELECT fault_id, link, kind, since, operation_id FROM node_faults WHERE node_id = ? AND active = 0 AND episode_state IS NOT NULL').get(nodeId);
+        if (retained !== undefined && retained.link === fault.link && retained.kind === fault.kind && retained.since === fault.since && retained.operation_id === fault.operation_id) {
+            db.prepare('UPDATE node_faults SET active = 1, updated_at = ? WHERE fault_id = ?').run(now(), retained.fault_id);
+            return updateFault(db, retained.fault_id, fault);
+        }
+        return insertFault(db, nodeId, fault, true, episode);
+    }
+    if (current.link === fault.link && current.kind === fault.kind) {
+        const retry = { ...current.retry, ...fault.retry };
+        if (fault.retry.disposition === 'fatal' && !('nextAt' in fault.retry))
+            delete retry.nextAt;
+        return updateFault(db, current.faultId, { ...fault, since: current.since, anchorEntryId: current.anchorEntryId ?? fault.anchorEntryId, retry }, episode);
+    }
+    if (faultPriority(current) > faultPriority(fault))
+        return null;
+    db.prepare('UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?').run(now(), current.faultId);
+    deleteIfUnretained(db, current.faultId);
+    return insertFault(db, nodeId, fault, true, episode);
+}
+function liveBinding(db, nodeId, sessionId) {
+    const row = db.prepare('SELECT binding_id FROM sessions WHERE node_id = ? AND session_id = ? AND unbound_at IS NULL').get(nodeId, sessionId);
+    return row?.binding_id ?? null;
+}
+export function openTurn(executionId, _opts) {
+    return withCanvasWrite((db) => {
+        const at = now();
+        const open = db.prepare("UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(at, executionId);
+        if (open.changes > 0)
+            return true;
+        return db.prepare("UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ? AND state != 'exited'").run(at, at, executionId).changes > 0;
+    });
+}
+export function recordTurnProgress(executionId) {
+    return withCanvasWrite((db) => db.prepare("UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(now(), executionId).changes > 0);
+}
+export function closeTurn(executionId, opts) {
+    return withCanvasWrite((db) => db.prepare("UPDATE executions SET turn_closed_at = ?, turn_end_reason = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(now(), opts.endReason, executionId).changes > 0);
+}
+export function recordTransportPoisonVerdict(executionId, verdict) {
+    if (!Number.isInteger(verdict.consecutivePasses) || verdict.consecutivePasses < 1 || verdict.routes.length === 0 || !verdict.routes.every((route) => route !== ''))
+        throw new Error('invalid transport-poison verdict');
+    return withCanvasWrite((db) => db.prepare("UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ? AND state != 'exited'").run(JSON.stringify({ schema: 'crtr.execution-exit-diagnostic/v1', transport_poison: { consecutive_passes: verdict.consecutivePasses, routes: verdict.routes } }), executionId).changes > 0);
+}
+export function readTransportPoisonVerdict(nodeId) {
+    const row = openDb().prepare("SELECT exit_diagnostic_json FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(nodeId);
+    if (row === undefined || row.exit_diagnostic_json === null)
+        return null;
+    try {
+        const parsed = JSON.parse(row.exit_diagnostic_json);
+        const poison = parsed.transport_poison;
+        if (parsed.schema !== 'crtr.execution-exit-diagnostic/v1' || poison === undefined || !Number.isInteger(poison.consecutive_passes) || poison.consecutive_passes < 1 || !Array.isArray(poison.routes) || poison.routes.length === 0 || !poison.routes.every((route) => typeof route === 'string' && route !== ''))
+            return null;
+        return { consecutivePasses: poison.consecutive_passes, routes: poison.routes };
+    }
+    catch {
+        return null;
+    }
+}
+export function readLiveTurn(nodeId) {
+    const row = openDb().prepare("SELECT turn_open_at, turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND state != 'exited'").get(nodeId);
+    if (row === undefined)
+        return null;
+    return { busy: row.turn_open_at !== null && row.turn_closed_at === null, progressAt: row.turn_progress_at };
+}
+export function readInterruptedTurn(nodeId) {
+    const db = openDb();
+    const bearing = db.prepare("SELECT turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL ORDER BY rowid DESC LIMIT 1").get(nodeId);
+    if (bearing === undefined)
+        return null;
+    const latest = db.prepare('SELECT turn_end_reason FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1').get(nodeId);
+    return { interrupted: bearing.turn_closed_at === null || latest?.turn_end_reason === 'teardown-aborted', progressAt: bearing.turn_progress_at };
+}
+export function recordFault(nodeId, input) {
+    const fault = storedFault(input);
+    emitRecordedFault(nodeId, fault);
+    return withCanvasWrite((db) => replaceActiveFault(db, nodeId, fault));
+}
+export function clearFault(nodeId, opts) {
+    return withCanvasWrite((db) => {
+        const current = activeFault(db, nodeId);
+        if (current === null || (opts?.link !== undefined && current.link !== opts.link))
+            return false;
+        const episode = db.prepare('SELECT episode_state FROM node_faults WHERE fault_id = ?').get(current.faultId);
+        db.prepare('UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?').run(now(), current.faultId);
+        if (episode.episode_state === 'pending' && opts?.preserveEpisode !== true)
+            db.prepare('UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE fault_id = ?').run(now(), current.faultId);
+        deleteIfUnretained(db, current.faultId);
+        const operationId = resolveOperationId();
+        emitEvent({
+            level: 'info', event: 'fault.cleared', operation_id: operationId, node_id: nodeId, error_class: errorClassFromFault(current),
+            fields: { cleared_fault_operation_id: current.operation_id, link: current.link, op: current.op, kind: current.kind, retry: current.retry, message: current.message, since: current.since, ...(current.anchorEntryId === undefined ? {} : { anchor_entry_id: current.anchorEntryId }) },
+        });
+        return true;
+    });
+}
+export function readFault(nodeId) {
+    return activeFault(openDb(), nodeId);
+}
+export function recordPendingEpisode(nodeId, input) {
+    const fault = storedFault(input.input);
+    if (fault.link !== 'pi→provider' || fault.retry.disposition !== 'auto' || fault.retry.by !== 'daemon')
+        throw new Error('provider retry episode requires an automatic daemon provider fault');
+    emitRecordedFault(nodeId, fault);
+    return withCanvasWrite((db) => {
+        const bindingId = liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return null;
+        const current = activeFault(db, nodeId);
+        if (current !== null && (current.link !== fault.link || current.kind !== fault.kind) && faultPriority(current) > faultPriority(fault))
+            return null;
+        const existingEpisodes = db.prepare('SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_state IS NOT NULL').all(nodeId);
+        db.prepare('UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_state IS NOT NULL').run(now(), nodeId);
+        for (const row of existingEpisodes)
+            deleteIfUnretained(db, row.fault_id);
+        const stored = replaceActiveFault(db, nodeId, fault, { state: 'pending', bindingId });
+        if (stored === null)
+            return null;
+        db.prepare('UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?').run('pending', bindingId, now(), stored.faultId);
+        return stored;
+    });
+}
+export function admitEpisode(nodeId, input) {
+    return withCanvasWrite((db) => {
+        const bindingId = liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return false;
+        return db.prepare("UPDATE node_faults SET episode_state = 'admitted', retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, updated_at = ? WHERE node_id = ? AND fault_id = ? AND episode_state = 'pending' AND episode_binding_id = ?").run(input.retry.disposition, input.retry.by ?? null, input.retry.attempt ?? null, input.retry.max ?? null, input.retry.nextAt ?? null, now(), nodeId, input.faultId, bindingId).changes > 0;
+    });
+}
+export function reopenEpisode(nodeId, input) {
+    return withCanvasWrite((db) => {
+        const bindingId = liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return false;
+        return db.prepare("UPDATE node_faults SET episode_state = 'pending', updated_at = ? WHERE node_id = ? AND episode_state = 'admitted' AND episode_binding_id = ?").run(now(), nodeId, bindingId).changes > 0;
+    });
+}
+export function invalidateEpisode(nodeId, input) {
+    return withCanvasWrite((db) => {
+        const bindingId = liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return false;
+        return db.prepare("UPDATE node_faults SET episode_state = 'invalidated', updated_at = ? WHERE node_id = ? AND episode_state != 'invalidated' AND episode_state IS NOT NULL AND episode_binding_id = ?").run(now(), nodeId, bindingId).changes > 0;
+    });
+}
+export function clearEpisode(nodeId, input) {
+    return withCanvasWrite((db) => {
+        const bindingId = liveBinding(db, nodeId, input.sessionId);
+        if (bindingId === null)
+            return false;
+        const rows = db.prepare('SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL').all(nodeId, bindingId);
+        if (rows.length === 0)
+            return false;
+        db.prepare('UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL').run(now(), nodeId, bindingId);
+        for (const row of rows)
+            deleteIfUnretained(db, row.fault_id);
+        return true;
+    });
+}
+export function readEpisode(nodeId) {
+    const row = openDb().prepare('SELECT f.*, s.session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.node_id = ? AND f.episode_state IS NOT NULL').get(nodeId);
+    if (row === undefined)
+        return null;
+    return { state: row['episode_state'], sessionId: row['session_id'], fault: hydrateFault(row) };
+}
+export function beginBootFaultAttempt(nodeId) {
+    const preLaunchClear = readFault(nodeId) === null;
+    return {
+        explanation() {
+            if (!preLaunchClear)
+                return null;
+            const fault = readFault(nodeId);
+            if (fault === null || fault.link !== 'crtr→pi' || fault.retry.disposition !== 'fatal')
+                return null;
+            const op = fault.op.trim();
+            const message = fault.message.trim();
+            return op === '' || message === '' ? null : `${op}: ${message}`;
+        },
+    };
+}
+function markerMtime(path, fallback) {
+    try {
+        return statSync(path).mtime.toISOString();
+    }
+    catch {
+        return fallback;
+    }
+}
+function legacyFile(path, required) {
+    if (!existsSync(path))
+        return null;
+    try {
+        return readFileSync(path, 'utf8');
+    }
+    catch (error) {
+        if (!required)
+            return null;
+        throw new Error(`cannot import recovery input ${path}: ${error.message}`);
+    }
+}
+function latestExecution(db, nodeId) {
+    const row = db.prepare('SELECT execution_id, started_at FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1').get(nodeId);
+    return row ?? null;
+}
+function syntheticExecution(db, nodeId) {
+    const columns = tableColumns(db, 'nodes');
+    const launched = columns.has('launched_at')
+        ? db.prepare('SELECT COALESCE(launched_at, created) AS started_at FROM nodes WHERE node_id = ?').get(nodeId)
+        : db.prepare('SELECT created AS started_at FROM nodes WHERE node_id = ?').get(nodeId);
+    if (launched === undefined)
+        throw new Error(`cannot import recovery witness for unknown node ${nodeId}`);
+    const executionId = randomUUID();
+    db.prepare("INSERT INTO executions (execution_id, node_id, daemon_epoch, pid, process_identity, state, started_at, ended_at, exit_reason) VALUES (?, ?, NULL, NULL, NULL, 'exited', ?, ?, 'pre-consolidation')").run(executionId, nodeId, launched.started_at, launched.started_at);
+    return { execution_id: executionId, started_at: launched.started_at };
+}
+function insertMigratedFault(db, nodeId, fault, active, episode) {
+    return insertFault(db, nodeId, fault, active, episode);
+}
+function migrationBinding(db, nodeId, transcriptPath, legacyPath) {
+    const current = db.prepare('SELECT binding_id, node_id FROM sessions WHERE transcript_path = ? AND unbound_at IS NULL').get(transcriptPath);
+    if (current !== undefined) {
+        if (current.node_id !== nodeId)
+            throw new Error(`cannot import provider retry ${legacyPath}: transcript is live-owned by ${current.node_id}`);
+        return current.binding_id;
+    }
+    const known = db.prepare('SELECT binding_id FROM sessions WHERE node_id = ? AND transcript_path = ? ORDER BY rowid DESC LIMIT 1').get(nodeId, transcriptPath);
+    if (known !== undefined)
+        return known.binding_id;
+    const created = db.prepare('SELECT created FROM nodes WHERE node_id = ?').get(nodeId);
+    if (created === undefined)
+        throw new Error(`cannot import provider retry ${legacyPath}: unknown node ${nodeId}`);
+    const bindingId = randomUUID();
+    db.prepare('INSERT INTO sessions (binding_id, session_id, node_id, transcript_path, source_session_id, fork_entry_id, created_at, bound_at, unbound_at) VALUES (?, NULL, ?, ?, NULL, NULL, ?, ?, ?)').run(bindingId, nodeId, transcriptPath, created.created, created.created, now());
+    return bindingId;
+}
+/** Import legacy witness and recovery files inside the parent's one shared consolidation transaction. */
+export function migrateRecoveryWitnesses(db) {
+    addExecutionColumns(db);
+    createFaultTable(db);
+    if (!existsSync(nodesRoot()))
+        return;
+    const entries = readdirSync(nodesRoot(), { withFileTypes: true }).filter((entry) => entry.isDirectory()).sort((a, b) => a.name.localeCompare(b.name));
+    const nodeExists = db.prepare('SELECT 1 FROM nodes WHERE node_id = ?');
+    for (const entry of entries) {
+        const nodeId = entry.name;
+        const directory = jobDir(nodeId);
+        const busyPath = join(directory, 'busy');
+        const abortPath = join(directory, 'clean-abort');
+        const poisonPath = join(directory, 'transport-poison.json');
+        const faultPath = join(directory, 'fault');
+        const episodePath = join(directory, 'provider-retry');
+        const busy = existsSync(busyPath);
+        const aborted = existsSync(abortPath);
+        if (!busy && !aborted && !existsSync(poisonPath) && !existsSync(faultPath) && !existsSync(episodePath))
+            continue;
+        if (!existsSync(join(nodesRoot(), nodeId, 'meta.json')) && nodeExists.get(nodeId) === undefined)
+            continue;
+        const poison = legacyFile(poisonPath, false);
+        const legacyFault = legacyFile(faultPath, true);
+        const legacyEpisode = legacyFile(episodePath, true);
+        if (!busy && !aborted && poison === null && legacyFault === null && legacyEpisode === null)
+            continue;
+        let execution = latestExecution(db, nodeId);
+        if (execution === null)
+            execution = syntheticExecution(db, nodeId);
+        if (busy) {
+            const at = markerMtime(busyPath, execution.started_at);
+            db.prepare('UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ?').run(at, at, execution.execution_id);
+        }
+        else if (aborted) {
+            const at = markerMtime(abortPath, execution.started_at);
+            db.prepare("UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = ?, turn_end_reason = 'teardown-aborted' WHERE execution_id = ?").run(at, at, at, execution.execution_id);
+        }
+        if (poison !== null) {
+            try {
+                const parsed = JSON.parse(poison);
+                if (Number.isInteger(parsed.consecutivePasses) && parsed.consecutivePasses >= 1 && Array.isArray(parsed.routes) && parsed.routes.length > 0 && parsed.routes.every((route) => typeof route === 'string' && route !== '')) {
+                    db.prepare('UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ?').run(JSON.stringify({ schema: 'crtr.execution-exit-diagnostic/v1', transport_poison: { consecutive_passes: parsed.consecutivePasses, routes: parsed.routes } }), execution.execution_id);
+                }
+            }
+            catch {
+                // Diagnostic-only legacy context is intentionally omitted when corrupt.
+            }
+        }
+        const active = legacyFault === null ? null : parseLegacyFault(legacyFault);
+        if (legacyFault !== null && active === null)
+            throw new Error(`cannot import recovery fault ${faultPath}`);
+        const episode = legacyEpisode === null ? null : parseLegacyEpisode(legacyEpisode);
+        if (legacyEpisode !== null && episode === null)
+            throw new Error(`cannot import provider retry ${episodePath}`);
+        let activeStored = active === null || active.link === 'broker↔crtrd' ? null : insertMigratedFault(db, nodeId, active, true);
+        if (episode !== null) {
+            const bindingId = migrationBinding(db, nodeId, episode.sessionFile, episodePath);
+            if (activeStored !== null && activeStored.since === episode.fault.since) {
+                db.prepare('UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?').run(episode.state, bindingId, now(), activeStored.faultId);
+            }
+            else {
+                insertMigratedFault(db, nodeId, episode.fault, false, { state: episode.state, bindingId });
+            }
+        }
+    }
+}

package/dist/core/canvas/remote-canvas-source.js

@@ -216,6 +216,10 @@ export class RemoteCanvasSource {
             window: n.window ?? null,
             tmux_session: n.tmux_session ?? null,
             pane: n.pane ?? null,
+            // Recovery state projected by the REMOTE canvas's own daemon (stale-provider
+            // overlay already applied there), never a read of this host's files.
+            fault: n.fault ?? null,
+            streaming: n.streaming === true,
         };
     }
     async listNodes(filter) {