@north-light/api 0.3.580 → 0.3.588

package/dist/core/canvas/person.js

@@ -0,0 +1,244 @@
+// person.ts — the install's person (`user:<id>`), the one routine that replaces one person with another in every
+// known column, and the boot step that retires the legacy local person. Design: cycle 2 local identity.
+import { randomBytes } from 'node:crypto';
+import { CrtrError } from '../errors.js';
+import { PersonUnresolvedError, PersonOutsideRequestError, setPersonId, markCloudRole, isCloudRole, runAsPerson, personId, bootPersonId, requestPersonId } from './person-state.js';
+import { readConfig } from '../config.js';
+import { joinedIssuer, joinedOwner } from '../directory/join.js';
+import { installLocalTerminalGrant } from '../grants/local.js';
+import { dropLocalGrantMirror } from '../grants/mirror.js';
+import { ownerForGrantee, ensureOwner } from '../graph/objects.js';
+import { canvasDatabaseUrl, CanvasConstraintError, withCanvasWrite } from './db.js';
+import { LEGACY_LOCAL_PERSON, LEGACY_LOCAL_TERMINAL_GRANT } from './legacy-person.js';
+import { rebuildTotalsFor } from '../ledger/totals.js';
+export { LEGACY_LOCAL_PERSON };
+const PERSON_KEY = 'person_id';
+const RETIRED_KEY = 'local_person_retired';
+const PERSON_SHAPE = /^user:[A-Za-z0-9._-]+$/;
+const ALPHABET = 'abcdefghijklmnopqrstuvwxyz0123456789';
+export { PersonUnresolvedError, PersonOutsideRequestError, setPersonId, markCloudRole, isCloudRole, runAsPerson, personId };
+const lazyPersonByHome = new Map();
+/** As `personId()`, but a process that is not the daemon gets a lazy get-or-create keyed by the active canvas database. */
+export async function personIdAsync() {
+    const request = requestPersonId();
+    if (request !== undefined)
+        return request;
+    if (isCloudRole())
+        throw new PersonOutsideRequestError();
+    const boot = bootPersonId();
+    if (boot !== undefined)
+        return boot;
+    const home = await canvasDatabaseUrl();
+    const cached = lazyPersonByHome.get(home);
+    if (cached !== undefined)
+        return cached;
+    const resolved = await resolvePersonId();
+    if (resolved === null)
+        throw new PersonUnresolvedError();
+    lazyPersonByHome.set(home, resolved);
+    return resolved;
+}
+function mintPersonId() {
+    const bytes = randomBytes(12);
+    let out = 'user:';
+    for (let i = 0; i < 12; i++)
+        out += ALPHABET[bytes[i] % ALPHABET.length];
+    return out;
+}
+async function readMeta(db, key) {
+    const row = await db.prepare('SELECT value FROM canvas_meta WHERE key = ?').get(key);
+    return row?.value ?? null;
+}
+async function writeMeta(db, key, value) {
+    await db.prepare('INSERT INTO canvas_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);
+}
+/** Get-or-create the person and say where it came from. Order: stored, the join file's owner, the single `sub` of the
+ *  mirrored grants (a runtime with an issuer and no join file), else a minted id on a runtime without an issuer.
+ *  Null when an issuer runtime has no mirrored grants yet; nothing is stored then. */
+export async function resolvePerson() {
+    return await withCanvasWrite(async (db) => {
+        const stored = await readMeta(db, PERSON_KEY);
+        if (stored !== null)
+            return { id: stored, source: 'stored' };
+        const joined = joinedOwner();
+        if (joined !== null) {
+            await writeMeta(db, PERSON_KEY, joined);
+            return { id: joined, source: 'joined' };
+        }
+        if (readConfig('user').runtime?.issuer) {
+            const subs = await db.prepare('SELECT DISTINCT sub FROM grants').all();
+            if (subs.length !== 1)
+                return null;
+            await writeMeta(db, PERSON_KEY, subs[0].sub);
+            return { id: subs[0].sub, source: 'grants' };
+        }
+        const minted = mintPersonId();
+        await writeMeta(db, PERSON_KEY, minted);
+        return { id: minted, source: 'minted' };
+    });
+}
+export async function resolvePersonId() {
+    return (await resolvePerson())?.id ?? null;
+}
+const changes = (result) => Number(result.changes);
+/** Plain `UPDATE <table> SET <column> = to WHERE <column> = from` columns of the substitution. */
+const PLAIN_COLUMNS = [
+    ['nodes', 'sub'], ['node_tokens', 'sub'], ['grants', 'sub'], ['crons', 'sub'], ['audit_rows', 'sub'],
+    ['provider_approvals', 'sub'], ['audit_rows', 'grantee'], ['run_events', 'grantee'], ['node_tokens', 'grantee'],
+    ['uploads', 'grantee'], ['provider_approvals', 'grantee'],
+];
+/** The person stores (`002-fleet.sql`, "person stores"): every table whose rows belong to one person by a `person` column. */
+const PERSON_STORE_TABLES = [
+    'profiles', 'profile_defaults', 'scope_settings', 'scope_bookkeeping', 'scope_marketplaces', 'scope_plugins', 'person_settings',
+    'model_accounts', 'model_account_cooldowns', 'model_api_keys', 'model_settings', 'model_sign_in_flows', 'env_vars',
+];
+/** The ledger's person columns, renamed in place. */
+const LEDGER_COLUMNS = [
+    ['charges', 'sub'], ['charges', 'payer'], ['charges', 'grantee'], ['limits', 'payer'], ['limits', 'sub'], ['limits', 'grantee'],
+];
+function assertPerson(name, value) {
+    if (!PERSON_SHAPE.test(value)) {
+        throw new CrtrError('invalid_person', `${name} must be a user:<id> principal, got ${JSON.stringify(value)}`);
+    }
+}
+/** Whether a rollup row or a moved-month chunk still names `person` (a person whose every month was moved has no `charges` rows). */
+async function namesPerson(db, person) {
+    return await db.prepare('SELECT 1 AS found FROM charge_totals WHERE payer = ? OR sub = ? OR grantee = ? UNION ALL SELECT 1 FROM charge_chunks WHERE payer = ? LIMIT 1')
+        .get(person, person, person, person) !== undefined;
+}
+/** Replace person `from` with `to` in every known column, inside the caller's `withCanvasWrite` transaction. Idempotent.
+ *  Rows keyed by the principal merge on collision. Returns rows changed per column. */
+export async function substitutePerson(db, from, to) {
+    assertPerson('from', from);
+    assertPerson('to', to);
+    if (from === to)
+        throw new CrtrError('invalid_person', 'cannot substitute a person with itself');
+    const counts = {};
+    try {
+        counts['identities'] = changes(await db.prepare("UPDATE identities SET scheme = 'user', external_id = ? WHERE scheme IN ('local-user', 'user') AND external_id = ?").run(to, from));
+    }
+    catch (error) {
+        if (error instanceof CanvasConstraintError) {
+            throw new CrtrError('person_conflict', `another person object already holds ${to}; merging two person objects is not supported (${error.message})`, undefined, { from, to });
+        }
+        throw error;
+    }
+    counts['daemon_actions.principal_sub'] = changes(await db.prepare(`UPDATE daemon_actions SET adapter_json = jsonb_set(adapter_json::jsonb, '{principal,sub}', to_jsonb(?::text))::text
+      WHERE state IN ('pending', 'handed_off') AND adapter_json::jsonb ->> 'adapter' = 'cron_spawn'
+        AND adapter_json::jsonb #>> '{principal,sub}' = ?`).run(to, from));
+    for (const [table, column] of PLAIN_COLUMNS) {
+        counts[`${table}.${column}`] = changes(await db.prepareUncached(`UPDATE ${table} SET ${column} = ? WHERE ${column} = ?`).run(to, from));
+    }
+    // A key the new person already holds keeps its row; the old person's duplicate is dropped.
+    counts['idempotency_keys.grantee'] = changes(await db.prepare('DELETE FROM idempotency_keys WHERE grantee = ? AND key IN (SELECT key FROM idempotency_keys WHERE grantee = ?)').run(from, to))
+        + changes(await db.prepare('UPDATE idempotency_keys SET grantee = ? WHERE grantee = ?').run(to, from));
+    // Permission facts name a user by its bare id (`relations.subject_id`, subject_type `user`). A fact the new person already holds keeps its row;
+    // the old person's duplicate is dropped. Nothing the old person did not hold is granted. `written_by` and `revisions.acting_for` record who
+    // acted: the old person is the same human as the new one, so they move too, and no principal that no longer exists stays in the history.
+    const fromId = from.replace(/^user:/, '');
+    const toId = to.replace(/^user:/, '');
+    counts['relations.subject_id'] = changes(await db.prepare(`DELETE FROM relations r WHERE r.subject_type = 'user' AND r.subject_id = ?
+    AND EXISTS (SELECT 1 FROM relations n WHERE n.subject_type = 'user' AND n.subject_id = ? AND n.resource_type = r.resource_type
+      AND n.resource_id = r.resource_id AND n.relation = r.relation AND n.subject_relation = r.subject_relation)`).run(fromId, toId))
+        + changes(await db.prepare("UPDATE relations SET subject_id = ? WHERE subject_type = 'user' AND subject_id = ?").run(toId, fromId));
+    counts['relations.written_by'] = changes(await db.prepare('UPDATE relations SET written_by = ? WHERE written_by = ?').run(to, from));
+    counts['revisions.acting_for'] = changes(await db.prepare('UPDATE revisions SET acting_for = ? WHERE acting_for = ?').run(to, from));
+    // The ledger renames the person in place. The `charge_totals` rows (and `charge_chunks` rows of months `ledger-r2` moved out of
+    // `charges`) that name the old person are renamed too, and two rollup rows that now share a key are summed once.
+    for (const [table, column] of LEDGER_COLUMNS) {
+        counts[`${table}.${column}`] = changes(await db.prepareUncached(`UPDATE ${table} SET ${column} = ? WHERE ${column} = ?`).run(to, from));
+    }
+    // The person's stores move with it. A person that already holds rows in a table keeps them; the old person's rows in that table are dropped.
+    for (const table of PERSON_STORE_TABLES) {
+        const dropped = changes(await db.prepareUncached(`DELETE FROM ${table} WHERE person = ? AND EXISTS (SELECT 1 FROM ${table} WHERE person = ?)`).run(from, to));
+        const moved = dropped + changes(await db.prepareUncached(`UPDATE ${table} SET person = ? WHERE person = ?`).run(to, from));
+        if (moved !== 0)
+            counts[`${table}.person`] = moved;
+    }
+    if (Object.entries(counts).some(([key, n]) => key.startsWith('charges.') && n !== 0) || await namesPerson(db, from)) {
+        await rebuildTotalsFor(db, from, to);
+        counts['charge_totals.rebuilt'] = 1;
+    }
+    return counts;
+}
+/** Flow 2 part B step 1: the legacy local grantee becomes the terminal app in nodes (the node object's owner), node_tokens,
+ *  run_events and idempotency_keys. Only a node with a local `nodes` row moves: a peer stub of a remote watcher is owned by
+ *  the person on purpose (`ensureNodeStub`). */
+async function rewriteLegacyGrantees(db) {
+    const localNodes = `SELECT o.id FROM objects o JOIN identities i ON i.object = o.owner JOIN nodes n ON n.node_id = o.id
+    WHERE o.type = 'node' AND i.scheme = 'local-user' AND i.external_id = '${LEGACY_LOCAL_PERSON}'`;
+    const moved = await db.prepareUncached(`${localNodes} LIMIT 1`).get() !== undefined
+        ? changes(await db.prepareUncached(`UPDATE objects SET owner = ? WHERE id IN (${localNodes})`).run((await ownerForGrantee(db, 'app:terminal')).id))
+        : 0;
+    return moved
+        + changes(await db.prepare("UPDATE node_tokens SET grantee = 'app:terminal' WHERE grantee = ?").run(LEGACY_LOCAL_PERSON))
+        + changes(await db.prepare("UPDATE run_events SET grantee = 'app:terminal' WHERE grantee = ?").run(LEGACY_LOCAL_PERSON))
+        // A key the terminal app already holds keeps its row; the legacy duplicate is dropped (PK (grantee, key)).
+        + changes(await db.prepare("DELETE FROM idempotency_keys WHERE grantee = ? AND key IN (SELECT key FROM idempotency_keys WHERE grantee = 'app:terminal')").run(LEGACY_LOCAL_PERSON))
+        + changes(await db.prepare("UPDATE idempotency_keys SET grantee = 'app:terminal' WHERE grantee = ?").run(LEGACY_LOCAL_PERSON));
+}
+/** Flow 2 part B step 3: grant ids that no longer exist (the legacy local terminal grant, or `terminal:<install>` on a runtime that joined) point at
+ *  `target` (this install's terminal grant) or at nothing. */
+async function repointGrantReferences(db, installId, target) {
+    const stale = [LEGACY_LOCAL_TERMINAL_GRANT, `terminal:${installId}`].filter((id) => id !== target);
+    const marks = stale.map(() => '?').join(', ');
+    const out = {};
+    out['nodes.grant_id'] = changes(await db.prepareUncached(`UPDATE nodes SET grant_id = ? WHERE grant_id IN (${marks})`).run(target, ...stale));
+    out['node_tokens.grant'] = changes(await db.prepareUncached(`UPDATE node_tokens SET "grant" = ? WHERE "grant" IN (${marks})`).run(target, ...stale));
+    out['crons.grant_id'] = changes(await db.prepareUncached(`UPDATE crons SET grant_id = ? WHERE grant_id IN (${marks})`).run(target, ...stale));
+    const value = target === null ? "'null'::jsonb" : 'to_jsonb(?::text)';
+    out['daemon_actions.principal_grant_id'] = changes(await db.prepareUncached(`UPDATE daemon_actions SET adapter_json = jsonb_set(adapter_json::jsonb, '{principal,grant_id}', ${value})::text
+      WHERE state IN ('pending', 'handed_off') AND adapter_json::jsonb ->> 'adapter' = 'cron_spawn'
+        AND adapter_json::jsonb #>> '{principal,grant_id}' IN (${marks})`).run(...(target === null ? [] : [target]), ...stale));
+    return out;
+}
+/** The boot step (flow 2), one transaction. Part A applies a join that landed after the person was minted; part B retires the
+ *  legacy local person once per person; part C installs the local terminal grant on a runtime without an issuer. */
+export async function retireLocalPerson() {
+    const result = await withCanvasWrite(async (db) => {
+        const columns = {};
+        const add = (counts, prefix = '') => { for (const [k, v] of Object.entries(counts))
+            if (v !== 0)
+                columns[prefix + k] = (columns[prefix + k] ?? 0) + v; };
+        let person = await readMeta(db, PERSON_KEY);
+        if (person === null)
+            throw new PersonUnresolvedError();
+        let joinedFrom = null;
+        const owner = joinedOwner();
+        if (owner !== null && owner !== person) {
+            add(await substitutePerson(db, person, owner), 'join.');
+            await writeMeta(db, PERSON_KEY, owner);
+            joinedFrom = person;
+            person = owner;
+        }
+        // The join file's issuer, not the settings': the daemon applies the joined settings after this step.
+        const issuer = joinedIssuer() ?? readConfig('user').runtime?.issuer;
+        const installId = await readMeta(db, 'install_id');
+        if (installId === null)
+            throw new Error('retireLocalPerson: the install id is not stored');
+        const retired = await readMeta(db, RETIRED_KEY) !== person;
+        if (retired) {
+            add({ legacy_grantees: await rewriteLegacyGrantees(db) });
+            add(await substitutePerson(db, LEGACY_LOCAL_PERSON, person));
+            // A node stub that receives events with no app identity (an earlier start moved it to the terminal app) belongs to the person.
+            const damaged = `SELECT o.id FROM objects o JOIN peer_stubs s ON s.object = o.id JOIN identities i ON i.object = o.owner
+        WHERE o.type = 'node' AND o.reception = 2 AND s.envelope_json ->> 'app' IS NULL AND i.scheme = 'app' AND i.external_id = 'app:terminal'`;
+            if (await db.prepareUncached(`${damaged} LIMIT 1`).get() !== undefined) {
+                const personObject = await ensureOwner(db, 'person', 'user', person, 'user');
+                add({ repaired_stubs: changes(await db.prepareUncached(`UPDATE objects SET owner = ? WHERE id IN (${damaged})`).run(personObject.id)) });
+            }
+            add({ 'nodes.sub_null_terminal': changes(await db.prepare(`UPDATE nodes SET sub = ? WHERE sub IS NULL AND node_id IN (
+        SELECT n.node_id FROM nodes n JOIN identities i ON i.object = n.app
+        WHERE i.scheme = 'app' AND i.external_id = 'app:terminal')`).run(person)) });
+            add(await repointGrantReferences(db, installId, issuer ? null : `terminal:${installId}`));
+            if (issuer)
+                add({ local_grant_mirror: await dropLocalGrantMirror(db) });
+            await writeMeta(db, RETIRED_KEY, person);
+        }
+        if (!issuer)
+            await installLocalTerminalGrant(installId, person);
+        return { person, joinedFrom, retired, columns };
+    });
+    setPersonId(result.person);
+    return result;
+}

package/dist/core/canvas/pid.d.ts

@@ -153,6 +153,8 @@ export declare function composeIdentity(pid: number, lstart: string, deps?: {
  *  probe itself failed; callers preserve that as unknown rather than absence. */
 export interface PsLivenessProcess {
     zombie: boolean;
+    /** The process group the process is in. */
+    pgid: number;
     identity: string;
     command: string;
 }
@@ -173,6 +175,7 @@ interface ProcStat {
     comm: string;
     state: string;
     ppid: number;
+    pgid: number;
     starttime: string;
 }
 export declare function parseProcStat(raw: string): ProcStat | null;
@@ -233,6 +236,9 @@ export declare const procfsBackendForTest: {
  *  effort: returns `[]` (never throws) on any `ps` failure, exactly like
  *  `killProcessGroup`'s ESRCH-swallowing. */
 export declare function descendantPids(rootPid: number): number[];
+/** The pids of the live (not zombie) processes in process group `pgid`, from one process-table sample. `null` when the table cannot be read: an unreadable
+ *  table is unknown, never an empty group. */
+export declare function pidsInProcessGroup(pgid: number): number[] | null;
 /** Capture a portable process-identity fingerprint — `ps`'s `lstart` column,
  *  the process's full wall-clock start time — for each of `pids`, in ONE
  *  batched `ps -p <list>` call. `lstart` is a standard format keyword on BOTH

package/dist/core/canvas/pid.js

@@ -352,7 +352,7 @@ export function composeIdentity(pid, lstart, deps = {}) {
 }
 /** The `ps` invocation both snapshot captures share, so the synchronous and
  *  asynchronous forms can never sample different columns. */
-const PS_SNAPSHOT_ARGS = ['-Ao', 'pid=,ppid=,stat=,lstart=,command='];
+const PS_SNAPSHOT_ARGS = ['-Ao', 'pid=,ppid=,pgid=,stat=,lstart=,command='];
 const PS_SNAPSHOT_TIMEOUT_MS = 2000;
 export function captureLivenessSnapshot() {
     if (procfsUsable())
@@ -419,15 +419,16 @@ export function parseProcStat(raw) {
     if (open === -1 || close === -1 || close < open)
         return null;
     const rest = raw.slice(close + 2).trim().split(/\s+/);
-    // state(0) ppid(1) ... starttime(19) — proc(5) fields 3, 4 and 22.
+    // state(0) ppid(1) pgrp(2) ... starttime(19) — proc(5) fields 3, 4 and 22.
     const state = rest[0];
     const ppid = Number(rest[1]);
+    const pgid = Number(rest[2]);
     const starttime = rest[19];
-    if (state === undefined || state === '' || !Number.isInteger(ppid) || ppid < 0)
+    if (state === undefined || state === '' || !Number.isInteger(ppid) || ppid < 0 || !Number.isInteger(pgid) || pgid < 0)
         return null;
     if (starttime === undefined || !/^\d+$/.test(starttime))
         return null;
-    return { comm: raw.slice(open + 1, close), state, ppid, starttime };
+    return { comm: raw.slice(open + 1, close), state, ppid, pgid, starttime };
 }
 /** `/proc/<pid>/cmdline` (NUL-separated argv) as the single space-joined line
  *  `ps -o command=` prints; kernel threads and zombies, whose cmdline is
@@ -543,7 +544,7 @@ function addProcfsRow(src, rows, pid, stat, rawCmdline) {
     const identity = procIdentity(stat, src);
     if (identity === null)
         return false;
-    rows.processes.set(pid, { zombie: stat.state === 'Z' || stat.state === 'X', identity, command: procCommandLine(rawCmdline, stat.comm) });
+    rows.processes.set(pid, { zombie: stat.state === 'Z' || stat.state === 'X', pgid: stat.pgid, identity, command: procCommandLine(rawCmdline, stat.comm) });
     const siblings = rows.childrenOf.get(stat.ppid);
     if (siblings === undefined)
         rows.childrenOf.set(stat.ppid, [pid]);
@@ -675,17 +676,18 @@ function parseLivenessTable(stdout) {
         if (trimmed === '')
             continue;
         // lstart is five whitespace-delimited fields on BSD ps and procps-ng.
-        const m = /^(\d+)\s+(\d+)\s+(\S+)\s+(\S+(?:\s+\S+){4})\s+(.+)$/.exec(trimmed);
+        const m = /^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+(?:\s+\S+){4})\s+(.+)$/.exec(trimmed);
         if (m === null)
             continue;
         const pid = Number(m[1]);
         const ppid = Number(m[2]);
-        const stat = m[3];
-        const lstart = m[4];
-        const command = m[5].trim();
+        const pgid = Number(m[3]);
+        const stat = m[4];
+        const lstart = m[5];
+        const command = m[6].trim();
         if (!Number.isInteger(pid) || pid <= 0 || !Number.isInteger(ppid) || ppid < 0 || command === '')
             continue;
-        processes.set(pid, { zombie: stat.startsWith('Z'), identity: composeIdentity(pid, lstart), command });
+        processes.set(pid, { zombie: stat.startsWith('Z'), pgid, identity: composeIdentity(pid, lstart), command });
         const siblings = childrenOf.get(ppid);
         if (siblings === undefined)
             childrenOf.set(ppid, [pid]);
@@ -735,6 +737,14 @@ export function descendantPids(rootPid) {
         return [];
     return descendantPidsFrom(rootPid, table.childrenOf);
 }
+/** The pids of the live (not zombie) processes in process group `pgid`, from one process-table sample. `null` when the table cannot be read: an unreadable
+ *  table is unknown, never an empty group. */
+export function pidsInProcessGroup(pgid) {
+    const table = captureLivenessSnapshot();
+    if (table === null)
+        return null;
+    return [...table.processes].filter(([, process]) => process.pgid === pgid && !process.zombie).map(([pid]) => pid);
+}
 /** Capture a portable process-identity fingerprint — `ps`'s `lstart` column,
  *  the process's full wall-clock start time — for each of `pids`, in ONE
  *  batched `ps -p <list>` call. `lstart` is a standard format keyword on BOTH

package/dist/core/canvas/reconciler-leases.d.ts

@@ -0,0 +1,12 @@
+/** A claim owner is a fresh token for each attempt, not a process id: a timed-out worker cannot renew a successor's claim. */
+export interface ReconcilerLease {
+    cursor: string | null;
+}
+/** Claim an unowned or expired lane/key; retain its cursor across owner changes. */
+export declare function claimReconcilerLease(lane: string, key: string, owner: string, ttlMs: number): Promise<ReconcilerLease | undefined>;
+/** Only the still-current claim may extend its lifetime. */
+export declare function renewReconcilerLease(lane: string, key: string, owner: string, ttlMs: number): Promise<boolean>;
+/** An expired owner cannot advance the cursor. Cursor changes and row work that require exact-once must share a transaction. */
+export declare function advanceReconcilerCursor(lane: string, key: string, owner: string, cursor: string | null): Promise<boolean>;
+/** Release only the current claim, preserving its cursor for the next worker. */
+export declare function releaseReconcilerLease(lane: string, key: string, owner: string): Promise<boolean>;

package/dist/core/canvas/reconciler-leases.js

@@ -0,0 +1,40 @@
+import { openDb } from './db.js';
+function expiry(ttlMs) {
+    if (!Number.isFinite(ttlMs) || ttlMs <= 0)
+        throw new Error('reconciler lease duration must be positive');
+    return new Date(Date.now() + ttlMs).toISOString();
+}
+/** Claim an unowned or expired lane/key; retain its cursor across owner changes. */
+export async function claimReconcilerLease(lane, key, owner, ttlMs) {
+    const until = expiry(ttlMs);
+    const now = new Date().toISOString();
+    const db = await openDb();
+    return db.prepare(`INSERT INTO reconciler_leases (lane, key, owner, expires_at)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT (lane, key) DO UPDATE SET owner = EXCLUDED.owner, expires_at = EXCLUDED.expires_at
+      WHERE reconciler_leases.expires_at <= ?
+    RETURNING cursor`).get(lane, key, owner, until, now);
+}
+/** Only the still-current claim may extend its lifetime. */
+export async function renewReconcilerLease(lane, key, owner, ttlMs) {
+    const until = expiry(ttlMs);
+    const now = new Date().toISOString();
+    const db = await openDb();
+    const result = await db.prepare(`UPDATE reconciler_leases SET expires_at = ?
+    WHERE lane = ? AND key = ? AND owner = ? AND expires_at > ?`).run(until, lane, key, owner, now);
+    return result.changes === 1;
+}
+/** An expired owner cannot advance the cursor. Cursor changes and row work that require exact-once must share a transaction. */
+export async function advanceReconcilerCursor(lane, key, owner, cursor) {
+    const db = await openDb();
+    const result = await db.prepare(`UPDATE reconciler_leases SET cursor = ?
+    WHERE lane = ? AND key = ? AND owner = ? AND expires_at > ?`).run(cursor, lane, key, owner, new Date().toISOString());
+    return result.changes === 1;
+}
+/** Release only the current claim, preserving its cursor for the next worker. */
+export async function releaseReconcilerLease(lane, key, owner) {
+    const db = await openDb();
+    const result = await db.prepare(`UPDATE reconciler_leases SET expires_at = ?
+    WHERE lane = ? AND key = ? AND owner = ? AND expires_at > ?`).run(new Date(0).toISOString(), lane, key, owner, new Date().toISOString());
+    return result.changes === 1;
+}

package/dist/core/canvas/recovery.d.ts

@@ -0,0 +1,127 @@
+import type { DatabaseSync } from 'node:sqlite';
+import { type Fault, type FaultKind, type FaultLink, type FaultProviderError, type FaultRetry } from '../fault-classifier.js';
+import type { ExecutionExitDiagnosticDTO, ExecutionLaunchFailureDiagnostic } from '../../api/dto/recovery.js';
+export type { Fault, FaultKind, FaultLink, FaultProviderError, FaultRetry } from '../fault-classifier.js';
+export declare const FAULT_QUIET_MS: number;
+export interface FaultRecordInput {
+    link: FaultLink;
+    op: string;
+    kind: FaultKind;
+    retry: FaultRetry;
+    message: string;
+    operation_id?: string;
+    since?: string;
+    anchorEntryId?: string;
+    providerError?: FaultProviderError;
+}
+export interface StoredFault extends Fault {
+    faultId: string;
+    /** The episode start (`since`) a fatal-fault notice was last published for; unset until one is sent. */
+    notifiedSince?: string;
+}
+export interface ProviderRetryEpisode {
+    state: 'pending' | 'admitted' | 'invalidated';
+    sessionId: string | null;
+    fault: StoredFault;
+}
+export interface TransportPoisonVerdict {
+    consecutivePasses: number;
+    routes: string[];
+}
+export interface BootFaultAttempt {
+    explanation(): Promise<string | null>;
+}
+export declare function parseProviderError(value: unknown): FaultProviderError | undefined;
+export declare function hydrateFault(row: Record<string, unknown>): StoredFault;
+export declare function openTurn(executionId: string, _opts: {
+    reason: 'agent-start' | 'prompt-unsettled';
+}): Promise<boolean>;
+export declare function recordTurnProgress(executionId: string): Promise<boolean>;
+/** Closes the open turn. A `settled` close also stamps the node's `reply_at` (the dashboard's unread marker) in the same
+ *  write, and `read_at` too when a viewer was shown the conversation while the turn ended. A close that changes no row writes neither. */
+export declare function closeTurn(executionId: string, opts: {
+    endReason: 'settled' | 'teardown-aborted';
+    viewerPresent?: boolean;
+}): Promise<boolean>;
+/** Stamps `read_at`: a viewer has just been shown this conversation. */
+export declare function markNodeRead(nodeId: string): Promise<boolean>;
+/** The one writer of an `executions.exit_diagnostic_json` envelope: the schema tag plus one payload the DTOs declare. */
+export declare function exitDiagnosticJson(payload: Omit<ExecutionExitDiagnosticDTO, 'schema'> | Omit<ExecutionLaunchFailureDiagnostic, 'schema'>): string;
+export declare function recordTransportPoisonVerdict(executionId: string, verdict: TransportPoisonVerdict): Promise<boolean>;
+export declare function readTransportPoisonVerdict(nodeId: string): Promise<TransportPoisonVerdict | null>;
+export declare function readLiveTurn(nodeId: string): Promise<{
+    busy: boolean;
+    progressAt: string | null;
+} | null>;
+export declare function readInterruptedTurn(nodeId: string): Promise<{
+    interrupted: boolean;
+    progressAt: string | null;
+} | null>;
+/** Whether any execution of this node ever admitted a turn (agent_start recorded
+ *  by the daemon). This, not the transcript, is the durable fact that a node's
+ *  birth task was delivered: a session file holding only the bearings message
+ *  exists before the kickoff is admitted. */
+export declare function hasAdmittedTurn(nodeId: string): Promise<boolean>;
+export declare function recordFault(nodeId: string, input: FaultRecordInput): Promise<StoredFault | null>;
+/** Record that the fatal-fault notice for this fault row's current episode has been published. Callers run it in the write that publishes. */
+export declare function markFaultNotified(faultId: string): Promise<boolean>;
+/** Nodes whose active fatal provider fault has not been announced yet, read from the fault table so a node with no live broker is still found. */
+export declare function listUnnotifiedFatalFaultNodes(): Promise<string[]>;
+/** The daemon registers the one function that publishes a fatal provider fault's notice. Every clear of an unannounced fatal fault runs it first, so no clear can drop a notice. */
+export declare function setFatalFaultAnnouncer(announce: ((nodeId: string) => Promise<void>) | null): void;
+export declare function clearFault(nodeId: string, opts?: {
+    link?: FaultLink;
+    preserveEpisode?: boolean;
+}): Promise<boolean>;
+export declare function readFault(nodeId: string): Promise<StoredFault | null>;
+export declare function recordPendingEpisode(nodeId: string, input: {
+    sessionId: string;
+    input: FaultRecordInput;
+}): Promise<StoredFault | null>;
+export declare function admitEpisode(nodeId: string, input: {
+    sessionId: string;
+    faultId: string;
+    retry: FaultRetry;
+}): Promise<boolean>;
+/** Atomically settle only the admitted attempt that rejected its recovery prompt. A newer turn or episode must never be overwritten by its late result. */
+export declare function settleRejectedEpisode(nodeId: string, input: {
+    sessionId: string;
+    faultId: string;
+    fault: FaultRecordInput;
+    reopen: boolean;
+}): Promise<boolean>;
+export declare function reopenEpisode(nodeId: string, input: {
+    sessionId: string;
+}): Promise<boolean>;
+export declare function invalidateEpisode(nodeId: string, input: {
+    sessionId: string;
+}): Promise<boolean>;
+export declare function clearEpisode(nodeId: string, input: {
+    sessionId: string;
+}): Promise<boolean>;
+export declare function readEpisode(nodeId: string): Promise<ProviderRetryEpisode | null>;
+/** One active automatic provider-retry fault whose `pending` episode is bound to
+ *  its node's live session — the shape a broker's in-process retry timer owns. */
+export interface PendingProviderRetry {
+    nodeId: string;
+    sessionId: string;
+    fault: StoredFault;
+    /** When the fault row last changed (ISO) — the daemon's anchor for "due". */
+    updatedAt: string;
+    /** When a reconciler last claimed this fault for a nudge (ISO), or null. */
+    nudgedAt: string | null;
+}
+/** Every node with an active daemon-retried provider fault (connection,
+ *  rate-limit, overloaded) and a pending episode on its live session binding.
+ *  The daemon backstop reads this to find retries whose broker timer died. */
+export declare function listPendingProviderRetries(): Promise<PendingProviderRetry[]>;
+/** Claim one nudge of a fault: only the caller whose conditional UPDATE changes the row may send it, and a later claim waits out `graceMs`. */
+export declare function claimFaultNudge(faultId: string, nowMs: number, graceMs: number): Promise<boolean>;
+export declare function beginBootFaultAttempt(nodeId: string): Promise<BootFaultAttempt>;
+/** Import legacy witness and recovery files inside the parent's one shared consolidation transaction. */
+/** v98 — `notified_since` on a fault row: the episode start its fatal notice was published for.
+ *  No backfill: a fatal fault already active at upgrade reads as unannounced and is announced once more,
+ *  because the old in-memory record cannot be rebuilt and a backfill could silence a fault never announced.
+ *  Guarded like v95: a fixture that starts past v39 never created the table. */
+export declare function addFaultNotifiedSince(db: DatabaseSync): void;
+export declare function migrateRecoveryWitnesses(db: DatabaseSync): void;