@north-light/cli 0.0.0 → 0.3.413

package/dist/core/canvas/pid.d.ts

@@ -0,0 +1,391 @@
+type PsProcessState = 'zombie' | 'gone' | 'live' | 'unknown';
+/** True if a process with `pid` is currently alive. `kill(pid, 0)` provides the
+ *  initial existence check; EPERM means the foreign process exists. For our
+ *  processes, the later `ps` read is authoritative because it also distinguishes
+ *  zombies and processes reaped between the two probes. Only an unavailable
+ *  `ps` probe fails open to alive. A null/undefined pid reads dead. */
+export declare function isPidAlive(pid: number | null | undefined): boolean;
+/** Existence-only liveness for READ-ONLY display projections: the signal-0
+ *  probe alone, no `ps`. A zombie reads alive here, which a dashboard column
+ *  tolerates and which no lifecycle decision may rely on — revive, teardown,
+ *  and every other authority stays on `isPidAlive`. Exists because the
+ *  dashboard snapshot runs this per live row per request; a `ps` spawn each
+ *  time was a third of daemon CPU on a large canvas. */
+export declare function isPidPresent(pid: number | null | undefined): boolean;
+/** THE single precision-aware identity compare, used EVERYWHERE two
+ *  `composeIdentity` fingerprints are compared (this module's own
+ *  `recordedPidLiveness`, `captureTeardownSnapshot`'s reuse check, and
+ *  `killProcessTreePids`'s reuse check) — centralized so no call site can
+ *  drift from this rule.
+ *
+ *  Two hazards this closes, both in the false-DEAD direction (the dangerous
+ *  one — a false DEAD lets `reviveNode`'s double-launch guard relaunch a
+ *  SECOND broker onto a still-live session):
+ *
+ *  1. `composeIdentity` appends the `#<ticks>` Linux discriminator only when
+ *     `/proc/<pid>/stat` happens to be readable at capture time — so the SAME
+ *     live process can be recorded plain `lstart` on one capture and
+ *     `lstart#ticks` on another (a transient `/proc` read failure, or
+ *     capturing on a platform/container without procfs). A naive `===` reads
+ *     that as a reuse (different string) and reports `dead` for a broker that
+ *     never died.
+ *  2. `lstart` is NOT stable for a live process on every host. It is derived
+ *     from `/proc/stat` btime (wall-clock boot epoch) + starttime jiffies, so
+ *     a clock correction / suspend-resume shifts btime and re-anchors the
+ *     SAME live process's `lstart` by seconds between two captures. Observed
+ *     on Blaxel unikraft guests: pid+ticks identical across probes, `lstart`
+ *     drifting +81s — a base-first compare then reads the live home-node
+ *     broker as reused/dead every cycle, driving an uncapped revive loop.
+ *
+ *  A third hazard the base itself once carried: `ticks` (jiffies-since-boot)
+ *  RESET on every boot and can repeat across boots, so ticks-alone is only
+ *  safe WITHIN one boot. `composeIdentity` therefore boot-SCOPES the base on
+ *  Linux-with-procfs — the base IS the kernel `boot_id` UUID — so a
+ *  new-format identity is `<bootId>#<ticks>` and equality means same boot AND
+ *  same process. This closes the cross-boot false match at the root, without
+ *  plumbing boot reconciliation ahead of every caller.
+ *
+ *  Rule: split both on `#`.
+ *   - BOTH sides carry `#ticks` AND both bases are `boot_id` UUIDs
+ *     (new-format vs new-format): match iff base AND ticks are equal. A
+ *     different boot_id with equal ticks is NOT a match (the cross-boot
+ *     collision).
+ *   - BOTH sides carry `#ticks` but a base is NOT a boot_id UUID on EITHER
+ *     side — a LEGACY `lstart#ticks` row recorded before this repo boot-scoped
+ *     the base, or a Linux host that has procfs ticks but no readable boot_id:
+ *     ticks DECIDE it alone (the prior landed semantics). This is a NARROW
+ *     format-migration path so an in-place upgrade never false-kills a live
+ *     broker whose row still holds `lstart#ticks`; it is NOT a general
+ *     fallback — new-format vs new-format always demands base+ticks above.
+ *   - EITHER side lacks a valid `#ticks` suffix (macOS / no procfs / transient
+ *     `/proc` miss / malformed suffix): fall back to the coarse base —
+ *     matching iff the bases agree (fail-open to same-process, matching this
+ *     module's universal discipline: only a POSITIVE mismatch counts as reuse,
+ *     never an absence of evidence). */
+export declare function identitiesMatch(a: string, b: string): boolean;
+/** Three-valued liveness verdict for a RECORDED node pid, identity-guarded
+ *  against PID REUSE using the launch-time baseline (`pi_pid_identity`,
+ *  captured by `recordPid`). On a heavily-forking host — esp. Linux, where
+ *  low pids recycle fast — a broker's recorded pid can be reused by an
+ *  unrelated process (or the `ps` probe confirming it can itself
+ *  intermittently fail) within milliseconds of the broker dying; collapsing
+ *  "confirmed alive", "confirmed dead/reused", and "couldn't confirm either
+ *  way" into one boolean forces every caller to fail open to ALIVE on the
+ *  third case — which, upstream in the daemon's grace-clock bookkeeping, is
+ *  worse than misreporting one tick: a single such read WIPES the daemon's
+ *  memory that the node had already been observed dead, restarting the whole
+ *  grace window from scratch (the confirmed Round-2 crash-revive wedge, in
+ *  the pre-fleet daemon's liveness bookkeeping). Exposing the third state lets callers
+ *  decide for themselves whether "can't confirm" should act like alive or
+ *  dead for their own direction of risk.
+ *
+ *  Without a snapshot this preserves the historical per-pid probes. With a
+ *  snapshot it classifies process presence, zombie state, and identity from
+ *  that one authoritative table read:
+ *    - absent or zombie pid → `'dead'`. Load-bearing: a SIGKILLed detached
+ *      broker awaiting reap must still be revived rather than read as live.
+ *    - `expectedIdentity == null` (legacy row / launch capture failed) →
+ *      `'alive'` (preserve pre-identity behavior — no baseline to check
+ *      against).
+ *    - `ps` probe itself failed (`capturePidIdentities` → `null`) →
+ *      `'indeterminate'`. THIS is the branch the Round-2 fix changes: it used
+ *      to fail open to `true` (alive) here, which is the confirmed root cause
+ *      of the wedge.
+ *    - probe succeeded but has no row for this pid (alive per signal-0 a
+ *      moment ago, gone now) → `'dead'`.
+ *    - probe succeeded and the identity is present → `'alive'` iff
+ *      `identitiesMatch` says so, else `'dead'` (positively reused). */
+export type RecordedPidLiveness = 'alive' | 'dead' | 'indeterminate';
+export declare function recordedPidLiveness(pid: number | null | undefined, expectedIdentity: string | null | undefined, snapshot?: PsLivenessSnapshot | null): RecordedPidLiveness;
+/** SIGTERM a process GROUP by pid (the negative-pid convention) — best-effort,
+ *  swallowing ESRCH (already gone). Used by cron cancellation, timeout, and
+ *  stale-lease recovery. Lives beside isPidAlive: canvas/ is
+ *  the lowest shared layer every process-liveness/-teardown primitive sits at.
+ *  Guards `pid` before signaling: never hit pid 0
+ *  (which means "my own group") or a negative/non-integer value. */
+export declare function killProcessGroup(pid: number, signal?: NodeJS.Signals): void;
+/** The kernel `boot_id` (`/proc/sys/kernel/random/boot_id`) is stable for the
+ *  whole lifetime of this process — it only changes on a kernel boot, which
+ *  necessarily kills this process — so once a real value is read it is cached
+ *  for good, rather than re-read for every row of a whole-process-table `ps`
+ *  scan. Only SUCCESSFUL reads are cached forever: a `null` read is negative
+ *  for `BOOT_ID_NEGATIVE_TTL_MS`, after which the next call retries — so a
+ *  transient `/proc` miss on a boot_id-capable host still recovers (never
+ *  permanently downgraded to legacy `lstart#ticks`), while a genuinely
+ *  no-boot_id host is not hammered with a synchronous throwing read on every
+ *  identity composition. `now` is injectable for tests. */
+export declare function makeBootIdCache(read: () => string | null, now?: () => number, negativeTtlMs?: number): () => string | null;
+/** Is `identity` a LEGACY (pre-boot-scoped) baseline — a `<lstart>#<ticks>` row
+ *  whose base is NOT a boot_id UUID but which DOES carry ticks? These are the
+ *  rows the one-time startup migration re-records once this platform can
+ *  compose boot-scoped identities (see `migrateLegacyPidIdentities`). */
+export declare function isLegacyTicksIdentity(identity: string): boolean;
+/** Is `identity` a NEW-format boot-scoped baseline — `<bootId>#<ticks>` with a
+ *  real boot_id UUID base AND ticks? The startup migration only re-records a
+ *  legacy row when the pid's CURRENT identity comes out in this shape (i.e. the
+ *  platform now composes new-format), never otherwise. */
+export declare function isBootScopedIdentity(identity: string): boolean;
+/** Compose the stable, BOOT-SCOPED process-identity fingerprint used
+ *  everywhere in this module. ALWAYS used to build an identity string, never a
+ *  raw field directly, so every identity captured anywhere (launch-time
+ *  `recordPid`, the teardown snapshot, and the escalation-window re-check) is
+ *  comparable apples-to-apples.
+ *
+ *  Platform capability, not layered fallbacks — the base is the finest STABLE
+ *  per-boot anchor available:
+ *   - Linux-with-procfs (the production guest shape): `<bootId>#<ticks>` — the
+ *     per-boot `boot_id` UUID plus the per-process jiffies-since-boot. Ticks
+ *     reset per boot and can repeat, so scoping the base to the boot is what
+ *     makes an equal-ticks compare safe ACROSS boots (see `identitiesMatch`).
+ *   - Linux-with-procfs but no readable `boot_id` (if that combination
+ *     exists): `<lstart>#<ticks>` — the prior landed behavior.
+ *   - No procfs ticks (macOS / containers without procfs): `<lstart>` alone —
+ *     unchanged coarse behavior. The base is only ever a `boot_id` when a
+ *     `ticks` discriminator is ALSO present; a bare `boot_id` (shared by every
+ *     process on the boot) would be a useless per-process identity. */
+export declare function composeIdentity(pid: number, lstart: string, deps?: {
+    ticks?: string | null;
+    bootId?: string | null;
+}): string;
+/** One authoritative process-table sample. Supervision, broker census,
+ *  teardown tree walks, identity guards, and zombie detection can all classify
+ *  from this one read instead of spawning a `ps` probe per pid. `null` means the
+ *  probe itself failed; callers preserve that as unknown rather than absence. */
+export interface PsLivenessProcess {
+    zombie: boolean;
+    identity: string;
+    command: string;
+}
+export interface PsLivenessSnapshot {
+    processes: ReadonlyMap<number, PsLivenessProcess>;
+    childrenOf: ReadonlyMap<number, number[]>;
+}
+export declare function captureLivenessSnapshot(): PsLivenessSnapshot | null;
+/** The same sample without blocking the event loop, for callers running inside
+ *  crtrd's detached work: one `ps` read classifies every pid they care about,
+ *  where a per-pid `spawnSync` probe would stall broker supervision and the API
+ *  socket for the duration of each probe. `null` means the probe failed, which
+ *  every caller must preserve as unknown rather than absence. */
+export declare function captureLivenessSnapshotAsync(): Promise<PsLivenessSnapshot | null>;
+/** Parsed `/proc/<pid>/stat`: `comm` is parenthesized and may itself contain
+ *  spaces/parens, so fields are counted after the LAST `)`. */
+interface ProcStat {
+    comm: string;
+    state: string;
+    ppid: number;
+    starttime: string;
+}
+export declare function parseProcStat(raw: string): ProcStat | null;
+/** `/proc/<pid>/cmdline` (NUL-separated argv) as the single space-joined line
+ *  `ps -o command=` prints; kernel threads and zombies, whose cmdline is
+ *  empty, show as `[comm]` exactly as `ps` renders them. */
+export declare function procCommandLine(rawCmdline: string, comm: string): string;
+/** Where a procfs read comes from: the real `/proc` in production, a fake tree
+ *  plus a fixed boot id in tests. */
+interface ProcSource {
+    root: string;
+    bootId: () => string | null;
+    /** The uid this process runs as; defaults to `process.getuid()`. Tests inject
+     *  one so a fake tree (owned by the test user) can hold a "foreign" entry. */
+    selfUid?: () => number | null;
+}
+declare function procProcessState(pid: number, src?: ProcSource): PsProcessState;
+declare function procIdentities(pids: readonly number[], src?: ProcSource): Map<number, string> | null;
+declare function procCommand(pid: number, src?: ProcSource): string | null;
+/** One whole-table sample from procfs. Skipped as absent, as in a `ps` sample:
+ *  a pid that vanishes between the listing and its read, and a pid whose stat
+ *  is refused (EACCES/EPERM — `hidepid=1`) when its `/proc/<pid>` directory is
+ *  owned by another uid (this user cannot see it, and `ps` would not list it).
+ *  Every other read failure — including a refusal on a process we own or whose
+ *  owner we cannot establish — fails the whole sample, which every caller
+ *  treats as unknown. */
+export declare function readProcfsSnapshot(src?: ProcSource): PsLivenessSnapshot | null;
+/** The same sample without blocking the event loop (crtrd's detached work). */
+export declare function readProcfsSnapshotAsync(src?: ProcSource): Promise<PsLivenessSnapshot | null>;
+/** Every live process's pid, parent pid and full command line, for callers that
+ *  only match command text or walk parentage (stray-daemon sweep, viewer-pane
+ *  detection). `null` when no sample could be taken. Same backend rule as the
+ *  rest of this module: `/proc` where usable, `ps` elsewhere. */
+export declare function listProcessTable(): {
+    pid: number;
+    ppid: number;
+    command: string;
+}[] | null;
+/** Test seam: exercise the `/proc` backend against a fake tree. */
+export declare const procfsBackendForTest: {
+    procIdentities: typeof procIdentities;
+    procProcessState: typeof procProcessState;
+    procCommand: typeof procCommand;
+    readProcfsSnapshot: typeof readProcfsSnapshot;
+    readProcfsSnapshotAsync: typeof readProcfsSnapshotAsync;
+};
+/** Every transitive descendant of `rootPid`, discovered by walking `ps`'s
+ *  pid/ppid table (BFS) — NOT process-GROUP membership. The pi SDK's bash tool
+ *  spawns its shell child with
+ *  `detached: true` (verified in `@earendil-works/pi-agent-core`'s
+ *  `harness/env/nodejs.js`), which calls `setsid()` and puts that child in a
+ *  brand-new process GROUP of its own — so `kill(-rootPid)` never reaches it.
+ *  `setsid()` changes only SID/PGID, NEVER ppid, so the child (and its own
+ *  descendants) is still discoverable by walking parentage from `rootPid`, as
+ *  long as `rootPid`'s OS process is still alive when we walk (a fully-dead
+ *  parent is reparented away by the kernel immediately on exit, severing this
+ *  link — the wedged-but-alive broker this fix targets never hits that). Best-
+ *  effort: returns `[]` (never throws) on any `ps` failure, exactly like
+ *  `killProcessGroup`'s ESRCH-swallowing. */
+export declare function descendantPids(rootPid: number): number[];
+/** Capture a portable process-identity fingerprint — `ps`'s `lstart` column,
+ *  the process's full wall-clock start time — for each of `pids`, in ONE
+ *  batched `ps -p <list>` call. `lstart` is a standard format keyword on BOTH
+ *  BSD ps (macOS) and procps-ng (Linux), so this needs no `/proc` parsing or
+ *  other Linux-only machinery. Used to guard against PID REUSE during the
+ *  multi-second teardown escalation ladder: a
+ *  captured pid can exit and the OS can recycle it for an unrelated process
+ *  before the SIGKILL rung fires, and a bare `kill(pid, 0)` / `kill(pid, sig)`
+ *  cannot tell the difference.
+ *
+ *  Returns `null` — NOT an empty map — when the `ps` PROBE ITSELF failed (it
+ *  could not be spawned, or was killed by the timeout): a probe failure means
+ *  we have NO information, which is a fundamentally different outcome from a
+ *  probe that SUCCEEDED and simply found no row for a given pid (that pid is
+ *  genuinely gone). Collapsing these two into the same empty map is unsafe:
+ *  `killProcessTreePids` cannot tell "pid confirmed gone" from "we don't know"
+ *  without this
+ *  distinction, and misreading a failed probe as positive proof of
+ *  absence/reuse either signals a reused stranger or — worse — silently skips
+ *  reaping a real live orphan. A successful call may still return a map with
+ *  no entry for some/all of `pids` (those pids have no current row — genuinely
+ *  gone); that is the valid empty-or-partial case and is NOT a failure.
+ *
+ *  EXIT STATUS IS NOT THE FAILURE SIGNAL. `ps` exits 1 — on BOTH BSD ps and
+ *  procps-ng — when it matched no process at all, which is precisely the
+ *  "probe succeeded, that pid is gone" case, and exits 1 with PARTIAL stdout
+ *  when only some of a batch's pids still exist. Reading a non-zero exit as a
+ *  probe failure therefore turned every confirmed-dead pid into
+ *  `'indeterminate'` (`recordedPidLiveness`), so a just-killed broker read as
+ *  "can't confirm" — never as dead. Callers that fail open on indeterminate
+ *  (`ensureAttach`'s revive decision, the daemon's exit policy) then treat a
+ *  dead broker as live forever. Discriminate on whether the PROCESS RAN
+ *  (`r.error`/`r.signal`) and whether it complained (`stderr`), never on what
+ *  it exited with: a real argument/usage failure writes a diagnostic to stderr
+ *  (`ps: process id too large`), while "matched nothing" is silent. */
+export declare function capturePidIdentities(pids: readonly number[]): Map<number, string> | null;
+/** Read one pid's current full command line for a destructive caller that
+ *  must freshly re-confirm process ownership immediately before signaling.
+ *  Failure and absence both return null, so uncertainty never authorizes a
+ *  signal. */
+export declare function capturePidCommand(pid: number): string | null;
+/** The full pre-signal snapshot `teardown()` needs, taken ONCE, from ONE `ps`
+ *  table: the process TREE rooted at `rootPid`
+ *  (itself + every transitive descendant) alongside each member's identity
+ *  fingerprint — PLUS, load-bearing, a launch-time-identity REUSE check on
+ *  `rootPid` itself before the tree is even walked.
+ *
+ *  The hazard this closes: the PID-reuse guard elsewhere in this module (see
+ *  `killProcessTreePids`) only ever compared identities captured across the
+ *  teardown ESCALATION window (a few seconds) — it had no idea whether
+ *  `rootPid` (the broker pid recorded on the node row at LAUNCH time, possibly
+ *  hours or days earlier) had ALREADY been recycled for an unrelated process
+ *  before `teardown()` was ever called. In that case the naive snapshot would
+ *  capture the STRANGER's current identity as the "expected" baseline and
+ *  cheerfully SIGTERM/SIGKILL it and whatever the stranger's OWN process tree
+ *  turns out to be — exactly the wrong-victim risk finding 1 targets.
+ *
+ *  `expectedIdentity` is the LAUNCH-time identity recorded on the node row
+ *  (`recordPid`'s `capturePidIdentities([pid])` snapshot, taken the instant the
+ *  broker was spawned/bound) — `null` for a node booted before this field
+ *  existed, or if that original capture itself failed. When `expectedIdentity`
+ *  is present AND the CURRENT probe for `rootPid` succeeds AND finds a DIFFERENT
+ *  identity, `rootPid` is provably a different OS process than the broker this
+ *  node ever launched: `reused: true`, `tree: []`, `identities: null` — the
+ *  caller must treat this exactly like "nothing to tear down", and must NOT
+ *  walk `descendantPids` from this poisoned root (those "descendants" would be
+ *  the STRANGER's children, not this node's). Every other case (no baseline,
+ *  the pid is simply gone, the probe itself failed) falls through to the
+ *  normal walk — fail-open, matching every other guard in this module: a
+ *  MISMATCH is the only thing that skips, never an absence of evidence. */
+export interface TeardownSnapshot {
+    /** `[rootPid, ...descendants]`, or `[]` when `reused` is true (nothing safe
+     *  to signal) or `rootPid` itself is invalid. */
+    tree: number[];
+    /** Identity fingerprint per pid in `tree`, from the SAME `ps` table read as
+     *  the tree walk — or `null` when that probe failed outright (fail-open: the
+     *  caller's downstream `killProcessTreePids` signals unguarded in that case,
+     *  same as an omitted `identities` arg). Always `null` when `reused` is true
+     *  (there is no tree to guard). */
+    identities: Map<number, string> | null;
+    /** True iff `rootPid`'s CURRENT identity is POSITIVELY known to differ from
+     *  `expectedIdentity` — the pid was already reused by an unrelated process
+     *  before this snapshot was taken. Callers must skip ALL signaling when true. */
+    reused: boolean;
+}
+export declare function captureTeardownSnapshot(rootPid: number, expectedIdentity: string | null, excludePid?: number): TeardownSnapshot;
+/** Best-effort signal against every pid in an ALREADY-CAPTURED tree snapshot
+ *  (typically `[rootPid, ...descendantPids(rootPid)]`, taken ONCE by the
+ *  caller before sending any signal — see the doc on `isProcessTreeAlive` for
+ *  why re-deriving via `ps` after signaling starts is unsafe). Signals EACH
+ *  pid's OWN process group (a detached SDK-spawned descendant is its own
+ *  group leader — the pi bash tool's shell child)
+ *  PLUS a plain `kill(pid)` on each (belt-and-suspenders for a pid that is
+ *  NOT its own group leader). Every signal is best-effort — `killProcessGroup`
+ *  swallows ESRCH and the plain `kill` here does too — so firing this against
+ *  an already-fully-dead snapshot is a harmless no-op.
+ *
+ *  `identities`, when supplied, is a
+ *  `capturePidIdentities` snapshot taken alongside the pid list at the SAME
+ *  moment — `null` if that INITIAL capture's `ps` probe itself failed (see
+ *  `capturePidIdentities`'s doc). Before signaling EITHER the group or the
+ *  plain kill for a pid, this re-derives CURRENT identities in one fresh
+ *  batched call and compares — but ONLY skips on POSITIVE proof of reuse; any
+ *  failure or absence of evidence falls through to the normal signal path
+ *  (fail-open, matching this module's unguarded behavior):
+ *
+ *    - baseline identity present AND the current probe SUCCEEDED AND the
+ *      current identity for this pid is present AND DIFFERENT — the OS has
+ *      recycled this pid for an unrelated process since the snapshot — SKIP
+ *      (the sole skip condition; this is the actual reused-pid hazard finding
+ *      3 targets).
+ *    - baseline identity present but the pid is now simply GONE (current
+ *      probe succeeded, no row for this pid) — it died naturally in the
+ *      meantime, not reused — SIGNAL as normal (harmless no-op on the plain
+ *      kill; load-bearing for the group kill, which can still reach a
+ *      surviving process that shares this now-dead pid's process GROUP).
+ *    - baseline had NO identity at all for this pid (the pid was ALREADY dead
+ *      at snapshot time — e.g. a broker that crashed before `teardown()` ever
+ *      ran) — SIGNAL as normal regardless of the current probe: this is
+ *      exactly how `killProcessGroup` on an already-dead root still reaches a
+ *      live process sharing its process group (the pgid persists after the
+ *      group leader exits; see `host-teardown-process-group.test.ts`'s "dead
+ *      broker" case).
+ *    - the INITIAL capture's probe failed outright (`identities === null`) —
+ *      we have NO baseline to compare against for ANY pid — SIGNAL as normal
+ *      for the whole list; the CURRENT probe is not even attempted (nothing
+ *      to compare it to).
+ *    - the baseline succeeded but the CURRENT probe (taken here) fails — we
+ *      have NO current information — SIGNAL as normal for the whole list; a
+ *      failed probe must never be read as "gone" (would risk reaching a
+ *      reused stranger, the residual risk this guard cannot close) NOR as
+ *      "reused" (would wrongly disable the reap for every real live pid).
+ *
+ *  Callers that omit `identities` entirely (pre-existing call sites) keep the
+ *  prior unguarded behavior. */
+export declare function killProcessTreePids(pids: readonly number[], signal?: NodeJS.Signals, identities?: ReadonlyMap<number, string> | null): void;
+/** Is `rootPid`'s process TREE — itself or any CURRENT descendant — still
+ *  alive, re-deriving the descendant set fresh via `ps` on every call? Safe
+ *  ONLY before any signal has been sent to this tree. Once a signal kills the
+ *  root (it's typically its own process-group leader — see `launch()` in
+ *  host.ts), the kernel reparents any surviving child away from `rootPid`
+ *  IMMEDIATELY (standard orphan handling, confirmed empirically: a killed
+ *  parent's child shows `ppid=1` within the same tick, not lazily) —
+ *  silently "losing" that child from a fresh `descendantPids` walk even
+ *  though it is still very much alive. Use this ONLY to watch a tree you have
+ *  not touched yet (e.g. waiting out a graceful-shutdown grace window); once
+ *  you start signaling, capture the pid list ONCE and check it with
+ *  `isAnyPidAlive` instead — see `killProcessTreePids`. */
+export declare function isProcessTreeAlive(rootPid: number): boolean;
+/** Is any pid in an ALREADY-CAPTURED pid list (see `killProcessTreePids`)
+ *  still alive? Pure signal-0 check against the fixed list — NEVER re-walks
+ *  `ps` — the correct liveness check once you've started signaling a tree
+ *  (see `isProcessTreeAlive`'s doc for why re-deriving after that point would
+ *  silently drop a still-alive orphan). */
+export declare function isAnyPidAlive(pids: readonly number[]): boolean;
+export {};

package/dist/core/canvas/pid.js

@@ -0,0 +1,5 @@
+var J=Object.defineProperty;var s=(n,t)=>J(n,"name",{value:t,configurable:!0});import{spawn as V,spawnSync as b}from"node:child_process";import{readFileSync as h,readdirSync as K,statSync as X}from"node:fs";import{readFile as x}from"node:fs/promises";import{readKernelBootId as P}from"./boot-id.js";function q(n){if(m())return D(n);try{const t=b("ps",["-o","stat=","-p",String(n)],{encoding:"utf8",timeout:2e3});if(t.error!=null||t.signal!=null||typeof t.stdout!="string"||typeof t.stderr=="string"&&t.stderr.trim()!=="")return"unknown";const e=t.stdout.trim();return e===""?"gone":e.charAt(0)==="Z"?"zombie":"live"}catch{return"unknown"}}s(q,"psProcessState");function S(n){if(n==null)return!1;try{process.kill(n,0)}catch(e){return e.code==="EPERM"}const t=q(n);return t!=="zombie"&&t!=="gone"}s(S,"isPidAlive");function fn(n){if(n==null)return!1;try{return process.kill(n,0),!0}catch(t){return t.code==="EPERM"}}s(fn,"isPidPresent");const y=/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;function w(n){const t=n.indexOf("#");if(t===-1)return{base:n,ticks:void 0};const e=n.slice(t+1);return{base:n.slice(0,t),ticks:/^\d+$/.test(e)?e:void 0}}s(w,"splitIdentity");function k(n,t){const e=w(n),r=w(t);return e.ticks!==void 0&&r.ticks!==void 0?y.test(e.base)&&y.test(r.base)?e.base===r.base&&e.ticks===r.ticks:e.ticks===r.ticks:e.base===r.base}s(k,"identitiesMatch");function dn(n,t,e){const r=!!process.env.CRTR_DEBUG_PID_LIVENESS;if(n==null)return"dead";if(e===null)return r&&p(n,!1,t,"probe-null","ps snapshot failed \u2192 indeterminate","indeterminate"),"indeterminate";const o=e?.processes.get(n),i=e===void 0?S(n):o!==void 0&&!o.zombie;if(!i)return r&&p(n,i,t,e===void 0?"no-probe":"no-entry","process absent/zombie \u2192 dead","dead"),"dead";if(t==null)return r&&p(n,i,t,"no-probe","no baseline \u2192 trust process presence","alive"),"alive";let u;if(e===void 0){const l=Z([n]);if(l==null)return r&&p(n,i,t,"probe-null","ps probe failed \u2192 indeterminate","indeterminate"),"indeterminate";if(u=l.get(n),u===void 0)return r&&p(n,i,t,"no-entry","no identity row for pid (race) \u2192 dead","dead"),"dead"}else u=o.identity;const c=k(u,t);return r&&p(n,i,t,"value",c?"identity matches expected \u2192 alive":"identity differs from expected \u2192 REUSED \u2192 dead",c?"alive":"dead",u),c?"alive":"dead"}s(dn,"recordedPidLiveness");function p(n,t,e,r,o,i,u){const c=[`pid=${String(n)}`,`isPidAlive=${t}`,`expectedIdentity=${e===void 0?"undefined":e===null?"null":JSON.stringify(e)}`,`capturePidIdentities=${r}${r==="value"?`(${JSON.stringify(u)})`:""}`,`branch=${JSON.stringify(o)}`,`returned=${i}`];process.stderr.write(`[recordedPidLiveness] ${c.join(" ")}
+`)}s(p,"logPidLivenessDecision");function W(n,t="SIGTERM"){if(!(!Number.isInteger(n)||n<=0))try{process.kill(-n,t)}catch{}}s(W,"killProcessGroup");function Q(n){try{const t=h(`/proc/${n}/stat`,"utf8"),e=t.lastIndexOf(")");if(e===-1)return null;const o=t.slice(e+2).trim().split(/\s+/)[19];return o!==void 0&&/^\d+$/.test(o)?o:null}catch{return null}}s(Q,"procStartTicks");const Y=6e4;function nn(n,t=Date.now,e=Y){let r=null,o=0;return()=>{if(r!==null)return r;const i=t();if(i<o)return null;const u=n();return u!==null?r=u:(o=i+e,null)}}s(nn,"makeBootIdCache");const T=nn(P);function an(n){const{base:t,ticks:e}=w(n);return e!==void 0&&!y.test(t)}s(an,"isLegacyTicksIdentity");function pn(n){const{base:t,ticks:e}=w(n);return e!==void 0&&y.test(t)}s(pn,"isBootScopedIdentity");function v(n,t,e={}){const r=e.ticks!==void 0?e.ticks:Q(n);return r===null?t:`${(e.bootId!==void 0?e.bootId:T())??t}#${r}`}s(v,"composeIdentity");const M=["-Ao","pid=,ppid=,stat=,lstart=,command="],A=2e3;function R(){if(m())return N();let n;try{const t=b("ps",[...M],{encoding:"utf8",timeout:A,maxBuffer:8388608});if(t.status!==0||typeof t.stdout!="string")return null;n=t.stdout}catch{return null}return F(n)}s(R,"captureLivenessSnapshot");async function mn(){if(m())return j();const n=await new Promise(t=>{let e;try{e=V("ps",[...M])}catch{return t(null)}let r="",o=!1;const i=s(c=>{o||(o=!0,clearTimeout(u),t(c))},"settle"),u=setTimeout(()=>{try{e.kill("SIGKILL")}catch{}i(null)},A);u.unref?.(),e.stdout.on("data",c=>r+=c.toString()),e.stderr.on("data",()=>{}),e.on("error",()=>i(null)),e.on("close",c=>i(c===0?r:null))});return n===null?null:F(n)}s(mn,"captureLivenessSnapshotAsync");function _(n){const t=n.indexOf("("),e=n.lastIndexOf(")");if(t===-1||e===-1||e<t)return null;const r=n.slice(e+2).trim().split(/\s+/),o=r[0],i=Number(r[1]),u=r[19];return o===void 0||o===""||!Number.isInteger(i)||i<0||u===void 0||!/^\d+$/.test(u)?null:{comm:n.slice(t+1,e),state:o,ppid:i,starttime:u}}s(_,"parseProcStat");function L(n,t){const e=n.replace(/\0+$/,"").split("\0").join(" ").trim();return e===""?`[${t}]`:e}s(L,"procCommandLine");let I;function m(){return I===void 0&&(I=process.platform==="linux"&&P()!==null&&tn(process.pid)!==null),I}s(m,"procfsUsable");const a={root:"/proc",bootId:s(()=>T(),"bootId")};function tn(n,t=a){const e=g(n,t);return e==="unreadable"||e==="denied"||e==="gone"?null:e}s(tn,"procStatOf");function g(n,t=a){let e;try{e=h(`${t.root}/${n}/stat`,"utf8")}catch(r){return $(r)}return _(e)??"unreadable"}s(g,"readProcStat");function $(n){const t=n.code;return t==="ENOENT"||t==="ESRCH"?"gone":t==="EACCES"||t==="EPERM"?"denied":"unreadable"}s($,"classifyProcReadError");function en(n,t){const e=t.selfUid!==void 0?t.selfUid():process.getuid?.()??null;if(e===null)return"own-or-unknown";try{return X(`${t.root}/${n}`).uid===e?"own-or-unknown":"foreign"}catch(r){return $(r)==="gone"?"gone":"own-or-unknown"}}s(en,"procOwnership");function B(n,t,e){return n==="gone"?"skip":n==="unreadable"||en(t,e)==="own-or-unknown"?"fail":"skip"}s(B,"scanDisposition");function C(n,t=a){const e=t.bootId();return e===null?null:`${e}#${n.starttime}`}s(C,"procIdentity");function D(n,t=a){const e=g(n,t);return e==="gone"?"gone":e==="denied"||e==="unreadable"?"unknown":e.state==="Z"||e.state==="X"?"zombie":"live"}s(D,"procProcessState");function z(n,t=a){const e=new Map;for(const r of n){const o=g(r,t);if(o==="gone")continue;if(o==="unreadable"||o==="denied")return null;const i=C(o,t);if(i===null)return null;e.set(r,i)}return e}s(z,"procIdentities");function G(n,t=a){const e=g(n,t);if(e==="gone"||e==="unreadable"||e==="denied")return null;try{const r=L(h(`${t.root}/${n}/cmdline`,"utf8"),e.comm);return r===""?null:r}catch{return null}}s(G,"procCommand");function H(n,t,e,r,o){const i=C(r,n);if(i===null)return!1;t.processes.set(e,{zombie:r.state==="Z"||r.state==="X",identity:i,command:L(o,r.comm)});const u=t.childrenOf.get(r.ppid);return u===void 0?t.childrenOf.set(r.ppid,[e]):u.push(e),!0}s(H,"addProcfsRow");function U(n){try{return K(n).filter(t=>/^\d+$/.test(t)).map(Number).filter(t=>t>0)}catch{return null}}s(U,"procPids");function N(n=a){const t=U(n.root);if(t===null)return null;const e={processes:new Map,childrenOf:new Map};for(const r of t){const o=g(r,n);if(o==="gone"||o==="denied"||o==="unreadable"){if(B(o==="gone"?"gone":o,r,n)==="skip")continue;return null}let i="";try{i=h(`${n.root}/${r}/cmdline`,"utf8")}catch(u){const c=u.code;if(c==="ENOENT"||c==="ESRCH")continue}if(!H(n,e,r,o,i))return null}return e}s(N,"readProcfsSnapshot");async function j(n=a){const t=U(n.root);if(t===null)return null;const e={processes:new Map,childrenOf:new Map},r=64;for(let o=0;o<t.length;o+=r){const i=t.slice(o,o+r),u=await Promise.all(i.map(async c=>{try{const l=_(await x(`${n.root}/${c}/stat`,"utf8"));if(l===null)return"failed";const d=await x(`${n.root}/${c}/cmdline`,"utf8").catch(f=>{if(f.code==="ENOENT"||f.code==="ESRCH")throw f;return""});return{pid:c,stat:l,cmdline:d}}catch(l){return B($(l),c,n)==="skip"?"gone":"failed"}}));for(const c of u)if(c!=="gone"&&(c==="failed"||!H(n,e,c.pid,c.stat,c.cmdline)))return null}return e}s(j,"readProcfsSnapshotAsync");function bn(){if(m()){const n=N();if(n===null)return null;const t=new Map;for(const[e,r]of n.childrenOf)for(const o of r)t.set(o,e);return[...n.processes].map(([e,r])=>({pid:e,ppid:t.get(e)??0,command:r.command}))}try{const n=b("ps",["-ax","-o","pid=,ppid=,command="],{encoding:"utf8",timeout:2e3,maxBuffer:8388608});if(n.status!==0||typeof n.stdout!="string")return null;const t=[];for(const e of n.stdout.split(`
+`)){const r=/^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(e);r!==null&&t.push({pid:Number(r[1]),ppid:Number(r[2]),command:r[3].trim()})}return t}catch{return null}}s(bn,"listProcessTable");const gn={procIdentities:z,procProcessState:D,procCommand:G,readProcfsSnapshot:N,readProcfsSnapshotAsync:j};function F(n){const t=new Map,e=new Map;for(const r of n.split(`
+`)){const o=r.trim();if(o==="")continue;const i=/^(\d+)\s+(\d+)\s+(\S+)\s+(\S+(?:\s+\S+){4})\s+(.+)$/.exec(o);if(i===null)continue;const u=Number(i[1]),c=Number(i[2]),l=i[3],d=i[4],f=i[5].trim();if(!Number.isInteger(u)||u<=0||!Number.isInteger(c)||c<0||f==="")continue;t.set(u,{zombie:l.startsWith("Z"),identity:v(u,d),command:f});const O=e.get(c);O===void 0?e.set(c,[u]):O.push(u)}return{processes:t,childrenOf:e}}s(F,"parseLivenessTable");function E(n,t){const e=[],r=new Set([n]),o=[...t.get(n)??[]];for(;o.length>0;){const i=o.shift();if(r.has(i))continue;r.add(i),e.push(i);const u=t.get(i);u!==void 0&&o.push(...u)}return e}s(E,"descendantPidsFrom");function rn(n){if(!Number.isInteger(n)||n<=0)return[];const t=R();return t===null?[]:E(n,t.childrenOf)}s(rn,"descendantPids");function Z(n){const t=n.filter(r=>Number.isInteger(r)&&r>0),e=new Map;if(t.length===0)return e;if(m())return z(t);try{const r=b("ps",["-o","pid=,lstart=","-p",t.join(",")],{encoding:"utf8",timeout:2e3});if(r.error!=null||r.signal!=null||typeof r.stdout!="string"||typeof r.stderr=="string"&&r.stderr.trim()!=="")return null;for(const o of r.stdout.split(`
+`)){const i=o.trim();if(i==="")continue;const u=i.indexOf(" ");if(u===-1)continue;const c=Number(i.slice(0,u)),l=i.slice(u+1).trim();!Number.isInteger(c)||c<=0||l===""||e.set(c,v(c,l))}return e}catch{return null}}s(Z,"capturePidIdentities");function hn(n){if(!Number.isInteger(n)||n<=0)return null;if(m())return G(n);try{const t=b("ps",["-o","command=","-p",String(n)],{encoding:"utf8",timeout:2e3});if(t.status!==0||typeof t.stdout!="string")return null;const e=t.stdout.trim();return e===""?null:e}catch{return null}}s(hn,"capturePidCommand");function Sn(n,t,e){if(!Number.isInteger(n)||n<=0)return{tree:[],identities:null,reused:!1};const r=R();if(r===null)return{tree:[n],identities:null,reused:!1};const o=r.processes.get(n)?.identity;if(t!=null&&o!==void 0&&!k(o,t))return{tree:[],identities:null,reused:!0};const u=e!==void 0&&Number.isInteger(e)&&e>0?new Set([e,...E(e,r.childrenOf)]):void 0,c=[n,...E(n,r.childrenOf)].filter(d=>!u?.has(d)),l=new Map;for(const d of c){const f=r.processes.get(d)?.identity;f!==void 0&&l.set(d,f)}return{tree:c,identities:l,reused:!1}}s(Sn,"captureTeardownSnapshot");function yn(n,t="SIGTERM",e){const r=e!=null?Z(n):void 0;for(const o of n){if(e!=null&&r!=null){const i=e.get(o),u=r.get(o);if(i!==void 0&&u!==void 0&&!k(u,i))continue}if(W(o,t),!(!Number.isInteger(o)||o<=0))try{process.kill(o,t)}catch{}}}s(yn,"killProcessTreePids");function wn(n){return S(n)?!0:rn(n).some(t=>S(t))}s(wn,"isProcessTreeAlive");function kn(n){return n.some(t=>S(t))}s(kn,"isAnyPidAlive");export{R as captureLivenessSnapshot,mn as captureLivenessSnapshotAsync,hn as capturePidCommand,Z as capturePidIdentities,Sn as captureTeardownSnapshot,v as composeIdentity,rn as descendantPids,k as identitiesMatch,kn as isAnyPidAlive,pn as isBootScopedIdentity,an as isLegacyTicksIdentity,S as isPidAlive,fn as isPidPresent,wn as isProcessTreeAlive,W as killProcessGroup,yn as killProcessTreePids,bn as listProcessTable,nn as makeBootIdCache,_ as parseProcStat,L as procCommandLine,gn as procfsBackendForTest,N as readProcfsSnapshot,j as readProcfsSnapshotAsync,dn as recordedPidLiveness};

package/dist/core/canvas/recovery.d.ts

@@ -0,0 +1,90 @@
+import type { DatabaseSync } from 'node:sqlite';
+import { type Fault, type FaultKind, type FaultLink, type FaultProviderError, type FaultRetry } from '../fault-classifier.js';
+export type { Fault, FaultKind, FaultLink, FaultProviderError, FaultRetry } from '../fault-classifier.js';
+export declare const FAULT_QUIET_MS: number;
+export interface FaultRecordInput {
+    link: FaultLink;
+    op: string;
+    kind: FaultKind;
+    retry: FaultRetry;
+    message: string;
+    operation_id?: string;
+    since?: string;
+    anchorEntryId?: string;
+    providerError?: FaultProviderError;
+}
+export interface StoredFault extends Fault {
+    faultId: string;
+}
+export interface ProviderRetryEpisode {
+    state: 'pending' | 'admitted' | 'invalidated';
+    sessionId: string | null;
+    fault: StoredFault;
+}
+export interface TransportPoisonVerdict {
+    consecutivePasses: number;
+    routes: string[];
+}
+export interface BootFaultAttempt {
+    explanation(): string | null;
+}
+export declare function parseProviderError(value: unknown): FaultProviderError | undefined;
+export declare function hydrateFault(row: Record<string, unknown>): StoredFault;
+export declare function openTurn(executionId: string, _opts: {
+    reason: 'agent-start' | 'prompt-unsettled';
+}): boolean;
+export declare function recordTurnProgress(executionId: string): boolean;
+export declare function closeTurn(executionId: string, opts: {
+    endReason: 'settled' | 'teardown-aborted';
+}): boolean;
+export declare function recordTransportPoisonVerdict(executionId: string, verdict: TransportPoisonVerdict): boolean;
+export declare function readTransportPoisonVerdict(nodeId: string): TransportPoisonVerdict | null;
+export declare function readLiveTurn(nodeId: string): {
+    busy: boolean;
+    progressAt: string | null;
+} | null;
+export declare function readInterruptedTurn(nodeId: string): {
+    interrupted: boolean;
+    progressAt: string | null;
+} | null;
+export declare function recordFault(nodeId: string, input: FaultRecordInput): StoredFault | null;
+export declare function clearFault(nodeId: string, opts?: {
+    link?: FaultLink;
+    preserveEpisode?: boolean;
+}): boolean;
+export declare function readFault(nodeId: string): StoredFault | null;
+export declare function recordPendingEpisode(nodeId: string, input: {
+    sessionId: string;
+    input: FaultRecordInput;
+}): StoredFault | null;
+export declare function admitEpisode(nodeId: string, input: {
+    sessionId: string;
+    faultId: string;
+    retry: FaultRetry;
+}): boolean;
+export declare function reopenEpisode(nodeId: string, input: {
+    sessionId: string;
+}): boolean;
+export declare function invalidateEpisode(nodeId: string, input: {
+    sessionId: string;
+}): boolean;
+export declare function clearEpisode(nodeId: string, input: {
+    sessionId: string;
+}): boolean;
+export declare function readEpisode(nodeId: string): ProviderRetryEpisode | null;
+/** One active automatic provider-retry fault whose `pending` episode is bound to
+ *  its node's live session — the shape a broker's in-process retry timer owns. */
+export interface PendingProviderRetry {
+    nodeId: string;
+    sessionId: string;
+    fault: StoredFault;
+    /** When the fault row last changed (ISO) — the daemon's anchor for "due". */
+    updatedAt: string;
+}
+/** Every node with an active daemon-retried provider fault (connection,
+ *  rate-limit, overloaded) and a pending episode on its live session binding.
+ *  The daemon backstop reads this to find retries whose broker timer died. */
+export declare function listPendingProviderRetries(): PendingProviderRetry[];
+export declare function beginBootFaultAttempt(nodeId: string): BootFaultAttempt;
+/** Import legacy witness and recovery files inside the parent's one shared consolidation transaction. */
+export declare function migrateRecoveryWitnesses(db: DatabaseSync): void;

package/dist/core/canvas/recovery.js

@@ -0,0 +1,26 @@
+var re=Object.defineProperty;var o=(e,n)=>re(e,"name",{value:n,configurable:!0});import{prepared as s}from"../sqlite-statements.js";import{randomUUID as b}from"node:crypto";import{existsSync as p,readdirSync as ie,readFileSync as oe,realpathSync as se,statSync as de}from"node:fs";import{join as N,resolve as ae}from"node:path";import{openDb as T,prepared as ue,withCanvasWrite as l}from"./db.js";import{jobDir as ce,nodesRoot as C}from"./paths.js";import{errorClassFromFault as M}from"../fault-classifier.js";import{parseProviderRateLimit as _e}from"../provider-failure.js";import{emitEvent as X}from"../events/emit.js";import{operationIdContext as j,validateOperationId as $}from"../events/operation-id.js";const We=5*6e4,le=500,B=["pi\u2192provider","viewer\u2194broker","relay\u2194broker","viewer\u2194crtrd","daemon\u2192node","crtr\u2192pi"],pe=[...B,"broker\u2194crtrd"],Ee=["rate-limit","overloaded","connection","auth","protocol","context-overflow","other","wedged","model-not-found"],fe=["auto","manual","fatal"],ye=["sdk","daemon","client"],Te=["pending","admitted","invalidated"];function c(){return new Date().toISOString()}o(c,"now");function J(e,n){return new Set(e.prepare(`PRAGMA table_info(${n})`).all().map(({name:t})=>t))}o(J,"tableColumns");function Ne(e){const n=J(e,"executions"),t=[["turn_open_at","TEXT"],["turn_progress_at","TEXT"],["turn_closed_at","TEXT"],["turn_end_reason","TEXT CHECK (turn_end_reason IS NULL OR turn_end_reason IN ('settled', 'teardown-aborted'))"],["exit_diagnostic_json","TEXT"]];for(const[r,i]of t)n.has(r)||e.exec(`ALTER TABLE executions ADD COLUMN ${r} ${i}`)}o(Ne,"addExecutionColumns");function ge(e){e.exec(`
+CREATE TABLE IF NOT EXISTS node_faults (
+  fault_id TEXT PRIMARY KEY,
+  node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE,
+  link TEXT NOT NULL CHECK (link IN ('pi\u2192provider', 'viewer\u2194broker', 'relay\u2194broker', 'viewer\u2194crtrd', 'daemon\u2192node', 'crtr\u2192pi')),
+  op TEXT NOT NULL,
+  kind TEXT NOT NULL CHECK (kind IN ('rate-limit', 'overloaded', 'connection', 'auth', 'protocol', 'context-overflow', 'other', 'wedged', 'model-not-found')),
+  message TEXT NOT NULL,
+  since TEXT NOT NULL,
+  operation_id TEXT NOT NULL,
+  anchor_entry_id TEXT,
+  retry_disposition TEXT NOT NULL CHECK (retry_disposition IN ('auto', 'manual', 'fatal')),
+  retry_by TEXT CHECK (retry_by IS NULL OR retry_by IN ('sdk', 'daemon', 'client')),
+  retry_attempt INTEGER,
+  retry_max INTEGER,
+  retry_next_at TEXT,
+  provider_error_json TEXT,
+  active INTEGER NOT NULL DEFAULT 0 CHECK (active IN (0, 1)),
+  episode_state TEXT CHECK (episode_state IS NULL OR episode_state IN ('pending', 'admitted', 'invalidated')),
+  episode_binding_id TEXT REFERENCES sessions(binding_id),
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_active ON node_faults(node_id) WHERE active = 1;
+CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_episode ON node_faults(node_id) WHERE episode_state IS NOT NULL;
+`)}o(ge,"createFaultTable");function g(e,n){return typeof n=="string"&&e.includes(n)}o(g,"contains");function F(e){return typeof e=="object"&&e!==null&&!Array.isArray(e)}o(F,"isRecord");function K(e){if(!F(e)||typeof e.provider!="string"||e.provider===""||typeof e.model!="string"||e.model==="")return;const n=typeof e.status=="number"&&Number.isFinite(e.status)?e.status:void 0,t=typeof e.code=="string"||typeof e.code=="number"?e.code:void 0,r=typeof e.errno=="string"?e.errno:void 0,i=typeof e.body=="string"?e.body:void 0,d=_e(e.ratelimit),a=Array.isArray(e.causes)?e.causes.flatMap(u=>{if(!F(u))return[];const E={...typeof u.message=="string"?{message:u.message}:{},...typeof u.status=="number"&&Number.isFinite(u.status)?{status:u.status}:{},...typeof u.code=="string"||typeof u.code=="number"?{code:u.code}:{},...typeof u.errno=="string"?{errno:u.errno}:{},...typeof u.body=="string"?{body:u.body}:{}};return Object.keys(E).length===0?[]:[E]}):void 0;return{provider:e.provider,model:e.model,...n===void 0?{}:{status:n},...t===void 0?{}:{code:t},...r===void 0?{}:{errno:r},...i===void 0?{}:{body:i},...a===void 0||a.length===0?{}:{causes:a},...d===void 0?{}:{ratelimit:d}}}o(K,"parseProviderError");function Y(e){try{const n=JSON.parse(e);if(!g(pe,n.link)||typeof n.op!="string"||n.op===""||!g(Ee,n.kind)||typeof n.message!="string"||typeof n.since!="string")return null;const t=$(n.operation_id);if(!F(n.retry)||!g(fe,n.retry.disposition)||n.retry.by!==void 0&&!g(ye,n.retry.by)||n.retry.attempt!==void 0&&typeof n.retry.attempt!="number"||n.retry.max!==void 0&&typeof n.retry.max!="number"||n.retry.nextAt!==void 0&&typeof n.retry.nextAt!="string"||n.anchorEntryId!==void 0&&(typeof n.anchorEntryId!="string"||n.anchorEntryId===""))return null;const r=n.providerError===void 0?void 0:K(n.providerError);return n.providerError!==void 0&&r===void 0?null:{link:n.link,op:n.op,kind:n.kind,retry:{disposition:n.retry.disposition,...n.retry.by===void 0?{}:{by:n.retry.by},...n.retry.attempt===void 0?{}:{attempt:n.retry.attempt},...n.retry.max===void 0?{}:{max:n.retry.max},...n.retry.nextAt===void 0?{}:{nextAt:n.retry.nextAt}},message:n.message,since:n.since,operation_id:t,...n.anchorEntryId===void 0?{}:{anchorEntryId:n.anchorEntryId},...r===void 0?{}:{providerError:r}}}catch{return null}}o(Y,"parseLegacyFault");function Le(e){const n=ae(e);try{return se(n)}catch{return n}}o(Le,"canonicalPath");function me(e){try{const n=JSON.parse(e);if(!g(Te,n.state)||typeof n.sessionFile!="string"||n.sessionFile==="")return null;const t=JSON.stringify(n.fault),r=Y(t);if(r===null||r.link!=="pi\u2192provider"||r.retry.disposition!=="auto"||r.retry.by!=="daemon")return null;const i=n.anchorEntryId;return i!==null&&typeof i!="string"||(r.anchorEntryId??null)!==i?null:{state:n.state,sessionFile:Le(n.sessionFile),fault:r}}catch{return null}}o(me,"parseLegacyEpisode");function G(e){return e!==void 0?$(e):j.current()??j.fresh(t=>t)}o(G,"resolveOperationId");function Ae(e){return e.slice(0,le)}o(Ae,"clampMessage");function R(e){return e.link==="pi\u2192provider"&&e.retry.disposition==="auto"&&e.retry.by==="daemon"?3:e.retry.disposition==="fatal"?2:1}o(R,"faultPriority");function V(e){return{link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operationId:e.operation_id,anchorEntryId:e.anchorEntryId??null,retryDisposition:e.retry.disposition,retryBy:e.retry.by??null,retryAttempt:e.retry.attempt??null,retryMax:e.retry.max??null,retryNextAt:e.retry.nextAt??null,providerError:e.providerError===void 0?null:JSON.stringify(e.providerError)}}o(V,"encodeFault");function H(e){const n=e.provider_error_json,t=typeof n=="string"?K(JSON.parse(n)):void 0;if(typeof n=="string"&&t===void 0)throw new Error(`invalid provider error on stored fault ${e.fault_id}`);const r={disposition:e.retry_disposition,...e.retry_by===null?{}:{by:e.retry_by},...e.retry_attempt===null?{}:{attempt:e.retry_attempt},...e.retry_max===null?{}:{max:e.retry_max},...e.retry_next_at===null?{}:{nextAt:e.retry_next_at}};return{faultId:e.fault_id,link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operation_id:e.operation_id,retry:r,...e.anchor_entry_id===null?{}:{anchorEntryId:e.anchor_entry_id},...t===void 0?{}:{providerError:t}}}o(H,"hydrateFault");function L(e,n){const t=ue(e,"SELECT * FROM node_faults WHERE node_id = ? AND active = 1").get(n);return t===void 0?null:H(t)}o(L,"activeFault");function Q(e){if(!g(B,e.link))throw new Error(`fault link is not durable: ${e.link}`);return{link:e.link,op:e.op,kind:e.kind,retry:e.retry,message:Ae(e.message),since:e.since??c(),operation_id:G(e.operation_id),...typeof e.anchorEntryId=="string"&&e.anchorEntryId!==""?{anchorEntryId:e.anchorEntryId}:{},...e.providerError===void 0?{}:{providerError:e.providerError}}}o(Q,"storedFault");function q(e,n){X({level:n.retry.disposition==="fatal"?"error":"warn",event:"fault.recorded",operation_id:n.operation_id,node_id:e,error_class:M(n),fields:{link:n.link,op:n.op,kind:n.kind,retry:n.retry,message:n.message,...n.anchorEntryId===void 0?{}:{anchor_entry_id:n.anchorEntryId},...n.providerError===void 0?{}:{provider_error:n.providerError}}})}o(q,"emitRecordedFault");function W(e,n,t,r,i){const d=b(),a=V(t),u=c();return s(e,"INSERT INTO node_faults (fault_id, node_id, link, op, kind, message, since, operation_id, anchor_entry_id, retry_disposition, retry_by, retry_attempt, retry_max, retry_next_at, provider_error_json, active, episode_state, episode_binding_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(d,n,a.link,a.op,a.kind,a.message,a.since,a.operationId,a.anchorEntryId,a.retryDisposition,a.retryBy,a.retryAttempt,a.retryMax,a.retryNextAt,a.providerError,r?1:0,i?.state??null,i?.bindingId??null,u,u),{...t,faultId:d}}o(W,"insertFault");function z(e,n,t,r){const i=V(t);return s(e,"UPDATE node_faults SET link = ?, op = ?, kind = ?, message = ?, since = ?, operation_id = ?, anchor_entry_id = ?, retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, provider_error_json = ?, episode_state = COALESCE(?, episode_state), episode_binding_id = COALESCE(?, episode_binding_id), updated_at = ? WHERE fault_id = ?").run(i.link,i.op,i.kind,i.message,i.since,i.operationId,i.anchorEntryId,i.retryDisposition,i.retryBy,i.retryAttempt,i.retryMax,i.retryNextAt,i.providerError,r?.state??null,r?.bindingId??null,c(),n),{...t,faultId:n}}o(z,"updateFault");function D(e,n){s(e,"DELETE FROM node_faults WHERE fault_id = ? AND active = 0 AND episode_state IS NULL").run(n)}o(D,"deleteIfUnretained");function Z(e,n,t,r){const i=L(e,n);if(i===null){const d=s(e,"SELECT fault_id, link, kind, since, operation_id FROM node_faults WHERE node_id = ? AND active = 0 AND episode_state IS NOT NULL").get(n);return d!==void 0&&d.link===t.link&&d.kind===t.kind&&d.since===t.since&&d.operation_id===t.operation_id?(s(e,"UPDATE node_faults SET active = 1, updated_at = ? WHERE fault_id = ?").run(c(),d.fault_id),z(e,d.fault_id,t)):W(e,n,t,!0,r)}if(i.link===t.link&&i.kind===t.kind){const d={...i.retry,...t.retry};return t.retry.disposition==="fatal"&&!("nextAt"in t.retry)&&delete d.nextAt,z(e,i.faultId,{...t,since:i.since,anchorEntryId:i.anchorEntryId??t.anchorEntryId,retry:d},r)}return R(i)>R(t)?null:(s(e,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(e,i.faultId),W(e,n,t,!0,r))}o(Z,"replaceActiveFault");function m(e,n,t){return s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND session_id = ? AND unbound_at IS NULL").get(n,t)?.binding_id??null}o(m,"liveBinding");function Pe(e,n){return l(t=>{const r=c();return s(t,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(r,e).changes>0?!0:s(t,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ? AND state != 'exited'").run(r,r,e).changes>0})}o(Pe,"openTurn");function we(e){return l(n=>s(n,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(c(),e).changes>0)}o(we,"recordTurnProgress");function Me(e,n){return l(t=>s(t,"UPDATE executions SET turn_closed_at = ?, turn_end_reason = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(c(),n.endReason,e).changes>0)}o(Me,"closeTurn");function Xe(e,n){if(!Number.isInteger(n.consecutivePasses)||n.consecutivePasses<1||n.routes.length===0||!n.routes.every(t=>t!==""))throw new Error("invalid transport-poison verdict");return l(t=>s(t,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ? AND state != 'exited'").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:n.consecutivePasses,routes:n.routes}}),e).changes>0)}o(Xe,"recordTransportPoisonVerdict");function je(e){const n=s(T(),"SELECT exit_diagnostic_json FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);if(n===void 0||n.exit_diagnostic_json===null)return null;try{const t=JSON.parse(n.exit_diagnostic_json),r=t.transport_poison;return t.schema!=="crtr.execution-exit-diagnostic/v1"||r===void 0||!Number.isInteger(r.consecutive_passes)||r.consecutive_passes<1||!Array.isArray(r.routes)||r.routes.length===0||!r.routes.every(i=>typeof i=="string"&&i!=="")?null:{consecutivePasses:r.consecutive_passes,routes:r.routes}}catch{return null}}o(je,"readTransportPoisonVerdict");function $e(e){const n=s(T(),"SELECT turn_open_at, turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND state != 'exited'").get(e);return n===void 0?null:{busy:n.turn_open_at!==null&&n.turn_closed_at===null,progressAt:n.turn_progress_at}}o($e,"readLiveTurn");function Be(e){const n=T(),t=s(n,"SELECT turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL ORDER BY rowid DESC LIMIT 1").get(e);if(t===void 0)return null;const r=s(n,"SELECT turn_end_reason FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);return{interrupted:t.turn_closed_at===null||r?.turn_end_reason==="teardown-aborted",progressAt:t.turn_progress_at}}o(Be,"readInterruptedTurn");function Je(e,n){const t=Q(n);return q(e,t),l(r=>Z(r,e,t))}o(Je,"recordFault");function Ke(e,n){const t=L(T(),e);return t===null||n?.link!==void 0&&t.link!==n.link?!1:l(r=>{const i=L(r,e);if(i===null||n?.link!==void 0&&i.link!==n.link)return!1;const d=s(r,"SELECT episode_state FROM node_faults WHERE fault_id = ?").get(i.faultId);s(r,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),d.episode_state==="pending"&&n?.preserveEpisode!==!0&&s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(r,i.faultId);const a=G();return X({level:"info",event:"fault.cleared",operation_id:a,node_id:e,error_class:M(i),fields:{cleared_fault_operation_id:i.operation_id,link:i.link,op:i.op,kind:i.kind,retry:i.retry,message:i.message,since:i.since,...i.anchorEntryId===void 0?{}:{anchor_entry_id:i.anchorEntryId}}}),!0})}o(Ke,"clearFault");function ee(e){return L(T(),e)}o(ee,"readFault");function Ye(e,n){const t=Q(n.input);if(t.link!=="pi\u2192provider"||t.retry.disposition!=="auto"||t.retry.by!=="daemon")throw new Error("provider retry episode requires an automatic daemon provider fault");return q(e,t),l(r=>{const i=m(r,e,n.sessionId);if(i===null)return null;const d=L(r,e);if(d!==null&&(d.link!==t.link||d.kind!==t.kind)&&R(d)>R(t))return null;const a=s(r,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_state IS NOT NULL").all(e);s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_state IS NOT NULL").run(c(),e);for(const E of a)D(r,E.fault_id);const u=Z(r,e,t,{state:"pending",bindingId:i});return u===null?null:(s(r,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run("pending",i,c(),u.faultId),u)})}o(Ye,"recordPendingEpisode");function Ge(e,n){return l(t=>{if(s(t,"SELECT 1 FROM nodes WHERE node_id = ? AND intent IS 'refresh'").get(e)!==void 0)return!1;const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'admitted', retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, updated_at = ? WHERE node_id = ? AND fault_id = ? AND episode_state = 'pending' AND episode_binding_id = ?").run(n.retry.disposition,n.retry.by??null,n.retry.attempt??null,n.retry.max??null,n.retry.nextAt??null,c(),e,n.faultId,r).changes>0})}o(Ge,"admitEpisode");function Ve(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'pending', updated_at = ? WHERE node_id = ? AND episode_state = 'admitted' AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(Ve,"reopenEpisode");function Qe(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'invalidated', updated_at = ? WHERE node_id = ? AND episode_state != 'invalidated' AND episode_state IS NOT NULL AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(Qe,"invalidateEpisode");function qe(e,n){return l(t=>{const r=m(t,e,n.sessionId);if(r===null)return!1;const i=s(t,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").all(e,r);if(i.length===0)return!1;s(t,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").run(c(),e,r);for(const d of i)D(t,d.fault_id);return!0})}o(qe,"clearEpisode");function ze(e){const n=s(T(),"SELECT f.*, s.session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.node_id = ? AND f.episode_state IS NOT NULL").get(e);return n===void 0?null:{state:n.episode_state,sessionId:n.session_id,fault:H(n)}}o(ze,"readEpisode");function Ze(){return s(T(),"SELECT f.*, s.session_id AS bound_session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.active = 1 AND f.episode_state = 'pending' AND s.unbound_at IS NULL AND s.session_id IS NOT NULL AND f.link = 'pi\u2192provider' AND f.retry_disposition = 'auto' AND f.retry_by = 'daemon' AND f.kind IN ('connection', 'rate-limit', 'overloaded')").all().map(n=>({nodeId:n.node_id,sessionId:n.bound_session_id,fault:H(n),updatedAt:n.updated_at}))}o(Ze,"listPendingProviderRetries");function en(e){const n=ee(e)===null;return{explanation(){if(!n)return null;const t=ee(e);if(t===null||t.link!=="crtr\u2192pi"||t.retry.disposition!=="fatal")return null;const r=t.op.trim(),i=t.message.trim();return r===""||i===""?null:`${r}: ${i}`}}}o(en,"beginBootFaultAttempt");function ne(e,n){try{return de(e).mtime.toISOString()}catch{return n}}o(ne,"markerMtime");function P(e,n){if(!p(e))return null;try{return oe(e,"utf8")}catch(t){if(!n)return null;throw new Error(`cannot import recovery input ${e}: ${t.message}`)}}o(P,"legacyFile");function Se(e,n){return s(e,"SELECT execution_id, started_at FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(n)??null}o(Se,"latestExecution");function xe(e,n){const r=J(e,"nodes").has("launched_at")?s(e,"SELECT COALESCE(launched_at, created) AS started_at FROM nodes WHERE node_id = ?").get(n):s(e,"SELECT created AS started_at FROM nodes WHERE node_id = ?").get(n);if(r===void 0)throw new Error(`cannot import recovery witness for unknown node ${n}`);const i=b();return s(e,"INSERT INTO executions (execution_id, node_id, daemon_epoch, pid, process_identity, state, started_at, ended_at, exit_reason) VALUES (?, ?, NULL, NULL, NULL, 'exited', ?, ?, 'pre-consolidation')").run(i,n,r.started_at,r.started_at),{execution_id:i,started_at:r.started_at}}o(xe,"syntheticExecution");function te(e,n,t,r,i){return W(e,n,t,r,i)}o(te,"insertMigratedFault");function Ie(e,n,t,r){const i=s(e,"SELECT binding_id, node_id FROM sessions WHERE transcript_path = ? AND unbound_at IS NULL").get(t);if(i!==void 0){if(i.node_id!==n)throw new Error(`cannot import provider retry ${r}: transcript is live-owned by ${i.node_id}`);return i.binding_id}const d=s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND transcript_path = ? ORDER BY rowid DESC LIMIT 1").get(n,t);if(d!==void 0)return d.binding_id;const a=s(e,"SELECT created FROM nodes WHERE node_id = ?").get(n);if(a===void 0)throw new Error(`cannot import provider retry ${r}: unknown node ${n}`);const u=b();return s(e,"INSERT INTO sessions (binding_id, session_id, node_id, transcript_path, source_session_id, fork_entry_id, created_at, bound_at, unbound_at) VALUES (?, NULL, ?, ?, NULL, NULL, ?, ?, ?)").run(u,n,t,a.created,a.created,c()),u}o(Ie,"migrationBinding");function nn(e){if(Ne(e),ge(e),!p(C()))return;const n=ie(C(),{withFileTypes:!0}).filter(r=>r.isDirectory()).sort((r,i)=>r.name.localeCompare(i.name)),t=s(e,"SELECT 1 FROM nodes WHERE node_id = ?");for(const r of n){const i=r.name,d=ce(i),a=N(d,"busy"),u=N(d,"clean-abort"),E=N(d,"transport-poison.json"),h=N(d,"fault"),A=N(d,"provider-retry"),O=p(a),U=p(u);if(!O&&!U&&!p(E)&&!p(h)&&!p(A)||!p(N(C(),i,"meta.json"))&&t.get(i)===void 0)continue;const k=P(E,!1),S=P(h,!0),x=P(A,!0);if(!O&&!U&&k===null&&S===null&&x===null)continue;let f=Se(e,i);if(f===null&&(f=xe(e,i)),O){const _=ne(a,f.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ?").run(_,_,f.execution_id)}else if(U){const _=ne(u,f.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = ?, turn_end_reason = 'teardown-aborted' WHERE execution_id = ?").run(_,_,_,f.execution_id)}if(k!==null)try{const _=JSON.parse(k);Number.isInteger(_.consecutivePasses)&&_.consecutivePasses>=1&&Array.isArray(_.routes)&&_.routes.length>0&&_.routes.every(w=>typeof w=="string"&&w!=="")&&s(e,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ?").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:_.consecutivePasses,routes:_.routes}}),f.execution_id)}catch{}const I=S===null?null:Y(S);if(S!==null&&I===null)throw new Error(`cannot import recovery fault ${h}`);const y=x===null?null:me(x);if(x!==null&&y===null)throw new Error(`cannot import provider retry ${A}`);let v=I===null||I.link==="broker\u2194crtrd"?null:te(e,i,I,!0);if(y!==null){const _=Ie(e,i,y.sessionFile,A);v!==null&&v.since===y.fault.since?s(e,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run(y.state,_,c(),v.faultId):te(e,i,y.fault,!1,{state:y.state,bindingId:_})}}}o(nn,"migrateRecoveryWitnesses");export{We as FAULT_QUIET_MS,Ge as admitEpisode,en as beginBootFaultAttempt,qe as clearEpisode,Ke as clearFault,Me as closeTurn,H as hydrateFault,Qe as invalidateEpisode,Ze as listPendingProviderRetries,nn as migrateRecoveryWitnesses,Pe as openTurn,K as parseProviderError,ze as readEpisode,ee as readFault,Be as readInterruptedTurn,$e as readLiveTurn,je as readTransportPoisonVerdict,Je as recordFault,Ye as recordPendingEpisode,Xe as recordTransportPoisonVerdict,we as recordTurnProgress,Ve as reopenEpisode};