@crouton-kit/crouter 0.3.32 → 0.3.34

package/dist/daemon/crtrd.js

@@ -10,8 +10,8 @@
 // reaps a node on pane/window state. All node liveness routes through
 // handleNodeLiveness:
 //   • Poll every intervalMs (default 2000ms).
-//   • pid alive → leave; but enforce a stuck refresh-yield (§H) and a
-//     wedged-engine error-stall (§I, see below).
+//   • pid alive → leave; but enforce a stuck refresh-yield and active-fault
+//     recovery when a parked broker is due.
 //   • pid null + pi_session_id set → relaunch in flight; leave.
 //   • pid null + pi_session_id null → after a boot grace, crash + surface boot
 //     failure (the broker exited before session_start).
@@ -28,35 +28,46 @@
 // grace, it force-kills the engine itself (handleYieldStall) so the dead-pid
 // refresh path then revives it fresh.
 //
-// §I error-stall: an engine whose turn ends with stopReason 'error' (e.g. a
-// provider outage exhausted the SDK retry budget) is left PARKED by the
-// stophook ("stay alive for re-steering") — but a headless node has no human to
-// re-steer it, so the broker stays alive, status stays 'active', and the pid-only
-// liveness check reads it healthy FOREVER (gh issue #4: a 9h zombie). The daemon
-// is the authority here too: when a live, not-busy node with no pending intent
-// has a session file that is QUIET past a grace AND whose last assistant message
-// stopped with 'error', it kills the engine (handleErrorStall) so the ordinary
-// dead-pid crash path grace-revives RESUME on the saved session — the exact
-// manual recovery (kill broker + canvas revive) that fixes these by hand.
+// An engine whose turn ends with stopReason 'error' (e.g. a provider outage
+// exhausted the SDK retry budget) is left parked by the stophook ("stay alive
+// for re-steering") — but a headless node has no human to re-steer it, so the
+// broker stays alive, status stays 'active', and the pid-only liveness check
+// reads it healthy forever. The daemon is the authority here too: when a live,
+// not-busy node with no pending intent has a session file that is quiet past
+// the recovery window and whose last assistant message stops with 'error', the
+// daemon keeps the live broker parked and retries it via the inbox-based
+// fault-recovery pass so the ordinary dead-pid crash path never needs an
+// out-of-band launch.
+// dead-pid crash path grace-revives RESUME on the saved session — the manual
+// recovery (kill broker + canvas revive) that fixes these by hand.
 //
 // Single-instance guarantee
-//   A PID file at crtrHome()/crtrd.pid prevents double-runs. On start, if the
-//   file exists and the recorded pid is alive, we refuse to start (exit 0).
-//   On stop (SIGINT/SIGTERM/exit) we remove the file.
-import { writeFileSync, readFileSync, rmSync, existsSync, mkdirSync, statSync, openSync, readSync, closeSync, } from 'node:fs';
-import { join } from 'node:path';
+//   A PID file prevents double-runs. On start, if the file exists and the
+//   recorded pid is alive, we refuse to start (exit 0). Hearth guests set
+//   CRTR_PIDFILE to keep this process-local on recreate; otherwise it defaults
+//   to crtrHome()/crtrd.pid. On stop (SIGINT/SIGTERM/exit) we remove the file.
+import { writeFileSync, readFileSync, rmSync, existsSync, mkdirSync, statSync, } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { homedir } from 'node:os';
 import { crtrHome, nodeDir, jobDir } from '../core/canvas/paths.js';
-import { listNodes, getRow, getNode, dueWakes, consumeWake, advanceWake, listFocuses, closeFocusRow, } from '../core/canvas/index.js';
+import { listNodes, getRow, getNode, dueClockTriggers, consumeTrigger, advanceTrigger, listFocuses, closeFocusRow, pruneToLimit, } from '../core/canvas/index.js';
 import { transition } from '../core/runtime/lifecycle.js';
 import { isBusy } from '../core/runtime/busy.js';
+import { FAULT_QUIET_MS } from '../core/runtime/fault.js';
+import { logger } from '../core/log.js';
+export { FAULT_QUIET_MS };
 import { isPidAlive } from '../core/canvas/pid.js';
+import { reconcileBootLiveness } from '../core/canvas/boot.js';
 import { listLivePanes, tearDownNode } from '../core/runtime/placement.js';
 import { reviveNode } from '../core/runtime/revive.js';
+import { isBrokerLive } from '../core/runtime/model-swap.js';
+import { reloadAuthLive } from '../core/runtime/auth-reload.js';
 import { spawnChild } from '../core/runtime/spawn.js';
 import { wakeOriginFrom } from '../core/runtime/bearings.js';
 import { pushUrgent } from '../core/feed/feed.js';
 import { appendInbox, readInboxSince, readCursor } from '../core/feed/inbox.js';
 import { nextSlotAfter } from '../core/wake.js';
+import { evaluateDuePredicates, reapTimedOutPredicates, recoverStalePredicateLeases, predicateCapacity, } from './predicate-eval.js';
 /** Surface a vehicle that never booted.
  *
  *  `spawnChild` returns status="active" the instant the tmux window opens — it
@@ -78,6 +89,19 @@ async function surfaceBootFailure(meta) {
     await pushUrgent(meta.node_id, body, { from: meta.node_id });
 }
 const DEFAULT_INTERVAL_MS = 2000;
+// Count-based auto-prune of the canvas (pruneToLimit). The canvas grows
+// unbounded — the TTL `crtr canvas prune` is manual, so without this a long-lived
+// daemon's canvas accumulates thousands of terminal nodes, making every scan +
+// `canvas snapshot` expensive. Keep the AUTO_PRUNE_MAX most-recently-active nodes
+// (live nodes are NEVER deleted, so the real floor is max(AUTO_PRUNE_MAX,
+// liveCount)). It's a DB sweep, not per-tick work — throttled to PRUNE_INTERVAL_MS.
+const AUTO_PRUNE_MAX = 2000;
+const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
+// Last auto-prune wall-clock. -Infinity so the FIRST tick after start runs it
+// (the canvas may already be huge at boot); thereafter every PRUNE_INTERVAL_MS.
+// In-memory only — a daemon restart re-runs it once, which is harmless.
+let lastPruneAt = Number.NEGATIVE_INFINITY;
+const log = logger('crtrd');
 // How long a node's broker pid may be observed dead before the daemon revives
 // it. MUST exceed worst-case broker boot time: a refresh / crash-revive
 // transiently shows a dead OLD pid for the gap between the old broker exiting
@@ -106,13 +130,13 @@ const KILL_ESCALATE_MS = 20_000;
 // unhealthySince — a daemon restart resets them (worst case: one extra grace).
 const yieldStallSince = new Map();
 const yieldTermAt = new Map();
-// Wake-failure dedup latch (third pass): wakeup_ids we've already fail-loud
+// Trigger-failure dedup latch (third pass): trigger_ids we've already fail-loud
 // notified about, so a permanently-broken spawn-cron (or a quarantined recurrence)
 // notifies the armer ONCE and then stays quiet until the next success or cancel —
 // the unrotated, append-only inbox.jsonl must not be flooded (Min-9). In-memory
 // only, like unhealthySince; a daemon restart resets it (worst case: one repeat
 // notice). A successful spawn clears its entry so a future failure re-notifies.
-const notifiedWakeFailures = new Set();
+const notifiedTriggerFailures = new Set();
 /** Refresh-stall decision (§H refresh-authority; audit 2026-06-09, Bug 1 — node
  *  mq5v9hfa-74493d45 and siblings; stuck-refresh — parent mq7s5o93): a `node
  *  yield` / refresh records intent='refresh' and relies on the in-process
@@ -184,151 +208,77 @@ function handleYieldStall(row, pid, now) {
         }
     }
 }
-// §I error-stall grace: how long a node's session .jsonl must have been QUIET
-// (no append — mtime is the clock, so it survives daemon restarts) before a
-// trailing-error engine is declared wedged and killed. Generous enough that an
-// in-flight recovery (the human re-steering, a parent's message waking a new
-// turn via the live inbox-watcher — both of which append and reset the clock)
-// always wins, while a true zombie recovers in minutes instead of never.
-export const ERROR_STALL_QUIET_MS = 5 * 60_000;
-// Per-node SIGTERM timestamps for §I kill escalation, mirroring yieldTermAt.
-const errorTermAt = new Map();
-// Memoized tail inspection per node, keyed on the session file's mtime so the
-// (rare) 64KB tail read happens once per file change, not once per 2s tick.
-const errorTailMemo = new Map();
-const TAIL_BYTES = 64 * 1024;
-/** Read the last TAIL_BYTES of a file. null on any failure (absent file reads
- *  as "nothing to inspect", never as evidence of a wedge). */
-function readTail(path) {
-    try {
-        const size = statSync(path).size;
-        const len = Math.min(TAIL_BYTES, size);
-        if (len === 0)
-            return null;
-        const fd = openSync(path, 'r');
-        try {
-            const buf = Buffer.alloc(len);
-            readSync(fd, buf, 0, len, size - len);
-            return buf.toString('utf8');
-        }
-        finally {
-            closeSync(fd);
-        }
-    }
-    catch {
-        return null;
-    }
-}
-/** The §I wedge signature, from a session-jsonl tail: the LAST assistant
- *  message stopped with 'error' (retry budget exhausted — pi appends the error
- *  turn and parks). Returns its errorMessage for the daemon log (the "why it
- *  wedged" trace), or null when the last assistant turn ended any other way —
- *  'stop'/'length' is a healthy park, 'aborted' is a human Esc, and trailing
- *  non-assistant lines (toolResults, an injected user message) are walked past.
- *  The first line of a mid-file tail cut fails JSON.parse and is skipped. */
-export function trailingEngineError(tail) {
-    if (tail === null)
-        return null;
-    const lines = tail.split('\n');
-    for (let i = lines.length - 1; i >= 0; i--) {
-        const line = lines[i].trim();
-        if (line === '')
-            continue;
-        let msg;
-        try {
-            msg = JSON.parse(line)?.message;
-        }
-        catch {
-            continue; // a cut fragment (tail boundary) or non-JSON line
-        }
-        if (msg?.role !== 'assistant')
-            continue;
-        if (msg.stopReason !== 'error')
-            return null;
-        return typeof msg.errorMessage === 'string' && msg.errorMessage !== ''
-            ? msg.errorMessage
-            : 'engine error (no errorMessage recorded)';
-    }
-    return null;
+// ---------------------------------------------------------------------------
+// Auth-reload fan (one /login → every live broker follows)
+// ---------------------------------------------------------------------------
+/** Path of the shared pi credentials file every broker reads. VENDORED from pi
+ *  `config.getAuthPath()` (= `<agentDir>/auth.json`) the same way spawn.ts
+ *  vendors the sessions root: a ROOT import of pi's `getAgentDir` would eager-
+ *  load the heavy pi SDK index, so we mirror its resolution — `PI_CODING_AGENT_DIR`
+ *  env, else `~/.pi/agent`. Re-sync on a pi SDK bump that moves auth.json. */
+function piAuthPath() {
+    const env = process.env['PI_CODING_AGENT_DIR'];
+    const agentDir = env !== undefined && env !== '' ? env : join(homedir(), '.pi', 'agent');
+    return join(agentDir, 'auth.json');
 }
-/** Error-stall decision (§I; gh issue #4 — node mqaeqzdu-0746b3b7, a 9h zombie
- *  after an Anthropic outage). Pure, mirroring yieldStallVerdict; the kill side
- *  effects live in handleErrorStall.
+// Last-observed auth.json mtime, for the daemon's change-watch. Lazily seeded on
+// the first poll that sees the file (so daemon start never fans a spurious
+// reload), then updated on every detected change. In-memory only — a daemon
+// restart re-seeds from the current file, so at most one missed change spans the
+// restart gap (re-login during a ~1s restart), which a human just redoes.
+let knownAuthMtimeMs = null;
+/** Watch the shared auth.json mtime and, on a change, fan a live `reload_auth`
+ *  to every live broker so one `/login` propagates across the whole canvas. The
+ *  broker that performed the login already self-reloaded; re-reloading it is
+ *  harmless (authStorage.reload() is idempotent), so we don't exclude it.
  *
- *  'kill' ONLY when ALL of: engine alive, NO pending intent (refresh is §H's
- *  domain; idle-release chose dormancy), the turn is over (busy absent — a node
- *  wedged mid-bash must be recovered by killing the subprocess, never the node),
- *  the session file has been quiet past ERROR_STALL_QUIET_MS, and its last
- *  assistant message stopped with 'error'. */
-export function errorStallVerdict(piPidAlive, intent, busy, quietForMs, trailingError) {
-    if (piPidAlive !== true)
-        return 'leave';
-    if (intent !== null)
-        return 'leave';
-    if (busy)
-        return 'leave';
-    if (quietForMs === null || quietForMs < ERROR_STALL_QUIET_MS)
-        return 'leave';
-    if (!trailingError)
-        return 'leave';
-    return 'kill';
-}
-/** Enact an error-stall (§I): when the verdict is 'kill', SIGTERM the parked
- *  engine (escalating to SIGKILL after KILL_ESCALATE_MS, like §H — a broker
- *  wedged after an outage may not run graceful handlers). intent is null, so
- *  once the pid dies the ordinary crash branch in handleNodeLiveness grace-
- *  revives RESUME on the saved session — the conversation is preserved and the
- *  revive kickoff prompts the node to continue. No grace clock of our own: the
- *  session file's mtime IS the clock (any append — a re-steer, an inbox-watcher
- *  wake, a new turn — resets it). */
-function handleErrorStall(row, pid, now) {
-    const id = row.node_id;
-    if (row.intent !== null || isBusy(id)) {
-        errorTermAt.delete(id);
-        return;
-    }
-    const file = getNode(id)?.pi_session_file;
-    if (file == null)
-        return;
+ *  Rides the existing ~2s poll, so several rapid writes from one login collapse
+ *  to a single observed mtime change → a single fan. Each broker is reloaded
+ *  under its own try/catch (Promise.allSettled): a broker mid-shutdown or whose
+ *  socket refuses connect is logged and skipped, never aborting the others. */
+async function handleAuthReload(now) {
     let mtimeMs;
     try {
-        mtimeMs = statSync(file).mtimeMs;
+        mtimeMs = statSync(piAuthPath()).mtimeMs;
     }
     catch {
-        return; // session file unreadable — no evidence, never kill on "can't tell"
+        return; // no auth.json yet (never logged in) — nothing to watch
     }
-    const quietFor = now - mtimeMs;
-    if (quietFor < ERROR_STALL_QUIET_MS) {
-        errorTermAt.delete(id);
+    if (knownAuthMtimeMs === null) {
+        knownAuthMtimeMs = mtimeMs; // seed on first sight; never fan on daemon start
         return;
     }
-    let memo = errorTailMemo.get(id);
-    if (memo === undefined || memo.mtimeMs !== mtimeMs) {
-        memo = { mtimeMs, error: trailingEngineError(readTail(file)) };
-        errorTailMemo.set(id, memo);
+    if (mtimeMs === knownAuthMtimeMs)
+        return;
+    knownAuthMtimeMs = mtimeMs;
+    // Enumerate live brokers (pid alive + view.sock present) only when auth
+    // actually changed — the rare path, so the extra list/stat cost is paid only
+    // on a real login, never every tick.
+    let live;
+    try {
+        live = listNodes({ status: ['active', 'idle'] })
+            .filter((r) => r.kind !== 'human')
+            .map((r) => getNode(r.node_id))
+            .filter((m) => m !== null && isBrokerLive(m))
+            .map((m) => m.node_id);
     }
-    if (errorStallVerdict(true, row.intent, false, quietFor, memo.error !== null) !== 'kill')
+    catch (err) {
+        process.stderr.write(`[crtrd] auth-reload enumerate error: ${err.message}\n`);
         return;
-    const termed = errorTermAt.get(id);
-    if (termed === undefined) {
-        process.stderr.write(`[crtrd] kill ${id} (error-stall: engine parked on a trailing error turn, session quiet ${Math.round(quietFor / 1000)}s — SIGTERM; last error: ${String(memo.error).slice(0, 200)})\n`);
-        try {
-            process.kill(pid, 'SIGTERM');
-        }
-        catch {
-            /* already gone */
-        }
-        errorTermAt.set(id, now);
     }
-    else if (now - termed >= KILL_ESCALATE_MS) {
-        process.stderr.write(`[crtrd] kill ${id} (error-stall: pi ${pid} survived SIGTERM — SIGKILL)\n`);
+    if (live.length === 0)
+        return;
+    const results = await Promise.allSettled(live.map(async (id) => {
         try {
-            process.kill(pid, 'SIGKILL');
+            await reloadAuthLive(id);
         }
-        catch {
-            /* already gone */
+        catch (err) {
+            process.stderr.write(`[crtrd] auth-reload ${id} failed: ${err.message}\n`);
+            throw err;
         }
-    }
+    }));
+    const ok = results.filter((r) => r.status === 'fulfilled').length;
+    process.stderr.write(`[crtrd] auth-reload: auth.json changed — fanned to ${live.length} live broker(s) (${ok} ok, ${live.length - ok} failed)\n`);
 }
 /** Decide what to do with a node whose engine pid is DEAD, from how long it's
  *  been dead. Pure — the time/revive side effects live in handleNodeLiveness;
@@ -354,12 +304,19 @@ export function livenessVerdict(piPidAlive, deadFor) {
  *      with the turn over) and, once the engine has sat alive past the grace,
  *      kills it — so the daemon, not the (possibly stale) in-process stophook, is
  *      the authority on refresh, and the dead-pid refresh branch below then
- *      revives it fresh. AND enforce §I: handleErrorStall self-guards (no-op
- *      unless intent is null with the turn over) and kills an engine parked on a
- *      trailing-error turn whose session has been quiet past the grace, so the
- *      crash branch below grace-revives it RESUME.
+ *      revives it fresh. Fault recovery now runs in a separate pass: live,
+ *      daemon-owned provider faults are retried by inbox nudge, while dead-pid
+ *      crashes still fall through the ordinary resume path below.
+ *    • pid null + intent==='idle-release' → dormant by choice; its pid may have
+ *      just been cleared by boot reconciliation, but a null pid here is still
+ *      that chosen dormancy, not a stranded relaunch — leave it, the second pass
+ *      revives (resume) on the next unseen inbox entry (mirrors the dead-pid
+ *      idle-release case).
  *    • pid null + pi_session_id set (a relaunch in flight — reviveNode clears
- *      pi_pid right after launch) → leave; the fresh engine re-records its pid.
+ *      pi_pid right after launch) → leave through a boot grace; a healthy engine
+ *      re-records its pid well within it. If the grace ELAPSES with still no pid,
+ *      the relaunch died after the pid-clear → grace-revive RESUME on the saved
+ *      session (else the node strands 'active' with no engine forever).
  *    • pid null + pi_session_id null (NEVER booted) → normally the sub-second SDK
  *      boot gap, but a broker that throws BEFORE session_start records no pid and
  *      no session ever — so after a boot grace with STILL nothing, crash +
@@ -380,37 +337,62 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
     if (pid != null && isPidAlive(pid)) {
         unhealthySince.delete(id);
         handleYieldStall(row, pid, now);
-        handleErrorStall(row, pid, now);
         return;
     }
     if (pid == null) {
-        // No supervised pid recorded. Two very different cases turn on pi_session_id:
-        //   • a relaunch in flight — reviveNode clears pi_pid right after launch, but
-        //     the node ALREADY booted once (pi_session_id captured), so the fresh
-        //     engine re-records its pid within a tick or two; leave it.
-        //   • a NEVER-BOOTED broker (pi_session_id null) — normally the sub-second SDK
-        //     boot gap, BUT a broker that THROWS before session_start (malformed
-        //     broker-launch.json, SessionManager.open on a missing .jsonl, a loader/
-        //     registry/createAgentSession failure, the fork / bare-id guards, or
-        //     broker-cli's own fatal catch) records NO pid and NO session — EVER.
-        //     With pid==null read unconditionally as "still booting" that strands the
-        //     node 'active' with no engine forever and its parent waits on a dead
-        //     child. After a boot grace with STILL no pid AND no session, crash +
+        // No supervised pid recorded. BOTH sub-cases run the SAME boot grace clock and
+        // differ only in how it ENDS, keyed on pi_session_id:
+        //   • session SET — either a relaunch in flight (reviveNode clears pi_pid right
+        //     after launch; the fresh engine re-records within a tick or two, well
+        //     inside the grace, and the pid-alive branch above clears the clock) OR a
+        //     relaunch whose broker DIED after the pid-clear and before it re-recorded.
+        //     The healthy case never reaches the grace; if the grace DOES elapse with
+        //     still no pid, the relaunch is dead → grace-revive RESUME on the saved
+        //     session. Returning unconditionally here (the old behavior) read null+
+        //     session as "in flight" FOREVER and stranded such a node 'active' with no
+        //     engine — its parent waiting on a dead child indefinitely.
+        //   • session NULL — normally the sub-second SDK boot gap, BUT a broker that
+        //     THROWS before session_start (malformed broker-launch.json, SessionManager
+        //     .open on a missing .jsonl, a loader/registry/createAgentSession failure,
+        //     the fork / bare-id guards, or broker-cli's own fatal catch) records NO
+        //     pid and NO session — EVER. After the grace with STILL nothing, crash +
         //     surfaceBootFailure up the spine (M-1).
+        // The 20s grace doubles as the double-spawn guard: a healthy relaunch always
+        // re-records its pid well within it, so neither branch can fire on one.
         const meta = getNode(id);
-        if (meta === null || meta.pi_session_id != null) {
-            unhealthySince.delete(id); // relaunch in flight (or identity already bound)
+        if (meta === null) {
+            unhealthySince.delete(id); // row gone from meta — nothing to supervise
+            return;
+        }
+        // Dormant by choice: an idle-release node freed itself while dormant. Boot
+        // reconciliation may have just cleared its recorded pid to null, but a null
+        // pid here is still that chosen dormancy — NOT a stranded relaunch and NOT a
+        // failed boot. Honor it exactly like the dead-pid idle-release branch below:
+        // leave it idle; the second pass revives it (resume) the moment its inbox
+        // gains an unseen entry. Without this, boot reconciliation clearing the pid
+        // would drop this node into the boot-grace path and spuriously wake it after
+        // the grace with no work waiting.
+        if (row.intent === 'idle-release') {
+            unhealthySince.delete(id);
             return;
         }
         const since = unhealthySince.get(id);
         if (since === undefined) {
-            unhealthySince.set(id, now); // start the boot-grace clock
+            unhealthySince.set(id, now); // start the boot grace clock
             return;
         }
         if (now - since < REVIVE_GRACE_MS)
             return; // still inside the boot grace
-        // Boot grace elapsed, still no pid and no session → the broker never booted.
         unhealthySince.delete(id);
+        if (meta.pi_session_id != null) {
+            // Grace elapsed with a session but still no pid → a relaunch that died after
+            // the pid-clear, before re-record. Resume it on the saved session.
+            process.stderr.write(`[crtrd] revive ${id} (stranded relaunch — pid never re-recorded)\n`);
+            reviveNode(id, { resume: true });
+            revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
+            return;
+        }
+        // Grace elapsed with no pid AND no session → the broker never booted.
         process.stderr.write(`[crtrd] boot-failed ${id} (broker exited before session_start)\n`);
         transition(id, 'crash');
         try {
@@ -421,12 +403,10 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
         }
         return;
     }
-    // The engine pid is DEAD — any pending refresh/error-stall clock is moot (the
-    // dead-pid paths below own recovery now). Branch on intent.
+    // The engine pid is DEAD — the dead-pid paths below own recovery now. Branch
+    // on intent.
     yieldStallSince.delete(id);
     yieldTermAt.delete(id);
-    errorTermAt.delete(id);
-    errorTailMemo.delete(id);
     if (row.intent === 'refresh') {
         // Clean yield — or the §H force-kill above just landed — → respawn FRESH,
         // immediately (no grace wait).
@@ -456,19 +436,20 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
     reviveNode(id, { resume: true });
     revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
 }
-/** Fail loud for a drifted/broken wake (design §6.6/Q5): wake the ARMER DIRECTLY
- *  via appendInbox — NOT pushUrgent, which fans to subscribersOf(owner), and a
- *  DETACHED spawn has no subscribers (so the armer would never be reached); the
- *  literal {from:null} is also a type error for pushUrgent. Deduped per wakeup_id
- *  (notify once, suppress until the next success/cancel via notifiedWakeFailures)
- *  so a broken spawn-cron can't flood the unrotated inbox.jsonl. Falls back to a
- *  loud daemon-log line when the owner no longer resolves to a node. */
-function failLoudWake(w, label, body) {
-    if (notifiedWakeFailures.has(w.wakeup_id))
+/** Fail loud for a drifted/broken trigger (design §6.6/Q5): wake the ARMER
+ *  DIRECTLY via appendInbox — NOT pushUrgent, which fans to subscribersOf(owner),
+ *  and a DETACHED spawn has no subscribers (so the armer would never be
+ *  reached); the literal {from:null} is also a type error for pushUrgent.
+ *  Deduped per trigger_id (notify once, suppress until the next success/cancel
+ *  via notifiedTriggerFailures) so a broken spawn-cron can't flood the
+ *  unrotated inbox.jsonl. Falls back to a loud daemon-log line when the owner
+ *  no longer resolves to a node. */
+function failLoudTrigger(t, label, body) {
+    if (notifiedTriggerFailures.has(t.trigger_id))
         return;
-    notifiedWakeFailures.add(w.wakeup_id);
-    if (getRow(w.owner_id) !== null) {
-        appendInbox(w.owner_id, {
+    notifiedTriggerFailures.add(t.trigger_id);
+    if (getRow(t.owner_id) !== null) {
+        appendInbox(t.owner_id, {
             from: null,
             tier: 'urgent',
             kind: 'urgent',
@@ -477,7 +458,63 @@ function failLoudWake(w, label, body) {
         });
     }
     else {
-        process.stderr.write(`[crtrd] wake ${w.wakeup_id} failed (owner ${w.owner_id} gone): ${body}\n`);
+        process.stderr.write(`[crtrd] trigger ${t.trigger_id} failed (owner ${t.owner_id} gone): ${body}\n`);
+    }
+}
+/** Notify the owner once for a settled (already-consumed) predicate outcome —
+ *  never deduped like failLoudTrigger, because the trigger is gone by the time
+ *  this fires so there is no repeat-notification risk to guard against. */
+function notifyOwnerUrgent(t, label, body) {
+    if (getRow(t.owner_id) !== null) {
+        appendInbox(t.owner_id, { from: null, tier: 'urgent', kind: 'urgent', label, data: { body } });
+    }
+    else {
+        process.stderr.write(`[crtrd] trigger ${t.trigger_id} notice undelivered (owner ${t.owner_id} gone): ${body}\n`);
+    }
+}
+/** Settle a predicate outcome onto the SAME enactment paths pass 3 uses
+ *  (appendInbox / spawnChild) — fresh_revive/deadline_message never reach here
+ *  (the command surface never pairs `--when` with those action_modes; see the
+ *  spec's valid flag matrix). stdout_overflow/target_gone are terminal
+ *  failures the trigger was already consumed for by predicate-eval.ts —  this
+ *  only sends the owner notice. */
+async function handlePredicateOutcome(outcome) {
+    const t = outcome.trigger;
+    if (outcome.kind === 'stdout_overflow') {
+        notifyOwnerUrgent(t, '⚠ predicate stdout too large', `Trigger ${t.trigger_id}'s predicate exceeded the 16 KiB stdout limit — the action was not delivered and the trigger has been canceled.`);
+        return;
+    }
+    if (outcome.kind === 'target_gone') {
+        notifyOwnerUrgent(t, '⚠ predicate target gone', `Trigger ${t.trigger_id}'s target no longer exists — the trigger has been canceled.`);
+        return;
+    }
+    // outcome.kind === 'fired'
+    if (t.action === 'msg' && t.action_mode === 'inbox_message') {
+        const p = t.payload;
+        const body = (p.body ?? outcome.stdout).trim();
+        if (body === '') {
+            notifyOwnerUrgent(t, '⚠ predicate fired with no message', `Trigger ${t.trigger_id}'s predicate exited 0 but produced no body (no supplied body, empty stdout) — nothing was delivered.`);
+            return;
+        }
+        if (t.node_id == null)
+            return; // integrity backstop only — armTrigger requires a target here
+        appendInbox(t.node_id, {
+            from: t.owner_id ?? null,
+            tier: p.tier ?? 'normal',
+            kind: 'message',
+            label: `⏰ predicate fired — ${p.label}`,
+            data: { body },
+        });
+    }
+    else if (t.action === 'new' && t.action_mode === 'node_birth') {
+        const recipe = t.payload;
+        try {
+            await spawnChild({ ...recipe, wakeOrigin: wakeOriginFrom(t) });
+        }
+        catch (err) {
+            failLoudTrigger(t, '⚠ deferred predicate spawn failed', `Deferred spawn (${recipe.kind} @ ${recipe.cwd}) failed after its predicate fired: ` +
+                `${err.message}\n\nRe-home (fix the cwd/parent) or re-arm the trigger.`);
+        }
     }
 }
 // How long a dead node's on-disk record must have been QUIET (no meta/telemetry
@@ -578,11 +615,12 @@ function gcStaleFocuses() {
 // Pidfile
 // ---------------------------------------------------------------------------
 function pidfilePath() {
-    return join(crtrHome(), 'crtrd.pid');
+    const override = process.env['CRTR_PIDFILE'];
+    return override !== undefined && override !== '' ? override : join(crtrHome(), 'crtrd.pid');
 }
 function writePidfile() {
-    // Ensure the canvas home exists before writing.
-    mkdirSync(crtrHome(), { recursive: true });
+    // Ensure the pidfile parent exists before writing.
+    mkdirSync(dirname(pidfilePath()), { recursive: true });
     writeFileSync(pidfilePath(), String(process.pid), 'utf8');
 }
 function removePidfile() {
@@ -615,7 +653,18 @@ export function isDaemonRunning() {
 // ---------------------------------------------------------------------------
 // Supervisor tick
 // ---------------------------------------------------------------------------
+/** Supervision tick over the daemon's current live set. */
 export async function superviseTick(now = Date.now()) {
+    // Auth-reload fan FIRST, independent of node supervision: a change to the
+    // shared auth.json propagates a live credential reload to every broker. Kept
+    // ahead of the listNodes guard below so a transient list error in the
+    // supervision passes never skips an auth fan (it does its own enumeration).
+    try {
+        await handleAuthReload(now);
+    }
+    catch (err) {
+        process.stderr.write(`[crtrd] handleAuthReload error: ${err.message}\n`);
+    }
     let rows;
     try {
         rows = listNodes({ status: ['active', 'idle'] });
@@ -681,143 +730,142 @@ export async function superviseTick(now = Date.now()) {
             process.stderr.write(`[crtrd] error polling inbox ${row.node_id}: ${err.message}\n`);
         }
     }
-    // Third pass: FIRE DUE SCHEDULED WAKES (the wakeups engine, migration v7).
+    // Third pass: FIRE DUE CLOCK TRIGGERS (the triggers engine, migration v9).
     // Runs immediately AFTER the inbox second pass — the ordering is load-bearing
     // (design §6.4): a same-tick inbox revive runs cancelDeadlinesFor (inside
-    // reviveNode) BEFORE this pass's dueWakes query, so an event that wins a
-    // deadline deletes its row before it can fire. The pass-2 inbox loop above is
-    // otherwise untouched (design D3) — it stays the generic wake-on-message path
-    // that noted/deadline rely on; the only addition is the revivedThisTick.add
-    // observation that the bare branch below needs to avoid a double-spawn.
+    // reviveNode) BEFORE this pass's dueClockTriggers query, so an event that wins
+    // a deadline deletes its row before it can fire. The pass-2 inbox loop above
+    // is otherwise untouched (design D3) — it stays the generic wake-on-message
+    // path that inbox_message/deadline_message rely on; the only addition is the
+    // revivedThisTick.add observation that the fresh_revive branch below needs to
+    // avoid a double-spawn.
     // For each due row, in fire_at order, settle the row FIRST (advance-or-consume
-    // — crash-safety, design D4) THEN enact by kind, wrapped per-row so one bad
-    // wake never kills the tick.
+    // — crash-safety, design D4) THEN enact by action/action_mode, wrapped per-row
+    // so one bad trigger never kills the tick.
     const nowIso = new Date(now).toISOString();
     const nowDate = new Date(now);
     let due;
     try {
-        due = dueWakes(nowIso);
+        due = dueClockTriggers(nowIso);
     }
     catch (err) {
-        process.stderr.write(`[crtrd] dueWakes error: ${err.message}\n`);
+        process.stderr.write(`[crtrd] dueClockTriggers error: ${err.message}\n`);
         due = [];
     }
-    for (const w of due) {
+    for (const t of due) {
         try {
-            // 1. Advance-or-consume FIRST (design D4): settle the row's pending state
-            //    before ANY side effect, so a daemon crash mid-fire can never double-
-            //    fire (a double SPAWN is the worst case — structurally prevented here).
-            if (w.recur == null) {
-                consumeWake(w.wakeup_id);
+            // 1. Settle FIRST: crash-safety — advance/consume the trigger before
+            //    enacting it, so a crash between settle and enact never re-fires.
+            if (t.recur == null) {
+                consumeTrigger(t.trigger_id);
             }
             else {
                 let nextFire;
                 try {
-                    nextFire = nextSlotAfter(w.recur, nowDate);
+                    nextFire = nextSlotAfter(t.recur, nowDate);
                 }
                 catch (err) {
-                    // The recur is unparseable (parseCadence validates at arm time, so this
-                    // is a rare backstop — a corrupted/foreign row). Leaving a past-fire_at
-                    // row pending would re-qualify it EVERY tick (silent CPU spin, enact
-                    // never reached), so quarantine it (consume) + fail loud ONCE; never
-                    // re-query it each tick (Min-8).
-                    consumeWake(w.wakeup_id);
-                    failLoudWake(w, '⚠ recurrence quarantined', `Scheduled wake ${w.wakeup_id} has an unparseable recurrence and was removed: ` +
+                    consumeTrigger(t.trigger_id);
+                    failLoudTrigger(t, '⚠ recurrence quarantined', `Scheduled trigger ${t.trigger_id} has an unparseable recurrence and was removed: ` +
                         `${err.message}\n\nRe-arm it with a valid cadence.`);
                     continue;
                 }
-                advanceWake(w.wakeup_id, nextFire);
+                advanceTrigger(t.trigger_id, nextFire);
             }
-            // 2. Enact by kind.
-            if (w.kind === 'bare') {
-                // A bare wake leaves NO inbox entry, so pass 2 can never wake it — it
-                // must revive DIRECTLY with resume:false (re-reads roadmap; AC-N2).
-                const target = w.node_id == null ? null : getRow(w.node_id);
-                if (target === null)
-                    continue; // target gone (the FK cascade reaped it)
-                // Same-tick double-spawn guard (Maj-4): pass 1/2 (or an earlier bare row
-                // this pass) may have revived this node THIS tick — reviveNode clears
-                // pi_pid to NULL before the fresh pi re-records it, so the pi_pid==null
-                // liveness check below would falsely read "not live" and a second
-                // reviveNode would launch a SECOND pi on the same .jsonl (corruption).
-                // (Pane-existence is the WRONG guard — it would no-op a frozen-focus bare
-                // wake, which is pane-alive/pi-dead by design.)
+            // 2. Enact by action/action_mode.
+            if (t.action === 'msg' && t.action_mode === 'fresh_revive') {
+                const target = t.node_id == null ? null : getRow(t.node_id);
+                if (target === null) {
+                    notifyOwnerUrgent(t, '⚠ scheduled trigger target gone', `Trigger ${t.trigger_id}'s target no longer exists — no revive happened.`);
+                    continue;
+                }
                 if (revivedThisTick.has(target.node_id))
                     continue;
-                // pi live → no-op (AC-E3 bare): an already-awake node needs no revive.
                 if (target.pi_pid != null && isPidAlive(target.pi_pid))
                     continue;
                 process.stderr.write(`[crtrd] wake ${target.node_id} (bare alarm)\n`);
-                // wakeReason carries the timer provenance so the fresh-revive kickoff
-                // leads with a <crtr-wake> block (Invariant B/D: the node learns a clock,
-                // not an event, woke it) instead of a generic context-refresh framing.
-                reviveNode(target.node_id, { resume: false, wakeReason: wakeOriginFrom(w) });
+                reviveNode(target.node_id, { resume: false, wakeReason: wakeOriginFrom(t) });
                 revivedThisTick.add(target.node_id);
             }
-            else if (w.kind === 'noted') {
-                // Deliver the note; the UNMODIFIED pass 2 wakes the dormant target next
-                // tick (AC-N1), or its live in-process watcher delivers it (AC-E3 noted).
-                if (w.node_id == null)
+            else if (t.action === 'msg' && t.action_mode === 'inbox_message') {
+                if (t.node_id == null) {
+                    notifyOwnerUrgent(t, '⚠ scheduled trigger target gone', `Trigger ${t.trigger_id}'s target no longer exists — nothing was delivered.`);
                     continue;
-                const p = w.payload;
-                // Mark the delivery as a SCHEDULED wake (Invariant D): the woken node sees
-                // only the rendered digest (inbox inlines label/body, never arbitrary
-                // data keys), so the timer signal MUST ride the visible label — the same
-                // ⏰ family deadline uses — so a noted wake is distinguishable from a plain
-                // `node msg`, not indistinguishable from it.
-                appendInbox(w.node_id, {
-                    from: w.owner_id ?? null,
-                    tier: 'normal',
+                }
+                const p = t.payload;
+                appendInbox(t.node_id, {
+                    from: t.owner_id ?? null,
+                    tier: p.tier ?? 'normal',
                     kind: 'message',
                     label: `⏰ scheduled wake — ${p.label}`,
-                    data: { body: p.body },
+                    data: { body: p.body ?? '' },
                 });
             }
-            else if (w.kind === 'deadline') {
-                // Same delivery as noted but URGENT tier. The woken node sees only the
-                // RENDERED digest (inbox inlines label/body, NEVER arbitrary data keys),
-                // so the timeout signal MUST ride the visible label/body (Maj-8);
-                // data.timeout is a machine-readable mirror only. body is always non-
-                // empty (the surface supplies a default timeout note when none is
-                // authored, Maj-7). One-shot — already consumed in step 1.
-                if (w.node_id == null)
+            else if (t.action === 'msg' && t.action_mode === 'deadline_message') {
+                if (t.node_id == null) {
+                    notifyOwnerUrgent(t, '⚠ scheduled trigger target gone', `Trigger ${t.trigger_id}'s target no longer exists — nothing was delivered.`);
                     continue;
-                const p = w.payload;
-                appendInbox(w.node_id, {
-                    from: w.owner_id ?? null,
+                }
+                const p = t.payload;
+                appendInbox(t.node_id, {
+                    from: t.owner_id ?? null,
                     tier: 'urgent',
                     kind: 'message',
                     label: `⏰ deadline reached — ${p.label}`,
                     data: { body: p.body, timeout: true },
                 });
             }
-            else if (w.kind === 'spawn') {
-                // Re-derives LaunchSpec live (AC-W2); payload.parent is the non-null
-                // resolved armer id (T2/T7 guarantee it), so spawnChild never throws the
-                // "requires a calling node" error even for a --root recipe.
-                const recipe = w.payload;
+            else if (t.action === 'new' && t.action_mode === 'node_birth') {
+                const recipe = t.payload;
                 try {
-                    // Spread in the wake provenance at fire time (in-memory only; never the
-                    // stored recipe) so the born node's kickoff leads with a <crtr-wake>
-                    // block — it learns, by construction, that a timer birthed it (and, for
-                    // a spawn-cron, that it is one run of a standing job).
-                    spawnChild({ ...recipe, wakeOrigin: wakeOriginFrom(w) });
-                    notifiedWakeFailures.delete(w.wakeup_id); // success clears the dedup latch
+                    await spawnChild({ ...recipe, wakeOrigin: wakeOriginFrom(t) });
+                    notifiedTriggerFailures.delete(t.trigger_id);
                 }
                 catch (err) {
-                    // Fail loud by waking the ARMER DIRECTLY (design §6.6/Q5). The row was
-                    // already settled in step 1, so this is never a hot retry loop; a
-                    // permanently-broken spawn-cron stays deduped-noisy until cancel.
-                    failLoudWake(w, '⚠ deferred spawn failed', `Deferred spawn (${recipe.kind} @ ${recipe.cwd}) failed: ` +
-                        `${err.message}\n\nRe-home (fix the cwd/parent) or re-arm the wake.`);
+                    failLoudTrigger(t, '⚠ deferred spawn failed', `Deferred spawn (${recipe.kind} @ ${recipe.cwd}) failed: ` +
+                        `${err.message}\n\nRe-home (fix the cwd/parent) or re-arm the trigger.`);
                 }
             }
         }
         catch (err) {
-            // One bad wake never kills the tick.
-            process.stderr.write(`[crtrd] error firing wake ${w.wakeup_id}: ${err.message}\n`);
+            process.stderr.write(`[crtrd] error firing trigger ${t.trigger_id}: ${err.message}\n`);
         }
     }
+    // Pass 3a: recover stale predicate leases stranded by a PRIOR daemon
+    // instance (crash/restart mid-evaluation) — BEFORE 3b/3c, so a row a dead
+    // daemon left `eval_state='running'` forever can rejoin the idle pool
+    // instead of being wedged out of duePredicateTriggers' idle-only filter
+    // (spec §Daemon-behavior step 1, acceptance criterion 9: survives restart).
+    try {
+        recoverStalePredicateLeases(now, predicateCapacity());
+    }
+    catch (err) {
+        process.stderr.write(`[crtrd] recoverStalePredicateLeases error: ${err.message}\n`);
+    }
+    // Pass 3b: wall-clock predicate timeouts — reap regardless of eval_state,
+    // BEFORE selecting fresh evaluations (so a timed-out row can never also be
+    // evaluated this same tick — consumeTrigger removes it from the due query).
+    try {
+        reapTimedOutPredicates(now, (t) => {
+            notifyOwnerUrgent(t, '⚠ predicate timed out', `Trigger ${t.trigger_id}'s wall-clock timeout was reached before its predicate ever fired — it has been canceled.`);
+        });
+    }
+    catch (err) {
+        process.stderr.write(`[crtrd] reapTimedOutPredicates error: ${err.message}\n`);
+    }
+    // Pass 3c: kick off due predicate evaluations. Fire-and-forget — settlement
+    // happens async via onDone, decoupled from this tick (never awaited; a slow/
+    // hung predicate must never block supervision, spec §Predicate triggers).
+    try {
+        evaluateDuePredicates(now, (outcome) => {
+            handlePredicateOutcome(outcome).catch((err) => {
+                process.stderr.write(`[crtrd] predicate outcome error: ${err.message}\n`);
+            });
+        });
+    }
+    catch (err) {
+        process.stderr.write(`[crtrd] evaluateDuePredicates error: ${err.message}\n`);
+    }
     // Fourth pass: reap the leftover placement of long-dead nodes (Bug 3), then
     // sweep focus rows whose pane died outside the sanctioned teardown paths
     // (Bug 5). Reap first so the GC also catches any focus row a reaped pane
@@ -825,6 +873,25 @@ export async function superviseTick(now = Date.now()) {
     // re-anchored.
     reapDeadResidue(now);
     gcStaleFocuses();
+    // Fifth pass: count-based auto-prune, throttled to once per PRUNE_INTERVAL_MS
+    // (NOT every 2s tick — it's a full-canvas DB sweep). Keeps the AUTO_PRUNE_MAX
+    // most-recently-active nodes; live nodes are never deleted (pruneToLimit's
+    // floor is max(AUTO_PRUNE_MAX, liveCount)). Wrapped so a sweep failure never
+    // crashes the tick; the tick is already serialized (recursive setTimeout), so
+    // this runs sequentially within it.
+    if (now - lastPruneAt >= PRUNE_INTERVAL_MS) {
+        lastPruneAt = now;
+        try {
+            const before = listNodes().length;
+            const { pruned } = pruneToLimit({ maxNodes: AUTO_PRUNE_MAX });
+            if (pruned.length > 0) {
+                process.stderr.write(`[crtrd] auto-prune: removed ${pruned.length} nodes (kept ${before - pruned.length})\n`);
+            }
+        }
+        catch (err) {
+            process.stderr.write(`[crtrd] auto-prune error: ${err.message}\n`);
+        }
+    }
 }
 /** Start the supervisor loop.
  *
@@ -841,8 +908,21 @@ export function runDaemon(opts = {}) {
         process.exit(0);
     }
     const interval = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
+    // Once, before the first tick ever inspects a single `pi_pid`: detect a boot
+    // change (a host reboot, or a sandbox/VM destroy+recreate — canvas.db is
+    // durable and survives either, but every `pi_pid` it carries is a raw OS pid
+    // from the process table THAT reset) and clear every stale pid up front. This
+    // must run before the tick loop, not lazily inside reviveNode: the daemon's
+    // OWN liveness poll (handleNodeLiveness) treats `isPidAlive(pid) === true` as
+    // "leave it, nothing to revive" and never calls reviveNode at all — a
+    // pid-reuse collision on the first post-boot tick would silently no-op
+    // forever with no revive attempt to even hit reviveNode's own reconcile call.
+    const { changed, cleared } = reconcileBootLiveness();
+    if (changed) {
+        process.stderr.write(`[crtrd] boot change detected — cleared ${cleared.length} stale pi_pid(s) recorded before this boot\n`);
+    }
     writePidfile();
-    process.stderr.write(`[crtrd] started (pid ${process.pid}, interval ${interval}ms)\n`);
+    log.info(`started (pid ${process.pid}, interval ${interval}ms)`);
     let running = true;
     // Cleanup — idempotent.
     const cleanup = () => {