@crouton-kit/crouter 0.3.35 → 0.3.37

package/dist/daemon/crtrd.js

@@ -49,11 +49,14 @@
 import { writeFileSync, readFileSync, rmSync, existsSync, mkdirSync, statSync, } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { homedir } from 'node:os';
+import { spawnSync } from 'node:child_process';
 import { crtrHome, nodeDir, jobDir } from '../core/canvas/paths.js';
-import { listNodes, getRow, getNode, dueClockTriggers, consumeTrigger, advanceTrigger, listFocuses, closeFocusRow, pruneToLimit, } from '../core/canvas/index.js';
+import { listNodes, getRow, getNode, dueClockTriggers, consumeTrigger, advanceTrigger, listFocuses, closeFocusRow, pruneToLimit, subscribersOf, } from '../core/canvas/index.js';
+import { fullName } from '../core/canvas/labels.js';
 import { transition } from '../core/runtime/lifecycle.js';
-import { isBusy } from '../core/runtime/busy.js';
-import { FAULT_QUIET_MS } from '../core/runtime/fault.js';
+import { isBusy, busySince } from '../core/runtime/busy.js';
+import { FAULT_QUIET_MS, recordFault } from '../core/runtime/fault.js';
+import { fanDoctrineWake } from '../core/runtime/close.js';
 import { logger } from '../core/log.js';
 export { FAULT_QUIET_MS };
 import { isPidAlive } from '../core/canvas/pid.js';
@@ -209,6 +212,207 @@ function handleYieldStall(row, pid, now) {
     }
 }
 // ---------------------------------------------------------------------------
+// Wedge detection (issue #110): a broker whose pid is alive but whose engine is
+// frozen MID-TURN (a runaway subprocess, a hung stream) is invisible to every
+// other guarantee — the pid-alive branch above returns early, yield-stall only
+// fires on intent='refresh', and provider-fault recovery needs a completed
+// ERROR turn. A node in this state never pushes, so its parent waits forever
+// with no wake and no visible fault marker (`node inspect list --hanging`
+// returns nothing — there's no fault recorded, just silence).
+//
+// Detection, not recovery: this NOTIFIES (fans a doctrine wake + records a
+// `daemon→node`/'wedged' fault so the node shows ⌘ “wedged · needs you” on the
+// canvas graph views, same surface `node inspect list --hanging` already reads)
+// — it never kills or restarts the engine. Killing the runaway SUBPROCESS (not
+// the node) is the sanctioned recovery; see `crtr memory read
+// wedged-child-on-runaway-bash`.
+//
+// Two corroborating signals, both generous, so a routine long-running bash/
+// build/test tool call is never mistaken for a wedge:
+//   1. quietForMs — ms since the busy marker's mtime last moved. canvas-
+//      stophook re-touches that marker (`touchBusy`) on every engine-level sign
+//      of life within a turn: a tool starting/streaming/ending, a streamed
+//      assistant token, or the turn ending. A tool that is actively streaming
+//      output, or a turn producing tokens, keeps this at ~0 no matter how long
+//      the OVERALL turn runs. Only a turn with NO observed engine event at all
+//      for WEDGE_QUIET_MS is even a candidate.
+//   2. cpuPercent — the broker's WHOLE PROCESS TREE (the broker pid + every
+//      descendant), summed, sampled ONLY once a node is already a time-based
+//      candidate (never every tick — the `ps` shell-out is cheap but pointless
+//      on a healthy node). Sampling the tree, not just the broker pid, matters
+//      because the broker often sits idle awaiting a CHILD subprocess (bash,
+//      a build, a test run) that is the one actually burning CPU — sampling
+//      the broker alone would read that as 0% and false-fire on a perfectly
+//      healthy long-running tool call. A pi process (or any descendant)
+//      genuinely computing shows non-trivial tree CPU and is exonerated even
+//      past the quiet grace; a pi merely awaiting a subprocess or a stalled
+//      I/O call — wedged OR legitimately blocked on something equally silent —
+//      reads ~0% across the whole tree, matching the filed issue's own
+//      evidence (pid alive, 0% CPU, session file not advancing). A failed/
+//      unreadable CPU sample is treated as UNKNOWN, never as “low” — the
+//      verdict stays 'pending' rather than risk a false fire on an unreadable
+//      signal.
+//
+//      ACCEPTED residual: a legitimate subprocess blocked on silent I/O (e.g. a
+//      20-minute download emitting no output, a long `sleep`) reads ~0% tree
+//      CPU and is indistinguishable from a real wedge at the OS level — no CPU
+//      signal can tell a hung child from an I/O-blocked one. This is accepted,
+//      not deferred, because detection only NOTIFIES and never kills: the cost
+//      of that rare false positive is one doctrine wake to a parent, and 20+
+//      minutes of total silence with near-zero tree CPU is itself worth
+//      surfacing either way.
+// ---------------------------------------------------------------------------
+// Generous: routine long tool calls settle well under this; only a turn with
+// zero observed engine progress for 20+ minutes is even a wedge CANDIDATE.
+export const WEDGE_QUIET_MS = 20 * 60_000;
+// CPU% ceiling for the corroborating sample, taken once quietForMs already
+// clears the grace above. Small but nonzero — tolerates measurement noise
+// without accepting a process that's visibly still doing work.
+export const WEDGE_CPU_MAX_PERCENT = 2;
+/** Pure wedge decision — mirrors livenessVerdict/yieldStallVerdict: a total
+ *  function of the observed signals, no clock or process access of its own.
+ *  `busy` false — the turn already ended (however it routed) — is never a
+ *  wedge, regardless of how stale the marker looks. `quietForMs` null means
+ *  the heartbeat marker couldn't be read (treated as “not enough evidence”, not
+ *  “wedged”). `cpuPercent` null (sample failed/unavailable) or above the
+ *  ceiling — EITHER keeps the verdict at 'pending' (keep watching next tick)
+ *  rather than firing on an unconfirmed or contradicted signal. */
+export function wedgeVerdict(input) {
+    if (!input.busy)
+        return 'leave';
+    if (input.quietForMs === null || input.quietForMs < WEDGE_QUIET_MS)
+        return 'leave';
+    if (input.cpuPercent === null || input.cpuPercent > WEDGE_CPU_MAX_PERCENT)
+        return 'pending';
+    return 'wedged';
+}
+/** Notified-episode dedup (one notice per wedge, per §Detection-not-repeat-fire):
+ *  keyed on node id → the busy-marker mtime we already notified for. The marker
+ *  only advances on genuine engine progress (a heartbeat) or a fresh turn
+ *  starting, so a stale wedge that later recovers (or a brand-new turn) always
+ *  carries a DIFFERENT mtime and is free to notify again. In-memory only, like
+ *  every other daemon dedup latch here — a restart re-arms it (worst case: one
+ *  repeat notice), which is the same tradeoff `notifiedTriggerFailures` and
+ *  `unhealthySince` already accept. */
+const wedgeNotifiedAt = new Map();
+/** Pure: sum CPU% across a process's entire tree (itself + every descendant).
+ *  Parses `ps -ax -o pid=,ppid=,pcpu=` output, builds a ppid→children map, and
+ *  DFS's from `rootPid` collecting root + all descendants; malformed/blank
+ *  lines are skipped. Returns `null` if `rootPid` never appears in the output
+ *  (can't measure at all — treated as UNKNOWN by wedgeVerdict, never as
+ *  “low”) or the output has no parsable rows. Exported for direct unit testing
+ *  — no process access of its own, mirroring wedgeVerdict's purity. */
+export function sumTreeCpu(psOutput, rootPid) {
+    const byPid = new Map(); // pid → pcpu
+    const childrenOf = new Map(); // ppid → child pids
+    for (const line of psOutput.split('\n')) {
+        const trimmed = line.trim();
+        if (trimmed === '')
+            continue;
+        const parts = trimmed.split(/\s+/);
+        if (parts.length < 3)
+            continue;
+        const pid = Number(parts[0]);
+        const ppid = Number(parts[1]);
+        const pcpu = Number(parts[2]);
+        if (!Number.isFinite(pid) || !Number.isFinite(ppid) || !Number.isFinite(pcpu))
+            continue;
+        byPid.set(pid, pcpu);
+        const siblings = childrenOf.get(ppid);
+        if (siblings === undefined)
+            childrenOf.set(ppid, [pid]);
+        else
+            siblings.push(pid);
+    }
+    if (!byPid.has(rootPid))
+        return null;
+    let total = 0;
+    const stack = [rootPid];
+    const seen = new Set();
+    while (stack.length > 0) {
+        const pid = stack.pop();
+        if (seen.has(pid))
+            continue; // guard against a malformed/cyclic ps snapshot
+        seen.add(pid);
+        total += byPid.get(pid) ?? 0;
+        const kids = childrenOf.get(pid);
+        if (kids !== undefined)
+            stack.push(...kids);
+    }
+    return total;
+}
+/** Sample the broker's WHOLE PROCESS TREE — the broker pid plus every
+ *  descendant, summed — via one `ps` shell-out, the OS-level signal that
+ *  corroborates the heartbeat silence (see the section banner above). Null on
+ *  ANY failure (the shell-out itself failing, `ps` missing, or `sumTreeCpu`
+ *  finding the root pid absent/output unparseable) — wedgeVerdict treats null
+ *  exactly like “too high”, so a broken probe can only ever suppress a false
+ *  fire, never cause one. */
+function readTreeCpuPercent(pid) {
+    try {
+        const r = spawnSync('ps', ['-ax', '-o', 'pid=,ppid=,pcpu='], { encoding: 'utf8', timeout: 2000 });
+        if (r.status !== 0 || typeof r.stdout !== 'string')
+            return null;
+        return sumTreeCpu(r.stdout, pid);
+    }
+    catch {
+        return null;
+    }
+}
+/** Enact wedge detection for a LIVE, busy node (called from the pid-alive
+ *  branch of handleNodeLiveness, alongside handleYieldStall — the two never
+ *  overlap in practice: yield-stall requires the turn OVER, wedge requires it
+ *  mid-flight). Reads the heartbeat + samples CPU only when already past the
+ *  time grace, fans one doctrine wake + records one `daemon→node` fault per
+ *  episode, and never touches the engine itself. */
+function handleWedgeDetection(id, pid, now) {
+    if (!isBusy(id)) {
+        wedgeNotifiedAt.delete(id); // turn ended (however it routed) — next turn starts a fresh episode
+        return;
+    }
+    const since = busySince(id);
+    const quietForMs = since === null ? null : now - since;
+    const cpuPercent = quietForMs !== null && quietForMs >= WEDGE_QUIET_MS ? readTreeCpuPercent(pid) : null;
+    const verdict = wedgeVerdict({ busy: true, quietForMs, cpuPercent });
+    if (verdict !== 'wedged')
+        return;
+    if (since !== null && wedgeNotifiedAt.get(id) === since)
+        return; // already notified THIS episode
+    if (since !== null)
+        wedgeNotifiedAt.set(id, since);
+    const meta = getNode(id);
+    const name = meta !== null ? fullName(meta) : id;
+    const minutes = Math.round((quietForMs ?? WEDGE_QUIET_MS) / 60_000);
+    const label = `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
+        `${minutes}+ minutes and its process is near-idle. It will NOT recover on its own — read ` +
+        `\`crtr memory read wedged-child-on-runaway-bash\` and, if a runaway subprocess is to blame, kill ` +
+        `the SUBPROCESS (not this node) so its turn can resume with no lost context.`;
+    process.stderr.write(`[crtrd] wedged ${id} (busy ${minutes}+ min with no engine progress, cpu~0%)\n`);
+    try {
+        recordFault(id, {
+            link: 'daemon→node',
+            op: 'engine wedge',
+            kind: 'wedged',
+            retry: { disposition: 'manual' },
+            message: label,
+        });
+    }
+    catch {
+        /* best-effort — the doctrine wake below is the load-bearing notice */
+    }
+    try {
+        fanDoctrineWake(id, subscribersOf(id), label, {
+            reason: 'child-wedged',
+            child: id,
+            quiet_ms: quietForMs,
+            cpu_percent: cpuPercent,
+        });
+    }
+    catch {
+        /* best-effort, mirrors every other fan-out in this file */
+    }
+}
+// ---------------------------------------------------------------------------
 // Auth-reload fan (one /login → every live broker follows)
 // ---------------------------------------------------------------------------
 /** Path of the shared pi credentials file every broker reads. VENDORED from pi
@@ -306,7 +510,10 @@ export function livenessVerdict(piPidAlive, deadFor) {
  *      the authority on refresh, and the dead-pid refresh branch below then
  *      revives it fresh. Fault recovery now runs in a separate pass: live,
  *      daemon-owned provider faults are retried by inbox nudge, while dead-pid
- *      crashes still fall through the ordinary resume path below.
+ *      crashes still fall through the ordinary resume path below. Wedge
+ *      detection (issue #110) runs here too: handleWedgeDetection no-ops unless
+ *      the node is BUSY (mid-turn) and its heartbeat has gone quiet past a
+ *      generous grace, corroborated by near-zero CPU.
  *    • pid null + intent==='idle-release' → dormant by choice; its pid may have
  *      just been cleared by boot reconciliation, but a null pid here is still
  *      that chosen dormancy, not a stranded relaunch — leave it, the second pass
@@ -337,6 +544,7 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
     if (pid != null && isPidAlive(pid)) {
         unhealthySince.delete(id);
         handleYieldStall(row, pid, now);
+        handleWedgeDetection(id, pid, now);
         return;
     }
     if (pid == null) {
@@ -642,7 +850,7 @@ export function readPidfile() {
 }
 // isPidAlive now lives in canvas/pid.ts (the one shared signal-0 probe at the
 // lowest layer, so canvas/, runtime/, AND daemon/ all import it down). Re-
-// exported here to preserve the public surface consumed by commands/daemon.ts
+// exported here to preserve the public surface consumed by commands/sys/daemon.ts
 // and the grace-clock test.
 export { isPidAlive };
 /** True when a crtrd process is already running (pidfile exists + pid alive). */

package/dist/daemon/manage.js

@@ -1,6 +1,6 @@
 // Daemon management helpers — importable without the full command tree.
 //
-// spawnDaemon() is the low-level spawn call shared by `crtr canvas daemon start` and
+// spawnDaemon() is the low-level spawn call shared by `crtr sys daemon start` and
 // ensureDaemon(). ensureDaemon() is the silent "start if not running" front-
 // door helper called by the canvas runtime before spawning child nodes.
 import { spawn } from 'node:child_process';
@@ -68,7 +68,7 @@ export async function spawnDaemon() {
     // Launch from the crouter-branded host binary so the daemon shows "crouter"
     // in macOS Full Disk Access, not "node" (see branded-host). NOTE: when crtrd
     // is launchd-owned, the plist's ProgramArguments must point at the branded
-    // binary too — this path only covers a manual `crtr canvas daemon start`.
+    // binary too — this path only covers a manual `crtr sys daemon start`.
     const entry = resolveCrtrdEntry();
     const child = spawn(hostExecPath(), [entry], {
         detached: true,

package/dist/pi-extensions/canvas-nav.js

@@ -479,8 +479,8 @@ export function registerCanvasNav(pi) {
         }
         if (isPlain(data, 'e')) {
             // Expand → tmux: spread the cursor node's local subtree into a tiled tmux
-            // window (same action as the alt+c → e menu chord), for parity.
-            shellCrtr(['canvas', 'tmux-spread', cursorId ?? nodeId], render);
+            // window (`crtr surface tmux-spread`).
+            shellCrtr(['surface', 'tmux-spread', cursorId ?? nodeId], render);
             render();
             return { consume: true };
         }

package/dist/pi-extensions/canvas-stophook.d.ts

@@ -1,4 +1,4 @@
-type PiEvents = 'agent_start' | 'turn_end' | 'agent_end' | 'session_shutdown' | 'session_start' | 'tool_execution_start' | 'tool_execution_end';
+type PiEvents = 'agent_start' | 'turn_end' | 'agent_end' | 'session_shutdown' | 'session_start' | 'tool_execution_start' | 'tool_execution_end' | 'tool_execution_update' | 'message_update';
 interface PiLike {
     on: (event: PiEvents, handler: (event: any, ctx: any) => void | Promise<void>) => void;
     sendUserMessage: (content: string, options?: {

package/dist/pi-extensions/canvas-stophook.js

@@ -30,7 +30,7 @@ import { writeFileSync, mkdirSync, existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { getNode, jobDir, updateNode, recordPid, setPresence } from '../core/canvas/index.js';
 import { transition } from '../core/runtime/lifecycle.js';
-import { markBusy, clearBusy } from '../core/runtime/busy.js';
+import { markBusy, clearBusy, touchBusy } from '../core/runtime/busy.js';
 import { recordFault, clearFault } from '../core/runtime/fault.js';
 import { classify } from '../core/fault-classifier.js';
 import { evaluateStop } from '../core/runtime/stop-guard.js';
@@ -241,6 +241,25 @@ export function registerCanvasStophook(pi) {
     // checkpoint roadmap + yield; a resident base (root chat) to promote-or-wrap-
     // up (it has no roadmap); a terminal worker to promote / push final.
     const firedBands = new Set();
+    // Wedge-heartbeat throttle: message_update/tool_execution_update can fire many
+    // times per second while streaming, and touchBusy is a disk write — rate-limit
+    // it to at most once per HEARTBEAT_THROTTLE_MS. The daemon's wedge grace is
+    // measured in minutes, so this throttle costs it nothing. Also clears any
+    // stale `daemon→node` wedge fault at the same throttled cadence: a sign of
+    // life should both refresh the heartbeat marker AND clear a stale wedge
+    // notice as one coherent act, same as every coarser progress site below
+    // (tool_execution_start/end, turn_end, agent_end) already does. clearFault is
+    // a cheap no-op when no fault exists, so this stays safe to call every time.
+    const HEARTBEAT_THROTTLE_MS = 3_000;
+    let lastHeartbeatAt = 0;
+    const heartbeat = () => {
+        const now = Date.now();
+        if (now - lastHeartbeatAt < HEARTBEAT_THROTTLE_MS)
+            return;
+        lastHeartbeatAt = now;
+        touchBusy(nodeId);
+        clearFault(nodeId, { link: 'daemon→node' });
+    };
     // ---------------------------------------------------------------------------
     // session_start — capture pi's session id, and detect `/new`.
     //
@@ -336,6 +355,36 @@ export function registerCanvasStophook(pi) {
         try {
             lastActivity = summarizeActivity(String(event?.toolName ?? ''), event?.args);
             flushTelemetry(jobDirPath, totalIn, totalOut, model, lastContext, lastActivity);
+            touchBusy(nodeId); // wedge heartbeat: a fresh tool starting is a sign of life
+            clearFault(nodeId, { link: 'daemon→node' }); // a tool actually starting clears any prior wedge notice
+        }
+        catch { /* best-effort */ }
+    });
+    // ---------------------------------------------------------------------------
+    // tool_execution_update — wedge heartbeat + fault clear only. Fired with
+    // partial/streaming tool output (when a tool reports progress); throttled via
+    // `heartbeat()` since it can arrive many times per second. Most tools (e.g.
+    // bash) don't stream progress today, so this is currently a light-touch no-op
+    // for them — it exists so any tool that DOES stream partial output (now or in
+    // the future) counts as live progress instead of tripping the wedge detector,
+    // AND clears any wedge notice that already fired before this progress resumed.
+    // ---------------------------------------------------------------------------
+    pi.on('tool_execution_update', () => {
+        try {
+            heartbeat();
+        }
+        catch { /* best-effort */ }
+    });
+    // ---------------------------------------------------------------------------
+    // message_update — wedge heartbeat + fault clear only. Fired token-by-token
+    // while the assistant is actively streaming a reply — genuine progress,
+    // throttled the same way, and equally clears a stale wedge notice if streaming
+    // resumes before a coarser boundary. A node stuck streaming forever without
+    // ever firing this is exactly the daemon's busy+quiet wedge signature.
+    // ---------------------------------------------------------------------------
+    pi.on('message_update', () => {
+        try {
+            heartbeat();
         }
         catch { /* best-effort */ }
     });
@@ -353,6 +402,8 @@ export function registerCanvasStophook(pi) {
     // refreshes on its first tool — the desired outcome.)
     pi.on('tool_execution_end', (_event, ctx) => {
         try {
+            touchBusy(nodeId); // wedge heartbeat: the tool returning is a sign of life
+            clearFault(nodeId, { link: 'daemon→node' }); // a tool that was "stuck" just returned — the wedge notice was resolved
             if (getNode(nodeId)?.intent === 'refresh') {
                 clearBusy(nodeId); // agent_end won't run to clear it; harmless if it lingers
                 try {
@@ -422,6 +473,8 @@ export function registerCanvasStophook(pi) {
             lastContext = contextTokens;
         // Fire-and-forget: flushTelemetry uses synchronous fs writes and never throws.
         flushTelemetry(jobDirPath, totalIn, totalOut, model, contextTokens, lastActivity);
+        touchBusy(nodeId); // wedge heartbeat: a turn ending is a sign of life
+        clearFault(nodeId, { link: 'daemon→node' }); // reaching turn_end proves this turn wasn't wedged
         // Context-size steering: fire the current band once, with lifecycle-specific
         // guidance (lifecycle is read live — a terminal worker may have promoted to
         // resident since launch).
@@ -473,8 +526,12 @@ export function registerCanvasStophook(pi) {
         // removed, so the handler no longer needs to be async — the node reaches its
         // subscribers ONLY through its own explicit `crtr push` calls.
         // The turn has ended regardless of how it routes below — clear the mid-turn
-        // marker FIRST so a focus-away from this now-parked node despawns it.
+        // marker FIRST so a focus-away from this now-parked node despawns it. Also
+        // clear any wedge notice: reaching agent_end at all proves the turn ended
+        // and the engine was never permanently stuck (covers an abort/error path
+        // that skips the tool_execution_end/turn_end clear sites above).
         clearBusy(nodeId);
+        clearFault(nodeId, { link: 'daemon→node' });
         try {
             const messages = Array.isArray(event?.messages) ? event.messages : [];
             // Accumulate tokens from the final batch (edge case: a turn that fired