@crouton-kit/crouter 0.3.32 → 0.3.33

package/dist/core/__tests__/context-intro.test.js

@@ -1,6 +1,6 @@
-// Tests for the <crtr-context> bearings preamble:
-//   1. Worker and orchestrator bearings carry the `<knowledge>` block
-//      (substrate knowledge docs + node-local docs as a file tree).
+// Tests for the <crtr-bearings> preamble:
+//   1. Worker and orchestrator bearings carry the `<memory kind="knowledge">`
+//      block (substrate knowledge docs + node-local docs as a file tree).
 //      Orchestrators add the across-cycles framing; promotion delivers that
 //      same orchestrator context-dir framing to a node that spawned base.
 //   2. canvas-context-intro injects the block as its own session message at
@@ -20,9 +20,14 @@ import { personaDrift } from '../runtime/persona.js';
 import { memoryDir } from '../runtime/memory.js';
 import registerCanvasContextIntro, { buildContextIntro, renderContextMessage, CONTEXT_INTRO_CUSTOM_TYPE, } from '../../pi-extensions/canvas-context-intro.js';
 let home;
+let prevAgentDir;
 before(() => {
     home = mkdtempSync(join(tmpdir(), 'crtr-ctxintro-'));
     process.env['CRTR_HOME'] = home;
+    // Isolate pi's global agent dir to an empty path so buildProjectContextBlock
+    // can't pick up the dev machine's real ~/.pi/agent/AGENTS.md (hermeticity).
+    prevAgentDir = process.env['PI_CODING_AGENT_DIR'];
+    process.env['PI_CODING_AGENT_DIR'] = join(home, 'no-agent-dir');
 });
 beforeEach(() => {
     closeDb();
@@ -33,6 +38,10 @@ after(() => {
     rmSync(home, { recursive: true, force: true });
     delete process.env['CRTR_HOME'];
     delete process.env['CRTR_NODE_ID'];
+    if (prevAgentDir === undefined)
+        delete process.env['PI_CODING_AGENT_DIR'];
+    else
+        process.env['PI_CODING_AGENT_DIR'] = prevAgentDir;
 });
 test('worker bearings: base framing + <knowledge> block, NO across-cycles framing', () => {
     // Bug-regression: review finding M1 — buildContextBearings renders the
@@ -44,23 +53,49 @@ test('worker bearings: base framing + <knowledge> block, NO across-cycles framin
     mkdirSync(dir, { recursive: true });
     writeFileSync(join(dir, 'test-ref.md'), '---\nkind: knowledge\nwhen-and-why-to-read: When testing, this reference should be read because it is a regression fixture\nsystem-prompt-visibility: preview\n---\nTest body.\n');
     const block = buildContextIntro(meta.node_id);
-    assert.match(block, new RegExp(`<crtr-context dir="${contextDir(meta.node_id)}">`));
+    assert.match(block, /<crtr-bearings>/, 'opens with the bearings section');
+    assert.ok(block.includes(contextDir(meta.node_id)), 'names the context-dir path inline (no dir= attribute)');
+    assert.doesNotMatch(block, /<crtr-context/, 'no legacy <crtr-context dir=…> wrapper');
+    assert.match(block, /## Your context directory/, 'context dir is its own markdown subsection');
     assert.match(block, /shared document store, not a task tracker/, 'base = shared docs, not tasks');
-    // Knowledge content renders ONLY as the <knowledge> file-tree block.
-    assert.doesNotMatch(block, /<memory>/, 'no <memory> block');
+    // The catalog is a sibling <memory kind="knowledge"> block, never nested
+    // under anything dir-scoped.
+    assert.match(block, /<memory kind="knowledge">/, 'knowledge catalog present as a memory block');
     // Per-store stanza headers (label · dir) never appear.
     assert.doesNotMatch(block, /user-global · /, 'no user-global label·dir stanza');
     assert.doesNotMatch(block, /node-local · /, 'no node-local label·dir stanza');
     // No (empty) placeholder marker.
     assert.doesNotMatch(block, /\(empty\)/, 'no (empty) placeholder');
-    assert.match(block, /<knowledge>/, '<knowledge> block present');
     assert.match(block, /\nknowledge\n/, 'tree headed by the `knowledge` root label');
     // The node-local preview-rung doc renders as a tree entry with a `# read when:`
     // routing line (verbatim previewLine output).
     assert.match(block, /test-ref {2}# read when: When testing, this reference should be read because it is a regression fixture\./, 'node-local preview doc renders its routing line');
+    assert.match(block, /<project_context>/, 'project_context now rides the bearings even without project files');
+    assert.match(block, /<environment cwd="\/tmp\/work">/, 'environment block is always present');
+    assert.match(block, /Directory listing: unavailable \(cwd missing or unreadable\)\./, 'missing cwd degrades the directory listing');
+    assert.match(block, /Git: not a git repository\./, 'missing cwd degrades git info');
     // A terminal worker must NOT carry the orchestrator across-cycles framing.
     assert.doesNotMatch(block, /refresh cycles/, 'no across-cycles framing for a terminal worker');
-    assert.match(block, /<\/crtr-context>/);
+    assert.match(block, /<\/crtr-bearings>/);
+});
+test('project instructions (AGENTS.md/CLAUDE.md) ride the bearings, not the system prompt', () => {
+    // pi's <project_context> is suppressed in the system prompt (broker.ts
+    // noContextFiles) and re-rendered into the first-message bearings here. The
+    // discovery walks the node cwd ancestry, so a CLAUDE.md in the node's cwd must
+    // surface inside a <project_context> block AFTER the <crtr-bearings> block.
+    const proj = mkdtempSync(join(tmpdir(), 'crtr-proj-'));
+    writeFileSync(join(proj, 'CLAUDE.md'), 'PROJECT_MARKER: build then commit.\n');
+    const meta = spawnNode({ kind: 'general', cwd: proj, parent: null });
+    const block = buildContextIntro(meta.node_id);
+    assert.match(block, /<project_context>/, 'project_context block present in the bearings');
+    assert.match(block, new RegExp(`<project_instructions path="${join(proj, 'CLAUDE.md')}">`), 'the project CLAUDE.md is rendered with its path');
+    assert.match(block, /PROJECT_MARKER: build then commit\./, 'project file content rides the message');
+    assert.match(block, /<environment cwd=".*crtr-proj-.*">/, 'environment block is appended inside project_context');
+    assert.match(block, /Directory:\n  CLAUDE\.md\n/, 'environment block includes an ls-style listing');
+    assert.match(block, /Git: not a git repository\./, 'git snapshot degrades cleanly for a non-repo');
+    assert.match(block, /<environment[\s\S]*<\/environment>\n<\/project_context>$/, 'environment block closes project_context');
+    assert.ok(block.indexOf('</crtr-bearings>') < block.indexOf('<project_context>'), 'project_context follows the crtr-bearings block');
+    rmSync(proj, { recursive: true, force: true });
 });
 test('orchestrator bearings: across-cycles framing + node-local substrate docs ride into <knowledge>; a non-substrate .md file is never inlined', () => {
     // Bug-regression: review finding M1 — buildContextBearings renders <knowledge>.
@@ -78,21 +113,20 @@ test('orchestrator bearings: across-cycles framing + node-local substrate docs r
     const block = buildContextIntro(meta.node_id);
     assert.match(block, /shared document store, not a task tracker/, 'still carries the base framing');
     assert.match(block, /refresh cycles/, 'orchestrator gets the across-cycles framing');
-    assert.doesNotMatch(block, /<memory>/, 'no <memory> block');
-    assert.match(block, /<knowledge>/, 'knowledge block present');
+    assert.match(block, /<memory kind="knowledge">/, 'knowledge catalog present as a memory block');
     assert.match(block, /flaky-build {2}# read when: When the build flakes, this reference should be read because the first run fails\./, 'node-local doc renders as a tree entry with its routing line');
     // The non-substrate file never renders: no header line, no pointer line, no path.
     assert.ok(!block.includes('# memory index'), 'the index header comment is NOT inlined');
     assert.ok(!block.includes('- [Flaky build](flaky-build.md)'), 'index pointer lines are NOT inlined');
     assert.ok(!block.includes(legacyIndexPath), 'no absolute index (MEMORY.md) path');
     assert.ok(!block.includes('node-local · '), 'no label·dir stanza header');
-    assert.match(block, /<\/crtr-context>/);
+    assert.match(block, /<\/crtr-bearings>/);
 });
 test('orchestrator bearings: no per-store stanzas or (empty) markers; a rung-none node-local doc still surfaces as a bare-name tree entry', () => {
     // Bug-regression: review findings M1 + M6 — the <knowledge> block carries no
     // per-store `label · dir` stanzas or (empty) markers, and node-local docs are
     // NOT filtered on rung: a migrated node-local doc defaults
-    // system-prompt-visibility: none and must still ride into <knowledge> as
+    // system-prompt-visibility: none and must still ride into the knowledge memory block as
     // its bare name (floored to the `name` rung; never its body).
     const meta = spawnNode({ kind: 'general', cwd: '/tmp/work', parent: null });
     promote(meta.node_id); // flip to orchestrator mode
@@ -147,7 +181,7 @@ test('session_start injects the block as the first message of an empty session',
     assert.equal(pi.sent.length, 1, 'one message injected');
     assert.equal(pi.sent[0].customType, CONTEXT_INTRO_CUSTOM_TYPE);
     assert.equal(pi.sent[0].display, true);
-    assert.match(pi.sent[0].content, /<crtr-context dir=/);
+    assert.match(pi.sent[0].content, /<crtr-bearings>/);
 });
 test('session_start is idempotent across resume (skips if already in history)', () => {
     const meta = spawnNode({ kind: 'general', cwd: '/tmp/work', parent: null });
@@ -177,7 +211,7 @@ test('renderer registered for the customType; collapsed hides body, ctrl+o expan
     registerCanvasContextIntro(pi);
     const renderer = pi.renderers[CONTEXT_INTRO_CUSTOM_TYPE];
     assert.ok(renderer, 'message renderer registered for crtr-context');
-    const body = '<crtr-context dir="/x">\nshared document store\n</crtr-context>';
+    const body = '<crtr-bearings>\nshared document store\n</crtr-bearings>';
     const message = { customType: CONTEXT_INTRO_CUSTOM_TYPE, content: body };
     const theme = {}; // no fg → plain text, easy to assert
     // Collapsed (default): a single stub line, NONE of the body.

package/dist/core/__tests__/daemon-boot.test.js

@@ -1,13 +1,18 @@
 import { test, before, after, beforeEach } from 'node:test';
 import assert from 'node:assert/strict';
-import { mkdtempSync, rmSync } from 'node:fs';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
-import { spawnSync } from 'node:child_process';
+import { spawnSync, spawn } from 'node:child_process';
+import { createServer } from 'node:net';
 import { createNode, getNode, subscribe } from '../canvas/canvas.js';
 import { closeDb } from '../canvas/db.js';
 import { readInboxSince } from '../feed/inbox.js';
-import { superviseTick } from '../../daemon/crtrd.js';
+import { superviseTick, readPidfile } from '../../daemon/crtrd.js';
+import { verifyDaemonStartup } from '../../daemon/manage.js';
+import { waitForBrokerViewSocket } from '../runtime/placement.js';
+import { viewSocketPath } from '../canvas/paths.js';
+import { signBrandedHostArtifacts, brandedHostWorks } from '../runtime/branded-host.js';
 import { markBusy, clearBusy } from '../runtime/busy.js';
 let home;
 function node(id, over = {}) {
@@ -104,13 +109,181 @@ test('a crashed booted child is revivable, not reaped, and raises no false boot-
     assert.equal(readInboxSince('P2').length, 0, 'no false boot-failure / child-death push on a crash (wake relocated to push/close)');
     clearBusy('C2');
 });
-// A still-booting node whose window is alive must be left untouched — boot is
-// slow, and an alive window means pi may still be coming up.
-test('a node with a live window is left alone even before it boots', async () => {
-    // No tmux_session/window → treated as an inline root and skipped entirely,
-    // which exercises the "no placement → not daemon-managed" guard rather than a
-    // false reap. (We avoid depending on a real live tmux window in the test env.)
+// An unplaced active node is not daemon-managed; the daemon leaves it untouched
+// rather than treating the missing broker/session fields as a boot failure.
+test('an unplaced active node is left alone before it boots', async () => {
     createNode(node('S', { status: 'active', pi_session_id: null }));
     await superviseTick();
     assert.equal(getNode('S').status, 'active', 'unplaced active node is not reaped');
 });
+// Daemon start guard: spawnDaemon must not report success before the daemon is
+// plausibly alive and has written its pidfile. This is a pure bounded poll over
+// injected clocks/probes, so we can lock it down deterministically without a
+// real daemon process.
+test('verifyDaemonStartup waits for pidfile + live pid, then reports success for the spawned pid', async () => {
+    let now = 0;
+    let pidfileReady = false;
+    let pidAliveReady = false;
+    let slept = 0;
+    const existingPid = await verifyDaemonStartup(42, 50, {
+        now: () => now,
+        readPidfile: () => (pidfileReady ? 42 : null),
+        isPidAlive: () => pidAliveReady,
+        sleepMs: (ms) => {
+            slept += ms;
+            now += ms;
+            if (now >= 20)
+                pidfileReady = true;
+            if (now >= 30)
+                pidAliveReady = true;
+        },
+    });
+    assert.equal(existingPid, null, 'the spawned daemon owns the pidfile when startup succeeds');
+    assert.equal(pidfileReady, true, 'pidfile must have appeared before success');
+    assert.equal(pidAliveReady, true, 'pid must have been alive before success');
+    assert.equal(slept > 0, true, 'the guard must actually poll');
+});
+// Regression for the cold-guest recreate blocker: the production startup window
+// must tolerate a slower cold start than the old 500ms bound.
+test('verifyDaemonStartup default window tolerates a slower cold start', async () => {
+    let now = 0;
+    let pidfileReady = false;
+    let pidAliveReady = false;
+    const existingPid = await verifyDaemonStartup(42, undefined, {
+        now: () => now,
+        readPidfile: () => (pidfileReady ? 42 : null),
+        isPidAlive: () => pidAliveReady,
+        sleepMs: (ms) => {
+            now += ms;
+            if (now >= 600)
+                pidfileReady = true;
+            if (now >= 700)
+                pidAliveReady = true;
+        },
+    });
+    assert.equal(existingPid, null, 'the production startup window must allow a cold guest to settle');
+});
+// A concurrent start race is success when another live daemon already owns the
+// pidfile; the loser must return that existing pid instead of timing out.
+test('verifyDaemonStartup returns the existing live daemon when another pid won the startup race', async () => {
+    let now = 0;
+    let slept = 0;
+    const existingPid = await verifyDaemonStartup(42, 50, {
+        now: () => now,
+        readPidfile: () => 7,
+        isPidAlive: (pid) => pid === 7,
+        sleepMs: (ms) => {
+            slept += ms;
+            now += ms;
+        },
+    });
+    assert.equal(existingPid, 7, 'the live pidfile owner must be reported back');
+    assert.equal(slept, 0, 'no polling is needed once another live daemon is visible');
+});
+// The guard stays bounded: if the daemon never becomes ready, it throws instead
+// of claiming "started" forever.
+test('verifyDaemonStartup throws after its tiny startup window expires', async () => {
+    let now = 0;
+    await assert.rejects(async () => verifyDaemonStartup(99, 25, {
+        now: () => now,
+        readPidfile: () => null,
+        isPidAlive: () => false,
+        sleepMs: (ms) => {
+            now += ms;
+        },
+    }), /did not become ready within 25ms/);
+});
+// REGRESSION (bug #102): the verifier polls with a real async sleep, NOT a
+// synchronous Atomics.wait, so the event loop stays free to dispatch the spawned
+// child's `exit` event. We spawn a REAL child that exits immediately and let the
+// real wall clock + real setTimeout sleep drive the loop: the exit event fires
+// mid-wait, childExited() flips non-null, and the verifier fails fast with the
+// exit reason — well inside the 2s window. If the loop blocked the event loop
+// (the old bug), the exit event would never fire and this would time out with the
+// generic message instead.
+test('verifyDaemonStartup throws when the spawned daemon exits before it can become ready', async () => {
+    let exited = null;
+    const child = spawn(process.execPath, ['-e', 'process.exit(1)'], { stdio: 'ignore' });
+    child.once('exit', (code, signal) => {
+        exited = { code, signal };
+    });
+    const started = Date.now();
+    await assert.rejects(async () => verifyDaemonStartup(77, 2_000, {
+        readPidfile: () => null,
+        isPidAlive: () => false,
+        childExited: () => exited,
+    }), /exited before becoming ready \(with exit code 1\)/);
+    assert.ok(Date.now() - started < 1_500, 'must fail fast on the real child exit, not run out the window');
+});
+test('the daemon honours an explicit CRTR_PIDFILE override', () => {
+    const home = mkdtempSync(join(tmpdir(), 'crtr-pidfile-home-'));
+    const pidfile = join(home, 'crtrd.pid');
+    const prevHome = process.env['CRTR_HOME'];
+    const prevPidfile = process.env['CRTR_PIDFILE'];
+    process.env['CRTR_HOME'] = mkdtempSync(join(tmpdir(), 'crtr-home-'));
+    process.env['CRTR_PIDFILE'] = pidfile;
+    writeFileSync(pidfile, '321');
+    try {
+        assert.equal(readPidfile(), 321, 'the daemon must read its pidfile from CRTR_PIDFILE, not CRTR_HOME');
+    }
+    finally {
+        if (prevHome === undefined)
+            delete process.env['CRTR_HOME'];
+        else
+            process.env['CRTR_HOME'] = prevHome;
+        if (prevPidfile === undefined)
+            delete process.env['CRTR_PIDFILE'];
+        else
+            process.env['CRTR_PIDFILE'] = prevPidfile;
+    }
+});
+// REGRESSION (bug #102 follow-up): with NO exit promise, the readiness loop must
+// still observe a real `connect`. A buggy `?? Promise.resolve('retry')` race lost
+// to the probe instantly, so live-broker callers (focus on an already-alive node,
+// canvas-tmux-spread) could never return true and stalled the full 30s deadline.
+test('waitForBrokerViewSocket returns true on a live socket when given no exit promise', async () => {
+    // CRTR_SOCK_DIR override keeps the AF_UNIX path short — the temp CRTR_HOME path
+    // would blow past macOS's 104-byte sun_path limit (listen EINVAL).
+    const sockDir = mkdtempSync('/tmp/crtr-sock-');
+    process.env['CRTR_SOCK_DIR'] = sockDir;
+    const nodeId = 'lb';
+    const sockPath = viewSocketPath(nodeId);
+    const server = createServer();
+    await new Promise((resolve) => server.listen(sockPath, () => resolve()));
+    try {
+        const started = Date.now();
+        const ready = await waitForBrokerViewSocket(nodeId);
+        assert.equal(ready, true, 'a live accepting socket must report ready even without an exit promise');
+        assert.ok(Date.now() - started < 1_000, 'readiness must resolve promptly, not stall the full deadline');
+    }
+    finally {
+        await new Promise((resolve) => server.close(() => resolve()));
+        delete process.env['CRTR_SOCK_DIR'];
+        rmSync(sockDir, { recursive: true, force: true });
+    }
+});
+test('waitForBrokerViewSocket returns false as soon as the real broker process exits', async () => {
+    const child = spawn(process.execPath, ['-e', 'process.exit(1)'], { stdio: 'ignore' });
+    const exited = new Promise((resolve) => {
+        child.once('exit', (code, signal) => {
+            resolve({ code, signal });
+        });
+    });
+    const started = Date.now();
+    const ready = await waitForBrokerViewSocket('dead-broker-for-readiness-test', exited);
+    assert.equal(ready, false, 'an exited broker must fail readiness');
+    assert.ok(Date.now() - started < 1000, 'the readiness guard must fail fast instead of waiting for the socket deadline');
+});
+test('signBrandedHostArtifacts signs the copied libnode before the branded binary', () => {
+    const signed = [];
+    signBrandedHostArtifacts('/tmp/crouter', '/tmp/libnode.dylib', (path) => signed.push(path));
+    assert.deepEqual(signed, ['/tmp/libnode.dylib', '/tmp/crouter']);
+});
+test('brandedHostWorks health-checks if a binary is executable', () => {
+    // Issue #102: the branded host binary can be corrupted or killed by macOS
+    // before exec, causing spawn() to fail silently. The health-check detects this.
+    const goodBin = process.execPath; // a real, executable binary
+    const badBin = '/nonexistent/crouter-broken';
+    assert.equal(brandedHostWorks(goodBin), true, 'health-check passes for a real executable binary');
+    assert.equal(brandedHostWorks(badBin), false, 'health-check fails for a nonexistent binary');
+});

package/dist/core/__tests__/editor-label.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/editor-label.test.js

@@ -0,0 +1,26 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { editorLabel } from '../canvas/labels.js';
+function meta(over = {}) {
+    return {
+        node_id: 'n',
+        name: 'alpha-beta',
+        created: '2026-01-01T00:00:00.000Z',
+        cwd: '/tmp/work',
+        kind: 'developer',
+        mode: 'orchestrator',
+        lifecycle: 'resident',
+        status: 'active',
+        cycles: 7,
+        ...over,
+    };
+}
+test('editorLabel orders full name before kind and omits the cycle number', () => {
+    const label = editorLabel(meta());
+    assert.equal(label, 'alpha-beta developer (orchestrator)');
+    assert.equal(label.includes(String(meta().cycles)), false);
+});
+test('editorLabel falls back to kind when fullName collapses to kind or empty', () => {
+    assert.equal(editorLabel(meta({ name: 'developer', kind: 'developer', mode: 'base' })), 'developer (base)');
+    assert.equal(editorLabel(meta({ name: '', kind: 'developer', mode: 'base' })), 'developer (base)');
+});

package/dist/core/__tests__/fault-marker.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/fault-marker.test.js

@@ -0,0 +1,112 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/fault-marker.test.ts
+//
+// BUG-REGRESSION: active-fault visibility. When a node's engine exhausts its
+// retry budget on a rate-limit / overloaded / connection error, the broker
+// stays alive and the node is indistinguishable from a healthy dormant node for
+// up to the fault retry quiet window.
+// The active-fault MARKER makes that invisible window visible on the canvas-graph
+// views. Two legs:
+//   M1 — classify: each kind + the fallback (the heuristic, order-sensitive:
+//        rate-limit/overloaded checked before connection).
+//   M2 — the marker round-trip through dashboardRowsAll: record → the row shows
+//        hanging (streaming forced false); clear → gone; a DEAD pid → not hanging
+//        (the pid-AND gate makes a stale marker from a crashed broker harmless).
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { createHeadlessHarness } from './helpers/harness.js';
+import { classify } from '../fault-classifier.js';
+import { clearFault, recordFault, isModelNotFoundError } from '../runtime/fault.js';
+import { dashboardRowsAll } from '../canvas/render.js';
+// ---------------------------------------------------------------------------
+// M1 — the heuristic classifier.
+// ---------------------------------------------------------------------------
+test('M1 classify: each kind + fallback, order-sensitive', () => {
+    // rate-limit.
+    assert.equal(classify('pi→provider', 'Rate limit exceeded').kind, 'rate-limit');
+    assert.equal(classify('pi→provider', 'HTTP 429 Too Many Requests').kind, 'rate-limit');
+    assert.equal(classify('pi→provider', 'you have exceeded your quota').kind, 'rate-limit');
+    // overloaded.
+    assert.equal(classify('pi→provider', 'Overloaded').kind, 'overloaded');
+    assert.equal(classify('pi→provider', 'Error 529: server is temporarily unavailable').kind, 'overloaded');
+    assert.equal(classify('pi→provider', '503 Service Unavailable').kind, 'overloaded');
+    assert.equal(classify('pi→provider', 'the server busy, try again').kind, 'overloaded');
+    // connection.
+    assert.equal(classify('pi→provider', 'Connection error.').kind, 'connection');
+    assert.equal(classify('pi→provider', 'ECONNRESET').kind, 'connection');
+    assert.equal(classify('pi→provider', 'fetch failed').kind, 'connection');
+    assert.equal(classify('pi→provider', 'request timed out').kind, 'connection');
+    assert.equal(classify('viewer↔broker', { code: 'ENOENT' }).kind, 'connection');
+    assert.equal(classify('viewer↔broker', { code: 1011, reason: 'no running broker for n1' }).kind, 'connection');
+    // ORDER: a 429/529 whose text ALSO mentions connection/network must NOT be
+    // swallowed by the connection match (rate-limit/overloaded are checked first).
+    assert.equal(classify('pi→provider', '429 rate limit on this network connection').kind, 'rate-limit');
+    assert.equal(classify('pi→provider', '529 overloaded — connection to upstream lost').kind, 'overloaded');
+    // fallback.
+    assert.equal(classify('pi→provider', 'some unrecognized engine failure').kind, 'other');
+    assert.equal(classify('pi→provider', 'some unrecognized engine failure').disposition, 'fatal');
+    assert.equal(classify('pi→provider', '').kind, 'other');
+});
+// ---------------------------------------------------------------------------
+// BUG-REGRESSION: a 404 not_found (configured model decommissioned/unavailable)
+// must be detected so the broker fails the node over to the strong-anthropic
+// ladder model instead of wedging on the same dead model forever. The verbatim
+// text below is the engine error a node hit when its `anthropic/ultra` ladder
+// cell pointed at a model the account could not reach.
+// ---------------------------------------------------------------------------
+test('isModelNotFoundError detects the provider 404 not_found that triggers strong-anthropic fallback', () => {
+    const verbatim = 'Error: 404 {"type":"error","error":{"type":"not_found_error","message":"Claude Fable 5 is not available. Please use Opus 4.8. Learn more: https://www.anthropic.com/news/fable-mythos-access"},"request_id":"req_011CcHgqfwD2Hjg193Ym1Sys"}';
+    assert.equal(isModelNotFoundError(verbatim), true);
+    assert.equal(isModelNotFoundError('Error: 404 model not found'), true);
+    assert.equal(isModelNotFoundError('the requested model does not exist'), true);
+    assert.equal(isModelNotFoundError('HTTP 429 Too Many Requests'), false);
+    assert.equal(isModelNotFoundError('503 Service Unavailable'), false);
+    assert.equal(isModelNotFoundError('Connection error.'), false);
+    assert.equal(isModelNotFoundError('Error: 404 unexpected endpoint /v1/foo'), false);
+    assert.equal(isModelNotFoundError(''), false);
+});
+// ---------------------------------------------------------------------------
+// M2 — the marker round-trip through dashboardRowsAll.
+// ---------------------------------------------------------------------------
+test('M2 dashboardRowsAll surfaces a hanging marker (pid-gated); clear removes it', async () => {
+    const h = await createHeadlessHarness({ sessionPrefix: 'crtr-esm' });
+    try {
+        // A LIVE node (the test process pid is definitely alive).
+        const liveId = h.fabricateBrokerNode({ kind: 'developer', status: 'active', pi_pid: process.pid });
+        // A node whose recorded pid is DEAD (the pid-gate must read it not-hanging).
+        // A child we spawn-then-let-exit gives a pid that is reliably gone.
+        const dead = spawnSync('true', [], { stdio: 'ignore' });
+        const deadPid = dead.pid ?? 2147483646; // `true` has already exited
+        const deadId = h.fabricateBrokerNode({ kind: 'developer', status: 'active', pi_pid: deadPid });
+        const find = (id) => dashboardRowsAll().find((r) => r.node_id === id);
+        // No marker yet → not hanging.
+        assert.equal(find(liveId)?.hanging ?? null, null, 'no marker → not hanging');
+        // Mark the live node → its row shows hanging, classified, streaming forced off.
+        recordFault(liveId, {
+            link: 'pi→provider',
+            op: 'generation turn',
+            kind: classify('pi→provider', 'Rate limit exceeded (429)').kind,
+            retry: { disposition: 'auto', by: 'daemon' },
+            message: 'Rate limit exceeded (429)',
+        });
+        const row = find(liveId);
+        assert.notEqual(row?.hanging ?? null, null, 'marked live node → hanging set');
+        assert.equal(row?.hanging?.kind, 'rate-limit', 'kind classified from the message');
+        assert.equal(row?.streaming, false, 'hanging forces streaming false (mutually exclusive)');
+        // A DEAD pid + a marker → NOT hanging (the pid-AND gate).
+        recordFault(deadId, {
+            link: 'pi→provider',
+            op: 'generation turn',
+            kind: classify('pi→provider', 'Connection error.').kind,
+            retry: { disposition: 'auto', by: 'daemon' },
+            message: 'Connection error.',
+        });
+        assert.equal(find(deadId)?.hanging ?? null, null, 'dead pid → marker ignored (pid gate)');
+        // Clear → gone.
+        clearFault(liveId);
+        assert.equal(find(liveId)?.hanging ?? null, null, 'cleared marker → not hanging');
+    }
+    finally {
+        await h.dispose();
+    }
+});

package/dist/core/__tests__/fault-retry-rewind.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/fault-retry-rewind.test.js

@@ -0,0 +1,123 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/fault-retry-rewind.test.ts
+//
+// BUG-REGRESSION: broker-owned fault retries rewind the session tree before
+// re-sending the recovery nudge, so the active branch keeps exactly one nudge
+// and the first failed attempt stays on an abandoned sibling branch.
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { createHeadlessHarness } from './helpers/harness.js';
+import { CONNECTION_FAULT_RECOVERY_BODY } from '../runtime/fault-recovery-nudge.js';
+const TASK = 'fault-retry rewind regression task';
+function messageText(message) {
+    if (typeof message.content === 'string')
+        return message.content;
+    const content = message.content ?? [];
+    for (const part of content) {
+        if (typeof part?.text === 'string' && part.text !== '')
+            return part.text;
+    }
+    return '';
+}
+function flatten(nodes) {
+    const out = [];
+    const walk = (node) => {
+        out.push(node);
+        for (const child of node.children)
+            walk(child);
+    };
+    for (const node of nodes)
+        walk(node);
+    return out;
+}
+function pathToLeaf(nodes, leafId) {
+    if (leafId === null)
+        return [];
+    const byId = new Map(nodes.map((node) => [node.entry.id, node]));
+    const path = [];
+    let cur = byId.get(leafId);
+    while (cur !== undefined) {
+        path.push(cur);
+        cur = cur.entry.parentId !== null ? byId.get(cur.entry.parentId) : undefined;
+    }
+    return path.reverse();
+}
+function readSnapshot(home, nodeId) {
+    const path = join(home, 'nodes', nodeId, 'fake-pi.tree.json');
+    return JSON.parse(readFileSync(path, 'utf8'));
+}
+function agentEndCount(home, nodeId) {
+    const path = join(home, 'nodes', nodeId, 'fake-pi.events.jsonl');
+    try {
+        return readFileSync(path, 'utf8')
+            .split('\n')
+            .filter((line) => line.trim() !== '')
+            .filter((line) => {
+            try {
+                return JSON.parse(line).event === 'agent_end';
+            }
+            catch {
+                return false;
+            }
+        }).length;
+    }
+    catch {
+        return 0;
+    }
+}
+test('broker fault retries rewind before replaying the recovery nudge', { timeout: 30_000 }, async () => {
+    const prevProbe = process.env['CRTR_FAULT_RETRY_PROBE'];
+    const prevFails = process.env['CRTR_FAKE_ENGINE_PROMPT_FAILS'];
+    process.env['CRTR_FAULT_RETRY_PROBE'] = 'always';
+    process.env['CRTR_FAKE_ENGINE_PROMPT_FAILS'] = '2';
+    const h = await createHeadlessHarness({ sessionPrefix: 'crtr-fault-rewind' });
+    try {
+        const root = h.spawnRoot('fault-retry rewind root');
+        const nodeId = await h.spawnHeadlessChild(root, TASK);
+        await h.waitFor(() => (agentEndCount(h.home, nodeId) >= 2 ? true : null), { timeoutMs: 20_000, label: 'second failed attempt to settle' });
+        const snapshot = await h.waitFor(() => {
+            const fresh = readSnapshot(h.home, nodeId);
+            const flat = flatten(fresh.tree);
+            const path = pathToLeaf(flat, fresh.leafId);
+            const pathIds = new Set(path.map((node) => node.entry.id));
+            const nudgeOnPath = path.filter((node) => node.entry.message.role === 'user' && messageText(node.entry.message) === CONNECTION_FAULT_RECOVERY_BODY);
+            if (nudgeOnPath.length !== 1)
+                return null;
+            const nudgeNode = nudgeOnPath[0];
+            const failedAttempt = flat.find((node) => node.entry.message.role === 'assistant'
+                && node.entry.message.stopReason === 'error'
+                && messageText(node.entry.message) === TASK);
+            if (failedAttempt === undefined)
+                return null;
+            if (pathIds.has(failedAttempt.entry.id))
+                return null;
+            if (failedAttempt.entry.parentId !== nudgeNode.entry.parentId)
+                return null;
+            return fresh;
+        }, { timeoutMs: 20_000, label: 'rewound active branch snapshot' });
+        const flat = flatten(snapshot.tree);
+        const path = pathToLeaf(flat, snapshot.leafId);
+        const pathMessages = path.map((node) => node.entry.message);
+        assert.equal(pathMessages.filter((message) => message.role === 'user' && messageText(message) === CONNECTION_FAULT_RECOVERY_BODY).length, 1, 'the active branch carries exactly one recovery nudge');
+        const failedAttempt = flat.find((node) => node.entry.message.role === 'assistant'
+            && node.entry.message.stopReason === 'error'
+            && messageText(node.entry.message) === TASK);
+        assert.ok(failedAttempt !== undefined, 'the first failed attempt is still present in the tree');
+        assert.ok(!path.some((node) => node.entry.id === failedAttempt.entry.id), 'the first failed attempt is not on the active path');
+        const nudgeNode = path.find((node) => node.entry.message.role === 'user' && messageText(node.entry.message) === CONNECTION_FAULT_RECOVERY_BODY);
+        assert.ok(nudgeNode !== undefined, 'the active branch includes the recovery nudge');
+        assert.equal(failedAttempt.entry.parentId, nudgeNode.entry.parentId, 'the failed attempt is a sibling branch of the active nudge');
+    }
+    finally {
+        if (prevProbe === undefined)
+            delete process.env['CRTR_FAULT_RETRY_PROBE'];
+        else
+            process.env['CRTR_FAULT_RETRY_PROBE'] = prevProbe;
+        if (prevFails === undefined)
+            delete process.env['CRTR_FAKE_ENGINE_PROMPT_FAILS'];
+        else
+            process.env['CRTR_FAKE_ENGINE_PROMPT_FAILS'] = prevFails;
+        await h.dispose();
+    }
+});