@north-light/crouter 0.3.154

package/dist/core/__tests__/daemon-boot.test.js

@@ -0,0 +1,374 @@
+import { test, before, after, beforeEach } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { spawnSync, spawn } from 'node:child_process';
+import { createServer } from 'node:net';
+import { createNode, getNode, subscribe } from '../canvas/canvas.js';
+import { closeDb } from '../canvas/db.js';
+import { readInboxSince } from '../feed/inbox.js';
+import { superviseTick, readPidfile } from '../../daemon/crtrd.js';
+import { verifyDaemonStartup, waitForDaemonExit } from '../../daemon/manage.js';
+import { waitForBrokerViewSocket } from '../runtime/placement-tmux.js';
+import { viewSocketPath } from '../canvas/paths.js';
+import { signBrandedHostArtifacts, brandedHostWorks, codesignIdentitiesForBrandedHost } from '../runtime/branded-host.js';
+import { markBusy, clearBusy } from '../runtime/busy.js';
+import { bindTestFleet } from './helpers/fleet.js';
+import { resetFleetForTests } from '../runtime/fleet.js';
+let home;
+let tf;
+function node(id, over = {}) {
+    return {
+        node_id: id,
+        name: id,
+        created: new Date().toISOString(),
+        cwd: '/tmp/work',
+        kind: 'general',
+        mode: 'base',
+        lifecycle: 'terminal',
+        status: 'active',
+        ...over,
+    };
+}
+/** Wire `parent subscribes_to child` — the spawn-time edge (parent wakes on the
+ *  child's pushes), so the daemon's boot-failure push reaches the parent. */
+function spawnEdge(parent, child) {
+    subscribe(parent, child, true);
+}
+/** A pid that is guaranteed dead (models a crashed engine). */
+function deadPid() {
+    const r = spawnSync('true', [], { stdio: 'ignore' });
+    return r.pid ?? 0x7ffffffe;
+}
+before(() => {
+    home = mkdtempSync(join(tmpdir(), 'crtr-daemon-boot-'));
+    process.env['CRTR_HOME'] = home;
+    // superviseTick + exit-policy delivery require a bound fleet (D-2).
+    tf = bindTestFleet();
+});
+beforeEach(() => {
+    closeDb();
+    rmSync(home, { recursive: true, force: true });
+});
+after(() => {
+    closeDb();
+    rmSync(home, { recursive: true, force: true });
+    delete process.env['CRTR_HOME'];
+    tf.fleet.dispose();
+    resetFleetForTests();
+});
+// A broker that exits while pi_session_id was NEVER set: the vehicle died
+// before pi ever booted. The spawner already returned status="active", so the
+// parent must be told — the exit-policy job (table row 8) marks it dead AND
+// pushes urgent up. The old null-pid boot-grace clock is gone: the exit event
+// names the exact death, so there is nothing to bridge.
+test('never-booted broker exit is marked dead and surfaced to the parent as urgent', async () => {
+    createNode(node('P', { kind: 'developer', lifecycle: 'resident' }));
+    createNode(node('C', {
+        parent: 'P',
+        kind: 'explore',
+        pi_session_id: null, // never booted
+        intent: null,
+    }));
+    spawnEdge('P', 'C');
+    // Exit-event authority: the launch's real child exit (delivered synthetically
+    // here — same path) classifies as table row 8: pi_session_id null, no fork
+    // claim → boot failure. No grace, no retry.
+    tf.fleet.deliverExit('C', { code: 1, signal: null });
+    await tf.drain();
+    assert.equal(getNode('C').status, 'dead', 'never-booted child is dead');
+    // Row 8 fans TWO entries: the urgent surfaceBootFailure push and the normal
+    // doctrine wake (design flow B row 8 — crash + surfaceBootFailure + doctrine
+    // wake), both from C.
+    const inbox = readInboxSince('P');
+    assert.equal(inbox.length, 2, 'parent received the boot-failure push and the doctrine wake');
+    const urgent = inbox.find((e) => e.tier === 'urgent');
+    assert.ok(urgent, 'boot failure is delivered as urgent');
+    assert.equal(urgent.from, 'C');
+    assert.match(urgent.label, /Spawn failed/);
+    const wake = inbox.find((e) => e.tier === 'normal');
+    assert.ok(wake, 'the doctrine wake rides beside the urgent push');
+    assert.match(wake.label, /crashed/i);
+});
+// BROKER CUT: surfaceChildDeath was DELETED and the tick runs no liveness pass.
+// A crashed booted child (engine pid DEAD, pi_session_id set) is never marked
+// `dead` by a tick and never fans a "child died" notice — recovery rides the
+// broker's real exit event (policy rows 5/6 revive it on its saved session), so
+// a fabricated row with no fleet entry is simply left revivable and the doctrine wake
+// relocated (the revived child pushes when it truly finishes; a deliberate close
+// is woken by close.ts). This test preserves the discriminating intent in the
+// new model: a crash is never reaped to `dead` and never raises a false
+// boot-failure alarm.
+test('a crashed booted child is revivable, not reaped, and raises no false boot-failure alarm', async () => {
+    createNode(node('P2', { kind: 'developer', lifecycle: 'resident' }));
+    createNode(node('C2', {
+        parent: 'P2',
+        kind: 'explore',
+        pi_pid: deadPid(), // engine crashed — its recorded pid is dead
+        pi_session_id: '019e8f00-booted-once', // it booted before dying
+        intent: null,
+    }));
+    spawnEdge('P2', 'C2');
+    markBusy('C2'); // died mid-generation
+    await superviseTick();
+    // The tick has no liveness pass: a stale recorded pid is metadata, not a
+    // death signal — the row stays active/revivable, never reaped to `dead`.
+    assert.equal(getNode('C2').status, 'active', 'crashed booted child stays revivable, NOT marked dead');
+    // surfaceChildDeath is gone and this is not a never-booted node, so the daemon
+    // fans NO push here — no false boot-failure / child-death alarm on a crash.
+    assert.equal(readInboxSince('P2').length, 0, 'no false boot-failure / child-death push on a crash (wake relocated to push/close)');
+    clearBusy('C2');
+});
+// An unplaced active node is not daemon-managed; the daemon leaves it untouched
+// rather than treating the missing broker/session fields as a boot failure.
+test('an unplaced active node is left alone before it boots', async () => {
+    createNode(node('S', { status: 'active', pi_session_id: null }));
+    await superviseTick();
+    assert.equal(getNode('S').status, 'active', 'unplaced active node is not reaped');
+});
+// Major-4 (kept under exit-event authority): a correctly-dormant idle-release
+// node with no fleet entry must NOT be spuriously revived by the tick. Pass-2
+// is the ONLY wake path for a released node, and it fires solely on an unseen
+// inbox entry — a null recorded pid plus a saved session must never read as a
+// "stranded relaunch" (that inference branch is deleted).
+test('idle-release node with a boot-cleared (null) pid and no unseen inbox stays dormant', async () => {
+    createNode(node('D', {
+        status: 'idle',
+        intent: 'idle-release',
+        pi_pid: null, // cleared by boot reconciliation
+        pi_session_id: '019e8f00-freed-while-dormant', // set → would look like a stranded relaunch
+    }));
+    const t0 = Date.now();
+    await superviseTick(t0); // first observation
+    await superviseTick(t0 + 60_000); // well past the boot grace
+    const after = getNode('D');
+    assert.equal(after.status, 'idle', 'dormant idle-release node is left idle, not revived');
+    assert.equal(after.intent, 'idle-release', 'intent unchanged — no revive fired');
+    assert.equal(after.pi_pid, null, 'no fresh pid — reviveNode was never invoked');
+});
+// Daemon start guard: spawnDaemon must not report success before the daemon is
+// plausibly alive and has written its pidfile. This is a pure bounded poll over
+// injected clocks/probes, so we can lock it down deterministically without a
+// real daemon process.
+test('verifyDaemonStartup waits for pidfile + live pid, then reports success for the spawned pid', async () => {
+    let now = 0;
+    let pidfileReady = false;
+    let pidAliveReady = false;
+    let slept = 0;
+    const existingPid = await verifyDaemonStartup(42, 50, {
+        now: () => now,
+        readPidfile: () => (pidfileReady ? 42 : null),
+        isPidAlive: () => pidAliveReady,
+        sleepMs: (ms) => {
+            slept += ms;
+            now += ms;
+            if (now >= 20)
+                pidfileReady = true;
+            if (now >= 30)
+                pidAliveReady = true;
+        },
+    });
+    assert.equal(existingPid, null, 'the spawned daemon owns the pidfile when startup succeeds');
+    assert.equal(pidfileReady, true, 'pidfile must have appeared before success');
+    assert.equal(pidAliveReady, true, 'pid must have been alive before success');
+    assert.equal(slept > 0, true, 'the guard must actually poll');
+});
+// Regression for the cold-guest recreate blocker: the production startup window
+// must tolerate a slower cold start than the old 500ms bound.
+test('verifyDaemonStartup default window tolerates a slower cold start', async () => {
+    let now = 0;
+    let pidfileReady = false;
+    let pidAliveReady = false;
+    const existingPid = await verifyDaemonStartup(42, undefined, {
+        now: () => now,
+        readPidfile: () => (pidfileReady ? 42 : null),
+        isPidAlive: () => pidAliveReady,
+        sleepMs: (ms) => {
+            now += ms;
+            if (now >= 600)
+                pidfileReady = true;
+            if (now >= 700)
+                pidAliveReady = true;
+        },
+    });
+    assert.equal(existingPid, null, 'the production startup window must allow a cold guest to settle');
+});
+// REGRESSION (#508): a fixed 5s startup window falsely failed a valid cold
+// boot right after `install-runtime`, where the daemon's first-ever module
+// load + possible db migration can push writePidfile() past 5s even though
+// the daemon is starting up fine. The production window must comfortably
+// tolerate a pidfile that only appears well past the old 5s bound.
+test('verifyDaemonStartup default window tolerates a cold start slower than the old 5s bound', async () => {
+    let now = 0;
+    let pidfileReady = false;
+    let pidAliveReady = false;
+    const existingPid = await verifyDaemonStartup(42, undefined, {
+        now: () => now,
+        readPidfile: () => (pidfileReady ? 42 : null),
+        isPidAlive: () => pidAliveReady,
+        sleepMs: (ms) => {
+            now += ms;
+            if (now >= 12_000)
+                pidfileReady = true;
+            if (now >= 12_100)
+                pidAliveReady = true;
+        },
+    });
+    assert.equal(existingPid, null, 'the production startup window must survive a pidfile that appears well past the old 5s bound');
+});
+// A concurrent start race is success when another live daemon already owns the
+// pidfile; the loser must return that existing pid instead of timing out.
+test('verifyDaemonStartup returns the existing live daemon when another pid won the startup race', async () => {
+    let now = 0;
+    let slept = 0;
+    const existingPid = await verifyDaemonStartup(42, 50, {
+        now: () => now,
+        readPidfile: () => 7,
+        isPidAlive: (pid) => pid === 7,
+        sleepMs: (ms) => {
+            slept += ms;
+            now += ms;
+        },
+    });
+    assert.equal(existingPid, 7, 'the live pidfile owner must be reported back');
+    assert.equal(slept, 0, 'no polling is needed once another live daemon is visible');
+});
+// A stop acknowledgement is the synchronization point for a following start:
+// it waits for both the old process and its pidfile ownership to disappear.
+test('waitForDaemonExit holds stop→start until the old pid is dead and the pidfile is released', async () => {
+    let now = 0;
+    let pidAlive = true;
+    let pidfileOwner = 42;
+    await waitForDaemonExit(42, 50, {
+        now: () => now,
+        isPidAlive: () => pidAlive,
+        readPidfile: () => pidfileOwner,
+        sleepMs: (ms) => {
+            now += ms;
+            if (now >= 10)
+                pidfileOwner = null;
+            if (now >= 20)
+                pidAlive = false;
+        },
+    });
+    assert.equal(now, 20, 'stop does not acknowledge while the signaled process is still alive');
+    assert.equal(pidfileOwner, null, 'a following start sees no owner in the pidfile');
+});
+// The guard stays bounded: if the daemon never becomes ready, it throws instead
+// of claiming "started" forever.
+test('verifyDaemonStartup throws after its tiny startup window expires', async () => {
+    let now = 0;
+    await assert.rejects(async () => verifyDaemonStartup(99, 25, {
+        now: () => now,
+        readPidfile: () => null,
+        isPidAlive: () => false,
+        sleepMs: (ms) => {
+            now += ms;
+        },
+    }), /did not become ready within 25ms/);
+});
+// REGRESSION (bug #102): the verifier polls with a real async sleep, NOT a
+// synchronous Atomics.wait, so the event loop stays free to dispatch the spawned
+// child's `exit` event. We spawn a REAL child that exits immediately and let the
+// real wall clock + real setTimeout sleep drive the loop: the exit event fires
+// mid-wait, childExited() flips non-null, and the verifier fails fast with the
+// exit reason — well inside the 2s window. If the loop blocked the event loop
+// (the old bug), the exit event would never fire and this would time out with the
+// generic message instead.
+test('verifyDaemonStartup throws when the spawned daemon exits before it can become ready', async () => {
+    let exited = null;
+    const child = spawn(process.execPath, ['-e', 'process.exit(1)'], { stdio: 'ignore' });
+    child.once('exit', (code, signal) => {
+        exited = { code, signal };
+    });
+    const started = Date.now();
+    await assert.rejects(async () => verifyDaemonStartup(77, 2_000, {
+        readPidfile: () => null,
+        isPidAlive: () => false,
+        childExited: () => exited,
+    }), /exited before becoming ready \(with exit code 1\)/);
+    assert.ok(Date.now() - started < 1_500, 'must fail fast on the real child exit, not run out the window');
+});
+test('the daemon honours an explicit CRTR_PIDFILE override', () => {
+    const home = mkdtempSync(join(tmpdir(), 'crtr-pidfile-home-'));
+    const pidfile = join(home, 'crtrd.pid');
+    const prevHome = process.env['CRTR_HOME'];
+    const prevPidfile = process.env['CRTR_PIDFILE'];
+    process.env['CRTR_HOME'] = mkdtempSync(join(tmpdir(), 'crtr-home-'));
+    process.env['CRTR_PIDFILE'] = pidfile;
+    writeFileSync(pidfile, '321');
+    try {
+        assert.equal(readPidfile(), 321, 'the daemon must read its pidfile from CRTR_PIDFILE, not CRTR_HOME');
+    }
+    finally {
+        if (prevHome === undefined)
+            delete process.env['CRTR_HOME'];
+        else
+            process.env['CRTR_HOME'] = prevHome;
+        if (prevPidfile === undefined)
+            delete process.env['CRTR_PIDFILE'];
+        else
+            process.env['CRTR_PIDFILE'] = prevPidfile;
+    }
+});
+// REGRESSION (bug #102 follow-up): with NO exit promise, the readiness loop must
+// still observe a real `connect`. A buggy `?? Promise.resolve('retry')` race lost
+// to the probe instantly, so live-broker callers (focus on an already-alive node,
+// surface tmux-spread) could never return true and stalled the full 30s deadline.
+test('waitForBrokerViewSocket returns true on a live socket when given no exit promise', async () => {
+    // A short /tmp CRTR_HOME keeps the AF_UNIX path under macOS's 104-byte
+    // sun_path limit (listen EINVAL) — the suite's default temp home is too long.
+    const shortHome = mkdtempSync('/tmp/crtr-sock-');
+    const prevHome = process.env['CRTR_HOME'];
+    process.env['CRTR_HOME'] = shortHome;
+    const nodeId = 'lb';
+    const sockPath = viewSocketPath(nodeId);
+    mkdirSync(dirname(sockPath), { recursive: true });
+    const server = createServer();
+    await new Promise((resolve) => server.listen(sockPath, () => resolve()));
+    try {
+        const started = Date.now();
+        const ready = await waitForBrokerViewSocket(nodeId);
+        assert.equal(ready, true, 'a live accepting socket must report ready even without an exit promise');
+        assert.ok(Date.now() - started < 1_000, 'readiness must resolve promptly, not stall the full deadline');
+    }
+    finally {
+        await new Promise((resolve) => server.close(() => resolve()));
+        if (prevHome === undefined)
+            delete process.env['CRTR_HOME'];
+        else
+            process.env['CRTR_HOME'] = prevHome;
+        rmSync(shortHome, { recursive: true, force: true });
+    }
+});
+test('waitForBrokerViewSocket returns false as soon as the real broker process exits', async () => {
+    const child = spawn(process.execPath, ['-e', 'process.exit(1)'], { stdio: 'ignore' });
+    const exited = new Promise((resolve) => {
+        child.once('exit', (code, signal) => {
+            resolve({ code, signal });
+        });
+    });
+    const started = Date.now();
+    const ready = await waitForBrokerViewSocket('dead-broker-for-readiness-test', exited);
+    assert.equal(ready, false, 'an exited broker must fail readiness');
+    assert.ok(Date.now() - started < 1000, 'the readiness guard must fail fast instead of waiting for the socket deadline');
+});
+test('signBrandedHostArtifacts signs the copied libnode before the branded binary', () => {
+    const signed = [];
+    signBrandedHostArtifacts('/tmp/crouter', '/tmp/libnode.dylib', (path) => signed.push(path));
+    assert.deepEqual(signed, ['/tmp/libnode.dylib', '/tmp/crouter']);
+});
+test('codesignIdentitiesForBrandedHost prefers configured or discovered identities and otherwise ad-hoc signs', () => {
+    assert.deepEqual(codesignIdentitiesForBrandedHost({ configured: 'Developer ID Application: Example', discovered: 'Apple Development: Local' }), ['Developer ID Application: Example']);
+    assert.deepEqual(codesignIdentitiesForBrandedHost({ discovered: 'Apple Development: Local' }), ['Apple Development: Local']);
+    assert.deepEqual(codesignIdentitiesForBrandedHost(), ['-']);
+});
+test('brandedHostWorks health-checks if a binary is executable', () => {
+    // Issue #102: the branded host binary can be corrupted or killed by macOS
+    // before exec, causing spawn() to fail silently. The health-check detects this.
+    const goodBin = process.execPath; // a real, executable binary
+    const badBin = '/nonexistent/crouter-broken';
+    assert.equal(brandedHostWorks(goodBin), true, 'health-check passes for a real executable binary');
+    assert.equal(brandedHostWorks(badBin), false, 'health-check fails for a nonexistent binary');
+});

package/dist/core/__tests__/daemon-wedge.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/daemon-wedge.test.js

@@ -0,0 +1,104 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/daemon-wedge.test.ts
+//
+// Issue #110 — pid-alive WEDGED node detection. The pure decision core:
+// wedgeVerdict(busy, quietForMs, cpuPercent). No wall-clock sleeps, no ps
+// shell-outs, no fake pids — every input is injected directly, mirroring the
+// yieldStallVerdict truth-table suite in refresh-stall-recycle.test.ts.
+//
+// LOCKS: (1) a routine long-running BUT ACTIVE turn (heartbeat recent, or CPU
+// non-trivial) must never verdict 'wedged' — false positives are the failure
+// mode this feature exists to avoid. (2) only busy+quiet+near-zero-CPU, all at
+// once, past the generous grace, fires. (3) a null/failed CPU sample never
+// fires on its own (fails toward 'pending', not 'wedged').
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { wedgeVerdict, WEDGE_QUIET_MS, WEDGE_CPU_MAX_PERCENT, sumTreeCpu, treeHasDescendants } from '../../daemon/crtrd.js';
+test('wedgeVerdict: a node whose turn already ended is never wedged, regardless of the other signals', () => {
+    assert.equal(wedgeVerdict({ busy: false, quietForMs: null, cpuPercent: null }), 'leave');
+    assert.equal(wedgeVerdict({ busy: false, quietForMs: 10 * 60 * 60_000, cpuPercent: 0 }), 'leave');
+});
+test('wedgeVerdict: a busy node with an unreadable heartbeat is left alone (not enough evidence)', () => {
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: null, cpuPercent: null }), 'leave');
+});
+test('wedgeVerdict: routine turns — even long ones — never trip before the grace elapses', () => {
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: 0, cpuPercent: 0 }), 'leave', 'just started');
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: 5 * 60_000, cpuPercent: 0 }), 'leave', 'a routine multi-minute tool call');
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS - 1, cpuPercent: 0 }), 'leave', 'just under the grace, even with 0% cpu');
+});
+test('wedgeVerdict: past the grace with NO cpu corroboration only pends — never fires on time alone', () => {
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS, cpuPercent: null }), 'pending', 'cpu sample failed/unavailable — never treated as confirmation');
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS * 10, cpuPercent: null }), 'pending', 'no amount of extra quiet time substitutes for the corroborating signal');
+});
+test('wedgeVerdict: a long-quiet turn that is STILL VISIBLY WORKING (real cpu) is exonerated, not wedged', () => {
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS * 3, cpuPercent: WEDGE_CPU_MAX_PERCENT + 0.1 }), 'pending', 'cpu just above the ceiling — a genuinely churning long build — must not fire');
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS * 3, cpuPercent: 40 }), 'pending', 'clearly-active cpu never fires regardless of how quiet the heartbeat looks');
+});
+test('wedgeVerdict: fires ONLY when busy + quiet-past-grace + near-zero cpu all corroborate', () => {
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS, cpuPercent: 0 }), 'wedged');
+    assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS + 60_000, cpuPercent: WEDGE_CPU_MAX_PERCENT }), 'wedged', 'at the cpu ceiling (inclusive)');
+});
+// ---------------------------------------------------------------------------
+// sumTreeCpu (pure) — process-tree CPU sampling. Review fix for issue #110:
+// sampling only the broker pid false-fired on a broker awaiting a busy CHILD
+// subprocess. These lock the tree-sum behavior directly, independent of any
+// `ps` shell-out.
+// ---------------------------------------------------------------------------
+test('sumTreeCpu: a busy CHILD subprocess exonerates an idle broker (the false-positive this fixes)', () => {
+    const ps = [
+        '  100     1  0.0', // broker, idle
+        '  200   100 85.0', // child of the broker, busy
+    ].join('\n');
+    assert.equal(sumTreeCpu(ps, 100), 85.0);
+});
+test('sumTreeCpu: broker + all descendants idle sums to ~0 (the real wedge is still detected)', () => {
+    const ps = [
+        '  100     1  0.0',
+        '  200   100  0.0',
+        '  201   100  0.0',
+    ].join('\n');
+    assert.equal(sumTreeCpu(ps, 100), 0);
+});
+test('sumTreeCpu: sums across multiple generations (grandchild busy)', () => {
+    const ps = [
+        '  100     1  0.0', // broker
+        '  200   100  0.0', // child, idle
+        '  300   200 40.0', // grandchild, busy
+    ].join('\n');
+    assert.equal(sumTreeCpu(ps, 100), 40.0);
+});
+test('sumTreeCpu: root pid absent from the ps output returns null (can\'t measure)', () => {
+    const ps = [
+        '  200     1  0.0',
+        '  300   200 10.0',
+    ].join('\n');
+    assert.equal(sumTreeCpu(ps, 100), null);
+});
+test('sumTreeCpu: malformed/blank lines are skipped; empty output returns null', () => {
+    const ps = [
+        '',
+        '   ',
+        'garbage line not ps shaped',
+        '  100     1  5.0',
+        '  bad   100  1.0',
+    ].join('\n');
+    assert.equal(sumTreeCpu(ps, 100), 5.0);
+    assert.equal(sumTreeCpu('', 100), null);
+    assert.equal(sumTreeCpu('   \n  \n', 100), null);
+});
+// ---------------------------------------------------------------------------
+// treeHasDescendants (pure) — issue #119: the broker-alone case has no
+// subprocess to kill, so remediation must branch on this instead of assuming
+// a runaway child always exists.
+// ---------------------------------------------------------------------------
+test('treeHasDescendants: broker alone in its own tree — nothing to kill', () => {
+    const ps = ['  100     1  0.0'].join('\n');
+    assert.equal(treeHasDescendants(ps, 100), false);
+});
+test('treeHasDescendants: a child present — a subprocess to kill', () => {
+    const ps = ['  100     1  0.0', '  200   100  0.0'].join('\n');
+    assert.equal(treeHasDescendants(ps, 100), true);
+});
+test('treeHasDescendants: root pid absent from the ps output returns null (can\'t tell)', () => {
+    const ps = ['  200     1  0.0'].join('\n');
+    assert.equal(treeHasDescendants(ps, 100), null);
+});

package/dist/core/__tests__/dead-node-policy-table.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/dead-node-policy-table.test.js

@@ -0,0 +1,103 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/dead-node-policy-table.test.ts
+//
+// The invariant-5 policy table (design flow B): `classifyDeadNode` is the ONE
+// pure function mapping a known-dead node's durable row state to its recovery
+// action, used at exactly two call sites — the exit-policy job and the startup
+// recovery sweep — so exit recovery and startup recovery can never diverge.
+// This file pins every table row directly, replacing the scattered pins the
+// five deleted liveness-inference branches used to carry (grace-clock,
+// stranded-relaunch, daemon-liveness), plus the `retryResumeMode` policy pins
+// ported verbatim from the deleted grace-clock suite (the function is pure and
+// unchanged — only its caller moved into the table).
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { classifyDeadNode, retryResumeMode, respawnBackoffMs, HEALTHY_UPTIME_MS, MAX_RESPAWN_ATTEMPTS, } from '../../daemon/fleet.js';
+/** A table-row fixture: an active, booted-or-not node in the named state. */
+function row(over = {}) {
+    return {
+        status: 'active',
+        intent: null,
+        pi_session_id: null,
+        cycle_pending: null,
+        private_fork_phase: null,
+        ...over,
+    };
+}
+// ===========================================================================
+// The nine table rows, in evaluation order (design flow B).
+// ===========================================================================
+test('row 1: a deleted row (orphan of a pruned/closed node) is forgotten', () => {
+    assert.equal(classifyDeadNode(null, false), 'forget');
+    assert.equal(classifyDeadNode(null, true), 'forget', 'a stale busy marker cannot resurrect a deleted row');
+});
+test('row 2: settled terminal exits (done/canceled/dead) are forgotten — fanouts already happened at the transition site', () => {
+    assert.equal(classifyDeadNode(row({ status: 'done' }), false), 'forget');
+    assert.equal(classifyDeadNode(row({ status: 'canceled' }), false), 'forget');
+    assert.equal(classifyDeadNode(row({ status: 'dead' }), false), 'forget');
+    assert.equal(classifyDeadNode(row({ status: 'done' }), true), 'forget', 'terminal status outranks a leftover busy marker');
+});
+test('row 3: intent=refresh (yield/refresh exit) respawns fresh — no grace, no capacity check', () => {
+    assert.equal(classifyDeadNode(row({ intent: 'refresh' }), false), 'respawn-fresh');
+    assert.equal(classifyDeadNode(row({ intent: 'refresh', pi_session_id: 'sess' }), true), 'respawn-fresh', 'refresh outranks the crash rows below it — the exit was deliberate');
+});
+test('row 4: intent=idle-release (chosen dormancy) takes no action — pass-2 inbox wake owns the next revive', () => {
+    assert.equal(classifyDeadNode(row({ intent: 'idle-release' }), false), 'dormant');
+    assert.equal(classifyDeadNode(row({ intent: 'idle-release', pi_session_id: 'sess' }), false), 'dormant', 'a saved session does not turn chosen dormancy into a crash');
+});
+test('row 5: a broker-certified clean abort resumes its persisted session', () => {
+    assert.equal(classifyDeadNode(row({ pi_session_id: 'sess' }), true, true), 'respawn-resume');
+    assert.equal(classifyDeadNode(row({ pi_session_id: 'sess', cycle_pending: true }), true, true), 'respawn-resume', 'the clean-abort certificate proves this interrupted turn is resumable, even if a prior cycle was pending');
+    assert.equal(classifyDeadNode(row(), true, true), 'respawn-cycle', 'a certificate without a saved session cannot turn a dirty busy death into a resume');
+});
+test('row 6: a set busy marker without clean-abort evidence is a dirty mid-turn crash — ALWAYS a recovery cycle', () => {
+    assert.equal(classifyDeadNode(row(), true), 'respawn-cycle');
+    assert.equal(classifyDeadNode(row({ pi_session_id: 'sess' }), true), 'respawn-cycle');
+    assert.equal(classifyDeadNode(row({ pi_session_id: 'sess', cycle_pending: true }), true), 'respawn-cycle');
+});
+test('row 7: a saved session with no crash markers resumes strictly; a pending cycle retries AS a cycle', () => {
+    assert.equal(classifyDeadNode(row({ pi_session_id: 'sess' }), false), 'respawn-resume');
+    assert.equal(classifyDeadNode(row({ pi_session_id: 'sess', cycle_pending: true }), false), 'respawn-cycle');
+    assert.equal(classifyDeadNode(row({ pi_session_id: 'sess', cycle_pending: false }), false), 'respawn-resume');
+});
+test('row 8: a private fork dead before binding its child session retries the immutable fork birth', () => {
+    assert.equal(classifyDeadNode(row({ private_fork_phase: 'fork_pending' }), false), 'respawn-fresh');
+});
+test('row 9: sessionless with no fork claim — a fresh launch died before session_start — is a boot failure', () => {
+    assert.equal(classifyDeadNode(row(), false), 'boot-failure');
+    assert.equal(classifyDeadNode(row({ private_fork_phase: 'session_started' }), false), 'boot-failure', 'a bound fork whose session identity was cleared for a fresh launch classifies like any fresh launch');
+});
+// ===========================================================================
+// retryResumeMode (pure) — ported verbatim from the deleted grace-clock suite.
+// A crash-retry retries AS A CYCLE (resume:false) when the dead attempt's own
+// launch was itself a cycling revive that never reached session_start
+// (cycle_pending), OR when a surviving busy marker proves the broker died
+// mid-turn. Idle crashes resume STRICTLY (resume:true). See canvas.test.ts's
+// updateNode/getNode round-trip for the companion guarantee that cycle_pending
+// survives the persistence the policy job reads it back from.
+// ===========================================================================
+test('retryResumeMode: interrupted turns and pending cycles recover as cycles; idle crashes resume strictly', () => {
+    assert.equal(retryResumeMode({ cycle_pending: true }, false), false, 'a pending cycling attempt retries AS a cycle');
+    assert.equal(retryResumeMode({ cycle_pending: false }, true), false, 'a surviving busy marker starts a recovery cycle');
+    assert.equal(retryResumeMode({ cycle_pending: true }, true), false, 'both recovery signals still produce one cycle');
+    assert.equal(retryResumeMode({ cycle_pending: false }, false), true, 'an idle ordinary crash resumes strictly');
+    assert.equal(retryResumeMode({ cycle_pending: null }, false), true, 'cycle_pending=null idle crash resumes strictly');
+    assert.equal(retryResumeMode({}, false), true, 'cycle_pending undefined idle crash resumes strictly');
+    assert.equal(retryResumeMode(null, false), true, 'no meta and no busy marker resumes strictly');
+});
+// ===========================================================================
+// The throttle's backoff schedule (pure): attempt n waits min(5s·2^(n-1), 10min);
+// zero consecutive failures respawn immediately (the sub-second yield respawn
+// D-3 buys). The stateful throttle behavior lives in respawn-throttle.test.ts.
+// ===========================================================================
+test('respawnBackoffMs: immediate at zero failures, doubling from 5s, capped at 10 minutes', () => {
+    assert.equal(respawnBackoffMs(0), 0);
+    assert.equal(respawnBackoffMs(1), 5_000);
+    assert.equal(respawnBackoffMs(2), 10_000);
+    assert.equal(respawnBackoffMs(3), 20_000);
+    assert.equal(respawnBackoffMs(5), 80_000);
+    assert.equal(respawnBackoffMs(20), 600_000, 'the backoff is capped, never unbounded');
+});
+test('throttle constants: healthy uptime is 60s; the loop terminalizes at 6 consecutive short-lived exits', () => {
+    assert.equal(HEALTHY_UPTIME_MS, 60_000);
+    assert.equal(MAX_RESPAWN_ATTEMPTS, 6);
+});

package/dist/core/__tests__/deferred-no-wake.test.d.ts

@@ -0,0 +1 @@
+export {};