@north-light/crouter 0.3.154

package/dist/core/__tests__/helpers/harness.js

@@ -0,0 +1,867 @@
+// helpers/harness.ts — a reusable, FAITHFUL integration-test driver for the
+// node/canvas runtime.
+//
+// It drives the REAL `crtr` CLI (as subprocesses, AS specific nodes) into a
+// REAL but isolated tmux session, substitutes the fake-pi vehicle (fixtures/
+// fake-pi-host.ts) for the LLM `pi` via the CRTR_PI_BINARY seam, fires the REAL
+// extension hooks inside that fake-pi over a polled control channel, and runs
+// the daemon decision pass in-process via superviseTick(now). Every assertion
+// reads straight off the canvas data layer. NOTHING here mocks the runtime.
+//
+// See harness-design.md §4/§5 and vehicle-and-hooks.md §6 for the architecture
+// this implements. The ONE production seam used is CRTR_PI_BINARY in
+// piCommand (src/core/runtime/tmux.ts) — no production file is modified.
+//
+// Isolation contract (harness-design.md §4a):
+//   • The harness itself runs AS a canvas node, so its OWN process.env carries
+//     the REAL canvas vars. We override CRTR_HOME (+ CRTR_PI_BINARY) for our own
+//     in-process reads/revives, and scrub every canvas var from each subprocess
+//     env. closeDb() rebinds sqlite to the isolated home before every read.
+//   • The isolated tmux session lives on the DEFAULT server (the runtime shells
+//     `tmux` with no -L, so an -L server would be invisible to the real CLI and
+//     to superviseTick). We only ever kill-session, never kill-server.
+import { spawnSync, spawn } from 'node:child_process';
+import { mkdtempSync, rmSync, existsSync, readFileSync, readdirSync, writeFileSync, renameSync, } from 'node:fs';
+import { homedir, tmpdir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createRequire } from 'node:module';
+import { createNode, getNode, recordPid, subscribersOf, subscriptionsOf, } from '../../canvas/canvas.js';
+import { closeDb } from '../../canvas/db.js';
+import { isFocused } from '../../runtime/placement.js';
+import { reviveNode } from '../../runtime/revive.js';
+import { superviseTick } from '../../../daemon/crtrd.js';
+import { DaemonFleet } from '../../../daemon/fleet.js';
+import { apiSocketPath } from '../../canvas/paths.js';
+import { bindDaemonEventSource, resetEventSourceForTests } from '../../events/source.js';
+import { bindFleet, resetFleetForTests } from '../../runtime/fleet.js';
+import { readInboxSince } from '../../feed/inbox.js';
+import { capturePidCommand, recordedPidLiveness } from '../../canvas/pid.js';
+// --- locations --------------------------------------------------------------
+const HERE = dirname(fileURLToPath(import.meta.url));
+const TREE_ROOT = join(HERE, '..', '..', '..'); // src/ or dist/
+const CROUTER = join(TREE_ROOT, '..'); // package root
+const COMPILED = TREE_ROOT === join(CROUTER, 'dist');
+const MODULE_EXT = COMPILED ? '.js' : '.ts';
+const CLI_ENTRY = join(TREE_ROOT, `cli${MODULE_EXT}`);
+const FAKE_PI_HOST = join(HERE, '..', 'fixtures', `fake-pi-host${MODULE_EXT}`);
+// The standalone API-server host (separate process — see the fixture header for
+// why it cannot be in-process alongside the harness's spawnSync cli() driver).
+const API_SERVER_HOST = join(HERE, '..', 'fixtures', `api-server-host${MODULE_EXT}`);
+// The in-process fake BrokerEngine (the SDK analog of the fake-pi vehicle). The
+// REAL broker loads it via the CRTR_BROKER_ENGINE seam (broker-sdk.ts). Source
+// tests load the TypeScript fixture through tsx; compiled tests load its emitted
+// JavaScript directly so every integration child exercises the built runtime.
+const FAKE_ENGINE = join(HERE, '..', 'fixtures', `fake-engine${MODULE_EXT}`);
+const TSX_ESM = COMPILED ? null : createRequire(import.meta.url).resolve('tsx/esm');
+// A multi-word launcher, baked verbatim ahead of the (shell-quoted) argv by the
+// seam. Absolute paths so it works regardless of the spawned window's cwd.
+const FAKE_PI_BINARY = [process.execPath, ...(TSX_ESM === null ? [] : ['--import', TSX_ESM]), FAKE_PI_HOST].join(' ');
+// Every raw `tmux` call the harness/full-tier tests make against the shared
+// DEFAULT server is synchronous (spawnSync). Node's spawnSync blocks the WHOLE
+// worker (no event loop, no `node --test` timeout can preempt it) until the
+// child exits — with no `timeout` option it can wait FOREVER. A wedged/CPU-
+// starved tmux server (the acknowledged "intermittent Linux-only crash-revive
+// wedge", CI-only) has been observed to hang one of these calls indefinitely,
+// which permanently consumes one of the `--test-concurrency` slots; as more
+// full-tier files each hit the SAME wedged server, every slot eventually gets
+// consumed the same way and the whole suite flatlines with zero further TAP
+// output (confirmed 3/3 in CI, see ci-full-suite-hang-blocker.md). TMUX_TIMEOUT_MS
+// bounds every raw tmux spawnSync call so a wedge fails that ONE call (and the
+// test it belongs to) fast instead of hanging the run.
+export const TMUX_TIMEOUT_MS = 10_000;
+/** True when a usable tmux is on PATH — tests gate on this and SKIP otherwise. */
+export function hasTmux() {
+    return spawnSync('tmux', ['-V'], { stdio: 'ignore', timeout: TMUX_TIMEOUT_MS }).status === 0;
+}
+function tmuxSessionExists(session) {
+    return (spawnSync('tmux', ['has-session', '-t', session], {
+        stdio: 'ignore',
+        timeout: TMUX_TIMEOUT_MS,
+    }).status === 0);
+}
+// Every canvas/tmux var the harness itself runs under — scrubbed from each child
+// env so a spawned CLI cannot leak the REAL canvas into the isolated test.
+const CANVAS_ENV_KEYS = [
+    'CRTR_NODE_ID',
+    'CRTR_HOME',
+    'CRTR_ROOT_SESSION',
+    'CRTR_SUBTREE',
+    'CRTR_NODE_SESSION',
+    'CRTR_PARENT_NODE_ID',
+    'CRTR_FRONT_DOOR',
+    'CRTR_KIND',
+    'CRTR_MODE',
+    'CRTR_LIFECYCLE',
+    'CRTR_NODE_CWD',
+    'CRTR_PI_BINARY',
+    // Scrubbed then re-added (controlled) in childEnv, mirroring CRTR_PI_BINARY —
+    // so the headless-broker seam can't leak the REAL engine into the isolated
+    // test, and the CLI subprocess that spawns a broker carries the FAKE one.
+    'CRTR_BROKER_ENGINE',
+    'TMUX',
+    'TMUX_PANE',
+];
+function cleanBaseEnv() {
+    const e = {};
+    for (const [k, v] of Object.entries(process.env))
+        if (v !== undefined)
+            e[k] = v;
+    for (const k of CANVAS_ENV_KEYS)
+        delete e[k];
+    // Contain per-invocation bootstrap + auto-update side effects (they write to
+    // ~/.crouter / ~/.claude / ~/.pi, NOT under CRTR_HOME — HOME is contained too).
+    e['CRTR_NO_BOOTSTRAP'] = '1';
+    e['CRTR_NO_AUTO_UPDATE'] = '1';
+    e['CRTR_NO_EXPORTS'] = '1';
+    e['CRTR_NO_AUTO_INIT'] = '1';
+    return e;
+}
+async function waitFor(probe, opts = {}) {
+    const timeoutMs = opts.timeoutMs ?? 20_000;
+    const intervalMs = opts.intervalMs ?? 100;
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        const v = probe();
+        if (v)
+            return v;
+        if (Date.now() > deadline)
+            throw new Error(`waitFor timed out: ${opts.label ?? 'condition'}`);
+        await new Promise((r) => setTimeout(r, intervalMs));
+    }
+}
+/** Thin wrapper: a headless-only harness (broker-hosted, paneless nodes). */
+export async function createHeadlessHarness(opts = {}) {
+    return createHarness({ ...opts, headless: true });
+}
+export async function createHarness(opts = {}) {
+    const headless = opts.headless === true;
+    if (!headless && !hasTmux())
+        throw new Error('createHarness: tmux not available');
+    const origHome = process.env['CRTR_HOME'];
+    const origAgentHome = process.env['HOME'];
+    const origPiBinary = process.env['CRTR_PI_BINARY'];
+    const origNodeSession = process.env['CRTR_NODE_SESSION'];
+    const origNodeOptions = process.env['NODE_OPTIONS'];
+    const origBrokerEngine = process.env['CRTR_BROKER_ENGINE'];
+    // The headless broker binds a unix socket at <home>/nodes/<id>/view.sock. A
+    // unix socket path (sun_path) has a ~104-char OS limit, and macOS's per-user
+    // $TMPDIR (/var/folders/<…>/T/, ~49 chars) alone pushes that path past the
+    // limit — server.listen() then fails EINVAL and the broker's loop drains
+    // (boots, never serves). NOT a production concern (~/.crouter/canvas/… is
+    // short); a test-only artifact of the long temp base. Use a SHORT base + short
+    // prefix for the canvas home so broker sockets fit. tmpHome (HOME) hosts no
+    // socket, so it can stay on the standard temp base.
+    const shortTmpBase = existsSync('/tmp') ? '/tmp' : tmpdir();
+    const home = mkdtempSync(join(shortTmpBase, 'crh-'));
+    const tmpHome = mkdtempSync(join(tmpdir(), 'crtr-harness-HOME-'));
+    const session = `${opts.sessionPrefix ?? 'crtr-harness'}-${process.pid}-${Date.now().toString(36)}`;
+    // The harness reads/writes the isolated canvas in-process. CRTR_PI_BINARY in
+    // OUR env makes in-process revives (superviseTick → reviveNode → openNodeWindow)
+    // bake the fake-pi into the command string.
+    process.env['CRTR_HOME'] = home;
+    // HOME must be isolated in THIS process too, not just in the subprocess envs
+    // below: an in-process launch (spawnHeadlessChild's reviveNode, superviseTick
+    // pass-2 wakes) builds its broker env from process.env, so a real HOME here
+    // makes every broker read the developer's OWN ~/.pi/agent — auth.json and the
+    // managed subscription pools — instead of agentHome, and provider tests then
+    // pass or fail on whatever credentials the machine happens to hold.
+    // …but the isolated HOME must NOT hide the installed runtime generation from
+    // the `crtr` chrome panes tmux forks. `bin/crtr` resolves its generation from
+    // `$CRTR_RUNTIME_HOME` else `$HOME/.crouter/runtime`, and a viewer pane bakes
+    // in the BARE command `crtr surface attach to <id>` — so with HOME rewritten to
+    // an empty temp dir every viewer pane died with `invalid runtime selection:
+    // missing <tmpHome>/.crouter/runtime/selected` before it could self-tag.
+    // Pin the REAL runtime home (captured before the HOME rewrite) so panes load
+    // the generation `npm run install-runtime` selected. Runner-only until now
+    // purely by accident: a dev box already has a tmux server running with the
+    // developer's real HOME, and a new session inherits the SERVER's env — on a
+    // pristine runner the harness's own `new-session` starts the server, so the
+    // rewritten HOME becomes every pane's HOME.
+    const realRuntimeHome = process.env['CRTR_RUNTIME_HOME'] ?? join(process.env['HOME'] ?? homedir(), '.crouter', 'runtime');
+    process.env['HOME'] = tmpHome;
+    process.env['CRTR_RUNTIME_HOME'] = realRuntimeHome;
+    process.env['CRTR_PI_BINARY'] = FAKE_PI_BINARY;
+    delete process.env['CRTR_NODE_SESSION'];
+    // Source tests need tsx in every spawned broker because host.ts resolves its
+    // sibling broker-cli.js to broker-cli.ts and the fake engine is TypeScript.
+    // Compiled integration tests execute emitted JavaScript directly and scrub an
+    // ambient NODE_OPTIONS so their children do not accidentally pay for or depend
+    // on a source loader. dispose() restores the caller's value in both cases.
+    if (TSX_ESM === null) {
+        delete process.env['NODE_OPTIONS'];
+    }
+    else {
+        process.env['NODE_OPTIONS'] = [process.env['NODE_OPTIONS'], `--import ${TSX_ESM}`]
+            .filter((s) => s !== undefined && s !== '')
+            .join(' ');
+    }
+    process.env['CRTR_BROKER_ENGINE'] = FAKE_ENGINE;
+    closeDb();
+    // The harness drives superviseTick() in-process, so it stands in for the
+    // daemon process (crtrd.ts:2490 binds the same way at real-daemon startup).
+    // Reset first: `bound` is a single-assignment process global with no other
+    // unbind path, and each harness instance points at a different isolated
+    // `home`, so a bare re-bind() across sequential harnesses in one test
+    // process would throw "already bound". Binding daemon (not broker) here
+    // keeps in-process reviveNode()'s host.ts:299 `eventSource() === undefined`
+    // check false, so it never binds a broker source over this one.
+    resetEventSourceForTests();
+    bindDaemonEventSource();
+    // The harness also stands in for the daemon's FLEET: every in-harness broker
+    // launch (reviveNode from a test, superviseTick pass-2 wakes) registers its
+    // ChildProcess handle here, and real child exits enqueue exit-policy jobs on
+    // this serial work lane — the same single-lane discipline crtrd runs, but
+    // drainable to a deterministic quiescent point. Reset first for the same
+    // reason as the event source above (process-global, sequential harnesses).
+    resetFleetForTests();
+    let laneTail = Promise.resolve();
+    let laneClosed = false;
+    function enqueueLaneJob(job) {
+        // Same admission rule as crtrd's shutdown: once the harness is disposed,
+        // late exit events (dispose SIGKILLs every broker; their exit callbacks
+        // fire on later event-loop turns, after CRTR_HOME is restored) must NOT
+        // run policy — they would read the REAL canvas home.
+        if (laneClosed)
+            return laneTail;
+        // Drop the cached SQLite handle before every policy job. crtrd owns canvas
+        // state in-process, so its exit policy always reads its own writes; the
+        // harness runs lifecycle verbs as SUBPROCESSES (`h.cli`), whose WAL writes
+        // are invisible to a cached handle. Without this, an exit landing just
+        // after `node lifecycle close` reads the pre-close row (still active, with
+        // a session id), classifies `respawn-resume`, and RELAUNCHES a node the
+        // close just canceled — the intermittent "broker engine destroyed by
+        // close" failure, which only reproduced when the exit lost the race.
+        const run = laneTail.then(() => { closeDb(); return job(); }).then(() => { }, (err) => {
+            // Policy jobs contain their own error handling; anything escaping is a
+            // genuine bug — surface it, never wedge the lane.
+            console.error('[harness] fleet lane job failed:', err);
+        });
+        laneTail = run;
+        return run;
+    }
+    async function drainLane() {
+        let tail;
+        do {
+            tail = laneTail;
+            await tail;
+        } while (tail !== laneTail);
+    }
+    const fleet = new DaemonFleet({ epoch: 'harness-epoch', enqueue: enqueueLaneJob });
+    bindFleet(fleet);
+    // Neutralize ensureDaemon (spawn.ts calls it on every `node new`): write a
+    // fake-live-daemon pidfile pointing at OUR (always-alive) pid, so
+    // isDaemonRunning() is true and no REAL crtrd is spawned against the isolated
+    // home. Source-mode broker launches propagate NODE_OPTIONS=--import tsx to
+    // subprocesses, so allowing ensureDaemon to start a real supervisor would race
+    // the in-process superviseTick + fixed-clock crash/grace scenarios. The same
+    // containment keeps compiled runs identical. The harness drives the daemon pass
+    // itself via `tick()`. (fakeLiveDaemon pattern, home-session.test.ts.) home/ is
+    // rmSync'd in dispose.
+    writeFileSync(join(home, 'crtrd.pid'), String(process.pid), 'utf8');
+    const pidsToKill = new Set();
+    let nextRootSeq = 0;
+    // Every REAL `crtr` CLI subprocess this harness spawns (`node new`,
+    // `node promote`, `node message send`, `push final`, …) routes its canvas reads/mutations
+    // through cliClient() over crtrd's socket. The fake pidfile above makes
+    // isDaemonRunning() true so ensureDaemon never spawns a REAL crtrd against the
+    // isolated home — but that leaves the socket UNBOUND, so a subprocess client
+    // finds nothing (exit 5, daemon_unavailable). Stand up an API server on THIS
+    // isolated home's socket. It MUST run in a SEPARATE process: cli() drives the
+    // CLI via spawnSync, which blocks this process's event loop until the child
+    // exits, so an in-process server could never answer that child's request
+    // (request_timeout). The api-server-host fixture runs createApiServer() ONLY
+    // (no supervision loop — the harness drives superviseTick itself), carrying
+    // CRTR_HOME=home so apiSocketPath() resolves to this isolated socket. Killed
+    // in dispose(). No real subprocess daemon is ever autostarted; no client is
+    // mocked.
+    const apiServerProc = spawn(process.execPath, [...(TSX_ESM === null ? [] : ['--import', TSX_ESM]), API_SERVER_HOST], {
+        cwd: CROUTER,
+        // `node new`/`node promote`/… route to THIS server, whose spawnChild/
+        // reviveNode launch the broker ENGINE as a detached child of this very
+        // process (launchBrokerOrCrash, spawn.ts). That broker must load the FAKE
+        // pi vehicle + fake BrokerEngine, so carry the same two seams childEnv
+        // gives every in-process harness launch (cleanBaseEnv scrubs them). HOME
+        // is pinned to the isolated tmpHome so the server + the brokers it spawns
+        // never read the real user home. Without these, spawnChild launches the
+        // REAL pi and the child never boots (awaitBoot: pi_pid=null, bootCount=0).
+        env: {
+            ...cleanBaseEnv(),
+            CRTR_HOME: home,
+            HOME: tmpHome,
+            CRTR_PI_BINARY: FAKE_PI_BINARY,
+            CRTR_BROKER_ENGINE: FAKE_ENGINE,
+        },
+        stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let apiServerStderr = '';
+    apiServerProc.stderr?.on('data', (chunk) => {
+        apiServerStderr += chunk.toString();
+    });
+    if (apiServerProc.pid !== undefined)
+        pidsToKill.add(apiServerProc.pid);
+    try {
+        await waitFor(() => existsSync(apiSocketPath()), {
+            timeoutMs: 30_000,
+            intervalMs: 25,
+            label: 'api server socket bind',
+        });
+    }
+    catch (err) {
+        try {
+            apiServerProc.kill('SIGKILL');
+        }
+        catch {
+            /* already gone */
+        }
+        throw new Error(`api-server-host never bound ${apiSocketPath()}\n--server stderr--\n${apiServerStderr}`, { cause: err });
+    }
+    // Pre-create the isolated session on the DEFAULT server so teardown always
+    // has a kill-sessions target.
+    // ISOLATION ASSUMPTION (see header + MINOR-6): isolation is by SESSION NAME on
+    // the DEFAULT tmux server only. The runtime CLI shells `tmux` with no `-L`, so
+    // a custom-socket server (`tmux -L foo`) would be invisible to it; this harness
+    // therefore assumes the default socket and only ever kill-sessions, never the
+    // server.
+    // HEADLESS: skip ALL eager tmux setup. No node in a headless run opens a
+    // window, so there is no pane to pre-seed an env into and no session to pre-
+    // create.
+    if (!headless) {
+        // -x/-y explicitly: a DETACHED session with no client attached inherits
+        // tmux's 80x24 default, and these tests split it several times before
+        // `surface node focus` splits once more for the viewer. At 24 rows the
+        // viewer lands in a 2-row pane (or the split fails outright), the attach
+        // TUI exits, and tmux closes the pane — leaving the node with no tagged
+        // viewer and the test with nothing to look at. A dev box's session is as
+        // big as the developer's terminal, which is why this was hosted-runner-only.
+        spawnSync('tmux', ['new-session', '-d', '-s', session, '-x', '250', '-y', '60', '-c', CROUTER, 'sleep 100000'], {
+            stdio: 'ignore',
+            timeout: TMUX_TIMEOUT_MS,
+        });
+        // Put CRTR_PI_BINARY in the SESSION environment so EVERY pane/process spawned
+        // in this session inherits it — so any pi the runtime launches (boot, revive)
+        // resolves to the fake-pi vehicle, not a real pi.
+        spawnSync('tmux', ['set-environment', '-t', session, 'CRTR_PI_BINARY', FAKE_PI_BINARY], {
+            stdio: 'ignore',
+            timeout: TMUX_TIMEOUT_MS,
+        });
+        spawnSync('tmux', ['set-environment', '-t', session, 'CRTR_HOME', home], {
+            stdio: 'ignore',
+            timeout: TMUX_TIMEOUT_MS,
+        });
+        // Every `crtr` pane this session forks (the attach viewers above all) must
+        // find the installed runtime generation despite the isolated HOME.
+        spawnSync('tmux', ['set-environment', '-t', session, 'CRTR_RUNTIME_HOME', realRuntimeHome], {
+            stdio: 'ignore',
+            timeout: TMUX_TIMEOUT_MS,
+        });
+    }
+    // -- env for a subprocess CLI invocation -----------------------------------
+    function childEnv(nodeId) {
+        const e = cleanBaseEnv();
+        e['CRTR_HOME'] = home;
+        e['HOME'] = tmpHome;
+        e['CRTR_RUNTIME_HOME'] = realRuntimeHome;
+        e['CRTR_NODE_SESSION'] = session;
+        e['CRTR_PI_BINARY'] = FAKE_PI_BINARY;
+        e['CRTR_BROKER_ENGINE'] = FAKE_ENGINE;
+        if (nodeId !== null)
+            e['CRTR_NODE_ID'] = nodeId;
+        return e;
+    }
+    function nodeDir(id) {
+        return join(home, 'nodes', id);
+    }
+    function nodeDirs() {
+        try {
+            // Node ids are minted by newNodeId() (runtime/nodes.ts) as
+            // `<install>-<time36>-<rand4hex>` (a72554e3 stamped the canvas-install id
+            // onto the historical 2-segment `<time36>-<rand4hex>` shape, so the id is
+            // opaque and its segment count has already changed once — mirror
+            // profiles/manifest.ts's ID_SHAPE rather than hard-coding a segment
+            // count). Scope the scan to "one or more lowercase-alnum segments ending
+            // in an 8-hex random suffix" so a stray non-node sibling under nodes/ is
+            // never miscounted as a new node dir — e.g. a `reports/` dir created for
+            // an empty node id (the rare CI flake on full-tier run 27459304455 that
+            // broke the spawn counters with `expected exactly 1 new node dir, got
+            // [<id>, reports]`), or literal `reports` (no hyphen, no hex suffix).
+            return readdirSync(join(home, 'nodes')).filter((d) => /^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{8}$/.test(d));
+        }
+        catch {
+            return [];
+        }
+    }
+    function readLines(path) {
+        try {
+            return readFileSync(path, 'utf8')
+                .split('\n')
+                .filter((l) => l.trim() !== '');
+        }
+        catch {
+            return [];
+        }
+    }
+    function cli(nodeId, args) {
+        const runtimeArgs = TSX_ESM === null ? [] : ['--import', TSX_ESM];
+        const res = spawnSync(process.execPath, [...runtimeArgs, CLI_ENTRY, ...args], {
+            cwd: CROUTER,
+            env: childEnv(nodeId),
+            encoding: 'utf8',
+            stdio: ['ignore', 'pipe', 'pipe'],
+            timeout: 60_000,
+        });
+        closeDb();
+        let json;
+        try {
+            json = JSON.parse(res.stdout ?? '');
+        }
+        catch {
+            /* not json */
+        }
+        return { code: res.status ?? -1, stdout: res.stdout ?? '', stderr: res.stderr ?? '', json };
+    }
+    // -- control channel -------------------------------------------------------
+    function sendCmd(nodeId, cmd) {
+        const dir = nodeDir(nodeId);
+        const tmp = join(dir, 'fake-pi.cmd.tmp');
+        writeFileSync(tmp, JSON.stringify(cmd));
+        renameSync(tmp, join(dir, 'fake-pi.cmd')); // atomic: host never reads a partial
+    }
+    function eventCount(nodeId, event) {
+        return readLines(join(nodeDir(nodeId), 'fake-pi.events.jsonl')).filter((l) => {
+            try {
+                return JSON.parse(l).event === event;
+            }
+            catch {
+                return false;
+            }
+        }).length;
+    }
+    function bootCount(nodeId) {
+        return readLines(join(nodeDir(nodeId), 'fake-pi.boots.jsonl')).length;
+    }
+    function injected(nodeId) {
+        return readLines(join(nodeDir(nodeId), 'fake-pi.injected.jsonl'))
+            .map((l) => {
+            try {
+                return JSON.parse(l);
+            }
+            catch {
+                return null;
+            }
+        })
+            .filter((x) => x !== null);
+    }
+    // Wait for the host to have BEGUN dispatching agent_end (recorded before its
+    // handlers run, so it survives a handler that tears the process down).
+    async function awaitAgentEnd(nodeId, base, label) {
+        await waitFor(() => eventCount(nodeId, 'agent_end') > base, {
+            timeoutMs: 20_000,
+            label,
+        });
+    }
+    const harness = {
+        home,
+        agentHome: tmpHome,
+        session,
+        spawnRoot(task, o = {}) {
+            const id = o.id ?? `root-${process.pid}-${nextRootSeq++}`;
+            const meta = {
+                node_id: id,
+                name: o.id ?? (task.slice(0, 24) || id),
+                created: new Date().toISOString(),
+                cwd: CROUTER,
+                kind: o.kind ?? 'general',
+                mode: o.mode ?? 'base',
+                lifecycle: o.lifecycle ?? 'resident',
+                status: 'active',
+                parent: null,
+            };
+            createNode(meta);
+            // Every real bootRoot records its broker's pid synchronously at launch
+            // (spawn.ts's bootRoot calls recordPid right after launchBrokerOrCrash,
+            // well before any daemon tick) — a production root's pi_pid is never
+            // null for longer than an instant. This in-process fixture has no real
+            // broker to launch, so mint it a pid that is genuinely alive for the
+            // rest of the test: this harness process's own pid (already threaded
+            // into the id scheme above). The pid is metadata post-cutover (fleet
+            // entries are the liveness authority), but status surfaces and identity
+            // rechecks still read it, so keep the fixture row shaped like a real one.
+            recordPid(id, process.pid);
+            closeDb();
+            return id;
+        },
+        fabricateBrokerNode(o = {}) {
+            const id = o.id ?? `brk-${process.pid}-${nextRootSeq++}`;
+            const meta = {
+                node_id: id,
+                name: id,
+                created: new Date().toISOString(),
+                cwd: CROUTER,
+                host_kind: 'broker',
+                kind: o.kind ?? 'developer',
+                mode: o.mode ?? 'base',
+                lifecycle: o.lifecycle ?? 'terminal',
+                parent: o.parent ?? null,
+                pi_session_id: o.pi_session_id ?? null,
+                pi_session_file: o.pi_session_file ?? null,
+                status: o.status ?? 'active',
+                intent: o.intent ?? null,
+                pi_pid: o.pi_pid ?? null,
+            };
+            createNode(meta);
+            closeDb();
+            return id;
+        },
+        async spawnChild(parentId, task, o = {}) {
+            const before = new Set(nodeDirs());
+            const args = ['node', 'new', task, '--parent', parentId, '--cwd', CROUTER];
+            if (o.kind)
+                args.push('--kind', o.kind);
+            if (o.mode)
+                args.push('--mode', o.mode);
+            if (o.model)
+                args.push('--model', o.model);
+            const res = cli(parentId, args);
+            if (res.code !== 0) {
+                throw new Error(`spawnChild(${parentId}) failed (code ${res.code})\n--stdout--\n${res.stdout}\n--stderr--\n${res.stderr}`);
+            }
+            const added = nodeDirs().filter((d) => !before.has(d));
+            if (added.length !== 1) {
+                throw new Error(`spawnChild: expected exactly 1 new node dir, got [${added.join(', ')}]`);
+            }
+            const childId = added[0];
+            await harness.awaitBoot(childId);
+            return childId;
+        },
+        async spawnHeadlessChild(parentId, task, o = {}) {
+            const before = new Set(nodeDirs());
+            // The synchronous CLI must reach the separate API fixture process, which
+            // creates the row and necessarily starts a fixture-owned broker before
+            // returning. Production has one daemon/fleet; the harness split exists
+            // only because spawnSync blocks this process's event loop. Retire that
+            // fixture launch immediately, then perform the authoritative boot through
+            // THIS process's reviveNode so the harness fleet owns the ChildProcess
+            // handle and its real exit event (§H harness mode a).
+            const args = ['node', 'new', task, '--parent', parentId, '--cwd', CROUTER];
+            if (o.kind)
+                args.push('--kind', o.kind);
+            if (o.mode)
+                args.push('--mode', o.mode);
+            if (o.model)
+                args.push('--model', o.model);
+            const res = cli(parentId, args);
+            if (res.code !== 0) {
+                throw new Error(`spawnHeadlessChild(${parentId}) failed (code ${res.code})\n--stdout--\n${res.stdout}\n--stderr--\n${res.stderr}`);
+            }
+            const added = nodeDirs().filter((d) => !before.has(d));
+            if (added.length !== 1) {
+                throw new Error(`spawnHeadlessChild: expected exactly 1 new node dir, got [${added.join(', ')}]`);
+            }
+            const childId = added[0];
+            closeDb();
+            const fixtureRow = getNode(childId);
+            const fixturePid = fixtureRow?.pi_pid ?? null;
+            const fixtureIdentity = fixtureRow?.pi_pid_identity ?? null;
+            const priorBoots = harness.bootCount(childId);
+            if (fixturePid !== null && fixtureIdentity !== null && recordedPidLiveness(fixturePid, fixtureIdentity) === 'alive') {
+                process.kill(fixturePid, 'SIGTERM');
+                await waitFor(() => recordedPidLiveness(fixturePid, fixtureIdentity) !== 'alive', {
+                    timeoutMs: 10_000,
+                    intervalMs: 25,
+                    label: `API-fixture broker exit for ${childId}`,
+                });
+            }
+            reviveNode(childId, { resume: false });
+            await harness.awaitBoot(childId, { minCount: priorBoots + 1 });
+            return childId;
+        },
+        async spawnHeadlessChildNoBoot(parentId, task, o = {}) {
+            const before = new Set(nodeDirs());
+            const args = ['node', 'new', task, '--parent', parentId, '--cwd', CROUTER];
+            if (o.kind)
+                args.push('--kind', o.kind);
+            if (o.mode)
+                args.push('--mode', o.mode);
+            if (o.model)
+                args.push('--model', o.model);
+            const res = cli(parentId, args);
+            if (res.code !== 0) {
+                throw new Error(`spawnHeadlessChildNoBoot(${parentId}) failed (code ${res.code})\n--stdout--\n${res.stdout}\n--stderr--\n${res.stderr}`);
+            }
+            const added = nodeDirs().filter((d) => !before.has(d));
+            if (added.length !== 1) {
+                throw new Error(`spawnHeadlessChildNoBoot: expected exactly 1 new node dir, got [${added.join(', ')}]`);
+            }
+            return added[0];
+        },
+        cli,
+        async turn(nodeId, text = '') {
+            const base = eventCount(nodeId, 'agent_end');
+            sendCmd(nodeId, { cmd: 'turn', id: `turn-${Date.now()}`, text });
+            await awaitAgentEnd(nodeId, base, `turn agent_end for ${nodeId}`);
+        },
+        async stop(nodeId, reason = 'stop', errorMessage) {
+            const base = eventCount(nodeId, 'agent_end');
+            sendCmd(nodeId, { cmd: 'stop', id: `stop-${Date.now()}`, reason, errorMessage });
+            await awaitAgentEnd(nodeId, base, `stop agent_end for ${nodeId}`);
+        },
+        async finish(nodeId, finalText) {
+            const res = cli(nodeId, ['push', 'final', finalText]);
+            if (res.code !== 0) {
+                throw new Error(`finish(${nodeId}): push final failed (code ${res.code})\n${res.stderr}`);
+            }
+            // Fire agent_end so the now-done node runs the real (b) done branch
+            // (null presence + ctx.shutdown → window closes), exactly as real pi would.
+            const base = eventCount(nodeId, 'agent_end');
+            sendCmd(nodeId, { cmd: 'stop', id: `finish-${Date.now()}` });
+            await awaitAgentEnd(nodeId, base, `finish agent_end for ${nodeId}`);
+            await harness.waitForPaneGone(nodeId);
+        },
+        async tick(now) {
+            closeDb();
+            // On the lane, after every already-enqueued exit job — the same strict
+            // serialization crtrd's enqueueTick gives the real daemon.
+            await enqueueLaneJob(() => superviseTick(now));
+            closeDb();
+        },
+        fleet,
+        async deliverExit(nodeId, status = { code: null, signal: 'SIGKILL' }) {
+            closeDb();
+            fleet.deliverExit(nodeId, status);
+            await drainLane();
+            closeDb();
+        },
+        drainFleet: drainLane,
+        async awaitFleetExit(nodeId, timeoutMs = 15_000) {
+            await waitFor(() => !fleet.has(nodeId), {
+                timeoutMs,
+                intervalMs: 20,
+                label: `fleet forgets ${nodeId} (child exit delivered)`,
+            });
+            await drainLane();
+            closeDb();
+        },
+        async awaitBoot(nodeId, o = {}) {
+            const minCount = o.minCount ?? 1;
+            const bootsPath = join(nodeDir(nodeId), 'fake-pi.boots.jsonl');
+            const errPath = join(nodeDir(nodeId), 'fake-pi.error');
+            let lines;
+            try {
+                lines = await waitFor(() => {
+                    const ls = readLines(bootsPath);
+                    return ls.length >= minCount ? ls : null;
+                }, {
+                    timeoutMs: o.timeoutMs ?? 30_000,
+                    label: `fake-pi boot >= ${minCount} for ${nodeId}` +
+                        (existsSync(errPath) ? ` (error file: ${readFileSync(errPath, 'utf8')})` : ''),
+                });
+            }
+            catch (err) {
+                // Failure-path-only diagnostic: on boot timeout, dump the node row's recorded
+                // liveness fields, recorded-vs-live identity verdict, tail of broker.log, and
+                // late error file so the failure is self-explaining from the CI log alone.
+                closeDb();
+                const n = getNode(nodeId);
+                process.stderr.write(`[awaitBoot] timeout for ${nodeId}: pi_pid=${n?.pi_pid ?? 'null'} ` +
+                    `pi_pid_identity=${n?.pi_pid_identity ?? 'null'} status=${n?.status ?? 'null'} ` +
+                    `intent=${n?.intent ?? 'null'} pi_session_id=${n?.pi_session_id ?? 'null'} ` +
+                    `bootCount=${bootCount(nodeId)} minCount=${minCount}\n`);
+                // Identity-aware liveness: what does the current process state say about recorded identity?
+                if (n?.pi_pid !== null && n?.pi_pid !== undefined) {
+                    const liveness = recordedPidLiveness(n.pi_pid, n.pi_pid_identity);
+                    process.stderr.write(`[awaitBoot] identity-liveness: pid=${n.pi_pid} recorded-identity=${n.pi_pid_identity ?? 'null'} liveness=${liveness}\n`);
+                }
+                // Tail of broker.log if it exists (shows pre-session startup issues)
+                const brokerLogPath = join(nodeDir(nodeId), 'job', 'broker.log');
+                if (existsSync(brokerLogPath)) {
+                    const logContent = readFileSync(brokerLogPath, 'utf8');
+                    const lines = logContent.split('\n');
+                    const tail = lines.slice(-10).join('\n');
+                    process.stderr.write(`[awaitBoot] broker.log tail:\n${tail}\n`);
+                }
+                // Late error file (written during boot, e.g., fake-pi init failure)
+                const lateErrPath = join(nodeDir(nodeId), 'fake-pi.error');
+                if (existsSync(lateErrPath)) {
+                    const lateErr = readFileSync(lateErrPath, 'utf8');
+                    process.stderr.write(`[awaitBoot] fake-pi.error: ${lateErr}\n`);
+                }
+                throw err;
+            }
+            const boot = JSON.parse(lines[lines.length - 1]);
+            if (typeof boot.pid === 'number')
+                pidsToKill.add(boot.pid);
+            return boot;
+        },
+        async awaitWake(nodeId, o = {}) {
+            const sinceCount = o.sinceCount ?? 0;
+            const match = o.match;
+            const fresh = await waitFor(() => {
+                const all = injected(nodeId).slice(sinceCount);
+                if (all.length === 0)
+                    return null;
+                if (match && !all.some((e) => match.test(e.content)))
+                    return null;
+                return all;
+            }, { timeoutMs: o.timeoutMs ?? 15_000, label: `wake delivery for ${nodeId}` });
+            return fresh.map((e) => e.content);
+        },
+        async waitForStatus(nodeId, status, timeoutMs = 20_000) {
+            await waitFor(() => {
+                closeDb();
+                return getNode(nodeId)?.status === status;
+            }, { timeoutMs, label: `status=${status} for ${nodeId}` });
+        },
+        async waitForPaneGone(nodeId, timeoutMs = 20_000) {
+            await waitFor(() => {
+                closeDb();
+                return !isFocused(nodeId);
+            }, { timeoutMs, label: `viewer gone for ${nodeId}` });
+        },
+        waitFor,
+        node(nodeId) {
+            closeDb();
+            return getNode(nodeId);
+        },
+        status(nodeId) {
+            closeDb();
+            return getNode(nodeId)?.status ?? null;
+        },
+        paneAlive(nodeId) {
+            closeDb();
+            return isFocused(nodeId);
+        },
+        inbox(nodeId) {
+            closeDb();
+            return readInboxSince(nodeId);
+        },
+        injected,
+        fakeCmd(nodeId, cmd) {
+            sendCmd(nodeId, cmd);
+        },
+        dialogResults(nodeId) {
+            return readLines(join(nodeDir(nodeId), 'fake-pi.dialog.jsonl'))
+                .map((l) => {
+                try {
+                    return JSON.parse(l);
+                }
+                catch {
+                    return null;
+                }
+            })
+                .filter((x) => x !== null);
+        },
+        brokerSock(nodeId) {
+            return join(nodeDir(nodeId), 'view.sock');
+        },
+        bootCount,
+        subscribers(nodeId) {
+            closeDb();
+            return subscribersOf(nodeId).map((s) => ({ node_id: s.node_id, active: s.active }));
+        },
+        subscriptions(nodeId) {
+            closeDb();
+            return subscriptionsOf(nodeId).map((s) => ({ node_id: s.node_id, active: s.active }));
+        },
+        async dispose() {
+            laneClosed = true;
+            spawnSync('tmux', ['kill-session', '-t', session], {
+                stdio: 'ignore',
+                timeout: TMUX_TIMEOUT_MS,
+            });
+            // Best-effort: a superviseTick-triggered revive of a fabricated node fires
+            // a DETACHED throwaway broker (it loads the real SDK; the rmSync of home
+            // below makes a mid-boot one self-terminate when its files vanish). Kill
+            // any that recorded a pid first — scan every node's row pid + recorded
+            // boot pids and add them to the kill set.
+            closeDb();
+            for (const d of nodeDirs()) {
+                try {
+                    const n = getNode(d);
+                    if (n?.pi_pid != null)
+                        pidsToKill.add(n.pi_pid);
+                }
+                catch {
+                    /* row gone — fine */
+                }
+                for (const line of readLines(join(nodeDir(d), 'fake-pi.boots.jsonl'))) {
+                    try {
+                        const p = JSON.parse(line).pid;
+                        if (typeof p === 'number')
+                            pidsToKill.add(p);
+                    }
+                    catch {
+                        /* skip */
+                    }
+                }
+            }
+            // Every pid here was recorded EARLIER (a row's pi_pid, a fake-pi boot
+            // line, the API-server child) and may since have exited and had its pid
+            // REUSED by an unrelated process — including a real, just-spawned broker
+            // on the developer's own canvas, which a blind SIGKILL then murders (the
+            // 2026-07-27 "brokers die at startup" RCA). Re-read each pid's command
+            // at the signal boundary (same fresh-read guard crtrd's reapOrphanBrokers
+            // uses) and signal ONLY what still looks like one of THIS fixture's
+            // processes: a broker/CLI declaring this isolated canvas home, or one of
+            // the fixture entrypoints. Anything else is skipped, not killed.
+            const fixtureFingerprints = [home, tmpHome, API_SERVER_HOST, FAKE_ENGINE, FAKE_PI_HOST];
+            for (const p of pidsToKill) {
+                const command = capturePidCommand(p);
+                if (command === null)
+                    continue; // gone already — nothing to signal
+                if (!fixtureFingerprints.some((mark) => command.includes(mark)))
+                    continue; // pid reused by a stranger
+                try {
+                    process.kill(p, 'SIGKILL');
+                }
+                catch {
+                    /* already gone */
+                }
+            }
+            closeDb();
+            // Stop the standalone API-server process (its socket lives under home/,
+            // removed below). SIGTERM lets it close() its listener and unlink the
+            // socket; the pid is also in pidsToKill as a backstop.
+            try {
+                apiServerProc.kill('SIGTERM');
+            }
+            catch {
+                /* already gone */
+            }
+            // Leave the process-global event source + fleet unbound so the next
+            // harness instance in this same test process can bind cleanly (see setup
+            // above). dispose() the fleet first: a pending backoff-respawn timer
+            // would otherwise hold the test process's event loop open.
+            fleet.dispose();
+            resetFleetForTests();
+            resetEventSourceForTests();
+            rmSync(home, { recursive: true, force: true });
+            rmSync(tmpHome, { recursive: true, force: true });
+            if (origHome === undefined)
+                delete process.env['CRTR_HOME'];
+            else
+                process.env['CRTR_HOME'] = origHome;
+            if (origAgentHome === undefined)
+                delete process.env['HOME'];
+            else
+                process.env['HOME'] = origAgentHome;
+            if (origPiBinary === undefined)
+                delete process.env['CRTR_PI_BINARY'];
+            else
+                process.env['CRTR_PI_BINARY'] = origPiBinary;
+            if (origNodeSession === undefined)
+                delete process.env['CRTR_NODE_SESSION'];
+            else
+                process.env['CRTR_NODE_SESSION'] = origNodeSession;
+            if (origNodeOptions === undefined)
+                delete process.env['NODE_OPTIONS'];
+            else
+                process.env['NODE_OPTIONS'] = origNodeOptions;
+            if (origBrokerEngine === undefined)
+                delete process.env['CRTR_BROKER_ENGINE'];
+            else
+                process.env['CRTR_BROKER_ENGINE'] = origBrokerEngine;
+        },
+    };
+    return harness;
+}