@north-light/crouter 0.3.318 → 0.3.320

package/dist/core/__tests__/seam/broker-crash-teardown.test.js

@@ -1,131 +0,0 @@
-// Run with: node --import tsx/esm --test src/core/__tests__/seam/broker-crash-teardown.test.ts
-//
-// Daemon-epoch broker ownership, real-process crash and teardown. The broker's
-// supervision tick is the death authority after the broker exit is observed: a SIGTERM of a streaming broker
-// aborts cleanly and resumes the saved session, and a
-// SIGKILL mid-turn — with the durable open-turn record — resumes
-// that same saved session in place and re-drives the stopped turn with the
-// same continuation (both policy row 5). Each replacement starts only after the
-// prior process has exited, preserving one writer per session. A clean `node
-// lifecycle close` then tears the replacement down (shutdown frame → exit,
-// socket unlinked, status canceled, daemon leaves it).
-//
-// spawnHeadlessChild calls the production birth path in the harness process, so
-// every broker below is harness-fleet-owned from its first and only boot and
-// every SIGKILL's REAL ChildProcess exit is observed by the fleet; the harness
-// then drives the production supervision tick — no synthetic delivery or grace.
-// Each crash has its own real broker setup, so either top-level test remains a
-// complete journey under name filtering and pays only its own first respawn
-// backoff.
-import { test, before, after } from 'node:test';
-import assert from 'node:assert/strict';
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-import { createHarness } from '../helpers/harness.js';
-import { isPidAlive } from '../../canvas/pid.js';
-import { readLiveTurn } from '../../canvas/recovery.js';
-import { RUNTIME_RESTART_CONTINUATION } from '../../runtime/kickoff.js';
-let h;
-let root;
-let priorHoldTurn;
-before(async () => {
-    priorHoldTurn = process.env['PI_FAKE_ENGINE_HOLD_TURN'];
-    process.env['PI_FAKE_ENGINE_HOLD_TURN'] = '1';
-    h = await createHarness({ headless: true, sessionPrefix: 'crtr-brkcrash' });
-    root = h.spawnRoot('broker-crash suite root');
-});
-after(async () => {
-    if (h !== undefined)
-        await h.dispose();
-    if (priorHoldTurn === undefined)
-        delete process.env['PI_FAKE_ENGINE_HOLD_TURN'];
-    else
-        process.env['PI_FAKE_ENGINE_HOLD_TURN'] = priorHoldTurn;
-});
-// Items 4 + 5 + 6 — two independent real-process journeys:
-//   graceful busy teardown → strict resume;
-//   dirty interrupted-turn crash → recovery cycle → close.
-test('SIGTERM of a busy broker aborts cleanly and resumes the saved session', { timeout: 30_000 }, async () => {
-    const id = await h.spawnHeadlessChild(root, 'headless worker — graceful crash recovery');
-    const boot1 = await h.awaitBoot(id);
-    const oldPid = h.node(id).pi_pid;
-    assert.equal(oldPid, boot1.pid, 'canvas pid identifies the broker that booted the session');
-    assert.equal(isPidAlive(oldPid), true, 'busy broker alive before graceful teardown');
-    await h.waitFor(() => existsSync(h.brokerSock(id)), { label: 'broker socket bound' });
-    await h.waitFor(() => readLiveTurn(id)?.busy === true, { label: 'fixture turn is active before SIGTERM' });
-    assert.equal(h.node(id).intent ?? null, null, 'fresh broker has a null intent (not refresh/idle-release)');
-    const boots = h.bootCount(id);
-    // SIGTERM exercises broker.ts's bounded session.abort() path. Its REAL child
-    // exit closes the broker's durable turn as teardown-aborted; the supervision tick
-    // classifies it and persists the first-failure backoff.
-    process.kill(oldPid, 'SIGTERM');
-    await h.waitFor(() => !isPidAlive(oldPid), { label: 'gracefully torn-down broker pid is dead' });
-    await h.awaitFleetExit(id);
-    await h.tick();
-    const gracefulBackoff = h.node(id);
-    assert.equal(gracefulBackoff.respawn_failures, 1, 'the first failed launch is durably counted');
-    assert.ok(gracefulBackoff.respawn_not_before != null, 'the first retry is durably deferred');
-    assert.equal(gracefulBackoff.pi_pid, null, 'classification clears the dead broker pid');
-    await h.tick(Date.parse(gracefulBackoff.respawn_not_before) + 1);
-    // This is the file's SECOND real broker boot, after the durable 5s backoff
-    // matures on a supervision tick.
-    const boot2 = await h.awaitBoot(id, { minCount: boots + 1, timeoutMs: 25_000 });
-    assert.equal(boot2.resuming, true, 'clean-abort tick-driven recovery RESUMES the saved .jsonl');
-    assert.ok(!boot2.rawArgv.includes('--crtr-new-cycle'), 'clean abort never starts a recovery cycle');
-    const revivedPid = h.node(id).pi_pid;
-    assert.ok(revivedPid != null && isPidAlive(revivedPid), 'the revived broker pid is alive');
-    assert.notEqual(revivedPid, oldPid, 'the revived pid is distinct from the crashed one');
-    assert.equal(isPidAlive(oldPid), false, 'one-writer: the crashed pid never resurrected');
-});
-test('SIGKILL of a busy broker resumes the interrupted turn in place; close tears down the replacement', { timeout: 30_000 }, async () => {
-    const id = await h.spawnHeadlessChild(root, 'headless worker — dirty crash+teardown');
-    const boot1 = await h.awaitBoot(id);
-    const boots = h.bootCount(id);
-    const busyPid = h.node(id).pi_pid;
-    assert.equal(busyPid, boot1.pid, 'canvas pid identifies the broker that owns the dirty-crash session');
-    assert.equal(isPidAlive(busyPid), true, 'busy broker is alive before SIGKILL');
-    // This broker has an open durable turn — SIGKILL cannot run the abort handler
-    // to close it. The saved transcript is still
-    // resumable (pi persists whole entries), so policy resumes it strictly and
-    // the respawn continuation re-drives the stopped turn, after this journey's
-    // first 5s backoff. A dirty death is not a reason to throw the branch away.
-    await h.waitFor(() => readLiveTurn(id)?.busy === true, { label: 'fixture turn is active before SIGKILL' });
-    process.kill(busyPid, 'SIGKILL');
-    await h.waitFor(() => !isPidAlive(busyPid), { label: 'mid-turn crashed broker pid is dead' });
-    await h.awaitFleetExit(id);
-    await h.tick();
-    const dirtyBackoff = h.node(id);
-    assert.equal(dirtyBackoff.respawn_failures, 1, 'the first failed launch is durably counted');
-    assert.ok(dirtyBackoff.respawn_not_before != null, 'the first retry is durably deferred');
-    assert.equal(dirtyBackoff.pi_pid, null, 'classification clears the dead broker pid');
-    await h.tick(Date.parse(dirtyBackoff.respawn_not_before) + 1);
-    const boot2 = await h.awaitBoot(id, { minCount: boots + 1, timeoutMs: 25_000 });
-    // The fake SessionManager reports `resuming=true` for ANY existing session
-    // path, including a cycle; production distinguishes cycling with the argv
-    // token and non-null kickoff. Assert those load-bearing inputs directly.
-    const recoveryArgv = boot2.rawArgv;
-    assert.ok(!recoveryArgv.includes('--crtr-new-cycle'), 'a surviving open turn resumes the saved session — it never opens a fresh cycle');
-    assert.equal(boot2.prompt, RUNTIME_RESTART_CONTINUATION, 'the strict resume injects the runtime-restart continuation that re-drives the interrupted turn');
-    const recoveryPid = h.node(id).pi_pid;
-    assert.ok(recoveryPid != null && isPidAlive(recoveryPid), 'the recovered broker pid is alive');
-    assert.notEqual(recoveryPid, busyPid, 'the recovery uses a distinct broker pid');
-    assert.equal(isPidAlive(busyPid), false, 'one-writer: the interrupted pid never resurrected');
-    // --- Item 5: clean teardown of the (revived, live) broker. node lifecycle close marks
-    //     canceled BEFORE teardown (crash-safe order), then hostFor(broker).teardown
-    //     sends the `shutdown` frame → the broker disposes + exits, socket unlinked. ---
-    const sessionFile = h.node(id).pi_session_file;
-    assert.ok(sessionFile !== null && sessionFile !== undefined, 'revived broker has a session file');
-    mkdirSync(join(sessionFile, '..'), { recursive: true });
-    writeFileSync(sessionFile, JSON.stringify({ type: 'message', message: { role: 'assistant', content: [{ type: 'text', text: 'hi' }] } }) + '\n');
-    const sock = h.brokerSock(id);
-    await h.waitFor(() => existsSync(sock), { label: 'revived broker created its view.sock' });
-    const res = h.cli(root, ['node', 'lifecycle', 'close', '--node', id]);
-    assert.equal(res.code, 0, `node lifecycle close should exit 0\n--stderr--\n${res.stderr}`);
-    await h.waitFor(() => !isPidAlive(recoveryPid), { label: 'broker process exited on shutdown frame' });
-    await h.waitFor(() => !existsSync(sock), { label: 'broker unlinked its socket on exit' });
-    assert.equal(h.status(id), 'canceled', 'the closed node is canceled');
-    // listNodes only surfaces active|idle, so a canceled broker is never supervised.
-    await h.tick();
-    assert.equal(h.status(id), 'canceled', 'still canceled after a daemon tick');
-    assert.equal(isPidAlive(recoveryPid), false, 'the broker stays dead — never revived');
-});

package/dist/core/__tests__/seam/broker-pane-resolution.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/core/__tests__/seam/broker-pane-resolution.test.js

@@ -1,204 +0,0 @@
-// Run with: node --import tsx/esm --test src/core/__tests__/seam/broker-pane-resolution.test.ts
-//
-// REGRESSION — broker-hosted nodes were second-class in the tmux command
-// surface (audit: findings-headless-default.md, Gaps 1 + 3). Both regressions
-// are observable only with a REAL tmux pane, so this is a compiled-seam test.
-//
-// (1) Gap 1 — `nodeInPane()` (src/commands/surface/node/placement.ts) resolved a pane via
-//     window→node ONLY. A broker engine runs DETACHED with window=null, and its
-//     on-screen presence is a `crtr surface attach` VIEWER pane, so the window lookup
-//     never found it: viewer navigation and node lifecycle operations all
-//     broke when the current pane was a broker viewer. The fix has attach
-//     self-tag its pane `@crtr_node=<id>` and nodeInPane resolve that tag first.
-//     LOCKED: an untagged pane over a broker resolves undefined (the bug); the
-//     SAME pane, once tagged, resolves the broker (the fix).
-//
-// (3) Gap 3 (tests-lens MAJOR-1) — the §A.4 viewer-DEDUP, the headline new
-//     behavior of the broker-universal cut, had ZERO end-to-end coverage: a
-//     SECOND `surface node focus` on a node must REUSE/move its one viewer, never
-//     STACK a second. Only the DB UNIQUE(node_id)/GC layer was tested; `focus()` itself
-//     (placement.ts) was unexercised. LOCKED here over a REAL broker viewer: two
-//     `surface node focus` calls leave EXACTLY ONE viewer pane + ONE focus row for the
-//     node, the second reused IN PLACE (in_place=true). A focus() that always
-//     opened a fresh pane would yield two tagged panes (or crash on the UNIQUE
-//     constraint) — either way this goes RED.
-//
-// (4) `node lifecycle demote --detach` (the Alt+C → D menu path) CLOSES the
-//     node's viewer pane and its `@crtr_node` tag while the broker ENGINE keeps
-//     running untouched, reconnectable by a later focus. Only the pane-closing
-//     half needs tmux — "detach leaves the broker running" is already proved
-//     headlessly by broker-attach-stream G2 — so it rides this file's pane, tag
-//     helpers and budget instead of its own harness.
-import { test, before, after } from 'node:test';
-import assert from 'node:assert/strict';
-import { spawnSync } from 'node:child_process';
-import { mkdirSync, writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-import { createHarness, hasTmux, TMUX_TIMEOUT_MS } from '../helpers/harness.js';
-import { paneTag, taggedViewerPanes, waitForOneTaggedViewer } from '../helpers/viewer-panes.js';
-import { closeDb } from '../../canvas/db.js';
-import { isPidAlive } from '../../canvas/pid.js';
-import { nodeInPane } from '../../../commands/surface/node/placement.js';
-let h;
-let root;
-before(async () => {
-    h = await createHarness({ sessionPrefix: 'crtr-brkpane' });
-    root = h.spawnRoot('host root');
-});
-after(async () => {
-    if (h !== undefined)
-        await h.dispose();
-});
-/** Split a throwaway pane in the harness session; return its %id. */
-function makePane() {
-    const r = spawnSync('tmux', ['split-window', '-d', '-t', h.session, '-P', '-F', '#{pane_id}', 'sleep 100000'], { encoding: 'utf8', timeout: TMUX_TIMEOUT_MS });
-    const pane = (r.stdout ?? '').trim();
-    assert.ok(pane.startsWith('%'), `expected a %pane_id, got "${pane}" (stderr: ${r.stderr})`);
-    return pane;
-}
-function tagPane(pane, value) {
-    spawnSync('tmux', ['set-option', '-p', '-t', pane, '@crtr_node', value], {
-        stdio: 'ignore',
-        timeout: TMUX_TIMEOUT_MS,
-    });
-}
-/** Parse `crtr`'s human-list stdout (`- key: value` lines) into a map. */
-function parseList(stdout) {
-    const out = {};
-    for (const line of stdout.split('\n')) {
-        const m = /^- ([^:]+):\s*(.*)$/.exec(line.trim());
-        if (m)
-            out[m[1].trim()] = m[2].trim();
-    }
-    return out;
-}
-// `taggedViewerPanes` / `waitForOneTaggedViewer` live in helpers/viewer-panes.ts:
-// the attach client sets `@crtr_node` asynchronously on CONNECT (not
-// synchronously by openViewerWindow), and the second focus's reuse branch keys
-// on that tag, so the test must let attach connect before reading the registry.
-test('Gap 1 — nodeInPane resolves a broker node from its tagged viewer pane (undefined when untagged)', async (t) => {
-    if (!hasTmux())
-        return t.skip('tmux unavailable');
-    const broker = await h.spawnHeadlessChild(root, 'broker worker');
-    closeDb();
-    assert.equal(h.node(broker).host_kind, 'broker', 'spawned a broker-hosted node');
-    assert.equal(h.node(broker).window ?? null, null, 'broker engine has window=null (paneless)');
-    const viewer = makePane();
-    // BUG REPRODUCTION: an untagged pane over a broker is invisible — no node owns
-    // this pane's window, and a broker has no window to find, so resolution fails.
-    closeDb();
-    assert.equal(nodeInPane(viewer), undefined, 'untagged viewer pane → broker is invisible (the bug)');
-    // FIX: the viewer self-tag (`@crtr_node`) is the broker's pane handle.
-    tagPane(viewer, broker);
-    closeDb();
-    assert.equal(nodeInPane(viewer), broker, 'tagged viewer pane → resolves the broker (the fix)');
-    // A stale tag pointing at a DONE node is ignored (must be live: active/idle).
-    spawnSync('tmux', ['kill-pane', '-t', viewer], { stdio: 'ignore', timeout: TMUX_TIMEOUT_MS });
-});
-// 30s, not the 10s default: this test's own `waitForOneTaggedViewer` budget is
-// 15s (the attach client self-tags its pane asynchronously on connect), so a
-// 10s cap kills the test BEFORE its internal wait can even expire — it passed
-// locally only because a dev box tags in ~1s. A hosted runner is slower, and
-// the run then reported `test timed out after 10000ms` with no real signal.
-test('Gap 3 — a second `surface node focus` REUSES the one viewer (in place), never stacks a second pane/row', { timeout: 30_000 }, async (t) => {
-    if (!hasTmux())
-        return t.skip('tmux unavailable');
-    const broker = await h.spawnHeadlessChild(root, 'broker to focus twice');
-    closeDb();
-    assert.equal(h.node(broker).host_kind, 'broker', 'precondition: broker-hosted node');
-    // A caller pane in the harness session to focus BESIDE (the user's pane the
-    // viewer splits next to). The viewer lands in this same session, so the SECOND
-    // focus's reuse takes the in-session NAVIGATE path (in_place), not the
-    // cross-session MOVE path.
-    const caller = makePane();
-    // FOCUS #1 — a managed child boots with NO viewer (spawn no longer auto-opens
-    // one), so this OPENS the broker's single viewer beside the caller.
-    const f1 = h.cli(broker, ['surface', 'node', 'focus', broker, '--pane', caller]);
-    assert.equal(f1.code, 0, `first focus exit 0\n${f1.stderr}`);
-    assert.equal(parseList(f1.stdout).focused, 'true', 'first focus brought the viewer on screen');
-    // The `@crtr_node` pane tag is the viewer registry (P1: `surface node focus` no longer
-    // writes the focuses DB table — the CLI can't reach canvas.db). Let the attach
-    // client connect + self-tag its one viewer pane before the second focus — the
-    // reuse branch keys on `@crtr_node`; without the tag focus() would (correctly)
-    // treat it as stale and open fresh. This wait is a test-timing concern, not a
-    // product gap (in real use the second focus is a much-later user action).
-    const pane1 = await waitForOneTaggedViewer(h.session, broker);
-    assert.deepEqual(taggedViewerPanes(h.session, broker), [pane1], 'exactly ONE tagged viewer pane on screen after focus #1');
-    // FOCUS #2 — the headline dedup: the node already has a live, tagged viewer in
-    // the caller's session, so focus NAVIGATES to it (no new pane), never stacks.
-    const f2 = h.cli(broker, ['surface', 'node', 'focus', broker, '--pane', caller]);
-    assert.equal(f2.code, 0, `second focus exit 0\n${f2.stderr}`);
-    const f2out = parseList(f2.stdout);
-    assert.equal(f2out.focused, 'true', 'second focus also on screen');
-    assert.equal(f2out.in_place, 'true', 'second focus REUSED the viewer in place (navigate, no new pane)');
-    // The invariant: STILL exactly one viewer pane, the SAME pane (never stacked).
-    assert.deepEqual(taggedViewerPanes(h.session, broker), [pane1], 'STILL exactly ONE tagged viewer pane after focus #2 — the SAME pane (reused, not stacked)');
-});
-// 30s for the same reason as Gap 3 above: a 15s internal viewer-tag wait
-// cannot live under a 10s per-test cap.
-test('surface node focus reuses a viewer in another session instead of reopening it beside the caller', { timeout: 30_000 }, async (t) => {
-    if (!hasTmux())
-        return t.skip('tmux unavailable');
-    const broker = await h.spawnHeadlessChild(root, 'cross-session viewer reuse');
-    const firstCaller = makePane();
-    const first = h.cli(broker, ['surface', 'node', 'focus', broker, '--pane', firstCaller]);
-    assert.equal(first.code, 0, `first focus exit 0\n${first.stderr}`);
-    const originalPane = await waitForOneTaggedViewer(h.session, broker);
-    const otherSession = `${h.session}-other`;
-    const created = spawnSync('tmux', 
-    // -x/-y for the same reason the harness session sets them: a detached
-    // session defaults to 80x24, and the viewer this test MOVES here needs a
-    // pane it can actually live in.
-    ['new-session', '-d', '-s', otherSession, '-x', '250', '-y', '60', '-P', '-F', '#{pane_id}', 'sleep 100000'], { encoding: 'utf8', timeout: TMUX_TIMEOUT_MS });
-    assert.equal(created.status, 0, `created caller session: ${created.stderr}`);
-    const otherCaller = (created.stdout ?? '').trim();
-    try {
-        const second = h.cli(broker, ['surface', 'node', 'focus', broker, '--pane', otherCaller]);
-        assert.equal(second.code, 0, `cross-session focus exit 0\n${second.stderr}`);
-        assert.equal(parseList(second.stdout).in_place, 'true', 'cross-session focus navigated to the existing viewer');
-        assert.deepEqual(taggedViewerPanes(h.session, broker), [originalPane], 'the existing viewer remained open and attached');
-        assert.deepEqual(taggedViewerPanes(otherSession, broker), [], 'no duplicate viewer opened beside the caller');
-    }
-    finally {
-        spawnSync('tmux', ['kill-session', '-t', otherSession], { stdio: 'ignore', timeout: TMUX_TIMEOUT_MS });
-    }
-});
-// 30s for the same reason as the two tests above: the viewer self-tags
-// asynchronously on connect, and this test waits for that tag before detaching.
-test('node lifecycle demote --detach closes the viewer pane + tag while the broker engine keeps running', { timeout: 30_000 }, async (t) => {
-    if (!hasTmux())
-        return t.skip('tmux unavailable');
-    const broker = await h.spawnHeadlessChild(root, 'detach while the broker keeps running');
-    closeDb();
-    // Make the node NON-EMPTY (it produced an assistant message) so `--detach`
-    // exercises the "close the viewer, broker survives" contract and NOT the
-    // reap-empty path, which tears an output-less shell down.
-    const sessionFile = h.node(broker).pi_session_file;
-    assert.ok(sessionFile != null, 'the node has a session file');
-    mkdirSync(join(sessionFile, '..'), { recursive: true });
-    writeFileSync(sessionFile, JSON.stringify({ type: 'message', message: { role: 'assistant', content: [{ type: 'text', text: 'hi' }] } }) + '\n');
-    // Put the node genuinely on screen: one viewer pane beside a caller pane.
-    const caller = makePane();
-    const focused = h.cli(broker, ['surface', 'node', 'focus', broker, '--pane', caller]);
-    assert.equal(focused.code, 0, `surface node focus exit 0\n${focused.stderr}`);
-    const viewerPane = await waitForOneTaggedViewer(h.session, broker);
-    // The engine the detach must NOT disturb is the one serving the viewer that is
-    // now on screen, so read its pid here — a headless child that idle-released
-    // after its kickoff turn is revived BY the focus, under a new pid.
-    closeDb();
-    const brokerPid = h.node(broker).pi_pid;
-    assert.ok(isPidAlive(brokerPid), 'broker engine alive before the detach');
-    // Drive the REAL verb on the focused node (the Alt+C → D menu path).
-    const res = h.cli(broker, ['node', 'lifecycle', 'demote', '--node', broker, '--pane', viewerPane, '--detach']);
-    assert.equal(res.code, 0, `node lifecycle demote --detach exit 0\n${res.stderr}`);
-    assert.match(res.stdout, /viewer closed \(broker still running off-screen\)/, `the verb reports the viewer closed with the broker alive\n${res.stdout}`);
-    closeDb();
-    const after = h.node(broker);
-    assert.equal(after.status, 'active', 'detach does NOT end the node');
-    assert.equal(after.lifecycle, 'terminal', 'lifecycle flips terminal IN PLACE');
-    assert.equal(after.pi_pid, brokerPid, 'the broker pid is UNCHANGED (engine not killed or respawned)');
-    assert.equal(isPidAlive(brokerPid), true, 'the broker engine keeps running off-screen');
-    // The viewer pane is CLOSED and untagged — no orphaned/phantom viewport.
-    await h.waitFor(() => (taggedViewerPanes(h.session, broker).length === 0 ? true : null), { label: 'the detached viewer pane is gone from the tag registry' });
-    assert.equal(paneTag(viewerPane), '', 'NO phantom @crtr_node tag resolves on the closed viewer %id');
-});

package/dist/core/__tests__/seam/broker-provider-retry.test.d.ts

@@ -1 +0,0 @@
-export {};