@crouton-kit/crouter 0.3.19 → 0.3.20

package/dist/core/__tests__/broker-crash-teardown.test.js

@@ -0,0 +1,116 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/broker-crash-teardown.test.ts
+//
+// Broker crash / teardown / boot-failure — acceptance items 4–6 of the headless-
+// broker migration plus the M-1/M-2 boot-failure regression. Split out of
+// broker-lifecycle.test.ts (see its header for the full file map); the tests are
+// the original acceptance gate, unchanged, on their own isolated harness:
+//   4 survive a broker crash via grace-revive RESUME on the saved .jsonl         → "CRASH"
+//   5 tear down cleanly — close → shutdown frame → exit, socket unlinked         → "clean teardown"
+//   6 ONE-WRITER (R2) — never two engine pids alive across crash→grace-revive    → "CRASH"
+//   M-1/M-2 a broker that dies before session_start is reaped, not stranded      → "boot failure"
+import { test, before, after } from 'node:test';
+import assert from 'node:assert/strict';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { createHarness, hasTmux } from './helpers/harness.js';
+import { isPidAlive } from '../canvas/pid.js';
+import { FAIL_BEFORE_SESSION_START } from './fixtures/fake-engine.js';
+// crtrd.ts module const (not exported). The fresh-pi-boot grace window the daemon
+// waits before grace-reviving a pi observed dead. Reference: crtrd.ts
+// `REVIVE_GRACE_MS = 20_000`.
+const REVIVE_GRACE_MS = 20_000;
+let h;
+let root;
+before(async () => {
+    if (!hasTmux())
+        return;
+    h = await createHarness({ sessionPrefix: 'crtr-brkcrash' });
+    root = h.spawnRoot('broker-crash suite root');
+});
+after(async () => {
+    if (h !== undefined)
+        await h.dispose();
+});
+// ===========================================================================
+// Item 4 + 6 — broker CRASH → grace-revive RESUME on the saved .jsonl; ONE-
+// WRITER: the crashed pid is dead BEFORE the revive launches and never
+// resurrects, and the revived pid is distinct (never two engine pids at once).
+// ===========================================================================
+test('CRASH → grace-revive RESUME; one-writer (old pid dead before new, never resurrects)', { skip: !hasTmux() }, async () => {
+    const id = await h.spawnHeadlessChild(root, 'headless worker — crash');
+    const oldPid = h.node(id).pi_pid;
+    assert.equal(isPidAlive(oldPid), true, 'broker alive before the crash');
+    assert.equal(h.node(id).intent ?? null, null, 'fresh broker has a null intent (not refresh/idle-release)');
+    const boots = h.bootCount(id);
+    // Kill the broker out from under the daemon (a crash). Use a fixed clock so the
+    // grace window is exercised deterministically.
+    process.kill(oldPid, 'SIGKILL');
+    await h.waitFor(() => !isPidAlive(oldPid), { label: 'crashed broker pid is dead' });
+    // ONE-WRITER: the old engine pid is dead BEFORE any revive can launch.
+    assert.equal(isPidAlive(oldPid), false, 'crashed pid dead before the daemon revives');
+    const NOW = 5_000_000;
+    await h.tick(NOW); // pid dead, intent null → handleBrokerLiveness → handleLiveWindow marks pending
+    assert.equal(h.bootCount(id), boots, 'inside the grace window → NOT yet revived');
+    await h.tick(NOW + REVIVE_GRACE_MS + 1); // grace elapsed → reviveNode(resume:true)
+    const boot2 = await h.awaitBoot(id, { minCount: boots + 1 });
+    assert.equal(boot2.resuming, true, 'grace-revive RESUMES the saved .jsonl');
+    const newPid = h.node(id).pi_pid;
+    assert.ok(newPid != null && isPidAlive(newPid), 'the revived broker pid is alive');
+    assert.notEqual(newPid, oldPid, 'the revived pid is distinct from the crashed one');
+    assert.equal(isPidAlive(oldPid), false, 'one-writer: the crashed pid never resurrected');
+});
+// ===========================================================================
+// Item 5 — clean teardown: close → hostFor(meta).teardown → `shutdown` frame →
+// broker disposes + exits, socket unlinked, status canceled, daemon leaves it.
+// ===========================================================================
+test('clean teardown — node close → broker shutdown frame → exit, socket unlinked, not revived', { skip: !hasTmux() }, async () => {
+    const id = await h.spawnHeadlessChild(root, 'headless worker — teardown');
+    const pid = h.node(id).pi_pid;
+    const sock = h.brokerSock(id);
+    await h.waitFor(() => existsSync(sock), { label: 'broker created its view.sock' });
+    // Close via the REAL CLI (resolves the node from --node; the subprocess carries
+    // no TMUX_PANE). closeNode marks canceled BEFORE teardown (crash-safe order),
+    // then hostFor(broker).teardown sends the `shutdown` frame.
+    const res = h.cli(root, ['node', 'close', '--node', id]);
+    assert.equal(res.code, 0, `node close should exit 0\n--stderr--\n${res.stderr}`);
+    await h.waitFor(() => !isPidAlive(pid), { label: 'broker process exited on shutdown frame' });
+    await h.waitFor(() => !existsSync(sock), { label: 'broker unlinked its socket on exit' });
+    assert.equal(h.status(id), 'canceled', 'the closed node is canceled');
+    // listNodes only surfaces active|idle, so a canceled broker is never supervised.
+    await h.tick();
+    assert.equal(h.status(id), 'canceled', 'still canceled after a daemon tick');
+    assert.equal(isPidAlive(pid), false, 'the broker stays dead — never revived');
+});
+// ===========================================================================
+// M-1 regression — a broker that DIES before session_start must surface a boot
+// failure, not strand the node forever. Before the fix handleBrokerLiveness read
+// pid==null UNCONDITIONALLY as "still booting", so a broker that threw before
+// recording a pid/session left the node 'active' with no engine FOREVER and the
+// parent waited on a dead child. The daemon must instead crash it past the boot
+// grace + push a boot-failure up the spine. (Also proves M-2: the broker's fatal
+// diagnostic reaches job/broker.log instead of /dev/null.)
+// ===========================================================================
+test('boot failure — a broker that dies before session_start is reaped, not stranded (review M-1/M-2)', { skip: !hasTmux() }, async () => {
+    // A headless child whose broker throws before session_start (no pid, no session
+    // EVER recorded). spawnHeadlessChildNoBoot does not awaitBoot (there is none).
+    const id = await h.spawnHeadlessChildNoBoot(root, `headless worker — boot fail ${FAIL_BEFORE_SESSION_START}`);
+    // M-2: the broker logs its fatal stack to job/broker.log and exits(1). Without
+    // the stdio redirect this diagnostic would be lost to /dev/null.
+    const brokerLog = join(h.home, 'nodes', id, 'job', 'broker.log');
+    await h.waitFor(() => existsSync(brokerLog) && /simulated pre-session_start boot failure/.test(readFileSync(brokerLog, 'utf8')), { label: 'broker logged its pre-session_start failure to job/broker.log' });
+    // The node is stuck 'active' with a null pid and a null session — the strand.
+    const n0 = h.node(id);
+    assert.equal(n0.pi_pid ?? null, null, 'no broker pid was ever recorded');
+    assert.equal(n0.pi_session_id ?? null, null, 'no session was ever recorded');
+    // Inside the boot grace the daemon LEAVES it (could be the sub-second boot gap).
+    const NOW = 9_000_000;
+    await h.tick(NOW);
+    assert.equal(h.status(id), 'active', 'inside the boot grace → daemon leaves it (boot gap)');
+    // Past the boot grace with STILL no pid and no session → crash + boot failure.
+    await h.tick(NOW + REVIVE_GRACE_MS + 1);
+    await h.waitForStatus(id, 'dead');
+    assert.equal(h.status(id), 'dead', 'a never-booted broker is reaped, not stranded active forever');
+    // …and the parent (an active subscriber of the child) was told up the spine.
+    const note = h.inbox(root).find((e) => /never started/.test(e.label ?? ''));
+    assert.ok(note !== undefined, 'the parent received a boot-failure notice up the spine');
+});

package/dist/core/__tests__/broker-dialogs.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/broker-dialogs.test.js

@@ -0,0 +1,126 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/broker-dialogs.test.ts
+//
+// Broker dialog handling — the §5.4 C2 forward-progress proof plus the T8 attach
+// gates G5/G5b/G6. Split out of broker-lifecycle.test.ts (see its header for the
+// full file map); the tests are the original acceptance gate, unchanged, on their
+// own isolated harness. The attach-client helpers live in
+// helpers/broker-clients.ts (the PRODUCTION ViewSocketClient, §0 one-writer: a
+// viewer holds ONLY a socket).
+import { test, before, after, afterEach } from 'node:test';
+import assert from 'node:assert/strict';
+import { createHarness, hasTmux } from './helpers/harness.js';
+import { isPidAlive } from '../canvas/pid.js';
+import { createAttachKit } from './helpers/broker-clients.js';
+let h;
+let root;
+const kit = createAttachKit(() => h);
+const { attach, attachUntil } = kit;
+before(async () => {
+    if (!hasTmux())
+        return;
+    h = await createHarness({ sessionPrefix: 'crtr-brkdlg' });
+    root = h.spawnRoot('broker-dialogs suite root');
+});
+after(async () => {
+    if (h !== undefined)
+        await h.dispose();
+});
+afterEach(() => {
+    kit.closeAll();
+});
+const brokerPid = (id) => h.node(id).pi_pid;
+// ===========================================================================
+// C2 forward-progress (zero-viewer path) — an unattended blocking dialog
+// resolves to its default (false) IMMEDIATELY: with no controller connected the
+// broker's REAL makeBrokerUiContext falls back to noOp resolution, so the engine
+// never deadlocks AND never waits on a per-dialog timeout (the design §5.4
+// timeout premise is false). Red/green: the OLD broker armed a setTimeout and
+// resolved only after `timeout` ms; the fixed broker resolves in ~0ms. (Supports
+// acceptance item 7: the existing suite stays green; this only ADDS coverage.)
+// ===========================================================================
+test('C2 — unattended dialog resolves to its default IMMEDIATELY (noOp), never waits on a timeout', { skip: !hasTmux() }, async () => {
+    const id = await h.spawnHeadlessChild(root, 'headless worker — dialog');
+    const pid = h.node(id).pi_pid;
+    // Drive the fake engine to call uiContext.confirm(..., { timeout: 5000 }) with
+    // NO controller. C2 fix: makeBrokerUiContext resolves the default (false) AT
+    // ONCE — it does NOT arm/await the timeout. A generous < 2000ms bound is still a
+    // hard fail against the old ~5000ms timeout-wait while staying robust on slow CI.
+    h.fakeCmd(id, { cmd: 'dialog', timeout: 5000 });
+    const results = await h.waitFor(() => {
+        const r = h.dialogResults(id);
+        return r.length > 0 ? r : null;
+    }, { timeoutMs: 15_000, label: 'unattended dialog resolved' });
+    assert.equal(results[0].resolved, false, 'unattended confirm resolves to its default (false / deny)');
+    assert.ok(results[0].ms < 2000, `C2: resolved IMMEDIATELY (noOp), not after the 5000ms timeout — got ${results[0].ms}ms`);
+    assert.equal(isPidAlive(pid), true, 'the broker made forward progress (still alive, did not hang or exit)');
+});
+// ---------------------------------------------------------------------------
+// G5 — dialog forward + answer. Guards: a blocking dialog reaches the controller
+// as extension_ui_request and the controller's extension_ui_response unblocks the
+// engine with ITS answer (not the default). Failure mode: a dialog the controller
+// can't see/answer (silent deadlock).
+// ---------------------------------------------------------------------------
+test('G5 — controller receives an extension_ui_request, answers it, and the engine proceeds with that answer', { skip: !hasTmux() }, async () => {
+    const id = await h.spawnHeadlessChild(root, 'headless worker — G5a');
+    const c = await attach(id, 'controller', 'g5-ctrl');
+    h.fakeCmd(id, { cmd: 'dialog', timeout: 20_000 }); // generous: the controller answers first
+    const req = await c.waitFrame((f) => f.type === 'extension_ui_request', 'G5 dialog forwarded to controller');
+    assert.equal(req.method, 'confirm', 'G5: the forwarded dialog is the confirm() the engine raised');
+    const reqId = req.id;
+    c.send({ type: 'extension_ui_response', id: reqId, confirmed: true });
+    const results = await h.waitFor(() => {
+        const r = h.dialogResults(id);
+        return r.length > 0 ? r : null;
+    }, { label: 'G5 dialog resolved by the controller', timeoutMs: 15_000 });
+    assert.equal(results[0].resolved, true, 'G5: the engine proceeded with the controller answer (true), not the default (false)');
+});
+// ---------------------------------------------------------------------------
+// G5 (mid-dialog attach) — Guards: a dialog raised under a prior controller stays
+// pending across that controller's detach (M2) and is delivered to whoever takes
+// control next via welcome.pending_dialog. Failure mode: a pending dialog lost on
+// controller handoff.
+// ---------------------------------------------------------------------------
+test('G5 — a controller attaching MID-dialog receives the pending dialog via welcome.pending_dialog', { skip: !hasTmux() }, async () => {
+    const id = await h.spawnHeadlessChild(root, 'headless worker — G5b');
+    const a = await attach(id, 'controller', 'g5b-A');
+    h.fakeCmd(id, { cmd: 'dialog', timeout: 30_000 }); // stays pending long enough for the handoff
+    const reqA = await a.waitFrame((f) => f.type === 'extension_ui_request', 'G5b dialog forwarded to controller A');
+    const reqId = reqA.id;
+    a.send({ type: 'bye' });
+    a.close(); // M2: detach frees control but does NOT cancel the pending dialog
+    // Controller B takes control (retry covers the close→controllerId=null beat) and
+    // its welcome carries the still-pending dialog.
+    const b = await attachUntil(id, 'controller', 'g5b-B', (x) => x.welcome.role === 'controller' && x.welcome.pending_dialog != null, 'G5b controller B takes control with the pending dialog');
+    assert.equal(b.welcome.pending_dialog.id, reqId, 'G5b: welcome.pending_dialog is the same dialog raised under A');
+    assert.equal(b.welcome.pending_dialog.method, 'confirm', 'G5b: the pending dialog is the confirm()');
+    b.send({ type: 'extension_ui_response', id: reqId, confirmed: true });
+    const results = await h.waitFor(() => {
+        const r = h.dialogResults(id);
+        return r.length > 0 ? r : null;
+    }, { label: 'G5b dialog resolved by controller B', timeoutMs: 15_000 });
+    assert.equal(results[0].resolved, true, 'G5b: controller B answered the handed-off dialog and the engine proceeded');
+});
+// ---------------------------------------------------------------------------
+// G6 — anti-deadlock. (a) a zero-viewer dialog resolves to its default AT ONCE
+// (noOp). (b) an ATTENDED dialog the controller never answers resolves on a SHORT
+// per-dialog broker timeout. Guards: the engine never hangs on a dialog with no
+// answerer. Failure mode: a forever-blocked turn.
+// ---------------------------------------------------------------------------
+test('G6 — zero-viewer dialog resolves immediately; an unanswered attended dialog resolves on the broker timeout', { skip: !hasTmux() }, async () => {
+    const id = await h.spawnHeadlessChild(root, 'headless worker — G6');
+    const pid = brokerPid(id);
+    // (a) zero viewers → immediate noOp default (NOT the 5000ms timeout).
+    h.fakeCmd(id, { cmd: 'dialog', timeout: 5000 });
+    const r1 = await h.waitFor(() => (h.dialogResults(id).length >= 1 ? h.dialogResults(id) : null), { label: 'G6a zero-viewer dialog resolved', timeoutMs: 15_000 });
+    assert.equal(r1[0].resolved, false, 'G6a: zero-viewer dialog resolves to the default (deny)');
+    assert.ok(r1[0].ms < 2000, `G6a: resolved immediately (noOp), not after the 5000ms timeout — got ${r1[0].ms}ms`);
+    // (b) controller attached but silent → the broker resolves on the SHORT explicit
+    // per-dialog timeout (800ms), never the 120s default.
+    const c = await attach(id, 'controller', 'g6-ctrl');
+    h.fakeCmd(id, { cmd: 'dialog', timeout: 800 });
+    await c.waitFrame((f) => f.type === 'extension_ui_request', 'G6b dialog forwarded to controller'); // received, deliberately NOT answered
+    const r2 = await h.waitFor(() => (h.dialogResults(id).length >= 2 ? h.dialogResults(id) : null), { label: 'G6b attended dialog resolved on timeout', timeoutMs: 15_000 });
+    assert.equal(r2[1].resolved, false, 'G6b: an unanswered attended dialog resolves to the default (deny)');
+    assert.ok(r2[1].ms >= 600 && r2[1].ms < 5000, `G6b: resolved on the ~800ms per-dialog timeout, not instantly and not the 120s default — got ${r2[1].ms}ms`);
+    assert.equal(isPidAlive(pid), true, 'G6: the engine made forward progress on both dialogs (still alive)');
+});

package/dist/core/__tests__/broker-dormant-wake.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/broker-dormant-wake.test.js

@@ -0,0 +1,51 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/broker-dormant-wake.test.ts
+//
+// Acceptance item 3 (DORMANT) of the headless-broker migration. Split out of
+// broker-lifecycle.test.ts (see its header for the full file map); the test is
+// the original acceptance gate, unchanged, on its own isolated harness.
+import { test, before, after } from 'node:test';
+import assert from 'node:assert/strict';
+import { createHarness, hasTmux } from './helpers/harness.js';
+import { subscribe } from '../canvas/canvas.js';
+import { appendInbox } from '../feed/inbox.js';
+import { isPidAlive } from '../canvas/pid.js';
+let h;
+let root;
+before(async () => {
+    if (!hasTmux())
+        return;
+    h = await createHarness({ sessionPrefix: 'crtr-brkdorm' });
+    // An active resident root: the spawn parent AND the live node a child can hold
+    // an active subscription to (so its natural stop classifies 'awaiting').
+    root = h.spawnRoot('broker-dormant suite root');
+});
+after(async () => {
+    if (h !== undefined)
+        await h.dispose();
+});
+// ===========================================================================
+// Item 3 (DORMANT) — a natural stop while awaiting a live subscription idle-
+// RELEASES the broker (it exits); the daemon's second pass revives it (RESUME)
+// on the next unseen inbox entry.
+// ===========================================================================
+test('DORMANT wake — idle-release → broker exits → daemon pass-2 reviveNode(resume) on inbox', { skip: !hasTmux() }, async () => {
+    const id = await h.spawnHeadlessChild(root, 'headless worker — dormant wake');
+    const pid = h.node(id).pi_pid;
+    // Give the (terminal) child an ACTIVE subscription to the LIVE root, so the
+    // stop-guard classifies its natural stop as 'awaiting' (a dormant orchestrator
+    // awaiting a worker) → the stophook idle-releases it (paneless → no focus).
+    subscribe(id, root, true);
+    await h.stop(id, 'stop');
+    await h.waitForStatus(id, 'idle');
+    assert.equal(h.node(id).intent, 'idle-release', 'awaiting + unfocused → idle-release');
+    await h.waitFor(() => !isPidAlive(pid), { label: 'released broker process exited' });
+    // Daemon owns wake-on-message for a dormant (pi-dead) broker: an unseen inbox
+    // entry + a tick → pass 2 → reviveNode(resume:true) → a fresh broker boots.
+    appendInbox(id, { from: 'tester', tier: 'normal', kind: 'message', label: 'resume-me', data: { body: 'work after release' } });
+    await h.tick();
+    const boot2 = await h.awaitBoot(id, { minCount: 2 });
+    assert.equal(boot2.resuming, true, 'the dormant revive RESUMES the saved session');
+    const newPid = h.node(id).pi_pid;
+    assert.ok(newPid != null && isPidAlive(newPid), 'the revived broker pid is alive');
+    assert.notEqual(newPid, pid, 'the revived broker is a fresh process');
+});