@crouton-kit/crouter 0.3.19 → 0.3.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/builtin-personas/design/PERSONA.md +1 -0
  2. package/dist/builtin-personas/design/orchestrator.md +1 -0
  3. package/dist/builtin-personas/developer/PERSONA.md +1 -0
  4. package/dist/builtin-personas/developer/orchestrator.md +1 -0
  5. package/dist/builtin-personas/explore/PERSONA.md +1 -0
  6. package/dist/builtin-personas/explore/orchestrator.md +4 -0
  7. package/dist/builtin-personas/general/PERSONA.md +1 -0
  8. package/dist/builtin-personas/general/orchestrator.md +4 -0
  9. package/dist/builtin-personas/plan/PERSONA.md +1 -0
  10. package/dist/builtin-personas/plan/orchestrator.md +1 -0
  11. package/dist/builtin-personas/plan/reviewers/architecture-fit/PERSONA.md +1 -0
  12. package/dist/builtin-personas/plan/reviewers/code-smells/PERSONA.md +1 -0
  13. package/dist/builtin-personas/plan/reviewers/pattern-consistency/PERSONA.md +1 -0
  14. package/dist/builtin-personas/plan/reviewers/requirements-coverage/PERSONA.md +1 -0
  15. package/dist/builtin-personas/plan/reviewers/security/PERSONA.md +1 -0
  16. package/dist/builtin-personas/review/PERSONA.md +1 -0
  17. package/dist/builtin-personas/review/orchestrator.md +4 -0
  18. package/dist/builtin-personas/spec/PERSONA.md +1 -0
  19. package/dist/builtin-personas/spec/orchestrator.md +1 -0
  20. package/dist/commands/human/queue.js +11 -0
  21. package/dist/commands/node.js +4 -1
  22. package/dist/core/__tests__/broker-attach-limits.test.d.ts +1 -0
  23. package/dist/core/__tests__/broker-attach-limits.test.js +157 -0
  24. package/dist/core/__tests__/broker-attach-stream.test.d.ts +1 -0
  25. package/dist/core/__tests__/broker-attach-stream.test.js +125 -0
  26. package/dist/core/__tests__/broker-crash-teardown.test.d.ts +1 -0
  27. package/dist/core/__tests__/broker-crash-teardown.test.js +116 -0
  28. package/dist/core/__tests__/broker-dialogs.test.d.ts +1 -0
  29. package/dist/core/__tests__/broker-dialogs.test.js +126 -0
  30. package/dist/core/__tests__/broker-dormant-wake.test.d.ts +1 -0
  31. package/dist/core/__tests__/broker-dormant-wake.test.js +51 -0
  32. package/dist/core/__tests__/broker-lifecycle.test.js +14 -604
  33. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.d.ts +1 -0
  34. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +102 -0
  35. package/dist/core/__tests__/canvas-inbox-watcher.test.js +2 -36
  36. package/dist/core/__tests__/helpers/broker-clients.d.ts +43 -0
  37. package/dist/core/__tests__/helpers/broker-clients.js +178 -0
  38. package/dist/core/__tests__/live-mutation-verbs.test.d.ts +1 -0
  39. package/dist/core/__tests__/live-mutation-verbs.test.js +175 -0
  40. package/dist/core/__tests__/live-mutation.test.js +4 -147
  41. package/dist/core/canvas/canvas.js +1 -1
  42. package/dist/core/canvas/types.d.ts +6 -0
  43. package/dist/core/runtime/launch.d.ts +9 -2
  44. package/dist/core/runtime/launch.js +20 -3
  45. package/dist/core/runtime/nodes.d.ts +4 -0
  46. package/dist/core/runtime/nodes.js +1 -0
  47. package/dist/core/runtime/promote.js +3 -0
  48. package/dist/core/runtime/reset.js +3 -2
  49. package/dist/core/runtime/spawn.d.ts +4 -0
  50. package/dist/core/runtime/spawn.js +2 -1
  51. package/package.json +1 -1
@@ -9,44 +9,33 @@
9
9
  // the inbox-watcher, the stop-guard, or the daemon (superviseTick) is mocked.
10
10
  // Every assertion reads straight off the canvas data layer.
11
11
  //
12
+ // The suite is SPLIT across sibling files (one isolated harness each) so
13
+ // node:test's file-level parallelism applies — the tests themselves are the
14
+ // original acceptance gate, unchanged:
15
+ // THIS FILE — items 1–2 + 3 (LIVE): spawn, LIVE wake
16
+ // broker-dormant-wake.test.ts — item 3 (DORMANT): idle-release → daemon revive
17
+ // broker-crash-teardown.test.ts — items 4–6 + M-1/M-2: crash, teardown, boot failure
18
+ // broker-dialogs.test.ts — §5.4 C2 + gates G5/G5b/G6 (dialogs)
19
+ // broker-attach-stream.test.ts — gates G1/G1b/G2/G3 (drive + relay + snapshot)
20
+ // broker-attach-limits.test.ts — gates G4/G7/G8/G9 (arbitration + caps + one-writer)
21
+ //
12
22
  // Acceptance items (plan §4) → the asserting test:
13
23
  // 1 spawn --headless → host_kind='broker', null placement, live broker pid → "spawn"
14
24
  // 2 supervised by broker-pid signal-0 (stophook recordPid = broker pid) → "spawn"
15
- // 3 inbox wakes LIVE (in-broker watcher) AND DORMANT (idle-release → daemon) → "live wake" + "dormant wake"
16
- // 4 survive a broker crash via grace-revive RESUME on the saved .jsonl → "crash"
17
- // 5 tear down cleanly close shutdown frame exit, socket unlinked → "teardown"
18
- // 6 ONE-WRITER (R2) never two engine pids alive across crash→grace-revive → "crash"
19
- // 7 tmux stays default — the existing suite passes UNMODIFIED (this file only
20
- // ADDS a test file + ADDS harness wiring); the "dialog" test is the §5.4
21
- // forward-progress bonus proof of the zero-viewer path.
22
- import { test, before, after, afterEach } from 'node:test';
25
+ // 3 inbox wakes LIVE (in-broker watcher) AND DORMANT (idle-release → daemon) → "live wake" here + "dormant wake" (broker-dormant-wake.test.ts)
26
+ // 7 tmux stays default the existing suite passes UNMODIFIED (this split only
27
+ // MOVES tests between files + extracts harness wiring; nothing is rewritten).
28
+ import { test, before, after } from 'node:test';
23
29
  import assert from 'node:assert/strict';
24
- import { existsSync, readFileSync } from 'node:fs';
25
- import { join } from 'node:path';
26
- import { createConnection } from 'node:net';
27
- import { spawnSync } from 'node:child_process';
28
30
  import { createHarness, hasTmux } from './helpers/harness.js';
29
- import { subscribe } from '../canvas/canvas.js';
30
31
  import { appendInbox } from '../feed/inbox.js';
31
32
  import { isPidAlive } from '../canvas/pid.js';
32
- import { FAIL_BEFORE_SESSION_START } from './fixtures/fake-engine.js';
33
- // The PRODUCTION attach client (pure node:net + the broker codec, no TUI) reused
34
- // as the in-test controller/observer so the G1–G9 gate exercises the REAL client
35
- // too (§0 one-writer: a viewer holds ONLY a socket).
36
- import { ViewSocketClient } from '../../clients/attach/view-socket.js';
37
- import { CLIENT_READ_CAPS, FrameDecoder, encodeFrame, } from '../runtime/broker-protocol.js';
38
- // crtrd.ts module const (not exported). The fresh-pi-boot grace window the daemon
39
- // waits before grace-reviving a pi observed dead. Reference: crtrd.ts
40
- // `REVIVE_GRACE_MS = 20_000`.
41
- const REVIVE_GRACE_MS = 20_000;
42
33
  let h;
43
34
  let root;
44
35
  before(async () => {
45
36
  if (!hasTmux())
46
37
  return;
47
38
  h = await createHarness({ sessionPrefix: 'crtr-broker' });
48
- // An active resident root: the spawn parent AND the live node a child can hold
49
- // an active subscription to (so its natural stop classifies 'awaiting').
50
39
  root = h.spawnRoot('broker-suite root');
51
40
  });
52
41
  after(async () => {
@@ -96,582 +85,3 @@ test('LIVE wake — in-broker watcher injects an inbox push with no exit/revive'
96
85
  assert.equal(h.node(id).pi_pid, pid, 'pi_pid unchanged — the broker never exited');
97
86
  assert.equal(isPidAlive(pid), true, 'the broker is still alive after the live wake');
98
87
  });
99
- // ===========================================================================
100
- // Item 3 (DORMANT) — a natural stop while awaiting a live subscription idle-
101
- // RELEASES the broker (it exits); the daemon's second pass revives it (RESUME)
102
- // on the next unseen inbox entry.
103
- // ===========================================================================
104
- test('DORMANT wake — idle-release → broker exits → daemon pass-2 reviveNode(resume) on inbox', { skip: !hasTmux() }, async () => {
105
- const id = await h.spawnHeadlessChild(root, 'headless worker — dormant wake');
106
- const pid = h.node(id).pi_pid;
107
- // Give the (terminal) child an ACTIVE subscription to the LIVE root, so the
108
- // stop-guard classifies its natural stop as 'awaiting' (a dormant orchestrator
109
- // awaiting a worker) → the stophook idle-releases it (paneless → no focus).
110
- subscribe(id, root, true);
111
- await h.stop(id, 'stop');
112
- await h.waitForStatus(id, 'idle');
113
- assert.equal(h.node(id).intent, 'idle-release', 'awaiting + unfocused → idle-release');
114
- await h.waitFor(() => !isPidAlive(pid), { label: 'released broker process exited' });
115
- // Daemon owns wake-on-message for a dormant (pi-dead) broker: an unseen inbox
116
- // entry + a tick → pass 2 → reviveNode(resume:true) → a fresh broker boots.
117
- appendInbox(id, { from: 'tester', tier: 'normal', kind: 'message', label: 'resume-me', data: { body: 'work after release' } });
118
- await h.tick();
119
- const boot2 = await h.awaitBoot(id, { minCount: 2 });
120
- assert.equal(boot2.resuming, true, 'the dormant revive RESUMES the saved session');
121
- const newPid = h.node(id).pi_pid;
122
- assert.ok(newPid != null && isPidAlive(newPid), 'the revived broker pid is alive');
123
- assert.notEqual(newPid, pid, 'the revived broker is a fresh process');
124
- });
125
- // ===========================================================================
126
- // Item 4 + 6 — broker CRASH → grace-revive RESUME on the saved .jsonl; ONE-
127
- // WRITER: the crashed pid is dead BEFORE the revive launches and never
128
- // resurrects, and the revived pid is distinct (never two engine pids at once).
129
- // ===========================================================================
130
- test('CRASH → grace-revive RESUME; one-writer (old pid dead before new, never resurrects)', { skip: !hasTmux() }, async () => {
131
- const id = await h.spawnHeadlessChild(root, 'headless worker — crash');
132
- const oldPid = h.node(id).pi_pid;
133
- assert.equal(isPidAlive(oldPid), true, 'broker alive before the crash');
134
- assert.equal(h.node(id).intent ?? null, null, 'fresh broker has a null intent (not refresh/idle-release)');
135
- const boots = h.bootCount(id);
136
- // Kill the broker out from under the daemon (a crash). Use a fixed clock so the
137
- // grace window is exercised deterministically.
138
- process.kill(oldPid, 'SIGKILL');
139
- await h.waitFor(() => !isPidAlive(oldPid), { label: 'crashed broker pid is dead' });
140
- // ONE-WRITER: the old engine pid is dead BEFORE any revive can launch.
141
- assert.equal(isPidAlive(oldPid), false, 'crashed pid dead before the daemon revives');
142
- const NOW = 5_000_000;
143
- await h.tick(NOW); // pid dead, intent null → handleBrokerLiveness → handleLiveWindow marks pending
144
- assert.equal(h.bootCount(id), boots, 'inside the grace window → NOT yet revived');
145
- await h.tick(NOW + REVIVE_GRACE_MS + 1); // grace elapsed → reviveNode(resume:true)
146
- const boot2 = await h.awaitBoot(id, { minCount: boots + 1 });
147
- assert.equal(boot2.resuming, true, 'grace-revive RESUMES the saved .jsonl');
148
- const newPid = h.node(id).pi_pid;
149
- assert.ok(newPid != null && isPidAlive(newPid), 'the revived broker pid is alive');
150
- assert.notEqual(newPid, oldPid, 'the revived pid is distinct from the crashed one');
151
- assert.equal(isPidAlive(oldPid), false, 'one-writer: the crashed pid never resurrected');
152
- });
153
- // ===========================================================================
154
- // Item 5 — clean teardown: close → hostFor(meta).teardown → `shutdown` frame →
155
- // broker disposes + exits, socket unlinked, status canceled, daemon leaves it.
156
- // ===========================================================================
157
- test('clean teardown — node close → broker shutdown frame → exit, socket unlinked, not revived', { skip: !hasTmux() }, async () => {
158
- const id = await h.spawnHeadlessChild(root, 'headless worker — teardown');
159
- const pid = h.node(id).pi_pid;
160
- const sock = h.brokerSock(id);
161
- await h.waitFor(() => existsSync(sock), { label: 'broker created its view.sock' });
162
- // Close via the REAL CLI (resolves the node from --node; the subprocess carries
163
- // no TMUX_PANE). closeNode marks canceled BEFORE teardown (crash-safe order),
164
- // then hostFor(broker).teardown sends the `shutdown` frame.
165
- const res = h.cli(root, ['node', 'close', '--node', id]);
166
- assert.equal(res.code, 0, `node close should exit 0\n--stderr--\n${res.stderr}`);
167
- await h.waitFor(() => !isPidAlive(pid), { label: 'broker process exited on shutdown frame' });
168
- await h.waitFor(() => !existsSync(sock), { label: 'broker unlinked its socket on exit' });
169
- assert.equal(h.status(id), 'canceled', 'the closed node is canceled');
170
- // listNodes only surfaces active|idle, so a canceled broker is never supervised.
171
- await h.tick();
172
- assert.equal(h.status(id), 'canceled', 'still canceled after a daemon tick');
173
- assert.equal(isPidAlive(pid), false, 'the broker stays dead — never revived');
174
- });
175
- // ===========================================================================
176
- // C2 forward-progress (zero-viewer path) — an unattended blocking dialog
177
- // resolves to its default (false) IMMEDIATELY: with no controller connected the
178
- // broker's REAL makeBrokerUiContext falls back to noOp resolution, so the engine
179
- // never deadlocks AND never waits on a per-dialog timeout (the design §5.4
180
- // timeout premise is false). Red/green: the OLD broker armed a setTimeout and
181
- // resolved only after `timeout` ms; the fixed broker resolves in ~0ms. (Supports
182
- // acceptance item 7: the existing suite stays green; this only ADDS coverage.)
183
- // ===========================================================================
184
- test('C2 — unattended dialog resolves to its default IMMEDIATELY (noOp), never waits on a timeout', { skip: !hasTmux() }, async () => {
185
- const id = await h.spawnHeadlessChild(root, 'headless worker — dialog');
186
- const pid = h.node(id).pi_pid;
187
- // Drive the fake engine to call uiContext.confirm(..., { timeout: 5000 }) with
188
- // NO controller. C2 fix: makeBrokerUiContext resolves the default (false) AT
189
- // ONCE — it does NOT arm/await the timeout. A generous < 2000ms bound is still a
190
- // hard fail against the old ~5000ms timeout-wait while staying robust on slow CI.
191
- h.fakeCmd(id, { cmd: 'dialog', timeout: 5000 });
192
- const results = await h.waitFor(() => {
193
- const r = h.dialogResults(id);
194
- return r.length > 0 ? r : null;
195
- }, { timeoutMs: 15_000, label: 'unattended dialog resolved' });
196
- assert.equal(results[0].resolved, false, 'unattended confirm resolves to its default (false / deny)');
197
- assert.ok(results[0].ms < 2000, `C2: resolved IMMEDIATELY (noOp), not after the 5000ms timeout — got ${results[0].ms}ms`);
198
- assert.equal(isPidAlive(pid), true, 'the broker made forward progress (still alive, did not hang or exit)');
199
- });
200
- // ===========================================================================
201
- // M-1 regression — a broker that DIES before session_start must surface a boot
202
- // failure, not strand the node forever. Before the fix handleBrokerLiveness read
203
- // pid==null UNCONDITIONALLY as "still booting", so a broker that threw before
204
- // recording a pid/session left the node 'active' with no engine FOREVER and the
205
- // parent waited on a dead child. The daemon must instead crash it past the boot
206
- // grace + push a boot-failure up the spine. (Also proves M-2: the broker's fatal
207
- // diagnostic reaches job/broker.log instead of /dev/null.)
208
- // ===========================================================================
209
- test('boot failure — a broker that dies before session_start is reaped, not stranded (review M-1/M-2)', { skip: !hasTmux() }, async () => {
210
- // A headless child whose broker throws before session_start (no pid, no session
211
- // EVER recorded). spawnHeadlessChildNoBoot does not awaitBoot (there is none).
212
- const id = await h.spawnHeadlessChildNoBoot(root, `headless worker — boot fail ${FAIL_BEFORE_SESSION_START}`);
213
- // M-2: the broker logs its fatal stack to job/broker.log and exits(1). Without
214
- // the stdio redirect this diagnostic would be lost to /dev/null.
215
- const brokerLog = join(h.home, 'nodes', id, 'job', 'broker.log');
216
- await h.waitFor(() => existsSync(brokerLog) && /simulated pre-session_start boot failure/.test(readFileSync(brokerLog, 'utf8')), { label: 'broker logged its pre-session_start failure to job/broker.log' });
217
- // The node is stuck 'active' with a null pid and a null session — the strand.
218
- const n0 = h.node(id);
219
- assert.equal(n0.pi_pid ?? null, null, 'no broker pid was ever recorded');
220
- assert.equal(n0.pi_session_id ?? null, null, 'no session was ever recorded');
221
- // Inside the boot grace the daemon LEAVES it (could be the sub-second boot gap).
222
- const NOW = 9_000_000;
223
- await h.tick(NOW);
224
- assert.equal(h.status(id), 'active', 'inside the boot grace → daemon leaves it (boot gap)');
225
- // Past the boot grace with STILL no pid and no session → crash + boot failure.
226
- await h.tick(NOW + REVIVE_GRACE_MS + 1);
227
- await h.waitForStatus(id, 'dead');
228
- assert.equal(h.status(id), 'dead', 'a never-booted broker is reaped, not stranded active forever');
229
- // …and the parent (an active subscriber of the child) was told up the spine.
230
- const note = h.inbox(root).find((e) => /never started/.test(e.label ?? ''));
231
- assert.ok(note !== undefined, 'the parent received a boot-failure notice up the spine');
232
- });
233
- // ===========================================================================
234
- // T8 — the `crtr attach` acceptance gate (G1–G9). Each test below drives the REAL
235
- // detached broker process + REAL view.sock with the enriched fake engine, using
236
- // the PRODUCTION ViewSocketClient as the controller/observer (raw node:net only
237
- // where the client lifecycle is awkward — G7 oversized line, G8 stalled viewer).
238
- // The engine is hosted IN-PROCESS by the broker, so engine pid == broker pid ==
239
- // node.pi_pid == boot.pid; "engine pid unchanged" == broker pid unchanged + no new
240
- // boot. Each test's lead comment names its gate # and the failure mode it guards.
241
- // ===========================================================================
242
- const liveClients = [];
243
- afterEach(() => {
244
- for (const c of liveClients.splice(0)) {
245
- try {
246
- c.close();
247
- }
248
- catch {
249
- /* already gone */
250
- }
251
- }
252
- });
253
- const delay = (ms) => new Promise((r) => setTimeout(r, ms));
254
- const tok = (s) => `${s}-${Math.random().toString(36).slice(2, 8)}`;
255
- const frameHas = (f, token) => JSON.stringify(f).includes(token);
256
- const brokerPid = (id) => h.node(id).pi_pid;
257
- function brokerLogText(id) {
258
- try {
259
- return readFileSync(join(h.home, 'nodes', id, 'job', 'broker.log'), 'utf8');
260
- }
261
- catch {
262
- return '';
263
- }
264
- }
265
- // Connect a ViewSocketClient to a node's running broker, hello, await welcome.
266
- // awaitBoot returns once the boot proof is written — which is BEFORE the broker's
267
- // server.listen() binds view.sock — so a fresh attach can momentarily race the
268
- // bind; retry the connect on BrokerUnavailable until it is listening.
269
- async function attach(id, role, clientId) {
270
- await h.waitFor(() => existsSync(h.brokerSock(id)), { label: `view.sock for ${id}`, timeoutMs: 20_000 });
271
- const frames = [];
272
- let client;
273
- for (let attempt = 0;; attempt++) {
274
- client = new ViewSocketClient(id);
275
- client.on('frame', (f) => frames.push(f));
276
- try {
277
- await new Promise((resolve, reject) => {
278
- client.once('connect', resolve);
279
- client.once('error', reject);
280
- client.connect();
281
- });
282
- break;
283
- }
284
- catch (err) {
285
- client.close();
286
- if (attempt >= 30)
287
- throw err;
288
- await delay(100);
289
- }
290
- }
291
- client.on('error', () => { }); // post-connect error sink (never throw uncaught)
292
- // Register cleanup the instant the socket is connected — BEFORE the hello/welcome
293
- // round-trip — so a welcome timeout cannot leak a connected socket past the test.
294
- liveClients.push({ close: () => client.close() });
295
- const waitFrame = (pred, label, timeoutMs = 15_000) => h.waitFor(() => frames.find(pred) ?? null, { label, timeoutMs });
296
- client.send({ type: 'hello', role, client_id: clientId });
297
- let welcome;
298
- try {
299
- welcome = (await waitFrame((f) => f.type === 'welcome', `welcome for ${clientId}`));
300
- }
301
- catch (err) {
302
- client.close();
303
- throw err;
304
- }
305
- return { client, frames, welcome, send: (f) => client.send(f), waitFrame, close: () => client.close() };
306
- }
307
- // Attach (as `role`) and retry until the welcome satisfies `pred` — used where the
308
- // observable lands a beat after a prior action (G3 snapshot accrual, G5b control
309
- // handoff after a controller detaches). Deterministic: it polls an observable.
310
- async function attachUntil(id, role, clientId, pred, label) {
311
- for (let attempt = 0;; attempt++) {
312
- const a = await attach(id, role, `${clientId}-${attempt}`);
313
- if (pred(a))
314
- return a;
315
- a.close();
316
- if (attempt >= 40)
317
- throw new Error(`attachUntil timed out: ${label}`);
318
- await delay(150);
319
- }
320
- }
321
- // A raw node:net peer. read:true decodes incoming frames; read:false leaves the
322
- // socket PAUSED (no 'data' listener, never resumed) so it never drains — the
323
- // stalled viewer (G8) whose backlog the broker must shed at the HWM.
324
- async function connectRaw(id, opts) {
325
- await h.waitFor(() => existsSync(h.brokerSock(id)), { label: `view.sock for ${id}`, timeoutMs: 20_000 });
326
- const frames = [];
327
- const decoder = new FrameDecoder(CLIENT_READ_CAPS);
328
- let isClosed = false;
329
- const socket = await new Promise((resolve, reject) => {
330
- const s = createConnection(h.brokerSock(id));
331
- s.once('connect', () => resolve(s));
332
- s.once('error', reject);
333
- });
334
- socket.on('close', () => {
335
- isClosed = true;
336
- });
337
- socket.on('error', () => {
338
- /* close follows */
339
- });
340
- if (opts.read) {
341
- socket.on('data', (chunk) => {
342
- try {
343
- for (const f of decoder.push(chunk))
344
- frames.push(f);
345
- }
346
- catch {
347
- /* a client-side overflow is irrelevant here */
348
- }
349
- });
350
- }
351
- const rc = {
352
- socket,
353
- frames,
354
- closed: () => isClosed,
355
- send: (f) => {
356
- try {
357
- socket.write(encodeFrame(f));
358
- }
359
- catch {
360
- /* dead */
361
- }
362
- },
363
- writeRaw: (d) => {
364
- try {
365
- socket.write(d);
366
- }
367
- catch {
368
- /* dead */
369
- }
370
- },
371
- waitClosed: (label, timeoutMs = 15_000) => h.waitFor(() => (isClosed ? true : null), { label, timeoutMs }).then(() => undefined),
372
- close: () => {
373
- try {
374
- socket.destroy();
375
- }
376
- catch {
377
- /* ignore */
378
- }
379
- },
380
- };
381
- liveClients.push(rc);
382
- return rc;
383
- }
384
- /** lsof the holders of `path`, or null when lsof is unavailable (skip the fd
385
- * check). Exit-non-zero with empty stdout means "no holders". */
386
- function lsofHolders(path) {
387
- if (spawnSync('which', ['lsof'], { stdio: 'ignore' }).status !== 0)
388
- return null;
389
- const out = (spawnSync('lsof', ['-t', '--', path], { encoding: 'utf8' }).stdout ?? '').trim();
390
- if (out === '')
391
- return [];
392
- return out
393
- .split('\n')
394
- .map((l) => Number(l.trim()))
395
- .filter((n) => Number.isFinite(n));
396
- }
397
- // ---------------------------------------------------------------------------
398
- // G1 — controller drive + live relay. Guards: a controller's `prompt` runs the
399
- // engine AND the streaming AgentSessionEvents are fanned out to the client
400
- // VERBATIM. Failure mode: a broken relay/fan-out (viewer sees nothing).
401
- // ---------------------------------------------------------------------------
402
- test('G1 — controller prompt runs the engine and the streamed AgentSessionEvents relay to the client', { skip: !hasTmux() }, async () => {
403
- const id = await h.spawnHeadlessChild(root, 'headless worker — G1');
404
- const c = await attach(id, 'controller', 'g1-ctrl');
405
- assert.equal(c.welcome.role, 'controller', 'first controller is admitted as controller');
406
- const token = tok('G1-PROMPT');
407
- c.send({ type: 'prompt', text: token });
408
- // The full streaming turn relays: agent_end carrying the token proves the turn
409
- // ran AND its terminal frame reached the client; the intermediate types prove
410
- // the stream (not just a final blob) was fanned out.
411
- await c.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G1 agent_end carrying the prompt token');
412
- for (const t of ['message_start', 'message_update', 'tool_execution_start', 'tool_execution_end', 'message_end', 'turn_end', 'agent_start', 'turn_start']) {
413
- assert.ok(c.frames.some((f) => f.type === t), `G1: client received a relayed ${t} frame`);
414
- }
415
- assert.ok(c.frames.some((f) => f.type === 'message_start' && frameHas(f, token)), 'G1: a relayed message carries the prompt token');
416
- });
417
- // ---------------------------------------------------------------------------
418
- // G1b — message_update coalescing preserves ordering (regression for the F2
419
- // attach typing-lag fix, 2026-06-09). The broker holds the latest message_update
420
- // on a ~75ms timer; any OTHER event must flush it FIRST, so a viewer can never
421
- // observe message_update AFTER its message_end (which would resurrect stale
422
- // streaming text over the final message), and the LAST coalesced update must not
423
- // be silently dropped at end-of-turn. Failure modes: a flush ordered after the
424
- // non-update event, or a pending update discarded when message_end wins the race.
425
- // ---------------------------------------------------------------------------
426
- test('G1b — coalesced message_update never arrives after message_end; updates still relayed', { skip: !hasTmux() }, async () => {
427
- const id = await h.spawnHeadlessChild(root, 'headless worker — G1b');
428
- const c = await attach(id, 'controller', 'g1b-ctrl');
429
- const token = tok('G1B');
430
- // 12 updates at the fake engine's setImmediate pace — far faster than the 75ms
431
- // coalesce window, so coalescing genuinely engages (fewer relayed than emitted).
432
- h.fakeCmd(id, { cmd: 'stream', text: token, updates: 12 });
433
- await c.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G1b turn relayed');
434
- const types = c.frames.map((f) => f.type);
435
- const lastUpdate = types.lastIndexOf('message_update');
436
- const msgEnd = types.indexOf('message_end');
437
- assert.ok(lastUpdate >= 0, 'G1b: at least one message_update relayed (coalescing must not starve updates)');
438
- assert.ok(msgEnd >= 0, 'G1b: message_end relayed');
439
- assert.ok(lastUpdate < msgEnd, `G1b: a message_update arrived AFTER message_end (update@${lastUpdate} vs end@${msgEnd}) — the coalescer flushed out of order`);
440
- // tool/turn/agent frames must also never precede a stale held update.
441
- for (const t of ['tool_execution_start', 'turn_end', 'agent_end']) {
442
- const i = types.indexOf(t);
443
- assert.ok(i < 0 || lastUpdate < i, `G1b: message_update relayed after ${t}`);
444
- }
445
- });
446
- // ---------------------------------------------------------------------------
447
- // G2 — detach leaves the engine running. Guards: `bye`/close drops ONE listener,
448
- // never the engine. Failure mode: a detach that disposes the broker/engine.
449
- // ---------------------------------------------------------------------------
450
- test('G2 — detach (bye+close) leaves the broker alive + the engine still emitting; engine pid unchanged', { skip: !hasTmux() }, async () => {
451
- const id = await h.spawnHeadlessChild(root, 'headless worker — G2');
452
- const pid = brokerPid(id);
453
- const boots = h.bootCount(id);
454
- const c = await attach(id, 'controller', 'g2-ctrl');
455
- const p = tok('G2-PROMPT');
456
- c.send({ type: 'prompt', text: p });
457
- await c.waitFrame((f) => f.type === 'agent_end' && frameHas(f, p), 'G2 turn relayed before detach');
458
- c.send({ type: 'bye' });
459
- c.close();
460
- await delay(300); // let the broker process the 'close'
461
- assert.equal(isPidAlive(pid), true, 'G2: broker still alive after detach');
462
- assert.equal(brokerPid(id), pid, 'G2: engine (== broker) pid UNCHANGED across detach');
463
- assert.equal(h.bootCount(id), boots, 'G2: no reboot — no second engine spawned');
464
- // Still emitting: drive a turn with no client, then a fresh observer's snapshot
465
- // carries it (the engine kept running and processing after the detach).
466
- const after = tok('G2-AFTER-DETACH');
467
- h.fakeCmd(id, { cmd: 'stream', text: after });
468
- const o = await attachUntil(id, 'observer', 'g2-obs', (a) => JSON.stringify(a.welcome.snapshot.messages).includes(after), 'G2 post-detach message in snapshot');
469
- assert.ok(o.welcome.snapshot.messages.length > 0, 'G2: the engine produced a message after the detach');
470
- assert.equal(isPidAlive(pid), true, 'G2: broker still alive after the post-detach turn');
471
- });
472
- // ---------------------------------------------------------------------------
473
- // G3 — catch-up snapshot. Guards: messages produced with NO viewer attached are
474
- // in welcome.snapshot on reattach, and live events resume. Failure mode: a viewer
475
- // that reattaches blind to history / gets no further events.
476
- // ---------------------------------------------------------------------------
477
- test('G3 — messages produced while detached appear in welcome.snapshot on reattach; live events resume', { skip: !hasTmux() }, async () => {
478
- const id = await h.spawnHeadlessChild(root, 'headless worker — G3');
479
- const past = tok('G3-DETACHED');
480
- h.fakeCmd(id, { cmd: 'stream', text: past }); // produced with zero viewers attached
481
- const o = await attachUntil(id, 'observer', 'g3-obs', (a) => JSON.stringify(a.welcome.snapshot.messages).includes(past), 'G3 detached message in snapshot');
482
- assert.ok(JSON.stringify(o.welcome.snapshot.messages).includes(past), 'G3: welcome.snapshot.messages contains the message produced while detached');
483
- const live = tok('G3-LIVE');
484
- h.fakeCmd(id, { cmd: 'stream', text: live });
485
- await o.waitFrame((f) => f.type === 'agent_end' && frameHas(f, live), 'G3 live events resume after reattach');
486
- });
487
- // ---------------------------------------------------------------------------
488
- // G4 — arbitration + observer read. Guards: 2nd client is admitted observer, an
489
- // observer prompt is rejected not_controller, BOTH clients receive the relay.
490
- // Failure mode: two controllers, or an observer driving the engine, or fan-out
491
- // that misses a viewer.
492
- // ---------------------------------------------------------------------------
493
- test('G4 — second client is observer; observer prompt → error{not_controller}; both receive the stream', { skip: !hasTmux() }, async () => {
494
- const id = await h.spawnHeadlessChild(root, 'headless worker — G4');
495
- const c1 = await attach(id, 'controller', 'g4-ctrl');
496
- assert.equal(c1.welcome.role, 'controller', 'first client holds control');
497
- const c2 = await attach(id, 'controller', 'g4-second'); // requests control; held → observer
498
- assert.equal(c2.welcome.role, 'observer', 'second client is admitted read-only observer (first-attach-wins)');
499
- c2.send({ type: 'prompt', text: 'observer must not drive' });
500
- const err = await c2.waitFrame((f) => f.type === 'error', 'G4 observer prompt rejected');
501
- assert.equal(err.code, 'not_controller', 'G4: observer prompt → error{not_controller}');
502
- const token = tok('G4-BROADCAST');
503
- c1.send({ type: 'prompt', text: token });
504
- await c1.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G4 controller received the stream');
505
- await c2.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G4 observer ALSO received the stream');
506
- });
507
- // ---------------------------------------------------------------------------
508
- // G5 — dialog forward + answer. Guards: a blocking dialog reaches the controller
509
- // as extension_ui_request and the controller's extension_ui_response unblocks the
510
- // engine with ITS answer (not the default). Failure mode: a dialog the controller
511
- // can't see/answer (silent deadlock).
512
- // ---------------------------------------------------------------------------
513
- test('G5 — controller receives an extension_ui_request, answers it, and the engine proceeds with that answer', { skip: !hasTmux() }, async () => {
514
- const id = await h.spawnHeadlessChild(root, 'headless worker — G5a');
515
- const c = await attach(id, 'controller', 'g5-ctrl');
516
- h.fakeCmd(id, { cmd: 'dialog', timeout: 20_000 }); // generous: the controller answers first
517
- const req = await c.waitFrame((f) => f.type === 'extension_ui_request', 'G5 dialog forwarded to controller');
518
- assert.equal(req.method, 'confirm', 'G5: the forwarded dialog is the confirm() the engine raised');
519
- const reqId = req.id;
520
- c.send({ type: 'extension_ui_response', id: reqId, confirmed: true });
521
- const results = await h.waitFor(() => {
522
- const r = h.dialogResults(id);
523
- return r.length > 0 ? r : null;
524
- }, { label: 'G5 dialog resolved by the controller', timeoutMs: 15_000 });
525
- assert.equal(results[0].resolved, true, 'G5: the engine proceeded with the controller answer (true), not the default (false)');
526
- });
527
- // ---------------------------------------------------------------------------
528
- // G5 (mid-dialog attach) — Guards: a dialog raised under a prior controller stays
529
- // pending across that controller's detach (M2) and is delivered to whoever takes
530
- // control next via welcome.pending_dialog. Failure mode: a pending dialog lost on
531
- // controller handoff.
532
- // ---------------------------------------------------------------------------
533
- test('G5 — a controller attaching MID-dialog receives the pending dialog via welcome.pending_dialog', { skip: !hasTmux() }, async () => {
534
- const id = await h.spawnHeadlessChild(root, 'headless worker — G5b');
535
- const a = await attach(id, 'controller', 'g5b-A');
536
- h.fakeCmd(id, { cmd: 'dialog', timeout: 30_000 }); // stays pending long enough for the handoff
537
- const reqA = await a.waitFrame((f) => f.type === 'extension_ui_request', 'G5b dialog forwarded to controller A');
538
- const reqId = reqA.id;
539
- a.send({ type: 'bye' });
540
- a.close(); // M2: detach frees control but does NOT cancel the pending dialog
541
- // Controller B takes control (retry covers the close→controllerId=null beat) and
542
- // its welcome carries the still-pending dialog.
543
- const b = await attachUntil(id, 'controller', 'g5b-B', (x) => x.welcome.role === 'controller' && x.welcome.pending_dialog != null, 'G5b controller B takes control with the pending dialog');
544
- assert.equal(b.welcome.pending_dialog.id, reqId, 'G5b: welcome.pending_dialog is the same dialog raised under A');
545
- assert.equal(b.welcome.pending_dialog.method, 'confirm', 'G5b: the pending dialog is the confirm()');
546
- b.send({ type: 'extension_ui_response', id: reqId, confirmed: true });
547
- const results = await h.waitFor(() => {
548
- const r = h.dialogResults(id);
549
- return r.length > 0 ? r : null;
550
- }, { label: 'G5b dialog resolved by controller B', timeoutMs: 15_000 });
551
- assert.equal(results[0].resolved, true, 'G5b: controller B answered the handed-off dialog and the engine proceeded');
552
- });
553
- // ---------------------------------------------------------------------------
554
- // G6 — anti-deadlock. (a) a zero-viewer dialog resolves to its default AT ONCE
555
- // (noOp). (b) an ATTENDED dialog the controller never answers resolves on a SHORT
556
- // per-dialog broker timeout. Guards: the engine never hangs on a dialog with no
557
- // answerer. Failure mode: a forever-blocked turn.
558
- // ---------------------------------------------------------------------------
559
- test('G6 — zero-viewer dialog resolves immediately; an unanswered attended dialog resolves on the broker timeout', { skip: !hasTmux() }, async () => {
560
- const id = await h.spawnHeadlessChild(root, 'headless worker — G6');
561
- const pid = brokerPid(id);
562
- // (a) zero viewers → immediate noOp default (NOT the 5000ms timeout).
563
- h.fakeCmd(id, { cmd: 'dialog', timeout: 5000 });
564
- const r1 = await h.waitFor(() => (h.dialogResults(id).length >= 1 ? h.dialogResults(id) : null), { label: 'G6a zero-viewer dialog resolved', timeoutMs: 15_000 });
565
- assert.equal(r1[0].resolved, false, 'G6a: zero-viewer dialog resolves to the default (deny)');
566
- assert.ok(r1[0].ms < 2000, `G6a: resolved immediately (noOp), not after the 5000ms timeout — got ${r1[0].ms}ms`);
567
- // (b) controller attached but silent → the broker resolves on the SHORT explicit
568
- // per-dialog timeout (800ms), never the 120s default.
569
- const c = await attach(id, 'controller', 'g6-ctrl');
570
- h.fakeCmd(id, { cmd: 'dialog', timeout: 800 });
571
- await c.waitFrame((f) => f.type === 'extension_ui_request', 'G6b dialog forwarded to controller'); // received, deliberately NOT answered
572
- const r2 = await h.waitFor(() => (h.dialogResults(id).length >= 2 ? h.dialogResults(id) : null), { label: 'G6b attended dialog resolved on timeout', timeoutMs: 15_000 });
573
- assert.equal(r2[1].resolved, false, 'G6b: an unanswered attended dialog resolves to the default (deny)');
574
- assert.ok(r2[1].ms >= 600 && r2[1].ms < 5000, `G6b: resolved on the ~800ms per-dialog timeout, not instantly and not the 120s default — got ${r2[1].ms}ms`);
575
- assert.equal(isPidAlive(pid), true, 'G6: the engine made forward progress on both dialogs (still alive)');
576
- });
577
- // ---------------------------------------------------------------------------
578
- // G7 — decoder overflow (guards C5 OOM). A client line over BROKER_READ_CAPS is
579
- // cap-and-dropped; the broker survives and other clients are unaffected. Failure
580
- // mode: an unbounded decoder buffer growing the broker to OOM.
581
- // ---------------------------------------------------------------------------
582
- test('G7 — an oversized client line is dropped (frame_overflow), the broker survives, other clients unaffected', { skip: !hasTmux() }, async () => {
583
- const id = await h.spawnHeadlessChild(root, 'headless worker — G7');
584
- const pid = brokerPid(id);
585
- const boots = h.bootCount(id);
586
- const survivor = await attach(id, 'observer', 'g7-survivor');
587
- // 26 MiB with NO newline > BROKER_READ_CAPS.maxLineBytes (24 MiB) → the bounded
588
- // FrameDecoder throws FrameOverflowError; the broker drops the peer.
589
- const bad = await connectRaw(id, { read: true });
590
- bad.writeRaw(Buffer.alloc(26 * 1024 * 1024, 0x78));
591
- await bad.waitClosed('G7 oversized peer dropped by the broker');
592
- await h.waitFor(() => /frame overflow/.test(brokerLogText(id)) || null, { label: 'G7 broker logged the frame-overflow drop' });
593
- // The broker survives and the other client is unaffected: a fresh turn relays.
594
- const token = tok('G7-AFTER');
595
- h.fakeCmd(id, { cmd: 'stream', text: token });
596
- await survivor.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G7 survivor still receives live frames');
597
- assert.equal(isPidAlive(pid), true, 'G7: the broker survived the overflow');
598
- assert.equal(brokerPid(id), pid, 'G7: broker pid unchanged');
599
- assert.equal(h.bootCount(id), boots, 'G7: no reboot');
600
- });
601
- // ---------------------------------------------------------------------------
602
- // G8 — backpressure leak (guards M1). A stalled (non-reading) viewer is shed at
603
- // the HWM (32 MiB byte cap) while the broker + other viewers are unaffected.
604
- // Failure mode: an indefinitely-growing per-viewer queue (broker OOM) or a slow
605
- // viewer back-pressuring the shared engine.
606
- // ---------------------------------------------------------------------------
607
- test('G8 — a stalled viewer is dropped at the backpressure HWM; the broker + fast viewers are unaffected', { skip: !hasTmux() }, async () => {
608
- const id = await h.spawnHeadlessChild(root, 'headless worker — G8');
609
- const pid = brokerPid(id);
610
- const fast = await attach(id, 'observer', 'g8-fast'); // reads normally
611
- // A helloed viewer that NEVER reads (paused socket). It must be in the broadcast
612
- // set before the flood; hello, then a short beat for the broker to process it.
613
- const stalled = await connectRaw(id, { read: false });
614
- stalled.send({ type: 'hello', role: 'observer', client_id: 'g8-stalled' });
615
- await delay(400);
616
- // A fast event stream: ~60 MiB across 240 message_update frames. The stalled
617
- // viewer's per-viewer backlog crosses the 32 MiB byte cap → dropped; the fast
618
- // viewer drains between frames (per-update yield) and survives.
619
- //
620
- // The DROP SIGNAL is the broker's own log line, NOT the stalled socket's 'close':
621
- // a PAUSED node socket does not surface the peer FIN/close until it is resumed,
622
- // so the broker-side `backpressure high-water mark exceeded` line is the
623
- // deterministic, race-free proof that the slow viewer was shed at the HWM.
624
- const token = tok('G8-FLOOD');
625
- h.fakeCmd(id, { cmd: 'stream', text: token, updates: 240, padBytes: 256 * 1024, tool: false });
626
- await h.waitFor(() => /backpressure high-water mark exceeded/.test(brokerLogText(id)) || null, { label: 'G8 broker shed the stalled viewer at the HWM', timeoutMs: 30_000 });
627
- // The fast viewer drained the WHOLE stream (agent_end is the terminal frame) —
628
- // proving only the stalled viewer was shed, while the fast one was unaffected.
629
- await fast.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G8 fast viewer received the whole stream', 30_000);
630
- assert.equal(isPidAlive(pid), true, 'G8: the broker survived the slow-viewer flood');
631
- assert.equal(brokerPid(id), pid, 'G8: broker pid unchanged (the engine was not back-pressured into a restart)');
632
- });
633
- // ---------------------------------------------------------------------------
634
- // G9 — the load-bearing ONE-WRITER assertion. Across attach→detach→reattach the
635
- // broker pid AND engine pid (the same, in-process) are UNCHANGED and no second
636
- // engine is ever spawned; the viewer holds ONLY the socket (never the .jsonl).
637
- // Failure mode: a viewer that spawns/forks a second engine or opens the session
638
- // .jsonl — the corruption the headless design exists to prevent.
639
- // ---------------------------------------------------------------------------
640
- test('G9 — one-writer: broker/engine pid stable across attach→detach→reattach; no second engine; viewer holds only the socket', { skip: !hasTmux() }, async () => {
641
- const id = await h.spawnHeadlessChild(root, 'headless worker — G9');
642
- const pid0 = brokerPid(id);
643
- const boots0 = h.bootCount(id);
644
- const c1 = await attach(id, 'controller', 'g9-a');
645
- const a = tok('G9-A');
646
- c1.send({ type: 'prompt', text: a });
647
- await c1.waitFrame((f) => f.type === 'agent_end' && frameHas(f, a), 'G9 first turn relayed');
648
- c1.send({ type: 'bye' });
649
- c1.close();
650
- await delay(300);
651
- assert.equal(brokerPid(id), pid0, 'G9: broker/engine pid unchanged after detach');
652
- assert.equal(isPidAlive(pid0), true, 'G9: the one engine is still alive after detach');
653
- assert.equal(h.bootCount(id), boots0, 'G9: no second engine spawned across detach');
654
- // attachUntil (not a fixed sleep) synchronizes on the controller handoff: the
655
- // detach's controllerId=null lands a beat after close, so retry until the
656
- // reattach is admitted controller before driving the second prompt.
657
- const c2 = await attachUntil(id, 'controller', 'g9-b', (x) => x.welcome.role === 'controller', 'G9 reattach re-takes control');
658
- assert.equal(c2.welcome.role, 'controller', 'G9: the reattached client drives the SAME engine as controller');
659
- const b = tok('G9-B');
660
- c2.send({ type: 'prompt', text: b });
661
- await c2.waitFrame((f) => f.type === 'agent_end' && frameHas(f, b), 'G9 reattached controller drives the SAME engine');
662
- assert.equal(brokerPid(id), pid0, 'G9: STILL the same broker/engine pid after reattach (one writer, never two)');
663
- assert.equal(isPidAlive(pid0), true, 'G9: the single engine is alive across the full cycle');
664
- assert.equal(h.bootCount(id), boots0, 'G9: exactly one engine boot across attach→detach→reattach');
665
- // fd-check (best-effort): the viewer host (this test process) must NOT hold the
666
- // session .jsonl — only the broker may. With the fake engine the .jsonl is not
667
- // held open continuously, so the meaningful assertion is that the VIEWER never
668
- // appears among its holders (and any holder that exists is the broker).
669
- const jsonl = join(h.home, 'nodes', id, 'fake-session.jsonl');
670
- const holders = lsofHolders(jsonl);
671
- if (holders !== null) {
672
- assert.ok(!holders.includes(process.pid), 'G9: the attach client (this process) does NOT hold the session .jsonl');
673
- for (const holder of holders) {
674
- assert.equal(holder, pid0, 'G9: the ONLY holder of the session .jsonl is the broker/engine');
675
- }
676
- }
677
- });