@crouton-kit/crouter 0.3.19 → 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin-personas/design/PERSONA.md +1 -0
- package/dist/builtin-personas/design/orchestrator.md +1 -0
- package/dist/builtin-personas/developer/PERSONA.md +1 -0
- package/dist/builtin-personas/developer/orchestrator.md +1 -0
- package/dist/builtin-personas/explore/PERSONA.md +1 -0
- package/dist/builtin-personas/explore/orchestrator.md +4 -0
- package/dist/builtin-personas/general/PERSONA.md +1 -0
- package/dist/builtin-personas/general/orchestrator.md +4 -0
- package/dist/builtin-personas/plan/PERSONA.md +1 -0
- package/dist/builtin-personas/plan/orchestrator.md +1 -0
- package/dist/builtin-personas/plan/reviewers/architecture-fit/PERSONA.md +1 -0
- package/dist/builtin-personas/plan/reviewers/code-smells/PERSONA.md +1 -0
- package/dist/builtin-personas/plan/reviewers/pattern-consistency/PERSONA.md +1 -0
- package/dist/builtin-personas/plan/reviewers/requirements-coverage/PERSONA.md +1 -0
- package/dist/builtin-personas/plan/reviewers/security/PERSONA.md +1 -0
- package/dist/builtin-personas/review/PERSONA.md +1 -0
- package/dist/builtin-personas/review/orchestrator.md +4 -0
- package/dist/builtin-personas/spec/PERSONA.md +1 -0
- package/dist/builtin-personas/spec/orchestrator.md +1 -0
- package/dist/commands/human/queue.js +11 -0
- package/dist/commands/node.js +4 -1
- package/dist/core/__tests__/broker-attach-limits.test.d.ts +1 -0
- package/dist/core/__tests__/broker-attach-limits.test.js +157 -0
- package/dist/core/__tests__/broker-attach-stream.test.d.ts +1 -0
- package/dist/core/__tests__/broker-attach-stream.test.js +125 -0
- package/dist/core/__tests__/broker-crash-teardown.test.d.ts +1 -0
- package/dist/core/__tests__/broker-crash-teardown.test.js +116 -0
- package/dist/core/__tests__/broker-dialogs.test.d.ts +1 -0
- package/dist/core/__tests__/broker-dialogs.test.js +126 -0
- package/dist/core/__tests__/broker-dormant-wake.test.d.ts +1 -0
- package/dist/core/__tests__/broker-dormant-wake.test.js +51 -0
- package/dist/core/__tests__/broker-lifecycle.test.js +14 -604
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.d.ts +1 -0
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +102 -0
- package/dist/core/__tests__/canvas-inbox-watcher.test.js +2 -36
- package/dist/core/__tests__/helpers/broker-clients.d.ts +43 -0
- package/dist/core/__tests__/helpers/broker-clients.js +178 -0
- package/dist/core/__tests__/live-mutation-verbs.test.d.ts +1 -0
- package/dist/core/__tests__/live-mutation-verbs.test.js +175 -0
- package/dist/core/__tests__/live-mutation.test.js +4 -147
- package/dist/core/canvas/canvas.js +1 -1
- package/dist/core/canvas/types.d.ts +6 -0
- package/dist/core/runtime/launch.d.ts +9 -2
- package/dist/core/runtime/launch.js +20 -3
- package/dist/core/runtime/nodes.d.ts +4 -0
- package/dist/core/runtime/nodes.js +1 -0
- package/dist/core/runtime/promote.js +3 -0
- package/dist/core/runtime/reset.js +3 -2
- package/dist/core/runtime/spawn.d.ts +4 -0
- package/dist/core/runtime/spawn.js +2 -1
- package/package.json +1 -1
|
@@ -9,44 +9,33 @@
|
|
|
9
9
|
// the inbox-watcher, the stop-guard, or the daemon (superviseTick) is mocked.
|
|
10
10
|
// Every assertion reads straight off the canvas data layer.
|
|
11
11
|
//
|
|
12
|
+
// The suite is SPLIT across sibling files (one isolated harness each) so
|
|
13
|
+
// node:test's file-level parallelism applies — the tests themselves are the
|
|
14
|
+
// original acceptance gate, unchanged:
|
|
15
|
+
// THIS FILE — items 1–2 + 3 (LIVE): spawn, LIVE wake
|
|
16
|
+
// broker-dormant-wake.test.ts — item 3 (DORMANT): idle-release → daemon revive
|
|
17
|
+
// broker-crash-teardown.test.ts — items 4–6 + M-1/M-2: crash, teardown, boot failure
|
|
18
|
+
// broker-dialogs.test.ts — §5.4 C2 + gates G5/G5b/G6 (dialogs)
|
|
19
|
+
// broker-attach-stream.test.ts — gates G1/G1b/G2/G3 (drive + relay + snapshot)
|
|
20
|
+
// broker-attach-limits.test.ts — gates G4/G7/G8/G9 (arbitration + caps + one-writer)
|
|
21
|
+
//
|
|
12
22
|
// Acceptance items (plan §4) → the asserting test:
|
|
13
23
|
// 1 spawn --headless → host_kind='broker', null placement, live broker pid → "spawn"
|
|
14
24
|
// 2 supervised by broker-pid signal-0 (stophook recordPid = broker pid) → "spawn"
|
|
15
|
-
// 3 inbox wakes LIVE (in-broker watcher) AND DORMANT (idle-release → daemon) → "live wake" + "dormant wake"
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
|
|
19
|
-
// 7 tmux stays default — the existing suite passes UNMODIFIED (this file only
|
|
20
|
-
// ADDS a test file + ADDS harness wiring); the "dialog" test is the §5.4
|
|
21
|
-
// forward-progress bonus proof of the zero-viewer path.
|
|
22
|
-
import { test, before, after, afterEach } from 'node:test';
|
|
25
|
+
// 3 inbox wakes LIVE (in-broker watcher) AND DORMANT (idle-release → daemon) → "live wake" here + "dormant wake" (broker-dormant-wake.test.ts)
|
|
26
|
+
// 7 tmux stays default — the existing suite passes UNMODIFIED (this split only
|
|
27
|
+
// MOVES tests between files + extracts harness wiring; nothing is rewritten).
|
|
28
|
+
import { test, before, after } from 'node:test';
|
|
23
29
|
import assert from 'node:assert/strict';
|
|
24
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
25
|
-
import { join } from 'node:path';
|
|
26
|
-
import { createConnection } from 'node:net';
|
|
27
|
-
import { spawnSync } from 'node:child_process';
|
|
28
30
|
import { createHarness, hasTmux } from './helpers/harness.js';
|
|
29
|
-
import { subscribe } from '../canvas/canvas.js';
|
|
30
31
|
import { appendInbox } from '../feed/inbox.js';
|
|
31
32
|
import { isPidAlive } from '../canvas/pid.js';
|
|
32
|
-
import { FAIL_BEFORE_SESSION_START } from './fixtures/fake-engine.js';
|
|
33
|
-
// The PRODUCTION attach client (pure node:net + the broker codec, no TUI) reused
|
|
34
|
-
// as the in-test controller/observer so the G1–G9 gate exercises the REAL client
|
|
35
|
-
// too (§0 one-writer: a viewer holds ONLY a socket).
|
|
36
|
-
import { ViewSocketClient } from '../../clients/attach/view-socket.js';
|
|
37
|
-
import { CLIENT_READ_CAPS, FrameDecoder, encodeFrame, } from '../runtime/broker-protocol.js';
|
|
38
|
-
// crtrd.ts module const (not exported). The fresh-pi-boot grace window the daemon
|
|
39
|
-
// waits before grace-reviving a pi observed dead. Reference: crtrd.ts
|
|
40
|
-
// `REVIVE_GRACE_MS = 20_000`.
|
|
41
|
-
const REVIVE_GRACE_MS = 20_000;
|
|
42
33
|
let h;
|
|
43
34
|
let root;
|
|
44
35
|
before(async () => {
|
|
45
36
|
if (!hasTmux())
|
|
46
37
|
return;
|
|
47
38
|
h = await createHarness({ sessionPrefix: 'crtr-broker' });
|
|
48
|
-
// An active resident root: the spawn parent AND the live node a child can hold
|
|
49
|
-
// an active subscription to (so its natural stop classifies 'awaiting').
|
|
50
39
|
root = h.spawnRoot('broker-suite root');
|
|
51
40
|
});
|
|
52
41
|
after(async () => {
|
|
@@ -96,582 +85,3 @@ test('LIVE wake — in-broker watcher injects an inbox push with no exit/revive'
|
|
|
96
85
|
assert.equal(h.node(id).pi_pid, pid, 'pi_pid unchanged — the broker never exited');
|
|
97
86
|
assert.equal(isPidAlive(pid), true, 'the broker is still alive after the live wake');
|
|
98
87
|
});
|
|
99
|
-
// ===========================================================================
|
|
100
|
-
// Item 3 (DORMANT) — a natural stop while awaiting a live subscription idle-
|
|
101
|
-
// RELEASES the broker (it exits); the daemon's second pass revives it (RESUME)
|
|
102
|
-
// on the next unseen inbox entry.
|
|
103
|
-
// ===========================================================================
|
|
104
|
-
test('DORMANT wake — idle-release → broker exits → daemon pass-2 reviveNode(resume) on inbox', { skip: !hasTmux() }, async () => {
|
|
105
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — dormant wake');
|
|
106
|
-
const pid = h.node(id).pi_pid;
|
|
107
|
-
// Give the (terminal) child an ACTIVE subscription to the LIVE root, so the
|
|
108
|
-
// stop-guard classifies its natural stop as 'awaiting' (a dormant orchestrator
|
|
109
|
-
// awaiting a worker) → the stophook idle-releases it (paneless → no focus).
|
|
110
|
-
subscribe(id, root, true);
|
|
111
|
-
await h.stop(id, 'stop');
|
|
112
|
-
await h.waitForStatus(id, 'idle');
|
|
113
|
-
assert.equal(h.node(id).intent, 'idle-release', 'awaiting + unfocused → idle-release');
|
|
114
|
-
await h.waitFor(() => !isPidAlive(pid), { label: 'released broker process exited' });
|
|
115
|
-
// Daemon owns wake-on-message for a dormant (pi-dead) broker: an unseen inbox
|
|
116
|
-
// entry + a tick → pass 2 → reviveNode(resume:true) → a fresh broker boots.
|
|
117
|
-
appendInbox(id, { from: 'tester', tier: 'normal', kind: 'message', label: 'resume-me', data: { body: 'work after release' } });
|
|
118
|
-
await h.tick();
|
|
119
|
-
const boot2 = await h.awaitBoot(id, { minCount: 2 });
|
|
120
|
-
assert.equal(boot2.resuming, true, 'the dormant revive RESUMES the saved session');
|
|
121
|
-
const newPid = h.node(id).pi_pid;
|
|
122
|
-
assert.ok(newPid != null && isPidAlive(newPid), 'the revived broker pid is alive');
|
|
123
|
-
assert.notEqual(newPid, pid, 'the revived broker is a fresh process');
|
|
124
|
-
});
|
|
125
|
-
// ===========================================================================
|
|
126
|
-
// Item 4 + 6 — broker CRASH → grace-revive RESUME on the saved .jsonl; ONE-
|
|
127
|
-
// WRITER: the crashed pid is dead BEFORE the revive launches and never
|
|
128
|
-
// resurrects, and the revived pid is distinct (never two engine pids at once).
|
|
129
|
-
// ===========================================================================
|
|
130
|
-
test('CRASH → grace-revive RESUME; one-writer (old pid dead before new, never resurrects)', { skip: !hasTmux() }, async () => {
|
|
131
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — crash');
|
|
132
|
-
const oldPid = h.node(id).pi_pid;
|
|
133
|
-
assert.equal(isPidAlive(oldPid), true, 'broker alive before the crash');
|
|
134
|
-
assert.equal(h.node(id).intent ?? null, null, 'fresh broker has a null intent (not refresh/idle-release)');
|
|
135
|
-
const boots = h.bootCount(id);
|
|
136
|
-
// Kill the broker out from under the daemon (a crash). Use a fixed clock so the
|
|
137
|
-
// grace window is exercised deterministically.
|
|
138
|
-
process.kill(oldPid, 'SIGKILL');
|
|
139
|
-
await h.waitFor(() => !isPidAlive(oldPid), { label: 'crashed broker pid is dead' });
|
|
140
|
-
// ONE-WRITER: the old engine pid is dead BEFORE any revive can launch.
|
|
141
|
-
assert.equal(isPidAlive(oldPid), false, 'crashed pid dead before the daemon revives');
|
|
142
|
-
const NOW = 5_000_000;
|
|
143
|
-
await h.tick(NOW); // pid dead, intent null → handleBrokerLiveness → handleLiveWindow marks pending
|
|
144
|
-
assert.equal(h.bootCount(id), boots, 'inside the grace window → NOT yet revived');
|
|
145
|
-
await h.tick(NOW + REVIVE_GRACE_MS + 1); // grace elapsed → reviveNode(resume:true)
|
|
146
|
-
const boot2 = await h.awaitBoot(id, { minCount: boots + 1 });
|
|
147
|
-
assert.equal(boot2.resuming, true, 'grace-revive RESUMES the saved .jsonl');
|
|
148
|
-
const newPid = h.node(id).pi_pid;
|
|
149
|
-
assert.ok(newPid != null && isPidAlive(newPid), 'the revived broker pid is alive');
|
|
150
|
-
assert.notEqual(newPid, oldPid, 'the revived pid is distinct from the crashed one');
|
|
151
|
-
assert.equal(isPidAlive(oldPid), false, 'one-writer: the crashed pid never resurrected');
|
|
152
|
-
});
|
|
153
|
-
// ===========================================================================
|
|
154
|
-
// Item 5 — clean teardown: close → hostFor(meta).teardown → `shutdown` frame →
|
|
155
|
-
// broker disposes + exits, socket unlinked, status canceled, daemon leaves it.
|
|
156
|
-
// ===========================================================================
|
|
157
|
-
test('clean teardown — node close → broker shutdown frame → exit, socket unlinked, not revived', { skip: !hasTmux() }, async () => {
|
|
158
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — teardown');
|
|
159
|
-
const pid = h.node(id).pi_pid;
|
|
160
|
-
const sock = h.brokerSock(id);
|
|
161
|
-
await h.waitFor(() => existsSync(sock), { label: 'broker created its view.sock' });
|
|
162
|
-
// Close via the REAL CLI (resolves the node from --node; the subprocess carries
|
|
163
|
-
// no TMUX_PANE). closeNode marks canceled BEFORE teardown (crash-safe order),
|
|
164
|
-
// then hostFor(broker).teardown sends the `shutdown` frame.
|
|
165
|
-
const res = h.cli(root, ['node', 'close', '--node', id]);
|
|
166
|
-
assert.equal(res.code, 0, `node close should exit 0\n--stderr--\n${res.stderr}`);
|
|
167
|
-
await h.waitFor(() => !isPidAlive(pid), { label: 'broker process exited on shutdown frame' });
|
|
168
|
-
await h.waitFor(() => !existsSync(sock), { label: 'broker unlinked its socket on exit' });
|
|
169
|
-
assert.equal(h.status(id), 'canceled', 'the closed node is canceled');
|
|
170
|
-
// listNodes only surfaces active|idle, so a canceled broker is never supervised.
|
|
171
|
-
await h.tick();
|
|
172
|
-
assert.equal(h.status(id), 'canceled', 'still canceled after a daemon tick');
|
|
173
|
-
assert.equal(isPidAlive(pid), false, 'the broker stays dead — never revived');
|
|
174
|
-
});
|
|
175
|
-
// ===========================================================================
|
|
176
|
-
// C2 forward-progress (zero-viewer path) — an unattended blocking dialog
|
|
177
|
-
// resolves to its default (false) IMMEDIATELY: with no controller connected the
|
|
178
|
-
// broker's REAL makeBrokerUiContext falls back to noOp resolution, so the engine
|
|
179
|
-
// never deadlocks AND never waits on a per-dialog timeout (the design §5.4
|
|
180
|
-
// timeout premise is false). Red/green: the OLD broker armed a setTimeout and
|
|
181
|
-
// resolved only after `timeout` ms; the fixed broker resolves in ~0ms. (Supports
|
|
182
|
-
// acceptance item 7: the existing suite stays green; this only ADDS coverage.)
|
|
183
|
-
// ===========================================================================
|
|
184
|
-
test('C2 — unattended dialog resolves to its default IMMEDIATELY (noOp), never waits on a timeout', { skip: !hasTmux() }, async () => {
|
|
185
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — dialog');
|
|
186
|
-
const pid = h.node(id).pi_pid;
|
|
187
|
-
// Drive the fake engine to call uiContext.confirm(..., { timeout: 5000 }) with
|
|
188
|
-
// NO controller. C2 fix: makeBrokerUiContext resolves the default (false) AT
|
|
189
|
-
// ONCE — it does NOT arm/await the timeout. A generous < 2000ms bound is still a
|
|
190
|
-
// hard fail against the old ~5000ms timeout-wait while staying robust on slow CI.
|
|
191
|
-
h.fakeCmd(id, { cmd: 'dialog', timeout: 5000 });
|
|
192
|
-
const results = await h.waitFor(() => {
|
|
193
|
-
const r = h.dialogResults(id);
|
|
194
|
-
return r.length > 0 ? r : null;
|
|
195
|
-
}, { timeoutMs: 15_000, label: 'unattended dialog resolved' });
|
|
196
|
-
assert.equal(results[0].resolved, false, 'unattended confirm resolves to its default (false / deny)');
|
|
197
|
-
assert.ok(results[0].ms < 2000, `C2: resolved IMMEDIATELY (noOp), not after the 5000ms timeout — got ${results[0].ms}ms`);
|
|
198
|
-
assert.equal(isPidAlive(pid), true, 'the broker made forward progress (still alive, did not hang or exit)');
|
|
199
|
-
});
|
|
200
|
-
// ===========================================================================
|
|
201
|
-
// M-1 regression — a broker that DIES before session_start must surface a boot
|
|
202
|
-
// failure, not strand the node forever. Before the fix handleBrokerLiveness read
|
|
203
|
-
// pid==null UNCONDITIONALLY as "still booting", so a broker that threw before
|
|
204
|
-
// recording a pid/session left the node 'active' with no engine FOREVER and the
|
|
205
|
-
// parent waited on a dead child. The daemon must instead crash it past the boot
|
|
206
|
-
// grace + push a boot-failure up the spine. (Also proves M-2: the broker's fatal
|
|
207
|
-
// diagnostic reaches job/broker.log instead of /dev/null.)
|
|
208
|
-
// ===========================================================================
|
|
209
|
-
test('boot failure — a broker that dies before session_start is reaped, not stranded (review M-1/M-2)', { skip: !hasTmux() }, async () => {
|
|
210
|
-
// A headless child whose broker throws before session_start (no pid, no session
|
|
211
|
-
// EVER recorded). spawnHeadlessChildNoBoot does not awaitBoot (there is none).
|
|
212
|
-
const id = await h.spawnHeadlessChildNoBoot(root, `headless worker — boot fail ${FAIL_BEFORE_SESSION_START}`);
|
|
213
|
-
// M-2: the broker logs its fatal stack to job/broker.log and exits(1). Without
|
|
214
|
-
// the stdio redirect this diagnostic would be lost to /dev/null.
|
|
215
|
-
const brokerLog = join(h.home, 'nodes', id, 'job', 'broker.log');
|
|
216
|
-
await h.waitFor(() => existsSync(brokerLog) && /simulated pre-session_start boot failure/.test(readFileSync(brokerLog, 'utf8')), { label: 'broker logged its pre-session_start failure to job/broker.log' });
|
|
217
|
-
// The node is stuck 'active' with a null pid and a null session — the strand.
|
|
218
|
-
const n0 = h.node(id);
|
|
219
|
-
assert.equal(n0.pi_pid ?? null, null, 'no broker pid was ever recorded');
|
|
220
|
-
assert.equal(n0.pi_session_id ?? null, null, 'no session was ever recorded');
|
|
221
|
-
// Inside the boot grace the daemon LEAVES it (could be the sub-second boot gap).
|
|
222
|
-
const NOW = 9_000_000;
|
|
223
|
-
await h.tick(NOW);
|
|
224
|
-
assert.equal(h.status(id), 'active', 'inside the boot grace → daemon leaves it (boot gap)');
|
|
225
|
-
// Past the boot grace with STILL no pid and no session → crash + boot failure.
|
|
226
|
-
await h.tick(NOW + REVIVE_GRACE_MS + 1);
|
|
227
|
-
await h.waitForStatus(id, 'dead');
|
|
228
|
-
assert.equal(h.status(id), 'dead', 'a never-booted broker is reaped, not stranded active forever');
|
|
229
|
-
// …and the parent (an active subscriber of the child) was told up the spine.
|
|
230
|
-
const note = h.inbox(root).find((e) => /never started/.test(e.label ?? ''));
|
|
231
|
-
assert.ok(note !== undefined, 'the parent received a boot-failure notice up the spine');
|
|
232
|
-
});
|
|
233
|
-
// ===========================================================================
|
|
234
|
-
// T8 — the `crtr attach` acceptance gate (G1–G9). Each test below drives the REAL
|
|
235
|
-
// detached broker process + REAL view.sock with the enriched fake engine, using
|
|
236
|
-
// the PRODUCTION ViewSocketClient as the controller/observer (raw node:net only
|
|
237
|
-
// where the client lifecycle is awkward — G7 oversized line, G8 stalled viewer).
|
|
238
|
-
// The engine is hosted IN-PROCESS by the broker, so engine pid == broker pid ==
|
|
239
|
-
// node.pi_pid == boot.pid; "engine pid unchanged" == broker pid unchanged + no new
|
|
240
|
-
// boot. Each test's lead comment names its gate # and the failure mode it guards.
|
|
241
|
-
// ===========================================================================
|
|
242
|
-
const liveClients = [];
|
|
243
|
-
afterEach(() => {
|
|
244
|
-
for (const c of liveClients.splice(0)) {
|
|
245
|
-
try {
|
|
246
|
-
c.close();
|
|
247
|
-
}
|
|
248
|
-
catch {
|
|
249
|
-
/* already gone */
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
});
|
|
253
|
-
const delay = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
254
|
-
const tok = (s) => `${s}-${Math.random().toString(36).slice(2, 8)}`;
|
|
255
|
-
const frameHas = (f, token) => JSON.stringify(f).includes(token);
|
|
256
|
-
const brokerPid = (id) => h.node(id).pi_pid;
|
|
257
|
-
function brokerLogText(id) {
|
|
258
|
-
try {
|
|
259
|
-
return readFileSync(join(h.home, 'nodes', id, 'job', 'broker.log'), 'utf8');
|
|
260
|
-
}
|
|
261
|
-
catch {
|
|
262
|
-
return '';
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
// Connect a ViewSocketClient to a node's running broker, hello, await welcome.
|
|
266
|
-
// awaitBoot returns once the boot proof is written — which is BEFORE the broker's
|
|
267
|
-
// server.listen() binds view.sock — so a fresh attach can momentarily race the
|
|
268
|
-
// bind; retry the connect on BrokerUnavailable until it is listening.
|
|
269
|
-
async function attach(id, role, clientId) {
|
|
270
|
-
await h.waitFor(() => existsSync(h.brokerSock(id)), { label: `view.sock for ${id}`, timeoutMs: 20_000 });
|
|
271
|
-
const frames = [];
|
|
272
|
-
let client;
|
|
273
|
-
for (let attempt = 0;; attempt++) {
|
|
274
|
-
client = new ViewSocketClient(id);
|
|
275
|
-
client.on('frame', (f) => frames.push(f));
|
|
276
|
-
try {
|
|
277
|
-
await new Promise((resolve, reject) => {
|
|
278
|
-
client.once('connect', resolve);
|
|
279
|
-
client.once('error', reject);
|
|
280
|
-
client.connect();
|
|
281
|
-
});
|
|
282
|
-
break;
|
|
283
|
-
}
|
|
284
|
-
catch (err) {
|
|
285
|
-
client.close();
|
|
286
|
-
if (attempt >= 30)
|
|
287
|
-
throw err;
|
|
288
|
-
await delay(100);
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
client.on('error', () => { }); // post-connect error sink (never throw uncaught)
|
|
292
|
-
// Register cleanup the instant the socket is connected — BEFORE the hello/welcome
|
|
293
|
-
// round-trip — so a welcome timeout cannot leak a connected socket past the test.
|
|
294
|
-
liveClients.push({ close: () => client.close() });
|
|
295
|
-
const waitFrame = (pred, label, timeoutMs = 15_000) => h.waitFor(() => frames.find(pred) ?? null, { label, timeoutMs });
|
|
296
|
-
client.send({ type: 'hello', role, client_id: clientId });
|
|
297
|
-
let welcome;
|
|
298
|
-
try {
|
|
299
|
-
welcome = (await waitFrame((f) => f.type === 'welcome', `welcome for ${clientId}`));
|
|
300
|
-
}
|
|
301
|
-
catch (err) {
|
|
302
|
-
client.close();
|
|
303
|
-
throw err;
|
|
304
|
-
}
|
|
305
|
-
return { client, frames, welcome, send: (f) => client.send(f), waitFrame, close: () => client.close() };
|
|
306
|
-
}
|
|
307
|
-
// Attach (as `role`) and retry until the welcome satisfies `pred` — used where the
|
|
308
|
-
// observable lands a beat after a prior action (G3 snapshot accrual, G5b control
|
|
309
|
-
// handoff after a controller detaches). Deterministic: it polls an observable.
|
|
310
|
-
async function attachUntil(id, role, clientId, pred, label) {
|
|
311
|
-
for (let attempt = 0;; attempt++) {
|
|
312
|
-
const a = await attach(id, role, `${clientId}-${attempt}`);
|
|
313
|
-
if (pred(a))
|
|
314
|
-
return a;
|
|
315
|
-
a.close();
|
|
316
|
-
if (attempt >= 40)
|
|
317
|
-
throw new Error(`attachUntil timed out: ${label}`);
|
|
318
|
-
await delay(150);
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
// A raw node:net peer. read:true decodes incoming frames; read:false leaves the
|
|
322
|
-
// socket PAUSED (no 'data' listener, never resumed) so it never drains — the
|
|
323
|
-
// stalled viewer (G8) whose backlog the broker must shed at the HWM.
|
|
324
|
-
async function connectRaw(id, opts) {
|
|
325
|
-
await h.waitFor(() => existsSync(h.brokerSock(id)), { label: `view.sock for ${id}`, timeoutMs: 20_000 });
|
|
326
|
-
const frames = [];
|
|
327
|
-
const decoder = new FrameDecoder(CLIENT_READ_CAPS);
|
|
328
|
-
let isClosed = false;
|
|
329
|
-
const socket = await new Promise((resolve, reject) => {
|
|
330
|
-
const s = createConnection(h.brokerSock(id));
|
|
331
|
-
s.once('connect', () => resolve(s));
|
|
332
|
-
s.once('error', reject);
|
|
333
|
-
});
|
|
334
|
-
socket.on('close', () => {
|
|
335
|
-
isClosed = true;
|
|
336
|
-
});
|
|
337
|
-
socket.on('error', () => {
|
|
338
|
-
/* close follows */
|
|
339
|
-
});
|
|
340
|
-
if (opts.read) {
|
|
341
|
-
socket.on('data', (chunk) => {
|
|
342
|
-
try {
|
|
343
|
-
for (const f of decoder.push(chunk))
|
|
344
|
-
frames.push(f);
|
|
345
|
-
}
|
|
346
|
-
catch {
|
|
347
|
-
/* a client-side overflow is irrelevant here */
|
|
348
|
-
}
|
|
349
|
-
});
|
|
350
|
-
}
|
|
351
|
-
const rc = {
|
|
352
|
-
socket,
|
|
353
|
-
frames,
|
|
354
|
-
closed: () => isClosed,
|
|
355
|
-
send: (f) => {
|
|
356
|
-
try {
|
|
357
|
-
socket.write(encodeFrame(f));
|
|
358
|
-
}
|
|
359
|
-
catch {
|
|
360
|
-
/* dead */
|
|
361
|
-
}
|
|
362
|
-
},
|
|
363
|
-
writeRaw: (d) => {
|
|
364
|
-
try {
|
|
365
|
-
socket.write(d);
|
|
366
|
-
}
|
|
367
|
-
catch {
|
|
368
|
-
/* dead */
|
|
369
|
-
}
|
|
370
|
-
},
|
|
371
|
-
waitClosed: (label, timeoutMs = 15_000) => h.waitFor(() => (isClosed ? true : null), { label, timeoutMs }).then(() => undefined),
|
|
372
|
-
close: () => {
|
|
373
|
-
try {
|
|
374
|
-
socket.destroy();
|
|
375
|
-
}
|
|
376
|
-
catch {
|
|
377
|
-
/* ignore */
|
|
378
|
-
}
|
|
379
|
-
},
|
|
380
|
-
};
|
|
381
|
-
liveClients.push(rc);
|
|
382
|
-
return rc;
|
|
383
|
-
}
|
|
384
|
-
/** lsof the holders of `path`, or null when lsof is unavailable (skip the fd
|
|
385
|
-
* check). Exit-non-zero with empty stdout means "no holders". */
|
|
386
|
-
function lsofHolders(path) {
|
|
387
|
-
if (spawnSync('which', ['lsof'], { stdio: 'ignore' }).status !== 0)
|
|
388
|
-
return null;
|
|
389
|
-
const out = (spawnSync('lsof', ['-t', '--', path], { encoding: 'utf8' }).stdout ?? '').trim();
|
|
390
|
-
if (out === '')
|
|
391
|
-
return [];
|
|
392
|
-
return out
|
|
393
|
-
.split('\n')
|
|
394
|
-
.map((l) => Number(l.trim()))
|
|
395
|
-
.filter((n) => Number.isFinite(n));
|
|
396
|
-
}
|
|
397
|
-
// ---------------------------------------------------------------------------
|
|
398
|
-
// G1 — controller drive + live relay. Guards: a controller's `prompt` runs the
|
|
399
|
-
// engine AND the streaming AgentSessionEvents are fanned out to the client
|
|
400
|
-
// VERBATIM. Failure mode: a broken relay/fan-out (viewer sees nothing).
|
|
401
|
-
// ---------------------------------------------------------------------------
|
|
402
|
-
test('G1 — controller prompt runs the engine and the streamed AgentSessionEvents relay to the client', { skip: !hasTmux() }, async () => {
|
|
403
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G1');
|
|
404
|
-
const c = await attach(id, 'controller', 'g1-ctrl');
|
|
405
|
-
assert.equal(c.welcome.role, 'controller', 'first controller is admitted as controller');
|
|
406
|
-
const token = tok('G1-PROMPT');
|
|
407
|
-
c.send({ type: 'prompt', text: token });
|
|
408
|
-
// The full streaming turn relays: agent_end carrying the token proves the turn
|
|
409
|
-
// ran AND its terminal frame reached the client; the intermediate types prove
|
|
410
|
-
// the stream (not just a final blob) was fanned out.
|
|
411
|
-
await c.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G1 agent_end carrying the prompt token');
|
|
412
|
-
for (const t of ['message_start', 'message_update', 'tool_execution_start', 'tool_execution_end', 'message_end', 'turn_end', 'agent_start', 'turn_start']) {
|
|
413
|
-
assert.ok(c.frames.some((f) => f.type === t), `G1: client received a relayed ${t} frame`);
|
|
414
|
-
}
|
|
415
|
-
assert.ok(c.frames.some((f) => f.type === 'message_start' && frameHas(f, token)), 'G1: a relayed message carries the prompt token');
|
|
416
|
-
});
|
|
417
|
-
// ---------------------------------------------------------------------------
|
|
418
|
-
// G1b — message_update coalescing preserves ordering (regression for the F2
|
|
419
|
-
// attach typing-lag fix, 2026-06-09). The broker holds the latest message_update
|
|
420
|
-
// on a ~75ms timer; any OTHER event must flush it FIRST, so a viewer can never
|
|
421
|
-
// observe message_update AFTER its message_end (which would resurrect stale
|
|
422
|
-
// streaming text over the final message), and the LAST coalesced update must not
|
|
423
|
-
// be silently dropped at end-of-turn. Failure modes: a flush ordered after the
|
|
424
|
-
// non-update event, or a pending update discarded when message_end wins the race.
|
|
425
|
-
// ---------------------------------------------------------------------------
|
|
426
|
-
test('G1b — coalesced message_update never arrives after message_end; updates still relayed', { skip: !hasTmux() }, async () => {
|
|
427
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G1b');
|
|
428
|
-
const c = await attach(id, 'controller', 'g1b-ctrl');
|
|
429
|
-
const token = tok('G1B');
|
|
430
|
-
// 12 updates at the fake engine's setImmediate pace — far faster than the 75ms
|
|
431
|
-
// coalesce window, so coalescing genuinely engages (fewer relayed than emitted).
|
|
432
|
-
h.fakeCmd(id, { cmd: 'stream', text: token, updates: 12 });
|
|
433
|
-
await c.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G1b turn relayed');
|
|
434
|
-
const types = c.frames.map((f) => f.type);
|
|
435
|
-
const lastUpdate = types.lastIndexOf('message_update');
|
|
436
|
-
const msgEnd = types.indexOf('message_end');
|
|
437
|
-
assert.ok(lastUpdate >= 0, 'G1b: at least one message_update relayed (coalescing must not starve updates)');
|
|
438
|
-
assert.ok(msgEnd >= 0, 'G1b: message_end relayed');
|
|
439
|
-
assert.ok(lastUpdate < msgEnd, `G1b: a message_update arrived AFTER message_end (update@${lastUpdate} vs end@${msgEnd}) — the coalescer flushed out of order`);
|
|
440
|
-
// tool/turn/agent frames must also never precede a stale held update.
|
|
441
|
-
for (const t of ['tool_execution_start', 'turn_end', 'agent_end']) {
|
|
442
|
-
const i = types.indexOf(t);
|
|
443
|
-
assert.ok(i < 0 || lastUpdate < i, `G1b: message_update relayed after ${t}`);
|
|
444
|
-
}
|
|
445
|
-
});
|
|
446
|
-
// ---------------------------------------------------------------------------
|
|
447
|
-
// G2 — detach leaves the engine running. Guards: `bye`/close drops ONE listener,
|
|
448
|
-
// never the engine. Failure mode: a detach that disposes the broker/engine.
|
|
449
|
-
// ---------------------------------------------------------------------------
|
|
450
|
-
test('G2 — detach (bye+close) leaves the broker alive + the engine still emitting; engine pid unchanged', { skip: !hasTmux() }, async () => {
|
|
451
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G2');
|
|
452
|
-
const pid = brokerPid(id);
|
|
453
|
-
const boots = h.bootCount(id);
|
|
454
|
-
const c = await attach(id, 'controller', 'g2-ctrl');
|
|
455
|
-
const p = tok('G2-PROMPT');
|
|
456
|
-
c.send({ type: 'prompt', text: p });
|
|
457
|
-
await c.waitFrame((f) => f.type === 'agent_end' && frameHas(f, p), 'G2 turn relayed before detach');
|
|
458
|
-
c.send({ type: 'bye' });
|
|
459
|
-
c.close();
|
|
460
|
-
await delay(300); // let the broker process the 'close'
|
|
461
|
-
assert.equal(isPidAlive(pid), true, 'G2: broker still alive after detach');
|
|
462
|
-
assert.equal(brokerPid(id), pid, 'G2: engine (== broker) pid UNCHANGED across detach');
|
|
463
|
-
assert.equal(h.bootCount(id), boots, 'G2: no reboot — no second engine spawned');
|
|
464
|
-
// Still emitting: drive a turn with no client, then a fresh observer's snapshot
|
|
465
|
-
// carries it (the engine kept running and processing after the detach).
|
|
466
|
-
const after = tok('G2-AFTER-DETACH');
|
|
467
|
-
h.fakeCmd(id, { cmd: 'stream', text: after });
|
|
468
|
-
const o = await attachUntil(id, 'observer', 'g2-obs', (a) => JSON.stringify(a.welcome.snapshot.messages).includes(after), 'G2 post-detach message in snapshot');
|
|
469
|
-
assert.ok(o.welcome.snapshot.messages.length > 0, 'G2: the engine produced a message after the detach');
|
|
470
|
-
assert.equal(isPidAlive(pid), true, 'G2: broker still alive after the post-detach turn');
|
|
471
|
-
});
|
|
472
|
-
// ---------------------------------------------------------------------------
|
|
473
|
-
// G3 — catch-up snapshot. Guards: messages produced with NO viewer attached are
|
|
474
|
-
// in welcome.snapshot on reattach, and live events resume. Failure mode: a viewer
|
|
475
|
-
// that reattaches blind to history / gets no further events.
|
|
476
|
-
// ---------------------------------------------------------------------------
|
|
477
|
-
test('G3 — messages produced while detached appear in welcome.snapshot on reattach; live events resume', { skip: !hasTmux() }, async () => {
|
|
478
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G3');
|
|
479
|
-
const past = tok('G3-DETACHED');
|
|
480
|
-
h.fakeCmd(id, { cmd: 'stream', text: past }); // produced with zero viewers attached
|
|
481
|
-
const o = await attachUntil(id, 'observer', 'g3-obs', (a) => JSON.stringify(a.welcome.snapshot.messages).includes(past), 'G3 detached message in snapshot');
|
|
482
|
-
assert.ok(JSON.stringify(o.welcome.snapshot.messages).includes(past), 'G3: welcome.snapshot.messages contains the message produced while detached');
|
|
483
|
-
const live = tok('G3-LIVE');
|
|
484
|
-
h.fakeCmd(id, { cmd: 'stream', text: live });
|
|
485
|
-
await o.waitFrame((f) => f.type === 'agent_end' && frameHas(f, live), 'G3 live events resume after reattach');
|
|
486
|
-
});
|
|
487
|
-
// ---------------------------------------------------------------------------
|
|
488
|
-
// G4 — arbitration + observer read. Guards: 2nd client is admitted observer, an
|
|
489
|
-
// observer prompt is rejected not_controller, BOTH clients receive the relay.
|
|
490
|
-
// Failure mode: two controllers, or an observer driving the engine, or fan-out
|
|
491
|
-
// that misses a viewer.
|
|
492
|
-
// ---------------------------------------------------------------------------
|
|
493
|
-
test('G4 — second client is observer; observer prompt → error{not_controller}; both receive the stream', { skip: !hasTmux() }, async () => {
|
|
494
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G4');
|
|
495
|
-
const c1 = await attach(id, 'controller', 'g4-ctrl');
|
|
496
|
-
assert.equal(c1.welcome.role, 'controller', 'first client holds control');
|
|
497
|
-
const c2 = await attach(id, 'controller', 'g4-second'); // requests control; held → observer
|
|
498
|
-
assert.equal(c2.welcome.role, 'observer', 'second client is admitted read-only observer (first-attach-wins)');
|
|
499
|
-
c2.send({ type: 'prompt', text: 'observer must not drive' });
|
|
500
|
-
const err = await c2.waitFrame((f) => f.type === 'error', 'G4 observer prompt rejected');
|
|
501
|
-
assert.equal(err.code, 'not_controller', 'G4: observer prompt → error{not_controller}');
|
|
502
|
-
const token = tok('G4-BROADCAST');
|
|
503
|
-
c1.send({ type: 'prompt', text: token });
|
|
504
|
-
await c1.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G4 controller received the stream');
|
|
505
|
-
await c2.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G4 observer ALSO received the stream');
|
|
506
|
-
});
|
|
507
|
-
// ---------------------------------------------------------------------------
|
|
508
|
-
// G5 — dialog forward + answer. Guards: a blocking dialog reaches the controller
|
|
509
|
-
// as extension_ui_request and the controller's extension_ui_response unblocks the
|
|
510
|
-
// engine with ITS answer (not the default). Failure mode: a dialog the controller
|
|
511
|
-
// can't see/answer (silent deadlock).
|
|
512
|
-
// ---------------------------------------------------------------------------
|
|
513
|
-
test('G5 — controller receives an extension_ui_request, answers it, and the engine proceeds with that answer', { skip: !hasTmux() }, async () => {
|
|
514
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G5a');
|
|
515
|
-
const c = await attach(id, 'controller', 'g5-ctrl');
|
|
516
|
-
h.fakeCmd(id, { cmd: 'dialog', timeout: 20_000 }); // generous: the controller answers first
|
|
517
|
-
const req = await c.waitFrame((f) => f.type === 'extension_ui_request', 'G5 dialog forwarded to controller');
|
|
518
|
-
assert.equal(req.method, 'confirm', 'G5: the forwarded dialog is the confirm() the engine raised');
|
|
519
|
-
const reqId = req.id;
|
|
520
|
-
c.send({ type: 'extension_ui_response', id: reqId, confirmed: true });
|
|
521
|
-
const results = await h.waitFor(() => {
|
|
522
|
-
const r = h.dialogResults(id);
|
|
523
|
-
return r.length > 0 ? r : null;
|
|
524
|
-
}, { label: 'G5 dialog resolved by the controller', timeoutMs: 15_000 });
|
|
525
|
-
assert.equal(results[0].resolved, true, 'G5: the engine proceeded with the controller answer (true), not the default (false)');
|
|
526
|
-
});
|
|
527
|
-
// ---------------------------------------------------------------------------
|
|
528
|
-
// G5 (mid-dialog attach) — Guards: a dialog raised under a prior controller stays
|
|
529
|
-
// pending across that controller's detach (M2) and is delivered to whoever takes
|
|
530
|
-
// control next via welcome.pending_dialog. Failure mode: a pending dialog lost on
|
|
531
|
-
// controller handoff.
|
|
532
|
-
// ---------------------------------------------------------------------------
|
|
533
|
-
test('G5 — a controller attaching MID-dialog receives the pending dialog via welcome.pending_dialog', { skip: !hasTmux() }, async () => {
|
|
534
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G5b');
|
|
535
|
-
const a = await attach(id, 'controller', 'g5b-A');
|
|
536
|
-
h.fakeCmd(id, { cmd: 'dialog', timeout: 30_000 }); // stays pending long enough for the handoff
|
|
537
|
-
const reqA = await a.waitFrame((f) => f.type === 'extension_ui_request', 'G5b dialog forwarded to controller A');
|
|
538
|
-
const reqId = reqA.id;
|
|
539
|
-
a.send({ type: 'bye' });
|
|
540
|
-
a.close(); // M2: detach frees control but does NOT cancel the pending dialog
|
|
541
|
-
// Controller B takes control (retry covers the close→controllerId=null beat) and
|
|
542
|
-
// its welcome carries the still-pending dialog.
|
|
543
|
-
const b = await attachUntil(id, 'controller', 'g5b-B', (x) => x.welcome.role === 'controller' && x.welcome.pending_dialog != null, 'G5b controller B takes control with the pending dialog');
|
|
544
|
-
assert.equal(b.welcome.pending_dialog.id, reqId, 'G5b: welcome.pending_dialog is the same dialog raised under A');
|
|
545
|
-
assert.equal(b.welcome.pending_dialog.method, 'confirm', 'G5b: the pending dialog is the confirm()');
|
|
546
|
-
b.send({ type: 'extension_ui_response', id: reqId, confirmed: true });
|
|
547
|
-
const results = await h.waitFor(() => {
|
|
548
|
-
const r = h.dialogResults(id);
|
|
549
|
-
return r.length > 0 ? r : null;
|
|
550
|
-
}, { label: 'G5b dialog resolved by controller B', timeoutMs: 15_000 });
|
|
551
|
-
assert.equal(results[0].resolved, true, 'G5b: controller B answered the handed-off dialog and the engine proceeded');
|
|
552
|
-
});
|
|
553
|
-
// ---------------------------------------------------------------------------
|
|
554
|
-
// G6 — anti-deadlock. (a) a zero-viewer dialog resolves to its default AT ONCE
|
|
555
|
-
// (noOp). (b) an ATTENDED dialog the controller never answers resolves on a SHORT
|
|
556
|
-
// per-dialog broker timeout. Guards: the engine never hangs on a dialog with no
|
|
557
|
-
// answerer. Failure mode: a forever-blocked turn.
|
|
558
|
-
// ---------------------------------------------------------------------------
|
|
559
|
-
test('G6 — zero-viewer dialog resolves immediately; an unanswered attended dialog resolves on the broker timeout', { skip: !hasTmux() }, async () => {
|
|
560
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G6');
|
|
561
|
-
const pid = brokerPid(id);
|
|
562
|
-
// (a) zero viewers → immediate noOp default (NOT the 5000ms timeout).
|
|
563
|
-
h.fakeCmd(id, { cmd: 'dialog', timeout: 5000 });
|
|
564
|
-
const r1 = await h.waitFor(() => (h.dialogResults(id).length >= 1 ? h.dialogResults(id) : null), { label: 'G6a zero-viewer dialog resolved', timeoutMs: 15_000 });
|
|
565
|
-
assert.equal(r1[0].resolved, false, 'G6a: zero-viewer dialog resolves to the default (deny)');
|
|
566
|
-
assert.ok(r1[0].ms < 2000, `G6a: resolved immediately (noOp), not after the 5000ms timeout — got ${r1[0].ms}ms`);
|
|
567
|
-
// (b) controller attached but silent → the broker resolves on the SHORT explicit
|
|
568
|
-
// per-dialog timeout (800ms), never the 120s default.
|
|
569
|
-
const c = await attach(id, 'controller', 'g6-ctrl');
|
|
570
|
-
h.fakeCmd(id, { cmd: 'dialog', timeout: 800 });
|
|
571
|
-
await c.waitFrame((f) => f.type === 'extension_ui_request', 'G6b dialog forwarded to controller'); // received, deliberately NOT answered
|
|
572
|
-
const r2 = await h.waitFor(() => (h.dialogResults(id).length >= 2 ? h.dialogResults(id) : null), { label: 'G6b attended dialog resolved on timeout', timeoutMs: 15_000 });
|
|
573
|
-
assert.equal(r2[1].resolved, false, 'G6b: an unanswered attended dialog resolves to the default (deny)');
|
|
574
|
-
assert.ok(r2[1].ms >= 600 && r2[1].ms < 5000, `G6b: resolved on the ~800ms per-dialog timeout, not instantly and not the 120s default — got ${r2[1].ms}ms`);
|
|
575
|
-
assert.equal(isPidAlive(pid), true, 'G6: the engine made forward progress on both dialogs (still alive)');
|
|
576
|
-
});
|
|
577
|
-
// ---------------------------------------------------------------------------
|
|
578
|
-
// G7 — decoder overflow (guards C5 OOM). A client line over BROKER_READ_CAPS is
|
|
579
|
-
// cap-and-dropped; the broker survives and other clients are unaffected. Failure
|
|
580
|
-
// mode: an unbounded decoder buffer growing the broker to OOM.
|
|
581
|
-
// ---------------------------------------------------------------------------
|
|
582
|
-
test('G7 — an oversized client line is dropped (frame_overflow), the broker survives, other clients unaffected', { skip: !hasTmux() }, async () => {
|
|
583
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G7');
|
|
584
|
-
const pid = brokerPid(id);
|
|
585
|
-
const boots = h.bootCount(id);
|
|
586
|
-
const survivor = await attach(id, 'observer', 'g7-survivor');
|
|
587
|
-
// 26 MiB with NO newline > BROKER_READ_CAPS.maxLineBytes (24 MiB) → the bounded
|
|
588
|
-
// FrameDecoder throws FrameOverflowError; the broker drops the peer.
|
|
589
|
-
const bad = await connectRaw(id, { read: true });
|
|
590
|
-
bad.writeRaw(Buffer.alloc(26 * 1024 * 1024, 0x78));
|
|
591
|
-
await bad.waitClosed('G7 oversized peer dropped by the broker');
|
|
592
|
-
await h.waitFor(() => /frame overflow/.test(brokerLogText(id)) || null, { label: 'G7 broker logged the frame-overflow drop' });
|
|
593
|
-
// The broker survives and the other client is unaffected: a fresh turn relays.
|
|
594
|
-
const token = tok('G7-AFTER');
|
|
595
|
-
h.fakeCmd(id, { cmd: 'stream', text: token });
|
|
596
|
-
await survivor.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G7 survivor still receives live frames');
|
|
597
|
-
assert.equal(isPidAlive(pid), true, 'G7: the broker survived the overflow');
|
|
598
|
-
assert.equal(brokerPid(id), pid, 'G7: broker pid unchanged');
|
|
599
|
-
assert.equal(h.bootCount(id), boots, 'G7: no reboot');
|
|
600
|
-
});
|
|
601
|
-
// ---------------------------------------------------------------------------
|
|
602
|
-
// G8 — backpressure leak (guards M1). A stalled (non-reading) viewer is shed at
|
|
603
|
-
// the HWM (32 MiB byte cap) while the broker + other viewers are unaffected.
|
|
604
|
-
// Failure mode: an indefinitely-growing per-viewer queue (broker OOM) or a slow
|
|
605
|
-
// viewer back-pressuring the shared engine.
|
|
606
|
-
// ---------------------------------------------------------------------------
|
|
607
|
-
test('G8 — a stalled viewer is dropped at the backpressure HWM; the broker + fast viewers are unaffected', { skip: !hasTmux() }, async () => {
|
|
608
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G8');
|
|
609
|
-
const pid = brokerPid(id);
|
|
610
|
-
const fast = await attach(id, 'observer', 'g8-fast'); // reads normally
|
|
611
|
-
// A helloed viewer that NEVER reads (paused socket). It must be in the broadcast
|
|
612
|
-
// set before the flood; hello, then a short beat for the broker to process it.
|
|
613
|
-
const stalled = await connectRaw(id, { read: false });
|
|
614
|
-
stalled.send({ type: 'hello', role: 'observer', client_id: 'g8-stalled' });
|
|
615
|
-
await delay(400);
|
|
616
|
-
// A fast event stream: ~60 MiB across 240 message_update frames. The stalled
|
|
617
|
-
// viewer's per-viewer backlog crosses the 32 MiB byte cap → dropped; the fast
|
|
618
|
-
// viewer drains between frames (per-update yield) and survives.
|
|
619
|
-
//
|
|
620
|
-
// The DROP SIGNAL is the broker's own log line, NOT the stalled socket's 'close':
|
|
621
|
-
// a PAUSED node socket does not surface the peer FIN/close until it is resumed,
|
|
622
|
-
// so the broker-side `backpressure high-water mark exceeded` line is the
|
|
623
|
-
// deterministic, race-free proof that the slow viewer was shed at the HWM.
|
|
624
|
-
const token = tok('G8-FLOOD');
|
|
625
|
-
h.fakeCmd(id, { cmd: 'stream', text: token, updates: 240, padBytes: 256 * 1024, tool: false });
|
|
626
|
-
await h.waitFor(() => /backpressure high-water mark exceeded/.test(brokerLogText(id)) || null, { label: 'G8 broker shed the stalled viewer at the HWM', timeoutMs: 30_000 });
|
|
627
|
-
// The fast viewer drained the WHOLE stream (agent_end is the terminal frame) —
|
|
628
|
-
// proving only the stalled viewer was shed, while the fast one was unaffected.
|
|
629
|
-
await fast.waitFrame((f) => f.type === 'agent_end' && frameHas(f, token), 'G8 fast viewer received the whole stream', 30_000);
|
|
630
|
-
assert.equal(isPidAlive(pid), true, 'G8: the broker survived the slow-viewer flood');
|
|
631
|
-
assert.equal(brokerPid(id), pid, 'G8: broker pid unchanged (the engine was not back-pressured into a restart)');
|
|
632
|
-
});
|
|
633
|
-
// ---------------------------------------------------------------------------
|
|
634
|
-
// G9 — the load-bearing ONE-WRITER assertion. Across attach→detach→reattach the
|
|
635
|
-
// broker pid AND engine pid (the same, in-process) are UNCHANGED and no second
|
|
636
|
-
// engine is ever spawned; the viewer holds ONLY the socket (never the .jsonl).
|
|
637
|
-
// Failure mode: a viewer that spawns/forks a second engine or opens the session
|
|
638
|
-
// .jsonl — the corruption the headless design exists to prevent.
|
|
639
|
-
// ---------------------------------------------------------------------------
|
|
640
|
-
test('G9 — one-writer: broker/engine pid stable across attach→detach→reattach; no second engine; viewer holds only the socket', { skip: !hasTmux() }, async () => {
|
|
641
|
-
const id = await h.spawnHeadlessChild(root, 'headless worker — G9');
|
|
642
|
-
const pid0 = brokerPid(id);
|
|
643
|
-
const boots0 = h.bootCount(id);
|
|
644
|
-
const c1 = await attach(id, 'controller', 'g9-a');
|
|
645
|
-
const a = tok('G9-A');
|
|
646
|
-
c1.send({ type: 'prompt', text: a });
|
|
647
|
-
await c1.waitFrame((f) => f.type === 'agent_end' && frameHas(f, a), 'G9 first turn relayed');
|
|
648
|
-
c1.send({ type: 'bye' });
|
|
649
|
-
c1.close();
|
|
650
|
-
await delay(300);
|
|
651
|
-
assert.equal(brokerPid(id), pid0, 'G9: broker/engine pid unchanged after detach');
|
|
652
|
-
assert.equal(isPidAlive(pid0), true, 'G9: the one engine is still alive after detach');
|
|
653
|
-
assert.equal(h.bootCount(id), boots0, 'G9: no second engine spawned across detach');
|
|
654
|
-
// attachUntil (not a fixed sleep) synchronizes on the controller handoff: the
|
|
655
|
-
// detach's controllerId=null lands a beat after close, so retry until the
|
|
656
|
-
// reattach is admitted controller before driving the second prompt.
|
|
657
|
-
const c2 = await attachUntil(id, 'controller', 'g9-b', (x) => x.welcome.role === 'controller', 'G9 reattach re-takes control');
|
|
658
|
-
assert.equal(c2.welcome.role, 'controller', 'G9: the reattached client drives the SAME engine as controller');
|
|
659
|
-
const b = tok('G9-B');
|
|
660
|
-
c2.send({ type: 'prompt', text: b });
|
|
661
|
-
await c2.waitFrame((f) => f.type === 'agent_end' && frameHas(f, b), 'G9 reattached controller drives the SAME engine');
|
|
662
|
-
assert.equal(brokerPid(id), pid0, 'G9: STILL the same broker/engine pid after reattach (one writer, never two)');
|
|
663
|
-
assert.equal(isPidAlive(pid0), true, 'G9: the single engine is alive across the full cycle');
|
|
664
|
-
assert.equal(h.bootCount(id), boots0, 'G9: exactly one engine boot across attach→detach→reattach');
|
|
665
|
-
// fd-check (best-effort): the viewer host (this test process) must NOT hold the
|
|
666
|
-
// session .jsonl — only the broker may. With the fake engine the .jsonl is not
|
|
667
|
-
// held open continuously, so the meaningful assertion is that the VIEWER never
|
|
668
|
-
// appears among its holders (and any holder that exists is the broker).
|
|
669
|
-
const jsonl = join(h.home, 'nodes', id, 'fake-session.jsonl');
|
|
670
|
-
const holders = lsofHolders(jsonl);
|
|
671
|
-
if (holders !== null) {
|
|
672
|
-
assert.ok(!holders.includes(process.pid), 'G9: the attach client (this process) does NOT hold the session .jsonl');
|
|
673
|
-
for (const holder of holders) {
|
|
674
|
-
assert.equal(holder, pid0, 'G9: the ONLY holder of the session .jsonl is the broker/engine');
|
|
675
|
-
}
|
|
676
|
-
}
|
|
677
|
-
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|