@north-light/crouter 0.3.248 → 0.3.250
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/__tests__/integration/client.test.js +28 -7
- package/dist/api/client.d.ts +1 -1
- package/dist/api/client.js +2 -2
- package/dist/api/dto/lifecycle.d.ts +3 -0
- package/dist/api/dto/messages.d.ts +4 -0
- package/dist/api/dto/nodes.d.ts +4 -0
- package/dist/api/dto/reports.d.ts +7 -2
- package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
- package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
- package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +1 -0
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
- package/dist/clients/inbox/controller.js +32 -5
- package/dist/clients/inbox/tui/ansi.d.ts +3 -1
- package/dist/clients/inbox/tui/ansi.js +14 -10
- package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
- package/dist/commands/api-client.d.ts +2 -2
- package/dist/commands/api-client.js +5 -5
- package/dist/commands/canvas-browse.js +1 -1
- package/dist/commands/node/create.js +3 -1
- package/dist/commands/node/inspect.js +1 -0
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node/message.js +6 -4
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/push.js +36 -37
- package/dist/commands/sys/daemon.js +21 -8
- package/dist/commands/sys/setup-core.js +2 -2
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
- package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
- package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +5 -1
- package/dist/core/__tests__/child-death-wake.test.js +11 -7
- package/dist/core/__tests__/collectible.test.d.ts +1 -0
- package/dist/core/__tests__/collectible.test.js +88 -0
- package/dist/core/__tests__/context-intro.test.js +4 -41
- package/dist/core/__tests__/daemon-boot.test.js +30 -14
- package/dist/core/__tests__/daemon-wedge.test.js +25 -19
- package/dist/core/__tests__/fixtures/fake-engine.js +11 -0
- package/dist/core/__tests__/human-deliver.test.js +5 -5
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
- package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
- package/dist/core/__tests__/integration/revive.test.js +48 -0
- package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
- package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
- package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
- package/dist/core/__tests__/migration.test.js +30 -0
- package/dist/core/__tests__/passive-subscription.test.js +39 -1
- package/dist/core/__tests__/prune.test.js +12 -3
- package/dist/core/__tests__/relaunch-root.test.js +1 -1
- package/dist/core/__tests__/respawn-throttle.test.js +16 -146
- package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
- package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
- package/dist/core/canvas/canvas.d.ts +9 -15
- package/dist/core/canvas/canvas.js +41 -47
- package/dist/core/canvas/crons.d.ts +5 -8
- package/dist/core/canvas/crons.js +5 -8
- package/dist/core/canvas/history.d.ts +5 -0
- package/dist/core/canvas/history.js +12 -1
- package/dist/core/canvas/migrations.js +58 -1
- package/dist/core/canvas/pid.d.ts +1 -1
- package/dist/core/canvas/pid.js +5 -2
- package/dist/core/canvas/types.d.ts +4 -0
- package/dist/core/feed/feed.d.ts +4 -0
- package/dist/core/feed/feed.js +13 -8
- package/dist/core/feed/inbox.d.ts +2 -1
- package/dist/core/feed/inbox.js +22 -3
- package/dist/core/git.d.ts +3 -0
- package/dist/core/git.js +3 -1
- package/dist/core/human/feedback-companion.js +1 -0
- package/dist/core/preview-registry.js +0 -1
- package/dist/core/review/companion.js +1 -0
- package/dist/core/review/realize.js +1 -0
- package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
- package/dist/core/runtime/broker/auth-reload.js +9 -3
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
- package/dist/core/runtime/broker/engine-drive.js +28 -23
- package/dist/core/runtime/broker/event-projection.d.ts +4 -0
- package/dist/core/runtime/broker/event-projection.js +43 -23
- package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
- package/dist/core/runtime/broker/fault-retry.js +94 -8
- package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
- package/dist/core/runtime/broker/frame-dispatch.js +4 -1
- package/dist/core/runtime/broker/inbox.js +4 -0
- package/dist/core/runtime/broker/rebind.js +18 -0
- package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
- package/dist/core/runtime/broker/turn-admission.js +31 -0
- package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
- package/dist/core/runtime/broker/turn-ignition.js +25 -15
- package/dist/core/runtime/broker.js +21 -2
- package/dist/core/runtime/close.d.ts +11 -9
- package/dist/core/runtime/close.js +23 -19
- package/dist/core/runtime/fault.d.ts +21 -0
- package/dist/core/runtime/fault.js +129 -19
- package/dist/core/runtime/fleet.d.ts +0 -6
- package/dist/core/runtime/host.d.ts +7 -1
- package/dist/core/runtime/host.js +20 -2
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +24 -1
- package/dist/core/runtime/placement.d.ts +6 -6
- package/dist/core/runtime/placement.js +15 -8
- package/dist/core/runtime/reset.js +2 -2
- package/dist/core/runtime/revive.js +40 -4
- package/dist/core/runtime/spawn.d.ts +3 -0
- package/dist/core/runtime/spawn.js +1 -0
- package/dist/core/runtime/warm-pool.d.ts +4 -0
- package/dist/core/runtime/warm-pool.js +2 -0
- package/dist/core/worktree.d.ts +4 -3
- package/dist/core/worktree.js +73 -27
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
- package/dist/daemon/api/handlers/bash-jobs.js +2 -0
- package/dist/daemon/api/handlers/broker-ops.js +23 -2
- package/dist/daemon/api/handlers/human.js +1 -1
- package/dist/daemon/api/handlers/messages.js +11 -2
- package/dist/daemon/api/handlers/nodes.js +13 -1
- package/dist/daemon/api/handlers/reports.js +14 -6
- package/dist/daemon/api/map.js +1 -0
- package/dist/daemon/api/server.d.ts +9 -12
- package/dist/daemon/api/server.js +30 -19
- package/dist/daemon/companion-retire.js +3 -9
- package/dist/daemon/cron/sinks.js +7 -2
- package/dist/daemon/crtrd.js +4 -1
- package/dist/daemon/fleet.d.ts +2 -16
- package/dist/daemon/fleet.js +63 -150
- package/dist/daemon/human/finish.js +2 -0
- package/dist/daemon/manage.d.ts +11 -0
- package/dist/daemon/manage.js +32 -3
- package/dist/daemon/messaging/node-message.js +7 -1
- package/dist/daemon/reconcilers/bash-deadline.js +2 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
- package/dist/daemon/reconcilers/broker-supervision.js +22 -9
- package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
- package/dist/daemon/reconcilers/live-obligation.js +18 -6
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
- package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
- package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
- package/dist/pi-extensions/canvas-context-intro.js +4 -76
- package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
- package/dist/pi-extensions/canvas-review-boundary.js +4 -29
- package/dist/pi-extensions/canvas-stophook.js +1 -6
- package/dist/shared/birth-announcement.d.ts +12 -0
- package/dist/shared/birth-announcement.js +25 -0
- package/dist/shared/generated-context.d.ts +8 -4
- package/dist/shared/generated-context.js +15 -9
- package/package.json +4 -4
- package/runtime.lock.json +2 -2
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
- package/dist/clients/inbox/tui/clipboard.d.ts +0 -2
- package/dist/clients/inbox/tui/clipboard.js +0 -26
- package/dist/pi-extensions/truncate.d.ts +0 -5
- package/dist/pi-extensions/truncate.js +0 -14
|
@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
|
|
|
3
3
|
import { mkdtempSync, rmSync, existsSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
-
import { createNode, getRow, subscribe, recordPid, recordSpawn, subscribersOf, subscriptionsOf, pruneNodes, pruneToLimit, } from '../canvas/canvas.js';
|
|
6
|
+
import { createNode, getRow, subscribe, recordPid, recordSpawn, subscribersOf, subscriptionsOf, pruneNodes, pruneToLimit, transitionStatusIntent, } from '../canvas/canvas.js';
|
|
7
7
|
import { closeDb, openDb } from '../canvas/db.js';
|
|
8
8
|
import { nodeDir } from '../canvas/paths.js';
|
|
9
9
|
let home;
|
|
@@ -30,6 +30,9 @@ function node(id, over = {}) {
|
|
|
30
30
|
function totalEdges() {
|
|
31
31
|
return openDb().prepare('SELECT COUNT(*) AS n FROM edges').get().n;
|
|
32
32
|
}
|
|
33
|
+
function markCollectibleTerminal(nodeId) {
|
|
34
|
+
transitionStatusIntent(nodeId, { terminal_reason: 'closed' });
|
|
35
|
+
}
|
|
33
36
|
/** The standard fixture: three prunable terminal nodes (old), three survivors
|
|
34
37
|
* (two old-but-live, one recent-but-dead), and edges that touch a pruned node
|
|
35
38
|
* so cascade is observable. */
|
|
@@ -147,6 +150,8 @@ test('count pruning keeps all live nodes even when liveCount exceeds the cap', (
|
|
|
147
150
|
}
|
|
148
151
|
createNode(node('old-a', { status: 'dead', created: at(100) }));
|
|
149
152
|
createNode(node('old-b', { status: 'done', created: at(101) }));
|
|
153
|
+
markCollectibleTerminal('old-a');
|
|
154
|
+
markCollectibleTerminal('old-b');
|
|
150
155
|
const result = pruneToLimit({ maxNodes: 3 });
|
|
151
156
|
assert.equal(result.dryRun, false);
|
|
152
157
|
assert.deepEqual(result.pruned.map((p) => p.node_id).sort(), ['old-a', 'old-b']);
|
|
@@ -161,7 +166,9 @@ test('count pruning deletes the oldest terminal nodes beyond max(cap, liveCount)
|
|
|
161
166
|
createNode(node('live-0', { status: 'idle', created: at(0) }));
|
|
162
167
|
recordPid('live-0', process.pid);
|
|
163
168
|
for (let i = 0; i < 5; i++) {
|
|
164
|
-
|
|
169
|
+
const id = `term-${i}`;
|
|
170
|
+
createNode(node(id, { status: 'dead', created: at(10 + i) }));
|
|
171
|
+
markCollectibleTerminal(id);
|
|
165
172
|
}
|
|
166
173
|
const result = pruneToLimit({ maxNodes: 3 });
|
|
167
174
|
assert.deepEqual(result.pruned.map((p) => p.node_id).sort(), ['term-2', 'term-3', 'term-4']);
|
|
@@ -180,7 +187,9 @@ test('count pruning dry-run reports candidates without deleting rows or dirs', (
|
|
|
180
187
|
createNode(node('live-0', { status: 'idle', created: at(0) }));
|
|
181
188
|
recordPid('live-0', process.pid);
|
|
182
189
|
for (let i = 0; i < 5; i++) {
|
|
183
|
-
|
|
190
|
+
const id = `term-${i}`;
|
|
191
|
+
createNode(node(id, { status: 'dead', created: at(10 + i) }));
|
|
192
|
+
markCollectibleTerminal(id);
|
|
184
193
|
}
|
|
185
194
|
const result = pruneToLimit({ maxNodes: 3, dryRun: true });
|
|
186
195
|
assert.equal(result.dryRun, true);
|
|
@@ -173,7 +173,7 @@ test('root /new capacity refusal reaches the attached human through broker UI no
|
|
|
173
173
|
sessionManager: {
|
|
174
174
|
getSessionId: () => 'root-new-session',
|
|
175
175
|
getSessionFile: () => null,
|
|
176
|
-
|
|
176
|
+
getBranch: () => [],
|
|
177
177
|
},
|
|
178
178
|
ui: { notify: (message) => resolveNotice(message) },
|
|
179
179
|
});
|
|
@@ -1,150 +1,20 @@
|
|
|
1
|
-
// Run with: node --import tsx/esm --test src/core/__tests__/respawn-throttle.test.ts
|
|
2
|
-
//
|
|
3
|
-
// The event-driven respawn throttle (design flow B) — the replacement for the
|
|
4
|
-
// deleted stranded-relaunch counters. Each respawn action consults the node's
|
|
5
|
-
// consecutive short-lived-exit count: attempt n defers min(5s·2^(n-1), 10min)
|
|
6
|
-
// via a REAL backoff timer that re-enqueues a fresh policy job; at
|
|
7
|
-
// MAX_RESPAWN_ATTEMPTS (6) the loop terminalizes (transition('crash') +
|
|
8
|
-
// surfaceBootFailure + doctrine wake, counter cleared). An exit with uptime ≥
|
|
9
|
-
// HEALTHY_UPTIME_MS (60s) resets the counter BEFORE classification, restoring
|
|
10
|
-
// the immediate (sub-second) respawn.
|
|
11
|
-
//
|
|
12
|
-
// The backoff timers are real setTimeout — only the throttle's uptime clock is
|
|
13
|
-
// injected — so this file asserts the DEFERRAL (the emitted
|
|
14
|
-
// `broker.respawn.deferred` event + no launch), never timer maturity: the
|
|
15
|
-
// shortest deferral is 5s, well past this file's lifetime, and dispose()
|
|
16
|
-
// cancels every pending timer.
|
|
17
|
-
import { test, before, after } from 'node:test';
|
|
18
1
|
import assert from 'node:assert/strict';
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
node_id: id,
|
|
36
|
-
name: id,
|
|
37
|
-
created: new Date().toISOString(),
|
|
38
|
-
cwd: home,
|
|
39
|
-
kind: 'developer',
|
|
40
|
-
mode: 'base',
|
|
41
|
-
lifecycle: 'terminal',
|
|
42
|
-
status: 'active',
|
|
43
|
-
...over,
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
/** Every daemon event this process emitted, oldest first. */
|
|
47
|
-
function events() {
|
|
48
|
-
let raw;
|
|
49
|
-
try {
|
|
50
|
-
raw = readFileSync(join(home, 'crtrd.log'), 'utf8');
|
|
51
|
-
}
|
|
52
|
-
catch {
|
|
53
|
-
return [];
|
|
54
|
-
}
|
|
55
|
-
return raw
|
|
56
|
-
.split('\n')
|
|
57
|
-
.filter((l) => l.trim() !== '')
|
|
58
|
-
.map((l) => JSON.parse(l));
|
|
59
|
-
}
|
|
60
|
-
before(() => {
|
|
61
|
-
home = mkdtempSync(join(tmpdir(), 'crtr-respawn-throttle-'));
|
|
62
|
-
mkdirSync(home, { recursive: true });
|
|
63
|
-
process.env['CRTR_HOME'] = home;
|
|
64
|
-
resetEventSourceForTests();
|
|
65
|
-
bindDaemonEventSource(); // events land in <home>/crtrd.log, like the daemon's
|
|
66
|
-
tf = bindTestFleet({ now: () => nowMs });
|
|
67
|
-
});
|
|
68
|
-
after(() => {
|
|
69
|
-
tf.fleet.dispose(); // cancels every pending real backoff timer
|
|
70
|
-
resetFleetForTests();
|
|
71
|
-
resetEventSourceForTests();
|
|
72
|
-
closeDb();
|
|
73
|
-
rmSync(home, { recursive: true, force: true });
|
|
74
|
-
delete process.env['CRTR_HOME'];
|
|
75
|
-
});
|
|
76
|
-
// Short-lived exits 1-5 defer with doubling backoff; the 6th terminalizes.
|
|
77
|
-
test('consecutive short-lived exits defer respawn on a doubling backoff, then terminalize at the cap', async () => {
|
|
78
|
-
// A booted row (saved session, no crash markers) — table row 6, respawn-resume.
|
|
79
|
-
createNode(node('T', { pi_session_id: 'sess-T' }));
|
|
80
|
-
// Five synthetic exits (no known launch time → each counts as short-lived).
|
|
81
|
-
for (let i = 0; i < 5; i++) {
|
|
82
|
-
tf.fleet.deliverExit('T', { code: 1, signal: null });
|
|
83
|
-
await tf.drain();
|
|
84
|
-
}
|
|
85
|
-
const deferred = events().filter((e) => e.event === 'broker.respawn.deferred');
|
|
86
|
-
assert.deepEqual(deferred.map((e) => e.fields?.['backoff_ms']), [5_000, 10_000, 20_000, 40_000, 80_000], 'attempts 1-5 defer on the doubling 5s·2^(n-1) schedule');
|
|
87
|
-
assert.deepEqual(deferred.map((e) => e.fields?.['consecutive_failures']), [1, 2, 3, 4, 5], 'the counter grows once per short-lived exit');
|
|
88
|
-
assert.equal(getNode('T').status, 'active', 'the node stays revivable while attempts remain');
|
|
89
|
-
assert.equal(events().some((e) => e.event === 'broker.respawn.started'), false, 'no launch fires while every attempt is deferred');
|
|
90
|
-
// The 6th short-lived exit exhausts the budget: crash + surface, no respawn.
|
|
91
|
-
tf.fleet.deliverExit('T', { code: 1, signal: null });
|
|
92
|
-
await tf.drain();
|
|
93
|
-
assert.equal(getNode('T').status, 'dead', 'the respawn loop terminalizes at MAX_RESPAWN_ATTEMPTS');
|
|
94
|
-
assert.equal(events().some((e) => e.event === 'broker.respawn.exhausted'), true, 'exhaustion is surfaced as its own event');
|
|
95
|
-
});
|
|
96
|
-
// A healthy-uptime exit resets the counter: the next respawn is immediate.
|
|
97
|
-
test('a pre-launch respawn refusal terminalizes the stranded node and wakes its parent', async () => {
|
|
98
|
-
const missingCwd = join(home, 'reaped-worktree');
|
|
99
|
-
createNode(node('P', { lifecycle: 'resident' }));
|
|
100
|
-
createNode(node('R', { cwd: missingCwd, parent: 'P', pi_session_id: 'sess-R' }));
|
|
101
|
-
subscribe('P', 'R', true);
|
|
102
|
-
// A healthy real entry makes this exit respawn immediately (rather than
|
|
103
|
-
// waiting for the throttle), reaching reviveNode's cwd preflight through
|
|
104
|
-
// DaemonFleet.#respawn.
|
|
105
|
-
nowMs = 10_000_000;
|
|
106
|
-
tf.fleet.register('R', { pid: 424_244, exited: new Promise(() => { }) });
|
|
107
|
-
nowMs += HEALTHY_UPTIME_MS;
|
|
108
|
-
tf.fleet.deliverExit('R', { code: 1, signal: null });
|
|
109
|
-
await tf.drain();
|
|
110
|
-
assert.equal(getNode('R').status, 'dead', 'a pre-launch refusal cannot strand an active node without a fleet entry');
|
|
111
|
-
const inbox = readInboxSince('P');
|
|
112
|
-
assert.equal(inbox.length, 2, 'the terminal outcome surfaces through both the urgent failure and doctrine wake');
|
|
113
|
-
const urgent = inbox.find((entry) => entry.tier === 'urgent');
|
|
114
|
-
assert.ok(urgent, 'the parent receives the recovery-refusal surface');
|
|
115
|
-
assert.match(urgent.label, /Recovery failed/, 'the surface describes the refusal instead of falsely claiming a fresh boot failed');
|
|
116
|
-
assert.ok(inbox.some((entry) => entry.tier === 'normal'), 'the parent receives the doctrine wake');
|
|
117
|
-
});
|
|
118
|
-
test('an exit after HEALTHY_UPTIME_MS of uptime resets the counter and respawns immediately', async () => {
|
|
119
|
-
const originalLaunch = headlessBrokerHost.launch;
|
|
120
|
-
const launches = [];
|
|
121
|
-
try {
|
|
122
|
-
createNode(node('H', { pi_session_id: 'sess-H' }));
|
|
123
|
-
// Build up one failure first: a short-lived exit defers (5s timer, no launch).
|
|
124
|
-
tf.fleet.deliverExit('H', { code: 1, signal: null });
|
|
125
|
-
await tf.drain();
|
|
126
|
-
assert.equal(events().filter((e) => e.event === 'broker.respawn.deferred' && e.fields?.['backoff_ms'] === 5_000).length >= 1, true, 'precondition: the first short-lived exit deferred at 5s');
|
|
127
|
-
// A live registration (a manual revive winning the race) supersedes the
|
|
128
|
-
// pending backoff timer and stamps the current failure count on the entry.
|
|
129
|
-
const t0 = nowMs;
|
|
130
|
-
tf.fleet.register('H', { pid: 424_242, exited: new Promise(() => { }) });
|
|
131
|
-
assert.equal(tf.fleet.get('H').consecutiveFailures, 1, 'the entry mirrors the throttle count at registration');
|
|
132
|
-
// The relaunch runs healthy past the 60s bar, then exits: the counter
|
|
133
|
-
// resets BEFORE classification, so the respawn is immediate (backoff 0).
|
|
134
|
-
headlessBrokerHost.launch = (nodeId) => {
|
|
135
|
-
launches.push(nodeId);
|
|
136
|
-
const handle = { pid: 424_243, exited: new Promise(() => { }) };
|
|
137
|
-
tf.fleet.register(nodeId, handle); // the real host contract: registration at the launch seam
|
|
138
|
-
return handle;
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
import { decideRespawnAttempt, MAX_RESPAWN_ATTEMPTS, respawnBackoffMs } from '../../daemon/reconcilers/node-lifecycle/respawn-policy.js';
|
|
4
|
+
test('respawn policy resets after settlement and exhausts six no-settlement wedge relaunches', () => {
|
|
5
|
+
const launch = Date.parse('2026-01-01T00:00:00.000Z');
|
|
6
|
+
assert.deepEqual(decideRespawnAttempt({ last_settled_at: new Date(launch + 1).toISOString(), launched_at: new Date(launch).toISOString(), respawn_failures: 4 }, launch + 2), { failures: 0, notBefore: null, exhausted: false }, 'a turn settled after launch proves progress and resets the sequence');
|
|
7
|
+
let row = { last_settled_at: null, launched_at: new Date(launch).toISOString(), respawn_failures: 0 };
|
|
8
|
+
for (let failures = 1; failures <= MAX_RESPAWN_ATTEMPTS; failures++) {
|
|
9
|
+
const now = launch + failures * 1_000;
|
|
10
|
+
const attempt = decideRespawnAttempt(row, now);
|
|
11
|
+
assert.equal(attempt.failures, failures);
|
|
12
|
+
assert.equal(Date.parse(attempt.notBefore), now + respawnBackoffMs(failures));
|
|
13
|
+
assert.equal(attempt.exhausted, failures === MAX_RESPAWN_ATTEMPTS);
|
|
14
|
+
row = {
|
|
15
|
+
last_settled_at: null,
|
|
16
|
+
launched_at: new Date(now + respawnBackoffMs(failures)).toISOString(),
|
|
17
|
+
respawn_failures: attempt.failures,
|
|
139
18
|
};
|
|
140
|
-
nowMs = t0 + 61_000;
|
|
141
|
-
tf.fleet.deliverExit('H', { code: 0, signal: null });
|
|
142
|
-
await tf.drain();
|
|
143
|
-
assert.deepEqual(launches, ['H'], 'a healthy-uptime exit respawns immediately — no deferral');
|
|
144
|
-
assert.equal(tf.fleet.get('H').consecutiveFailures, 0, 'the healthy run reset the consecutive-failure counter');
|
|
145
|
-
assert.equal(getNode('H').status, 'active', 'the node is live again, never terminalized');
|
|
146
|
-
}
|
|
147
|
-
finally {
|
|
148
|
-
headlessBrokerHost.launch = originalLaunch;
|
|
149
19
|
}
|
|
150
20
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Run with: node --import tsx/esm --test src/core/__tests__/seam/broker-crash-teardown.test.ts
|
|
2
2
|
//
|
|
3
3
|
// Daemon-epoch broker ownership, real-process crash and teardown. The broker's
|
|
4
|
-
//
|
|
4
|
+
// supervision tick is the death authority after the broker exit is observed: a SIGTERM of a streaming broker
|
|
5
5
|
// aborts cleanly and resumes the saved session with a continuation, and a
|
|
6
6
|
// SIGKILL mid-turn — no certificate, only the durable busy marker — resumes
|
|
7
7
|
// that same saved session in place and re-drives the stopped turn with the
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
//
|
|
13
13
|
// spawnHeadlessChild calls the production birth path in the harness process, so
|
|
14
14
|
// every broker below is harness-fleet-owned from its first and only boot and
|
|
15
|
-
// every SIGKILL's REAL ChildProcess exit
|
|
16
|
-
//
|
|
15
|
+
// every SIGKILL's REAL ChildProcess exit is observed by the fleet; the harness
|
|
16
|
+
// then drives the production supervision tick — no synthetic delivery or grace.
|
|
17
17
|
// Each crash has its own real broker setup, so either top-level test remains a
|
|
18
18
|
// complete journey under name filtering and pays only its own first respawn
|
|
19
19
|
// backoff.
|
|
@@ -57,17 +57,22 @@ test('SIGTERM of a busy broker aborts cleanly and resumes with a continuation',
|
|
|
57
57
|
assert.equal(h.node(id).intent ?? null, null, 'fresh broker has a null intent (not refresh/idle-release)');
|
|
58
58
|
const boots = h.bootCount(id);
|
|
59
59
|
// SIGTERM exercises broker.ts's bounded session.abort() path. Its REAL child
|
|
60
|
-
// exit
|
|
61
|
-
// the first-failure backoff
|
|
60
|
+
// exit leaves the broker-written clean-abort certificate; the supervision tick
|
|
61
|
+
// classifies it and persists the first-failure backoff.
|
|
62
62
|
process.kill(oldPid, 'SIGTERM');
|
|
63
63
|
await h.waitFor(() => !isPidAlive(oldPid), { label: 'gracefully torn-down broker pid is dead' });
|
|
64
|
-
assert.equal(existsSync(join(jobDir(id), 'clean-abort')), true, 'settled abort leaves durable clean-abort evidence before
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
64
|
+
assert.equal(existsSync(join(jobDir(id), 'clean-abort')), true, 'settled abort leaves durable clean-abort evidence before supervision classifies it');
|
|
65
|
+
await h.awaitFleetExit(id);
|
|
66
|
+
await h.tick();
|
|
67
|
+
const gracefulBackoff = h.node(id);
|
|
68
|
+
assert.equal(gracefulBackoff.respawn_failures, 1, 'the first failed launch is durably counted');
|
|
69
|
+
assert.ok(gracefulBackoff.respawn_not_before != null, 'the first retry is durably deferred');
|
|
70
|
+
assert.equal(gracefulBackoff.pi_pid, null, 'classification clears the dead broker pid');
|
|
71
|
+
await h.tick(Date.parse(gracefulBackoff.respawn_not_before) + 1);
|
|
72
|
+
// This is the file's SECOND real broker boot, after the durable 5s backoff
|
|
73
|
+
// matures on a supervision tick.
|
|
69
74
|
const boot2 = await h.awaitBoot(id, { minCount: boots + 1, timeoutMs: 25_000 });
|
|
70
|
-
assert.equal(boot2.resuming, true, 'clean-abort
|
|
75
|
+
assert.equal(boot2.resuming, true, 'clean-abort tick-driven recovery RESUMES the saved .jsonl');
|
|
71
76
|
assert.equal(boot2.prompt, RUNTIME_RESTART_CONTINUATION, 'the strict resume injects the runtime-restart continuation');
|
|
72
77
|
assert.ok(!boot2.rawArgv.includes('--crtr-new-cycle'), 'clean abort never starts a recovery cycle');
|
|
73
78
|
const revivedPid = h.node(id).pi_pid;
|
|
@@ -90,6 +95,13 @@ test('SIGKILL of a busy broker resumes the interrupted turn in place; close tear
|
|
|
90
95
|
markBusy(id);
|
|
91
96
|
process.kill(busyPid, 'SIGKILL');
|
|
92
97
|
await h.waitFor(() => !isPidAlive(busyPid), { label: 'mid-turn crashed broker pid is dead' });
|
|
98
|
+
await h.awaitFleetExit(id);
|
|
99
|
+
await h.tick();
|
|
100
|
+
const dirtyBackoff = h.node(id);
|
|
101
|
+
assert.equal(dirtyBackoff.respawn_failures, 1, 'the first failed launch is durably counted');
|
|
102
|
+
assert.ok(dirtyBackoff.respawn_not_before != null, 'the first retry is durably deferred');
|
|
103
|
+
assert.equal(dirtyBackoff.pi_pid, null, 'classification clears the dead broker pid');
|
|
104
|
+
await h.tick(Date.parse(dirtyBackoff.respawn_not_before) + 1);
|
|
93
105
|
const boot2 = await h.awaitBoot(id, { minCount: boots + 1, timeoutMs: 25_000 });
|
|
94
106
|
// The fake SessionManager reports `resuming=true` for ANY existing session
|
|
95
107
|
// path, including a cycle; production distinguishes cycling with the argv
|
|
@@ -7,12 +7,12 @@
|
|
|
7
7
|
// its helpers in isolation.
|
|
8
8
|
import { test, before, after, afterEach } from 'node:test';
|
|
9
9
|
import assert from 'node:assert/strict';
|
|
10
|
-
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
10
|
+
import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
|
11
11
|
import { join } from 'node:path';
|
|
12
12
|
import { createHeadlessHarness } from '../helpers/harness.js';
|
|
13
13
|
import { createAttachKit } from '../helpers/broker-clients.js';
|
|
14
14
|
import { defaultModelLaddersConfig } from '../../../types.js';
|
|
15
|
-
import { readFault } from '../../runtime/fault.js';
|
|
15
|
+
import { admitProviderRetryEpisode, readFault, readProviderRetryEpisode } from '../../runtime/fault.js';
|
|
16
16
|
import { buildPiArgv, normalizeModel } from '../../runtime/launch.js';
|
|
17
17
|
import { encodeFrame } from '../../runtime/broker-protocol.js';
|
|
18
18
|
const MODEL_LADDERS = defaultModelLaddersConfig();
|
|
@@ -79,6 +79,18 @@ function agentStartCount(id) {
|
|
|
79
79
|
return 0;
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
|
+
async function crashAndRespawn(id, boots) {
|
|
83
|
+
const pid = h.node(id)?.pi_pid;
|
|
84
|
+
assert.ok(pid != null, 'the broker has a live pid before its physical death');
|
|
85
|
+
process.kill(pid, 'SIGKILL');
|
|
86
|
+
await h.awaitFleetExit(id);
|
|
87
|
+
await h.tick();
|
|
88
|
+
const deferred = h.node(id);
|
|
89
|
+
assert.ok(deferred.respawn_not_before != null, 'the daemon records respawn backoff after the broker exit');
|
|
90
|
+
await h.tick(Date.parse(deferred.respawn_not_before) + 1);
|
|
91
|
+
await h.awaitBoot(id, { minCount: boots + 1 });
|
|
92
|
+
await h.waitFor(() => existsSync(h.brokerSock(id)), { label: 'replacement broker listener bound' });
|
|
93
|
+
}
|
|
82
94
|
before(async () => {
|
|
83
95
|
h = await createHeadlessHarness({ sessionPrefix: 'crtr-provider-retry' });
|
|
84
96
|
});
|
|
@@ -130,6 +142,73 @@ test('a managed-cooling terminal authors a rate-limit auto fault whose deadline
|
|
|
130
142
|
await new Promise((resolve) => setTimeout(resolve, 2500));
|
|
131
143
|
assert.equal(agentStartCount(child), baselineStarts + 1, 'the fault-retry timer re-drives the turn exactly once');
|
|
132
144
|
});
|
|
145
|
+
test('a crashed broker re-drives only its pending provider episode, never admitted or invalidated episodes', { timeout: 90_000 }, async () => {
|
|
146
|
+
const parent = h.spawnRoot('durable provider episode parent');
|
|
147
|
+
seedPool('anthropic', false);
|
|
148
|
+
seedPool('openai-codex', false);
|
|
149
|
+
const pendingNode = await h.spawnHeadlessChild(parent, 'pending provider episode', { model: OPENAI_STRONG.spec });
|
|
150
|
+
await h.waitFor(() => agentStartCount(pendingNode) >= 1, { label: 'pending episode fixture started' });
|
|
151
|
+
const pendingStarts = agentStartCount(pendingNode);
|
|
152
|
+
h.fakeCmd(pendingNode, {
|
|
153
|
+
cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 2_000,
|
|
154
|
+
});
|
|
155
|
+
const pending = await h.waitFor(() => readProviderRetryEpisode(pendingNode)?.state === 'pending' ? readProviderRetryEpisode(pendingNode) : null, { label: 'pending provider episode persisted before crash' });
|
|
156
|
+
assert.equal(pending.sessionFile, realpathSync(h.node(pendingNode).pi_session_file), 'the pending episode is tied to its transcript path');
|
|
157
|
+
await crashAndRespawn(pendingNode, h.bootCount(pendingNode));
|
|
158
|
+
await h.waitFor(() => agentStartCount(pendingNode) >= pendingStarts + 1, { label: 'replacement re-drove the pending episode' });
|
|
159
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
160
|
+
assert.equal(agentStartCount(pendingNode), pendingStarts + 1, 'a pending episode re-drives exactly once after broker death');
|
|
161
|
+
const admittedNode = await h.spawnHeadlessChild(parent, 'admitted provider episode', { model: OPENAI_STRONG.spec });
|
|
162
|
+
await h.waitFor(() => agentStartCount(admittedNode) >= 1, { label: 'admitted episode fixture started' });
|
|
163
|
+
const admittedStarts = agentStartCount(admittedNode);
|
|
164
|
+
h.fakeCmd(admittedNode, {
|
|
165
|
+
cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 60_000,
|
|
166
|
+
});
|
|
167
|
+
const admittedPending = await h.waitFor(() => readProviderRetryEpisode(admittedNode)?.state === 'pending' ? readProviderRetryEpisode(admittedNode) : null, { label: 'admitted episode pending record persisted' });
|
|
168
|
+
const admittedFault = readFault(admittedNode);
|
|
169
|
+
assert.ok(admittedFault !== null, 'the pending episode retains its retry fault before admission');
|
|
170
|
+
assert.equal(admitProviderRetryEpisode(admittedNode, admittedPending.sessionFile, admittedFault), true, 'the dispatch boundary durably admits the matching episode');
|
|
171
|
+
assert.equal(readProviderRetryEpisode(admittedNode)?.state, 'admitted');
|
|
172
|
+
await crashAndRespawn(admittedNode, h.bootCount(admittedNode));
|
|
173
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
174
|
+
assert.equal(agentStartCount(admittedNode), admittedStarts, 'an admitted episode is never replayed by replacement startup');
|
|
175
|
+
const invalidatedNode = await h.spawnHeadlessChild(parent, 'invalidated provider episode', { model: OPENAI_STRONG.spec });
|
|
176
|
+
await h.waitFor(() => agentStartCount(invalidatedNode) >= 1, { label: 'invalidated episode fixture started' });
|
|
177
|
+
const invalidatedStarts = agentStartCount(invalidatedNode);
|
|
178
|
+
h.fakeCmd(invalidatedNode, {
|
|
179
|
+
cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 60_000,
|
|
180
|
+
});
|
|
181
|
+
await h.waitFor(() => readProviderRetryEpisode(invalidatedNode)?.state === 'pending' ? true : null, { label: 'invalidated episode pending record persisted' });
|
|
182
|
+
const controller = await kit.attach(invalidatedNode, 'controller', 'invalidated-provider-episode');
|
|
183
|
+
controller.send({ type: 'new_session' });
|
|
184
|
+
await controller.waitFrame((frame) => frame.type === 'ack' && frame.for === 'new_session', 'replacement session acknowledged');
|
|
185
|
+
assert.equal(readFault(invalidatedNode), null, 'replacement discards the old daemon-owned auto fault instead of inheriting its anchor');
|
|
186
|
+
await h.waitFor(() => readProviderRetryEpisode(invalidatedNode)?.state === 'invalidated' ? true : null, { label: 'rebind durably invalidated the old episode' });
|
|
187
|
+
controller.close();
|
|
188
|
+
await crashAndRespawn(invalidatedNode, h.bootCount(invalidatedNode));
|
|
189
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
190
|
+
assert.equal(agentStartCount(invalidatedNode), invalidatedStarts, 'an invalidated rebound-session episode is never replayed by replacement startup');
|
|
191
|
+
const cyclingNode = await h.spawnHeadlessChild(parent, 'fresh-cycle provider episode', { model: OPENAI_STRONG.spec });
|
|
192
|
+
await h.waitFor(() => agentStartCount(cyclingNode) >= 1, { label: 'fresh-cycle episode fixture started' });
|
|
193
|
+
h.fakeCmd(cyclingNode, {
|
|
194
|
+
cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 60_000,
|
|
195
|
+
});
|
|
196
|
+
await h.waitFor(() => readProviderRetryEpisode(cyclingNode)?.state === 'pending' ? true : null, { label: 'fresh-cycle episode pending record persisted' });
|
|
197
|
+
const cyclingBoots = h.bootCount(cyclingNode);
|
|
198
|
+
const yieldResult = h.cli(cyclingNode, ['node', 'yield', 'begin a fresh cycle']);
|
|
199
|
+
assert.equal(yieldResult.code, 0, `node yield should succeed\n--stderr--\n${yieldResult.stderr}`);
|
|
200
|
+
const cyclingPid = h.node(cyclingNode)?.pi_pid;
|
|
201
|
+
assert.ok(cyclingPid != null, 'the refresh-pending broker remains live until its physical death');
|
|
202
|
+
process.kill(cyclingPid, 'SIGKILL');
|
|
203
|
+
await h.awaitFleetExit(cyclingNode);
|
|
204
|
+
await h.tick();
|
|
205
|
+
const cyclingBackoff = h.node(cyclingNode);
|
|
206
|
+
assert.ok(cyclingBackoff.respawn_not_before != null, 'the fresh-cycle relaunch receives daemon backoff');
|
|
207
|
+
await h.tick(Date.parse(cyclingBackoff.respawn_not_before) + 1);
|
|
208
|
+
await h.awaitBoot(cyclingNode, { minCount: cyclingBoots + 1 });
|
|
209
|
+
await h.waitFor(() => existsSync(h.brokerSock(cyclingNode)), { label: 'fresh-cycle replacement broker listener bound' });
|
|
210
|
+
assert.equal(readProviderRetryEpisode(cyclingNode)?.state, 'invalidated', 'new-cycle startup invalidates the old transcript episode');
|
|
211
|
+
});
|
|
133
212
|
test('reload_auth re-drives a parked fatal provider-auth turn exactly once', { timeout: 30_000 }, async () => {
|
|
134
213
|
const parent = h.spawnRoot('auth reload parent');
|
|
135
214
|
seedPool('anthropic', false);
|
|
@@ -194,7 +194,7 @@ test('an unattended resident with no live obligation completes on the daemon clo
|
|
|
194
194
|
assert.equal(h.node(nodeId)?.status, 'done', 'the row is terminal and eligible for history pruning');
|
|
195
195
|
const wake = readInboxSince(root).find((entry) => entry.from === nodeId && entry.data?.['reason'] === 'child-auto-done');
|
|
196
196
|
assert.ok(wake, 'the subscribing root is woken when the unattended clock completes the resident');
|
|
197
|
-
assert.match(wake.label, /
|
|
197
|
+
assert.match(wake.label, /submitted no final report/, 'the wake distinguishes automatic completion from a final report');
|
|
198
198
|
await h.tick();
|
|
199
199
|
assert.equal(h.fleet.has(nodeId), false, 'a supervision tick does not revive completed history');
|
|
200
200
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// Run with: npm run build && node --test dist/core/__tests__/seam/self-close-teardown.test.js
|
|
2
|
+
import { after, before, test } from 'node:test';
|
|
3
|
+
import assert from 'node:assert/strict';
|
|
4
|
+
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { isPidAlive } from '../../canvas/pid.js';
|
|
7
|
+
import { nodeDir } from '../../canvas/paths.js';
|
|
8
|
+
import { createHarness } from '../helpers/harness.js';
|
|
9
|
+
let h;
|
|
10
|
+
let root;
|
|
11
|
+
before(async () => {
|
|
12
|
+
h = await createHarness({ headless: true, sessionPrefix: 'crtr-self-close' });
|
|
13
|
+
root = h.spawnRoot('self-close teardown root');
|
|
14
|
+
});
|
|
15
|
+
after(async () => {
|
|
16
|
+
if (h !== undefined)
|
|
17
|
+
await h.dispose();
|
|
18
|
+
});
|
|
19
|
+
test('a node closes itself without killing its long-lived calling shell', { timeout: 60_000 }, async () => {
|
|
20
|
+
const id = await h.spawnHeadlessChild(root, 'self-close worker');
|
|
21
|
+
const brokerPid = h.node(id)?.pi_pid;
|
|
22
|
+
assert.ok(brokerPid != null && isPidAlive(brokerPid), 'precondition: worker broker is alive');
|
|
23
|
+
const session = h.node(id)?.pi_session_file;
|
|
24
|
+
assert.ok(session != null, 'worker has a session file');
|
|
25
|
+
writeFileSync(session, JSON.stringify({ type: 'message', message: { role: 'assistant', content: [{ type: 'text', text: 'work' }] } }) + '\n');
|
|
26
|
+
const dir = nodeDir(id);
|
|
27
|
+
const resultPath = join(dir, 'self-close-result');
|
|
28
|
+
const shellPidPath = join(dir, 'self-close-shell.pid');
|
|
29
|
+
let shellPid;
|
|
30
|
+
try {
|
|
31
|
+
h.fakeCmd(id, { cmd: 'self_close' });
|
|
32
|
+
await h.waitFor(() => existsSync(resultPath), { label: 'self-close CLI result' });
|
|
33
|
+
assert.equal(readFileSync(resultPath, 'utf8').trim(), '0', 'the shell received a successful close result');
|
|
34
|
+
shellPid = Number(readFileSync(shellPidPath, 'utf8').trim());
|
|
35
|
+
assert.ok(Number.isInteger(shellPid) && shellPid > 0, 'fixture recorded the long-lived calling shell pid');
|
|
36
|
+
assert.equal(isPidAlive(shellPid), true, 'the calling shell survives the daemon teardown');
|
|
37
|
+
await h.waitFor(() => !isPidAlive(brokerPid), { label: 'self-closing broker exits' });
|
|
38
|
+
assert.equal(isPidAlive(shellPid), true, 'the calling shell remains alive after the broker is gone');
|
|
39
|
+
assert.equal(h.status(id), 'canceled', 'close writes the terminal transition before teardown');
|
|
40
|
+
}
|
|
41
|
+
finally {
|
|
42
|
+
if (shellPid !== undefined) {
|
|
43
|
+
try {
|
|
44
|
+
process.kill(shellPid, 'SIGKILL');
|
|
45
|
+
}
|
|
46
|
+
catch { /* shell already exited */ }
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
});
|
|
@@ -24,8 +24,8 @@ after(async () => {
|
|
|
24
24
|
if (h !== undefined)
|
|
25
25
|
await h.dispose();
|
|
26
26
|
});
|
|
27
|
-
// 30s (approved compiled-seam cap): three real broker boots plus the
|
|
28
|
-
// respawn
|
|
27
|
+
// 30s (approved compiled-seam cap): three real broker boots plus the durable
|
|
28
|
+
// respawn backoff's first ~5s delay before the successful retry.
|
|
29
29
|
test('yield-message survives one pre-session_start retry and clears on session_start', { timeout: 30_000 }, async () => {
|
|
30
30
|
const note = `Issue #146 pre-session_start replay ${FAIL_BEFORE_SESSION_START_ONCE} keeps the wake note`;
|
|
31
31
|
const id = await h.spawnHeadlessChild(root, 'headless worker — yield message transactional retry');
|
|
@@ -35,29 +35,37 @@ test('yield-message survives one pre-session_start retry and clears on session_s
|
|
|
35
35
|
const rc = h.cli(id, ['node', 'yield', note]);
|
|
36
36
|
assert.equal(rc.code, 0, `node yield should succeed\n--stderr--\n${rc.stderr}`);
|
|
37
37
|
// `node yield` arms intent=refresh; the current turn ending is what lets the
|
|
38
|
-
// stophook release the old broker.
|
|
39
|
-
//
|
|
40
|
-
//
|
|
38
|
+
// stophook release the old broker. The supervision tick classifies that real
|
|
39
|
+
// ChildProcess exit, persists its backoff, and launches the fresh cycle once
|
|
40
|
+
// the backoff matures.
|
|
41
41
|
await h.stop(id);
|
|
42
42
|
await h.waitFor(() => !isPidAlive(oldPid), { label: 'yielded broker exited before session_start' });
|
|
43
|
+
await h.awaitFleetExit(id);
|
|
44
|
+
await h.tick();
|
|
45
|
+
const initialRefreshBackoff = h.node(id);
|
|
46
|
+
assert.equal(initialRefreshBackoff.respawn_failures, 1, 'the old broker death is durably counted');
|
|
47
|
+
assert.ok(initialRefreshBackoff.respawn_not_before != null, 'the initial refresh retry is durably deferred');
|
|
48
|
+
assert.equal(initialRefreshBackoff.pi_pid, null, 'classification clears the old broker pid');
|
|
49
|
+
await h.tick(Date.parse(initialRefreshBackoff.respawn_not_before) + 1);
|
|
43
50
|
// The initial refresh attempt launches and fails before session_start,
|
|
44
51
|
// leaving the note still present in node context.
|
|
45
52
|
const failedCyclePid = await h.waitFor(() => {
|
|
46
53
|
const pid = h.node(id)?.pi_pid;
|
|
47
54
|
return pid != null && pid !== oldPid ? pid : null;
|
|
48
|
-
}, { label: '
|
|
55
|
+
}, { label: 'tick-driven refresh attempt recorded its broker pid', timeoutMs: 30_000 });
|
|
49
56
|
await h.waitFor(() => !isPidAlive(failedCyclePid), { label: 'one-shot cycling broker died before session_start' });
|
|
50
57
|
// PID death precedes Node's ChildProcess `exit` delivery. Wait through the
|
|
51
|
-
// authoritative fleet barrier
|
|
52
|
-
// below reaches refresh arbitration
|
|
53
|
-
// The policy has only armed its backoff timer at this point; no recovery broker
|
|
54
|
-
// owns the slot yet.
|
|
58
|
+
// authoritative fleet barrier, then classify the failure once so the contender
|
|
59
|
+
// below reaches refresh arbitration while durable backoff is pending.
|
|
55
60
|
await h.awaitFleetExit(id);
|
|
61
|
+
await h.tick();
|
|
56
62
|
const afterFirstExit = h.node(id);
|
|
57
63
|
assert.equal(h.fleet.has(id), false, 'the failed broker has left the authoritative fleet before focus competes');
|
|
58
64
|
assert.equal(afterFirstExit?.cycles, baseCycles + 1, 'the first refresh attempt advances cycles');
|
|
59
65
|
assert.equal(afterFirstExit?.intent, 'refresh', 'the durable refresh intent still confronts the strict-resume contender');
|
|
60
66
|
assert.equal(afterFirstExit?.cycle_pending, true, 'cycle_pending remains true until session_start confirms');
|
|
67
|
+
assert.equal(afterFirstExit?.respawn_failures, 2, 'the failed refresh attempt is durably counted');
|
|
68
|
+
assert.ok(afterFirstExit?.respawn_not_before != null, 'focus competes while the durable refresh backoff is pending');
|
|
61
69
|
assert.equal(readYieldMessage(id), note, 'yield note stays present after pre-session_start failure');
|
|
62
70
|
assert.equal(existsSync(yieldMessagePath(id)), true, 'yield file survives the failed cycle');
|
|
63
71
|
// A real `surface node focus` on the node right now asks the revive arbiter for
|
|
@@ -334,6 +334,10 @@ export interface PruneResult {
|
|
|
334
334
|
pruned: PrunedNode[];
|
|
335
335
|
dryRun: boolean;
|
|
336
336
|
}
|
|
337
|
+
/** Whether a row may be removed by an automatic collection sweep. */
|
|
338
|
+
export declare function isCollectible(row: NodeRow, opts?: {
|
|
339
|
+
pastTtl?: boolean;
|
|
340
|
+
}): boolean;
|
|
337
341
|
/** {@link deleteNode}'s result: whether it actually deleted the node. */
|
|
338
342
|
export interface DeleteNodeResult {
|
|
339
343
|
deleted: boolean;
|
|
@@ -396,24 +400,14 @@ export declare function pruneNodes(opts: {
|
|
|
396
400
|
* store rather than per-node loss, and reaps nothing. Returns the reaped ids. */
|
|
397
401
|
export declare function reapGhostRows(): string[];
|
|
398
402
|
/** Count-based retention sweep: retain the `maxNodes` MOST-RECENTLY-ACTIVE
|
|
399
|
-
* nodes plus every
|
|
400
|
-
* `max(maxNodes,
|
|
401
|
-
* TTL sweep can't (a canvas that never crosses the TTL still grows without
|
|
403
|
+
* nodes plus every row the collection authority retains, so the real retained
|
|
404
|
+
* floor is `max(maxNodes, retainedCount)`. Bounds the otherwise-unbounded growth
|
|
405
|
+
* the TTL sweep can't (a canvas that never crosses the TTL still grows without
|
|
402
406
|
* limit).
|
|
403
407
|
*
|
|
404
|
-
* Protected (never a prune candidate, regardless of recency rank): any
|
|
405
|
-
* non-terminal node — status active|idle — REGARDLESS of `pi_pid` liveness,
|
|
406
|
-
* or a node with an open managed worktree. A crashed-but-non-terminal row
|
|
407
|
-
* (dead pid, status still active/idle) must survive for daemon crash-recovery
|
|
408
|
-
* to resume, and an open worktree must retain its node ownership until
|
|
409
|
-
* `node worktree close`; neither is eligible for count-based pruning. The caller
|
|
410
|
-
* ($CRTR_NODE_ID) is also always protected.
|
|
411
|
-
*
|
|
412
408
|
* Algorithm: order ALL nodes by recency DESC (telemetry.json mtime, falling back
|
|
413
|
-
* to `created`); `target = max(maxNodes,
|
|
414
|
-
* ranked beyond `target` (0-indexed rank >= target) that is
|
|
415
|
-
* NOT the caller. Retained = all protected + the newest terminal nodes up to
|
|
416
|
-
* `target`.
|
|
409
|
+
* to `created`); `target = max(maxNodes, retainedCount)`; delete every node
|
|
410
|
+
* ranked beyond `target` (0-indexed rank >= target) that is collectible.
|
|
417
411
|
*
|
|
418
412
|
* Mirrors {@link pruneNodes} exactly for the delete (shared `deletePrunedNodes`:
|
|
419
413
|
* transactioned row sweep + FK edge cascade + best-effort dir removal). `dryRun`
|