@north-light/crouter 0.3.249 → 0.3.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/__tests__/integration/client.test.js +28 -7
- package/dist/api/client.d.ts +1 -1
- package/dist/api/client.js +2 -2
- package/dist/api/dto/lifecycle.d.ts +3 -0
- package/dist/api/dto/messages.d.ts +4 -0
- package/dist/api/dto/nodes.d.ts +4 -0
- package/dist/api/dto/reports.d.ts +7 -2
- package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
- package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
- package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +1 -0
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
- package/dist/clients/inbox/controller.js +32 -5
- package/dist/clients/inbox/tui/ansi.d.ts +3 -1
- package/dist/clients/inbox/tui/ansi.js +14 -10
- package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
- package/dist/commands/api-client.d.ts +2 -2
- package/dist/commands/api-client.js +5 -5
- package/dist/commands/canvas-browse.js +1 -1
- package/dist/commands/node/create.js +3 -1
- package/dist/commands/node/inspect.js +1 -0
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node/message.js +6 -4
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/push.js +36 -37
- package/dist/commands/sys/daemon.js +21 -8
- package/dist/commands/sys/setup-core.js +2 -2
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
- package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
- package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +100 -1
- package/dist/core/__tests__/child-death-wake.test.js +11 -7
- package/dist/core/__tests__/collectible.test.d.ts +1 -0
- package/dist/core/__tests__/collectible.test.js +88 -0
- package/dist/core/__tests__/context-intro.test.js +4 -41
- package/dist/core/__tests__/daemon-boot.test.js +30 -14
- package/dist/core/__tests__/daemon-wedge.test.js +25 -19
- package/dist/core/__tests__/fixtures/fake-engine.js +20 -2
- package/dist/core/__tests__/human-deliver.test.js +5 -5
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
- package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
- package/dist/core/__tests__/integration/revive.test.js +48 -0
- package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
- package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
- package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
- package/dist/core/__tests__/migration.test.js +30 -0
- package/dist/core/__tests__/passive-subscription.test.js +39 -1
- package/dist/core/__tests__/prune.test.js +12 -3
- package/dist/core/__tests__/relaunch-root.test.js +1 -1
- package/dist/core/__tests__/respawn-throttle.test.js +16 -146
- package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
- package/dist/core/canvas/canvas.d.ts +9 -15
- package/dist/core/canvas/canvas.js +41 -47
- package/dist/core/canvas/crons.d.ts +5 -8
- package/dist/core/canvas/crons.js +5 -8
- package/dist/core/canvas/history.d.ts +5 -0
- package/dist/core/canvas/history.js +12 -1
- package/dist/core/canvas/migrations.js +58 -1
- package/dist/core/canvas/pid.d.ts +1 -1
- package/dist/core/canvas/pid.js +5 -2
- package/dist/core/canvas/types.d.ts +4 -0
- package/dist/core/feed/feed.d.ts +4 -0
- package/dist/core/feed/feed.js +13 -8
- package/dist/core/feed/inbox.d.ts +2 -1
- package/dist/core/feed/inbox.js +22 -3
- package/dist/core/git.d.ts +3 -0
- package/dist/core/git.js +3 -1
- package/dist/core/human/feedback-companion.js +1 -0
- package/dist/core/preview-registry.js +0 -1
- package/dist/core/review/companion.js +1 -0
- package/dist/core/review/realize.js +1 -0
- package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
- package/dist/core/runtime/broker/auth-reload.js +9 -3
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
- package/dist/core/runtime/broker/engine-drive.js +39 -23
- package/dist/core/runtime/broker/event-projection.d.ts +4 -0
- package/dist/core/runtime/broker/event-projection.js +43 -23
- package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
- package/dist/core/runtime/broker/fault-retry.js +94 -8
- package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
- package/dist/core/runtime/broker/frame-dispatch.js +4 -1
- package/dist/core/runtime/broker/held-deferred-inbox.d.ts +9 -0
- package/dist/core/runtime/broker/held-deferred-inbox.js +8 -0
- package/dist/core/runtime/broker/inbox.js +4 -0
- package/dist/core/runtime/broker/rebind.js +18 -0
- package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
- package/dist/core/runtime/broker/turn-admission.js +31 -0
- package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
- package/dist/core/runtime/broker/turn-ignition.js +25 -15
- package/dist/core/runtime/broker.js +21 -2
- package/dist/core/runtime/close.d.ts +11 -9
- package/dist/core/runtime/close.js +23 -19
- package/dist/core/runtime/fault.d.ts +21 -0
- package/dist/core/runtime/fault.js +129 -19
- package/dist/core/runtime/fleet.d.ts +0 -6
- package/dist/core/runtime/host.d.ts +7 -1
- package/dist/core/runtime/host.js +20 -2
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +24 -1
- package/dist/core/runtime/placement.d.ts +6 -6
- package/dist/core/runtime/placement.js +15 -8
- package/dist/core/runtime/reset.js +2 -2
- package/dist/core/runtime/revive.js +40 -4
- package/dist/core/runtime/spawn.d.ts +3 -0
- package/dist/core/runtime/spawn.js +1 -0
- package/dist/core/runtime/warm-pool.d.ts +4 -0
- package/dist/core/runtime/warm-pool.js +2 -0
- package/dist/core/worktree.d.ts +4 -3
- package/dist/core/worktree.js +73 -27
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
- package/dist/daemon/api/handlers/bash-jobs.js +2 -0
- package/dist/daemon/api/handlers/broker-ops.js +23 -2
- package/dist/daemon/api/handlers/human.js +1 -1
- package/dist/daemon/api/handlers/messages.js +11 -2
- package/dist/daemon/api/handlers/nodes.js +13 -1
- package/dist/daemon/api/handlers/reports.js +14 -6
- package/dist/daemon/api/map.js +1 -0
- package/dist/daemon/api/server.d.ts +9 -12
- package/dist/daemon/api/server.js +30 -19
- package/dist/daemon/companion-retire.js +3 -9
- package/dist/daemon/cron/sinks.js +7 -2
- package/dist/daemon/crtrd.js +4 -1
- package/dist/daemon/fleet.d.ts +2 -16
- package/dist/daemon/fleet.js +63 -150
- package/dist/daemon/human/finish.js +2 -0
- package/dist/daemon/manage.d.ts +11 -0
- package/dist/daemon/manage.js +32 -3
- package/dist/daemon/messaging/node-message.js +7 -1
- package/dist/daemon/reconcilers/bash-deadline.js +2 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
- package/dist/daemon/reconcilers/broker-supervision.js +22 -9
- package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
- package/dist/daemon/reconcilers/live-obligation.js +18 -6
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
- package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
- package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
- package/dist/pi-extensions/canvas-context-intro.js +4 -76
- package/dist/pi-extensions/canvas-inbox-watcher.js +105 -4
- package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
- package/dist/pi-extensions/canvas-review-boundary.js +4 -29
- package/dist/pi-extensions/canvas-stophook.js +1 -6
- package/dist/shared/birth-announcement.d.ts +12 -0
- package/dist/shared/birth-announcement.js +25 -0
- package/dist/shared/generated-context.d.ts +8 -4
- package/dist/shared/generated-context.js +15 -9
- package/package.json +4 -4
- package/runtime.lock.json +2 -2
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
- package/dist/pi-extensions/truncate.d.ts +0 -5
- package/dist/pi-extensions/truncate.js +0 -14
|
@@ -71,7 +71,11 @@ test('closeManagedWorktree leaves the active caller checkout intact with named d
|
|
|
71
71
|
assert.equal(wt.base_ref, 'main', 'default base must remain backward-compatible with persisted main worktrees');
|
|
72
72
|
assert.equal(wt.path, managedWorktreePath(nodeId));
|
|
73
73
|
assert.ok(existsSync(wt.path), 'worktree checkout should exist after create');
|
|
74
|
-
|
|
74
|
+
const sessionDir = join(home, 'sessions');
|
|
75
|
+
mkdirSync(sessionDir, { recursive: true });
|
|
76
|
+
const sessionFile = join(sessionDir, 'session.jsonl');
|
|
77
|
+
writeFileSync(sessionFile, `${JSON.stringify({ type: 'session', version: 3, id: 'wt-session', cwd: wt.path })}\n`);
|
|
78
|
+
createNode(node(nodeId, { cwd: wt.path, managed_worktree: wt, pi_session_id: 'wt-session', pi_session_file: sessionFile }));
|
|
75
79
|
const result = closeManagedWorktree(nodeId);
|
|
76
80
|
assert.equal(result.worktree_removed, false);
|
|
77
81
|
assert.match(result.worktree_remove_error ?? '', /cleanup pending/);
|
|
@@ -83,6 +87,8 @@ test('closeManagedWorktree leaves the active caller checkout intact with named d
|
|
|
83
87
|
assert.equal(meta?.managed_worktree?.state, 'closed');
|
|
84
88
|
assert.equal(meta?.managed_worktree?.cleanup, 'pending');
|
|
85
89
|
assert.equal(meta?.cwd, wt.repo_root, 'a later revive uses the durable base checkout');
|
|
90
|
+
assert.equal(meta?.pi_session_id, null, 'the live transcript remains sole-writer-owned, so close invalidates its stale resume identity');
|
|
91
|
+
assert.equal(meta?.pi_session_file, null, 'a later revive launches fresh rather than loading the header that still names this checkout');
|
|
86
92
|
});
|
|
87
93
|
test('two concurrent creates from linked worktrees serialize on their shared repository', async () => {
|
|
88
94
|
const worktreeModule = new URL('../../worktree.ts', import.meta.url).href;
|
|
@@ -334,7 +340,7 @@ test('rejects remote and nonexistent worktree bases before creating any resource
|
|
|
334
340
|
assert.equal(gitSync(['show-ref', '--verify', '--quiet', `refs/heads/crtr/${nodeId}`], sourceDir).status, 1, `${base} must not leave a dedicated branch`);
|
|
335
341
|
}
|
|
336
342
|
});
|
|
337
|
-
test('refuses to close when the recorded base was deleted
|
|
343
|
+
test('refuses to close a committed worktree when the recorded base was deleted', () => {
|
|
338
344
|
const nodeId = 'wt-deleted-base';
|
|
339
345
|
run(['branch', 'feature/deleted-base'], sourceDir);
|
|
340
346
|
const mainBefore = gitSync(['rev-parse', 'main'], sourceDir).stdout.trim();
|
|
@@ -345,10 +351,83 @@ test('refuses to close when the recorded base was deleted and leaves the worktre
|
|
|
345
351
|
run(['-c', 'user.email=test@example.com', '-c', 'user.name=Test', 'commit', '-m', 'deleted base work'], wt.path);
|
|
346
352
|
run(['branch', '-D', 'feature/deleted-base'], sourceDir);
|
|
347
353
|
assert.throws(() => closeManagedWorktree(nodeId), (error) => error instanceof Error && 'code' in error && error.code === 'missing_base_branch');
|
|
348
|
-
assert.equal(getNode(nodeId)?.managed_worktree?.state, 'open', 'a missing base must
|
|
354
|
+
assert.equal(getNode(nodeId)?.managed_worktree?.state, 'open', 'a missing base with committed work must remain open for a human decision');
|
|
349
355
|
assert.ok(existsSync(wt.path), 'the worktree must remain available for a human decision');
|
|
350
356
|
assert.equal(gitSync(['rev-parse', 'main'], sourceDir).stdout.trim(), mainBefore, 'missing base must never auto-land onto main');
|
|
351
357
|
});
|
|
358
|
+
// NL-153/NL-154: the parent can land and finish before its clean child. Its
|
|
359
|
+
// checkout and branch then disappear, leaving `repo_root` as an empty path;
|
|
360
|
+
// the child itself still resolves the shared repository and is exactly at its
|
|
361
|
+
// recorded creation SHA, so close is a no-op with only deferred cleanup left.
|
|
362
|
+
test('close concludes a clean child after parent-first closure removes its checkout and base branch', () => {
|
|
363
|
+
const repo = mkdtempSync(join(tmpdir(), 'crtr-worktree-parent-first-'));
|
|
364
|
+
try {
|
|
365
|
+
run(['init', '-b', 'main'], repo);
|
|
366
|
+
run(['config', 'user.email', 'test@example.com'], repo);
|
|
367
|
+
run(['config', 'user.name', 'Test'], repo);
|
|
368
|
+
writeFileSync(join(repo, 'README.md'), 'hello\n');
|
|
369
|
+
run(['add', '.'], repo);
|
|
370
|
+
run(['commit', '-m', 'init'], repo);
|
|
371
|
+
const parent = createManagedWorktree(repo, 'wt-parent-first-parent');
|
|
372
|
+
createNode(node('wt-parent-first-parent', { cwd: parent.path, managed_worktree: parent }));
|
|
373
|
+
const child = createManagedWorktree(parent.path, 'wt-parent-first-child', parent.branch);
|
|
374
|
+
createNode(node('wt-parent-first-child', { cwd: child.path, managed_worktree: child }));
|
|
375
|
+
assert.equal(gitSync(['rev-parse', 'HEAD'], child.path).stdout.trim(), child.base_sha, 'the child must be a clean zero-commit checkout at its recorded base');
|
|
376
|
+
closeManagedWorktree('wt-parent-first-parent');
|
|
377
|
+
run(['worktree', 'remove', parent.path], repo);
|
|
378
|
+
run(['branch', '-D', parent.branch], repo);
|
|
379
|
+
mkdirSync(parent.path, { recursive: true });
|
|
380
|
+
assert.notEqual(gitSync(['rev-parse', '--git-dir'], parent.path).status, 0, 'the recorded parent path must be an empty non-Git directory');
|
|
381
|
+
assert.equal(gitSync(['show-ref', '--verify', '--quiet', `refs/heads/${parent.branch}`], child.path).status, 1, 'the parent base branch must be gone');
|
|
382
|
+
const result = closeManagedWorktree('wt-parent-first-child');
|
|
383
|
+
assert.equal(result.landed_sha, child.base_sha, 'a no-op close reports the recorded creation SHA');
|
|
384
|
+
assert.equal(result.worktree_removed, false, 'the live caller checkout remains for deferred cleanup');
|
|
385
|
+
assert.equal(getNode('wt-parent-first-child')?.managed_worktree?.state, 'closed');
|
|
386
|
+
run(['worktree', 'remove', child.path], repo);
|
|
387
|
+
run(['branch', '-D', child.branch], repo);
|
|
388
|
+
}
|
|
389
|
+
finally {
|
|
390
|
+
rmSync(repo, { recursive: true, force: true });
|
|
391
|
+
}
|
|
392
|
+
});
|
|
393
|
+
// A failed ref inspection is not proof that a clean child has nothing to
|
|
394
|
+
// land: only show-ref's completed absent-ref result may take that no-op path.
|
|
395
|
+
test('close surfaces a failed base ref inspection instead of closing a clean worktree', () => {
|
|
396
|
+
const nodeId = 'wt-base-ref-inspection-failure';
|
|
397
|
+
const wt = createManagedWorktree(sourceDir, nodeId);
|
|
398
|
+
createNode(node(nodeId, { cwd: wt.path, managed_worktree: wt }));
|
|
399
|
+
const wrapperDir = mkdtempSync(join(tmpdir(), 'crtr-git-wrapper-'));
|
|
400
|
+
const wrapperPath = join(wrapperDir, 'git');
|
|
401
|
+
const originalPath = process.env['PATH'] ?? '';
|
|
402
|
+
writeFileSync(wrapperPath, `#!/bin/sh
|
|
403
|
+
if [ "$1" = "show-ref" ] && [ "$2" = "--verify" ] && [ "$3" = "--quiet" ] && [ "$4" = "refs/heads/main" ]; then
|
|
404
|
+
echo "simulated unreadable ref store" >&2
|
|
405
|
+
exit 128
|
|
406
|
+
fi
|
|
407
|
+
PATH="$CRTR_TEST_REAL_PATH" exec git "$@"
|
|
408
|
+
`);
|
|
409
|
+
chmodSync(wrapperPath, 0o755);
|
|
410
|
+
process.env['CRTR_TEST_REAL_PATH'] = originalPath;
|
|
411
|
+
process.env['PATH'] = `${wrapperDir}:${originalPath}`;
|
|
412
|
+
try {
|
|
413
|
+
assert.throws(() => closeManagedWorktree(nodeId), (error) => error instanceof Error
|
|
414
|
+
&& 'code' in error
|
|
415
|
+
&& error.code === 'base_branch_inspection_failed'
|
|
416
|
+
&& 'detail' in error
|
|
417
|
+
&& typeof error.detail === 'string'
|
|
418
|
+
&& error.detail.includes('status: 128')
|
|
419
|
+
&& error.detail.includes('stderr: simulated unreadable ref store'));
|
|
420
|
+
}
|
|
421
|
+
finally {
|
|
422
|
+
process.env['PATH'] = originalPath;
|
|
423
|
+
delete process.env['CRTR_TEST_REAL_PATH'];
|
|
424
|
+
rmSync(wrapperDir, { recursive: true, force: true });
|
|
425
|
+
run(['worktree', 'remove', wt.path], sourceDir);
|
|
426
|
+
run(['branch', '-D', wt.branch], sourceDir);
|
|
427
|
+
}
|
|
428
|
+
assert.equal(getNode(nodeId)?.managed_worktree?.state, 'open', 'the failed inspection must not mark a clean worktree closed');
|
|
429
|
+
assert.ok(!existsSync(wt.path), 'the test must clean up its temporary checkout');
|
|
430
|
+
});
|
|
352
431
|
// Regression for #330/#331: crtr's own untracked `.crouter/config.json` /
|
|
353
432
|
// `state.json` bookkeeping (written by ordinary project-scope init) must
|
|
354
433
|
// never block `close`'s dirty check — Fix A seeds `.git/info/exclude` for
|
|
@@ -261,10 +261,9 @@ test('autoDropCleanManagedWorktree drops a branch with real commits already cont
|
|
|
261
261
|
assert.equal(gitSync(['merge-base', '--is-ancestor', branchSha, 'main'], repo).status, 0, 'the landed commit must remain reachable from main after the drop');
|
|
262
262
|
rmSync(repo, { recursive: true, force: true });
|
|
263
263
|
});
|
|
264
|
-
// A
|
|
265
|
-
//
|
|
266
|
-
|
|
267
|
-
test('autoDropCleanManagedWorktree refuses when the recorded base branch no longer resolves', async () => {
|
|
264
|
+
// A clean child whose recorded base branch vanished has nothing to land only
|
|
265
|
+
// when its live HEAD exactly matches the persisted creation SHA.
|
|
266
|
+
test('autoDropCleanManagedWorktree drops a clean worktree at base_sha when its recorded base branch no longer resolves', async () => {
|
|
268
267
|
const { autoDropCleanManagedWorktree } = await import('../../worktree.js');
|
|
269
268
|
const repo = mkdtempSync(join(tmpdir(), 'crtr-worktree-missingbase-'));
|
|
270
269
|
run(['init', '-b', 'main'], repo);
|
|
@@ -279,11 +278,50 @@ test('autoDropCleanManagedWorktree refuses when the recorded base branch no long
|
|
|
279
278
|
// The recorded base branch ('main') has been renamed away since creation.
|
|
280
279
|
run(['branch', '-m', 'main', 'renamed-main'], repo);
|
|
281
280
|
const result = autoDropCleanManagedWorktree(nodeId);
|
|
282
|
-
assert.
|
|
283
|
-
assert.equal(getNode(nodeId)?.managed_worktree?.state, '
|
|
281
|
+
assert.ok(result !== null, 'a clean worktree exactly at its recorded base SHA has nothing to land');
|
|
282
|
+
assert.equal(getNode(nodeId)?.managed_worktree?.state, 'closed');
|
|
284
283
|
assert.ok(existsSync(wt.path));
|
|
285
284
|
rmSync(repo, { recursive: true, force: true });
|
|
286
285
|
});
|
|
286
|
+
// Reap may take the same missing-base no-op only after show-ref completed with
|
|
287
|
+
// its documented absent-ref status; inspection failures must remain visible.
|
|
288
|
+
test('autoDropCleanManagedWorktree surfaces a failed base ref inspection', async () => {
|
|
289
|
+
const { autoDropCleanManagedWorktree } = await import('../../worktree.js');
|
|
290
|
+
const nodeId = 'wt-reap-base-ref-inspection-failure';
|
|
291
|
+
const wt = createManagedWorktree(sourceDir, nodeId);
|
|
292
|
+
createNode(node(nodeId, { cwd: wt.path, managed_worktree: wt }));
|
|
293
|
+
const wrapperDir = mkdtempSync(join(tmpdir(), 'crtr-git-wrapper-'));
|
|
294
|
+
const wrapperPath = join(wrapperDir, 'git');
|
|
295
|
+
const originalPath = process.env['PATH'] ?? '';
|
|
296
|
+
writeFileSync(wrapperPath, `#!/bin/sh
|
|
297
|
+
if [ "$1" = "show-ref" ] && [ "$2" = "--verify" ] && [ "$3" = "--quiet" ] && [ "$4" = "refs/heads/main" ]; then
|
|
298
|
+
echo "simulated unreadable ref store" >&2
|
|
299
|
+
exit 128
|
|
300
|
+
fi
|
|
301
|
+
PATH="$CRTR_TEST_REAL_PATH" exec git "$@"
|
|
302
|
+
`);
|
|
303
|
+
chmodSync(wrapperPath, 0o755);
|
|
304
|
+
process.env['CRTR_TEST_REAL_PATH'] = originalPath;
|
|
305
|
+
process.env['PATH'] = `${wrapperDir}:${originalPath}`;
|
|
306
|
+
try {
|
|
307
|
+
assert.throws(() => autoDropCleanManagedWorktree(nodeId), (error) => error instanceof Error
|
|
308
|
+
&& 'code' in error
|
|
309
|
+
&& error.code === 'base_branch_inspection_failed'
|
|
310
|
+
&& 'detail' in error
|
|
311
|
+
&& typeof error.detail === 'string'
|
|
312
|
+
&& error.detail.includes('status: 128')
|
|
313
|
+
&& error.detail.includes('stderr: simulated unreadable ref store'));
|
|
314
|
+
}
|
|
315
|
+
finally {
|
|
316
|
+
process.env['PATH'] = originalPath;
|
|
317
|
+
delete process.env['CRTR_TEST_REAL_PATH'];
|
|
318
|
+
rmSync(wrapperDir, { recursive: true, force: true });
|
|
319
|
+
run(['worktree', 'remove', wt.path], sourceDir);
|
|
320
|
+
run(['branch', '-D', wt.branch], sourceDir);
|
|
321
|
+
}
|
|
322
|
+
assert.equal(getNode(nodeId)?.managed_worktree?.state, 'open', 'the failed inspection must not mark a clean worktree closed');
|
|
323
|
+
assert.ok(!existsSync(wt.path), 'the test must clean up its temporary checkout');
|
|
324
|
+
});
|
|
287
325
|
// The Critical fix: the TOCTOU race between proving clean-and-contained and
|
|
288
326
|
// the destructive branch delete. `proveCleanEmptyManagedWorktree` and
|
|
289
327
|
// `autoDropCleanManagedWorktreeLocked` are exported specifically so these
|
|
@@ -215,6 +215,36 @@ test('a fresh db has the four additive runtime columns', () => {
|
|
|
215
215
|
assert.equal(rt['window'], null);
|
|
216
216
|
assert.equal(rt['tmux_session'], null);
|
|
217
217
|
});
|
|
218
|
+
test('session-file ownership migration deterministically clears duplicate claimants before creating its UNIQUE index', () => {
|
|
219
|
+
ensureHome();
|
|
220
|
+
const raw = new DatabaseSync(canvasDbPath());
|
|
221
|
+
migrateToVersion(raw, 35);
|
|
222
|
+
const sessionFile = '/tmp/preexisting-duplicate.jsonl';
|
|
223
|
+
const aliasedSessionFile = '/tmp/./preexisting-duplicate.jsonl';
|
|
224
|
+
for (const id of ['first-owner', 'later-claimant']) {
|
|
225
|
+
const nodeDir = join(home, 'nodes', id);
|
|
226
|
+
mkdirSync(nodeDir, { recursive: true });
|
|
227
|
+
writeFileSync(join(nodeDir, 'meta.json'), JSON.stringify({
|
|
228
|
+
node_id: id, name: id, created: id === 'first-owner' ? '2026-01-01T00:00:00.000Z' : '2026-01-02T00:00:00.000Z',
|
|
229
|
+
cwd: '/tmp', kind: 'developer', mode: 'base', lifecycle: 'terminal', pi_session_id: `${id}-session`, pi_session_file: id === 'first-owner' ? sessionFile : aliasedSessionFile,
|
|
230
|
+
}));
|
|
231
|
+
}
|
|
232
|
+
raw.prepare('INSERT INTO nodes (node_id, name, kind, mode, lifecycle, status, cwd, created, pi_session_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
|
|
233
|
+
.run('first-owner', 'first-owner', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-01T00:00:00.000Z', sessionFile);
|
|
234
|
+
raw.prepare('INSERT INTO nodes (node_id, name, kind, mode, lifecycle, status, cwd, created, pi_session_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
|
|
235
|
+
.run('later-claimant', 'later-claimant', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-02T00:00:00.000Z', aliasedSessionFile);
|
|
236
|
+
assert.doesNotThrow(() => migrate(raw), 'a real canvas with pre-existing duplicates must migrate');
|
|
237
|
+
const rows = raw.prepare('SELECT node_id, pi_session_file FROM nodes ORDER BY created, node_id').all();
|
|
238
|
+
assert.deepEqual(rows.map((row) => [row.node_id, row.pi_session_file]), [
|
|
239
|
+
['first-owner', sessionFile],
|
|
240
|
+
['later-claimant', null],
|
|
241
|
+
], 'the oldest created row wins after aliases are canonicalized; later claimants are cleared');
|
|
242
|
+
const loser = JSON.parse(readFileSync(join(home, 'nodes', 'later-claimant', 'meta.json'), 'utf8'));
|
|
243
|
+
assert.equal(loser.pi_session_id, null, 'the loser meta cannot reindex its stale session id');
|
|
244
|
+
assert.equal(loser.pi_session_file, null, 'the loser meta cannot restore the duplicate projection');
|
|
245
|
+
assert.throws(() => raw.prepare('UPDATE nodes SET pi_session_file = ? WHERE node_id = ?').run(sessionFile, 'later-claimant'), /UNIQUE/, 'the schema enforces the repaired invariant');
|
|
246
|
+
raw.close();
|
|
247
|
+
});
|
|
218
248
|
test('node lifecycle substrate migration adds six columns and backfills terminal reasons', () => {
|
|
219
249
|
ensureHome();
|
|
220
250
|
const raw = new DatabaseSync(canvasDbPath());
|
|
@@ -13,6 +13,7 @@ import { mkdtempSync, mkdirSync, rmSync, existsSync, readFileSync, appendFileSyn
|
|
|
13
13
|
import { tmpdir } from 'node:os';
|
|
14
14
|
import { basename, join } from 'node:path';
|
|
15
15
|
import { createNode, subscribe } from '../canvas/canvas.js';
|
|
16
|
+
import { armCron, getCron } from '../canvas/crons.js';
|
|
16
17
|
import { closeDb } from '../canvas/db.js';
|
|
17
18
|
import { inboxPath, passivePath, apiSocketPath } from '../canvas/paths.js';
|
|
18
19
|
import { createApiServer } from '../../daemon/api/server.js';
|
|
@@ -99,16 +100,53 @@ test('passive push accumulates in passive.jsonl, not inbox.jsonl', async () => {
|
|
|
99
100
|
assert.equal(acc[0].label, 'first observation');
|
|
100
101
|
assert.match(acc[0].ref ?? '', /^pub:reports\/\d{8}T\d{9}-[0-9a-f]{8}-update\.md$/, 'report refs use a compact random suffix');
|
|
101
102
|
});
|
|
102
|
-
test('active push still lands in inbox.jsonl (wakes)', async () => {
|
|
103
|
+
test('active push still lands in inbox.jsonl (wakes) and clears the manager deadline', async () => {
|
|
103
104
|
createNode(node('pub'));
|
|
104
105
|
createNode(node('worker-mgr'));
|
|
105
106
|
subscribe('worker-mgr', 'pub', true); // ACTIVE
|
|
107
|
+
armCron({
|
|
108
|
+
cron_id: 'manager-deadline',
|
|
109
|
+
name: 'manager deadline',
|
|
110
|
+
command: 'echo stale deadline',
|
|
111
|
+
fire_at: new Date(Date.now() + 60_000).toISOString(),
|
|
112
|
+
recur: null,
|
|
113
|
+
tz: null,
|
|
114
|
+
expires_at: null,
|
|
115
|
+
anchor_node: 'worker-mgr',
|
|
116
|
+
cancel_on_wake: true,
|
|
117
|
+
created_by: null,
|
|
118
|
+
cwd: home,
|
|
119
|
+
env_json: null,
|
|
120
|
+
profile: null,
|
|
121
|
+
scope: 'profile',
|
|
122
|
+
run_timeout_s: 30,
|
|
123
|
+
overlap: 'skip',
|
|
124
|
+
on_output: 'silent',
|
|
125
|
+
sink: '{}',
|
|
126
|
+
tier: 'normal',
|
|
127
|
+
});
|
|
106
128
|
await push('pub', { kind: 'update', body: 'active report' });
|
|
107
129
|
assert.equal(existsSync(passivePath('worker-mgr')), false);
|
|
108
130
|
const inbox = readInboxSince('worker-mgr');
|
|
109
131
|
assert.equal(inbox.length, 1);
|
|
110
132
|
assert.equal(inbox[0].from, 'pub');
|
|
111
133
|
assert.match(inbox[0].ref ?? '', /^pub:reports\/.+-update\.md$/);
|
|
134
|
+
assert.equal(getCron('manager-deadline'), null, 'the child report won the deadline race');
|
|
135
|
+
});
|
|
136
|
+
test('a deferred update still lands in an active subscriber inbox at deferred tier', async () => {
|
|
137
|
+
createNode(node('pub'));
|
|
138
|
+
// Resident: deferred mail to a TERMINAL target is bumped to urgent by the
|
|
139
|
+
// shipped tier bump, so only a resident subscriber shows the requested tier.
|
|
140
|
+
createNode(node('worker-mgr', { lifecycle: 'resident' }));
|
|
141
|
+
subscribe('worker-mgr', 'pub', true); // ACTIVE
|
|
142
|
+
await push('pub', { kind: 'update', deliveryTier: 'deferred', body: 'active report' });
|
|
143
|
+
assert.equal(existsSync(passivePath('worker-mgr')), false);
|
|
144
|
+
const inbox = readInboxSince('worker-mgr');
|
|
145
|
+
assert.equal(inbox.length, 1);
|
|
146
|
+
assert.equal(inbox[0].from, 'pub');
|
|
147
|
+
assert.equal(inbox[0].tier, 'deferred', 'requested delivery urgency reaches the subscriber unchanged');
|
|
148
|
+
assert.equal(inbox[0].kind, 'update', 'the stored report kind is untouched by delivery urgency');
|
|
149
|
+
assert.match(inbox[0].ref ?? '', /^pub:reports\/.+-update\.md$/);
|
|
112
150
|
});
|
|
113
151
|
test('mixed active + passive subscribers route to their own stores', async () => {
|
|
114
152
|
createNode(node('pub'));
|
|
@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
|
|
|
3
3
|
import { mkdtempSync, rmSync, existsSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
-
import { createNode, getRow, subscribe, recordPid, recordSpawn, subscribersOf, subscriptionsOf, pruneNodes, pruneToLimit, } from '../canvas/canvas.js';
|
|
6
|
+
import { createNode, getRow, subscribe, recordPid, recordSpawn, subscribersOf, subscriptionsOf, pruneNodes, pruneToLimit, transitionStatusIntent, } from '../canvas/canvas.js';
|
|
7
7
|
import { closeDb, openDb } from '../canvas/db.js';
|
|
8
8
|
import { nodeDir } from '../canvas/paths.js';
|
|
9
9
|
let home;
|
|
@@ -30,6 +30,9 @@ function node(id, over = {}) {
|
|
|
30
30
|
function totalEdges() {
|
|
31
31
|
return openDb().prepare('SELECT COUNT(*) AS n FROM edges').get().n;
|
|
32
32
|
}
|
|
33
|
+
function markCollectibleTerminal(nodeId) {
|
|
34
|
+
transitionStatusIntent(nodeId, { terminal_reason: 'closed' });
|
|
35
|
+
}
|
|
33
36
|
/** The standard fixture: three prunable terminal nodes (old), three survivors
|
|
34
37
|
* (two old-but-live, one recent-but-dead), and edges that touch a pruned node
|
|
35
38
|
* so cascade is observable. */
|
|
@@ -147,6 +150,8 @@ test('count pruning keeps all live nodes even when liveCount exceeds the cap', (
|
|
|
147
150
|
}
|
|
148
151
|
createNode(node('old-a', { status: 'dead', created: at(100) }));
|
|
149
152
|
createNode(node('old-b', { status: 'done', created: at(101) }));
|
|
153
|
+
markCollectibleTerminal('old-a');
|
|
154
|
+
markCollectibleTerminal('old-b');
|
|
150
155
|
const result = pruneToLimit({ maxNodes: 3 });
|
|
151
156
|
assert.equal(result.dryRun, false);
|
|
152
157
|
assert.deepEqual(result.pruned.map((p) => p.node_id).sort(), ['old-a', 'old-b']);
|
|
@@ -161,7 +166,9 @@ test('count pruning deletes the oldest terminal nodes beyond max(cap, liveCount)
|
|
|
161
166
|
createNode(node('live-0', { status: 'idle', created: at(0) }));
|
|
162
167
|
recordPid('live-0', process.pid);
|
|
163
168
|
for (let i = 0; i < 5; i++) {
|
|
164
|
-
|
|
169
|
+
const id = `term-${i}`;
|
|
170
|
+
createNode(node(id, { status: 'dead', created: at(10 + i) }));
|
|
171
|
+
markCollectibleTerminal(id);
|
|
165
172
|
}
|
|
166
173
|
const result = pruneToLimit({ maxNodes: 3 });
|
|
167
174
|
assert.deepEqual(result.pruned.map((p) => p.node_id).sort(), ['term-2', 'term-3', 'term-4']);
|
|
@@ -180,7 +187,9 @@ test('count pruning dry-run reports candidates without deleting rows or dirs', (
|
|
|
180
187
|
createNode(node('live-0', { status: 'idle', created: at(0) }));
|
|
181
188
|
recordPid('live-0', process.pid);
|
|
182
189
|
for (let i = 0; i < 5; i++) {
|
|
183
|
-
|
|
190
|
+
const id = `term-${i}`;
|
|
191
|
+
createNode(node(id, { status: 'dead', created: at(10 + i) }));
|
|
192
|
+
markCollectibleTerminal(id);
|
|
184
193
|
}
|
|
185
194
|
const result = pruneToLimit({ maxNodes: 3, dryRun: true });
|
|
186
195
|
assert.equal(result.dryRun, true);
|
|
@@ -173,7 +173,7 @@ test('root /new capacity refusal reaches the attached human through broker UI no
|
|
|
173
173
|
sessionManager: {
|
|
174
174
|
getSessionId: () => 'root-new-session',
|
|
175
175
|
getSessionFile: () => null,
|
|
176
|
-
|
|
176
|
+
getBranch: () => [],
|
|
177
177
|
},
|
|
178
178
|
ui: { notify: (message) => resolveNotice(message) },
|
|
179
179
|
});
|
|
@@ -1,150 +1,20 @@
|
|
|
1
|
-
// Run with: node --import tsx/esm --test src/core/__tests__/respawn-throttle.test.ts
|
|
2
|
-
//
|
|
3
|
-
// The event-driven respawn throttle (design flow B) — the replacement for the
|
|
4
|
-
// deleted stranded-relaunch counters. Each respawn action consults the node's
|
|
5
|
-
// consecutive short-lived-exit count: attempt n defers min(5s·2^(n-1), 10min)
|
|
6
|
-
// via a REAL backoff timer that re-enqueues a fresh policy job; at
|
|
7
|
-
// MAX_RESPAWN_ATTEMPTS (6) the loop terminalizes (transition('crash') +
|
|
8
|
-
// surfaceBootFailure + doctrine wake, counter cleared). An exit with uptime ≥
|
|
9
|
-
// HEALTHY_UPTIME_MS (60s) resets the counter BEFORE classification, restoring
|
|
10
|
-
// the immediate (sub-second) respawn.
|
|
11
|
-
//
|
|
12
|
-
// The backoff timers are real setTimeout — only the throttle's uptime clock is
|
|
13
|
-
// injected — so this file asserts the DEFERRAL (the emitted
|
|
14
|
-
// `broker.respawn.deferred` event + no launch), never timer maturity: the
|
|
15
|
-
// shortest deferral is 5s, well past this file's lifetime, and dispose()
|
|
16
|
-
// cancels every pending timer.
|
|
17
|
-
import { test, before, after } from 'node:test';
|
|
18
1
|
import assert from 'node:assert/strict';
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
node_id: id,
|
|
36
|
-
name: id,
|
|
37
|
-
created: new Date().toISOString(),
|
|
38
|
-
cwd: home,
|
|
39
|
-
kind: 'developer',
|
|
40
|
-
mode: 'base',
|
|
41
|
-
lifecycle: 'terminal',
|
|
42
|
-
status: 'active',
|
|
43
|
-
...over,
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
/** Every daemon event this process emitted, oldest first. */
|
|
47
|
-
function events() {
|
|
48
|
-
let raw;
|
|
49
|
-
try {
|
|
50
|
-
raw = readFileSync(join(home, 'crtrd.log'), 'utf8');
|
|
51
|
-
}
|
|
52
|
-
catch {
|
|
53
|
-
return [];
|
|
54
|
-
}
|
|
55
|
-
return raw
|
|
56
|
-
.split('\n')
|
|
57
|
-
.filter((l) => l.trim() !== '')
|
|
58
|
-
.map((l) => JSON.parse(l));
|
|
59
|
-
}
|
|
60
|
-
before(() => {
|
|
61
|
-
home = mkdtempSync(join(tmpdir(), 'crtr-respawn-throttle-'));
|
|
62
|
-
mkdirSync(home, { recursive: true });
|
|
63
|
-
process.env['CRTR_HOME'] = home;
|
|
64
|
-
resetEventSourceForTests();
|
|
65
|
-
bindDaemonEventSource(); // events land in <home>/crtrd.log, like the daemon's
|
|
66
|
-
tf = bindTestFleet({ now: () => nowMs });
|
|
67
|
-
});
|
|
68
|
-
after(() => {
|
|
69
|
-
tf.fleet.dispose(); // cancels every pending real backoff timer
|
|
70
|
-
resetFleetForTests();
|
|
71
|
-
resetEventSourceForTests();
|
|
72
|
-
closeDb();
|
|
73
|
-
rmSync(home, { recursive: true, force: true });
|
|
74
|
-
delete process.env['CRTR_HOME'];
|
|
75
|
-
});
|
|
76
|
-
// Short-lived exits 1-5 defer with doubling backoff; the 6th terminalizes.
|
|
77
|
-
test('consecutive short-lived exits defer respawn on a doubling backoff, then terminalize at the cap', async () => {
|
|
78
|
-
// A booted row (saved session, no crash markers) — table row 6, respawn-resume.
|
|
79
|
-
createNode(node('T', { pi_session_id: 'sess-T' }));
|
|
80
|
-
// Five synthetic exits (no known launch time → each counts as short-lived).
|
|
81
|
-
for (let i = 0; i < 5; i++) {
|
|
82
|
-
tf.fleet.deliverExit('T', { code: 1, signal: null });
|
|
83
|
-
await tf.drain();
|
|
84
|
-
}
|
|
85
|
-
const deferred = events().filter((e) => e.event === 'broker.respawn.deferred');
|
|
86
|
-
assert.deepEqual(deferred.map((e) => e.fields?.['backoff_ms']), [5_000, 10_000, 20_000, 40_000, 80_000], 'attempts 1-5 defer on the doubling 5s·2^(n-1) schedule');
|
|
87
|
-
assert.deepEqual(deferred.map((e) => e.fields?.['consecutive_failures']), [1, 2, 3, 4, 5], 'the counter grows once per short-lived exit');
|
|
88
|
-
assert.equal(getNode('T').status, 'active', 'the node stays revivable while attempts remain');
|
|
89
|
-
assert.equal(events().some((e) => e.event === 'broker.respawn.started'), false, 'no launch fires while every attempt is deferred');
|
|
90
|
-
// The 6th short-lived exit exhausts the budget: crash + surface, no respawn.
|
|
91
|
-
tf.fleet.deliverExit('T', { code: 1, signal: null });
|
|
92
|
-
await tf.drain();
|
|
93
|
-
assert.equal(getNode('T').status, 'dead', 'the respawn loop terminalizes at MAX_RESPAWN_ATTEMPTS');
|
|
94
|
-
assert.equal(events().some((e) => e.event === 'broker.respawn.exhausted'), true, 'exhaustion is surfaced as its own event');
|
|
95
|
-
});
|
|
96
|
-
// A healthy-uptime exit resets the counter: the next respawn is immediate.
|
|
97
|
-
test('a pre-launch respawn refusal terminalizes the stranded node and wakes its parent', async () => {
|
|
98
|
-
const missingCwd = join(home, 'reaped-worktree');
|
|
99
|
-
createNode(node('P', { lifecycle: 'resident' }));
|
|
100
|
-
createNode(node('R', { cwd: missingCwd, parent: 'P', pi_session_id: 'sess-R' }));
|
|
101
|
-
subscribe('P', 'R', true);
|
|
102
|
-
// A healthy real entry makes this exit respawn immediately (rather than
|
|
103
|
-
// waiting for the throttle), reaching reviveNode's cwd preflight through
|
|
104
|
-
// DaemonFleet.#respawn.
|
|
105
|
-
nowMs = 10_000_000;
|
|
106
|
-
tf.fleet.register('R', { pid: 424_244, exited: new Promise(() => { }) });
|
|
107
|
-
nowMs += HEALTHY_UPTIME_MS;
|
|
108
|
-
tf.fleet.deliverExit('R', { code: 1, signal: null });
|
|
109
|
-
await tf.drain();
|
|
110
|
-
assert.equal(getNode('R').status, 'dead', 'a pre-launch refusal cannot strand an active node without a fleet entry');
|
|
111
|
-
const inbox = readInboxSince('P');
|
|
112
|
-
assert.equal(inbox.length, 2, 'the terminal outcome surfaces through both the urgent failure and doctrine wake');
|
|
113
|
-
const urgent = inbox.find((entry) => entry.tier === 'urgent');
|
|
114
|
-
assert.ok(urgent, 'the parent receives the recovery-refusal surface');
|
|
115
|
-
assert.match(urgent.label, /Recovery failed/, 'the surface describes the refusal instead of falsely claiming a fresh boot failed');
|
|
116
|
-
assert.ok(inbox.some((entry) => entry.tier === 'normal'), 'the parent receives the doctrine wake');
|
|
117
|
-
});
|
|
118
|
-
test('an exit after HEALTHY_UPTIME_MS of uptime resets the counter and respawns immediately', async () => {
|
|
119
|
-
const originalLaunch = headlessBrokerHost.launch;
|
|
120
|
-
const launches = [];
|
|
121
|
-
try {
|
|
122
|
-
createNode(node('H', { pi_session_id: 'sess-H' }));
|
|
123
|
-
// Build up one failure first: a short-lived exit defers (5s timer, no launch).
|
|
124
|
-
tf.fleet.deliverExit('H', { code: 1, signal: null });
|
|
125
|
-
await tf.drain();
|
|
126
|
-
assert.equal(events().filter((e) => e.event === 'broker.respawn.deferred' && e.fields?.['backoff_ms'] === 5_000).length >= 1, true, 'precondition: the first short-lived exit deferred at 5s');
|
|
127
|
-
// A live registration (a manual revive winning the race) supersedes the
|
|
128
|
-
// pending backoff timer and stamps the current failure count on the entry.
|
|
129
|
-
const t0 = nowMs;
|
|
130
|
-
tf.fleet.register('H', { pid: 424_242, exited: new Promise(() => { }) });
|
|
131
|
-
assert.equal(tf.fleet.get('H').consecutiveFailures, 1, 'the entry mirrors the throttle count at registration');
|
|
132
|
-
// The relaunch runs healthy past the 60s bar, then exits: the counter
|
|
133
|
-
// resets BEFORE classification, so the respawn is immediate (backoff 0).
|
|
134
|
-
headlessBrokerHost.launch = (nodeId) => {
|
|
135
|
-
launches.push(nodeId);
|
|
136
|
-
const handle = { pid: 424_243, exited: new Promise(() => { }) };
|
|
137
|
-
tf.fleet.register(nodeId, handle); // the real host contract: registration at the launch seam
|
|
138
|
-
return handle;
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
import { decideRespawnAttempt, MAX_RESPAWN_ATTEMPTS, respawnBackoffMs } from '../../daemon/reconcilers/node-lifecycle/respawn-policy.js';
|
|
4
|
+
test('respawn policy resets after settlement and exhausts six no-settlement wedge relaunches', () => {
|
|
5
|
+
const launch = Date.parse('2026-01-01T00:00:00.000Z');
|
|
6
|
+
assert.deepEqual(decideRespawnAttempt({ last_settled_at: new Date(launch + 1).toISOString(), launched_at: new Date(launch).toISOString(), respawn_failures: 4 }, launch + 2), { failures: 0, notBefore: null, exhausted: false }, 'a turn settled after launch proves progress and resets the sequence');
|
|
7
|
+
let row = { last_settled_at: null, launched_at: new Date(launch).toISOString(), respawn_failures: 0 };
|
|
8
|
+
for (let failures = 1; failures <= MAX_RESPAWN_ATTEMPTS; failures++) {
|
|
9
|
+
const now = launch + failures * 1_000;
|
|
10
|
+
const attempt = decideRespawnAttempt(row, now);
|
|
11
|
+
assert.equal(attempt.failures, failures);
|
|
12
|
+
assert.equal(Date.parse(attempt.notBefore), now + respawnBackoffMs(failures));
|
|
13
|
+
assert.equal(attempt.exhausted, failures === MAX_RESPAWN_ATTEMPTS);
|
|
14
|
+
row = {
|
|
15
|
+
last_settled_at: null,
|
|
16
|
+
launched_at: new Date(now + respawnBackoffMs(failures)).toISOString(),
|
|
17
|
+
respawn_failures: attempt.failures,
|
|
139
18
|
};
|
|
140
|
-
nowMs = t0 + 61_000;
|
|
141
|
-
tf.fleet.deliverExit('H', { code: 0, signal: null });
|
|
142
|
-
await tf.drain();
|
|
143
|
-
assert.deepEqual(launches, ['H'], 'a healthy-uptime exit respawns immediately — no deferral');
|
|
144
|
-
assert.equal(tf.fleet.get('H').consecutiveFailures, 0, 'the healthy run reset the consecutive-failure counter');
|
|
145
|
-
assert.equal(getNode('H').status, 'active', 'the node is live again, never terminalized');
|
|
146
|
-
}
|
|
147
|
-
finally {
|
|
148
|
-
headlessBrokerHost.launch = originalLaunch;
|
|
149
19
|
}
|
|
150
20
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Run with: node --import tsx/esm --test src/core/__tests__/seam/broker-crash-teardown.test.ts
|
|
2
2
|
//
|
|
3
3
|
// Daemon-epoch broker ownership, real-process crash and teardown. The broker's
|
|
4
|
-
//
|
|
4
|
+
// supervision tick is the death authority after the broker exit is observed: a SIGTERM of a streaming broker
|
|
5
5
|
// aborts cleanly and resumes the saved session with a continuation, and a
|
|
6
6
|
// SIGKILL mid-turn — no certificate, only the durable busy marker — resumes
|
|
7
7
|
// that same saved session in place and re-drives the stopped turn with the
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
//
|
|
13
13
|
// spawnHeadlessChild calls the production birth path in the harness process, so
|
|
14
14
|
// every broker below is harness-fleet-owned from its first and only boot and
|
|
15
|
-
// every SIGKILL's REAL ChildProcess exit
|
|
16
|
-
//
|
|
15
|
+
// every SIGKILL's REAL ChildProcess exit is observed by the fleet; the harness
|
|
16
|
+
// then drives the production supervision tick — no synthetic delivery or grace.
|
|
17
17
|
// Each crash has its own real broker setup, so either top-level test remains a
|
|
18
18
|
// complete journey under name filtering and pays only its own first respawn
|
|
19
19
|
// backoff.
|
|
@@ -57,17 +57,22 @@ test('SIGTERM of a busy broker aborts cleanly and resumes with a continuation',
|
|
|
57
57
|
assert.equal(h.node(id).intent ?? null, null, 'fresh broker has a null intent (not refresh/idle-release)');
|
|
58
58
|
const boots = h.bootCount(id);
|
|
59
59
|
// SIGTERM exercises broker.ts's bounded session.abort() path. Its REAL child
|
|
60
|
-
// exit
|
|
61
|
-
// the first-failure backoff
|
|
60
|
+
// exit leaves the broker-written clean-abort certificate; the supervision tick
|
|
61
|
+
// classifies it and persists the first-failure backoff.
|
|
62
62
|
process.kill(oldPid, 'SIGTERM');
|
|
63
63
|
await h.waitFor(() => !isPidAlive(oldPid), { label: 'gracefully torn-down broker pid is dead' });
|
|
64
|
-
assert.equal(existsSync(join(jobDir(id), 'clean-abort')), true, 'settled abort leaves durable clean-abort evidence before
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
64
|
+
assert.equal(existsSync(join(jobDir(id), 'clean-abort')), true, 'settled abort leaves durable clean-abort evidence before supervision classifies it');
|
|
65
|
+
await h.awaitFleetExit(id);
|
|
66
|
+
await h.tick();
|
|
67
|
+
const gracefulBackoff = h.node(id);
|
|
68
|
+
assert.equal(gracefulBackoff.respawn_failures, 1, 'the first failed launch is durably counted');
|
|
69
|
+
assert.ok(gracefulBackoff.respawn_not_before != null, 'the first retry is durably deferred');
|
|
70
|
+
assert.equal(gracefulBackoff.pi_pid, null, 'classification clears the dead broker pid');
|
|
71
|
+
await h.tick(Date.parse(gracefulBackoff.respawn_not_before) + 1);
|
|
72
|
+
// This is the file's SECOND real broker boot, after the durable 5s backoff
|
|
73
|
+
// matures on a supervision tick.
|
|
69
74
|
const boot2 = await h.awaitBoot(id, { minCount: boots + 1, timeoutMs: 25_000 });
|
|
70
|
-
assert.equal(boot2.resuming, true, 'clean-abort
|
|
75
|
+
assert.equal(boot2.resuming, true, 'clean-abort tick-driven recovery RESUMES the saved .jsonl');
|
|
71
76
|
assert.equal(boot2.prompt, RUNTIME_RESTART_CONTINUATION, 'the strict resume injects the runtime-restart continuation');
|
|
72
77
|
assert.ok(!boot2.rawArgv.includes('--crtr-new-cycle'), 'clean abort never starts a recovery cycle');
|
|
73
78
|
const revivedPid = h.node(id).pi_pid;
|
|
@@ -90,6 +95,13 @@ test('SIGKILL of a busy broker resumes the interrupted turn in place; close tear
|
|
|
90
95
|
markBusy(id);
|
|
91
96
|
process.kill(busyPid, 'SIGKILL');
|
|
92
97
|
await h.waitFor(() => !isPidAlive(busyPid), { label: 'mid-turn crashed broker pid is dead' });
|
|
98
|
+
await h.awaitFleetExit(id);
|
|
99
|
+
await h.tick();
|
|
100
|
+
const dirtyBackoff = h.node(id);
|
|
101
|
+
assert.equal(dirtyBackoff.respawn_failures, 1, 'the first failed launch is durably counted');
|
|
102
|
+
assert.ok(dirtyBackoff.respawn_not_before != null, 'the first retry is durably deferred');
|
|
103
|
+
assert.equal(dirtyBackoff.pi_pid, null, 'classification clears the dead broker pid');
|
|
104
|
+
await h.tick(Date.parse(dirtyBackoff.respawn_not_before) + 1);
|
|
93
105
|
const boot2 = await h.awaitBoot(id, { minCount: boots + 1, timeoutMs: 25_000 });
|
|
94
106
|
// The fake SessionManager reports `resuming=true` for ANY existing session
|
|
95
107
|
// path, including a cycle; production distinguishes cycling with the argv
|