@north-light/crouter 0.3.249 → 0.3.250
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/__tests__/integration/client.test.js +28 -7
- package/dist/api/client.d.ts +1 -1
- package/dist/api/client.js +2 -2
- package/dist/api/dto/lifecycle.d.ts +3 -0
- package/dist/api/dto/messages.d.ts +4 -0
- package/dist/api/dto/nodes.d.ts +4 -0
- package/dist/api/dto/reports.d.ts +7 -2
- package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
- package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
- package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +1 -0
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
- package/dist/clients/inbox/controller.js +32 -5
- package/dist/clients/inbox/tui/ansi.d.ts +3 -1
- package/dist/clients/inbox/tui/ansi.js +14 -10
- package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
- package/dist/commands/api-client.d.ts +2 -2
- package/dist/commands/api-client.js +5 -5
- package/dist/commands/canvas-browse.js +1 -1
- package/dist/commands/node/create.js +3 -1
- package/dist/commands/node/inspect.js +1 -0
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node/message.js +6 -4
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/push.js +36 -37
- package/dist/commands/sys/daemon.js +21 -8
- package/dist/commands/sys/setup-core.js +2 -2
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
- package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
- package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +5 -1
- package/dist/core/__tests__/child-death-wake.test.js +11 -7
- package/dist/core/__tests__/collectible.test.d.ts +1 -0
- package/dist/core/__tests__/collectible.test.js +88 -0
- package/dist/core/__tests__/context-intro.test.js +4 -41
- package/dist/core/__tests__/daemon-boot.test.js +30 -14
- package/dist/core/__tests__/daemon-wedge.test.js +25 -19
- package/dist/core/__tests__/fixtures/fake-engine.js +11 -0
- package/dist/core/__tests__/human-deliver.test.js +5 -5
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
- package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
- package/dist/core/__tests__/integration/revive.test.js +48 -0
- package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
- package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
- package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
- package/dist/core/__tests__/migration.test.js +30 -0
- package/dist/core/__tests__/passive-subscription.test.js +39 -1
- package/dist/core/__tests__/prune.test.js +12 -3
- package/dist/core/__tests__/relaunch-root.test.js +1 -1
- package/dist/core/__tests__/respawn-throttle.test.js +16 -146
- package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
- package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
- package/dist/core/canvas/canvas.d.ts +9 -15
- package/dist/core/canvas/canvas.js +41 -47
- package/dist/core/canvas/crons.d.ts +5 -8
- package/dist/core/canvas/crons.js +5 -8
- package/dist/core/canvas/history.d.ts +5 -0
- package/dist/core/canvas/history.js +12 -1
- package/dist/core/canvas/migrations.js +58 -1
- package/dist/core/canvas/pid.d.ts +1 -1
- package/dist/core/canvas/pid.js +5 -2
- package/dist/core/canvas/types.d.ts +4 -0
- package/dist/core/feed/feed.d.ts +4 -0
- package/dist/core/feed/feed.js +13 -8
- package/dist/core/feed/inbox.d.ts +2 -1
- package/dist/core/feed/inbox.js +22 -3
- package/dist/core/git.d.ts +3 -0
- package/dist/core/git.js +3 -1
- package/dist/core/human/feedback-companion.js +1 -0
- package/dist/core/preview-registry.js +0 -1
- package/dist/core/review/companion.js +1 -0
- package/dist/core/review/realize.js +1 -0
- package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
- package/dist/core/runtime/broker/auth-reload.js +9 -3
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
- package/dist/core/runtime/broker/engine-drive.js +28 -23
- package/dist/core/runtime/broker/event-projection.d.ts +4 -0
- package/dist/core/runtime/broker/event-projection.js +43 -23
- package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
- package/dist/core/runtime/broker/fault-retry.js +94 -8
- package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
- package/dist/core/runtime/broker/frame-dispatch.js +4 -1
- package/dist/core/runtime/broker/inbox.js +4 -0
- package/dist/core/runtime/broker/rebind.js +18 -0
- package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
- package/dist/core/runtime/broker/turn-admission.js +31 -0
- package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
- package/dist/core/runtime/broker/turn-ignition.js +25 -15
- package/dist/core/runtime/broker.js +21 -2
- package/dist/core/runtime/close.d.ts +11 -9
- package/dist/core/runtime/close.js +23 -19
- package/dist/core/runtime/fault.d.ts +21 -0
- package/dist/core/runtime/fault.js +129 -19
- package/dist/core/runtime/fleet.d.ts +0 -6
- package/dist/core/runtime/host.d.ts +7 -1
- package/dist/core/runtime/host.js +20 -2
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +24 -1
- package/dist/core/runtime/placement.d.ts +6 -6
- package/dist/core/runtime/placement.js +15 -8
- package/dist/core/runtime/reset.js +2 -2
- package/dist/core/runtime/revive.js +40 -4
- package/dist/core/runtime/spawn.d.ts +3 -0
- package/dist/core/runtime/spawn.js +1 -0
- package/dist/core/runtime/warm-pool.d.ts +4 -0
- package/dist/core/runtime/warm-pool.js +2 -0
- package/dist/core/worktree.d.ts +4 -3
- package/dist/core/worktree.js +73 -27
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
- package/dist/daemon/api/handlers/bash-jobs.js +2 -0
- package/dist/daemon/api/handlers/broker-ops.js +23 -2
- package/dist/daemon/api/handlers/human.js +1 -1
- package/dist/daemon/api/handlers/messages.js +11 -2
- package/dist/daemon/api/handlers/nodes.js +13 -1
- package/dist/daemon/api/handlers/reports.js +14 -6
- package/dist/daemon/api/map.js +1 -0
- package/dist/daemon/api/server.d.ts +9 -12
- package/dist/daemon/api/server.js +30 -19
- package/dist/daemon/companion-retire.js +3 -9
- package/dist/daemon/cron/sinks.js +7 -2
- package/dist/daemon/crtrd.js +4 -1
- package/dist/daemon/fleet.d.ts +2 -16
- package/dist/daemon/fleet.js +63 -150
- package/dist/daemon/human/finish.js +2 -0
- package/dist/daemon/manage.d.ts +11 -0
- package/dist/daemon/manage.js +32 -3
- package/dist/daemon/messaging/node-message.js +7 -1
- package/dist/daemon/reconcilers/bash-deadline.js +2 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
- package/dist/daemon/reconcilers/broker-supervision.js +22 -9
- package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
- package/dist/daemon/reconcilers/live-obligation.js +18 -6
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
- package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
- package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
- package/dist/pi-extensions/canvas-context-intro.js +4 -76
- package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
- package/dist/pi-extensions/canvas-review-boundary.js +4 -29
- package/dist/pi-extensions/canvas-stophook.js +1 -6
- package/dist/shared/birth-announcement.d.ts +12 -0
- package/dist/shared/birth-announcement.js +25 -0
- package/dist/shared/generated-context.d.ts +8 -4
- package/dist/shared/generated-context.js +15 -9
- package/package.json +4 -4
- package/runtime.lock.json +2 -2
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
- package/dist/pi-extensions/truncate.d.ts +0 -5
- package/dist/pi-extensions/truncate.js +0 -14
|
@@ -8,22 +8,25 @@
|
|
|
8
8
|
// This subtree starts, checks, and stops the daemon process.
|
|
9
9
|
import { defineLeaf, defineBranch } from '../../core/command.js';
|
|
10
10
|
import { InputError } from '../../core/io.js';
|
|
11
|
-
import { spawnDaemon, stopDaemonProcess, sweepStrayDaemons } from '../../daemon/manage.js';
|
|
11
|
+
import { isDaemonServing, spawnDaemon, stopDaemonProcess, sweepStrayDaemons } from '../../daemon/manage.js';
|
|
12
12
|
import { isDaemonRunning, readPidfile, isPidAlive } from '../../daemon/pidfile.js';
|
|
13
13
|
import { cliClient } from '../api-client.js';
|
|
14
14
|
// daemon start
|
|
15
15
|
const daemonStart = defineLeaf({
|
|
16
16
|
name: 'start',
|
|
17
17
|
description: 'start the daemon in the background',
|
|
18
|
-
whenToUse: 'bringing the crtrd supervisor up for the first time in a session so node window-exits get handled (no-op
|
|
18
|
+
whenToUse: 'bringing the crtrd supervisor up for the first time in a session so node window-exits get handled (no-op only when its Unix API socket already serves)',
|
|
19
19
|
help: {
|
|
20
20
|
name: 'sys daemon start',
|
|
21
|
-
summary: 'start the crtrd supervisor daemon in the background (no-op if
|
|
21
|
+
summary: 'start the crtrd supervisor daemon in the background (no-op only if its Unix API socket serves)',
|
|
22
22
|
params: [],
|
|
23
23
|
output: [
|
|
24
24
|
{ name: 'started', type: 'boolean', required: true, constraint: 'True when a new daemon process was spawned; false when one was already running.' },
|
|
25
25
|
{ name: 'pid', type: 'number', required: false, constraint: 'PID of the newly spawned daemon.' },
|
|
26
26
|
{ name: 'existing_pid', type: 'number', required: false, constraint: 'PID of the already-running daemon (when started=false).' },
|
|
27
|
+
{ name: 'running', type: 'boolean', required: false, constraint: 'True only when a live pidfile owner does not serve its Unix API socket.' },
|
|
28
|
+
{ name: 'serving', type: 'boolean', required: false, constraint: 'False only when a live pidfile owner does not answer /healthz.' },
|
|
29
|
+
{ name: 'next', type: 'string', required: false, constraint: 'Recovery instruction when running=true and serving=false.' },
|
|
27
30
|
],
|
|
28
31
|
outputKind: 'object',
|
|
29
32
|
effects: [
|
|
@@ -46,18 +49,28 @@ const daemonStatus = defineLeaf({
|
|
|
46
49
|
summary: 'check whether the crtrd supervisor daemon is currently running',
|
|
47
50
|
params: [],
|
|
48
51
|
output: [
|
|
49
|
-
{ name: 'running', type: 'boolean', required: true, constraint: 'True when the daemon is alive.' },
|
|
52
|
+
{ name: 'running', type: 'boolean', required: true, constraint: 'True when the daemon pidfile owner is alive.' },
|
|
53
|
+
{ name: 'serving', type: 'boolean', required: false, constraint: 'True when the live daemon answers /healthz; false when it does not.' },
|
|
50
54
|
{ name: 'pid', type: 'number', required: false, constraint: 'PID of the running daemon.' },
|
|
55
|
+
{ name: 'next', type: 'string', required: false, constraint: 'Recovery instruction when running=true and serving=false.' },
|
|
51
56
|
],
|
|
52
57
|
outputKind: 'object',
|
|
53
|
-
effects: ['Read-only: reads the pidfile
|
|
58
|
+
effects: ['Read-only: reads the pidfile, probes the pid, and requests /healthz without autostart.'],
|
|
54
59
|
},
|
|
55
60
|
run: async (_input) => {
|
|
56
61
|
const pid = readPidfile();
|
|
57
62
|
const running = pid !== null && isPidAlive(pid);
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
63
|
+
if (!running)
|
|
64
|
+
return { running: false };
|
|
65
|
+
const serving = await isDaemonServing();
|
|
66
|
+
return serving
|
|
67
|
+
? { running: true, serving: true, pid }
|
|
68
|
+
: {
|
|
69
|
+
running: true,
|
|
70
|
+
serving: false,
|
|
71
|
+
pid,
|
|
72
|
+
next: 'Run `crtr sys daemon stop` to release the non-serving daemon before starting it again.',
|
|
73
|
+
};
|
|
61
74
|
},
|
|
62
75
|
});
|
|
63
76
|
// daemon restart
|
|
@@ -178,8 +178,8 @@ const PACKAGE_COPY = {
|
|
|
178
178
|
},
|
|
179
179
|
'crtr-extensions': {
|
|
180
180
|
name: 'pi-crtr-extensions',
|
|
181
|
-
shortDescription: 'crtr companion extensions — command palette, help gating,
|
|
182
|
-
longDescription: 'Bundles the extensions built for crtr: a crtr slash-command palette, crtr/crouter help gating, surfacing of Claude Code marketplace plugin commands, nested AGENTS.md/CLAUDE.md loading,
|
|
181
|
+
shortDescription: 'crtr companion extensions — command palette, help gating, subscription/OAuth rotation, nested context.',
|
|
182
|
+
longDescription: 'Bundles the extensions built for crtr: a crtr slash-command palette, crtr/crouter help gating, surfacing of Claude Code marketplace plugin commands, nested AGENTS.md/CLAUDE.md loading, OAuth login + rotation across multiple Claude subscriptions (provider-rotation), and system-prompt trimming. The OAuth/rotation piece is what lets crtr authenticate your Claude account.',
|
|
183
183
|
},
|
|
184
184
|
'oauth-adapter': {
|
|
185
185
|
name: 'pi-claude-oauth-adapter',
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
import { EventProjection } from '../runtime/broker/event-projection.js';
|
|
4
|
+
import { RebindDriver } from '../runtime/broker/rebind.js';
|
|
5
|
+
function generationFor(session) {
|
|
6
|
+
return {
|
|
7
|
+
id: 1,
|
|
8
|
+
session,
|
|
9
|
+
fallback: { blocked: false },
|
|
10
|
+
stagedProviderAgentEnd: null,
|
|
11
|
+
stagedWatchdogAbort: false,
|
|
12
|
+
stagedRefreshAbort: false,
|
|
13
|
+
stagedOverflowFailure: null,
|
|
14
|
+
executingTools: new Set(),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
function projectionFor(session, generation) {
|
|
18
|
+
return new EventProjection({
|
|
19
|
+
registry: { broadcast: () => undefined },
|
|
20
|
+
faultRetry: { onEvent: () => undefined, clearTimer: () => undefined },
|
|
21
|
+
installedGeneration: () => generation,
|
|
22
|
+
currentSession: () => session,
|
|
23
|
+
notifyTurnAccepted: () => undefined,
|
|
24
|
+
emitStartupMilestone: () => undefined,
|
|
25
|
+
formatModelSpec: () => null,
|
|
26
|
+
refreshIntent: async () => false,
|
|
27
|
+
onRefreshIntentError: () => undefined,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
test('agent_start arms the provider watchdog before another engine event arrives', (t) => {
|
|
31
|
+
t.mock.timers.enable({ apis: ['setTimeout'] });
|
|
32
|
+
let aborts = 0;
|
|
33
|
+
const session = { isStreaming: true, abort: async () => { aborts++; } };
|
|
34
|
+
const generation = generationFor(session);
|
|
35
|
+
const projection = projectionFor(session, generation);
|
|
36
|
+
projection.relayEvent({ type: 'agent_start' }, generation);
|
|
37
|
+
t.mock.timers.tick(5 * 60_000);
|
|
38
|
+
assert.equal(aborts, 1, 'the initial streaming event leaves a watchdog floor even if the engine falls silent immediately');
|
|
39
|
+
assert.equal(generation.stagedWatchdogAbort, true, 'the floor retains watchdog provenance for provider-fault recovery');
|
|
40
|
+
});
|
|
41
|
+
test('tool execution replaces the provider watchdog with the long ceiling', (t) => {
|
|
42
|
+
t.mock.timers.enable({ apis: ['setTimeout'] });
|
|
43
|
+
let aborts = 0;
|
|
44
|
+
const session = { isStreaming: true, abort: async () => { aborts++; } };
|
|
45
|
+
const generation = generationFor(session);
|
|
46
|
+
const projection = projectionFor(session, generation);
|
|
47
|
+
projection.relayEvent({ type: 'agent_start' }, generation);
|
|
48
|
+
projection.relayEvent({ type: 'tool_execution_start', toolCallId: 'stuck-tool' }, generation);
|
|
49
|
+
t.mock.timers.tick(5 * 60_000);
|
|
50
|
+
assert.equal(aborts, 0, 'a tool outlives the ordinary five-minute provider watchdog');
|
|
51
|
+
t.mock.timers.tick(15 * 60_000);
|
|
52
|
+
assert.equal(aborts, 1, 'a silent in-process tool cannot clear all watchdog coverage indefinitely');
|
|
53
|
+
});
|
|
54
|
+
test('rebind rearms the watchdog floor for a still-streaming replacement session', async () => {
|
|
55
|
+
const calls = [];
|
|
56
|
+
const projection = {
|
|
57
|
+
reset: () => calls.push('reset'),
|
|
58
|
+
rearmIfStreaming: (generation) => calls.push(`rearm:${generation.id}`),
|
|
59
|
+
};
|
|
60
|
+
const session = {
|
|
61
|
+
isStreaming: true,
|
|
62
|
+
systemPrompt: '',
|
|
63
|
+
bindExtensions: async () => undefined,
|
|
64
|
+
subscribe: () => () => undefined,
|
|
65
|
+
};
|
|
66
|
+
const driver = new RebindDriver({
|
|
67
|
+
nodeId: 'watchdog-floor-rebind',
|
|
68
|
+
cfg: {},
|
|
69
|
+
session,
|
|
70
|
+
services: {},
|
|
71
|
+
runtime: undefined,
|
|
72
|
+
uiContext: () => ({}),
|
|
73
|
+
registry: {},
|
|
74
|
+
toolGroups: { resetForSession: () => undefined },
|
|
75
|
+
faultRetry: () => ({ reset: () => undefined }),
|
|
76
|
+
projection: () => projection,
|
|
77
|
+
setActiveSession: () => undefined,
|
|
78
|
+
disposeAndExit: () => undefined,
|
|
79
|
+
onSessionRebind: () => undefined,
|
|
80
|
+
reWelcomeAll: () => undefined,
|
|
81
|
+
});
|
|
82
|
+
await driver.enqueueRebind(session, {});
|
|
83
|
+
assert.deepEqual(calls, ['reset', 'rearm:1'], 'a rebind that resets projection state immediately restores coverage for its live stream');
|
|
84
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
import { createTurnAdmissionGate } from '../runtime/broker/turn-admission.js';
|
|
4
|
+
function deferred() {
|
|
5
|
+
let resolve;
|
|
6
|
+
const promise = new Promise((done) => {
|
|
7
|
+
resolve = done;
|
|
8
|
+
});
|
|
9
|
+
return { promise, resolve };
|
|
10
|
+
}
|
|
11
|
+
test('concurrent starts admit one turn and reject the loser without a spurious settlement', async () => {
|
|
12
|
+
const gate = createTurnAdmissionGate();
|
|
13
|
+
const preflight = deferred();
|
|
14
|
+
const turn = deferred();
|
|
15
|
+
const enteredPreflight = deferred();
|
|
16
|
+
let streaming = false;
|
|
17
|
+
let admittedTurns = 0;
|
|
18
|
+
let settlements = 0;
|
|
19
|
+
const prompt = async (release) => {
|
|
20
|
+
enteredPreflight.resolve();
|
|
21
|
+
await preflight.promise;
|
|
22
|
+
if (streaming) {
|
|
23
|
+
release();
|
|
24
|
+
throw new Error('Agent is already processing');
|
|
25
|
+
}
|
|
26
|
+
streaming = true;
|
|
27
|
+
admittedTurns++;
|
|
28
|
+
release();
|
|
29
|
+
await turn.promise;
|
|
30
|
+
streaming = false;
|
|
31
|
+
settlements++;
|
|
32
|
+
};
|
|
33
|
+
const winner = gate.admit(prompt);
|
|
34
|
+
await enteredPreflight.promise;
|
|
35
|
+
const loser = gate.admit(prompt);
|
|
36
|
+
preflight.resolve();
|
|
37
|
+
await assert.rejects(loser, /Agent is already processing/);
|
|
38
|
+
assert.equal(admittedTurns, 1, 'only the winner entered a turn');
|
|
39
|
+
assert.equal(settlements, 0, 'the rejected call did not settle the live turn');
|
|
40
|
+
assert.equal(streaming, true, 'the winner remains live after the loser rejects');
|
|
41
|
+
turn.resolve();
|
|
42
|
+
await winner;
|
|
43
|
+
assert.equal(settlements, 1, 'the admitted turn alone settled');
|
|
44
|
+
});
|
|
@@ -105,7 +105,11 @@ function ensureWatcherNode() {
|
|
|
105
105
|
cwd: '/tmp',
|
|
106
106
|
kind: 'general',
|
|
107
107
|
mode: 'base',
|
|
108
|
-
|
|
108
|
+
// Resident: these cases are about the watcher's own cursor/handoff/hold
|
|
109
|
+
// contract at each tier as sent. A terminal node's mail is raised to the
|
|
110
|
+
// steering tier on append (feed/inbox.ts), which would route every case
|
|
111
|
+
// here as a steer.
|
|
112
|
+
lifecycle: 'resident',
|
|
109
113
|
status: 'active',
|
|
110
114
|
});
|
|
111
115
|
recordPid(nodeId, process.pid);
|
|
@@ -48,9 +48,8 @@ function deadPid() {
|
|
|
48
48
|
const GONE_PANE = '%999999';
|
|
49
49
|
/** A booted child whose engine pid is DEAD and whose viewer pane is gone — the
|
|
50
50
|
* shape a crashed/finished child leaves behind. `pi_session_id` set so it is
|
|
51
|
-
* "booted"; `intent:null`
|
|
52
|
-
*
|
|
53
|
-
* alone must not touch it. */
|
|
51
|
+
* "booted"; `intent:null` selects policy row 6 (strict resume) when the
|
|
52
|
+
* authoritative tick observes the missing engine. */
|
|
54
53
|
function deadEngineChild(id, over = {}) {
|
|
55
54
|
createNode(node(id, {
|
|
56
55
|
pane: GONE_PANE,
|
|
@@ -86,7 +85,7 @@ after(() => {
|
|
|
86
85
|
delete process.env['CRTR_HOME'];
|
|
87
86
|
resetFleetForTests();
|
|
88
87
|
});
|
|
89
|
-
test('
|
|
88
|
+
test('a dead-engine booted child enters durable backoff before recovery, is NOT reaped, and fans NO wake to its dormant parent', async () => {
|
|
90
89
|
const originalLaunch = headlessBrokerHost.launch;
|
|
91
90
|
const launches = [];
|
|
92
91
|
headlessBrokerHost.launch = (nodeId) => {
|
|
@@ -100,9 +99,14 @@ test('exit-event authority: a dead-engine booted child is recovered, NOT reaped,
|
|
|
100
99
|
deadEngineChild('CHILD', { cwd: home }); // a cwd that exists — reviveNode's preflight refuses a missing one
|
|
101
100
|
subscribe('PARENT', 'CHILD', true); // the spawn-time spine edge
|
|
102
101
|
await superviseTick();
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
assert.
|
|
102
|
+
const afterFirstTick = getNode('CHILD');
|
|
103
|
+
assert.deepEqual(launches, [], 'the first tick records the failed launch without relaunching');
|
|
104
|
+
assert.equal(afterFirstTick.respawn_failures, 1, 'the first tick records failure one durably');
|
|
105
|
+
assert.ok(afterFirstTick.respawn_not_before, 'the first tick records durable backoff');
|
|
106
|
+
await superviseTick(Date.parse(afterFirstTick.respawn_not_before));
|
|
107
|
+
// A stale recorded pid is metadata, not a verdict: once backoff matures,
|
|
108
|
+
// the row is relaunched on its saved session, never reaped to dead/done/idle.
|
|
109
|
+
assert.deepEqual(launches, ['CHILD'], 'the matured tick recovers the dead engine through the policy table');
|
|
106
110
|
assert.equal(getNode('CHILD').status, 'active', 'CHILD is recovered, NOT reaped');
|
|
107
111
|
assert.equal(readInboxSince('PARENT').length, 0, 'PARENT inbox EMPTY — no daemon liveness wake');
|
|
108
112
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { after, before, beforeEach, test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { createNode, getRow, isCollectible, setFrozen, transitionStatusIntent, } from '../canvas/canvas.js';
|
|
7
|
+
import { closeDb } from '../canvas/db.js';
|
|
8
|
+
let home;
|
|
9
|
+
function node(id, over = {}) {
|
|
10
|
+
return {
|
|
11
|
+
node_id: id,
|
|
12
|
+
name: id,
|
|
13
|
+
created: new Date().toISOString(),
|
|
14
|
+
cwd: '/tmp/work',
|
|
15
|
+
kind: 'general',
|
|
16
|
+
mode: 'base',
|
|
17
|
+
lifecycle: 'terminal',
|
|
18
|
+
status: 'active',
|
|
19
|
+
...over,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
function terminalRow(id, reason, status = 'done') {
|
|
23
|
+
createNode(node(id));
|
|
24
|
+
transitionStatusIntent(id, { status, terminal_reason: reason });
|
|
25
|
+
return getRow(id);
|
|
26
|
+
}
|
|
27
|
+
before(() => {
|
|
28
|
+
home = mkdtempSync(join(tmpdir(), 'crtr-collectible-'));
|
|
29
|
+
process.env['CRTR_HOME'] = home;
|
|
30
|
+
});
|
|
31
|
+
beforeEach(() => {
|
|
32
|
+
closeDb();
|
|
33
|
+
rmSync(home, { recursive: true, force: true });
|
|
34
|
+
});
|
|
35
|
+
after(() => {
|
|
36
|
+
closeDb();
|
|
37
|
+
rmSync(home, { recursive: true, force: true });
|
|
38
|
+
delete process.env['CRTR_HOME'];
|
|
39
|
+
});
|
|
40
|
+
test('isCollectible applies the terminal reason truth table and absolute retains', () => {
|
|
41
|
+
const terminalCases = [
|
|
42
|
+
['finalized', 'done', true, true],
|
|
43
|
+
['finished', 'done', true, true],
|
|
44
|
+
['parked', 'done', true, true],
|
|
45
|
+
['closed', 'dead', true, true],
|
|
46
|
+
['retired', 'canceled', true, true],
|
|
47
|
+
['crashed', 'dead', false, true],
|
|
48
|
+
['stranded', 'canceled', false, true],
|
|
49
|
+
[null, 'done', false, true],
|
|
50
|
+
];
|
|
51
|
+
for (const [reason, status, beforeTtl, pastTtl] of terminalCases) {
|
|
52
|
+
const row = terminalRow(`reason-${reason ?? 'null'}`, reason, status);
|
|
53
|
+
assert.equal(isCollectible(row), beforeTtl, `${reason ?? 'null'} before TTL`);
|
|
54
|
+
assert.equal(isCollectible(row, { pastTtl: true }), pastTtl, `${reason ?? 'null'} past TTL`);
|
|
55
|
+
}
|
|
56
|
+
const frozen = terminalRow('frozen', 'closed');
|
|
57
|
+
setFrozen(frozen.node_id, new Date().toISOString());
|
|
58
|
+
createNode(node('worktree', {
|
|
59
|
+
managed_worktree: {
|
|
60
|
+
state: 'open',
|
|
61
|
+
path: '/tmp/worktree',
|
|
62
|
+
branch: 'crtr/worktree',
|
|
63
|
+
repo_root: '/tmp/repo',
|
|
64
|
+
base_ref: 'main',
|
|
65
|
+
base_sha: 'base',
|
|
66
|
+
created: new Date().toISOString(),
|
|
67
|
+
},
|
|
68
|
+
}));
|
|
69
|
+
transitionStatusIntent('worktree', { status: 'done', terminal_reason: 'closed' });
|
|
70
|
+
const caller = terminalRow('caller', 'closed');
|
|
71
|
+
createNode(node('active'));
|
|
72
|
+
const nonTerminal = getRow('active');
|
|
73
|
+
const absoluteRows = [getRow(frozen.node_id), getRow('worktree'), caller, nonTerminal];
|
|
74
|
+
const previousNodeId = process.env['CRTR_NODE_ID'];
|
|
75
|
+
process.env['CRTR_NODE_ID'] = 'caller';
|
|
76
|
+
try {
|
|
77
|
+
for (const row of absoluteRows) {
|
|
78
|
+
assert.equal(isCollectible(row), false, `${row.node_id} before TTL`);
|
|
79
|
+
assert.equal(isCollectible(row, { pastTtl: true }), false, `${row.node_id} past TTL`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
finally {
|
|
83
|
+
if (previousNodeId === undefined)
|
|
84
|
+
delete process.env['CRTR_NODE_ID'];
|
|
85
|
+
else
|
|
86
|
+
process.env['CRTR_NODE_ID'] = previousNodeId;
|
|
87
|
+
}
|
|
88
|
+
});
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// 1. Situational context rides the bearings as a sibling block, and only
|
|
3
3
|
// once something has been set.
|
|
4
4
|
// 2. canvas-context-intro injects the block as its own session message at
|
|
5
|
-
// session_start (before the first prompt), idempotent across resumes,
|
|
6
|
-
// inert without a node id
|
|
5
|
+
// session_start (before the first prompt), idempotent across resumes, and
|
|
6
|
+
// inert without a node id.
|
|
7
7
|
//
|
|
8
8
|
// Run: node --import tsx/esm --test src/core/__tests__/context-intro.test.ts
|
|
9
9
|
import { test, before, beforeEach, after, afterEach } from 'node:test';
|
|
@@ -15,7 +15,7 @@ import { closeDb } from '../canvas/db.js';
|
|
|
15
15
|
import { apiSocketPath } from '../canvas/paths.js';
|
|
16
16
|
import { spawnNode } from '../runtime/nodes.js';
|
|
17
17
|
import { appendSituationalContext, formatSituationalProse, situationalContextEnvelope, } from '../runtime/situational-context.js';
|
|
18
|
-
import registerCanvasContextIntro, {
|
|
18
|
+
import registerCanvasContextIntro, { CONTEXT_INTRO_CUSTOM_TYPE, } from '../../pi-extensions/canvas-context-intro.js';
|
|
19
19
|
import { buildContextBearings as buildContextIntro } from '../runtime/bearings.js';
|
|
20
20
|
import { createApiServer } from '../../daemon/api/server.js';
|
|
21
21
|
let home;
|
|
@@ -61,18 +61,16 @@ function makeFakePi(entries = []) {
|
|
|
61
61
|
return {
|
|
62
62
|
sent: [],
|
|
63
63
|
entries,
|
|
64
|
-
renderers: {},
|
|
65
64
|
on(e, h) { if (e === 'session_start')
|
|
66
65
|
this.handler = h; },
|
|
67
66
|
sendMessage(m) {
|
|
68
67
|
this.sent.push(m);
|
|
69
68
|
this.entries.push({ type: 'custom_message', ...m });
|
|
70
69
|
},
|
|
71
|
-
registerMessageRenderer(t, r) { this.renderers[t] = r; },
|
|
72
70
|
};
|
|
73
71
|
}
|
|
74
72
|
function fakeCtx(pi) {
|
|
75
|
-
return { sessionManager: {
|
|
73
|
+
return { sessionManager: { getBranch: () => pi.entries } };
|
|
76
74
|
}
|
|
77
75
|
test('session_start injects the block as the first message of an empty session', async () => {
|
|
78
76
|
const meta = spawnNode({ kind: 'general', cwd: '/tmp/work', parent: null });
|
|
@@ -111,38 +109,3 @@ test('session_start is inert when CRTR_NODE_ID is absent', async () => {
|
|
|
111
109
|
await pi.handler({ reason: 'startup' }, fakeCtx(pi));
|
|
112
110
|
assert.equal(pi.sent.length, 0);
|
|
113
111
|
});
|
|
114
|
-
test('renderer registered for the customType; collapsed hides body, ctrl+o expand reveals it', () => {
|
|
115
|
-
const pi = makeFakePi();
|
|
116
|
-
registerCanvasContextIntro(pi);
|
|
117
|
-
const renderer = pi.renderers[CONTEXT_INTRO_CUSTOM_TYPE];
|
|
118
|
-
assert.ok(renderer, 'message renderer registered for crtr-context');
|
|
119
|
-
const body = '<crtr-bearings>\nshared document store\n</crtr-bearings>';
|
|
120
|
-
const message = { customType: CONTEXT_INTRO_CUSTOM_TYPE, content: body };
|
|
121
|
-
const theme = {}; // no fg → plain text, easy to assert
|
|
122
|
-
// Collapsed (default): a single stub line, NONE of the body.
|
|
123
|
-
const collapsed = renderContextMessage(message, { expanded: false }, theme).render(80);
|
|
124
|
-
assert.equal(collapsed.length, 1, 'collapsed is one line');
|
|
125
|
-
assert.match(collapsed[0], /ctrl\+o to expand/);
|
|
126
|
-
assert.ok(!collapsed.join('\n').includes('shared document store'), 'body hidden when collapsed');
|
|
127
|
-
// Expanded (Ctrl+O): label + full body.
|
|
128
|
-
const expanded = renderContextMessage(message, { expanded: true }, theme).render(80);
|
|
129
|
-
const joined = expanded.join('\n');
|
|
130
|
-
assert.ok(joined.includes('shared document store'), 'body shown when expanded');
|
|
131
|
-
assert.ok(joined.includes(`[${CONTEXT_INTRO_CUSTOM_TYPE}]`), 'expanded shows the label');
|
|
132
|
-
});
|
|
133
|
-
test('renderer never emits a line wider than the terminal (truncates the collapsed stub)', () => {
|
|
134
|
-
// Regression: a fixed-width stub crashed pi's TUI at narrow widths
|
|
135
|
-
// ("Rendered line 1 exceeds terminal width"). Every emitted line — collapsed
|
|
136
|
-
// or expanded — must fit within the width handed to render().
|
|
137
|
-
const body = Array.from({ length: 6 }, () => 'x'.repeat(200)).join('\n');
|
|
138
|
-
const message = { customType: CONTEXT_INTRO_CUSTOM_TYPE, content: body };
|
|
139
|
-
const theme = {}; // no fg → plain text, so .length == visible width
|
|
140
|
-
for (const w of [1, 2, 5, 10, 20, 42, 80]) {
|
|
141
|
-
for (const expanded of [false, true]) {
|
|
142
|
-
const lines = renderContextMessage(message, { expanded }, theme).render(w);
|
|
143
|
-
for (const line of lines) {
|
|
144
|
-
assert.ok([...line].length <= w, `width=${w} expanded=${expanded}: line ${[...line].length} cols exceeds ${w}: ${JSON.stringify(line)}`);
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
});
|
|
@@ -62,23 +62,24 @@ after(() => {
|
|
|
62
62
|
});
|
|
63
63
|
// A broker that exits while pi_session_id was NEVER set: the vehicle died
|
|
64
64
|
// before pi ever booted. The spawner already returned status="active", so the
|
|
65
|
-
//
|
|
66
|
-
//
|
|
67
|
-
|
|
68
|
-
test('never-booted broker exit is marked dead and surfaced to the parent as urgent', async () => {
|
|
65
|
+
// authoritative tick applies table row 8 and tells the parent. The delivered
|
|
66
|
+
// exit is observation-only.
|
|
67
|
+
test('never-booted broker is marked dead by the tick and surfaced to the parent as urgent', async () => {
|
|
69
68
|
createNode(node('P', { kind: 'developer', lifecycle: 'resident' }));
|
|
70
69
|
createNode(node('C', {
|
|
71
70
|
parent: 'P',
|
|
72
71
|
kind: 'explore',
|
|
72
|
+
pi_pid: deadPid(), // launch recorded its process before it died
|
|
73
73
|
pi_session_id: null, // never booted
|
|
74
74
|
intent: null,
|
|
75
75
|
}));
|
|
76
76
|
spawnEdge('P', 'C');
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
// claim → boot failure. No grace, no retry.
|
|
77
|
+
// The launch's real child exit is delivered synthetically here, then the
|
|
78
|
+
// authoritative tick classifies the dead recorded pid as table row 8:
|
|
79
|
+
// pi_session_id null, no fork claim → boot failure. No grace, no retry.
|
|
80
80
|
tf.fleet.deliverExit('C', { code: 1, signal: null });
|
|
81
81
|
await tf.drain();
|
|
82
|
+
await superviseTick();
|
|
82
83
|
assert.equal(getNode('C').status, 'dead', 'never-booted child is dead');
|
|
83
84
|
// Row 8 fans TWO entries: the urgent surfaceBootFailure push and the normal
|
|
84
85
|
// doctrine wake (design flow B row 8 — crash + surfaceBootFailure + doctrine
|
|
@@ -115,7 +116,12 @@ test('a crashed booted child is relaunched on its saved session, not reaped, and
|
|
|
115
116
|
spawnEdge('P2', 'C2');
|
|
116
117
|
markBusy('C2'); // died mid-generation
|
|
117
118
|
await superviseTick();
|
|
118
|
-
|
|
119
|
+
const afterFirstTick = getNode('C2');
|
|
120
|
+
assert.deepEqual(launches, [], 'the first tick records the failed launch without relaunching');
|
|
121
|
+
assert.equal(afterFirstTick.respawn_failures, 1, 'the first tick records failure one durably');
|
|
122
|
+
assert.ok(afterFirstTick.respawn_not_before, 'the first tick records durable backoff');
|
|
123
|
+
await superviseTick(Date.parse(afterFirstTick.respawn_not_before));
|
|
124
|
+
assert.deepEqual(launches, ['C2'], 'the matured tick recovers the crash through the policy table');
|
|
119
125
|
assert.equal(getNode('C2').status, 'active', 'crashed booted child is relaunched, NOT marked dead');
|
|
120
126
|
assert.equal(readInboxSince('P2').length, 0, 'no false boot-failure / child-death push on a crash (wake relocated to push/close)');
|
|
121
127
|
clearBusy('C2');
|
|
@@ -155,19 +161,21 @@ test('idle-release node with a boot-cleared (null) pid and no unseen inbox is se
|
|
|
155
161
|
assert.equal(after.intent, null, 'no revive fired — the row is settled, not relaunched');
|
|
156
162
|
assert.equal(after.pi_pid, null, 'no fresh pid — reviveNode was never invoked');
|
|
157
163
|
});
|
|
158
|
-
// Daemon start guard: spawnDaemon must not report success before the daemon
|
|
159
|
-
//
|
|
160
|
-
// injected clocks/probes, so we can lock it down deterministically
|
|
161
|
-
// real daemon process.
|
|
162
|
-
test('verifyDaemonStartup waits for pidfile
|
|
164
|
+
// Daemon start guard: spawnDaemon must not report success before the daemon's
|
|
165
|
+
// pidfile owner is live AND its Unix API socket serves. This is a pure bounded
|
|
166
|
+
// poll over injected clocks/probes, so we can lock it down deterministically
|
|
167
|
+
// without a real daemon process.
|
|
168
|
+
test('verifyDaemonStartup waits for pidfile, live pid, and serving socket before reporting success', async () => {
|
|
163
169
|
let now = 0;
|
|
164
170
|
let pidfileReady = false;
|
|
165
171
|
let pidAliveReady = false;
|
|
172
|
+
let socketServing = false;
|
|
166
173
|
let slept = 0;
|
|
167
174
|
const existingPid = await verifyDaemonStartup(42, 50, {
|
|
168
175
|
now: () => now,
|
|
169
176
|
readPidfile: () => (pidfileReady ? 42 : null),
|
|
170
177
|
isPidAlive: () => pidAliveReady,
|
|
178
|
+
isDaemonServing: async () => socketServing,
|
|
171
179
|
sleepMs: (ms) => {
|
|
172
180
|
slept += ms;
|
|
173
181
|
now += ms;
|
|
@@ -175,12 +183,15 @@ test('verifyDaemonStartup waits for pidfile + live pid, then reports success for
|
|
|
175
183
|
pidfileReady = true;
|
|
176
184
|
if (now >= 30)
|
|
177
185
|
pidAliveReady = true;
|
|
186
|
+
if (now >= 40)
|
|
187
|
+
socketServing = true;
|
|
178
188
|
},
|
|
179
189
|
});
|
|
180
190
|
assert.equal(existingPid, null, 'the spawned daemon owns the pidfile when startup succeeds');
|
|
181
191
|
assert.equal(pidfileReady, true, 'pidfile must have appeared before success');
|
|
182
192
|
assert.equal(pidAliveReady, true, 'pid must have been alive before success');
|
|
183
|
-
assert.equal(
|
|
193
|
+
assert.equal(socketServing, true, 'the Unix API socket must serve before success');
|
|
194
|
+
assert.equal(slept >= 40, true, 'the guard must actually poll through socket readiness');
|
|
184
195
|
});
|
|
185
196
|
test('runDaemon awaits activation before ownership and readiness', () => {
|
|
186
197
|
const source = readFileSync(fileURLToPath(new URL('../../daemon/crtrd.ts', import.meta.url)), 'utf8');
|
|
@@ -204,6 +215,7 @@ test('verifyDaemonStartup default window tolerates a slower cold start', async (
|
|
|
204
215
|
now: () => now,
|
|
205
216
|
readPidfile: () => (pidfileReady ? 42 : null),
|
|
206
217
|
isPidAlive: () => pidAliveReady,
|
|
218
|
+
isDaemonServing: async () => pidAliveReady,
|
|
207
219
|
sleepMs: (ms) => {
|
|
208
220
|
now += ms;
|
|
209
221
|
if (now >= 600)
|
|
@@ -221,6 +233,7 @@ test('startup verification waits for delayed activation inside the shared window
|
|
|
221
233
|
now: () => now,
|
|
222
234
|
readPidfile: () => (activationComplete ? 42 : null),
|
|
223
235
|
isPidAlive: () => activationComplete,
|
|
236
|
+
isDaemonServing: async () => activationComplete,
|
|
224
237
|
sleepMs: (ms) => {
|
|
225
238
|
now += ms;
|
|
226
239
|
if (now >= 25_000)
|
|
@@ -239,6 +252,7 @@ test('verifyDaemonStartup returns the existing live daemon when another pid won
|
|
|
239
252
|
now: () => now,
|
|
240
253
|
readPidfile: () => 7,
|
|
241
254
|
isPidAlive: (pid) => pid === 7,
|
|
255
|
+
isDaemonServing: async () => true,
|
|
242
256
|
sleepMs: (ms) => {
|
|
243
257
|
slept += ms;
|
|
244
258
|
now += ms;
|
|
@@ -276,6 +290,7 @@ test('verifyDaemonStartup throws after its tiny startup window expires', async (
|
|
|
276
290
|
now: () => now,
|
|
277
291
|
readPidfile: () => null,
|
|
278
292
|
isPidAlive: () => false,
|
|
293
|
+
isDaemonServing: async () => false,
|
|
279
294
|
sleepMs: (ms) => {
|
|
280
295
|
now += ms;
|
|
281
296
|
},
|
|
@@ -299,6 +314,7 @@ test('verifyDaemonStartup throws when the spawned daemon exits before it can bec
|
|
|
299
314
|
await assert.rejects(async () => verifyDaemonStartup(77, 2_000, {
|
|
300
315
|
readPidfile: () => null,
|
|
301
316
|
isPidAlive: () => false,
|
|
317
|
+
isDaemonServing: async () => false,
|
|
302
318
|
childExited: () => exited,
|
|
303
319
|
}), /exited before becoming ready \(with exit code 1\)/);
|
|
304
320
|
assert.ok(Date.now() - started < 1_500, 'must fail fast on the real child exit, not run out the window');
|
|
@@ -1,41 +1,47 @@
|
|
|
1
1
|
// Run with: node --import tsx/esm --test src/core/__tests__/daemon-wedge.test.ts
|
|
2
2
|
//
|
|
3
3
|
// Issue #110 — pid-alive WEDGED node detection. The pure decision core:
|
|
4
|
-
// wedgeVerdict(busy, quietForMs, cpuPercent). No wall-clock sleeps, no ps
|
|
4
|
+
// wedgeVerdict(busy, quietForMs, cpuPercent, hasDescendants). No wall-clock sleeps, no ps
|
|
5
5
|
// shell-outs, no fake pids — every input is injected directly, mirroring the
|
|
6
6
|
// yieldStallVerdict truth-table suite in refresh-stall-recycle.test.ts.
|
|
7
7
|
//
|
|
8
|
-
// LOCKS: (1) a routine long-running
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
8
|
+
// LOCKS: (1) a routine long-running turn with a CPU-active subprocess must
|
|
9
|
+
// never verdict 'wedged' — false positives are the failure mode this feature
|
|
10
|
+
// exists to avoid. (2) a childless busy broker that is quiet past grace is
|
|
11
|
+
// wedged even when its own CPU is high. (3) a null/failed CPU sample never
|
|
12
12
|
// fires on its own (fails toward 'pending', not 'wedged').
|
|
13
13
|
import { test } from 'node:test';
|
|
14
14
|
import assert from 'node:assert/strict';
|
|
15
15
|
import { wedgeVerdict, WEDGE_QUIET_MS, WEDGE_CPU_MAX_PERCENT, sumTreeCpu, treeHasDescendants, } from '../../daemon/reconcilers/broker-supervision.js';
|
|
16
16
|
test('wedgeVerdict: a node whose turn already ended is never wedged, regardless of the other signals', () => {
|
|
17
|
-
assert.equal(wedgeVerdict({ busy: false, quietForMs: null, cpuPercent: null }), 'leave');
|
|
18
|
-
assert.equal(wedgeVerdict({ busy: false, quietForMs: 10 * 60 * 60_000, cpuPercent: 0 }), 'leave');
|
|
17
|
+
assert.equal(wedgeVerdict({ busy: false, quietForMs: null, cpuPercent: null, hasDescendants: null }), 'leave');
|
|
18
|
+
assert.equal(wedgeVerdict({ busy: false, quietForMs: 10 * 60 * 60_000, cpuPercent: 0, hasDescendants: null }), 'leave');
|
|
19
19
|
});
|
|
20
20
|
test('wedgeVerdict: a busy node with an unreadable heartbeat is left alone (not enough evidence)', () => {
|
|
21
|
-
assert.equal(wedgeVerdict({ busy: true, quietForMs: null, cpuPercent: null }), 'leave');
|
|
21
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: null, cpuPercent: null, hasDescendants: null }), 'leave');
|
|
22
22
|
});
|
|
23
23
|
test('wedgeVerdict: routine turns — even long ones — never trip before the grace elapses', () => {
|
|
24
|
-
assert.equal(wedgeVerdict({ busy: true, quietForMs: 0, cpuPercent: 0 }), 'leave', 'just started');
|
|
25
|
-
assert.equal(wedgeVerdict({ busy: true, quietForMs: 5 * 60_000, cpuPercent: 0 }), 'leave', 'a routine multi-minute tool call');
|
|
26
|
-
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS - 1, cpuPercent: 0 }), 'leave', 'just under the grace, even with 0% cpu');
|
|
24
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: 0, cpuPercent: 0, hasDescendants: null }), 'leave', 'just started');
|
|
25
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: 5 * 60_000, cpuPercent: 0, hasDescendants: null }), 'leave', 'a routine multi-minute tool call');
|
|
26
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS - 1, cpuPercent: 0, hasDescendants: null }), 'leave', 'just under the grace, even with 0% cpu');
|
|
27
27
|
});
|
|
28
28
|
test('wedgeVerdict: past the grace with NO cpu corroboration only pends — never fires on time alone', () => {
|
|
29
|
-
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS, cpuPercent: null }), 'pending', 'cpu sample failed/unavailable — never treated as confirmation');
|
|
30
|
-
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS * 10, cpuPercent: null }), 'pending', 'no amount of extra quiet time substitutes for the corroborating signal');
|
|
29
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS, cpuPercent: null, hasDescendants: null }), 'pending', 'cpu sample failed/unavailable — never treated as confirmation');
|
|
30
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS * 10, cpuPercent: null, hasDescendants: null }), 'pending', 'no amount of extra quiet time substitutes for the corroborating signal');
|
|
31
31
|
});
|
|
32
|
-
test('wedgeVerdict:
|
|
33
|
-
assert.equal(wedgeVerdict({
|
|
34
|
-
|
|
32
|
+
test('wedgeVerdict: high CPU exonerates a broker with descendants but not a childless stalled engine', () => {
|
|
33
|
+
assert.equal(wedgeVerdict({
|
|
34
|
+
busy: true,
|
|
35
|
+
quietForMs: WEDGE_QUIET_MS * 3,
|
|
36
|
+
cpuPercent: WEDGE_CPU_MAX_PERCENT + 0.1,
|
|
37
|
+
hasDescendants: true,
|
|
38
|
+
}), 'pending', 'cpu just above the ceiling — a genuinely churning long build — must not fire');
|
|
39
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS * 3, cpuPercent: 40, hasDescendants: true }), 'pending', 'clearly-active child cpu never fires regardless of how quiet the heartbeat looks');
|
|
40
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS * 3, cpuPercent: 40, hasDescendants: false }), 'wedged', 'a childless broker spinning in-process is itself stalled');
|
|
35
41
|
});
|
|
36
|
-
test('wedgeVerdict: fires
|
|
37
|
-
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS, cpuPercent: 0 }), 'wedged');
|
|
38
|
-
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS + 60_000, cpuPercent: WEDGE_CPU_MAX_PERCENT }), 'wedged', 'at the cpu ceiling (inclusive)');
|
|
42
|
+
test('wedgeVerdict: fires when busy + quiet-past-grace + near-zero cpu all corroborate', () => {
|
|
43
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS, cpuPercent: 0, hasDescendants: true }), 'wedged');
|
|
44
|
+
assert.equal(wedgeVerdict({ busy: true, quietForMs: WEDGE_QUIET_MS + 60_000, cpuPercent: WEDGE_CPU_MAX_PERCENT, hasDescendants: true }), 'wedged', 'at the cpu ceiling (inclusive)');
|
|
39
45
|
});
|
|
40
46
|
// sumTreeCpu (pure) — process-tree CPU sampling. Review fix for issue #110:
|
|
41
47
|
// sampling only the broker pid false-fired on a broker awaiting a busy CHILD
|