@north-light/crouter 0.3.250 → 0.3.252
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin-memory/02-turn-lifecycle/00-ending-a-turn.md +1 -5
- package/dist/builtin-memory/05-kinds/review/security-findings.md +1 -1
- package/dist/clients/attach/__tests__/completion-frecency.test.d.ts +1 -0
- package/dist/clients/attach/__tests__/completion-frecency.test.js +35 -0
- package/dist/clients/attach/__tests__/ref-autocomplete.test.js +3 -1
- package/dist/clients/attach/__tests__/titled-editor-preview.test.js +2 -1
- package/dist/clients/attach/input/completion-frecency.d.ts +45 -0
- package/dist/clients/attach/input/completion-frecency.js +141 -0
- package/dist/clients/attach/input/controller.d.ts +17 -0
- package/dist/clients/attach/input/controller.js +40 -0
- package/dist/clients/attach/input/ref-autocomplete.d.ts +3 -1
- package/dist/clients/attach/input/ref-autocomplete.js +49 -8
- package/dist/clients/attach/input/titled-editor.d.ts +3 -0
- package/dist/clients/attach/input/titled-editor.js +5 -0
- package/dist/clients/attach/session/editor-inventory.d.ts +3 -0
- package/dist/clients/attach/session/editor-inventory.js +1 -1
- package/dist/clients/attach/session/input-wiring.d.ts +3 -0
- package/dist/clients/attach/session/input-wiring.js +2 -0
- package/dist/clients/attach/viewer.js +578 -578
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +98 -1
- package/dist/core/__tests__/canvas-inbox-watcher-naming.test.js +4 -1
- package/dist/core/__tests__/canvas-inbox-watcher.test.js +4 -1
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +3 -1
- package/dist/core/__tests__/fixtures/fake-engine.js +16 -3
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +4 -1
- package/dist/core/__tests__/migration.test.js +57 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +17 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -0
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
- package/dist/core/__tests__/watchdog-abort-arms-retry.test.js +9 -2
- package/dist/core/canvas/migrations.js +63 -24
- package/dist/core/runtime/broker/engine-drive.js +20 -3
- package/dist/core/runtime/broker/fault-retry.js +31 -13
- package/dist/core/runtime/broker/held-deferred-inbox.d.ts +13 -0
- package/dist/core/runtime/broker/held-deferred-inbox.js +13 -0
- package/dist/core/runtime/broker.js +19 -0
- package/dist/core/runtime/close.js +2 -2
- package/dist/core/runtime/fault.js +8 -5
- package/dist/core/runtime/recycle.js +63 -41
- package/dist/core/runtime/reset.d.ts +1 -1
- package/dist/core/runtime/reset.js +25 -2
- package/dist/daemon/__tests__/helpers/source-daemon.js +1 -0
- package/dist/daemon/api/handlers/messages.js +1 -1
- package/dist/daemon/api/handlers/reports.js +1 -1
- package/dist/daemon/cron/sinks.js +1 -1
- package/dist/daemon/messaging/node-message.js +1 -1
- package/dist/pi-extensions/canvas-inbox-watcher.js +109 -5
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
|
@@ -13,6 +13,9 @@ import registerCanvasInboxWatcher from '../../pi-extensions/canvas-inbox-watcher
|
|
|
13
13
|
import { appendInbox, readCursor } from '../feed/inbox.js';
|
|
14
14
|
import { drainBearings } from '../runtime/kickoff.js';
|
|
15
15
|
import { appendSituationalContext } from '../runtime/situational-context.js';
|
|
16
|
+
import { createEngineDriver } from '../runtime/broker/engine-drive.js';
|
|
17
|
+
import { createTurnAdmissionGate } from '../runtime/broker/turn-admission.js';
|
|
18
|
+
import { BROKER_IDLE_TURN_START, heldDeferredInboxPromptJoin, } from '../runtime/broker/held-deferred-inbox.js';
|
|
16
19
|
import { createNode, getNode, recordPid, setIntent } from '../canvas/canvas.js';
|
|
17
20
|
import { createProfile, pauseProfile, resumeProfile } from '../profiles/manifest.js';
|
|
18
21
|
import { closeDb } from '../canvas/db.js';
|
|
@@ -44,7 +47,7 @@ function makeFakePi() {
|
|
|
44
47
|
const handlers = {};
|
|
45
48
|
let resolveFirstInjection;
|
|
46
49
|
const firstInjection = new Promise((resolve) => { resolveFirstInjection = resolve; });
|
|
47
|
-
|
|
50
|
+
const pi = {
|
|
48
51
|
injected: [],
|
|
49
52
|
firstInjection,
|
|
50
53
|
sentMessages: [],
|
|
@@ -56,6 +59,8 @@ function makeFakePi() {
|
|
|
56
59
|
sendMessage(message) { this.sentMessages.push(message); },
|
|
57
60
|
async fire(e, ev, ctx) { await handlers[e]?.(ev, ctx); },
|
|
58
61
|
};
|
|
62
|
+
globalThis[BROKER_IDLE_TURN_START] = (content) => pi.sendUserMessage(content);
|
|
63
|
+
return pi;
|
|
59
64
|
}
|
|
60
65
|
// Fire agent_settled — the authoritative durable-commit boundary in the
|
|
61
66
|
// at-least-once model. `pending` maps to pi's `hasPendingMessages()` and is
|
|
@@ -275,6 +280,98 @@ describe('canvas inbox watcher — hold + idle delivery', () => {
|
|
|
275
280
|
assert.equal(pi.injected.length, 1);
|
|
276
281
|
assert.equal(pi.injected[0].deliverAs, undefined, 'idle → sendUserMessage triggers a turn, no deliverAs');
|
|
277
282
|
});
|
|
283
|
+
test('the broker prompt boundary prepends held deferred mail before the human body', async () => {
|
|
284
|
+
freshNode('node-deferred-engine-prompt');
|
|
285
|
+
const pi = makeFakePi();
|
|
286
|
+
disposers.push(registerWatcher(pi));
|
|
287
|
+
await wait(TICK_MS + 100);
|
|
288
|
+
const entry = appendInbox('node-deferred-engine-prompt', {
|
|
289
|
+
from: 'child-4',
|
|
290
|
+
tier: 'deferred',
|
|
291
|
+
kind: 'update',
|
|
292
|
+
label: 'held update',
|
|
293
|
+
});
|
|
294
|
+
await wait(SETTLE_MS);
|
|
295
|
+
const order = [];
|
|
296
|
+
const session = {
|
|
297
|
+
isStreaming: false,
|
|
298
|
+
sendCustomMessage: async ({ content }) => { order.push(content); },
|
|
299
|
+
prompt: async (text, options) => {
|
|
300
|
+
order.push(text);
|
|
301
|
+
options.preflightResult?.(true);
|
|
302
|
+
},
|
|
303
|
+
};
|
|
304
|
+
const drive = createEngineDriver({
|
|
305
|
+
registry: { sendFrame: () => undefined },
|
|
306
|
+
replies: {},
|
|
307
|
+
session: () => session,
|
|
308
|
+
dispatches: { track: (promise) => promise },
|
|
309
|
+
memoryRefs: { guidanceFor: () => null },
|
|
310
|
+
authReloadGate: { inFlight: () => undefined },
|
|
311
|
+
rebind: { installedGeneration: () => 0 },
|
|
312
|
+
projection: {},
|
|
313
|
+
toolGroups: {},
|
|
314
|
+
turnAdmission: createTurnAdmissionGate(),
|
|
315
|
+
});
|
|
316
|
+
await drive({}, { type: 'prompt', text: 'where are we?' });
|
|
317
|
+
assert.equal(order.length, 2, 'one held card and the one human prompt share the admitted operation');
|
|
318
|
+
assert.match(order[0], /held update/, 'the held inbox card is first');
|
|
319
|
+
assert.equal(order[1], 'where are we?', 'the human body follows the held card');
|
|
320
|
+
assert.equal(readCursor('node-deferred-engine-prompt'), undefined, 'the joined handoff waits for settlement');
|
|
321
|
+
await pi.fire('agent_start', { type: 'agent_start' }, { isIdle: () => false });
|
|
322
|
+
await settle(pi);
|
|
323
|
+
assert.equal(readCursor('node-deferred-engine-prompt'), entry.entry_id, 'agent_settled commits the broker-joined handoff');
|
|
324
|
+
const staleFollowUpEntry = appendInbox('node-deferred-engine-prompt', {
|
|
325
|
+
from: 'child-4',
|
|
326
|
+
tier: 'deferred',
|
|
327
|
+
kind: 'update',
|
|
328
|
+
label: 'held after a stale client snapshot',
|
|
329
|
+
});
|
|
330
|
+
await wait(SETTLE_MS);
|
|
331
|
+
const beforeFollowUp = order.length;
|
|
332
|
+
await drive({}, { type: 'follow_up', text: 'continue from there' });
|
|
333
|
+
assert.equal(order.length, beforeFollowUp + 2, 'an idle-routed follow_up shares the same prompt boundary');
|
|
334
|
+
assert.match(order.at(-2), /held after a stale client snapshot/, 'the held card still precedes the stale follow_up body');
|
|
335
|
+
assert.equal(order.at(-1), 'continue from there');
|
|
336
|
+
await pi.fire('agent_start', { type: 'agent_start' }, { isIdle: () => false });
|
|
337
|
+
await settle(pi);
|
|
338
|
+
assert.equal(readCursor('node-deferred-engine-prompt'), staleFollowUpEntry.entry_id, 'the m-C prompt settles its joined handoff too');
|
|
339
|
+
});
|
|
340
|
+
test('a held deferred batch is prepended to an admitted human prompt and replays if that prompt reloads before settlement', async () => {
|
|
341
|
+
freshNode('node-deferred-human-prompt');
|
|
342
|
+
const pi = makeFakePi();
|
|
343
|
+
disposers.push(registerWatcher(pi));
|
|
344
|
+
await wait(TICK_MS + 100);
|
|
345
|
+
const entry = appendInbox('node-deferred-human-prompt', {
|
|
346
|
+
from: 'child-4',
|
|
347
|
+
tier: 'deferred',
|
|
348
|
+
kind: 'update',
|
|
349
|
+
label: 'held update',
|
|
350
|
+
});
|
|
351
|
+
await wait(SETTLE_MS);
|
|
352
|
+
assert.equal(pi.injected.length, 0, 'deferred-only mail remains held while no turn starts');
|
|
353
|
+
const firstPromptPrefix = [];
|
|
354
|
+
await heldDeferredInboxPromptJoin()?.({
|
|
355
|
+
sendCustomMessage: async ({ content }) => { firstPromptPrefix.push(content); },
|
|
356
|
+
});
|
|
357
|
+
assert.equal(firstPromptPrefix.length, 1, 'the held card is sent before the human prompt starts');
|
|
358
|
+
assert.match(firstPromptPrefix[0], /held update/);
|
|
359
|
+
assert.equal(readCursor('node-deferred-human-prompt'), undefined, 'the pre-prompt handoff is not committed');
|
|
360
|
+
await shutdown(pi);
|
|
361
|
+
assert.equal(readCursor('node-deferred-human-prompt'), undefined, 'a reload before agent_settled leaves the held entry replayable');
|
|
362
|
+
const replacement = makeFakePi();
|
|
363
|
+
disposers.push(registerWatcher(replacement));
|
|
364
|
+
await wait(SETTLE_MS);
|
|
365
|
+
const replayedPrefix = [];
|
|
366
|
+
await heldDeferredInboxPromptJoin()?.({
|
|
367
|
+
sendCustomMessage: async ({ content }) => { replayedPrefix.push(content); },
|
|
368
|
+
});
|
|
369
|
+
assert.equal(replayedPrefix.length, 1, 'the replacement prompt receives the uncommitted held card again');
|
|
370
|
+
assert.match(replayedPrefix[0], /held update/);
|
|
371
|
+
await replacement.fire('agent_start', { type: 'agent_start' }, { isIdle: () => false });
|
|
372
|
+
await settle(replacement);
|
|
373
|
+
assert.equal(readCursor('node-deferred-human-prompt'), entry.entry_id, 'only agent_settled commits the prompt-joined handoff');
|
|
374
|
+
});
|
|
278
375
|
// Pause makes the profile inert while its broker is still alive: the entry
|
|
279
376
|
// must be RETAINED, not consumed. A critical entry is the sharpest case —
|
|
280
377
|
// its route aborts the live turn, exactly the punch-through pause forbids.
|
|
@@ -30,6 +30,7 @@ import { createNode } from '../canvas/canvas.js';
|
|
|
30
30
|
import { closeDb } from '../canvas/db.js';
|
|
31
31
|
import { apiSocketPath } from '../canvas/paths.js';
|
|
32
32
|
import { appendInbox } from '../feed/inbox.js';
|
|
33
|
+
import { BROKER_IDLE_TURN_START } from '../runtime/broker/held-deferred-inbox.js';
|
|
33
34
|
const TICK_MS = 20;
|
|
34
35
|
const DEBOUNCE_MS = 25;
|
|
35
36
|
let home;
|
|
@@ -76,7 +77,7 @@ function getRequestCount(nodeId) {
|
|
|
76
77
|
}
|
|
77
78
|
function makeFakePi() {
|
|
78
79
|
const handlers = {};
|
|
79
|
-
|
|
80
|
+
const pi = {
|
|
80
81
|
injected: [],
|
|
81
82
|
sentMessages: [],
|
|
82
83
|
on(e, h) { handlers[e] = h; },
|
|
@@ -84,6 +85,8 @@ function makeFakePi() {
|
|
|
84
85
|
sendMessage(message) { this.sentMessages.push(message); },
|
|
85
86
|
fire(e, ev, ctx) { handlers[e]?.(ev, ctx); },
|
|
86
87
|
};
|
|
88
|
+
globalThis[BROKER_IDLE_TURN_START] = (content) => pi.sendUserMessage(content);
|
|
89
|
+
return pi;
|
|
87
90
|
}
|
|
88
91
|
const wait = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
89
92
|
async function waitFor(predicate, timeoutMs = 3000, stepMs = 10) {
|
|
@@ -18,6 +18,7 @@ import { closeDb } from '../canvas/db.js';
|
|
|
18
18
|
import { appendInbox, coalesce, finalizeInboxEntry } from '../feed/inbox.js';
|
|
19
19
|
import { appendSituationalContext, SITUATIONAL_CONTEXT_CUSTOM_TYPE } from '../runtime/situational-context.js';
|
|
20
20
|
import { parseCard } from '../../shared/generated-context.js';
|
|
21
|
+
import { BROKER_IDLE_TURN_START } from '../runtime/broker/held-deferred-inbox.js';
|
|
21
22
|
// Drive the watcher's injectable cadence seam (CRTR_WATCHER_TICK_MS /
|
|
22
23
|
// CRTR_WATCHER_DEBOUNCE_MS) at a fast tempo so the watcher delivers in tens of
|
|
23
24
|
// milliseconds, not seconds — exactly the resolve+seed tick, read tick, and
|
|
@@ -40,7 +41,7 @@ function freshNode(nodeId) {
|
|
|
40
41
|
}
|
|
41
42
|
function makeFakePi() {
|
|
42
43
|
const handlers = {};
|
|
43
|
-
|
|
44
|
+
const pi = {
|
|
44
45
|
injected: [],
|
|
45
46
|
sentMessages: [],
|
|
46
47
|
on(e, h) { handlers[e] = h; },
|
|
@@ -48,6 +49,8 @@ function makeFakePi() {
|
|
|
48
49
|
sendMessage(message) { this.sentMessages.push(message); },
|
|
49
50
|
fire(e, ev, ctx) { handlers[e]?.(ev, ctx); },
|
|
50
51
|
};
|
|
52
|
+
globalThis[BROKER_IDLE_TURN_START] = (content) => pi.sendUserMessage(content);
|
|
53
|
+
return pi;
|
|
51
54
|
}
|
|
52
55
|
const wait = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
53
56
|
/** The text of a batch that must coalesce into exactly one delivery. */
|
|
@@ -193,7 +193,9 @@ declare class FakeSession {
|
|
|
193
193
|
getSessionStats(): Record<string, unknown>;
|
|
194
194
|
private syncTreeSnapshot;
|
|
195
195
|
setSessionName(_name: string): void;
|
|
196
|
-
prompt(text: string,
|
|
196
|
+
prompt(text: string, options?: {
|
|
197
|
+
preflightResult?: (success: boolean) => void;
|
|
198
|
+
}): Promise<void>;
|
|
197
199
|
/** The unrun queues the welcome snapshot reports (`snapshot.queued`). Real
|
|
198
200
|
* pi holds a steer + followUp queue and drains it when the next turn runs;
|
|
199
201
|
* the fake records the same texts and clears them in emitTurn, so a client
|
|
@@ -448,12 +448,18 @@ class FakeSession {
|
|
|
448
448
|
/* no-op recording stub */
|
|
449
449
|
}
|
|
450
450
|
// controller-drive stubs (no controller connects in these tests)
|
|
451
|
-
prompt(text,
|
|
451
|
+
prompt(text, options) {
|
|
452
452
|
// Rejection seam: the engine refuses the call itself (pi's no-provider-
|
|
453
453
|
// credential shape) — the promise rejects and NO lifecycle event fires.
|
|
454
454
|
if (text.includes(REJECT_PROMPT_CALL)) {
|
|
455
455
|
return Promise.reject(new Error('No API key found for anthropic'));
|
|
456
456
|
}
|
|
457
|
+
options?.preflightResult?.(true);
|
|
458
|
+
// The inbox watcher now enters through the broker-owned prompt bridge, not
|
|
459
|
+
// the Pi extension runtime. Keep its existing observable wake record for
|
|
460
|
+
// the compiled dormancy seam while the prompt drives the real lifecycle.
|
|
461
|
+
if (text.startsWith('<runtime kind="inbox"'))
|
|
462
|
+
this.recordInjected({ content: text });
|
|
457
463
|
// The broker calls this for the controller-less retry path. Emit the turn
|
|
458
464
|
// on the broker's subscribe channel and also run the real extension hooks so
|
|
459
465
|
// the canvas stophook records the fault marker and the broker can schedule
|
|
@@ -494,12 +500,19 @@ class FakeSession {
|
|
|
494
500
|
* a resolvable `/name` token must not crash on a missing method. */
|
|
495
501
|
customMessages = [];
|
|
496
502
|
sendCustomMessage(message, options) {
|
|
497
|
-
|
|
503
|
+
const customMessage = {
|
|
498
504
|
customType: message.customType,
|
|
499
505
|
content: message.content,
|
|
500
506
|
display: message.display,
|
|
501
507
|
deliverAs: options?.deliverAs,
|
|
502
|
-
}
|
|
508
|
+
};
|
|
509
|
+
this.customMessages.push(customMessage);
|
|
510
|
+
try {
|
|
511
|
+
appendFileSync(join(this.dir, 'fake-pi.custom-messages.jsonl'), JSON.stringify(customMessage) + '\n');
|
|
512
|
+
}
|
|
513
|
+
catch {
|
|
514
|
+
/* best effort test proof */
|
|
515
|
+
}
|
|
503
516
|
return Promise.resolve();
|
|
504
517
|
}
|
|
505
518
|
abort() {
|
|
@@ -50,6 +50,7 @@ import { canvasDbPath, nodeDir, apiSocketPath } from '../../canvas/paths.js';
|
|
|
50
50
|
import { closeDb } from '../../canvas/db.js';
|
|
51
51
|
import { createApiServer } from '../../../daemon/api/server.js';
|
|
52
52
|
import { CrtrClient } from '../../../api/index.js';
|
|
53
|
+
import { BROKER_IDLE_TURN_START } from '../../runtime/broker/held-deferred-inbox.js';
|
|
53
54
|
import { registerNode } from '../../../commands/node.js';
|
|
54
55
|
import { pushFinal } from '../../feed/feed.js';
|
|
55
56
|
import { InputError } from '../../io.js';
|
|
@@ -500,13 +501,15 @@ test('the lifecycle tick parks a resident holding deferred-only mail — mail th
|
|
|
500
501
|
});
|
|
501
502
|
function makeFakePi() {
|
|
502
503
|
const handlers = {};
|
|
503
|
-
|
|
504
|
+
const pi = {
|
|
504
505
|
injected: [],
|
|
505
506
|
on(e, h) { handlers[e] = h; },
|
|
506
507
|
sendUserMessage(content, options) { this.injected.push({ content, deliverAs: options?.deliverAs }); },
|
|
507
508
|
sendMessage() { },
|
|
508
509
|
async fire(e, ev, ctx) { await handlers[e]?.(ev, ctx); },
|
|
509
510
|
};
|
|
511
|
+
globalThis[BROKER_IDLE_TURN_START] = (content) => pi.sendUserMessage(content);
|
|
512
|
+
return pi;
|
|
510
513
|
}
|
|
511
514
|
const wait = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
512
515
|
test('the real inbox watcher holds a deferred entry alone, then delivers it TOGETHER with a later normal entry with no duplicate/loss, and the durable cursor settles past both (gh #341 review: genuine delivery, not just the daemon decision)', { timeout: 30_000 }, async () => {
|
|
@@ -215,36 +215,82 @@ test('a fresh db has the four additive runtime columns', () => {
|
|
|
215
215
|
assert.equal(rt['window'], null);
|
|
216
216
|
assert.equal(rt['tmux_session'], null);
|
|
217
217
|
});
|
|
218
|
-
test('session-file ownership migration
|
|
218
|
+
test('session-file ownership migration reconciles authoritative metas before clearing duplicate claimants', () => {
|
|
219
219
|
ensureHome();
|
|
220
220
|
const raw = new DatabaseSync(canvasDbPath());
|
|
221
221
|
migrateToVersion(raw, 35);
|
|
222
222
|
const sessionFile = '/tmp/preexisting-duplicate.jsonl';
|
|
223
223
|
const aliasedSessionFile = '/tmp/./preexisting-duplicate.jsonl';
|
|
224
|
-
|
|
224
|
+
const recoveredSessionFile = '/tmp/recovered-session.jsonl';
|
|
225
|
+
const metas = [
|
|
226
|
+
['first-owner', '2026-01-01T00:00:00.000Z', sessionFile],
|
|
227
|
+
['later-claimant', '2026-01-02T00:00:00.000Z', aliasedSessionFile],
|
|
228
|
+
// The durable meta write survived, but the derived row below is stale.
|
|
229
|
+
['recovered-claimant', '2026-01-03T00:00:00.000Z', recoveredSessionFile],
|
|
230
|
+
];
|
|
231
|
+
for (const [id, created, pi_session_file] of metas) {
|
|
225
232
|
const nodeDir = join(home, 'nodes', id);
|
|
226
233
|
mkdirSync(nodeDir, { recursive: true });
|
|
227
234
|
writeFileSync(join(nodeDir, 'meta.json'), JSON.stringify({
|
|
228
|
-
node_id: id, name: id, created:
|
|
229
|
-
cwd: '/tmp', kind: 'developer', mode: 'base', lifecycle: 'terminal', pi_session_id: `${id}-session`, pi_session_file: id === 'first-owner' ? sessionFile : aliasedSessionFile,
|
|
235
|
+
node_id: id, name: id, created, cwd: '/tmp', kind: 'developer', mode: 'base', lifecycle: 'terminal', pi_session_id: `${id}-session`, pi_session_file,
|
|
230
236
|
}));
|
|
231
237
|
}
|
|
232
|
-
raw.prepare('INSERT INTO nodes (node_id, name, kind, mode, lifecycle, status, cwd, created, pi_session_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
238
|
+
const insert = raw.prepare('INSERT INTO nodes (node_id, name, kind, mode, lifecycle, status, cwd, created, pi_session_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
|
|
239
|
+
insert.run('first-owner', 'first-owner', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-01T00:00:00.000Z', sessionFile);
|
|
240
|
+
insert.run('later-claimant', 'later-claimant', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-02T00:00:00.000Z', aliasedSessionFile);
|
|
241
|
+
insert.run('recovered-claimant', 'recovered-claimant', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-03T00:00:00.000Z', sessionFile);
|
|
236
242
|
assert.doesNotThrow(() => migrate(raw), 'a real canvas with pre-existing duplicates must migrate');
|
|
237
243
|
const rows = raw.prepare('SELECT node_id, pi_session_file FROM nodes ORDER BY created, node_id').all();
|
|
238
244
|
assert.deepEqual(rows.map((row) => [row.node_id, row.pi_session_file]), [
|
|
239
245
|
['first-owner', sessionFile],
|
|
240
246
|
['later-claimant', null],
|
|
241
|
-
|
|
247
|
+
['recovered-claimant', recoveredSessionFile],
|
|
248
|
+
], 'the oldest duplicate is retained while a stale row is reconciled from its authoritative meta');
|
|
242
249
|
const loser = JSON.parse(readFileSync(join(home, 'nodes', 'later-claimant', 'meta.json'), 'utf8'));
|
|
243
|
-
assert.equal(loser.pi_session_id, null, 'the loser meta cannot reindex its stale session id');
|
|
244
|
-
assert.equal(loser.pi_session_file, null, 'the loser meta cannot restore the duplicate projection');
|
|
250
|
+
assert.equal(loser.pi_session_id, null, 'the duplicate loser meta cannot reindex its stale session id');
|
|
251
|
+
assert.equal(loser.pi_session_file, null, 'the duplicate loser meta cannot restore the duplicate projection');
|
|
252
|
+
const recovered = JSON.parse(readFileSync(join(home, 'nodes', 'recovered-claimant', 'meta.json'), 'utf8'));
|
|
253
|
+
assert.equal(recovered.pi_session_id, 'recovered-claimant-session', 'a stale row cannot clear the authoritative session id');
|
|
254
|
+
assert.equal(recovered.pi_session_file, recoveredSessionFile, 'a stale row cannot clear the authoritative session file');
|
|
245
255
|
assert.throws(() => raw.prepare('UPDATE nodes SET pi_session_file = ? WHERE node_id = ?').run(sessionFile, 'later-claimant'), /UNIQUE/, 'the schema enforces the repaired invariant');
|
|
246
256
|
raw.close();
|
|
247
257
|
});
|
|
258
|
+
test('forward session-file repair reprojects a surviving v36 meta coordinate before restoring its UNIQUE index', () => {
|
|
259
|
+
ensureHome();
|
|
260
|
+
const raw = new DatabaseSync(canvasDbPath());
|
|
261
|
+
migrateToVersion(raw, 35);
|
|
262
|
+
const recoveredSessionFile = '/tmp/surviving-meta-coordinate.jsonl';
|
|
263
|
+
const staleSessionFile = '/tmp/stale-v36-coordinate.jsonl';
|
|
264
|
+
for (const [id, pi_session_id, pi_session_file] of [
|
|
265
|
+
['surviving-owner', 'surviving-session', recoveredSessionFile],
|
|
266
|
+
['stale-row', null, null],
|
|
267
|
+
]) {
|
|
268
|
+
const nodeDir = join(home, 'nodes', id);
|
|
269
|
+
mkdirSync(nodeDir, { recursive: true });
|
|
270
|
+
writeFileSync(join(nodeDir, 'meta.json'), JSON.stringify({
|
|
271
|
+
node_id: id, name: id, created: id === 'surviving-owner' ? '2026-01-01T00:00:00.000Z' : '2026-01-02T00:00:00.000Z',
|
|
272
|
+
cwd: '/tmp', kind: 'developer', mode: 'base', lifecycle: 'terminal', pi_session_id, pi_session_file,
|
|
273
|
+
}));
|
|
274
|
+
}
|
|
275
|
+
const insert = raw.prepare('INSERT INTO nodes (node_id, name, kind, mode, lifecycle, status, cwd, created, pi_session_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
|
|
276
|
+
// This is the v0.3.250 shape after v36: the owner’s surviving meta coordinate
|
|
277
|
+
// disagrees with its indexed row, while the row carrying it has no meta claim.
|
|
278
|
+
insert.run('surviving-owner', 'surviving-owner', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-01T00:00:00.000Z', staleSessionFile);
|
|
279
|
+
insert.run('stale-row', 'stale-row', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-02T00:00:00.000Z', recoveredSessionFile);
|
|
280
|
+
raw.exec('DROP INDEX IF EXISTS idx_nodes_pi_session_file; CREATE UNIQUE INDEX idx_nodes_pi_session_file ON nodes(pi_session_file); PRAGMA user_version = 36;');
|
|
281
|
+
assert.doesNotThrow(() => migrate(raw));
|
|
282
|
+
assert.equal(userVersion(raw), MIGRATIONS.length);
|
|
283
|
+
const rows = raw.prepare('SELECT node_id, pi_session_file FROM nodes ORDER BY created, node_id').all();
|
|
284
|
+
assert.deepEqual(rows.map((row) => [row.node_id, row.pi_session_file]), [
|
|
285
|
+
['surviving-owner', recoveredSessionFile],
|
|
286
|
+
['stale-row', null],
|
|
287
|
+
]);
|
|
288
|
+
const survivor = JSON.parse(readFileSync(join(home, 'nodes', 'surviving-owner', 'meta.json'), 'utf8'));
|
|
289
|
+
assert.equal(survivor.pi_session_id, 'surviving-session');
|
|
290
|
+
assert.equal(survivor.pi_session_file, recoveredSessionFile);
|
|
291
|
+
assert.throws(() => raw.prepare('UPDATE nodes SET pi_session_file = ? WHERE node_id = ?').run(recoveredSessionFile, 'stale-row'), /UNIQUE/);
|
|
292
|
+
raw.close();
|
|
293
|
+
});
|
|
248
294
|
test('node lifecycle substrate migration adds six columns and backfills terminal reasons', () => {
|
|
249
295
|
ensureHome();
|
|
250
296
|
const raw = new DatabaseSync(canvasDbPath());
|
|
@@ -148,6 +148,8 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
|
|
|
148
148
|
seedPool('openai-codex', false);
|
|
149
149
|
const pendingNode = await h.spawnHeadlessChild(parent, 'pending provider episode', { model: OPENAI_STRONG.spec });
|
|
150
150
|
await h.waitFor(() => agentStartCount(pendingNode) >= 1, { label: 'pending episode fixture started' });
|
|
151
|
+
const pendingKickoffObserver = await attachUntil(pendingNode, 'observer', 'pending-provider-kickoff-settled', (a) => a.welcome.snapshot.state.isStreaming === false, 'pending provider episode kickoff settled');
|
|
152
|
+
pendingKickoffObserver.close();
|
|
151
153
|
const pendingStarts = agentStartCount(pendingNode);
|
|
152
154
|
h.fakeCmd(pendingNode, {
|
|
153
155
|
cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 2_000,
|
|
@@ -160,6 +162,8 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
|
|
|
160
162
|
assert.equal(agentStartCount(pendingNode), pendingStarts + 1, 'a pending episode re-drives exactly once after broker death');
|
|
161
163
|
const admittedNode = await h.spawnHeadlessChild(parent, 'admitted provider episode', { model: OPENAI_STRONG.spec });
|
|
162
164
|
await h.waitFor(() => agentStartCount(admittedNode) >= 1, { label: 'admitted episode fixture started' });
|
|
165
|
+
const admittedKickoffObserver = await attachUntil(admittedNode, 'observer', 'admitted-provider-kickoff-settled', (a) => a.welcome.snapshot.state.isStreaming === false, 'admitted provider episode kickoff settled');
|
|
166
|
+
admittedKickoffObserver.close();
|
|
163
167
|
const admittedStarts = agentStartCount(admittedNode);
|
|
164
168
|
h.fakeCmd(admittedNode, {
|
|
165
169
|
cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 60_000,
|
|
@@ -174,6 +178,8 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
|
|
|
174
178
|
assert.equal(agentStartCount(admittedNode), admittedStarts, 'an admitted episode is never replayed by replacement startup');
|
|
175
179
|
const invalidatedNode = await h.spawnHeadlessChild(parent, 'invalidated provider episode', { model: OPENAI_STRONG.spec });
|
|
176
180
|
await h.waitFor(() => agentStartCount(invalidatedNode) >= 1, { label: 'invalidated episode fixture started' });
|
|
181
|
+
const invalidatedKickoffObserver = await attachUntil(invalidatedNode, 'observer', 'invalidated-provider-kickoff-settled', (a) => a.welcome.snapshot.state.isStreaming === false, 'invalidated provider episode kickoff settled');
|
|
182
|
+
invalidatedKickoffObserver.close();
|
|
177
183
|
const invalidatedStarts = agentStartCount(invalidatedNode);
|
|
178
184
|
h.fakeCmd(invalidatedNode, {
|
|
179
185
|
cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 60_000,
|
|
@@ -190,10 +196,13 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
|
|
|
190
196
|
assert.equal(agentStartCount(invalidatedNode), invalidatedStarts, 'an invalidated rebound-session episode is never replayed by replacement startup');
|
|
191
197
|
const cyclingNode = await h.spawnHeadlessChild(parent, 'fresh-cycle provider episode', { model: OPENAI_STRONG.spec });
|
|
192
198
|
await h.waitFor(() => agentStartCount(cyclingNode) >= 1, { label: 'fresh-cycle episode fixture started' });
|
|
199
|
+
const cyclingKickoffObserver = await attachUntil(cyclingNode, 'observer', 'fresh-cycle-provider-kickoff-settled', (a) => a.welcome.snapshot.state.isStreaming === false, 'fresh-cycle provider episode kickoff settled');
|
|
200
|
+
cyclingKickoffObserver.close();
|
|
193
201
|
h.fakeCmd(cyclingNode, {
|
|
194
|
-
cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() +
|
|
202
|
+
cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 2_000,
|
|
195
203
|
});
|
|
196
204
|
await h.waitFor(() => readProviderRetryEpisode(cyclingNode)?.state === 'pending' ? true : null, { label: 'fresh-cycle episode pending record persisted' });
|
|
205
|
+
const cyclingStarts = agentStartCount(cyclingNode);
|
|
197
206
|
const cyclingBoots = h.bootCount(cyclingNode);
|
|
198
207
|
const yieldResult = h.cli(cyclingNode, ['node', 'yield', 'begin a fresh cycle']);
|
|
199
208
|
assert.equal(yieldResult.code, 0, `node yield should succeed\n--stderr--\n${yieldResult.stderr}`);
|
|
@@ -207,7 +216,13 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
|
|
|
207
216
|
await h.tick(Date.parse(cyclingBackoff.respawn_not_before) + 1);
|
|
208
217
|
await h.awaitBoot(cyclingNode, { minCount: cyclingBoots + 1 });
|
|
209
218
|
await h.waitFor(() => existsSync(h.brokerSock(cyclingNode)), { label: 'fresh-cycle replacement broker listener bound' });
|
|
210
|
-
|
|
219
|
+
await h.waitFor(() => readProviderRetryEpisode(cyclingNode) === null, { label: 'fresh-cycle kickoff settled successfully' });
|
|
220
|
+
assert.equal(readProviderRetryEpisode(cyclingNode), null, 'the settled fresh-cycle kickoff clears the old provider episode');
|
|
221
|
+
assert.equal(agentStartCount(cyclingNode), cyclingStarts + 1, 'the fresh-cycle kickoff starts exactly once');
|
|
222
|
+
// A retained episode would re-arm immediately once its deadline becomes due.
|
|
223
|
+
// Wait past that poll interval to prove the completed kickoff cannot replay.
|
|
224
|
+
await new Promise((resolve) => setTimeout(resolve, 2500));
|
|
225
|
+
assert.equal(agentStartCount(cyclingNode), cyclingStarts + 1, 'the cleared fresh-cycle episode never replays');
|
|
211
226
|
});
|
|
212
227
|
test('reload_auth re-drives a parked fatal provider-auth turn exactly once', { timeout: 30_000 }, async () => {
|
|
213
228
|
const parent = h.spawnRoot('auth reload parent');
|
|
@@ -72,6 +72,7 @@ test('idle-release settles its real watcher before exit; daemon revives exactly
|
|
|
72
72
|
tools: [],
|
|
73
73
|
systemPrompt: '',
|
|
74
74
|
env: {
|
|
75
|
+
CRTR_RUNTIME_HOME: process.env['CRTR_RUNTIME_HOME'],
|
|
75
76
|
CRTR_FAKE_ENGINE_PENDING_AT_SETTLE: '1',
|
|
76
77
|
CRTR_WATCHER_TICK_MS: '20',
|
|
77
78
|
CRTR_WATCHER_DEBOUNCE_MS: '25',
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// Run with: npm run build && node --test --test-timeout=180000 dist/core/__tests__/seam/held-deferred-human-prompt.test.js
|
|
2
|
+
//
|
|
3
|
+
// Regression: Pi loads canvas extensions through Jiti with moduleCache:false,
|
|
4
|
+
// while broker frame dispatch is native ESM. Deferred mail held by the Jiti
|
|
5
|
+
// watcher must still join the next human prompt driven by native broker code.
|
|
6
|
+
import { after, before, test } from 'node:test';
|
|
7
|
+
import assert from 'node:assert/strict';
|
|
8
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
9
|
+
import { join } from 'node:path';
|
|
10
|
+
import { appendInbox, readCursor } from '../../feed/inbox.js';
|
|
11
|
+
import { updateNode } from '../../canvas/index.js';
|
|
12
|
+
import { reviveNode } from '../../runtime/revive.js';
|
|
13
|
+
import { createHeadlessHarness } from '../helpers/harness.js';
|
|
14
|
+
import { createAttachKit, delay, frameHas, tok } from '../helpers/broker-clients.js';
|
|
15
|
+
let h;
|
|
16
|
+
let root;
|
|
17
|
+
const kit = createAttachKit(() => h);
|
|
18
|
+
before(async () => {
|
|
19
|
+
h = await createHeadlessHarness({ sessionPrefix: 'crtr-held-deferred-prompt' });
|
|
20
|
+
root = h.spawnRoot('held deferred prompt seam root');
|
|
21
|
+
});
|
|
22
|
+
after(async () => {
|
|
23
|
+
kit.closeAll();
|
|
24
|
+
if (h !== undefined)
|
|
25
|
+
await h.dispose();
|
|
26
|
+
});
|
|
27
|
+
test('a Jiti-loaded watcher prepends its held deferred batch to a native broker human prompt', { timeout: 30_000 }, async () => {
|
|
28
|
+
const nodeId = h.fabricateBrokerNode({
|
|
29
|
+
id: 'held-deferred-human-prompt',
|
|
30
|
+
parent: root,
|
|
31
|
+
lifecycle: 'resident',
|
|
32
|
+
status: 'idle',
|
|
33
|
+
});
|
|
34
|
+
updateNode(nodeId, {
|
|
35
|
+
launch: {
|
|
36
|
+
extensions: [],
|
|
37
|
+
tools: [],
|
|
38
|
+
systemPrompt: '',
|
|
39
|
+
env: {
|
|
40
|
+
CRTR_WATCHER_TICK_MS: '20',
|
|
41
|
+
CRTR_WATCHER_DEBOUNCE_MS: '25',
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
});
|
|
45
|
+
reviveNode(nodeId, { resume: true, capacity: 'freeze' });
|
|
46
|
+
await h.awaitBoot(nodeId);
|
|
47
|
+
const entry = appendInbox(nodeId, {
|
|
48
|
+
from: 'child',
|
|
49
|
+
tier: 'deferred',
|
|
50
|
+
kind: 'update',
|
|
51
|
+
label: 'held Jiti deferred update',
|
|
52
|
+
});
|
|
53
|
+
await delay(150);
|
|
54
|
+
const customPath = join(h.home, 'nodes', nodeId, 'fake-pi.custom-messages.jsonl');
|
|
55
|
+
assert.equal(existsSync(customPath), false, 'deferred-only mail stays held and does not wake the broker');
|
|
56
|
+
const controller = await kit.attach(nodeId, 'controller', 'held-deferred-prompt-controller');
|
|
57
|
+
const prompt = tok('HELD-DEFERRED-HUMAN-PROMPT');
|
|
58
|
+
controller.send({ type: 'prompt', text: prompt });
|
|
59
|
+
await controller.waitFrame((frame) => frame.type === 'agent_end' && frameHas(frame, prompt), 'the human prompt turn');
|
|
60
|
+
const customMessages = readFileSync(customPath, 'utf8');
|
|
61
|
+
assert.match(customMessages, /held Jiti deferred update/, 'the Jiti extension instance joined its held batch before the native broker prompt');
|
|
62
|
+
assert.equal(readCursor(nodeId), entry.entry_id, 'agent_settled commits the joined handoff');
|
|
63
|
+
});
|
|
@@ -4,7 +4,7 @@ import { mkdtempSync, rmSync } from 'node:fs';
|
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
6
|
import { FaultRetry } from '../runtime/broker/fault-retry.js';
|
|
7
|
-
import { readFault,
|
|
7
|
+
import { admitProviderRetryEpisode, clearFault, readFault, readProviderRetryEpisode, recordPendingProviderRetryFault, } from '../runtime/fault.js';
|
|
8
8
|
let home;
|
|
9
9
|
before(() => {
|
|
10
10
|
home = mkdtempSync(join(tmpdir(), 'crtr-watchdog-abort-retry-'));
|
|
@@ -70,12 +70,18 @@ test('only a watchdog-stamped aborted settlement records an automatic connection
|
|
|
70
70
|
test('a failed retry settlement carries the episode forward instead of restarting it', () => {
|
|
71
71
|
// The episode marker a daemon retry records before re-prompting: original
|
|
72
72
|
// anchor, start time, and the attempt just consumed.
|
|
73
|
-
|
|
73
|
+
const sessionFile = 'retry-episode-session';
|
|
74
|
+
const admittedFault = recordPendingProviderRetryFault('retry-episode-node', sessionFile, {
|
|
74
75
|
link: 'pi→provider', op: 'generation turn', kind: 'connection',
|
|
75
76
|
retry: { disposition: 'auto', by: 'daemon', attempt: 2, max: 8 },
|
|
76
77
|
message: 'connection reset', since: '2026-01-01T00:00:00.000Z', anchorEntryId: 'original-anchor',
|
|
77
78
|
});
|
|
79
|
+
assert.ok(admittedFault, 'the pre-prompt retry fault persists');
|
|
80
|
+
assert.equal(admitProviderRetryEpisode('retry-episode-node', sessionFile, admittedFault), true, 'the retry is durably admitted before prompt');
|
|
81
|
+
clearFault('retry-episode-node', { link: 'pi→provider' }); // error turn_end clears its ordinary marker
|
|
82
|
+
assert.equal(readProviderRetryEpisode('retry-episode-node')?.state, 'admitted', 'error turn_end preserves the admitted retry for settlement');
|
|
78
83
|
const session = {
|
|
84
|
+
sessionFile,
|
|
79
85
|
sessionManager: { getLeafId: () => 'leaf-after-failed-retry' },
|
|
80
86
|
model: undefined,
|
|
81
87
|
};
|
|
@@ -100,4 +106,5 @@ test('a failed retry settlement carries the episode forward instead of restartin
|
|
|
100
106
|
assert.equal(fault?.anchorEntryId, 'original-anchor', 'each retry must rewind to the original fork point, not stack prompts');
|
|
101
107
|
assert.equal(fault?.since, '2026-01-01T00:00:00.000Z', 'the episode start survives a failed retry');
|
|
102
108
|
assert.equal(fault?.retry.attempt, 2, 'the attempt count accumulates toward exhaustion');
|
|
109
|
+
assert.equal(readProviderRetryEpisode('retry-episode-node')?.state, 'pending', 'the failed admitted retry becomes the next durable pending attempt');
|
|
103
110
|
});
|