@north-light/crouter 0.3.251 → 0.3.252

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/builtin-memory/02-turn-lifecycle/00-ending-a-turn.md +1 -5
  2. package/dist/builtin-memory/05-kinds/review/security-findings.md +1 -1
  3. package/dist/clients/attach/__tests__/completion-frecency.test.d.ts +1 -0
  4. package/dist/clients/attach/__tests__/completion-frecency.test.js +35 -0
  5. package/dist/clients/attach/__tests__/ref-autocomplete.test.js +3 -1
  6. package/dist/clients/attach/__tests__/titled-editor-preview.test.js +2 -1
  7. package/dist/clients/attach/input/completion-frecency.d.ts +45 -0
  8. package/dist/clients/attach/input/completion-frecency.js +141 -0
  9. package/dist/clients/attach/input/controller.d.ts +17 -0
  10. package/dist/clients/attach/input/controller.js +40 -0
  11. package/dist/clients/attach/input/ref-autocomplete.d.ts +3 -1
  12. package/dist/clients/attach/input/ref-autocomplete.js +49 -8
  13. package/dist/clients/attach/input/titled-editor.d.ts +3 -0
  14. package/dist/clients/attach/input/titled-editor.js +5 -0
  15. package/dist/clients/attach/session/editor-inventory.d.ts +3 -0
  16. package/dist/clients/attach/session/editor-inventory.js +1 -1
  17. package/dist/clients/attach/session/input-wiring.d.ts +3 -0
  18. package/dist/clients/attach/session/input-wiring.js +2 -0
  19. package/dist/clients/attach/viewer.js +578 -578
  20. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +4 -2
  21. package/dist/core/__tests__/canvas-inbox-watcher-naming.test.js +4 -1
  22. package/dist/core/__tests__/canvas-inbox-watcher.test.js +4 -1
  23. package/dist/core/__tests__/fixtures/fake-engine.d.ts +3 -1
  24. package/dist/core/__tests__/fixtures/fake-engine.js +7 -1
  25. package/dist/core/__tests__/integration/deferred-no-wake.test.js +4 -1
  26. package/dist/core/__tests__/migration.test.js +57 -11
  27. package/dist/core/__tests__/seam/broker-provider-retry.test.js +17 -2
  28. package/dist/core/__tests__/seam/dormancy-release.test.js +1 -0
  29. package/dist/core/__tests__/watchdog-abort-arms-retry.test.js +9 -2
  30. package/dist/core/canvas/migrations.js +63 -24
  31. package/dist/core/runtime/broker/engine-drive.js +7 -1
  32. package/dist/core/runtime/broker/fault-retry.js +31 -13
  33. package/dist/core/runtime/broker/held-deferred-inbox.d.ts +4 -0
  34. package/dist/core/runtime/broker/held-deferred-inbox.js +7 -2
  35. package/dist/core/runtime/broker.js +19 -0
  36. package/dist/core/runtime/close.js +2 -2
  37. package/dist/core/runtime/fault.js +8 -5
  38. package/dist/core/runtime/recycle.js +63 -41
  39. package/dist/core/runtime/reset.d.ts +1 -1
  40. package/dist/core/runtime/reset.js +25 -2
  41. package/dist/daemon/__tests__/helpers/source-daemon.js +1 -0
  42. package/dist/daemon/api/handlers/messages.js +1 -1
  43. package/dist/daemon/api/handlers/reports.js +1 -1
  44. package/dist/daemon/cron/sinks.js +1 -1
  45. package/dist/daemon/messaging/node-message.js +1 -1
  46. package/dist/pi-extensions/canvas-inbox-watcher.js +5 -2
  47. package/package.json +1 -1
  48. package/runtime.lock.json +2 -2
@@ -15,7 +15,7 @@ import { drainBearings } from '../runtime/kickoff.js';
15
15
  import { appendSituationalContext } from '../runtime/situational-context.js';
16
16
  import { createEngineDriver } from '../runtime/broker/engine-drive.js';
17
17
  import { createTurnAdmissionGate } from '../runtime/broker/turn-admission.js';
18
- import { heldDeferredInboxPromptJoin } from '../runtime/broker/held-deferred-inbox.js';
18
+ import { BROKER_IDLE_TURN_START, heldDeferredInboxPromptJoin, } from '../runtime/broker/held-deferred-inbox.js';
19
19
  import { createNode, getNode, recordPid, setIntent } from '../canvas/canvas.js';
20
20
  import { createProfile, pauseProfile, resumeProfile } from '../profiles/manifest.js';
21
21
  import { closeDb } from '../canvas/db.js';
@@ -47,7 +47,7 @@ function makeFakePi() {
47
47
  const handlers = {};
48
48
  let resolveFirstInjection;
49
49
  const firstInjection = new Promise((resolve) => { resolveFirstInjection = resolve; });
50
- return {
50
+ const pi = {
51
51
  injected: [],
52
52
  firstInjection,
53
53
  sentMessages: [],
@@ -59,6 +59,8 @@ function makeFakePi() {
59
59
  sendMessage(message) { this.sentMessages.push(message); },
60
60
  async fire(e, ev, ctx) { await handlers[e]?.(ev, ctx); },
61
61
  };
62
+ globalThis[BROKER_IDLE_TURN_START] = (content) => pi.sendUserMessage(content);
63
+ return pi;
62
64
  }
63
65
  // Fire agent_settled — the authoritative durable-commit boundary in the
64
66
  // at-least-once model. `pending` maps to pi's `hasPendingMessages()` and is
@@ -30,6 +30,7 @@ import { createNode } from '../canvas/canvas.js';
30
30
  import { closeDb } from '../canvas/db.js';
31
31
  import { apiSocketPath } from '../canvas/paths.js';
32
32
  import { appendInbox } from '../feed/inbox.js';
33
+ import { BROKER_IDLE_TURN_START } from '../runtime/broker/held-deferred-inbox.js';
33
34
  const TICK_MS = 20;
34
35
  const DEBOUNCE_MS = 25;
35
36
  let home;
@@ -76,7 +77,7 @@ function getRequestCount(nodeId) {
76
77
  }
77
78
  function makeFakePi() {
78
79
  const handlers = {};
79
- return {
80
+ const pi = {
80
81
  injected: [],
81
82
  sentMessages: [],
82
83
  on(e, h) { handlers[e] = h; },
@@ -84,6 +85,8 @@ function makeFakePi() {
84
85
  sendMessage(message) { this.sentMessages.push(message); },
85
86
  fire(e, ev, ctx) { handlers[e]?.(ev, ctx); },
86
87
  };
88
+ globalThis[BROKER_IDLE_TURN_START] = (content) => pi.sendUserMessage(content);
89
+ return pi;
87
90
  }
88
91
  const wait = (ms) => new Promise((r) => setTimeout(r, ms));
89
92
  async function waitFor(predicate, timeoutMs = 3000, stepMs = 10) {
@@ -18,6 +18,7 @@ import { closeDb } from '../canvas/db.js';
18
18
  import { appendInbox, coalesce, finalizeInboxEntry } from '../feed/inbox.js';
19
19
  import { appendSituationalContext, SITUATIONAL_CONTEXT_CUSTOM_TYPE } from '../runtime/situational-context.js';
20
20
  import { parseCard } from '../../shared/generated-context.js';
21
+ import { BROKER_IDLE_TURN_START } from '../runtime/broker/held-deferred-inbox.js';
21
22
  // Drive the watcher's injectable cadence seam (CRTR_WATCHER_TICK_MS /
22
23
  // CRTR_WATCHER_DEBOUNCE_MS) at a fast tempo so the watcher delivers in tens of
23
24
  // milliseconds, not seconds — exactly the resolve+seed tick, read tick, and
@@ -40,7 +41,7 @@ function freshNode(nodeId) {
40
41
  }
41
42
  function makeFakePi() {
42
43
  const handlers = {};
43
- return {
44
+ const pi = {
44
45
  injected: [],
45
46
  sentMessages: [],
46
47
  on(e, h) { handlers[e] = h; },
@@ -48,6 +49,8 @@ function makeFakePi() {
48
49
  sendMessage(message) { this.sentMessages.push(message); },
49
50
  fire(e, ev, ctx) { handlers[e]?.(ev, ctx); },
50
51
  };
52
+ globalThis[BROKER_IDLE_TURN_START] = (content) => pi.sendUserMessage(content);
53
+ return pi;
51
54
  }
52
55
  const wait = (ms) => new Promise((r) => setTimeout(r, ms));
53
56
  /** The text of a batch that must coalesce into exactly one delivery. */
@@ -193,7 +193,9 @@ declare class FakeSession {
193
193
  getSessionStats(): Record<string, unknown>;
194
194
  private syncTreeSnapshot;
195
195
  setSessionName(_name: string): void;
196
- prompt(text: string, _options?: unknown): Promise<void>;
196
+ prompt(text: string, options?: {
197
+ preflightResult?: (success: boolean) => void;
198
+ }): Promise<void>;
197
199
  /** The unrun queues the welcome snapshot reports (`snapshot.queued`). Real
198
200
  * pi holds a steer + followUp queue and drains it when the next turn runs;
199
201
  * the fake records the same texts and clears them in emitTurn, so a client
@@ -448,12 +448,18 @@ class FakeSession {
448
448
  /* no-op recording stub */
449
449
  }
450
450
  // controller-drive stubs (no controller connects in these tests)
451
- prompt(text, _options) {
451
+ prompt(text, options) {
452
452
  // Rejection seam: the engine refuses the call itself (pi's no-provider-
453
453
  // credential shape) — the promise rejects and NO lifecycle event fires.
454
454
  if (text.includes(REJECT_PROMPT_CALL)) {
455
455
  return Promise.reject(new Error('No API key found for anthropic'));
456
456
  }
457
+ options?.preflightResult?.(true);
458
+ // The inbox watcher now enters through the broker-owned prompt bridge, not
459
+ // the Pi extension runtime. Keep its existing observable wake record for
460
+ // the compiled dormancy seam while the prompt drives the real lifecycle.
461
+ if (text.startsWith('<runtime kind="inbox"'))
462
+ this.recordInjected({ content: text });
457
463
  // The broker calls this for the controller-less retry path. Emit the turn
458
464
  // on the broker's subscribe channel and also run the real extension hooks so
459
465
  // the canvas stophook records the fault marker and the broker can schedule
@@ -50,6 +50,7 @@ import { canvasDbPath, nodeDir, apiSocketPath } from '../../canvas/paths.js';
50
50
  import { closeDb } from '../../canvas/db.js';
51
51
  import { createApiServer } from '../../../daemon/api/server.js';
52
52
  import { CrtrClient } from '../../../api/index.js';
53
+ import { BROKER_IDLE_TURN_START } from '../../runtime/broker/held-deferred-inbox.js';
53
54
  import { registerNode } from '../../../commands/node.js';
54
55
  import { pushFinal } from '../../feed/feed.js';
55
56
  import { InputError } from '../../io.js';
@@ -500,13 +501,15 @@ test('the lifecycle tick parks a resident holding deferred-only mail — mail th
500
501
  });
501
502
  function makeFakePi() {
502
503
  const handlers = {};
503
- return {
504
+ const pi = {
504
505
  injected: [],
505
506
  on(e, h) { handlers[e] = h; },
506
507
  sendUserMessage(content, options) { this.injected.push({ content, deliverAs: options?.deliverAs }); },
507
508
  sendMessage() { },
508
509
  async fire(e, ev, ctx) { await handlers[e]?.(ev, ctx); },
509
510
  };
511
+ globalThis[BROKER_IDLE_TURN_START] = (content) => pi.sendUserMessage(content);
512
+ return pi;
510
513
  }
511
514
  const wait = (ms) => new Promise((r) => setTimeout(r, ms));
512
515
  test('the real inbox watcher holds a deferred entry alone, then delivers it TOGETHER with a later normal entry with no duplicate/loss, and the durable cursor settles past both (gh #341 review: genuine delivery, not just the daemon decision)', { timeout: 30_000 }, async () => {
@@ -215,36 +215,82 @@ test('a fresh db has the four additive runtime columns', () => {
215
215
  assert.equal(rt['window'], null);
216
216
  assert.equal(rt['tmux_session'], null);
217
217
  });
218
- test('session-file ownership migration deterministically clears duplicate claimants before creating its UNIQUE index', () => {
218
+ test('session-file ownership migration reconciles authoritative metas before clearing duplicate claimants', () => {
219
219
  ensureHome();
220
220
  const raw = new DatabaseSync(canvasDbPath());
221
221
  migrateToVersion(raw, 35);
222
222
  const sessionFile = '/tmp/preexisting-duplicate.jsonl';
223
223
  const aliasedSessionFile = '/tmp/./preexisting-duplicate.jsonl';
224
- for (const id of ['first-owner', 'later-claimant']) {
224
+ const recoveredSessionFile = '/tmp/recovered-session.jsonl';
225
+ const metas = [
226
+ ['first-owner', '2026-01-01T00:00:00.000Z', sessionFile],
227
+ ['later-claimant', '2026-01-02T00:00:00.000Z', aliasedSessionFile],
228
+ // The durable meta write survived, but the derived row below is stale.
229
+ ['recovered-claimant', '2026-01-03T00:00:00.000Z', recoveredSessionFile],
230
+ ];
231
+ for (const [id, created, pi_session_file] of metas) {
225
232
  const nodeDir = join(home, 'nodes', id);
226
233
  mkdirSync(nodeDir, { recursive: true });
227
234
  writeFileSync(join(nodeDir, 'meta.json'), JSON.stringify({
228
- node_id: id, name: id, created: id === 'first-owner' ? '2026-01-01T00:00:00.000Z' : '2026-01-02T00:00:00.000Z',
229
- cwd: '/tmp', kind: 'developer', mode: 'base', lifecycle: 'terminal', pi_session_id: `${id}-session`, pi_session_file: id === 'first-owner' ? sessionFile : aliasedSessionFile,
235
+ node_id: id, name: id, created, cwd: '/tmp', kind: 'developer', mode: 'base', lifecycle: 'terminal', pi_session_id: `${id}-session`, pi_session_file,
230
236
  }));
231
237
  }
232
- raw.prepare('INSERT INTO nodes (node_id, name, kind, mode, lifecycle, status, cwd, created, pi_session_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
233
- .run('first-owner', 'first-owner', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-01T00:00:00.000Z', sessionFile);
234
- raw.prepare('INSERT INTO nodes (node_id, name, kind, mode, lifecycle, status, cwd, created, pi_session_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
235
- .run('later-claimant', 'later-claimant', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-02T00:00:00.000Z', aliasedSessionFile);
238
+ const insert = raw.prepare('INSERT INTO nodes (node_id, name, kind, mode, lifecycle, status, cwd, created, pi_session_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
239
+ insert.run('first-owner', 'first-owner', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-01T00:00:00.000Z', sessionFile);
240
+ insert.run('later-claimant', 'later-claimant', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-02T00:00:00.000Z', aliasedSessionFile);
241
+ insert.run('recovered-claimant', 'recovered-claimant', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-03T00:00:00.000Z', sessionFile);
236
242
  assert.doesNotThrow(() => migrate(raw), 'a real canvas with pre-existing duplicates must migrate');
237
243
  const rows = raw.prepare('SELECT node_id, pi_session_file FROM nodes ORDER BY created, node_id').all();
238
244
  assert.deepEqual(rows.map((row) => [row.node_id, row.pi_session_file]), [
239
245
  ['first-owner', sessionFile],
240
246
  ['later-claimant', null],
241
- ], 'the oldest created row wins after aliases are canonicalized; later claimants are cleared');
247
+ ['recovered-claimant', recoveredSessionFile],
248
+ ], 'the oldest duplicate is retained while a stale row is reconciled from its authoritative meta');
242
249
  const loser = JSON.parse(readFileSync(join(home, 'nodes', 'later-claimant', 'meta.json'), 'utf8'));
243
- assert.equal(loser.pi_session_id, null, 'the loser meta cannot reindex its stale session id');
244
- assert.equal(loser.pi_session_file, null, 'the loser meta cannot restore the duplicate projection');
250
+ assert.equal(loser.pi_session_id, null, 'the duplicate loser meta cannot reindex its stale session id');
251
+ assert.equal(loser.pi_session_file, null, 'the duplicate loser meta cannot restore the duplicate projection');
252
+ const recovered = JSON.parse(readFileSync(join(home, 'nodes', 'recovered-claimant', 'meta.json'), 'utf8'));
253
+ assert.equal(recovered.pi_session_id, 'recovered-claimant-session', 'a stale row cannot clear the authoritative session id');
254
+ assert.equal(recovered.pi_session_file, recoveredSessionFile, 'a stale row cannot clear the authoritative session file');
245
255
  assert.throws(() => raw.prepare('UPDATE nodes SET pi_session_file = ? WHERE node_id = ?').run(sessionFile, 'later-claimant'), /UNIQUE/, 'the schema enforces the repaired invariant');
246
256
  raw.close();
247
257
  });
258
+ test('forward session-file repair reprojects a surviving v36 meta coordinate before restoring its UNIQUE index', () => {
259
+ ensureHome();
260
+ const raw = new DatabaseSync(canvasDbPath());
261
+ migrateToVersion(raw, 35);
262
+ const recoveredSessionFile = '/tmp/surviving-meta-coordinate.jsonl';
263
+ const staleSessionFile = '/tmp/stale-v36-coordinate.jsonl';
264
+ for (const [id, pi_session_id, pi_session_file] of [
265
+ ['surviving-owner', 'surviving-session', recoveredSessionFile],
266
+ ['stale-row', null, null],
267
+ ]) {
268
+ const nodeDir = join(home, 'nodes', id);
269
+ mkdirSync(nodeDir, { recursive: true });
270
+ writeFileSync(join(nodeDir, 'meta.json'), JSON.stringify({
271
+ node_id: id, name: id, created: id === 'surviving-owner' ? '2026-01-01T00:00:00.000Z' : '2026-01-02T00:00:00.000Z',
272
+ cwd: '/tmp', kind: 'developer', mode: 'base', lifecycle: 'terminal', pi_session_id, pi_session_file,
273
+ }));
274
+ }
275
+ const insert = raw.prepare('INSERT INTO nodes (node_id, name, kind, mode, lifecycle, status, cwd, created, pi_session_file) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
276
+ // This is the v0.3.250 shape after v36: the owner’s surviving meta coordinate
277
+ // disagrees with its indexed row, while the row carrying it has no meta claim.
278
+ insert.run('surviving-owner', 'surviving-owner', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-01T00:00:00.000Z', staleSessionFile);
279
+ insert.run('stale-row', 'stale-row', 'developer', 'base', 'terminal', 'active', '/tmp', '2026-01-02T00:00:00.000Z', recoveredSessionFile);
280
+ raw.exec('DROP INDEX IF EXISTS idx_nodes_pi_session_file; CREATE UNIQUE INDEX idx_nodes_pi_session_file ON nodes(pi_session_file); PRAGMA user_version = 36;');
281
+ assert.doesNotThrow(() => migrate(raw));
282
+ assert.equal(userVersion(raw), MIGRATIONS.length);
283
+ const rows = raw.prepare('SELECT node_id, pi_session_file FROM nodes ORDER BY created, node_id').all();
284
+ assert.deepEqual(rows.map((row) => [row.node_id, row.pi_session_file]), [
285
+ ['surviving-owner', recoveredSessionFile],
286
+ ['stale-row', null],
287
+ ]);
288
+ const survivor = JSON.parse(readFileSync(join(home, 'nodes', 'surviving-owner', 'meta.json'), 'utf8'));
289
+ assert.equal(survivor.pi_session_id, 'surviving-session');
290
+ assert.equal(survivor.pi_session_file, recoveredSessionFile);
291
+ assert.throws(() => raw.prepare('UPDATE nodes SET pi_session_file = ? WHERE node_id = ?').run(recoveredSessionFile, 'stale-row'), /UNIQUE/);
292
+ raw.close();
293
+ });
248
294
  test('node lifecycle substrate migration adds six columns and backfills terminal reasons', () => {
249
295
  ensureHome();
250
296
  const raw = new DatabaseSync(canvasDbPath());
@@ -148,6 +148,8 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
148
148
  seedPool('openai-codex', false);
149
149
  const pendingNode = await h.spawnHeadlessChild(parent, 'pending provider episode', { model: OPENAI_STRONG.spec });
150
150
  await h.waitFor(() => agentStartCount(pendingNode) >= 1, { label: 'pending episode fixture started' });
151
+ const pendingKickoffObserver = await attachUntil(pendingNode, 'observer', 'pending-provider-kickoff-settled', (a) => a.welcome.snapshot.state.isStreaming === false, 'pending provider episode kickoff settled');
152
+ pendingKickoffObserver.close();
151
153
  const pendingStarts = agentStartCount(pendingNode);
152
154
  h.fakeCmd(pendingNode, {
153
155
  cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 2_000,
@@ -160,6 +162,8 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
160
162
  assert.equal(agentStartCount(pendingNode), pendingStarts + 1, 'a pending episode re-drives exactly once after broker death');
161
163
  const admittedNode = await h.spawnHeadlessChild(parent, 'admitted provider episode', { model: OPENAI_STRONG.spec });
162
164
  await h.waitFor(() => agentStartCount(admittedNode) >= 1, { label: 'admitted episode fixture started' });
165
+ const admittedKickoffObserver = await attachUntil(admittedNode, 'observer', 'admitted-provider-kickoff-settled', (a) => a.welcome.snapshot.state.isStreaming === false, 'admitted provider episode kickoff settled');
166
+ admittedKickoffObserver.close();
163
167
  const admittedStarts = agentStartCount(admittedNode);
164
168
  h.fakeCmd(admittedNode, {
165
169
  cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 60_000,
@@ -174,6 +178,8 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
174
178
  assert.equal(agentStartCount(admittedNode), admittedStarts, 'an admitted episode is never replayed by replacement startup');
175
179
  const invalidatedNode = await h.spawnHeadlessChild(parent, 'invalidated provider episode', { model: OPENAI_STRONG.spec });
176
180
  await h.waitFor(() => agentStartCount(invalidatedNode) >= 1, { label: 'invalidated episode fixture started' });
181
+ const invalidatedKickoffObserver = await attachUntil(invalidatedNode, 'observer', 'invalidated-provider-kickoff-settled', (a) => a.welcome.snapshot.state.isStreaming === false, 'invalidated provider episode kickoff settled');
182
+ invalidatedKickoffObserver.close();
177
183
  const invalidatedStarts = agentStartCount(invalidatedNode);
178
184
  h.fakeCmd(invalidatedNode, {
179
185
  cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 60_000,
@@ -190,10 +196,13 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
190
196
  assert.equal(agentStartCount(invalidatedNode), invalidatedStarts, 'an invalidated rebound-session episode is never replayed by replacement startup');
191
197
  const cyclingNode = await h.spawnHeadlessChild(parent, 'fresh-cycle provider episode', { model: OPENAI_STRONG.spec });
192
198
  await h.waitFor(() => agentStartCount(cyclingNode) >= 1, { label: 'fresh-cycle episode fixture started' });
199
+ const cyclingKickoffObserver = await attachUntil(cyclingNode, 'observer', 'fresh-cycle-provider-kickoff-settled', (a) => a.welcome.snapshot.state.isStreaming === false, 'fresh-cycle provider episode kickoff settled');
200
+ cyclingKickoffObserver.close();
193
201
  h.fakeCmd(cyclingNode, {
194
- cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 60_000,
202
+ cmd: 'stop', reason: 'error', errorMessage: 'Managed provider pools are temporarily unavailable.', deadline: Date.now() + 2_000,
195
203
  });
196
204
  await h.waitFor(() => readProviderRetryEpisode(cyclingNode)?.state === 'pending' ? true : null, { label: 'fresh-cycle episode pending record persisted' });
205
+ const cyclingStarts = agentStartCount(cyclingNode);
197
206
  const cyclingBoots = h.bootCount(cyclingNode);
198
207
  const yieldResult = h.cli(cyclingNode, ['node', 'yield', 'begin a fresh cycle']);
199
208
  assert.equal(yieldResult.code, 0, `node yield should succeed\n--stderr--\n${yieldResult.stderr}`);
@@ -207,7 +216,13 @@ test('a crashed broker re-drives only its pending provider episode, never admitt
207
216
  await h.tick(Date.parse(cyclingBackoff.respawn_not_before) + 1);
208
217
  await h.awaitBoot(cyclingNode, { minCount: cyclingBoots + 1 });
209
218
  await h.waitFor(() => existsSync(h.brokerSock(cyclingNode)), { label: 'fresh-cycle replacement broker listener bound' });
210
- assert.equal(readProviderRetryEpisode(cyclingNode)?.state, 'invalidated', 'new-cycle startup invalidates the old transcript episode');
219
+ await h.waitFor(() => readProviderRetryEpisode(cyclingNode) === null, { label: 'fresh-cycle kickoff settled successfully' });
220
+ assert.equal(readProviderRetryEpisode(cyclingNode), null, 'the settled fresh-cycle kickoff clears the old provider episode');
221
+ assert.equal(agentStartCount(cyclingNode), cyclingStarts + 1, 'the fresh-cycle kickoff starts exactly once');
222
+ // A retained episode would re-arm immediately once its deadline becomes due.
223
+ // Wait past that poll interval to prove the completed kickoff cannot replay.
224
+ await new Promise((resolve) => setTimeout(resolve, 2500));
225
+ assert.equal(agentStartCount(cyclingNode), cyclingStarts + 1, 'the cleared fresh-cycle episode never replays');
211
226
  });
212
227
  test('reload_auth re-drives a parked fatal provider-auth turn exactly once', { timeout: 30_000 }, async () => {
213
228
  const parent = h.spawnRoot('auth reload parent');
@@ -72,6 +72,7 @@ test('idle-release settles its real watcher before exit; daemon revives exactly
72
72
  tools: [],
73
73
  systemPrompt: '',
74
74
  env: {
75
+ CRTR_RUNTIME_HOME: process.env['CRTR_RUNTIME_HOME'],
75
76
  CRTR_FAKE_ENGINE_PENDING_AT_SETTLE: '1',
76
77
  CRTR_WATCHER_TICK_MS: '20',
77
78
  CRTR_WATCHER_DEBOUNCE_MS: '25',
@@ -4,7 +4,7 @@ import { mkdtempSync, rmSync } from 'node:fs';
4
4
  import { tmpdir } from 'node:os';
5
5
  import { join } from 'node:path';
6
6
  import { FaultRetry } from '../runtime/broker/fault-retry.js';
7
- import { readFault, recordFault } from '../runtime/fault.js';
7
+ import { admitProviderRetryEpisode, clearFault, readFault, readProviderRetryEpisode, recordPendingProviderRetryFault, } from '../runtime/fault.js';
8
8
  let home;
9
9
  before(() => {
10
10
  home = mkdtempSync(join(tmpdir(), 'crtr-watchdog-abort-retry-'));
@@ -70,12 +70,18 @@ test('only a watchdog-stamped aborted settlement records an automatic connection
70
70
  test('a failed retry settlement carries the episode forward instead of restarting it', () => {
71
71
  // The episode marker a daemon retry records before re-prompting: original
72
72
  // anchor, start time, and the attempt just consumed.
73
- recordFault('retry-episode-node', {
73
+ const sessionFile = 'retry-episode-session';
74
+ const admittedFault = recordPendingProviderRetryFault('retry-episode-node', sessionFile, {
74
75
  link: 'pi→provider', op: 'generation turn', kind: 'connection',
75
76
  retry: { disposition: 'auto', by: 'daemon', attempt: 2, max: 8 },
76
77
  message: 'connection reset', since: '2026-01-01T00:00:00.000Z', anchorEntryId: 'original-anchor',
77
78
  });
79
+ assert.ok(admittedFault, 'the pre-prompt retry fault persists');
80
+ assert.equal(admitProviderRetryEpisode('retry-episode-node', sessionFile, admittedFault), true, 'the retry is durably admitted before prompt');
81
+ clearFault('retry-episode-node', { link: 'pi→provider' }); // error turn_end clears its ordinary marker
82
+ assert.equal(readProviderRetryEpisode('retry-episode-node')?.state, 'admitted', 'error turn_end preserves the admitted retry for settlement');
78
83
  const session = {
84
+ sessionFile,
79
85
  sessionManager: { getLeafId: () => 'leaf-after-failed-retry' },
80
86
  model: undefined,
81
87
  };
@@ -100,4 +106,5 @@ test('a failed retry settlement carries the episode forward instead of restartin
100
106
  assert.equal(fault?.anchorEntryId, 'original-anchor', 'each retry must rewind to the original fork point, not stack prompts');
101
107
  assert.equal(fault?.since, '2026-01-01T00:00:00.000Z', 'the episode start survives a failed retry');
102
108
  assert.equal(fault?.retry.attempt, 2, 'the attempt count accumulates toward exhaustion');
109
+ assert.equal(readProviderRetryEpisode('retry-episode-node')?.state, 'pending', 'the failed admitted retry becomes the next durable pending attempt');
103
110
  });
@@ -26,14 +26,19 @@ import { basename, join, resolve } from 'node:path';
26
26
  import { crtrHome, isSafeNodeId, nodesRoot, nodeMetaPath } from './paths.js';
27
27
  import { STATE_MIGRATIONS } from '../../migrations/registry.js';
28
28
  import { customExtensionPaths } from './extensions.js';
29
- import { updateJsonFileDurably } from './meta-file.js';
29
+ import { readMetaObject, updateJsonFileDurably, withMetaLock, writeMetaObjectLocked } from './meta-file.js';
30
30
  // Schema as a forward-only migration list
31
31
  //
32
32
  // The schema is the migration list: one place a schema change is expressed,
33
33
  // one gate (`PRAGMA user_version`) that applies it. `migrate()` runs every
34
34
  // pending step in order and bumps `user_version` after each, so a fresh db and
35
35
  // the live fleet (all at `user_version 0`) converge on the same final shape.
36
- // Migrations are append-only and forward-only — never edit a shipped step.
36
+ // Migrations are append-only and forward-only — never edit a shipped step. The
37
+ // v36 body below is the sole corrective exception: v0.3.250 shipped a destructive
38
+ // stale-row decision, so v36 is narrowed for databases still below its gate while
39
+ // appended v37 repairs databases that already crossed it. Do not repeat this
40
+ // exception; an appended migration is required unless it would run too late to
41
+ // prevent destruction at an existing version boundary.
37
42
  /** v1 — the baseline tables + indexes. `IF NOT EXISTS` makes this a no-op on
38
43
  * any existing db (the live fleet already has these tables). */
39
44
  function baselineSchema(db) {
@@ -1215,32 +1220,43 @@ function addNodeSessionFileIndex(db) {
1215
1220
  }
1216
1221
  db.exec('CREATE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);');
1217
1222
  }
1218
- /** v36 — one Pi transcript may have exactly one node owner. Normalize every
1219
- * stored path first (including meta), then retain the earliest created row
1220
- * (node_id breaks equal timestamps) and clear both durable session coordinates
1221
- * from every later claimant before creating the UNIQUE index. */
1222
- function makeNodeSessionFileUnique(db) {
1223
+ /** Reproject the derived session-file index from authoritative node metadata.
1224
+ * v36 may resolve duplicate authoritative claimants because it runs before the
1225
+ * first UNIQUE index exists. v37 must retain every surviving meta claim: a
1226
+ * conflict at that later boundary is irreconcilable and fails instead. */
1227
+ function reconcileNodeSessionFileOwners(db, clearDuplicateMetaClaimants) {
1223
1228
  const canonical = (sessionFile) => {
1224
1229
  const absolute = resolve(sessionFile);
1225
1230
  return existsSync(absolute) ? realpathSync(absolute) : absolute;
1226
1231
  };
1227
- const rows = db.prepare(`
1228
- SELECT node_id, pi_session_file
1229
- FROM nodes
1230
- WHERE pi_session_file IS NOT NULL
1231
- ORDER BY created ASC, node_id ASC
1232
- `).all();
1233
1232
  const setFile = db.prepare('UPDATE nodes SET pi_session_file = ? WHERE node_id = ?');
1234
- for (const row of rows) {
1235
- const sessionFile = canonical(row.pi_session_file);
1236
- if (sessionFile === row.pi_session_file)
1233
+ db.exec('DROP INDEX IF EXISTS idx_nodes_pi_session_file;');
1234
+ const rows = db.prepare('SELECT node_id FROM nodes ORDER BY created ASC, node_id ASC').all();
1235
+ for (const { node_id } of rows) {
1236
+ const metaPath = nodeMetaPath(node_id);
1237
+ // Missing or unreadable metadata has no authoritative coordinate. Avoid
1238
+ // taking a file lock there: old rows can outlive a removed node directory.
1239
+ if (readMetaObject(metaPath) === null) {
1240
+ setFile.run(null, node_id);
1237
1241
  continue;
1238
- // Identity is authoritative, so normalize both stores before grouping.
1239
- updateJsonFileDurably(nodeMetaPath(row.node_id), (meta) => {
1240
- meta['pi_session_file'] = sessionFile;
1241
- return true;
1242
+ }
1243
+ const sessionFile = withMetaLock(metaPath, () => {
1244
+ const meta = readMetaObject(metaPath);
1245
+ if (meta === null)
1246
+ throw new Error(`node ${node_id} session metadata changed during ownership migration`);
1247
+ if (typeof meta['pi_session_file'] !== 'string')
1248
+ return null;
1249
+ const storedSessionFile = meta['pi_session_file'];
1250
+ const normalizedSessionFile = canonical(storedSessionFile);
1251
+ if (normalizedSessionFile !== storedSessionFile) {
1252
+ meta['pi_session_file'] = normalizedSessionFile;
1253
+ writeMetaObjectLocked(metaPath, meta);
1254
+ }
1255
+ return normalizedSessionFile;
1242
1256
  });
1243
- setFile.run(sessionFile, row.node_id);
1257
+ // meta.json is authoritative: a crash after its write can leave this row
1258
+ // claiming another node's old session coordinate.
1259
+ setFile.run(sessionFile, node_id);
1244
1260
  }
1245
1261
  const duplicates = db.prepare(`
1246
1262
  SELECT pi_session_file
@@ -1249,6 +1265,10 @@ WHERE pi_session_file IS NOT NULL
1249
1265
  GROUP BY pi_session_file
1250
1266
  HAVING COUNT(*) > 1
1251
1267
  `).all();
1268
+ if (!clearDuplicateMetaClaimants && duplicates.length > 0) {
1269
+ const coordinates = duplicates.map(({ pi_session_file }) => pi_session_file).join(', ');
1270
+ throw new Error(`cannot restore unique Pi session ownership: authoritative metadata has concurrent claimants for ${coordinates}`);
1271
+ }
1252
1272
  const clearRow = db.prepare('UPDATE nodes SET pi_session_file = NULL WHERE node_id = ?');
1253
1273
  for (const { pi_session_file } of duplicates) {
1254
1274
  const owners = db.prepare(`
@@ -1259,18 +1279,36 @@ ORDER BY created ASC, node_id ASC
1259
1279
  `).all(pi_session_file);
1260
1280
  for (const { node_id } of owners.slice(1)) {
1261
1281
  // Clear both durable coordinates so a later reindex cannot restore the
1262
- // duplicate projection.
1263
- updateJsonFileDurably(nodeMetaPath(node_id), (meta) => {
1282
+ // duplicate projection. The checked write fails rather than erasing a
1283
+ // concurrent metadata change.
1284
+ const cleared = updateJsonFileDurably(nodeMetaPath(node_id), (meta) => {
1285
+ const currentSessionFile = meta['pi_session_file'];
1286
+ if (typeof currentSessionFile !== 'string' || canonical(currentSessionFile) !== pi_session_file) {
1287
+ throw new Error(`node ${node_id} session metadata changed during ownership migration`);
1288
+ }
1264
1289
  meta['pi_session_id'] = null;
1265
1290
  meta['pi_session_file'] = null;
1266
1291
  return true;
1267
1292
  });
1293
+ if (!cleared)
1294
+ throw new Error(`node ${node_id} session metadata changed during ownership migration`);
1268
1295
  clearRow.run(node_id);
1269
1296
  }
1270
1297
  }
1271
- db.exec('DROP INDEX IF EXISTS idx_nodes_pi_session_file;');
1272
1298
  db.exec('CREATE UNIQUE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);');
1273
1299
  }
1300
+ /** v36 — one Pi transcript may have exactly one node owner. This is the narrow
1301
+ * corrective body for clients below v36; it reprojects metadata before choosing
1302
+ * the earliest created duplicate claimant (node_id breaks equal timestamps). */
1303
+ function makeNodeSessionFileUnique(db) {
1304
+ reconcileNodeSessionFileOwners(db, true);
1305
+ }
1306
+ /** v37 — repair v0.3.250's already-applied v36 projection without inventing
1307
+ * coordinates v36 already cleared. Surviving authoritative metadata wins over
1308
+ * stale rows; concurrent surviving claims fail loudly rather than clearing one. */
1309
+ function repairNodeSessionFileUniqueIndex(db) {
1310
+ reconcileNodeSessionFileOwners(db, false);
1311
+ }
1274
1312
  /** v35 — durable node lifecycle execution state and terminal cause. */
1275
1313
  function addNodeLifecycleColumns(db) {
1276
1314
  const columns = nodeColumns(db);
@@ -1340,6 +1378,7 @@ export const MIGRATIONS = [
1340
1378
  /* v34 */ addNodeSessionFileIndex,
1341
1379
  /* v35 */ addNodeLifecycleColumns,
1342
1380
  /* v36 */ makeNodeSessionFileUnique,
1381
+ /* v37 */ repairNodeSessionFileUniqueIndex,
1343
1382
  ...JOURNALED_STATE_MIGRATIONS,
1344
1383
  ];
1345
1384
  /** Migration indexes that manage their OWN transaction and therefore must not
@@ -52,7 +52,13 @@ export function createEngineDriver(deps) {
52
52
  // claims held deferred mail before the prompt body, but its cursor stays
53
53
  // uncommitted until the resulting turn's agent_settled event.
54
54
  if (route.call === 'prompt' && !session().isStreaming) {
55
- await heldDeferredInboxPromptJoin()?.(session());
55
+ try {
56
+ await heldDeferredInboxPromptJoin()?.(session());
57
+ }
58
+ catch {
59
+ // The watcher emitted the failed durable delivery and retains it for
60
+ // replay. A person's live prompt is not replayable, so it must run.
61
+ }
56
62
  }
57
63
  await session().prompt(frame.text, { ...promptOptions, preflightResult: release });
58
64
  };
@@ -58,19 +58,26 @@ export class FaultRetry {
58
58
  generation.stagedRefreshAbort = false;
59
59
  generation.stagedOverflowFailure = null;
60
60
  const settledSession = generation.session;
61
- // A failed daemon retry settles here with the episode's fault marker still
62
- // present (a successful provider round-trip would have cleared it). Carry
63
- // the episode forward — original anchor, start time, and attempt count — so
64
- // each retry rewinds to the same fork point instead of stacking recovery
65
- // prompts, and so backoff/exhaustion actually accumulate.
61
+ // turn_end clears the ordinary marker before this settlement. A failed
62
+ // admitted retry therefore carries from its durable episode, while an
63
+ // initial provider failure still reads from the ordinary marker.
66
64
  const prior = readFault(this.deps.nodeId);
67
- const episode = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
68
- ? { since: prior.since, anchorEntryId: prior.anchorEntryId, attempt: prior.retry.attempt }
69
- : null;
65
+ const durableEpisode = readProviderRetryEpisode(this.deps.nodeId);
66
+ const episodeFault = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
67
+ ? prior
68
+ : durableEpisode?.state === 'admitted' &&
69
+ durableEpisode.sessionFile === this.sessionFile(settledSession) &&
70
+ this.isActiveAutoFault(durableEpisode.fault)
71
+ ? durableEpisode.fault
72
+ : null;
73
+ const episode = episodeFault === null
74
+ ? null
75
+ : { since: episodeFault.since, anchorEntryId: episodeFault.anchorEntryId, attempt: episodeFault.retry.attempt };
70
76
  const messages = Array.isArray(agentEnd?.messages) ? agentEnd.messages : [];
71
77
  const last = [...messages].reverse().find((message) => typeof message === 'object' && message !== null && message.role === 'assistant');
72
78
  if (overflowFailure !== null) {
73
79
  clearFault(this.deps.nodeId, { link: 'pi→provider' });
80
+ clearProviderRetryEpisode(this.deps.nodeId);
74
81
  recordFault(this.deps.nodeId, {
75
82
  link: 'pi→provider', op: 'context overflow recovery', kind: 'context-overflow', retry: { disposition: 'fatal' },
76
83
  message: overflowFailure.errorMessage, anchorEntryId: settledSession.sessionManager.getLeafId?.() ?? undefined,
@@ -87,21 +94,30 @@ export class FaultRetry {
87
94
  });
88
95
  return;
89
96
  }
90
- if (last?.stopReason !== 'error')
97
+ if (last?.stopReason !== 'error') {
98
+ if (last !== undefined)
99
+ clearProviderRetryEpisode(this.deps.nodeId);
91
100
  return;
92
- if (refreshAbort)
101
+ }
102
+ if (refreshAbort) {
103
+ clearProviderRetryEpisode(this.deps.nodeId);
93
104
  return;
105
+ }
94
106
  const raw = typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)';
95
107
  // An abort landing between requests reaches pi's catch as a bare AbortError
96
108
  // and settles as an error rather than `aborted`. Something asked that turn
97
109
  // to stop, so there is no provider fault to record.
98
- if (endedByAbort({ stopReason: 'error', errorMessage: raw }))
110
+ if (endedByAbort({ stopReason: 'error', errorMessage: raw })) {
111
+ clearProviderRetryEpisode(this.deps.nodeId);
99
112
  return;
113
+ }
100
114
  const uncompactedOverflow = isContextOverflow(last, settledSession.model?.contextWindow);
101
115
  const classified = classify('pi→provider', last);
102
116
  const coolingDeadline = extractCoolingDeadline(last);
103
- if (uncompactedOverflow)
117
+ if (uncompactedOverflow) {
104
118
  clearFault(this.deps.nodeId, { link: 'pi→provider' });
119
+ clearProviderRetryEpisode(this.deps.nodeId);
120
+ }
105
121
  const faultInput = {
106
122
  link: 'pi→provider', op: 'generation turn', kind: uncompactedOverflow ? 'context-overflow' : classified.kind,
107
123
  retry: uncompactedOverflow
@@ -121,8 +137,10 @@ export class FaultRetry {
121
137
  };
122
138
  if (faultInput.retry.disposition === 'auto')
123
139
  this.recordPendingProviderFault(settledSession, faultInput);
124
- else
140
+ else {
141
+ clearProviderRetryEpisode(this.deps.nodeId);
125
142
  recordFault(this.deps.nodeId, faultInput);
143
+ }
126
144
  }
127
145
  /** Startup re-drive reads only a durable pending episode. Revive deliberately
128
146
  * clears the ordinary fault marker before this boundary, so the episode is
@@ -1,4 +1,5 @@
1
1
  export declare const HELD_DEFERRED_INBOX_PROMPT_JOIN: unique symbol;
2
+ export declare const BROKER_IDLE_TURN_START: unique symbol;
2
3
  export type HeldDeferredInboxPromptJoin = (session: {
3
4
  sendCustomMessage: (message: {
4
5
  customType: string;
@@ -7,3 +8,6 @@ export type HeldDeferredInboxPromptJoin = (session: {
7
8
  }) => Promise<void>;
8
9
  }) => Promise<void>;
9
10
  export declare function heldDeferredInboxPromptJoin(): HeldDeferredInboxPromptJoin | undefined;
11
+ /** Starts an inbox-delivered idle turn through the broker's admission gate. */
12
+ export type BrokerIdleTurnStart = (text: string) => void;
13
+ export declare function brokerIdleTurnStart(): BrokerIdleTurnStart | undefined;
@@ -1,8 +1,13 @@
1
1
  // The inbox watcher is a Pi extension loaded through Jiti, while the broker is
2
- // native ESM. They therefore cannot share a module singleton. This process-global
3
- // symbol names the watcher-owned join function across that loader boundary.
2
+ // native ESM. They therefore cannot share a module singleton. Process-global
3
+ // symbols carry their narrow bridge functions across that loader boundary.
4
4
  export const HELD_DEFERRED_INBOX_PROMPT_JOIN = Symbol.for('@crouton-kit/crtr:held-deferred-inbox-prompt-join');
5
+ export const BROKER_IDLE_TURN_START = Symbol.for('@crouton-kit/crtr:broker-idle-turn-start');
5
6
  export function heldDeferredInboxPromptJoin() {
6
7
  const join = globalThis[HELD_DEFERRED_INBOX_PROMPT_JOIN];
7
8
  return typeof join === 'function' ? join : undefined;
8
9
  }
10
+ export function brokerIdleTurnStart() {
11
+ const start = globalThis[BROKER_IDLE_TURN_START];
12
+ return typeof start === 'function' ? start : undefined;
13
+ }