@north-light/crouter 0.3.250 → 0.3.252

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/builtin-memory/02-turn-lifecycle/00-ending-a-turn.md +1 -5
  2. package/dist/builtin-memory/05-kinds/review/security-findings.md +1 -1
  3. package/dist/clients/attach/__tests__/completion-frecency.test.d.ts +1 -0
  4. package/dist/clients/attach/__tests__/completion-frecency.test.js +35 -0
  5. package/dist/clients/attach/__tests__/ref-autocomplete.test.js +3 -1
  6. package/dist/clients/attach/__tests__/titled-editor-preview.test.js +2 -1
  7. package/dist/clients/attach/input/completion-frecency.d.ts +45 -0
  8. package/dist/clients/attach/input/completion-frecency.js +141 -0
  9. package/dist/clients/attach/input/controller.d.ts +17 -0
  10. package/dist/clients/attach/input/controller.js +40 -0
  11. package/dist/clients/attach/input/ref-autocomplete.d.ts +3 -1
  12. package/dist/clients/attach/input/ref-autocomplete.js +49 -8
  13. package/dist/clients/attach/input/titled-editor.d.ts +3 -0
  14. package/dist/clients/attach/input/titled-editor.js +5 -0
  15. package/dist/clients/attach/session/editor-inventory.d.ts +3 -0
  16. package/dist/clients/attach/session/editor-inventory.js +1 -1
  17. package/dist/clients/attach/session/input-wiring.d.ts +3 -0
  18. package/dist/clients/attach/session/input-wiring.js +2 -0
  19. package/dist/clients/attach/viewer.js +578 -578
  20. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +98 -1
  21. package/dist/core/__tests__/canvas-inbox-watcher-naming.test.js +4 -1
  22. package/dist/core/__tests__/canvas-inbox-watcher.test.js +4 -1
  23. package/dist/core/__tests__/fixtures/fake-engine.d.ts +3 -1
  24. package/dist/core/__tests__/fixtures/fake-engine.js +16 -3
  25. package/dist/core/__tests__/integration/deferred-no-wake.test.js +4 -1
  26. package/dist/core/__tests__/migration.test.js +57 -11
  27. package/dist/core/__tests__/seam/broker-provider-retry.test.js +17 -2
  28. package/dist/core/__tests__/seam/dormancy-release.test.js +1 -0
  29. package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
  30. package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
  31. package/dist/core/__tests__/watchdog-abort-arms-retry.test.js +9 -2
  32. package/dist/core/canvas/migrations.js +63 -24
  33. package/dist/core/runtime/broker/engine-drive.js +20 -3
  34. package/dist/core/runtime/broker/fault-retry.js +31 -13
  35. package/dist/core/runtime/broker/held-deferred-inbox.d.ts +13 -0
  36. package/dist/core/runtime/broker/held-deferred-inbox.js +13 -0
  37. package/dist/core/runtime/broker.js +19 -0
  38. package/dist/core/runtime/close.js +2 -2
  39. package/dist/core/runtime/fault.js +8 -5
  40. package/dist/core/runtime/recycle.js +63 -41
  41. package/dist/core/runtime/reset.d.ts +1 -1
  42. package/dist/core/runtime/reset.js +25 -2
  43. package/dist/daemon/__tests__/helpers/source-daemon.js +1 -0
  44. package/dist/daemon/api/handlers/messages.js +1 -1
  45. package/dist/daemon/api/handlers/reports.js +1 -1
  46. package/dist/daemon/cron/sinks.js +1 -1
  47. package/dist/daemon/messaging/node-message.js +1 -1
  48. package/dist/pi-extensions/canvas-inbox-watcher.js +109 -5
  49. package/package.json +1 -1
  50. package/runtime.lock.json +2 -2
@@ -26,14 +26,19 @@ import { basename, join, resolve } from 'node:path';
26
26
  import { crtrHome, isSafeNodeId, nodesRoot, nodeMetaPath } from './paths.js';
27
27
  import { STATE_MIGRATIONS } from '../../migrations/registry.js';
28
28
  import { customExtensionPaths } from './extensions.js';
29
- import { updateJsonFileDurably } from './meta-file.js';
29
+ import { readMetaObject, updateJsonFileDurably, withMetaLock, writeMetaObjectLocked } from './meta-file.js';
30
30
  // Schema as a forward-only migration list
31
31
  //
32
32
  // The schema is the migration list: one place a schema change is expressed,
33
33
  // one gate (`PRAGMA user_version`) that applies it. `migrate()` runs every
34
34
  // pending step in order and bumps `user_version` after each, so a fresh db and
35
35
  // the live fleet (all at `user_version 0`) converge on the same final shape.
36
- // Migrations are append-only and forward-only — never edit a shipped step.
36
+ // Migrations are append-only and forward-only — never edit a shipped step. The
37
+ // v36 body below is the sole corrective exception: v0.3.250 shipped a destructive
38
+ // stale-row decision, so v36 is narrowed for databases still below its gate while
39
+ // appended v37 repairs databases that already crossed it. Do not repeat this
40
+ // exception; an appended migration is required unless it would run too late to
41
+ // prevent destruction at an existing version boundary.
37
42
  /** v1 — the baseline tables + indexes. `IF NOT EXISTS` makes this a no-op on
38
43
  * any existing db (the live fleet already has these tables). */
39
44
  function baselineSchema(db) {
@@ -1215,32 +1220,43 @@ function addNodeSessionFileIndex(db) {
1215
1220
  }
1216
1221
  db.exec('CREATE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);');
1217
1222
  }
1218
- /** v36 — one Pi transcript may have exactly one node owner. Normalize every
1219
- * stored path first (including meta), then retain the earliest created row
1220
- * (node_id breaks equal timestamps) and clear both durable session coordinates
1221
- * from every later claimant before creating the UNIQUE index. */
1222
- function makeNodeSessionFileUnique(db) {
1223
+ /** Reproject the derived session-file index from authoritative node metadata.
1224
+ * v36 may resolve duplicate authoritative claimants because it runs before the
1225
+ * first UNIQUE index exists. v37 must retain every surviving meta claim: a
1226
+ * conflict at that later boundary is irreconcilable and fails instead. */
1227
+ function reconcileNodeSessionFileOwners(db, clearDuplicateMetaClaimants) {
1223
1228
  const canonical = (sessionFile) => {
1224
1229
  const absolute = resolve(sessionFile);
1225
1230
  return existsSync(absolute) ? realpathSync(absolute) : absolute;
1226
1231
  };
1227
- const rows = db.prepare(`
1228
- SELECT node_id, pi_session_file
1229
- FROM nodes
1230
- WHERE pi_session_file IS NOT NULL
1231
- ORDER BY created ASC, node_id ASC
1232
- `).all();
1233
1232
  const setFile = db.prepare('UPDATE nodes SET pi_session_file = ? WHERE node_id = ?');
1234
- for (const row of rows) {
1235
- const sessionFile = canonical(row.pi_session_file);
1236
- if (sessionFile === row.pi_session_file)
1233
+ db.exec('DROP INDEX IF EXISTS idx_nodes_pi_session_file;');
1234
+ const rows = db.prepare('SELECT node_id FROM nodes ORDER BY created ASC, node_id ASC').all();
1235
+ for (const { node_id } of rows) {
1236
+ const metaPath = nodeMetaPath(node_id);
1237
+ // Missing or unreadable metadata has no authoritative coordinate. Avoid
1238
+ // taking a file lock there: old rows can outlive a removed node directory.
1239
+ if (readMetaObject(metaPath) === null) {
1240
+ setFile.run(null, node_id);
1237
1241
  continue;
1238
- // Identity is authoritative, so normalize both stores before grouping.
1239
- updateJsonFileDurably(nodeMetaPath(row.node_id), (meta) => {
1240
- meta['pi_session_file'] = sessionFile;
1241
- return true;
1242
+ }
1243
+ const sessionFile = withMetaLock(metaPath, () => {
1244
+ const meta = readMetaObject(metaPath);
1245
+ if (meta === null)
1246
+ throw new Error(`node ${node_id} session metadata changed during ownership migration`);
1247
+ if (typeof meta['pi_session_file'] !== 'string')
1248
+ return null;
1249
+ const storedSessionFile = meta['pi_session_file'];
1250
+ const normalizedSessionFile = canonical(storedSessionFile);
1251
+ if (normalizedSessionFile !== storedSessionFile) {
1252
+ meta['pi_session_file'] = normalizedSessionFile;
1253
+ writeMetaObjectLocked(metaPath, meta);
1254
+ }
1255
+ return normalizedSessionFile;
1242
1256
  });
1243
- setFile.run(sessionFile, row.node_id);
1257
+ // meta.json is authoritative: a crash after its write can leave this row
1258
+ // claiming another node's old session coordinate.
1259
+ setFile.run(sessionFile, node_id);
1244
1260
  }
1245
1261
  const duplicates = db.prepare(`
1246
1262
  SELECT pi_session_file
@@ -1249,6 +1265,10 @@ WHERE pi_session_file IS NOT NULL
1249
1265
  GROUP BY pi_session_file
1250
1266
  HAVING COUNT(*) > 1
1251
1267
  `).all();
1268
+ if (!clearDuplicateMetaClaimants && duplicates.length > 0) {
1269
+ const coordinates = duplicates.map(({ pi_session_file }) => pi_session_file).join(', ');
1270
+ throw new Error(`cannot restore unique Pi session ownership: authoritative metadata has concurrent claimants for ${coordinates}`);
1271
+ }
1252
1272
  const clearRow = db.prepare('UPDATE nodes SET pi_session_file = NULL WHERE node_id = ?');
1253
1273
  for (const { pi_session_file } of duplicates) {
1254
1274
  const owners = db.prepare(`
@@ -1259,18 +1279,36 @@ ORDER BY created ASC, node_id ASC
1259
1279
  `).all(pi_session_file);
1260
1280
  for (const { node_id } of owners.slice(1)) {
1261
1281
  // Clear both durable coordinates so a later reindex cannot restore the
1262
- // duplicate projection.
1263
- updateJsonFileDurably(nodeMetaPath(node_id), (meta) => {
1282
+ // duplicate projection. The checked write fails rather than erasing a
1283
+ // concurrent metadata change.
1284
+ const cleared = updateJsonFileDurably(nodeMetaPath(node_id), (meta) => {
1285
+ const currentSessionFile = meta['pi_session_file'];
1286
+ if (typeof currentSessionFile !== 'string' || canonical(currentSessionFile) !== pi_session_file) {
1287
+ throw new Error(`node ${node_id} session metadata changed during ownership migration`);
1288
+ }
1264
1289
  meta['pi_session_id'] = null;
1265
1290
  meta['pi_session_file'] = null;
1266
1291
  return true;
1267
1292
  });
1293
+ if (!cleared)
1294
+ throw new Error(`node ${node_id} session metadata changed during ownership migration`);
1268
1295
  clearRow.run(node_id);
1269
1296
  }
1270
1297
  }
1271
- db.exec('DROP INDEX IF EXISTS idx_nodes_pi_session_file;');
1272
1298
  db.exec('CREATE UNIQUE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);');
1273
1299
  }
1300
+ /** v36 — one Pi transcript may have exactly one node owner. This is the narrow
1301
+ * corrective body for clients below v36; it reprojects metadata before choosing
1302
+ * the earliest created duplicate claimant (node_id breaks equal timestamps). */
1303
+ function makeNodeSessionFileUnique(db) {
1304
+ reconcileNodeSessionFileOwners(db, true);
1305
+ }
1306
+ /** v37 — repair v0.3.250's already-applied v36 projection without inventing
1307
+ * coordinates v36 already cleared. Surviving authoritative metadata wins over
1308
+ * stale rows; concurrent surviving claims fail loudly rather than clearing one. */
1309
+ function repairNodeSessionFileUniqueIndex(db) {
1310
+ reconcileNodeSessionFileOwners(db, false);
1311
+ }
1274
1312
  /** v35 — durable node lifecycle execution state and terminal cause. */
1275
1313
  function addNodeLifecycleColumns(db) {
1276
1314
  const columns = nodeColumns(db);
@@ -1340,6 +1378,7 @@ export const MIGRATIONS = [
1340
1378
  /* v34 */ addNodeSessionFileIndex,
1341
1379
  /* v35 */ addNodeLifecycleColumns,
1342
1380
  /* v36 */ makeNodeSessionFileUnique,
1381
+ /* v37 */ repairNodeSessionFileUniqueIndex,
1343
1382
  ...JOURNALED_STATE_MIGRATIONS,
1344
1383
  ];
1345
1384
  /** Migration indexes that manage their OWN transaction and therefore must not
@@ -1,6 +1,7 @@
1
1
  import { emitEvent } from '../../events/emit.js';
2
2
  import { operationIdContext } from '../../events/operation-id.js';
3
3
  import { promptWithAdvertisedCommandInvocation } from '../advertised-command-invocation.js';
4
+ import { heldDeferredInboxPromptJoin } from './held-deferred-inbox.js';
4
5
  import { chooseGuidanceDeliveryMode, promptWithNextTurnGuidance, resolveEngineRoute, resolveSteerCall, } from './engine-routing.js';
5
6
  export function createEngineDriver(deps) {
6
7
  const { registry, replies, session, dispatches, memoryRefs, authReloadGate, rebind, projection, toolGroups, turnAdmission, } = deps;
@@ -44,14 +45,30 @@ export function createEngineDriver(deps) {
44
45
  // false. Admission waits for that preflight, then routes against the
45
46
  // state it actually established instead of its stale snapshot.
46
47
  const route = resolveEngineRoute(frame, session().isStreaming);
48
+ const promptOptions = route.call === 'prompt' ? route.options : undefined;
47
49
  const guidance = memoryRefs.guidanceFor(frame.text);
50
+ const prompt = async () => {
51
+ // Both human surfaces converge at this admitted boundary. The watcher
52
+ // claims held deferred mail before the prompt body, but its cursor stays
53
+ // uncommitted until the resulting turn's agent_settled event.
54
+ if (route.call === 'prompt' && !session().isStreaming) {
55
+ try {
56
+ await heldDeferredInboxPromptJoin()?.(session());
57
+ }
58
+ catch {
59
+ // The watcher emitted the failed durable delivery and retains it for
60
+ // replay. A person's live prompt is not replayable, so it must run.
61
+ }
62
+ }
63
+ await session().prompt(frame.text, { ...promptOptions, preflightResult: release });
64
+ };
48
65
  if (guidance === null) {
49
66
  if (route.call === 'prompt') {
50
67
  // A routed prompt STARTS a turn — including an idle follow_up
51
68
  // escalated by m-C — so it gets a fresh operation id. A true
52
69
  // (mid-stream) followUp rides the current turn's operation, like
53
70
  // steer.
54
- return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, { ...route.options, preflightResult: release })).catch(relayError))));
71
+ return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt).catch(relayError))));
55
72
  }
56
73
  // C1: pi's followUp() takes images as a POSITIONAL 2nd arg (`followUp(text, images?)`).
57
74
  release();
@@ -67,9 +84,9 @@ export function createEngineDriver(deps) {
67
84
  // ATOMICALLY clears its own pending entry if prompt() settles
68
85
  // without ever reaching that drain (an early throw/return leaves
69
86
  // nothing stranded to leak into a later, unrelated turn).
70
- return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, { ...route.options, preflightResult: release })))).catch(relayError)));
87
+ return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt))).catch(relayError)));
71
88
  }
72
- return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, { ...route.options, preflightResult: release }))
89
+ return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt)
73
90
  .catch(relayError)
74
91
  .then(() => sendGuidance(guidance, deliverAs)))));
75
92
  }
@@ -58,19 +58,26 @@ export class FaultRetry {
58
58
  generation.stagedRefreshAbort = false;
59
59
  generation.stagedOverflowFailure = null;
60
60
  const settledSession = generation.session;
61
- // A failed daemon retry settles here with the episode's fault marker still
62
- // present (a successful provider round-trip would have cleared it). Carry
63
- // the episode forward — original anchor, start time, and attempt count — so
64
- // each retry rewinds to the same fork point instead of stacking recovery
65
- // prompts, and so backoff/exhaustion actually accumulate.
61
+ // turn_end clears the ordinary marker before this settlement. A failed
62
+ // admitted retry therefore carries from its durable episode, while an
63
+ // initial provider failure still reads from the ordinary marker.
66
64
  const prior = readFault(this.deps.nodeId);
67
- const episode = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
68
- ? { since: prior.since, anchorEntryId: prior.anchorEntryId, attempt: prior.retry.attempt }
69
- : null;
65
+ const durableEpisode = readProviderRetryEpisode(this.deps.nodeId);
66
+ const episodeFault = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
67
+ ? prior
68
+ : durableEpisode?.state === 'admitted' &&
69
+ durableEpisode.sessionFile === this.sessionFile(settledSession) &&
70
+ this.isActiveAutoFault(durableEpisode.fault)
71
+ ? durableEpisode.fault
72
+ : null;
73
+ const episode = episodeFault === null
74
+ ? null
75
+ : { since: episodeFault.since, anchorEntryId: episodeFault.anchorEntryId, attempt: episodeFault.retry.attempt };
70
76
  const messages = Array.isArray(agentEnd?.messages) ? agentEnd.messages : [];
71
77
  const last = [...messages].reverse().find((message) => typeof message === 'object' && message !== null && message.role === 'assistant');
72
78
  if (overflowFailure !== null) {
73
79
  clearFault(this.deps.nodeId, { link: 'pi→provider' });
80
+ clearProviderRetryEpisode(this.deps.nodeId);
74
81
  recordFault(this.deps.nodeId, {
75
82
  link: 'pi→provider', op: 'context overflow recovery', kind: 'context-overflow', retry: { disposition: 'fatal' },
76
83
  message: overflowFailure.errorMessage, anchorEntryId: settledSession.sessionManager.getLeafId?.() ?? undefined,
@@ -87,21 +94,30 @@ export class FaultRetry {
87
94
  });
88
95
  return;
89
96
  }
90
- if (last?.stopReason !== 'error')
97
+ if (last?.stopReason !== 'error') {
98
+ if (last !== undefined)
99
+ clearProviderRetryEpisode(this.deps.nodeId);
91
100
  return;
92
- if (refreshAbort)
101
+ }
102
+ if (refreshAbort) {
103
+ clearProviderRetryEpisode(this.deps.nodeId);
93
104
  return;
105
+ }
94
106
  const raw = typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)';
95
107
  // An abort landing between requests reaches pi's catch as a bare AbortError
96
108
  // and settles as an error rather than `aborted`. Something asked that turn
97
109
  // to stop, so there is no provider fault to record.
98
- if (endedByAbort({ stopReason: 'error', errorMessage: raw }))
110
+ if (endedByAbort({ stopReason: 'error', errorMessage: raw })) {
111
+ clearProviderRetryEpisode(this.deps.nodeId);
99
112
  return;
113
+ }
100
114
  const uncompactedOverflow = isContextOverflow(last, settledSession.model?.contextWindow);
101
115
  const classified = classify('pi→provider', last);
102
116
  const coolingDeadline = extractCoolingDeadline(last);
103
- if (uncompactedOverflow)
117
+ if (uncompactedOverflow) {
104
118
  clearFault(this.deps.nodeId, { link: 'pi→provider' });
119
+ clearProviderRetryEpisode(this.deps.nodeId);
120
+ }
105
121
  const faultInput = {
106
122
  link: 'pi→provider', op: 'generation turn', kind: uncompactedOverflow ? 'context-overflow' : classified.kind,
107
123
  retry: uncompactedOverflow
@@ -121,8 +137,10 @@ export class FaultRetry {
121
137
  };
122
138
  if (faultInput.retry.disposition === 'auto')
123
139
  this.recordPendingProviderFault(settledSession, faultInput);
124
- else
140
+ else {
141
+ clearProviderRetryEpisode(this.deps.nodeId);
125
142
  recordFault(this.deps.nodeId, faultInput);
143
+ }
126
144
  }
127
145
  /** Startup re-drive reads only a durable pending episode. Revive deliberately
128
146
  * clears the ordinary fault marker before this boundary, so the episode is
@@ -0,0 +1,13 @@
1
+ export declare const HELD_DEFERRED_INBOX_PROMPT_JOIN: unique symbol;
2
+ export declare const BROKER_IDLE_TURN_START: unique symbol;
3
+ export type HeldDeferredInboxPromptJoin = (session: {
4
+ sendCustomMessage: (message: {
5
+ customType: string;
6
+ content: string;
7
+ display: boolean;
8
+ }) => Promise<void>;
9
+ }) => Promise<void>;
10
+ export declare function heldDeferredInboxPromptJoin(): HeldDeferredInboxPromptJoin | undefined;
11
+ /** Starts an inbox-delivered idle turn through the broker's admission gate. */
12
+ export type BrokerIdleTurnStart = (text: string) => void;
13
+ export declare function brokerIdleTurnStart(): BrokerIdleTurnStart | undefined;
@@ -0,0 +1,13 @@
1
+ // The inbox watcher is a Pi extension loaded through Jiti, while the broker is
2
+ // native ESM. They therefore cannot share a module singleton. Process-global
3
+ // symbols carry their narrow bridge functions across that loader boundary.
4
+ export const HELD_DEFERRED_INBOX_PROMPT_JOIN = Symbol.for('@crouton-kit/crtr:held-deferred-inbox-prompt-join');
5
+ export const BROKER_IDLE_TURN_START = Symbol.for('@crouton-kit/crtr:broker-idle-turn-start');
6
+ export function heldDeferredInboxPromptJoin() {
7
+ const join = globalThis[HELD_DEFERRED_INBOX_PROMPT_JOIN];
8
+ return typeof join === 'function' ? join : undefined;
9
+ }
10
+ export function brokerIdleTurnStart() {
11
+ const start = globalThis[BROKER_IDLE_TURN_START];
12
+ return typeof start === 'function' ? start : undefined;
13
+ }
@@ -47,6 +47,7 @@ import { ToolGroupTracker } from './broker/tool-groups.js';
47
47
  import { onNodeNamed } from './broker/node-named.js';
48
48
  import { FaultRetry } from './broker/fault-retry.js';
49
49
  import { createTurnAdmissionGate } from './broker/turn-admission.js';
50
+ import { BROKER_IDLE_TURN_START } from './broker/held-deferred-inbox.js';
50
51
  import { EventProjection } from './broker/event-projection.js';
51
52
  import { createFrameDispatchContext, handleFrame, } from './broker/frame-dispatch.js';
52
53
  import { hydratePersistedAdvertisedCommandMessages, installAdvertisedCommandInvocationContract, } from './advertised-command-invocation.js';
@@ -327,6 +328,24 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
327
328
  registry.broadcast({ type: 'model_changed', model: isUnknownModel(liveSession.model) ? undefined : liveSession.model, spec });
328
329
  };
329
330
  const turnAdmission = createTurnAdmissionGate();
331
+ // The inbox watcher is Jiti-loaded and cannot import this native module's
332
+ // gate. Its idle delivery therefore calls this narrow process-global bridge:
333
+ // acquire admission first, then re-read the live session before prompt() so
334
+ // a viewer turn that won the race is joined rather than double-started.
335
+ const startIdleInboxTurn = (text) => {
336
+ void turnAdmission.admit((release) => {
337
+ const liveSession = rebind.session();
338
+ return liveSession
339
+ .prompt(text, {
340
+ preflightResult: release,
341
+ ...(liveSession.isStreaming ? { streamingBehavior: 'steer' } : {}),
342
+ })
343
+ .catch((error) => {
344
+ emitEvent({ level: 'error', event: 'broker.inbox.idle_prompt_failed', error });
345
+ });
346
+ });
347
+ };
348
+ globalThis[BROKER_IDLE_TURN_START] = startIdleInboxTurn;
330
349
  faultRetry = new FaultRetry({
331
350
  nodeId,
332
351
  cfg,
@@ -49,12 +49,12 @@ export function fanDoctrineWake(fromId, subscribers, label, data, tier = 'normal
49
49
  try {
50
50
  const notice = { from: fromId, tier, kind: 'message', label, data };
51
51
  if (sub.active) {
52
- appendInbox(sub.node_id, notice);
52
+ const entry = appendInbox(sub.node_id, notice);
53
53
  // A wake-capable notice consumes the subscriber's cancel-on-wake
54
54
  // deadline: the thing it was waiting for arrived. `deferred` is the
55
55
  // exception — it never wakes a node, riding the next natural cycle
56
56
  // instead, so the deadline it was racing must survive.
57
- if (tier !== 'deferred')
57
+ if (entry.tier !== 'deferred')
58
58
  cancelCronsOnWake(sub.node_id);
59
59
  }
60
60
  else
@@ -229,11 +229,14 @@ export function clearFault(nodeId, opts) {
229
229
  if (opts?.link !== undefined && current.link !== opts.link)
230
230
  return false;
231
231
  rmSync(faultPath(nodeId));
232
- if (opts?.preserveProviderRetryEpisode !== true) {
233
- const episode = currentProviderRetryEpisode(nodeId);
234
- if (episode?.fault.since === current.since)
235
- rmSync(providerRetryPath(nodeId), { force: true });
236
- }
232
+ const episode = currentProviderRetryEpisode(nodeId);
233
+ // turn_end precedes agent_settled. An admitted retry keeps its durable
234
+ // coordinate while the broker classifies an error settlement; an invalidated
235
+ // coordinate also remains durable so replacement startup cannot replay it.
236
+ if (opts?.preserveProviderRetryEpisode !== true &&
237
+ episode?.fault.since === current.since &&
238
+ episode.state === 'pending')
239
+ rmSync(providerRetryPath(nodeId), { force: true });
237
240
  const operationId = resolveOperationId();
238
241
  emitEvent({
239
242
  level: 'info',
@@ -5,8 +5,8 @@
5
5
  //
6
6
  // 1. Finalize — push the agent's last surfaced message as a `final` report so
7
7
  // every subscriber/manager waiting on it is unblocked, and mark it done.
8
- // 2. Close — tear the agent's broker engine down (the pane is only a viewer).
9
- // 3. Recycle — boot a fresh resident broker root the pane re-attaches to.
8
+ // 2. Recycle — boot a fresh resident broker root and prove it accepts viewers.
9
+ // 3. Close — tear the finalized broker down, then re-attach the pane to the replacement.
10
10
  //
11
11
  // NOT to be confused with `node lifecycle demote` (flip-to-terminal IN PLACE, which keeps
12
12
  // the agent focused and running): recycle ENDS this agent and boots a brand-new
@@ -17,7 +17,7 @@
17
17
  // message) — falling back to a short note when it never reported.
18
18
  import { readdirSync, readFileSync, statSync } from 'node:fs';
19
19
  import { join } from 'node:path';
20
- import { getNode, setPresence, setFocusOccupant, fullName } from '../canvas/index.js';
20
+ import { getNode, setPresence, setFocusOccupant, fullName, recordLaunch, recordPid } from '../canvas/index.js';
21
21
  import { reportsDir } from '../canvas/paths.js';
22
22
  import { pushFinal } from '../feed/feed.js';
23
23
  import { spawnNode, rootOfSpine } from './nodes.js';
@@ -25,6 +25,7 @@ import { buildLaunchSpecAsync, buildPiArgv } from './launch.js';
25
25
  import { focusOf, respawnPaneSync, setPaneOption } from './placement.js';
26
26
  import { waitForBrokerViewSocket, viewerSplitEnv } from './placement-tmux.js';
27
27
  import { headlessBrokerHost } from './host.js';
28
+ import { transition } from './lifecycle.js';
28
29
  import { ensureDaemon } from '../../daemon/manage.js';
29
30
  /** The agent's most recent surfaced message: the newest reports/*.md body with
30
31
  * its YAML frontmatter stripped. Empty string when the node never reported. */
@@ -74,23 +75,11 @@ export async function recycleNode(nodeId, callerPane) {
74
75
  finalized = true;
75
76
  }
76
77
  catch { /* recycle the pane even if the report failed */ }
77
- // The node's pane is a VIEWER (`crtr surface attach`), not its engine: the engine is
78
- // the detached broker process, so respawn-pane -k below would only kill the
79
- // viewer, never the engine. Tear the broker PROCESS down so it exits and
80
- // releases the sole .jsonl writer. Status is already flipped done by pushFinal
81
- // above (crash-safe order: the daemon won't revive a done node).
82
- try {
83
- headlessBrokerHost.teardown(nodeId);
84
- }
85
- catch { /* best-effort */ }
86
- // Capture M's focus viewport (if any) BEFORE nulling — the fresh root inherits
87
- // it (the SAME focus row + pane). The demoted node no longer holds a pane: it is
88
- // being reclaimed.
78
+ // Capture M's focus viewport (if any) before booting its replacement. The
79
+ // old broker remains live until the new broker proves it can accept a viewer;
80
+ // a failed replacement must not strand this finished conversation without an
81
+ // engine.
89
82
  const f = focusOf(nodeId);
90
- try {
91
- setPresence(nodeId, { pane: null, window: null, tmux_session: null });
92
- }
93
- catch { /* best-effort */ }
94
83
  // 2 + 3. Recycle — boot a fresh resident BROKER root for the SAME pane: the
95
84
  // viewer pane re-attaches to the fresh broker (broker-is-the-host — the viewer
96
85
  // pane stays a viewer, never becomes an engine pane).
@@ -113,36 +102,69 @@ export async function recycleNode(nodeId, callerPane) {
113
102
  profile_id: meta.profile_id,
114
103
  launch,
115
104
  });
116
- // Hand the viewport to the fresh root: reuse M's focus row over the SAME pane
117
- // (respawn-pane -k below keeps the %id), so the user keeps watching this slot.
105
+ const fresh = getNode(root.node_id);
106
+ const inv = buildPiArgv(fresh);
107
+ // CRTR_SUBTREE groups the fresh root's subtree; FRONT_DOOR is set by the broker
108
+ // host itself, so it is not added here.
109
+ inv.env = { ...inv.env, CRTR_SUBTREE: rootOfSpine(root.node_id) };
110
+ if (!(await launchRecycledBroker(fresh, inv))) {
111
+ return { recycled: false, finalized, newRoot: root.node_id, delivered };
112
+ }
113
+ // COMMIT after replacement readiness: the node's pane is a VIEWER, so tear
114
+ // down the old detached broker only now that the fresh broker can serve it.
115
+ try {
116
+ headlessBrokerHost.teardown(nodeId);
117
+ }
118
+ catch { /* best-effort */ }
119
+ try {
120
+ setPresence(nodeId, { pane: null, window: null, tmux_session: null });
121
+ }
122
+ catch { /* best-effort */ }
118
123
  if (f !== null) {
119
124
  try {
120
125
  setFocusOccupant(f.focus_id, root.node_id);
121
126
  }
122
127
  catch { /* best-effort */ }
123
128
  }
124
- const fresh = getNode(root.node_id);
125
- const inv = buildPiArgv(fresh);
126
- // CRTR_SUBTREE groups the fresh root's subtree; FRONT_DOOR is set by the broker
127
- // host itself, so it is not added here.
128
- inv.env = { ...inv.env, CRTR_SUBTREE: rootOfSpine(root.node_id) };
129
- const ok = await recycleBrokerViewer(fresh, pane, inv);
129
+ const ok = respawnRecycledViewer(fresh, pane);
130
130
  return { recycled: ok, finalized, newRoot: root.node_id, delivered };
131
131
  }
132
- /** Recycle a BROKER root into `pane`: the fresh root is broker-hosted, so its
133
- * engine runs in a DETACHED broker, not the pane. Birth-launch that broker via
134
- * the Host seam (mirrors spawnChild's birth path — the host records its pid),
135
- * wait for its view.sock to accept, then respawn the pane in place to the VIEWER
136
- * `crtr surface attach to <root>`. The pane stays a viewer (attach self-tags it
137
- * `@crtr_node`); it never hosts the engine. Returns false (recycle reports the
138
- * pane was not respawned) when the broker never serves — the fresh root row
139
- * still exists, broker-hosted, for the daemon to revive. */
140
- async function recycleBrokerViewer(fresh, pane, inv) {
141
- const placed = headlessBrokerHost.launch(fresh.node_id, inv, { cwd: fresh.cwd, name: fullName(fresh), resuming: false });
142
- if (placed.pid === null)
143
- return false;
144
- if (!(await waitForBrokerViewSocket(fresh.node_id, placed.exited)))
145
- return false;
132
+ /** Launch a recycled root and prove its viewer socket before recycling the old
133
+ * broker. This direct birth path writes the same launch/pid coordinates as
134
+ * reviveNode, and a failed readiness probe terminalizes and tears it down. */
135
+ async function launchRecycledBroker(fresh, inv) {
136
+ let placed;
137
+ try {
138
+ placed = headlessBrokerHost.launch(fresh.node_id, inv, { cwd: fresh.cwd, name: fullName(fresh), resuming: false });
139
+ if (placed.pid === null)
140
+ throw new Error('broker host returned no pid');
141
+ recordLaunch(fresh.node_id, new Date().toISOString());
142
+ recordPid(fresh.node_id, placed.pid);
143
+ if (await waitForBrokerViewSocket(fresh.node_id, placed.exited))
144
+ return true;
145
+ }
146
+ catch {
147
+ // The replacement never became usable; the finished root's broker remains
148
+ // intact and this half-born row cannot remain active without a fleet handle.
149
+ }
150
+ try {
151
+ transitionReplacementToDead(fresh.node_id);
152
+ }
153
+ catch { /* best-effort */ }
154
+ try {
155
+ headlessBrokerHost.teardown(fresh.node_id);
156
+ }
157
+ catch { /* best-effort */ }
158
+ return false;
159
+ }
160
+ function transitionReplacementToDead(nodeId) {
161
+ // spawnNode births active rows, so crash is the authoritative failed-launch
162
+ // outcome whether the host returned no pid or exited before readiness.
163
+ const replacement = getNode(nodeId);
164
+ if (replacement?.status === 'active' || replacement?.status === 'idle')
165
+ transition(nodeId, 'crash');
166
+ }
167
+ function respawnRecycledViewer(fresh, pane) {
146
168
  // Clear the finalized node's stale `@crtr_node` tag before respawn so the tag
147
169
  // never names a done node during the gap before the new `crtr surface attach` re-tags
148
170
  // on connect. Node ids are shell-safe identifiers; no quoting needed.
@@ -25,7 +25,7 @@ export interface RelaunchDeps {
25
25
  }>) => boolean | Promise<boolean>;
26
26
  /** Re-exec the viewer pane onto the new node. Default: respawnPaneSync. */
27
27
  respawnViewer?: typeof respawnPaneSync;
28
- /** Tear the old broker down. Default: headlessBrokerHost.teardown. */
28
+ /** Tear down a broker after a failed replacement or a committed old-root replacement. */
29
29
  teardownBroker?: typeof headlessBrokerHost.teardown;
30
30
  }
31
31
  export interface RelaunchRootResult {
@@ -20,7 +20,7 @@
20
20
  // reserved for finish).
21
21
  //
22
22
  // Best-effort throughout: a tmux/fs failure on one node never aborts the reap.
23
- import { getNode, updateNode, fullName, closeFocusRow, view, } from '../canvas/index.js';
23
+ import { getNode, updateNode, recordLaunch, recordPid, fullName, closeFocusRow, view, } from '../canvas/index.js';
24
24
  import { transition } from './lifecycle.js';
25
25
  import { headlessBrokerHost, requestBrokerTeardown } from './host.js';
26
26
  import { tearDownNode, focusOf, registerViewerFocus, respawnPaneSync, windowOfPane, renameWindow, } from './placement.js';
@@ -129,7 +129,30 @@ export async function relaunchRoot(oldId, deps = {}) {
129
129
  transition(newMeta.node_id, 'crash');
130
130
  return null;
131
131
  }
132
- await waitForViewSocket(newMeta.node_id, placed.exited); // best-effort; attach auto-redials on miss
132
+ // A replacement is a managed broker just like a revive: the fleet owns its
133
+ // real exit, while the row gets the matching launch and pid coordinates before
134
+ // readiness can observe an early exit.
135
+ recordLaunch(newMeta.node_id, new Date().toISOString());
136
+ recordPid(newMeta.node_id, placed.pid);
137
+ let ready = false;
138
+ try {
139
+ ready = await waitForViewSocket(newMeta.node_id, placed.exited);
140
+ }
141
+ catch {
142
+ // A readiness probe error means the replacement is not proven usable.
143
+ }
144
+ if (!ready) {
145
+ // Do not let a broker that can still bind later sit behind a dead row, and
146
+ // do not commit any old-root teardown without a usable replacement.
147
+ const replacement = getNode(newMeta.node_id);
148
+ if (replacement?.status === 'active' || replacement?.status === 'idle')
149
+ transition(newMeta.node_id, 'crash');
150
+ try {
151
+ teardownBroker(newMeta.node_id);
152
+ }
153
+ catch { /* best-effort */ }
154
+ return null;
155
+ }
133
156
  // --- COMMIT: park + reap the old root, re-point the viewer, kill the old
134
157
  // broker. Past this point the new node is the live root. ---
135
158
  reapDescendants(oldId); // old workers → canceled + torn down
@@ -57,6 +57,7 @@ export function startSourceDaemon(fixture) {
57
57
  };
58
58
  for (const key of [
59
59
  'CRTR_NODE_ID',
60
+ 'CRTR_SUBTREE',
60
61
  'CRTR_KIND',
61
62
  'CRTR_MODE',
62
63
  'CRTR_LIFECYCLE',
@@ -327,7 +327,7 @@ async function handleMessage(ctx) {
327
327
  const entry = deliver();
328
328
  // Deferred mail normally waits for a natural cycle, but a frozen row already
329
329
  // owes a thaw; consume its deadline when that durable future wake arrives.
330
- if (tier !== 'deferred' || meta.frozen_at !== null)
330
+ if (entry.tier !== 'deferred' || meta.frozen_at !== null)
331
331
  cancelCronsOnWake(id);
332
332
  // A wake-capable tier revives a dormant target so its inbox-watcher delivers
333
333
  // this; deferred never wakes — it rides the target's next natural cycle. The
@@ -33,7 +33,7 @@ function parsePushBody(body) {
33
33
  if (deliveryTier !== undefined && deliveryTier !== 'deferred' && deliveryTier !== 'normal' && deliveryTier !== 'urgent') {
34
34
  throw usage(`invalid delivery_tier: ${String(deliveryTier)} (expected deferred|normal|urgent)`);
35
35
  }
36
- if (tier === 'final' && deliveryTier !== undefined) {
36
+ if (tier !== 'update' && deliveryTier !== undefined) {
37
37
  throw usage('delivery_tier is only valid for update reports');
38
38
  }
39
39
  if (typeof text !== 'string' || text.trim() === '') {
@@ -31,7 +31,7 @@ function deliverNodeSink(c, target, stdout) {
31
31
  label: `⏰ cron ${c.name}`,
32
32
  data: { body: stdout },
33
33
  });
34
- if (c.tier !== 'deferred' || meta.frozen_at !== null)
34
+ if (entry.tier !== 'deferred' || meta.frozen_at !== null)
35
35
  cancelCronsOnWake(target);
36
36
  // The APPENDED tier decides the wake, not the cron's requested one: a deferred
37
37
  // cron aimed at a terminal node was raised out of `deferred` by the append.
@@ -59,7 +59,7 @@ export async function deliverNodeMessage(args) {
59
59
  label: args.label,
60
60
  data: { ...args.data, body: args.body },
61
61
  }));
62
- if (args.mode !== 'quiet' || target.frozen_at !== null)
62
+ if (entry.tier !== 'deferred' || target.frozen_at !== null)
63
63
  cancelCronsOnWake(args.node_id);
64
64
  // The APPENDED entry's tier decides the wake: a quiet send to a terminal
65
65
  // target was raised out of `deferred` by the append, and must be delivered.