@north-light/crouter 0.3.249 → 0.3.250

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/README.md +1 -1
  2. package/dist/api/__tests__/integration/client.test.js +28 -7
  3. package/dist/api/client.d.ts +1 -1
  4. package/dist/api/client.js +2 -2
  5. package/dist/api/dto/lifecycle.d.ts +3 -0
  6. package/dist/api/dto/messages.d.ts +4 -0
  7. package/dist/api/dto/nodes.d.ts +4 -0
  8. package/dist/api/dto/reports.d.ts +7 -2
  9. package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
  10. package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
  11. package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
  12. package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
  13. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
  14. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
  15. package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
  16. package/dist/clients/attach/session/chrome-refresh.js +1 -0
  17. package/dist/clients/attach/viewer.js +532 -532
  18. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
  19. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
  20. package/dist/clients/inbox/controller.js +32 -5
  21. package/dist/clients/inbox/tui/ansi.d.ts +3 -1
  22. package/dist/clients/inbox/tui/ansi.js +14 -10
  23. package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
  24. package/dist/commands/api-client.d.ts +2 -2
  25. package/dist/commands/api-client.js +5 -5
  26. package/dist/commands/canvas-browse.js +1 -1
  27. package/dist/commands/node/create.js +3 -1
  28. package/dist/commands/node/inspect.js +1 -0
  29. package/dist/commands/node/lifecycle.js +1 -1
  30. package/dist/commands/node/message.js +6 -4
  31. package/dist/commands/node/wait.js +1 -1
  32. package/dist/commands/push.js +36 -37
  33. package/dist/commands/sys/daemon.js +21 -8
  34. package/dist/commands/sys/setup-core.js +2 -2
  35. package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
  36. package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
  37. package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
  38. package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
  39. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +5 -1
  40. package/dist/core/__tests__/child-death-wake.test.js +11 -7
  41. package/dist/core/__tests__/collectible.test.d.ts +1 -0
  42. package/dist/core/__tests__/collectible.test.js +88 -0
  43. package/dist/core/__tests__/context-intro.test.js +4 -41
  44. package/dist/core/__tests__/daemon-boot.test.js +30 -14
  45. package/dist/core/__tests__/daemon-wedge.test.js +25 -19
  46. package/dist/core/__tests__/fixtures/fake-engine.js +11 -0
  47. package/dist/core/__tests__/human-deliver.test.js +5 -5
  48. package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
  49. package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
  50. package/dist/core/__tests__/integration/revive.test.js +48 -0
  51. package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
  52. package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
  53. package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
  54. package/dist/core/__tests__/migration.test.js +30 -0
  55. package/dist/core/__tests__/passive-subscription.test.js +39 -1
  56. package/dist/core/__tests__/prune.test.js +12 -3
  57. package/dist/core/__tests__/relaunch-root.test.js +1 -1
  58. package/dist/core/__tests__/respawn-throttle.test.js +16 -146
  59. package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
  60. package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
  61. package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
  62. package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
  63. package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
  64. package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
  65. package/dist/core/canvas/canvas.d.ts +9 -15
  66. package/dist/core/canvas/canvas.js +41 -47
  67. package/dist/core/canvas/crons.d.ts +5 -8
  68. package/dist/core/canvas/crons.js +5 -8
  69. package/dist/core/canvas/history.d.ts +5 -0
  70. package/dist/core/canvas/history.js +12 -1
  71. package/dist/core/canvas/migrations.js +58 -1
  72. package/dist/core/canvas/pid.d.ts +1 -1
  73. package/dist/core/canvas/pid.js +5 -2
  74. package/dist/core/canvas/types.d.ts +4 -0
  75. package/dist/core/feed/feed.d.ts +4 -0
  76. package/dist/core/feed/feed.js +13 -8
  77. package/dist/core/feed/inbox.d.ts +2 -1
  78. package/dist/core/feed/inbox.js +22 -3
  79. package/dist/core/git.d.ts +3 -0
  80. package/dist/core/git.js +3 -1
  81. package/dist/core/human/feedback-companion.js +1 -0
  82. package/dist/core/preview-registry.js +0 -1
  83. package/dist/core/review/companion.js +1 -0
  84. package/dist/core/review/realize.js +1 -0
  85. package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
  86. package/dist/core/runtime/broker/auth-reload.js +9 -3
  87. package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
  88. package/dist/core/runtime/broker/engine-drive.js +28 -23
  89. package/dist/core/runtime/broker/event-projection.d.ts +4 -0
  90. package/dist/core/runtime/broker/event-projection.js +43 -23
  91. package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
  92. package/dist/core/runtime/broker/fault-retry.js +94 -8
  93. package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
  94. package/dist/core/runtime/broker/frame-dispatch.js +4 -1
  95. package/dist/core/runtime/broker/inbox.js +4 -0
  96. package/dist/core/runtime/broker/rebind.js +18 -0
  97. package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
  98. package/dist/core/runtime/broker/turn-admission.js +31 -0
  99. package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
  100. package/dist/core/runtime/broker/turn-ignition.js +25 -15
  101. package/dist/core/runtime/broker.js +21 -2
  102. package/dist/core/runtime/close.d.ts +11 -9
  103. package/dist/core/runtime/close.js +23 -19
  104. package/dist/core/runtime/fault.d.ts +21 -0
  105. package/dist/core/runtime/fault.js +129 -19
  106. package/dist/core/runtime/fleet.d.ts +0 -6
  107. package/dist/core/runtime/host.d.ts +7 -1
  108. package/dist/core/runtime/host.js +20 -2
  109. package/dist/core/runtime/nodes.d.ts +5 -1
  110. package/dist/core/runtime/nodes.js +24 -1
  111. package/dist/core/runtime/placement.d.ts +6 -6
  112. package/dist/core/runtime/placement.js +15 -8
  113. package/dist/core/runtime/reset.js +2 -2
  114. package/dist/core/runtime/revive.js +40 -4
  115. package/dist/core/runtime/spawn.d.ts +3 -0
  116. package/dist/core/runtime/spawn.js +1 -0
  117. package/dist/core/runtime/warm-pool.d.ts +4 -0
  118. package/dist/core/runtime/warm-pool.js +2 -0
  119. package/dist/core/worktree.d.ts +4 -3
  120. package/dist/core/worktree.js +73 -27
  121. package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
  122. package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
  123. package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
  124. package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
  125. package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
  126. package/dist/daemon/api/handlers/bash-jobs.js +2 -0
  127. package/dist/daemon/api/handlers/broker-ops.js +23 -2
  128. package/dist/daemon/api/handlers/human.js +1 -1
  129. package/dist/daemon/api/handlers/messages.js +11 -2
  130. package/dist/daemon/api/handlers/nodes.js +13 -1
  131. package/dist/daemon/api/handlers/reports.js +14 -6
  132. package/dist/daemon/api/map.js +1 -0
  133. package/dist/daemon/api/server.d.ts +9 -12
  134. package/dist/daemon/api/server.js +30 -19
  135. package/dist/daemon/companion-retire.js +3 -9
  136. package/dist/daemon/cron/sinks.js +7 -2
  137. package/dist/daemon/crtrd.js +4 -1
  138. package/dist/daemon/fleet.d.ts +2 -16
  139. package/dist/daemon/fleet.js +63 -150
  140. package/dist/daemon/human/finish.js +2 -0
  141. package/dist/daemon/manage.d.ts +11 -0
  142. package/dist/daemon/manage.js +32 -3
  143. package/dist/daemon/messaging/node-message.js +7 -1
  144. package/dist/daemon/reconcilers/bash-deadline.js +2 -0
  145. package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
  146. package/dist/daemon/reconcilers/broker-supervision.js +22 -9
  147. package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
  148. package/dist/daemon/reconcilers/live-obligation.js +18 -6
  149. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
  150. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
  151. package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
  152. package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
  153. package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
  154. package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
  155. package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
  156. package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
  157. package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
  158. package/dist/pi-extensions/canvas-context-intro.js +4 -76
  159. package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
  160. package/dist/pi-extensions/canvas-review-boundary.js +4 -29
  161. package/dist/pi-extensions/canvas-stophook.js +1 -6
  162. package/dist/shared/birth-announcement.d.ts +12 -0
  163. package/dist/shared/birth-announcement.js +25 -0
  164. package/dist/shared/generated-context.d.ts +8 -4
  165. package/dist/shared/generated-context.js +15 -9
  166. package/package.json +4 -4
  167. package/runtime.lock.json +2 -2
  168. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
  169. package/dist/pi-extensions/truncate.d.ts +0 -5
  170. package/dist/pi-extensions/truncate.js +0 -14
@@ -3,7 +3,7 @@ import { operationIdContext } from '../../events/operation-id.js';
3
3
  import { promptWithAdvertisedCommandInvocation } from '../advertised-command-invocation.js';
4
4
  import { chooseGuidanceDeliveryMode, promptWithNextTurnGuidance, resolveEngineRoute, resolveSteerCall, } from './engine-routing.js';
5
5
  export function createEngineDriver(deps) {
6
- const { registry, replies, session, dispatches, memoryRefs, authReloadGate, rebind, projection, toolGroups, } = deps;
6
+ const { registry, replies, session, dispatches, memoryRefs, authReloadGate, rebind, projection, toolGroups, turnAdmission, } = deps;
7
7
  const { ackTo, engineErrorTo } = replies;
8
8
  const errorRelay = (client) => {
9
9
  let failure;
@@ -39,7 +39,10 @@ export function createEngineDriver(deps) {
39
39
  // before the caught promise resolves, so reading it after the chain is
40
40
  // the call's verdict.
41
41
  const settle = (p) => p.then(() => errors.failure());
42
- const dispatch = () => {
42
+ const dispatch = () => turnAdmission.admit((release) => {
43
+ // A prior prompt can be in asynchronous preflight while isStreaming is
44
+ // false. Admission waits for that preflight, then routes against the
45
+ // state it actually established instead of its stale snapshot.
43
46
  const route = resolveEngineRoute(frame, session().isStreaming);
44
47
  const guidance = memoryRefs.guidanceFor(frame.text);
45
48
  if (guidance === null) {
@@ -48,9 +51,10 @@ export function createEngineDriver(deps) {
48
51
  // escalated by m-C — so it gets a fresh operation id. A true
49
52
  // (mid-stream) followUp rides the current turn's operation, like
50
53
  // steer.
51
- return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, route.options)).catch(relayError))));
54
+ return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, { ...route.options, preflightResult: release })).catch(relayError))));
52
55
  }
53
56
  // C1: pi's followUp() takes images as a POSITIONAL 2nd arg (`followUp(text, images?)`).
57
+ release();
54
58
  return settle(session().followUp(frame.text, frame.images).catch(relayError));
55
59
  }
56
60
  const deliverAs = chooseGuidanceDeliveryMode(frame.type, route.call, session().isStreaming);
@@ -63,19 +67,20 @@ export function createEngineDriver(deps) {
63
67
  // ATOMICALLY clears its own pending entry if prompt() settles
64
68
  // without ever reaching that drain (an early throw/return leaves
65
69
  // nothing stranded to leak into a later, unrelated turn).
66
- return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, route.options)))).catch(relayError)));
70
+ return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, { ...route.options, preflightResult: release })))).catch(relayError)));
67
71
  }
68
- return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, route.options))
72
+ return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, { ...route.options, preflightResult: release }))
69
73
  .catch(relayError)
70
74
  .then(() => sendGuidance(guidance, deliverAs)))));
71
75
  }
72
76
  // deliverAs is always 'followUp' here per chooseGuidanceDeliveryMode's
73
77
  // contract (a followUp routeCall never yields 'nextTurn').
78
+ release();
74
79
  return settle(session()
75
80
  .followUp(frame.text, frame.images)
76
81
  .catch(relayError)
77
82
  .then(() => sendGuidance(guidance, deliverAs)));
78
- };
83
+ });
79
84
  // /login acknowledges before its asynchronous local refresh completes.
80
85
  // An idle turn must not overtake that refresh and launch on the stale
81
86
  // route; true mid-stream follow-ups keep their immediate queue behavior.
@@ -130,23 +135,23 @@ export function createEngineDriver(deps) {
130
135
  */
131
136
  const driveSteer = (frame, errors) => {
132
137
  const relayError = errors.relay;
133
- if (resolveSteerCall(frame.text, session()) === 'prompt') {
134
- void promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, {
135
- images: frame.images,
136
- streamingBehavior: 'steer',
137
- })).catch(relayError);
138
- return;
139
- }
140
- const guidance = memoryRefs.guidanceFor(frame.text);
141
- // C1: pi's steer() takes images as a POSITIONAL 2nd arg (`steer(text, images?)`).
142
- if (guidance === null) {
143
- void session().steer(frame.text, frame.images).catch(relayError);
144
- return;
145
- }
146
- void session()
147
- .steer(frame.text, frame.images)
148
- .catch(relayError)
149
- .then(() => sendGuidance(guidance, 'steer'));
138
+ void turnAdmission.admit((release) => {
139
+ if (resolveSteerCall(frame.text, session()) === 'prompt') {
140
+ return promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, {
141
+ images: frame.images,
142
+ streamingBehavior: 'steer',
143
+ preflightResult: release,
144
+ }));
145
+ }
146
+ release();
147
+ const guidance = memoryRefs.guidanceFor(frame.text);
148
+ // C1: pi's steer() takes images as a POSITIONAL 2nd arg (`steer(text, images?)`).
149
+ if (guidance === null)
150
+ return session().steer(frame.text, frame.images);
151
+ return session()
152
+ .steer(frame.text, frame.images)
153
+ .then(() => sendGuidance(guidance, 'steer'));
154
+ }).catch(relayError);
150
155
  };
151
156
  /**
152
157
  * esc/abort cancels whatever is live: a running `!` command (abortBash)
@@ -21,6 +21,7 @@ export declare class EventProjection {
21
21
  private pendingUpdateTimer;
22
22
  private workingActivity;
23
23
  private readonly streamWatchdog;
24
+ private readonly toolExecutionWatchdog;
24
25
  private readonly streamWatchdogMs;
25
26
  private pendingBroadcast;
26
27
  private broadcastGeneration;
@@ -30,8 +31,11 @@ export declare class EventProjection {
30
31
  flush(): void;
31
32
  reset(): void;
32
33
  dispose(): void;
34
+ /** Arms the applicable watchdog floor after a streaming event or rebind. */
35
+ rearmIfStreaming(generation: SubscriptionGeneration): void;
33
36
  knownStreamWait(deadlineMs: number): () => void;
34
37
  relayEvent(event: BrokerToClient, generation: SubscriptionGeneration): void;
38
+ private abortStalledStream;
35
39
  private clearPendingUpdate;
36
40
  private flushPendingUpdate;
37
41
  private invalidatePendingBroadcasts;
@@ -6,6 +6,9 @@ import { withoutYieldAbort } from '../session-cycles.js';
6
6
  import { DEFAULT_WORKING_ACTIVITY, randomWorkingActivity } from '../../../shared/working-activity.js';
7
7
  import { hydrateLiveAdvertisedCommandMessage, prepareAdvertisedCommandPersistence, } from '../advertised-command-invocation.js';
8
8
  const MESSAGE_UPDATE_COALESCE_MS = 75;
9
+ // Match the daemon's wedge grace: a tool may legitimately be silent longer than
10
+ // a provider stream, but it cannot leave the broker unguarded indefinitely.
11
+ const TOOL_EXECUTION_WATCHDOG_MS = 20 * 60_000;
9
12
  /** Projects one live engine event stream to broker viewers and owns its relay timers. */
10
13
  export class EventProjection {
11
14
  deps;
@@ -13,25 +16,18 @@ export class EventProjection {
13
16
  pendingUpdateTimer = null;
14
17
  workingActivity = DEFAULT_WORKING_ACTIVITY;
15
18
  streamWatchdog;
19
+ toolExecutionWatchdog;
16
20
  streamWatchdogMs = envStreamWatchdogMs();
17
21
  pendingBroadcast = null;
18
22
  broadcastGeneration = 0;
19
23
  constructor(deps) {
20
24
  this.deps = deps;
21
25
  this.streamWatchdog = new StreamWatchdog((armedGeneration) => {
22
- const generation = this.deps.installedGeneration();
23
- if (generation === undefined || armedGeneration !== generation.id)
24
- return;
25
- emitEvent({
26
- level: 'debug',
27
- event: 'broker.runtime.diagnostic',
28
- fields: { message: `stream watchdog: no engine event for ${this.streamWatchdogMs}ms while streaming — aborting stalled provider stream` },
29
- });
30
- generation.stagedWatchdogAbort = true;
31
- void generation.session.abort().catch((err) => {
32
- emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `stream watchdog abort failed: ${String(err)}` } });
33
- });
26
+ this.abortStalledStream(armedGeneration, this.streamWatchdogMs, 'while streaming — aborting stalled provider stream');
34
27
  }, this.streamWatchdogMs);
28
+ this.toolExecutionWatchdog = new StreamWatchdog((armedGeneration) => {
29
+ this.abortStalledStream(armedGeneration, TOOL_EXECUTION_WATCHDOG_MS, 'while a tool is executing — aborting stalled tool execution');
30
+ }, TOOL_EXECUTION_WATCHDOG_MS);
35
31
  }
36
32
  currentWorkingActivity() {
37
33
  return this.workingActivity;
@@ -46,11 +42,25 @@ export class EventProjection {
46
42
  this.clearPendingUpdate();
47
43
  this.invalidatePendingBroadcasts();
48
44
  this.streamWatchdog.reset();
45
+ this.toolExecutionWatchdog.reset();
49
46
  }
50
47
  dispose() {
51
48
  this.clearPendingUpdate();
52
49
  this.invalidatePendingBroadcasts();
53
50
  this.streamWatchdog.reset();
51
+ this.toolExecutionWatchdog.reset();
52
+ }
53
+ /** Arms the applicable watchdog floor after a streaming event or rebind. */
54
+ rearmIfStreaming(generation) {
55
+ if (!generation.session.isStreaming)
56
+ return;
57
+ if (generation.executingTools.size > 0) {
58
+ this.streamWatchdog.clear();
59
+ this.toolExecutionWatchdog.poke(generation.id);
60
+ return;
61
+ }
62
+ this.toolExecutionWatchdog.clear();
63
+ this.streamWatchdog.poke(generation.id);
54
64
  }
55
65
  knownStreamWait(deadlineMs) {
56
66
  if (!Number.isFinite(deadlineMs) || deadlineMs <= Date.now())
@@ -99,18 +109,14 @@ export class EventProjection {
99
109
  if (type === 'agent_end') {
100
110
  generation.executingTools.clear();
101
111
  this.streamWatchdog.reset();
112
+ this.toolExecutionWatchdog.reset();
102
113
  }
103
- else if (type === 'tool_execution_start') {
104
- generation.executingTools.add(event.toolCallId ?? '');
105
- this.streamWatchdog.clear();
106
- }
107
- else if (type === 'tool_execution_end') {
108
- generation.executingTools.delete(event.toolCallId ?? '');
109
- if (generation.executingTools.size === 0 && generation.session.isStreaming)
110
- this.streamWatchdog.poke(generation.id);
111
- }
112
- else if (generation.session.isStreaming && generation.executingTools.size === 0) {
113
- this.streamWatchdog.poke(generation.id);
114
+ else {
115
+ if (type === 'tool_execution_start')
116
+ generation.executingTools.add(event.toolCallId ?? '');
117
+ else if (type === 'tool_execution_end')
118
+ generation.executingTools.delete(event.toolCallId ?? '');
119
+ this.rearmIfStreaming(generation);
114
120
  }
115
121
  if (type === 'message_update') {
116
122
  this.pendingUpdate = event;
@@ -128,6 +134,20 @@ export class EventProjection {
128
134
  }
129
135
  this.broadcast(event);
130
136
  }
137
+ abortStalledStream(armedGeneration, timeoutMs, context) {
138
+ const generation = this.deps.installedGeneration();
139
+ if (generation === undefined || armedGeneration !== generation.id)
140
+ return;
141
+ emitEvent({
142
+ level: 'debug',
143
+ event: 'broker.runtime.diagnostic',
144
+ fields: { message: `stream watchdog: no engine event for ${timeoutMs}ms ${context}` },
145
+ });
146
+ generation.stagedWatchdogAbort = true;
147
+ void generation.session.abort().catch((err) => {
148
+ emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `stream watchdog abort failed: ${String(err)}` } });
149
+ });
150
+ }
131
151
  clearPendingUpdate() {
132
152
  if (this.pendingUpdateTimer !== null) {
133
153
  clearTimeout(this.pendingUpdateTimer);
@@ -4,6 +4,7 @@ import type { BrokerSdkConfig } from '../launch.js';
4
4
  import type { BrokerToClient } from '../broker-protocol.js';
5
5
  import type { BrokerSession } from './read-ops.js';
6
6
  import type { SubscriptionGeneration } from './rebind.js';
7
+ import type { TurnAdmissionGate } from './turn-admission.js';
7
8
  type FaultRetryDeps = {
8
9
  nodeId: string;
9
10
  cfg: BrokerSdkConfig;
@@ -14,6 +15,7 @@ type FaultRetryDeps = {
14
15
  registryOf: (services: AgentSessionServices) => ModelRegistry;
15
16
  formatModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
16
17
  broadcastModelChanged: (persist?: boolean) => Promise<void>;
18
+ turnAdmission: TurnAdmissionGate;
17
19
  };
18
20
  /** Owns provider fallback and daemon-driven fault retry state for one broker session. */
19
21
  export declare class FaultRetry {
@@ -29,9 +31,16 @@ export declare class FaultRetry {
29
31
  onEvent(event: BrokerToClient): void;
30
32
  isActiveAutoFault(fault: ReturnType<typeof readFault>): fault is NonNullable<ReturnType<typeof readFault>>;
31
33
  publishSettledProviderFault(generation: SubscriptionGeneration): void;
34
+ /** Startup re-drive reads only a durable pending episode. Revive deliberately
35
+ * clears the ordinary fault marker before this boundary, so the episode is
36
+ * the recovery authority and admitted/invalidated records remain inert. */
37
+ recoverPendingOnStartup(): void;
32
38
  schedule(): void;
33
39
  private maybeSwitchProviderForRetry;
34
40
  private maybeFallbackOnUnusableModel;
41
+ private recordPendingProviderFault;
42
+ private sessionFile;
43
+ private pendingEpisodeMatches;
35
44
  private faultRetryDelayMs;
36
45
  private clearAndArmFaultRetry;
37
46
  private runFaultRetry;
@@ -1,7 +1,8 @@
1
+ import { realpathSync } from 'node:fs';
1
2
  import { envModelIntent, envNodeCwd, envProfileId } from '../../../shared/env.js';
2
3
  import { isContextOverflow } from '@earendil-works/pi-ai';
3
4
  import { classify } from '../../fault-classifier.js';
4
- import { clearFault, isModelNotFoundError, isProviderUnconfiguredError, readFault, recordFault } from '../fault.js';
5
+ import { admitProviderRetryEpisode, clearFault, clearProviderRetryEpisode, invalidateProviderRetryEpisode, isModelNotFoundError, isProviderUnconfiguredError, readFault, readProviderRetryEpisode, recordFault, recordPendingProviderRetryFault, } from '../fault.js';
5
6
  import { extractCoolingDeadline } from '../managed-provider-cooling.js';
6
7
  import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from '../../model-routes.js';
7
8
  import { operationIdContext } from '../../events/operation-id.js';
@@ -77,7 +78,7 @@ export class FaultRetry {
77
78
  return;
78
79
  }
79
80
  if (watchdogAbort && last?.stopReason === 'aborted') {
80
- recordFault(this.deps.nodeId, {
81
+ this.recordPendingProviderFault(settledSession, {
81
82
  link: 'pi→provider', op: 'generation turn', kind: 'connection',
82
83
  retry: { disposition: 'auto', by: 'daemon', ...(episode?.attempt !== undefined ? { attempt: episode.attempt } : {}) },
83
84
  message: typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)',
@@ -101,7 +102,7 @@ export class FaultRetry {
101
102
  const coolingDeadline = extractCoolingDeadline(last);
102
103
  if (uncompactedOverflow)
103
104
  clearFault(this.deps.nodeId, { link: 'pi→provider' });
104
- recordFault(this.deps.nodeId, {
105
+ const faultInput = {
105
106
  link: 'pi→provider', op: 'generation turn', kind: uncompactedOverflow ? 'context-overflow' : classified.kind,
106
107
  retry: uncompactedOverflow
107
108
  ? { disposition: 'fatal' }
@@ -117,7 +118,35 @@ export class FaultRetry {
117
118
  anchorEntryId: !uncompactedOverflow && classified.disposition === 'auto' && episode?.anchorEntryId !== undefined
118
119
  ? episode.anchorEntryId
119
120
  : settledSession.sessionManager.getLeafId?.() ?? undefined,
120
- });
121
+ };
122
+ if (faultInput.retry.disposition === 'auto')
123
+ this.recordPendingProviderFault(settledSession, faultInput);
124
+ else
125
+ recordFault(this.deps.nodeId, faultInput);
126
+ }
127
+ /** Startup re-drive reads only a durable pending episode. Revive deliberately
128
+ * clears the ordinary fault marker before this boundary, so the episode is
129
+ * the recovery authority and admitted/invalidated records remain inert. */
130
+ recoverPendingOnStartup() {
131
+ const generation = this.deps.installedGeneration();
132
+ if (generation === undefined)
133
+ return;
134
+ const episode = readProviderRetryEpisode(this.deps.nodeId);
135
+ const sessionFile = this.sessionFile(generation.session);
136
+ // A new-cycle launch deliberately roots a new transcript branch even though
137
+ // it reuses the same .jsonl. Its old anchor is no longer a valid recovery
138
+ // coordinate, so mark the episode invalid rather than inferring that from
139
+ // the shared path or letting startup re-drive into the fresh cycle.
140
+ if (this.deps.cfg.newCycle === true) {
141
+ if (episode !== null && sessionFile !== null && episode.sessionFile === sessionFile) {
142
+ invalidateProviderRetryEpisode(this.deps.nodeId, sessionFile);
143
+ }
144
+ return;
145
+ }
146
+ if (episode === null || sessionFile === null || episode.state !== 'pending' || episode.sessionFile !== sessionFile)
147
+ return;
148
+ recordFault(this.deps.nodeId, episode.fault);
149
+ this.schedule();
121
150
  }
122
151
  schedule() {
123
152
  const generation = this.deps.installedGeneration();
@@ -125,7 +154,11 @@ export class FaultRetry {
125
154
  if (generation === undefined || session.isStreaming || this.timer !== null)
126
155
  return;
127
156
  const fault = readFault(this.deps.nodeId);
128
- if (!this.isActiveAutoFault(fault)) {
157
+ // A live session without a durable transcript path still owns its ordinary
158
+ // in-process timer; it simply cannot survive broker death. Once a path
159
+ // exists, require its pending episode so admitted/invalidated retries stay
160
+ // inert even before a replacement broker starts.
161
+ if (!this.isActiveAutoFault(fault) || (this.sessionFile(session) !== null && !this.pendingEpisodeMatches(session, fault))) {
129
162
  this.clearTimer();
130
163
  return;
131
164
  }
@@ -215,7 +248,7 @@ export class FaultRetry {
215
248
  return;
216
249
  if (unconfigured)
217
250
  clearFault(this.deps.nodeId, { link: 'pi→provider' });
218
- await operationIdContext.fresh(() => candidateSession.prompt(formatCard('recovery', {}, formatModelFallbackRecovery(currentSpec, resolved, unconfigured ? 'credential' : 'not-found'))));
251
+ await this.deps.turnAdmission.admit((release) => operationIdContext.fresh(() => candidateSession.prompt(formatCard('recovery', {}, formatModelFallbackRecovery(currentSpec, resolved, unconfigured ? 'credential' : 'not-found')), { preflightResult: release })));
219
252
  })().catch((err) => {
220
253
  emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `${unconfigured ? 'unconfigured-provider' : 'not_found'} fallback failed: ${err instanceof Error ? err.message : String(err)}` } });
221
254
  }).finally(() => {
@@ -224,6 +257,36 @@ export class FaultRetry {
224
257
  });
225
258
  this.notFoundFallbackInFlight = fallbackPromise;
226
259
  }
260
+ recordPendingProviderFault(session, input) {
261
+ const sessionFile = this.sessionFile(session);
262
+ // A pathless session cannot be crash-recovered, but its live broker still
263
+ // owns the ordinary auto-retry timer.
264
+ if (sessionFile === null) {
265
+ recordFault(this.deps.nodeId, input);
266
+ return;
267
+ }
268
+ recordPendingProviderRetryFault(this.deps.nodeId, sessionFile, input);
269
+ }
270
+ sessionFile(session) {
271
+ if (typeof session.sessionFile !== 'string' || session.sessionFile === '')
272
+ return null;
273
+ try {
274
+ return realpathSync(session.sessionFile);
275
+ }
276
+ catch {
277
+ return session.sessionFile;
278
+ }
279
+ }
280
+ pendingEpisodeMatches(session, fault) {
281
+ const sessionFile = this.sessionFile(session);
282
+ const episode = readProviderRetryEpisode(this.deps.nodeId);
283
+ return sessionFile !== null &&
284
+ episode !== null &&
285
+ episode.state === 'pending' &&
286
+ episode.sessionFile === sessionFile &&
287
+ episode.fault.since === fault.since &&
288
+ episode.anchorEntryId === (fault.anchorEntryId ?? null);
289
+ }
227
290
  faultRetryDelayMs(fault, now) {
228
291
  if (fault.retry.nextAt !== undefined) {
229
292
  const ms = Date.parse(fault.retry.nextAt) - now;
@@ -280,11 +343,13 @@ export class FaultRetry {
280
343
  const policy = policyFor(fault.kind);
281
344
  if (attempt >= policy.max) {
282
345
  recordFault(this.deps.nodeId, markFatalOnExhaust(fault));
346
+ clearProviderRetryEpisode(this.deps.nodeId);
283
347
  return;
284
348
  }
285
349
  const plan = nextFaultRetry(fault, Date.now());
286
350
  if (plan === null) {
287
351
  recordFault(this.deps.nodeId, markFatalOnExhaust(fault));
352
+ clearProviderRetryEpisode(this.deps.nodeId);
288
353
  return;
289
354
  }
290
355
  if (fault.anchorEntryId !== undefined) {
@@ -301,7 +366,28 @@ export class FaultRetry {
301
366
  }
302
367
  if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
303
368
  return;
304
- recordFault(this.deps.nodeId, plan.fault);
305
- await operationIdContext.fresh(() => candidateSession.prompt(formatCard('recovery', {}, fault.kind === 'connection' ? CONNECTION_FAULT_RECOVERY_BODY : PROVIDER_FAULT_RECOVERY_BODY)));
369
+ await this.deps.turnAdmission.admit((release) => {
370
+ const current = readFault(this.deps.nodeId);
371
+ if (this.deps.installedGeneration() !== generation ||
372
+ this.deps.currentSession() !== candidateSession ||
373
+ candidateSession.isStreaming ||
374
+ !this.isActiveAutoFault(current) ||
375
+ current.since !== fault.since) {
376
+ release();
377
+ return Promise.resolve();
378
+ }
379
+ // This is the admission point: all generation/session/streaming/fault
380
+ // revalidation above has passed, and only now may the durable episode
381
+ // become admitted. Writing it before prompt() fails closed across a broker
382
+ // death in the narrow handoff to the SDK — replaying there could duplicate
383
+ // an already accepted recovery turn.
384
+ const sessionFile = this.sessionFile(candidateSession);
385
+ if (sessionFile !== null && !admitProviderRetryEpisode(this.deps.nodeId, sessionFile, plan.fault)) {
386
+ release();
387
+ return Promise.resolve();
388
+ }
389
+ recordFault(this.deps.nodeId, plan.fault);
390
+ return operationIdContext.fresh(() => candidateSession.prompt(formatCard('recovery', {}, fault.kind === 'connection' ? CONNECTION_FAULT_RECOVERY_BODY : PROVIDER_FAULT_RECOVERY_BODY), { preflightResult: release }));
391
+ });
306
392
  }
307
393
  }
@@ -5,6 +5,7 @@ import type { BrokerClient, BrokerClientRegistry } from './client-registry.js';
5
5
  import type { BrokerSession } from './read-ops.js';
6
6
  import { type DriveFrame } from './engine-drive.js';
7
7
  import { type PendingDialog } from './frame-client.js';
8
+ import type { TurnAdmissionGate } from './turn-admission.js';
8
9
  import type { ToolGroupTracker } from './tool-groups.js';
9
10
  import type { EventProjection } from './event-projection.js';
10
11
  import type { RebindDriver } from './rebind.js';
@@ -28,6 +29,7 @@ export type FrameDispatchDeps = {
28
29
  registryOf: (services: AgentSessionServices) => ModelRegistry;
29
30
  formatModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
30
31
  formatExactModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
32
+ turnAdmission: TurnAdmissionGate;
31
33
  resolveLaunchModel: (services: AgentSessionServices, cfg: BrokerSdkConfig) => Promise<{
32
34
  model: NonNullable<ReturnType<ModelRegistry['find']>>;
33
35
  thinkingLevel: string | undefined;
@@ -9,7 +9,7 @@ import { createSessionHandler } from './frame-session.js';
9
9
  import { createKickoffDriver, createPromptDispatchTracker, createTurnAcceptanceGate, } from './turn-ignition.js';
10
10
  /** Owns frame-local input state while dispatching through explicit broker collaborators. */
11
11
  export function createFrameDispatchContext(deps) {
12
- const { nodeId, cfg, registry, toolGroups, projection, rebind, boundaryReviewId, pendingDialogs, sendWelcome, replayExtraPendingDialogsTo, reWelcomeAll, disposeAndExit, persistModelChoice, broadcastModelChanged, registryOf, formatModelSpec, formatExactModelSpec, resolveLaunchModel, } = deps;
12
+ const { nodeId, cfg, registry, toolGroups, projection, rebind, boundaryReviewId, pendingDialogs, sendWelcome, replayExtraPendingDialogsTo, reWelcomeAll, disposeAndExit, persistModelChoice, broadcastModelChanged, registryOf, formatModelSpec, formatExactModelSpec, turnAdmission, resolveLaunchModel, } = deps;
13
13
  const currentSession = () => rebind.session();
14
14
  const currentServices = () => rebind.services();
15
15
  // The reply vocabulary every handler below answers a client with.
@@ -21,6 +21,7 @@ export function createFrameDispatchContext(deps) {
21
21
  const kickoffDriver = createKickoffDriver({
22
22
  session: currentSession,
23
23
  dispatches,
24
+ turnAdmission,
24
25
  });
25
26
  const authReloadGate = createAuthReloadGate({
26
27
  nodeId,
@@ -33,6 +34,7 @@ export function createFrameDispatchContext(deps) {
33
34
  formatModelSpec,
34
35
  resolveLaunchModel,
35
36
  driveKickoff: kickoffDriver.drivePending,
37
+ turnAdmission,
36
38
  });
37
39
  const driveEngine = createEngineDriver({
38
40
  registry,
@@ -44,6 +46,7 @@ export function createFrameDispatchContext(deps) {
44
46
  rebind,
45
47
  projection,
46
48
  toolGroups,
49
+ turnAdmission,
47
50
  });
48
51
  const handleDriveFrame = (client, frame) => {
49
52
  if (notWritable(client, 'drive the engine'))
@@ -3,6 +3,7 @@
3
3
  import { existsSync, readFileSync } from 'node:fs';
4
4
  import { inboxPath, nodeDir } from '../../canvas/paths.js';
5
5
  import { formatInboxCard } from '../../../shared/generated-context.js';
6
+ import { renderBirthAnnouncementBody } from '../../../shared/birth-announcement.js';
6
7
  const ENTRY_ID = /^(?!0{32}$)[0-9a-f]{32}$/;
7
8
  const BODY_MAX_LINES = 12;
8
9
  const BODY_MAX_CHARS = 1000;
@@ -140,6 +141,9 @@ function inlineReport(entry, reportNodes) {
140
141
  /** One entry's card body: an inlined short report, a bounded inline message
141
142
  * body, or the label when the entry carries neither. */
142
143
  function entryBody(entry, reportNodes) {
144
+ const birth = renderBirthAnnouncementBody(entry.data, entry.ts);
145
+ if (birth !== null)
146
+ return birth;
143
147
  const report = inlineReport(entry, reportNodes);
144
148
  if (report !== null)
145
149
  return report.trimEnd();
@@ -5,6 +5,7 @@ import { emitEvent } from '../../events/emit.js';
5
5
  import { operationIdContext } from '../../events/operation-id.js';
6
6
  import { installPersonaTransitionGate } from '../broker-persona-guidance.js';
7
7
  import { abortSessionFromExtension } from './extension-abort.js';
8
+ import { clearFault, invalidateProviderRetryEpisode, readFault } from '../fault.js';
8
9
  const MAX_ADMITTED_REPLACEMENTS = 2;
9
10
  const REPLACEMENT_DEADLINE_MS = 30_000;
10
11
  const REPLACEMENT_DEADLINE_EXCEEDED = Symbol('replacement-deadline-exceeded');
@@ -95,6 +96,22 @@ export class RebindDriver {
95
96
  const task = this.rebindTail.then(async () => {
96
97
  if (ticket !== this.latestRebindRequest)
97
98
  return;
99
+ // A replacement changes the conversation coordinate. The initial bind has
100
+ // no prior generation and must retain a crash-surviving pending episode;
101
+ // every later rebind explicitly marks the old session's episode invalid.
102
+ if (this.installed !== undefined) {
103
+ if (typeof this.sessionValue.sessionFile === 'string' && this.sessionValue.sessionFile !== '') {
104
+ invalidateProviderRetryEpisode(this.deps.nodeId, this.sessionValue.sessionFile);
105
+ }
106
+ // A daemon-owned auto marker belongs to the outgoing session whether
107
+ // that session has a durable path or not. Retaining it lets the
108
+ // replacement inherit an old anchor, attempts, and backoff; fatal and
109
+ // manual markers stay visible.
110
+ const fault = readFault(this.deps.nodeId);
111
+ if (fault?.link === 'pi→provider' && fault.retry.disposition === 'auto' && fault.retry.by === 'daemon') {
112
+ clearFault(this.deps.nodeId, { link: 'pi→provider', preserveProviderRetryEpisode: true });
113
+ }
114
+ }
98
115
  this.installed?.executingTools.clear();
99
116
  this.removeInstalledSubscription();
100
117
  this.deps.setActiveSession(candidateSession);
@@ -192,6 +209,7 @@ export class RebindDriver {
192
209
  this.executeSubscriptionCallback(generation, event);
193
210
  });
194
211
  this.installed = generation;
212
+ this.deps.projection().rearmIfStreaming(generation);
195
213
  this.captureSystemPrompt(candidateSession);
196
214
  });
197
215
  this.rebindTail = task.catch(() => undefined);
@@ -0,0 +1,11 @@
1
+ /**
2
+ * Serializes entry into AgentSession.prompt() until its preflight has decided
3
+ * whether it will start (or join) a run. AgentSession.isStreaming stays false
4
+ * through asynchronous preflight, so checking it before entering prompt() lets
5
+ * two calls reach the SDK's active-run ownership race. Each waiter therefore
6
+ * re-reads live state only after the preceding prompt has reported preflight.
7
+ */
8
+ export type TurnAdmissionGate = {
9
+ admit<T>(start: (release: () => void) => Promise<T>): Promise<T>;
10
+ };
11
+ export declare function createTurnAdmissionGate(): TurnAdmissionGate;
@@ -0,0 +1,31 @@
1
+ export function createTurnAdmissionGate() {
2
+ let tail = Promise.resolve();
3
+ return {
4
+ admit(start) {
5
+ const previous = tail;
6
+ let resolveAdmission;
7
+ const admitted = new Promise((resolve) => {
8
+ resolveAdmission = resolve;
9
+ });
10
+ tail = previous.then(() => admitted);
11
+ return previous.then(() => {
12
+ let released = false;
13
+ const release = () => {
14
+ if (released)
15
+ return;
16
+ released = true;
17
+ resolveAdmission();
18
+ };
19
+ try {
20
+ const attempt = start(release);
21
+ void attempt.then(release, release);
22
+ return attempt;
23
+ }
24
+ catch (error) {
25
+ release();
26
+ throw error;
27
+ }
28
+ });
29
+ },
30
+ };
31
+ }
@@ -1,4 +1,5 @@
1
1
  import type { BrokerSession } from './read-ops.js';
2
+ import type { TurnAdmissionGate } from './turn-admission.js';
2
3
  /**
3
4
  * Prompt dispatches whose promise has not yet settled. `session.prompt()` does
4
5
  * async setup (input transforms, model/auth validation, compaction) BEFORE
@@ -39,4 +40,5 @@ export type KickoffDriver = {
39
40
  export declare function createKickoffDriver(deps: {
40
41
  session: () => BrokerSession;
41
42
  dispatches: PromptDispatchTracker;
43
+ turnAdmission: TurnAdmissionGate;
42
44
  }): KickoffDriver;
@@ -41,28 +41,38 @@ export function createTurnAcceptanceGate() {
41
41
  };
42
42
  }
43
43
  export function createKickoffDriver(deps) {
44
- const { session, dispatches } = deps;
44
+ const { session, dispatches, turnAdmission } = deps;
45
45
  let pendingPrompt;
46
46
  let inFlight = false;
47
47
  const sessionHasNoModel = () => session().model === undefined || isUnknownModel(session().model);
48
48
  const drivePending = () => {
49
49
  if (pendingPrompt === undefined || inFlight || sessionHasNoModel())
50
50
  return;
51
- const prompt = pendingPrompt;
52
- pendingPrompt = undefined;
53
51
  inFlight = true;
54
- dispatches.track(operationIdContext.fresh(() => session()
55
- .prompt(prompt)
56
- .catch((err) => {
57
- emitEvent({
58
- level: 'error',
59
- event: 'broker.prompt.first_failed',
60
- error: err,
61
- });
62
- })
63
- .finally(() => {
64
- inFlight = false;
65
- })));
52
+ void turnAdmission.admit((release) => {
53
+ const prompt = pendingPrompt;
54
+ if (prompt === undefined || sessionHasNoModel()) {
55
+ release();
56
+ inFlight = false;
57
+ return Promise.resolve();
58
+ }
59
+ pendingPrompt = undefined;
60
+ return dispatches.track(operationIdContext.fresh(() => session()
61
+ .prompt(prompt, {
62
+ preflightResult: release,
63
+ ...(session().isStreaming ? { streamingBehavior: 'steer' } : {}),
64
+ })
65
+ .catch((err) => {
66
+ emitEvent({
67
+ level: 'error',
68
+ event: 'broker.prompt.first_failed',
69
+ error: err,
70
+ });
71
+ })
72
+ .finally(() => {
73
+ inFlight = false;
74
+ })));
75
+ });
66
76
  };
67
77
  return {
68
78
  kickoff(prompt) {