@north-light/crouter 0.3.248 → 0.3.250

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/README.md +1 -1
  2. package/dist/api/__tests__/integration/client.test.js +28 -7
  3. package/dist/api/client.d.ts +1 -1
  4. package/dist/api/client.js +2 -2
  5. package/dist/api/dto/lifecycle.d.ts +3 -0
  6. package/dist/api/dto/messages.d.ts +4 -0
  7. package/dist/api/dto/nodes.d.ts +4 -0
  8. package/dist/api/dto/reports.d.ts +7 -2
  9. package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
  10. package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
  11. package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
  12. package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
  13. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
  14. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
  15. package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
  16. package/dist/clients/attach/session/chrome-refresh.js +1 -0
  17. package/dist/clients/attach/viewer.js +532 -532
  18. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
  19. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
  20. package/dist/clients/inbox/controller.js +32 -5
  21. package/dist/clients/inbox/tui/ansi.d.ts +3 -1
  22. package/dist/clients/inbox/tui/ansi.js +14 -10
  23. package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
  24. package/dist/commands/api-client.d.ts +2 -2
  25. package/dist/commands/api-client.js +5 -5
  26. package/dist/commands/canvas-browse.js +1 -1
  27. package/dist/commands/node/create.js +3 -1
  28. package/dist/commands/node/inspect.js +1 -0
  29. package/dist/commands/node/lifecycle.js +1 -1
  30. package/dist/commands/node/message.js +6 -4
  31. package/dist/commands/node/wait.js +1 -1
  32. package/dist/commands/push.js +36 -37
  33. package/dist/commands/sys/daemon.js +21 -8
  34. package/dist/commands/sys/setup-core.js +2 -2
  35. package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
  36. package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
  37. package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
  38. package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
  39. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +5 -1
  40. package/dist/core/__tests__/child-death-wake.test.js +11 -7
  41. package/dist/core/__tests__/collectible.test.d.ts +1 -0
  42. package/dist/core/__tests__/collectible.test.js +88 -0
  43. package/dist/core/__tests__/context-intro.test.js +4 -41
  44. package/dist/core/__tests__/daemon-boot.test.js +30 -14
  45. package/dist/core/__tests__/daemon-wedge.test.js +25 -19
  46. package/dist/core/__tests__/fixtures/fake-engine.js +11 -0
  47. package/dist/core/__tests__/human-deliver.test.js +5 -5
  48. package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
  49. package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
  50. package/dist/core/__tests__/integration/revive.test.js +48 -0
  51. package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
  52. package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
  53. package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
  54. package/dist/core/__tests__/migration.test.js +30 -0
  55. package/dist/core/__tests__/passive-subscription.test.js +39 -1
  56. package/dist/core/__tests__/prune.test.js +12 -3
  57. package/dist/core/__tests__/relaunch-root.test.js +1 -1
  58. package/dist/core/__tests__/respawn-throttle.test.js +16 -146
  59. package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
  60. package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
  61. package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
  62. package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
  63. package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
  64. package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
  65. package/dist/core/canvas/canvas.d.ts +9 -15
  66. package/dist/core/canvas/canvas.js +41 -47
  67. package/dist/core/canvas/crons.d.ts +5 -8
  68. package/dist/core/canvas/crons.js +5 -8
  69. package/dist/core/canvas/history.d.ts +5 -0
  70. package/dist/core/canvas/history.js +12 -1
  71. package/dist/core/canvas/migrations.js +58 -1
  72. package/dist/core/canvas/pid.d.ts +1 -1
  73. package/dist/core/canvas/pid.js +5 -2
  74. package/dist/core/canvas/types.d.ts +4 -0
  75. package/dist/core/feed/feed.d.ts +4 -0
  76. package/dist/core/feed/feed.js +13 -8
  77. package/dist/core/feed/inbox.d.ts +2 -1
  78. package/dist/core/feed/inbox.js +22 -3
  79. package/dist/core/git.d.ts +3 -0
  80. package/dist/core/git.js +3 -1
  81. package/dist/core/human/feedback-companion.js +1 -0
  82. package/dist/core/preview-registry.js +0 -1
  83. package/dist/core/review/companion.js +1 -0
  84. package/dist/core/review/realize.js +1 -0
  85. package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
  86. package/dist/core/runtime/broker/auth-reload.js +9 -3
  87. package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
  88. package/dist/core/runtime/broker/engine-drive.js +28 -23
  89. package/dist/core/runtime/broker/event-projection.d.ts +4 -0
  90. package/dist/core/runtime/broker/event-projection.js +43 -23
  91. package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
  92. package/dist/core/runtime/broker/fault-retry.js +94 -8
  93. package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
  94. package/dist/core/runtime/broker/frame-dispatch.js +4 -1
  95. package/dist/core/runtime/broker/inbox.js +4 -0
  96. package/dist/core/runtime/broker/rebind.js +18 -0
  97. package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
  98. package/dist/core/runtime/broker/turn-admission.js +31 -0
  99. package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
  100. package/dist/core/runtime/broker/turn-ignition.js +25 -15
  101. package/dist/core/runtime/broker.js +21 -2
  102. package/dist/core/runtime/close.d.ts +11 -9
  103. package/dist/core/runtime/close.js +23 -19
  104. package/dist/core/runtime/fault.d.ts +21 -0
  105. package/dist/core/runtime/fault.js +129 -19
  106. package/dist/core/runtime/fleet.d.ts +0 -6
  107. package/dist/core/runtime/host.d.ts +7 -1
  108. package/dist/core/runtime/host.js +20 -2
  109. package/dist/core/runtime/nodes.d.ts +5 -1
  110. package/dist/core/runtime/nodes.js +24 -1
  111. package/dist/core/runtime/placement.d.ts +6 -6
  112. package/dist/core/runtime/placement.js +15 -8
  113. package/dist/core/runtime/reset.js +2 -2
  114. package/dist/core/runtime/revive.js +40 -4
  115. package/dist/core/runtime/spawn.d.ts +3 -0
  116. package/dist/core/runtime/spawn.js +1 -0
  117. package/dist/core/runtime/warm-pool.d.ts +4 -0
  118. package/dist/core/runtime/warm-pool.js +2 -0
  119. package/dist/core/worktree.d.ts +4 -3
  120. package/dist/core/worktree.js +73 -27
  121. package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
  122. package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
  123. package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
  124. package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
  125. package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
  126. package/dist/daemon/api/handlers/bash-jobs.js +2 -0
  127. package/dist/daemon/api/handlers/broker-ops.js +23 -2
  128. package/dist/daemon/api/handlers/human.js +1 -1
  129. package/dist/daemon/api/handlers/messages.js +11 -2
  130. package/dist/daemon/api/handlers/nodes.js +13 -1
  131. package/dist/daemon/api/handlers/reports.js +14 -6
  132. package/dist/daemon/api/map.js +1 -0
  133. package/dist/daemon/api/server.d.ts +9 -12
  134. package/dist/daemon/api/server.js +30 -19
  135. package/dist/daemon/companion-retire.js +3 -9
  136. package/dist/daemon/cron/sinks.js +7 -2
  137. package/dist/daemon/crtrd.js +4 -1
  138. package/dist/daemon/fleet.d.ts +2 -16
  139. package/dist/daemon/fleet.js +63 -150
  140. package/dist/daemon/human/finish.js +2 -0
  141. package/dist/daemon/manage.d.ts +11 -0
  142. package/dist/daemon/manage.js +32 -3
  143. package/dist/daemon/messaging/node-message.js +7 -1
  144. package/dist/daemon/reconcilers/bash-deadline.js +2 -0
  145. package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
  146. package/dist/daemon/reconcilers/broker-supervision.js +22 -9
  147. package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
  148. package/dist/daemon/reconcilers/live-obligation.js +18 -6
  149. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
  150. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
  151. package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
  152. package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
  153. package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
  154. package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
  155. package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
  156. package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
  157. package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
  158. package/dist/pi-extensions/canvas-context-intro.js +4 -76
  159. package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
  160. package/dist/pi-extensions/canvas-review-boundary.js +4 -29
  161. package/dist/pi-extensions/canvas-stophook.js +1 -6
  162. package/dist/shared/birth-announcement.d.ts +12 -0
  163. package/dist/shared/birth-announcement.js +25 -0
  164. package/dist/shared/generated-context.d.ts +8 -4
  165. package/dist/shared/generated-context.js +15 -9
  166. package/package.json +4 -4
  167. package/runtime.lock.json +2 -2
  168. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
  169. package/dist/clients/inbox/tui/clipboard.d.ts +0 -2
  170. package/dist/clients/inbox/tui/clipboard.js +0 -26
  171. package/dist/pi-extensions/truncate.d.ts +0 -5
  172. package/dist/pi-extensions/truncate.js +0 -14
@@ -35,7 +35,7 @@ import { resolveBundledPiPackageDir } from './pi-cli.js';
35
35
  import { captureSessionChatCapabilities } from './command-surface.js';
36
36
  import { PoolCredentialStore } from '../pool-credential-store.js';
37
37
  import { clearCleanAbort, markBusy, markCleanAbort } from './busy.js';
38
- import { recordFaultForCurrentNode } from './fault.js';
38
+ import { clearFault, recordFaultForCurrentNode } from './fault.js';
39
39
  import { modelRequestFromConfig, selectInitialModelRoute, unregisteredConcreteModel } from '../model-routes.js';
40
40
  import { KNOWN_STREAM_WAIT_UI } from './stream-watchdog.js';
41
41
  import { probeOnline as probeOnlineReal } from './connectivity.js';
@@ -46,6 +46,7 @@ import { BrokerClientRegistry } from './broker/client-registry.js';
46
46
  import { ToolGroupTracker } from './broker/tool-groups.js';
47
47
  import { onNodeNamed } from './broker/node-named.js';
48
48
  import { FaultRetry } from './broker/fault-retry.js';
49
+ import { createTurnAdmissionGate } from './broker/turn-admission.js';
49
50
  import { EventProjection } from './broker/event-projection.js';
50
51
  import { createFrameDispatchContext, handleFrame, } from './broker/frame-dispatch.js';
51
52
  import { hydratePersistedAdvertisedCommandMessages, installAdvertisedCommandInvocationContract, } from './advertised-command-invocation.js';
@@ -325,6 +326,7 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
325
326
  const spec = persist ? await persistModelChoice(pinnedOverride, userSelected) : liveSpec;
326
327
  registry.broadcast({ type: 'model_changed', model: isUnknownModel(liveSession.model) ? undefined : liveSession.model, spec });
327
328
  };
329
+ const turnAdmission = createTurnAdmissionGate();
328
330
  faultRetry = new FaultRetry({
329
331
  nodeId,
330
332
  cfg,
@@ -335,6 +337,7 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
335
337
  registryOf,
336
338
  formatModelSpec,
337
339
  broadcastModelChanged,
340
+ turnAdmission,
338
341
  });
339
342
  projection = new EventProjection({
340
343
  registry,
@@ -527,7 +530,7 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
527
530
  nodeId, cfg, registry, toolGroups, projection, rebind, boundaryReviewId,
528
531
  pendingDialogs, sendWelcome, replayExtraPendingDialogsTo, reWelcomeAll, disposeAndExit,
529
532
  persistModelChoice, broadcastModelChanged, registryOf, formatModelSpec,
530
- formatExactModelSpec, resolveLaunchModel,
533
+ formatExactModelSpec, turnAdmission, resolveLaunchModel,
531
534
  });
532
535
  notifyTurnAccepted = frameDispatch.notifyTurnAccepted;
533
536
  resetFrameState = frameDispatch.resetForSession;
@@ -591,6 +594,22 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
591
594
  disposeAndExit('socket-error', 1);
592
595
  });
593
596
  server.listen(sockPath, () => {
597
+ // Binding view.sock is the exact inverse of a `connect ENOENT view.sock`
598
+ // fault, so the moment the listener accepts, any socket-absence marker a
599
+ // viewer/relay recorded during the revive→bind gap is provably stale. The
600
+ // broker owns this socket, so it clears those markers here rather than
601
+ // waiting for a client to reconnect — a headless node with nobody attached
602
+ // never gets that reconnect, which is how the stale ⚠ leaked. A genuine
603
+ // post-bind failure is re-recorded by the next failed client op.
604
+ clearFault(nodeId, { link: 'viewer↔broker' });
605
+ clearFault(nodeId, { link: 'relay↔broker' });
606
+ // The listener is also the broker startup boundary: a pending provider
607
+ // episode recovered after an instance death is re-armed only once this
608
+ // replacement can accept its normal viewer/daemon traffic. Independent of
609
+ // the link faults cleared above — those are viewer/relay socket markers,
610
+ // this is the provider retry episode. Admitted and invalidated records are
611
+ // intentionally inert inside FaultRetry.
612
+ faultRetry.recoverPendingOnStartup();
594
613
  // A launch prompt delivers only after the listener is accepting, so a
595
614
  // root-spawn readiness proof and prompt admission share one boundary. This
596
615
  // includes the one continuation carried by a clean runtime-abort resume.
@@ -1,13 +1,14 @@
1
1
  import { type SubscriptionRef } from '../canvas/index.js';
2
- /** The doctrine wake: fan a system notice to every given subscriber of a node
3
- * whose lifecycle just ended out from under it (closed, or crashed to dead) —
4
- * the runtime guarantee that a manager waiting dormant on a child always
5
- * learns of its terminal outcome instead of hanging forever. Active subscriber
6
- * → inbox (wakes a dormant manager via its live watcher / the daemon's
7
- * dormant-revive pass); passive → the passive accumulator (delivered, not
8
- * woken). Shared by closeNode (step 4, below) and reviveNode's launch-refusal
9
- * crash path (revive.ts). */
10
- export declare function fanDoctrineWake(fromId: string, subscribers: readonly SubscriptionRef[], label: string, data: Record<string, unknown>): void;
2
+ import { type InboxTier } from '../feed/inbox.js';
3
+ /** Fan a lifecycle or birth notice to every given subscriber. Active subscribers
4
+ * receive inbox mail; passive subscribers receive passive mail. Shared by
5
+ * terminal-outcome paths and externally created children, so callers preserve
6
+ * the existing subscription's delivery mode rather than inventing a wake path.
7
+ *
8
+ * `tier` defaults to a waking `normal`. A caller whose outcome is not worth
9
+ * starting a manager's cycle passes `deferred`, which rides the manager's next
10
+ * natural cycle instead. */
11
+ export declare function fanDoctrineWake(fromId: string, subscribers: readonly SubscriptionRef[], label: string, data: Record<string, unknown>, tier?: InboxTier): void;
11
12
  export interface CloseNodeResult {
12
13
  /** The focused node that was closed — the cascade root. */
13
14
  root: string;
@@ -31,4 +32,5 @@ export interface CloseNodeResult {
31
32
  * status keeps it (finish is only legal from a live status). */
32
33
  export declare function closeNode(rootId: string, opts?: {
33
34
  rootEvent?: 'cancel' | 'finish';
35
+ callerPid?: number;
34
36
  }): CloseNodeResult;
@@ -30,26 +30,33 @@
30
30
  // inside the closing set. A node still subscribed to by a manager outside the
31
31
  // subtree is left running — "only kill the children if they are only subscribed
32
32
  // to by the agent being closed", generalized to any depth via a fixpoint.
33
- import { getNode, subscriptionsOf, subscribersOf, } from '../canvas/index.js';
33
+ import { cancelCronsOnWake, getNode, subscriptionsOf, subscribersOf, } from '../canvas/index.js';
34
34
  import { transition } from './lifecycle.js';
35
- import { headlessBrokerHost } from './host.js';
35
+ import { requestBrokerTeardown } from './host.js';
36
36
  import { tearDownNode, reapIfEmpty } from './placement.js';
37
37
  import { appendInbox } from '../feed/inbox.js';
38
38
  import { appendPassive } from '../feed/passive.js';
39
- /** The doctrine wake: fan a system notice to every given subscriber of a node
40
- * whose lifecycle just ended out from under it (closed, or crashed to dead) —
41
- * the runtime guarantee that a manager waiting dormant on a child always
42
- * learns of its terminal outcome instead of hanging forever. Active subscriber
43
- * → inbox (wakes a dormant manager via its live watcher / the daemon's
44
- * dormant-revive pass); passive → the passive accumulator (delivered, not
45
- * woken). Shared by closeNode (step 4, below) and reviveNode's launch-refusal
46
- * crash path (revive.ts). */
47
- export function fanDoctrineWake(fromId, subscribers, label, data) {
39
+ /** Fan a lifecycle or birth notice to every given subscriber. Active subscribers
40
+ * receive inbox mail; passive subscribers receive passive mail. Shared by
41
+ * terminal-outcome paths and externally created children, so callers preserve
42
+ * the existing subscription's delivery mode rather than inventing a wake path.
43
+ *
44
+ * `tier` defaults to a waking `normal`. A caller whose outcome is not worth
45
+ * starting a manager's cycle passes `deferred`, which rides the manager's next
46
+ * natural cycle instead. */
47
+ export function fanDoctrineWake(fromId, subscribers, label, data, tier = 'normal') {
48
48
  for (const sub of subscribers) {
49
49
  try {
50
- const notice = { from: fromId, tier: 'normal', kind: 'message', label, data };
51
- if (sub.active)
50
+ const notice = { from: fromId, tier, kind: 'message', label, data };
51
+ if (sub.active) {
52
52
  appendInbox(sub.node_id, notice);
53
+ // A wake-capable notice consumes the subscriber's cancel-on-wake
54
+ // deadline: the thing it was waiting for arrived. `deferred` is the
55
+ // exception — it never wakes a node, riding the next natural cycle
56
+ // instead, so the deadline it was racing must survive.
57
+ if (tier !== 'deferred')
58
+ cancelCronsOnWake(sub.node_id);
59
+ }
53
60
  else
54
61
  appendPassive(sub.node_id, notice);
55
62
  }
@@ -190,12 +197,9 @@ export function closeNode(rootId, opts = {}) {
190
197
  else {
191
198
  transition(id, 'cancel', { reason: 'closed' });
192
199
  }
193
- // 2) Tear the node's ENGINE down: send the `shutdown` frame so the broker
194
- // PROCESS exits and releases the sole .jsonl writer. Then proactively
195
- // close the viewer pane + registry row — attach auto-reconnects, so on a
196
- // deliberate close the viewer must be torn down here or it lingers ~30s
197
- // showing a misleading "reconnecting…" instead of going away at once.
198
- headlessBrokerHost.teardown(id);
200
+ // 2) Ask the daemon to tear the engine down after this close response
201
+ // settles. The root caller's shell is excluded from its own broker tree.
202
+ requestBrokerTeardown(id, id === rootId ? { excludePid: opts.callerPid } : {});
199
203
  tearDownNode(id);
200
204
  // 3) Leave the resume notice AFTER the watcher is gone, so it survives.
201
205
  appendInbox(id, {
@@ -11,15 +11,36 @@ export interface FaultRecordInput {
11
11
  since?: string;
12
12
  anchorEntryId?: string;
13
13
  }
14
+ /** A provider-retry episode outlives its broker-local timer. Its transcript
15
+ * path and anchor are explicit coordinates, so replacement invalidation never
16
+ * has to guess which conversation a marker belongs to. */
17
+ export interface ProviderRetryEpisode {
18
+ state: 'pending' | 'admitted' | 'invalidated';
19
+ sessionFile: string;
20
+ anchorEntryId: string | null;
21
+ fault: Fault;
22
+ }
14
23
  export declare function recordFault(nodeId: string, input: FaultRecordInput): void;
24
+ /** Atomically establishes the crash-recovery authority before publishing its
25
+ * ordinary marker, so a broker death can never leave only a retryable fault. */
26
+ export declare function recordPendingProviderRetryFault(nodeId: string, sessionFile: string, input: FaultRecordInput): Fault | null;
15
27
  export declare function recordFaultForCurrentNode(input: FaultRecordInput): boolean;
16
28
  export declare function clearFault(nodeId: string, opts?: {
17
29
  link?: FaultLink;
30
+ preserveProviderRetryEpisode?: boolean;
18
31
  }): boolean;
19
32
  export declare function clearFaultForCurrentNode(opts?: {
20
33
  link?: FaultLink;
21
34
  }): boolean;
22
35
  export declare function readFault(nodeId: string): Fault | null;
36
+ export declare function readProviderRetryEpisode(nodeId: string): ProviderRetryEpisode | null;
37
+ /** Persist a newly scheduled provider retry before its broker-local timer is armed. */
38
+ export declare function recordPendingProviderRetryEpisode(nodeId: string, episode: Omit<ProviderRetryEpisode, 'state'>): boolean;
39
+ /** Admission is the irreversible boundary: an admitted retry is never replayed. */
40
+ export declare function admitProviderRetryEpisode(nodeId: string, sessionFile: string, fault: Fault): boolean;
41
+ /** A replacement explicitly invalidates its old transcript's retry episode. */
42
+ export declare function invalidateProviderRetryEpisode(nodeId: string, sessionFile: string): boolean;
43
+ export declare function clearProviderRetryEpisode(nodeId: string): void;
23
44
  export declare function readFaultAsync(nodeId: string): Promise<Fault | null>;
24
45
  export interface BootFaultAttempt {
25
46
  /** The recorded fault's operation + message (e.g. an unresolvable `--model`
@@ -1,5 +1,5 @@
1
1
  import { envNodeId } from '../../shared/env.js';
2
- import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
2
+ import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
3
3
  import { join } from 'node:path';
4
4
  import { jobDir } from '../canvas/paths.js';
5
5
  import { errorClassFromFault, } from '../fault-classifier.js';
@@ -10,6 +10,17 @@ const MESSAGE_CAP = 500;
10
10
  function faultPath(nodeId) {
11
11
  return join(jobDir(nodeId), 'fault');
12
12
  }
13
+ function providerRetryPath(nodeId) {
14
+ return join(jobDir(nodeId), 'provider-retry');
15
+ }
16
+ function canonicalSessionFile(sessionFile) {
17
+ try {
18
+ return realpathSync(sessionFile);
19
+ }
20
+ catch {
21
+ return sessionFile;
22
+ }
23
+ }
13
24
  function clampMessage(message) {
14
25
  return message.slice(0, MESSAGE_CAP);
15
26
  }
@@ -78,30 +89,64 @@ function currentFault(nodeId) {
78
89
  return null;
79
90
  }
80
91
  }
92
+ function currentProviderRetryEpisode(nodeId) {
93
+ try {
94
+ const parsed = JSON.parse(readFileSync(providerRetryPath(nodeId), 'utf8'));
95
+ if (parsed.state !== 'pending' && parsed.state !== 'admitted' && parsed.state !== 'invalidated')
96
+ return null;
97
+ if (typeof parsed.sessionFile !== 'string' || parsed.sessionFile === '')
98
+ return null;
99
+ if (parsed.anchorEntryId !== null && typeof parsed.anchorEntryId !== 'string')
100
+ return null;
101
+ const fault = parseFault(JSON.stringify(parsed.fault));
102
+ if (fault === null ||
103
+ fault.link !== 'pi→provider' ||
104
+ fault.retry.disposition !== 'auto' ||
105
+ fault.retry.by !== 'daemon' ||
106
+ (fault.anchorEntryId ?? null) !== parsed.anchorEntryId)
107
+ return null;
108
+ return { state: parsed.state, sessionFile: canonicalSessionFile(parsed.sessionFile), anchorEntryId: parsed.anchorEntryId, fault };
109
+ }
110
+ catch {
111
+ return null;
112
+ }
113
+ }
81
114
  function writeFault(nodeId, fault) {
82
115
  try {
83
116
  mkdirSync(jobDir(nodeId), { recursive: true });
84
117
  writeFileSync(faultPath(nodeId), JSON.stringify(fault));
118
+ return true;
85
119
  }
86
120
  catch {
87
- /* best-effort */
121
+ return false;
88
122
  }
89
123
  }
90
- function updateMarker(nodeId, fault) {
124
+ function writeProviderRetryEpisode(nodeId, episode) {
125
+ try {
126
+ mkdirSync(jobDir(nodeId), { recursive: true });
127
+ writeFileSync(providerRetryPath(nodeId), JSON.stringify(episode));
128
+ return true;
129
+ }
130
+ catch {
131
+ return false;
132
+ }
133
+ }
134
+ function resolvedMarker(nodeId, fault) {
91
135
  const current = currentFault(nodeId);
92
- if (current !== null) {
93
- if (current.link === fault.link && current.kind === fault.kind) {
94
- const retry = { ...current.retry, ...fault.retry };
95
- if (fault.retry.disposition === 'fatal' && !('nextAt' in fault.retry)) {
96
- delete retry.nextAt;
97
- }
98
- fault = { ...fault, since: current.since, anchorEntryId: current.anchorEntryId ?? fault.anchorEntryId, retry };
99
- }
100
- else if (faultPriority(current) > faultPriority(fault)) {
101
- return;
136
+ if (current === null)
137
+ return fault;
138
+ if (current.link === fault.link && current.kind === fault.kind) {
139
+ const retry = { ...current.retry, ...fault.retry };
140
+ if (fault.retry.disposition === 'fatal' && !('nextAt' in fault.retry)) {
141
+ delete retry.nextAt;
102
142
  }
143
+ return { ...fault, since: current.since, anchorEntryId: current.anchorEntryId ?? fault.anchorEntryId, retry };
103
144
  }
104
- writeFault(nodeId, fault);
145
+ return faultPriority(current) > faultPriority(fault) ? null : fault;
146
+ }
147
+ function updateMarker(nodeId, fault) {
148
+ const resolved = resolvedMarker(nodeId, fault);
149
+ return resolved !== null && writeFault(nodeId, resolved) ? resolved : null;
105
150
  }
106
151
  function resolveOperationId(explicit) {
107
152
  if (explicit !== undefined)
@@ -111,22 +156,23 @@ function resolveOperationId(explicit) {
111
156
  return current;
112
157
  return operationIdContext.fresh((operationId) => operationId);
113
158
  }
114
- export function recordFault(nodeId, input) {
115
- const operationId = resolveOperationId(input.operation_id);
116
- const fault = {
159
+ function storedFault(input) {
160
+ return {
117
161
  link: input.link,
118
162
  op: input.op,
119
163
  kind: input.kind,
120
164
  retry: input.retry,
121
165
  message: clampMessage(input.message),
122
166
  since: input.since ?? new Date().toISOString(),
123
- operation_id: operationId,
167
+ operation_id: resolveOperationId(input.operation_id),
124
168
  ...(typeof input.anchorEntryId === 'string' && input.anchorEntryId !== '' ? { anchorEntryId: input.anchorEntryId } : {}),
125
169
  };
170
+ }
171
+ function emitRecordedFault(nodeId, fault) {
126
172
  emitEvent({
127
173
  level: fault.retry.disposition === 'fatal' ? 'error' : 'warn',
128
174
  event: 'fault.recorded',
129
- operation_id: operationId,
175
+ operation_id: fault.operation_id,
130
176
  node_id: nodeId,
131
177
  error_class: errorClassFromFault(fault),
132
178
  fields: {
@@ -138,8 +184,29 @@ export function recordFault(nodeId, input) {
138
184
  ...(fault.anchorEntryId === undefined ? {} : { anchor_entry_id: fault.anchorEntryId }),
139
185
  },
140
186
  });
187
+ }
188
+ export function recordFault(nodeId, input) {
189
+ const fault = storedFault(input);
190
+ emitRecordedFault(nodeId, fault);
141
191
  updateMarker(nodeId, fault);
142
192
  }
193
+ /** Atomically establishes the crash-recovery authority before publishing its
194
+ * ordinary marker, so a broker death can never leave only a retryable fault. */
195
+ export function recordPendingProviderRetryFault(nodeId, sessionFile, input) {
196
+ const fault = storedFault(input);
197
+ emitRecordedFault(nodeId, fault);
198
+ const marker = resolvedMarker(nodeId, fault);
199
+ if (marker === null)
200
+ return null;
201
+ if (!writeProviderRetryEpisode(nodeId, {
202
+ state: 'pending',
203
+ sessionFile: canonicalSessionFile(sessionFile),
204
+ anchorEntryId: marker.anchorEntryId ?? null,
205
+ fault: marker,
206
+ }))
207
+ return null;
208
+ return writeFault(nodeId, marker) ? marker : null;
209
+ }
143
210
  export function recordFaultForCurrentNode(input) {
144
211
  const nodeId = envNodeId();
145
212
  if (nodeId === undefined || nodeId.trim() === '')
@@ -162,6 +229,11 @@ export function clearFault(nodeId, opts) {
162
229
  if (opts?.link !== undefined && current.link !== opts.link)
163
230
  return false;
164
231
  rmSync(faultPath(nodeId));
232
+ if (opts?.preserveProviderRetryEpisode !== true) {
233
+ const episode = currentProviderRetryEpisode(nodeId);
234
+ if (episode?.fault.since === current.since)
235
+ rmSync(providerRetryPath(nodeId), { force: true });
236
+ }
165
237
  const operationId = resolveOperationId();
166
238
  emitEvent({
167
239
  level: 'info',
@@ -197,6 +269,44 @@ export function clearFaultForCurrentNode(opts) {
197
269
  export function readFault(nodeId) {
198
270
  return currentFault(nodeId);
199
271
  }
272
+ export function readProviderRetryEpisode(nodeId) {
273
+ return currentProviderRetryEpisode(nodeId);
274
+ }
275
+ /** Persist a newly scheduled provider retry before its broker-local timer is armed. */
276
+ export function recordPendingProviderRetryEpisode(nodeId, episode) {
277
+ return writeProviderRetryEpisode(nodeId, { ...episode, sessionFile: canonicalSessionFile(episode.sessionFile), state: 'pending' });
278
+ }
279
+ /** Admission is the irreversible boundary: an admitted retry is never replayed. */
280
+ export function admitProviderRetryEpisode(nodeId, sessionFile, fault) {
281
+ const current = currentProviderRetryEpisode(nodeId);
282
+ if (current === null ||
283
+ current.state !== 'pending' ||
284
+ current.sessionFile !== canonicalSessionFile(sessionFile) ||
285
+ current.fault.since !== fault.since ||
286
+ current.anchorEntryId !== (fault.anchorEntryId ?? null))
287
+ return false;
288
+ return writeProviderRetryEpisode(nodeId, {
289
+ state: 'admitted',
290
+ sessionFile: current.sessionFile,
291
+ anchorEntryId: current.anchorEntryId,
292
+ fault,
293
+ });
294
+ }
295
+ /** A replacement explicitly invalidates its old transcript's retry episode. */
296
+ export function invalidateProviderRetryEpisode(nodeId, sessionFile) {
297
+ const current = currentProviderRetryEpisode(nodeId);
298
+ if (current === null || current.sessionFile !== canonicalSessionFile(sessionFile) || current.state === 'invalidated')
299
+ return false;
300
+ return writeProviderRetryEpisode(nodeId, { ...current, state: 'invalidated' });
301
+ }
302
+ export function clearProviderRetryEpisode(nodeId) {
303
+ try {
304
+ rmSync(providerRetryPath(nodeId), { force: true });
305
+ }
306
+ catch {
307
+ /* best-effort */
308
+ }
309
+ }
200
310
  export async function readFaultAsync(nodeId) {
201
311
  return readFault(nodeId);
202
312
  }
@@ -22,15 +22,9 @@ export interface FleetEntry {
22
22
  /** The broker's supervised pid (never null — a launch with no pid is not
23
23
  * registered). */
24
24
  pid: number;
25
- /** Date.now() at spawn — the respawn throttle's uptime baseline. */
26
- launchedAt: number;
27
25
  /** The ChildProcess-backed handle (host.ts) whose `exited` promise is the
28
26
  * liveness authority. */
29
27
  handle: HostHandle;
30
- /** Respawn-throttle state at registration (consecutive short-lived exits
31
- * observed before this launch). The authoritative counter lives in the
32
- * daemon implementation and survives the entry's removal at exit. */
33
- consecutiveFailures: number;
34
28
  }
35
29
  export interface FleetRegistry {
36
30
  /** This daemon's epoch id — minted at boot, stamped into every broker's argv
@@ -22,6 +22,9 @@ export interface HostHandle {
22
22
  signal: NodeJS.Signals | null;
23
23
  }>;
24
24
  }
25
+ export interface TeardownOptions {
26
+ excludePid?: number;
27
+ }
25
28
  export interface Host {
26
29
  /** Bring a node's ENGINE into existence from its launch recipe; return a
27
30
  * supervisable handle. */
@@ -31,7 +34,7 @@ export interface Host {
31
34
  * row) — share one selector. For the broker this IS isPidAlive(pi_pid). */
32
35
  isAlive(node: string | NodeRow): boolean;
33
36
  /** Tear the engine down (close/cancel teardown). */
34
- teardown(nodeId: string): void;
37
+ teardown(nodeId: string, opts?: TeardownOptions): void;
35
38
  /** Deliver an OS signal to the engine container — present so the daemon never
36
39
  * reaches around the abstraction. */
37
40
  signal(nodeId: string, sig: NodeJS.Signals): void;
@@ -40,4 +43,7 @@ export interface Host {
40
43
  * the fast-fail seam for known module/path errors: a bad engine resolve or a
41
44
  * missing plain session path throws here, before spawn(). */
42
45
  export declare function preflightBrokerLaunch(nodeId: string, inv: PiInvocation, cwd: string): void;
46
+ /** Queue broker teardown onto the daemon event loop after the close response can
47
+ * settle. Repeated terminal reconciliation coalesces on the same node. */
48
+ export declare function requestBrokerTeardown(nodeId: string, opts?: TeardownOptions): void;
43
49
  export declare const headlessBrokerHost: Host;
@@ -189,6 +189,24 @@ function escalateBrokerTeardown(tree, identities) {
189
189
  killProcessTreePids(tree, 'SIGTERM', identities);
190
190
  pollUntil(() => isAnyPidAlive(tree), BROKER_SIGKILL_GRACE_MS, () => killProcessTreePids(tree, 'SIGKILL', identities));
191
191
  }
192
+ const requestedTeardowns = new Map();
193
+ /** Queue broker teardown onto the daemon event loop after the close response can
194
+ * settle. Repeated terminal reconciliation coalesces on the same node. */
195
+ export function requestBrokerTeardown(nodeId, opts = {}) {
196
+ const existing = requestedTeardowns.get(nodeId);
197
+ if (existing !== undefined) {
198
+ if (existing.excludePid === undefined && opts.excludePid !== undefined)
199
+ existing.excludePid = opts.excludePid;
200
+ return;
201
+ }
202
+ requestedTeardowns.set(nodeId, { ...opts });
203
+ setImmediate(() => {
204
+ const request = requestedTeardowns.get(nodeId);
205
+ requestedTeardowns.delete(nodeId);
206
+ if (request !== undefined)
207
+ headlessBrokerHost.teardown(nodeId, request);
208
+ });
209
+ }
192
210
  export const headlessBrokerHost = {
193
211
  launch(nodeId, inv, opts) {
194
212
  // Fast-fail known launch errors before we detach a broker process.
@@ -336,7 +354,7 @@ export const headlessBrokerHost = {
336
354
  isAlive(node) {
337
355
  return isPidAlive((typeof node === 'string' ? getNode(node) : node)?.pi_pid);
338
356
  },
339
- teardown(nodeId) {
357
+ teardown(nodeId, opts = {}) {
340
358
  // Graceful: connect to view.sock + send a `shutdown` frame → the broker
341
359
  // dispose()s the engine, unlinks the socket, exits 0. Status is already
342
360
  // flipped done/canceled by the caller (crash-safe ordering), so the daemon
@@ -376,7 +394,7 @@ export const headlessBrokerHost = {
376
394
  // snapshot's `reused: true` comes back with an EMPTY tree, so nothing
377
395
  // below ever signals the stranger (or walks its process tree looking for
378
396
  // "descendants", which would be the stranger's children, not ours).
379
- const snapshot = pid != null ? captureTeardownSnapshot(pid, node?.pi_pid_identity ?? null) : null;
397
+ const snapshot = pid != null ? captureTeardownSnapshot(pid, node?.pi_pid_identity ?? null, opts.excludePid) : null;
380
398
  if (snapshot?.reused === true) {
381
399
  emitEvent({
382
400
  level: 'warn',
@@ -59,8 +59,12 @@ export interface SpawnNodeOpts {
59
59
  name?: string;
60
60
  /** Editor-label handle (2-4 word kebab-case) for the node's first prompt. */
61
61
  description?: string;
62
- /** Parent node id. Omit for a user-opened root. */
62
+ /** Parent node id: the spine manager. Omit for a user-opened root. */
63
63
  parent?: string | null;
64
+ /** Actor that created this node. Unlike parent (manager) and spawnedBy
65
+ * (audit provenance), it decides whether the parent gets a birth notice.
66
+ * Null/omitted means an external process. */
67
+ creator?: string | null;
64
68
  /** Who spawned me (the `spawned_by` provenance edge), when it differs from
65
69
  * `parent` — e.g. an independent root (parent=null) still records its
66
70
  * spawner. Defaults to `parent`. */
@@ -15,12 +15,14 @@
15
15
  import { envHomeOverride } from '../../shared/env.js';
16
16
  import { randomBytes } from 'node:crypto';
17
17
  import { mkdirSync } from 'node:fs';
18
- import { createNode, getNode, getRow, subscribe, recordSpawn, contextDir, } from '../canvas/index.js';
18
+ import { createNode, getNode, getRow, subscribe, recordSpawn, subscribersOf, contextDir, } from '../canvas/index.js';
19
19
  import { memoryDir } from './memory.js';
20
20
  import { resolveInstallId } from '../canvas/install-id.js';
21
21
  import { canvasDbPath } from '../canvas/paths.js';
22
22
  import { usage } from '../errors.js';
23
23
  import { assertProfileAvailableForBirth } from '../profiles/deletion-reservation.js';
24
+ import { fanDoctrineWake } from './close.js';
25
+ import { BIRTH_ANNOUNCEMENT_MARKER, birthAnnouncementBody } from '../../shared/birth-announcement.js';
24
26
  // crtrd sets this once at boot (setInstallId) and it wins unconditionally —
25
27
  // a daemon process only ever serves one canvas home for its whole lifetime,
26
28
  // so no keying is needed there. Every OTHER in-process caller (tests,
@@ -225,6 +227,7 @@ export function spawnNode(opts) {
225
227
  // Provenance is independent of the spine: a root has no parent but still
226
228
  // records who spawned it. A child's spawner is its parent unless overridden.
227
229
  const spawnedBy = opts.spawnedBy ?? parent;
230
+ const creator = opts.creator ?? null;
228
231
  const mode = opts.mode ?? 'base';
229
232
  // A user-opened root is resident (a conversation you live in); a spawned node
230
233
  // is terminal until it must persist (promotion handles that later).
@@ -248,6 +251,7 @@ export function spawnNode(opts) {
248
251
  status: 'active',
249
252
  parent,
250
253
  spawned_by: spawnedBy,
254
+ creator,
251
255
  fork_from: opts.forkFrom,
252
256
  fork_source_file: opts.forkSourceFile,
253
257
  review_binding: opts.reviewBinding,
@@ -265,6 +269,9 @@ export function spawnNode(opts) {
265
269
  if (parent !== null && getNode(parent) === null) {
266
270
  throw new Error(`cannot spawn under unknown parent node: ${parent}`);
267
271
  }
272
+ if (creator !== null && getNode(creator) === null) {
273
+ throw new Error(`cannot spawn from unknown creator node: ${creator}`);
274
+ }
268
275
  assertProfileAvailableForBirth(meta.profile_id ?? null);
269
276
  createNode(meta);
270
277
  // Create the node-local memory directory so substrate docs can be written
@@ -279,6 +286,22 @@ export function spawnNode(opts) {
279
286
  // A root (parent=null) gets NO subscription — nobody is woken by it.
280
287
  subscribe(parent, meta.node_id, true);
281
288
  }
289
+ if (parent !== null && creator !== parent) {
290
+ try {
291
+ fanDoctrineWake(meta.node_id, subscribersOf(meta.node_id), `Child created — ${meta.name} (${meta.node_id}, ${meta.kind})`, {
292
+ birth_announcement: BIRTH_ANNOUNCEMENT_MARKER,
293
+ body: birthAnnouncementBody({
294
+ name: meta.name,
295
+ nodeId: meta.node_id,
296
+ kind: meta.kind,
297
+ creator: creator === null ? 'an external process' : getNode(creator).name,
298
+ }),
299
+ }, 'deferred');
300
+ }
301
+ catch {
302
+ /* a failed birth announcement never prevents the child from existing */
303
+ }
304
+ }
282
305
  // Audit-only provenance edge — recorded for a root too (from its spawner).
283
306
  if (spawnedBy !== null && spawnedBy !== undefined && getNode(spawnedBy) !== null) {
284
307
  recordSpawn(meta.node_id, spawnedBy);
@@ -89,12 +89,12 @@ export declare function detachToBackground(nodeId: string, pane?: string): boole
89
89
  export declare function reapIfEmpty(nodeId: string, keepPane?: string): boolean;
90
90
  /** Sweep the whole canvas for empty shells and reap them — the bulk cleanup
91
91
  * behind `crtr canvas prune --empty`, for the husks that accumulated before
92
- * close/detach learned to reap. Skips: the caller ($CRTR_NODE_ID), nodes
93
- * mid-first-turn, and any node with a LIVE broker (pi_pid alive) — a running
94
- * engine is the daemon's domain and may have output not yet persisted, so bulk
95
- * GC never kills one (the per-action {@link reapIfEmpty} does, since the user
96
- * explicitly closed/detached that node). `dryRun` reports candidates without
97
- * deleting. Returns the reaped (or, under dryRun, reapable) node ids. */
92
+ * close/detach learned to reap. Skips: the caller ($CRTR_NODE_ID), dead rows,
93
+ * nodes mid-first-turn, and any node with a LIVE broker (pi_pid alive) — a
94
+ * running engine is the daemon's domain and may have output not yet persisted,
95
+ * so bulk GC never kills one (the per-action {@link reapIfEmpty} does, since
96
+ * the user explicitly closed/detached that node). `dryRun` reports candidates
97
+ * without deleting. Returns the reaped (or, under dryRun, reapable) node ids. */
98
98
  export declare function reapEmptyNodes(opts?: {
99
99
  dryRun?: boolean;
100
100
  }): string[];