@north-light/crouter 0.3.249 → 0.3.251

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/README.md +1 -1
  2. package/dist/api/__tests__/integration/client.test.js +28 -7
  3. package/dist/api/client.d.ts +1 -1
  4. package/dist/api/client.js +2 -2
  5. package/dist/api/dto/lifecycle.d.ts +3 -0
  6. package/dist/api/dto/messages.d.ts +4 -0
  7. package/dist/api/dto/nodes.d.ts +4 -0
  8. package/dist/api/dto/reports.d.ts +7 -2
  9. package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
  10. package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
  11. package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
  12. package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
  13. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
  14. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
  15. package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
  16. package/dist/clients/attach/session/chrome-refresh.js +1 -0
  17. package/dist/clients/attach/viewer.js +532 -532
  18. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
  19. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
  20. package/dist/clients/inbox/controller.js +32 -5
  21. package/dist/clients/inbox/tui/ansi.d.ts +3 -1
  22. package/dist/clients/inbox/tui/ansi.js +14 -10
  23. package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
  24. package/dist/commands/api-client.d.ts +2 -2
  25. package/dist/commands/api-client.js +5 -5
  26. package/dist/commands/canvas-browse.js +1 -1
  27. package/dist/commands/node/create.js +3 -1
  28. package/dist/commands/node/inspect.js +1 -0
  29. package/dist/commands/node/lifecycle.js +1 -1
  30. package/dist/commands/node/message.js +6 -4
  31. package/dist/commands/node/wait.js +1 -1
  32. package/dist/commands/push.js +36 -37
  33. package/dist/commands/sys/daemon.js +21 -8
  34. package/dist/commands/sys/setup-core.js +2 -2
  35. package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
  36. package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
  37. package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
  38. package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
  39. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +100 -1
  40. package/dist/core/__tests__/child-death-wake.test.js +11 -7
  41. package/dist/core/__tests__/collectible.test.d.ts +1 -0
  42. package/dist/core/__tests__/collectible.test.js +88 -0
  43. package/dist/core/__tests__/context-intro.test.js +4 -41
  44. package/dist/core/__tests__/daemon-boot.test.js +30 -14
  45. package/dist/core/__tests__/daemon-wedge.test.js +25 -19
  46. package/dist/core/__tests__/fixtures/fake-engine.js +20 -2
  47. package/dist/core/__tests__/human-deliver.test.js +5 -5
  48. package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
  49. package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
  50. package/dist/core/__tests__/integration/revive.test.js +48 -0
  51. package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
  52. package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
  53. package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
  54. package/dist/core/__tests__/migration.test.js +30 -0
  55. package/dist/core/__tests__/passive-subscription.test.js +39 -1
  56. package/dist/core/__tests__/prune.test.js +12 -3
  57. package/dist/core/__tests__/relaunch-root.test.js +1 -1
  58. package/dist/core/__tests__/respawn-throttle.test.js +16 -146
  59. package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
  60. package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
  61. package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
  62. package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
  63. package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
  64. package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
  65. package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
  66. package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
  67. package/dist/core/canvas/canvas.d.ts +9 -15
  68. package/dist/core/canvas/canvas.js +41 -47
  69. package/dist/core/canvas/crons.d.ts +5 -8
  70. package/dist/core/canvas/crons.js +5 -8
  71. package/dist/core/canvas/history.d.ts +5 -0
  72. package/dist/core/canvas/history.js +12 -1
  73. package/dist/core/canvas/migrations.js +58 -1
  74. package/dist/core/canvas/pid.d.ts +1 -1
  75. package/dist/core/canvas/pid.js +5 -2
  76. package/dist/core/canvas/types.d.ts +4 -0
  77. package/dist/core/feed/feed.d.ts +4 -0
  78. package/dist/core/feed/feed.js +13 -8
  79. package/dist/core/feed/inbox.d.ts +2 -1
  80. package/dist/core/feed/inbox.js +22 -3
  81. package/dist/core/git.d.ts +3 -0
  82. package/dist/core/git.js +3 -1
  83. package/dist/core/human/feedback-companion.js +1 -0
  84. package/dist/core/preview-registry.js +0 -1
  85. package/dist/core/review/companion.js +1 -0
  86. package/dist/core/review/realize.js +1 -0
  87. package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
  88. package/dist/core/runtime/broker/auth-reload.js +9 -3
  89. package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
  90. package/dist/core/runtime/broker/engine-drive.js +39 -23
  91. package/dist/core/runtime/broker/event-projection.d.ts +4 -0
  92. package/dist/core/runtime/broker/event-projection.js +43 -23
  93. package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
  94. package/dist/core/runtime/broker/fault-retry.js +94 -8
  95. package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
  96. package/dist/core/runtime/broker/frame-dispatch.js +4 -1
  97. package/dist/core/runtime/broker/held-deferred-inbox.d.ts +9 -0
  98. package/dist/core/runtime/broker/held-deferred-inbox.js +8 -0
  99. package/dist/core/runtime/broker/inbox.js +4 -0
  100. package/dist/core/runtime/broker/rebind.js +18 -0
  101. package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
  102. package/dist/core/runtime/broker/turn-admission.js +31 -0
  103. package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
  104. package/dist/core/runtime/broker/turn-ignition.js +25 -15
  105. package/dist/core/runtime/broker.js +21 -2
  106. package/dist/core/runtime/close.d.ts +11 -9
  107. package/dist/core/runtime/close.js +23 -19
  108. package/dist/core/runtime/fault.d.ts +21 -0
  109. package/dist/core/runtime/fault.js +129 -19
  110. package/dist/core/runtime/fleet.d.ts +0 -6
  111. package/dist/core/runtime/host.d.ts +7 -1
  112. package/dist/core/runtime/host.js +20 -2
  113. package/dist/core/runtime/nodes.d.ts +5 -1
  114. package/dist/core/runtime/nodes.js +24 -1
  115. package/dist/core/runtime/placement.d.ts +6 -6
  116. package/dist/core/runtime/placement.js +15 -8
  117. package/dist/core/runtime/reset.js +2 -2
  118. package/dist/core/runtime/revive.js +40 -4
  119. package/dist/core/runtime/spawn.d.ts +3 -0
  120. package/dist/core/runtime/spawn.js +1 -0
  121. package/dist/core/runtime/warm-pool.d.ts +4 -0
  122. package/dist/core/runtime/warm-pool.js +2 -0
  123. package/dist/core/worktree.d.ts +4 -3
  124. package/dist/core/worktree.js +73 -27
  125. package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
  126. package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
  127. package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
  128. package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
  129. package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
  130. package/dist/daemon/api/handlers/bash-jobs.js +2 -0
  131. package/dist/daemon/api/handlers/broker-ops.js +23 -2
  132. package/dist/daemon/api/handlers/human.js +1 -1
  133. package/dist/daemon/api/handlers/messages.js +11 -2
  134. package/dist/daemon/api/handlers/nodes.js +13 -1
  135. package/dist/daemon/api/handlers/reports.js +14 -6
  136. package/dist/daemon/api/map.js +1 -0
  137. package/dist/daemon/api/server.d.ts +9 -12
  138. package/dist/daemon/api/server.js +30 -19
  139. package/dist/daemon/companion-retire.js +3 -9
  140. package/dist/daemon/cron/sinks.js +7 -2
  141. package/dist/daemon/crtrd.js +4 -1
  142. package/dist/daemon/fleet.d.ts +2 -16
  143. package/dist/daemon/fleet.js +63 -150
  144. package/dist/daemon/human/finish.js +2 -0
  145. package/dist/daemon/manage.d.ts +11 -0
  146. package/dist/daemon/manage.js +32 -3
  147. package/dist/daemon/messaging/node-message.js +7 -1
  148. package/dist/daemon/reconcilers/bash-deadline.js +2 -0
  149. package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
  150. package/dist/daemon/reconcilers/broker-supervision.js +22 -9
  151. package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
  152. package/dist/daemon/reconcilers/live-obligation.js +18 -6
  153. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
  154. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
  155. package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
  156. package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
  157. package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
  158. package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
  159. package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
  160. package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
  161. package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
  162. package/dist/pi-extensions/canvas-context-intro.js +4 -76
  163. package/dist/pi-extensions/canvas-inbox-watcher.js +105 -4
  164. package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
  165. package/dist/pi-extensions/canvas-review-boundary.js +4 -29
  166. package/dist/pi-extensions/canvas-stophook.js +1 -6
  167. package/dist/shared/birth-announcement.d.ts +12 -0
  168. package/dist/shared/birth-announcement.js +25 -0
  169. package/dist/shared/generated-context.d.ts +8 -4
  170. package/dist/shared/generated-context.js +15 -9
  171. package/package.json +4 -4
  172. package/runtime.lock.json +2 -2
  173. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
  174. package/dist/pi-extensions/truncate.d.ts +0 -5
  175. package/dist/pi-extensions/truncate.js +0 -14
@@ -1,6 +1,7 @@
1
1
  // server.ts — the crtrd HTTP+WS API server (spec §4, esp. §4.1/§5). A-9.
2
2
  //
3
- // ONE `http.Server`, bound TWICE (the dockerd model, §4.1):
3
+ // Two `http.Server` listeners share one logical API (a Node server can bind one
4
+ // handle only):
4
5
  // - the unix socket at `apiSocketPath()` — ALWAYS bound, owner-only (mode
5
6
  // 0600 = the fs-perms-are-auth model);
6
7
  // - an opt-in TCP listener, bound only when `--tcp <host:port>` / `CRTRD_TCP`
@@ -121,19 +122,14 @@ function closeServer(server) {
121
122
  server.close(() => resolve());
122
123
  });
123
124
  }
124
- /** Create and start the crtrd API server (A-9). Returns immediately with a
125
- * `close()` handle; the listeners bind asynchronously (a unix socket's file is
126
- * created on `listen`, so its 0600 chmod happens in the listen callback). A
127
- * bind failure or a post-listen accept error is emitted and swallowed — it must
128
- * NEVER become an unhandled 'error' that crashes the daemon.
125
+ /** Create and start the crtrd API server. Returns immediately with a `close()`
126
+ * handle and a `ready` promise for the mandatory Unix socket bind. TCP remains
127
+ * opt-in, and its bind or post-listen accept errors are emitted but non-fatal.
129
128
  *
130
- * NOTE (correction to the plan's "one http.Server bound twice"): a Node
131
- * `net.Server`/`http.Server` binds exactly ONE handle — a second `.listen()`
132
- * on the same instance throws `ERR_SERVER_ALREADY_LISTEN`. So the two listeners
133
- * are two `http.Server` instances SHARING the same request + upgrade handler
134
- * functions (the standard way to bind one logical API to two transports). The
135
- * Router and attach bridge are process-wide singletons, so both servers route
136
- * into identical state. */
129
+ * A Node `net.Server`/`http.Server` binds exactly ONE handle, so the two
130
+ * listeners are separate `http.Server` instances sharing request and upgrade
131
+ * handlers. The Router and attach bridge are process-wide singletons, so both
132
+ * servers route into identical state. */
137
133
  export function createApiServer(opts = {}) {
138
134
  const router = buildRouter();
139
135
  const sockPath = apiSocketPath();
@@ -199,17 +195,32 @@ export function createApiServer(opts = {}) {
199
195
  }
200
196
  };
201
197
  const bindErrorHandler = (server) => {
202
- // A bind failure or post-listen accept error must not be an unhandled
203
- // 'error' that crashes the process — emit and carry on.
198
+ // TCP is opt-in, so neither its bind failure nor a later accept error can
199
+ // make the mandatory Unix API unavailable.
204
200
  server.on('error', (err) => {
205
201
  emitEvent({ event: 'api.server.failed', level: 'error', error: err });
206
202
  });
207
203
  };
208
- // Unix socket — ALWAYS bound. Unlink any stale socket first (mirror
209
- // broker.ts's listener setup), listen, then chmod 0600 once the file exists.
204
+ // Unix socket — ALWAYS bound. Its bind result is startup readiness; after
205
+ // that point, accept errors remain observable but do not tear the daemon down.
210
206
  const socketServer = createServer(onRequest);
211
207
  socketServer.on('upgrade', onUpgrade);
212
- bindErrorHandler(socketServer);
208
+ let socketBound = false;
209
+ let resolveReady;
210
+ let rejectReady;
211
+ const ready = new Promise((resolve, reject) => {
212
+ resolveReady = resolve;
213
+ rejectReady = reject;
214
+ });
215
+ socketServer.once('listening', () => {
216
+ socketBound = true;
217
+ resolveReady();
218
+ });
219
+ socketServer.on('error', (err) => {
220
+ emitEvent({ event: 'api.server.failed', level: 'error', error: err });
221
+ if (!socketBound)
222
+ rejectReady(err);
223
+ });
213
224
  try {
214
225
  if (existsSync(sockPath))
215
226
  unlinkSync(sockPath);
@@ -265,5 +276,5 @@ export function createApiServer(opts = {}) {
265
276
  /* best-effort cleanup */
266
277
  }
267
278
  };
268
- return { close };
279
+ return { ready, close };
269
280
  }
@@ -5,7 +5,7 @@
5
5
  import { getNode } from '../core/canvas/canvas.js';
6
6
  import { FinalizationError, pushFinal } from '../core/feed/feed.js';
7
7
  import { transition } from '../core/runtime/lifecycle.js';
8
- import { headlessBrokerHost } from '../core/runtime/host.js';
8
+ import { requestBrokerTeardown } from '../core/runtime/host.js';
9
9
  /** Retire a companion only through the terminal transition legal for its current status. */
10
10
  export async function retireCompanion(nodeId, outcome) {
11
11
  const node = getNode(nodeId);
@@ -23,18 +23,12 @@ export async function retireCompanion(nodeId, outcome) {
23
23
  throw error;
24
24
  result = 'already';
25
25
  }
26
- try {
27
- headlessBrokerHost.teardown(nodeId);
28
- }
29
- catch { /* the canonical final is authoritative */ }
26
+ requestBrokerTeardown(nodeId);
30
27
  return result;
31
28
  }
32
29
  transition(nodeId, 'cancel', { reason: 'retired' });
33
30
  if (node.status === 'active' || node.status === 'idle') {
34
- try {
35
- headlessBrokerHost.teardown(nodeId);
36
- }
37
- catch { /* the terminal transition is authoritative */ }
31
+ requestBrokerTeardown(nodeId);
38
32
  }
39
33
  return 'retired';
40
34
  }
@@ -5,6 +5,7 @@ import { ticketDir } from '../../core/human/root.js';
5
5
  import { preparePage, publishPage } from '../../core/human/tickets.js';
6
6
  import { resolvePageComponents } from '../../core/config.js';
7
7
  import { getNode } from '../../core/canvas/canvas.js';
8
+ import { cancelCronsOnWake } from '../../core/canvas/crons.js';
8
9
  import { appendInbox } from '../../core/feed/inbox.js';
9
10
  import { hasNoNaturalCycle } from '../../core/runtime/revive-all.js';
10
11
  import { isParked } from '../../core/runtime/lifecycle.js';
@@ -23,15 +24,19 @@ function deliverNodeSink(c, target, stdout) {
23
24
  throw new Error(`node sink ${target} has no natural cycle ahead (done or finalized)`);
24
25
  }
25
26
  const from = c.created_by != null && getNode(c.created_by) !== null ? c.created_by : null;
26
- appendInbox(target, {
27
+ const entry = appendInbox(target, {
27
28
  from,
28
29
  tier: c.tier,
29
30
  kind: 'message',
30
31
  label: `⏰ cron ${c.name}`,
31
32
  data: { body: stdout },
32
33
  });
34
+ if (c.tier !== 'deferred' || meta.frozen_at !== null)
35
+ cancelCronsOnWake(target);
36
+ // The APPENDED tier decides the wake, not the cron's requested one: a deferred
37
+ // cron aimed at a terminal node was raised out of `deferred` by the append.
33
38
  // The durable append succeeds even if its best-effort wake fails.
34
- if (c.tier !== 'deferred' && !isBrokerLive(meta)) {
39
+ if (entry.tier !== 'deferred' && !isBrokerLive(meta)) {
35
40
  try {
36
41
  reviveNode(target, { resume: true, capacity: 'freeze' });
37
42
  }
@@ -668,8 +668,11 @@ export async function runDaemon(opts = {}) {
668
668
  // no adoption across epochs, ever.
669
669
  await operationIdContext.fresh(() => reapPriorEpochBrokers(epoch));
670
670
  writePidfile();
671
- // Start the HTTP+WS API server (§4): unix socket always, TCP when opted in.
671
+ // The Unix API socket is the daemon's readiness boundary. A rejected bind
672
+ // enters the existing startup catch, which closes resources and releases
673
+ // this invocation's pidfile mirror and ownership claim before propagating.
672
674
  apiServer = createApiServer({ tcp: opts.tcp });
675
+ await apiServer.ready;
673
676
  // Pre-warm the node pool for the most recent project recipes, so the first
674
677
  // create after a boot is warm too. Fire-and-forget (the mints run off the
675
678
  // work lane), so it never delays supervision. Boot-time dead rows are the
@@ -1,7 +1,7 @@
1
1
  import { type NodeMeta } from '../core/canvas/index.js';
2
2
  import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
3
3
  import type { HostHandle } from '../core/runtime/host.js';
4
- /** Surface a vehicle that never booted (or exhausted its respawn budget).
4
+ /** Surface a vehicle that never booted.
5
5
  *
6
6
  * A pi that dies before its first session_start (so `pi_session_id` was never
7
7
  * recorded) is invisible to its parent — the spawn returned optimistically.
@@ -43,16 +43,6 @@ export type DeadNodeRow = Pick<NodeMeta, 'status' | 'intent' | 'pi_session_id' |
43
43
  * would drive a turn if the engine came back. It selects row 4b and defaults
44
44
  * to `true`, so an uninformed caller keeps the conservative respawn. */
45
45
  export declare function classifyDeadNode(row: DeadNodeRow | null, busy: boolean, pendingWake?: boolean): DeadNodeAction;
46
- /** An exit with uptime at or above this resets the consecutive-failure
47
- * counter before classification; a shorter-lived exit increments it. */
48
- export declare const HEALTHY_UPTIME_MS = 60000;
49
- /** At this many consecutive short-lived exits the respawn loop terminalizes:
50
- * transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
51
- export declare const MAX_RESPAWN_ATTEMPTS = 6;
52
- /** Backoff before the next respawn given the consecutive-failure count.
53
- * Zero failures → immediate (the sub-second yield respawn D-3 buys); the
54
- * n-th delayed attempt waits min(5s·2^n, 10min). */
55
- export declare function respawnBackoffMs(consecutiveFailures: number): number;
56
46
  export interface DaemonFleetOpts {
57
47
  /** This daemon's epoch id (randomBytes(8).toString('hex'), minted by
58
48
  * runDaemon after the ownership claim). */
@@ -78,16 +68,12 @@ export declare class DaemonFleet implements FleetRegistry {
78
68
  forget(nodeId: string): void;
79
69
  onChildExit(nodeId: string, status: FleetExitStatus): void;
80
70
  deliverExit(nodeId: string, status: FleetExitStatus): void;
81
- /** The node-lifecycle tick's enactment entry: classify a known-dead row and
82
- * enact immediately through the SAME policy table exit events use, so tick
83
- * recovery and exit recovery can never diverge. */
71
+ /** The node-lifecycle tick's enactment entry. */
84
72
  applyDeadRowPolicy(nodeId: string): Promise<void>;
85
73
  /** A revive refusal before its launch try/catch leaves the row active but
86
74
  * with no fleet entry. Convert that otherwise-silent strand into the same
87
75
  * terminal outcome as a boot failure. The fresh read avoids duplicating
88
76
  * reviveNode's launch-failure crash path. */
89
77
  terminalizeIfStillNonTerminal(nodeId: string, event: string, error: unknown): Promise<boolean>;
90
- /** Daemon shutdown: cancel every pending backoff respawn. Live processes
91
- * are torn down by the daemon's fleet-teardown stage, not here. */
92
78
  dispose(): void;
93
79
  }
@@ -1,32 +1,31 @@
1
1
  // src/daemon/fleet.ts — the daemon's broker fleet: the ChildProcess-handle
2
2
  // registry (the ONE normal liveness authority), the exit-policy classifier
3
- // (invariant-5 policy table), and the event-driven respawn throttle.
3
+ // (invariant-5 policy table), and durable respawn recovery.
4
4
  //
5
5
  // Constructed by runDaemon with the serial work lane's `enqueue` and this
6
6
  // daemon's epoch id, then injected into core via `bindFleet` BEFORE the API
7
- // server or any tick starts. Exit events never run policy inline on the Node
8
- // event callback — they enqueue a job on the serial lane, so policy never
9
- // interleaves with a tick's row reads (design D-3).
7
+ // server or any tick starts. Exit events enqueue observation only; the durable
8
+ // row tick owns recovery policy.
10
9
  //
11
10
  // Row state (status/intent/busy marker/session identity), never exit codes,
12
11
  // is the policy input (design D-4): the stophook writes the exit's explanation
13
12
  // before every deliberate exit, and that durable state is robust to every kill
14
- // path. Codes/signals are logged for diagnostics only. The single timing
15
- // input is uptime, used only by the throttle.
16
- import { getNode, subscribersOf } from '../core/canvas/index.js';
13
+ // path. Codes/signals are logged for diagnostics only.
14
+ import { getNode, getRow, subscribersOf } from '../core/canvas/index.js';
17
15
  import { fullName } from '../core/canvas/labels.js';
18
16
  import { isSafeNodeId } from '../core/canvas/paths.js';
19
17
  import { transition } from '../core/runtime/lifecycle.js';
20
18
  import { fanDoctrineWake } from '../core/runtime/close.js';
21
19
  import { hasCleanAbort, isBusy } from '../core/runtime/busy.js';
22
20
  import { hasPendingWake } from './reconcilers/live-obligation.js';
21
+ import { clearRespawnPolicy, MAX_RESPAWN_ATTEMPTS, recordRespawnPolicy, respawnBackoffMs } from './reconcilers/node-lifecycle/respawn-policy.js';
23
22
  import { reviveNode } from '../core/runtime/revive.js';
24
23
  import { pushUrgent } from '../core/feed/feed.js';
25
24
  import { emitEvent } from '../core/events/emit.js';
26
25
  import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
27
26
  // Shared crash helpers (moved from crtrd.ts — the fleet's policy paths are now
28
27
  // their primary consumers; crtrd imports them back for its surviving passes).
29
- /** Surface a vehicle that never booted (or exhausted its respawn budget).
28
+ /** Surface a vehicle that never booted.
30
29
  *
31
30
  * A pi that dies before its first session_start (so `pi_session_id` was never
32
31
  * recorded) is invisible to its parent — the spawn returned optimistically.
@@ -42,6 +41,12 @@ export async function surfaceBootFailure(meta) {
42
41
  `If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`;
43
42
  await pushUrgent(meta.node_id, body, { from: meta.node_id });
44
43
  }
44
+ async function surfaceCrashLoop(meta) {
45
+ const body = `⚠ Crash loop — \`${meta.name}\` (${meta.kind}) kept dying without settling a turn.\n\n` +
46
+ `Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.\n\n` +
47
+ `Fix the crash cause, then revive the node if the work still needs doing.`;
48
+ await pushUrgent(meta.node_id, body, { from: meta.node_id });
49
+ }
45
50
  /** Surface a recovery refusal that happened before a broker could launch. */
46
51
  async function surfaceRecoveryFailure(meta, error) {
47
52
  const message = error instanceof Error ? error.message : String(error);
@@ -110,23 +115,6 @@ export function classifyDeadNode(row, busy, pendingWake = true) {
110
115
  return 'respawn-fresh'; // row 7: bounded ordinary fork-birth retry
111
116
  return 'boot-failure'; // row 8
112
117
  }
113
- // Respawn throttle (replaces the stranded-relaunch counters, event-driven)
114
- /** An exit with uptime at or above this resets the consecutive-failure
115
- * counter before classification; a shorter-lived exit increments it. */
116
- export const HEALTHY_UPTIME_MS = 60_000;
117
- /** At this many consecutive short-lived exits the respawn loop terminalizes:
118
- * transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
119
- export const MAX_RESPAWN_ATTEMPTS = 6;
120
- const RESPAWN_BACKOFF_BASE_MS = 5_000;
121
- const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
122
- /** Backoff before the next respawn given the consecutive-failure count.
123
- * Zero failures → immediate (the sub-second yield respawn D-3 buys); the
124
- * n-th delayed attempt waits min(5s·2^n, 10min). */
125
- export function respawnBackoffMs(consecutiveFailures) {
126
- if (consecutiveFailures <= 0)
127
- return 0;
128
- return Math.min(RESPAWN_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESPAWN_BACKOFF_MAX_MS);
129
- }
130
118
  /** How long a capacity denial stays quiet before it is worth logging again. */
131
119
  const CAPACITY_LOG_THROTTLE_MS = 30_000;
132
120
  export class DaemonFleet {
@@ -134,7 +122,6 @@ export class DaemonFleet {
134
122
  #enqueue;
135
123
  #now;
136
124
  #map = new Map();
137
- #throttle = new Map();
138
125
  /** Slots claimed but not yet held by a registered broker: a launch in flight,
139
126
  * or a broker that exited and whose exit-policy job has not yet decided
140
127
  * whether to relaunch it. Counting both against the cap is what keeps a
@@ -156,21 +143,8 @@ export class DaemonFleet {
156
143
  if (handle.pid === null) {
157
144
  throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
158
145
  }
159
- // A live registration supersedes any scheduled backoff respawn: the node
160
- // is alive again (a manual revive won the race), so the pending respawn
161
- // must not fire a second launch.
162
- const throttle = this.#throttle.get(nodeId);
163
- if (throttle?.timer !== undefined) {
164
- clearTimeout(throttle.timer);
165
- throttle.timer = undefined;
166
- }
167
146
  this.#reservations.delete(nodeId);
168
- this.#map.set(nodeId, {
169
- pid: handle.pid,
170
- launchedAt: this.#now(),
171
- handle,
172
- consecutiveFailures: throttle?.failures ?? 0,
173
- });
147
+ this.#map.set(nodeId, { pid: handle.pid, handle });
174
148
  }
175
149
  has(nodeId) {
176
150
  return this.#map.has(nodeId);
@@ -212,28 +186,22 @@ export class DaemonFleet {
212
186
  forget(nodeId) {
213
187
  this.#map.delete(nodeId);
214
188
  this.#reservations.delete(nodeId);
215
- this.#clearThrottle(nodeId);
216
189
  }
217
190
  onChildExit(nodeId, status) {
218
191
  // Move the slot from the registry to a reservation in ONE synchronous
219
192
  // step: the exit-policy job about to be enqueued may relaunch this node,
220
- // and holding the slot for it is what keeps a 1:1 respawn from losing its
221
- // own capacity to a launcher that raced onto the lane first. `#applyPolicy`
222
- // releases it on every path that does not relaunch.
223
- const entry = this.#map.get(nodeId);
193
+ // and holding the slot through the exit observation prevents a concurrent
194
+ // launch from entering before the registry reflects the exit.
224
195
  this.#map.delete(nodeId);
225
196
  this.#reservations.add(nodeId);
226
- const launchedAt = entry?.launchedAt ?? null;
227
- this.#enqueue(() => this.#runExitPolicy(nodeId, status, launchedAt));
197
+ this.#enqueue(() => this.#runExitPolicy(nodeId, status));
228
198
  }
229
199
  deliverExit(nodeId, status) {
230
200
  this.onChildExit(nodeId, status);
231
201
  }
232
- /** The node-lifecycle tick's enactment entry: classify a known-dead row and
233
- * enact immediately through the SAME policy table exit events use, so tick
234
- * recovery and exit recovery can never diverge. */
202
+ /** The node-lifecycle tick's enactment entry. */
235
203
  async applyDeadRowPolicy(nodeId) {
236
- await this.#applyPolicy(nodeId, 'tick');
204
+ await this.#applyPolicy(nodeId);
237
205
  }
238
206
  /** A revive refusal before its launch try/catch leaves the row active but
239
207
  * with no fleet entry. Convert that otherwise-silent strand into the same
@@ -251,49 +219,17 @@ export class DaemonFleet {
251
219
  });
252
220
  return true;
253
221
  }
254
- /** Daemon shutdown: cancel every pending backoff respawn. Live processes
255
- * are torn down by the daemon's fleet-teardown stage, not here. */
256
- dispose() {
257
- for (const state of this.#throttle.values()) {
258
- if (state.timer !== undefined) {
259
- clearTimeout(state.timer);
260
- state.timer = undefined;
261
- }
262
- }
263
- }
264
- #clearThrottle(nodeId) {
265
- const state = this.#throttle.get(nodeId);
266
- if (state?.timer !== undefined)
267
- clearTimeout(state.timer);
268
- this.#throttle.delete(nodeId);
269
- }
270
- async #runExitPolicy(nodeId, status, launchedAt) {
271
- const uptimeMs = launchedAt === null ? null : Math.max(0, this.#now() - launchedAt);
272
- // A newer launch may own this node by the time the lane reaches this job
273
- // (an API revive raced the exit event onto the lane). The exit belonged to
274
- // the OLD process; the node is alive — nothing to recover, but preserve the
275
- // observation without charging the fresh launch's throttle budget.
276
- if (this.#map.has(nodeId)) {
277
- this.#emitExitObserved(nodeId, status, uptimeMs, this.#throttle.get(nodeId)?.failures ?? 0);
278
- return;
279
- }
280
- const state = this.#throttle.get(nodeId) ?? { failures: 0 };
281
- if (state.timer !== undefined) {
282
- clearTimeout(state.timer);
283
- state.timer = undefined;
284
- }
285
- // Healthy uptime resets the counter BEFORE classification; a short-lived
286
- // exit (or a synthetic exit with no known launch time) increments it.
287
- if (uptimeMs !== null && uptimeMs >= HEALTHY_UPTIME_MS) {
288
- state.failures = 0;
289
- this.#throttle.delete(nodeId);
222
+ dispose() { }
223
+ async #runExitPolicy(nodeId, status) {
224
+ try {
225
+ const launchedAt = getNode(nodeId)?.launched_at;
226
+ const launchedMs = launchedAt == null ? Number.NaN : Date.parse(launchedAt);
227
+ const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
228
+ this.#emitExitObserved(nodeId, status, uptimeMs, getNode(nodeId)?.respawn_failures ?? 0);
290
229
  }
291
- else {
292
- state.failures += 1;
293
- this.#throttle.set(nodeId, state);
230
+ finally {
231
+ this.releaseReservation(nodeId);
294
232
  }
295
- this.#emitExitObserved(nodeId, status, uptimeMs, state.failures);
296
- await this.#applyPolicy(nodeId, 'exit');
297
233
  }
298
234
  #emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures) {
299
235
  emitEvent({
@@ -309,19 +245,10 @@ export class DaemonFleet {
309
245
  },
310
246
  });
311
247
  }
312
- async #applyPolicy(nodeId, origin) {
313
- try {
314
- return await this.#decideAndEnact(nodeId, origin);
315
- }
316
- finally {
317
- // The reservation this node has held since its exit is either consumed by
318
- // a successful `register` (making this a no-op) or given back here — the
319
- // one release path covering a declined policy, a deferred backoff, and
320
- // terminalization alike.
321
- this.releaseReservation(nodeId);
322
- }
248
+ async #applyPolicy(nodeId) {
249
+ return this.#decideAndEnact(nodeId);
323
250
  }
324
- async #decideAndEnact(nodeId, origin) {
251
+ async #decideAndEnact(nodeId) {
325
252
  const meta = getNode(nodeId);
326
253
  // Two durable witnesses mean the dead broker had a turn in flight. A dirty
327
254
  // death leaves the busy marker behind; a bounded graceful abort clears busy
@@ -332,19 +259,19 @@ export class DaemonFleet {
332
259
  switch (action) {
333
260
  case 'forget':
334
261
  // Fanouts for terminal exits already happened at the transition site.
335
- this.#clearThrottle(nodeId);
262
+ clearRespawnPolicy(nodeId);
336
263
  break;
337
264
  case 'dormant':
338
265
  // Chosen dormancy is not a failure — the counter does not survive it.
339
266
  // The pathological wake→fault→release→wake loop is bounded by the
340
- // tick's pass-2 cursor-keyed retry cap (D-11), not this throttle.
341
- this.#clearThrottle(nodeId);
267
+ // tick's pass-2 cursor-keyed retry cap (D-11), not this policy.
268
+ clearRespawnPolicy(nodeId);
342
269
  break;
343
270
  case 'release-idle':
344
271
  // Dormancy the daemon chose ON the node's behalf, and the same
345
272
  // non-failure: the row lands exactly where a stophook idle-release
346
273
  // would have left it, so pass-2 inbox wakes own its next revive.
347
- this.#clearThrottle(nodeId);
274
+ clearRespawnPolicy(nodeId);
348
275
  transition(nodeId, 'release');
349
276
  emitEvent({
350
277
  level: 'info',
@@ -352,7 +279,6 @@ export class DaemonFleet {
352
279
  ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
353
280
  fields: {
354
281
  ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
355
- origin,
356
282
  },
357
283
  });
358
284
  break;
@@ -366,51 +292,42 @@ export class DaemonFleet {
366
292
  case 'respawn-fresh':
367
293
  case 'respawn-cycle':
368
294
  case 'respawn-resume':
369
- await this.#respawn(nodeId, meta, action, origin, interruptedTurn);
295
+ await this.#respawn(nodeId, meta, action, interruptedTurn);
370
296
  break;
371
297
  }
372
298
  return action;
373
299
  }
374
- async #respawn(nodeId, meta, action, origin, interruptedTurn) {
375
- const state = this.#throttle.get(nodeId) ?? { failures: 0 };
376
- if (state.failures >= MAX_RESPAWN_ATTEMPTS) {
300
+ async #respawn(nodeId, meta, action, interruptedTurn) {
301
+ const row = getRow(nodeId);
302
+ if (row === null)
303
+ return;
304
+ const assess = row.respawn_not_before == null;
305
+ const priorFailures = row.respawn_failures ?? 0;
306
+ const attempt = assess
307
+ ? recordRespawnPolicy(row, this.#now())
308
+ : { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
309
+ if (attempt.exhausted) {
377
310
  await this.#terminalize(nodeId, meta, {
378
311
  event: 'broker.respawn.exhausted',
379
- wake: `Child crashed — ${fullName(meta)} (${nodeId}) kept dying shortly after launch (${state.failures} consecutive ` +
380
- `short-lived exits) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
312
+ wake: `Child crashed — ${fullName(meta)} (${nodeId}) kept dying without settling a turn (${attempt.failures} consecutive ` +
313
+ `launches) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
314
+ ...(meta.pi_session_id != null ? { surface: () => surfaceCrashLoop(meta) } : {}),
381
315
  });
382
316
  return;
383
317
  }
384
- // A backoff timer already owns this node's next respawn; a tick that
385
- // enacted the same policy meanwhile would defeat the throttle entirely.
386
- if (origin === 'tick' && state.timer !== undefined)
318
+ if (assess && attempt.notBefore !== null) {
319
+ emitEvent({
320
+ level: 'info',
321
+ event: 'broker.respawn.deferred',
322
+ ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
323
+ fields: {
324
+ ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
325
+ action,
326
+ backoff_ms: respawnBackoffMs(attempt.failures),
327
+ consecutive_failures: attempt.failures,
328
+ },
329
+ });
387
330
  return;
388
- // Only a fresh exit consults the backoff: a matured timer and a tick
389
- // enactment relaunch immediately (the timer already WAS the backoff, and a
390
- // tick candidate reached the lane only after its own gating).
391
- if (origin === 'exit') {
392
- const delay = respawnBackoffMs(state.failures);
393
- if (delay > 0) {
394
- state.timer = setTimeout(() => {
395
- state.timer = undefined;
396
- // A fresh policy job re-reads the row, so a node closed/canceled
397
- // during the backoff lands in table row 2 and the respawn evaporates.
398
- this.#enqueue(async () => { await this.#applyPolicy(nodeId, 'timer'); });
399
- }, delay);
400
- this.#throttle.set(nodeId, state);
401
- emitEvent({
402
- level: 'info',
403
- event: 'broker.respawn.deferred',
404
- ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
405
- fields: {
406
- ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
407
- action,
408
- backoff_ms: delay,
409
- consecutive_failures: state.failures,
410
- },
411
- });
412
- return;
413
- }
414
331
  }
415
332
  emitEvent({
416
333
  level: 'info',
@@ -419,8 +336,7 @@ export class DaemonFleet {
419
336
  fields: {
420
337
  ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
421
338
  action,
422
- origin,
423
- consecutive_failures: state.failures,
339
+ consecutive_failures: attempt.failures,
424
340
  },
425
341
  });
426
342
  try {
@@ -450,7 +366,6 @@ export class DaemonFleet {
450
366
  fields: {
451
367
  ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
452
368
  action,
453
- origin,
454
369
  },
455
370
  });
456
371
  // Launch failures terminalize inside reviveNode, but its preflight
@@ -459,10 +374,8 @@ export class DaemonFleet {
459
374
  await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err);
460
375
  }
461
376
  }
462
- /** Terminal outcome shared by table row 8 and throttle exhaustion:
463
- * transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
377
+ /** Terminal outcome shared by table row 8 and respawn exhaustion. */
464
378
  async #terminalize(nodeId, meta, opts) {
465
- this.#clearThrottle(nodeId);
466
379
  emitEvent({
467
380
  level: 'error',
468
381
  event: opts.event,
@@ -1,5 +1,6 @@
1
1
  import { ApiError } from '../../api/index.js';
2
2
  import { getRow, subscribersOf } from '../../core/canvas/canvas.js';
3
+ import { cancelCronsOnWake } from '../../core/canvas/crons.js';
3
4
  import { opaqueInboxTicketId, ticketDir } from '../../core/human/root.js';
4
5
  import { readTicketActionBinding } from '../../core/human/action-binding.js';
5
6
  import { completionEventFor, composeHumanCompletion } from '../../core/human/completion.js';
@@ -275,6 +276,7 @@ export async function deliverTerminalResult(ticketId) {
275
276
  data: { body },
276
277
  disposition: 'human-canceled',
277
278
  });
279
+ cancelCronsOnWake(subscriber.node_id);
278
280
  }
279
281
  catch {
280
282
  failedTo.push(subscriber.node_id);
@@ -46,7 +46,17 @@ export interface SpawnDaemonResult {
46
46
  pid?: number;
47
47
  /** PID of the already-running daemon, if it was already up. */
48
48
  existing_pid?: number;
49
+ /** Present only when a live pidfile owner does not answer `/healthz`. */
50
+ running?: true;
51
+ /** Present with `running:true` when the Unix API socket is not serving. */
52
+ serving?: false;
53
+ /** Recovery instruction for a live but non-serving daemon. */
54
+ next?: string;
49
55
  }
56
+ /** True only when the mandatory Unix API socket answers `/healthz`. This is
57
+ * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
58
+ * identifying the process an operator must stop, but it is not readiness. */
59
+ export declare function isDaemonServing(): Promise<boolean>;
50
60
  export interface DaemonWaitDeps {
51
61
  readPidfile?: () => number | null;
52
62
  isPidAlive?: (pid: number) => boolean;
@@ -58,6 +68,7 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
58
68
  code: number | null;
59
69
  signal: NodeJS.Signals | null;
60
70
  } | null;
71
+ isDaemonServing?: () => Promise<boolean>;
61
72
  }
62
73
  /** Tiny bounded post-spawn guard: wait for the daemon pidfile and live pid to
63
74
  * appear before reporting success. This catches the "started:true but not yet