@north-light/crouter 0.3.248 → 0.3.250

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/README.md +1 -1
  2. package/dist/api/__tests__/integration/client.test.js +28 -7
  3. package/dist/api/client.d.ts +1 -1
  4. package/dist/api/client.js +2 -2
  5. package/dist/api/dto/lifecycle.d.ts +3 -0
  6. package/dist/api/dto/messages.d.ts +4 -0
  7. package/dist/api/dto/nodes.d.ts +4 -0
  8. package/dist/api/dto/reports.d.ts +7 -2
  9. package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
  10. package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
  11. package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
  12. package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
  13. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
  14. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
  15. package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
  16. package/dist/clients/attach/session/chrome-refresh.js +1 -0
  17. package/dist/clients/attach/viewer.js +532 -532
  18. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
  19. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
  20. package/dist/clients/inbox/controller.js +32 -5
  21. package/dist/clients/inbox/tui/ansi.d.ts +3 -1
  22. package/dist/clients/inbox/tui/ansi.js +14 -10
  23. package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
  24. package/dist/commands/api-client.d.ts +2 -2
  25. package/dist/commands/api-client.js +5 -5
  26. package/dist/commands/canvas-browse.js +1 -1
  27. package/dist/commands/node/create.js +3 -1
  28. package/dist/commands/node/inspect.js +1 -0
  29. package/dist/commands/node/lifecycle.js +1 -1
  30. package/dist/commands/node/message.js +6 -4
  31. package/dist/commands/node/wait.js +1 -1
  32. package/dist/commands/push.js +36 -37
  33. package/dist/commands/sys/daemon.js +21 -8
  34. package/dist/commands/sys/setup-core.js +2 -2
  35. package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
  36. package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
  37. package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
  38. package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
  39. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +5 -1
  40. package/dist/core/__tests__/child-death-wake.test.js +11 -7
  41. package/dist/core/__tests__/collectible.test.d.ts +1 -0
  42. package/dist/core/__tests__/collectible.test.js +88 -0
  43. package/dist/core/__tests__/context-intro.test.js +4 -41
  44. package/dist/core/__tests__/daemon-boot.test.js +30 -14
  45. package/dist/core/__tests__/daemon-wedge.test.js +25 -19
  46. package/dist/core/__tests__/fixtures/fake-engine.js +11 -0
  47. package/dist/core/__tests__/human-deliver.test.js +5 -5
  48. package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
  49. package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
  50. package/dist/core/__tests__/integration/revive.test.js +48 -0
  51. package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
  52. package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
  53. package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
  54. package/dist/core/__tests__/migration.test.js +30 -0
  55. package/dist/core/__tests__/passive-subscription.test.js +39 -1
  56. package/dist/core/__tests__/prune.test.js +12 -3
  57. package/dist/core/__tests__/relaunch-root.test.js +1 -1
  58. package/dist/core/__tests__/respawn-throttle.test.js +16 -146
  59. package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
  60. package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
  61. package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
  62. package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
  63. package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
  64. package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
  65. package/dist/core/canvas/canvas.d.ts +9 -15
  66. package/dist/core/canvas/canvas.js +41 -47
  67. package/dist/core/canvas/crons.d.ts +5 -8
  68. package/dist/core/canvas/crons.js +5 -8
  69. package/dist/core/canvas/history.d.ts +5 -0
  70. package/dist/core/canvas/history.js +12 -1
  71. package/dist/core/canvas/migrations.js +58 -1
  72. package/dist/core/canvas/pid.d.ts +1 -1
  73. package/dist/core/canvas/pid.js +5 -2
  74. package/dist/core/canvas/types.d.ts +4 -0
  75. package/dist/core/feed/feed.d.ts +4 -0
  76. package/dist/core/feed/feed.js +13 -8
  77. package/dist/core/feed/inbox.d.ts +2 -1
  78. package/dist/core/feed/inbox.js +22 -3
  79. package/dist/core/git.d.ts +3 -0
  80. package/dist/core/git.js +3 -1
  81. package/dist/core/human/feedback-companion.js +1 -0
  82. package/dist/core/preview-registry.js +0 -1
  83. package/dist/core/review/companion.js +1 -0
  84. package/dist/core/review/realize.js +1 -0
  85. package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
  86. package/dist/core/runtime/broker/auth-reload.js +9 -3
  87. package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
  88. package/dist/core/runtime/broker/engine-drive.js +28 -23
  89. package/dist/core/runtime/broker/event-projection.d.ts +4 -0
  90. package/dist/core/runtime/broker/event-projection.js +43 -23
  91. package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
  92. package/dist/core/runtime/broker/fault-retry.js +94 -8
  93. package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
  94. package/dist/core/runtime/broker/frame-dispatch.js +4 -1
  95. package/dist/core/runtime/broker/inbox.js +4 -0
  96. package/dist/core/runtime/broker/rebind.js +18 -0
  97. package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
  98. package/dist/core/runtime/broker/turn-admission.js +31 -0
  99. package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
  100. package/dist/core/runtime/broker/turn-ignition.js +25 -15
  101. package/dist/core/runtime/broker.js +21 -2
  102. package/dist/core/runtime/close.d.ts +11 -9
  103. package/dist/core/runtime/close.js +23 -19
  104. package/dist/core/runtime/fault.d.ts +21 -0
  105. package/dist/core/runtime/fault.js +129 -19
  106. package/dist/core/runtime/fleet.d.ts +0 -6
  107. package/dist/core/runtime/host.d.ts +7 -1
  108. package/dist/core/runtime/host.js +20 -2
  109. package/dist/core/runtime/nodes.d.ts +5 -1
  110. package/dist/core/runtime/nodes.js +24 -1
  111. package/dist/core/runtime/placement.d.ts +6 -6
  112. package/dist/core/runtime/placement.js +15 -8
  113. package/dist/core/runtime/reset.js +2 -2
  114. package/dist/core/runtime/revive.js +40 -4
  115. package/dist/core/runtime/spawn.d.ts +3 -0
  116. package/dist/core/runtime/spawn.js +1 -0
  117. package/dist/core/runtime/warm-pool.d.ts +4 -0
  118. package/dist/core/runtime/warm-pool.js +2 -0
  119. package/dist/core/worktree.d.ts +4 -3
  120. package/dist/core/worktree.js +73 -27
  121. package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
  122. package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
  123. package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
  124. package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
  125. package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
  126. package/dist/daemon/api/handlers/bash-jobs.js +2 -0
  127. package/dist/daemon/api/handlers/broker-ops.js +23 -2
  128. package/dist/daemon/api/handlers/human.js +1 -1
  129. package/dist/daemon/api/handlers/messages.js +11 -2
  130. package/dist/daemon/api/handlers/nodes.js +13 -1
  131. package/dist/daemon/api/handlers/reports.js +14 -6
  132. package/dist/daemon/api/map.js +1 -0
  133. package/dist/daemon/api/server.d.ts +9 -12
  134. package/dist/daemon/api/server.js +30 -19
  135. package/dist/daemon/companion-retire.js +3 -9
  136. package/dist/daemon/cron/sinks.js +7 -2
  137. package/dist/daemon/crtrd.js +4 -1
  138. package/dist/daemon/fleet.d.ts +2 -16
  139. package/dist/daemon/fleet.js +63 -150
  140. package/dist/daemon/human/finish.js +2 -0
  141. package/dist/daemon/manage.d.ts +11 -0
  142. package/dist/daemon/manage.js +32 -3
  143. package/dist/daemon/messaging/node-message.js +7 -1
  144. package/dist/daemon/reconcilers/bash-deadline.js +2 -0
  145. package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
  146. package/dist/daemon/reconcilers/broker-supervision.js +22 -9
  147. package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
  148. package/dist/daemon/reconcilers/live-obligation.js +18 -6
  149. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
  150. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
  151. package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
  152. package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
  153. package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
  154. package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
  155. package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
  156. package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
  157. package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
  158. package/dist/pi-extensions/canvas-context-intro.js +4 -76
  159. package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
  160. package/dist/pi-extensions/canvas-review-boundary.js +4 -29
  161. package/dist/pi-extensions/canvas-stophook.js +1 -6
  162. package/dist/shared/birth-announcement.d.ts +12 -0
  163. package/dist/shared/birth-announcement.js +25 -0
  164. package/dist/shared/generated-context.d.ts +8 -4
  165. package/dist/shared/generated-context.js +15 -9
  166. package/package.json +4 -4
  167. package/runtime.lock.json +2 -2
  168. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
  169. package/dist/clients/inbox/tui/clipboard.d.ts +0 -2
  170. package/dist/clients/inbox/tui/clipboard.js +0 -26
  171. package/dist/pi-extensions/truncate.d.ts +0 -5
  172. package/dist/pi-extensions/truncate.js +0 -14
@@ -1,32 +1,31 @@
1
1
  // src/daemon/fleet.ts — the daemon's broker fleet: the ChildProcess-handle
2
2
  // registry (the ONE normal liveness authority), the exit-policy classifier
3
- // (invariant-5 policy table), and the event-driven respawn throttle.
3
+ // (invariant-5 policy table), and durable respawn recovery.
4
4
  //
5
5
  // Constructed by runDaemon with the serial work lane's `enqueue` and this
6
6
  // daemon's epoch id, then injected into core via `bindFleet` BEFORE the API
7
- // server or any tick starts. Exit events never run policy inline on the Node
8
- // event callback — they enqueue a job on the serial lane, so policy never
9
- // interleaves with a tick's row reads (design D-3).
7
+ // server or any tick starts. Exit events enqueue observation only; the durable
8
+ // row tick owns recovery policy.
10
9
  //
11
10
  // Row state (status/intent/busy marker/session identity), never exit codes,
12
11
  // is the policy input (design D-4): the stophook writes the exit's explanation
13
12
  // before every deliberate exit, and that durable state is robust to every kill
14
- // path. Codes/signals are logged for diagnostics only. The single timing
15
- // input is uptime, used only by the throttle.
16
- import { getNode, subscribersOf } from '../core/canvas/index.js';
13
+ // path. Codes/signals are logged for diagnostics only.
14
+ import { getNode, getRow, subscribersOf } from '../core/canvas/index.js';
17
15
  import { fullName } from '../core/canvas/labels.js';
18
16
  import { isSafeNodeId } from '../core/canvas/paths.js';
19
17
  import { transition } from '../core/runtime/lifecycle.js';
20
18
  import { fanDoctrineWake } from '../core/runtime/close.js';
21
19
  import { hasCleanAbort, isBusy } from '../core/runtime/busy.js';
22
20
  import { hasPendingWake } from './reconcilers/live-obligation.js';
21
+ import { clearRespawnPolicy, MAX_RESPAWN_ATTEMPTS, recordRespawnPolicy, respawnBackoffMs } from './reconcilers/node-lifecycle/respawn-policy.js';
23
22
  import { reviveNode } from '../core/runtime/revive.js';
24
23
  import { pushUrgent } from '../core/feed/feed.js';
25
24
  import { emitEvent } from '../core/events/emit.js';
26
25
  import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
27
26
  // Shared crash helpers (moved from crtrd.ts — the fleet's policy paths are now
28
27
  // their primary consumers; crtrd imports them back for its surviving passes).
29
- /** Surface a vehicle that never booted (or exhausted its respawn budget).
28
+ /** Surface a vehicle that never booted.
30
29
  *
31
30
  * A pi that dies before its first session_start (so `pi_session_id` was never
32
31
  * recorded) is invisible to its parent — the spawn returned optimistically.
@@ -42,6 +41,12 @@ export async function surfaceBootFailure(meta) {
42
41
  `If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`;
43
42
  await pushUrgent(meta.node_id, body, { from: meta.node_id });
44
43
  }
44
+ async function surfaceCrashLoop(meta) {
45
+ const body = `⚠ Crash loop — \`${meta.name}\` (${meta.kind}) kept dying without settling a turn.\n\n` +
46
+ `Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.\n\n` +
47
+ `Fix the crash cause, then revive the node if the work still needs doing.`;
48
+ await pushUrgent(meta.node_id, body, { from: meta.node_id });
49
+ }
45
50
  /** Surface a recovery refusal that happened before a broker could launch. */
46
51
  async function surfaceRecoveryFailure(meta, error) {
47
52
  const message = error instanceof Error ? error.message : String(error);
@@ -110,23 +115,6 @@ export function classifyDeadNode(row, busy, pendingWake = true) {
110
115
  return 'respawn-fresh'; // row 7: bounded ordinary fork-birth retry
111
116
  return 'boot-failure'; // row 8
112
117
  }
113
- // Respawn throttle (replaces the stranded-relaunch counters, event-driven)
114
- /** An exit with uptime at or above this resets the consecutive-failure
115
- * counter before classification; a shorter-lived exit increments it. */
116
- export const HEALTHY_UPTIME_MS = 60_000;
117
- /** At this many consecutive short-lived exits the respawn loop terminalizes:
118
- * transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
119
- export const MAX_RESPAWN_ATTEMPTS = 6;
120
- const RESPAWN_BACKOFF_BASE_MS = 5_000;
121
- const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
122
- /** Backoff before the next respawn given the consecutive-failure count.
123
- * Zero failures → immediate (the sub-second yield respawn D-3 buys); the
124
- * n-th delayed attempt waits min(5s·2^n, 10min). */
125
- export function respawnBackoffMs(consecutiveFailures) {
126
- if (consecutiveFailures <= 0)
127
- return 0;
128
- return Math.min(RESPAWN_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESPAWN_BACKOFF_MAX_MS);
129
- }
130
118
  /** How long a capacity denial stays quiet before it is worth logging again. */
131
119
  const CAPACITY_LOG_THROTTLE_MS = 30_000;
132
120
  export class DaemonFleet {
@@ -134,7 +122,6 @@ export class DaemonFleet {
134
122
  #enqueue;
135
123
  #now;
136
124
  #map = new Map();
137
- #throttle = new Map();
138
125
  /** Slots claimed but not yet held by a registered broker: a launch in flight,
139
126
  * or a broker that exited and whose exit-policy job has not yet decided
140
127
  * whether to relaunch it. Counting both against the cap is what keeps a
@@ -156,21 +143,8 @@ export class DaemonFleet {
156
143
  if (handle.pid === null) {
157
144
  throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
158
145
  }
159
- // A live registration supersedes any scheduled backoff respawn: the node
160
- // is alive again (a manual revive won the race), so the pending respawn
161
- // must not fire a second launch.
162
- const throttle = this.#throttle.get(nodeId);
163
- if (throttle?.timer !== undefined) {
164
- clearTimeout(throttle.timer);
165
- throttle.timer = undefined;
166
- }
167
146
  this.#reservations.delete(nodeId);
168
- this.#map.set(nodeId, {
169
- pid: handle.pid,
170
- launchedAt: this.#now(),
171
- handle,
172
- consecutiveFailures: throttle?.failures ?? 0,
173
- });
147
+ this.#map.set(nodeId, { pid: handle.pid, handle });
174
148
  }
175
149
  has(nodeId) {
176
150
  return this.#map.has(nodeId);
@@ -212,28 +186,22 @@ export class DaemonFleet {
212
186
  forget(nodeId) {
213
187
  this.#map.delete(nodeId);
214
188
  this.#reservations.delete(nodeId);
215
- this.#clearThrottle(nodeId);
216
189
  }
217
190
  onChildExit(nodeId, status) {
218
191
  // Move the slot from the registry to a reservation in ONE synchronous
219
192
  // step: the exit-policy job about to be enqueued may relaunch this node,
220
- // and holding the slot for it is what keeps a 1:1 respawn from losing its
221
- // own capacity to a launcher that raced onto the lane first. `#applyPolicy`
222
- // releases it on every path that does not relaunch.
223
- const entry = this.#map.get(nodeId);
193
+ // and holding the slot through the exit observation prevents a concurrent
194
+ // launch from entering before the registry reflects the exit.
224
195
  this.#map.delete(nodeId);
225
196
  this.#reservations.add(nodeId);
226
- const launchedAt = entry?.launchedAt ?? null;
227
- this.#enqueue(() => this.#runExitPolicy(nodeId, status, launchedAt));
197
+ this.#enqueue(() => this.#runExitPolicy(nodeId, status));
228
198
  }
229
199
  deliverExit(nodeId, status) {
230
200
  this.onChildExit(nodeId, status);
231
201
  }
232
- /** The node-lifecycle tick's enactment entry: classify a known-dead row and
233
- * enact immediately through the SAME policy table exit events use, so tick
234
- * recovery and exit recovery can never diverge. */
202
+ /** The node-lifecycle tick's enactment entry. */
235
203
  async applyDeadRowPolicy(nodeId) {
236
- await this.#applyPolicy(nodeId, 'tick');
204
+ await this.#applyPolicy(nodeId);
237
205
  }
238
206
  /** A revive refusal before its launch try/catch leaves the row active but
239
207
  * with no fleet entry. Convert that otherwise-silent strand into the same
@@ -251,49 +219,17 @@ export class DaemonFleet {
251
219
  });
252
220
  return true;
253
221
  }
254
- /** Daemon shutdown: cancel every pending backoff respawn. Live processes
255
- * are torn down by the daemon's fleet-teardown stage, not here. */
256
- dispose() {
257
- for (const state of this.#throttle.values()) {
258
- if (state.timer !== undefined) {
259
- clearTimeout(state.timer);
260
- state.timer = undefined;
261
- }
262
- }
263
- }
264
- #clearThrottle(nodeId) {
265
- const state = this.#throttle.get(nodeId);
266
- if (state?.timer !== undefined)
267
- clearTimeout(state.timer);
268
- this.#throttle.delete(nodeId);
269
- }
270
- async #runExitPolicy(nodeId, status, launchedAt) {
271
- const uptimeMs = launchedAt === null ? null : Math.max(0, this.#now() - launchedAt);
272
- // A newer launch may own this node by the time the lane reaches this job
273
- // (an API revive raced the exit event onto the lane). The exit belonged to
274
- // the OLD process; the node is alive — nothing to recover, but preserve the
275
- // observation without charging the fresh launch's throttle budget.
276
- if (this.#map.has(nodeId)) {
277
- this.#emitExitObserved(nodeId, status, uptimeMs, this.#throttle.get(nodeId)?.failures ?? 0);
278
- return;
279
- }
280
- const state = this.#throttle.get(nodeId) ?? { failures: 0 };
281
- if (state.timer !== undefined) {
282
- clearTimeout(state.timer);
283
- state.timer = undefined;
284
- }
285
- // Healthy uptime resets the counter BEFORE classification; a short-lived
286
- // exit (or a synthetic exit with no known launch time) increments it.
287
- if (uptimeMs !== null && uptimeMs >= HEALTHY_UPTIME_MS) {
288
- state.failures = 0;
289
- this.#throttle.delete(nodeId);
222
+ dispose() { }
223
+ async #runExitPolicy(nodeId, status) {
224
+ try {
225
+ const launchedAt = getNode(nodeId)?.launched_at;
226
+ const launchedMs = launchedAt == null ? Number.NaN : Date.parse(launchedAt);
227
+ const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
228
+ this.#emitExitObserved(nodeId, status, uptimeMs, getNode(nodeId)?.respawn_failures ?? 0);
290
229
  }
291
- else {
292
- state.failures += 1;
293
- this.#throttle.set(nodeId, state);
230
+ finally {
231
+ this.releaseReservation(nodeId);
294
232
  }
295
- this.#emitExitObserved(nodeId, status, uptimeMs, state.failures);
296
- await this.#applyPolicy(nodeId, 'exit');
297
233
  }
298
234
  #emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures) {
299
235
  emitEvent({
@@ -309,19 +245,10 @@ export class DaemonFleet {
309
245
  },
310
246
  });
311
247
  }
312
- async #applyPolicy(nodeId, origin) {
313
- try {
314
- return await this.#decideAndEnact(nodeId, origin);
315
- }
316
- finally {
317
- // The reservation this node has held since its exit is either consumed by
318
- // a successful `register` (making this a no-op) or given back here — the
319
- // one release path covering a declined policy, a deferred backoff, and
320
- // terminalization alike.
321
- this.releaseReservation(nodeId);
322
- }
248
+ async #applyPolicy(nodeId) {
249
+ return this.#decideAndEnact(nodeId);
323
250
  }
324
- async #decideAndEnact(nodeId, origin) {
251
+ async #decideAndEnact(nodeId) {
325
252
  const meta = getNode(nodeId);
326
253
  // Two durable witnesses mean the dead broker had a turn in flight. A dirty
327
254
  // death leaves the busy marker behind; a bounded graceful abort clears busy
@@ -332,19 +259,19 @@ export class DaemonFleet {
332
259
  switch (action) {
333
260
  case 'forget':
334
261
  // Fanouts for terminal exits already happened at the transition site.
335
- this.#clearThrottle(nodeId);
262
+ clearRespawnPolicy(nodeId);
336
263
  break;
337
264
  case 'dormant':
338
265
  // Chosen dormancy is not a failure — the counter does not survive it.
339
266
  // The pathological wake→fault→release→wake loop is bounded by the
340
- // tick's pass-2 cursor-keyed retry cap (D-11), not this throttle.
341
- this.#clearThrottle(nodeId);
267
+ // tick's pass-2 cursor-keyed retry cap (D-11), not this policy.
268
+ clearRespawnPolicy(nodeId);
342
269
  break;
343
270
  case 'release-idle':
344
271
  // Dormancy the daemon chose ON the node's behalf, and the same
345
272
  // non-failure: the row lands exactly where a stophook idle-release
346
273
  // would have left it, so pass-2 inbox wakes own its next revive.
347
- this.#clearThrottle(nodeId);
274
+ clearRespawnPolicy(nodeId);
348
275
  transition(nodeId, 'release');
349
276
  emitEvent({
350
277
  level: 'info',
@@ -352,7 +279,6 @@ export class DaemonFleet {
352
279
  ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
353
280
  fields: {
354
281
  ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
355
- origin,
356
282
  },
357
283
  });
358
284
  break;
@@ -366,51 +292,42 @@ export class DaemonFleet {
366
292
  case 'respawn-fresh':
367
293
  case 'respawn-cycle':
368
294
  case 'respawn-resume':
369
- await this.#respawn(nodeId, meta, action, origin, interruptedTurn);
295
+ await this.#respawn(nodeId, meta, action, interruptedTurn);
370
296
  break;
371
297
  }
372
298
  return action;
373
299
  }
374
- async #respawn(nodeId, meta, action, origin, interruptedTurn) {
375
- const state = this.#throttle.get(nodeId) ?? { failures: 0 };
376
- if (state.failures >= MAX_RESPAWN_ATTEMPTS) {
300
+ async #respawn(nodeId, meta, action, interruptedTurn) {
301
+ const row = getRow(nodeId);
302
+ if (row === null)
303
+ return;
304
+ const assess = row.respawn_not_before == null;
305
+ const priorFailures = row.respawn_failures ?? 0;
306
+ const attempt = assess
307
+ ? recordRespawnPolicy(row, this.#now())
308
+ : { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
309
+ if (attempt.exhausted) {
377
310
  await this.#terminalize(nodeId, meta, {
378
311
  event: 'broker.respawn.exhausted',
379
- wake: `Child crashed — ${fullName(meta)} (${nodeId}) kept dying shortly after launch (${state.failures} consecutive ` +
380
- `short-lived exits) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
312
+ wake: `Child crashed — ${fullName(meta)} (${nodeId}) kept dying without settling a turn (${attempt.failures} consecutive ` +
313
+ `launches) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
314
+ ...(meta.pi_session_id != null ? { surface: () => surfaceCrashLoop(meta) } : {}),
381
315
  });
382
316
  return;
383
317
  }
384
- // A backoff timer already owns this node's next respawn; a tick that
385
- // enacted the same policy meanwhile would defeat the throttle entirely.
386
- if (origin === 'tick' && state.timer !== undefined)
318
+ if (assess && attempt.notBefore !== null) {
319
+ emitEvent({
320
+ level: 'info',
321
+ event: 'broker.respawn.deferred',
322
+ ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
323
+ fields: {
324
+ ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
325
+ action,
326
+ backoff_ms: respawnBackoffMs(attempt.failures),
327
+ consecutive_failures: attempt.failures,
328
+ },
329
+ });
387
330
  return;
388
- // Only a fresh exit consults the backoff: a matured timer and a tick
389
- // enactment relaunch immediately (the timer already WAS the backoff, and a
390
- // tick candidate reached the lane only after its own gating).
391
- if (origin === 'exit') {
392
- const delay = respawnBackoffMs(state.failures);
393
- if (delay > 0) {
394
- state.timer = setTimeout(() => {
395
- state.timer = undefined;
396
- // A fresh policy job re-reads the row, so a node closed/canceled
397
- // during the backoff lands in table row 2 and the respawn evaporates.
398
- this.#enqueue(async () => { await this.#applyPolicy(nodeId, 'timer'); });
399
- }, delay);
400
- this.#throttle.set(nodeId, state);
401
- emitEvent({
402
- level: 'info',
403
- event: 'broker.respawn.deferred',
404
- ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
405
- fields: {
406
- ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
407
- action,
408
- backoff_ms: delay,
409
- consecutive_failures: state.failures,
410
- },
411
- });
412
- return;
413
- }
414
331
  }
415
332
  emitEvent({
416
333
  level: 'info',
@@ -419,8 +336,7 @@ export class DaemonFleet {
419
336
  fields: {
420
337
  ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
421
338
  action,
422
- origin,
423
- consecutive_failures: state.failures,
339
+ consecutive_failures: attempt.failures,
424
340
  },
425
341
  });
426
342
  try {
@@ -450,7 +366,6 @@ export class DaemonFleet {
450
366
  fields: {
451
367
  ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
452
368
  action,
453
- origin,
454
369
  },
455
370
  });
456
371
  // Launch failures terminalize inside reviveNode, but its preflight
@@ -459,10 +374,8 @@ export class DaemonFleet {
459
374
  await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err);
460
375
  }
461
376
  }
462
- /** Terminal outcome shared by table row 8 and throttle exhaustion:
463
- * transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
377
+ /** Terminal outcome shared by table row 8 and respawn exhaustion. */
464
378
  async #terminalize(nodeId, meta, opts) {
465
- this.#clearThrottle(nodeId);
466
379
  emitEvent({
467
380
  level: 'error',
468
381
  event: opts.event,
@@ -1,5 +1,6 @@
1
1
  import { ApiError } from '../../api/index.js';
2
2
  import { getRow, subscribersOf } from '../../core/canvas/canvas.js';
3
+ import { cancelCronsOnWake } from '../../core/canvas/crons.js';
3
4
  import { opaqueInboxTicketId, ticketDir } from '../../core/human/root.js';
4
5
  import { readTicketActionBinding } from '../../core/human/action-binding.js';
5
6
  import { completionEventFor, composeHumanCompletion } from '../../core/human/completion.js';
@@ -275,6 +276,7 @@ export async function deliverTerminalResult(ticketId) {
275
276
  data: { body },
276
277
  disposition: 'human-canceled',
277
278
  });
279
+ cancelCronsOnWake(subscriber.node_id);
278
280
  }
279
281
  catch {
280
282
  failedTo.push(subscriber.node_id);
@@ -46,7 +46,17 @@ export interface SpawnDaemonResult {
46
46
  pid?: number;
47
47
  /** PID of the already-running daemon, if it was already up. */
48
48
  existing_pid?: number;
49
+ /** Present only when a live pidfile owner does not answer `/healthz`. */
50
+ running?: true;
51
+ /** Present with `running:true` when the Unix API socket is not serving. */
52
+ serving?: false;
53
+ /** Recovery instruction for a live but non-serving daemon. */
54
+ next?: string;
49
55
  }
56
+ /** True only when the mandatory Unix API socket answers `/healthz`. This is
57
+ * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
58
+ * identifying the process an operator must stop, but it is not readiness. */
59
+ export declare function isDaemonServing(): Promise<boolean>;
50
60
  export interface DaemonWaitDeps {
51
61
  readPidfile?: () => number | null;
52
62
  isPidAlive?: (pid: number) => boolean;
@@ -58,6 +68,7 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
58
68
  code: number | null;
59
69
  signal: NodeJS.Signals | null;
60
70
  } | null;
71
+ isDaemonServing?: () => Promise<boolean>;
61
72
  }
62
73
  /** Tiny bounded post-spawn guard: wait for the daemon pidfile and live pid to
63
74
  * appear before reporting success. This catches the "started:true but not yet
@@ -16,6 +16,7 @@ import { hostExecPath } from '../core/runtime/branded-host.js';
16
16
  import { isDaemonRunning, readPidfile, isPidAlive } from './pidfile.js';
17
17
  import { DAEMON_STARTUP_WINDOW_MS } from './startup-policy.js';
18
18
  import { envNoDaemonAutostart } from '../shared/env.js';
19
+ import { CrtrClient } from '../api/index.js';
19
20
  // Daemon env sanitization
20
21
  /** Env keys that must NEVER reach the daemon process. Restarting crtrd is
21
22
  * overwhelmingly done from inside an agent node's own bash tool (`crtr sys
@@ -201,6 +202,24 @@ export async function execDaemon() {
201
202
  // Shutdown stays short even though startup may include corpus migration.
202
203
  const DAEMON_SHUTDOWN_WINDOW_MS = 20_000;
203
204
  const DAEMON_VERIFY_POLL_MS = 10;
205
+ /** A status/start readiness probe must not hang behind a daemon that accepts a
206
+ * Unix connection but never answers it. */
207
+ const DAEMON_HEALTH_PROBE_TIMEOUT_MS = 1_000;
208
+ /** True only when the mandatory Unix API socket answers `/healthz`. This is
209
+ * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
210
+ * identifying the process an operator must stop, but it is not readiness. */
211
+ export async function isDaemonServing() {
212
+ try {
213
+ const health = await CrtrClient.forLocalSocket({
214
+ autostart: false,
215
+ timeoutMs: DAEMON_HEALTH_PROBE_TIMEOUT_MS,
216
+ }).healthz();
217
+ return health.ok;
218
+ }
219
+ catch {
220
+ return false;
221
+ }
222
+ }
204
223
  function sleepMs(ms) {
205
224
  return new Promise((resolve) => {
206
225
  setTimeout(resolve, ms);
@@ -223,12 +242,13 @@ export async function verifyDaemonStartup(pid, timeoutMs = DAEMON_STARTUP_WINDOW
223
242
  const read = deps.readPidfile ?? readPidfile;
224
243
  const alive = deps.isPidAlive ?? isPidAlive;
225
244
  const exited = deps.childExited;
245
+ const serving = deps.isDaemonServing ?? isDaemonServing;
226
246
  const sleep = deps.sleepMs ?? sleepMs;
227
247
  const now = deps.now ?? Date.now;
228
248
  const deadline = now() + timeoutMs;
229
249
  while (now() <= deadline) {
230
250
  const owner = read();
231
- if (owner !== null && alive(owner))
251
+ if (owner !== null && alive(owner) && await serving())
232
252
  return owner === pid ? null : owner;
233
253
  const exitState = exited?.();
234
254
  if (exitState !== null && exitState !== undefined) {
@@ -314,8 +334,17 @@ export async function stopDaemonProcess(pid, timeoutMs = DAEMON_SHUTDOWN_WINDOW_
314
334
  * If the daemon is already running, returns {started:false, existing_pid}.
315
335
  * If spawning fails (e.g. missing dist — run `npm run build` first), throws. */
316
336
  export async function spawnDaemon() {
317
- if (isDaemonRunning()) {
318
- return { started: false, existing_pid: readPidfile() ?? undefined };
337
+ const recordedPid = readPidfile();
338
+ if (recordedPid !== null && isPidAlive(recordedPid)) {
339
+ if (await isDaemonServing())
340
+ return { started: false, existing_pid: recordedPid };
341
+ return {
342
+ started: false,
343
+ existing_pid: recordedPid,
344
+ running: true,
345
+ serving: false,
346
+ next: 'Run `crtr sys daemon stop` to release the non-serving daemon before starting it again.',
347
+ };
319
348
  }
320
349
  // Ensure the canvas home directory exists so the daemon can write its pidfile.
321
350
  mkdirSync(crtrHome(), { recursive: true });
@@ -1,4 +1,5 @@
1
1
  import { getNode } from '../../core/canvas/canvas.js';
2
+ import { cancelCronsOnWake } from '../../core/canvas/crons.js';
2
3
  import { withFreshTerminalGuard } from '../../core/canvas/canvas.js';
3
4
  import { appendInbox } from '../../core/feed/inbox.js';
4
5
  import { BrokerUnreachableError } from '../../core/runtime/broker-request.js';
@@ -43,6 +44,7 @@ export async function deliverNodeMessage(args) {
43
44
  // before reporting an engine receipt so a final that committed during the
44
45
  // request is not represented as a routable delivery.
45
46
  guardRoutable(args.node_id, () => undefined);
47
+ cancelCronsOnWake(args.node_id);
46
48
  return { entry_id: null, via: 'engine', revived: false };
47
49
  }
48
50
  catch (error) {
@@ -57,9 +59,13 @@ export async function deliverNodeMessage(args) {
57
59
  label: args.label,
58
60
  data: { ...args.data, body: args.body },
59
61
  }));
62
+ if (args.mode !== 'quiet' || target.frozen_at !== null)
63
+ cancelCronsOnWake(args.node_id);
64
+ // The APPENDED entry's tier decides the wake: a quiet send to a terminal
65
+ // target was raised out of `deferred` by the append, and must be delivered.
60
66
  let revived = false;
61
67
  let reason;
62
- if (args.mode !== 'quiet') {
68
+ if (entry.tier !== 'deferred') {
63
69
  target = getNode(args.node_id);
64
70
  if (target === null)
65
71
  deliveryUnroutable(args.node_id, 'origin_missing');
@@ -1,6 +1,7 @@
1
1
  import { bashJobPaths, expiredAutoBackgroundBashJobs, formatBashElapsed, stopBackgroundBashJob, } from '../../core/bash-jobs.js';
2
2
  import { isSafeNodeId } from '../../core/canvas/paths.js';
3
3
  import { contextDir } from '../../core/canvas/paths.js';
4
+ import { cancelCronsOnWake } from '../../core/canvas/crons.js';
4
5
  import { appendInbox } from '../../core/feed/inbox.js';
5
6
  import { emitEvent } from '../../core/events/emit.js';
6
7
  import { operationIdContext } from '../../core/events/operation-id.js';
@@ -64,6 +65,7 @@ export class BashDeadlineReconciler {
64
65
  `Log: ${paths.jobLog} Command: ${paths.cmdSh}`,
65
66
  },
66
67
  });
68
+ cancelCronsOnWake(nodeId);
67
69
  }
68
70
  catch (err) {
69
71
  emitEvent({ level: 'error', event: 'bash_deadline.notice_failed', ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}), error: err, fields: { job_id: jobId } });
@@ -36,6 +36,7 @@ export declare function wedgeVerdict(input: {
36
36
  busy: boolean;
37
37
  quietForMs: number | null;
38
38
  cpuPercent: number | null;
39
+ hasDescendants: boolean | null;
39
40
  }): WedgeVerdict;
40
41
  export declare function sumTreeCpu(psOutput: string, rootPid: number): number | null;
41
42
  export declare function treeHasDescendants(psOutput: string, rootPid: number): boolean | null;
@@ -61,7 +61,9 @@ export function wedgeVerdict(input) {
61
61
  return 'leave';
62
62
  if (input.quietForMs === null || input.quietForMs < WEDGE_QUIET_MS)
63
63
  return 'leave';
64
- if (input.cpuPercent === null || input.cpuPercent > WEDGE_CPU_MAX_PERCENT)
64
+ if (input.cpuPercent === null)
65
+ return 'pending';
66
+ if (input.hasDescendants !== false && input.cpuPercent > WEDGE_CPU_MAX_PERCENT)
65
67
  return 'pending';
66
68
  return 'wedged';
67
69
  }
@@ -471,18 +473,20 @@ export class BrokerSupervisionReconciler {
471
473
  const quietForMs = since === null ? null : now - since;
472
474
  const sample = quietForMs !== null && quietForMs >= WEDGE_QUIET_MS ? await this.readTreeSample(pid) : null;
473
475
  const cpuPercent = sample?.cpuPercent ?? null;
474
- if (wedgeVerdict({ busy: true, quietForMs, cpuPercent }) !== 'wedged')
476
+ const sampleHasDescendants = sample?.hasDescendants ?? null;
477
+ if (wedgeVerdict({ busy: true, quietForMs, cpuPercent, hasDescendants: sampleHasDescendants }) !== 'wedged')
475
478
  return;
476
479
  if (since !== null && this.wedgeNotifiedAt.get(id) === since)
477
480
  return;
478
- if (sample?.hasDescendants === false && (expectedIdentity == null || recordedPidLiveness(pid, expectedIdentity) !== 'alive'))
479
- return;
480
481
  if (since !== null)
481
482
  this.wedgeNotifiedAt.set(id, since);
482
483
  const meta = getNode(id);
483
484
  const name = meta !== null ? fullName(meta) : id;
484
485
  const minutes = Math.round((quietForMs ?? WEDGE_QUIET_MS) / 60_000);
485
- const hasDescendants = sample?.hasDescendants ?? true;
486
+ const hasDescendants = sampleHasDescendants ?? true;
487
+ const kickSuppressionReason = !hasDescendants && (expectedIdentity == null || recordedPidLiveness(pid, expectedIdentity) !== 'alive')
488
+ ? 'pid_identity_unproven'
489
+ : null;
486
490
  let label;
487
491
  if (hasDescendants) {
488
492
  label =
@@ -491,11 +495,18 @@ export class BrokerSupervisionReconciler {
491
495
  `\`crtr memory read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this ` +
492
496
  `node) so its turn can resume with no lost context.`;
493
497
  }
498
+ else if (kickSuppressionReason !== null) {
499
+ label =
500
+ `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
501
+ `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did ` +
502
+ `not SIGTERM the broker because its PID identity cannot be ` +
503
+ `confirmed alive.`;
504
+ }
494
505
  else {
495
506
  label =
496
507
  `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
497
- `${minutes}+ minutes, its process is near-idle, AND it has no subprocess to kill (the engine ` +
498
- `itself stalled). Kicking it now: SIGTERM to the broker — the daemon will resume a cleanly ` +
508
+ `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). Kicking it ` +
509
+ `now: SIGTERM to the broker — the daemon will resume a cleanly ` +
499
510
  `aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`;
500
511
  try {
501
512
  process.kill(pid, 'SIGTERM');
@@ -514,7 +525,8 @@ export class BrokerSupervisionReconciler {
514
525
  quiet_ms: quietForMs,
515
526
  cpu_percent: cpuPercent,
516
527
  has_descendants: hasDescendants,
517
- kicked: !hasDescendants,
528
+ kicked: !hasDescendants && kickSuppressionReason === null,
529
+ ...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
518
530
  },
519
531
  });
520
532
  try {
@@ -539,7 +551,8 @@ export class BrokerSupervisionReconciler {
539
551
  child: id,
540
552
  quiet_ms: quietForMs,
541
553
  cpu_percent: cpuPercent,
542
- kicked: !hasDescendants,
554
+ kicked: !hasDescendants && kickSuppressionReason === null,
555
+ ...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
543
556
  });
544
557
  }
545
558
  handleFatalFault(id) {
@@ -25,16 +25,26 @@ export declare function hasPendingWake(nodeId: string): boolean;
25
25
  * report is an outstanding deliverable, so such a node keeps its
26
26
  * wakeable idle row instead of being written off.
27
27
  *
28
- * Undelivered mail of ANY tier also holds the row. A deferred-only unseen set
29
- * never wakes a node on its own — it rides the next natural cycle — but a
30
- * terminalized row no longer takes that cycle, so writing one off would drop
31
- * content the sender already handed over.
28
+ * Undelivered mail holds the row, with one carve-out: a deferred-only unseen
29
+ * set does not hold a RESIDENT. Deferred mail never wakes a node on its own,
30
+ * and parking a resident is reversible — a message sent to it revives it and
31
+ * flushes the held mail on that cycle. A terminalized row takes no such
32
+ * cycle, so unread mail of any tier keeps it from being written off: the
33
+ * invariant is that content a sender already handed over is never dropped.
32
34
  */
33
35
  export type ParkEvent = Extract<LifecycleEvent, 'release' | 'park' | 'cancel'>;
34
36
  export declare function parkEvent(nodeId: string, lifecycle: Lifecycle): ParkEvent;
35
37
  /** Apply an unattended decision and notify subscribers of automatic completion.
36
38
  * This is the DEGRADED enactment: no summary turn ran, so subscribers get the
37
- * doctrine wake instead of the node's own report. The live clock's good path
39
+ * doctrine notice instead of the node's own report. The live clock's good path
38
40
  * (broker settlement after a parking turn) enacts `park` directly and fans
39
- * nothing — the node's report is the notice. */
41
+ * nothing — the node's report is the notice.
42
+ *
43
+ * The notice is DEFERRED: a node settling quietly at the end of its unattended
44
+ * grace is not an event worth starting a manager's cycle for, so it rides the
45
+ * manager's next natural cycle and arrives alongside whatever does.
46
+ *
47
+ * Only `park` fans, and only a resident parks, so the notice speaks about a
48
+ * resident throughout: its conversation ended without a final report, its state
49
+ * lives in whatever report it did leave, and a message reopens it. */
40
50
  export declare function applyParkEvent(nodeId: string, event: ParkEvent): void;