@north-light/crouter 0.3.236 → 0.3.238

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/dist/api/client.d.ts +2 -1
  2. package/dist/api/client.js +3 -0
  3. package/dist/api/dto/broker-ops.d.ts +1 -1
  4. package/dist/api/dto/common.d.ts +2 -0
  5. package/dist/api/dto/lifecycle.d.ts +9 -2
  6. package/dist/api/dto/messages.d.ts +5 -0
  7. package/dist/api/dto/nodes.d.ts +10 -2
  8. package/dist/api/dto/worktree.d.ts +10 -0
  9. package/dist/api/routes.d.ts +1 -0
  10. package/dist/api/routes.js +1 -0
  11. package/dist/builtin-memory/00-runtime-base/00-authoring.md +1 -1
  12. package/dist/builtin-memory/internal/agent-shaping.md +1 -1
  13. package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +1 -1
  14. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/sysprompt-window.ts +1 -1
  15. package/dist/clients/attach/render/tool-calls.js +18 -3
  16. package/dist/clients/attach/viewer.js +442 -442
  17. package/dist/clients/inbox/review/__tests__/editor-roundtrip.test.js +6 -0
  18. package/dist/clients/inbox/review/roundtrip.js +1 -1
  19. package/dist/commands/memory/lint.js +22 -13
  20. package/dist/commands/memory/shared.js +1 -1
  21. package/dist/commands/node/create.d.ts +1 -1
  22. package/dist/commands/node/create.js +1 -1
  23. package/dist/commands/node/lifecycle.js +1 -1
  24. package/dist/commands/node-worktree.js +39 -8
  25. package/dist/commands/revive.js +11 -2
  26. package/dist/commands/sys/context/admin/actions.d.ts +48 -0
  27. package/dist/commands/sys/context/admin/actions.js +246 -0
  28. package/dist/commands/sys/context/admin/axes-panel.d.ts +21 -0
  29. package/dist/commands/sys/context/admin/axes-panel.js +117 -0
  30. package/dist/commands/sys/context/admin/detail-panel.d.ts +53 -0
  31. package/dist/commands/sys/context/admin/detail-panel.js +178 -0
  32. package/dist/commands/sys/context/admin/docs-panel.d.ts +45 -0
  33. package/dist/commands/sys/context/admin/docs-panel.js +186 -0
  34. package/dist/commands/sys/context/admin/model.d.ts +75 -0
  35. package/dist/commands/sys/context/admin/model.js +209 -0
  36. package/dist/commands/sys/context/admin/shell.d.ts +82 -0
  37. package/dist/commands/sys/context/admin/shell.js +669 -0
  38. package/dist/commands/sys/context/admin.d.ts +1 -0
  39. package/dist/commands/sys/context/admin.js +124 -0
  40. package/dist/commands/sys/context/doc.d.ts +1 -0
  41. package/dist/commands/sys/context/doc.js +196 -0
  42. package/dist/commands/sys/context/prompt-review.d.ts +1 -0
  43. package/dist/commands/sys/{prompt-review.js → context/prompt-review.js} +10 -10
  44. package/dist/commands/sys/context/resolve.d.ts +55 -0
  45. package/dist/commands/sys/context/resolve.js +428 -0
  46. package/dist/commands/sys/context/sysprompt.d.ts +1 -0
  47. package/dist/commands/sys/{sysprompt.js → context/sysprompt.js} +11 -11
  48. package/dist/commands/sys/context.d.ts +2 -0
  49. package/dist/commands/sys/context.js +20 -0
  50. package/dist/commands/sys.js +3 -4
  51. package/dist/core/__tests__/child-death-wake.test.js +38 -22
  52. package/dist/core/__tests__/cron-self-cancel-delivery.test.js +72 -0
  53. package/dist/core/__tests__/daemon-boot.test.js +40 -28
  54. package/dist/core/__tests__/human-deliver.test.js +1 -1
  55. package/dist/core/__tests__/human-node-not-supervised.test.js +10 -15
  56. package/dist/core/__tests__/integration/revive.test.js +12 -12
  57. package/dist/core/__tests__/integration/spawn-root.test.js +6 -0
  58. package/dist/core/__tests__/integration/worktree-reap.test.js +32 -0
  59. package/dist/core/__tests__/lifecycle-terminal-reasons.test.d.ts +1 -0
  60. package/dist/core/__tests__/lifecycle-terminal-reasons.test.js +61 -0
  61. package/dist/core/__tests__/migration.test.js +38 -1
  62. package/dist/core/__tests__/push-final-guard.test.js +1 -1
  63. package/dist/core/__tests__/revive-capacity.test.d.ts +1 -0
  64. package/dist/core/__tests__/revive-capacity.test.js +127 -0
  65. package/dist/core/__tests__/revive-parked-fresh.test.js +7 -7
  66. package/dist/core/__tests__/seam/broker-cap-freeze.test.d.ts +1 -0
  67. package/dist/core/__tests__/seam/broker-cap-freeze.test.js +81 -0
  68. package/dist/core/__tests__/seam/dormancy-release.test.js +7 -7
  69. package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +1 -1
  70. package/dist/core/canvas/browse/render.js +11 -3
  71. package/dist/core/canvas/canvas.d.ts +15 -3
  72. package/dist/core/canvas/canvas.js +52 -16
  73. package/dist/core/canvas/migrations.js +21 -0
  74. package/dist/core/canvas/nav-render.js +3 -1
  75. package/dist/core/canvas/remote-canvas-source.js +2 -0
  76. package/dist/core/canvas/render-source.d.ts +3 -1
  77. package/dist/core/canvas/render-source.js +7 -3
  78. package/dist/core/canvas/status-glyph.d.ts +10 -1
  79. package/dist/core/canvas/status-glyph.js +19 -3
  80. package/dist/core/canvas/types.d.ts +20 -1
  81. package/dist/core/feed/feed.js +1 -1
  82. package/dist/core/human/feedback-companion.js +5 -2
  83. package/dist/core/keybindings/catalog.d.ts +1 -1
  84. package/dist/core/keybindings/catalog.js +6 -2
  85. package/dist/core/memory/lint.d.ts +19 -0
  86. package/dist/core/memory/lint.js +97 -0
  87. package/dist/core/memory-resolver.d.ts +4 -0
  88. package/dist/core/memory-resolver.js +7 -7
  89. package/dist/core/preview-registry.js +4 -4
  90. package/dist/core/review/realize.js +4 -2
  91. package/dist/core/runtime/close.js +1 -1
  92. package/dist/core/runtime/fleet.d.ts +29 -1
  93. package/dist/core/runtime/fleet.js +23 -0
  94. package/dist/core/runtime/host.js +32 -11
  95. package/dist/core/runtime/launch-prompt.d.ts +1 -1
  96. package/dist/core/runtime/launch-prompt.js +2 -2
  97. package/dist/core/runtime/lifecycle.d.ts +4 -2
  98. package/dist/core/runtime/lifecycle.js +12 -1
  99. package/dist/core/runtime/memory.js +1 -1
  100. package/dist/core/runtime/reopen.js +10 -1
  101. package/dist/core/runtime/reset.js +1 -1
  102. package/dist/core/runtime/revive-all.d.ts +3 -0
  103. package/dist/core/runtime/revive-all.js +10 -4
  104. package/dist/core/runtime/revive.d.ts +12 -0
  105. package/dist/core/runtime/revive.js +62 -3
  106. package/dist/core/runtime/spawn.d.ts +4 -0
  107. package/dist/core/runtime/spawn.js +36 -7
  108. package/dist/core/runtime/tmux-bindings.js +4 -0
  109. package/dist/core/substrate/frontmatter-validation.js +1 -1
  110. package/dist/core/substrate/index.d.ts +6 -3
  111. package/dist/core/substrate/index.js +4 -3
  112. package/dist/core/substrate/on-read.d.ts +0 -8
  113. package/dist/core/substrate/on-read.js +45 -124
  114. package/dist/core/substrate/plan.d.ts +94 -0
  115. package/dist/core/substrate/plan.js +266 -0
  116. package/dist/core/substrate/render.d.ts +5 -6
  117. package/dist/core/substrate/render.js +76 -129
  118. package/dist/core/substrate/schema.d.ts +1 -6
  119. package/dist/core/substrate/schema.js +1 -15
  120. package/dist/core/substrate/session-cache.d.ts +2 -10
  121. package/dist/core/substrate/session-cache.js +9 -47
  122. package/dist/core/substrate/subject-fields.d.ts +1 -4
  123. package/dist/core/substrate/subject-fields.js +2 -5
  124. package/dist/core/substrate/surface-match.d.ts +37 -17
  125. package/dist/core/substrate/surface-match.js +44 -53
  126. package/dist/core/tui/page-host.d.ts +6 -0
  127. package/dist/core/tui/page-host.js +12 -3
  128. package/dist/core/worktree.d.ts +8 -0
  129. package/dist/core/worktree.js +65 -0
  130. package/dist/daemon/__tests__/reconciler-signature.test.js +3 -2
  131. package/dist/daemon/__tests__/thaw-order.test.d.ts +1 -0
  132. package/dist/daemon/__tests__/thaw-order.test.js +55 -0
  133. package/dist/daemon/api/__tests__/canvas-snapshot-fields.test.js +6 -0
  134. package/dist/daemon/api/__tests__/node-create-description.test.js +6 -0
  135. package/dist/daemon/api/__tests__/profile-launch-gates.test.js +1 -1
  136. package/dist/daemon/api/bridge.js +5 -1
  137. package/dist/daemon/api/handlers/attach.js +5 -1
  138. package/dist/daemon/api/handlers/bash-jobs.js +1 -1
  139. package/dist/daemon/api/handlers/messages.js +10 -2
  140. package/dist/daemon/api/handlers/nodes.js +8 -1
  141. package/dist/daemon/api/handlers/worktree.js +10 -1
  142. package/dist/daemon/api/map.js +7 -0
  143. package/dist/daemon/companion-retire.js +1 -1
  144. package/dist/daemon/cron/passes.d.ts +0 -5
  145. package/dist/daemon/cron/passes.js +0 -20
  146. package/dist/daemon/cron/sinks.js +1 -1
  147. package/dist/daemon/cron-run.d.ts +5 -6
  148. package/dist/daemon/cron-run.js +7 -16
  149. package/dist/daemon/crtrd.d.ts +0 -10
  150. package/dist/daemon/crtrd.js +23 -61
  151. package/dist/daemon/fleet.d.ts +10 -7
  152. package/dist/daemon/fleet.js +71 -12
  153. package/dist/daemon/messaging/node-message.js +3 -1
  154. package/dist/daemon/profile-delete.js +1 -1
  155. package/dist/daemon/reconcilers/broker-supervision.d.ts +26 -9
  156. package/dist/daemon/reconcilers/broker-supervision.js +39 -52
  157. package/dist/daemon/reconcilers/controller-death.d.ts +7 -0
  158. package/dist/daemon/reconcilers/controller-death.js +26 -0
  159. package/dist/daemon/reconcilers/cron-lane.d.ts +0 -3
  160. package/dist/daemon/reconcilers/cron-lane.js +0 -3
  161. package/dist/daemon/reconcilers/live-obligation.js +3 -1
  162. package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.d.ts +53 -0
  163. package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +170 -0
  164. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +5 -0
  165. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +16 -0
  166. package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -0
  167. package/dist/daemon/reconcilers/node-lifecycle/terminating.js +13 -0
  168. package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +26 -0
  169. package/dist/daemon/reconcilers/node-lifecycle/tick.js +140 -0
  170. package/package.json +1 -1
  171. package/runtime.lock.json +2 -2
  172. package/dist/commands/sys/prompt-review.d.ts +0 -1
  173. package/dist/commands/sys/sysprompt.d.ts +0 -1
  174. package/dist/core/__tests__/cron-broker-capacity.test.js +0 -119
  175. package/dist/daemon/reconcilers/dormant-inbox.d.ts +0 -18
  176. package/dist/daemon/reconcilers/dormant-inbox.js +0 -157
  177. /package/dist/core/__tests__/{cron-broker-capacity.test.d.ts → cron-self-cancel-delivery.test.d.ts} +0 -0
@@ -2,7 +2,7 @@ import { type RunDueCronOptions } from './cron/passes.js';
2
2
  import { type Cron, type CronRunRecord } from '../core/canvas/crons.js';
3
3
  export { CRON_OUTPUT_HEAD_BYTES } from './cron/capture.js';
4
4
  export { cronDaemonInstanceId, markSelfCanceledCronRun } from './cron/live-runs.js';
5
- export { cronLaunchesNode, type RunDueCronOptions } from './cron/passes.js';
5
+ export { type RunDueCronOptions } from './cron/passes.js';
6
6
  export { recoverStaleCronLeases } from './cron/lease-recovery.js';
7
7
  /** Kick off ONE cron run and return its settlement promise, resolving with
8
8
  * the settled run record (null = the idle→running lease was lost, or the row
@@ -24,9 +24,8 @@ export declare function executeCron(c: Cron, opts: {
24
24
  * but is NEVER awaited in the tick; returns one settlement promise per
25
25
  * admitted run.
26
26
  *
27
- * A run that would launch or revive a node (`cronLaunchesNode`) is admitted
28
- * only while the fleet is under its automatic-revive cap. An unadmitted run is
29
- * DEFERRED, never consumed: nothing is recorded, no recurrence advances, no
30
- * one-shot is deleted, and no failure escalates — the row keeps its past-due
31
- * `fire_at` and is retried on the next poll once capacity frees up. */
27
+ * A run that launches or revives a node is admitted unconditionally: the
28
+ * broker slot ledger is the one admission gate, so such a run's node is born
29
+ * or revived frozen at the cap and the node-lifecycle tick starts it when a
30
+ * slot frees. The cron itself always fires and settles. */
32
31
  export declare function runDueCrons(now: number, options?: RunDueCronOptions): Promise<void>[];
@@ -2,8 +2,8 @@
2
2
  // surface of the cron lane: subprocess spawn/capture/timeout/lease mechanics
3
3
  // for due `crons` rows (the cron-spec scheduler), and the tick pass that fires
4
4
  // them. The collaborators it composes live in `cron/`: arm-time context,
5
- // output capture, the live-run registry, the pre-run passes and capacity
6
- // admission, stale-lease recovery, the owed-gate decision, delivery sinks,
5
+ // output capture, the live-run registry, the pre-run passes, stale-lease
6
+ // recovery, the owed-gate decision, delivery sinks,
7
7
  // escalation, output disposition, and the one settle tail.
8
8
  //
9
9
  // A cron RUN is the enactment — there is no separate action to fire —
@@ -40,7 +40,7 @@ import { clearLiveRun, cronDaemonInstanceId, liveRunFor, registerLiveRun } from
40
40
  import { buildArmedRunContext } from './cron/armed-context.js';
41
41
  import { decideHeldSettle, EXIT_HELD, HELD_DELIVERED } from './cron/held.js';
42
42
  import { settleRun } from './cron/settle.js';
43
- import { applyOverlapPolicies, cronLaunchesNode, emitCapacityDeferral, expireDueCrons, } from './cron/passes.js';
43
+ import { applyOverlapPolicies, expireDueCrons, } from './cron/passes.js';
44
44
  import { dueClockCrons, acquireCronRunLease, advanceCron, setCronState, } from '../core/canvas/crons.js';
45
45
  import { killProcessGroup } from '../core/canvas/pid.js';
46
46
  import { emitEvent } from '../core/events/emit.js';
@@ -51,7 +51,6 @@ import { resolveAdmittedHostCommand } from '../core/runtime/spawn-env.js';
51
51
  // decomposed into.
52
52
  export { CRON_OUTPUT_HEAD_BYTES } from './cron/capture.js';
53
53
  export { cronDaemonInstanceId, markSelfCanceledCronRun } from './cron/live-runs.js';
54
- export { cronLaunchesNode } from './cron/passes.js';
55
54
  export { recoverStaleCronLeases } from './cron/lease-recovery.js';
56
55
  /** Settle one closed child: choose the outcome (replaced short-circuit, owed
57
56
  * gate, or the full disposition), then hand it to the one settle tail. */
@@ -350,22 +349,16 @@ export function executeCron(c, opts) {
350
349
  * but is NEVER awaited in the tick; returns one settlement promise per
351
350
  * admitted run.
352
351
  *
353
- * A run that would launch or revive a node (`cronLaunchesNode`) is admitted
354
- * only while the fleet is under its automatic-revive cap. An unadmitted run is
355
- * DEFERRED, never consumed: nothing is recorded, no recurrence advances, no
356
- * one-shot is deleted, and no failure escalates — the row keeps its past-due
357
- * `fire_at` and is retried on the next poll once capacity frees up. */
352
+ * A run that launches or revives a node is admitted unconditionally: the
353
+ * broker slot ledger is the one admission gate, so such a run's node is born
354
+ * or revived frozen at the cap and the node-lifecycle tick starts it when a
355
+ * slot frees. The cron itself always fires and settles. */
358
356
  export function runDueCrons(now, options = {}) {
359
357
  const nowIso = new Date(now).toISOString();
360
358
  expireDueCrons(nowIso);
361
359
  applyOverlapPolicies(now, nowIso);
362
360
  const settlements = [];
363
- let deferred = 0;
364
361
  for (const c of dueClockCrons(nowIso)) {
365
- if (options.hasBrokerCapacity !== undefined && cronLaunchesNode(c) && !options.hasBrokerCapacity()) {
366
- deferred += 1;
367
- continue; // still due — retried next poll, at no cost to the row
368
- }
369
362
  const settlement = executeCron(c, { outOfBand: false }).then(() => undefined);
370
363
  settlements.push(settlement);
371
364
  options.register?.(settlement);
@@ -374,7 +367,5 @@ export function runDueCrons(now, options = {}) {
374
367
  // as a process-level unhandled rejection.
375
368
  for (const s of settlements)
376
369
  void s.catch(() => { });
377
- if (deferred > 0)
378
- emitCapacityDeferral(now, deferred);
379
370
  return settlements;
380
371
  }
@@ -1,18 +1,8 @@
1
1
  import { type DetachedWorkLifecycle } from './reconcilers/broker-supervision.js';
2
- import { DaemonFleet } from './fleet.js';
3
2
  export declare function brokerCommandDeclaresCanvasHome(command: string, home: string): boolean;
4
3
  /** The `--epoch <id>` tag host.ts stamps into every broker's argv, or null
5
4
  * for a legacy/untagged broker (which the census treats as prior-epoch). */
6
5
  export declare function brokerCommandEpoch(command: string): string | null;
7
- /** Startup recovery sweep (design flow C step 2): the FIRST job on the serial
8
- * work lane. The census already reaped every prior-epoch broker, so every
9
- * active|idle row at boot is known-dead — classify each through the SAME
10
- * policy table exit events use (fleet.recoverDeadNode → classifyDeadNode,
11
- * design D-12), so exit recovery and startup recovery can never diverge. A
12
- * raced API revive is safe: reviveNode's fleet.has guard makes the loser a
13
- * no-op. Capacity-gated: past the automatic-revive cap the sweep stops — the
14
- * tick's pass-2 inbox wakes own the remainder as capacity frees up. */
15
- export declare function runStartupRecoverySweep(fleet: DaemonFleet): Promise<void>;
16
6
  interface DaemonTickLifecycle extends DetachedWorkLifecycle {
17
7
  }
18
8
  /** Supervision tick over the daemon's current live set. */
@@ -46,7 +46,7 @@
46
46
  import { writeFileSync, readFileSync, rmSync, mkdirSync, } from 'node:fs';
47
47
  import { dirname } from 'node:path';
48
48
  import { randomBytes } from 'node:crypto';
49
- import { crtrHome, isSafeNodeId } from '../core/canvas/paths.js';
49
+ import { crtrHome } from '../core/canvas/paths.js';
50
50
  import { listNodes, migrateLegacyPidIdentities, } from '../core/canvas/index.js';
51
51
  import { createApiServer } from './api/server.js';
52
52
  import { reconcileUndeliveredTickets } from './human/sweep.js';
@@ -56,7 +56,8 @@ import { pidfilePath } from './pidfile.js';
56
56
  import { rendererWarning } from '../core/runtime/package-health.js';
57
57
  import { DEFAULT_INTERVAL_MS } from './supervise-cadence.js';
58
58
  import { BrokerSupervisionReconciler } from './reconcilers/broker-supervision.js';
59
- import { ControllerDeathReconciler, DormantInboxReconciler } from './reconcilers/dormant-inbox.js';
59
+ import { ControllerDeathReconciler } from './reconcilers/controller-death.js';
60
+ import { NodeLifecycleReconciler } from './reconcilers/node-lifecycle/tick.js';
60
61
  import { CronLaneReconciler } from './reconcilers/cron-lane.js';
61
62
  import { HumanDeliveryLaneReconciler } from './reconcilers/human-delivery-lane.js';
62
63
  import { PendingReviewSubmitReconciler } from './reconcilers/pending-review-submit.js';
@@ -229,47 +230,6 @@ async function teardownFleet(fleet) {
229
230
  fields: { brokers: items.length, sigkilled },
230
231
  });
231
232
  }
232
- /** Startup recovery sweep (design flow C step 2): the FIRST job on the serial
233
- * work lane. The census already reaped every prior-epoch broker, so every
234
- * active|idle row at boot is known-dead — classify each through the SAME
235
- * policy table exit events use (fleet.recoverDeadNode → classifyDeadNode,
236
- * design D-12), so exit recovery and startup recovery can never diverge. A
237
- * raced API revive is safe: reviveNode's fleet.has guard makes the loser a
238
- * no-op. Capacity-gated: past the automatic-revive cap the sweep stops — the
239
- * tick's pass-2 inbox wakes own the remainder as capacity frees up. */
240
- export async function runStartupRecoverySweep(fleet) {
241
- let rows;
242
- try {
243
- rows = listNodes({ status: ['active', 'idle'] });
244
- }
245
- catch (err) {
246
- emitEvent({ level: 'error', event: 'daemon.recovery_sweep.failed', error: err });
247
- return;
248
- }
249
- for (const row of rows) {
250
- if (row.kind === 'human')
251
- continue; // human bridge rows — never broker-hosted
252
- if (!brokerSupervision.hasStartupRecoveryCapacity(fleet))
253
- break;
254
- await operationIdContext.fresh(async () => {
255
- try {
256
- await fleet.recoverDeadNode(row.node_id);
257
- }
258
- catch (err) {
259
- emitEvent({
260
- level: 'error',
261
- event: 'daemon.recovery_sweep.node_failed',
262
- ...(isSafeNodeId(row.node_id) ? { node_id: row.node_id } : { fields: { affected_node_id: row.node_id } }),
263
- error: err,
264
- });
265
- // reviveNode launch failures terminalize themselves, but preflight
266
- // refusals happen before its catch. Re-read and surface those rows
267
- // instead of leaving an active node with no broker to retry it.
268
- await fleet.terminalizeIfStillNonTerminal(row.node_id, 'daemon.recovery_sweep.node_terminalized', err);
269
- }
270
- });
271
- }
272
- }
273
233
  // Pidfile
274
234
  function writePidfile() {
275
235
  // Ensure the pidfile parent exists before writing.
@@ -311,7 +271,7 @@ const directTickLifecycle = {
311
271
  };
312
272
  const brokerSupervision = new BrokerSupervisionReconciler();
313
273
  const controllerDeath = new ControllerDeathReconciler();
314
- const dormantInbox = new DormantInboxReconciler();
274
+ const nodeLifecycle = new NodeLifecycleReconciler();
315
275
  const cronLane = new CronLaneReconciler();
316
276
  const humanDeliveryLane = new HumanDeliveryLaneReconciler();
317
277
  const pendingReviewSubmit = new PendingReviewSubmitReconciler();
@@ -321,26 +281,32 @@ export async function superviseTick(now = Date.now(), lifecycle = directTickLife
321
281
  const fleet = boundFleet();
322
282
  // Auth reload must precede the row-list guard: it is fleet-wide and does not
323
283
  // depend on a successful canvas scan.
324
- const thresholds = await brokerSupervision.run(now, { fleet, lifecycle });
284
+ const thresholds = await brokerSupervision.runFleetPreamble(fleet);
325
285
  let rows;
286
+ // A terminal row that still holds a process is the Terminating state, and the
287
+ // lifecycle tick is the only pass that owes it anything.
288
+ let terminating;
326
289
  try {
327
- rows = listNodes({ status: ['active', 'idle'] });
290
+ // Human bridge rows have no broker engine and participate in none of the
291
+ // recurring row-driven reconciler passes.
292
+ rows = listNodes({ status: ['active', 'idle'] }).filter((row) => row.kind !== 'human');
293
+ terminating = listNodes({ status: ['done', 'canceled', 'dead'], withRecordedPid: true })
294
+ .filter((row) => row.kind !== 'human');
328
295
  }
329
296
  catch (err) {
330
297
  emitEvent({ level: 'error', event: 'canvas.node_list.failed', error: err });
331
298
  return;
332
299
  }
333
- // Human bridge rows have no broker engine and participate in none of the
334
- // recurring row-driven reconciler passes. Startup recovery applies the same
335
- // classification boundary.
336
- rows = rows.filter((row) => row.kind !== 'human');
337
- const hasBrokerCapacity = () => brokerSupervision.hasBrokerCapacity(thresholds.automaticReviveCap, fleet);
300
+ await brokerSupervision.run(now, { rows, fleet, lifecycle });
338
301
  controllerDeath.run(now, { rows });
339
- dormantInbox.run(now, { rows, fleet, hasBrokerCapacity });
340
- cronLane.run(now, { lifecycle, hasBrokerCapacity });
302
+ const frozen = await nodeLifecycle.run(now, { rows: [...rows, ...terminating], fleet });
303
+ cronLane.run(now, { lifecycle });
341
304
  humanDeliveryLane.run(now, { lifecycle });
342
305
  pendingReviewSubmit.run(now, { lifecycle });
343
306
  storageMaintenance.run(now, {});
307
+ // Last: the saturation figure is only true once the freeze lane has spent
308
+ // this tick's free slots.
309
+ brokerSupervision.warnIfSaturated({ fleet, lifecycle, thresholds, frozen });
344
310
  }
345
311
  /** Start the supervisor loop after winning the authoritative canvas claim.
346
312
  *
@@ -704,15 +670,11 @@ export async function runDaemon(opts = {}) {
704
670
  writePidfile();
705
671
  // Start the HTTP+WS API server (§4): unix socket always, TCP when opted in.
706
672
  apiServer = createApiServer({ tcp: opts.tcp });
707
- // The startup recovery sweep is the FIRST job on the lane (flow C step 2):
708
- // it classifies every boot-time dead row through the same policy table
709
- // exit events use, before the first tick or any exit event interleaves.
710
- const bootFleet = fleet;
711
- enqueue(() => runStartupRecoverySweep(bootFleet));
712
673
  // Pre-warm the node pool for the most recent project recipes, so the first
713
- // create after a boot is warm too. Scheduled behind the recovery sweep and
714
- // fire-and-forget (the mints run off the work lane), so it never delays
715
- // supervision.
674
+ // create after a boot is warm too. Fire-and-forget (the mints run off the
675
+ // work lane), so it never delays supervision. Boot-time dead rows are the
676
+ // node-lifecycle tick's business: the first tick classifies every one of
677
+ // them through the same policy table exit events use, under the same cap.
716
678
  // Collect the spares this daemon's predecessor left behind BEFORE minting
717
679
  // new ones: every broker of the prior epoch was torn down, and no sweep
718
680
  // resumes a spare, so each is a dead row holding a node dir.
@@ -1,5 +1,5 @@
1
1
  import { type NodeMeta } from '../core/canvas/index.js';
2
- import type { FleetEntry, FleetExitStatus, FleetRegistry } from '../core/runtime/fleet.js';
2
+ import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
3
3
  import type { HostHandle } from '../core/runtime/host.js';
4
4
  /** Surface a vehicle that never booted (or exhausted its respawn budget).
5
5
  *
@@ -25,8 +25,8 @@ export type DeadNodeAction = 'forget' | 'respawn-fresh' | 'dormant' | 'release-i
25
25
  * NodeMeta itself) so tests fabricate table rows directly. */
26
26
  export type DeadNodeRow = Pick<NodeMeta, 'status' | 'intent' | 'pi_session_id' | 'cycle_pending' | 'fork_from' | 'lifecycle'>;
27
27
  /** One pure function producing the action for a node known-dead. Used at
28
- * exactly two call sites — an exit-policy job and the startup recovery sweep
29
- * (design D-12) — so exit recovery and startup recovery can never diverge.
28
+ * exactly two call sites — an exit-policy job and the node-lifecycle tick
29
+ * (design D-12) — so exit recovery and tick recovery can never diverge.
30
30
  * Evaluated in table order; `busy` is the on-disk busy marker.
31
31
  *
32
32
  * `busy` no longer selects the ACTION. Whether the engine died mid-turn or
@@ -71,14 +71,17 @@ export declare class DaemonFleet implements FleetRegistry {
71
71
  has(nodeId: string): boolean;
72
72
  get(nodeId: string): FleetEntry | undefined;
73
73
  size(): number;
74
+ occupancy(): number;
75
+ reserveSlot(nodeId: string): boolean;
76
+ releaseReservation(nodeId: string): void;
74
77
  entries(): Iterable<[string, FleetEntry]>;
75
78
  forget(nodeId: string): void;
76
79
  onChildExit(nodeId: string, status: FleetExitStatus): void;
77
80
  deliverExit(nodeId: string, status: FleetExitStatus): void;
78
- /** Startup recovery sweep entry (design flow C step 2 / D-12): classify a
79
- * known-dead row and enact immediately through the SAME table exit events
80
- * use. Returns the action so the caller can capacity-gate launches. */
81
- recoverDeadNode(nodeId: string): Promise<DeadNodeAction>;
81
+ /** The node-lifecycle tick's enactment entry: classify a known-dead row and
82
+ * enact immediately through the SAME policy table exit events use, so tick
83
+ * recovery and exit recovery can never diverge. */
84
+ applyDeadRowPolicy(nodeId: string): Promise<void>;
82
85
  /** A revive refusal before its launch try/catch leaves the row active but
83
86
  * with no fleet entry. Convert that otherwise-silent strand into the same
84
87
  * terminal outcome as a boot failure. The fresh read avoids duplicating
@@ -23,6 +23,7 @@ import { hasPendingWake } from './reconcilers/live-obligation.js';
23
23
  import { reviveNode } from '../core/runtime/revive.js';
24
24
  import { pushUrgent } from '../core/feed/feed.js';
25
25
  import { emitEvent } from '../core/events/emit.js';
26
+ import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
26
27
  // Shared crash helpers (moved from crtrd.ts — the fleet's policy paths are now
27
28
  // their primary consumers; crtrd imports them back for its surviving passes).
28
29
  /** Surface a vehicle that never booted (or exhausted its respawn budget).
@@ -63,8 +64,8 @@ export function retryResumeMode(meta) {
63
64
  return meta?.cycle_pending !== true;
64
65
  }
65
66
  /** One pure function producing the action for a node known-dead. Used at
66
- * exactly two call sites — an exit-policy job and the startup recovery sweep
67
- * (design D-12) — so exit recovery and startup recovery can never diverge.
67
+ * exactly two call sites — an exit-policy job and the node-lifecycle tick
68
+ * (design D-12) — so exit recovery and tick recovery can never diverge.
68
69
  * Evaluated in table order; `busy` is the on-disk busy marker.
69
70
  *
70
71
  * `busy` no longer selects the ACTION. Whether the engine died mid-turn or
@@ -126,12 +127,20 @@ export function respawnBackoffMs(consecutiveFailures) {
126
127
  return 0;
127
128
  return Math.min(RESPAWN_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESPAWN_BACKOFF_MAX_MS);
128
129
  }
130
+ /** How long a capacity denial stays quiet before it is worth logging again. */
131
+ const CAPACITY_LOG_THROTTLE_MS = 30_000;
129
132
  export class DaemonFleet {
130
133
  #epoch;
131
134
  #enqueue;
132
135
  #now;
133
136
  #map = new Map();
134
137
  #throttle = new Map();
138
+ /** Slots claimed but not yet held by a registered broker: a launch in flight,
139
+ * or a broker that exited and whose exit-policy job has not yet decided
140
+ * whether to relaunch it. Counting both against the cap is what keeps a
141
+ * 1:1 respawn from losing its own slot to a concurrent launcher. */
142
+ #reservations = new Set();
143
+ #lastCapacityLogAt = Number.NEGATIVE_INFINITY;
135
144
  constructor(opts) {
136
145
  this.#epoch = opts.epoch;
137
146
  this.#enqueue = opts.enqueue;
@@ -155,6 +164,7 @@ export class DaemonFleet {
155
164
  clearTimeout(throttle.timer);
156
165
  throttle.timer = undefined;
157
166
  }
167
+ this.#reservations.delete(nodeId);
158
168
  this.#map.set(nodeId, {
159
169
  pid: handle.pid,
160
170
  launchedAt: this.#now(),
@@ -171,29 +181,59 @@ export class DaemonFleet {
171
181
  size() {
172
182
  return this.#map.size;
173
183
  }
184
+ occupancy() {
185
+ return this.#map.size + this.#reservations.size;
186
+ }
187
+ reserveSlot(nodeId) {
188
+ if (this.#reservations.has(nodeId) || this.#map.has(nodeId))
189
+ return true;
190
+ const { automaticReviveCap } = brokerThresholdsForDaemon();
191
+ if (this.occupancy() < automaticReviveCap) {
192
+ this.#reservations.add(nodeId);
193
+ return true;
194
+ }
195
+ const now = this.#now();
196
+ if (now - this.#lastCapacityLogAt >= CAPACITY_LOG_THROTTLE_MS) {
197
+ this.#lastCapacityLogAt = now;
198
+ emitEvent({
199
+ level: 'warn',
200
+ event: 'broker.capacity.revive_deferred',
201
+ fields: { live_count: this.occupancy(), hard_cap: automaticReviveCap },
202
+ });
203
+ }
204
+ return false;
205
+ }
206
+ releaseReservation(nodeId) {
207
+ this.#reservations.delete(nodeId);
208
+ }
174
209
  entries() {
175
210
  return this.#map.entries();
176
211
  }
177
212
  forget(nodeId) {
178
213
  this.#map.delete(nodeId);
214
+ this.#reservations.delete(nodeId);
179
215
  this.#clearThrottle(nodeId);
180
216
  }
181
217
  onChildExit(nodeId, status) {
182
- // Remove the entry BEFORE enqueuing the policy job: the slot is freed at
183
- // exit, which is what makes exit-driven 1:1 respawns cap-exempt (D-7).
218
+ // Move the slot from the registry to a reservation in ONE synchronous
219
+ // step: the exit-policy job about to be enqueued may relaunch this node,
220
+ // and holding the slot for it is what keeps a 1:1 respawn from losing its
221
+ // own capacity to a launcher that raced onto the lane first. `#applyPolicy`
222
+ // releases it on every path that does not relaunch.
184
223
  const entry = this.#map.get(nodeId);
185
224
  this.#map.delete(nodeId);
225
+ this.#reservations.add(nodeId);
186
226
  const launchedAt = entry?.launchedAt ?? null;
187
227
  this.#enqueue(() => this.#runExitPolicy(nodeId, status, launchedAt));
188
228
  }
189
229
  deliverExit(nodeId, status) {
190
230
  this.onChildExit(nodeId, status);
191
231
  }
192
- /** Startup recovery sweep entry (design flow C step 2 / D-12): classify a
193
- * known-dead row and enact immediately through the SAME table exit events
194
- * use. Returns the action so the caller can capacity-gate launches. */
195
- async recoverDeadNode(nodeId) {
196
- return this.#applyPolicy(nodeId, 'sweep');
232
+ /** The node-lifecycle tick's enactment entry: classify a known-dead row and
233
+ * enact immediately through the SAME policy table exit events use, so tick
234
+ * recovery and exit recovery can never diverge. */
235
+ async applyDeadRowPolicy(nodeId) {
236
+ await this.#applyPolicy(nodeId, 'tick');
197
237
  }
198
238
  /** A revive refusal before its launch try/catch leaves the row active but
199
239
  * with no fleet entry. Convert that otherwise-silent strand into the same
@@ -270,6 +310,18 @@ export class DaemonFleet {
270
310
  });
271
311
  }
272
312
  async #applyPolicy(nodeId, origin) {
313
+ try {
314
+ return await this.#decideAndEnact(nodeId, origin);
315
+ }
316
+ finally {
317
+ // The reservation this node has held since its exit is either consumed by
318
+ // a successful `register` (making this a no-op) or given back here — the
319
+ // one release path covering a declined policy, a deferred backoff, and
320
+ // terminalization alike.
321
+ this.releaseReservation(nodeId);
322
+ }
323
+ }
324
+ async #decideAndEnact(nodeId, origin) {
273
325
  const meta = getNode(nodeId);
274
326
  // Two durable witnesses mean the dead broker had a turn in flight. A dirty
275
327
  // death leaves the busy marker behind; a bounded graceful abort clears busy
@@ -329,9 +381,13 @@ export class DaemonFleet {
329
381
  });
330
382
  return;
331
383
  }
332
- // Only a fresh exit consults the backoff: a matured timer and the startup
333
- // sweep enact immediately (the timer already WAS the backoff, and startup
334
- // counters are empty for a fresh epoch).
384
+ // A backoff timer already owns this node's next respawn; a tick that
385
+ // enacted the same policy meanwhile would defeat the throttle entirely.
386
+ if (origin === 'tick' && state.timer !== undefined)
387
+ return;
388
+ // Only a fresh exit consults the backoff: a matured timer and a tick
389
+ // enactment relaunch immediately (the timer already WAS the backoff, and a
390
+ // tick candidate reached the lane only after its own gating).
335
391
  if (origin === 'exit') {
336
392
  const delay = respawnBackoffMs(state.failures);
337
393
  if (delay > 0) {
@@ -369,6 +425,9 @@ export class DaemonFleet {
369
425
  });
370
426
  try {
371
427
  reviveNode(nodeId, {
428
+ // A recovery that loses the race for the last slot freezes and waits
429
+ // for one — the tick thaws it — rather than erroring the node away.
430
+ capacity: 'freeze',
372
431
  resume: action === 'respawn-resume',
373
432
  // A resumed session whose turn was interrupted comes back parked at an
374
433
  // idle prompt with the dead turn unfinished. The continuation re-drives
@@ -64,7 +64,9 @@ export async function deliverNodeMessage(args) {
64
64
  deliveryUnroutable(args.node_id, 'origin_missing');
65
65
  if (!isBrokerLive(target)) {
66
66
  try {
67
- revived = reviveNode(args.node_id, { resume: true }).launch !== undefined;
67
+ // Mail is durable, so a wake denied for capacity waits for a slot
68
+ // rather than being lost: the row freezes and the tick relaunches it.
69
+ revived = reviveNode(args.node_id, { resume: true, capacity: 'freeze' }).launch !== undefined;
68
70
  }
69
71
  catch {
70
72
  // The durable entry remains authoritative; one wake attempt is enough.
@@ -216,7 +216,7 @@ async function detachProfile(plan) {
216
216
  const detached = new Set(detachedNodeIds);
217
217
  for (const nodeId of restartNodeIds) {
218
218
  if (detached.has(nodeId))
219
- reviveNode(nodeId, { resume: true, recovery: true });
219
+ reviveNode(nodeId, { resume: true, recovery: true, capacity: 'freeze' });
220
220
  }
221
221
  return {
222
222
  ...resultForPlan(plan),
@@ -1,17 +1,30 @@
1
1
  import { type NodeRow } from '../../core/canvas/index.js';
2
- import type { FleetRegistry } from '../../core/runtime/fleet.js';
2
+ import { type FleetRegistry } from '../../core/runtime/fleet.js';
3
3
  export interface DetachedWorkLifecycle {
4
4
  acceptsDetachedWork(): boolean;
5
5
  registerDetached(work: Promise<void>): void;
6
6
  }
7
7
  export interface BrokerSupervisionContext {
8
+ /** The rows the tick loaded — supervision walks these and looks each one's
9
+ * fleet entry up, rather than re-reading a row per live entry. */
10
+ rows: readonly NodeRow[];
8
11
  fleet: FleetRegistry;
9
12
  lifecycle: DetachedWorkLifecycle;
10
13
  }
11
- export declare function brokerThresholdsForDaemon(): {
12
- warning: number;
13
- automaticReviveCap: number;
14
- };
14
+ export interface SaturationContext {
15
+ fleet: FleetRegistry;
16
+ lifecycle: DetachedWorkLifecycle;
17
+ thresholds: {
18
+ warning: number;
19
+ automaticReviveCap: number;
20
+ };
21
+ /** What the freeze lane left frozen this tick, split into rows that stalled
22
+ * and rows waiting on mail. */
23
+ frozen: {
24
+ stalled: number;
25
+ waitingOnMail: number;
26
+ };
27
+ }
15
28
  export declare const YIELD_STALL_GRACE_MS: number;
16
29
  export declare const WEDGE_QUIET_MS: number;
17
30
  export declare const WEDGE_CPU_MAX_PERCENT = 2;
@@ -34,11 +47,17 @@ export declare class BrokerSupervisionReconciler {
34
47
  private readonly unattendedSince;
35
48
  private knownAuthMtimeMs;
36
49
  private lastOverCapWarnAt;
37
- private lastCapacityLogAt;
38
- run(now: number, ctx: BrokerSupervisionContext): Promise<{
50
+ /** The fleet-wide preamble, run BEFORE the tick's row-list guard: an auth
51
+ * reload does not depend on a successful canvas scan, and the thresholds it
52
+ * returns are read once for the whole tick. */
53
+ runFleetPreamble(fleet: FleetRegistry): Promise<{
39
54
  warning: number;
40
55
  automaticReviveCap: number;
41
56
  }>;
57
+ /** The saturation notice, run LAST in the tick: the frozen split it reports
58
+ * is only known once the freeze lane has spent this tick's free slots. */
59
+ warnIfSaturated(ctx: SaturationContext): void;
60
+ run(now: number, ctx: BrokerSupervisionContext): Promise<void>;
42
61
  /** Every condition that resets the unattended clock EXCEPT `isBusy` — the
43
62
  * node is unavailable to park, attended, or has work waiting. Short-circuits
44
63
  * in cost order so the common tick does no file reads it can avoid.
@@ -58,6 +77,4 @@ export declare class BrokerSupervisionReconciler {
58
77
  private handleWedgeDetection;
59
78
  private handleFatalFault;
60
79
  private handleAuthReload;
61
- hasBrokerCapacity(max: number, fleet: FleetRegistry): boolean;
62
- hasStartupRecoveryCapacity(fleet: FleetRegistry): boolean;
63
80
  }