@north-light/crouter 0.3.236 → 0.3.238

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/dist/api/client.d.ts +2 -1
  2. package/dist/api/client.js +3 -0
  3. package/dist/api/dto/broker-ops.d.ts +1 -1
  4. package/dist/api/dto/common.d.ts +2 -0
  5. package/dist/api/dto/lifecycle.d.ts +9 -2
  6. package/dist/api/dto/messages.d.ts +5 -0
  7. package/dist/api/dto/nodes.d.ts +10 -2
  8. package/dist/api/dto/worktree.d.ts +10 -0
  9. package/dist/api/routes.d.ts +1 -0
  10. package/dist/api/routes.js +1 -0
  11. package/dist/builtin-memory/00-runtime-base/00-authoring.md +1 -1
  12. package/dist/builtin-memory/internal/agent-shaping.md +1 -1
  13. package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +1 -1
  14. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/sysprompt-window.ts +1 -1
  15. package/dist/clients/attach/render/tool-calls.js +18 -3
  16. package/dist/clients/attach/viewer.js +442 -442
  17. package/dist/clients/inbox/review/__tests__/editor-roundtrip.test.js +6 -0
  18. package/dist/clients/inbox/review/roundtrip.js +1 -1
  19. package/dist/commands/memory/lint.js +22 -13
  20. package/dist/commands/memory/shared.js +1 -1
  21. package/dist/commands/node/create.d.ts +1 -1
  22. package/dist/commands/node/create.js +1 -1
  23. package/dist/commands/node/lifecycle.js +1 -1
  24. package/dist/commands/node-worktree.js +39 -8
  25. package/dist/commands/revive.js +11 -2
  26. package/dist/commands/sys/context/admin/actions.d.ts +48 -0
  27. package/dist/commands/sys/context/admin/actions.js +246 -0
  28. package/dist/commands/sys/context/admin/axes-panel.d.ts +21 -0
  29. package/dist/commands/sys/context/admin/axes-panel.js +117 -0
  30. package/dist/commands/sys/context/admin/detail-panel.d.ts +53 -0
  31. package/dist/commands/sys/context/admin/detail-panel.js +178 -0
  32. package/dist/commands/sys/context/admin/docs-panel.d.ts +45 -0
  33. package/dist/commands/sys/context/admin/docs-panel.js +186 -0
  34. package/dist/commands/sys/context/admin/model.d.ts +75 -0
  35. package/dist/commands/sys/context/admin/model.js +209 -0
  36. package/dist/commands/sys/context/admin/shell.d.ts +82 -0
  37. package/dist/commands/sys/context/admin/shell.js +669 -0
  38. package/dist/commands/sys/context/admin.d.ts +1 -0
  39. package/dist/commands/sys/context/admin.js +124 -0
  40. package/dist/commands/sys/context/doc.d.ts +1 -0
  41. package/dist/commands/sys/context/doc.js +196 -0
  42. package/dist/commands/sys/context/prompt-review.d.ts +1 -0
  43. package/dist/commands/sys/{prompt-review.js → context/prompt-review.js} +10 -10
  44. package/dist/commands/sys/context/resolve.d.ts +55 -0
  45. package/dist/commands/sys/context/resolve.js +428 -0
  46. package/dist/commands/sys/context/sysprompt.d.ts +1 -0
  47. package/dist/commands/sys/{sysprompt.js → context/sysprompt.js} +11 -11
  48. package/dist/commands/sys/context.d.ts +2 -0
  49. package/dist/commands/sys/context.js +20 -0
  50. package/dist/commands/sys.js +3 -4
  51. package/dist/core/__tests__/child-death-wake.test.js +38 -22
  52. package/dist/core/__tests__/cron-self-cancel-delivery.test.js +72 -0
  53. package/dist/core/__tests__/daemon-boot.test.js +40 -28
  54. package/dist/core/__tests__/human-deliver.test.js +1 -1
  55. package/dist/core/__tests__/human-node-not-supervised.test.js +10 -15
  56. package/dist/core/__tests__/integration/revive.test.js +12 -12
  57. package/dist/core/__tests__/integration/spawn-root.test.js +6 -0
  58. package/dist/core/__tests__/integration/worktree-reap.test.js +32 -0
  59. package/dist/core/__tests__/lifecycle-terminal-reasons.test.d.ts +1 -0
  60. package/dist/core/__tests__/lifecycle-terminal-reasons.test.js +61 -0
  61. package/dist/core/__tests__/migration.test.js +38 -1
  62. package/dist/core/__tests__/push-final-guard.test.js +1 -1
  63. package/dist/core/__tests__/revive-capacity.test.d.ts +1 -0
  64. package/dist/core/__tests__/revive-capacity.test.js +127 -0
  65. package/dist/core/__tests__/revive-parked-fresh.test.js +7 -7
  66. package/dist/core/__tests__/seam/broker-cap-freeze.test.d.ts +1 -0
  67. package/dist/core/__tests__/seam/broker-cap-freeze.test.js +81 -0
  68. package/dist/core/__tests__/seam/dormancy-release.test.js +7 -7
  69. package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +1 -1
  70. package/dist/core/canvas/browse/render.js +11 -3
  71. package/dist/core/canvas/canvas.d.ts +15 -3
  72. package/dist/core/canvas/canvas.js +52 -16
  73. package/dist/core/canvas/migrations.js +21 -0
  74. package/dist/core/canvas/nav-render.js +3 -1
  75. package/dist/core/canvas/remote-canvas-source.js +2 -0
  76. package/dist/core/canvas/render-source.d.ts +3 -1
  77. package/dist/core/canvas/render-source.js +7 -3
  78. package/dist/core/canvas/status-glyph.d.ts +10 -1
  79. package/dist/core/canvas/status-glyph.js +19 -3
  80. package/dist/core/canvas/types.d.ts +20 -1
  81. package/dist/core/feed/feed.js +1 -1
  82. package/dist/core/human/feedback-companion.js +5 -2
  83. package/dist/core/keybindings/catalog.d.ts +1 -1
  84. package/dist/core/keybindings/catalog.js +6 -2
  85. package/dist/core/memory/lint.d.ts +19 -0
  86. package/dist/core/memory/lint.js +97 -0
  87. package/dist/core/memory-resolver.d.ts +4 -0
  88. package/dist/core/memory-resolver.js +7 -7
  89. package/dist/core/preview-registry.js +4 -4
  90. package/dist/core/review/realize.js +4 -2
  91. package/dist/core/runtime/close.js +1 -1
  92. package/dist/core/runtime/fleet.d.ts +29 -1
  93. package/dist/core/runtime/fleet.js +23 -0
  94. package/dist/core/runtime/host.js +32 -11
  95. package/dist/core/runtime/launch-prompt.d.ts +1 -1
  96. package/dist/core/runtime/launch-prompt.js +2 -2
  97. package/dist/core/runtime/lifecycle.d.ts +4 -2
  98. package/dist/core/runtime/lifecycle.js +12 -1
  99. package/dist/core/runtime/memory.js +1 -1
  100. package/dist/core/runtime/reopen.js +10 -1
  101. package/dist/core/runtime/reset.js +1 -1
  102. package/dist/core/runtime/revive-all.d.ts +3 -0
  103. package/dist/core/runtime/revive-all.js +10 -4
  104. package/dist/core/runtime/revive.d.ts +12 -0
  105. package/dist/core/runtime/revive.js +62 -3
  106. package/dist/core/runtime/spawn.d.ts +4 -0
  107. package/dist/core/runtime/spawn.js +36 -7
  108. package/dist/core/runtime/tmux-bindings.js +4 -0
  109. package/dist/core/substrate/frontmatter-validation.js +1 -1
  110. package/dist/core/substrate/index.d.ts +6 -3
  111. package/dist/core/substrate/index.js +4 -3
  112. package/dist/core/substrate/on-read.d.ts +0 -8
  113. package/dist/core/substrate/on-read.js +45 -124
  114. package/dist/core/substrate/plan.d.ts +94 -0
  115. package/dist/core/substrate/plan.js +266 -0
  116. package/dist/core/substrate/render.d.ts +5 -6
  117. package/dist/core/substrate/render.js +76 -129
  118. package/dist/core/substrate/schema.d.ts +1 -6
  119. package/dist/core/substrate/schema.js +1 -15
  120. package/dist/core/substrate/session-cache.d.ts +2 -10
  121. package/dist/core/substrate/session-cache.js +9 -47
  122. package/dist/core/substrate/subject-fields.d.ts +1 -4
  123. package/dist/core/substrate/subject-fields.js +2 -5
  124. package/dist/core/substrate/surface-match.d.ts +37 -17
  125. package/dist/core/substrate/surface-match.js +44 -53
  126. package/dist/core/tui/page-host.d.ts +6 -0
  127. package/dist/core/tui/page-host.js +12 -3
  128. package/dist/core/worktree.d.ts +8 -0
  129. package/dist/core/worktree.js +65 -0
  130. package/dist/daemon/__tests__/reconciler-signature.test.js +3 -2
  131. package/dist/daemon/__tests__/thaw-order.test.d.ts +1 -0
  132. package/dist/daemon/__tests__/thaw-order.test.js +55 -0
  133. package/dist/daemon/api/__tests__/canvas-snapshot-fields.test.js +6 -0
  134. package/dist/daemon/api/__tests__/node-create-description.test.js +6 -0
  135. package/dist/daemon/api/__tests__/profile-launch-gates.test.js +1 -1
  136. package/dist/daemon/api/bridge.js +5 -1
  137. package/dist/daemon/api/handlers/attach.js +5 -1
  138. package/dist/daemon/api/handlers/bash-jobs.js +1 -1
  139. package/dist/daemon/api/handlers/messages.js +10 -2
  140. package/dist/daemon/api/handlers/nodes.js +8 -1
  141. package/dist/daemon/api/handlers/worktree.js +10 -1
  142. package/dist/daemon/api/map.js +7 -0
  143. package/dist/daemon/companion-retire.js +1 -1
  144. package/dist/daemon/cron/passes.d.ts +0 -5
  145. package/dist/daemon/cron/passes.js +0 -20
  146. package/dist/daemon/cron/sinks.js +1 -1
  147. package/dist/daemon/cron-run.d.ts +5 -6
  148. package/dist/daemon/cron-run.js +7 -16
  149. package/dist/daemon/crtrd.d.ts +0 -10
  150. package/dist/daemon/crtrd.js +23 -61
  151. package/dist/daemon/fleet.d.ts +10 -7
  152. package/dist/daemon/fleet.js +71 -12
  153. package/dist/daemon/messaging/node-message.js +3 -1
  154. package/dist/daemon/profile-delete.js +1 -1
  155. package/dist/daemon/reconcilers/broker-supervision.d.ts +26 -9
  156. package/dist/daemon/reconcilers/broker-supervision.js +39 -52
  157. package/dist/daemon/reconcilers/controller-death.d.ts +7 -0
  158. package/dist/daemon/reconcilers/controller-death.js +26 -0
  159. package/dist/daemon/reconcilers/cron-lane.d.ts +0 -3
  160. package/dist/daemon/reconcilers/cron-lane.js +0 -3
  161. package/dist/daemon/reconcilers/live-obligation.js +3 -1
  162. package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.d.ts +53 -0
  163. package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +170 -0
  164. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +5 -0
  165. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +16 -0
  166. package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -0
  167. package/dist/daemon/reconcilers/node-lifecycle/terminating.js +13 -0
  168. package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +26 -0
  169. package/dist/daemon/reconcilers/node-lifecycle/tick.js +140 -0
  170. package/package.json +1 -1
  171. package/runtime.lock.json +2 -2
  172. package/dist/commands/sys/prompt-review.d.ts +0 -1
  173. package/dist/commands/sys/sysprompt.d.ts +0 -1
  174. package/dist/core/__tests__/cron-broker-capacity.test.js +0 -119
  175. package/dist/daemon/reconcilers/dormant-inbox.d.ts +0 -18
  176. package/dist/daemon/reconcilers/dormant-inbox.js +0 -157
  177. /package/dist/core/__tests__/{cron-broker-capacity.test.d.ts → cron-self-cancel-delivery.test.d.ts} +0 -0
@@ -3,7 +3,7 @@ import { join } from 'node:path';
3
3
  import { homedir } from 'node:os';
4
4
  import { execFile } from 'node:child_process';
5
5
  import { promisify } from 'node:util';
6
- import { getRow, getNode, isWarmSpare, subscribersOf } from '../../core/canvas/index.js';
6
+ import { getNode, isWarmSpare, subscribersOf } from '../../core/canvas/index.js';
7
7
  import { contextDir, isSafeNodeId } from '../../core/canvas/paths.js';
8
8
  import { fullName } from '../../core/canvas/labels.js';
9
9
  import { recordedPidLiveness } from '../../core/canvas/pid.js';
@@ -14,7 +14,6 @@ import { nextFaultRetry } from '../../core/runtime/fault-recovery.js';
14
14
  import { fanDoctrineWake } from '../../core/runtime/close.js';
15
15
  import { headlessBrokerHost } from '../../core/runtime/host.js';
16
16
  import { reloadAuthLive } from '../../core/runtime/auth-reload.js';
17
- import { readConfig } from '../../core/config.js';
18
17
  import { emitEvent } from '../../core/events/emit.js';
19
18
  import { operationIdContext } from '../../core/events/operation-id.js';
20
19
  import { hasUnseenWakeMail } from '../../core/feed/inbox.js';
@@ -24,22 +23,16 @@ import { hasAttachedBrokerViewers } from '../../core/runtime/broker/client-regis
24
23
  import { deliverLive } from '../../core/runtime/interactive-deliver.js';
25
24
  import { BrokerUnreachableError } from '../../core/runtime/broker-request.js';
26
25
  import { formatCard, PARK_SUMMARY_PROMPT } from '../../shared/generated-context.js';
26
+ import { brokerThresholdsForDaemon } from '../../core/runtime/fleet.js';
27
27
  import { beginPendingPark, consumePendingPark, pendingParkIds, pendingParkStartedAt } from '../park-pending.js';
28
28
  import { applyParkEvent, parkEvent } from './live-obligation.js';
29
- import { envMaxLiveBrokers, envNoDaemonAutostart, envTestParkSummaryGraceMs, envTestUnattendedParkMs, envWarnLiveBrokers, } from '../../shared/env.js';
29
+ import { envNoDaemonAutostart, envTestParkSummaryGraceMs, envTestUnattendedParkMs, } from '../../shared/env.js';
30
30
  const execFileAsync = promisify(execFile);
31
- export function brokerThresholdsForDaemon() {
32
- const configured = readConfig('user').brokerThresholds;
33
- const automaticReviveCap = envMaxLiveBrokers() ?? configured.automaticReviveCap;
34
- const warning = envWarnLiveBrokers() ?? configured.warning;
35
- return { warning: Math.min(warning, automaticReviveCap), automaticReviveCap };
36
- }
37
31
  export const YIELD_STALL_GRACE_MS = 3 * 60_000;
38
32
  const KILL_ESCALATE_MS = 20_000;
39
33
  export const WEDGE_QUIET_MS = 20 * 60_000;
40
34
  export const WEDGE_CPU_MAX_PERCENT = 2;
41
35
  const OVER_CAP_WARN_THROTTLE_MS = 5 * 60_000;
42
- const CAPACITY_LOG_THROTTLE_MS = 30_000;
43
36
  const DEFAULT_UNATTENDED_PARK_MS = 15 * 60_000;
44
37
  // Longer than a summary turn and shorter than the unattended clock itself, so a
45
38
  // stuck parking turn cannot hold a broker for a whole extra clock period.
@@ -138,22 +131,31 @@ export class BrokerSupervisionReconciler {
138
131
  unattendedSince = new Map();
139
132
  knownAuthMtimeMs = null;
140
133
  lastOverCapWarnAt = Number.NEGATIVE_INFINITY;
141
- lastCapacityLogAt = Number.NEGATIVE_INFINITY;
134
+ /** The fleet-wide preamble, run BEFORE the tick's row-list guard: an auth
135
+ * reload does not depend on a successful canvas scan, and the thresholds it
136
+ * returns are read once for the whole tick. */
137
+ async runFleetPreamble(fleet) {
138
+ await this.handleAuthReload(fleet);
139
+ return brokerThresholdsForDaemon();
140
+ }
141
+ /** The saturation notice, run LAST in the tick: the frozen split it reports
142
+ * is only known once the freeze lane has spent this tick's free slots. */
143
+ warnIfSaturated(ctx) {
144
+ if (ctx.fleet.size() < ctx.thresholds.warning || !ctx.lifecycle.acceptsDetachedWork())
145
+ return;
146
+ const warning = operationIdContext.fresh(() => this.warnOverCapInTmux(ctx.fleet.size(), ctx.thresholds, ctx.frozen));
147
+ ctx.lifecycle.registerDetached(warning);
148
+ }
142
149
  async run(now, ctx) {
143
- await this.handleAuthReload(ctx.fleet);
144
- const thresholds = brokerThresholdsForDaemon();
145
- if (ctx.fleet.size() >= thresholds.warning && ctx.lifecycle.acceptsDetachedWork()) {
146
- const warning = operationIdContext.fresh(() => this.warnOverCapInTmux(ctx.fleet.size(), thresholds.warning, thresholds.automaticReviveCap));
147
- ctx.lifecycle.registerDetached(warning);
148
- }
149
- for (const [id, entry] of Array.from(ctx.fleet.entries())) {
150
+ for (const row of ctx.rows) {
151
+ const id = row.node_id;
152
+ const entry = ctx.fleet.get(id);
153
+ if (entry === undefined || (row.status !== 'active' && row.status !== 'idle')) {
154
+ this.unattendedSince.delete(id);
155
+ continue;
156
+ }
150
157
  await operationIdContext.fresh(async () => {
151
158
  try {
152
- const row = getRow(id);
153
- if (row === null || (row.status !== 'active' && row.status !== 'idle')) {
154
- this.unattendedSince.delete(id);
155
- return;
156
- }
157
159
  this.handleYieldStall(row, entry.pid, now);
158
160
  await this.handleWedgeDetection(id, entry.pid, row.pi_pid_identity, now);
159
161
  this.handleFatalFault(id);
@@ -180,7 +182,6 @@ export class BrokerSupervisionReconciler {
180
182
  if (!ctx.fleet.has(id))
181
183
  consumePendingPark(id);
182
184
  }
183
- return thresholds;
184
185
  }
185
186
  /** Every condition that resets the unattended clock EXCEPT `isBusy` — the
186
187
  * node is unavailable to park, attended, or has work waiting. Short-circuits
@@ -325,17 +326,28 @@ export class BrokerSupervisionReconciler {
325
326
  });
326
327
  }
327
328
  }
328
- async warnOverCapInTmux(count, warning, max) {
329
+ async warnOverCapInTmux(count, thresholds, frozen) {
329
330
  const now = Date.now();
330
331
  if (now - this.lastOverCapWarnAt < OVER_CAP_WARN_THROTTLE_MS)
331
332
  return;
332
333
  this.lastOverCapWarnAt = now;
334
+ const { warning, automaticReviveCap: max } = thresholds;
335
+ const waiting = frozen.stalled + frozen.waitingOnMail;
333
336
  emitEvent({
334
337
  level: 'warn',
335
338
  event: 'broker.capacity.warning',
336
- fields: { live_count: count, warning_threshold: warning, hard_cap: max },
339
+ fields: {
340
+ live_count: count,
341
+ warning_threshold: warning,
342
+ hard_cap: max,
343
+ frozen_stalled: frozen.stalled,
344
+ frozen_waiting_on_mail: frozen.waitingOnMail,
345
+ },
337
346
  });
338
- const msg = `⚠ crtrd: ${count} live brokers (warning ${warning}; hard cap ${max}) — possible broker leak. ps -ax | grep broker-cli.js`;
347
+ const frozenNote = waiting === 0
348
+ ? ''
349
+ : ` — ${waiting} frozen (${frozen.stalled} stalled, ${frozen.waitingOnMail} waiting on mail)`;
350
+ const msg = `⚠ crtrd: ${count} live brokers (warning ${warning}; hard cap ${max})${frozenNote} — possible broker leak. ps -ax | grep broker-cli.js`;
339
351
  try {
340
352
  const clients = await execFileAsync('tmux', ['list-clients', '-F', '#{client_name}'], {
341
353
  encoding: 'utf8',
@@ -637,29 +649,4 @@ export class BrokerSupervisionReconciler {
637
649
  }
638
650
  });
639
651
  }
640
- hasBrokerCapacity(max, fleet) {
641
- if (fleet.size() < max)
642
- return true;
643
- const now = Date.now();
644
- if (now - this.lastCapacityLogAt >= CAPACITY_LOG_THROTTLE_MS) {
645
- this.lastCapacityLogAt = now;
646
- emitEvent({
647
- level: 'warn',
648
- event: 'broker.capacity.revive_deferred',
649
- fields: { live_count: fleet.size(), hard_cap: max },
650
- });
651
- }
652
- return false;
653
- }
654
- hasStartupRecoveryCapacity(fleet) {
655
- const { automaticReviveCap } = brokerThresholdsForDaemon();
656
- if (fleet.size() < automaticReviveCap)
657
- return true;
658
- emitEvent({
659
- level: 'warn',
660
- event: 'daemon.recovery_sweep.capacity_deferred',
661
- fields: { live_count: fleet.size() },
662
- });
663
- return false;
664
- }
665
652
  }
@@ -0,0 +1,7 @@
1
+ import { type NodeRow } from '../../core/canvas/index.js';
2
+ export interface ControllerDeathContext {
3
+ rows: readonly NodeRow[];
4
+ }
5
+ export declare class ControllerDeathReconciler {
6
+ run(now: number, ctx: ControllerDeathContext): void;
7
+ }
@@ -0,0 +1,26 @@
1
+ import { getRow } from '../../core/canvas/index.js';
2
+ import { operationIdContext } from '../../core/events/operation-id.js';
3
+ import { appendControllerDeathNotice } from '../../core/feed/inbox.js';
4
+ export class ControllerDeathReconciler {
5
+ run(now, ctx) {
6
+ void now; // no time-gating here — the death check is already-controller-status-driven; the param is kept for six-way call-site uniformity, matching every other reconciler's run(now, ctx) shape.
7
+ // Settle every wait whose controller has ended, independent of the waiter's
8
+ // current status. Active and focused terminal nodes own a live inbox
9
+ // watcher; an idle-released node is revived by the node-lifecycle tick.
10
+ for (const row of ctx.rows) {
11
+ const current = getRow(row.node_id);
12
+ const controllerId = current?.waiting_for;
13
+ if (controllerId === null || controllerId === undefined)
14
+ continue;
15
+ // Read-gate before the write boundary: only a dead controller warrants a
16
+ // notice. appendControllerDeathNotice re-checks liveness inside its
17
+ // transaction, so this gate cannot misfire.
18
+ const controller = getRow(controllerId);
19
+ if (controller !== null && (controller.status === 'active' || controller.status === 'idle'))
20
+ continue;
21
+ operationIdContext.fresh(() => {
22
+ appendControllerDeathNotice(row.node_id, controllerId);
23
+ });
24
+ }
25
+ }
26
+ }
@@ -1,9 +1,6 @@
1
1
  import type { DetachedWorkLifecycle } from './broker-supervision.js';
2
2
  export interface CronLaneContext {
3
3
  lifecycle: DetachedWorkLifecycle;
4
- /** P3.4's capacity authority, bound to the thresholds returned by the live
5
- * supervision pass for this same tick. */
6
- hasBrokerCapacity(): boolean;
7
4
  }
8
5
  export declare class CronLaneReconciler {
9
6
  run(now: number, ctx: CronLaneContext): void;
@@ -20,9 +20,6 @@ export class CronLaneReconciler {
20
20
  try {
21
21
  runDueCrons(now, {
22
22
  register: (settlement) => ctx.lifecycle.registerDetached(settlement),
23
- // A due run that would launch or revive a node stays due until P3.4's
24
- // capacity authority admits it.
25
- hasBrokerCapacity: ctx.hasBrokerCapacity,
26
23
  });
27
24
  }
28
25
  catch (err) {
@@ -45,7 +45,9 @@ export function parkEvent(nodeId, lifecycle) {
45
45
  * (broker settlement after a parking turn) enacts `park` directly and fans
46
46
  * nothing — the node's report is the notice. */
47
47
  export function applyParkEvent(nodeId, event) {
48
- const node = transition(nodeId, event);
48
+ const node = event === 'cancel'
49
+ ? transition(nodeId, event, { reason: 'stranded' })
50
+ : transition(nodeId, event);
49
51
  if (event !== 'park')
50
52
  return;
51
53
  fanDoctrineWake(nodeId, subscribersOf(nodeId), `Child done — ${fullName(node)} (${nodeId}) was automatically marked done after its unattended grace with nothing left to wake it. It sent no final report.`, { reason: 'child-auto-done', child: nodeId });
@@ -0,0 +1,53 @@
1
+ import { type NodeRow } from '../../../core/canvas/index.js';
2
+ import type { FleetRegistry } from '../../../core/runtime/fleet.js';
3
+ /** How a candidate spends a slot when its turn comes. `thaw` simply launches a
4
+ * row the cap turned away — its disposition was decided when it froze, and the
5
+ * dead-row table has no jurisdiction over a broker that never existed.
6
+ * `policy` re-enacts the dead-row policy table (which relaunches through
7
+ * `reviveNode`); `idle-release` runs the stuck-delivery episode rules before
8
+ * relaunching. */
9
+ export type ThawEnactment = 'thaw' | 'policy' | 'idle-release';
10
+ export interface ThawCandidate {
11
+ row: NodeRow;
12
+ enact: ThawEnactment;
13
+ }
14
+ /** Tier 1 is a row with nothing waiting for it that simply stopped executing —
15
+ * the stalled set, served first. Tier 2 is a row waiting on mail, plus a row
16
+ * that held a slot last time and settled nothing with it. */
17
+ export type ThawTier = 1 | 2;
18
+ export interface ThawEntry extends ThawCandidate {
19
+ tier: ThawTier;
20
+ }
21
+ export interface FreezeLaneContext {
22
+ fleet: FleetRegistry;
23
+ candidates: readonly ThawCandidate[];
24
+ }
25
+ /** What stayed frozen this tick, split the way the saturation warning reports
26
+ * it. Rows in backoff never reach this lane and are excluded by construction. */
27
+ export interface FrozenCounts {
28
+ stalled: number;
29
+ waitingOnMail: number;
30
+ }
31
+ /** Which tier a candidate competes in. A row that was launched and never
32
+ * settled a turn drops to tier 2: it already spent a slot without producing
33
+ * anything, so it must not keep outranking the mail it stalled behind. A row
34
+ * that never launched at all has spent nothing and stays tier 1. */
35
+ export declare function thawTier(row: NodeRow, pendingWake: boolean): ThawTier;
36
+ /** Thaw order: tier first, then longest-frozen first. A candidate with no
37
+ * `frozen_at` has not waited at all and sorts last within its tier. */
38
+ export declare function compareThaw(a: ThawEntry, b: ThawEntry): number;
39
+ export declare class FreezeLaneReconciler {
40
+ private readonly relaunchState;
41
+ /** End a stuck-delivery episode the tick decided is over — the node's profile
42
+ * paused, or nothing wake-worthy is left in its inbox. */
43
+ endEpisode(nodeId: string): void;
44
+ run(now: number, ctx: FreezeLaneContext): Promise<FrozenCounts>;
45
+ private enact;
46
+ /** A frozen row is owed exactly one thing: the launch the cap refused. Strict
47
+ * resume is the request, and `reviveNode` itself vetoes it for a pending
48
+ * refresh or an unconfirmed cycle — the same modes the dead-row table would
49
+ * have chosen. It is recovery, not a wake: nothing arrived for this node
50
+ * while it waited, so an armed deadline survives the thaw. */
51
+ private enactThaw;
52
+ private enactIdleRelease;
53
+ }
@@ -0,0 +1,170 @@
1
+ // freeze-lane.ts — who gets the free slots, in what order, and what happens to
2
+ // everyone else.
3
+ //
4
+ // The tick hands this lane every row that wants execution and has no engine.
5
+ // Enactment is strictly sequential on the daemon's serial work lane, so the
6
+ // ledger inside `headlessBrokerHost.launch` hands slots out in exactly the
7
+ // order this lane enacts: ordering IS the allocation policy, and no candidate
8
+ // needs to pre-reserve anything. A candidate that finds the ledger full freezes
9
+ // (`reviveNode` with `capacity: 'freeze'` is the single freeze writer) and waits
10
+ // for the next tick.
11
+ //
12
+ // The idle-release episode rules below are the former DormantInboxReconciler's,
13
+ // moved here whole: a relaunch that only ever tried to deliver mail is a launch
14
+ // like any other, and it now competes for slots through the same lane.
15
+ import { getRow, subscribersOf, } from '../../../core/canvas/index.js';
16
+ import { fullName } from '../../../core/canvas/labels.js';
17
+ import { isSafeNodeId } from '../../../core/canvas/paths.js';
18
+ import { emitEvent } from '../../../core/events/emit.js';
19
+ import { operationIdContext } from '../../../core/events/operation-id.js';
20
+ import { readCursor } from '../../../core/feed/inbox.js';
21
+ import { fanDoctrineWake } from '../../../core/runtime/close.js';
22
+ import { transition } from '../../../core/runtime/lifecycle.js';
23
+ import { reviveNode } from '../../../core/runtime/revive.js';
24
+ import { hasPendingWake } from '../live-obligation.js';
25
+ // Idle-release/inbox retry cap (2026-07-10 diagnosis, round 2): the SAME
26
+ // livelock shape as the stranded-relaunch case, but on the pass that resumes a
27
+ // dormant idle-released node when its inbox gains an unseen entry. The counter
28
+ // is keyed on inbox-cursor progress, not row-state residency or pid liveness: a
29
+ // transiently-live broker does not prove that delivery completed.
30
+ const IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS = 6;
31
+ const IDLE_RELEASE_RELAUNCH_MAX_BACKOFF_MS = 10 * 60_000;
32
+ const IDLE_RELEASE_RELAUNCH_BASE_MS = 20_000;
33
+ /** Which tier a candidate competes in. A row that was launched and never
34
+ * settled a turn drops to tier 2: it already spent a slot without producing
35
+ * anything, so it must not keep outranking the mail it stalled behind. A row
36
+ * that never launched at all has spent nothing and stays tier 1. */
37
+ export function thawTier(row, pendingWake) {
38
+ if (pendingWake)
39
+ return 2;
40
+ const launched = row.launched_at;
41
+ if (launched == null)
42
+ return 1;
43
+ const settled = row.last_settled_at;
44
+ if (settled == null)
45
+ return 2;
46
+ return Date.parse(settled) > Date.parse(launched) ? 1 : 2;
47
+ }
48
+ /** Thaw order: tier first, then longest-frozen first. A candidate with no
49
+ * `frozen_at` has not waited at all and sorts last within its tier. */
50
+ export function compareThaw(a, b) {
51
+ if (a.tier !== b.tier)
52
+ return a.tier - b.tier;
53
+ const af = a.row.frozen_at;
54
+ const bf = b.row.frozen_at;
55
+ if (af == null)
56
+ return bf == null ? 0 : 1;
57
+ if (bf == null)
58
+ return -1;
59
+ return Date.parse(af) - Date.parse(bf);
60
+ }
61
+ export class FreezeLaneReconciler {
62
+ relaunchState = new Map();
63
+ /** End a stuck-delivery episode the tick decided is over — the node's profile
64
+ * paused, or nothing wake-worthy is left in its inbox. */
65
+ endEpisode(nodeId) {
66
+ this.relaunchState.delete(nodeId);
67
+ }
68
+ async run(now, ctx) {
69
+ const entries = ctx.candidates.map((candidate) => ({
70
+ ...candidate,
71
+ tier: thawTier(candidate.row, hasPendingWake(candidate.row.node_id)),
72
+ }));
73
+ entries.sort(compareThaw);
74
+ const counts = { stalled: 0, waitingOnMail: 0 };
75
+ for (const entry of entries) {
76
+ const outcome = await operationIdContext.fresh(() => this.enact(now, entry, ctx.fleet));
77
+ if (outcome !== 'frozen')
78
+ continue;
79
+ if (entry.tier === 1)
80
+ counts.stalled += 1;
81
+ else
82
+ counts.waitingOnMail += 1;
83
+ }
84
+ return counts;
85
+ }
86
+ async enact(now, entry, fleet) {
87
+ const id = entry.row.node_id;
88
+ try {
89
+ if (entry.enact === 'thaw')
90
+ return this.enactThaw(id);
91
+ if (entry.enact === 'idle-release')
92
+ return this.enactIdleRelease(now, entry.row);
93
+ // The SAME policy table an exit event runs, so tick recovery and exit
94
+ // recovery can never diverge. It relaunches through
95
+ // `reviveNode(capacity: 'freeze')`, which freezes the row when the ledger
96
+ // is full — the outcome this lane reports as saturation.
97
+ await fleet.applyDeadRowPolicy(id);
98
+ if (fleet.has(id))
99
+ return 'launched';
100
+ return getRow(id)?.frozen_at != null ? 'frozen' : 'skipped';
101
+ }
102
+ catch (err) {
103
+ emitEvent({
104
+ level: 'error',
105
+ event: 'node.lifecycle.enact_failed',
106
+ ...(isSafeNodeId(id) ? { node_id: id } : {}),
107
+ error: err,
108
+ fields: { ...(isSafeNodeId(id) ? {} : { affected_node_id: id }), enact: entry.enact },
109
+ });
110
+ return 'skipped';
111
+ }
112
+ }
113
+ /** A frozen row is owed exactly one thing: the launch the cap refused. Strict
114
+ * resume is the request, and `reviveNode` itself vetoes it for a pending
115
+ * refresh or an unconfirmed cycle — the same modes the dead-row table would
116
+ * have chosen. It is recovery, not a wake: nothing arrived for this node
117
+ * while it waited, so an armed deadline survives the thaw. */
118
+ enactThaw(nodeId) {
119
+ return reviveNode(nodeId, { resume: true, capacity: 'freeze', recovery: true }).outcome === 'frozen'
120
+ ? 'frozen'
121
+ : 'launched';
122
+ }
123
+ enactIdleRelease(now, row) {
124
+ const id = row.node_id;
125
+ const cursorNow = readCursor(id);
126
+ const prior = this.relaunchState.get(id);
127
+ const cursorChanged = prior === undefined || prior.cursor !== cursorNow;
128
+ const attempts = cursorChanged ? 0 : prior.attempts;
129
+ if (attempts >= IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS) {
130
+ this.relaunchState.delete(id);
131
+ emitEvent({
132
+ level: 'warn',
133
+ event: 'broker.idle_release.exhausted',
134
+ ...(isSafeNodeId(id) ? { node_id: id } : {}),
135
+ fields: {
136
+ ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
137
+ attempts,
138
+ max_attempts: IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS,
139
+ },
140
+ });
141
+ transition(id, 'crash');
142
+ fanDoctrineWake(id, subscribersOf(id), `Child crashed — ${fullName(row)} (${id}) failed to relaunch after ${attempts} attempts while trying to ` +
143
+ `deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays ` +
144
+ `dead until you revive it — \`crtr node lifecycle revive ${id}\`.`, { reason: 'child-crashed', child: id });
145
+ return 'skipped';
146
+ }
147
+ const lastAttempt = cursorChanged ? undefined : prior?.lastAttempt;
148
+ const backoff = Math.min(IDLE_RELEASE_RELAUNCH_BASE_MS * 2 ** attempts, IDLE_RELEASE_RELAUNCH_MAX_BACKOFF_MS);
149
+ if (lastAttempt !== undefined && now - lastAttempt < backoff)
150
+ return 'skipped';
151
+ // A relaunch the broker cap froze got no slot and therefore proved nothing
152
+ // about this delivery, so it must not spend an attempt: only a launch that
153
+ // actually happened advances the episode toward exhaustion.
154
+ if (reviveNode(id, { resume: true, capacity: 'freeze' }).outcome === 'frozen')
155
+ return 'frozen';
156
+ this.relaunchState.set(id, { cursor: cursorNow, attempts: attempts + 1, lastAttempt: now });
157
+ emitEvent({
158
+ level: 'info',
159
+ event: 'broker.revive.idle_release',
160
+ ...(isSafeNodeId(id) ? { node_id: id } : {}),
161
+ attempt: attempts + 1,
162
+ fields: {
163
+ ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
164
+ attempts: attempts + 1,
165
+ max_attempts: IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS,
166
+ },
167
+ });
168
+ return 'launched';
169
+ }
170
+ }
@@ -0,0 +1,5 @@
1
+ import type { NodeRow } from '../../../core/canvas/index.js';
2
+ /** Whether a durable backoff still holds this row back from a relaunch. A row
3
+ * in backoff holds no slot and is excluded from the saturation figure — it is
4
+ * waiting on a clock, not on capacity. */
5
+ export declare function isInBackoff(row: NodeRow, now: number): boolean;
@@ -0,0 +1,16 @@
1
+ // respawn-policy.ts — the tick's Backoff gate, and P5's file to own outright.
2
+ //
3
+ // P3 only READS `respawn_not_before`: nothing writes it yet, so the gate is
4
+ // inert in effect. P5 replaces the in-memory throttle in `daemon/fleet.ts` with
5
+ // the durable counter writes (`respawn_failures` + `respawn_not_before`) that
6
+ // make it bite, and adds them here rather than in `tick.ts`.
7
+ /** Whether a durable backoff still holds this row back from a relaunch. A row
8
+ * in backoff holds no slot and is excluded from the saturation figure — it is
9
+ * waiting on a clock, not on capacity. */
10
+ export function isInBackoff(row, now) {
11
+ const notBefore = row.respawn_not_before;
12
+ if (notBefore == null)
13
+ return false;
14
+ const at = Date.parse(notBefore);
15
+ return Number.isFinite(at) && now < at;
16
+ }
@@ -0,0 +1,4 @@
1
+ import type { NodeRow } from '../../../core/canvas/index.js';
2
+ /** Handle a terminal row that still holds a process. Deliberately inert until
3
+ * P4: a row reaching here is already excluded from every other tick duty. */
4
+ export declare function handleTerminatingRow(row: NodeRow): void;
@@ -0,0 +1,13 @@
1
+ // terminating.ts — the Terminating state's owner, and P4's file to own outright.
2
+ //
3
+ // A terminal row that still holds a process is a node whose teardown has been
4
+ // requested but not finished. P4 gives it one bounded shutdown request and an
5
+ // escalation after its grace, moving the inline `headlessBrokerHost.teardown`
6
+ // calls at the close/reset sites into this daemon-side lane. P3 changes no
7
+ // teardown behavior: it only OBSERVES such rows (the tick queries terminal rows
8
+ // carrying a recorded pid) and hands them here, so P4 edits this file alone.
9
+ /** Handle a terminal row that still holds a process. Deliberately inert until
10
+ * P4: a row reaching here is already excluded from every other tick duty. */
11
+ export function handleTerminatingRow(row) {
12
+ void row;
13
+ }
@@ -0,0 +1,26 @@
1
+ import { type NodeRow } from '../../../core/canvas/index.js';
2
+ import type { FleetRegistry } from '../../../core/runtime/fleet.js';
3
+ import { type FrozenCounts } from './freeze-lane.js';
4
+ export interface NodeLifecycleContext {
5
+ /** Every active and idle row (human rows excluded), plus terminal rows that
6
+ * still hold a recorded pid. */
7
+ rows: readonly NodeRow[];
8
+ fleet: FleetRegistry;
9
+ }
10
+ /** Whether this row should have a live engine right now. An active row always
11
+ * should — that status IS the claim that it is executing. An idle row was
12
+ * deliberately released and holds no engine on purpose: only mail it has not
13
+ * seen brings one back, because a live obligation keeps a node WAKEABLE rather
14
+ * than awake, and relaunching on one would re-release it every tick. */
15
+ export declare function wantsExecution(row: NodeRow): boolean;
16
+ export declare class NodeLifecycleReconciler {
17
+ private readonly freezeLane;
18
+ run(now: number, ctx: NodeLifecycleContext): Promise<FrozenCounts>;
19
+ private classifyRow;
20
+ private thawMark;
21
+ /** An idle row with no wake-worthy inbox entry still needs the lifecycle
22
+ * settlement the live park clock applies: once nothing live remains, this is
23
+ * history rather than a permanently protected idle row. The row is already
24
+ * parked, so only a terminalizing verdict is worth writing. */
25
+ private settleParked;
26
+ }