@north-light/crouter 0.3.249 → 0.3.251

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/README.md +1 -1
  2. package/dist/api/__tests__/integration/client.test.js +28 -7
  3. package/dist/api/client.d.ts +1 -1
  4. package/dist/api/client.js +2 -2
  5. package/dist/api/dto/lifecycle.d.ts +3 -0
  6. package/dist/api/dto/messages.d.ts +4 -0
  7. package/dist/api/dto/nodes.d.ts +4 -0
  8. package/dist/api/dto/reports.d.ts +7 -2
  9. package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
  10. package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
  11. package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
  12. package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
  13. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
  14. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
  15. package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
  16. package/dist/clients/attach/session/chrome-refresh.js +1 -0
  17. package/dist/clients/attach/viewer.js +532 -532
  18. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
  19. package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
  20. package/dist/clients/inbox/controller.js +32 -5
  21. package/dist/clients/inbox/tui/ansi.d.ts +3 -1
  22. package/dist/clients/inbox/tui/ansi.js +14 -10
  23. package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
  24. package/dist/commands/api-client.d.ts +2 -2
  25. package/dist/commands/api-client.js +5 -5
  26. package/dist/commands/canvas-browse.js +1 -1
  27. package/dist/commands/node/create.js +3 -1
  28. package/dist/commands/node/inspect.js +1 -0
  29. package/dist/commands/node/lifecycle.js +1 -1
  30. package/dist/commands/node/message.js +6 -4
  31. package/dist/commands/node/wait.js +1 -1
  32. package/dist/commands/push.js +36 -37
  33. package/dist/commands/sys/daemon.js +21 -8
  34. package/dist/commands/sys/setup-core.js +2 -2
  35. package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
  36. package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
  37. package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
  38. package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
  39. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +100 -1
  40. package/dist/core/__tests__/child-death-wake.test.js +11 -7
  41. package/dist/core/__tests__/collectible.test.d.ts +1 -0
  42. package/dist/core/__tests__/collectible.test.js +88 -0
  43. package/dist/core/__tests__/context-intro.test.js +4 -41
  44. package/dist/core/__tests__/daemon-boot.test.js +30 -14
  45. package/dist/core/__tests__/daemon-wedge.test.js +25 -19
  46. package/dist/core/__tests__/fixtures/fake-engine.js +20 -2
  47. package/dist/core/__tests__/human-deliver.test.js +5 -5
  48. package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
  49. package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
  50. package/dist/core/__tests__/integration/revive.test.js +48 -0
  51. package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
  52. package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
  53. package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
  54. package/dist/core/__tests__/migration.test.js +30 -0
  55. package/dist/core/__tests__/passive-subscription.test.js +39 -1
  56. package/dist/core/__tests__/prune.test.js +12 -3
  57. package/dist/core/__tests__/relaunch-root.test.js +1 -1
  58. package/dist/core/__tests__/respawn-throttle.test.js +16 -146
  59. package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
  60. package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
  61. package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
  62. package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
  63. package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
  64. package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
  65. package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
  66. package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
  67. package/dist/core/canvas/canvas.d.ts +9 -15
  68. package/dist/core/canvas/canvas.js +41 -47
  69. package/dist/core/canvas/crons.d.ts +5 -8
  70. package/dist/core/canvas/crons.js +5 -8
  71. package/dist/core/canvas/history.d.ts +5 -0
  72. package/dist/core/canvas/history.js +12 -1
  73. package/dist/core/canvas/migrations.js +58 -1
  74. package/dist/core/canvas/pid.d.ts +1 -1
  75. package/dist/core/canvas/pid.js +5 -2
  76. package/dist/core/canvas/types.d.ts +4 -0
  77. package/dist/core/feed/feed.d.ts +4 -0
  78. package/dist/core/feed/feed.js +13 -8
  79. package/dist/core/feed/inbox.d.ts +2 -1
  80. package/dist/core/feed/inbox.js +22 -3
  81. package/dist/core/git.d.ts +3 -0
  82. package/dist/core/git.js +3 -1
  83. package/dist/core/human/feedback-companion.js +1 -0
  84. package/dist/core/preview-registry.js +0 -1
  85. package/dist/core/review/companion.js +1 -0
  86. package/dist/core/review/realize.js +1 -0
  87. package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
  88. package/dist/core/runtime/broker/auth-reload.js +9 -3
  89. package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
  90. package/dist/core/runtime/broker/engine-drive.js +39 -23
  91. package/dist/core/runtime/broker/event-projection.d.ts +4 -0
  92. package/dist/core/runtime/broker/event-projection.js +43 -23
  93. package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
  94. package/dist/core/runtime/broker/fault-retry.js +94 -8
  95. package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
  96. package/dist/core/runtime/broker/frame-dispatch.js +4 -1
  97. package/dist/core/runtime/broker/held-deferred-inbox.d.ts +9 -0
  98. package/dist/core/runtime/broker/held-deferred-inbox.js +8 -0
  99. package/dist/core/runtime/broker/inbox.js +4 -0
  100. package/dist/core/runtime/broker/rebind.js +18 -0
  101. package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
  102. package/dist/core/runtime/broker/turn-admission.js +31 -0
  103. package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
  104. package/dist/core/runtime/broker/turn-ignition.js +25 -15
  105. package/dist/core/runtime/broker.js +21 -2
  106. package/dist/core/runtime/close.d.ts +11 -9
  107. package/dist/core/runtime/close.js +23 -19
  108. package/dist/core/runtime/fault.d.ts +21 -0
  109. package/dist/core/runtime/fault.js +129 -19
  110. package/dist/core/runtime/fleet.d.ts +0 -6
  111. package/dist/core/runtime/host.d.ts +7 -1
  112. package/dist/core/runtime/host.js +20 -2
  113. package/dist/core/runtime/nodes.d.ts +5 -1
  114. package/dist/core/runtime/nodes.js +24 -1
  115. package/dist/core/runtime/placement.d.ts +6 -6
  116. package/dist/core/runtime/placement.js +15 -8
  117. package/dist/core/runtime/reset.js +2 -2
  118. package/dist/core/runtime/revive.js +40 -4
  119. package/dist/core/runtime/spawn.d.ts +3 -0
  120. package/dist/core/runtime/spawn.js +1 -0
  121. package/dist/core/runtime/warm-pool.d.ts +4 -0
  122. package/dist/core/runtime/warm-pool.js +2 -0
  123. package/dist/core/worktree.d.ts +4 -3
  124. package/dist/core/worktree.js +73 -27
  125. package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
  126. package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
  127. package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
  128. package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
  129. package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
  130. package/dist/daemon/api/handlers/bash-jobs.js +2 -0
  131. package/dist/daemon/api/handlers/broker-ops.js +23 -2
  132. package/dist/daemon/api/handlers/human.js +1 -1
  133. package/dist/daemon/api/handlers/messages.js +11 -2
  134. package/dist/daemon/api/handlers/nodes.js +13 -1
  135. package/dist/daemon/api/handlers/reports.js +14 -6
  136. package/dist/daemon/api/map.js +1 -0
  137. package/dist/daemon/api/server.d.ts +9 -12
  138. package/dist/daemon/api/server.js +30 -19
  139. package/dist/daemon/companion-retire.js +3 -9
  140. package/dist/daemon/cron/sinks.js +7 -2
  141. package/dist/daemon/crtrd.js +4 -1
  142. package/dist/daemon/fleet.d.ts +2 -16
  143. package/dist/daemon/fleet.js +63 -150
  144. package/dist/daemon/human/finish.js +2 -0
  145. package/dist/daemon/manage.d.ts +11 -0
  146. package/dist/daemon/manage.js +32 -3
  147. package/dist/daemon/messaging/node-message.js +7 -1
  148. package/dist/daemon/reconcilers/bash-deadline.js +2 -0
  149. package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
  150. package/dist/daemon/reconcilers/broker-supervision.js +22 -9
  151. package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
  152. package/dist/daemon/reconcilers/live-obligation.js +18 -6
  153. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
  154. package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
  155. package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
  156. package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
  157. package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
  158. package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
  159. package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
  160. package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
  161. package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
  162. package/dist/pi-extensions/canvas-context-intro.js +4 -76
  163. package/dist/pi-extensions/canvas-inbox-watcher.js +105 -4
  164. package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
  165. package/dist/pi-extensions/canvas-review-boundary.js +4 -29
  166. package/dist/pi-extensions/canvas-stophook.js +1 -6
  167. package/dist/shared/birth-announcement.d.ts +12 -0
  168. package/dist/shared/birth-announcement.js +25 -0
  169. package/dist/shared/generated-context.d.ts +8 -4
  170. package/dist/shared/generated-context.js +15 -9
  171. package/package.json +4 -4
  172. package/runtime.lock.json +2 -2
  173. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
  174. package/dist/pi-extensions/truncate.d.ts +0 -5
  175. package/dist/pi-extensions/truncate.js +0 -14
@@ -16,6 +16,7 @@ import { hostExecPath } from '../core/runtime/branded-host.js';
16
16
  import { isDaemonRunning, readPidfile, isPidAlive } from './pidfile.js';
17
17
  import { DAEMON_STARTUP_WINDOW_MS } from './startup-policy.js';
18
18
  import { envNoDaemonAutostart } from '../shared/env.js';
19
+ import { CrtrClient } from '../api/index.js';
19
20
  // Daemon env sanitization
20
21
  /** Env keys that must NEVER reach the daemon process. Restarting crtrd is
21
22
  * overwhelmingly done from inside an agent node's own bash tool (`crtr sys
@@ -201,6 +202,24 @@ export async function execDaemon() {
201
202
  // Shutdown stays short even though startup may include corpus migration.
202
203
  const DAEMON_SHUTDOWN_WINDOW_MS = 20_000;
203
204
  const DAEMON_VERIFY_POLL_MS = 10;
205
+ /** A status/start readiness probe must not hang behind a daemon that accepts a
206
+ * Unix connection but never answers it. */
207
+ const DAEMON_HEALTH_PROBE_TIMEOUT_MS = 1_000;
208
+ /** True only when the mandatory Unix API socket answers `/healthz`. This is
209
+ * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
210
+ * identifying the process an operator must stop, but it is not readiness. */
211
+ export async function isDaemonServing() {
212
+ try {
213
+ const health = await CrtrClient.forLocalSocket({
214
+ autostart: false,
215
+ timeoutMs: DAEMON_HEALTH_PROBE_TIMEOUT_MS,
216
+ }).healthz();
217
+ return health.ok;
218
+ }
219
+ catch {
220
+ return false;
221
+ }
222
+ }
204
223
  function sleepMs(ms) {
205
224
  return new Promise((resolve) => {
206
225
  setTimeout(resolve, ms);
@@ -223,12 +242,13 @@ export async function verifyDaemonStartup(pid, timeoutMs = DAEMON_STARTUP_WINDOW
223
242
  const read = deps.readPidfile ?? readPidfile;
224
243
  const alive = deps.isPidAlive ?? isPidAlive;
225
244
  const exited = deps.childExited;
245
+ const serving = deps.isDaemonServing ?? isDaemonServing;
226
246
  const sleep = deps.sleepMs ?? sleepMs;
227
247
  const now = deps.now ?? Date.now;
228
248
  const deadline = now() + timeoutMs;
229
249
  while (now() <= deadline) {
230
250
  const owner = read();
231
- if (owner !== null && alive(owner))
251
+ if (owner !== null && alive(owner) && await serving())
232
252
  return owner === pid ? null : owner;
233
253
  const exitState = exited?.();
234
254
  if (exitState !== null && exitState !== undefined) {
@@ -314,8 +334,17 @@ export async function stopDaemonProcess(pid, timeoutMs = DAEMON_SHUTDOWN_WINDOW_
314
334
  * If the daemon is already running, returns {started:false, existing_pid}.
315
335
  * If spawning fails (e.g. missing dist — run `npm run build` first), throws. */
316
336
  export async function spawnDaemon() {
317
- if (isDaemonRunning()) {
318
- return { started: false, existing_pid: readPidfile() ?? undefined };
337
+ const recordedPid = readPidfile();
338
+ if (recordedPid !== null && isPidAlive(recordedPid)) {
339
+ if (await isDaemonServing())
340
+ return { started: false, existing_pid: recordedPid };
341
+ return {
342
+ started: false,
343
+ existing_pid: recordedPid,
344
+ running: true,
345
+ serving: false,
346
+ next: 'Run `crtr sys daemon stop` to release the non-serving daemon before starting it again.',
347
+ };
319
348
  }
320
349
  // Ensure the canvas home directory exists so the daemon can write its pidfile.
321
350
  mkdirSync(crtrHome(), { recursive: true });
@@ -1,4 +1,5 @@
1
1
  import { getNode } from '../../core/canvas/canvas.js';
2
+ import { cancelCronsOnWake } from '../../core/canvas/crons.js';
2
3
  import { withFreshTerminalGuard } from '../../core/canvas/canvas.js';
3
4
  import { appendInbox } from '../../core/feed/inbox.js';
4
5
  import { BrokerUnreachableError } from '../../core/runtime/broker-request.js';
@@ -43,6 +44,7 @@ export async function deliverNodeMessage(args) {
43
44
  // before reporting an engine receipt so a final that committed during the
44
45
  // request is not represented as a routable delivery.
45
46
  guardRoutable(args.node_id, () => undefined);
47
+ cancelCronsOnWake(args.node_id);
46
48
  return { entry_id: null, via: 'engine', revived: false };
47
49
  }
48
50
  catch (error) {
@@ -57,9 +59,13 @@ export async function deliverNodeMessage(args) {
57
59
  label: args.label,
58
60
  data: { ...args.data, body: args.body },
59
61
  }));
62
+ if (args.mode !== 'quiet' || target.frozen_at !== null)
63
+ cancelCronsOnWake(args.node_id);
64
+ // The APPENDED entry's tier decides the wake: a quiet send to a terminal
65
+ // target was raised out of `deferred` by the append, and must be delivered.
60
66
  let revived = false;
61
67
  let reason;
62
- if (args.mode !== 'quiet') {
68
+ if (entry.tier !== 'deferred') {
63
69
  target = getNode(args.node_id);
64
70
  if (target === null)
65
71
  deliveryUnroutable(args.node_id, 'origin_missing');
@@ -1,6 +1,7 @@
1
1
  import { bashJobPaths, expiredAutoBackgroundBashJobs, formatBashElapsed, stopBackgroundBashJob, } from '../../core/bash-jobs.js';
2
2
  import { isSafeNodeId } from '../../core/canvas/paths.js';
3
3
  import { contextDir } from '../../core/canvas/paths.js';
4
+ import { cancelCronsOnWake } from '../../core/canvas/crons.js';
4
5
  import { appendInbox } from '../../core/feed/inbox.js';
5
6
  import { emitEvent } from '../../core/events/emit.js';
6
7
  import { operationIdContext } from '../../core/events/operation-id.js';
@@ -64,6 +65,7 @@ export class BashDeadlineReconciler {
64
65
  `Log: ${paths.jobLog} Command: ${paths.cmdSh}`,
65
66
  },
66
67
  });
68
+ cancelCronsOnWake(nodeId);
67
69
  }
68
70
  catch (err) {
69
71
  emitEvent({ level: 'error', event: 'bash_deadline.notice_failed', ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}), error: err, fields: { job_id: jobId } });
@@ -36,6 +36,7 @@ export declare function wedgeVerdict(input: {
36
36
  busy: boolean;
37
37
  quietForMs: number | null;
38
38
  cpuPercent: number | null;
39
+ hasDescendants: boolean | null;
39
40
  }): WedgeVerdict;
40
41
  export declare function sumTreeCpu(psOutput: string, rootPid: number): number | null;
41
42
  export declare function treeHasDescendants(psOutput: string, rootPid: number): boolean | null;
@@ -61,7 +61,9 @@ export function wedgeVerdict(input) {
61
61
  return 'leave';
62
62
  if (input.quietForMs === null || input.quietForMs < WEDGE_QUIET_MS)
63
63
  return 'leave';
64
- if (input.cpuPercent === null || input.cpuPercent > WEDGE_CPU_MAX_PERCENT)
64
+ if (input.cpuPercent === null)
65
+ return 'pending';
66
+ if (input.hasDescendants !== false && input.cpuPercent > WEDGE_CPU_MAX_PERCENT)
65
67
  return 'pending';
66
68
  return 'wedged';
67
69
  }
@@ -471,18 +473,20 @@ export class BrokerSupervisionReconciler {
471
473
  const quietForMs = since === null ? null : now - since;
472
474
  const sample = quietForMs !== null && quietForMs >= WEDGE_QUIET_MS ? await this.readTreeSample(pid) : null;
473
475
  const cpuPercent = sample?.cpuPercent ?? null;
474
- if (wedgeVerdict({ busy: true, quietForMs, cpuPercent }) !== 'wedged')
476
+ const sampleHasDescendants = sample?.hasDescendants ?? null;
477
+ if (wedgeVerdict({ busy: true, quietForMs, cpuPercent, hasDescendants: sampleHasDescendants }) !== 'wedged')
475
478
  return;
476
479
  if (since !== null && this.wedgeNotifiedAt.get(id) === since)
477
480
  return;
478
- if (sample?.hasDescendants === false && (expectedIdentity == null || recordedPidLiveness(pid, expectedIdentity) !== 'alive'))
479
- return;
480
481
  if (since !== null)
481
482
  this.wedgeNotifiedAt.set(id, since);
482
483
  const meta = getNode(id);
483
484
  const name = meta !== null ? fullName(meta) : id;
484
485
  const minutes = Math.round((quietForMs ?? WEDGE_QUIET_MS) / 60_000);
485
- const hasDescendants = sample?.hasDescendants ?? true;
486
+ const hasDescendants = sampleHasDescendants ?? true;
487
+ const kickSuppressionReason = !hasDescendants && (expectedIdentity == null || recordedPidLiveness(pid, expectedIdentity) !== 'alive')
488
+ ? 'pid_identity_unproven'
489
+ : null;
486
490
  let label;
487
491
  if (hasDescendants) {
488
492
  label =
@@ -491,11 +495,18 @@ export class BrokerSupervisionReconciler {
491
495
  `\`crtr memory read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this ` +
492
496
  `node) so its turn can resume with no lost context.`;
493
497
  }
498
+ else if (kickSuppressionReason !== null) {
499
+ label =
500
+ `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
501
+ `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did ` +
502
+ `not SIGTERM the broker because its PID identity cannot be ` +
503
+ `confirmed alive.`;
504
+ }
494
505
  else {
495
506
  label =
496
507
  `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
497
- `${minutes}+ minutes, its process is near-idle, AND it has no subprocess to kill (the engine ` +
498
- `itself stalled). Kicking it now: SIGTERM to the broker — the daemon will resume a cleanly ` +
508
+ `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). Kicking it ` +
509
+ `now: SIGTERM to the broker — the daemon will resume a cleanly ` +
499
510
  `aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`;
500
511
  try {
501
512
  process.kill(pid, 'SIGTERM');
@@ -514,7 +525,8 @@ export class BrokerSupervisionReconciler {
514
525
  quiet_ms: quietForMs,
515
526
  cpu_percent: cpuPercent,
516
527
  has_descendants: hasDescendants,
517
- kicked: !hasDescendants,
528
+ kicked: !hasDescendants && kickSuppressionReason === null,
529
+ ...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
518
530
  },
519
531
  });
520
532
  try {
@@ -539,7 +551,8 @@ export class BrokerSupervisionReconciler {
539
551
  child: id,
540
552
  quiet_ms: quietForMs,
541
553
  cpu_percent: cpuPercent,
542
- kicked: !hasDescendants,
554
+ kicked: !hasDescendants && kickSuppressionReason === null,
555
+ ...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
543
556
  });
544
557
  }
545
558
  handleFatalFault(id) {
@@ -25,16 +25,26 @@ export declare function hasPendingWake(nodeId: string): boolean;
25
25
  * report is an outstanding deliverable, so such a node keeps its
26
26
  * wakeable idle row instead of being written off.
27
27
  *
28
- * Undelivered mail of ANY tier also holds the row. A deferred-only unseen set
29
- * never wakes a node on its own — it rides the next natural cycle — but a
30
- * terminalized row no longer takes that cycle, so writing one off would drop
31
- * content the sender already handed over.
28
+ * Undelivered mail holds the row, with one carve-out: a deferred-only unseen
29
+ * set does not hold a RESIDENT. Deferred mail never wakes a node on its own,
30
+ * and parking a resident is reversible — a message sent to it revives it and
31
+ * flushes the held mail on that cycle. A terminalized row takes no such
32
+ * cycle, so unread mail of any tier keeps it from being written off: the
33
+ * invariant is that content a sender already handed over is never dropped.
32
34
  */
33
35
  export type ParkEvent = Extract<LifecycleEvent, 'release' | 'park' | 'cancel'>;
34
36
  export declare function parkEvent(nodeId: string, lifecycle: Lifecycle): ParkEvent;
35
37
  /** Apply an unattended decision and notify subscribers of automatic completion.
36
38
  * This is the DEGRADED enactment: no summary turn ran, so subscribers get the
37
- * doctrine wake instead of the node's own report. The live clock's good path
39
+ * doctrine notice instead of the node's own report. The live clock's good path
38
40
  * (broker settlement after a parking turn) enacts `park` directly and fans
39
- * nothing — the node's report is the notice. */
41
+ * nothing — the node's report is the notice.
42
+ *
43
+ * The notice is DEFERRED: a node settling quietly at the end of its unattended
44
+ * grace is not an event worth starting a manager's cycle for, so it rides the
45
+ * manager's next natural cycle and arrives alongside whatever does.
46
+ *
47
+ * Only `park` fans, and only a resident parks, so the notice speaks about a
48
+ * resident throughout: its conversation ended without a final report, its state
49
+ * lives in whatever report it did leave, and a message reopens it. */
40
50
  export declare function applyParkEvent(nodeId: string, event: ParkEvent): void;
@@ -1,4 +1,4 @@
1
- import { hasActiveLiveSubscription, hasLiveMessageWait, hasLiveSubscriber, hasPendingCancelOnWakeCron, subscribersOf, } from '../../core/canvas/index.js';
1
+ import { newestReportRef, hasActiveLiveSubscription, hasLiveMessageWait, hasLiveSubscriber, hasPendingCancelOnWakeCron, subscribersOf, } from '../../core/canvas/index.js';
2
2
  import { fullName } from '../../core/canvas/labels.js';
3
3
  import { countTickets } from '../../core/canvas/attention.js';
4
4
  import { hasUnseenWakeMail, readCursor, readInboxSince } from '../../core/feed/inbox.js';
@@ -33,22 +33,34 @@ export function hasPendingWake(nodeId) {
33
33
  export function parkEvent(nodeId, lifecycle) {
34
34
  if (hasLiveObligation(nodeId))
35
35
  return 'release';
36
+ if (lifecycle === 'resident')
37
+ return hasUnseenWakeMail(nodeId) ? 'release' : 'park';
36
38
  if (readInboxSince(nodeId, readCursor(nodeId)).length > 0)
37
39
  return 'release';
38
- if (lifecycle === 'resident')
39
- return 'park';
40
40
  return hasLiveSubscriber(nodeId) ? 'release' : 'cancel';
41
41
  }
42
42
  /** Apply an unattended decision and notify subscribers of automatic completion.
43
43
  * This is the DEGRADED enactment: no summary turn ran, so subscribers get the
44
- * doctrine wake instead of the node's own report. The live clock's good path
44
+ * doctrine notice instead of the node's own report. The live clock's good path
45
45
  * (broker settlement after a parking turn) enacts `park` directly and fans
46
- * nothing — the node's report is the notice. */
46
+ * nothing — the node's report is the notice.
47
+ *
48
+ * The notice is DEFERRED: a node settling quietly at the end of its unattended
49
+ * grace is not an event worth starting a manager's cycle for, so it rides the
50
+ * manager's next natural cycle and arrives alongside whatever does.
51
+ *
52
+ * Only `park` fans, and only a resident parks, so the notice speaks about a
53
+ * resident throughout: its conversation ended without a final report, its state
54
+ * lives in whatever report it did leave, and a message reopens it. */
47
55
  export function applyParkEvent(nodeId, event) {
48
56
  const node = event === 'cancel'
49
57
  ? transition(nodeId, event, { reason: 'stranded' })
50
58
  : transition(nodeId, event);
51
59
  if (event !== 'park')
52
60
  return;
53
- fanDoctrineWake(nodeId, subscribersOf(nodeId), `Child done — ${fullName(node)} (${nodeId}) was automatically marked done after its unattended grace with nothing left to wake it. It sent no final report.`, { reason: 'child-auto-done', child: nodeId });
61
+ const ref = newestReportRef(nodeId);
62
+ const pointer = ref === null
63
+ ? `It left no report behind; \`crtr node inspect show ${nodeId}\` shows where it stands.`
64
+ : `Its latest report is where it left things: \`crtr canvas history read ${ref}\`.`;
65
+ fanDoctrineWake(nodeId, subscribersOf(nodeId), `Resident concluded — ${fullName(node)} (${nodeId}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${pointer} A message to it reopens it on a fresh cycle.`, { reason: 'child-auto-done', child: nodeId }, 'deferred');
54
66
  }
@@ -1,5 +1,16 @@
1
- import type { NodeRow } from '../../../core/canvas/index.js';
2
- /** Whether a durable backoff still holds this row back from a relaunch. A row
3
- * in backoff holds no slot and is excluded from the saturation figure — it is
4
- * waiting on a clock, not on capacity. */
1
+ import { type NodeRow } from '../../../core/canvas/index.js';
2
+ export declare const MAX_RESPAWN_ATTEMPTS = 6;
3
+ export declare function respawnBackoffMs(consecutiveFailures: number): number;
4
+ export interface RespawnAttempt {
5
+ failures: number;
6
+ notBefore: string | null;
7
+ exhausted: boolean;
8
+ }
9
+ /** Whether a durable backoff still holds this row back from a relaunch. */
5
10
  export declare function isInBackoff(row: NodeRow, now: number): boolean;
11
+ /** The only respawn-failure policy: progress since launch clears the sequence;
12
+ * every other missing-engine observation advances it and records its backoff. */
13
+ export declare function decideRespawnAttempt(row: Pick<NodeRow, 'last_settled_at' | 'launched_at' | 'respawn_failures'>, now: number): RespawnAttempt;
14
+ /** Persist the policy decision through the daemon-only canvas writer. */
15
+ export declare function recordRespawnPolicy(row: NodeRow, now: number): RespawnAttempt;
16
+ export declare function clearRespawnPolicy(nodeId: string): void;
@@ -1,6 +1,13 @@
1
- /** Whether a durable backoff still holds this row back from a relaunch. A row
2
- * in backoff holds no slot and is excluded from the saturation figure — it is
3
- * waiting on a clock, not on capacity. */
1
+ import { recordRespawnAttempt } from '../../../core/canvas/index.js';
2
+ export const MAX_RESPAWN_ATTEMPTS = 6;
3
+ const RESPAWN_BACKOFF_BASE_MS = 5_000;
4
+ const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
5
+ export function respawnBackoffMs(consecutiveFailures) {
6
+ if (consecutiveFailures <= 0)
7
+ return 0;
8
+ return Math.min(RESPAWN_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESPAWN_BACKOFF_MAX_MS);
9
+ }
10
+ /** Whether a durable backoff still holds this row back from a relaunch. */
4
11
  export function isInBackoff(row, now) {
5
12
  const notBefore = row.respawn_not_before;
6
13
  if (notBefore == null)
@@ -8,3 +15,27 @@ export function isInBackoff(row, now) {
8
15
  const at = Date.parse(notBefore);
9
16
  return Number.isFinite(at) && now < at;
10
17
  }
18
+ /** The only respawn-failure policy: progress since launch clears the sequence;
19
+ * every other missing-engine observation advances it and records its backoff. */
20
+ export function decideRespawnAttempt(row, now) {
21
+ const settledAt = row.last_settled_at == null ? Number.NaN : Date.parse(row.last_settled_at);
22
+ const launchedAt = row.launched_at == null ? Number.NaN : Date.parse(row.launched_at);
23
+ if (Number.isFinite(settledAt) && Number.isFinite(launchedAt) && settledAt > launchedAt) {
24
+ return { failures: 0, notBefore: null, exhausted: false };
25
+ }
26
+ const failures = (row.respawn_failures ?? 0) + 1;
27
+ return {
28
+ failures,
29
+ notBefore: new Date(now + respawnBackoffMs(failures)).toISOString(),
30
+ exhausted: failures >= MAX_RESPAWN_ATTEMPTS,
31
+ };
32
+ }
33
+ /** Persist the policy decision through the daemon-only canvas writer. */
34
+ export function recordRespawnPolicy(row, now) {
35
+ const attempt = decideRespawnAttempt(row, now);
36
+ recordRespawnAttempt(row.node_id, { failures: attempt.failures, notBefore: attempt.notBefore });
37
+ return attempt;
38
+ }
39
+ export function clearRespawnPolicy(nodeId) {
40
+ recordRespawnAttempt(nodeId, { failures: 0, notBefore: null });
41
+ }
@@ -1,2 +1,5 @@
1
1
  import type { NodeRow } from '../../../core/canvas/index.js';
2
- export declare function handleTerminatingRow(row: NodeRow): void;
2
+ /** Request teardown only while a terminal row still has a daemon handle or a
3
+ * provably live recorded process. An indeterminate pid is left for the next
4
+ * tick: uncertainty never authorizes a signal. */
5
+ export declare function handleTerminatingRow(row: NodeRow, hasFleetHandle: boolean): boolean;
@@ -1,3 +1,12 @@
1
- export function handleTerminatingRow(row) {
2
- void row;
1
+ import { recordedPidLiveness } from '../../../core/canvas/pid.js';
2
+ import { requestBrokerTeardown } from '../../../core/runtime/host.js';
3
+ /** Request teardown only while a terminal row still has a daemon handle or a
4
+ * provably live recorded process. An indeterminate pid is left for the next
5
+ * tick: uncertainty never authorizes a signal. */
6
+ export function handleTerminatingRow(row, hasFleetHandle) {
7
+ if (!hasFleetHandle && (row.pi_pid == null || recordedPidLiveness(row.pi_pid, row.pi_pid_identity) !== 'alive')) {
8
+ return false;
9
+ }
10
+ requestBrokerTeardown(row.node_id);
11
+ return true;
3
12
  }
@@ -59,18 +59,21 @@ export class NodeLifecycleReconciler {
59
59
  }
60
60
  async classifyRow(now, row, ctx, candidates) {
61
61
  const id = row.node_id;
62
+ if (isTerminal(row)) {
63
+ if (handleTerminatingRow(row, ctx.fleet.has(id)))
64
+ return;
65
+ if (row.pi_pid != null && recordedPidLiveness(row.pi_pid, row.pi_pid_identity) === 'dead') {
66
+ clearRecordedPid(id);
67
+ }
68
+ if (row.frozen_at != null)
69
+ candidates.push({ row, enact: 'thaw' });
70
+ return;
71
+ }
62
72
  // 1. A fleet handle IS execution. Whatever froze this row is over.
63
73
  if (ctx.fleet.has(id)) {
64
74
  this.thawMark(row);
65
75
  return;
66
76
  }
67
- if (isTerminal(row)) {
68
- if (row.pi_pid != null)
69
- handleTerminatingRow(row);
70
- else if (row.frozen_at != null)
71
- candidates.push({ row, enact: 'thaw' });
72
- return;
73
- }
74
77
  // A paused profile launches nothing, and its stuck-delivery episode is over
75
78
  // rather than merely stalled — the pause is not a delivery failure.
76
79
  if (isProfilePaused(row.profile_id)) {
@@ -101,6 +104,10 @@ export class NodeLifecycleReconciler {
101
104
  // its way to the launch seam, which owns its own launch and must be left
102
105
  // alone rather than raced or written off as a boot failure.
103
106
  if (row.pi_pid == null) {
107
+ if (row.respawn_not_before != null) {
108
+ candidates.push({ row, enact: 'policy' });
109
+ return;
110
+ }
104
111
  if (row.frozen_at != null)
105
112
  candidates.push({ row, enact: 'thaw' });
106
113
  return;
@@ -13,8 +13,8 @@ export declare class StorageMaintenanceReconciler {
13
13
  /** Milliseconds since meta/telemetry was last written. Inbox activity is
14
14
  * deliberately excluded: children can keep writing a dead manager's inbox. */
15
15
  private msSinceDiskActivity;
16
- /** Reap only the physical placement residue of long-dead nodes. The durable
17
- * row, graph, and on-disk state remain available for a later manual revive. */
16
+ /** Reap only physical placement residue of terminal nodes. The durable row,
17
+ * graph, and on-disk state remain available for a later manual revive. */
18
18
  private reapDeadResidue;
19
19
  /** Delete focus rows whose recorded tmux pane no longer exists. A failed tmux
20
20
  * probe is "can't tell" and never authorizes deleting every viewport. */
@@ -66,12 +66,12 @@ export class StorageMaintenanceReconciler {
66
66
  }
67
67
  return latest === 0 ? Number.POSITIVE_INFINITY : now - latest;
68
68
  }
69
- /** Reap only the physical placement residue of long-dead nodes. The durable
70
- * row, graph, and on-disk state remain available for a later manual revive. */
69
+ /** Reap only physical placement residue of terminal nodes. The durable row,
70
+ * graph, and on-disk state remain available for a later manual revive. */
71
71
  reapDeadResidue(now) {
72
72
  let rows;
73
73
  try {
74
- rows = listNodes({ status: ['dead'] });
74
+ rows = listNodes({ status: ['done', 'canceled', 'dead'] });
75
75
  }
76
76
  catch (err) {
77
77
  operationIdContext.fresh(() => {
@@ -186,7 +186,7 @@ test('gh-451: finished does NOT shut down while a non-deferred inbox entry sits
186
186
  assert.equal(getNode('mgr')?.intent ?? null, null, 'no idle-release intent while a non-deferred entry is unseen');
187
187
  assert.equal(shutdown, false, 'pi stays alive so the in-process watcher can still deliver/commit');
188
188
  });
189
- test('gh-451: a DEFERRED-only unseen entry does not block idle-release (matches the daemon\'s own wake gate)', { timeout: 1_000 }, async () => {
189
+ test('a deferred send to a TERMINAL node was raised on append, so it holds the settle instead of riding a later cycle', { timeout: 1_000 }, async () => {
190
190
  createNode(node('root', { parent: null, lifecycle: 'resident' }));
191
191
  createNode(node('mgr', { parent: 'root', lifecycle: 'terminal', mode: 'orchestrator' }));
192
192
  createNode(node('worker', { parent: 'mgr', lifecycle: 'terminal', status: 'active' }));
@@ -199,9 +199,9 @@ test('gh-451: a DEFERRED-only unseen entry does not block idle-release (matches
199
199
  let shutdown = false;
200
200
  const ctx = { shutdown: () => { shutdown = true; } };
201
201
  await settle(pi, stopEvent('still waiting on the worker'), ctx);
202
- assert.equal(getNode('mgr')?.intent, 'idle-release', 'a deferred-only unseen entry does not block release (#341 parity)');
203
- assert.equal(getNode('mgr')?.status, 'idle');
204
- assert.equal(shutdown, true, 'pi shuts down — deferred entries ride the next natural cycle, not this settle');
202
+ assert.equal(getNode('mgr')?.intent ?? null, null, 'a terminal node takes no reliable later cycle, so its raised entry blocks the release');
203
+ assert.equal(getNode('mgr')?.status, 'active', 'mgr stays active for the entry the watcher still owes it');
204
+ assert.equal(shutdown, false, 'pi stays alive so the in-process watcher can deliver it');
205
205
  });
206
206
  test('queued continuation finishes before natural-stop routing commits', { timeout: 1_000 }, async () => {
207
207
  createNode(node('leaf', { parent: null, lifecycle: 'terminal' }));
@@ -23,8 +23,9 @@ interface SessionEntryLike {
23
23
  }
24
24
  interface SessionStartCtxLike {
25
25
  sessionManager: {
26
- getEntries: () => SessionEntryLike[];
27
- getBranch?: (fromId?: string) => SessionEntryLike[];
26
+ /** Required by pi's `ReadonlySessionManager`: the active branch, which is
27
+ * the context the next turn actually sees. */
28
+ getBranch: (fromId?: string) => SessionEntryLike[];
28
29
  };
29
30
  }
30
31
  interface CustomMessageLike {
@@ -37,35 +38,12 @@ interface CustomMessageLike {
37
38
  nodeId: string;
38
39
  };
39
40
  }
40
- /** The message handed to a message renderer. `content` is normally the string we
41
- * sent, but pi types it as string-or-blocks, so we handle both. */
42
- interface RenderedMessageLike {
43
- customType: string;
44
- content: string | Array<{
45
- type: string;
46
- text?: string;
47
- }>;
48
- }
49
- /** Minimal structural match for pi-tui's `Component` (render + invalidate). A
50
- * plain object of this shape is a valid child for pi's Container. */
51
- interface ComponentLike {
52
- render: (width: number) => string[];
53
- invalidate: () => void;
54
- }
55
- /** Subset of pi's `Theme` we touch — `fg(color, text)` wraps text in ANSI. Used
56
- * defensively (falls back to plain text if absent). */
57
- interface ThemeLike {
58
- fg?: (color: string, text: string) => string;
59
- }
60
41
  interface PiLike {
61
42
  on: (event: 'session_start', handler: (event: unknown, ctx: SessionStartCtxLike) => void | Promise<void>) => void;
62
43
  sendMessage: (message: CustomMessageLike, options?: {
63
44
  deliverAs?: string;
64
45
  triggerTurn?: boolean;
65
46
  }) => void;
66
- registerMessageRenderer: (customType: string, renderer: (message: RenderedMessageLike, options: {
67
- expanded?: boolean;
68
- }, theme: ThemeLike) => ComponentLike | undefined) => void;
69
47
  }
70
48
  /** Build the full bearings baseline for a conversation without inherited
71
49
  * crouter context. Exported for testing. */
@@ -73,15 +51,6 @@ export declare function buildContextIntro(nodeId: string, target?: ExposureTarge
73
51
  /** Build the compact node-scoped update for a fork whose copied branch already
74
52
  * carries a full crouter baseline. Exported for testing. */
75
53
  export declare function buildForkContextIntro(nodeId: string): Promise<string>;
76
- /**
77
- * Renderer for `crtr-context` messages. Collapsed (default) shows a one-line
78
- * stub; expanded (Ctrl+O) shows the label + full body. Returns a plain object
79
- * matching pi's structural `Component` interface — no pi-tui import. Exported for
80
- * testing.
81
- */
82
- export declare function renderContextMessage(message: RenderedMessageLike, options: {
83
- expanded?: boolean;
84
- }, theme: ThemeLike): ComponentLike;
85
54
  /** Register new-context baseline assembly; a strict resume changes nothing. */
86
55
  export declare function registerCanvasContextIntro(pi: PiLike): void;
87
56
  export default registerCanvasContextIntro;