@north-light/crouter 0.3.282 → 0.3.284

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/dist/api/client.d.ts +22 -4
  2. package/dist/api/client.js +34 -0
  3. package/dist/api/dto/broker-ops.d.ts +11 -0
  4. package/dist/api/dto/broker.d.ts +3 -8
  5. package/dist/api/dto/common.d.ts +1 -1
  6. package/dist/api/dto/messages.d.ts +2 -2
  7. package/dist/api/dto/node-outcomes.d.ts +80 -0
  8. package/dist/api/dto/node-outcomes.js +2 -0
  9. package/dist/api/dto/nodes.d.ts +9 -0
  10. package/dist/api/dto/profiles.d.ts +5 -0
  11. package/dist/api/dto/reports.d.ts +45 -0
  12. package/dist/api/index.d.ts +1 -0
  13. package/dist/api/index.js +1 -0
  14. package/dist/api/routes.d.ts +7 -0
  15. package/dist/api/routes.js +7 -0
  16. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +24 -3
  17. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +31 -4
  18. package/dist/clients/attach/overlays/graph.d.ts +5 -0
  19. package/dist/clients/attach/overlays/graph.js +65 -47
  20. package/dist/clients/attach/overlays/pickers.js +8 -9
  21. package/dist/clients/attach/render/card-presentation.js +1 -1
  22. package/dist/clients/attach/viewer.js +1286 -1260
  23. package/dist/commands/api-client.js +22 -23
  24. package/dist/commands/node/create.js +38 -3
  25. package/dist/commands/node/inspect.js +3 -7
  26. package/dist/commands/node/message.js +3 -3
  27. package/dist/commands/node/outcome.d.ts +2 -0
  28. package/dist/commands/node/outcome.js +134 -0
  29. package/dist/commands/node.js +2 -1
  30. package/dist/commands/profile/pause.js +17 -14
  31. package/dist/commands/push.js +102 -1
  32. package/dist/commands/sys/__tests__/setup-core.test.js +4 -1
  33. package/dist/commands/sys/panels/broker-limits-panel.js +7 -1
  34. package/dist/commands/sys/panels/models-panel.js +2 -2
  35. package/dist/commands/sys/settings-shell.d.ts +1 -1
  36. package/dist/commands/sys/settings-shell.js +6 -0
  37. package/dist/commands/sys/settings.js +3 -3
  38. package/dist/core/__tests__/fixtures/fake-engine.d.ts +2 -0
  39. package/dist/core/__tests__/fixtures/fake-engine.js +42 -4
  40. package/dist/core/__tests__/helpers/harness.d.ts +1 -0
  41. package/dist/core/__tests__/helpers/harness.js +9 -0
  42. package/dist/core/__tests__/integration/profile-pause.test.d.ts +1 -0
  43. package/dist/core/__tests__/integration/profile-pause.test.js +115 -0
  44. package/dist/core/__tests__/node-outcome.test.d.ts +1 -0
  45. package/dist/core/__tests__/node-outcome.test.js +184 -0
  46. package/dist/core/__tests__/revive-parked-fresh.test.js +13 -20
  47. package/dist/core/__tests__/seam/broker-provider-retry.test.js +6 -2
  48. package/dist/core/__tests__/seam/dormancy-release.test.js +3 -3
  49. package/dist/core/__tests__/seam/node-outcome-delivery.test.d.ts +1 -0
  50. package/dist/core/__tests__/seam/node-outcome-delivery.test.js +153 -0
  51. package/dist/core/bash-jobs.d.ts +6 -0
  52. package/dist/core/bash-jobs.js +22 -0
  53. package/dist/core/canvas/canvas.d.ts +41 -3
  54. package/dist/core/canvas/canvas.js +153 -39
  55. package/dist/core/canvas/crons.js +2 -1
  56. package/dist/core/canvas/index.d.ts +1 -0
  57. package/dist/core/canvas/index.js +1 -0
  58. package/dist/core/canvas/migrations.js +47 -0
  59. package/dist/core/canvas/node-outcome-deliveries.d.ts +80 -0
  60. package/dist/core/canvas/node-outcome-deliveries.js +163 -0
  61. package/dist/core/canvas/pid.d.ts +7 -0
  62. package/dist/core/canvas/pid.js +17 -0
  63. package/dist/core/canvas/render-source.js +2 -2
  64. package/dist/core/canvas/types.d.ts +42 -1
  65. package/dist/core/canvas/types.js +2 -1
  66. package/dist/core/config.d.ts +4 -1
  67. package/dist/core/config.js +25 -1
  68. package/dist/core/feed/feed.d.ts +8 -0
  69. package/dist/core/feed/feed.js +25 -4
  70. package/dist/core/feed/inbox.d.ts +5 -0
  71. package/dist/core/feed/inbox.js +13 -0
  72. package/dist/core/keybindings/catalog.d.ts +1 -1
  73. package/dist/core/keybindings/catalog.js +3 -0
  74. package/dist/core/profiles/manifest.d.ts +3 -1
  75. package/dist/core/profiles/manifest.js +6 -1
  76. package/dist/core/runtime/broker/daemon-ops.d.ts +3 -1
  77. package/dist/core/runtime/broker/daemon-ops.js +6 -0
  78. package/dist/core/runtime/broker/engine-drive.d.ts +2 -3
  79. package/dist/core/runtime/broker/engine-drive.js +74 -27
  80. package/dist/core/runtime/broker/frame-dispatch.js +1 -0
  81. package/dist/core/runtime/broker/read-ops.js +14 -0
  82. package/dist/core/runtime/broker/rebind.js +0 -7
  83. package/dist/core/runtime/broker-protocol.d.ts +1 -6
  84. package/dist/core/runtime/fleet.d.ts +7 -0
  85. package/dist/core/runtime/headless-pi.d.ts +13 -0
  86. package/dist/core/runtime/headless-pi.js +10 -2
  87. package/dist/core/runtime/lifecycle.d.ts +6 -1
  88. package/dist/core/runtime/lifecycle.js +50 -1
  89. package/dist/core/runtime/nodes.d.ts +5 -1
  90. package/dist/core/runtime/nodes.js +2 -1
  91. package/dist/core/runtime/outcome-document.d.ts +70 -0
  92. package/dist/core/runtime/outcome-document.js +97 -0
  93. package/dist/core/runtime/revive.js +16 -21
  94. package/dist/core/runtime/spawn.d.ts +5 -1
  95. package/dist/core/runtime/spawn.js +18 -7
  96. package/dist/core/runtime/stamp/channel.d.ts +3 -5
  97. package/dist/core/runtime/stamp/channel.js +3 -10
  98. package/dist/core/runtime/stop-guard.js +2 -2
  99. package/dist/core/runtime/structured-output.d.ts +24 -3
  100. package/dist/core/runtime/structured-output.js +42 -3
  101. package/dist/core/runtime/tmux-bindings.js +4 -0
  102. package/dist/core/runtime/warm-pool.d.ts +5 -2
  103. package/dist/core/runtime/warm-pool.js +5 -5
  104. package/dist/core/subscription-state.js +7 -5
  105. package/dist/core/substrate/render.js +10 -10
  106. package/dist/core/user-settings.d.ts +18 -1
  107. package/dist/core/user-settings.js +37 -0
  108. package/dist/daemon/__tests__/node-deadline-lane.test.d.ts +1 -0
  109. package/dist/daemon/__tests__/node-deadline-lane.test.js +76 -0
  110. package/dist/daemon/__tests__/node-outcome-birth-invariants.test.d.ts +1 -0
  111. package/dist/daemon/__tests__/node-outcome-birth-invariants.test.js +122 -0
  112. package/dist/daemon/__tests__/node-outcome-delivery.test.d.ts +1 -0
  113. package/dist/daemon/__tests__/node-outcome-delivery.test.js +86 -0
  114. package/dist/daemon/api/__tests__/broker-settle-park.test.js +86 -18
  115. package/dist/daemon/api/__tests__/profile-launch-gates.test.js +10 -1
  116. package/dist/daemon/api/__tests__/seam/node-outcome-api.test.d.ts +1 -0
  117. package/dist/daemon/api/__tests__/seam/node-outcome-api.test.js +159 -0
  118. package/dist/daemon/api/__tests__/submit-result.test.d.ts +1 -0
  119. package/dist/daemon/api/__tests__/submit-result.test.js +214 -0
  120. package/dist/daemon/api/handlers/broker-ops.js +53 -24
  121. package/dist/daemon/api/handlers/canvas.js +2 -2
  122. package/dist/daemon/api/handlers/messages.js +12 -2
  123. package/dist/daemon/api/handlers/node-outcomes.d.ts +2 -0
  124. package/dist/daemon/api/handlers/node-outcomes.js +136 -0
  125. package/dist/daemon/api/handlers/nodes.js +67 -1
  126. package/dist/daemon/api/handlers/profiles.js +53 -1
  127. package/dist/daemon/api/handlers/reports.js +197 -33
  128. package/dist/daemon/api/map.d.ts +7 -2
  129. package/dist/daemon/api/map.js +78 -4
  130. package/dist/daemon/api/server.js +74 -7
  131. package/dist/daemon/crtrd.js +12 -0
  132. package/dist/daemon/fleet.d.ts +4 -8
  133. package/dist/daemon/fleet.js +48 -9
  134. package/dist/daemon/messaging/node-message.js +2 -1
  135. package/dist/daemon/node-outcome/deliver-outcome.d.ts +14 -0
  136. package/dist/daemon/node-outcome/deliver-outcome.js +155 -0
  137. package/dist/daemon/park-activity.d.ts +9 -0
  138. package/dist/daemon/park-activity.js +30 -0
  139. package/dist/daemon/park-pending.d.ts +10 -5
  140. package/dist/daemon/park-pending.js +18 -8
  141. package/dist/daemon/reconcilers/bash-deadline.js +6 -0
  142. package/dist/daemon/reconcilers/broker-supervision.d.ts +3 -7
  143. package/dist/daemon/reconcilers/broker-supervision.js +29 -33
  144. package/dist/daemon/reconcilers/node-deadline.d.ts +5 -0
  145. package/dist/daemon/reconcilers/node-deadline.js +82 -0
  146. package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +9 -1
  147. package/dist/daemon/reconcilers/node-lifecycle/tick.js +12 -3
  148. package/dist/daemon/reconcilers/node-outcome-delivery-lane.d.ts +10 -0
  149. package/dist/daemon/reconcilers/node-outcome-delivery-lane.js +41 -0
  150. package/dist/pi-extensions/__tests__/canvas-structured-output.test.js +66 -66
  151. package/dist/pi-extensions/canvas-structured-output.d.ts +9 -5
  152. package/dist/pi-extensions/canvas-structured-output.js +23 -297
  153. package/dist/shared/env.d.ts +8 -0
  154. package/dist/shared/env.js +14 -0
  155. package/dist/shared/generated-context.d.ts +3 -9
  156. package/dist/shared/generated-context.js +8 -16
  157. package/dist/types.d.ts +12 -0
  158. package/dist/types.js +1 -0
  159. package/package.json +9 -8
  160. package/runtime.lock.json +227 -383
@@ -3,10 +3,14 @@
3
3
  // Two `http.Server` listeners share one logical API (a Node server can bind one
4
4
  // handle only):
5
5
  // - the unix socket at `apiSocketPath()` — ALWAYS bound, owner-only (mode
6
- // 0600 = the fs-perms-are-auth model);
6
+ // 0600 = the fs-perms-are-auth model); NEVER token-checked, token set or
7
+ // not;
7
8
  // - an opt-in TCP listener, bound only when `--tcp <host:port>` / `CRTRD_TCP`
8
- // is set (a fronting edge owns auth there).
9
- // crtrd applies NO auth to either listener (hard constraint, §1).
9
+ // is set. When `CRTRD_TOKEN` is ALSO set, every TCP request and WS upgrade
10
+ // must carry `Authorization: Bearer <token>` or the daemon answers 401
11
+ // with the standard `ErrorBody` (errors.ts) — otherwise (the default) TCP
12
+ // gets no auth check at all, unchanged from before this guard existed (a
13
+ // fronting edge owns auth there instead).
10
14
  //
11
15
  // Non-upgrade requests flow through the hand-rolled `Router` (A-6) loaded with
12
16
  // every handler route table (A-7). `GET /v1/nodes/:id/attach` upgrades are
@@ -17,7 +21,7 @@ import { createServer } from 'node:http';
17
21
  import { chmodSync, existsSync, unlinkSync } from 'node:fs';
18
22
  import { apiSocketPath } from '../../core/canvas/paths.js';
19
23
  import { emitEvent } from '../../core/events/emit.js';
20
- import { envCrtrdTcp } from '../../shared/env.js';
24
+ import { envCrtrdTcp, envCrtrdToken } from '../../shared/env.js';
21
25
  import { Router } from './router.js';
22
26
  import { handleAttachUpgrade } from './bridge.js';
23
27
  import { attachRoutes } from './handlers/attach.js';
@@ -38,6 +42,7 @@ import { messageRoutes } from './handlers/messages.js';
38
42
  import { modelAuthRoutes } from './handlers/modelauth.js';
39
43
  import { modelConfigRoutes } from './handlers/model-config.js';
40
44
  import { nodeRoutes } from './handlers/nodes.js';
45
+ import { nodeOutcomeRoutes } from './handlers/node-outcomes.js';
41
46
  import { profileRoutes } from './handlers/profiles.js';
42
47
  import { prospectiveChatInventoryRoutes } from './handlers/prospective-chat-inventory.js';
43
48
  import { reportRoutes } from './handlers/reports.js';
@@ -54,6 +59,7 @@ function buildRouter() {
54
59
  .registerAll(healthRoutes)
55
60
  .registerAll(daemonRoutes)
56
61
  .registerAll(nodeRoutes)
62
+ .registerAll(nodeOutcomeRoutes)
57
63
  .registerAll(bashJobRoutes)
58
64
  .registerAll(brokerOperationRoutes)
59
65
  .registerAll(worktreeRoutes)
@@ -112,6 +118,46 @@ function parseTcp(spec) {
112
118
  return null;
113
119
  return { host: host === '' ? '0.0.0.0' : host, port };
114
120
  }
121
+ /** The standard `ErrorBody` (errors.ts) for a TCP request/upgrade missing or
122
+ * carrying a wrong `Authorization: Bearer <token>` header. Only ever written
123
+ * when `CRTRD_TOKEN` is set — an unset token never reaches this path. */
124
+ const UNAUTHORIZED_BODY = {
125
+ error: { code: 'unauthorized', message: 'missing or invalid bearer token' },
126
+ };
127
+ /** Does `req` carry the exact `Authorization: Bearer <token>` this TCP
128
+ * listener requires? */
129
+ function isAuthorized(req, token) {
130
+ return req.headers['authorization'] === `Bearer ${token}`;
131
+ }
132
+ /** Reject an unauthorized non-upgrade TCP request with the standard 401 body. */
133
+ function writeUnauthorized(res) {
134
+ const payload = JSON.stringify(UNAUTHORIZED_BODY);
135
+ res.writeHead(401, { 'content-type': 'application/json; charset=utf-8' });
136
+ res.end(payload);
137
+ }
138
+ /** Reject an unauthorized TCP WS-upgrade attempt: this is a raw pre-upgrade
139
+ * socket (no `ServerResponse`), so the 401 is written as a literal HTTP
140
+ * response — mirroring `bridge.ts`'s `refuseUpgrade` (`end()`, not
141
+ * `write()`+`destroy()`, so a slow peer can't truncate the body) — before the
142
+ * socket closes. */
143
+ function destroyUpgradeUnauthorized(socket) {
144
+ const payload = JSON.stringify(UNAUTHORIZED_BODY);
145
+ const response = `HTTP/1.1 401 Unauthorized\r\n` +
146
+ `content-type: application/json; charset=utf-8\r\n` +
147
+ `content-length: ${Buffer.byteLength(payload)}\r\n` +
148
+ `connection: close\r\n\r\n${payload}`;
149
+ try {
150
+ socket.end(response);
151
+ }
152
+ catch {
153
+ try {
154
+ socket.destroy();
155
+ }
156
+ catch {
157
+ /* already gone */
158
+ }
159
+ }
160
+ }
115
161
  /** Close one http.Server, dropping idle keep-alives so the callback fires. */
116
162
  function closeServer(server) {
117
163
  return new Promise((resolve) => {
@@ -242,7 +288,9 @@ export function createApiServer(opts = {}) {
242
288
  });
243
289
  }
244
290
  });
245
- // TCP — opt-in only. A SECOND http.Server sharing the same handlers.
291
+ // TCP — opt-in only. A SECOND http.Server sharing the same handlers, gated
292
+ // by CRTRD_TOKEN when set (never the unix socket, which shares onRequest/
293
+ // onUpgrade directly above with no wrapping at all).
246
294
  let tcpServer;
247
295
  const tcpSpec = opts.tcp ?? envCrtrdTcp();
248
296
  if (tcpSpec !== undefined && tcpSpec !== '') {
@@ -255,8 +303,27 @@ export function createApiServer(opts = {}) {
255
303
  });
256
304
  }
257
305
  else {
258
- tcpServer = createServer(onRequest);
259
- tcpServer.on('upgrade', onUpgrade);
306
+ const token = envCrtrdToken();
307
+ const tcpOnRequest = token === undefined
308
+ ? onRequest
309
+ : (req, res) => {
310
+ if (!isAuthorized(req, token)) {
311
+ writeUnauthorized(res);
312
+ return;
313
+ }
314
+ onRequest(req, res);
315
+ };
316
+ const tcpOnUpgrade = token === undefined
317
+ ? onUpgrade
318
+ : (req, socket, head) => {
319
+ if (!isAuthorized(req, token)) {
320
+ destroyUpgradeUnauthorized(socket);
321
+ return;
322
+ }
323
+ onUpgrade(req, socket, head);
324
+ };
325
+ tcpServer = createServer(tcpOnRequest);
326
+ tcpServer.on('upgrade', tcpOnUpgrade);
260
327
  bindErrorHandler(tcpServer);
261
328
  tcpServer.listen(parsed.port, parsed.host);
262
329
  }
@@ -50,6 +50,7 @@ import { listNodes, migrateLegacyPidIdentities, } from '../core/canvas/index.js'
50
50
  import { createApiServer } from './api/server.js';
51
51
  import { reconcileUndeliveredTickets } from './human/sweep.js';
52
52
  import { recoverRunningActionDeliveries } from '../core/canvas/human-deliveries.js';
53
+ import { recoverRunningOutcomeDeliveries } from '../core/canvas/node-outcome-deliveries.js';
53
54
  import { sweepReviews } from './review/sweep.js';
54
55
  import { pidfilePath } from './pidfile.js';
55
56
  import { rendererWarning } from '../core/runtime/package-health.js';
@@ -57,8 +58,10 @@ import { DEFAULT_INTERVAL_MS } from './supervise-cadence.js';
57
58
  import { BrokerSupervisionReconciler } from './reconcilers/broker-supervision.js';
58
59
  import { ControllerDeathReconciler } from './reconcilers/controller-death.js';
59
60
  import { NodeLifecycleReconciler } from './reconcilers/node-lifecycle/tick.js';
61
+ import { NodeDeadlineReconciler } from './reconcilers/node-deadline.js';
60
62
  import { CronLaneReconciler } from './reconcilers/cron-lane.js';
61
63
  import { HumanDeliveryLaneReconciler } from './reconcilers/human-delivery-lane.js';
64
+ import { NodeOutcomeDeliveryLaneReconciler } from './reconcilers/node-outcome-delivery-lane.js';
62
65
  import { PendingReviewSubmitReconciler } from './reconcilers/pending-review-submit.js';
63
66
  import { StorageMaintenanceReconciler } from './reconcilers/storage-maintenance.js';
64
67
  import { BashDeadlineReconciler } from './reconcilers/bash-deadline.js';
@@ -272,8 +275,10 @@ const directTickLifecycle = {
272
275
  const brokerSupervision = new BrokerSupervisionReconciler();
273
276
  const controllerDeath = new ControllerDeathReconciler();
274
277
  const nodeLifecycle = new NodeLifecycleReconciler();
278
+ const nodeDeadline = new NodeDeadlineReconciler();
275
279
  const cronLane = new CronLaneReconciler();
276
280
  const humanDeliveryLane = new HumanDeliveryLaneReconciler();
281
+ const nodeOutcomeDeliveryLane = new NodeOutcomeDeliveryLaneReconciler();
277
282
  const pendingReviewSubmit = new PendingReviewSubmitReconciler();
278
283
  const storageMaintenance = new StorageMaintenanceReconciler();
279
284
  const bashDeadline = new BashDeadlineReconciler();
@@ -301,8 +306,12 @@ export async function superviseTick(now = Date.now(), lifecycle = directTickLife
301
306
  await brokerSupervision.run(now, { rows, fleet, lifecycle });
302
307
  controllerDeath.run(now, { rows });
303
308
  const frozen = await nodeLifecycle.run(now, { rows: [...rows, ...terminating], fleet });
309
+ // The universal backstop: a node past its deadline with no outcome is
310
+ // canceled and terminalized here, after the lifecycle pass has had its say.
311
+ nodeDeadline.run(now);
304
312
  cronLane.run(now, { lifecycle });
305
313
  humanDeliveryLane.run(now, { lifecycle });
314
+ nodeOutcomeDeliveryLane.run(now, { lifecycle });
306
315
  pendingReviewSubmit.run(now, { lifecycle });
307
316
  storageMaintenance.run(now, { lifecycle, fleet });
308
317
  await bashDeadline.run(now, { rows });
@@ -640,6 +649,9 @@ export async function runDaemon(opts = {}) {
640
649
  await reconcileUndeliveredTickets();
641
650
  // Unlike a cron one-shot, a completion is at-least-once: destinations deduplicate request IDs, so a dead daemon's running delivery is re-queued immediately.
642
651
  recoverRunningActionDeliveries(Date.now());
652
+ // Same at-least-once contract for node outcome deliveries: a delivery the
653
+ // dead daemon had marked running is re-queued rather than stranded.
654
+ recoverRunningOutcomeDeliveries(Date.now());
643
655
  });
644
656
  // Boot reconciliation clears stale process identities before any tick can
645
657
  // inspect them, then migrates legacy identities only with proven same-boot
@@ -1,13 +1,7 @@
1
1
  import { type NodeMeta } from '../core/canvas/index.js';
2
2
  import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
3
+ import type { NodeOutcomeDetailV1, TerminalReason } from '../core/canvas/types.js';
3
4
  import type { HostHandle } from '../core/runtime/host.js';
4
- /** Surface a vehicle that never booted.
5
- *
6
- * A pi that dies before its first session_start (so `pi_session_id` was never
7
- * recorded) is invisible to its parent — the spawn returned optimistically.
8
- * When the daemon observes that death it errors LOUDLY up the spine: an
9
- * urgent push so the parent learns the child failed to launch instead of just
10
- * seeing a silent `dead`. */
11
5
  export declare function surfaceBootFailure(meta: NodeMeta): Promise<void>;
12
6
  /** Whether a dead relaunch should resume the saved session strictly, or retry
13
7
  * as a fresh cycle instead. A cycling attempt (cycle_pending) that dies before
@@ -69,6 +63,8 @@ export declare class DaemonFleet implements FleetRegistry {
69
63
  reserveSlot(nodeId: string): boolean;
70
64
  releaseReservation(nodeId: string): void;
71
65
  entries(): Iterable<[string, FleetEntry]>;
66
+ interruptForProfilePause(nodeId: string, pid?: number | undefined): void;
67
+ hasProfilePauseInterruption(nodeId: string): boolean;
72
68
  forget(nodeId: string): void;
73
69
  onChildExit(nodeId: string, status: FleetExitStatus): void;
74
70
  deliverExit(nodeId: string, status: FleetExitStatus): void;
@@ -78,6 +74,6 @@ export declare class DaemonFleet implements FleetRegistry {
78
74
  * with no fleet entry. Convert that otherwise-silent strand into the same
79
75
  * terminal outcome as a boot failure. The fresh read avoids duplicating
80
76
  * reviveNode's launch-failure crash path. */
81
- terminalizeIfStillNonTerminal(nodeId: string, event: string, error: unknown): Promise<boolean>;
77
+ terminalizeIfStillNonTerminal(nodeId: string, event: string, error: unknown, reason: TerminalReason, detail: NodeOutcomeDetailV1): Promise<boolean>;
82
78
  dispose(): void;
83
79
  }
@@ -12,6 +12,7 @@
12
12
  // before every deliberate exit, and that durable state is robust to every kill
13
13
  // path. Codes/signals are logged for diagnostics only.
14
14
  import { getNode, getRow, subscribersOf } from '../core/canvas/index.js';
15
+ import { isProfilePaused } from '../core/profiles/manifest.js';
15
16
  import { fullName } from '../core/canvas/labels.js';
16
17
  import { isSafeNodeId } from '../core/canvas/paths.js';
17
18
  import { transition } from '../core/runtime/lifecycle.js';
@@ -33,14 +34,16 @@ import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
33
34
  * When the daemon observes that death it errors LOUDLY up the spine: an
34
35
  * urgent push so the parent learns the child failed to launch instead of just
35
36
  * seeing a silent `dead`. */
36
- export async function surfaceBootFailure(meta) {
37
- const body = `⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
37
+ function bootFailureBody(meta) {
38
+ return (`⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
38
39
  `Its pi vehicle exited before the session came up (no pi_session_id was ever ` +
39
40
  `recorded), so the node produced no output. This is almost always a transient ` +
40
41
  `launch failure — e.g. resource pressure when several nodes boot at once — not ` +
41
42
  `a fault in the task itself.\n\n` +
42
- `If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`;
43
- await pushUrgent(meta.node_id, body, { from: meta.node_id });
43
+ `If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`);
44
+ }
45
+ export async function surfaceBootFailure(meta) {
46
+ await pushUrgent(meta.node_id, bootFailureBody(meta), { from: meta.node_id });
44
47
  }
45
48
  async function surfaceCrashLoop(meta) {
46
49
  const transportPoison = readTransportPoisonVerdict(meta.node_id);
@@ -131,6 +134,9 @@ export class DaemonFleet {
131
134
  #map = new Map();
132
135
  /** Slots claimed by launches that have not registered a broker yet. */
133
136
  #reservations = new Set();
137
+ /** Brokers deliberately interrupted by profile pause. This belongs to the
138
+ * exact child, so a resume cannot turn its later SIGTERM exit into a crash. */
139
+ #profilePauseInterruptions = new Set();
134
140
  #lastCapacityLogAt = Number.NEGATIVE_INFINITY;
135
141
  constructor(opts) {
136
142
  this.#epoch = opts.epoch;
@@ -191,6 +197,21 @@ export class DaemonFleet {
191
197
  entries() {
192
198
  return this.#map.entries();
193
199
  }
200
+ interruptForProfilePause(nodeId, pid = this.#map.get(nodeId)?.pid) {
201
+ if (pid === undefined)
202
+ return;
203
+ this.#profilePauseInterruptions.add(nodeId);
204
+ try {
205
+ process.kill(pid, 'SIGTERM');
206
+ }
207
+ catch {
208
+ // A live handle or identity-checked PID can exit between its observation
209
+ // and signal. The marker still makes that exact observed exit harmless.
210
+ }
211
+ }
212
+ hasProfilePauseInterruption(nodeId) {
213
+ return this.#profilePauseInterruptions.has(nodeId);
214
+ }
194
215
  forget(nodeId) {
195
216
  this.#map.delete(nodeId);
196
217
  this.#reservations.delete(nodeId);
@@ -217,12 +238,14 @@ export class DaemonFleet {
217
238
  * with no fleet entry. Convert that otherwise-silent strand into the same
218
239
  * terminal outcome as a boot failure. The fresh read avoids duplicating
219
240
  * reviveNode's launch-failure crash path. */
220
- async terminalizeIfStillNonTerminal(nodeId, event, error) {
241
+ async terminalizeIfStillNonTerminal(nodeId, event, error, reason, detail) {
221
242
  const meta = getNode(nodeId);
222
243
  if (meta === null || meta.status === 'done' || meta.status === 'canceled' || meta.status === 'dead')
223
244
  return false;
224
245
  await this.#terminalize(nodeId, meta, {
225
246
  event,
247
+ reason,
248
+ detail,
226
249
  wake: `Child crashed — ${fullName(meta)} (${nodeId}) could not be relaunched and is now dead. ` +
227
250
  `It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
228
251
  surface: () => surfaceRecoveryFailure(meta, error),
@@ -249,6 +272,17 @@ export class DaemonFleet {
249
272
  }
250
273
  async #decideAndEnact(nodeId) {
251
274
  const meta = getNode(nodeId);
275
+ if (meta !== null && isProfilePaused(meta.profile_id)) {
276
+ clearRespawnPolicy(nodeId);
277
+ return 'forget';
278
+ }
279
+ // Consume one unpaused policy pass for the exact broker profile pause
280
+ // interrupted. The following tick owns ordinary recovery regardless of
281
+ // which dead-row verdict the still-recorded PID would otherwise select.
282
+ if (this.#profilePauseInterruptions.delete(nodeId)) {
283
+ clearRespawnPolicy(nodeId);
284
+ return 'forget';
285
+ }
252
286
  // Two durable witnesses mean the dead broker had a turn in flight. A dirty
253
287
  // death leaves the busy marker behind; a bounded graceful abort clears busy
254
288
  // as the run settles, then writes the clean-abort certificate. Both need the
@@ -284,6 +318,8 @@ export class DaemonFleet {
284
318
  case 'boot-failure':
285
319
  await this.#terminalize(nodeId, meta, {
286
320
  event: 'broker.boot.failed',
321
+ reason: 'boot_failed',
322
+ detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta) },
287
323
  wake: `Child crashed — ${fullName(meta)} (${nodeId}) exited before its session ever came up and is now dead. ` +
288
324
  `It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
289
325
  });
@@ -328,10 +364,13 @@ export class DaemonFleet {
328
364
  ? recordRespawnPolicy(row, this.#now())
329
365
  : { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
330
366
  if (attempt.exhausted) {
367
+ const wake = `Child crashed — ${fullName(meta)} (${nodeId}) kept dying without settling a turn (${attempt.failures} consecutive ` +
368
+ `launches) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`;
331
369
  await this.#terminalize(nodeId, meta, {
332
370
  event: 'broker.respawn.exhausted',
333
- wake: `Child crashed — ${fullName(meta)} (${nodeId}) kept dying without settling a turn (${attempt.failures} consecutive ` +
334
- `launches) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
371
+ reason: 'crash_looped',
372
+ detail: { schema: 'crtr.node-outcome-detail/v1', message: wake, respawn_failures: attempt.failures },
373
+ wake,
335
374
  ...(meta.pi_session_id != null ? { surface: () => surfaceCrashLoop(meta) } : {}),
336
375
  });
337
376
  return;
@@ -392,7 +431,7 @@ export class DaemonFleet {
392
431
  // Launch failures terminalize inside reviveNode, but its preflight
393
432
  // refusals (for example cwd_missing) happen before that catch. Re-read
394
433
  // before terminalizing so the launch path's settled terminal row wins.
395
- await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err);
434
+ await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err, 'launch_failed', { schema: 'crtr.node-outcome-detail/v1', message: err instanceof Error ? err.message : String(err) });
396
435
  }
397
436
  }
398
437
  /** Terminal outcome shared by table row 8 and respawn exhaustion. */
@@ -402,7 +441,7 @@ export class DaemonFleet {
402
441
  event: opts.event,
403
442
  ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : { fields: { affected_node_id: nodeId } }),
404
443
  });
405
- transition(nodeId, 'crash');
444
+ transition(nodeId, 'crash', { reason: opts.reason, outcome: { detail: opts.detail } });
406
445
  try {
407
446
  await (opts.surface ?? (() => surfaceBootFailure(meta)))();
408
447
  }
@@ -9,6 +9,7 @@ import { reviveNode } from '../../core/runtime/revive.js';
9
9
  import { hasNoNaturalCycle } from '../../core/runtime/revive-all.js';
10
10
  import { ReviewOperationError } from '../../core/review/types.js';
11
11
  import { formatCard } from '../../shared/generated-context.js';
12
+ import { isProfilePaused } from '../../core/profiles/manifest.js';
12
13
  function deliveryUnroutable(nodeId, reason) {
13
14
  throw new ReviewOperationError('delivery_unroutable', reason === 'origin_missing' ? 'message target was not found' : 'message target has no natural delivery cycle', { node_id: nodeId, reason });
14
15
  }
@@ -33,7 +34,7 @@ export async function deliverNodeMessage(args) {
33
34
  let target = getNode(args.node_id);
34
35
  if (target === null)
35
36
  deliveryUnroutable(args.node_id, 'origin_missing');
36
- if (args.mode === 'interactive' && isBrokerLive(target)) {
37
+ if (args.mode === 'interactive' && !isProfilePaused(target.profile_id) && isBrokerLive(target)) {
37
38
  try {
38
39
  // Inbox delivery supplies the raw body as an entry in its outer inbox
39
40
  // card; only live delivery needs this message's own envelope.
@@ -0,0 +1,14 @@
1
+ import { type OutcomeDeliveryRecord, type OutcomeDeliverySettlement } from '../../core/canvas/node-outcome-deliveries.js';
2
+ export declare const OUTCOME_DELIVERY_TIMEOUT_MS = 120000;
3
+ export declare const OUTCOME_DELIVERY_STDERR_TAIL_BYTES: number;
4
+ export interface OutcomeDeliveryProcessOutcome {
5
+ exitCode: number | null;
6
+ signal: string | null;
7
+ timedOut: boolean;
8
+ spawnError?: Error;
9
+ stderr: string;
10
+ }
11
+ /** The delivery target's exit code is its complete acknowledgment protocol. */
12
+ export declare function outcomeDeliverySettlementForOutcome(outcome: OutcomeDeliveryProcessOutcome, attempt: number, now: number): OutcomeDeliverySettlement;
13
+ /** Spawn and settle one already-claimed outcome delivery. */
14
+ export declare function deliverOutcome(delivery: OutcomeDeliveryRecord, claimOwner: string): Promise<void>;
@@ -0,0 +1,155 @@
1
+ import { spawn } from 'node:child_process';
2
+ import { envHomeOverride } from '../../shared/env.js';
3
+ import { outcomeDeliveryBackoffMs, settleOutcomeDelivery, } from '../../core/canvas/node-outcome-deliveries.js';
4
+ import { killProcessGroup } from '../../core/canvas/pid.js';
5
+ import { buildOperationalEnvBase } from '../../core/runtime/spawn-env.js';
6
+ export const OUTCOME_DELIVERY_TIMEOUT_MS = 120_000;
7
+ const OUTCOME_DELIVERY_KILL_GRACE_MS = 2_000;
8
+ export const OUTCOME_DELIVERY_STDERR_TAIL_BYTES = 4 * 1024;
9
+ /** The delivery target's exit code is its complete acknowledgment protocol. */
10
+ export function outcomeDeliverySettlementForOutcome(outcome, attempt, now) {
11
+ if (outcome.spawnError !== undefined) {
12
+ return {
13
+ state: 'pending',
14
+ nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
15
+ failure: failure('spawn_error', { message: outcome.spawnError.message, stderr: outcome.stderr }),
16
+ };
17
+ }
18
+ if (outcome.timedOut) {
19
+ return {
20
+ state: 'pending',
21
+ nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
22
+ failure: failure('timeout', { ...(outcome.signal === null ? {} : { signal: outcome.signal }), stderr: outcome.stderr }),
23
+ };
24
+ }
25
+ if (outcome.signal !== null) {
26
+ return {
27
+ state: 'pending',
28
+ nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
29
+ failure: failure('signal', { signal: outcome.signal, stderr: outcome.stderr }),
30
+ };
31
+ }
32
+ if (outcome.exitCode === 0)
33
+ return { state: 'accepted' };
34
+ if (outcome.exitCode === 78) {
35
+ return { state: 'permanent_failed', failure: failure('exit', { exitCode: 78, stderr: outcome.stderr }) };
36
+ }
37
+ return {
38
+ state: 'pending',
39
+ nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
40
+ failure: failure('exit', { exitCode: outcome.exitCode ?? undefined, stderr: outcome.stderr }),
41
+ };
42
+ }
43
+ function failure(kind, fields) {
44
+ return { kind, ...fields };
45
+ }
46
+ function stderrTailCapture() {
47
+ let tail = Buffer.alloc(0);
48
+ return {
49
+ take(chunk) {
50
+ tail = chunk.length >= OUTCOME_DELIVERY_STDERR_TAIL_BYTES
51
+ ? chunk.subarray(chunk.length - OUTCOME_DELIVERY_STDERR_TAIL_BYTES)
52
+ : Buffer.concat([tail, chunk]).subarray(Math.max(0, tail.length + chunk.length - OUTCOME_DELIVERY_STDERR_TAIL_BYTES));
53
+ },
54
+ decode() {
55
+ return tail.toString('utf8');
56
+ },
57
+ };
58
+ }
59
+ /** The stored document is immutable; attempt is its only per-delivery mutation. */
60
+ function documentForAttempt(document, attempt) {
61
+ const outcome = document;
62
+ const delivery = outcome.delivery;
63
+ return JSON.stringify({ ...outcome, delivery: { ...delivery, attempt } });
64
+ }
65
+ /** Spawn and settle one already-claimed outcome delivery. */
66
+ export async function deliverOutcome(delivery, claimOwner) {
67
+ const stderr = stderrTailCapture();
68
+ let document;
69
+ try {
70
+ document = documentForAttempt(delivery.document, delivery.attempt);
71
+ }
72
+ catch (error) {
73
+ settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome({
74
+ exitCode: null,
75
+ signal: null,
76
+ timedOut: false,
77
+ spawnError: error,
78
+ stderr: stderr.decode(),
79
+ }, delivery.attempt, Date.now()));
80
+ return;
81
+ }
82
+ let child;
83
+ try {
84
+ child = spawn(delivery.argv[0], delivery.argv.slice(1), {
85
+ cwd: delivery.cwd,
86
+ env: outcomeDeliveryEnv(delivery.cwd),
87
+ detached: true,
88
+ stdio: ['pipe', 'pipe', 'pipe'],
89
+ });
90
+ }
91
+ catch (error) {
92
+ settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome({
93
+ exitCode: null,
94
+ signal: null,
95
+ timedOut: false,
96
+ spawnError: error,
97
+ stderr: stderr.decode(),
98
+ }, delivery.attempt, Date.now()));
99
+ return;
100
+ }
101
+ await new Promise((resolve) => {
102
+ let settled = false;
103
+ let timedOut = false;
104
+ let killTimer;
105
+ let timeout;
106
+ const finish = (outcome) => {
107
+ if (settled)
108
+ return;
109
+ settled = true;
110
+ if (timeout !== undefined)
111
+ clearTimeout(timeout);
112
+ // A timed-out leader can exit while a SIGTERM-ignoring group member remains.
113
+ if (!outcome.timedOut && killTimer !== undefined)
114
+ clearTimeout(killTimer);
115
+ child.stdin?.destroy();
116
+ child.stdout?.destroy();
117
+ child.stderr?.destroy();
118
+ settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome(outcome, delivery.attempt, Date.now()));
119
+ resolve();
120
+ };
121
+ child.stdout?.on('data', () => { }); // Drain output; stdout is never a control channel.
122
+ child.stderr?.on('data', (chunk) => stderr.take(chunk));
123
+ child.stdin?.on('error', () => { }); // The child may exit before consuming stdin; its exit code is authoritative.
124
+ child.on('error', (error) => {
125
+ finish({ exitCode: null, signal: null, timedOut: false, spawnError: error, stderr: stderr.decode() });
126
+ });
127
+ child.on('exit', (exitCode, signal) => {
128
+ finish({ exitCode, signal, timedOut, stderr: stderr.decode() });
129
+ });
130
+ const killWithEscalation = () => {
131
+ if (child.pid === undefined)
132
+ return;
133
+ killProcessGroup(child.pid);
134
+ killTimer = setTimeout(() => killProcessGroup(child.pid, 'SIGKILL'), OUTCOME_DELIVERY_KILL_GRACE_MS);
135
+ killTimer.unref?.();
136
+ };
137
+ timeout = setTimeout(() => {
138
+ timedOut = true;
139
+ killWithEscalation();
140
+ }, OUTCOME_DELIVERY_TIMEOUT_MS);
141
+ timeout.unref?.();
142
+ child.stdin?.end(document, 'utf8');
143
+ });
144
+ }
145
+ /** Outcome targets cross the same consent boundary as every other crouter spawn. */
146
+ function outcomeDeliveryEnv(cwd) {
147
+ const env = buildOperationalEnvBase({ targetCwd: cwd, targetProfileId: null });
148
+ const canvasHome = envHomeOverride();
149
+ if (canvasHome !== undefined && canvasHome !== '')
150
+ env['CRTR_HOME'] = canvasHome;
151
+ return env;
152
+ }
153
+ function settle(delivery, claimOwner, settlement) {
154
+ settleOutcomeDelivery(delivery.nodeId, delivery.attempt, claimOwner, settlement, Date.now());
155
+ }
@@ -0,0 +1,9 @@
1
+ import type { NodeMeta } from '../core/canvas/types.js';
2
+ import type { PendingPark } from './park-pending.js';
3
+ /** A main-session write after parking was armed is durable evidence of live
4
+ * input even when the turn has since settled and the viewer detached. */
5
+ export declare function mainSessionActivitySince(sessionFile: string | null | undefined, startedAt: number): boolean;
6
+ /** One verdict shared by normal completion and grace expiry. A broker records
7
+ * accepted live input against the process-local marker before it drives Pi, so
8
+ * the verdict cannot race a viewer, live delivery, or durable wake. */
9
+ export declare function shouldParkAfterSummary(nodeId: string, node: Pick<NodeMeta, 'lifecycle' | 'pi_session_file'>, pending: PendingPark): boolean;
@@ -0,0 +1,30 @@
1
+ import { statSync } from 'node:fs';
2
+ import { emitEvent } from '../core/events/emit.js';
3
+ import { hasWakeMailSince } from '../core/feed/inbox.js';
4
+ import { isBusy } from '../core/runtime/busy.js';
5
+ import { parkEvent } from './reconcilers/live-obligation.js';
6
+ /** A main-session write after parking was armed is durable evidence of live
7
+ * input even when the turn has since settled and the viewer detached. */
8
+ export function mainSessionActivitySince(sessionFile, startedAt) {
9
+ if (sessionFile === null || sessionFile === undefined)
10
+ return false;
11
+ try {
12
+ return statSync(sessionFile).mtimeMs >= startedAt;
13
+ }
14
+ catch (error) {
15
+ if (error.code === 'ENOENT')
16
+ return false;
17
+ emitEvent({ level: 'warn', event: 'broker.park.session_activity_scan_failed', error });
18
+ return true;
19
+ }
20
+ }
21
+ /** One verdict shared by normal completion and grace expiry. A broker records
22
+ * accepted live input against the process-local marker before it drives Pi, so
23
+ * the verdict cannot race a viewer, live delivery, or durable wake. */
24
+ export function shouldParkAfterSummary(nodeId, node, pending) {
25
+ return !pending.activity
26
+ && !isBusy(nodeId)
27
+ && !mainSessionActivitySince(node.pi_session_file, pending.startedAt)
28
+ && !hasWakeMailSince(nodeId, pending.startedAt)
29
+ && parkEvent(nodeId, node.lifecycle) === 'park';
30
+ }
@@ -1,13 +1,18 @@
1
+ export type PendingPark = {
2
+ startedAt: number;
3
+ activity: boolean;
4
+ };
1
5
  /** Arm a pending park for `nodeId`. The caller delivers the parking prompt. */
2
6
  export declare function beginPendingPark(nodeId: string, now: number): void;
7
+ /** Record main-engine input accepted while the isolated turn is in flight. */
8
+ export declare function recordPendingParkActivity(nodeId: string): boolean;
3
9
  export declare function isPendingPark(nodeId: string): boolean;
4
10
  /** When the pending park was armed, or undefined when none is armed. */
5
11
  export declare function pendingParkStartedAt(nodeId: string): number | undefined;
6
- /** Delete-and-report: true only for the caller that actually took the pending
7
- * park. Both the supervision tick and the settle handler run on the daemon
8
- * thread with no await between their check and this call, so exactly one of
9
- * them ever enacts a given park. */
10
- export declare function consumePendingPark(nodeId: string): boolean;
12
+ /** Delete-and-report the pending marker. Both the supervision tick and the
13
+ * settle handler run on the daemon thread with no await between their check and
14
+ * this call, so exactly one of them ever enacts a given park. */
15
+ export declare function consumePendingPark(nodeId: string): PendingPark | undefined;
11
16
  /** Every node with a parking turn in flight — the reconciler's sweep drops the
12
17
  * ones whose broker has left the fleet. */
13
18
  export declare function pendingParkIds(): string[];
@@ -15,25 +15,35 @@
15
15
  // and nothing needs repair. A durable in-flight flag would instead need its own
16
16
  // reconciliation on every daemon start to avoid stranding a node that never got
17
17
  // its prompt.
18
- /** node id → the timestamp its parking turn was requested at. */
18
+ /** node id → the pending parking turn and any accepted main-engine input. */
19
19
  const pending = new Map();
20
20
  /** Arm a pending park for `nodeId`. The caller delivers the parking prompt. */
21
21
  export function beginPendingPark(nodeId, now) {
22
- pending.set(nodeId, now);
22
+ pending.set(nodeId, { startedAt: now, activity: false });
23
+ }
24
+ /** Record main-engine input accepted while the isolated turn is in flight. */
25
+ export function recordPendingParkActivity(nodeId) {
26
+ const marker = pending.get(nodeId);
27
+ if (marker === undefined)
28
+ return false;
29
+ marker.activity = true;
30
+ return true;
23
31
  }
24
32
  export function isPendingPark(nodeId) {
25
33
  return pending.has(nodeId);
26
34
  }
27
35
  /** When the pending park was armed, or undefined when none is armed. */
28
36
  export function pendingParkStartedAt(nodeId) {
29
- return pending.get(nodeId);
37
+ return pending.get(nodeId)?.startedAt;
30
38
  }
31
- /** Delete-and-report: true only for the caller that actually took the pending
32
- * park. Both the supervision tick and the settle handler run on the daemon
33
- * thread with no await between their check and this call, so exactly one of
34
- * them ever enacts a given park. */
39
+ /** Delete-and-report the pending marker. Both the supervision tick and the
40
+ * settle handler run on the daemon thread with no await between their check and
41
+ * this call, so exactly one of them ever enacts a given park. */
35
42
  export function consumePendingPark(nodeId) {
36
- return pending.delete(nodeId);
43
+ const marker = pending.get(nodeId);
44
+ if (marker !== undefined)
45
+ pending.delete(nodeId);
46
+ return marker;
37
47
  }
38
48
  /** Every node with a parking turn in flight — the reconciler's sweep drops the
39
49
  * ones whose broker has left the fleet. */