@crouton-kit/crouter 0.3.40 → 0.3.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/dist/build-root.js +1 -0
  2. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/provider-rotation.test.ts +472 -9
  3. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +284 -76
  4. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/strip-skills-docs.ts +31 -24
  5. package/dist/builtin-pi-packages/pi-crtr-extensions/lib/subscription-state.ts +52 -15
  6. package/dist/clients/attach/attach-cmd.js +720 -747
  7. package/dist/clients/attach/chat-view.js +11 -0
  8. package/dist/clients/attach/view-socket.d.ts +8 -1
  9. package/dist/clients/attach/view-socket.js +15 -1
  10. package/dist/clients/web/server.js +5 -28
  11. package/dist/clients/web/web-cmd.js +1 -1
  12. package/dist/commands/__tests__/revive-now-gate.test.js +60 -0
  13. package/dist/commands/canvas-rebuild-index.js +5 -5
  14. package/dist/commands/memory/shared.d.ts +7 -3
  15. package/dist/commands/memory/shared.js +35 -5
  16. package/dist/commands/memory/write.js +5 -3
  17. package/dist/commands/node-snapshot.js +9 -1
  18. package/dist/commands/node.js +37 -17
  19. package/dist/commands/push.js +8 -0
  20. package/dist/commands/revive.d.ts +10 -0
  21. package/dist/commands/revive.js +28 -10
  22. package/dist/commands/sys/__tests__/setup-core.test.js +19 -0
  23. package/dist/commands/sys/doctor.js +1 -1
  24. package/dist/commands/sys/setup-core.js +3 -2
  25. package/dist/commands/sys/setup.js +1 -1
  26. package/dist/commands/worktree.d.ts +2 -0
  27. package/dist/commands/worktree.js +94 -0
  28. package/dist/core/__tests__/boot.test.js +4 -4
  29. package/dist/core/__tests__/canvas.test.js +19 -7
  30. package/dist/core/__tests__/child-followup.test.js +15 -5
  31. package/dist/core/__tests__/daemon-boot.test.js +6 -1
  32. package/dist/core/__tests__/daemon-wedge.test.js +18 -1
  33. package/dist/core/__tests__/fault-classifier.test.js +30 -0
  34. package/dist/core/__tests__/fixtures/fake-engine.d.ts +13 -0
  35. package/dist/core/__tests__/fixtures/fake-engine.js +20 -0
  36. package/dist/core/__tests__/full/spike-harness.test.js +8 -5
  37. package/dist/core/__tests__/grace-clock.test.js +18 -2
  38. package/dist/core/__tests__/host-teardown-process-group.test.js +465 -0
  39. package/dist/core/__tests__/prune-to-limit.test.js +14 -0
  40. package/dist/core/__tests__/review-model-floor.test.js +32 -0
  41. package/dist/core/__tests__/revive.test.js +18 -4
  42. package/dist/core/__tests__/session-cycles.test.js +77 -0
  43. package/dist/core/__tests__/worktree.test.js +85 -0
  44. package/dist/core/canvas/boot.js +12 -7
  45. package/dist/core/canvas/canvas.d.ts +27 -8
  46. package/dist/core/canvas/canvas.js +54 -26
  47. package/dist/core/canvas/db.js +14 -0
  48. package/dist/core/canvas/history.js +1 -0
  49. package/dist/core/canvas/paths.d.ts +10 -9
  50. package/dist/core/canvas/paths.js +10 -9
  51. package/dist/core/canvas/pid.d.ts +155 -1
  52. package/dist/core/canvas/pid.js +306 -1
  53. package/dist/core/canvas/status-glyph.d.ts +7 -0
  54. package/dist/core/canvas/status-glyph.js +10 -1
  55. package/dist/core/canvas/types.d.ts +34 -0
  56. package/dist/core/fault-classifier.js +5 -1
  57. package/dist/core/profiles/select.d.ts +4 -2
  58. package/dist/core/profiles/select.js +30 -4
  59. package/dist/core/runtime/bearings.js +4 -0
  60. package/dist/core/runtime/branded-host.d.ts +7 -0
  61. package/dist/core/runtime/branded-host.js +44 -17
  62. package/dist/core/runtime/broker-sdk.js +28 -68
  63. package/dist/core/runtime/broker.js +35 -4
  64. package/dist/core/runtime/host.d.ts +3 -3
  65. package/dist/core/runtime/host.js +148 -33
  66. package/dist/core/runtime/launch.d.ts +15 -15
  67. package/dist/core/runtime/launch.js +58 -4
  68. package/dist/core/runtime/naming.js +3 -2
  69. package/dist/core/runtime/nodes.d.ts +3 -1
  70. package/dist/core/runtime/nodes.js +1 -0
  71. package/dist/core/runtime/pi-cli.d.ts +2 -0
  72. package/dist/core/runtime/pi-cli.js +51 -0
  73. package/dist/core/runtime/placement.d.ts +1 -1
  74. package/dist/core/runtime/placement.js +5 -2
  75. package/dist/core/runtime/recap.js +2 -1
  76. package/dist/core/runtime/revive.d.ts +21 -15
  77. package/dist/core/runtime/revive.js +87 -52
  78. package/dist/core/runtime/session-cycles.d.ts +30 -0
  79. package/dist/core/runtime/session-cycles.js +77 -0
  80. package/dist/core/runtime/spawn.d.ts +4 -0
  81. package/dist/core/runtime/spawn.js +152 -100
  82. package/dist/core/runtime/tmux.d.ts +7 -10
  83. package/dist/core/runtime/tmux.js +9 -11
  84. package/dist/core/worktree.d.ts +35 -0
  85. package/dist/core/worktree.js +158 -0
  86. package/dist/daemon/crtrd.d.ts +17 -8
  87. package/dist/daemon/crtrd.js +191 -40
  88. package/dist/index.d.ts +3 -0
  89. package/dist/index.js +7 -0
  90. package/dist/pi-extensions/canvas-context-intro.d.ts +1 -0
  91. package/dist/pi-extensions/canvas-context-intro.js +34 -23
  92. package/dist/pi-extensions/canvas-stophook.js +11 -5
  93. package/dist/web-client/assets/{index-CbO8L0mN.js → index-B00YpRQ1.js} +20 -20
  94. package/dist/web-client/assets/index-DrkcvANq.css +2 -0
  95. package/dist/web-client/index.html +2 -2
  96. package/docs/compat/hearth-crtr-v1.md +191 -0
  97. package/docs/public-api.md +75 -0
  98. package/package.json +3 -4
  99. package/dist/core/__tests__/hearth-bootstrap.test.js +0 -136
  100. package/dist/core/hearth/__tests__/model-auth-guest.test.js +0 -151
  101. package/dist/core/hearth/config.d.ts +0 -3
  102. package/dist/core/hearth/config.js +0 -108
  103. package/dist/core/hearth/guest-env.d.ts +0 -9
  104. package/dist/core/hearth/guest-env.js +0 -27
  105. package/dist/core/hearth/index.d.ts +0 -4
  106. package/dist/core/hearth/index.js +0 -4
  107. package/dist/core/hearth/model-auth-guest.d.ts +0 -8
  108. package/dist/core/hearth/model-auth-guest.js +0 -430
  109. package/dist/core/hearth/provider.d.ts +0 -36
  110. package/dist/core/hearth/provider.js +0 -10
  111. package/dist/core/hearth/providers/__tests__/sweep-and-release.test.js +0 -362
  112. package/dist/core/hearth/providers/blaxel-bootstrap.d.ts +0 -12
  113. package/dist/core/hearth/providers/blaxel-bootstrap.js +0 -147
  114. package/dist/core/hearth/providers/blaxel-home.d.ts +0 -60
  115. package/dist/core/hearth/providers/blaxel-home.js +0 -405
  116. package/dist/core/hearth/providers/blaxel.d.ts +0 -36
  117. package/dist/core/hearth/providers/blaxel.js +0 -364
  118. package/dist/core/hearth/providers/types.d.ts +0 -93
  119. package/dist/core/hearth/types.d.ts +0 -155
  120. package/dist/hearth/control-plane/__tests__/error-serialization.test.js +0 -29
  121. package/dist/hearth/control-plane/__tests__/node-message.test.js +0 -60
  122. package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.d.ts +0 -1
  123. package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.js +0 -44
  124. package/dist/hearth/control-plane/__tests__/rate-limit-recurrence.test.d.ts +0 -1
  125. package/dist/hearth/control-plane/__tests__/rate-limit-recurrence.test.js +0 -49
  126. package/dist/hearth/control-plane/__tests__/relay-security.test.d.ts +0 -1
  127. package/dist/hearth/control-plane/__tests__/relay-security.test.js +0 -314
  128. package/dist/hearth/control-plane/__tests__/scheduler-scan-loop.test.d.ts +0 -1
  129. package/dist/hearth/control-plane/__tests__/scheduler-scan-loop.test.js +0 -133
  130. package/dist/hearth/control-plane/__tests__/trigger-delivery.test.d.ts +0 -1
  131. package/dist/hearth/control-plane/__tests__/trigger-delivery.test.js +0 -170
  132. package/dist/hearth/control-plane/__tests__/wake-roll.test.d.ts +0 -1
  133. package/dist/hearth/control-plane/__tests__/wake-roll.test.js +0 -230
  134. package/dist/hearth/control-plane/__tests__/webhook-ingress.test.d.ts +0 -1
  135. package/dist/hearth/control-plane/__tests__/webhook-ingress.test.js +0 -167
  136. package/dist/hearth/control-plane/config.d.ts +0 -21
  137. package/dist/hearth/control-plane/config.js +0 -77
  138. package/dist/hearth/control-plane/db.d.ts +0 -30
  139. package/dist/hearth/control-plane/db.js +0 -561
  140. package/dist/hearth/control-plane/hearth-target.d.ts +0 -23
  141. package/dist/hearth/control-plane/hearth-target.js +0 -68
  142. package/dist/hearth/control-plane/ingress/rate-limit.d.ts +0 -24
  143. package/dist/hearth/control-plane/ingress/rate-limit.js +0 -100
  144. package/dist/hearth/control-plane/ingress/route-store.d.ts +0 -31
  145. package/dist/hearth/control-plane/ingress/route-store.js +0 -61
  146. package/dist/hearth/control-plane/ingress/webhook-delivery-store.d.ts +0 -41
  147. package/dist/hearth/control-plane/ingress/webhook-delivery-store.js +0 -69
  148. package/dist/hearth/control-plane/ingress/webhook-route.d.ts +0 -55
  149. package/dist/hearth/control-plane/ingress/webhook-route.js +0 -285
  150. package/dist/hearth/control-plane/main.d.ts +0 -1
  151. package/dist/hearth/control-plane/main.js +0 -88
  152. package/dist/hearth/control-plane/node-message.d.ts +0 -31
  153. package/dist/hearth/control-plane/node-message.js +0 -98
  154. package/dist/hearth/control-plane/register.d.ts +0 -15
  155. package/dist/hearth/control-plane/register.js +0 -34
  156. package/dist/hearth/control-plane/registry.d.ts +0 -22
  157. package/dist/hearth/control-plane/registry.js +0 -168
  158. package/dist/hearth/control-plane/relay.d.ts +0 -44
  159. package/dist/hearth/control-plane/relay.js +0 -711
  160. package/dist/hearth/control-plane/scheduler/fire-store.d.ts +0 -36
  161. package/dist/hearth/control-plane/scheduler/fire-store.js +0 -73
  162. package/dist/hearth/control-plane/scheduler/recurrence.d.ts +0 -7
  163. package/dist/hearth/control-plane/scheduler/recurrence.js +0 -58
  164. package/dist/hearth/control-plane/scheduler/scan-loop.d.ts +0 -38
  165. package/dist/hearth/control-plane/scheduler/scan-loop.js +0 -138
  166. package/dist/hearth/control-plane/scheduler/schedule-store.d.ts +0 -32
  167. package/dist/hearth/control-plane/scheduler/schedule-store.js +0 -66
  168. package/dist/hearth/control-plane/secrets.d.ts +0 -31
  169. package/dist/hearth/control-plane/secrets.js +0 -134
  170. package/dist/hearth/control-plane/server.d.ts +0 -27
  171. package/dist/hearth/control-plane/server.js +0 -482
  172. package/dist/hearth/control-plane/serving.d.ts +0 -15
  173. package/dist/hearth/control-plane/serving.js +0 -106
  174. package/dist/hearth/control-plane/session.d.ts +0 -68
  175. package/dist/hearth/control-plane/session.js +0 -273
  176. package/dist/hearth/control-plane/triggers/acl.d.ts +0 -14
  177. package/dist/hearth/control-plane/triggers/acl.js +0 -52
  178. package/dist/hearth/control-plane/triggers/audit-store.d.ts +0 -38
  179. package/dist/hearth/control-plane/triggers/audit-store.js +0 -79
  180. package/dist/hearth/control-plane/triggers/deliver.d.ts +0 -43
  181. package/dist/hearth/control-plane/triggers/deliver.js +0 -76
  182. package/dist/hearth/control-plane/triggers/envelope.d.ts +0 -29
  183. package/dist/hearth/control-plane/triggers/envelope.js +0 -38
  184. package/dist/hearth/control-plane/types.d.ts +0 -86
  185. package/dist/hearth/control-plane/types.js +0 -1
  186. package/dist/hearth/control-plane/wake.d.ts +0 -86
  187. package/dist/hearth/control-plane/wake.js +0 -550
  188. package/dist/web-client/assets/index-DwO46Cs5.css +0 -2
  189. /package/dist/{core/__tests__/hearth-bootstrap.test.d.ts → commands/__tests__/revive-now-gate.test.d.ts} +0 -0
  190. /package/dist/{core/hearth/__tests__/model-auth-guest.test.d.ts → commands/sys/__tests__/setup-core.test.d.ts} +0 -0
  191. /package/dist/core/{hearth/providers/__tests__/sweep-and-release.test.d.ts → __tests__/fault-classifier.test.d.ts} +0 -0
  192. /package/dist/core/{hearth/providers/types.js → __tests__/host-teardown-process-group.test.d.ts} +0 -0
  193. /package/dist/core/{hearth/types.js → __tests__/review-model-floor.test.d.ts} +0 -0
  194. /package/dist/{hearth/control-plane/__tests__/error-serialization.test.d.ts → core/__tests__/session-cycles.test.d.ts} +0 -0
  195. /package/dist/{hearth/control-plane/__tests__/node-message.test.d.ts → core/__tests__/worktree.test.d.ts} +0 -0
@@ -1,550 +0,0 @@
1
- // control-plane/wake.ts — CP wake orchestration (P4.2 / part-plan T3.1).
2
- //
3
- // Generalizes M0 `wake-proxy/home.ts` (`ensureAwake`/`resolveRunningHome`/
4
- // `probeRelayServing`) onto the CP `home` registry, with one deliberate
5
- // behavior drop: NO idle-pause machinery (S3 — the provider owns
6
- // idle->suspend; M0's `scheduleIdlePause`/`pauseNow` timer chain is not
7
- // reproduced here, by design, not oversight).
8
- //
9
- // Pipeline (R-WAKE-1): read the `home` row -> resolve the relay token handle
10
- // -> call the provider wake/resume seam (`HomeBackend.wake`, owned by the
11
- // home/provider slice, IP-1) -> ready-probe the in-guest web server on
12
- // `/node/<home_agent_target>` over a short retried window, carrying the
13
- // relay token -> if the provider returned a new sandbox/preview (a
14
- // recreate), persist the refreshed pointers; `home_agent_target` is NEVER
15
- // part of that patch (R-WAKE-6 — `refreshProviderPointers` doesn't accept it)
16
- // -> record wake/ready timing -> return the route. Readiness here means the
17
- // guest web server answers `/node/<target>` AND relays to a live broker
18
- // `view.sock` (R-WAKE-2 — not "the SPA shell renders"). `src/clients/web/
19
- // server.ts` only proves that second half on a WS upgrade: `bridgeConnection`
20
- // opens `view.sock` AFTER the 101 response and closes the socket with 1011
21
- // the instant that fails (ECONNREFUSED/ENOENT — "no running broker"), so an
22
- // HTTP GET 200 on `/node/<target>` (the SPA static/fallback route) proves
23
- // only that the guest web server is up, not that the broker is. The probe
24
- // therefore opens a real WebSocket to `/node/<target>` and requires a
25
- // successful upgrade that SURVIVES a short settle window (or yields a first
26
- // frame) before declaring readiness.
27
- //
28
- // All timers used during a probe retry window are cleared before the
29
- // function that started them returns — there is no `setTimeout`/`setInterval`
30
- // left armed once `ensureAwake`/`resolveRunning` resolve (S3 regression bar).
31
- import { setTimeout as delay } from 'node:timers/promises';
32
- import { WebSocket } from 'ws';
33
- import { general } from '../../core/errors.js';
34
- import { hearthHomeImageRefForVersion, loadM0Config } from '../../core/hearth/config.js';
35
- import { getHomeBackend } from '../../core/hearth/provider.js';
36
- import { loadControlPlaneConfig } from './config.js';
37
- import { getHearthTarget, recordBadTarget } from './hearth-target.js';
38
- import { getHomeById, refreshProviderPointers, setHealthVerdict, setHomeStatus, touchReady, touchWake } from './registry.js';
39
- import { isHomeServing as isHomeServingNow } from './serving.js';
40
- import { resolveSecret } from './secrets.js';
41
- // Individual probe attempt timeout — bounded short so a dead route fails
42
- // fast and the retry loop gets multiple attempts inside the overall window,
43
- // rather than one attempt eating the whole window (mirrors M0's
44
- // `READY_PROBE_TIMEOUT_MS`).
45
- const READY_PROBE_ATTEMPT_TIMEOUT_MS = 1_500;
46
- // Backoff between probe attempts (Q1: "500ms backoff between attempts").
47
- const READY_PROBE_BACKOFF_MS = 500;
48
- // How long a confirmed-ready verdict is trusted before `resolveRunning`/
49
- // `ensureAwake` re-checks instead of trusting a stale answer (Q1: "10s TTL,
50
- // in-process"). Single CP process, no distributed cache (S2).
51
- const WARM_VERDICT_TTL_MS = 10_000;
52
- /** Normalize a provider-observed runtime status into the CP `HomeStatus`
53
- * vocabulary before it is ever persisted (Q3 / P3.4 review Minor finding):
54
- * the provider's `MachineState` is broad (and may include future
55
- * pause/suspend/standby spellings the SDK doesn't expose yet); CP's
56
- * `HomeStatus` enum does not have a `'paused'` member. Anything
57
- * pause/suspend/standby-shaped collapses to `'suspended'`; an exact
58
- * `'running'` passes through; everything else (provisioning, terminating,
59
- * terminated, restoring, unrecognized) becomes `'unknown'` rather than
60
- * guessing a more specific CP status from a provider-specific string. */
61
- export function normalizeProviderStatus(providerStatus) {
62
- if (providerStatus === null || providerStatus === undefined)
63
- return 'unknown';
64
- const normalized = providerStatus.trim().toLowerCase();
65
- if (normalized === 'paused' || normalized === 'suspended' || normalized === 'standby')
66
- return 'suspended';
67
- if (normalized === 'running')
68
- return 'running';
69
- return 'unknown';
70
- }
71
- function requireHomeField(value, field, home_id) {
72
- if (value === null || value === undefined || (typeof value === 'string' && value.trim() === '')) {
73
- throw general(`home ${home_id} is missing ${field} — cannot wake`, { home_id, field });
74
- }
75
- return value;
76
- }
77
- function buildDescriptor(home, relayToken) {
78
- return {
79
- homeId: home.home_id,
80
- tenantId: home.tenant_id,
81
- userId: home.user_id,
82
- providerName: home.provider,
83
- providerSandboxId: requireHomeField(home.provider_sandbox_id, 'provider_sandbox_id', home.home_id),
84
- providerVolumeId: requireHomeField(home.provider_volume_id, 'provider_volume_id', home.home_id),
85
- previewEndpoint: home.preview_endpoint,
86
- homeAgentTarget: requireHomeField(home.home_agent_target, 'home_agent_target', home.home_id),
87
- relayAuthRef: home.relay_auth_ref,
88
- relayToken,
89
- status: home.status,
90
- templateVersion: home.template_version,
91
- };
92
- }
93
- // After a successful 101 upgrade, `bridgeConnection` (server.ts) still has to
94
- // dial the broker's `view.sock` and can close the socket (1011, "no running
95
- // broker") the instant that dial fails. That happens fast on a local unix
96
- // socket (ECONNREFUSED/ENOENT are immediate), so a settle window this short
97
- // is enough to let a dead-broker close win the race against a false-positive
98
- // `open` before the probe declares readiness.
99
- const READY_PROBE_WS_SETTLE_MS = 300;
100
- function wsUrlForNode(previewEndpoint, target) {
101
- const url = new URL(`/node/${encodeURIComponent(target)}`, previewEndpoint);
102
- url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
103
- return url;
104
- }
105
- /** One bounded attempt: open a WebSocket to `/node/<target>` carrying the
106
- * relay token and require the upgrade to both succeed AND survive a short
107
- * settle window (or yield a first frame) within `attemptTimeoutMs` — proof
108
- * the in-guest web server answered AND relayed to a live broker `view.sock`
109
- * (R-WAKE-2), not just that the SPA shell route is up. Every branch
110
- * (success, handshake rejection, error, premature close, attempt timeout)
111
- * closes the socket and resolves exactly once — a success closes gracefully
112
- * (code 1000) so the guest relay does not record a false fault, anything
113
- * else terminates it. */
114
- async function probeNodeOnce(previewEndpoint, target, relayToken, attemptTimeoutMs) {
115
- return new Promise((resolveProbe) => {
116
- let settled = false;
117
- let settleTimer = null;
118
- let ws;
119
- try {
120
- ws = new WebSocket(wsUrlForNode(previewEndpoint, target), {
121
- headers: { authorization: `Bearer ${relayToken}` },
122
- perMessageDeflate: false,
123
- });
124
- }
125
- catch {
126
- resolveProbe(false);
127
- return;
128
- }
129
- const finish = (ready) => {
130
- if (settled)
131
- return;
132
- settled = true;
133
- if (settleTimer !== null)
134
- clearTimeout(settleTimer);
135
- clearTimeout(attemptTimer);
136
- try {
137
- // A confirmed-ready probe closes gracefully (code 1000) so the guest
138
- // relay's `bridgeConnection` sees a normal browser close and does NOT
139
- // record a false browser<->relay fault (P4.2 re-review Major). `close()`
140
- // sends the close frame synchronously off the OPEN socket, so resolving
141
- // right after calling it still delivers a normal close to the guest —
142
- // the probe does not need to wait for the closing handshake to finish.
143
- // Failure/timeout/error paths still abort hard via `terminate()`: there
144
- // is nothing graceful to negotiate when the socket never proved ready.
145
- if (ready) {
146
- ws.close(1000, 'readiness probe complete');
147
- }
148
- else {
149
- ws.terminate();
150
- }
151
- }
152
- catch {
153
- /* ignore */
154
- }
155
- resolveProbe(ready);
156
- };
157
- const attemptTimer = setTimeout(() => finish(false), attemptTimeoutMs);
158
- ws.once('open', () => {
159
- // The attempt timer (above) still bounds the total wait, so it is safe
160
- // to always arm the full settle window here — whichever timer fires
161
- // first wins via `finish`'s once-only guard.
162
- settleTimer = setTimeout(() => finish(true), Math.min(READY_PROBE_WS_SETTLE_MS, attemptTimeoutMs));
163
- });
164
- ws.once('message', () => finish(true));
165
- ws.once('unexpected-response', () => finish(false));
166
- ws.once('error', () => finish(false));
167
- ws.once('close', () => finish(false));
168
- });
169
- }
170
- /** Retry the single-shot probe until it confirms `/node/<target>` is serving
171
- * or `windowMs` elapses. Always runs at least one attempt (R-WAKE-1: "a
172
- * single failed probe does not abort"). Every timer this loop starts
173
- * (the per-attempt abort timeout, the inter-attempt backoff) is cleared/
174
- * resolved before the loop returns — nothing is left armed. */
175
- async function probeNodeReady(previewEndpoint, target, relayToken, windowMs) {
176
- const deadline = Date.now() + Math.max(0, windowMs);
177
- let firstAttempt = true;
178
- for (;;) {
179
- const remaining = deadline - Date.now();
180
- // Past the first attempt, an already-expired budget ends the loop here
181
- // instead of spending one more full attempt timeout (P4.2 re-review
182
- // Minor) — only the very first attempt is unconditional (R-WAKE-1: "a
183
- // single failed probe does not abort"/at-least-one-attempt).
184
- if (!firstAttempt && remaining <= 0)
185
- return false;
186
- const attemptTimeoutMs = firstAttempt
187
- ? Math.max(250, Math.min(READY_PROBE_ATTEMPT_TIMEOUT_MS, windowMs))
188
- : Math.min(READY_PROBE_ATTEMPT_TIMEOUT_MS, remaining);
189
- firstAttempt = false;
190
- if (await wakeInternals.probeNodeOnce(previewEndpoint, target, relayToken, attemptTimeoutMs))
191
- return true;
192
- const remainingAfter = deadline - Date.now();
193
- if (remainingAfter <= 0)
194
- return false;
195
- await delay(Math.min(READY_PROBE_BACKOFF_MS, remainingAfter));
196
- }
197
- }
198
- /** Indirection seam so a smoke/test script can swap any step (the provider
199
- * backend, the registry setters, the secret resolver, the probe itself, the
200
- * clock) without touching a real DB/provider/filesystem. Production code
201
- * never reassigns these; only ad hoc verification scripts do. */
202
- export const wakeInternals = {
203
- getHomeById,
204
- getHearthTarget,
205
- recordBadTarget,
206
- refreshProviderPointers,
207
- setHomeStatus,
208
- setHealthVerdict,
209
- touchWake,
210
- touchReady,
211
- resolveSecret,
212
- getHomeBackend: (config) => getHomeBackend(config),
213
- loadM0Config: () => loadM0Config(),
214
- loadControlPlaneConfig: () => loadControlPlaneConfig(),
215
- probeNodeOnce,
216
- probeNodeReady,
217
- now: () => Date.now(),
218
- };
219
- const warmVerdicts = new Map();
220
- const inflightWakes = new Map();
221
- /** Cheap, non-probing fast path: return a still-fresh warm-verdict cache hit
222
- * for `home_id`, or `null` if there is none (expired or never confirmed).
223
- * Never reads the registry and never calls the provider — callers fall back
224
- * to `ensureAwake` on a `null` (Q1: "no call to provider"). */
225
- export function resolveRunning(home_id) {
226
- const warm = warmVerdicts.get(home_id);
227
- if (warm === undefined)
228
- return null;
229
- if (wakeInternals.now() - warm.confirmedAt >= WARM_VERDICT_TTL_MS)
230
- return null;
231
- return warm.result;
232
- }
233
- /** Drop the cached warm verdict for a home (e.g. an upstream relay forward
234
- * failed) so the next `ensureAwake`/`resolveRunning` re-checks instead of
235
- * trusting a stale "ready" answer. */
236
- export function invalidateWarmVerdict(home_id) {
237
- warmVerdicts.delete(home_id);
238
- }
239
- /** Clear all warm verdicts and in-flight wake coalescing. Test/smoke-script
240
- * hygiene only — production never needs this (one CP process, S2). */
241
- export function resetWakeCacheForTests() {
242
- warmVerdicts.clear();
243
- inflightWakes.clear();
244
- }
245
- /** Is this home currently serving, so a destructive roll would interrupt it?
246
- * Delegates entirely to `serving.ts`'s independent live-session counters,
247
- * bumped at the relay's HTTP/WS lifecycle seams and the CP's OAuth
248
- * start/finish dispatch — never derived from any TTL/cache. `home.status` is
249
- * intentionally NOT consulted: a suspended home has no live counters anyway
250
- * (nothing to bump them), so this stays a single source of truth. */
251
- function homeIsServing(home) {
252
- return isHomeServingNow(home.home_id);
253
- }
254
- /** Is this home due to roll onto the registered target? Stale = it isn't
255
- * already on the target version AND the target is not a known-bad release.
256
- * The bad-target clause is the ONLY brake in the zero-touch loop (roadmap
257
- * decision 1: never auto-advance onto a release that already failed a roll):
258
- * once `recordBadTarget` poisons the current target, no home rolls onto it
259
- * until a NEW release advances the target past it.
260
- *
261
- * NOTE (design divergence, flagged): design §3's literal snippet compared
262
- * `home.template_version !== last_bad_target_version`, which would THRASH —
263
- * after a failed roll a home sits on the PRIOR version, so that clause stays
264
- * true and it would re-roll onto the same broken target forever. The roadmap's
265
- * plain-English brake and design §5's own comment both mean `target !==
266
- * last_bad`; that is what is implemented here. */
267
- function isHomeStale(home, target) {
268
- if (home.template_version === null)
269
- return false;
270
- if (home.template_version === target.target_template_version)
271
- return false;
272
- if (target.last_bad_target_version !== null && target.target_template_version === target.last_bad_target_version)
273
- return false;
274
- return true;
275
- }
276
- // Baked crtr path (deploy/hearth-blaxel/Dockerfile symlink) — target it
277
- // directly for the in-guest version assert rather than a bare PATH lookup,
278
- // exactly as blaxel-bootstrap.ts does, so a stale legacy binary earlier on
279
- // PATH can't answer instead.
280
- const BAKED_CRTR_PATH = '/usr/local/bin/crtr';
281
- /** Assert what crtr the recreated guest is ACTUALLY running (design §4 step 7)
282
- * by reading `crtr --json sys version` in-guest — stronger than trusting the
283
- * provider's config version, since it proves the booted binary. Parses the
284
- * `.version` field from `{ "version": "X.Y.Z" }`. The same drift-guard env
285
- * vars the bootstrap uses keep this a pure read (no bootstrap/auto-init/
286
- * auto-update side effects). */
287
- async function assertGuestVersion(backend, sandboxName) {
288
- const result = await backend.exec(sandboxName, {
289
- command: `CRTR_NO_BOOTSTRAP=1 CRTR_NO_AUTO_INIT=1 CRTR_NO_AUTO_UPDATE=1 ${BAKED_CRTR_PATH} --json sys version`,
290
- timeoutMs: 20_000,
291
- });
292
- // A parseable stdout is not proof the command itself succeeded — require a
293
- // clean exit before trusting anything it printed.
294
- if (result.exitCode !== 0 || result.timedOut === true) {
295
- throw general(`in-guest crtr sys version did not exit cleanly (exitCode ${result.exitCode}, timedOut ${result.timedOut})`, {
296
- sandboxName,
297
- exitCode: result.exitCode,
298
- timedOut: result.timedOut,
299
- stdout: result.stdout.slice(0, 200),
300
- stderr: result.stderr.slice(0, 200),
301
- });
302
- }
303
- const stdout = result.stdout.trim();
304
- try {
305
- const parsed = JSON.parse(stdout);
306
- if (typeof parsed.version === 'string' && parsed.version.trim() !== '')
307
- return parsed.version.trim();
308
- }
309
- catch {
310
- /* fall through to a tolerant field extraction below */
311
- }
312
- const match = /"version"\s*:\s*"([^"]+)"/.exec(stdout);
313
- if (match !== null && match[1].trim() !== '')
314
- return match[1].trim();
315
- throw general(`in-guest crtr sys version returned no parseable .version (exit ${result.exitCode})`, {
316
- sandboxName,
317
- stdout: stdout.slice(0, 200),
318
- });
319
- }
320
- /** Structured HIGH-SEVERITY alert lines. The CP is a standalone Fly process
321
- * and cannot `crtr push`; alerting is a distinct `level: 'error'` stderr line
322
- * with an `alert` marker that ops log-alerting keys on. (Wiring these to a
323
- * real notification channel is future work — the marker is the seam.) */
324
- function alertRollRolledBack(home, target, restoredVersion) {
325
- process.stderr.write(`${JSON.stringify({
326
- level: 'error',
327
- alert: 'hearth.roll.rolled_back',
328
- message: `Hearth auto-upgrade roll FAILED and was rolled back — release ${target.target_template_version} is broken`,
329
- home_id: home.home_id,
330
- bad_target_version: target.target_template_version,
331
- restored_version: restoredVersion,
332
- })}\n`);
333
- }
334
- export function serializeError(error) {
335
- if (error instanceof Error)
336
- return error.message;
337
- try {
338
- // `JSON.stringify` returns `undefined` (does not throw) for `undefined`, a function, or a
339
- // symbol — falling through to that would OMIT the `error` field from the incident alert's
340
- // JSON line entirely, which is worse than the "[object Object]" bug this rider fixes.
341
- const json = JSON.stringify(error);
342
- return json === undefined ? String(error) : json;
343
- }
344
- catch {
345
- return String(error);
346
- }
347
- }
348
- function alertRollIncident(home, target, priorVersion, error) {
349
- process.stderr.write(`${JSON.stringify({
350
- level: 'error',
351
- alert: 'hearth.roll.incident',
352
- message: `Hearth auto-upgrade roll AND rollback both failed — home ${home.home_id} is DEGRADED, manual recovery needed`,
353
- home_id: home.home_id,
354
- bad_target_version: target.target_template_version,
355
- prior_version: priorVersion,
356
- error: serializeError(error),
357
- })}\n`);
358
- }
359
- /** Fall back onto the PRIOR pinned image after a failed roll (design §5). The
360
- * volume was never touched by the roll, so this is deterministic: recreate on
361
- * the prior image name (still in the registry — images are never pruned),
362
- * revive, ready-probe, assert `crtr sys version === priorVersion`. On success,
363
- * ALERT (the release is broken) — the bad target is already poisoned by
364
- * `rollHome`'s catch (the roll's own failure is what poisons the target, not
365
- * rollback's success, so the brake is set even when rollback ALSO fails).
366
- * On rollback also failing: mark the home
367
- * `unknown`/`degraded` and alert — a genuine incident (the volume is intact,
368
- * so manual recovery is always possible). */
369
- async function rollbackHome(args) {
370
- const { home, target, descriptor, priorImageRef, priorVersion, relayToken, backend, readyTimeoutMs } = args;
371
- try {
372
- const refresh = await backend.recreateOnImage(descriptor, priorImageRef, priorVersion);
373
- const previewEndpoint = requireHomeField(refresh.previewEndpoint ?? null, 'preview_endpoint', home.home_id);
374
- const ready = await wakeInternals.probeNodeReady(previewEndpoint, descriptor.homeAgentTarget, relayToken, readyTimeoutMs);
375
- if (!ready)
376
- throw general(`rollback ready-probe timed out for ${home.home_id}`, { home_id: home.home_id });
377
- // Assert against the sandbox the recreate actually reports, not a stale name captured at
378
- // entry — under generation naming (recreate-race-fix.md §3.5) the pre-recreate sandbox is
379
- // destroyed, so exec-ing into anything else would target a gone/wrong generation.
380
- const asserted = await assertGuestVersion(backend, refresh.providerSandboxId);
381
- if (asserted !== priorVersion) {
382
- throw general(`rollback version mismatch: expected ${priorVersion}, got ${asserted}`, { home_id: home.home_id, expected: priorVersion, asserted });
383
- }
384
- wakeInternals.refreshProviderPointers(home.home_id, {
385
- provider_sandbox_id: refresh.providerSandboxId,
386
- preview_endpoint: previewEndpoint,
387
- template_version: priorVersion,
388
- status: 'running',
389
- });
390
- wakeInternals.setHealthVerdict(home.home_id, 'healthy');
391
- alertRollRolledBack(home, target, priorVersion);
392
- return { preview_endpoint: previewEndpoint, relayToken };
393
- }
394
- catch (rollbackError) {
395
- wakeInternals.setHomeStatus(home.home_id, 'unknown');
396
- wakeInternals.setHealthVerdict(home.home_id, 'degraded');
397
- alertRollIncident(home, target, priorVersion, rollbackError);
398
- throw general(`hearth roll AND rollback both failed for ${home.home_id}`, {
399
- home_id: home.home_id,
400
- target: target.target_template_version,
401
- priorVersion,
402
- });
403
- }
404
- }
405
- /** CP-driven destroy→recreate of a stale home onto the registered target
406
- * image, reattaching the same durable volume (design §4). Captures rollback
407
- * anchors first, marks the home `restoring` (so a concurrent wake coalesces
408
- * rather than races), then recreates on the target image, ready-probes, and
409
- * ASSERTS the in-guest `crtr sys version` equals the target before committing
410
- * pointers + `health_verdict = healthy`. A roll never writes
411
- * `home_agent_target` (canvas.db is reattached byte-identical;
412
- * `refreshProviderPointers` has no such field). Any of recreate/probe/assert
413
- * failing falls back to the prior image (§5). Returns the route so the
414
- * triggering wake serves the just-upgraded home in the same call. */
415
- async function rollHome(home, target, relayToken, descriptor) {
416
- const priorVersion = requireHomeField(home.template_version, 'template_version', home.home_id);
417
- const priorImageRef = hearthHomeImageRefForVersion(priorVersion);
418
- const m0 = wakeInternals.loadM0Config();
419
- const backend = wakeInternals.getHomeBackend(m0);
420
- const cpConfig = wakeInternals.loadControlPlaneConfig();
421
- wakeInternals.setHomeStatus(home.home_id, 'restoring');
422
- invalidateWarmVerdict(home.home_id);
423
- const startedAt = wakeInternals.now();
424
- try {
425
- const refresh = await backend.recreateOnImage(descriptor, target.target_image_ref, target.target_template_version);
426
- const previewEndpoint = requireHomeField(refresh.previewEndpoint ?? null, 'preview_endpoint', home.home_id);
427
- const ready = await wakeInternals.probeNodeReady(previewEndpoint, descriptor.homeAgentTarget, relayToken, cpConfig.readyTimeoutMs);
428
- if (!ready)
429
- throw general(`roll ready-probe timed out for ${home.home_id}`, { home_id: home.home_id });
430
- // Assert against the sandbox the recreate actually reports (recreate-race-fix.md §3.5) —
431
- // never the row's pre-recreate `provider_sandbox_id`, which under generation naming is
432
- // already destroyed by the time this runs.
433
- const asserted = await assertGuestVersion(backend, refresh.providerSandboxId);
434
- if (asserted !== target.target_template_version) {
435
- throw general(`roll version mismatch: expected ${target.target_template_version}, got ${asserted}`, {
436
- home_id: home.home_id,
437
- expected: target.target_template_version,
438
- asserted,
439
- });
440
- }
441
- wakeInternals.refreshProviderPointers(home.home_id, {
442
- provider_sandbox_id: refresh.providerSandboxId,
443
- preview_endpoint: previewEndpoint,
444
- template_version: asserted,
445
- status: 'running',
446
- });
447
- wakeInternals.setHealthVerdict(home.home_id, 'healthy');
448
- wakeInternals.touchWake(home.home_id, { at: new Date(startedAt).toISOString(), latency_ms: wakeInternals.now() - startedAt });
449
- wakeInternals.touchReady(home.home_id);
450
- return { preview_endpoint: previewEndpoint, relayToken };
451
- }
452
- catch {
453
- // The target is known-bad the INSTANT its own roll fails, independent of
454
- // whether rollback then recovers — poison it here, before delegating to
455
- // rollbackHome, so a roll-fails+rollback-also-fails home doesn't leave the
456
- // brake unset (which would let a later wake, of this home or any other,
457
- // re-roll onto the same broken target).
458
- wakeInternals.recordBadTarget(target.target_template_version);
459
- return rollbackHome({ home, target, descriptor, priorImageRef, priorVersion, relayToken, backend, readyTimeoutMs: cpConfig.readyTimeoutMs });
460
- }
461
- }
462
- async function runEnsureAwake(home_id) {
463
- const home = wakeInternals.getHomeById(home_id);
464
- if (home === null)
465
- throw general(`home not found: ${home_id}`, { home_id });
466
- const relayAuthRef = requireHomeField(home.relay_auth_ref, 'relay_auth_ref', home_id);
467
- const relayToken = wakeInternals.resolveSecret(relayAuthRef);
468
- const target = requireHomeField(home.home_agent_target, 'home_agent_target', home_id);
469
- const descriptor = buildDescriptor(home, relayToken);
470
- // Auto-upgrade lazy roll (design §3/§4): if a newer image target is
471
- // registered, this home is stale, and it is NOT currently serving, roll it
472
- // onto the target image now instead of the normal wake. At wake-entry an
473
- // idle home is definitionally safe to tear down; a serving home doesn't roll
474
- // this pass (it rolls on its next cold wake). This is the ONLY new branch —
475
- // the normal wake path below is unchanged.
476
- const hearthTarget = wakeInternals.getHearthTarget();
477
- if (hearthTarget !== null && isHomeStale(home, hearthTarget) && !homeIsServing(home)) {
478
- return rollHome(home, hearthTarget, relayToken, descriptor);
479
- }
480
- const startedAt = wakeInternals.now();
481
- const m0 = wakeInternals.loadM0Config();
482
- const refresh = await wakeInternals.getHomeBackend(m0).wake(descriptor);
483
- const normalizedStatus = normalizeProviderStatus(refresh.status);
484
- const recreated = refresh.providerSandboxId !== home.provider_sandbox_id || (refresh.previewEndpoint ?? null) !== home.preview_endpoint;
485
- let previewEndpoint;
486
- if (recreated) {
487
- // R-WAKE-6, non-negotiable: persist the new sandbox/preview pointers but
488
- // never the home node target — `refreshProviderPointers`'s input shape
489
- // has no `home_agent_target` field, so there is no way for this call to
490
- // touch it even by accident.
491
- const updated = wakeInternals.refreshProviderPointers(home_id, {
492
- provider_sandbox_id: refresh.providerSandboxId,
493
- preview_endpoint: refresh.previewEndpoint ?? null,
494
- status: normalizedStatus,
495
- template_version: refresh.templateVersion ?? home.template_version,
496
- });
497
- previewEndpoint = updated.preview_endpoint;
498
- }
499
- else {
500
- wakeInternals.setHomeStatus(home_id, normalizedStatus);
501
- previewEndpoint = home.preview_endpoint;
502
- }
503
- const resolvedPreviewEndpoint = requireHomeField(previewEndpoint, 'preview_endpoint', home_id);
504
- const controlPlaneConfig = wakeInternals.loadControlPlaneConfig();
505
- const ready = await wakeInternals.probeNodeReady(resolvedPreviewEndpoint, target, relayToken, controlPlaneConfig.readyTimeoutMs);
506
- if (!ready) {
507
- throw general(`guest web relay did not become ready before timeout: ${home_id}`, { home_id, previewEndpoint: resolvedPreviewEndpoint });
508
- }
509
- const readyAt = new Date().toISOString();
510
- wakeInternals.touchWake(home_id, { at: new Date(startedAt).toISOString(), latency_ms: wakeInternals.now() - startedAt });
511
- wakeInternals.touchReady(home_id, readyAt);
512
- // A successful ready-probe means the home is healthy right now, regardless of what the
513
- // verdict was before this wake (e.g. left `degraded` by an earlier failed rollback) —
514
- // clear it here the same way rollHome/rollbackHome clear it on their own success, or a
515
- // stale degraded/unknown verdict never heals on the normal wake path.
516
- wakeInternals.setHealthVerdict(home_id, 'healthy');
517
- return { preview_endpoint: resolvedPreviewEndpoint, relayToken };
518
- }
519
- /** Bring a home up and ready, returning the route to relay to once it is.
520
- * Reads the registry, resolves the relay token, calls the provider
521
- * wake/resume seam, ready-probes `/node/<home_agent_target>`, refreshes
522
- * provider pointers on a recreate, records wake/ready timing, clears the
523
- * health verdict to `healthy` on success, and returns
524
- * `{ preview_endpoint, relayToken }`. Throws on a ready-probe timeout
525
- * (wake-failure UX is deferred — gap note 2 — a throw is the accepted
526
- * cutover behavior).
527
- *
528
- * A fresh warm-verdict cache hit short-circuits without re-entering this
529
- * pipeline at all. Otherwise concurrent callers for the SAME `home_id`
530
- * coalesce onto one in-flight wake (Q1) instead of each driving their own
531
- * provider call. */
532
- export async function ensureAwake(home_id) {
533
- const warm = resolveRunning(home_id);
534
- if (warm !== null)
535
- return warm;
536
- const inflight = inflightWakes.get(home_id);
537
- if (inflight !== undefined)
538
- return inflight;
539
- const promise = runEnsureAwake(home_id)
540
- .then((result) => {
541
- warmVerdicts.set(home_id, { result, confirmedAt: wakeInternals.now() });
542
- return result;
543
- })
544
- .finally(() => {
545
- if (inflightWakes.get(home_id) === promise)
546
- inflightWakes.delete(home_id);
547
- });
548
- inflightWakes.set(home_id, promise);
549
- return promise;
550
- }