@crouton-kit/crouter 0.3.40 → 0.3.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-root.js +1 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/provider-rotation.test.ts +472 -9
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +284 -76
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/strip-skills-docs.ts +31 -24
- package/dist/builtin-pi-packages/pi-crtr-extensions/lib/subscription-state.ts +52 -15
- package/dist/clients/attach/attach-cmd.js +720 -747
- package/dist/clients/attach/chat-view.js +11 -0
- package/dist/clients/attach/view-socket.d.ts +8 -1
- package/dist/clients/attach/view-socket.js +15 -1
- package/dist/clients/web/server.js +5 -28
- package/dist/clients/web/web-cmd.js +1 -1
- package/dist/commands/__tests__/revive-now-gate.test.js +60 -0
- package/dist/commands/canvas-rebuild-index.js +5 -5
- package/dist/commands/memory/shared.d.ts +7 -3
- package/dist/commands/memory/shared.js +35 -5
- package/dist/commands/memory/write.js +5 -3
- package/dist/commands/node-snapshot.js +9 -1
- package/dist/commands/node.js +37 -17
- package/dist/commands/push.js +8 -0
- package/dist/commands/revive.d.ts +10 -0
- package/dist/commands/revive.js +28 -10
- package/dist/commands/sys/__tests__/setup-core.test.js +19 -0
- package/dist/commands/sys/doctor.js +1 -1
- package/dist/commands/sys/setup-core.js +3 -2
- package/dist/commands/sys/setup.js +1 -1
- package/dist/commands/worktree.d.ts +2 -0
- package/dist/commands/worktree.js +94 -0
- package/dist/core/__tests__/boot.test.js +4 -4
- package/dist/core/__tests__/canvas.test.js +19 -7
- package/dist/core/__tests__/child-followup.test.js +15 -5
- package/dist/core/__tests__/daemon-boot.test.js +6 -1
- package/dist/core/__tests__/daemon-wedge.test.js +18 -1
- package/dist/core/__tests__/fault-classifier.test.js +30 -0
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +13 -0
- package/dist/core/__tests__/fixtures/fake-engine.js +20 -0
- package/dist/core/__tests__/full/spike-harness.test.js +8 -5
- package/dist/core/__tests__/grace-clock.test.js +18 -2
- package/dist/core/__tests__/host-teardown-process-group.test.js +465 -0
- package/dist/core/__tests__/prune-to-limit.test.js +14 -0
- package/dist/core/__tests__/review-model-floor.test.js +32 -0
- package/dist/core/__tests__/revive.test.js +18 -4
- package/dist/core/__tests__/session-cycles.test.js +77 -0
- package/dist/core/__tests__/worktree.test.js +85 -0
- package/dist/core/canvas/boot.js +12 -7
- package/dist/core/canvas/canvas.d.ts +27 -8
- package/dist/core/canvas/canvas.js +54 -26
- package/dist/core/canvas/db.js +14 -0
- package/dist/core/canvas/history.js +1 -0
- package/dist/core/canvas/paths.d.ts +10 -9
- package/dist/core/canvas/paths.js +10 -9
- package/dist/core/canvas/pid.d.ts +155 -1
- package/dist/core/canvas/pid.js +306 -1
- package/dist/core/canvas/status-glyph.d.ts +7 -0
- package/dist/core/canvas/status-glyph.js +10 -1
- package/dist/core/canvas/types.d.ts +34 -0
- package/dist/core/fault-classifier.js +5 -1
- package/dist/core/profiles/select.d.ts +4 -2
- package/dist/core/profiles/select.js +30 -4
- package/dist/core/runtime/bearings.js +4 -0
- package/dist/core/runtime/branded-host.d.ts +7 -0
- package/dist/core/runtime/branded-host.js +44 -17
- package/dist/core/runtime/broker-sdk.js +28 -68
- package/dist/core/runtime/broker.js +35 -4
- package/dist/core/runtime/host.d.ts +3 -3
- package/dist/core/runtime/host.js +148 -33
- package/dist/core/runtime/launch.d.ts +15 -15
- package/dist/core/runtime/launch.js +58 -4
- package/dist/core/runtime/naming.js +3 -2
- package/dist/core/runtime/nodes.d.ts +3 -1
- package/dist/core/runtime/nodes.js +1 -0
- package/dist/core/runtime/pi-cli.d.ts +2 -0
- package/dist/core/runtime/pi-cli.js +51 -0
- package/dist/core/runtime/placement.d.ts +1 -1
- package/dist/core/runtime/placement.js +5 -2
- package/dist/core/runtime/recap.js +2 -1
- package/dist/core/runtime/revive.d.ts +21 -15
- package/dist/core/runtime/revive.js +87 -52
- package/dist/core/runtime/session-cycles.d.ts +30 -0
- package/dist/core/runtime/session-cycles.js +77 -0
- package/dist/core/runtime/spawn.d.ts +4 -0
- package/dist/core/runtime/spawn.js +152 -100
- package/dist/core/runtime/tmux.d.ts +7 -10
- package/dist/core/runtime/tmux.js +9 -11
- package/dist/core/worktree.d.ts +35 -0
- package/dist/core/worktree.js +158 -0
- package/dist/daemon/crtrd.d.ts +17 -8
- package/dist/daemon/crtrd.js +191 -40
- package/dist/index.d.ts +3 -0
- package/dist/index.js +7 -0
- package/dist/pi-extensions/canvas-context-intro.d.ts +1 -0
- package/dist/pi-extensions/canvas-context-intro.js +34 -23
- package/dist/pi-extensions/canvas-stophook.js +11 -5
- package/dist/web-client/assets/{index-CbO8L0mN.js → index-B00YpRQ1.js} +20 -20
- package/dist/web-client/assets/index-DrkcvANq.css +2 -0
- package/dist/web-client/index.html +2 -2
- package/docs/compat/hearth-crtr-v1.md +191 -0
- package/docs/public-api.md +75 -0
- package/package.json +3 -4
- package/dist/core/__tests__/hearth-bootstrap.test.js +0 -136
- package/dist/core/hearth/__tests__/model-auth-guest.test.js +0 -151
- package/dist/core/hearth/config.d.ts +0 -3
- package/dist/core/hearth/config.js +0 -108
- package/dist/core/hearth/guest-env.d.ts +0 -9
- package/dist/core/hearth/guest-env.js +0 -27
- package/dist/core/hearth/index.d.ts +0 -4
- package/dist/core/hearth/index.js +0 -4
- package/dist/core/hearth/model-auth-guest.d.ts +0 -8
- package/dist/core/hearth/model-auth-guest.js +0 -430
- package/dist/core/hearth/provider.d.ts +0 -36
- package/dist/core/hearth/provider.js +0 -10
- package/dist/core/hearth/providers/__tests__/sweep-and-release.test.js +0 -362
- package/dist/core/hearth/providers/blaxel-bootstrap.d.ts +0 -12
- package/dist/core/hearth/providers/blaxel-bootstrap.js +0 -147
- package/dist/core/hearth/providers/blaxel-home.d.ts +0 -60
- package/dist/core/hearth/providers/blaxel-home.js +0 -405
- package/dist/core/hearth/providers/blaxel.d.ts +0 -36
- package/dist/core/hearth/providers/blaxel.js +0 -364
- package/dist/core/hearth/providers/types.d.ts +0 -93
- package/dist/core/hearth/types.d.ts +0 -155
- package/dist/hearth/control-plane/__tests__/error-serialization.test.js +0 -29
- package/dist/hearth/control-plane/__tests__/node-message.test.js +0 -60
- package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.js +0 -44
- package/dist/hearth/control-plane/__tests__/rate-limit-recurrence.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/rate-limit-recurrence.test.js +0 -49
- package/dist/hearth/control-plane/__tests__/relay-security.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/relay-security.test.js +0 -314
- package/dist/hearth/control-plane/__tests__/scheduler-scan-loop.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/scheduler-scan-loop.test.js +0 -133
- package/dist/hearth/control-plane/__tests__/trigger-delivery.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/trigger-delivery.test.js +0 -170
- package/dist/hearth/control-plane/__tests__/wake-roll.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/wake-roll.test.js +0 -230
- package/dist/hearth/control-plane/__tests__/webhook-ingress.test.d.ts +0 -1
- package/dist/hearth/control-plane/__tests__/webhook-ingress.test.js +0 -167
- package/dist/hearth/control-plane/config.d.ts +0 -21
- package/dist/hearth/control-plane/config.js +0 -77
- package/dist/hearth/control-plane/db.d.ts +0 -30
- package/dist/hearth/control-plane/db.js +0 -561
- package/dist/hearth/control-plane/hearth-target.d.ts +0 -23
- package/dist/hearth/control-plane/hearth-target.js +0 -68
- package/dist/hearth/control-plane/ingress/rate-limit.d.ts +0 -24
- package/dist/hearth/control-plane/ingress/rate-limit.js +0 -100
- package/dist/hearth/control-plane/ingress/route-store.d.ts +0 -31
- package/dist/hearth/control-plane/ingress/route-store.js +0 -61
- package/dist/hearth/control-plane/ingress/webhook-delivery-store.d.ts +0 -41
- package/dist/hearth/control-plane/ingress/webhook-delivery-store.js +0 -69
- package/dist/hearth/control-plane/ingress/webhook-route.d.ts +0 -55
- package/dist/hearth/control-plane/ingress/webhook-route.js +0 -285
- package/dist/hearth/control-plane/main.d.ts +0 -1
- package/dist/hearth/control-plane/main.js +0 -88
- package/dist/hearth/control-plane/node-message.d.ts +0 -31
- package/dist/hearth/control-plane/node-message.js +0 -98
- package/dist/hearth/control-plane/register.d.ts +0 -15
- package/dist/hearth/control-plane/register.js +0 -34
- package/dist/hearth/control-plane/registry.d.ts +0 -22
- package/dist/hearth/control-plane/registry.js +0 -168
- package/dist/hearth/control-plane/relay.d.ts +0 -44
- package/dist/hearth/control-plane/relay.js +0 -711
- package/dist/hearth/control-plane/scheduler/fire-store.d.ts +0 -36
- package/dist/hearth/control-plane/scheduler/fire-store.js +0 -73
- package/dist/hearth/control-plane/scheduler/recurrence.d.ts +0 -7
- package/dist/hearth/control-plane/scheduler/recurrence.js +0 -58
- package/dist/hearth/control-plane/scheduler/scan-loop.d.ts +0 -38
- package/dist/hearth/control-plane/scheduler/scan-loop.js +0 -138
- package/dist/hearth/control-plane/scheduler/schedule-store.d.ts +0 -32
- package/dist/hearth/control-plane/scheduler/schedule-store.js +0 -66
- package/dist/hearth/control-plane/secrets.d.ts +0 -31
- package/dist/hearth/control-plane/secrets.js +0 -134
- package/dist/hearth/control-plane/server.d.ts +0 -27
- package/dist/hearth/control-plane/server.js +0 -482
- package/dist/hearth/control-plane/serving.d.ts +0 -15
- package/dist/hearth/control-plane/serving.js +0 -106
- package/dist/hearth/control-plane/session.d.ts +0 -68
- package/dist/hearth/control-plane/session.js +0 -273
- package/dist/hearth/control-plane/triggers/acl.d.ts +0 -14
- package/dist/hearth/control-plane/triggers/acl.js +0 -52
- package/dist/hearth/control-plane/triggers/audit-store.d.ts +0 -38
- package/dist/hearth/control-plane/triggers/audit-store.js +0 -79
- package/dist/hearth/control-plane/triggers/deliver.d.ts +0 -43
- package/dist/hearth/control-plane/triggers/deliver.js +0 -76
- package/dist/hearth/control-plane/triggers/envelope.d.ts +0 -29
- package/dist/hearth/control-plane/triggers/envelope.js +0 -38
- package/dist/hearth/control-plane/types.d.ts +0 -86
- package/dist/hearth/control-plane/types.js +0 -1
- package/dist/hearth/control-plane/wake.d.ts +0 -86
- package/dist/hearth/control-plane/wake.js +0 -550
- package/dist/web-client/assets/index-DwO46Cs5.css +0 -2
- /package/dist/{core/__tests__/hearth-bootstrap.test.d.ts → commands/__tests__/revive-now-gate.test.d.ts} +0 -0
- /package/dist/{core/hearth/__tests__/model-auth-guest.test.d.ts → commands/sys/__tests__/setup-core.test.d.ts} +0 -0
- /package/dist/core/{hearth/providers/__tests__/sweep-and-release.test.d.ts → __tests__/fault-classifier.test.d.ts} +0 -0
- /package/dist/core/{hearth/providers/types.js → __tests__/host-teardown-process-group.test.d.ts} +0 -0
- /package/dist/core/{hearth/types.js → __tests__/review-model-floor.test.d.ts} +0 -0
- /package/dist/{hearth/control-plane/__tests__/error-serialization.test.d.ts → core/__tests__/session-cycles.test.d.ts} +0 -0
- /package/dist/{hearth/control-plane/__tests__/node-message.test.d.ts → core/__tests__/worktree.test.d.ts} +0 -0
|
@@ -1,550 +0,0 @@
|
|
|
1
|
-
// control-plane/wake.ts — CP wake orchestration (P4.2 / part-plan T3.1).
|
|
2
|
-
//
|
|
3
|
-
// Generalizes M0 `wake-proxy/home.ts` (`ensureAwake`/`resolveRunningHome`/
|
|
4
|
-
// `probeRelayServing`) onto the CP `home` registry, with one deliberate
|
|
5
|
-
// behavior drop: NO idle-pause machinery (S3 — the provider owns
|
|
6
|
-
// idle->suspend; M0's `scheduleIdlePause`/`pauseNow` timer chain is not
|
|
7
|
-
// reproduced here, by design, not oversight).
|
|
8
|
-
//
|
|
9
|
-
// Pipeline (R-WAKE-1): read the `home` row -> resolve the relay token handle
|
|
10
|
-
// -> call the provider wake/resume seam (`HomeBackend.wake`, owned by the
|
|
11
|
-
// home/provider slice, IP-1) -> ready-probe the in-guest web server on
|
|
12
|
-
// `/node/<home_agent_target>` over a short retried window, carrying the
|
|
13
|
-
// relay token -> if the provider returned a new sandbox/preview (a
|
|
14
|
-
// recreate), persist the refreshed pointers; `home_agent_target` is NEVER
|
|
15
|
-
// part of that patch (R-WAKE-6 — `refreshProviderPointers` doesn't accept it)
|
|
16
|
-
// -> record wake/ready timing -> return the route. Readiness here means the
|
|
17
|
-
// guest web server answers `/node/<target>` AND relays to a live broker
|
|
18
|
-
// `view.sock` (R-WAKE-2 — not "the SPA shell renders"). `src/clients/web/
|
|
19
|
-
// server.ts` only proves that second half on a WS upgrade: `bridgeConnection`
|
|
20
|
-
// opens `view.sock` AFTER the 101 response and closes the socket with 1011
|
|
21
|
-
// the instant that fails (ECONNREFUSED/ENOENT — "no running broker"), so an
|
|
22
|
-
// HTTP GET 200 on `/node/<target>` (the SPA static/fallback route) proves
|
|
23
|
-
// only that the guest web server is up, not that the broker is. The probe
|
|
24
|
-
// therefore opens a real WebSocket to `/node/<target>` and requires a
|
|
25
|
-
// successful upgrade that SURVIVES a short settle window (or yields a first
|
|
26
|
-
// frame) before declaring readiness.
|
|
27
|
-
//
|
|
28
|
-
// All timers used during a probe retry window are cleared before the
|
|
29
|
-
// function that started them returns — there is no `setTimeout`/`setInterval`
|
|
30
|
-
// left armed once `ensureAwake`/`resolveRunning` resolve (S3 regression bar).
|
|
31
|
-
import { setTimeout as delay } from 'node:timers/promises';
|
|
32
|
-
import { WebSocket } from 'ws';
|
|
33
|
-
import { general } from '../../core/errors.js';
|
|
34
|
-
import { hearthHomeImageRefForVersion, loadM0Config } from '../../core/hearth/config.js';
|
|
35
|
-
import { getHomeBackend } from '../../core/hearth/provider.js';
|
|
36
|
-
import { loadControlPlaneConfig } from './config.js';
|
|
37
|
-
import { getHearthTarget, recordBadTarget } from './hearth-target.js';
|
|
38
|
-
import { getHomeById, refreshProviderPointers, setHealthVerdict, setHomeStatus, touchReady, touchWake } from './registry.js';
|
|
39
|
-
import { isHomeServing as isHomeServingNow } from './serving.js';
|
|
40
|
-
import { resolveSecret } from './secrets.js';
|
|
41
|
-
// Individual probe attempt timeout — bounded short so a dead route fails
|
|
42
|
-
// fast and the retry loop gets multiple attempts inside the overall window,
|
|
43
|
-
// rather than one attempt eating the whole window (mirrors M0's
|
|
44
|
-
// `READY_PROBE_TIMEOUT_MS`).
|
|
45
|
-
const READY_PROBE_ATTEMPT_TIMEOUT_MS = 1_500;
|
|
46
|
-
// Backoff between probe attempts (Q1: "500ms backoff between attempts").
|
|
47
|
-
const READY_PROBE_BACKOFF_MS = 500;
|
|
48
|
-
// How long a confirmed-ready verdict is trusted before `resolveRunning`/
|
|
49
|
-
// `ensureAwake` re-checks instead of trusting a stale answer (Q1: "10s TTL,
|
|
50
|
-
// in-process"). Single CP process, no distributed cache (S2).
|
|
51
|
-
const WARM_VERDICT_TTL_MS = 10_000;
|
|
52
|
-
/** Normalize a provider-observed runtime status into the CP `HomeStatus`
|
|
53
|
-
* vocabulary before it is ever persisted (Q3 / P3.4 review Minor finding):
|
|
54
|
-
* the provider's `MachineState` is broad (and may include future
|
|
55
|
-
* pause/suspend/standby spellings the SDK doesn't expose yet); CP's
|
|
56
|
-
* `HomeStatus` enum does not have a `'paused'` member. Anything
|
|
57
|
-
* pause/suspend/standby-shaped collapses to `'suspended'`; an exact
|
|
58
|
-
* `'running'` passes through; everything else (provisioning, terminating,
|
|
59
|
-
* terminated, restoring, unrecognized) becomes `'unknown'` rather than
|
|
60
|
-
* guessing a more specific CP status from a provider-specific string. */
|
|
61
|
-
export function normalizeProviderStatus(providerStatus) {
|
|
62
|
-
if (providerStatus === null || providerStatus === undefined)
|
|
63
|
-
return 'unknown';
|
|
64
|
-
const normalized = providerStatus.trim().toLowerCase();
|
|
65
|
-
if (normalized === 'paused' || normalized === 'suspended' || normalized === 'standby')
|
|
66
|
-
return 'suspended';
|
|
67
|
-
if (normalized === 'running')
|
|
68
|
-
return 'running';
|
|
69
|
-
return 'unknown';
|
|
70
|
-
}
|
|
71
|
-
function requireHomeField(value, field, home_id) {
|
|
72
|
-
if (value === null || value === undefined || (typeof value === 'string' && value.trim() === '')) {
|
|
73
|
-
throw general(`home ${home_id} is missing ${field} — cannot wake`, { home_id, field });
|
|
74
|
-
}
|
|
75
|
-
return value;
|
|
76
|
-
}
|
|
77
|
-
function buildDescriptor(home, relayToken) {
|
|
78
|
-
return {
|
|
79
|
-
homeId: home.home_id,
|
|
80
|
-
tenantId: home.tenant_id,
|
|
81
|
-
userId: home.user_id,
|
|
82
|
-
providerName: home.provider,
|
|
83
|
-
providerSandboxId: requireHomeField(home.provider_sandbox_id, 'provider_sandbox_id', home.home_id),
|
|
84
|
-
providerVolumeId: requireHomeField(home.provider_volume_id, 'provider_volume_id', home.home_id),
|
|
85
|
-
previewEndpoint: home.preview_endpoint,
|
|
86
|
-
homeAgentTarget: requireHomeField(home.home_agent_target, 'home_agent_target', home.home_id),
|
|
87
|
-
relayAuthRef: home.relay_auth_ref,
|
|
88
|
-
relayToken,
|
|
89
|
-
status: home.status,
|
|
90
|
-
templateVersion: home.template_version,
|
|
91
|
-
};
|
|
92
|
-
}
|
|
93
|
-
// After a successful 101 upgrade, `bridgeConnection` (server.ts) still has to
|
|
94
|
-
// dial the broker's `view.sock` and can close the socket (1011, "no running
|
|
95
|
-
// broker") the instant that dial fails. That happens fast on a local unix
|
|
96
|
-
// socket (ECONNREFUSED/ENOENT are immediate), so a settle window this short
|
|
97
|
-
// is enough to let a dead-broker close win the race against a false-positive
|
|
98
|
-
// `open` before the probe declares readiness.
|
|
99
|
-
const READY_PROBE_WS_SETTLE_MS = 300;
|
|
100
|
-
function wsUrlForNode(previewEndpoint, target) {
|
|
101
|
-
const url = new URL(`/node/${encodeURIComponent(target)}`, previewEndpoint);
|
|
102
|
-
url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
|
103
|
-
return url;
|
|
104
|
-
}
|
|
105
|
-
/** One bounded attempt: open a WebSocket to `/node/<target>` carrying the
|
|
106
|
-
* relay token and require the upgrade to both succeed AND survive a short
|
|
107
|
-
* settle window (or yield a first frame) within `attemptTimeoutMs` — proof
|
|
108
|
-
* the in-guest web server answered AND relayed to a live broker `view.sock`
|
|
109
|
-
* (R-WAKE-2), not just that the SPA shell route is up. Every branch
|
|
110
|
-
* (success, handshake rejection, error, premature close, attempt timeout)
|
|
111
|
-
* closes the socket and resolves exactly once — a success closes gracefully
|
|
112
|
-
* (code 1000) so the guest relay does not record a false fault, anything
|
|
113
|
-
* else terminates it. */
|
|
114
|
-
async function probeNodeOnce(previewEndpoint, target, relayToken, attemptTimeoutMs) {
|
|
115
|
-
return new Promise((resolveProbe) => {
|
|
116
|
-
let settled = false;
|
|
117
|
-
let settleTimer = null;
|
|
118
|
-
let ws;
|
|
119
|
-
try {
|
|
120
|
-
ws = new WebSocket(wsUrlForNode(previewEndpoint, target), {
|
|
121
|
-
headers: { authorization: `Bearer ${relayToken}` },
|
|
122
|
-
perMessageDeflate: false,
|
|
123
|
-
});
|
|
124
|
-
}
|
|
125
|
-
catch {
|
|
126
|
-
resolveProbe(false);
|
|
127
|
-
return;
|
|
128
|
-
}
|
|
129
|
-
const finish = (ready) => {
|
|
130
|
-
if (settled)
|
|
131
|
-
return;
|
|
132
|
-
settled = true;
|
|
133
|
-
if (settleTimer !== null)
|
|
134
|
-
clearTimeout(settleTimer);
|
|
135
|
-
clearTimeout(attemptTimer);
|
|
136
|
-
try {
|
|
137
|
-
// A confirmed-ready probe closes gracefully (code 1000) so the guest
|
|
138
|
-
// relay's `bridgeConnection` sees a normal browser close and does NOT
|
|
139
|
-
// record a false browser<->relay fault (P4.2 re-review Major). `close()`
|
|
140
|
-
// sends the close frame synchronously off the OPEN socket, so resolving
|
|
141
|
-
// right after calling it still delivers a normal close to the guest —
|
|
142
|
-
// the probe does not need to wait for the closing handshake to finish.
|
|
143
|
-
// Failure/timeout/error paths still abort hard via `terminate()`: there
|
|
144
|
-
// is nothing graceful to negotiate when the socket never proved ready.
|
|
145
|
-
if (ready) {
|
|
146
|
-
ws.close(1000, 'readiness probe complete');
|
|
147
|
-
}
|
|
148
|
-
else {
|
|
149
|
-
ws.terminate();
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
catch {
|
|
153
|
-
/* ignore */
|
|
154
|
-
}
|
|
155
|
-
resolveProbe(ready);
|
|
156
|
-
};
|
|
157
|
-
const attemptTimer = setTimeout(() => finish(false), attemptTimeoutMs);
|
|
158
|
-
ws.once('open', () => {
|
|
159
|
-
// The attempt timer (above) still bounds the total wait, so it is safe
|
|
160
|
-
// to always arm the full settle window here — whichever timer fires
|
|
161
|
-
// first wins via `finish`'s once-only guard.
|
|
162
|
-
settleTimer = setTimeout(() => finish(true), Math.min(READY_PROBE_WS_SETTLE_MS, attemptTimeoutMs));
|
|
163
|
-
});
|
|
164
|
-
ws.once('message', () => finish(true));
|
|
165
|
-
ws.once('unexpected-response', () => finish(false));
|
|
166
|
-
ws.once('error', () => finish(false));
|
|
167
|
-
ws.once('close', () => finish(false));
|
|
168
|
-
});
|
|
169
|
-
}
|
|
170
|
-
/** Retry the single-shot probe until it confirms `/node/<target>` is serving
|
|
171
|
-
* or `windowMs` elapses. Always runs at least one attempt (R-WAKE-1: "a
|
|
172
|
-
* single failed probe does not abort"). Every timer this loop starts
|
|
173
|
-
* (the per-attempt abort timeout, the inter-attempt backoff) is cleared/
|
|
174
|
-
* resolved before the loop returns — nothing is left armed. */
|
|
175
|
-
async function probeNodeReady(previewEndpoint, target, relayToken, windowMs) {
|
|
176
|
-
const deadline = Date.now() + Math.max(0, windowMs);
|
|
177
|
-
let firstAttempt = true;
|
|
178
|
-
for (;;) {
|
|
179
|
-
const remaining = deadline - Date.now();
|
|
180
|
-
// Past the first attempt, an already-expired budget ends the loop here
|
|
181
|
-
// instead of spending one more full attempt timeout (P4.2 re-review
|
|
182
|
-
// Minor) — only the very first attempt is unconditional (R-WAKE-1: "a
|
|
183
|
-
// single failed probe does not abort"/at-least-one-attempt).
|
|
184
|
-
if (!firstAttempt && remaining <= 0)
|
|
185
|
-
return false;
|
|
186
|
-
const attemptTimeoutMs = firstAttempt
|
|
187
|
-
? Math.max(250, Math.min(READY_PROBE_ATTEMPT_TIMEOUT_MS, windowMs))
|
|
188
|
-
: Math.min(READY_PROBE_ATTEMPT_TIMEOUT_MS, remaining);
|
|
189
|
-
firstAttempt = false;
|
|
190
|
-
if (await wakeInternals.probeNodeOnce(previewEndpoint, target, relayToken, attemptTimeoutMs))
|
|
191
|
-
return true;
|
|
192
|
-
const remainingAfter = deadline - Date.now();
|
|
193
|
-
if (remainingAfter <= 0)
|
|
194
|
-
return false;
|
|
195
|
-
await delay(Math.min(READY_PROBE_BACKOFF_MS, remainingAfter));
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
/** Indirection seam so a smoke/test script can swap any step (the provider
|
|
199
|
-
* backend, the registry setters, the secret resolver, the probe itself, the
|
|
200
|
-
* clock) without touching a real DB/provider/filesystem. Production code
|
|
201
|
-
* never reassigns these; only ad hoc verification scripts do. */
|
|
202
|
-
export const wakeInternals = {
|
|
203
|
-
getHomeById,
|
|
204
|
-
getHearthTarget,
|
|
205
|
-
recordBadTarget,
|
|
206
|
-
refreshProviderPointers,
|
|
207
|
-
setHomeStatus,
|
|
208
|
-
setHealthVerdict,
|
|
209
|
-
touchWake,
|
|
210
|
-
touchReady,
|
|
211
|
-
resolveSecret,
|
|
212
|
-
getHomeBackend: (config) => getHomeBackend(config),
|
|
213
|
-
loadM0Config: () => loadM0Config(),
|
|
214
|
-
loadControlPlaneConfig: () => loadControlPlaneConfig(),
|
|
215
|
-
probeNodeOnce,
|
|
216
|
-
probeNodeReady,
|
|
217
|
-
now: () => Date.now(),
|
|
218
|
-
};
|
|
219
|
-
const warmVerdicts = new Map();
|
|
220
|
-
const inflightWakes = new Map();
|
|
221
|
-
/** Cheap, non-probing fast path: return a still-fresh warm-verdict cache hit
|
|
222
|
-
* for `home_id`, or `null` if there is none (expired or never confirmed).
|
|
223
|
-
* Never reads the registry and never calls the provider — callers fall back
|
|
224
|
-
* to `ensureAwake` on a `null` (Q1: "no call to provider"). */
|
|
225
|
-
export function resolveRunning(home_id) {
|
|
226
|
-
const warm = warmVerdicts.get(home_id);
|
|
227
|
-
if (warm === undefined)
|
|
228
|
-
return null;
|
|
229
|
-
if (wakeInternals.now() - warm.confirmedAt >= WARM_VERDICT_TTL_MS)
|
|
230
|
-
return null;
|
|
231
|
-
return warm.result;
|
|
232
|
-
}
|
|
233
|
-
/** Drop the cached warm verdict for a home (e.g. an upstream relay forward
|
|
234
|
-
* failed) so the next `ensureAwake`/`resolveRunning` re-checks instead of
|
|
235
|
-
* trusting a stale "ready" answer. */
|
|
236
|
-
export function invalidateWarmVerdict(home_id) {
|
|
237
|
-
warmVerdicts.delete(home_id);
|
|
238
|
-
}
|
|
239
|
-
/** Clear all warm verdicts and in-flight wake coalescing. Test/smoke-script
|
|
240
|
-
* hygiene only — production never needs this (one CP process, S2). */
|
|
241
|
-
export function resetWakeCacheForTests() {
|
|
242
|
-
warmVerdicts.clear();
|
|
243
|
-
inflightWakes.clear();
|
|
244
|
-
}
|
|
245
|
-
/** Is this home currently serving, so a destructive roll would interrupt it?
|
|
246
|
-
* Delegates entirely to `serving.ts`'s independent live-session counters,
|
|
247
|
-
* bumped at the relay's HTTP/WS lifecycle seams and the CP's OAuth
|
|
248
|
-
* start/finish dispatch — never derived from any TTL/cache. `home.status` is
|
|
249
|
-
* intentionally NOT consulted: a suspended home has no live counters anyway
|
|
250
|
-
* (nothing to bump them), so this stays a single source of truth. */
|
|
251
|
-
function homeIsServing(home) {
|
|
252
|
-
return isHomeServingNow(home.home_id);
|
|
253
|
-
}
|
|
254
|
-
/** Is this home due to roll onto the registered target? Stale = it isn't
|
|
255
|
-
* already on the target version AND the target is not a known-bad release.
|
|
256
|
-
* The bad-target clause is the ONLY brake in the zero-touch loop (roadmap
|
|
257
|
-
* decision 1: never auto-advance onto a release that already failed a roll):
|
|
258
|
-
* once `recordBadTarget` poisons the current target, no home rolls onto it
|
|
259
|
-
* until a NEW release advances the target past it.
|
|
260
|
-
*
|
|
261
|
-
* NOTE (design divergence, flagged): design §3's literal snippet compared
|
|
262
|
-
* `home.template_version !== last_bad_target_version`, which would THRASH —
|
|
263
|
-
* after a failed roll a home sits on the PRIOR version, so that clause stays
|
|
264
|
-
* true and it would re-roll onto the same broken target forever. The roadmap's
|
|
265
|
-
* plain-English brake and design §5's own comment both mean `target !==
|
|
266
|
-
* last_bad`; that is what is implemented here. */
|
|
267
|
-
function isHomeStale(home, target) {
|
|
268
|
-
if (home.template_version === null)
|
|
269
|
-
return false;
|
|
270
|
-
if (home.template_version === target.target_template_version)
|
|
271
|
-
return false;
|
|
272
|
-
if (target.last_bad_target_version !== null && target.target_template_version === target.last_bad_target_version)
|
|
273
|
-
return false;
|
|
274
|
-
return true;
|
|
275
|
-
}
|
|
276
|
-
// Baked crtr path (deploy/hearth-blaxel/Dockerfile symlink) — target it
|
|
277
|
-
// directly for the in-guest version assert rather than a bare PATH lookup,
|
|
278
|
-
// exactly as blaxel-bootstrap.ts does, so a stale legacy binary earlier on
|
|
279
|
-
// PATH can't answer instead.
|
|
280
|
-
const BAKED_CRTR_PATH = '/usr/local/bin/crtr';
|
|
281
|
-
/** Assert what crtr the recreated guest is ACTUALLY running (design §4 step 7)
|
|
282
|
-
* by reading `crtr --json sys version` in-guest — stronger than trusting the
|
|
283
|
-
* provider's config version, since it proves the booted binary. Parses the
|
|
284
|
-
* `.version` field from `{ "version": "X.Y.Z" }`. The same drift-guard env
|
|
285
|
-
* vars the bootstrap uses keep this a pure read (no bootstrap/auto-init/
|
|
286
|
-
* auto-update side effects). */
|
|
287
|
-
async function assertGuestVersion(backend, sandboxName) {
|
|
288
|
-
const result = await backend.exec(sandboxName, {
|
|
289
|
-
command: `CRTR_NO_BOOTSTRAP=1 CRTR_NO_AUTO_INIT=1 CRTR_NO_AUTO_UPDATE=1 ${BAKED_CRTR_PATH} --json sys version`,
|
|
290
|
-
timeoutMs: 20_000,
|
|
291
|
-
});
|
|
292
|
-
// A parseable stdout is not proof the command itself succeeded — require a
|
|
293
|
-
// clean exit before trusting anything it printed.
|
|
294
|
-
if (result.exitCode !== 0 || result.timedOut === true) {
|
|
295
|
-
throw general(`in-guest crtr sys version did not exit cleanly (exitCode ${result.exitCode}, timedOut ${result.timedOut})`, {
|
|
296
|
-
sandboxName,
|
|
297
|
-
exitCode: result.exitCode,
|
|
298
|
-
timedOut: result.timedOut,
|
|
299
|
-
stdout: result.stdout.slice(0, 200),
|
|
300
|
-
stderr: result.stderr.slice(0, 200),
|
|
301
|
-
});
|
|
302
|
-
}
|
|
303
|
-
const stdout = result.stdout.trim();
|
|
304
|
-
try {
|
|
305
|
-
const parsed = JSON.parse(stdout);
|
|
306
|
-
if (typeof parsed.version === 'string' && parsed.version.trim() !== '')
|
|
307
|
-
return parsed.version.trim();
|
|
308
|
-
}
|
|
309
|
-
catch {
|
|
310
|
-
/* fall through to a tolerant field extraction below */
|
|
311
|
-
}
|
|
312
|
-
const match = /"version"\s*:\s*"([^"]+)"/.exec(stdout);
|
|
313
|
-
if (match !== null && match[1].trim() !== '')
|
|
314
|
-
return match[1].trim();
|
|
315
|
-
throw general(`in-guest crtr sys version returned no parseable .version (exit ${result.exitCode})`, {
|
|
316
|
-
sandboxName,
|
|
317
|
-
stdout: stdout.slice(0, 200),
|
|
318
|
-
});
|
|
319
|
-
}
|
|
320
|
-
/** Structured HIGH-SEVERITY alert lines. The CP is a standalone Fly process
|
|
321
|
-
* and cannot `crtr push`; alerting is a distinct `level: 'error'` stderr line
|
|
322
|
-
* with an `alert` marker that ops log-alerting keys on. (Wiring these to a
|
|
323
|
-
* real notification channel is future work — the marker is the seam.) */
|
|
324
|
-
function alertRollRolledBack(home, target, restoredVersion) {
|
|
325
|
-
process.stderr.write(`${JSON.stringify({
|
|
326
|
-
level: 'error',
|
|
327
|
-
alert: 'hearth.roll.rolled_back',
|
|
328
|
-
message: `Hearth auto-upgrade roll FAILED and was rolled back — release ${target.target_template_version} is broken`,
|
|
329
|
-
home_id: home.home_id,
|
|
330
|
-
bad_target_version: target.target_template_version,
|
|
331
|
-
restored_version: restoredVersion,
|
|
332
|
-
})}\n`);
|
|
333
|
-
}
|
|
334
|
-
export function serializeError(error) {
|
|
335
|
-
if (error instanceof Error)
|
|
336
|
-
return error.message;
|
|
337
|
-
try {
|
|
338
|
-
// `JSON.stringify` returns `undefined` (does not throw) for `undefined`, a function, or a
|
|
339
|
-
// symbol — falling through to that would OMIT the `error` field from the incident alert's
|
|
340
|
-
// JSON line entirely, which is worse than the "[object Object]" bug this rider fixes.
|
|
341
|
-
const json = JSON.stringify(error);
|
|
342
|
-
return json === undefined ? String(error) : json;
|
|
343
|
-
}
|
|
344
|
-
catch {
|
|
345
|
-
return String(error);
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
function alertRollIncident(home, target, priorVersion, error) {
|
|
349
|
-
process.stderr.write(`${JSON.stringify({
|
|
350
|
-
level: 'error',
|
|
351
|
-
alert: 'hearth.roll.incident',
|
|
352
|
-
message: `Hearth auto-upgrade roll AND rollback both failed — home ${home.home_id} is DEGRADED, manual recovery needed`,
|
|
353
|
-
home_id: home.home_id,
|
|
354
|
-
bad_target_version: target.target_template_version,
|
|
355
|
-
prior_version: priorVersion,
|
|
356
|
-
error: serializeError(error),
|
|
357
|
-
})}\n`);
|
|
358
|
-
}
|
|
359
|
-
/** Fall back onto the PRIOR pinned image after a failed roll (design §5). The
|
|
360
|
-
* volume was never touched by the roll, so this is deterministic: recreate on
|
|
361
|
-
* the prior image name (still in the registry — images are never pruned),
|
|
362
|
-
* revive, ready-probe, assert `crtr sys version === priorVersion`. On success,
|
|
363
|
-
* ALERT (the release is broken) — the bad target is already poisoned by
|
|
364
|
-
* `rollHome`'s catch (the roll's own failure is what poisons the target, not
|
|
365
|
-
* rollback's success, so the brake is set even when rollback ALSO fails).
|
|
366
|
-
* On rollback also failing: mark the home
|
|
367
|
-
* `unknown`/`degraded` and alert — a genuine incident (the volume is intact,
|
|
368
|
-
* so manual recovery is always possible). */
|
|
369
|
-
async function rollbackHome(args) {
|
|
370
|
-
const { home, target, descriptor, priorImageRef, priorVersion, relayToken, backend, readyTimeoutMs } = args;
|
|
371
|
-
try {
|
|
372
|
-
const refresh = await backend.recreateOnImage(descriptor, priorImageRef, priorVersion);
|
|
373
|
-
const previewEndpoint = requireHomeField(refresh.previewEndpoint ?? null, 'preview_endpoint', home.home_id);
|
|
374
|
-
const ready = await wakeInternals.probeNodeReady(previewEndpoint, descriptor.homeAgentTarget, relayToken, readyTimeoutMs);
|
|
375
|
-
if (!ready)
|
|
376
|
-
throw general(`rollback ready-probe timed out for ${home.home_id}`, { home_id: home.home_id });
|
|
377
|
-
// Assert against the sandbox the recreate actually reports, not a stale name captured at
|
|
378
|
-
// entry — under generation naming (recreate-race-fix.md §3.5) the pre-recreate sandbox is
|
|
379
|
-
// destroyed, so exec-ing into anything else would target a gone/wrong generation.
|
|
380
|
-
const asserted = await assertGuestVersion(backend, refresh.providerSandboxId);
|
|
381
|
-
if (asserted !== priorVersion) {
|
|
382
|
-
throw general(`rollback version mismatch: expected ${priorVersion}, got ${asserted}`, { home_id: home.home_id, expected: priorVersion, asserted });
|
|
383
|
-
}
|
|
384
|
-
wakeInternals.refreshProviderPointers(home.home_id, {
|
|
385
|
-
provider_sandbox_id: refresh.providerSandboxId,
|
|
386
|
-
preview_endpoint: previewEndpoint,
|
|
387
|
-
template_version: priorVersion,
|
|
388
|
-
status: 'running',
|
|
389
|
-
});
|
|
390
|
-
wakeInternals.setHealthVerdict(home.home_id, 'healthy');
|
|
391
|
-
alertRollRolledBack(home, target, priorVersion);
|
|
392
|
-
return { preview_endpoint: previewEndpoint, relayToken };
|
|
393
|
-
}
|
|
394
|
-
catch (rollbackError) {
|
|
395
|
-
wakeInternals.setHomeStatus(home.home_id, 'unknown');
|
|
396
|
-
wakeInternals.setHealthVerdict(home.home_id, 'degraded');
|
|
397
|
-
alertRollIncident(home, target, priorVersion, rollbackError);
|
|
398
|
-
throw general(`hearth roll AND rollback both failed for ${home.home_id}`, {
|
|
399
|
-
home_id: home.home_id,
|
|
400
|
-
target: target.target_template_version,
|
|
401
|
-
priorVersion,
|
|
402
|
-
});
|
|
403
|
-
}
|
|
404
|
-
}
|
|
405
|
-
/** CP-driven destroy→recreate of a stale home onto the registered target
|
|
406
|
-
* image, reattaching the same durable volume (design §4). Captures rollback
|
|
407
|
-
* anchors first, marks the home `restoring` (so a concurrent wake coalesces
|
|
408
|
-
* rather than races), then recreates on the target image, ready-probes, and
|
|
409
|
-
* ASSERTS the in-guest `crtr sys version` equals the target before committing
|
|
410
|
-
* pointers + `health_verdict = healthy`. A roll never writes
|
|
411
|
-
* `home_agent_target` (canvas.db is reattached byte-identical;
|
|
412
|
-
* `refreshProviderPointers` has no such field). Any of recreate/probe/assert
|
|
413
|
-
* failing falls back to the prior image (§5). Returns the route so the
|
|
414
|
-
* triggering wake serves the just-upgraded home in the same call. */
|
|
415
|
-
async function rollHome(home, target, relayToken, descriptor) {
|
|
416
|
-
const priorVersion = requireHomeField(home.template_version, 'template_version', home.home_id);
|
|
417
|
-
const priorImageRef = hearthHomeImageRefForVersion(priorVersion);
|
|
418
|
-
const m0 = wakeInternals.loadM0Config();
|
|
419
|
-
const backend = wakeInternals.getHomeBackend(m0);
|
|
420
|
-
const cpConfig = wakeInternals.loadControlPlaneConfig();
|
|
421
|
-
wakeInternals.setHomeStatus(home.home_id, 'restoring');
|
|
422
|
-
invalidateWarmVerdict(home.home_id);
|
|
423
|
-
const startedAt = wakeInternals.now();
|
|
424
|
-
try {
|
|
425
|
-
const refresh = await backend.recreateOnImage(descriptor, target.target_image_ref, target.target_template_version);
|
|
426
|
-
const previewEndpoint = requireHomeField(refresh.previewEndpoint ?? null, 'preview_endpoint', home.home_id);
|
|
427
|
-
const ready = await wakeInternals.probeNodeReady(previewEndpoint, descriptor.homeAgentTarget, relayToken, cpConfig.readyTimeoutMs);
|
|
428
|
-
if (!ready)
|
|
429
|
-
throw general(`roll ready-probe timed out for ${home.home_id}`, { home_id: home.home_id });
|
|
430
|
-
// Assert against the sandbox the recreate actually reports (recreate-race-fix.md §3.5) —
|
|
431
|
-
// never the row's pre-recreate `provider_sandbox_id`, which under generation naming is
|
|
432
|
-
// already destroyed by the time this runs.
|
|
433
|
-
const asserted = await assertGuestVersion(backend, refresh.providerSandboxId);
|
|
434
|
-
if (asserted !== target.target_template_version) {
|
|
435
|
-
throw general(`roll version mismatch: expected ${target.target_template_version}, got ${asserted}`, {
|
|
436
|
-
home_id: home.home_id,
|
|
437
|
-
expected: target.target_template_version,
|
|
438
|
-
asserted,
|
|
439
|
-
});
|
|
440
|
-
}
|
|
441
|
-
wakeInternals.refreshProviderPointers(home.home_id, {
|
|
442
|
-
provider_sandbox_id: refresh.providerSandboxId,
|
|
443
|
-
preview_endpoint: previewEndpoint,
|
|
444
|
-
template_version: asserted,
|
|
445
|
-
status: 'running',
|
|
446
|
-
});
|
|
447
|
-
wakeInternals.setHealthVerdict(home.home_id, 'healthy');
|
|
448
|
-
wakeInternals.touchWake(home.home_id, { at: new Date(startedAt).toISOString(), latency_ms: wakeInternals.now() - startedAt });
|
|
449
|
-
wakeInternals.touchReady(home.home_id);
|
|
450
|
-
return { preview_endpoint: previewEndpoint, relayToken };
|
|
451
|
-
}
|
|
452
|
-
catch {
|
|
453
|
-
// The target is known-bad the INSTANT its own roll fails, independent of
|
|
454
|
-
// whether rollback then recovers — poison it here, before delegating to
|
|
455
|
-
// rollbackHome, so a roll-fails+rollback-also-fails home doesn't leave the
|
|
456
|
-
// brake unset (which would let a later wake, of this home or any other,
|
|
457
|
-
// re-roll onto the same broken target).
|
|
458
|
-
wakeInternals.recordBadTarget(target.target_template_version);
|
|
459
|
-
return rollbackHome({ home, target, descriptor, priorImageRef, priorVersion, relayToken, backend, readyTimeoutMs: cpConfig.readyTimeoutMs });
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
async function runEnsureAwake(home_id) {
|
|
463
|
-
const home = wakeInternals.getHomeById(home_id);
|
|
464
|
-
if (home === null)
|
|
465
|
-
throw general(`home not found: ${home_id}`, { home_id });
|
|
466
|
-
const relayAuthRef = requireHomeField(home.relay_auth_ref, 'relay_auth_ref', home_id);
|
|
467
|
-
const relayToken = wakeInternals.resolveSecret(relayAuthRef);
|
|
468
|
-
const target = requireHomeField(home.home_agent_target, 'home_agent_target', home_id);
|
|
469
|
-
const descriptor = buildDescriptor(home, relayToken);
|
|
470
|
-
// Auto-upgrade lazy roll (design §3/§4): if a newer image target is
|
|
471
|
-
// registered, this home is stale, and it is NOT currently serving, roll it
|
|
472
|
-
// onto the target image now instead of the normal wake. At wake-entry an
|
|
473
|
-
// idle home is definitionally safe to tear down; a serving home doesn't roll
|
|
474
|
-
// this pass (it rolls on its next cold wake). This is the ONLY new branch —
|
|
475
|
-
// the normal wake path below is unchanged.
|
|
476
|
-
const hearthTarget = wakeInternals.getHearthTarget();
|
|
477
|
-
if (hearthTarget !== null && isHomeStale(home, hearthTarget) && !homeIsServing(home)) {
|
|
478
|
-
return rollHome(home, hearthTarget, relayToken, descriptor);
|
|
479
|
-
}
|
|
480
|
-
const startedAt = wakeInternals.now();
|
|
481
|
-
const m0 = wakeInternals.loadM0Config();
|
|
482
|
-
const refresh = await wakeInternals.getHomeBackend(m0).wake(descriptor);
|
|
483
|
-
const normalizedStatus = normalizeProviderStatus(refresh.status);
|
|
484
|
-
const recreated = refresh.providerSandboxId !== home.provider_sandbox_id || (refresh.previewEndpoint ?? null) !== home.preview_endpoint;
|
|
485
|
-
let previewEndpoint;
|
|
486
|
-
if (recreated) {
|
|
487
|
-
// R-WAKE-6, non-negotiable: persist the new sandbox/preview pointers but
|
|
488
|
-
// never the home node target — `refreshProviderPointers`'s input shape
|
|
489
|
-
// has no `home_agent_target` field, so there is no way for this call to
|
|
490
|
-
// touch it even by accident.
|
|
491
|
-
const updated = wakeInternals.refreshProviderPointers(home_id, {
|
|
492
|
-
provider_sandbox_id: refresh.providerSandboxId,
|
|
493
|
-
preview_endpoint: refresh.previewEndpoint ?? null,
|
|
494
|
-
status: normalizedStatus,
|
|
495
|
-
template_version: refresh.templateVersion ?? home.template_version,
|
|
496
|
-
});
|
|
497
|
-
previewEndpoint = updated.preview_endpoint;
|
|
498
|
-
}
|
|
499
|
-
else {
|
|
500
|
-
wakeInternals.setHomeStatus(home_id, normalizedStatus);
|
|
501
|
-
previewEndpoint = home.preview_endpoint;
|
|
502
|
-
}
|
|
503
|
-
const resolvedPreviewEndpoint = requireHomeField(previewEndpoint, 'preview_endpoint', home_id);
|
|
504
|
-
const controlPlaneConfig = wakeInternals.loadControlPlaneConfig();
|
|
505
|
-
const ready = await wakeInternals.probeNodeReady(resolvedPreviewEndpoint, target, relayToken, controlPlaneConfig.readyTimeoutMs);
|
|
506
|
-
if (!ready) {
|
|
507
|
-
throw general(`guest web relay did not become ready before timeout: ${home_id}`, { home_id, previewEndpoint: resolvedPreviewEndpoint });
|
|
508
|
-
}
|
|
509
|
-
const readyAt = new Date().toISOString();
|
|
510
|
-
wakeInternals.touchWake(home_id, { at: new Date(startedAt).toISOString(), latency_ms: wakeInternals.now() - startedAt });
|
|
511
|
-
wakeInternals.touchReady(home_id, readyAt);
|
|
512
|
-
// A successful ready-probe means the home is healthy right now, regardless of what the
|
|
513
|
-
// verdict was before this wake (e.g. left `degraded` by an earlier failed rollback) —
|
|
514
|
-
// clear it here the same way rollHome/rollbackHome clear it on their own success, or a
|
|
515
|
-
// stale degraded/unknown verdict never heals on the normal wake path.
|
|
516
|
-
wakeInternals.setHealthVerdict(home_id, 'healthy');
|
|
517
|
-
return { preview_endpoint: resolvedPreviewEndpoint, relayToken };
|
|
518
|
-
}
|
|
519
|
-
/** Bring a home up and ready, returning the route to relay to once it is.
|
|
520
|
-
* Reads the registry, resolves the relay token, calls the provider
|
|
521
|
-
* wake/resume seam, ready-probes `/node/<home_agent_target>`, refreshes
|
|
522
|
-
* provider pointers on a recreate, records wake/ready timing, clears the
|
|
523
|
-
* health verdict to `healthy` on success, and returns
|
|
524
|
-
* `{ preview_endpoint, relayToken }`. Throws on a ready-probe timeout
|
|
525
|
-
* (wake-failure UX is deferred — gap note 2 — a throw is the accepted
|
|
526
|
-
* cutover behavior).
|
|
527
|
-
*
|
|
528
|
-
* A fresh warm-verdict cache hit short-circuits without re-entering this
|
|
529
|
-
* pipeline at all. Otherwise concurrent callers for the SAME `home_id`
|
|
530
|
-
* coalesce onto one in-flight wake (Q1) instead of each driving their own
|
|
531
|
-
* provider call. */
|
|
532
|
-
export async function ensureAwake(home_id) {
|
|
533
|
-
const warm = resolveRunning(home_id);
|
|
534
|
-
if (warm !== null)
|
|
535
|
-
return warm;
|
|
536
|
-
const inflight = inflightWakes.get(home_id);
|
|
537
|
-
if (inflight !== undefined)
|
|
538
|
-
return inflight;
|
|
539
|
-
const promise = runEnsureAwake(home_id)
|
|
540
|
-
.then((result) => {
|
|
541
|
-
warmVerdicts.set(home_id, { result, confirmedAt: wakeInternals.now() });
|
|
542
|
-
return result;
|
|
543
|
-
})
|
|
544
|
-
.finally(() => {
|
|
545
|
-
if (inflightWakes.get(home_id) === promise)
|
|
546
|
-
inflightWakes.delete(home_id);
|
|
547
|
-
});
|
|
548
|
-
inflightWakes.set(home_id, promise);
|
|
549
|
-
return promise;
|
|
550
|
-
}
|