@crouton-kit/crouter 0.3.38 → 0.3.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/hearth/provider.d.ts +8 -0
- package/dist/core/hearth/providers/__tests__/sweep-and-release.test.d.ts +1 -0
- package/dist/core/hearth/providers/__tests__/sweep-and-release.test.js +254 -0
- package/dist/core/hearth/providers/blaxel-home.d.ts +39 -0
- package/dist/core/hearth/providers/blaxel-home.js +271 -21
- package/dist/core/hearth/providers/blaxel.d.ts +5 -0
- package/dist/core/hearth/providers/blaxel.js +86 -4
- package/dist/core/hearth/providers/types.d.ts +16 -0
- package/dist/hearth/control-plane/__tests__/error-serialization.test.d.ts +1 -0
- package/dist/hearth/control-plane/__tests__/error-serialization.test.js +29 -0
- package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.d.ts +1 -0
- package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.js +44 -0
- package/dist/hearth/control-plane/__tests__/wake-roll.test.d.ts +1 -0
- package/dist/hearth/control-plane/__tests__/wake-roll.test.js +230 -0
- package/dist/hearth/control-plane/db.js +27 -0
- package/dist/hearth/control-plane/hearth-target.d.ts +23 -0
- package/dist/hearth/control-plane/hearth-target.js +68 -0
- package/dist/hearth/control-plane/registry.d.ts +6 -1
- package/dist/hearth/control-plane/registry.js +7 -0
- package/dist/hearth/control-plane/relay.d.ts +4 -0
- package/dist/hearth/control-plane/relay.js +72 -3
- package/dist/hearth/control-plane/secrets.d.ts +14 -0
- package/dist/hearth/control-plane/secrets.js +21 -0
- package/dist/hearth/control-plane/server.js +140 -4
- package/dist/hearth/control-plane/serving.d.ts +15 -0
- package/dist/hearth/control-plane/serving.js +106 -0
- package/dist/hearth/control-plane/types.d.ts +20 -0
- package/dist/hearth/control-plane/wake.d.ts +8 -2
- package/dist/hearth/control-plane/wake.js +241 -3
- package/package.json +1 -1
|
@@ -7,6 +7,7 @@ interface BlaxelMachineProviderOptions {
|
|
|
7
7
|
templateVersion?: string;
|
|
8
8
|
volumeSizeGb?: number;
|
|
9
9
|
}
|
|
10
|
+
export declare function isMissingResourceError(error: unknown): boolean;
|
|
10
11
|
export declare class BlaxelMachineProvider implements MachineProvider {
|
|
11
12
|
readonly providerName: "blaxel";
|
|
12
13
|
readonly imageRef: string;
|
|
@@ -18,6 +19,9 @@ export declare class BlaxelMachineProvider implements MachineProvider {
|
|
|
18
19
|
name: string;
|
|
19
20
|
}): Promise<VolumeRef>;
|
|
20
21
|
createMachine(input: CreateMachineInput): Promise<MachineRef>;
|
|
22
|
+
listMachines(filter?: {
|
|
23
|
+
externalId?: string;
|
|
24
|
+
}): Promise<MachineRef[]>;
|
|
21
25
|
getMachine(machineId: string): Promise<MachineRef>;
|
|
22
26
|
exec(machineId: string, input: ExecInput): Promise<ExecResult>;
|
|
23
27
|
waitForPort(machineId: string, port: number, opts?: {
|
|
@@ -27,5 +31,6 @@ export declare class BlaxelMachineProvider implements MachineProvider {
|
|
|
27
31
|
health(): Promise<HealthResult>;
|
|
28
32
|
destroy(machineId: string): Promise<DestroyOutcome>;
|
|
29
33
|
destroyVolume(volumeId: string): Promise<DestroyOutcome>;
|
|
34
|
+
getVolumeAttachment(volumeId: string): Promise<string | null>;
|
|
30
35
|
}
|
|
31
36
|
export {};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { readFileSync } from 'node:fs';
|
|
3
|
-
import { SandboxInstance, VolumeInstance } from '@blaxel/core';
|
|
3
|
+
import { SandboxInstance, VolumeInstance, getVolume, listSandboxes } from '@blaxel/core';
|
|
4
4
|
import { general } from '../../errors.js';
|
|
5
5
|
const TERMINATED_STATES = new Set(['TERMINATED', 'DELETING', 'DEACTIVATED', 'DEACTIVATING', 'FAILED']);
|
|
6
6
|
const PROVISIONING_STATES = new Set(['UPLOADING', 'BUILDING', 'DEPLOYING', 'BUILT']);
|
|
@@ -81,10 +81,28 @@ function resolveExecCommand(input) {
|
|
|
81
81
|
}
|
|
82
82
|
return argv.map(shellQuote).join(' ');
|
|
83
83
|
}
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
// The Blaxel SDK (@blaxel/core 0.2.94) does not always reject with an `Error` instance — a 404
|
|
85
|
+
// comes back as a plain object (e.g. `{code:404,error:"Sandbox not found"}`), and `String()` on
|
|
86
|
+
// such an object yields the useless literal "[object Object]", which matches none of the regex
|
|
87
|
+
// below. Recognize the structured shape directly (a numeric-or-string `404` status field) AND
|
|
88
|
+
// fall back to JSON-serializing the value (mirroring wake.ts's `serializeError`) before running
|
|
89
|
+
// the message regex, so both an `Error("... not found")` and a plain-object 404 are caught.
|
|
90
|
+
export function isMissingResourceError(error) {
|
|
91
|
+
const fields = record(error);
|
|
92
|
+
if (fields !== null) {
|
|
93
|
+
for (const key of ['code', 'status', 'status_code', 'statusCode']) {
|
|
94
|
+
const value = fields[key];
|
|
95
|
+
if (value === 404 || value === '404')
|
|
96
|
+
return true;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const message = error instanceof Error ? error.message : (JSON.stringify(error) ?? String(error));
|
|
86
100
|
return /\b(not found|404|does not exist|missing)\b/i.test(message);
|
|
87
101
|
}
|
|
102
|
+
// Accepts either a live `SandboxInstance` or a raw wire `Sandbox` page item — only the
|
|
103
|
+
// `metadata`/`status`/`spec` fields are read, so mapping raw list pages avoids constructing
|
|
104
|
+
// `SandboxInstance`s (whose `list()` eagerly warms an H2 session per sandbox — pointless for a
|
|
105
|
+
// destroy sweep).
|
|
88
106
|
function toMachineRef(sandbox) {
|
|
89
107
|
const metadata = sandbox.metadata;
|
|
90
108
|
const status = sandbox.status ?? undefined;
|
|
@@ -142,7 +160,16 @@ export class BlaxelMachineProvider {
|
|
|
142
160
|
async createMachine(input) {
|
|
143
161
|
const sandbox = await SandboxInstance.createIfNotExists({
|
|
144
162
|
name: input.name ?? input.tenantId,
|
|
145
|
-
|
|
163
|
+
// Tag every sandbox with the home's stable volume id so the recreate sweep can scope its
|
|
164
|
+
// destructive list to this home's family only (orphan-cleanup-fix MAJOR-1). Unset on
|
|
165
|
+
// callers that pass no externalId.
|
|
166
|
+
...(input.externalId === undefined ? {} : { externalId: input.externalId }),
|
|
167
|
+
// A targeted recreate (auto-upgrade roll / rollback, design §4.1) passes
|
|
168
|
+
// an explicit `image`; the normal provision/wake path leaves it unset
|
|
169
|
+
// and uses the provider's construction-time config image. `createIfNotExists`
|
|
170
|
+
// only honors this on genuine creation — the roll path destroys the old
|
|
171
|
+
// sandbox first so the name is free (blaxel-home.ts `recreateOnImage`).
|
|
172
|
+
image: input.image ?? this.config.image,
|
|
146
173
|
memory: this.config.memory,
|
|
147
174
|
region: this.config.region,
|
|
148
175
|
volumes: input.volumeMount === undefined || input.volumeMount === null ? undefined : [{ name: input.volumeMount.volumeName, mountPath: input.volumeMount.path, readOnly: false }],
|
|
@@ -158,6 +185,38 @@ export class BlaxelMachineProvider {
|
|
|
158
185
|
await sandbox.wait({ maxWait: 90_000, interval: 500 }).catch(() => sandbox);
|
|
159
186
|
return toMachineRef(sandbox);
|
|
160
187
|
}
|
|
188
|
+
// orphan-cleanup-fix.md §3.2 + MAJOR-1/MINOR-1: the deterministic name-family sweep enumerates
|
|
189
|
+
// a home's generations via LIST scoped to the caller-owned `externalId` tag, then destroys the
|
|
190
|
+
// ones whose name is in the family. Over-listing is harmless — a FAILED/detached record still
|
|
191
|
+
// shows up here and destroy tolerates not_found, which is exactly what closes the autonomous-
|
|
192
|
+
// standby-detach orphan gap. Drives the generated `listSandboxes` client directly (bypassing
|
|
193
|
+
// `SandboxInstance.list()`, which never follows a cursor and warms an H2 session per sandbox):
|
|
194
|
+
// loops until exhausted so a future SDK version crossing Blaxel-Version 2026-04-28 (cursor
|
|
195
|
+
// pagination, 50/page) cannot silently truncate the sweep to page 1. At the SDK's pinned
|
|
196
|
+
// 2026-04-16 the server returns a bare array (the full set) and the loop breaks on the first
|
|
197
|
+
// page.
|
|
198
|
+
async listMachines(filter) {
|
|
199
|
+
const machines = [];
|
|
200
|
+
let cursor;
|
|
201
|
+
for (;;) {
|
|
202
|
+
const { data: raw } = await listSandboxes({
|
|
203
|
+
query: {
|
|
204
|
+
limit: 200,
|
|
205
|
+
...(cursor === undefined ? {} : { cursor }),
|
|
206
|
+
...(filter?.externalId === undefined ? {} : { externalId: filter.externalId }),
|
|
207
|
+
},
|
|
208
|
+
throwOnError: true,
|
|
209
|
+
});
|
|
210
|
+
const page = Array.isArray(raw) ? raw : (raw?.data ?? []);
|
|
211
|
+
machines.push(...page.map(toMachineRef));
|
|
212
|
+
if (Array.isArray(raw))
|
|
213
|
+
break; // legacy bare array (Blaxel-Version < 2026-04-28) = full set
|
|
214
|
+
cursor = raw?.meta?.nextCursor || undefined; // empty when there are no more pages
|
|
215
|
+
if (cursor === undefined)
|
|
216
|
+
break;
|
|
217
|
+
}
|
|
218
|
+
return machines;
|
|
219
|
+
}
|
|
161
220
|
async getMachine(machineId) {
|
|
162
221
|
try {
|
|
163
222
|
const sandbox = await SandboxInstance.get(machineId);
|
|
@@ -279,4 +338,27 @@ export class BlaxelMachineProvider {
|
|
|
279
338
|
throw error;
|
|
280
339
|
}
|
|
281
340
|
}
|
|
341
|
+
// recreate-race-fix.md §3.1: read the platform's own positive attachment signal
|
|
342
|
+
// (`Volume.state.attachedTo`) rather than inferring detach from a sandbox GET/delete —
|
|
343
|
+
// that read/create consistency split is exactly what raced the old same-name recreate.
|
|
344
|
+
// `VolumeInstance.get`'s backing `volume` field is TS-private (no public state getter
|
|
345
|
+
// in @blaxel/core 0.2.94), so this calls the underlying `getVolume` SDK function
|
|
346
|
+
// directly instead — same request `VolumeInstance.get` makes, with `state` exposed.
|
|
347
|
+
// A 404 (or any other failure) throws, wrapped with the volumeId for context: a missing
|
|
348
|
+
// home volume is a real incident, never something to continue past (volume-is-sacred).
|
|
349
|
+
async getVolumeAttachment(volumeId) {
|
|
350
|
+
let attachedTo;
|
|
351
|
+
try {
|
|
352
|
+
const { data } = await getVolume({ path: { volumeName: volumeId }, throwOnError: true });
|
|
353
|
+
attachedTo = data.state?.attachedTo;
|
|
354
|
+
}
|
|
355
|
+
catch (error) {
|
|
356
|
+
throw general(`blaxel volume ${volumeId} attachment check failed`, {
|
|
357
|
+
volumeId,
|
|
358
|
+
cause: error instanceof Error ? error.message : String(error),
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
const trimmed = (attachedTo ?? '').trim();
|
|
362
|
+
return trimmed === '' ? null : trimmed;
|
|
363
|
+
}
|
|
282
364
|
}
|
|
@@ -22,10 +22,12 @@ export interface VolumeRef {
|
|
|
22
22
|
}
|
|
23
23
|
export interface CreateMachineInput {
|
|
24
24
|
tenantId: string;
|
|
25
|
+
externalId?: string;
|
|
25
26
|
template?: string | null;
|
|
26
27
|
envs?: Record<string, string>;
|
|
27
28
|
enableNetwork?: boolean;
|
|
28
29
|
name?: string | null;
|
|
30
|
+
image?: string | null;
|
|
29
31
|
ports?: {
|
|
30
32
|
target: number;
|
|
31
33
|
protocol: 'HTTP';
|
|
@@ -65,6 +67,16 @@ export interface MachineProvider {
|
|
|
65
67
|
}): Promise<VolumeRef>;
|
|
66
68
|
createMachine(input: CreateMachineInput): Promise<MachineRef>;
|
|
67
69
|
getMachine(machineId: MachineId): Promise<MachineRef>;
|
|
70
|
+
/** List machine records, optionally scoped to a caller-owned `externalId` tag. The recreate
|
|
71
|
+
* cleanup sweep passes the home's `providerVolumeId` as the filter so only sandboxes Hearth
|
|
72
|
+
* itself tagged for this home come back, then intersects that with the generation NAME FAMILY
|
|
73
|
+
* matcher — a tag∩name intersection on a destructive op, so no id-format or workspace name
|
|
74
|
+
* discipline is load-bearing for other homes' safety (orphan-cleanup-fix MAJOR-1). The result
|
|
75
|
+
* is NOT a promise of every machine in the workspace; without a filter it lists broadly, with
|
|
76
|
+
* one it lists only tagged matches. Implementations must page until exhausted (MINOR-1). */
|
|
77
|
+
listMachines(filter?: {
|
|
78
|
+
externalId?: string;
|
|
79
|
+
}): Promise<MachineRef[]>;
|
|
68
80
|
exec(machineId: MachineId, input: ExecInput): Promise<ExecResult>;
|
|
69
81
|
waitForPort(machineId: MachineId, port: number, opts?: {
|
|
70
82
|
maxWaitMs?: number;
|
|
@@ -73,5 +85,9 @@ export interface MachineProvider {
|
|
|
73
85
|
health(): Promise<HealthResult>;
|
|
74
86
|
destroy(machineId: MachineId): Promise<DestroyOutcome>;
|
|
75
87
|
destroyVolume(volumeId: string): Promise<DestroyOutcome>;
|
|
88
|
+
/** The resource currently holding this volume ("sandbox:<name>"), or null if detached.
|
|
89
|
+
* A missing volume (404) throws — a home volume disappearing is always an incident,
|
|
90
|
+
* never something to continue past (recreate-race-fix.md §3.1). */
|
|
91
|
+
getVolumeAttachment(volumeId: string): Promise<string | null>;
|
|
76
92
|
}
|
|
77
93
|
export type { MachineState, M0Config, RouteRef };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
// Bug-regression test (independent review of commit 6018050, recreate-race-fix.md §3.6):
|
|
4
|
+
// `JSON.stringify` returns `undefined` (does not throw) for `undefined`, a function, or a
|
|
5
|
+
// symbol — a naive `JSON.stringify(error)` rider would silently hand that `undefined` back to
|
|
6
|
+
// callers. In `relay.ts` that `undefined` message then makes `message.startsWith('relay ')`
|
|
7
|
+
// THROW inside the very catch block meant to handle it (a control-flow regression §3.6
|
|
8
|
+
// promised not to introduce); in `wake.ts` it would OMIT the `error` field from the
|
|
9
|
+
// `hearth.roll.incident` JSON line entirely. Both serializers must fall back to `String`
|
|
10
|
+
// whenever `JSON.stringify` yields `undefined`, not only when it throws.
|
|
11
|
+
test('serializeError (wake.ts) never returns undefined for undefined/function/symbol rejections', async () => {
|
|
12
|
+
const { serializeError } = await import('../wake.js');
|
|
13
|
+
for (const value of [undefined, function unnamed() { }, Symbol('boom')]) {
|
|
14
|
+
const result = serializeError(value);
|
|
15
|
+
assert.equal(typeof result, 'string', `serializeError(${String(value)}) must return a string, never JS \`undefined\``);
|
|
16
|
+
assert.ok(result.length > 0, `serializeError(${String(value)}) must be a non-empty string`);
|
|
17
|
+
}
|
|
18
|
+
});
|
|
19
|
+
test('serializeRelayError (relay.ts) never returns undefined for undefined/function/symbol rejections, and the relay control-flow checks it feeds never throw', async () => {
|
|
20
|
+
const { serializeRelayError } = await import('../relay.js');
|
|
21
|
+
for (const value of [undefined, function unnamed() { }, Symbol('boom')]) {
|
|
22
|
+
const result = serializeRelayError(value);
|
|
23
|
+
assert.equal(typeof result, 'string');
|
|
24
|
+
assert.ok(result.length > 0, `serializeRelayError(${String(value)}) must be a non-empty string`);
|
|
25
|
+
// The exact control flow `proxyHttp`'s catch runs on this value — must not throw.
|
|
26
|
+
assert.doesNotThrow(() => result.startsWith('relay '));
|
|
27
|
+
assert.doesNotThrow(() => /timed out/i.test(result));
|
|
28
|
+
}
|
|
29
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Regression test for the P2 review Major fix: a stale, abandoned OAuth
|
|
2
|
+
// marker must not make a later `beginOAuth` on the same home read as
|
|
3
|
+
// non-serving. See serving.ts's `beginOAuth`/`oauthMarkerLive`.
|
|
4
|
+
import { test } from 'node:test';
|
|
5
|
+
import assert from 'node:assert/strict';
|
|
6
|
+
import { mock } from 'node:test';
|
|
7
|
+
import { beginOAuth, endOAuth, isHomeServing, resetServingForTests } from '../serving.js';
|
|
8
|
+
const HOME = 'home-oauth-marker-test';
|
|
9
|
+
const OAUTH_MARKER_TTL_MS = 5 * 60_000;
|
|
10
|
+
test('a second beginOAuth refreshes a stale marker and stays live until both finishes land', () => {
|
|
11
|
+
resetServingForTests();
|
|
12
|
+
mock.timers.enable({ apis: ['Date'] });
|
|
13
|
+
try {
|
|
14
|
+
// Start A begins an OAuth window; it is immediately serving.
|
|
15
|
+
beginOAuth(HOME);
|
|
16
|
+
assert.equal(isHomeServing(HOME), true, 'start A should mark the home as serving');
|
|
17
|
+
// Start A is abandoned: advance the clock past the 5-minute TTL with no
|
|
18
|
+
// matching endOAuth. The marker goes stale, but the counter (1) is still
|
|
19
|
+
// positive because start A was never finished.
|
|
20
|
+
mock.timers.tick(OAUTH_MARKER_TTL_MS + 1_000);
|
|
21
|
+
assert.equal(isHomeServing(HOME), false, 'a stale marker with no refresh must read as not-serving');
|
|
22
|
+
// Start B is a fresh retry on the same home. Per the fix, beginOAuth
|
|
23
|
+
// refreshes oauthStartedAt on EVERY call, not only when the counter
|
|
24
|
+
// transitions from 0 — so B's own window must read as live immediately.
|
|
25
|
+
beginOAuth(HOME);
|
|
26
|
+
assert.equal(isHomeServing(HOME), true, 'a second beginOAuth must refresh the marker and read as serving');
|
|
27
|
+
// B's window survives well past when A's original (unrefreshed) TTL
|
|
28
|
+
// would have expired, proving the refresh — not the original timestamp
|
|
29
|
+
// — is what's being judged now.
|
|
30
|
+
mock.timers.tick(OAUTH_MARKER_TTL_MS - 1_000);
|
|
31
|
+
assert.equal(isHomeServing(HOME), true, 'B\'s window must still be live before its own TTL elapses');
|
|
32
|
+
// A late finish for start A arrives: counter floors from 2 -> 1. Still
|
|
33
|
+
// serving, because B's own start/finish window has not closed yet.
|
|
34
|
+
endOAuth(HOME);
|
|
35
|
+
assert.equal(isHomeServing(HOME), true, 'one matched finish (of two starts) must not end serving early');
|
|
36
|
+
// B finishes: counter 1 -> 0, marker cleared, no longer serving.
|
|
37
|
+
endOAuth(HOME);
|
|
38
|
+
assert.equal(isHomeServing(HOME), false, 'both starts matched by a finish must end serving');
|
|
39
|
+
}
|
|
40
|
+
finally {
|
|
41
|
+
mock.timers.reset();
|
|
42
|
+
resetServingForTests();
|
|
43
|
+
}
|
|
44
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { test } from 'node:test';
|
|
3
|
+
// Bug-regression tests for recreate-race-fix.md §3.3/§3.5: under generation naming, a roll
|
|
4
|
+
// or rollback recreate destroys the PRE-recreate sandbox and creates a fresh-named one.
|
|
5
|
+
// `rollHome`/`rollbackHome` must assert the in-guest `crtr sys version` against the sandbox
|
|
6
|
+
// name the recreate ACTUALLY reports (`refresh.providerSandboxId`) — asserting against a
|
|
7
|
+
// `sandboxName` captured from `home.provider_sandbox_id` at entry (the pre-fix behavior)
|
|
8
|
+
// would exec into the already-destroyed old generation instead. These tests fail if that
|
|
9
|
+
// regresses (they pin the exec target, not just that the call succeeds).
|
|
10
|
+
function fixtureHome(overrides = {}) {
|
|
11
|
+
return {
|
|
12
|
+
home_id: 'home-1',
|
|
13
|
+
tenant_id: 't1',
|
|
14
|
+
user_id: 'u1',
|
|
15
|
+
provider: 'blaxel',
|
|
16
|
+
provider_sandbox_id: 'hearth-home-1',
|
|
17
|
+
provider_volume_id: 'vol-home-1',
|
|
18
|
+
preview_endpoint: 'https://old.example',
|
|
19
|
+
home_agent_target: 'node-1',
|
|
20
|
+
relay_auth_ref: 'ref-1',
|
|
21
|
+
status: 'running',
|
|
22
|
+
template_version: '0.3.35',
|
|
23
|
+
last_ready_at: null,
|
|
24
|
+
last_wake_at: null,
|
|
25
|
+
last_suspend_at: null,
|
|
26
|
+
last_wake_latency_ms: null,
|
|
27
|
+
health_verdict: 'healthy',
|
|
28
|
+
created_at: '2026-07-01T00:00:00.000Z',
|
|
29
|
+
updated_at: '2026-07-01T00:00:00.000Z',
|
|
30
|
+
...overrides,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
function fixtureTarget(overrides = {}) {
|
|
34
|
+
return {
|
|
35
|
+
channel: 'home',
|
|
36
|
+
target_template_version: '0.3.36',
|
|
37
|
+
target_image_ref: 'crouter-hearth-home-0-3-36',
|
|
38
|
+
published_at: '2026-07-01T00:00:00.000Z',
|
|
39
|
+
source: 'test',
|
|
40
|
+
last_bad_target_version: null,
|
|
41
|
+
...overrides,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
test('a successful plain (non-roll) wake clears a stale degraded health verdict to healthy', async () => {
|
|
45
|
+
const { ensureAwake, wakeInternals, resetWakeCacheForTests } = await import('../wake.js');
|
|
46
|
+
const orig = { ...wakeInternals };
|
|
47
|
+
resetWakeCacheForTests();
|
|
48
|
+
const home = fixtureHome({ home_id: 'home-plain-wake', provider_sandbox_id: 'hearth-home-plain-wake', health_verdict: 'degraded' });
|
|
49
|
+
let healthVerdict = null;
|
|
50
|
+
wakeInternals.getHomeById = ((id) => (id === home.home_id ? home : null));
|
|
51
|
+
// No hearth target registered, so runEnsureAwake takes the normal wake path, not the roll branch.
|
|
52
|
+
wakeInternals.getHearthTarget = (() => null);
|
|
53
|
+
wakeInternals.resolveSecret = (() => 'relay-token');
|
|
54
|
+
wakeInternals.loadM0Config = (() => ({}));
|
|
55
|
+
wakeInternals.loadControlPlaneConfig = (() => ({ readyTimeoutMs: 1_000 }));
|
|
56
|
+
wakeInternals.probeNodeReady = (async () => true);
|
|
57
|
+
wakeInternals.setHomeStatus = (() => home);
|
|
58
|
+
wakeInternals.setHealthVerdict = ((_id, verdict) => {
|
|
59
|
+
healthVerdict = verdict;
|
|
60
|
+
return home;
|
|
61
|
+
});
|
|
62
|
+
wakeInternals.touchWake = (() => home);
|
|
63
|
+
wakeInternals.touchReady = (() => home);
|
|
64
|
+
wakeInternals.getHomeBackend = (() => ({
|
|
65
|
+
wake: async () => ({
|
|
66
|
+
providerName: 'blaxel',
|
|
67
|
+
providerSandboxId: home.provider_sandbox_id,
|
|
68
|
+
providerVolumeId: 'vol-home-1',
|
|
69
|
+
previewEndpoint: home.preview_endpoint,
|
|
70
|
+
route: null,
|
|
71
|
+
homeAgentTarget: 'node-1',
|
|
72
|
+
relayAuthRef: null,
|
|
73
|
+
templateVersion: home.template_version,
|
|
74
|
+
status: 'running',
|
|
75
|
+
machine: null,
|
|
76
|
+
updatedAt: new Date().toISOString(),
|
|
77
|
+
}),
|
|
78
|
+
recreateOnImage: async () => {
|
|
79
|
+
throw new Error('recreateOnImage must not be called on the plain-wake path');
|
|
80
|
+
},
|
|
81
|
+
suspend: async () => {
|
|
82
|
+
throw new Error('unused');
|
|
83
|
+
},
|
|
84
|
+
getMachine: async () => null,
|
|
85
|
+
exec: async () => {
|
|
86
|
+
throw new Error('exec must not be called on the plain-wake path');
|
|
87
|
+
},
|
|
88
|
+
}));
|
|
89
|
+
try {
|
|
90
|
+
const result = await ensureAwake(home.home_id);
|
|
91
|
+
assert.equal(result.preview_endpoint, home.preview_endpoint);
|
|
92
|
+
assert.equal(healthVerdict, 'healthy', 'a successful plain wake must clear a stale degraded verdict to healthy');
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
Object.assign(wakeInternals, orig);
|
|
96
|
+
resetWakeCacheForTests();
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
test('rollHome asserts guest version against the RECREATED sandbox name, not the stale pre-roll name', async () => {
|
|
100
|
+
const { ensureAwake, wakeInternals, resetWakeCacheForTests } = await import('../wake.js');
|
|
101
|
+
const orig = { ...wakeInternals };
|
|
102
|
+
resetWakeCacheForTests();
|
|
103
|
+
const execCalls = [];
|
|
104
|
+
const home = fixtureHome({ home_id: 'home-roll-ok', provider_sandbox_id: 'hearth-home-roll-ok' });
|
|
105
|
+
const target = fixtureTarget();
|
|
106
|
+
const freshName = 'hearth-home-roll-ok-gfresh1';
|
|
107
|
+
wakeInternals.getHomeById = ((id) => (id === home.home_id ? home : null));
|
|
108
|
+
wakeInternals.getHearthTarget = (() => target);
|
|
109
|
+
wakeInternals.resolveSecret = (() => 'relay-token');
|
|
110
|
+
wakeInternals.loadM0Config = (() => ({}));
|
|
111
|
+
wakeInternals.loadControlPlaneConfig = (() => ({ readyTimeoutMs: 1_000 }));
|
|
112
|
+
wakeInternals.probeNodeReady = (async () => true);
|
|
113
|
+
wakeInternals.refreshProviderPointers = ((_id, patch) => ({ ...home, ...patch }));
|
|
114
|
+
wakeInternals.setHomeStatus = (() => home);
|
|
115
|
+
wakeInternals.setHealthVerdict = (() => home);
|
|
116
|
+
wakeInternals.touchWake = (() => home);
|
|
117
|
+
wakeInternals.touchReady = (() => home);
|
|
118
|
+
wakeInternals.recordBadTarget = (() => null);
|
|
119
|
+
wakeInternals.getHomeBackend = (() => ({
|
|
120
|
+
wake: async () => {
|
|
121
|
+
throw new Error('wake() must not be called on the roll path');
|
|
122
|
+
},
|
|
123
|
+
recreateOnImage: async () => ({
|
|
124
|
+
providerName: 'blaxel',
|
|
125
|
+
providerSandboxId: freshName,
|
|
126
|
+
providerVolumeId: 'vol-home-1',
|
|
127
|
+
previewEndpoint: 'https://new.example',
|
|
128
|
+
route: null,
|
|
129
|
+
homeAgentTarget: 'node-1',
|
|
130
|
+
relayAuthRef: null,
|
|
131
|
+
templateVersion: target.target_template_version,
|
|
132
|
+
status: 'running',
|
|
133
|
+
machine: null,
|
|
134
|
+
updatedAt: new Date().toISOString(),
|
|
135
|
+
}),
|
|
136
|
+
suspend: async () => {
|
|
137
|
+
throw new Error('unused');
|
|
138
|
+
},
|
|
139
|
+
getMachine: async () => null,
|
|
140
|
+
exec: async (machineId) => {
|
|
141
|
+
execCalls.push(machineId);
|
|
142
|
+
return { exitCode: 0, timedOut: false, stdout: JSON.stringify({ version: target.target_template_version }), stderr: '' };
|
|
143
|
+
},
|
|
144
|
+
}));
|
|
145
|
+
try {
|
|
146
|
+
const result = await ensureAwake(home.home_id);
|
|
147
|
+
assert.equal(result.preview_endpoint, 'https://new.example');
|
|
148
|
+
assert.deepEqual(execCalls, [freshName], 'assertGuestVersion must exec into the RECREATED sandbox name, never the stale pre-roll name');
|
|
149
|
+
}
|
|
150
|
+
finally {
|
|
151
|
+
Object.assign(wakeInternals, orig);
|
|
152
|
+
resetWakeCacheForTests();
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
test('rollbackHome asserts guest version against the RECREATED rollback sandbox name, and rollback success sets healthy with no incident', async () => {
|
|
156
|
+
const { ensureAwake, wakeInternals, resetWakeCacheForTests } = await import('../wake.js');
|
|
157
|
+
const orig = { ...wakeInternals };
|
|
158
|
+
resetWakeCacheForTests();
|
|
159
|
+
const execCalls = [];
|
|
160
|
+
const home = fixtureHome({ home_id: 'home-roll-fail', provider_sandbox_id: 'hearth-home-roll-fail' });
|
|
161
|
+
const target = fixtureTarget();
|
|
162
|
+
const rollbackFreshName = 'hearth-home-roll-fail-gback1';
|
|
163
|
+
let recreateCalls = 0;
|
|
164
|
+
let badTargetRecorded = null;
|
|
165
|
+
let healthVerdict = null;
|
|
166
|
+
wakeInternals.getHomeById = ((id) => (id === home.home_id ? home : null));
|
|
167
|
+
wakeInternals.getHearthTarget = (() => target);
|
|
168
|
+
wakeInternals.resolveSecret = (() => 'relay-token');
|
|
169
|
+
wakeInternals.loadM0Config = (() => ({}));
|
|
170
|
+
wakeInternals.loadControlPlaneConfig = (() => ({ readyTimeoutMs: 1_000 }));
|
|
171
|
+
wakeInternals.probeNodeReady = (async () => true);
|
|
172
|
+
wakeInternals.refreshProviderPointers = ((_id, patch) => ({ ...home, ...patch }));
|
|
173
|
+
wakeInternals.setHomeStatus = (() => home);
|
|
174
|
+
wakeInternals.setHealthVerdict = ((_id, verdict) => {
|
|
175
|
+
healthVerdict = verdict;
|
|
176
|
+
return home;
|
|
177
|
+
});
|
|
178
|
+
wakeInternals.touchWake = (() => home);
|
|
179
|
+
wakeInternals.touchReady = (() => home);
|
|
180
|
+
wakeInternals.recordBadTarget = ((version) => {
|
|
181
|
+
badTargetRecorded = version;
|
|
182
|
+
return null;
|
|
183
|
+
});
|
|
184
|
+
wakeInternals.getHomeBackend = (() => ({
|
|
185
|
+
wake: async () => {
|
|
186
|
+
throw new Error('unused');
|
|
187
|
+
},
|
|
188
|
+
recreateOnImage: async (_descriptor, _imageRef, templateVersion) => {
|
|
189
|
+
recreateCalls += 1;
|
|
190
|
+
if (recreateCalls === 1) {
|
|
191
|
+
// Simulate the roll-forward recreate succeeding but the guest version assert
|
|
192
|
+
// failing downstream — rollHome's catch delegates to rollbackHome regardless of
|
|
193
|
+
// WHERE in the try it failed.
|
|
194
|
+
throw new Error('simulated roll failure (version mismatch)');
|
|
195
|
+
}
|
|
196
|
+
return {
|
|
197
|
+
providerName: 'blaxel',
|
|
198
|
+
providerSandboxId: rollbackFreshName,
|
|
199
|
+
providerVolumeId: 'vol-home-1',
|
|
200
|
+
previewEndpoint: 'https://rollback.example',
|
|
201
|
+
route: null,
|
|
202
|
+
homeAgentTarget: 'node-1',
|
|
203
|
+
relayAuthRef: null,
|
|
204
|
+
templateVersion,
|
|
205
|
+
status: 'running',
|
|
206
|
+
machine: null,
|
|
207
|
+
updatedAt: new Date().toISOString(),
|
|
208
|
+
};
|
|
209
|
+
},
|
|
210
|
+
suspend: async () => {
|
|
211
|
+
throw new Error('unused');
|
|
212
|
+
},
|
|
213
|
+
getMachine: async () => null,
|
|
214
|
+
exec: async (machineId) => {
|
|
215
|
+
execCalls.push(machineId);
|
|
216
|
+
return { exitCode: 0, timedOut: false, stdout: JSON.stringify({ version: home.template_version }), stderr: '' };
|
|
217
|
+
},
|
|
218
|
+
}));
|
|
219
|
+
try {
|
|
220
|
+
const result = await ensureAwake(home.home_id);
|
|
221
|
+
assert.equal(result.preview_endpoint, 'https://rollback.example');
|
|
222
|
+
assert.equal(badTargetRecorded, target.target_template_version, 'the roll failure must still poison the target (brake unconditional on roll failure)');
|
|
223
|
+
assert.equal(healthVerdict, 'healthy', 'a successful rollback must clear health to healthy, not leave/raise an incident');
|
|
224
|
+
assert.deepEqual(execCalls, [rollbackFreshName], "rollback must assert against ITS OWN recreated sandbox name — never the failed roll's name nor the stale pre-roll name");
|
|
225
|
+
}
|
|
226
|
+
finally {
|
|
227
|
+
Object.assign(wakeInternals, orig);
|
|
228
|
+
resetWakeCacheForTests();
|
|
229
|
+
}
|
|
230
|
+
});
|
|
@@ -457,6 +457,32 @@ BEGIN
|
|
|
457
457
|
END;
|
|
458
458
|
`);
|
|
459
459
|
}
|
|
460
|
+
/** v9 — the single global auto-upgrade target (design.md §2/§9). ONE row,
|
|
461
|
+
* `channel = 'home'`, holding the DESIRED crtr image line every home rolls
|
|
462
|
+
* onto: `target_template_version` is the semver the fleet should be running,
|
|
463
|
+
* `target_image_ref` its immutable Blaxel image name (must equal
|
|
464
|
+
* `hearthHomeImageRefForVersion(target_template_version)` — the admin route
|
|
465
|
+
* enforces that). `last_bad_target_version` is the ONLY brake in the
|
|
466
|
+
* zero-touch loop: a version recorded here already failed a live roll for
|
|
467
|
+
* the fleet, so the wake-path stale predicate excludes it until the target
|
|
468
|
+
* advances to a newer release. There is deliberately NO `roll_armed` column
|
|
469
|
+
* — the loop is full zero-touch (§9.1), so there is no per-release human arm
|
|
470
|
+
* gate to persist. `home.template_version` stays the ACTUAL running version;
|
|
471
|
+
* this row is the target. Single-tenant: one image line for all homes, so no
|
|
472
|
+
* per-home target column. */
|
|
473
|
+
function hearthTargetTable(db) {
|
|
474
|
+
db.exec(`
|
|
475
|
+
CREATE TABLE IF NOT EXISTS hearth_target (
|
|
476
|
+
channel TEXT PRIMARY KEY DEFAULT 'home',
|
|
477
|
+
target_template_version TEXT NOT NULL,
|
|
478
|
+
target_image_ref TEXT NOT NULL,
|
|
479
|
+
published_at TEXT NOT NULL,
|
|
480
|
+
source TEXT NOT NULL,
|
|
481
|
+
last_bad_target_version TEXT,
|
|
482
|
+
CHECK (channel = 'home')
|
|
483
|
+
);
|
|
484
|
+
`);
|
|
485
|
+
}
|
|
460
486
|
/** The ordered migration list. Index `i` is migration version `i + 1`; the
|
|
461
487
|
* db's `user_version` tracks how many have been applied. Append only. */
|
|
462
488
|
export const MIGRATIONS = [
|
|
@@ -468,6 +494,7 @@ export const MIGRATIONS = [
|
|
|
468
494
|
/* v6 */ triggerAuditTable,
|
|
469
495
|
/* v7 */ tenantIdentityTriggers,
|
|
470
496
|
/* v8 */ parentIdentityUpdateGuards,
|
|
497
|
+
/* v9 */ hearthTargetTable,
|
|
471
498
|
];
|
|
472
499
|
/** Bring `db` up to the latest schema version. Reads `user_version`, runs
|
|
473
500
|
* each pending migration in order, and bumps `user_version` after each so
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import type { HearthTarget, SetHearthTargetInput } from './types.js';
|
|
3
|
+
/** Read the single global target row, or `null` when no image has ever been
|
|
4
|
+
* registered (fresh CP). A `null` means the wake path treats every home as
|
|
5
|
+
* up to date (nothing to roll onto yet). */
|
|
6
|
+
export declare function getHearthTarget(db?: DatabaseSync): HearthTarget | null;
|
|
7
|
+
/** Upsert the global target to a newly-registered image. IDEMPOTENT: a
|
|
8
|
+
* re-post of the version already recorded is a no-op (the row, including its
|
|
9
|
+
* `published_at` and any `last_bad_target_version` brake, is left exactly as
|
|
10
|
+
* is) — the image workflow can fire the notify more than once for one
|
|
11
|
+
* release without churning state. Advancing to a DIFFERENT version rewrites
|
|
12
|
+
* the target/image/published_at/source but deliberately preserves
|
|
13
|
+
* `last_bad_target_version` (a brake recorded for an older release is
|
|
14
|
+
* historical and harmless once the target has moved past it; the wake
|
|
15
|
+
* predicate keys off `target !== last_bad`). */
|
|
16
|
+
export declare function setHearthTarget(input: SetHearthTargetInput, db?: DatabaseSync): HearthTarget;
|
|
17
|
+
/** Record a version whose live roll failed for the fleet (§5.4). After this,
|
|
18
|
+
* the wake-path stale predicate excludes the current target (target ===
|
|
19
|
+
* last_bad_target_version → no home rolls) until a NEW release advances the
|
|
20
|
+
* target past it. This is the ONLY brake in the zero-touch loop (roadmap
|
|
21
|
+
* decision 1: never auto-advance onto a release that already failed a roll).
|
|
22
|
+
* Returns `null` if no target row exists (nothing to brake). */
|
|
23
|
+
export declare function recordBadTarget(badVersion: string, db?: DatabaseSync): HearthTarget | null;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// control-plane/hearth-target.ts — the CP store for the single global
|
|
2
|
+
// auto-upgrade target row (`hearth_target`, db.ts v9). One row, `channel =
|
|
3
|
+
// 'home'`. This module is the ONLY reader/writer of that row; the admin
|
|
4
|
+
// notify route (server.ts) upserts it, the wake path (wake.ts) reads it to
|
|
5
|
+
// decide whether a home is stale, and the rollback path records a bad target
|
|
6
|
+
// through it. It goes through the shared CP db handle (`getControlPlaneDb`)
|
|
7
|
+
// exactly like `registry.ts` — never a second connection.
|
|
8
|
+
import { getControlPlaneDb } from './db.js';
|
|
9
|
+
const TARGET_COLUMNS = [
|
|
10
|
+
'channel',
|
|
11
|
+
'target_template_version',
|
|
12
|
+
'target_image_ref',
|
|
13
|
+
'published_at',
|
|
14
|
+
'source',
|
|
15
|
+
'last_bad_target_version',
|
|
16
|
+
];
|
|
17
|
+
const SELECT_TARGET = `SELECT ${TARGET_COLUMNS.join(', ')} FROM hearth_target WHERE channel = 'home'`;
|
|
18
|
+
function nowIso() {
|
|
19
|
+
return new Date().toISOString();
|
|
20
|
+
}
|
|
21
|
+
/** Read the single global target row, or `null` when no image has ever been
|
|
22
|
+
* registered (fresh CP). A `null` means the wake path treats every home as
|
|
23
|
+
* up to date (nothing to roll onto yet). */
|
|
24
|
+
export function getHearthTarget(db = getControlPlaneDb()) {
|
|
25
|
+
const row = db.prepare(SELECT_TARGET).get();
|
|
26
|
+
if (row === undefined || row === null)
|
|
27
|
+
return null;
|
|
28
|
+
return row;
|
|
29
|
+
}
|
|
30
|
+
/** Upsert the global target to a newly-registered image. IDEMPOTENT: a
|
|
31
|
+
* re-post of the version already recorded is a no-op (the row, including its
|
|
32
|
+
* `published_at` and any `last_bad_target_version` brake, is left exactly as
|
|
33
|
+
* is) — the image workflow can fire the notify more than once for one
|
|
34
|
+
* release without churning state. Advancing to a DIFFERENT version rewrites
|
|
35
|
+
* the target/image/published_at/source but deliberately preserves
|
|
36
|
+
* `last_bad_target_version` (a brake recorded for an older release is
|
|
37
|
+
* historical and harmless once the target has moved past it; the wake
|
|
38
|
+
* predicate keys off `target !== last_bad`). */
|
|
39
|
+
export function setHearthTarget(input, db = getControlPlaneDb()) {
|
|
40
|
+
const existing = getHearthTarget(db);
|
|
41
|
+
if (existing !== null && existing.target_template_version === input.target_template_version) {
|
|
42
|
+
return existing;
|
|
43
|
+
}
|
|
44
|
+
const published_at = input.published_at ?? nowIso();
|
|
45
|
+
db.prepare(`
|
|
46
|
+
INSERT INTO hearth_target (channel, target_template_version, target_image_ref, published_at, source, last_bad_target_version)
|
|
47
|
+
VALUES ('home', ?, ?, ?, ?, NULL)
|
|
48
|
+
ON CONFLICT(channel) DO UPDATE SET
|
|
49
|
+
target_template_version = excluded.target_template_version,
|
|
50
|
+
target_image_ref = excluded.target_image_ref,
|
|
51
|
+
published_at = excluded.published_at,
|
|
52
|
+
source = excluded.source
|
|
53
|
+
`).run(input.target_template_version, input.target_image_ref, published_at, input.source);
|
|
54
|
+
return getHearthTarget(db);
|
|
55
|
+
}
|
|
56
|
+
/** Record a version whose live roll failed for the fleet (§5.4). After this,
|
|
57
|
+
* the wake-path stale predicate excludes the current target (target ===
|
|
58
|
+
* last_bad_target_version → no home rolls) until a NEW release advances the
|
|
59
|
+
* target past it. This is the ONLY brake in the zero-touch loop (roadmap
|
|
60
|
+
* decision 1: never auto-advance onto a release that already failed a roll).
|
|
61
|
+
* Returns `null` if no target row exists (nothing to brake). */
|
|
62
|
+
export function recordBadTarget(badVersion, db = getControlPlaneDb()) {
|
|
63
|
+
const existing = getHearthTarget(db);
|
|
64
|
+
if (existing === null)
|
|
65
|
+
return null;
|
|
66
|
+
db.prepare(`UPDATE hearth_target SET last_bad_target_version = ? WHERE channel = 'home'`).run(badVersion);
|
|
67
|
+
return getHearthTarget(db);
|
|
68
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite';
|
|
2
|
-
import type { CreateHomeInput, Home, HomeId, HomePatch, HomeStatus, RefreshProviderPointersInput, ResolvedHome, TenantSession, TouchWakeInput } from './types.js';
|
|
2
|
+
import type { CreateHomeInput, Home, HomeHealthVerdict, HomeId, HomePatch, HomeStatus, RefreshProviderPointersInput, ResolvedHome, TenantSession, TouchWakeInput } from './types.js';
|
|
3
3
|
export declare function createHome(input: CreateHomeInput, db?: DatabaseSync): Home;
|
|
4
4
|
export declare function getHomeById(home_id: HomeId | null | undefined, db?: DatabaseSync): Home | null;
|
|
5
5
|
/** Resolve a tenant session to its registry row. A null return is a hard
|
|
@@ -12,6 +12,11 @@ export declare function getHomeBySandbox(provider_sandbox_id: string | null | un
|
|
|
12
12
|
export declare function updateHome(home_id: HomeId, mutate: (current: Home) => HomePatch, db?: DatabaseSync): Home;
|
|
13
13
|
export declare function refreshProviderPointers(home_id: HomeId, pointers: RefreshProviderPointersInput, db?: DatabaseSync): Home;
|
|
14
14
|
export declare function setHomeStatus(home_id: HomeId, status: HomeStatus, db?: DatabaseSync): Home;
|
|
15
|
+
/** Set the home's health verdict. The auto-upgrade roll (wake.ts `rollHome`)
|
|
16
|
+
* is the first natural producer of `health_verdict`: a completed roll/
|
|
17
|
+
* rollback writes `'healthy'`, a roll AND rollback both failing writes
|
|
18
|
+
* `'degraded'` (design §4/§5). */
|
|
19
|
+
export declare function setHealthVerdict(home_id: HomeId, health_verdict: HomeHealthVerdict, db?: DatabaseSync): Home;
|
|
15
20
|
export declare function touchWake(home_id: HomeId, input?: TouchWakeInput, db?: DatabaseSync): Home;
|
|
16
21
|
export declare function touchReady(home_id: HomeId, at?: string, db?: DatabaseSync): Home;
|
|
17
22
|
export declare function touchSuspend(home_id: HomeId, at?: string, db?: DatabaseSync): Home;
|