@crouton-kit/crouter 0.3.38 → 0.3.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/dist/core/hearth/provider.d.ts +8 -0
  2. package/dist/core/hearth/providers/__tests__/sweep-and-release.test.d.ts +1 -0
  3. package/dist/core/hearth/providers/__tests__/sweep-and-release.test.js +254 -0
  4. package/dist/core/hearth/providers/blaxel-home.d.ts +39 -0
  5. package/dist/core/hearth/providers/blaxel-home.js +271 -21
  6. package/dist/core/hearth/providers/blaxel.d.ts +5 -0
  7. package/dist/core/hearth/providers/blaxel.js +86 -4
  8. package/dist/core/hearth/providers/types.d.ts +16 -0
  9. package/dist/hearth/control-plane/__tests__/error-serialization.test.d.ts +1 -0
  10. package/dist/hearth/control-plane/__tests__/error-serialization.test.js +29 -0
  11. package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.d.ts +1 -0
  12. package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.js +44 -0
  13. package/dist/hearth/control-plane/__tests__/wake-roll.test.d.ts +1 -0
  14. package/dist/hearth/control-plane/__tests__/wake-roll.test.js +230 -0
  15. package/dist/hearth/control-plane/db.js +27 -0
  16. package/dist/hearth/control-plane/hearth-target.d.ts +23 -0
  17. package/dist/hearth/control-plane/hearth-target.js +68 -0
  18. package/dist/hearth/control-plane/registry.d.ts +6 -1
  19. package/dist/hearth/control-plane/registry.js +7 -0
  20. package/dist/hearth/control-plane/relay.d.ts +4 -0
  21. package/dist/hearth/control-plane/relay.js +72 -3
  22. package/dist/hearth/control-plane/secrets.d.ts +14 -0
  23. package/dist/hearth/control-plane/secrets.js +21 -0
  24. package/dist/hearth/control-plane/server.js +140 -4
  25. package/dist/hearth/control-plane/serving.d.ts +15 -0
  26. package/dist/hearth/control-plane/serving.js +106 -0
  27. package/dist/hearth/control-plane/types.d.ts +20 -0
  28. package/dist/hearth/control-plane/wake.d.ts +8 -2
  29. package/dist/hearth/control-plane/wake.js +241 -3
  30. package/package.json +1 -1
@@ -7,6 +7,7 @@ interface BlaxelMachineProviderOptions {
7
7
  templateVersion?: string;
8
8
  volumeSizeGb?: number;
9
9
  }
10
+ export declare function isMissingResourceError(error: unknown): boolean;
10
11
  export declare class BlaxelMachineProvider implements MachineProvider {
11
12
  readonly providerName: "blaxel";
12
13
  readonly imageRef: string;
@@ -18,6 +19,9 @@ export declare class BlaxelMachineProvider implements MachineProvider {
18
19
  name: string;
19
20
  }): Promise<VolumeRef>;
20
21
  createMachine(input: CreateMachineInput): Promise<MachineRef>;
22
+ listMachines(filter?: {
23
+ externalId?: string;
24
+ }): Promise<MachineRef[]>;
21
25
  getMachine(machineId: string): Promise<MachineRef>;
22
26
  exec(machineId: string, input: ExecInput): Promise<ExecResult>;
23
27
  waitForPort(machineId: string, port: number, opts?: {
@@ -27,5 +31,6 @@ export declare class BlaxelMachineProvider implements MachineProvider {
27
31
  health(): Promise<HealthResult>;
28
32
  destroy(machineId: string): Promise<DestroyOutcome>;
29
33
  destroyVolume(volumeId: string): Promise<DestroyOutcome>;
34
+ getVolumeAttachment(volumeId: string): Promise<string | null>;
30
35
  }
31
36
  export {};
@@ -1,6 +1,6 @@
1
1
  import { randomUUID } from 'node:crypto';
2
2
  import { readFileSync } from 'node:fs';
3
- import { SandboxInstance, VolumeInstance } from '@blaxel/core';
3
+ import { SandboxInstance, VolumeInstance, getVolume, listSandboxes } from '@blaxel/core';
4
4
  import { general } from '../../errors.js';
5
5
  const TERMINATED_STATES = new Set(['TERMINATED', 'DELETING', 'DEACTIVATED', 'DEACTIVATING', 'FAILED']);
6
6
  const PROVISIONING_STATES = new Set(['UPLOADING', 'BUILDING', 'DEPLOYING', 'BUILT']);
@@ -81,10 +81,28 @@ function resolveExecCommand(input) {
81
81
  }
82
82
  return argv.map(shellQuote).join(' ');
83
83
  }
84
- function isMissingResourceError(error) {
85
- const message = error instanceof Error ? error.message : String(error);
84
+ // The Blaxel SDK (@blaxel/core 0.2.94) does not always reject with an `Error` instance — a 404
85
+ // comes back as a plain object (e.g. `{code:404,error:"Sandbox not found"}`), and `String()` on
86
+ // such an object yields the useless literal "[object Object]", which matches none of the regex
87
+ // below. Recognize the structured shape directly (a numeric-or-string `404` status field) AND
88
+ // fall back to JSON-serializing the value (mirroring wake.ts's `serializeError`) before running
89
+ // the message regex, so both an `Error("... not found")` and a plain-object 404 are caught.
90
+ export function isMissingResourceError(error) {
91
+ const fields = record(error);
92
+ if (fields !== null) {
93
+ for (const key of ['code', 'status', 'status_code', 'statusCode']) {
94
+ const value = fields[key];
95
+ if (value === 404 || value === '404')
96
+ return true;
97
+ }
98
+ }
99
+ const message = error instanceof Error ? error.message : (JSON.stringify(error) ?? String(error));
86
100
  return /\b(not found|404|does not exist|missing)\b/i.test(message);
87
101
  }
102
+ // Accepts either a live `SandboxInstance` or a raw wire `Sandbox` page item — only the
103
+ // `metadata`/`status`/`spec` fields are read, so mapping raw list pages avoids constructing
104
+ // `SandboxInstance`s (whose `list()` eagerly warms an H2 session per sandbox — pointless for a
105
+ // destroy sweep).
88
106
  function toMachineRef(sandbox) {
89
107
  const metadata = sandbox.metadata;
90
108
  const status = sandbox.status ?? undefined;
@@ -142,7 +160,16 @@ export class BlaxelMachineProvider {
142
160
  async createMachine(input) {
143
161
  const sandbox = await SandboxInstance.createIfNotExists({
144
162
  name: input.name ?? input.tenantId,
145
- image: this.config.image,
163
+ // Tag every sandbox with the home's stable volume id so the recreate sweep can scope its
164
+ // destructive list to this home's family only (orphan-cleanup-fix MAJOR-1). Unset on
165
+ // callers that pass no externalId.
166
+ ...(input.externalId === undefined ? {} : { externalId: input.externalId }),
167
+ // A targeted recreate (auto-upgrade roll / rollback, design §4.1) passes
168
+ // an explicit `image`; the normal provision/wake path leaves it unset
169
+ // and uses the provider's construction-time config image. `createIfNotExists`
170
+ // only honors this on genuine creation — the roll path destroys the old
171
+ // sandbox first so the name is free (blaxel-home.ts `recreateOnImage`).
172
+ image: input.image ?? this.config.image,
146
173
  memory: this.config.memory,
147
174
  region: this.config.region,
148
175
  volumes: input.volumeMount === undefined || input.volumeMount === null ? undefined : [{ name: input.volumeMount.volumeName, mountPath: input.volumeMount.path, readOnly: false }],
@@ -158,6 +185,38 @@ export class BlaxelMachineProvider {
158
185
  await sandbox.wait({ maxWait: 90_000, interval: 500 }).catch(() => sandbox);
159
186
  return toMachineRef(sandbox);
160
187
  }
188
+ // orphan-cleanup-fix.md §3.2 + MAJOR-1/MINOR-1: the deterministic name-family sweep enumerates
189
+ // a home's generations via LIST scoped to the caller-owned `externalId` tag, then destroys the
190
+ // ones whose name is in the family. Over-listing is harmless — a FAILED/detached record still
191
+ // shows up here and destroy tolerates not_found, which is exactly what closes the autonomous-
192
+ // standby-detach orphan gap. Drives the generated `listSandboxes` client directly (bypassing
193
+ // `SandboxInstance.list()`, which never follows a cursor and warms an H2 session per sandbox):
194
+ // loops until exhausted so a future SDK version crossing Blaxel-Version 2026-04-28 (cursor
195
+ // pagination, 50/page) cannot silently truncate the sweep to page 1. At the SDK's pinned
196
+ // 2026-04-16 the server returns a bare array (the full set) and the loop breaks on the first
197
+ // page.
198
+ async listMachines(filter) {
199
+ const machines = [];
200
+ let cursor;
201
+ for (;;) {
202
+ const { data: raw } = await listSandboxes({
203
+ query: {
204
+ limit: 200,
205
+ ...(cursor === undefined ? {} : { cursor }),
206
+ ...(filter?.externalId === undefined ? {} : { externalId: filter.externalId }),
207
+ },
208
+ throwOnError: true,
209
+ });
210
+ const page = Array.isArray(raw) ? raw : (raw?.data ?? []);
211
+ machines.push(...page.map(toMachineRef));
212
+ if (Array.isArray(raw))
213
+ break; // legacy bare array (Blaxel-Version < 2026-04-28) = full set
214
+ cursor = raw?.meta?.nextCursor || undefined; // empty when there are no more pages
215
+ if (cursor === undefined)
216
+ break;
217
+ }
218
+ return machines;
219
+ }
161
220
  async getMachine(machineId) {
162
221
  try {
163
222
  const sandbox = await SandboxInstance.get(machineId);
@@ -279,4 +338,27 @@ export class BlaxelMachineProvider {
279
338
  throw error;
280
339
  }
281
340
  }
341
+ // recreate-race-fix.md §3.1: read the platform's own positive attachment signal
342
+ // (`Volume.state.attachedTo`) rather than inferring detach from a sandbox GET/delete —
343
+ // that read/create consistency split is exactly what raced the old same-name recreate.
344
+ // `VolumeInstance.get`'s backing `volume` field is TS-private (no public state getter
345
+ // in @blaxel/core 0.2.94), so this calls the underlying `getVolume` SDK function
346
+ // directly instead — same request `VolumeInstance.get` makes, with `state` exposed.
347
+ // A 404 (or any other failure) throws, wrapped with the volumeId for context: a missing
348
+ // home volume is a real incident, never something to continue past (volume-is-sacred).
349
+ async getVolumeAttachment(volumeId) {
350
+ let attachedTo;
351
+ try {
352
+ const { data } = await getVolume({ path: { volumeName: volumeId }, throwOnError: true });
353
+ attachedTo = data.state?.attachedTo;
354
+ }
355
+ catch (error) {
356
+ throw general(`blaxel volume ${volumeId} attachment check failed`, {
357
+ volumeId,
358
+ cause: error instanceof Error ? error.message : String(error),
359
+ });
360
+ }
361
+ const trimmed = (attachedTo ?? '').trim();
362
+ return trimmed === '' ? null : trimmed;
363
+ }
282
364
  }
@@ -22,10 +22,12 @@ export interface VolumeRef {
22
22
  }
23
23
  export interface CreateMachineInput {
24
24
  tenantId: string;
25
+ externalId?: string;
25
26
  template?: string | null;
26
27
  envs?: Record<string, string>;
27
28
  enableNetwork?: boolean;
28
29
  name?: string | null;
30
+ image?: string | null;
29
31
  ports?: {
30
32
  target: number;
31
33
  protocol: 'HTTP';
@@ -65,6 +67,16 @@ export interface MachineProvider {
65
67
  }): Promise<VolumeRef>;
66
68
  createMachine(input: CreateMachineInput): Promise<MachineRef>;
67
69
  getMachine(machineId: MachineId): Promise<MachineRef>;
70
+ /** List machine records, optionally scoped to a caller-owned `externalId` tag. The recreate
71
+ * cleanup sweep passes the home's `providerVolumeId` as the filter so only sandboxes Hearth
72
+ * itself tagged for this home come back, then intersects that with the generation NAME FAMILY
73
+ * matcher — a tag∩name intersection on a destructive op, so no id-format or workspace name
74
+ * discipline is load-bearing for other homes' safety (orphan-cleanup-fix MAJOR-1). The result
75
+ * is NOT a promise of every machine in the workspace; without a filter it lists broadly, with
76
+ * one it lists only tagged matches. Implementations must page until exhausted (MINOR-1). */
77
+ listMachines(filter?: {
78
+ externalId?: string;
79
+ }): Promise<MachineRef[]>;
68
80
  exec(machineId: MachineId, input: ExecInput): Promise<ExecResult>;
69
81
  waitForPort(machineId: MachineId, port: number, opts?: {
70
82
  maxWaitMs?: number;
@@ -73,5 +85,9 @@ export interface MachineProvider {
73
85
  health(): Promise<HealthResult>;
74
86
  destroy(machineId: MachineId): Promise<DestroyOutcome>;
75
87
  destroyVolume(volumeId: string): Promise<DestroyOutcome>;
88
+ /** The resource currently holding this volume ("sandbox:<name>"), or null if detached.
89
+ * A missing volume (404) throws — a home volume disappearing is always an incident,
90
+ * never something to continue past (recreate-race-fix.md §3.1). */
91
+ getVolumeAttachment(volumeId: string): Promise<string | null>;
76
92
  }
77
93
  export type { MachineState, M0Config, RouteRef };
@@ -0,0 +1,29 @@
1
+ import assert from 'node:assert/strict';
2
+ import { test } from 'node:test';
3
+ // Bug-regression test (independent review of commit 6018050, recreate-race-fix.md §3.6):
4
+ // `JSON.stringify` returns `undefined` (does not throw) for `undefined`, a function, or a
5
+ // symbol — a naive `JSON.stringify(error)` rider would silently hand that `undefined` back to
6
+ // callers. In `relay.ts` that `undefined` message then makes `message.startsWith('relay ')`
7
+ // THROW inside the very catch block meant to handle it (a control-flow regression §3.6
8
+ // promised not to introduce); in `wake.ts` it would OMIT the `error` field from the
9
+ // `hearth.roll.incident` JSON line entirely. Both serializers must fall back to `String`
10
+ // whenever `JSON.stringify` yields `undefined`, not only when it throws.
11
+ test('serializeError (wake.ts) never returns undefined for undefined/function/symbol rejections', async () => {
12
+ const { serializeError } = await import('../wake.js');
13
+ for (const value of [undefined, function unnamed() { }, Symbol('boom')]) {
14
+ const result = serializeError(value);
15
+ assert.equal(typeof result, 'string', `serializeError(${String(value)}) must return a string, never JS \`undefined\``);
16
+ assert.ok(result.length > 0, `serializeError(${String(value)}) must be a non-empty string`);
17
+ }
18
+ });
19
+ test('serializeRelayError (relay.ts) never returns undefined for undefined/function/symbol rejections, and the relay control-flow checks it feeds never throw', async () => {
20
+ const { serializeRelayError } = await import('../relay.js');
21
+ for (const value of [undefined, function unnamed() { }, Symbol('boom')]) {
22
+ const result = serializeRelayError(value);
23
+ assert.equal(typeof result, 'string');
24
+ assert.ok(result.length > 0, `serializeRelayError(${String(value)}) must be a non-empty string`);
25
+ // The exact control flow `proxyHttp`'s catch runs on this value — must not throw.
26
+ assert.doesNotThrow(() => result.startsWith('relay '));
27
+ assert.doesNotThrow(() => /timed out/i.test(result));
28
+ }
29
+ });
@@ -0,0 +1,44 @@
1
+ // Regression test for the P2 review Major fix: a stale, abandoned OAuth
2
+ // marker must not make a later `beginOAuth` on the same home read as
3
+ // non-serving. See serving.ts's `beginOAuth`/`oauthMarkerLive`.
4
+ import { test } from 'node:test';
5
+ import assert from 'node:assert/strict';
6
+ import { mock } from 'node:test';
7
+ import { beginOAuth, endOAuth, isHomeServing, resetServingForTests } from '../serving.js';
8
+ const HOME = 'home-oauth-marker-test';
9
+ const OAUTH_MARKER_TTL_MS = 5 * 60_000;
10
+ test('a second beginOAuth refreshes a stale marker and stays live until both finishes land', () => {
11
+ resetServingForTests();
12
+ mock.timers.enable({ apis: ['Date'] });
13
+ try {
14
+ // Start A begins an OAuth window; it is immediately serving.
15
+ beginOAuth(HOME);
16
+ assert.equal(isHomeServing(HOME), true, 'start A should mark the home as serving');
17
+ // Start A is abandoned: advance the clock past the 5-minute TTL with no
18
+ // matching endOAuth. The marker goes stale, but the counter (1) is still
19
+ // positive because start A was never finished.
20
+ mock.timers.tick(OAUTH_MARKER_TTL_MS + 1_000);
21
+ assert.equal(isHomeServing(HOME), false, 'a stale marker with no refresh must read as not-serving');
22
+ // Start B is a fresh retry on the same home. Per the fix, beginOAuth
23
+ // refreshes oauthStartedAt on EVERY call, not only when the counter
24
+ // transitions from 0 — so B's own window must read as live immediately.
25
+ beginOAuth(HOME);
26
+ assert.equal(isHomeServing(HOME), true, 'a second beginOAuth must refresh the marker and read as serving');
27
+ // B's window survives well past when A's original (unrefreshed) TTL
28
+ // would have expired, proving the refresh — not the original timestamp
29
+ // — is what's being judged now.
30
+ mock.timers.tick(OAUTH_MARKER_TTL_MS - 1_000);
31
+ assert.equal(isHomeServing(HOME), true, 'B\'s window must still be live before its own TTL elapses');
32
+ // A late finish for start A arrives: counter floors from 2 -> 1. Still
33
+ // serving, because B's own start/finish window has not closed yet.
34
+ endOAuth(HOME);
35
+ assert.equal(isHomeServing(HOME), true, 'one matched finish (of two starts) must not end serving early');
36
+ // B finishes: counter 1 -> 0, marker cleared, no longer serving.
37
+ endOAuth(HOME);
38
+ assert.equal(isHomeServing(HOME), false, 'both starts matched by a finish must end serving');
39
+ }
40
+ finally {
41
+ mock.timers.reset();
42
+ resetServingForTests();
43
+ }
44
+ });
@@ -0,0 +1,230 @@
1
+ import assert from 'node:assert/strict';
2
+ import { test } from 'node:test';
3
+ // Bug-regression tests for recreate-race-fix.md §3.3/§3.5: under generation naming, a roll
4
+ // or rollback recreate destroys the PRE-recreate sandbox and creates a fresh-named one.
5
+ // `rollHome`/`rollbackHome` must assert the in-guest `crtr sys version` against the sandbox
6
+ // name the recreate ACTUALLY reports (`refresh.providerSandboxId`) — asserting against a
7
+ // `sandboxName` captured from `home.provider_sandbox_id` at entry (the pre-fix behavior)
8
+ // would exec into the already-destroyed old generation instead. These tests fail if that
9
+ // regresses (they pin the exec target, not just that the call succeeds).
10
+ function fixtureHome(overrides = {}) {
11
+ return {
12
+ home_id: 'home-1',
13
+ tenant_id: 't1',
14
+ user_id: 'u1',
15
+ provider: 'blaxel',
16
+ provider_sandbox_id: 'hearth-home-1',
17
+ provider_volume_id: 'vol-home-1',
18
+ preview_endpoint: 'https://old.example',
19
+ home_agent_target: 'node-1',
20
+ relay_auth_ref: 'ref-1',
21
+ status: 'running',
22
+ template_version: '0.3.35',
23
+ last_ready_at: null,
24
+ last_wake_at: null,
25
+ last_suspend_at: null,
26
+ last_wake_latency_ms: null,
27
+ health_verdict: 'healthy',
28
+ created_at: '2026-07-01T00:00:00.000Z',
29
+ updated_at: '2026-07-01T00:00:00.000Z',
30
+ ...overrides,
31
+ };
32
+ }
33
+ function fixtureTarget(overrides = {}) {
34
+ return {
35
+ channel: 'home',
36
+ target_template_version: '0.3.36',
37
+ target_image_ref: 'crouter-hearth-home-0-3-36',
38
+ published_at: '2026-07-01T00:00:00.000Z',
39
+ source: 'test',
40
+ last_bad_target_version: null,
41
+ ...overrides,
42
+ };
43
+ }
44
+ test('a successful plain (non-roll) wake clears a stale degraded health verdict to healthy', async () => {
45
+ const { ensureAwake, wakeInternals, resetWakeCacheForTests } = await import('../wake.js');
46
+ const orig = { ...wakeInternals };
47
+ resetWakeCacheForTests();
48
+ const home = fixtureHome({ home_id: 'home-plain-wake', provider_sandbox_id: 'hearth-home-plain-wake', health_verdict: 'degraded' });
49
+ let healthVerdict = null;
50
+ wakeInternals.getHomeById = ((id) => (id === home.home_id ? home : null));
51
+ // No hearth target registered, so runEnsureAwake takes the normal wake path, not the roll branch.
52
+ wakeInternals.getHearthTarget = (() => null);
53
+ wakeInternals.resolveSecret = (() => 'relay-token');
54
+ wakeInternals.loadM0Config = (() => ({}));
55
+ wakeInternals.loadControlPlaneConfig = (() => ({ readyTimeoutMs: 1_000 }));
56
+ wakeInternals.probeNodeReady = (async () => true);
57
+ wakeInternals.setHomeStatus = (() => home);
58
+ wakeInternals.setHealthVerdict = ((_id, verdict) => {
59
+ healthVerdict = verdict;
60
+ return home;
61
+ });
62
+ wakeInternals.touchWake = (() => home);
63
+ wakeInternals.touchReady = (() => home);
64
+ wakeInternals.getHomeBackend = (() => ({
65
+ wake: async () => ({
66
+ providerName: 'blaxel',
67
+ providerSandboxId: home.provider_sandbox_id,
68
+ providerVolumeId: 'vol-home-1',
69
+ previewEndpoint: home.preview_endpoint,
70
+ route: null,
71
+ homeAgentTarget: 'node-1',
72
+ relayAuthRef: null,
73
+ templateVersion: home.template_version,
74
+ status: 'running',
75
+ machine: null,
76
+ updatedAt: new Date().toISOString(),
77
+ }),
78
+ recreateOnImage: async () => {
79
+ throw new Error('recreateOnImage must not be called on the plain-wake path');
80
+ },
81
+ suspend: async () => {
82
+ throw new Error('unused');
83
+ },
84
+ getMachine: async () => null,
85
+ exec: async () => {
86
+ throw new Error('exec must not be called on the plain-wake path');
87
+ },
88
+ }));
89
+ try {
90
+ const result = await ensureAwake(home.home_id);
91
+ assert.equal(result.preview_endpoint, home.preview_endpoint);
92
+ assert.equal(healthVerdict, 'healthy', 'a successful plain wake must clear a stale degraded verdict to healthy');
93
+ }
94
+ finally {
95
+ Object.assign(wakeInternals, orig);
96
+ resetWakeCacheForTests();
97
+ }
98
+ });
99
+ test('rollHome asserts guest version against the RECREATED sandbox name, not the stale pre-roll name', async () => {
100
+ const { ensureAwake, wakeInternals, resetWakeCacheForTests } = await import('../wake.js');
101
+ const orig = { ...wakeInternals };
102
+ resetWakeCacheForTests();
103
+ const execCalls = [];
104
+ const home = fixtureHome({ home_id: 'home-roll-ok', provider_sandbox_id: 'hearth-home-roll-ok' });
105
+ const target = fixtureTarget();
106
+ const freshName = 'hearth-home-roll-ok-gfresh1';
107
+ wakeInternals.getHomeById = ((id) => (id === home.home_id ? home : null));
108
+ wakeInternals.getHearthTarget = (() => target);
109
+ wakeInternals.resolveSecret = (() => 'relay-token');
110
+ wakeInternals.loadM0Config = (() => ({}));
111
+ wakeInternals.loadControlPlaneConfig = (() => ({ readyTimeoutMs: 1_000 }));
112
+ wakeInternals.probeNodeReady = (async () => true);
113
+ wakeInternals.refreshProviderPointers = ((_id, patch) => ({ ...home, ...patch }));
114
+ wakeInternals.setHomeStatus = (() => home);
115
+ wakeInternals.setHealthVerdict = (() => home);
116
+ wakeInternals.touchWake = (() => home);
117
+ wakeInternals.touchReady = (() => home);
118
+ wakeInternals.recordBadTarget = (() => null);
119
+ wakeInternals.getHomeBackend = (() => ({
120
+ wake: async () => {
121
+ throw new Error('wake() must not be called on the roll path');
122
+ },
123
+ recreateOnImage: async () => ({
124
+ providerName: 'blaxel',
125
+ providerSandboxId: freshName,
126
+ providerVolumeId: 'vol-home-1',
127
+ previewEndpoint: 'https://new.example',
128
+ route: null,
129
+ homeAgentTarget: 'node-1',
130
+ relayAuthRef: null,
131
+ templateVersion: target.target_template_version,
132
+ status: 'running',
133
+ machine: null,
134
+ updatedAt: new Date().toISOString(),
135
+ }),
136
+ suspend: async () => {
137
+ throw new Error('unused');
138
+ },
139
+ getMachine: async () => null,
140
+ exec: async (machineId) => {
141
+ execCalls.push(machineId);
142
+ return { exitCode: 0, timedOut: false, stdout: JSON.stringify({ version: target.target_template_version }), stderr: '' };
143
+ },
144
+ }));
145
+ try {
146
+ const result = await ensureAwake(home.home_id);
147
+ assert.equal(result.preview_endpoint, 'https://new.example');
148
+ assert.deepEqual(execCalls, [freshName], 'assertGuestVersion must exec into the RECREATED sandbox name, never the stale pre-roll name');
149
+ }
150
+ finally {
151
+ Object.assign(wakeInternals, orig);
152
+ resetWakeCacheForTests();
153
+ }
154
+ });
155
+ test('rollbackHome asserts guest version against the RECREATED rollback sandbox name, and rollback success sets healthy with no incident', async () => {
156
+ const { ensureAwake, wakeInternals, resetWakeCacheForTests } = await import('../wake.js');
157
+ const orig = { ...wakeInternals };
158
+ resetWakeCacheForTests();
159
+ const execCalls = [];
160
+ const home = fixtureHome({ home_id: 'home-roll-fail', provider_sandbox_id: 'hearth-home-roll-fail' });
161
+ const target = fixtureTarget();
162
+ const rollbackFreshName = 'hearth-home-roll-fail-gback1';
163
+ let recreateCalls = 0;
164
+ let badTargetRecorded = null;
165
+ let healthVerdict = null;
166
+ wakeInternals.getHomeById = ((id) => (id === home.home_id ? home : null));
167
+ wakeInternals.getHearthTarget = (() => target);
168
+ wakeInternals.resolveSecret = (() => 'relay-token');
169
+ wakeInternals.loadM0Config = (() => ({}));
170
+ wakeInternals.loadControlPlaneConfig = (() => ({ readyTimeoutMs: 1_000 }));
171
+ wakeInternals.probeNodeReady = (async () => true);
172
+ wakeInternals.refreshProviderPointers = ((_id, patch) => ({ ...home, ...patch }));
173
+ wakeInternals.setHomeStatus = (() => home);
174
+ wakeInternals.setHealthVerdict = ((_id, verdict) => {
175
+ healthVerdict = verdict;
176
+ return home;
177
+ });
178
+ wakeInternals.touchWake = (() => home);
179
+ wakeInternals.touchReady = (() => home);
180
+ wakeInternals.recordBadTarget = ((version) => {
181
+ badTargetRecorded = version;
182
+ return null;
183
+ });
184
+ wakeInternals.getHomeBackend = (() => ({
185
+ wake: async () => {
186
+ throw new Error('unused');
187
+ },
188
+ recreateOnImage: async (_descriptor, _imageRef, templateVersion) => {
189
+ recreateCalls += 1;
190
+ if (recreateCalls === 1) {
191
+ // Simulate the roll-forward recreate succeeding but the guest version assert
192
+ // failing downstream — rollHome's catch delegates to rollbackHome regardless of
193
+ // WHERE in the try it failed.
194
+ throw new Error('simulated roll failure (version mismatch)');
195
+ }
196
+ return {
197
+ providerName: 'blaxel',
198
+ providerSandboxId: rollbackFreshName,
199
+ providerVolumeId: 'vol-home-1',
200
+ previewEndpoint: 'https://rollback.example',
201
+ route: null,
202
+ homeAgentTarget: 'node-1',
203
+ relayAuthRef: null,
204
+ templateVersion,
205
+ status: 'running',
206
+ machine: null,
207
+ updatedAt: new Date().toISOString(),
208
+ };
209
+ },
210
+ suspend: async () => {
211
+ throw new Error('unused');
212
+ },
213
+ getMachine: async () => null,
214
+ exec: async (machineId) => {
215
+ execCalls.push(machineId);
216
+ return { exitCode: 0, timedOut: false, stdout: JSON.stringify({ version: home.template_version }), stderr: '' };
217
+ },
218
+ }));
219
+ try {
220
+ const result = await ensureAwake(home.home_id);
221
+ assert.equal(result.preview_endpoint, 'https://rollback.example');
222
+ assert.equal(badTargetRecorded, target.target_template_version, 'the roll failure must still poison the target (brake unconditional on roll failure)');
223
+ assert.equal(healthVerdict, 'healthy', 'a successful rollback must clear health to healthy, not leave/raise an incident');
224
+ assert.deepEqual(execCalls, [rollbackFreshName], "rollback must assert against ITS OWN recreated sandbox name — never the failed roll's name nor the stale pre-roll name");
225
+ }
226
+ finally {
227
+ Object.assign(wakeInternals, orig);
228
+ resetWakeCacheForTests();
229
+ }
230
+ });
@@ -457,6 +457,32 @@ BEGIN
457
457
  END;
458
458
  `);
459
459
  }
460
+ /** v9 — the single global auto-upgrade target (design.md §2/§9). ONE row,
461
+ * `channel = 'home'`, holding the DESIRED crtr image line every home rolls
462
+ * onto: `target_template_version` is the semver the fleet should be running,
463
+ * `target_image_ref` its immutable Blaxel image name (must equal
464
+ * `hearthHomeImageRefForVersion(target_template_version)` — the admin route
465
+ * enforces that). `last_bad_target_version` is the ONLY brake in the
466
+ * zero-touch loop: a version recorded here already failed a live roll for
467
+ * the fleet, so the wake-path stale predicate excludes it until the target
468
+ * advances to a newer release. There is deliberately NO `roll_armed` column
469
+ * — the loop is full zero-touch (§9.1), so there is no per-release human arm
470
+ * gate to persist. `home.template_version` stays the ACTUAL running version;
471
+ * this row is the target. Single-tenant: one image line for all homes, so no
472
+ * per-home target column. */
473
+ function hearthTargetTable(db) {
474
+ db.exec(`
475
+ CREATE TABLE IF NOT EXISTS hearth_target (
476
+ channel TEXT PRIMARY KEY DEFAULT 'home',
477
+ target_template_version TEXT NOT NULL,
478
+ target_image_ref TEXT NOT NULL,
479
+ published_at TEXT NOT NULL,
480
+ source TEXT NOT NULL,
481
+ last_bad_target_version TEXT,
482
+ CHECK (channel = 'home')
483
+ );
484
+ `);
485
+ }
460
486
  /** The ordered migration list. Index `i` is migration version `i + 1`; the
461
487
  * db's `user_version` tracks how many have been applied. Append only. */
462
488
  export const MIGRATIONS = [
@@ -468,6 +494,7 @@ export const MIGRATIONS = [
468
494
  /* v6 */ triggerAuditTable,
469
495
  /* v7 */ tenantIdentityTriggers,
470
496
  /* v8 */ parentIdentityUpdateGuards,
497
+ /* v9 */ hearthTargetTable,
471
498
  ];
472
499
  /** Bring `db` up to the latest schema version. Reads `user_version`, runs
473
500
  * each pending migration in order, and bumps `user_version` after each so
@@ -0,0 +1,23 @@
1
+ import { DatabaseSync } from 'node:sqlite';
2
+ import type { HearthTarget, SetHearthTargetInput } from './types.js';
3
+ /** Read the single global target row, or `null` when no image has ever been
4
+ * registered (fresh CP). A `null` means the wake path treats every home as
5
+ * up to date (nothing to roll onto yet). */
6
+ export declare function getHearthTarget(db?: DatabaseSync): HearthTarget | null;
7
+ /** Upsert the global target to a newly-registered image. IDEMPOTENT: a
8
+ * re-post of the version already recorded is a no-op (the row, including its
9
+ * `published_at` and any `last_bad_target_version` brake, is left exactly as
10
+ * is) — the image workflow can fire the notify more than once for one
11
+ * release without churning state. Advancing to a DIFFERENT version rewrites
12
+ * the target/image/published_at/source but deliberately preserves
13
+ * `last_bad_target_version` (a brake recorded for an older release is
14
+ * historical and harmless once the target has moved past it; the wake
15
+ * predicate keys off `target !== last_bad`). */
16
+ export declare function setHearthTarget(input: SetHearthTargetInput, db?: DatabaseSync): HearthTarget;
17
+ /** Record a version whose live roll failed for the fleet (§5.4). After this,
18
+ * the wake-path stale predicate excludes the current target (target ===
19
+ * last_bad_target_version → no home rolls) until a NEW release advances the
20
+ * target past it. This is the ONLY brake in the zero-touch loop (roadmap
21
+ * decision 1: never auto-advance onto a release that already failed a roll).
22
+ * Returns `null` if no target row exists (nothing to brake). */
23
+ export declare function recordBadTarget(badVersion: string, db?: DatabaseSync): HearthTarget | null;
@@ -0,0 +1,68 @@
1
+ // control-plane/hearth-target.ts — the CP store for the single global
2
+ // auto-upgrade target row (`hearth_target`, db.ts v9). One row, `channel =
3
+ // 'home'`. This module is the ONLY reader/writer of that row; the admin
4
+ // notify route (server.ts) upserts it, the wake path (wake.ts) reads it to
5
+ // decide whether a home is stale, and the rollback path records a bad target
6
+ // through it. It goes through the shared CP db handle (`getControlPlaneDb`)
7
+ // exactly like `registry.ts` — never a second connection.
8
+ import { getControlPlaneDb } from './db.js';
9
+ const TARGET_COLUMNS = [
10
+ 'channel',
11
+ 'target_template_version',
12
+ 'target_image_ref',
13
+ 'published_at',
14
+ 'source',
15
+ 'last_bad_target_version',
16
+ ];
17
+ const SELECT_TARGET = `SELECT ${TARGET_COLUMNS.join(', ')} FROM hearth_target WHERE channel = 'home'`;
18
+ function nowIso() {
19
+ return new Date().toISOString();
20
+ }
21
+ /** Read the single global target row, or `null` when no image has ever been
22
+ * registered (fresh CP). A `null` means the wake path treats every home as
23
+ * up to date (nothing to roll onto yet). */
24
+ export function getHearthTarget(db = getControlPlaneDb()) {
25
+ const row = db.prepare(SELECT_TARGET).get();
26
+ if (row === undefined || row === null)
27
+ return null;
28
+ return row;
29
+ }
30
+ /** Upsert the global target to a newly-registered image. IDEMPOTENT: a
31
+ * re-post of the version already recorded is a no-op (the row, including its
32
+ * `published_at` and any `last_bad_target_version` brake, is left exactly as
33
+ * is) — the image workflow can fire the notify more than once for one
34
+ * release without churning state. Advancing to a DIFFERENT version rewrites
35
+ * the target/image/published_at/source but deliberately preserves
36
+ * `last_bad_target_version` (a brake recorded for an older release is
37
+ * historical and harmless once the target has moved past it; the wake
38
+ * predicate keys off `target !== last_bad`). */
39
+ export function setHearthTarget(input, db = getControlPlaneDb()) {
40
+ const existing = getHearthTarget(db);
41
+ if (existing !== null && existing.target_template_version === input.target_template_version) {
42
+ return existing;
43
+ }
44
+ const published_at = input.published_at ?? nowIso();
45
+ db.prepare(`
46
+ INSERT INTO hearth_target (channel, target_template_version, target_image_ref, published_at, source, last_bad_target_version)
47
+ VALUES ('home', ?, ?, ?, ?, NULL)
48
+ ON CONFLICT(channel) DO UPDATE SET
49
+ target_template_version = excluded.target_template_version,
50
+ target_image_ref = excluded.target_image_ref,
51
+ published_at = excluded.published_at,
52
+ source = excluded.source
53
+ `).run(input.target_template_version, input.target_image_ref, published_at, input.source);
54
+ return getHearthTarget(db);
55
+ }
56
+ /** Record a version whose live roll failed for the fleet (§5.4). After this,
57
+ * the wake-path stale predicate excludes the current target (target ===
58
+ * last_bad_target_version → no home rolls) until a NEW release advances the
59
+ * target past it. This is the ONLY brake in the zero-touch loop (roadmap
60
+ * decision 1: never auto-advance onto a release that already failed a roll).
61
+ * Returns `null` if no target row exists (nothing to brake). */
62
+ export function recordBadTarget(badVersion, db = getControlPlaneDb()) {
63
+ const existing = getHearthTarget(db);
64
+ if (existing === null)
65
+ return null;
66
+ db.prepare(`UPDATE hearth_target SET last_bad_target_version = ? WHERE channel = 'home'`).run(badVersion);
67
+ return getHearthTarget(db);
68
+ }
@@ -1,5 +1,5 @@
1
1
  import { DatabaseSync } from 'node:sqlite';
2
- import type { CreateHomeInput, Home, HomeId, HomePatch, HomeStatus, RefreshProviderPointersInput, ResolvedHome, TenantSession, TouchWakeInput } from './types.js';
2
+ import type { CreateHomeInput, Home, HomeHealthVerdict, HomeId, HomePatch, HomeStatus, RefreshProviderPointersInput, ResolvedHome, TenantSession, TouchWakeInput } from './types.js';
3
3
  export declare function createHome(input: CreateHomeInput, db?: DatabaseSync): Home;
4
4
  export declare function getHomeById(home_id: HomeId | null | undefined, db?: DatabaseSync): Home | null;
5
5
  /** Resolve a tenant session to its registry row. A null return is a hard
@@ -12,6 +12,11 @@ export declare function getHomeBySandbox(provider_sandbox_id: string | null | un
12
12
  export declare function updateHome(home_id: HomeId, mutate: (current: Home) => HomePatch, db?: DatabaseSync): Home;
13
13
  export declare function refreshProviderPointers(home_id: HomeId, pointers: RefreshProviderPointersInput, db?: DatabaseSync): Home;
14
14
  export declare function setHomeStatus(home_id: HomeId, status: HomeStatus, db?: DatabaseSync): Home;
15
+ /** Set the home's health verdict. The auto-upgrade roll (wake.ts `rollHome`)
16
+ * is the first natural producer of `health_verdict`: a completed roll/
17
+ * rollback writes `'healthy'`, a roll AND rollback both failing writes
18
+ * `'degraded'` (design §4/§5). */
19
+ export declare function setHealthVerdict(home_id: HomeId, health_verdict: HomeHealthVerdict, db?: DatabaseSync): Home;
15
20
  export declare function touchWake(home_id: HomeId, input?: TouchWakeInput, db?: DatabaseSync): Home;
16
21
  export declare function touchReady(home_id: HomeId, at?: string, db?: DatabaseSync): Home;
17
22
  export declare function touchSuspend(home_id: HomeId, at?: string, db?: DatabaseSync): Home;