@crouton-kit/crouter 0.3.38 → 0.3.40

package/dist/core/hearth/providers/blaxel-home.js

@@ -1,10 +1,93 @@
+import { setTimeout as delay } from 'node:timers/promises';
 import { general } from '../../errors.js';
 import { nowIso } from '../../fs-utils.js';
 import { guestEnv } from '../guest-env.js';
 import { buildBootstrapScript } from './blaxel-bootstrap.js';
 import { BlaxelMachineProvider } from './blaxel.js';
-function stableRouteName(home) {
-    return `${home.providerSandboxId}-web`;
+// recreate-race-fix.md §3.2 — a volume GET is one cheap API call; 1s resolution is plenty
+// against an expected seconds-scale detach.
+const VOLUME_DETACH_POLL_MS = 1_000;
+// Expected wait is seconds (observed live); 120s is ~20-60x headroom over that, so this still
+// fails deterministically inside one wake's lifetime rather than hanging forever on a genuine
+// platform-side stuck detach.
+const VOLUME_DETACH_TIMEOUT_MS = 120_000;
+// orphan-cleanup-fix.md §4 — pause before the single bounded retry of a create that failed on a
+// server-side deployment transient, to let the platform-side condition clear.
+const CREATE_RETRY_DELAY_MS = 5_000;
+function stableRouteName(sandboxId) {
+    return `${sandboxId}-web`;
+}
+/** Mint a fresh per-generation sandbox name. `providerVolumeId` is the home's stable base
+ *  (never suffixed), so names stay bounded: `hearth-zt-test-1-gmchq3x2a`. Sequential recreates
+ *  are seconds apart, so a ms timestamp cannot collide for one home (recreate-race-fix.md §3.2). */
+function mintSandboxName(providerVolumeId) {
+    return `${providerVolumeId}-g${Date.now().toString(36)}`;
+}
+/** Escape a volumeId for safe use inside a RegExp so the generation-family anchor is exact —
+ *  a home base name is user-influenced, so a literal `.` or `+` in it must not become a
+ *  wildcard (orphan-cleanup-fix.md §3.1). */
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/** Predicate matching every generation of a home's name family: the stable base itself, or a
+ *  `<base>-g<suffix>` generation. The `-g` anchor is load-bearing — it prevents `<base>0-g…`
+ *  (e.g. `hearth-zt-test-10-g…`) from matching base `hearth-zt-test-1` (orphan-cleanup-fix.md
+ *  §3.1). Suffix chars are base36 (lowercase alnum), matching `mintSandboxName`. */
+function generationFamilyMatcher(volumeId) {
+    const gen = new RegExp(`^${escapeRegExp(volumeId)}-g[0-9a-z]+$`);
+    return (name) => name === volumeId || gen.test(name);
+}
+/** Serialize an arbitrary rejection to a non-empty marker string. `JSON.stringify` RETURNS
+ *  `undefined` (does not throw) for `undefined`, a function, or a symbol — falling through to
+ *  that would omit the `error` field from the marker's JSON line entirely, so fall back to
+ *  `String(...)` whenever it yields `undefined` (matches wake.ts/relay.ts `serializeError`). */
+function serialize(error) {
+    if (error instanceof Error)
+        return error.message;
+    try {
+        const json = JSON.stringify(error);
+        return json === undefined ? String(error) : json;
+    }
+    catch {
+        return String(error);
+    }
+}
+/** Emit one structured stderr line for a recreate-lifecycle marker (sweep / volume_wait /
+ *  cleanup_failed / create_retry). Single writer for all four so the JSON shape stays uniform
+ *  and ops log-alerting keys on `marker` (orphan-cleanup-fix.md §3.3/§3.4/§4). */
+function emitMarker(marker, fields, level = 'info') {
+    process.stderr.write(`${JSON.stringify({ level, marker, ...fields })}\n`);
+}
+/** Classify a failed create+deploy as a retryable server-side deployment transient
+ *  (orphan-cleanup-fix.md §4). Retryable IFF `status_code >= 500` OR `code === 'DEPLOYMENT_FAILED'`
+ *  on the rejection object, with a message-substring fallback for Error-wrapped variants whose
+ *  structured fields were flattened into `.message`. NEVER on 4xx — a 409 here is a real name
+ *  conflict the design must surface, not absorb, so an explicit 4xx status vetoes the retry even
+ *  if a `DEPLOYMENT_FAILED` code is also present. */
+function isRetryableDeploymentError(error) {
+    if (error !== null && typeof error === 'object') {
+        // Only the canonical HTTP-status keys — NOT a bare `status`, which is an overloaded field
+        // name some envelopes use for a non-HTTP state enum/count (MINOR-2).
+        const obj = error;
+        const status = [obj.status_code, obj.statusCode].find((v) => typeof v === 'number');
+        if (typeof status === 'number') {
+            if (status >= 400 && status < 500)
+                return false; // 4xx: real conflict, never absorb
+            if (status >= 500)
+                return true;
+        }
+        if (obj.code === 'DEPLOYMENT_FAILED')
+            return true;
+    }
+    // Fallback for Error-wrapped variants: only POSITIVE server-side signals, never inferring a 4xx.
+    const message = error instanceof Error ? error.message : typeof error === 'string' ? error : '';
+    if (message !== '') {
+        if (/DEPLOYMENT_FAILED/.test(message))
+            return true;
+        if (/\bstatus[_ ]?code\b\D{0,8}5\d\d/i.test(message))
+            return true;
+    }
+    return false;
 }
 function ensureReadyHome(input) {
     if (input.providerName !== 'blaxel') {
@@ -30,10 +113,10 @@ function isRunningMachine(machine) {
 function previewEndpoint(route, fallback) {
     return route.concreteUrl ?? route.url ?? fallback ?? null;
 }
-function refreshedPointers(home, provider, route, status, machine) {
+function refreshedPointers(home, provider, route, status, sandboxId, machine) {
     return {
         providerName: 'blaxel',
-        providerSandboxId: home.providerSandboxId,
+        providerSandboxId: sandboxId,
         providerVolumeId: home.providerVolumeId,
         previewEndpoint: route === null ? (home.previewEndpoint ?? null) : previewEndpoint(route, home.previewEndpoint),
         route,
@@ -80,25 +163,211 @@ export class BlaxelHomeBackend {
                 await this.provider.exec(current.providerSandboxId, { command: bootstrapScript, background: true, cwd: this.config.guestHome });
                 await this.provider.waitForPort(current.providerSandboxId, this.config.webPort, { maxWaitMs: 30_000 });
             }
-        }
-        else {
-            await this.provider.createMachine({
-                tenantId: current.tenantId,
-                name: current.providerSandboxId,
-                envs: guestEnv(this.config),
-                volumeMount: { volumeName: current.providerVolumeId, path: this.config.guestHome },
-                ports: [{ target: this.config.webPort, protocol: 'HTTP' }],
+            const route = await this.provider.publishHttpPort(current.providerSandboxId, {
+                machineId: current.providerSandboxId,
+                targetPort: this.config.webPort,
+                routeName: stableRouteName(current.providerSandboxId),
+                routeId: stableRouteName(current.providerSandboxId),
             });
-            await this.provider.exec(current.providerSandboxId, { command: bootstrapScript, background: true, cwd: this.config.guestHome });
-            await this.provider.waitForPort(current.providerSandboxId, this.config.webPort, { maxWaitMs: 60_000 });
+            return refreshedPointers(current, this.provider, route, 'running', current.providerSandboxId, await this.getMachine(current.providerSandboxId));
+        }
+        // Terminated-machine branch (recreate-race-fix.md §3.4, orphan-cleanup-fix.md §3.3/§3.4):
+        // the crash-recovery path for a sandbox deleted recently out-of-band OR by a crashed/failed
+        // roll (a Fly redeploy restarting the CP mid-roll lands here on the next wake).
+        // `sweepAndReleaseVolume` destroys EVERY generation in this home's name family — including
+        // an unpointed leftover a crashed roll left holding the volume, which the old point-in-time
+        // attachedTo read could miss once Blaxel autonomously detached it on idle→STANDBY — then
+        // polls until the volume is confirmed free before minting a fresh name and creating.
+        await this.sweepAndReleaseVolume(current.providerVolumeId, current.homeId);
+        const { sandboxName, route, machine: created } = await this.createGeneration({
+            homeId: current.homeId,
+            tenantId: current.tenantId,
+            volumeId: current.providerVolumeId,
+            bootstrapScript,
+        });
+        return refreshedPointers(current, this.provider, route, 'running', sandboxName, created);
+    }
+    /** Destroy EVERY generation of this home via a deterministic name-family sweep — never a
+     *  point-in-time `attachedTo` read, which Blaxel's autonomous idle→STANDBY detach can race
+     *  ahead of, leaving a live orphan the read cannot see (orphan-cleanup-fix.md §1/§3.3). The
+     *  destroy set is intentionally TAG-SCOPED (externalId = volumeId) and is never widened by a
+     *  name-family fallback (MAJOR-1) — so after destroying it, read the volume's current holder
+     *  ONCE: if the volume is still attached to a sandbox that is NOT one we just destroyed, that
+     *  holder is outside the strict sweep's reach (an untagged holder, or a foreign claim) and will
+     *  never detach on its own, so fail fast with an actionable error instead of silently spinning
+     *  the detach poll to its timeout. Otherwise poll
+     *  until the platform reports the volume detached. Safe because at sweep time the replacement
+     *  does not exist yet (its name is minted after) and any live family member is by construction
+     *  either the generation being replaced or an uncommitted leftover — the home's durable state
+     *  is the volume, never a sandbox; one wake per home runs at a time. */
+    async sweepAndReleaseVolume(volumeId, homeId) {
+        // Scope the list to sandboxes Hearth tagged for THIS home (externalId = volumeId), then keep
+        // the name-family matcher as an AND-intersection on top (orphan-cleanup-fix MAJOR-1). The
+        // destroy set requires BOTH keys to agree — the tag says "Hearth created this for home X",
+        // the name says "it is in X's generation family" — so a single fault (a mistagged create, a
+        // future API silently ignoring the filter param) cannot widen the blast radius back onto a
+        // foreign home. Fail-safe defense-in-depth on a destructive op, NOT a lenient fallback.
+        const inFamily = generationFamilyMatcher(volumeId);
+        const doomed = (await this.provider.listMachines({ externalId: volumeId })).map((m) => m.machineId).filter(inFamily);
+        for (const name of doomed)
+            await this.provider.destroy(name); // sequential; destroy tolerates not_found
+        if (doomed.length > 0)
+            emitMarker('hearth.recreate.sweep', { home_id: homeId, volume_id: volumeId, destroyed: doomed });
+        // Loud guard: the sweep above is strictly tag-scoped, so it can only ever destroy a sandbox
+        // that was created with `externalId = volumeId`. Read the volume's current holder ONCE,
+        // before the detach poll, to catch the case that sweep is structurally unable to fix — a
+        // holder that is NOT one of the sandboxes just destroyed (`doomed`) will never detach, and
+        // the poll below would spin the full VOLUME_DETACH_TIMEOUT_MS only to throw the same thing
+        // less clearly. Fail fast instead, naming the holder so an operator can act on it directly.
+        const holder = await this.provider.getVolumeAttachment(volumeId);
+        if (holder !== null) {
+            const holderName = holder.startsWith('sandbox:') ? holder.slice('sandbox:'.length) : holder;
+            if (!doomed.includes(holderName)) {
+                throw general(`legacy untagged sandbox ${holderName} holds volume ${volumeId} for home ${homeId} — the tag-scoped sweep cannot reclaim it (no externalId tag); migrate this home (delete the sandbox so a plain wake recreates it tagged) before rolling`, { homeId, volumeId, holder: holderName });
+            }
         }
-        const route = await this.provider.publishHttpPort(current.providerSandboxId, {
-            machineId: current.providerSandboxId,
-            targetPort: this.config.webPort,
-            routeName: stableRouteName(current),
-            routeId: stableRouteName(current),
+        await this.pollVolumeDetached(volumeId, homeId);
+    }
+    /** Poll the volume until the platform reports it detached — the create-precondition gate
+     *  (recreate-race-fix.md §3.2). Reads only; never destroys. Emits `hearth.recreate.volume_wait`
+     *  whenever it actually had to wait, so the re-proof can quantify headroom against the budget. */
+    async pollVolumeDetached(volumeId, homeId) {
+        const pollStartedAt = Date.now();
+        const deadline = pollStartedAt + VOLUME_DETACH_TIMEOUT_MS;
+        let waited = false;
+        for (;;) {
+            const attached = await this.provider.getVolumeAttachment(volumeId);
+            if (attached === null)
+                break;
+            waited = true;
+            if (Date.now() >= deadline) {
+                throw general(`volume ${volumeId} still attached to ${attached} after ${VOLUME_DETACH_TIMEOUT_MS}ms — cannot recreate home ${homeId}`, {
+                    homeId,
+                    volumeId,
+                    attached,
+                });
+            }
+            await delay(VOLUME_DETACH_POLL_MS);
+        }
+        if (waited) {
+            emitMarker('hearth.recreate.volume_wait', { home_id: homeId, volume_id: volumeId, waited_ms: Date.now() - pollStartedAt });
+        }
+    }
+    /** Create one fresh generation and bring it to a serving state (create → bootstrap exec →
+     *  waitForPort → publish → getMachine), with provider self-cleanup and a single bounded
+     *  retry (orphan-cleanup-fix.md §3.4/§4). Each attempt destroys the generation IT created if
+     *  anything from `createMachine` through the final `getMachine` fails — no code path exits
+     *  leaving a live generation it created and did not return — then rethrows the ORIGINAL error
+     *  (cleanup never masks it). On the first failure, if the error is a genuine server-side
+     *  deployment transient, pause, re-run ONLY the volume-free poll (the sweep already ran once),
+     *  mint a NEW name, and retry exactly once; a second failure throws. The sweep is the caller's
+     *  responsibility and runs before the first attempt only. */
+    async createGeneration(args) {
+        const attempt = async (sandboxName) => {
+            try {
+                await this.provider.createMachine({
+                    tenantId: args.tenantId,
+                    name: sandboxName,
+                    // Tag EVERY generation (first-name and retry-name alike) with the home's stable volume
+                    // id so the sweep's filtered list only ever sees this home's family (MAJOR-1).
+                    externalId: args.volumeId,
+                    ...(args.image === undefined ? {} : { image: args.image }),
+                    envs: guestEnv(this.config),
+                    volumeMount: { volumeName: args.volumeId, path: this.config.guestHome },
+                    ports: [{ target: this.config.webPort, protocol: 'HTTP' }],
+                });
+                await this.provider.exec(sandboxName, { command: args.bootstrapScript, background: true, cwd: this.config.guestHome });
+                await this.provider.waitForPort(sandboxName, this.config.webPort, { maxWaitMs: 60_000 });
+                const route = await this.provider.publishHttpPort(sandboxName, {
+                    machineId: sandboxName,
+                    targetPort: this.config.webPort,
+                    routeName: stableRouteName(sandboxName),
+                    routeId: stableRouteName(sandboxName),
+                });
+                return { route, machine: await this.getMachine(sandboxName) };
+            }
+            catch (error) {
+                // §3.4 self-cleanup: destroy the generation this attempt created; never mask the original.
+                await this.provider
+                    .destroy(sandboxName)
+                    .catch((cleanupErr) => emitMarker('hearth.recreate.cleanup_failed', { home_id: args.homeId, sandbox: sandboxName, error: serialize(cleanupErr) }, 'warn'));
+                throw error;
+            }
+        };
+        const firstName = mintSandboxName(args.volumeId);
+        try {
+            return { sandboxName: firstName, ...(await attempt(firstName)) };
+        }
+        catch (error) {
+            if (!isRetryableDeploymentError(error))
+                throw error;
+            emitMarker('hearth.recreate.create_retry', { home_id: args.homeId, discarded_generation: firstName, error: serialize(error) }, 'warn');
+            await delay(CREATE_RETRY_DELAY_MS);
+            // Re-run ONLY the poll — NOT a re-sweep. This sits outside attempt()'s try/catch, so a
+            // poll-timeout here would otherwise mask the ORIGINAL deployment error (MINOR-3). The
+            // original is what the caller must see (it is already recorded in the create_retry
+            // marker above); on a poll failure during the retry path, rethrow it, not the timeout.
+            try {
+                await this.pollVolumeDetached(args.volumeId, args.homeId);
+            }
+            catch {
+                throw error;
+            }
+            const secondName = mintSandboxName(args.volumeId);
+            return { sandboxName: secondName, ...(await attempt(secondName)) }; // second failure throws (self-cleaned)
+        }
+    }
+    /** Targeted destroy→recreate onto a SPECIFIC image (auto-upgrade roll and
+     *  rollback, design §4/§5). Unlike `wake()`'s standby-resume path, this is the ONE path
+     *  that force-destroys the home's sandboxes before recreating. Cleanup is a deterministic
+     *  name-family sweep (orphan-cleanup-fix.md §3.3): every generation of this home is destroyed
+     *  before the fresh one is minted, so a failed roll-forward generation can never survive as a
+     *  live orphan even after Blaxel autonomously detaches it from the volume. The recreate lands
+     *  on a FRESH sandbox name every call — a name that has never existed cannot 409 against a
+     *  lingering same-name delete. The durable volume is a separate resource and is NEVER touched
+     *  (flow-D durability) — user state is byte-identical across the flip, which is exactly what
+     *  makes both the upgrade and the rollback lossless. The new `providerSandboxId` is returned on
+     *  the refresh; callers persist it (`refreshProviderPointers` treats it as a refreshable
+     *  pointer). `templateVersion` is baked into the bootstrap's fail-fast version assert (the
+     *  baked crtr must match) AND reported back so the CP records the target version — the caller
+     *  still independently asserts `crtr sys version` in-guest before trusting it (wake.ts
+     *  `rollHome`/`rollbackHome`, against `refresh.providerSandboxId`, never a stale captured
+     *  name). */
+    async recreateOnImage(homeInput, imageRef, templateVersion) {
+        const current = ensureReadyHome(homeInput);
+        if (imageRef.trim() === '') {
+            throw general(`recreateOnImage for home ${current.homeId} requires a non-empty image ref`, { homeId: current.homeId });
+        }
+        if (templateVersion.trim() === '') {
+            throw general(`recreateOnImage for home ${current.homeId} requires a non-empty template version`, { homeId: current.homeId });
+        }
+        // Sweep the whole name family (destroys the failed roll-forward generation whether or not it
+        // still holds the volume — the exact Run-3 orphan gap) then wait until the volume is confirmed
+        // detached before minting a fresh name and creating (orphan-cleanup-fix.md §3.3).
+        await this.sweepAndReleaseVolume(current.providerVolumeId, current.homeId);
+        const bootstrapScript = buildBootstrapScript({
+            guestHome: this.config.guestHome,
+            crtrVersion: templateVersion,
+            relayToken: current.relayToken,
+            webPort: this.config.webPort,
+            nodeId: current.homeAgentTarget,
+            piAuthJson: this.config.piAuthJson,
+            piPackages: this.config.piPackages,
+            isFirstProvision: false,
+        });
+        const { sandboxName, route, machine } = await this.createGeneration({
+            homeId: current.homeId,
+            tenantId: current.tenantId,
+            volumeId: current.providerVolumeId,
+            bootstrapScript,
+            image: imageRef,
         });
-        return refreshedPointers(current, this.provider, route, 'running', await this.getMachine(current.providerSandboxId));
+        // Report the TARGET version, not the provider's construction-time value:
+        // the sandbox was recreated on `imageRef`, whose baked crtr is `templateVersion`.
+        return {
+            ...refreshedPointers(current, this.provider, route, 'running', sandboxName, machine),
+            templateVersion,
+        };
     }
     async suspend(homeInput) {
         const current = ensureReadyHome(homeInput);
@@ -121,7 +390,7 @@ export class BlaxelHomeBackend {
                 providerSandboxId: current.providerSandboxId,
             });
         }
-        return refreshedPointers(current, this.provider, null, machine.state, machine);
+        return refreshedPointers(current, this.provider, null, machine.state, current.providerSandboxId, machine);
     }
     async getMachine(machineId) {
         const machine = await this.provider.getMachine(machineId);

package/dist/core/hearth/providers/blaxel.d.ts

@@ -7,6 +7,7 @@ interface BlaxelMachineProviderOptions {
     templateVersion?: string;
     volumeSizeGb?: number;
 }
+export declare function isMissingResourceError(error: unknown): boolean;
 export declare class BlaxelMachineProvider implements MachineProvider {
     readonly providerName: "blaxel";
     readonly imageRef: string;
@@ -18,6 +19,9 @@ export declare class BlaxelMachineProvider implements MachineProvider {
         name: string;
     }): Promise<VolumeRef>;
     createMachine(input: CreateMachineInput): Promise<MachineRef>;
+    listMachines(filter?: {
+        externalId?: string;
+    }): Promise<MachineRef[]>;
     getMachine(machineId: string): Promise<MachineRef>;
     exec(machineId: string, input: ExecInput): Promise<ExecResult>;
     waitForPort(machineId: string, port: number, opts?: {
@@ -27,5 +31,6 @@ export declare class BlaxelMachineProvider implements MachineProvider {
     health(): Promise<HealthResult>;
     destroy(machineId: string): Promise<DestroyOutcome>;
     destroyVolume(volumeId: string): Promise<DestroyOutcome>;
+    getVolumeAttachment(volumeId: string): Promise<string | null>;
 }
 export {};

package/dist/core/hearth/providers/blaxel.js

@@ -1,6 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import { readFileSync } from 'node:fs';
-import { SandboxInstance, VolumeInstance } from '@blaxel/core';
+import { SandboxInstance, VolumeInstance, getVolume, listSandboxes } from '@blaxel/core';
 import { general } from '../../errors.js';
 const TERMINATED_STATES = new Set(['TERMINATED', 'DELETING', 'DEACTIVATED', 'DEACTIVATING', 'FAILED']);
 const PROVISIONING_STATES = new Set(['UPLOADING', 'BUILDING', 'DEPLOYING', 'BUILT']);
@@ -81,10 +81,28 @@ function resolveExecCommand(input) {
     }
     return argv.map(shellQuote).join(' ');
 }
-function isMissingResourceError(error) {
-    const message = error instanceof Error ? error.message : String(error);
+// The Blaxel SDK (@blaxel/core 0.2.94) does not always reject with an `Error` instance — a 404
+// comes back as a plain object (e.g. `{code:404,error:"Sandbox not found"}`), and `String()` on
+// such an object yields the useless literal "[object Object]", which matches none of the regex
+// below. Recognize the structured shape directly (a numeric-or-string `404` status field) AND
+// fall back to JSON-serializing the value (mirroring wake.ts's `serializeError`) before running
+// the message regex, so both an `Error("... not found")` and a plain-object 404 are caught.
+export function isMissingResourceError(error) {
+    const fields = record(error);
+    if (fields !== null) {
+        for (const key of ['code', 'status', 'status_code', 'statusCode']) {
+            const value = fields[key];
+            if (value === 404 || value === '404')
+                return true;
+        }
+    }
+    const message = error instanceof Error ? error.message : (JSON.stringify(error) ?? String(error));
     return /\b(not found|404|does not exist|missing)\b/i.test(message);
 }
+// Accepts either a live `SandboxInstance` or a raw wire `Sandbox` page item — only the
+// `metadata`/`status`/`spec` fields are read, so mapping raw list pages avoids constructing
+// `SandboxInstance`s (whose `list()` eagerly warms an H2 session per sandbox — pointless for a
+// destroy sweep).
 function toMachineRef(sandbox) {
     const metadata = sandbox.metadata;
     const status = sandbox.status ?? undefined;
@@ -142,7 +160,16 @@ export class BlaxelMachineProvider {
     async createMachine(input) {
         const sandbox = await SandboxInstance.createIfNotExists({
             name: input.name ?? input.tenantId,
-            image: this.config.image,
+            // Tag every sandbox with the home's stable volume id so the recreate sweep can scope its
+            // destructive list to this home's family only (orphan-cleanup-fix MAJOR-1). Unset on
+            // callers that pass no externalId.
+            ...(input.externalId === undefined ? {} : { externalId: input.externalId }),
+            // A targeted recreate (auto-upgrade roll / rollback, design §4.1) passes
+            // an explicit `image`; the normal provision/wake path leaves it unset
+            // and uses the provider's construction-time config image. `createIfNotExists`
+            // only honors this on genuine creation — the roll path destroys the old
+            // sandbox first so the name is free (blaxel-home.ts `recreateOnImage`).
+            image: input.image ?? this.config.image,
             memory: this.config.memory,
             region: this.config.region,
             volumes: input.volumeMount === undefined || input.volumeMount === null ? undefined : [{ name: input.volumeMount.volumeName, mountPath: input.volumeMount.path, readOnly: false }],
@@ -158,6 +185,38 @@ export class BlaxelMachineProvider {
         await sandbox.wait({ maxWait: 90_000, interval: 500 }).catch(() => sandbox);
         return toMachineRef(sandbox);
     }
+    // orphan-cleanup-fix.md §3.2 + MAJOR-1/MINOR-1: the deterministic name-family sweep enumerates
+    // a home's generations via LIST scoped to the caller-owned `externalId` tag, then destroys the
+    // ones whose name is in the family. Over-listing is harmless — a FAILED/detached record still
+    // shows up here and destroy tolerates not_found, which is exactly what closes the autonomous-
+    // standby-detach orphan gap. Drives the generated `listSandboxes` client directly (bypassing
+    // `SandboxInstance.list()`, which never follows a cursor and warms an H2 session per sandbox):
+    // loops until exhausted so a future SDK version crossing Blaxel-Version 2026-04-28 (cursor
+    // pagination, 50/page) cannot silently truncate the sweep to page 1. At the SDK's pinned
+    // 2026-04-16 the server returns a bare array (the full set) and the loop breaks on the first
+    // page.
+    async listMachines(filter) {
+        const machines = [];
+        let cursor;
+        for (;;) {
+            const { data: raw } = await listSandboxes({
+                query: {
+                    limit: 200,
+                    ...(cursor === undefined ? {} : { cursor }),
+                    ...(filter?.externalId === undefined ? {} : { externalId: filter.externalId }),
+                },
+                throwOnError: true,
+            });
+            const page = Array.isArray(raw) ? raw : (raw?.data ?? []);
+            machines.push(...page.map(toMachineRef));
+            if (Array.isArray(raw))
+                break; // legacy bare array (Blaxel-Version < 2026-04-28) = full set
+            cursor = raw?.meta?.nextCursor || undefined; // empty when there are no more pages
+            if (cursor === undefined)
+                break;
+        }
+        return machines;
+    }
     async getMachine(machineId) {
         try {
             const sandbox = await SandboxInstance.get(machineId);
@@ -279,4 +338,27 @@ export class BlaxelMachineProvider {
             throw error;
         }
     }
+    // recreate-race-fix.md §3.1: read the platform's own positive attachment signal
+    // (`Volume.state.attachedTo`) rather than inferring detach from a sandbox GET/delete —
+    // that read/create consistency split is exactly what raced the old same-name recreate.
+    // `VolumeInstance.get`'s backing `volume` field is TS-private (no public state getter
+    // in @blaxel/core 0.2.94), so this calls the underlying `getVolume` SDK function
+    // directly instead — same request `VolumeInstance.get` makes, with `state` exposed.
+    // A 404 (or any other failure) throws, wrapped with the volumeId for context: a missing
+    // home volume is a real incident, never something to continue past (volume-is-sacred).
+    async getVolumeAttachment(volumeId) {
+        let attachedTo;
+        try {
+            const { data } = await getVolume({ path: { volumeName: volumeId }, throwOnError: true });
+            attachedTo = data.state?.attachedTo;
+        }
+        catch (error) {
+            throw general(`blaxel volume ${volumeId} attachment check failed`, {
+                volumeId,
+                cause: error instanceof Error ? error.message : String(error),
+            });
+        }
+        const trimmed = (attachedTo ?? '').trim();
+        return trimmed === '' ? null : trimmed;
+    }
 }

package/dist/core/hearth/providers/types.d.ts

@@ -22,10 +22,12 @@ export interface VolumeRef {
 }
 export interface CreateMachineInput {
     tenantId: string;
+    externalId?: string;
     template?: string | null;
     envs?: Record<string, string>;
     enableNetwork?: boolean;
     name?: string | null;
+    image?: string | null;
     ports?: {
         target: number;
         protocol: 'HTTP';
@@ -65,6 +67,16 @@ export interface MachineProvider {
     }): Promise<VolumeRef>;
     createMachine(input: CreateMachineInput): Promise<MachineRef>;
     getMachine(machineId: MachineId): Promise<MachineRef>;
+    /** List machine records, optionally scoped to a caller-owned `externalId` tag. The recreate
+     *  cleanup sweep passes the home's `providerVolumeId` as the filter so only sandboxes Hearth
+     *  itself tagged for this home come back, then intersects that with the generation NAME FAMILY
+     *  matcher — a tag∩name intersection on a destructive op, so no id-format or workspace name
+     *  discipline is load-bearing for other homes' safety (orphan-cleanup-fix MAJOR-1). The result
+     *  is NOT a promise of every machine in the workspace; without a filter it lists broadly, with
+     *  one it lists only tagged matches. Implementations must page until exhausted (MINOR-1). */
+    listMachines(filter?: {
+        externalId?: string;
+    }): Promise<MachineRef[]>;
     exec(machineId: MachineId, input: ExecInput): Promise<ExecResult>;
     waitForPort(machineId: MachineId, port: number, opts?: {
         maxWaitMs?: number;
@@ -73,5 +85,9 @@ export interface MachineProvider {
     health(): Promise<HealthResult>;
     destroy(machineId: MachineId): Promise<DestroyOutcome>;
     destroyVolume(volumeId: string): Promise<DestroyOutcome>;
+    /** The resource currently holding this volume ("sandbox:<name>"), or null if detached.
+     *  A missing volume (404) throws — a home volume disappearing is always an incident,
+     *  never something to continue past (recreate-race-fix.md §3.1). */
+    getVolumeAttachment(volumeId: string): Promise<string | null>;
 }
 export type { MachineState, M0Config, RouteRef };

package/dist/hearth/control-plane/__tests__/error-serialization.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/hearth/control-plane/__tests__/error-serialization.test.js

@@ -0,0 +1,29 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+// Bug-regression test (independent review of commit 6018050, recreate-race-fix.md §3.6):
+// `JSON.stringify` returns `undefined` (does not throw) for `undefined`, a function, or a
+// symbol — a naive `JSON.stringify(error)` rider would silently hand that `undefined` back to
+// callers. In `relay.ts` that `undefined` message then makes `message.startsWith('relay ')`
+// THROW inside the very catch block meant to handle it (a control-flow regression §3.6
+// promised not to introduce); in `wake.ts` it would OMIT the `error` field from the
+// `hearth.roll.incident` JSON line entirely. Both serializers must fall back to `String`
+// whenever `JSON.stringify` yields `undefined`, not only when it throws.
+test('serializeError (wake.ts) never returns undefined for undefined/function/symbol rejections', async () => {
+    const { serializeError } = await import('../wake.js');
+    for (const value of [undefined, function unnamed() { }, Symbol('boom')]) {
+        const result = serializeError(value);
+        assert.equal(typeof result, 'string', `serializeError(${String(value)}) must return a string, never JS \`undefined\``);
+        assert.ok(result.length > 0, `serializeError(${String(value)}) must be a non-empty string`);
+    }
+});
+test('serializeRelayError (relay.ts) never returns undefined for undefined/function/symbol rejections, and the relay control-flow checks it feeds never throw', async () => {
+    const { serializeRelayError } = await import('../relay.js');
+    for (const value of [undefined, function unnamed() { }, Symbol('boom')]) {
+        const result = serializeRelayError(value);
+        assert.equal(typeof result, 'string');
+        assert.ok(result.length > 0, `serializeRelayError(${String(value)}) must be a non-empty string`);
+        // The exact control flow `proxyHttp`'s catch runs on this value — must not throw.
+        assert.doesNotThrow(() => result.startsWith('relay '));
+        assert.doesNotThrow(() => /timed out/i.test(result));
+    }
+});

package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/hearth/control-plane/__tests__/oauth-serving-marker.test.js

@@ -0,0 +1,44 @@
+// Regression test for the P2 review Major fix: a stale, abandoned OAuth
+// marker must not make a later `beginOAuth` on the same home read as
+// non-serving. See serving.ts's `beginOAuth`/`oauthMarkerLive`.
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mock } from 'node:test';
+import { beginOAuth, endOAuth, isHomeServing, resetServingForTests } from '../serving.js';
+const HOME = 'home-oauth-marker-test';
+const OAUTH_MARKER_TTL_MS = 5 * 60_000;
+test('a second beginOAuth refreshes a stale marker and stays live until both finishes land', () => {
+    resetServingForTests();
+    mock.timers.enable({ apis: ['Date'] });
+    try {
+        // Start A begins an OAuth window; it is immediately serving.
+        beginOAuth(HOME);
+        assert.equal(isHomeServing(HOME), true, 'start A should mark the home as serving');
+        // Start A is abandoned: advance the clock past the 5-minute TTL with no
+        // matching endOAuth. The marker goes stale, but the counter (1) is still
+        // positive because start A was never finished.
+        mock.timers.tick(OAUTH_MARKER_TTL_MS + 1_000);
+        assert.equal(isHomeServing(HOME), false, 'a stale marker with no refresh must read as not-serving');
+        // Start B is a fresh retry on the same home. Per the fix, beginOAuth
+        // refreshes oauthStartedAt on EVERY call, not only when the counter
+        // transitions from 0 — so B's own window must read as live immediately.
+        beginOAuth(HOME);
+        assert.equal(isHomeServing(HOME), true, 'a second beginOAuth must refresh the marker and read as serving');
+        // B's window survives well past when A's original (unrefreshed) TTL
+        // would have expired, proving the refresh — not the original timestamp
+        // — is what's being judged now.
+        mock.timers.tick(OAUTH_MARKER_TTL_MS - 1_000);
+        assert.equal(isHomeServing(HOME), true, 'B\'s window must still be live before its own TTL elapses');
+        // A late finish for start A arrives: counter floors from 2 -> 1. Still
+        // serving, because B's own start/finish window has not closed yet.
+        endOAuth(HOME);
+        assert.equal(isHomeServing(HOME), true, 'one matched finish (of two starts) must not end serving early');
+        // B finishes: counter 1 -> 0, marker cleared, no longer serving.
+        endOAuth(HOME);
+        assert.equal(isHomeServing(HOME), false, 'both starts matched by a finish must end serving');
+    }
+    finally {
+        mock.timers.reset();
+        resetServingForTests();
+    }
+});

package/dist/hearth/control-plane/__tests__/wake-roll.test.d.ts

@@ -0,0 +1 @@
+export {};