@crouton-kit/crouter 0.3.38 → 0.3.39

package/dist/core/hearth/provider.d.ts

@@ -11,6 +11,14 @@ export interface HomeBackend {
      * and return refreshed provider pointers for the CP registry to persist.
      */
     wake(home: HomeProviderDescriptor): Promise<HomeProviderRefresh>;
+    /**
+     * Targeted destroy→recreate of a home onto a SPECIFIC image, reattaching the same durable
+     * volume (auto-upgrade roll and rollback, design §4/§5). Force-destroys the sandbox first so
+     * the recreate genuinely lands on `imageRef` (the volume is never touched), reboots the guest,
+     * and reports `templateVersion` for the CP to record. The caller independently asserts the
+     * running `crtr sys version` in-guest before trusting the flip.
+     */
+    recreateOnImage(home: HomeProviderDescriptor, imageRef: string, templateVersion: string): Promise<HomeProviderRefresh>;
     /** Pause the home and return refreshed provider pointers for the CP registry to persist. */
     suspend(home: HomeProviderDescriptor): Promise<HomeProviderRefresh>;
     /** Machine resources (cpu/mem) for status, or null when unavailable. */

package/dist/core/hearth/providers/__tests__/sweep-and-release.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/hearth/providers/__tests__/sweep-and-release.test.js

@@ -0,0 +1,254 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { BlaxelHomeBackend } from '../blaxel-home.js';
+import { isMissingResourceError } from '../blaxel.js';
+// Regression for the Run-3 orphan gap (orphan-cleanup-fix.md §1/§3.3): cleanup used a
+// point-in-time `attachedTo` read to find "the sandbox holding the volume", but Blaxel
+// autonomously drops that claim on idle→STANDBY, so a failed roll-forward generation that had
+// already been detached was invisible to cleanup and lived on as a billed, route-registered,
+// volume-claiming orphan. The fix destroys the entire generation NAME FAMILY via `listMachines`
+// with ZERO dependence on attachment state — so a family member NOT holding the volume must
+// still be destroyed, and a same-prefixed foreign name must NOT.
+const VOLUME_ID = 'hearth-zt-test-1';
+function makeConfig() {
+    return {
+        providerName: 'blaxel',
+        guestUser: 'agent',
+        guestHome: '/home/agent',
+        guestCodePath: '/home/agent/code',
+        piPackages: [],
+        webPort: 8080,
+        nodeKind: 'developer',
+        imageRef: 'crouter-hearth-home-0-3-35',
+        templateVersion: '0.3.35',
+    };
+}
+function makeDescriptor() {
+    return {
+        homeId: 'home-1',
+        tenantId: 'tenant-1',
+        providerName: 'blaxel',
+        providerSandboxId: `${VOLUME_ID}-gold`,
+        providerVolumeId: VOLUME_ID,
+        homeAgentTarget: 'guest-node-1',
+        relayToken: 'relay-token',
+    };
+}
+/** A fake provider whose `listMachines` HONORS the `externalId` filter over a tagged inventory:
+ *  the row-pointed generation, a DETACHED/standby family member NOT holding the volume (the Run-3
+ *  miss), a FAILED family record (a create that 500'd but left a row, e.g. Blaxel's `gmr…azqr`),
+ *  and the bare base name — all tagged `externalId = VOLUME_ID`. Plus survivors: a same-prefixed
+ *  FOREIGN home (`hearth-zt-test-10-…`, numeric-adjacent, spared by the `-g` anchor too); the
+ *  suffix-extension collision `<base>-gfoo` that IS in the name family but belongs to a DIFFERENT
+ *  home (tagged with its own volume id) — destroyed by the old name-only sweep, spared now by the
+ *  tag filter; and a same-name-family-but-UNTAGGED record (a legacy pre-backfill member, no
+ *  externalId) — excluded by the filter, so it survives as the alerted self-heals orphan class,
+ *  NOT a collateral destroy. Records every destroy target. */
+function makeFakeProvider(destroyed) {
+    const inventory = [
+        { ref: { providerName: 'blaxel', machineId: `${VOLUME_ID}-gold`, state: 'running' }, externalId: VOLUME_ID }, // row pointer
+        { ref: { providerName: 'blaxel', machineId: `${VOLUME_ID}-gdetached`, state: 'running' }, externalId: VOLUME_ID }, // detached standby orphan — the miss
+        { ref: { providerName: 'blaxel', machineId: `${VOLUME_ID}-gfailed`, state: 'terminated' }, externalId: VOLUME_ID }, // FAILED leftover record
+        { ref: { providerName: 'blaxel', machineId: VOLUME_ID, state: 'running' }, externalId: VOLUME_ID }, // bare base name
+        { ref: { providerName: 'blaxel', machineId: 'hearth-zt-test-10-gxyz', state: 'running' }, externalId: 'hearth-zt-test-10' }, // FOREIGN numeric-adjacent home, must survive
+        { ref: { providerName: 'blaxel', machineId: `${VOLUME_ID}-gfoo`, state: 'running' }, externalId: `${VOLUME_ID}-gfoo` }, // suffix-extension collision, DIFFERENT home — must survive
+        { ref: { providerName: 'blaxel', machineId: `${VOLUME_ID}-gstale`, state: 'running' } }, // same-name-family but UNTAGGED (legacy) — must survive
+        { ref: { providerName: 'blaxel', machineId: 'unrelated-sandbox', state: 'running' } }, // unrelated, must survive
+    ];
+    const fake = {
+        providerName: 'blaxel',
+        imageRef: 'crouter-hearth-home-0-3-35',
+        templateVersion: '0.3.35',
+        volumeSizeGb: 4,
+        async listMachines(filter) {
+            return inventory.filter((e) => filter?.externalId === undefined || e.externalId === filter.externalId).map((e) => e.ref);
+        },
+        async destroy(machineId) {
+            destroyed.push(machineId);
+            return { kind: 'destroyed' };
+        },
+        // Volume already free (Blaxel detached it) — the poll passes on the first read.
+        async getVolumeAttachment() {
+            return null;
+        },
+        async createMachine(input) {
+            return { providerName: 'blaxel', machineId: input.name ?? VOLUME_ID, state: 'running' };
+        },
+        async exec(_id, _input) {
+            return { exitCode: 0, timedOut: false, stdout: '', stderr: '' };
+        },
+        async waitForPort() { },
+        async publishHttpPort(machineId, input) {
+            return { routeId: input.routeName ?? `${machineId}-web`, machineId, targetPort: input.targetPort, url: `https://${machineId}.example` };
+        },
+        async getMachine(machineId) {
+            return { providerName: 'blaxel', machineId, state: 'running' };
+        },
+    };
+    return fake;
+}
+// Regression for the recovery-wake live re-proof (SHA 567cd4f, backfill-recovery-evidence.md §3):
+// the Blaxel SDK (@blaxel/core 0.2.94) rejects a not-found lookup with a PLAIN OBJECT
+// (`{code:404,error:"Sandbox not found"}`), not an `Error` instance. `String()` on that object
+// yields the literal "[object Object]", which matched none of the message regex, so the
+// predicate returned `false` for a genuine 404 — breaking the `getMachine` terminated-machine
+// fallback (the sweep's entry point) AND the `destroy`/`destroyVolume` not_found tolerance the
+// sweep's per-name destroy loop depends on.
+test('isMissingResourceError recognizes the Blaxel SDK plain-object 404 shape', () => {
+    assert.equal(isMissingResourceError({ code: 404, error: 'Sandbox not found' }), true, 'plain-object 404 (the exact live shape) must be recognized');
+    assert.equal(isMissingResourceError({ code: 500, error: 'boom' }), false, 'an unrelated plain-object error must not be mistaken for not-found');
+});
+test('isMissingResourceError preserves existing Error-message-regex behavior', () => {
+    assert.equal(isMissingResourceError(new Error('sandbox hearth-zt-test-1-gxyz not found')), true, 'an Error whose message matches the regex must still be recognized');
+    assert.equal(isMissingResourceError(new Error('boom')), false, 'an unrelated Error must not be mistaken for not-found');
+});
+test('sweepAndReleaseVolume (via recreateOnImage) destroys EVERY name-family member — including a detached one — and spares foreign homes', async () => {
+    const destroyed = [];
+    const backend = new BlaxelHomeBackend(makeConfig(), makeFakeProvider(destroyed));
+    const refresh = await backend.recreateOnImage(makeDescriptor(), 'crouter-hearth-home-0-3-36', '0.3.36');
+    // Both the row-pointed generation AND the detached standby orphan (the exact Run-3 miss) are
+    // destroyed, plus the FAILED leftover and the bare base name.
+    assert.ok(destroyed.includes(`${VOLUME_ID}-gold`), 'row-pointed generation must be destroyed');
+    assert.ok(destroyed.includes(`${VOLUME_ID}-gdetached`), 'DETACHED standby orphan (Run-3 miss) must be destroyed');
+    assert.ok(destroyed.includes(`${VOLUME_ID}-gfailed`), 'FAILED leftover record must be destroyed');
+    assert.ok(destroyed.includes(VOLUME_ID), 'bare base-name generation must be destroyed');
+    // The `-g` anchor keeps a same-prefixed foreign home and unrelated sandboxes safe.
+    assert.ok(!destroyed.includes('hearth-zt-test-10-gxyz'), 'foreign home hearth-zt-test-10 must NOT be swept (-g anchor)');
+    assert.ok(!destroyed.includes('unrelated-sandbox'), 'unrelated sandbox must NOT be swept');
+    // MAJOR-1: the suffix-extension collision (`<base>-gfoo`, a DIFFERENT home) IS in the name
+    // family but carries a different externalId, so the tag-scoped list never surfaces it — the
+    // old name-only sweep would have collateral-destroyed it. And a same-name-family but UNTAGGED
+    // legacy record is likewise excluded by the filter, surviving as the self-heals orphan class.
+    assert.ok(!destroyed.includes(`${VOLUME_ID}-gfoo`), 'suffix-extension foreign home <base>-gfoo must NOT be swept (untagged — MAJOR-1)');
+    assert.ok(!destroyed.includes(`${VOLUME_ID}-gstale`), 'same-name-family but UNTAGGED record must NOT be swept (tag filter excludes it)');
+    // The four tagged family members are destroyed before the fresh generation is created.
+    assert.equal(destroyed.length, 4, 'exactly the four tagged family members are destroyed');
+    // A fresh `-g` generation is minted and reported on the target version.
+    assert.match(refresh.providerSandboxId, new RegExp(`^${VOLUME_ID}-g[0-9a-z]+$`));
+    assert.notEqual(refresh.providerSandboxId, `${VOLUME_ID}-gold`);
+    assert.equal(refresh.templateVersion, '0.3.36');
+});
+/** A fake modeling a provider that assembles its filtered result from MULTIPLE cursor pages
+ *  before returning (MINOR-1). The tagged family spans 3 pages; `listMachines` walks all pages,
+ *  concatenates, and honors the externalId filter — the same total the real paginated
+ *  `listSandboxes` loop assembles. Asserts the sweep destroys every tagged member across every
+ *  page, so a family member beyond page 1 can never survive as an orphan. */
+function makePaginatedFakeProvider(familyIds, destroyed) {
+    // Split the tagged family across three pages; sprinkle in a foreign untagged record per page.
+    const pages = [[], [], []];
+    familyIds.forEach((id, i) => pages[i % 3].push({ ref: { providerName: 'blaxel', machineId: id, state: 'running' }, externalId: VOLUME_ID }));
+    pages[0].push({ ref: { providerName: 'blaxel', machineId: 'foreign-a', state: 'running' }, externalId: 'other-home' });
+    pages[2].push({ ref: { providerName: 'blaxel', machineId: 'foreign-b', state: 'running' } });
+    const fake = {
+        providerName: 'blaxel',
+        imageRef: 'crouter-hearth-home-0-3-35',
+        templateVersion: '0.3.35',
+        volumeSizeGb: 4,
+        async listMachines(filter) {
+            // Assemble across all pages (what the real cursor loop does) then apply the filter.
+            return pages.flat().filter((e) => filter?.externalId === undefined || e.externalId === filter.externalId).map((e) => e.ref);
+        },
+        async destroy(machineId) {
+            destroyed.push(machineId);
+            return { kind: 'destroyed' };
+        },
+        async getVolumeAttachment() {
+            return null;
+        },
+        async createMachine(input) {
+            return { providerName: 'blaxel', machineId: input.name ?? VOLUME_ID, state: 'running' };
+        },
+        async exec() {
+            return { exitCode: 0, timedOut: false, stdout: '', stderr: '' };
+        },
+        async waitForPort() { },
+        async publishHttpPort(machineId, input) {
+            return { routeId: input.routeName ?? `${machineId}-web`, machineId, targetPort: input.targetPort, url: `https://${machineId}.example` };
+        },
+        async getMachine(machineId) {
+            return { providerName: 'blaxel', machineId, state: 'running' };
+        },
+    };
+    return fake;
+}
+test('MINOR-1 pagination: a family served across multiple pages is fully found+destroyed by the sweep', async () => {
+    const destroyed = [];
+    // A family that would straddle page boundaries: the base plus several `-g` generations.
+    const familyIds = [VOLUME_ID, `${VOLUME_ID}-ga`, `${VOLUME_ID}-gb`, `${VOLUME_ID}-gc`, `${VOLUME_ID}-gd`, `${VOLUME_ID}-ge`, `${VOLUME_ID}-gf`];
+    const backend = new BlaxelHomeBackend(makeConfig(), makePaginatedFakeProvider(familyIds, destroyed));
+    await backend.recreateOnImage(makeDescriptor(), 'crouter-hearth-home-0-3-36', '0.3.36');
+    // Every tagged family member across every page is destroyed — none survives beyond page 1.
+    for (const id of familyIds)
+        assert.ok(destroyed.includes(id), `family member ${id} (across pages) must be destroyed`);
+    assert.equal(destroyed.length, familyIds.length, 'exactly the tagged family is destroyed — foreign records on the same pages survive');
+    assert.ok(!destroyed.includes('foreign-a'), 'foreign tagged record must survive');
+    assert.ok(!destroyed.includes('foreign-b'), 'foreign untagged record must survive');
+});
+/** A fake whose `createMachine` runs a scripted sequence of outcomes ('ok' | a rejection value)
+ *  per call, with an empty family (isolating the §4 retry logic from the sweep). Records every
+ *  destroy so self-cleanup and the discarded generation are observable. */
+function makeCreateScriptedProvider(script, destroyed) {
+    let call = 0;
+    const fake = {
+        providerName: 'blaxel',
+        imageRef: 'crouter-hearth-home-0-3-35',
+        templateVersion: '0.3.35',
+        volumeSizeGb: 4,
+        async listMachines() {
+            return [];
+        },
+        async destroy(machineId) {
+            destroyed.push(machineId);
+            return { kind: 'destroyed' };
+        },
+        async getVolumeAttachment() {
+            return null;
+        },
+        async createMachine(input) {
+            const outcome = script[call++];
+            if (outcome !== 'ok')
+                throw outcome;
+            return { providerName: 'blaxel', machineId: input.name ?? VOLUME_ID, state: 'running' };
+        },
+        async exec() {
+            return { exitCode: 0, timedOut: false, stdout: '', stderr: '' };
+        },
+        async waitForPort() { },
+        async publishHttpPort(machineId, input) {
+            return { routeId: input.routeName ?? `${machineId}-web`, machineId, targetPort: input.targetPort, url: `https://${machineId}.example` };
+        },
+        async getMachine(machineId) {
+            return { providerName: 'blaxel', machineId, state: 'running' };
+        },
+    };
+    return fake;
+}
+test('§4 retry: a genuine 5xx DEPLOYMENT_FAILED create is self-cleaned and retried ONCE onto a fresh name', async () => {
+    const destroyed = [];
+    // First create fails with a server-side deployment transient; the bounded retry then succeeds.
+    const backend = new BlaxelHomeBackend(makeConfig(), makeCreateScriptedProvider([{ status_code: 500, code: 'DEPLOYMENT_FAILED', message: 'deployment failed, please try again' }, 'ok'], destroyed));
+    const refresh = await backend.recreateOnImage(makeDescriptor(), 'crouter-hearth-home-0-3-36', '0.3.36');
+    // The failed first generation is self-cleaned (destroyed) — no orphan — and exactly one
+    // discarded generation was destroyed (empty family means the only destroy is the self-cleanup).
+    assert.equal(destroyed.length, 1, 'exactly the failed first generation is self-cleaned');
+    // The returned generation is the SECOND minted name, not the discarded one.
+    assert.notEqual(refresh.providerSandboxId, destroyed[0]);
+    assert.match(refresh.providerSandboxId, new RegExp(`^${VOLUME_ID}-g[0-9a-z]+$`));
+    assert.equal(refresh.templateVersion, '0.3.36');
+});
+test('§4 retry: a 4xx create is NEVER retried — it self-cleans and rethrows the original error', async () => {
+    const destroyed = [];
+    const boom = { status_code: 409, code: 'SANDBOX_ALREADY_EXISTS', message: 'conflict' };
+    const backend = new BlaxelHomeBackend(makeConfig(), makeCreateScriptedProvider([boom, 'ok'], destroyed));
+    await assert.rejects(() => backend.recreateOnImage(makeDescriptor(), 'crouter-hearth-home-0-3-36', '0.3.36'), (err) => err === boom, 'a 4xx must rethrow the ORIGINAL rejection, never be absorbed by a retry');
+    // Self-cleanup destroyed the one failed generation, and the retry path never ran a second create.
+    assert.equal(destroyed.length, 1, 'the failed 4xx generation is still self-cleaned exactly once');
+});
+test('§4 retry: a second consecutive retryable failure throws (bounded to one retry) and both generations are self-cleaned', async () => {
+    const destroyed = [];
+    const boom = { status_code: 503, code: 'DEPLOYMENT_FAILED', message: 'still failing' };
+    const backend = new BlaxelHomeBackend(makeConfig(), makeCreateScriptedProvider([boom, boom], destroyed));
+    await assert.rejects(() => backend.recreateOnImage(makeDescriptor(), 'crouter-hearth-home-0-3-36', '0.3.36'), (err) => err === boom);
+    // Both the first and the retry generation are self-cleaned — no orphan survives a double failure.
+    assert.equal(destroyed.length, 2, 'both failed generations are self-cleaned');
+});

package/dist/core/hearth/providers/blaxel-home.d.ts

@@ -6,6 +6,45 @@ export declare class BlaxelHomeBackend {
     private readonly provider;
     constructor(config: M0Config, provider?: BlaxelMachineProvider);
     wake(homeInput: HomeProviderDescriptor): Promise<HomeProviderRefresh>;
+    /** Destroy EVERY generation of this home via a deterministic name-family sweep — never a
+     *  point-in-time `attachedTo` read, which Blaxel's autonomous idle→STANDBY detach can race
+     *  ahead of, leaving a live orphan the read cannot see (orphan-cleanup-fix.md §1/§3.3). Then
+     *  poll until the platform reports the volume detached. Safe because at sweep time the
+     *  replacement does not exist yet (its name is minted after) and any live family member is by
+     *  construction either the generation being replaced or an uncommitted leftover — the home's
+     *  durable state is the volume, never a sandbox; one wake per home runs at a time. */
+    private sweepAndReleaseVolume;
+    /** Poll the volume until the platform reports it detached — the create-precondition gate
+     *  (recreate-race-fix.md §3.2). Reads only; never destroys. Emits `hearth.recreate.volume_wait`
+     *  whenever it actually had to wait, so the re-proof can quantify headroom against the budget. */
+    private pollVolumeDetached;
+    /** Create one fresh generation and bring it to a serving state (create → bootstrap exec →
+     *  waitForPort → publish → getMachine), with provider self-cleanup and a single bounded
+     *  retry (orphan-cleanup-fix.md §3.4/§4). Each attempt destroys the generation IT created if
+     *  anything from `createMachine` through the final `getMachine` fails — no code path exits
+     *  leaving a live generation it created and did not return — then rethrows the ORIGINAL error
+     *  (cleanup never masks it). On the first failure, if the error is a genuine server-side
+     *  deployment transient, pause, re-run ONLY the volume-free poll (the sweep already ran once),
+     *  mint a NEW name, and retry exactly once; a second failure throws. The sweep is the caller's
+     *  responsibility and runs before the first attempt only. */
+    private createGeneration;
+    /** Targeted destroy→recreate onto a SPECIFIC image (auto-upgrade roll and
+     *  rollback, design §4/§5). Unlike `wake()`'s standby-resume path, this is the ONE path
+     *  that force-destroys the home's sandboxes before recreating. Cleanup is a deterministic
+     *  name-family sweep (orphan-cleanup-fix.md §3.3): every generation of this home is destroyed
+     *  before the fresh one is minted, so a failed roll-forward generation can never survive as a
+     *  live orphan even after Blaxel autonomously detaches it from the volume. The recreate lands
+     *  on a FRESH sandbox name every call — a name that has never existed cannot 409 against a
+     *  lingering same-name delete. The durable volume is a separate resource and is NEVER touched
+     *  (flow-D durability) — user state is byte-identical across the flip, which is exactly what
+     *  makes both the upgrade and the rollback lossless. The new `providerSandboxId` is returned on
+     *  the refresh; callers persist it (`refreshProviderPointers` treats it as a refreshable
+     *  pointer). `templateVersion` is baked into the bootstrap's fail-fast version assert (the
+     *  baked crtr must match) AND reported back so the CP records the target version — the caller
+     *  still independently asserts `crtr sys version` in-guest before trusting it (wake.ts
+     *  `rollHome`/`rollbackHome`, against `refresh.providerSandboxId`, never a stale captured
+     *  name). */
+    recreateOnImage(homeInput: HomeProviderDescriptor, imageRef: string, templateVersion: string): Promise<HomeProviderRefresh>;
     suspend(homeInput: HomeProviderDescriptor): Promise<HomeProviderRefresh>;
     getMachine(machineId: string): Promise<MachineRef | null>;
     /** P2.6 stdin-capable exec seam, passed through verbatim — the `HomeBackend` interface is the

package/dist/core/hearth/providers/blaxel-home.js

@@ -1,10 +1,93 @@
+import { setTimeout as delay } from 'node:timers/promises';
 import { general } from '../../errors.js';
 import { nowIso } from '../../fs-utils.js';
 import { guestEnv } from '../guest-env.js';
 import { buildBootstrapScript } from './blaxel-bootstrap.js';
 import { BlaxelMachineProvider } from './blaxel.js';
-function stableRouteName(home) {
-    return `${home.providerSandboxId}-web`;
+// recreate-race-fix.md §3.2 — a volume GET is one cheap API call; 1s resolution is plenty
+// against an expected seconds-scale detach.
+const VOLUME_DETACH_POLL_MS = 1_000;
+// Expected wait is seconds (observed live); 120s is ~20-60x headroom over that, so this still
+// fails deterministically inside one wake's lifetime rather than hanging forever on a genuine
+// platform-side stuck detach.
+const VOLUME_DETACH_TIMEOUT_MS = 120_000;
+// orphan-cleanup-fix.md §4 — pause before the single bounded retry of a create that failed on a
+// server-side deployment transient, to let the platform-side condition clear.
+const CREATE_RETRY_DELAY_MS = 5_000;
+function stableRouteName(sandboxId) {
+    return `${sandboxId}-web`;
+}
+/** Mint a fresh per-generation sandbox name. `providerVolumeId` is the home's stable base
+ *  (never suffixed), so names stay bounded: `hearth-zt-test-1-gmchq3x2a`. Sequential recreates
+ *  are seconds apart, so a ms timestamp cannot collide for one home (recreate-race-fix.md §3.2). */
+function mintSandboxName(providerVolumeId) {
+    return `${providerVolumeId}-g${Date.now().toString(36)}`;
+}
+/** Escape a volumeId for safe use inside a RegExp so the generation-family anchor is exact —
+ *  a home base name is user-influenced, so a literal `.` or `+` in it must not become a
+ *  wildcard (orphan-cleanup-fix.md §3.1). */
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/** Predicate matching every generation of a home's name family: the stable base itself, or a
+ *  `<base>-g<suffix>` generation. The `-g` anchor is load-bearing — it prevents `<base>0-g…`
+ *  (e.g. `hearth-zt-test-10-g…`) from matching base `hearth-zt-test-1` (orphan-cleanup-fix.md
+ *  §3.1). Suffix chars are base36 (lowercase alnum), matching `mintSandboxName`. */
+function generationFamilyMatcher(volumeId) {
+    const gen = new RegExp(`^${escapeRegExp(volumeId)}-g[0-9a-z]+$`);
+    return (name) => name === volumeId || gen.test(name);
+}
+/** Serialize an arbitrary rejection to a non-empty marker string. `JSON.stringify` RETURNS
+ *  `undefined` (does not throw) for `undefined`, a function, or a symbol — falling through to
+ *  that would omit the `error` field from the marker's JSON line entirely, so fall back to
+ *  `String(...)` whenever it yields `undefined` (matches wake.ts/relay.ts `serializeError`). */
+function serialize(error) {
+    if (error instanceof Error)
+        return error.message;
+    try {
+        const json = JSON.stringify(error);
+        return json === undefined ? String(error) : json;
+    }
+    catch {
+        return String(error);
+    }
+}
+/** Emit one structured stderr line for a recreate-lifecycle marker (sweep / volume_wait /
+ *  cleanup_failed / create_retry). Single writer for all four so the JSON shape stays uniform
+ *  and ops log-alerting keys on `marker` (orphan-cleanup-fix.md §3.3/§3.4/§4). */
+function emitMarker(marker, fields, level = 'info') {
+    process.stderr.write(`${JSON.stringify({ level, marker, ...fields })}\n`);
+}
+/** Classify a failed create+deploy as a retryable server-side deployment transient
+ *  (orphan-cleanup-fix.md §4). Retryable IFF `status_code >= 500` OR `code === 'DEPLOYMENT_FAILED'`
+ *  on the rejection object, with a message-substring fallback for Error-wrapped variants whose
+ *  structured fields were flattened into `.message`. NEVER on 4xx — a 409 here is a real name
+ *  conflict the design must surface, not absorb, so an explicit 4xx status vetoes the retry even
+ *  if a `DEPLOYMENT_FAILED` code is also present. */
+function isRetryableDeploymentError(error) {
+    if (error !== null && typeof error === 'object') {
+        // Only the canonical HTTP-status keys — NOT a bare `status`, which is an overloaded field
+        // name some envelopes use for a non-HTTP state enum/count (MINOR-2).
+        const obj = error;
+        const status = [obj.status_code, obj.statusCode].find((v) => typeof v === 'number');
+        if (typeof status === 'number') {
+            if (status >= 400 && status < 500)
+                return false; // 4xx: real conflict, never absorb
+            if (status >= 500)
+                return true;
+        }
+        if (obj.code === 'DEPLOYMENT_FAILED')
+            return true;
+    }
+    // Fallback for Error-wrapped variants: only POSITIVE server-side signals, never inferring a 4xx.
+    const message = error instanceof Error ? error.message : typeof error === 'string' ? error : '';
+    if (message !== '') {
+        if (/DEPLOYMENT_FAILED/.test(message))
+            return true;
+        if (/\bstatus[_ ]?code\b\D{0,8}5\d\d/i.test(message))
+            return true;
+    }
+    return false;
 }
 function ensureReadyHome(input) {
     if (input.providerName !== 'blaxel') {
@@ -30,10 +113,10 @@ function isRunningMachine(machine) {
 function previewEndpoint(route, fallback) {
     return route.concreteUrl ?? route.url ?? fallback ?? null;
 }
-function refreshedPointers(home, provider, route, status, machine) {
+function refreshedPointers(home, provider, route, status, sandboxId, machine) {
     return {
         providerName: 'blaxel',
-        providerSandboxId: home.providerSandboxId,
+        providerSandboxId: sandboxId,
         providerVolumeId: home.providerVolumeId,
         previewEndpoint: route === null ? (home.previewEndpoint ?? null) : previewEndpoint(route, home.previewEndpoint),
         route,
@@ -80,25 +163,192 @@ export class BlaxelHomeBackend {
                 await this.provider.exec(current.providerSandboxId, { command: bootstrapScript, background: true, cwd: this.config.guestHome });
                 await this.provider.waitForPort(current.providerSandboxId, this.config.webPort, { maxWaitMs: 30_000 });
             }
-        }
-        else {
-            await this.provider.createMachine({
-                tenantId: current.tenantId,
-                name: current.providerSandboxId,
-                envs: guestEnv(this.config),
-                volumeMount: { volumeName: current.providerVolumeId, path: this.config.guestHome },
-                ports: [{ target: this.config.webPort, protocol: 'HTTP' }],
+            const route = await this.provider.publishHttpPort(current.providerSandboxId, {
+                machineId: current.providerSandboxId,
+                targetPort: this.config.webPort,
+                routeName: stableRouteName(current.providerSandboxId),
+                routeId: stableRouteName(current.providerSandboxId),
             });
-            await this.provider.exec(current.providerSandboxId, { command: bootstrapScript, background: true, cwd: this.config.guestHome });
-            await this.provider.waitForPort(current.providerSandboxId, this.config.webPort, { maxWaitMs: 60_000 });
+            return refreshedPointers(current, this.provider, route, 'running', current.providerSandboxId, await this.getMachine(current.providerSandboxId));
+        }
+        // Terminated-machine branch (recreate-race-fix.md §3.4, orphan-cleanup-fix.md §3.3/§3.4):
+        // the crash-recovery path for a sandbox deleted recently out-of-band OR by a crashed/failed
+        // roll (a Fly redeploy restarting the CP mid-roll lands here on the next wake).
+        // `sweepAndReleaseVolume` destroys EVERY generation in this home's name family — including
+        // an unpointed leftover a crashed roll left holding the volume, which the old point-in-time
+        // attachedTo read could miss once Blaxel autonomously detached it on idle→STANDBY — then
+        // polls until the volume is confirmed free before minting a fresh name and creating.
+        await this.sweepAndReleaseVolume(current.providerVolumeId, current.homeId);
+        const { sandboxName, route, machine: created } = await this.createGeneration({
+            homeId: current.homeId,
+            tenantId: current.tenantId,
+            volumeId: current.providerVolumeId,
+            bootstrapScript,
+        });
+        return refreshedPointers(current, this.provider, route, 'running', sandboxName, created);
+    }
+    /** Destroy EVERY generation of this home via a deterministic name-family sweep — never a
+     *  point-in-time `attachedTo` read, which Blaxel's autonomous idle→STANDBY detach can race
+     *  ahead of, leaving a live orphan the read cannot see (orphan-cleanup-fix.md §1/§3.3). Then
+     *  poll until the platform reports the volume detached. Safe because at sweep time the
+     *  replacement does not exist yet (its name is minted after) and any live family member is by
+     *  construction either the generation being replaced or an uncommitted leftover — the home's
+     *  durable state is the volume, never a sandbox; one wake per home runs at a time. */
+    async sweepAndReleaseVolume(volumeId, homeId) {
+        // Scope the list to sandboxes Hearth tagged for THIS home (externalId = volumeId), then keep
+        // the name-family matcher as an AND-intersection on top (orphan-cleanup-fix MAJOR-1). The
+        // destroy set requires BOTH keys to agree — the tag says "Hearth created this for home X",
+        // the name says "it is in X's generation family" — so a single fault (a mistagged create, a
+        // future API silently ignoring the filter param) cannot widen the blast radius back onto a
+        // foreign home. Fail-safe defense-in-depth on a destructive op, NOT a lenient fallback.
+        const inFamily = generationFamilyMatcher(volumeId);
+        const doomed = (await this.provider.listMachines({ externalId: volumeId })).map((m) => m.machineId).filter(inFamily);
+        for (const name of doomed)
+            await this.provider.destroy(name); // sequential; destroy tolerates not_found
+        if (doomed.length > 0)
+            emitMarker('hearth.recreate.sweep', { home_id: homeId, volume_id: volumeId, destroyed: doomed });
+        await this.pollVolumeDetached(volumeId, homeId);
+    }
+    /** Poll the volume until the platform reports it detached — the create-precondition gate
+     *  (recreate-race-fix.md §3.2). Reads only; never destroys. Emits `hearth.recreate.volume_wait`
+     *  whenever it actually had to wait, so the re-proof can quantify headroom against the budget. */
+    async pollVolumeDetached(volumeId, homeId) {
+        const pollStartedAt = Date.now();
+        const deadline = pollStartedAt + VOLUME_DETACH_TIMEOUT_MS;
+        let waited = false;
+        for (;;) {
+            const attached = await this.provider.getVolumeAttachment(volumeId);
+            if (attached === null)
+                break;
+            waited = true;
+            if (Date.now() >= deadline) {
+                throw general(`volume ${volumeId} still attached to ${attached} after ${VOLUME_DETACH_TIMEOUT_MS}ms — cannot recreate home ${homeId}`, {
+                    homeId,
+                    volumeId,
+                    attached,
+                });
+            }
+            await delay(VOLUME_DETACH_POLL_MS);
+        }
+        if (waited) {
+            emitMarker('hearth.recreate.volume_wait', { home_id: homeId, volume_id: volumeId, waited_ms: Date.now() - pollStartedAt });
         }
-        const route = await this.provider.publishHttpPort(current.providerSandboxId, {
-            machineId: current.providerSandboxId,
-            targetPort: this.config.webPort,
-            routeName: stableRouteName(current),
-            routeId: stableRouteName(current),
+    }
+    /** Create one fresh generation and bring it to a serving state (create → bootstrap exec →
+     *  waitForPort → publish → getMachine), with provider self-cleanup and a single bounded
+     *  retry (orphan-cleanup-fix.md §3.4/§4). Each attempt destroys the generation IT created if
+     *  anything from `createMachine` through the final `getMachine` fails — no code path exits
+     *  leaving a live generation it created and did not return — then rethrows the ORIGINAL error
+     *  (cleanup never masks it). On the first failure, if the error is a genuine server-side
+     *  deployment transient, pause, re-run ONLY the volume-free poll (the sweep already ran once),
+     *  mint a NEW name, and retry exactly once; a second failure throws. The sweep is the caller's
+     *  responsibility and runs before the first attempt only. */
+    async createGeneration(args) {
+        const attempt = async (sandboxName) => {
+            try {
+                await this.provider.createMachine({
+                    tenantId: args.tenantId,
+                    name: sandboxName,
+                    // Tag EVERY generation (first-name and retry-name alike) with the home's stable volume
+                    // id so the sweep's filtered list only ever sees this home's family (MAJOR-1).
+                    externalId: args.volumeId,
+                    ...(args.image === undefined ? {} : { image: args.image }),
+                    envs: guestEnv(this.config),
+                    volumeMount: { volumeName: args.volumeId, path: this.config.guestHome },
+                    ports: [{ target: this.config.webPort, protocol: 'HTTP' }],
+                });
+                await this.provider.exec(sandboxName, { command: args.bootstrapScript, background: true, cwd: this.config.guestHome });
+                await this.provider.waitForPort(sandboxName, this.config.webPort, { maxWaitMs: 60_000 });
+                const route = await this.provider.publishHttpPort(sandboxName, {
+                    machineId: sandboxName,
+                    targetPort: this.config.webPort,
+                    routeName: stableRouteName(sandboxName),
+                    routeId: stableRouteName(sandboxName),
+                });
+                return { route, machine: await this.getMachine(sandboxName) };
+            }
+            catch (error) {
+                // §3.4 self-cleanup: destroy the generation this attempt created; never mask the original.
+                await this.provider
+                    .destroy(sandboxName)
+                    .catch((cleanupErr) => emitMarker('hearth.recreate.cleanup_failed', { home_id: args.homeId, sandbox: sandboxName, error: serialize(cleanupErr) }, 'warn'));
+                throw error;
+            }
+        };
+        const firstName = mintSandboxName(args.volumeId);
+        try {
+            return { sandboxName: firstName, ...(await attempt(firstName)) };
+        }
+        catch (error) {
+            if (!isRetryableDeploymentError(error))
+                throw error;
+            emitMarker('hearth.recreate.create_retry', { home_id: args.homeId, discarded_generation: firstName, error: serialize(error) }, 'warn');
+            await delay(CREATE_RETRY_DELAY_MS);
+            // Re-run ONLY the poll — NOT a re-sweep. This sits outside attempt()'s try/catch, so a
+            // poll-timeout here would otherwise mask the ORIGINAL deployment error (MINOR-3). The
+            // original is what the caller must see (it is already recorded in the create_retry
+            // marker above); on a poll failure during the retry path, rethrow it, not the timeout.
+            try {
+                await this.pollVolumeDetached(args.volumeId, args.homeId);
+            }
+            catch {
+                throw error;
+            }
+            const secondName = mintSandboxName(args.volumeId);
+            return { sandboxName: secondName, ...(await attempt(secondName)) }; // second failure throws (self-cleaned)
+        }
+    }
+    /** Targeted destroy→recreate onto a SPECIFIC image (auto-upgrade roll and
+     *  rollback, design §4/§5). Unlike `wake()`'s standby-resume path, this is the ONE path
+     *  that force-destroys the home's sandboxes before recreating. Cleanup is a deterministic
+     *  name-family sweep (orphan-cleanup-fix.md §3.3): every generation of this home is destroyed
+     *  before the fresh one is minted, so a failed roll-forward generation can never survive as a
+     *  live orphan even after Blaxel autonomously detaches it from the volume. The recreate lands
+     *  on a FRESH sandbox name every call — a name that has never existed cannot 409 against a
+     *  lingering same-name delete. The durable volume is a separate resource and is NEVER touched
+     *  (flow-D durability) — user state is byte-identical across the flip, which is exactly what
+     *  makes both the upgrade and the rollback lossless. The new `providerSandboxId` is returned on
+     *  the refresh; callers persist it (`refreshProviderPointers` treats it as a refreshable
+     *  pointer). `templateVersion` is baked into the bootstrap's fail-fast version assert (the
+     *  baked crtr must match) AND reported back so the CP records the target version — the caller
+     *  still independently asserts `crtr sys version` in-guest before trusting it (wake.ts
+     *  `rollHome`/`rollbackHome`, against `refresh.providerSandboxId`, never a stale captured
+     *  name). */
+    async recreateOnImage(homeInput, imageRef, templateVersion) {
+        const current = ensureReadyHome(homeInput);
+        if (imageRef.trim() === '') {
+            throw general(`recreateOnImage for home ${current.homeId} requires a non-empty image ref`, { homeId: current.homeId });
+        }
+        if (templateVersion.trim() === '') {
+            throw general(`recreateOnImage for home ${current.homeId} requires a non-empty template version`, { homeId: current.homeId });
+        }
+        // Sweep the whole name family (destroys the failed roll-forward generation whether or not it
+        // still holds the volume — the exact Run-3 orphan gap) then wait until the volume is confirmed
+        // detached before minting a fresh name and creating (orphan-cleanup-fix.md §3.3).
+        await this.sweepAndReleaseVolume(current.providerVolumeId, current.homeId);
+        const bootstrapScript = buildBootstrapScript({
+            guestHome: this.config.guestHome,
+            crtrVersion: templateVersion,
+            relayToken: current.relayToken,
+            webPort: this.config.webPort,
+            nodeId: current.homeAgentTarget,
+            piAuthJson: this.config.piAuthJson,
+            piPackages: this.config.piPackages,
+            isFirstProvision: false,
+        });
+        const { sandboxName, route, machine } = await this.createGeneration({
+            homeId: current.homeId,
+            tenantId: current.tenantId,
+            volumeId: current.providerVolumeId,
+            bootstrapScript,
+            image: imageRef,
         });
-        return refreshedPointers(current, this.provider, route, 'running', await this.getMachine(current.providerSandboxId));
+        // Report the TARGET version, not the provider's construction-time value:
+        // the sandbox was recreated on `imageRef`, whose baked crtr is `templateVersion`.
+        return {
+            ...refreshedPointers(current, this.provider, route, 'running', sandboxName, machine),
+            templateVersion,
+        };
     }
     async suspend(homeInput) {
         const current = ensureReadyHome(homeInput);
@@ -121,7 +371,7 @@ export class BlaxelHomeBackend {
                 providerSandboxId: current.providerSandboxId,
             });
         }
-        return refreshedPointers(current, this.provider, null, machine.state, machine);
+        return refreshedPointers(current, this.provider, null, machine.state, current.providerSandboxId, machine);
     }
     async getMachine(machineId) {
         const machine = await this.provider.getMachine(machineId);