@crouton-kit/crouter 0.3.38 → 0.3.40

package/dist/hearth/control-plane/server.js

@@ -53,10 +53,14 @@
 // handed to the dispatcher, which today simply reports "not implemented yet"
 // without contacting any home.
 import { createServer } from 'node:http';
+import { createHash, timingSafeEqual } from 'node:crypto';
+import { hearthHomeImageRefForVersion } from '../../core/hearth/config.js';
 import { getControlPlaneDb } from './db.js';
 import { resolveHome } from './registry.js';
+import { setHearthTarget } from './hearth-target.js';
 import { resolveTenantSession, resolveTenantWriteAuth, swapQueryTokenForCookie, tenantSessionAuthRedactedMessage } from './session.js';
-import { redactSecrets } from './secrets.js';
+import { redactSecrets, resolveCpAdminToken } from './secrets.js';
+import { beginOAuth, endOAuth } from './serving.js';
 import { handleWebhookRequest, isWebhookPath, isWebhookRequest } from './register.js';
 const notImplementedRelay = {
     proxyHttp(_req, res) {
@@ -89,6 +93,106 @@ function redirectChat(res, target) {
 function respondHealthz(res) {
     sendText(res, 200, 'ok\n');
 }
+const ADMIN_HEARTH_IMAGE_PATH = '/__admin/hearth-image';
+// The auto-upgrade notify body is tiny ({template_version, image_ref}); cap
+// hard so a malformed/oversized POST can't buffer unbounded.
+const ADMIN_BODY_MAX_BYTES = 4 * 1024;
+const STRICT_SEMVER_RE = /^[0-9]+\.[0-9]+\.[0-9]+$/;
+/** Fixed-length digest of a UTF-8 string — always 32 bytes (sha256), so
+ *  comparing two digests never branches on the ORIGINAL strings' lengths. */
+function sha256(value) {
+    return createHash('sha256').update(value, 'utf8').digest();
+}
+/** Constant-time bearer check against the configured CP admin token. Fails
+ *  CLOSED: no `Authorization: Bearer` header, or no token configured on the
+ *  CP, or a mismatch — all return false without leaking which. Compares
+ *  sha256 digests (always 32 bytes each) rather than the raw token buffers,
+ *  so there is no length-inequality short-circuit that would leak the
+ *  configured token's length before `timingSafeEqual` runs — equal plaintext
+ *  still implies equal digest, and every digest is the same size regardless
+ *  of what was presented. */
+function isAdminAuthorized(req) {
+    const configured = resolveCpAdminToken();
+    if (configured === null || configured === '')
+        return false;
+    const header = req.headers.authorization;
+    if (typeof header !== 'string')
+        return false;
+    const match = /^Bearer[ ]+(.+)$/.exec(header.trim());
+    if (match === null)
+        return false;
+    return timingSafeEqual(sha256(match[1]), sha256(configured));
+}
+function readRequestBody(req, maxBytes) {
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        let total = 0;
+        let tooLarge = false;
+        req.on('data', (chunk) => {
+            total += chunk.length;
+            if (total > maxBytes) {
+                tooLarge = true;
+                req.destroy();
+                return;
+            }
+            chunks.push(chunk);
+        });
+        req.on('end', () => resolve({ body: Buffer.concat(chunks), tooLarge }));
+        req.on('error', (error) => {
+            if (tooLarge) {
+                resolve({ body: Buffer.alloc(0), tooLarge: true });
+                return;
+            }
+            reject(error instanceof Error ? error : new Error(String(error)));
+        });
+    });
+}
+/** `POST /__admin/hearth-image` — the image-build workflow's push-notify
+ *  (design §2-C). Bearer-authed against the CP admin token (constant-time),
+ *  BEFORE and outside the session/registry gate (it is a machine ingress, not
+ *  a browser lane). Validates the body is strict semver + `image_ref ===
+ *  hearthHomeImageRefForVersion(template_version)` (the notify cannot register
+ *  an image name that doesn't match its version), then idempotently upserts
+ *  the single global target row — a re-post of the same version is a no-op. It
+ *  never contacts a home; the actual roll happens lazily on each home's next
+ *  wake (wake.ts). */
+async function respondAdminHearthImage(req, res, db) {
+    if (!isAdminAuthorized(req)) {
+        sendText(res, 401, 'admin authorization required\n');
+        return;
+    }
+    const { body, tooLarge } = await readRequestBody(req, ADMIN_BODY_MAX_BYTES);
+    if (tooLarge) {
+        sendText(res, 413, 'request body too large\n');
+        return;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(body.toString('utf8'));
+    }
+    catch {
+        sendText(res, 400, 'body must be valid JSON\n');
+        return;
+    }
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+        sendText(res, 400, 'body must be a JSON object with template_version and image_ref\n');
+        return;
+    }
+    const record = parsed;
+    const templateVersion = typeof record.template_version === 'string' ? record.template_version.trim() : '';
+    const imageRef = typeof record.image_ref === 'string' ? record.image_ref.trim() : '';
+    if (!STRICT_SEMVER_RE.test(templateVersion)) {
+        sendText(res, 400, 'template_version must be strict semver (X.Y.Z)\n');
+        return;
+    }
+    const expectedImageRef = hearthHomeImageRefForVersion(templateVersion);
+    if (imageRef !== expectedImageRef) {
+        sendText(res, 400, `image_ref must equal ${expectedImageRef} for template_version ${templateVersion}\n`);
+        return;
+    }
+    const target = setHearthTarget({ target_template_version: templateVersion, target_image_ref: imageRef, source: 'image-workflow' }, db);
+    sendText(res, 200, `hearth target = ${target.target_template_version} (${target.target_image_ref})\n`);
+}
 /** `?token=` is intentionally not accepted as durable auth (`session.ts`); the
  *  one allowed query-token use is the single-use M0 handoff swap-to-cookie,
  *  attempted here only when no durable cookie/bearer session is already
@@ -128,12 +232,14 @@ function respondRoot(req, res, db, config, store) {
     }
     redirectChat(res, target);
 }
+const MODEL_AUTH_START_PATH = '/__hearth/model-auth/anthropic/start';
+const MODEL_AUTH_FINISH_PATH = '/__hearth/model-auth/anthropic/finish';
 const DYNAMIC_ROUTES = [
     { method: 'POST', path: '/__crtr/source', auth: 'write' },
     { method: 'GET', path: '/__crtr/events', auth: 'read' },
     { method: 'GET', path: '/__hearth/model-auth/anthropic/status', auth: 'read' },
-    { method: 'POST', path: '/__hearth/model-auth/anthropic/start', auth: 'write' },
-    { method: 'POST', path: '/__hearth/model-auth/anthropic/finish', auth: 'write' },
+    { method: 'POST', path: MODEL_AUTH_START_PATH, auth: 'write' },
+    { method: 'POST', path: MODEL_AUTH_FINISH_PATH, auth: 'write' },
 ];
 const DYNAMIC_ROUTE_PATHS = new Set(DYNAMIC_ROUTES.map((route) => route.path));
 function findDynamicRoute(method, pathname) {
@@ -177,6 +283,13 @@ function isRelayableViewOrAssetPath(pathname) {
  *  (or, for `auth: 'write'`, an untrusted Origin) or an unresolved home
  *  rejects before the relay dispatcher (and therefore before any
  *  wake/provider/home contact) is ever invoked. */
+/** OAuth-in-progress marker: the CP's own proxied `start`/`finish` dispatch
+ *  is the seam — mark a window open the moment a
+ *  `start` is about to be proxied (regardless of what the guest does with
+ *  it), close it once the `finish` proxy call resolves (success OR failure;
+ *  a failed finish still ends the CP-visible window — the bounded TTL in
+ *  `serving.ts` is the backstop for a `start` that never gets a `finish` at
+ *  all, e.g. an abandoned browser tab). */
 async function respondDynamicLane(req, res, db, config, store, relay, auth) {
     const session = auth === 'write'
         ? resolveTenantWriteAuth(req, { store, publicOrigin: config.publicOrigin })
@@ -190,7 +303,18 @@ async function respondDynamicLane(req, res, db, config, store, relay, auth) {
         sendText(res, 404, 'home not found\n');
         return;
     }
-    await relay.proxyHttp(req, res, home);
+    const pathname = requestUrl(req).pathname;
+    const isOAuthStart = pathname === MODEL_AUTH_START_PATH;
+    const isOAuthFinish = pathname === MODEL_AUTH_FINISH_PATH;
+    if (isOAuthStart)
+        beginOAuth(home.home_id);
+    try {
+        await relay.proxyHttp(req, res, home);
+    }
+    finally {
+        if (isOAuthFinish)
+            endOAuth(home.home_id);
+    }
 }
 async function routeControlPlaneRequest(req, res, db, config, store, relay) {
     const url = requestUrl(req);
@@ -215,6 +339,18 @@ async function routeControlPlaneRequest(req, res, db, config, store, relay) {
         await handleWebhookRequest(req, res);
         return;
     }
+    // Admin push-notify lane. Authenticated by the CP admin bearer token inside
+    // the handler, so — like the webhook lane — it sits BEFORE and outside the
+    // session/registry gate. A wrong method on the admin path is a 405, never a
+    // fall-through to the relay.
+    if (url.pathname === ADMIN_HEARTH_IMAGE_PATH) {
+        if (req.method !== 'POST') {
+            sendText(res, 405, 'method not allowed\n');
+            return;
+        }
+        await respondAdminHearthImage(req, res, db);
+        return;
+    }
     if (DYNAMIC_ROUTE_PATHS.has(url.pathname)) {
         const route = findDynamicRoute(req.method, url.pathname);
         if (route === undefined) {

package/dist/hearth/control-plane/serving.d.ts

@@ -0,0 +1,15 @@
+import type { HomeId } from './types.js';
+export declare function beginHttp(id: HomeId): void;
+export declare function endHttp(id: HomeId): void;
+export declare function beginWs(id: HomeId): void;
+export declare function endWs(id: HomeId): void;
+export declare function beginOAuth(id: HomeId): void;
+export declare function endOAuth(id: HomeId): void;
+/** Is `id` currently serving — any active relay HTTP request, any open relay
+ *  WebSocket bridge, or a live (non-stale) OAuth start->finish window? This is
+ *  the ONLY signal `wake.ts`'s `homeIsServing` consults — it does not depend
+ *  on `resolveRunning`/`WARM_VERDICT_TTL_MS` at all. */
+export declare function isHomeServing(id: HomeId): boolean;
+/** Test/smoke-script hygiene only — production never needs this (one CP
+ *  process, S2, and every counter self-clears on its own end/dec path). */
+export declare function resetServingForTests(): void;

package/dist/hearth/control-plane/serving.js

@@ -0,0 +1,106 @@
+// control-plane/serving.ts — CP-layer active-session tracker, independent of
+// the wake path's warm-verdict TTL.
+//
+// `wake.ts`'s `resolveRunning`/`WARM_VERDICT_TTL_MS` answers "did we recently
+// confirm this home is up" for exactly 10s — a cheap fast-path cache, not a
+// live-session signal. A destructive roll must never fire while a home has
+// an open browser/broker WebSocket, an in-flight HTTP request, or an open
+// OAuth start->finish window, no matter how long that session has been
+// running (design §3's "no active turn / no in-flight OAuth" safety
+// invariant).
+//
+// Three independent live-count maps, keyed by HomeId, are bumped directly at
+// the relay's HTTP/WS lifecycle seams and at the CP's OAuth start/finish
+// dispatch — NOT derived from any cache/TTL. A home is "serving" whenever
+// any one of them is non-zero. Single CP process (S2), in-process only, no
+// persistence — a CP restart naturally drops to zero, which is correct (a
+// fresh process has no in-flight anything).
+const httpInflight = new Map();
+const wsOpen = new Map();
+const oauthInProgress = new Map();
+// Timestamp of the most recent `beginOAuth` call for this home — the TTL
+// clock for the bounded-safety rule below. Refreshed on every `beginOAuth`,
+// so each new `start` gets its own full TTL window regardless of how stale
+// an earlier, still-uncompleted start's marker had become. Cleared once the
+// counter returns to 0 (no window is open, nothing to bound).
+const oauthStartedAt = new Map();
+// Bounded safety — "do not let a dropped OAuth pin a home un-rollable
+// forever": a `start` with no matching `finish` (browser tab closed
+// mid-flow, network drop, abandoned auth) must not hold `isHomeServing` true
+// indefinitely. Chose a TTL over a "clear on idle" scheme because idle here
+// has no other natural signal to key off (no heartbeat on the OAuth lane)
+// and a TTL is simplest to reason about / test. 5 minutes is generous for a
+// real human completing an OAuth redirect+approve+callback round trip (the
+// M0 in-guest flow is a single localhost redirect, seconds in the happy
+// path) while still bounding the worst case to a single-digit-minutes window
+// rather than forever.
+const OAUTH_MARKER_TTL_MS = 5 * 60_000;
+function inc(map, id) {
+    map.set(id, (map.get(id) ?? 0) + 1);
+}
+function dec(map, id) {
+    const current = map.get(id) ?? 0;
+    const next = current - 1;
+    if (next <= 0) {
+        map.delete(id);
+    }
+    else {
+        map.set(id, next);
+    }
+}
+export function beginHttp(id) {
+    inc(httpInflight, id);
+}
+export function endHttp(id) {
+    dec(httpInflight, id);
+}
+export function beginWs(id) {
+    inc(wsOpen, id);
+}
+export function endWs(id) {
+    dec(wsOpen, id);
+}
+export function beginOAuth(id) {
+    // Every real OAuth `start` is a fresh auth window and earns its own full
+    // TTL, not just the first one since the counter hit 0 — otherwise a stale,
+    // abandoned start's marker would age out from under a brand-new retry that
+    // shares its counter, letting that retry's own wake see `!isHomeServing`.
+    oauthStartedAt.set(id, Date.now());
+    inc(oauthInProgress, id);
+}
+export function endOAuth(id) {
+    dec(oauthInProgress, id);
+    if ((oauthInProgress.get(id) ?? 0) === 0)
+        oauthStartedAt.delete(id);
+}
+/** Is an OAuth window open for `id` right now, per the counter, and still
+ *  within the bounded-safety TTL of the MOST RECENT `beginOAuth`? A marker
+ *  past TTL is treated as stale (does NOT count as serving) so a dropped
+ *  `start` can't pin a home un-rollable forever; the counter itself is left
+ *  alone (an eventual late `finish` still decrements it correctly, and `dec`
+ *  floors at 0 regardless), and a later `beginOAuth` on the same counter
+ *  refreshes the timestamp so its own window is judged live. */
+function oauthMarkerLive(id) {
+    const count = oauthInProgress.get(id) ?? 0;
+    if (count <= 0)
+        return false;
+    const startedAt = oauthStartedAt.get(id);
+    if (startedAt === undefined)
+        return true; // defensive: count>0 with no timestamp — treat as live
+    return Date.now() - startedAt < OAUTH_MARKER_TTL_MS;
+}
+/** Is `id` currently serving — any active relay HTTP request, any open relay
+ *  WebSocket bridge, or a live (non-stale) OAuth start->finish window? This is
+ *  the ONLY signal `wake.ts`'s `homeIsServing` consults — it does not depend
+ *  on `resolveRunning`/`WARM_VERDICT_TTL_MS` at all. */
+export function isHomeServing(id) {
+    return (httpInflight.get(id) ?? 0) > 0 || (wsOpen.get(id) ?? 0) > 0 || oauthMarkerLive(id);
+}
+/** Test/smoke-script hygiene only — production never needs this (one CP
+ *  process, S2, and every counter self-clears on its own end/dec path). */
+export function resetServingForTests() {
+    httpInflight.clear();
+    wsOpen.clear();
+    oauthInProgress.clear();
+    oauthStartedAt.clear();
+}

package/dist/hearth/control-plane/types.d.ts

@@ -64,3 +64,23 @@ export interface TenantSession {
     tenant_id: TenantId;
     user_id: UserId;
 }
+/** The single global auto-upgrade target row (`hearth_target`, db.ts v9).
+ *  `channel` is always `'home'` (single-tenant, one image line). The DESIRED
+ *  crtr version/image the fleet rolls onto; `home.template_version` is the
+ *  ACTUAL running version. `last_bad_target_version`, when set, is a release
+ *  whose live roll already failed — the wake-path stale predicate excludes it
+ *  so no home re-rolls onto it until the target advances. */
+export interface HearthTarget {
+    channel: 'home';
+    target_template_version: string;
+    target_image_ref: string;
+    published_at: string;
+    source: string;
+    last_bad_target_version: string | null;
+}
+export interface SetHearthTargetInput {
+    target_template_version: string;
+    target_image_ref: string;
+    source: string;
+    published_at?: string;
+}

package/dist/hearth/control-plane/wake.d.ts

@@ -1,7 +1,8 @@
 import { type HomeBackend } from '../../core/hearth/provider.js';
 import type { M0Config } from '../../core/hearth/types.js';
 import { type ControlPlaneConfig } from './config.js';
-import { getHomeById, refreshProviderPointers, setHomeStatus, touchReady, touchWake } from './registry.js';
+import { getHearthTarget, recordBadTarget } from './hearth-target.js';
+import { getHomeById, refreshProviderPointers, setHealthVerdict, setHomeStatus, touchReady, touchWake } from './registry.js';
 import { resolveSecret } from './secrets.js';
 import type { HomeId, HomeStatus } from './types.js';
 export interface WakeResult {
@@ -40,8 +41,11 @@ declare function probeNodeReady(previewEndpoint: string, target: string, relayTo
  *  never reassigns these; only ad hoc verification scripts do. */
 export declare const wakeInternals: {
     getHomeById: typeof getHomeById;
+    getHearthTarget: typeof getHearthTarget;
+    recordBadTarget: typeof recordBadTarget;
     refreshProviderPointers: typeof refreshProviderPointers;
     setHomeStatus: typeof setHomeStatus;
+    setHealthVerdict: typeof setHealthVerdict;
     touchWake: typeof touchWake;
     touchReady: typeof touchReady;
     resolveSecret: typeof resolveSecret;
@@ -64,10 +68,12 @@ export declare function invalidateWarmVerdict(home_id: HomeId): void;
 /** Clear all warm verdicts and in-flight wake coalescing. Test/smoke-script
  *  hygiene only — production never needs this (one CP process, S2). */
 export declare function resetWakeCacheForTests(): void;
+export declare function serializeError(error: unknown): string;
 /** Bring a home up and ready, returning the route to relay to once it is.
  *  Reads the registry, resolves the relay token, calls the provider
  *  wake/resume seam, ready-probes `/node/<home_agent_target>`, refreshes
- *  provider pointers on a recreate, records wake/ready timing, and returns
+ *  provider pointers on a recreate, records wake/ready timing, clears the
+ *  health verdict to `healthy` on success, and returns
  *  `{ preview_endpoint, relayToken }`. Throws on a ready-probe timeout
  *  (wake-failure UX is deferred — gap note 2 — a throw is the accepted
  *  cutover behavior).

package/dist/hearth/control-plane/wake.js

@@ -31,10 +31,12 @@
 import { setTimeout as delay } from 'node:timers/promises';
 import { WebSocket } from 'ws';
 import { general } from '../../core/errors.js';
-import { loadM0Config } from '../../core/hearth/config.js';
+import { hearthHomeImageRefForVersion, loadM0Config } from '../../core/hearth/config.js';
 import { getHomeBackend } from '../../core/hearth/provider.js';
 import { loadControlPlaneConfig } from './config.js';
-import { getHomeById, refreshProviderPointers, setHomeStatus, touchReady, touchWake } from './registry.js';
+import { getHearthTarget, recordBadTarget } from './hearth-target.js';
+import { getHomeById, refreshProviderPointers, setHealthVerdict, setHomeStatus, touchReady, touchWake } from './registry.js';
+import { isHomeServing as isHomeServingNow } from './serving.js';
 import { resolveSecret } from './secrets.js';
 // Individual probe attempt timeout — bounded short so a dead route fails
 // fast and the retry loop gets multiple attempts inside the overall window,
@@ -199,8 +201,11 @@ async function probeNodeReady(previewEndpoint, target, relayToken, windowMs) {
  *  never reassigns these; only ad hoc verification scripts do. */
 export const wakeInternals = {
     getHomeById,
+    getHearthTarget,
+    recordBadTarget,
     refreshProviderPointers,
     setHomeStatus,
+    setHealthVerdict,
     touchWake,
     touchReady,
     resolveSecret,
@@ -237,6 +242,223 @@ export function resetWakeCacheForTests() {
     warmVerdicts.clear();
     inflightWakes.clear();
 }
+/** Is this home currently serving, so a destructive roll would interrupt it?
+ *  Delegates entirely to `serving.ts`'s independent live-session counters,
+ *  bumped at the relay's HTTP/WS lifecycle seams and the CP's OAuth
+ *  start/finish dispatch — never derived from any TTL/cache. `home.status` is
+ *  intentionally NOT consulted: a suspended home has no live counters anyway
+ *  (nothing to bump them), so this stays a single source of truth. */
+function homeIsServing(home) {
+    return isHomeServingNow(home.home_id);
+}
+/** Is this home due to roll onto the registered target? Stale = it isn't
+ *  already on the target version AND the target is not a known-bad release.
+ *  The bad-target clause is the ONLY brake in the zero-touch loop (roadmap
+ *  decision 1: never auto-advance onto a release that already failed a roll):
+ *  once `recordBadTarget` poisons the current target, no home rolls onto it
+ *  until a NEW release advances the target past it.
+ *
+ *  NOTE (design divergence, flagged): design §3's literal snippet compared
+ *  `home.template_version !== last_bad_target_version`, which would THRASH —
+ *  after a failed roll a home sits on the PRIOR version, so that clause stays
+ *  true and it would re-roll onto the same broken target forever. The roadmap's
+ *  plain-English brake and design §5's own comment both mean `target !==
+ *  last_bad`; that is what is implemented here. */
+function isHomeStale(home, target) {
+    if (home.template_version === null)
+        return false;
+    if (home.template_version === target.target_template_version)
+        return false;
+    if (target.last_bad_target_version !== null && target.target_template_version === target.last_bad_target_version)
+        return false;
+    return true;
+}
+// Baked crtr path (deploy/hearth-blaxel/Dockerfile symlink) — target it
+// directly for the in-guest version assert rather than a bare PATH lookup,
+// exactly as blaxel-bootstrap.ts does, so a stale legacy binary earlier on
+// PATH can't answer instead.
+const BAKED_CRTR_PATH = '/usr/local/bin/crtr';
+/** Assert what crtr the recreated guest is ACTUALLY running (design §4 step 7)
+ *  by reading `crtr --json sys version` in-guest — stronger than trusting the
+ *  provider's config version, since it proves the booted binary. Parses the
+ *  `.version` field from `{ "version": "X.Y.Z" }`. The same drift-guard env
+ *  vars the bootstrap uses keep this a pure read (no bootstrap/auto-init/
+ *  auto-update side effects). */
+async function assertGuestVersion(backend, sandboxName) {
+    const result = await backend.exec(sandboxName, {
+        command: `CRTR_NO_BOOTSTRAP=1 CRTR_NO_AUTO_INIT=1 CRTR_NO_AUTO_UPDATE=1 ${BAKED_CRTR_PATH} --json sys version`,
+        timeoutMs: 20_000,
+    });
+    // A parseable stdout is not proof the command itself succeeded — require a
+    // clean exit before trusting anything it printed.
+    if (result.exitCode !== 0 || result.timedOut === true) {
+        throw general(`in-guest crtr sys version did not exit cleanly (exitCode ${result.exitCode}, timedOut ${result.timedOut})`, {
+            sandboxName,
+            exitCode: result.exitCode,
+            timedOut: result.timedOut,
+            stdout: result.stdout.slice(0, 200),
+            stderr: result.stderr.slice(0, 200),
+        });
+    }
+    const stdout = result.stdout.trim();
+    try {
+        const parsed = JSON.parse(stdout);
+        if (typeof parsed.version === 'string' && parsed.version.trim() !== '')
+            return parsed.version.trim();
+    }
+    catch {
+        /* fall through to a tolerant field extraction below */
+    }
+    const match = /"version"\s*:\s*"([^"]+)"/.exec(stdout);
+    if (match !== null && match[1].trim() !== '')
+        return match[1].trim();
+    throw general(`in-guest crtr sys version returned no parseable .version (exit ${result.exitCode})`, {
+        sandboxName,
+        stdout: stdout.slice(0, 200),
+    });
+}
+/** Structured HIGH-SEVERITY alert lines. The CP is a standalone Fly process
+ *  and cannot `crtr push`; alerting is a distinct `level: 'error'` stderr line
+ *  with an `alert` marker that ops log-alerting keys on. (Wiring these to a
+ *  real notification channel is future work — the marker is the seam.) */
+function alertRollRolledBack(home, target, restoredVersion) {
+    process.stderr.write(`${JSON.stringify({
+        level: 'error',
+        alert: 'hearth.roll.rolled_back',
+        message: `Hearth auto-upgrade roll FAILED and was rolled back — release ${target.target_template_version} is broken`,
+        home_id: home.home_id,
+        bad_target_version: target.target_template_version,
+        restored_version: restoredVersion,
+    })}\n`);
+}
+export function serializeError(error) {
+    if (error instanceof Error)
+        return error.message;
+    try {
+        // `JSON.stringify` returns `undefined` (does not throw) for `undefined`, a function, or a
+        // symbol — falling through to that would OMIT the `error` field from the incident alert's
+        // JSON line entirely, which is worse than the "[object Object]" bug this rider fixes.
+        const json = JSON.stringify(error);
+        return json === undefined ? String(error) : json;
+    }
+    catch {
+        return String(error);
+    }
+}
+function alertRollIncident(home, target, priorVersion, error) {
+    process.stderr.write(`${JSON.stringify({
+        level: 'error',
+        alert: 'hearth.roll.incident',
+        message: `Hearth auto-upgrade roll AND rollback both failed — home ${home.home_id} is DEGRADED, manual recovery needed`,
+        home_id: home.home_id,
+        bad_target_version: target.target_template_version,
+        prior_version: priorVersion,
+        error: serializeError(error),
+    })}\n`);
+}
+/** Fall back onto the PRIOR pinned image after a failed roll (design §5). The
+ *  volume was never touched by the roll, so this is deterministic: recreate on
+ *  the prior image name (still in the registry — images are never pruned),
+ *  revive, ready-probe, assert `crtr sys version === priorVersion`. On success,
+ *  ALERT (the release is broken) — the bad target is already poisoned by
+ *  `rollHome`'s catch (the roll's own failure is what poisons the target, not
+ *  rollback's success, so the brake is set even when rollback ALSO fails).
+ *  On rollback also failing: mark the home
+ *  `unknown`/`degraded` and alert — a genuine incident (the volume is intact,
+ *  so manual recovery is always possible). */
+async function rollbackHome(args) {
+    const { home, target, descriptor, priorImageRef, priorVersion, relayToken, backend, readyTimeoutMs } = args;
+    try {
+        const refresh = await backend.recreateOnImage(descriptor, priorImageRef, priorVersion);
+        const previewEndpoint = requireHomeField(refresh.previewEndpoint ?? null, 'preview_endpoint', home.home_id);
+        const ready = await wakeInternals.probeNodeReady(previewEndpoint, descriptor.homeAgentTarget, relayToken, readyTimeoutMs);
+        if (!ready)
+            throw general(`rollback ready-probe timed out for ${home.home_id}`, { home_id: home.home_id });
+        // Assert against the sandbox the recreate actually reports, not a stale name captured at
+        // entry — under generation naming (recreate-race-fix.md §3.5) the pre-recreate sandbox is
+        // destroyed, so exec-ing into anything else would target a gone/wrong generation.
+        const asserted = await assertGuestVersion(backend, refresh.providerSandboxId);
+        if (asserted !== priorVersion) {
+            throw general(`rollback version mismatch: expected ${priorVersion}, got ${asserted}`, { home_id: home.home_id, expected: priorVersion, asserted });
+        }
+        wakeInternals.refreshProviderPointers(home.home_id, {
+            provider_sandbox_id: refresh.providerSandboxId,
+            preview_endpoint: previewEndpoint,
+            template_version: priorVersion,
+            status: 'running',
+        });
+        wakeInternals.setHealthVerdict(home.home_id, 'healthy');
+        alertRollRolledBack(home, target, priorVersion);
+        return { preview_endpoint: previewEndpoint, relayToken };
+    }
+    catch (rollbackError) {
+        wakeInternals.setHomeStatus(home.home_id, 'unknown');
+        wakeInternals.setHealthVerdict(home.home_id, 'degraded');
+        alertRollIncident(home, target, priorVersion, rollbackError);
+        throw general(`hearth roll AND rollback both failed for ${home.home_id}`, {
+            home_id: home.home_id,
+            target: target.target_template_version,
+            priorVersion,
+        });
+    }
+}
+/** CP-driven destroy→recreate of a stale home onto the registered target
+ *  image, reattaching the same durable volume (design §4). Captures rollback
+ *  anchors first, marks the home `restoring` (so a concurrent wake coalesces
+ *  rather than races), then recreates on the target image, ready-probes, and
+ *  ASSERTS the in-guest `crtr sys version` equals the target before committing
+ *  pointers + `health_verdict = healthy`. A roll never writes
+ *  `home_agent_target` (canvas.db is reattached byte-identical;
+ *  `refreshProviderPointers` has no such field). Any of recreate/probe/assert
+ *  failing falls back to the prior image (§5). Returns the route so the
+ *  triggering wake serves the just-upgraded home in the same call. */
+async function rollHome(home, target, relayToken, descriptor) {
+    const priorVersion = requireHomeField(home.template_version, 'template_version', home.home_id);
+    const priorImageRef = hearthHomeImageRefForVersion(priorVersion);
+    const m0 = wakeInternals.loadM0Config();
+    const backend = wakeInternals.getHomeBackend(m0);
+    const cpConfig = wakeInternals.loadControlPlaneConfig();
+    wakeInternals.setHomeStatus(home.home_id, 'restoring');
+    invalidateWarmVerdict(home.home_id);
+    const startedAt = wakeInternals.now();
+    try {
+        const refresh = await backend.recreateOnImage(descriptor, target.target_image_ref, target.target_template_version);
+        const previewEndpoint = requireHomeField(refresh.previewEndpoint ?? null, 'preview_endpoint', home.home_id);
+        const ready = await wakeInternals.probeNodeReady(previewEndpoint, descriptor.homeAgentTarget, relayToken, cpConfig.readyTimeoutMs);
+        if (!ready)
+            throw general(`roll ready-probe timed out for ${home.home_id}`, { home_id: home.home_id });
+        // Assert against the sandbox the recreate actually reports (recreate-race-fix.md §3.5) —
+        // never the row's pre-recreate `provider_sandbox_id`, which under generation naming is
+        // already destroyed by the time this runs.
+        const asserted = await assertGuestVersion(backend, refresh.providerSandboxId);
+        if (asserted !== target.target_template_version) {
+            throw general(`roll version mismatch: expected ${target.target_template_version}, got ${asserted}`, {
+                home_id: home.home_id,
+                expected: target.target_template_version,
+                asserted,
+            });
+        }
+        wakeInternals.refreshProviderPointers(home.home_id, {
+            provider_sandbox_id: refresh.providerSandboxId,
+            preview_endpoint: previewEndpoint,
+            template_version: asserted,
+            status: 'running',
+        });
+        wakeInternals.setHealthVerdict(home.home_id, 'healthy');
+        wakeInternals.touchWake(home.home_id, { at: new Date(startedAt).toISOString(), latency_ms: wakeInternals.now() - startedAt });
+        wakeInternals.touchReady(home.home_id);
+        return { preview_endpoint: previewEndpoint, relayToken };
+    }
+    catch {
+        // The target is known-bad the INSTANT its own roll fails, independent of
+        // whether rollback then recovers — poison it here, before delegating to
+        // rollbackHome, so a roll-fails+rollback-also-fails home doesn't leave the
+        // brake unset (which would let a later wake, of this home or any other,
+        // re-roll onto the same broken target).
+        wakeInternals.recordBadTarget(target.target_template_version);
+        return rollbackHome({ home, target, descriptor, priorImageRef, priorVersion, relayToken, backend, readyTimeoutMs: cpConfig.readyTimeoutMs });
+    }
+}
 async function runEnsureAwake(home_id) {
     const home = wakeInternals.getHomeById(home_id);
     if (home === null)
@@ -245,6 +467,16 @@ async function runEnsureAwake(home_id) {
     const relayToken = wakeInternals.resolveSecret(relayAuthRef);
     const target = requireHomeField(home.home_agent_target, 'home_agent_target', home_id);
     const descriptor = buildDescriptor(home, relayToken);
+    // Auto-upgrade lazy roll (design §3/§4): if a newer image target is
+    // registered, this home is stale, and it is NOT currently serving, roll it
+    // onto the target image now instead of the normal wake. At wake-entry an
+    // idle home is definitionally safe to tear down; a serving home doesn't roll
+    // this pass (it rolls on its next cold wake). This is the ONLY new branch —
+    // the normal wake path below is unchanged.
+    const hearthTarget = wakeInternals.getHearthTarget();
+    if (hearthTarget !== null && isHomeStale(home, hearthTarget) && !homeIsServing(home)) {
+        return rollHome(home, hearthTarget, relayToken, descriptor);
+    }
     const startedAt = wakeInternals.now();
     const m0 = wakeInternals.loadM0Config();
     const refresh = await wakeInternals.getHomeBackend(m0).wake(descriptor);
@@ -277,12 +509,18 @@ async function runEnsureAwake(home_id) {
     const readyAt = new Date().toISOString();
     wakeInternals.touchWake(home_id, { at: new Date(startedAt).toISOString(), latency_ms: wakeInternals.now() - startedAt });
     wakeInternals.touchReady(home_id, readyAt);
+    // A successful ready-probe means the home is healthy right now, regardless of what the
+    // verdict was before this wake (e.g. left `degraded` by an earlier failed rollback) —
+    // clear it here the same way rollHome/rollbackHome clear it on their own success, or a
+    // stale degraded/unknown verdict never heals on the normal wake path.
+    wakeInternals.setHealthVerdict(home_id, 'healthy');
     return { preview_endpoint: resolvedPreviewEndpoint, relayToken };
 }
 /** Bring a home up and ready, returning the route to relay to once it is.
  *  Reads the registry, resolves the relay token, calls the provider
  *  wake/resume seam, ready-probes `/node/<home_agent_target>`, refreshes
- *  provider pointers on a recreate, records wake/ready timing, and returns
+ *  provider pointers on a recreate, records wake/ready timing, clears the
+ *  health verdict to `healthy` on success, and returns
  *  `{ preview_endpoint, relayToken }`. Throws on a ready-probe timeout
  *  (wake-failure UX is deferred — gap note 2 — a throw is the accepted
  *  cutover behavior).