@crouton-kit/crouter 0.3.38 → 0.3.39

package/dist/hearth/control-plane/wake.js

@@ -31,10 +31,12 @@
 import { setTimeout as delay } from 'node:timers/promises';
 import { WebSocket } from 'ws';
 import { general } from '../../core/errors.js';
-import { loadM0Config } from '../../core/hearth/config.js';
+import { hearthHomeImageRefForVersion, loadM0Config } from '../../core/hearth/config.js';
 import { getHomeBackend } from '../../core/hearth/provider.js';
 import { loadControlPlaneConfig } from './config.js';
-import { getHomeById, refreshProviderPointers, setHomeStatus, touchReady, touchWake } from './registry.js';
+import { getHearthTarget, recordBadTarget } from './hearth-target.js';
+import { getHomeById, refreshProviderPointers, setHealthVerdict, setHomeStatus, touchReady, touchWake } from './registry.js';
+import { isHomeServing as isHomeServingNow } from './serving.js';
 import { resolveSecret } from './secrets.js';
 // Individual probe attempt timeout — bounded short so a dead route fails
 // fast and the retry loop gets multiple attempts inside the overall window,
@@ -199,8 +201,11 @@ async function probeNodeReady(previewEndpoint, target, relayToken, windowMs) {
  *  never reassigns these; only ad hoc verification scripts do. */
 export const wakeInternals = {
     getHomeById,
+    getHearthTarget,
+    recordBadTarget,
     refreshProviderPointers,
     setHomeStatus,
+    setHealthVerdict,
     touchWake,
     touchReady,
     resolveSecret,
@@ -237,6 +242,223 @@ export function resetWakeCacheForTests() {
     warmVerdicts.clear();
     inflightWakes.clear();
 }
+/** Is this home currently serving, so a destructive roll would interrupt it?
+ *  Delegates entirely to `serving.ts`'s independent live-session counters,
+ *  bumped at the relay's HTTP/WS lifecycle seams and the CP's OAuth
+ *  start/finish dispatch — never derived from any TTL/cache. `home.status` is
+ *  intentionally NOT consulted: a suspended home has no live counters anyway
+ *  (nothing to bump them), so this stays a single source of truth. */
+function homeIsServing(home) {
+    return isHomeServingNow(home.home_id);
+}
+/** Is this home due to roll onto the registered target? Stale = it isn't
+ *  already on the target version AND the target is not a known-bad release.
+ *  The bad-target clause is the ONLY brake in the zero-touch loop (roadmap
+ *  decision 1: never auto-advance onto a release that already failed a roll):
+ *  once `recordBadTarget` poisons the current target, no home rolls onto it
+ *  until a NEW release advances the target past it.
+ *
+ *  NOTE (design divergence, flagged): design §3's literal snippet compared
+ *  `home.template_version !== last_bad_target_version`, which would THRASH —
+ *  after a failed roll a home sits on the PRIOR version, so that clause stays
+ *  true and it would re-roll onto the same broken target forever. The roadmap's
+ *  plain-English brake and design §5's own comment both mean `target !==
+ *  last_bad`; that is what is implemented here. */
+function isHomeStale(home, target) {
+    if (home.template_version === null)
+        return false;
+    if (home.template_version === target.target_template_version)
+        return false;
+    if (target.last_bad_target_version !== null && target.target_template_version === target.last_bad_target_version)
+        return false;
+    return true;
+}
+// Baked crtr path (deploy/hearth-blaxel/Dockerfile symlink) — target it
+// directly for the in-guest version assert rather than a bare PATH lookup,
+// exactly as blaxel-bootstrap.ts does, so a stale legacy binary earlier on
+// PATH can't answer instead.
+const BAKED_CRTR_PATH = '/usr/local/bin/crtr';
+/** Assert what crtr the recreated guest is ACTUALLY running (design §4 step 7)
+ *  by reading `crtr --json sys version` in-guest — stronger than trusting the
+ *  provider's config version, since it proves the booted binary. Parses the
+ *  `.version` field from `{ "version": "X.Y.Z" }`. The same drift-guard env
+ *  vars the bootstrap uses keep this a pure read (no bootstrap/auto-init/
+ *  auto-update side effects). */
+async function assertGuestVersion(backend, sandboxName) {
+    const result = await backend.exec(sandboxName, {
+        command: `CRTR_NO_BOOTSTRAP=1 CRTR_NO_AUTO_INIT=1 CRTR_NO_AUTO_UPDATE=1 ${BAKED_CRTR_PATH} --json sys version`,
+        timeoutMs: 20_000,
+    });
+    // A parseable stdout is not proof the command itself succeeded — require a
+    // clean exit before trusting anything it printed.
+    if (result.exitCode !== 0 || result.timedOut === true) {
+        throw general(`in-guest crtr sys version did not exit cleanly (exitCode ${result.exitCode}, timedOut ${result.timedOut})`, {
+            sandboxName,
+            exitCode: result.exitCode,
+            timedOut: result.timedOut,
+            stdout: result.stdout.slice(0, 200),
+            stderr: result.stderr.slice(0, 200),
+        });
+    }
+    const stdout = result.stdout.trim();
+    try {
+        const parsed = JSON.parse(stdout);
+        if (typeof parsed.version === 'string' && parsed.version.trim() !== '')
+            return parsed.version.trim();
+    }
+    catch {
+        /* fall through to a tolerant field extraction below */
+    }
+    const match = /"version"\s*:\s*"([^"]+)"/.exec(stdout);
+    if (match !== null && match[1].trim() !== '')
+        return match[1].trim();
+    throw general(`in-guest crtr sys version returned no parseable .version (exit ${result.exitCode})`, {
+        sandboxName,
+        stdout: stdout.slice(0, 200),
+    });
+}
+/** Structured HIGH-SEVERITY alert lines. The CP is a standalone Fly process
+ *  and cannot `crtr push`; alerting is a distinct `level: 'error'` stderr line
+ *  with an `alert` marker that ops log-alerting keys on. (Wiring these to a
+ *  real notification channel is future work — the marker is the seam.) */
+function alertRollRolledBack(home, target, restoredVersion) {
+    process.stderr.write(`${JSON.stringify({
+        level: 'error',
+        alert: 'hearth.roll.rolled_back',
+        message: `Hearth auto-upgrade roll FAILED and was rolled back — release ${target.target_template_version} is broken`,
+        home_id: home.home_id,
+        bad_target_version: target.target_template_version,
+        restored_version: restoredVersion,
+    })}\n`);
+}
+export function serializeError(error) {
+    if (error instanceof Error)
+        return error.message;
+    try {
+        // `JSON.stringify` returns `undefined` (does not throw) for `undefined`, a function, or a
+        // symbol — falling through to that would OMIT the `error` field from the incident alert's
+        // JSON line entirely, which is worse than the "[object Object]" bug this rider fixes.
+        const json = JSON.stringify(error);
+        return json === undefined ? String(error) : json;
+    }
+    catch {
+        return String(error);
+    }
+}
+function alertRollIncident(home, target, priorVersion, error) {
+    process.stderr.write(`${JSON.stringify({
+        level: 'error',
+        alert: 'hearth.roll.incident',
+        message: `Hearth auto-upgrade roll AND rollback both failed — home ${home.home_id} is DEGRADED, manual recovery needed`,
+        home_id: home.home_id,
+        bad_target_version: target.target_template_version,
+        prior_version: priorVersion,
+        error: serializeError(error),
+    })}\n`);
+}
+/** Fall back onto the PRIOR pinned image after a failed roll (design §5). The
+ *  volume was never touched by the roll, so this is deterministic: recreate on
+ *  the prior image name (still in the registry — images are never pruned),
+ *  revive, ready-probe, assert `crtr sys version === priorVersion`. On success,
+ *  ALERT (the release is broken) — the bad target is already poisoned by
+ *  `rollHome`'s catch (the roll's own failure is what poisons the target, not
+ *  rollback's success, so the brake is set even when rollback ALSO fails).
+ *  On rollback also failing: mark the home
+ *  `unknown`/`degraded` and alert — a genuine incident (the volume is intact,
+ *  so manual recovery is always possible). */
+async function rollbackHome(args) {
+    const { home, target, descriptor, priorImageRef, priorVersion, relayToken, backend, readyTimeoutMs } = args;
+    try {
+        const refresh = await backend.recreateOnImage(descriptor, priorImageRef, priorVersion);
+        const previewEndpoint = requireHomeField(refresh.previewEndpoint ?? null, 'preview_endpoint', home.home_id);
+        const ready = await wakeInternals.probeNodeReady(previewEndpoint, descriptor.homeAgentTarget, relayToken, readyTimeoutMs);
+        if (!ready)
+            throw general(`rollback ready-probe timed out for ${home.home_id}`, { home_id: home.home_id });
+        // Assert against the sandbox the recreate actually reports, not a stale name captured at
+        // entry — under generation naming (recreate-race-fix.md §3.5) the pre-recreate sandbox is
+        // destroyed, so exec-ing into anything else would target a gone/wrong generation.
+        const asserted = await assertGuestVersion(backend, refresh.providerSandboxId);
+        if (asserted !== priorVersion) {
+            throw general(`rollback version mismatch: expected ${priorVersion}, got ${asserted}`, { home_id: home.home_id, expected: priorVersion, asserted });
+        }
+        wakeInternals.refreshProviderPointers(home.home_id, {
+            provider_sandbox_id: refresh.providerSandboxId,
+            preview_endpoint: previewEndpoint,
+            template_version: priorVersion,
+            status: 'running',
+        });
+        wakeInternals.setHealthVerdict(home.home_id, 'healthy');
+        alertRollRolledBack(home, target, priorVersion);
+        return { preview_endpoint: previewEndpoint, relayToken };
+    }
+    catch (rollbackError) {
+        wakeInternals.setHomeStatus(home.home_id, 'unknown');
+        wakeInternals.setHealthVerdict(home.home_id, 'degraded');
+        alertRollIncident(home, target, priorVersion, rollbackError);
+        throw general(`hearth roll AND rollback both failed for ${home.home_id}`, {
+            home_id: home.home_id,
+            target: target.target_template_version,
+            priorVersion,
+        });
+    }
+}
+/** CP-driven destroy→recreate of a stale home onto the registered target
+ *  image, reattaching the same durable volume (design §4). Captures rollback
+ *  anchors first, marks the home `restoring` (so a concurrent wake coalesces
+ *  rather than races), then recreates on the target image, ready-probes, and
+ *  ASSERTS the in-guest `crtr sys version` equals the target before committing
+ *  pointers + `health_verdict = healthy`. A roll never writes
+ *  `home_agent_target` (canvas.db is reattached byte-identical;
+ *  `refreshProviderPointers` has no such field). Any of recreate/probe/assert
+ *  failing falls back to the prior image (§5). Returns the route so the
+ *  triggering wake serves the just-upgraded home in the same call. */
+async function rollHome(home, target, relayToken, descriptor) {
+    const priorVersion = requireHomeField(home.template_version, 'template_version', home.home_id);
+    const priorImageRef = hearthHomeImageRefForVersion(priorVersion);
+    const m0 = wakeInternals.loadM0Config();
+    const backend = wakeInternals.getHomeBackend(m0);
+    const cpConfig = wakeInternals.loadControlPlaneConfig();
+    wakeInternals.setHomeStatus(home.home_id, 'restoring');
+    invalidateWarmVerdict(home.home_id);
+    const startedAt = wakeInternals.now();
+    try {
+        const refresh = await backend.recreateOnImage(descriptor, target.target_image_ref, target.target_template_version);
+        const previewEndpoint = requireHomeField(refresh.previewEndpoint ?? null, 'preview_endpoint', home.home_id);
+        const ready = await wakeInternals.probeNodeReady(previewEndpoint, descriptor.homeAgentTarget, relayToken, cpConfig.readyTimeoutMs);
+        if (!ready)
+            throw general(`roll ready-probe timed out for ${home.home_id}`, { home_id: home.home_id });
+        // Assert against the sandbox the recreate actually reports (recreate-race-fix.md §3.5) —
+        // never the row's pre-recreate `provider_sandbox_id`, which under generation naming is
+        // already destroyed by the time this runs.
+        const asserted = await assertGuestVersion(backend, refresh.providerSandboxId);
+        if (asserted !== target.target_template_version) {
+            throw general(`roll version mismatch: expected ${target.target_template_version}, got ${asserted}`, {
+                home_id: home.home_id,
+                expected: target.target_template_version,
+                asserted,
+            });
+        }
+        wakeInternals.refreshProviderPointers(home.home_id, {
+            provider_sandbox_id: refresh.providerSandboxId,
+            preview_endpoint: previewEndpoint,
+            template_version: asserted,
+            status: 'running',
+        });
+        wakeInternals.setHealthVerdict(home.home_id, 'healthy');
+        wakeInternals.touchWake(home.home_id, { at: new Date(startedAt).toISOString(), latency_ms: wakeInternals.now() - startedAt });
+        wakeInternals.touchReady(home.home_id);
+        return { preview_endpoint: previewEndpoint, relayToken };
+    }
+    catch {
+        // The target is known-bad the INSTANT its own roll fails, independent of
+        // whether rollback then recovers — poison it here, before delegating to
+        // rollbackHome, so a roll-fails+rollback-also-fails home doesn't leave the
+        // brake unset (which would let a later wake, of this home or any other,
+        // re-roll onto the same broken target).
+        wakeInternals.recordBadTarget(target.target_template_version);
+        return rollbackHome({ home, target, descriptor, priorImageRef, priorVersion, relayToken, backend, readyTimeoutMs: cpConfig.readyTimeoutMs });
+    }
+}
 async function runEnsureAwake(home_id) {
     const home = wakeInternals.getHomeById(home_id);
     if (home === null)
@@ -245,6 +467,16 @@ async function runEnsureAwake(home_id) {
     const relayToken = wakeInternals.resolveSecret(relayAuthRef);
     const target = requireHomeField(home.home_agent_target, 'home_agent_target', home_id);
     const descriptor = buildDescriptor(home, relayToken);
+    // Auto-upgrade lazy roll (design §3/§4): if a newer image target is
+    // registered, this home is stale, and it is NOT currently serving, roll it
+    // onto the target image now instead of the normal wake. At wake-entry an
+    // idle home is definitionally safe to tear down; a serving home doesn't roll
+    // this pass (it rolls on its next cold wake). This is the ONLY new branch —
+    // the normal wake path below is unchanged.
+    const hearthTarget = wakeInternals.getHearthTarget();
+    if (hearthTarget !== null && isHomeStale(home, hearthTarget) && !homeIsServing(home)) {
+        return rollHome(home, hearthTarget, relayToken, descriptor);
+    }
     const startedAt = wakeInternals.now();
     const m0 = wakeInternals.loadM0Config();
     const refresh = await wakeInternals.getHomeBackend(m0).wake(descriptor);
@@ -277,12 +509,18 @@ async function runEnsureAwake(home_id) {
     const readyAt = new Date().toISOString();
     wakeInternals.touchWake(home_id, { at: new Date(startedAt).toISOString(), latency_ms: wakeInternals.now() - startedAt });
     wakeInternals.touchReady(home_id, readyAt);
+    // A successful ready-probe means the home is healthy right now, regardless of what the
+    // verdict was before this wake (e.g. left `degraded` by an earlier failed rollback) —
+    // clear it here the same way rollHome/rollbackHome clear it on their own success, or a
+    // stale degraded/unknown verdict never heals on the normal wake path.
+    wakeInternals.setHealthVerdict(home_id, 'healthy');
     return { preview_endpoint: resolvedPreviewEndpoint, relayToken };
 }
 /** Bring a home up and ready, returning the route to relay to once it is.
  *  Reads the registry, resolves the relay token, calls the provider
  *  wake/resume seam, ready-probes `/node/<home_agent_target>`, refreshes
- *  provider pointers on a recreate, records wake/ready timing, and returns
+ *  provider pointers on a recreate, records wake/ready timing, clears the
+ *  health verdict to `healthy` on success, and returns
  *  `{ preview_endpoint, relayToken }`. Throws on a ready-probe timeout
  *  (wake-failure UX is deferred — gap note 2 — a throw is the accepted
  *  cutover behavior).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crouton-kit/crouter",
-  "version": "0.3.38",
+  "version": "0.3.39",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",