@superblocksteam/sdk 2.0.151 → 2.0.152-next.1

package/dist/cli-replacement/dev.mjs

@@ -19,6 +19,7 @@ import { AutoConnectingRpcClient } from "@superblocksteam/vite-plugin-file-sync/
 import { SyncService } from "@superblocksteam/vite-plugin-file-sync/sync-service";
 import { devServerMetrics } from "../dev-utils/dev-server-metrics.mjs";
 import { createDevServer } from "../dev-utils/dev-server.mjs";
+import { buildFatalExitLog, FATAL_EXIT_EVENT, parseIsWarm, } from "../dev-utils/fatal-exit.mjs";
 import { AUTO_UPGRADE_EXIT_CODE } from "../index.js";
 import { getTracer, isCloudPrem, shutdownTelemetry, } from "../telemetry/index.js";
 import { getErrorMeta, getLogger } from "../telemetry/logging.js";
@@ -695,12 +696,321 @@ export function primeTokenManagerWithInitialToken(tokenManager, token) {
         tokenManager.updateToken(token);
     }
 }
+/**
+ * True when this dev server was launched as a CSB sandbox — `SUPERBLOCKS_IS_CSB
+ * === "true"`, the same variable that already gates the CSB-only behavior
+ * elsewhere in this file. That is two populations, not one:
+ *
+ *   - a disposable, orchestrator-managed SABS live-edit pod, and
+ *   - the LOCAL `csb-mock-server` emulator, which sets the identical variable
+ *     when it spawns `superblocks dev` for code-mode work
+ *     (`packages/csb-mock-server/src/app-manager.ts`, `startProcess` and
+ *     `restartSandbox`; neither passes anything that skips the auto-upgrade).
+ *
+ * So this is NOT "cloud, not a developer's machine". The emulator answering
+ * `true` is intended — it exists to emulate a pod, and an exit is the behavior a
+ * pod would show — but it emulates only half the story: `attachEventListeners`
+ * in that file merely logs the child's exit, so nothing restarts it. The
+ * practical consequence for a local dev whose auto-upgrade fails this way is a
+ * dead dev server until something calls `startProcess` again (reopening or
+ * restarting the app in code mode). Only a dev server the developer ran
+ * directly, with no `SUPERBLOCKS_IS_CSB`, still degrades.
+ *
+ * Read at call time, not module load, so tests can set the variable per-case.
+ */
+export function isDisposableSandbox() {
+    return process.env.SUPERBLOCKS_IS_CSB === "true";
+}
+/**
+ * The single implementation of "flush telemetry, but never wait forever".
+ *
+ * Two paths need it, for the same reason: the OTel metric reader exports on a
+ * 10s interval, so a sample recorded moments before the process goes away dies
+ * with it unless something forces the export — and neither path may block
+ * indefinitely on a slow or unreachable collector. The callers are the
+ * fatal-exit sequence (`flushTelemetryBeforeExit`, just below) and the
+ * auto-upgrade CLI restart in `dev()`, which used to carry its own copy of this
+ * race with the timer-clear missing.
+ *
+ * Resolves `true` only when a flush actually ran to completion — `false` on an
+ * internal shutdown error, when telemetry was never initialized, or when the
+ * timeout won the race.
+ *
+ * Reporting that boolean is left to the callers rather than parameterized here,
+ * because the useful part of the message is what an operator should read
+ * INSTEAD of the dropped samples, and that differs per path: the fatal paths
+ * point at their `event=dev_server_fatal_exit` line, which the restart path
+ * never writes.
+ *
+ * Rejects only if `shutdownTelemetry()` breaks its documented never-reject
+ * contract, so callers keep their own defensive catch.
+ */
+export async function flushTelemetryWithTimeout() {
+    let flushed = false;
+    let timer;
+    try {
+        await Promise.race([
+            shutdownTelemetry().then((didFlush) => {
+                flushed = didFlush;
+            }),
+            new Promise((resolve) => {
+                timer = setTimeout(resolve, TELEMETRY_FLUSH_TIMEOUT_MS);
+            }),
+        ]);
+    }
+    finally {
+        // The race is settled; leaving the timer armed would hold the event loop
+        // open for callers (tests, in particular) that do not immediately exit.
+        if (timer)
+            clearTimeout(timer);
+    }
+    return flushed;
+}
+/**
+ * Force a telemetry export before a fatal exit.
+ *
+ * The OTel metric reader exports on a 10s interval, so anything recorded
+ * moments before `process.exit(1)` dies with the process — the same drop
+ * already documented for the auto-upgrade restart path below. Any metric whose
+ * entire purpose is to make an exit observable has to be flushed here or it is
+ * never seen. The raced flush itself lives in `flushTelemetryWithTimeout`,
+ * shared with that restart path; this wrapper only adds the fatal-exit
+ * reporting.
+ *
+ * MUST BE CALLED LAST in any shutdown sequence. This does not merely flush: it
+ * awaits `shutdownTelemetry()`, which tears the provider down, so every signal
+ * emitted after it goes into a dead pipeline. It can also block for up to
+ * `TELEMETRY_FLUSH_TIMEOUT_MS`, and delaying `lockService.shutdown()` by that
+ * much delays the lock release that lets a replacement sandbox take over —
+ * which is the entire recovery story this exit path exists to enable.
+ *
+ * "Last" is only reachable because the callers drive the lock service with
+ * `shutdown()` rather than `shutdownAndExit()`. The latter ends in an
+ * unconditional `finally { process.exit(1) }`, and `process.exit` is
+ * synchronous, so anything sequenced after it never runs at all. See the two
+ * fatal-exit blocks in `dev()`.
+ *
+ * Never throws: a stuck exporter is raced against a timeout so it cannot wedge
+ * the exit path.
+ *
+ * `shutdownTelemetry()` resolves `true` only when a flush actually ran to
+ * completion — `false` on an internal shutdown error or when telemetry was
+ * never initialized — and the timeout branch leaves the flag `false`. Log that
+ * distinction rather than discarding it: when it is `false` the `outcome="exit"`
+ * sample did NOT make it out, and the only surviving evidence of the exit is the
+ * synchronous `event=dev_server_fatal_exit` line the caller already wrote. An
+ * operator who cannot tell those apart reads "zero exit samples" as "the exit
+ * path never ran". The auto-upgrade restart path reports the same distinction,
+ * against its own log line.
+ */
+async function flushTelemetryBeforeExit(logger) {
+    let flushed = false;
+    try {
+        flushed = await flushTelemetryWithTimeout();
+    }
+    catch (e) {
+        // Defensive only: `shutdownTelemetry()` catches internally and resolves
+        // `false` rather than rejecting. Kept so a future change to that contract
+        // cannot turn this into an unhandled rejection on the exit path.
+        logger.warn(`[dev-server] telemetry flush before exit failed: ${e}`);
+    }
+    if (!flushed) {
+        logger.warn(`[dev-server] telemetry flush did not complete (timed out after ${TELEMETRY_FLUSH_TIMEOUT_MS}ms, errored, or telemetry disabled) before fatal exit; ` +
+            `dependency-failure samples may be dropped — use the event=${FATAL_EXIT_EVENT} log line as the source of truth for this exit`);
+    }
+}
+/**
+ * Upper bound on any single teardown step run before a fatal exit.
+ *
+ * Each step talks to something that can stop answering (the ai-service over
+ * IPC, the lock service over the network). Without a bound, one unsettled
+ * promise means the `finally { process.exit(1) }` below it never runs and the
+ * pod stays Ready but zombied — the exact ENG-5297 symptom this exit path
+ * exists to end, one call earlier. The whole point of these paths is that the
+ * process goes away, so a step that has not answered in this long is abandoned
+ * rather than waited on.
+ */
+const FATAL_EXIT_STEP_TIMEOUT_MS = 5000;
+/**
+ * Run one best-effort teardown step on a fatal-exit path.
+ *
+ * Neither a throw nor a hang may stop the later steps: the lock release is what
+ * lets a replacement sandbox take over, and the telemetry flush is what makes
+ * the exit observable, so losing them to a failure in an earlier, less
+ * important step is strictly worse than the failure itself.
+ */
+async function runFatalExitStep(logger, name, step) {
+    if (!step) {
+        return;
+    }
+    let timer;
+    try {
+        await Promise.race([
+            // A synchronous throw from `step()` propagates out of the array literal
+            // before the race is constructed, and is caught by the same `catch`.
+            // `undefined` (the service was never constructed) is a non-thenable, so
+            // the race settles immediately and the step is a no-op.
+            step(),
+            new Promise((resolve) => {
+                timer = setTimeout(() => {
+                    logger.error(`[dev-server] ${name} did not settle within ${FATAL_EXIT_STEP_TIMEOUT_MS}ms before fatal exit; abandoning it and continuing`);
+                    resolve();
+                }, FATAL_EXIT_STEP_TIMEOUT_MS);
+            }),
+        ]);
+    }
+    catch (e) {
+        logger.error(`[dev-server] ${name} failed before fatal exit; continuing`, getErrorMeta(e));
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+/**
+ * The ordered teardown every fatal-exit path in `dev()` runs immediately before
+ * `process.exit(1)`. Stated once, here, so the ordering guarantee cannot drift
+ * between the two call sites.
+ *
+ * Order is load-bearing:
+ *   1. `removeIntegrationCache` — ai-service state that must not outlive us.
+ *   2. `shutdownLockService` — releases the AppBranchLock. Recovery-critical:
+ *      the heartbeat dies with the process, so skipping this makes the
+ *      replacement sandbox's `acquireLock()` fail with a ConflictError until
+ *      the server-side TTL expires, and the replacement is the entire point of
+ *      exiting.
+ *   3. `flushTelemetryBeforeExit` — LAST, because it tears the telemetry
+ *      provider down (anything after it is unobservable) and can block for
+ *      `TELEMETRY_FLUSH_TIMEOUT_MS`, which would otherwise delay the lock
+ *      release above.
+ *
+ * Every step is individually guarded and individually bounded, so neither a
+ * throw nor a hang in an earlier step can cost a later one. Never throws.
+ *
+ * This does NOT exit. The caller owns `process.exit(1)` from a `finally`, so
+ * the exit still happens even if this function is somehow bypassed. That is
+ * also why callers must drive the lock service through `shutdown()` and not
+ * `shutdownAndExit()`: the latter ends in its own unconditional
+ * `finally { process.exit(1) }`, and `process.exit` is synchronous, so
+ * everything sequenced after it — including the flush — silently never ran.
+ */
+export async function runFatalExitShutdown(steps) {
+    const { logger } = steps;
+    await runFatalExitStep(logger, "ai-service integration cache removal", steps.removeIntegrationCache);
+    await runFatalExitStep(logger, "lock service shutdown", steps.shutdownLockService);
+    await flushTelemetryBeforeExit(logger);
+}
+/**
+ * Upgrade-failure categories that are decided BEFORE npm can extract anything.
+ *
+ * npm builds the whole ideal tree from registry metadata before it writes a
+ * single byte into `node_modules`, so a run that died because the registry
+ * refused, could not be reached, could not be TLS-verified, or did not have the
+ * requested version never reached the extraction phase — the on-disk tree and
+ * the running esbuild service it feeds are provably untouched.
+ *
+ * These are also precisely the shape a registry/proxy outage takes. Treating
+ * them as fatal would exit every live-edit pod in the fleet simultaneously and
+ * hand SABS a queue of replacements that hit the same outage, converting a
+ * degraded-but-serving fleet into an unavailable one.
+ *
+ * Typed against `DependencyUpgradeError["category"]` so a future category added
+ * to that union has to be classified deliberately rather than drifting in.
+ *
+ * The union has exactly five members today, so holding the other four here
+ * makes `upgradeCategoryCouldTaintTree` presently equivalent to
+ * `category === "unknown"`. The set shape is kept anyway because the two forms
+ * diverge the moment a sixth category is added: an allowlist of known-safe
+ * categories fails CLOSED (a new, unclassified category exits, which is the
+ * conservative side), whereas `=== "unknown"` fails OPEN (a new category
+ * silently degrades, which is precisely the ENG-5297 zombie-pod bug). Naming
+ * the four safe categories also forces whoever adds the sixth to decide which
+ * side it belongs on.
+ *
+ * Residual risk, stated plainly — and this is the known hole in the gate, not a
+ * theoretical one. These categories are derived from npm/pnpm's error text,
+ * which carries no notion of install PHASE, so a later-phase failure can wear an
+ * early-phase label: `ERR_PNPM_FETCH_<5xx>` is pnpm's TARBALL fetch error yet
+ * maps to `registry_unreachable`, and `detectReasonFromText` matches
+ * ECONNRESET/ETIMEDOUT/ENOTFOUND anywhere in stderr. A CDN 502 or a mid-download
+ * reset therefore degrades even though extraction had begun. npm rolls the reify
+ * back, but rollback restores FILES only — it does not restart the esbuild
+ * service child process that died when its binary was replaced, which is the
+ * actual harm chain in ENG-5297. So this gate does not close that case.
+ *
+ * The trade is still taken knowingly: closing it properly means stamping the
+ * phase where it is actually known (a `treePossiblyMutated` flag set at the
+ * throw site in `upgradeCliWithPackageManager`) or probing esbuild/Vite health
+ * after an `UpgradeFailed`, both of which change the shared
+ * `DependencyUpgradeError` contract and belong in their own change. Meanwhile
+ * this gate is strictly better than the previous behavior, which degraded on
+ * every upgrade failure without exception, and exiting on these categories
+ * instead would amplify a registry outage into a fleet-wide restart.
+ */
+const RESOLUTION_PHASE_UPGRADE_CATEGORIES = new Set([
+    "not_in_registry",
+    "registry_auth_failed",
+    "registry_unreachable",
+    "tls_failed",
+]);
+/**
+ * True when a failed auto-upgrade could have mutated the live dependency tree,
+ * and therefore may have killed the running esbuild service out from under
+ * Vite. Unknown/unclassified categories deliberately answer `true`: ENG-5297
+ * arrived as `category: "unknown"`, and a category this function has never seen
+ * must fail closed (exit) rather than inherit the degrade path by default.
+ *
+ * This reduces to `category === "unknown"` against today's five-member union —
+ * see `RESOLUTION_PHASE_UPGRADE_CATEGORIES` for why the set is kept rather than
+ * collapsed to that comparison.
+ *
+ * The parameter is `string`, not `DependencyUpgradeError["category"]`, because
+ * that is the actual runtime contract: the payload is deserialized from the
+ * socket, so the five-member union is a promise about our own writers rather
+ * than a guarantee about the value — which is also why the set above is a
+ * `ReadonlySet<string>`. It is also what lets the fail-closed default be
+ * asserted for a category a FUTURE CLI adds; that case is not expressible
+ * through an `UpgradeFailed` marker without an `as` cast, and it is the one this
+ * gate most needs to keep.
+ *
+ * `"unknown"` is itself coarse; the two sub-cases sit on opposite sides of the
+ * question this function asks, and only one of them is a false positive:
+ *
+ *   no install command   `resolveCommand` returned nothing, so `child_process.exec`
+ *                        was never reached (automatic-upgrades.ts, the
+ *                        `if (!installCommand)` early throw). The tree is
+ *                        provably untouched and exiting is a false positive.
+ *   version validation   `getCurrentCliVersion()` returned nothing / mismatched.
+ *                        That check runs AFTER `await promise` resolves — i.e.
+ *                        after the global install completed — so the tree
+ *                        genuinely moved and exiting is correct.
+ *
+ * Only the first is a false positive, and it is not distinguishable from data
+ * already on `serverError`: both arrive as `category: "unknown"` with no
+ * `npmErrorCode`, and the only thing separating them is free-text English in
+ * `rawError`. Keying the exit decision on that string would be the same
+ * phase-blind text proxy this file already flags as the gate's weak point.
+ * Splitting them properly means a flag stamped at the throw site on
+ * `DependencyUpgradeError` — a change to a shared contract, deliberately not
+ * made here. Until then both land on exit, the direction this function is meant
+ * to err in.
+ *
+ * Note on cost if the false positive fires: SABS pods run `RestartPolicy: Never`,
+ * so the retry is a pod REPLACEMENT driven by the orchestrator, not an in-pod
+ * crash loop. A deterministic, environment-derived cause would therefore
+ * re-trigger in each replacement, and no per-process bound inside this file
+ * could stop that — a replacement is a fresh process with a fresh counter. Any
+ * real bound belongs in the orchestrator that decides to replace.
+ */
+export function upgradeCategoryCouldTaintTree(category) {
+    return !RESOLUTION_PHASE_UPGRADE_CATEGORIES.has(category);
+}
 /** Decide how the startup catch handles an error: degrade (record, keep Vite up)
  *  for an app-install failure (`InitialInstallFailed`) or a best-effort
  *  CLI/library auto-upgrade failure (`UpgradeFailed`), or exit for anything else
  *  (lock/sync/genuinely-unusable upgrade env). Pure + unit-tested. Does NOT call
  *  process.exit. */
-export function handleStartupError(error, status, logger) {
+export function handleStartupError(error, status, logger, options) {
     if (error instanceof InitialInstallFailed) {
         status.serverErrors.push(error.serverError);
         logger.error("[dev-server] initial dependency install failed; keeping dev server up", getErrorMeta(error));
@@ -712,17 +1022,56 @@ export function handleStartupError(error, status, logger) {
         return "degrade";
     }
     if (error instanceof UpgradeFailed) {
-        // The auto-upgrade is best-effort and the pod is still running the current
-        // version, so a failed upgrade degrades instead of crash-looping. Recording
-        // here also makes the failure observable: the process survives to the next
-        // metrics export tick, which the pre-exit `process.exit(1)` always lost.
         status.serverErrors.push(error.serverError);
-        logger.error("[dev-server] dependency upgrade failed; keeping the current version", getErrorMeta(error));
+        // The upgrade is an in-place `npm install` into the LIVE node_modules, and
+        // its install manifest includes esbuild. A failure part-way through can
+        // therefore have already killed the running esbuild service ("The service
+        // was stopped"), which rejects vitePromise and leaves a dev server that can
+        // never serve a request again. "Keeping the current version" is only true
+        // when the tree survived, and we cannot cheaply prove that it did.
+        //
+        // So the decision turns on who owns the process AND on whether the failure
+        // could have mutated the tree at all:
+        //   - disposable sandbox (SABS pod) + a failure that could have touched
+        //     `node_modules`: exit. The pod is cheap and SABS already reaps
+        //     terminal pods (they run RestartPolicy: Never), so exiting yields a
+        //     clean replacement. Degrading instead produced ENG-5297 — 85 minutes
+        //     of a Ready pod answering 500s.
+        //   - disposable sandbox + a resolution-phase failure: degrade. npm never
+        //     got as far as extraction, so the tree is intact and there is nothing
+        //     for a replacement pod to fix — see
+        //     `upgradeCategoryCouldTaintTree`, which also explains why exiting on
+        //     these would amplify a registry outage into a fleet-wide restart.
+        //   - anything not flagged as a CSB sandbox: degrade. Killing a dev server
+        //     the developer started themselves over a best-effort upgrade is worse
+        //     than running the older version. Note the carve-out is narrower than
+        //     "not in the cloud": the local `csb-mock-server` emulator also sets
+        //     `SUPERBLOCKS_IS_CSB=true`, so local code-mode dev servers exit here
+        //     too. That is intended (the emulator exists to behave like a pod), but
+        //     nothing in the emulator restarts the child — see
+        //     `isDisposableSandbox`.
+        const outcome = options?.disposableSandbox &&
+            upgradeCategoryCouldTaintTree(error.serverError.category)
+            ? "exit"
+            : "degrade";
         devServerMetrics.recordUpgradeFailure({
             category: error.serverError.category,
             npmErrorCode: error.serverError.npmErrorCode,
             hasAnyRegistryConfigured: error.serverError.hasAnyRegistryConfigured,
+            outcome,
         });
+        if (outcome === "exit") {
+            logger.error("[dev-server] dependency upgrade failed in a disposable sandbox; exiting so the sandbox is replaced (the live dependency tree may be tainted)", getErrorMeta(error));
+            // No `devServerMetrics.flush()` here: `record()` and `flush()` gate on
+            // the same `isTelemetryInitialized()` check and nothing async runs
+            // between them, so if the sample above was buffered, flush would bail on
+            // the identical condition — and if it was not buffered, there is nothing
+            // to drain. The caller's `flushTelemetryBeforeExit` (which awaits
+            // `shutdownTelemetry()`) is what actually forces the export before
+            // `process.exit(1)`.
+            return "exit";
+        }
+        logger.error("[dev-server] dependency upgrade failed; keeping the current version", getErrorMeta(error));
         return "degrade";
     }
     return "exit";
@@ -880,6 +1229,7 @@ export async function dev(options) {
                     superblocksBaseUrl: tokenConfig.superblocksBaseUrl,
                     rpcClient,
                     logger,
+                    removeManualCommitsEnabled: featureFlags.removeManualCommitsEnabled(),
                 });
                 logger.info("Checking if local files are synced with the server");
                 try {
@@ -1410,20 +1760,18 @@ export async function dev(options) {
                             // which is why that runner reports zero samples in every env
                             // despite real upgrades emitting it (spans survive because they
                             // flush on the faster batch processor; the metric does not).
-                            // shutdownTelemetry() never rejects; the catch is purely
-                            // defensive. It resolves to `true` only when a flush actually
-                            // ran — `false` on internal shutdown error or when telemetry was
-                            // disabled — and the timeout branch leaves the flag false. So
-                            // the log below reflects whether samples were really flushed, not
-                            // merely that the race settled.
+                            // The timeout-raced flush is shared with the fatal-exit path
+                            // (`flushTelemetryWithTimeout`) — this site used to carry its own
+                            // copy of the race, and that copy leaked its timer.
+                            // `flushTelemetryWithTimeout` resolves `true` only when a flush
+                            // actually ran — `false` on internal shutdown error, when
+                            // telemetry was disabled, or when the timeout won — so the log
+                            // below reflects whether samples were really flushed, not merely
+                            // that the race settled. It rejects only if shutdownTelemetry()
+                            // breaks its never-reject contract, so the catch stays defensive.
                             let telemetryFlushed = false;
                             try {
-                                await Promise.race([
-                                    shutdownTelemetry().then((flushed) => {
-                                        telemetryFlushed = flushed;
-                                    }),
-                                    new Promise((resolve) => setTimeout(resolve, TELEMETRY_FLUSH_TIMEOUT_MS)),
-                                ]);
+                                telemetryFlushed = await flushTelemetryWithTimeout();
                             }
                             catch (e) {
                                 logger.error("Error flushing telemetry before CLI restart", getErrorMeta(e));
@@ -1441,18 +1789,54 @@ export async function dev(options) {
                     });
                 }
                 catch (error) {
-                    if (handleStartupError(error, devServerStatus, logger) === "degrade") {
+                    if (handleStartupError(error, devServerStatus, logger, {
+                        disposableSandbox: isDisposableSandbox(),
+                    }) === "degrade") {
                         // app-install failure: do NOT exit — fall through to Vite startup below.
                         // upload + CLI-restart were already skipped (the rejecting join threw first).
                     }
                     else {
-                        logger.error("[dev-server] Startup failed during sync/lock/setup (exiting with code 1)", getErrorMeta(error));
+                        logger.error(
+                        // Names the stage, because `buildFatalExitLog` below carries only
+                        // the generic `handler=startupFailure`. Two stages reach here now:
+                        // sync/lock/setup, and (since ENG-5297) a dependency upgrade that
+                        // failed in a CSB sandbox and may have tainted the live tree.
+                        "[dev-server] Startup failed during sync/lock/setup, or a dependency upgrade failed in a CSB sandbox (exiting with code 1)", getErrorMeta(error));
+                        // Emitted BEFORE any teardown, and as a synchronous log line rather
+                        // than a counter, per the convention documented in
+                        // dev-utils/fatal-exit.mts: at exit time stdout is the only sink we
+                        // can rely on, so the reason has to land even if the flush below
+                        // never completes. The `outcome="exit"` metric is a complement to
+                        // this line, not a substitute for it.
+                        logger.error(buildFatalExitLog({
+                            handler: "startupFailure",
+                            exitCode: 1,
+                            isWarm: parseIsWarm(process.env.SUPERBLOCKS_WARM_STANDBY),
+                        }));
                         try {
-                            await aiService?.removeIntegrationCache();
-                            await lockService?.shutdownAndExit();
+                            // `shutdown()`, NOT `shutdownAndExit()`. The latter ends in an
+                            // unconditional `finally { process.exit(1) }`; `process.exit` is
+                            // synchronous, so every statement below it would be dead code
+                            // whenever `lockService` is set — which is the normal state in a
+                            // SABS pod, exactly the population this exit path targets. The
+                            // telemetry flush was silently never running. `shutdown()` does
+                            // the same teardown without owning the exit, matching how
+                            // `runGracefulShutdown` in dev-server.mts drives it; the
+                            // `finally` below performs the exit instead.
+                            //
+                            // Ordering, per-step error handling, and per-step timeouts all
+                            // live in `runFatalExitShutdown` so the two fatal paths cannot
+                            // drift apart.
+                            await runFatalExitShutdown({
+                                logger,
+                                removeIntegrationCache: () => aiService?.removeIntegrationCache(),
+                                shutdownLockService: () => lockService?.shutdown({ serverInitiated: false }),
+                            });
                         }
                         finally {
-                            // this is redundant, but it's here to make sure the lock service is shutdown and the process exits
+                            // The only exit on this path now that the lock service no longer
+                            // owns one. `runFatalExitShutdown` never throws and bounds every
+                            // step, so this runs regardless of what teardown did.
                             process.exit(1);
                         }
                     }
@@ -1485,15 +1869,32 @@ export async function dev(options) {
                 await joinUpgradeThenInstall("before Vite startup");
             }
             catch (error) {
-                if (handleStartupError(error, devServerStatus, logger) === "exit") {
+                if (handleStartupError(error, devServerStatus, logger, {
+                    disposableSandbox: isDisposableSandbox(),
+                }) === "exit") {
                     logger.error("[dev-server] Startup failed during pre-Vite install join (exiting with code 1)", getErrorMeta(error));
+                    // Same convention as the sync/lock catch above: a synchronous
+                    // `event=dev_server_fatal_exit` line, written before teardown, is the
+                    // thing oncall queries for this exit class.
+                    logger.error(buildFatalExitLog({
+                        handler: "startupFailure",
+                        exitCode: 1,
+                        isWarm: parseIsWarm(process.env.SUPERBLOCKS_WARM_STANDBY),
+                    }));
                     try {
-                        await aiService?.removeIntegrationCache();
-                        await lockService?.shutdownAndExit();
+                        // `shutdown()`, NOT `shutdownAndExit()` — same reason as the
+                        // sync/lock catch above: `shutdownAndExit()` exits the process
+                        // synchronously in its own `finally`, which made everything
+                        // sequenced after it unreachable.
+                        await runFatalExitShutdown({
+                            logger,
+                            removeIntegrationCache: () => aiService?.removeIntegrationCache(),
+                            shutdownLockService: () => lockService?.shutdown({ serverInitiated: false }),
+                        });
                     }
                     finally {
-                        // Redundant with `shutdownAndExit`; here so a thrown shutdown
-                        // path can't leave the process hanging on a stuck handle.
+                        // The only exit on this path; here so a thrown shutdown path can't
+                        // leave the process hanging on a stuck handle.
                         process.exit(1);
                     }
                 }