@superblocksteam/sdk 2.0.151 → 2.0.152-next.0

package/src/cli-replacement/dev.mts

@@ -11,7 +11,10 @@ import fs from "fs-extra";
 import { resolveCommand } from "package-manager-detector";
 import { detect } from "package-manager-detector/detect";
 
-import type { ServerError } from "@superblocksteam/library-shared/types";
+import type {
+  DependencyUpgradeError,
+  ServerError,
+} from "@superblocksteam/library-shared/types";
 import {
   buildGithubSuperblocksSyncWorkflow,
   buildGithubSuperblocksSyncWorkflowFromBaseUrl,
@@ -50,6 +53,11 @@ import { SyncService } from "@superblocksteam/vite-plugin-file-sync/sync-service
 
 import { devServerMetrics } from "../dev-utils/dev-server-metrics.mjs";
 import { createDevServer } from "../dev-utils/dev-server.mjs";
+import {
+  buildFatalExitLog,
+  FATAL_EXIT_EVENT,
+  parseIsWarm,
+} from "../dev-utils/fatal-exit.mjs";
 import { AUTO_UPGRADE_EXIT_CODE } from "../index.js";
 import type {
   AuthHotReloadServer,
@@ -901,6 +909,349 @@ export interface DevServerStatus {
   serverErrors: ServerError[];
 }
 
+/**
+ * True when this dev server was launched as a CSB sandbox — `SUPERBLOCKS_IS_CSB
+ * === "true"`, the same variable that already gates the CSB-only behavior
+ * elsewhere in this file. That is two populations, not one:
+ *
+ *   - a disposable, orchestrator-managed SABS live-edit pod, and
+ *   - the LOCAL `csb-mock-server` emulator, which sets the identical variable
+ *     when it spawns `superblocks dev` for code-mode work
+ *     (`packages/csb-mock-server/src/app-manager.ts`, `startProcess` and
+ *     `restartSandbox`; neither passes anything that skips the auto-upgrade).
+ *
+ * So this is NOT "cloud, not a developer's machine". The emulator answering
+ * `true` is intended — it exists to emulate a pod, and an exit is the behavior a
+ * pod would show — but it emulates only half the story: `attachEventListeners`
+ * in that file merely logs the child's exit, so nothing restarts it. The
+ * practical consequence for a local dev whose auto-upgrade fails this way is a
+ * dead dev server until something calls `startProcess` again (reopening or
+ * restarting the app in code mode). Only a dev server the developer ran
+ * directly, with no `SUPERBLOCKS_IS_CSB`, still degrades.
+ *
+ * Read at call time, not module load, so tests can set the variable per-case.
+ */
+export function isDisposableSandbox(): boolean {
+  return process.env.SUPERBLOCKS_IS_CSB === "true";
+}
+
+/**
+ * The single implementation of "flush telemetry, but never wait forever".
+ *
+ * Two paths need it, for the same reason: the OTel metric reader exports on a
+ * 10s interval, so a sample recorded moments before the process goes away dies
+ * with it unless something forces the export — and neither path may block
+ * indefinitely on a slow or unreachable collector. The callers are the
+ * fatal-exit sequence (`flushTelemetryBeforeExit`, just below) and the
+ * auto-upgrade CLI restart in `dev()`, which used to carry its own copy of this
+ * race with the timer-clear missing.
+ *
+ * Resolves `true` only when a flush actually ran to completion — `false` on an
+ * internal shutdown error, when telemetry was never initialized, or when the
+ * timeout won the race.
+ *
+ * Reporting that boolean is left to the callers rather than parameterized here,
+ * because the useful part of the message is what an operator should read
+ * INSTEAD of the dropped samples, and that differs per path: the fatal paths
+ * point at their `event=dev_server_fatal_exit` line, which the restart path
+ * never writes.
+ *
+ * Rejects only if `shutdownTelemetry()` breaks its documented never-reject
+ * contract, so callers keep their own defensive catch.
+ */
+export async function flushTelemetryWithTimeout(): Promise<boolean> {
+  let flushed = false;
+  let timer: NodeJS.Timeout | undefined;
+  try {
+    await Promise.race([
+      shutdownTelemetry().then((didFlush) => {
+        flushed = didFlush;
+      }),
+      new Promise<void>((resolve) => {
+        timer = setTimeout(resolve, TELEMETRY_FLUSH_TIMEOUT_MS);
+      }),
+    ]);
+  } finally {
+    // The race is settled; leaving the timer armed would hold the event loop
+    // open for callers (tests, in particular) that do not immediately exit.
+    if (timer) clearTimeout(timer);
+  }
+  return flushed;
+}
+
+/**
+ * Force a telemetry export before a fatal exit.
+ *
+ * The OTel metric reader exports on a 10s interval, so anything recorded
+ * moments before `process.exit(1)` dies with the process — the same drop
+ * already documented for the auto-upgrade restart path below. Any metric whose
+ * entire purpose is to make an exit observable has to be flushed here or it is
+ * never seen. The raced flush itself lives in `flushTelemetryWithTimeout`,
+ * shared with that restart path; this wrapper only adds the fatal-exit
+ * reporting.
+ *
+ * MUST BE CALLED LAST in any shutdown sequence. This does not merely flush: it
+ * awaits `shutdownTelemetry()`, which tears the provider down, so every signal
+ * emitted after it goes into a dead pipeline. It can also block for up to
+ * `TELEMETRY_FLUSH_TIMEOUT_MS`, and delaying `lockService.shutdown()` by that
+ * much delays the lock release that lets a replacement sandbox take over —
+ * which is the entire recovery story this exit path exists to enable.
+ *
+ * "Last" is only reachable because the callers drive the lock service with
+ * `shutdown()` rather than `shutdownAndExit()`. The latter ends in an
+ * unconditional `finally { process.exit(1) }`, and `process.exit` is
+ * synchronous, so anything sequenced after it never runs at all. See the two
+ * fatal-exit blocks in `dev()`.
+ *
+ * Never throws: a stuck exporter is raced against a timeout so it cannot wedge
+ * the exit path.
+ *
+ * `shutdownTelemetry()` resolves `true` only when a flush actually ran to
+ * completion — `false` on an internal shutdown error or when telemetry was
+ * never initialized — and the timeout branch leaves the flag `false`. Log that
+ * distinction rather than discarding it: when it is `false` the `outcome="exit"`
+ * sample did NOT make it out, and the only surviving evidence of the exit is the
+ * synchronous `event=dev_server_fatal_exit` line the caller already wrote. An
+ * operator who cannot tell those apart reads "zero exit samples" as "the exit
+ * path never ran". The auto-upgrade restart path reports the same distinction,
+ * against its own log line.
+ */
+async function flushTelemetryBeforeExit(logger: Logger): Promise<void> {
+  let flushed = false;
+  try {
+    flushed = await flushTelemetryWithTimeout();
+  } catch (e) {
+    // Defensive only: `shutdownTelemetry()` catches internally and resolves
+    // `false` rather than rejecting. Kept so a future change to that contract
+    // cannot turn this into an unhandled rejection on the exit path.
+    logger.warn(`[dev-server] telemetry flush before exit failed: ${e}`);
+  }
+  if (!flushed) {
+    logger.warn(
+      `[dev-server] telemetry flush did not complete (timed out after ${TELEMETRY_FLUSH_TIMEOUT_MS}ms, errored, or telemetry disabled) before fatal exit; ` +
+        `dependency-failure samples may be dropped — use the event=${FATAL_EXIT_EVENT} log line as the source of truth for this exit`,
+    );
+  }
+}
+
+/**
+ * Upper bound on any single teardown step run before a fatal exit.
+ *
+ * Each step talks to something that can stop answering (the ai-service over
+ * IPC, the lock service over the network). Without a bound, one unsettled
+ * promise means the `finally { process.exit(1) }` below it never runs and the
+ * pod stays Ready but zombied — the exact ENG-5297 symptom this exit path
+ * exists to end, one call earlier. The whole point of these paths is that the
+ * process goes away, so a step that has not answered in this long is abandoned
+ * rather than waited on.
+ */
+const FATAL_EXIT_STEP_TIMEOUT_MS = 5000;
+
+/**
+ * One teardown step. Returning `undefined` is allowed so callers can pass the
+ * optional-chained call directly (`() => lockService?.shutdown(...)`) without
+ * having to re-narrow a mutable binding at the call site.
+ */
+type FatalExitStep = () => Promise<unknown> | undefined;
+
+/**
+ * Run one best-effort teardown step on a fatal-exit path.
+ *
+ * Neither a throw nor a hang may stop the later steps: the lock release is what
+ * lets a replacement sandbox take over, and the telemetry flush is what makes
+ * the exit observable, so losing them to a failure in an earlier, less
+ * important step is strictly worse than the failure itself.
+ */
+async function runFatalExitStep(
+  logger: Logger,
+  name: string,
+  step: FatalExitStep | undefined,
+): Promise<void> {
+  if (!step) {
+    return;
+  }
+  let timer: NodeJS.Timeout | undefined;
+  try {
+    await Promise.race([
+      // A synchronous throw from `step()` propagates out of the array literal
+      // before the race is constructed, and is caught by the same `catch`.
+      // `undefined` (the service was never constructed) is a non-thenable, so
+      // the race settles immediately and the step is a no-op.
+      step(),
+      new Promise<void>((resolve) => {
+        timer = setTimeout(() => {
+          logger.error(
+            `[dev-server] ${name} did not settle within ${FATAL_EXIT_STEP_TIMEOUT_MS}ms before fatal exit; abandoning it and continuing`,
+          );
+          resolve();
+        }, FATAL_EXIT_STEP_TIMEOUT_MS);
+      }),
+    ]);
+  } catch (e) {
+    logger.error(
+      `[dev-server] ${name} failed before fatal exit; continuing`,
+      getErrorMeta(e),
+    );
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
+/**
+ * The ordered teardown every fatal-exit path in `dev()` runs immediately before
+ * `process.exit(1)`. Stated once, here, so the ordering guarantee cannot drift
+ * between the two call sites.
+ *
+ * Order is load-bearing:
+ *   1. `removeIntegrationCache` — ai-service state that must not outlive us.
+ *   2. `shutdownLockService` — releases the AppBranchLock. Recovery-critical:
+ *      the heartbeat dies with the process, so skipping this makes the
+ *      replacement sandbox's `acquireLock()` fail with a ConflictError until
+ *      the server-side TTL expires, and the replacement is the entire point of
+ *      exiting.
+ *   3. `flushTelemetryBeforeExit` — LAST, because it tears the telemetry
+ *      provider down (anything after it is unobservable) and can block for
+ *      `TELEMETRY_FLUSH_TIMEOUT_MS`, which would otherwise delay the lock
+ *      release above.
+ *
+ * Every step is individually guarded and individually bounded, so neither a
+ * throw nor a hang in an earlier step can cost a later one. Never throws.
+ *
+ * This does NOT exit. The caller owns `process.exit(1)` from a `finally`, so
+ * the exit still happens even if this function is somehow bypassed. That is
+ * also why callers must drive the lock service through `shutdown()` and not
+ * `shutdownAndExit()`: the latter ends in its own unconditional
+ * `finally { process.exit(1) }`, and `process.exit` is synchronous, so
+ * everything sequenced after it — including the flush — silently never ran.
+ */
+export async function runFatalExitShutdown(steps: {
+  logger: Logger;
+  removeIntegrationCache?: FatalExitStep;
+  shutdownLockService?: FatalExitStep;
+}): Promise<void> {
+  const { logger } = steps;
+  await runFatalExitStep(
+    logger,
+    "ai-service integration cache removal",
+    steps.removeIntegrationCache,
+  );
+  await runFatalExitStep(
+    logger,
+    "lock service shutdown",
+    steps.shutdownLockService,
+  );
+  await flushTelemetryBeforeExit(logger);
+}
+
+/**
+ * Upgrade-failure categories that are decided BEFORE npm can extract anything.
+ *
+ * npm builds the whole ideal tree from registry metadata before it writes a
+ * single byte into `node_modules`, so a run that died because the registry
+ * refused, could not be reached, could not be TLS-verified, or did not have the
+ * requested version never reached the extraction phase — the on-disk tree and
+ * the running esbuild service it feeds are provably untouched.
+ *
+ * These are also precisely the shape a registry/proxy outage takes. Treating
+ * them as fatal would exit every live-edit pod in the fleet simultaneously and
+ * hand SABS a queue of replacements that hit the same outage, converting a
+ * degraded-but-serving fleet into an unavailable one.
+ *
+ * Typed against `DependencyUpgradeError["category"]` so a future category added
+ * to that union has to be classified deliberately rather than drifting in.
+ *
+ * The union has exactly five members today, so holding the other four here
+ * makes `upgradeCategoryCouldTaintTree` presently equivalent to
+ * `category === "unknown"`. The set shape is kept anyway because the two forms
+ * diverge the moment a sixth category is added: an allowlist of known-safe
+ * categories fails CLOSED (a new, unclassified category exits, which is the
+ * conservative side), whereas `=== "unknown"` fails OPEN (a new category
+ * silently degrades, which is precisely the ENG-5297 zombie-pod bug). Naming
+ * the four safe categories also forces whoever adds the sixth to decide which
+ * side it belongs on.
+ *
+ * Residual risk, stated plainly — and this is the known hole in the gate, not a
+ * theoretical one. These categories are derived from npm/pnpm's error text,
+ * which carries no notion of install PHASE, so a later-phase failure can wear an
+ * early-phase label: `ERR_PNPM_FETCH_<5xx>` is pnpm's TARBALL fetch error yet
+ * maps to `registry_unreachable`, and `detectReasonFromText` matches
+ * ECONNRESET/ETIMEDOUT/ENOTFOUND anywhere in stderr. A CDN 502 or a mid-download
+ * reset therefore degrades even though extraction had begun. npm rolls the reify
+ * back, but rollback restores FILES only — it does not restart the esbuild
+ * service child process that died when its binary was replaced, which is the
+ * actual harm chain in ENG-5297. So this gate does not close that case.
+ *
+ * The trade is still taken knowingly: closing it properly means stamping the
+ * phase where it is actually known (a `treePossiblyMutated` flag set at the
+ * throw site in `upgradeCliWithPackageManager`) or probing esbuild/Vite health
+ * after an `UpgradeFailed`, both of which change the shared
+ * `DependencyUpgradeError` contract and belong in their own change. Meanwhile
+ * this gate is strictly better than the previous behavior, which degraded on
+ * every upgrade failure without exception, and exiting on these categories
+ * instead would amplify a registry outage into a fleet-wide restart.
+ */
+const RESOLUTION_PHASE_UPGRADE_CATEGORIES: ReadonlySet<string> = new Set<
+  DependencyUpgradeError["category"]
+>([
+  "not_in_registry",
+  "registry_auth_failed",
+  "registry_unreachable",
+  "tls_failed",
+]);
+
+/**
+ * True when a failed auto-upgrade could have mutated the live dependency tree,
+ * and therefore may have killed the running esbuild service out from under
+ * Vite. Unknown/unclassified categories deliberately answer `true`: ENG-5297
+ * arrived as `category: "unknown"`, and a category this function has never seen
+ * must fail closed (exit) rather than inherit the degrade path by default.
+ *
+ * This reduces to `category === "unknown"` against today's five-member union —
+ * see `RESOLUTION_PHASE_UPGRADE_CATEGORIES` for why the set is kept rather than
+ * collapsed to that comparison.
+ *
+ * The parameter is `string`, not `DependencyUpgradeError["category"]`, because
+ * that is the actual runtime contract: the payload is deserialized from the
+ * socket, so the five-member union is a promise about our own writers rather
+ * than a guarantee about the value — which is also why the set above is a
+ * `ReadonlySet<string>`. It is also what lets the fail-closed default be
+ * asserted for a category a FUTURE CLI adds; that case is not expressible
+ * through an `UpgradeFailed` marker without an `as` cast, and it is the one this
+ * gate most needs to keep.
+ *
+ * `"unknown"` is itself coarse; the two sub-cases sit on opposite sides of the
+ * question this function asks, and only one of them is a false positive:
+ *
+ *   no install command   `resolveCommand` returned nothing, so `child_process.exec`
+ *                        was never reached (automatic-upgrades.ts, the
+ *                        `if (!installCommand)` early throw). The tree is
+ *                        provably untouched and exiting is a false positive.
+ *   version validation   `getCurrentCliVersion()` returned nothing / mismatched.
+ *                        That check runs AFTER `await promise` resolves — i.e.
+ *                        after the global install completed — so the tree
+ *                        genuinely moved and exiting is correct.
+ *
+ * Only the first is a false positive, and it is not distinguishable from data
+ * already on `serverError`: both arrive as `category: "unknown"` with no
+ * `npmErrorCode`, and the only thing separating them is free-text English in
+ * `rawError`. Keying the exit decision on that string would be the same
+ * phase-blind text proxy this file already flags as the gate's weak point.
+ * Splitting them properly means a flag stamped at the throw site on
+ * `DependencyUpgradeError` — a change to a shared contract, deliberately not
+ * made here. Until then both land on exit, the direction this function is meant
+ * to err in.
+ *
+ * Note on cost if the false positive fires: SABS pods run `RestartPolicy: Never`,
+ * so the retry is a pod REPLACEMENT driven by the orchestrator, not an in-pod
+ * crash loop. A deterministic, environment-derived cause would therefore
+ * re-trigger in each replacement, and no per-process bound inside this file
+ * could stop that — a replacement is a fresh process with a fresh counter. Any
+ * real bound belongs in the orchestrator that decides to replace.
+ */
+export function upgradeCategoryCouldTaintTree(category: string): boolean {
+  return !RESOLUTION_PHASE_UPGRADE_CATEGORIES.has(category);
+}
+
 /** Decide how the startup catch handles an error: degrade (record, keep Vite up)
  *  for an app-install failure (`InitialInstallFailed`) or a best-effort
  *  CLI/library auto-upgrade failure (`UpgradeFailed`), or exit for anything else
@@ -910,6 +1261,17 @@ export function handleStartupError(
   error: unknown,
   status: DevServerStatus,
   logger: Logger,
+  options?: {
+    /**
+     * True when this process was launched as a CSB sandbox — a SABS live-edit
+     * pod OR the local `csb-mock-server` emulator, which sets the same
+     * `SUPERBLOCKS_IS_CSB` variable. Decides whether a failed best-effort
+     * upgrade may degrade or must be fatal — see the `UpgradeFailed` branch.
+     * `isDisposableSandbox` documents why the local emulator is deliberately
+     * included and what that costs a local dev.
+     */
+    disposableSandbox?: boolean;
+  },
 ): "degrade" | "exit" {
   if (error instanceof InitialInstallFailed) {
     status.serverErrors.push(error.serverError);
@@ -925,20 +1287,66 @@ export function handleStartupError(
     return "degrade";
   }
   if (error instanceof UpgradeFailed) {
-    // The auto-upgrade is best-effort and the pod is still running the current
-    // version, so a failed upgrade degrades instead of crash-looping. Recording
-    // here also makes the failure observable: the process survives to the next
-    // metrics export tick, which the pre-exit `process.exit(1)` always lost.
     status.serverErrors.push(error.serverError);
-    logger.error(
-      "[dev-server] dependency upgrade failed; keeping the current version",
-      getErrorMeta(error),
-    );
+
+    // The upgrade is an in-place `npm install` into the LIVE node_modules, and
+    // its install manifest includes esbuild. A failure part-way through can
+    // therefore have already killed the running esbuild service ("The service
+    // was stopped"), which rejects vitePromise and leaves a dev server that can
+    // never serve a request again. "Keeping the current version" is only true
+    // when the tree survived, and we cannot cheaply prove that it did.
+    //
+    // So the decision turns on who owns the process AND on whether the failure
+    // could have mutated the tree at all:
+    //   - disposable sandbox (SABS pod) + a failure that could have touched
+    //     `node_modules`: exit. The pod is cheap and SABS already reaps
+    //     terminal pods (they run RestartPolicy: Never), so exiting yields a
+    //     clean replacement. Degrading instead produced ENG-5297 — 85 minutes
+    //     of a Ready pod answering 500s.
+    //   - disposable sandbox + a resolution-phase failure: degrade. npm never
+    //     got as far as extraction, so the tree is intact and there is nothing
+    //     for a replacement pod to fix — see
+    //     `upgradeCategoryCouldTaintTree`, which also explains why exiting on
+    //     these would amplify a registry outage into a fleet-wide restart.
+    //   - anything not flagged as a CSB sandbox: degrade. Killing a dev server
+    //     the developer started themselves over a best-effort upgrade is worse
+    //     than running the older version. Note the carve-out is narrower than
+    //     "not in the cloud": the local `csb-mock-server` emulator also sets
+    //     `SUPERBLOCKS_IS_CSB=true`, so local code-mode dev servers exit here
+    //     too. That is intended (the emulator exists to behave like a pod), but
+    //     nothing in the emulator restarts the child — see
+    //     `isDisposableSandbox`.
+    const outcome =
+      options?.disposableSandbox &&
+      upgradeCategoryCouldTaintTree(error.serverError.category)
+        ? "exit"
+        : "degrade";
     devServerMetrics.recordUpgradeFailure({
       category: error.serverError.category,
       npmErrorCode: error.serverError.npmErrorCode,
       hasAnyRegistryConfigured: error.serverError.hasAnyRegistryConfigured,
+      outcome,
     });
+
+    if (outcome === "exit") {
+      logger.error(
+        "[dev-server] dependency upgrade failed in a disposable sandbox; exiting so the sandbox is replaced (the live dependency tree may be tainted)",
+        getErrorMeta(error),
+      );
+      // No `devServerMetrics.flush()` here: `record()` and `flush()` gate on
+      // the same `isTelemetryInitialized()` check and nothing async runs
+      // between them, so if the sample above was buffered, flush would bail on
+      // the identical condition — and if it was not buffered, there is nothing
+      // to drain. The caller's `flushTelemetryBeforeExit` (which awaits
+      // `shutdownTelemetry()`) is what actually forces the export before
+      // `process.exit(1)`.
+      return "exit";
+    }
+
+    logger.error(
+      "[dev-server] dependency upgrade failed; keeping the current version",
+      getErrorMeta(error),
+    );
     return "degrade";
   }
   return "exit";
@@ -1196,6 +1604,7 @@ export async function dev(options: {
           superblocksBaseUrl: tokenConfig.superblocksBaseUrl,
           rpcClient,
           logger,
+          removeManualCommitsEnabled: featureFlags.removeManualCommitsEnabled(),
         });
 
         logger.info("Checking if local files are synced with the server");
@@ -1902,22 +2311,18 @@ export async function dev(options: {
               // which is why that runner reports zero samples in every env
               // despite real upgrades emitting it (spans survive because they
               // flush on the faster batch processor; the metric does not).
-              // shutdownTelemetry() never rejects; the catch is purely
-              // defensive. It resolves to `true` only when a flush actually
-              // ran — `false` on internal shutdown error or when telemetry was
-              // disabled — and the timeout branch leaves the flag false. So
-              // the log below reflects whether samples were really flushed, not
-              // merely that the race settled.
+              // The timeout-raced flush is shared with the fatal-exit path
+              // (`flushTelemetryWithTimeout`) — this site used to carry its own
+              // copy of the race, and that copy leaked its timer.
+              // `flushTelemetryWithTimeout` resolves `true` only when a flush
+              // actually ran — `false` on internal shutdown error, when
+              // telemetry was disabled, or when the timeout won — so the log
+              // below reflects whether samples were really flushed, not merely
+              // that the race settled. It rejects only if shutdownTelemetry()
+              // breaks its never-reject contract, so the catch stays defensive.
               let telemetryFlushed = false;
               try {
-                await Promise.race([
-                  shutdownTelemetry().then((flushed) => {
-                    telemetryFlushed = flushed;
-                  }),
-                  new Promise<void>((resolve) =>
-                    setTimeout(resolve, TELEMETRY_FLUSH_TIMEOUT_MS),
-                  ),
-                ]);
+                telemetryFlushed = await flushTelemetryWithTimeout();
               } catch (e) {
                 logger.error(
                   "Error flushing telemetry before CLI restart",
@@ -1938,20 +2343,59 @@ export async function dev(options: {
           });
         } catch (error: any) {
           if (
-            handleStartupError(error, devServerStatus, logger) === "degrade"
+            handleStartupError(error, devServerStatus, logger, {
+              disposableSandbox: isDisposableSandbox(),
+            }) === "degrade"
           ) {
             // app-install failure: do NOT exit — fall through to Vite startup below.
             // upload + CLI-restart were already skipped (the rejecting join threw first).
           } else {
             logger.error(
-              "[dev-server] Startup failed during sync/lock/setup (exiting with code 1)",
+              // Names the stage, because `buildFatalExitLog` below carries only
+              // the generic `handler=startupFailure`. Two stages reach here now:
+              // sync/lock/setup, and (since ENG-5297) a dependency upgrade that
+              // failed in a CSB sandbox and may have tainted the live tree.
+              "[dev-server] Startup failed during sync/lock/setup, or a dependency upgrade failed in a CSB sandbox (exiting with code 1)",
               getErrorMeta(error),
             );
+            // Emitted BEFORE any teardown, and as a synchronous log line rather
+            // than a counter, per the convention documented in
+            // dev-utils/fatal-exit.mts: at exit time stdout is the only sink we
+            // can rely on, so the reason has to land even if the flush below
+            // never completes. The `outcome="exit"` metric is a complement to
+            // this line, not a substitute for it.
+            logger.error(
+              buildFatalExitLog({
+                handler: "startupFailure",
+                exitCode: 1,
+                isWarm: parseIsWarm(process.env.SUPERBLOCKS_WARM_STANDBY),
+              }),
+            );
             try {
-              await aiService?.removeIntegrationCache();
-              await lockService?.shutdownAndExit();
+              // `shutdown()`, NOT `shutdownAndExit()`. The latter ends in an
+              // unconditional `finally { process.exit(1) }`; `process.exit` is
+              // synchronous, so every statement below it would be dead code
+              // whenever `lockService` is set — which is the normal state in a
+              // SABS pod, exactly the population this exit path targets. The
+              // telemetry flush was silently never running. `shutdown()` does
+              // the same teardown without owning the exit, matching how
+              // `runGracefulShutdown` in dev-server.mts drives it; the
+              // `finally` below performs the exit instead.
+              //
+              // Ordering, per-step error handling, and per-step timeouts all
+              // live in `runFatalExitShutdown` so the two fatal paths cannot
+              // drift apart.
+              await runFatalExitShutdown({
+                logger,
+                removeIntegrationCache: () =>
+                  aiService?.removeIntegrationCache(),
+                shutdownLockService: () =>
+                  lockService?.shutdown({ serverInitiated: false }),
+              });
             } finally {
-              // this is redundant, but it's here to make sure the lock service is shutdown and the process exits
+              // The only exit on this path now that the lock service no longer
+              // owns one. `runFatalExitShutdown` never throws and bounds every
+              // step, so this runs regardless of what teardown did.
               process.exit(1);
             }
           }
@@ -1983,17 +2427,39 @@ export async function dev(options: {
       try {
         await joinUpgradeThenInstall("before Vite startup");
       } catch (error) {
-        if (handleStartupError(error, devServerStatus, logger) === "exit") {
+        if (
+          handleStartupError(error, devServerStatus, logger, {
+            disposableSandbox: isDisposableSandbox(),
+          }) === "exit"
+        ) {
           logger.error(
             "[dev-server] Startup failed during pre-Vite install join (exiting with code 1)",
             getErrorMeta(error),
           );
+          // Same convention as the sync/lock catch above: a synchronous
+          // `event=dev_server_fatal_exit` line, written before teardown, is the
+          // thing oncall queries for this exit class.
+          logger.error(
+            buildFatalExitLog({
+              handler: "startupFailure",
+              exitCode: 1,
+              isWarm: parseIsWarm(process.env.SUPERBLOCKS_WARM_STANDBY),
+            }),
+          );
           try {
-            await aiService?.removeIntegrationCache();
-            await lockService?.shutdownAndExit();
+            // `shutdown()`, NOT `shutdownAndExit()` — same reason as the
+            // sync/lock catch above: `shutdownAndExit()` exits the process
+            // synchronously in its own `finally`, which made everything
+            // sequenced after it unreachable.
+            await runFatalExitShutdown({
+              logger,
+              removeIntegrationCache: () => aiService?.removeIntegrationCache(),
+              shutdownLockService: () =>
+                lockService?.shutdown({ serverInitiated: false }),
+            });
           } finally {
-            // Redundant with `shutdownAndExit`; here so a thrown shutdown
-            // path can't leave the process hanging on a stuck handle.
+            // The only exit on this path; here so a thrown shutdown path can't
+            // leave the process hanging on a stuck handle.
             process.exit(1);
           }
         }

package/src/client.billing-usage.test.ts

@@ -57,6 +57,23 @@ describe("fetchBillingUsageRecordsResponse", () => {
           source: "seat",
         },
       ],
+      usageGroupRows: [
+        {
+          actorId: "user-1",
+          actorType: "user",
+          applicationId: "00000000-0000-4000-8000-000000000001",
+          applicationName: "Operations",
+          checkpointId: "commit-abc123",
+          checkpointLabel: "Add revenue chart",
+          creditsUsed: 40,
+          date: "2026-03-24",
+          email: "alice@example.com",
+          name: "Alice",
+          onDemandCreditsUsed: 0,
+          recordedAt: "2026-03-24 10:15:00",
+          unbilledCreditsUsed: 0,
+        },
+      ],
     };
     mockAxios.mockResolvedValue({ data: { data: payload } });
 
@@ -69,5 +86,14 @@ describe("fetchBillingUsageRecordsResponse", () => {
         token: "test-token",
       }),
     ).resolves.toEqual(payload);
+
+    const response = await fetchBillingUsageRecordsResponse({
+      cliVersion: "1.2.3",
+      endDate: "2026-03-25",
+      startDate: "2026-03-24",
+      superblocksBaseUrl: "https://staging.superblocks.com",
+      token: "test-token",
+    });
+    expect(response.usageGroupRows?.[0]?.checkpointId).toBe("commit-abc123");
   });
 });