@superblocksteam/sdk 2.0.157-next.4 → 2.0.157-next.5

package/src/dev-utils/dev-server.mts

@@ -69,11 +69,12 @@ import {
   type DevServerFailureType,
   devServerMetrics,
 } from "./dev-server-metrics.mjs";
+import { isRecoverableShutdownError, parseIsWarm } from "./fatal-exit.mjs";
 import {
-  buildFatalExitLog,
-  isRecoverableShutdownError,
-  parseIsWarm,
-} from "./fatal-exit.mjs";
+  fatalAwareExitCode,
+  installFatalProcessBarrier,
+  type FatalProcessHandler,
+} from "./fatal-process-barrier.mjs";
 import { OPTIMIZE_DEPS_CONFIG } from "./optimize-deps-config.mjs";
 import { prepareInPlaceRestart } from "./prepare-in-place-restart.mjs";
 import { handleRecoverableBuildError } from "./recoverable-build-error.mjs";
@@ -546,7 +547,7 @@ export function buildStatusPayload<T extends object>(
   return { ...base, serverErrors: devServerStatus?.serverErrors ?? [] };
 }
 
-type ProcessErrorHandler = "uncaughtException" | "unhandledRejection";
+type ProcessErrorHandler = FatalProcessHandler;
 
 // Both listeners log a recoverable shutdown, but with different wording, and
 // these lines are what on-call greps when a pod stays up instead of restarting.
@@ -557,6 +558,59 @@ const RECOVERABLE_SHUTDOWN_LOG_MESSAGE: Record<ProcessErrorHandler, string> = {
     "Ignoring recoverable shutdown rejection (unhandledRejection)",
 };
 
+/**
+ * What the dev server needs of the lock service on its way out. Structural so a
+ * test does not have to stand up a real one.
+ */
+type ExitingLockService = Pick<
+  LockService,
+  "notifyClosedByServer" | "relinquishLock"
+>;
+
+/**
+ * Give the AppBranchLock to whoever should have it next, then let the caller
+ * exit.
+ *
+ * Which of the two things to do is the dev server's knowledge, not the lock
+ * service's — it used to be a `serverInitiated` flag passed down into a
+ * `shutdown()` that branched on it, which is how a lock service came to have an
+ * opinion about why a process was stopping.
+ *
+ * Normally this process is the one stopping, so the lock goes back: the
+ * heartbeat dies with the process, and anything not released here makes the
+ * replacement sandbox's `acquireLock()` fail until the server-side TTL expires
+ * — and the replacement is the entire point of exiting.
+ *
+ * When the server closed us, the lock is already gone. `/_sb_disconnect` is
+ * usually a session moving across local↔cloud, so releasing would take the lock
+ * from whoever is taking over. The editor is told instead, because it has to
+ * distinguish "the server closed you" from a socket dropping.
+ *
+ * Extracted so both branches are testable: `runGracefulShutdown` is a closure
+ * over a booted dev server, so a unit test cannot otherwise reach either one.
+ */
+export async function handOverLockOnExit(
+  lockService: ExitingLockService | undefined,
+  {
+    serverInitiated,
+    switchingTo,
+    initiatedByEmail,
+  }: {
+    serverInitiated: boolean;
+    switchingTo?: "local" | "cloud" | "none";
+    initiatedByEmail?: string;
+  },
+): Promise<void> {
+  if (!lockService) {
+    return;
+  }
+  if (serverInitiated) {
+    await lockService.notifyClosedByServer({ switchingTo, initiatedByEmail });
+    return;
+  }
+  await lockService.relinquishLock();
+}
+
 /**
  * Decides whether a process-level error keeps the pod alive, and performs the
  * recording for the cases that do. Returns `true` when the caller must return
@@ -623,6 +677,22 @@ export function handleNonFatalProcessError(params: {
 export const DEV_SERVER_BOOT_ID = randomUUID();
 export const DEV_SERVER_STARTED_AT = new Date().toISOString();
 
+export function mountDevServerJsonParsers(app: IRouter): void {
+  app.use(
+    "/_sb_policy_gates",
+    express.json({
+      limit: "1mb",
+      strict: true,
+    }),
+  );
+  app.use(
+    express.json({
+      limit: "100kb",
+      strict: true,
+    }),
+  );
+}
+
 /**
  * Builds the `/_sb_health` payload, including the per-process boot identity.
  * Extracted as a pure function so the boot-identity contract can be unit
@@ -871,7 +941,7 @@ export async function createDevServer({
   }): Promise<void> {
     try {
       stopMemoryMetrics();
-      await lockService?.shutdown({
+      await handOverLockOnExit(lockService, {
         serverInitiated,
         switchingTo,
         initiatedByEmail,
@@ -882,16 +952,23 @@ export async function createDevServer({
     } catch (e) {
       logger.error(`Error during graceful shutdown: ${e}`);
     } finally {
+      // The statement about this process, made by the thing that owns its exit.
+      // It used to be the last line of `LockService.shutdown()`, prefixed
+      // `[lock-service]` — a lock service announcing a process exit, which is
+      // the tangle this change removes.
+      //
+      // Load bearing: cli-system-tests waits on this exact substring.
+      logger.info("[dev-server] shutting down and exiting dev server");
       const closeTimeout = setTimeout(() => {
         logger.info("Dev server closed");
-        process.exit(0);
+        process.exit(fatalAwareExitCode());
       }, 1000); // Fallback to exit if the HTTP server doesn't close within 1 second
 
       // Then close the HTTP server
       httpServer.close(() => {
         logger.info("Dev server closed");
         clearTimeout(closeTimeout);
-        process.exit(0);
+        process.exit(fatalAwareExitCode());
       });
     }
   }
@@ -902,11 +979,7 @@ export async function createDevServer({
       credentials: true,
     }),
   );
-  app.use(
-    express.json({
-      strict: true,
-    }),
-  );
+  mountDevServerJsonParsers(app);
 
   // Per-request metrics for dev-server endpoints. Identify the endpoint by URL
   // pattern, time the request, record duration + status + coarse failure type
@@ -1266,18 +1339,11 @@ export async function createDevServer({
   // even if the pod is activated later.
   const isWarm = parseIsWarm(process.env.SUPERBLOCKS_WARM_STANDBY);
 
-  // Only the first fatal event writes a fatal-exit line and starts shutdown.
-  // Without this, one crash could write several lines: an unhandled rejection
-  // fires once per rejected promise, so a burst of failures (say a dead DB pool
-  // rejecting every in-flight query) would each write a line and each start
-  // shutdown, making one crash look like many in the logs and the crash count.
-  let fatalExitHandled = false;
-
   // Signal handlers attach `.catch` so a synchronous throw in
   // `runGracefulShutdown` (e.g. logger init or lockService shutdown throwing
   // before the first `await`) is logged rather than surfacing as an
   // unhandled rejection across N handlers (one per fired signal). Symmetric
-  // with the `uncaughtException` handler below.
+  // with the process barrier below.
   process.on("SIGINT", () => {
     logger.info("SIGINT received");
     gracefulShutdown({
@@ -1311,86 +1377,37 @@ export async function createDevServer({
     );
   });
 
-  // The `.finally` always ends the process once shutdown settles. If an earlier
-  // signal already ran shutdown, this handler still needs to exit; without the
-  // final `process.exit` the process could hang instead of stopping.
-  process.on("uncaughtException", (error) => {
-    if (fatalExitHandled) {
-      return;
-    }
-    if (
+  // Preload may already have armed this from a separate module instance
+  // (node_modules export vs this esbuild bundle). A second call updates
+  // shutdown hooks on the shared host state and does not register another
+  // pair of listeners.
+  installFatalProcessBarrier({
+    isNonFatal: (handler, reason) =>
       handleNonFatalProcessError({
-        devServerStatus,
         devServerMetrics,
-        handler: "uncaughtException",
-        logger,
-        reason: error,
-      })
-    ) {
-      return;
-    }
-    fatalExitHandled = true;
-    // Write the fatal-exit line first, and synchronously, so the reason reaches
-    // the logs before anything else runs. Shutdown below may finish first and
-    // call `process.exit(0)`, so the container's real exit code can be 0 even
-    // though this was a crash. That is why `exit_code=1` here is the code this
-    // handler means to use, not a promise of what the container reports: for the
-    // JS-handler lines, trust that the line exists (and the `signal` field) over
-    // its `exit_code` when comparing against the pod's exit status.
-    logger.error(
-      buildFatalExitLog({ handler: "uncaughtException", exitCode: 1, isWarm }),
-      getErrorMeta(error),
-    );
-    gracefulShutdown({
-      logger,
-      serverInitiated: false,
-      source: "uncaughtException",
-    })
-      .catch((shutdownError) => {
-        logger.error(
-          "Error during shutdown after uncaught exception:",
-          getErrorMeta(shutdownError),
-        );
-      })
-      .finally(() => process.exit(1));
-  });
-
-  // Without its own listener, Node turns an unhandled promise rejection into an
-  // uncaught exception, which would log it under the wrong cause. Handle it here
-  // so a rejection is labelled as a rejection, then exit through the same
-  // shutdown path as an uncaught exception.
-  process.on("unhandledRejection", (reason) => {
-    if (fatalExitHandled) {
-      return;
-    }
-    if (
-      handleNonFatalProcessError({
         devServerStatus,
-        devServerMetrics,
-        handler: "unhandledRejection",
+        handler,
         logger,
         reason,
-      })
-    ) {
-      return;
-    }
-    fatalExitHandled = true;
-    logger.error(
-      buildFatalExitLog({ handler: "unhandledRejection", exitCode: 1, isWarm }),
-      getErrorMeta(reason),
-    );
-    gracefulShutdown({
-      logger,
-      serverInitiated: false,
-      source: "unhandledRejection",
-    })
-      .catch((shutdownError) => {
+      }),
+    isWarm,
+    log: (message, meta) => logger.error(message, meta),
+    onFatal: async (handler) => {
+      try {
+        await gracefulShutdown({
+          logger,
+          serverInitiated: false,
+          source: handler,
+        });
+      } catch (shutdownError) {
         logger.error(
-          "Error during shutdown after unhandled rejection:",
+          handler === "uncaughtException"
+            ? "Error during shutdown after uncaught exception:"
+            : "Error during shutdown after unhandled rejection:",
           getErrorMeta(shutdownError),
         );
-      })
-      .finally(() => process.exit(1));
+      }
+    },
   });
 
   if (existingServer) {

package/src/dev-utils/dev-server.shutdown-lock.test.mts

@@ -0,0 +1,58 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { handOverLockOnExit } from "./dev-server.mjs";
+
+/**
+ * ENG-6128. The dev server's exit path used to drive the lock service through a
+ * `shutdown({ serverInitiated })` that decided, inside the lock service, whether
+ * to release. That flag is the dev server's knowledge, not the lock service's,
+ * and getting it wrong in either direction is a live-edit outage: releasing a
+ * lock the server has already reassigned takes the app away from the session
+ * taking over, and failing to release on a normal exit makes the replacement
+ * sandbox wait out the TTL.
+ */
+describe("handOverLockOnExit", () => {
+  const makeLockService = () => ({
+    notifyClosedByServer: vi.fn(async () => undefined),
+    relinquishLock: vi.fn(async () => undefined),
+  });
+
+  it("hands the lock back when the dev server is the one stopping", async () => {
+    const lockService = makeLockService();
+
+    await handOverLockOnExit(lockService, { serverInitiated: false });
+
+    // Prompt release is the whole recovery story for this path: the heartbeat
+    // dies with the process, so anything not released here blocks the
+    // replacement sandbox for the two-minute TTL.
+    expect(lockService.relinquishLock).toHaveBeenCalledOnce();
+    expect(lockService.notifyClosedByServer).not.toHaveBeenCalled();
+  });
+
+  it("only notifies when the server closed the session", async () => {
+    const lockService = makeLockService();
+
+    await handOverLockOnExit(lockService, {
+      serverInitiated: true,
+      switchingTo: "local",
+      initiatedByEmail: "someone@example.com",
+    });
+
+    // The server has already taken the lock — on a local↔cloud switch it now
+    // belongs to the session taking over, so releasing would take it away from
+    // them.
+    expect(lockService.relinquishLock).not.toHaveBeenCalled();
+    expect(lockService.notifyClosedByServer).toHaveBeenCalledWith({
+      switchingTo: "local",
+      initiatedByEmail: "someone@example.com",
+    });
+  });
+
+  it("is a no-op when the dev server never had a lock service", async () => {
+    // Startup can fail before the lock service exists, and the exit path still
+    // runs.
+    await expect(
+      handOverLockOnExit(undefined, { serverInitiated: false }),
+    ).resolves.toBeUndefined();
+  });
+});

package/src/dev-utils/fatal-process-barrier.mts

@@ -0,0 +1,262 @@
+import {
+  sanitizeLogError,
+  sanitizeLogMessage,
+} from "@superblocksteam/telemetry";
+
+import { safeStringify } from "../telemetry/safe-stringify.js";
+import { buildFatalExitLog, parseIsWarm } from "./fatal-exit.mjs";
+
+export type FatalProcessHandler = "uncaughtException" | "unhandledRejection";
+
+export type FatalProcessListener = (...args: unknown[]) => void;
+
+/**
+ * The slice of `process` this needs, so a test can emit a crash without arming
+ * the real handlers and taking the runner down with them.
+ */
+export type FatalProcessHost = {
+  off: (event: string, handler: FatalProcessListener) => void;
+  on: (event: string, handler: FatalProcessListener) => void;
+};
+
+export type FatalErrorMeta = {
+  error: {
+    kind: string;
+    message: string;
+    stack?: string;
+  };
+};
+
+export type FatalProcessBarrierHooks = {
+  isNonFatal?: (handler: FatalProcessHandler, reason: unknown) => boolean;
+  /**
+   * Runs after the fatal-exit line is written and before `exit(1)`. The server
+   * path uses this for graceful shutdown. The early/preload path leaves it
+   * unset.
+   */
+  onFatal?: (
+    handler: FatalProcessHandler,
+    reason: unknown,
+  ) => void | Promise<void>;
+};
+
+export type FatalProcessBarrierOptions = FatalProcessBarrierHooks & {
+  exit?: (code: number) => void;
+  isWarm?: boolean;
+  log?: (message: string, meta?: FatalErrorMeta) => void;
+  process?: FatalProcessHost;
+};
+
+type BarrierState = {
+  fatalHandled: boolean;
+  hooks: {
+    exit: (code: number) => void;
+    isNonFatal?: FatalProcessBarrierHooks["isNonFatal"];
+    isWarm: boolean;
+    log: (message: string, meta?: FatalErrorMeta) => void;
+    onFatal?: FatalProcessBarrierHooks["onFatal"];
+  };
+  off: () => void;
+};
+
+/**
+ * Armed state lives on `globalThis`, not in a module-level WeakMap: the
+ * packaged CLI loads TWO copies of this file (bin/run.js imports the
+ * node_modules export; createDevServer uses the esbuild-bundled copy). A
+ * per-module map would make the second call register a new listener pair
+ * instead of attaching `isNonFatal` / `onFatal`, and the early copy would
+ * `exit(1)` on recoverable Vite/socket errors before shutdown ran.
+ */
+declare global {
+  var __superblocksFatalProcessBarrierArmed:
+    | WeakMap<FatalProcessHost, BarrierState>
+    | undefined;
+}
+
+function armedHosts(): WeakMap<FatalProcessHost, BarrierState> {
+  const existing = globalThis.__superblocksFatalProcessBarrierArmed;
+  if (existing) {
+    return existing;
+  }
+  const created = new WeakMap<FatalProcessHost, BarrierState>();
+  globalThis.__superblocksFatalProcessBarrierArmed = created;
+  return created;
+}
+
+function fatalErrorMeta(reason: unknown): FatalErrorMeta {
+  if (reason instanceof Error) {
+    return {
+      error: {
+        kind: reason.name,
+        message: reason.message,
+        stack: reason.stack,
+      },
+    };
+  }
+  return {
+    error: {
+      kind: "Unknown Error",
+      message: safeStringify(reason),
+    },
+  };
+}
+
+function stringField(value: unknown, key: string): string | undefined {
+  if (typeof value !== "object" || value === null || !(key in value)) {
+    return undefined;
+  }
+  const field = Reflect.get(value, key);
+  if (typeof field === "string") {
+    return field;
+  }
+  if (field === undefined) {
+    return undefined;
+  }
+  return safeStringify(field);
+}
+
+function defaultLog(message: string, meta?: FatalErrorMeta): void {
+  const safeMessage = sanitizeLogMessage(message);
+  if (!meta?.error) {
+    console.error(safeMessage);
+    return;
+  }
+  const sanitized = sanitizeLogError(meta.error);
+  console.error(safeMessage, {
+    error: {
+      kind:
+        stringField(sanitized, "kind") ??
+        stringField(sanitized, "name") ??
+        "Error",
+      message: stringField(sanitized, "message"),
+      stack: stringField(sanitized, "stack"),
+    },
+  });
+}
+
+function mergeHooks(
+  existing: BarrierState["hooks"],
+  options: FatalProcessBarrierOptions,
+): BarrierState["hooks"] {
+  return {
+    exit: options.exit ?? existing.exit,
+    isNonFatal: options.isNonFatal ?? existing.isNonFatal,
+    isWarm: options.isWarm ?? existing.isWarm,
+    log: options.log ?? existing.log,
+    onFatal: options.onFatal ?? existing.onFatal,
+  };
+}
+
+function handleCrash(
+  state: BarrierState,
+  handler: FatalProcessHandler,
+  reason: unknown,
+): void {
+  if (state.fatalHandled) {
+    return;
+  }
+  try {
+    if (state.hooks.isNonFatal?.(handler, reason)) {
+      return;
+    }
+  } catch (predicateError) {
+    // A throw inside the non-fatal check must not skip the fatal-exit line.
+    state.hooks.log(
+      "fatal process barrier isNonFatal threw; continuing as fatal",
+      fatalErrorMeta(predicateError),
+    );
+  }
+  state.fatalHandled = true;
+  try {
+    state.hooks.log(
+      buildFatalExitLog({
+        exitCode: 1,
+        handler,
+        isWarm: state.hooks.isWarm,
+      }),
+      fatalErrorMeta(reason),
+    );
+  } catch {
+    // Meta construction or the logger can throw; still run onFatal and exit(1).
+  }
+  const finish = (): void => {
+    state.hooks.exit(1);
+  };
+  const onFatal = state.hooks.onFatal;
+  if (!onFatal) {
+    finish();
+    return;
+  }
+  void Promise.resolve()
+    .then(() => onFatal(handler, reason))
+    .catch(() => undefined)
+    .finally(finish);
+}
+
+/**
+ * Arm process-level crash handlers once per host. A second call on the same
+ * host is a no-op for listeners and updates hooks when the caller supplies them
+ * (preload arms early; `createDevServer` later attaches shutdown). Shared
+ * across the node_modules and esbuild copies of this module.
+ */
+export function installFatalProcessBarrier(
+  options: FatalProcessBarrierOptions = {},
+): () => void {
+  const host = options.process ?? process;
+  const armed = armedHosts();
+  const existing = armed.get(host);
+  if (existing) {
+    existing.hooks = mergeHooks(existing.hooks, options);
+    return existing.off;
+  }
+
+  const state: BarrierState = {
+    fatalHandled: false,
+    hooks: {
+      exit: options.exit ?? ((code: number) => process.exit(code)),
+      isNonFatal: options.isNonFatal,
+      isWarm:
+        options.isWarm ?? parseIsWarm(process.env.SUPERBLOCKS_WARM_STANDBY),
+      log: options.log ?? defaultLog,
+      onFatal: options.onFatal,
+    },
+    off: () => undefined,
+  };
+
+  const onException = (error: unknown) => {
+    handleCrash(state, "uncaughtException", error);
+  };
+  const onRejection = (reason: unknown) => {
+    handleCrash(state, "unhandledRejection", reason);
+  };
+
+  host.on("uncaughtException", onException);
+  host.on("unhandledRejection", onRejection);
+
+  state.off = () => {
+    armed.delete(host);
+    host.off("uncaughtException", onException);
+    host.off("unhandledRejection", onRejection);
+  };
+  armed.set(host, state);
+  return state.off;
+}
+
+export function isFatalProcessHandled(
+  host: FatalProcessHost = process,
+): boolean {
+  return armedHosts().get(host)?.fatalHandled ?? false;
+}
+
+/** Exit code graceful shutdown should use so a crash is not reported as 0. */
+export function fatalAwareExitCode(host: FatalProcessHost = process): number {
+  return isFatalProcessHandled(host) ? 1 : 0;
+}
+
+/**
+ * Child/warm entry: arm before oclif/`dev()` so a startup crash still writes
+ * `event=dev_server_fatal_exit`. Safe to call again later from createDevServer.
+ */
+export function installEarlyFatalProcessBarrier(): void {
+  installFatalProcessBarrier();
+}

package/src/dev-utils/fatal-process-barrier.spawn-fixture.mts

@@ -0,0 +1,12 @@
+import { pathToFileURL } from "node:url";
+
+import { installEarlyFatalProcessBarrier } from "./fatal-process-barrier.mjs";
+
+const isMain =
+  process.argv[1] !== undefined &&
+  import.meta.url === pathToFileURL(process.argv[1]).href;
+
+if (isMain) {
+  installEarlyFatalProcessBarrier();
+  throw new TypeError("spawn-fixture crash");
+}