@superblocksteam/sdk 2.0.130-next.0 → 2.0.130-next.2

package/src/dev-utils/dev-server.mts

@@ -65,6 +65,7 @@ import {
   type DevServerFailureType,
   devServerMetrics,
 } from "./dev-server-metrics.mjs";
+import { buildFatalExitLog, parseIsWarm } from "./fatal-exit.mjs";
 import {
   formatViteDevServerStartedLog,
   logViteBuildError,
@@ -1101,6 +1102,18 @@ export async function createDevServer({
     }
   });
 
+  // Read once at startup. The pod's warm-or-cold launch mode is set when the pod
+  // starts and never changes, so this flag stays correct in the fatal-exit log
+  // even if the pod is activated later.
+  const isWarm = parseIsWarm(process.env.SUPERBLOCKS_WARM_STANDBY);
+
+  // Only the first fatal event writes a fatal-exit line and starts shutdown.
+  // Without this, one crash could write several lines: an unhandled rejection
+  // fires once per rejected promise, so a burst of failures (say a dead DB pool
+  // rejecting every in-flight query) would each write a line and each start
+  // shutdown, making one crash look like many in the logs and the crash count.
+  let fatalExitHandled = false;
+
   // Signal handlers attach `.catch` so a synchronous throw in
   // `runGracefulShutdown` (e.g. logger init or lockService shutdown throwing
   // before the first `await`) is logged rather than surfacing as an
@@ -1139,13 +1152,25 @@ export async function createDevServer({
     );
   });
 
-  // `uncaughtException` always force-exits in `.finally`, even when the
-  // underlying shutdown promise was already resolved by a prior signal. With
-  // the dedupe in place, awaiting the cached resolved promise short-circuits
-  // through `.then` without firing `.catch`; without the `.finally` the
-  // process would linger in event-loop limbo until the OS killed it.
+  // The `.finally` always ends the process once shutdown settles. If an earlier
+  // signal already ran shutdown, this handler still needs to exit; without the
+  // final `process.exit` the process could hang instead of stopping.
   process.on("uncaughtException", (error) => {
-    logger.error(`Uncaught exception: ${error.message}`, getErrorMeta(error));
+    if (fatalExitHandled) {
+      return;
+    }
+    fatalExitHandled = true;
+    // Write the fatal-exit line first, and synchronously, so the reason reaches
+    // the logs before anything else runs. Shutdown below may finish first and
+    // call `process.exit(0)`, so the container's real exit code can be 0 even
+    // though this was a crash. That is why `exit_code=1` here is the code this
+    // handler means to use, not a promise of what the container reports: for the
+    // JS-handler lines, trust that the line exists (and the `signal` field) over
+    // its `exit_code` when comparing against the pod's exit status.
+    logger.error(
+      buildFatalExitLog({ handler: "uncaughtException", exitCode: 1, isWarm }),
+      getErrorMeta(error),
+    );
     gracefulShutdown({
       logger,
       serverInitiated: false,
@@ -1154,7 +1179,34 @@ export async function createDevServer({
       .catch((shutdownError) => {
         logger.error(
           "Error during shutdown after uncaught exception:",
-          shutdownError,
+          getErrorMeta(shutdownError),
+        );
+      })
+      .finally(() => process.exit(1));
+  });
+
+  // Without its own listener, Node turns an unhandled promise rejection into an
+  // uncaught exception, which would log it under the wrong cause. Handle it here
+  // so a rejection is labelled as a rejection, then exit through the same
+  // shutdown path as an uncaught exception.
+  process.on("unhandledRejection", (reason) => {
+    if (fatalExitHandled) {
+      return;
+    }
+    fatalExitHandled = true;
+    logger.error(
+      buildFatalExitLog({ handler: "unhandledRejection", exitCode: 1, isWarm }),
+      getErrorMeta(reason),
+    );
+    gracefulShutdown({
+      logger,
+      serverInitiated: false,
+      source: "unhandledRejection",
+    })
+      .catch((shutdownError) => {
+        logger.error(
+          "Error during shutdown after unhandled rejection:",
+          getErrorMeta(shutdownError),
         );
       })
       .finally(() => process.exit(1));

package/src/dev-utils/fatal-exit.mts

@@ -0,0 +1,113 @@
+/**
+ * Structured fatal-exit logging for the dev server.
+ *
+ * The most common cause of live-edit pod deaths is a bare Node exit code 1 with
+ * no reason attached: the supervisor only sees the container exit code from
+ * Kubernetes, and the gVisor runtime reports no CPU/memory/event-loop metrics,
+ * so there is no way to tell why the process stopped. These helpers build one
+ * log line, easy to search for, that says why the process is exiting. It is
+ * written right before the process exits so the reason still lands in the pod
+ * logs even when the batched telemetry export never gets flushed.
+ *
+ * The fields go in the message body as `key=value` markers instead of as
+ * telemetry attributes, on purpose: at exit time the synchronous stdout write is
+ * the only sink we can rely on, while the batched OTel log exporter is usually
+ * dropped by `process.exit`. The existing warm-standby failure path in
+ * dev-server.mts already puts its markers in the body for the same reason. And
+ * for the same reason there is deliberately no OTel counter for fatal exits (a
+ * counter increment would be dropped at exit); `FATAL_EXIT_EVENT` below is the
+ * log token to query and alert on.
+ *
+ * Counting note for alerting: one crash can produce up to two of these lines.
+ * When a JS handler runs and its own `process.exit(1)` runs before shutdown can
+ * exit, the child emits a `handler=uncaughtException`/`unhandledRejection` line
+ * AND the supervisor (CLI parent, or the warm entrypoint shell) emits a
+ * `handler=containerExit` line for the same exit. A signal kill / OOM emits only
+ * the `containerExit` line (no JS handler ran); a JS crash where shutdown exits
+ * first with code 0 emits only the JS-handler line (the supervisor sees a clean
+ * exit). So a raw `count(event=dev_server_fatal_exit)` over-counts the double
+ * case: deduplicate by pod within a short time window for a true unique-crash
+ * count.
+ *
+ * Signal-kill field note: for a signal kill the cold parent emits
+ * `exit_code=null signal=SIGKILL` (Node's `child.on("exit")` gives a null code +
+ * the signal), while the warm entrypoint shell emits `exit_code=128+N
+ * signal=SIGKILL` (its `wait` returns the raw 128+N status). Both are correct
+ * for their context; query on the `signal` field, not `exit_code`, for
+ * signal-kill alerts.
+ */
+
+/**
+ * Where the fatal exit came from, so each kind can be told apart in the logs.
+ *
+ *   uncaughtException / unhandledRejection  A JS handler ran and knows the error
+ *                                           name/message/stack.
+ *   containerExit                           No JS handler ran (signal kill / OOM,
+ *                                           or the graceful-shutdown exit race);
+ *                                           the supervisor only knows the code +
+ *                                           signal. The CLI parent (dev-parent.mts)
+ *                                           and the warm entrypoint shell emit this
+ *                                           value. They cannot import this module,
+ *                                           so they repeat the same string.
+ */
+export type FatalExitHandler =
+  | "uncaughtException"
+  | "unhandledRejection"
+  | "containerExit";
+
+/** Fixed token so a log-based metric or a log search can pick out these lines. */
+export const FATAL_EXIT_EVENT = "dev_server_fatal_exit";
+
+export interface FatalExitFields {
+  /** Where the fatal exit came from. */
+  handler: FatalExitHandler;
+  /**
+   * Exit code the process is about to use (JS handler path) or the code the
+   * supervisor saw (container-exit path). `null` when the child was killed by a
+   * signal and has no exit code.
+   */
+  exitCode: number | null;
+  /** Signal that killed the process; absent for a JS-level exit. */
+  signal?: string | null;
+  /** Whether this pod was started as a warm-standby pod (entrypoint `--warm`). */
+  isWarm: boolean;
+}
+
+/**
+ * Parse the entrypoint's warm-standby env var the same way
+ * `scripts/dev-server-entrypoint.sh` does, so `is_warm` in the log matches the
+ * pod's launch mode whether or not it was activated later.
+ */
+export function parseIsWarm(value: string | undefined): boolean {
+  switch ((value ?? "").trim().toLowerCase()) {
+    case "true":
+    case "1":
+    case "yes":
+    case "on":
+      return true;
+    default:
+      return false;
+  }
+}
+
+/**
+ * Build the fatal-exit log message body. Every value is a simple primitive
+ * (enum / number / boolean), so the line is safe to write as-is even on the CSB
+ * log path.
+ *
+ * SYNC: this format is reproduced in two places that cannot import this module —
+ * `buildFatalExitLog` in `packages/cli/packages/cli/src/commands/dev-parent.mts`
+ * (the parent stays on `node:` builtins) and the `echo` line in
+ * `scripts/dev-server-entrypoint.sh` (bash). A field added here must be added in
+ * both, or `event=dev_server_fatal_exit` queries will miss one emitter.
+ */
+export function buildFatalExitLog(fields: FatalExitFields): string {
+  return [
+    "Dev server fatal exit",
+    `event=${FATAL_EXIT_EVENT}`,
+    `handler=${fields.handler}`,
+    `exit_code=${fields.exitCode ?? "null"}`,
+    `signal=${fields.signal ?? "none"}`,
+    `is_warm=${fields.isWarm}`,
+  ].join(" ");
+}

package/src/dev-utils/fatal-exit.test.mts

@@ -0,0 +1,81 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  buildFatalExitLog,
+  FATAL_EXIT_EVENT,
+  parseIsWarm,
+} from "./fatal-exit.mjs";
+
+describe("parseIsWarm", () => {
+  it("treats the same truthy spellings as the entrypoint shell", () => {
+    for (const value of ["true", "1", "yes", "on", "TRUE", "Yes", " on "]) {
+      expect(parseIsWarm(value)).toBe(true);
+    }
+  });
+
+  it("treats anything else (including undefined) as not warm", () => {
+    for (const value of ["false", "0", "no", "off", "", "cold", undefined]) {
+      expect(parseIsWarm(value)).toBe(false);
+    }
+  });
+});
+
+describe("buildFatalExitLog", () => {
+  it("carries the stable event marker so a log-based metric can isolate the line", () => {
+    const line = buildFatalExitLog({
+      handler: "uncaughtException",
+      exitCode: 1,
+      isWarm: true,
+    });
+    expect(line).toContain(`event=${FATAL_EXIT_EVENT}`);
+  });
+
+  it("encodes the handler, exit code, signal, and warm flag as snake_case markers", () => {
+    const line = buildFatalExitLog({
+      handler: "uncaughtException",
+      exitCode: 1,
+      signal: "SIGABRT",
+      isWarm: true,
+    });
+    expect(line).toContain("handler=uncaughtException");
+    expect(line).toContain("exit_code=1");
+    expect(line).toContain("signal=SIGABRT");
+    expect(line).toContain("is_warm=true");
+  });
+
+  it("defaults the signal to none for a JS-level exit and renders a null code", () => {
+    const line = buildFatalExitLog({
+      handler: "containerExit",
+      exitCode: null,
+      isWarm: false,
+    });
+    expect(line).toContain("signal=none");
+    expect(line).toContain("exit_code=null");
+    expect(line).toContain("is_warm=false");
+  });
+
+  it("renders an explicit null signal as none (interface allows null and undefined)", () => {
+    const line = buildFatalExitLog({
+      handler: "uncaughtException",
+      exitCode: 1,
+      signal: null,
+      isWarm: false,
+    });
+    expect(line).toContain("signal=none");
+  });
+
+  it("produces the exact format-contract string (field names, order, separators)", () => {
+    // The full line is the log-query/alerting contract shared across the three
+    // emitters. A reordered field, changed separator, or added prefix would slip
+    // past the per-field toContain checks but break queries; this locks it.
+    const line = buildFatalExitLog({
+      handler: "uncaughtException",
+      exitCode: 1,
+      signal: null,
+      isWarm: false,
+    });
+    expect(line).toBe(
+      "Dev server fatal exit event=dev_server_fatal_exit handler=uncaughtException exit_code=1 signal=none is_warm=false",
+    );
+  });
+});