@effectstream/db 0.100.22 → 0.100.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/db",
-  "version": "0.100.22",
+  "version": "0.100.24",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -21,11 +21,11 @@
     "./version": "./migrations/system-version.ts"
   },
   "dependencies": {
-    "@effectstream/coroutine": "0.100.22",
-    "@effectstream/log": "0.100.22",
-    "@effectstream/runtime": "0.100.22",
-    "@effectstream/sync": "0.100.22",
-    "@effectstream/utils": "0.100.22",
+    "@effectstream/coroutine": "0.100.24",
+    "@effectstream/log": "0.100.24",
+    "@effectstream/runtime": "0.100.24",
+    "@effectstream/sync": "0.100.24",
+    "@effectstream/utils": "0.100.24",
     "@electric-sql/pglite": "^0.3.14",
     "@pgtyped/runtime": "2.4.2",
     "@pgtyped/parser": "2.4.2",

package/scripts/start-pglite.ts

@@ -31,6 +31,10 @@ export async function startPglite(port = 5432): Promise<PgliteHandle> {
     },
   );
 
+  // Serialize every client's execProtocolRaw through one promise chain
+  // PGlite allows only one in-flight query, and per-process db_mutex can't coordinate across separate TCP clients.
+  let queue: Promise<unknown> = Promise.resolve();
+
   // TODO: consider switching to pglite-socket once it works
   //       https://discord.com/channels/933657521581858818/1371976702674075780/1371992712076595250
   const server = net.createServer(async (socket) => {
@@ -54,8 +58,11 @@ export async function startPglite(port = 5432): Promise<PgliteHandle> {
           return;
         }
 
-        // Forward raw message to PGlite and send response to client
-        return await db.execProtocolRaw(data);
+        // Forward raw message to PGlite and send response to client, serialized
+        // through the queue so the single WASM backend handles one at a time.
+        const run = queue.then(() => db.execProtocolRaw(data));
+        queue = run.catch(() => {}); // keep the chain alive even if a query errors
+        return await run;
       },
     });
   });

package/src/mod.ts

@@ -54,6 +54,7 @@ export * from "./sql/tables.queries.ts";
 
 export * from "./event-indexing.ts";
 export * from "./register-events.ts";
+export * from "./transient-pg-errors.ts";
 export * from "./pg-connection.ts";
 export * from "./scheduled-constructors.ts";
 export * from "./system.ts";

package/src/pg-connection.ts

@@ -1,10 +1,16 @@
 import type { Client, PoolClient, PoolConfig } from "pg";
 import pg from "pg";
-import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
+import { ComponentNames, installUnhandledRejectionLogger } from "@effectstream/log";
 import { type Operation, run, sleep } from "effection";
 import { ENV } from "@effectstream/utils/node-env";
+import { isTransientPgError } from "./transient-pg-errors.ts";
+import { poolErrors } from "./pool-error-tracker.ts";
+
+export { isTransientPgError } from "./transient-pg-errors.ts";
+export { poolErrors, PoolErrorTracker } from "./pool-error-tracker.ts";
 
 let readonlyDBConn: pg.Pool | null;
+let unhandledRejectionLoggerInstalled = false;
 
 // PGLite does not support multiple connections, so we need to use a mutex to ensure that only one query is executed at a time.
 // * For transactions use yield* acquireDBMutex(); ...Your Operations... releaseDBMutex();
@@ -138,6 +144,8 @@ export async function runPreparedQuery<T>(
     });
     result = await Promise.race([p, timeout]);
     clearTimeout(t);
+    // A successful query proves the DB is reachable end-to-end
+    poolErrors.markHealthy();
   } finally {
     releaseDBMutex(`pq:${name}`);
   }
@@ -148,6 +156,14 @@ export const getConnection = (
   creds?: PoolConfig,
   readonly = false,
 ): pg.Pool => {
+  if (!unhandledRejectionLoggerInstalled) {
+    unhandledRejectionLoggerInstalled = true;
+    installUnhandledRejectionLogger(
+      ComponentNames.EFFECTSTREAM_PGLITE,
+      isTransientPgError,
+    );
+  }
+
   if (!creds) {
     creds = {
       host: ENV.DB_HOST,
@@ -163,30 +179,21 @@ export const getConnection = (
   const MAX_CONNECTIONS = ENV.PGLITE ? 1 : 10;
 
   const pool = new pg.Pool({ ...creds, max: MAX_CONNECTIONS });
-  pool.on("error", (err: unknown) =>
-    log.remote(
-      ComponentNames.EFFECTSTREAM_PGLITE,
-      ["query"],
-      SeverityNumber.ERROR,
-      (log) => log(err),
-    ));
+  pool.on("error", (err: unknown) => poolErrors.log(err, ["query"]));
   pool.on("connect", (_client: PoolClient) => {
-    // On each new client initiated, need to register for error(this is a serious bug on pg, the client throw errors although it should not)
+    // Successful connect proves the DB is reachable — reset the failure clock.
+    poolErrors.markHealthy();
+    // Register a per-client error handler; pg can emit errors on idle clients.
     // https://github.com/brianc/node-postgres/issues/2499#issuecomment-805477725
-    _client.on("error", (err: Error) => {
-      log.remote(
-        ComponentNames.EFFECTSTREAM_PGLITE,
-        ["connect"],
-        SeverityNumber.ERROR,
-        (log) => log(err),
-      );
-    });
+    _client.on("error", (err: Error) => poolErrors.log(err, ["connect"]));
   });
 
   if (readonly) {
     const ensureReadOnly =
       `SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;`;
-    void pool.query(ensureReadOnly); // note: this query modifies the DB state
+    void pool.query(ensureReadOnly).catch((err: unknown) =>
+      poolErrors.log(err, ["readonly-setup"]),
+    );
     readonlyDBConn = pool;
   }
 
@@ -198,14 +205,6 @@ export const getPersistentConnection = (creds: PoolConfig): Client => {
   const client = new pg.Client(creds);
   client.connect(() => {});
   // https://github.com/brianc/node-postgres/issues/2499#issuecomment-805477725
-  // On each new client initiated, need to register for error(this is a serious bug on pg, the client throw errors although it should not)
-  client.on("error", (err: Error) => {
-    log.remote(
-      ComponentNames.EFFECTSTREAM_PGLITE,
-      ["query"],
-      SeverityNumber.ERROR,
-      (log) => log(err),
-    );
-  });
+  client.on("error", (err: Error) => poolErrors.log(err, ["persistent"]));
   return client;
 };

package/src/pool-error-tracker.test.ts

@@ -0,0 +1,65 @@
+import { afterEach, expect, test } from "bun:test";
+
+// Thresholds are read from env at module load (and `envMs` rejects 0, so use
+// small positive values). SUSTAINED=5ms → a failure older than 5ms is
+// "sustained"; FATAL=30ms → older than 30ms triggers the process.exit
+// escalation. Dynamic import so these are picked up before the module
+// computes its constants.
+process.env.POOL_SUSTAINED_THRESHOLD_MS = "5";
+process.env.POOL_FATAL_THRESHOLD_MS = "30";
+const { PoolErrorTracker } = await import("./pool-error-tracker.ts");
+
+const transient = () => ({
+  code: "ECONNRESET",
+  message: "Client network socket disconnected before secure TLS connection",
+  stack: "    at node_modules/pg-pool/index.js:45:11",
+});
+
+const realExit = process.exit;
+afterEach(() => {
+  process.exit = realExit;
+});
+
+test("ignores non-transient errors (auth/schema bugs)", () => {
+  const t = new PoolErrorTracker();
+  t.log(new Error("column foo does not exist"), ["sync-write"]);
+  expect(t.state().consecutive).toBe(0);
+  expect(t.state().sustained).toBe(false);
+});
+
+test("counts consecutive transient failures and reports sustained", async () => {
+  const t = new PoolErrorTracker();
+  t.log(transient(), ["sync-write"]);
+  expect(t.state().sustained).toBe(false); // fresh failure, < 5ms old
+  await Bun.sleep(15); // exceed SUSTAINED_THRESHOLD_MS=5, stay under FATAL=30
+  t.log(transient(), ["sync-write"]);
+  const s = t.state();
+  expect(s.consecutive).toBe(2);
+  expect(s.sustained).toBe(true);
+});
+
+test("markHealthy resets the failure clock", () => {
+  const t = new PoolErrorTracker();
+  t.log(transient(), ["sync-write"]);
+  t.markHealthy();
+  const s = t.state();
+  expect(s.consecutive).toBe(0);
+  expect(s.firstFailureAt).toBe(0);
+  expect(s.sustained).toBe(false);
+});
+
+test("exits the process once failures persist past the fatal threshold", async () => {
+  let exitCode: number | undefined;
+  process.exit = ((code?: number) => {
+    exitCode = code;
+  }) as typeof process.exit;
+
+  const t = new PoolErrorTracker();
+  t.log(transient(), ["sync-write"]); // starts the clock; not yet fatal
+  await Bun.sleep(40); // exceed FATAL_THRESHOLD_MS=20
+  t.log(transient(), ["sync-write"]); // now duration >= fatal → schedules exit
+
+  // exit is deferred via setImmediate so the fatal log line can flush first.
+  await new Promise((resolve) => setImmediate(resolve));
+  expect(exitCode).toBe(1);
+});

package/src/pool-error-tracker.ts

@@ -0,0 +1,87 @@
+import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
+import { isTransientPgError } from "./transient-pg-errors.ts";
+
+function envMs(name: string, fallback: number): number {
+  const raw = process.env[name];
+  const parsed = raw != null ? Number(raw) : NaN;
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+/** After this long, treat continuous transient failures as a real outage. */
+const SUSTAINED_THRESHOLD_MS = envMs("POOL_SUSTAINED_THRESHOLD_MS", 60_000);
+/** After this long, exit so the orchestrator restarts the process. */
+const FATAL_THRESHOLD_MS = envMs("POOL_FATAL_THRESHOLD_MS", 5 * 60_000);
+
+/**
+ * Tracks consecutive transient pool errors. Severity escalates WARN → ERROR
+ * once failures persist past `SUSTAINED_THRESHOLD_MS`, and the process exits
+ * past `FATAL_THRESHOLD_MS` (relying on the orchestrator's restart policy).
+ * Call `markHealthy()` on any positive signal to reset.
+ */
+export class PoolErrorTracker {
+  private consecutive = 0;
+  private firstFailureAt = 0;
+
+  log(err: unknown, tags: string[]): void {
+    // Non-transient errors are a different failure mode (auth, schema, etc.).
+    if (!isTransientPgError(err)) {
+      log.remote(
+        ComponentNames.EFFECTSTREAM_PGLITE,
+        tags,
+        SeverityNumber.ERROR,
+        (l) => l(err),
+      );
+      return;
+    }
+
+    if (this.consecutive === 0) this.firstFailureAt = Date.now();
+    this.consecutive++;
+
+    const durationMs = Date.now() - this.firstFailureAt;
+    const sustained = durationMs >= SUSTAINED_THRESHOLD_MS;
+    const fatal = durationMs >= FATAL_THRESHOLD_MS;
+
+    log.remote(
+      ComponentNames.EFFECTSTREAM_PGLITE,
+      fatal
+        ? [...tags, "sustained-failure", "fatal"]
+        : sustained
+        ? [...tags, "sustained-failure"]
+        : tags,
+      sustained ? SeverityNumber.ERROR : SeverityNumber.WARN,
+      (l) => l({ err, consecutive: this.consecutive, durationMs }),
+    );
+
+    if (fatal) {
+      // Defer to the next tick so the log line above has a chance to flush.
+      setImmediate(() => process.exit(1));
+    }
+  }
+
+  /** Call when the DB has proven reachable (e.g. successful pool connect). */
+  markHealthy(): void {
+    this.consecutive = 0;
+    this.firstFailureAt = 0;
+  }
+
+  /** Read-only snapshot for health checks, heartbeats, or external probes. */
+  state(): {
+    consecutive: number;
+    firstFailureAt: number;
+    sustainedDurationMs: number;
+    sustained: boolean;
+  } {
+    const durationMs = this.firstFailureAt === 0
+      ? 0
+      : Date.now() - this.firstFailureAt;
+    return {
+      consecutive: this.consecutive,
+      firstFailureAt: this.firstFailureAt,
+      sustainedDurationMs: durationMs,
+      sustained: durationMs >= SUSTAINED_THRESHOLD_MS,
+    };
+  }
+}
+
+/** Process-wide singleton — failures aggregate at DB-health granularity. */
+export const poolErrors = new PoolErrorTracker();

package/src/sql/rollup_inputs.queries.ts

@@ -383,6 +383,61 @@ const getFutureGameInputByMaxTimestampIR: any = {"usedParamSet":{"max_timestamp"
 export const getFutureGameInputByMaxTimestamp = new PreparedQuery<IGetFutureGameInputByMaxTimestampParams,IGetFutureGameInputByMaxTimestampResult>(getFutureGameInputByMaxTimestampIR);
 
 
+/** 'GetEarliestScheduledBlockHeight' parameters type */
+export type IGetEarliestScheduledBlockHeightParams = void;
+
+/** 'GetEarliestScheduledBlockHeight' return type */
+export interface IGetEarliestScheduledBlockHeightResult {
+  min_block_height: number | null;
+}
+
+/** 'GetEarliestScheduledBlockHeight' query type */
+export interface IGetEarliestScheduledBlockHeightQuery {
+  params: IGetEarliestScheduledBlockHeightParams;
+  result: IGetEarliestScheduledBlockHeightResult;
+}
+
+const getEarliestScheduledBlockHeightIR: any = {"usedParamSet":{},"params":[],"statement":"SELECT MIN(future_block_height) AS min_block_height\nFROM effectstream.rollup_input_future_block"};
+
+/**
+ * Query generated from SQL:
+ * ```
+ * SELECT MIN(future_block_height) AS min_block_height
+ * FROM effectstream.rollup_input_future_block
+ * ```
+ */
+export const getEarliestScheduledBlockHeight = new PreparedQuery<IGetEarliestScheduledBlockHeightParams,IGetEarliestScheduledBlockHeightResult>(getEarliestScheduledBlockHeightIR);
+
+
+/** 'GetEarliestScheduledTimestamp' parameters type */
+export type IGetEarliestScheduledTimestampParams = void;
+
+/** 'GetEarliestScheduledTimestamp' return type */
+export interface IGetEarliestScheduledTimestampResult {
+  min_timestamp: Date | null;
+}
+
+/** 'GetEarliestScheduledTimestamp' query type */
+export interface IGetEarliestScheduledTimestampQuery {
+  params: IGetEarliestScheduledTimestampParams;
+  result: IGetEarliestScheduledTimestampResult;
+}
+
+const getEarliestScheduledTimestampIR: any = {"usedParamSet":{},"params":[],"statement":"SELECT MIN(rollup_input_future_timestamp.future_ms_timestamp) AS min_timestamp\nFROM effectstream.rollup_input_future_timestamp\nLEFT OUTER JOIN effectstream.rollup_input_result\n  ON (effectstream.rollup_input_result.id = effectstream.rollup_input_future_timestamp.id)\nWHERE effectstream.rollup_input_result.id IS NULL"};
+
+/**
+ * Query generated from SQL:
+ * ```
+ * SELECT MIN(rollup_input_future_timestamp.future_ms_timestamp) AS min_timestamp
+ * FROM effectstream.rollup_input_future_timestamp
+ * LEFT OUTER JOIN effectstream.rollup_input_result
+ *   ON (effectstream.rollup_input_result.id = effectstream.rollup_input_future_timestamp.id)
+ * WHERE effectstream.rollup_input_result.id IS NULL
+ * ```
+ */
+export const getEarliestScheduledTimestamp = new PreparedQuery<IGetEarliestScheduledTimestampParams,IGetEarliestScheduledTimestampResult>(getEarliestScheduledTimestampIR);
+
+
 /** 'GetGameInputResultByBlockHeight' parameters type */
 export interface IGetGameInputResultByBlockHeightParams {
   block_height: number;

package/src/sql/rollup_inputs.sql

@@ -144,6 +144,17 @@ WHERE rollup_input_future_timestamp.future_ms_timestamp <= :max_timestamp! AND
      effectstream.rollup_input_result.id IS NULL
 ORDER BY rollup_input_future_timestamp.future_ms_timestamp ASC;
 
+/* @name getEarliestScheduledBlockHeight */
+SELECT MIN(future_block_height) AS min_block_height
+FROM effectstream.rollup_input_future_block;
+
+/* @name getEarliestScheduledTimestamp */
+SELECT MIN(rollup_input_future_timestamp.future_ms_timestamp) AS min_timestamp
+FROM effectstream.rollup_input_future_timestamp
+LEFT OUTER JOIN effectstream.rollup_input_result
+  ON (effectstream.rollup_input_result.id = effectstream.rollup_input_future_timestamp.id)
+WHERE effectstream.rollup_input_result.id IS NULL;
+
 /* @name getGameInputResultByBlockHeight */
 SELECT
   rollup_inputs.id,

package/src/transient-pg-errors.test.ts

@@ -0,0 +1,47 @@
+import { test, expect } from "bun:test";
+import { isTransientPgError } from "./transient-pg-errors.ts";
+
+test("isTransientPgError - Neon pg-pool TLS reset", () => {
+  expect(
+    isTransientPgError({
+      code: "ECONNRESET",
+      message:
+        "Client network socket disconnected before secure TLS connection was established",
+      stack: "    at /app/node_modules/.bun/pg-pool@3.14.0/node_modules/pg-pool/index.js:45:11",
+    }),
+  ).toBe(true);
+});
+
+test("isTransientPgError - rejects non-pg ETIMEDOUT", () => {
+  expect(
+    isTransientPgError({
+      code: "ETIMEDOUT",
+      message: "fetch failed",
+      stack: "    at node:internal/deps/undici/undici.js:1234:5",
+    }),
+  ).toBe(false);
+});
+
+test("isTransientPgError - walks cause chain", () => {
+  const inner = {
+    code: "ECONNRESET",
+    message:
+      "Client network socket disconnected before secure TLS connection was established",
+    stack: "    at node_modules/pg-pool/index.js:45:11",
+  };
+  expect(isTransientPgError({ message: "wrapper", cause: inner })).toBe(true);
+});
+
+test("isTransientPgError - rejects logic errors", () => {
+  expect(isTransientPgError(new Error("column foo does not exist"))).toBe(
+    false,
+  );
+});
+
+test("isTransientPgError - idle pool message without stack", () => {
+  expect(
+    isTransientPgError({
+      message: "Connection terminated unexpectedly",
+    }),
+  ).toBe(true);
+});

package/src/transient-pg-errors.ts

@@ -0,0 +1,56 @@
+const PG_STACK_RE = /node_modules\/(pg-pool|pg)\//;
+
+const TRANSIENT_PG_MESSAGES = [
+  "Connection terminated unexpectedly",
+  "Client has encountered a connection error",
+  "Client network socket disconnected before secure TLS connection",
+  "Query read timeout",
+] as const;
+
+function* walkErrors(reason: unknown): Generator<unknown> {
+  let current: unknown = reason;
+  const seen = new Set<unknown>();
+  while (current != null && typeof current === "object" && !seen.has(current)) {
+    seen.add(current);
+    yield current;
+    current = (current as { cause?: unknown }).cause;
+  }
+}
+
+function isFromPgStack(err: unknown): boolean {
+  if (err == null || typeof err !== "object" || !("stack" in err)) {
+    return false;
+  }
+  return PG_STACK_RE.test(String((err as Error).stack));
+}
+
+function hasTransientPgMessage(err: unknown): boolean {
+  if (err == null || typeof err !== "object") return false;
+  const msg = (err as { message?: string }).message;
+  if (typeof msg !== "string") return false;
+  return TRANSIENT_PG_MESSAGES.some((fragment) => msg.includes(fragment));
+}
+
+function isTransientPgErrorSingle(err: unknown): boolean {
+  if (err == null || typeof err !== "object") return false;
+  if (hasTransientPgMessage(err)) return true;
+  if (!isFromPgStack(err)) return false;
+  const code = (err as { code?: string }).code;
+  return code === "ECONNRESET" || code === "EPIPE" ||
+    code === "ECONNREFUSED";
+}
+
+/**
+ * Returns true for pg / network errors that are normal on serverless Postgres
+ * (Neon, etc.): cold-start RSTs, idle-socket evictions, transient TLS blips.
+ * The pool recovers on the next query — use with `installUnhandledRejectionLogger`
+ * from `@effectstream/log` so these do not crash the process.
+ *
+ * Intentionally narrow: known-recoverable conditions only; logic/SQL bugs still crash.
+ */
+export function isTransientPgError(reason: unknown): boolean {
+  for (const err of walkErrors(reason)) {
+    if (isTransientPgErrorSingle(err)) return true;
+  }
+  return false;
+}

package/src/unhandled-rejection-wiring.test.ts

@@ -0,0 +1,63 @@
+// Pins the PRODUCTION wiring used in pg-connection.ts:
+//   installUnhandledRejectionLogger(EFFECTSTREAM_PGLITE, isTransientPgError)
+// i.e. the live `unhandledRejection` handler treats `isTransientPgError` as its
+// survive predicate. This is the test @acedward asked for: prove that when
+// something other than a known-transient pg error escapes to the global
+// handler (e.g. a generic error torn out of the effection root), the process is
+// actually told to exit — it does NOT silently log-and-survive into a zombie
+// whose effection tree is dead but whose process is still up.
+
+import { test, expect } from "bun:test";
+import { buildUnhandledRejectionHandler } from "@effectstream/log";
+import { isTransientPgError } from "./transient-pg-errors.ts";
+
+const wireProductionHandler = () => {
+  const exits: number[] = [];
+  const handle = buildUnhandledRejectionHandler(
+    "pglite",
+    isTransientPgError,
+    (code) => exits.push(code),
+  );
+  return { handle, exits };
+};
+
+test("generic error escaping the effection root is fatal (no zombie)", () => {
+  const { handle, exits } = wireProductionHandler();
+
+  // What a halted effection scope surfaces: an ordinary Error, not a pg error.
+  handle(new Error("effection root task died"));
+
+  expect(exits).toEqual([1]);
+});
+
+test("a logic/SQL bug is fatal", () => {
+  const { handle, exits } = wireProductionHandler();
+  handle(new Error("column foo does not exist"));
+  expect(exits).toEqual([1]);
+});
+
+test("a genuine transient pg blip survives (the intended exemption)", () => {
+  const { handle, exits } = wireProductionHandler();
+
+  handle({
+    code: "ECONNRESET",
+    message:
+      "Client network socket disconnected before secure TLS connection was established",
+    stack: "    at node_modules/pg-pool/index.js:45:11",
+  });
+
+  expect(exits).toEqual([]);
+});
+
+test("a non-pg ECONNRESET-shaped rejection is still fatal", () => {
+  const { handle, exits } = wireProductionHandler();
+
+  // Same code, but not from the pg stack => isTransientPgError returns false.
+  handle({
+    code: "ETIMEDOUT",
+    message: "fetch failed",
+    stack: "    at node:internal/deps/undici/undici.js:1234:5",
+  });
+
+  expect(exits).toEqual([1]);
+});