@effectstream/db 0.100.19 → 0.100.21

package/migrations/system-up-v-0.0.0.sql

@@ -1,7 +1,5 @@
 CREATE SCHEMA IF NOT EXISTS effectstream;
 
-CREATE EXTENSION IF NOT EXISTS pg_ivm;
-
 CREATE TABLE effectstream.effectstream_expected_version (
   app_version_major INTEGER NOT NULL,
   app_version_minor INTEGER NOT NULL,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/db",
-  "version": "0.100.19",
+  "version": "0.100.21",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -21,15 +21,15 @@
     "./version": "./migrations/system-version.ts"
   },
   "dependencies": {
-    "@effectstream/coroutine": "0.100.19",
-    "@effectstream/log": "0.100.19",
-    "@effectstream/runtime": "0.100.19",
-    "@effectstream/sync": "0.100.19",
-    "@effectstream/utils": "0.100.19",
+    "@effectstream/coroutine": "0.100.21",
+    "@effectstream/log": "0.100.21",
+    "@effectstream/runtime": "0.100.21",
+    "@effectstream/sync": "0.100.21",
+    "@effectstream/utils": "0.100.21",
     "@electric-sql/pglite": "^0.3.14",
     "@pgtyped/runtime": "2.4.2",
     "@pgtyped/parser": "2.4.2",
-    "pg": "8.17.2",
+    "pg": "8.21.0",
     "pg-tx": "^1.0.1",
     "effection": "^3.5.0",
     "pg-gateway": "^0.3.0-beta.4"

package/src/MaterializedViewStrategy.ts

@@ -0,0 +1,50 @@
+/**
+ * Strategy for publishing the read-side relation that primitives query
+ * (e.g. `primitives.erc20_balances_view_<name>`).
+ *
+ * The intermediate table and the maintenance trigger are identical in both
+ * modes — only how the view itself is created differs. The query API
+ * (`SELECT * FROM primitives.<viewName>`) is unchanged across strategies.
+ */
+export interface MaterializedViewStrategy {
+  /**
+   * Stable identifier used to namespace migration rows per strategy.
+   * Empty string for the legacy pg_ivm strategy, so its migration rows keep
+   * the same names that existing deployments already have in their
+   * `effectstream_migration_history`. Only the new fallback strategy uses a
+   * suffixed name. This avoids re-applying dynamic-table DDL (which would
+   * fail on already-existing triggers/IMMVs) when an existing app upgrades.
+   */
+  readonly migrationSuffix: "" | "-view";
+  /** Emit DDL that publishes `viewName` as a queryable relation backed by `selectSql`. */
+  createView(viewName: string, selectSql: string): string;
+}
+
+/**
+ * Uses the `pg_ivm` extension to create an incrementally maintained
+ * materialized view. Requires the extension to be installed in the database.
+ *
+ * Migration rows for this strategy use the historical unsuffixed name
+ * (`dynamic-tables-<type>-<name>`) so deployments that were already running
+ * the engine before the strategy pattern existed keep matching their
+ * previously-applied rows on upgrade — no re-application, no DDL conflicts.
+ */
+export const pgIvmStrategy: MaterializedViewStrategy = {
+  migrationSuffix: "",
+  createView: (viewName, selectSql) =>
+    `SELECT pgivm.create_immv('${viewName}', $IVM$${selectSql}$IVM$);`,
+};
+
+/**
+ * Fallback: a plain SQL view computed on read. The intermediate table is
+ * still trigger-maintained, so the only extra cost is re-applying the
+ * (typically cheap) WHERE filter at query time.
+ *
+ * Migration rows use a `-view` suffix to keep this strategy's history
+ * distinct from the pg_ivm strategy's.
+ */
+export const plainViewStrategy: MaterializedViewStrategy = {
+  migrationSuffix: "-view",
+  createView: (viewName, selectSql) =>
+    `CREATE OR REPLACE VIEW ${viewName} AS ${selectSql};`,
+};

package/src/capabilities.ts

@@ -0,0 +1,118 @@
+import type { PoolClient } from "pg";
+import { ENV } from "@effectstream/utils/node-env";
+import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
+import {
+  type MaterializedViewStrategy,
+  pgIvmStrategy,
+  plainViewStrategy,
+} from "./MaterializedViewStrategy.ts";
+
+/**
+ * Optional database features the engine probes for at startup.
+ *
+ * - `pgIvm`: the `pg_ivm` extension that provides incrementally maintained
+ *   materialized views. When unavailable, the engine can fall back to plain
+ *   SQL views over the same trigger-maintained intermediate tables — but
+ *   only if the operator has explicitly opted in via `ENV.ALLOW_NO_PG_IVM`.
+ */
+export interface DatabaseCapabilities {
+  pgIvm: boolean;
+}
+
+/**
+ * Probes the connected database for optional features. Pure detection —
+ * no policy decision is made here; the caller (typically via
+ * {@link selectViewStrategy}) decides whether the resulting capability set
+ * is acceptable.
+ *
+ * - Best-effort `CREATE EXTENSION IF NOT EXISTS pg_ivm` (no-op if already
+ *   installed, soft-fail if the binary is missing on the server).
+ * - Then queries `pg_extension` to confirm the real state.
+ */
+export async function detectCapabilities(
+  client: Pick<PoolClient, "query">,
+): Promise<DatabaseCapabilities> {
+  try {
+    await client.query("CREATE EXTENSION IF NOT EXISTS pg_ivm");
+  } catch {
+    // Extension binary not installed on this server — fall through and let
+    // the pg_extension lookup confirm the absence.
+  }
+
+  const { rows } = await client.query<{ exists: boolean }>(
+    `SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_ivm') AS exists`,
+  );
+  return { pgIvm: rows[0]?.exists === true };
+}
+
+/**
+ * Decide which materialized-view strategy to use based on detected
+ * capabilities and operator policy.
+ *
+ * Behavior matrix:
+ *
+ * | pg_ivm available | ALLOW_NO_PG_IVM | Result                                   |
+ * | :--------------- | :-------------- | :--------------------------------------- |
+ * | yes              | any             | pgIvmStrategy (production path)          |
+ * | no               | false / unset   | **throws** — startup aborts loudly       |
+ * | no               | true            | plainViewStrategy + prominent startup WARN |
+ *
+ * The throw is intentional: silently degrading to a slower read path in
+ * production is much worse than failing early with a clear remediation
+ * message. Operators who genuinely need the dev/test fallback must opt in.
+ */
+export function selectViewStrategy(
+  capabilities: DatabaseCapabilities,
+): MaterializedViewStrategy {
+  if (capabilities.pgIvm) return pgIvmStrategy;
+
+  if (!ENV.ALLOW_NO_PG_IVM) {
+    throw new Error(formatMissingPgIvmError());
+  }
+
+  log.local(
+    ComponentNames.EFFECTSTREAM_RUNTIME,
+    "pg_ivm-fallback",
+    SeverityNumber.WARN,
+    (l) =>
+      l(
+        [
+          "─────────────────────────────────────────────────────────────",
+          "  effectstream: running WITHOUT pg_ivm (fallback mode)",
+          "  ALLOW_NO_PG_IVM=true is set; using plain SQL views over the",
+          "  trigger-maintained intermediate tables.",
+          "",
+          "  This path is intended for dev/test and low-volume apps only.",
+          "  Read-path performance degrades sharply on long-lived contracts",
+          "  with high address cardinality (intermediate tables grow",
+          "  monotonically; the plain view does a seq scan + filter per",
+          "  read). DO NOT run production on this configuration.",
+          "─────────────────────────────────────────────────────────────",
+        ].join("\n"),
+      ),
+  );
+  return plainViewStrategy;
+}
+
+function formatMissingPgIvmError(): string {
+  return [
+    "",
+    "Effectstream startup aborted: the `pg_ivm` PostgreSQL extension is not",
+    "available on this database.",
+    "",
+    "The engine uses `pg_ivm` for incrementally maintained materialized views,",
+    "which give compact, fast reads on long-lived contracts. Without it the",
+    "engine can fall back to plain SQL views over the trigger-maintained",
+    "intermediate tables — but that path degrades sharply on high-cardinality",
+    "data and is only suitable for dev/test and low-volume apps.",
+    "",
+    "To proceed:",
+    "  • Install pg_ivm in your PostgreSQL instance:",
+    "      https://github.com/sraoss/pg_ivm",
+    "  • OR explicitly accept the dev-mode fallback by setting:",
+    "      ALLOW_NO_PG_IVM=true",
+    "    (NOT recommended for production — see the database docs for the",
+    "     performance ceiling.)",
+    "",
+  ].join("\n");
+}

package/src/dynamic-tables.ts

@@ -6,26 +6,42 @@ import type { PoolClient } from "pg";
 import type { VersionInfo } from "../migrations/system-version.ts";
 import { applyMigrations } from "../scripts/apply-migrations.ts";
 import { findMigrationByName } from "./sql/system.queries.ts";
+import {
+  type MaterializedViewStrategy,
+  plainViewStrategy,
+} from "./MaterializedViewStrategy.ts";
 
 /**
  * Creates dynamic tables for the given sync protocols.
- * This is used to create the IVMs for the given sync protocols.
+ * This is used to create the read-side relations (incrementally maintained
+ * via pg_ivm when available, otherwise plain SQL views) for each primitive's
+ * intermediate state.
  *
  * For example for ERC20 it creates a table that holds the balance for each address.
  * For ERC721 it creates a table that holds the token id for each address.
  *
  * @param dbConn - The database connection.
  * @param syncProtocols - The sync protocols.
+ * @param strategy - How to publish the read-side view. Defaults to the plain
+ *   view fallback so callers that haven't probed capabilities yet stay safe
+ *   on managed Postgres without pg_ivm. The runtime detects the extension
+ *   and passes `pgIvmStrategy` explicitly when available.
  */
 export function* createDynamicTables(
   versionInfo: VersionInfo,
   lastBlockHeight: number,
   dbConn: PoolClient,
   syncProtocols: any[], // AllSyncProtocols[],
+  strategy: MaterializedViewStrategy = plainViewStrategy,
 ) {
   for (const syncProtocol of syncProtocols) {
     for (const primitive of syncProtocol.config.primitives) {
-      yield* createDynamicTableForPrimitive(primitive, lastBlockHeight, dbConn);
+      yield* createDynamicTableForPrimitive(
+        primitive,
+        lastBlockHeight,
+        dbConn,
+        strategy,
+      );
     }
   }
 }
@@ -39,6 +55,7 @@ function* createDynamicTableForPrimitive(
   },
   lastBlockHeight: number,
   dbConn: PoolClient,
+  strategy: MaterializedViewStrategy,
 ) {
   const type = p.primitive.type;
   const name = p.primitive.name;
@@ -54,10 +71,15 @@ function* createDynamicTableForPrimitive(
 
   // This primitive does not have dynamic tables.
   if (!sqlFunction) return;
-  const code = sqlFunction(name);
+  const code = sqlFunction(name, strategy);
   if (!code) return;
 
-  const migrationName = `dynamic-tables-${type}-${name}`;
+  // pgIvmStrategy uses the legacy unsuffixed name so existing deployments
+  // upgrading from a pre-strategy build keep matching their previously-
+  // applied migration rows. plainViewStrategy uses a `-view` suffix to keep
+  // its history distinct, so a deployment that later gains or loses pg_ivm
+  // re-applies the correct view-creation DDL.
+  const migrationName = `dynamic-tables-${type}-${name}${strategy.migrationSuffix}`;
   const [migration] = yield* until(findMigrationByName.run({
     name: migrationName,
     isSystemMigration: true,

package/src/mod.ts

@@ -64,6 +64,13 @@ export {
   getPrimitiveIntermediatePrefix,
   getPrimitivePrefix,
 } from "./dynamic-tables.ts";
+export { detectCapabilities, selectViewStrategy } from "./capabilities.ts";
+export type { DatabaseCapabilities } from "./capabilities.ts";
+export {
+  pgIvmStrategy,
+  plainViewStrategy,
+} from "./MaterializedViewStrategy.ts";
+export type { MaterializedViewStrategy } from "./MaterializedViewStrategy.ts";
 // export { startPGlite } from "./start-pglite.ts";
 
 export { getMigrationsForBlockHeight } from "./migrations.ts";

package/src/sql/rollup_inputs.queries.ts

@@ -383,6 +383,61 @@ const getFutureGameInputByMaxTimestampIR: any = {"usedParamSet":{"max_timestamp"
 export const getFutureGameInputByMaxTimestamp = new PreparedQuery<IGetFutureGameInputByMaxTimestampParams,IGetFutureGameInputByMaxTimestampResult>(getFutureGameInputByMaxTimestampIR);
 
 
+/** 'GetEarliestScheduledBlockHeight' parameters type */
+export type IGetEarliestScheduledBlockHeightParams = void;
+
+/** 'GetEarliestScheduledBlockHeight' return type */
+export interface IGetEarliestScheduledBlockHeightResult {
+  min_block_height: number | null;
+}
+
+/** 'GetEarliestScheduledBlockHeight' query type */
+export interface IGetEarliestScheduledBlockHeightQuery {
+  params: IGetEarliestScheduledBlockHeightParams;
+  result: IGetEarliestScheduledBlockHeightResult;
+}
+
+const getEarliestScheduledBlockHeightIR: any = {"usedParamSet":{},"params":[],"statement":"SELECT MIN(future_block_height) AS min_block_height\nFROM effectstream.rollup_input_future_block"};
+
+/**
+ * Query generated from SQL:
+ * ```
+ * SELECT MIN(future_block_height) AS min_block_height
+ * FROM effectstream.rollup_input_future_block
+ * ```
+ */
+export const getEarliestScheduledBlockHeight = new PreparedQuery<IGetEarliestScheduledBlockHeightParams,IGetEarliestScheduledBlockHeightResult>(getEarliestScheduledBlockHeightIR);
+
+
+/** 'GetEarliestScheduledTimestamp' parameters type */
+export type IGetEarliestScheduledTimestampParams = void;
+
+/** 'GetEarliestScheduledTimestamp' return type */
+export interface IGetEarliestScheduledTimestampResult {
+  min_timestamp: Date | null;
+}
+
+/** 'GetEarliestScheduledTimestamp' query type */
+export interface IGetEarliestScheduledTimestampQuery {
+  params: IGetEarliestScheduledTimestampParams;
+  result: IGetEarliestScheduledTimestampResult;
+}
+
+const getEarliestScheduledTimestampIR: any = {"usedParamSet":{},"params":[],"statement":"SELECT MIN(rollup_input_future_timestamp.future_ms_timestamp) AS min_timestamp\nFROM effectstream.rollup_input_future_timestamp\nLEFT OUTER JOIN effectstream.rollup_input_result\n  ON (effectstream.rollup_input_result.id = effectstream.rollup_input_future_timestamp.id)\nWHERE effectstream.rollup_input_result.id IS NULL"};
+
+/**
+ * Query generated from SQL:
+ * ```
+ * SELECT MIN(rollup_input_future_timestamp.future_ms_timestamp) AS min_timestamp
+ * FROM effectstream.rollup_input_future_timestamp
+ * LEFT OUTER JOIN effectstream.rollup_input_result
+ *   ON (effectstream.rollup_input_result.id = effectstream.rollup_input_future_timestamp.id)
+ * WHERE effectstream.rollup_input_result.id IS NULL
+ * ```
+ */
+export const getEarliestScheduledTimestamp = new PreparedQuery<IGetEarliestScheduledTimestampParams,IGetEarliestScheduledTimestampResult>(getEarliestScheduledTimestampIR);
+
+
 /** 'GetGameInputResultByBlockHeight' parameters type */
 export interface IGetGameInputResultByBlockHeightParams {
   block_height: number;

package/src/sql/rollup_inputs.sql

@@ -144,6 +144,17 @@ WHERE rollup_input_future_timestamp.future_ms_timestamp <= :max_timestamp! AND
      effectstream.rollup_input_result.id IS NULL
 ORDER BY rollup_input_future_timestamp.future_ms_timestamp ASC;
 
+/* @name getEarliestScheduledBlockHeight */
+SELECT MIN(future_block_height) AS min_block_height
+FROM effectstream.rollup_input_future_block;
+
+/* @name getEarliestScheduledTimestamp */
+SELECT MIN(rollup_input_future_timestamp.future_ms_timestamp) AS min_timestamp
+FROM effectstream.rollup_input_future_timestamp
+LEFT OUTER JOIN effectstream.rollup_input_result
+  ON (effectstream.rollup_input_result.id = effectstream.rollup_input_future_timestamp.id)
+WHERE effectstream.rollup_input_result.id IS NULL;
+
 /* @name getGameInputResultByBlockHeight */
 SELECT
   rollup_inputs.id,

package/src/unhandled-rejection-wiring.test.ts

@@ -0,0 +1,63 @@
+// Pins the PRODUCTION wiring used in pg-connection.ts:
+//   installUnhandledRejectionLogger(EFFECTSTREAM_PGLITE, isTransientPgError)
+// i.e. the live `unhandledRejection` handler treats `isTransientPgError` as its
+// survive predicate. This is the test @acedward asked for: prove that when
+// something other than a known-transient pg error escapes to the global
+// handler (e.g. a generic error torn out of the effection root), the process is
+// actually told to exit — it does NOT silently log-and-survive into a zombie
+// whose effection tree is dead but whose process is still up.
+
+import { test, expect } from "bun:test";
+import { buildUnhandledRejectionHandler } from "@effectstream/log";
+import { isTransientPgError } from "./transient-pg-errors.ts";
+
+const wireProductionHandler = () => {
+  const exits: number[] = [];
+  const handle = buildUnhandledRejectionHandler(
+    "pglite",
+    isTransientPgError,
+    (code) => exits.push(code),
+  );
+  return { handle, exits };
+};
+
+test("generic error escaping the effection root is fatal (no zombie)", () => {
+  const { handle, exits } = wireProductionHandler();
+
+  // What a halted effection scope surfaces: an ordinary Error, not a pg error.
+  handle(new Error("effection root task died"));
+
+  expect(exits).toEqual([1]);
+});
+
+test("a logic/SQL bug is fatal", () => {
+  const { handle, exits } = wireProductionHandler();
+  handle(new Error("column foo does not exist"));
+  expect(exits).toEqual([1]);
+});
+
+test("a genuine transient pg blip survives (the intended exemption)", () => {
+  const { handle, exits } = wireProductionHandler();
+
+  handle({
+    code: "ECONNRESET",
+    message:
+      "Client network socket disconnected before secure TLS connection was established",
+    stack: "    at node_modules/pg-pool/index.js:45:11",
+  });
+
+  expect(exits).toEqual([]);
+});
+
+test("a non-pg ECONNRESET-shaped rejection is still fatal", () => {
+  const { handle, exits } = wireProductionHandler();
+
+  // Same code, but not from the pg stack => isTransientPgError returns false.
+  handle({
+    code: "ETIMEDOUT",
+    message: "fetch failed",
+    stack: "    at node:internal/deps/undici/undici.js:1234:5",
+  });
+
+  expect(exits).toEqual([1]);
+});