@effectstream/db 0.100.0

package/src/pg-connection.ts

@@ -0,0 +1,211 @@
+import type { Client, PoolClient, PoolConfig } from "pg";
+import pg from "pg";
+import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
+import { type Operation, run, sleep } from "effection";
+import { ENV } from "@effectstream/utils/node-env";
+
+let readonlyDBConn: pg.Pool | null;
+
+// PGLite does not support multiple connections, so we need to use a mutex to ensure that only one query is executed at a time.
+// * For transactions use yield* acquireDBMutex(); ...Your Operations... releaseDBMutex();
+// * For single queries use await runPreparedQuery(myQuery.run(params, dbConn));
+// IMPORTANT: This is only for PGLite instances, for full pgsql servers, this is not needed.0
+// TODO This is a very simple mutex implementation.
+//      It releases the mutex randomly, and not in order.
+//      It only works for deno's single-threaded runtime.
+//      pglite exports async-mutex, so we can use that for a proper solution.
+let db_mutex: "free" | "locked" = "free";
+/**
+ * Acquires a mutex to ensure that only one query is executed at a time.
+ * If running in PGLite, it will acquire a mutex to ensure that only one query is executed at a time.
+ * If running in a full pgsql server, it will do nothing.
+ * Always run run `releaseDBMutex()` after the query locks no longer needed, other threads are blocked until the mutex is released.
+ */
+const _waitUntilFree = {
+  waiting: [] as {
+    priority: 'high' | 'low';
+    name: string;
+    date: string;
+  }[],
+  running: {
+    name: "",
+    date: "",
+  },
+  db_mutex,
+};
+
+// Get data for database.
+export const waitUntilFree = () => {
+  return {
+    ..._waitUntilFree,
+    db_mutex,
+  };
+};
+
+export function* acquireDBMutex(lockName: string, priority: 'high' | 'low' = 'low'): Operation<void> {
+  if (!ENV.PGLITE) return;
+  _waitUntilFree.waiting.push({
+    priority,
+    name: lockName,
+    date: new Date().toISOString(),
+  });
+
+  while (true) {
+    // If there is a high priority lock, wait for it to be released.
+    if (priority === 'low' && _waitUntilFree.waiting.find((w) => w.priority === 'high')) {
+      // do nothing
+    }
+
+    // If there is no lock, acquire it.
+    else if (db_mutex === "free") {
+      db_mutex = "locked";
+      _waitUntilFree.waiting.splice(
+        _waitUntilFree.waiting.findIndex((w) => w.name === lockName),
+        1,
+      );
+      _waitUntilFree.running = {
+        name: lockName,
+        date: new Date().toISOString(),
+      };
+      break;
+    }
+
+    // This is only for debugging purposes.
+    const i = _waitUntilFree.waiting.findIndex((w) => w.name === lockName);
+    if (i !== -1) {
+      const now = new Date().getTime();
+      const waitingSince = new Date(_waitUntilFree.waiting[i].date).getTime();
+      const waitingFor = now - waitingSince;
+      if (waitingFor > 2500) {
+        console.error(
+          `[DB Mutex] Waiting for ${waitingFor}[ms] for ${lockName}. 
+Locked by ${_waitUntilFree.running.name}. This is a critical error, please restart the sync service.`,
+        );
+      }
+    }
+
+    yield* sleep(10);
+  }
+}
+
+/**
+ * Releases a mutex to allow other queries to be executed.
+ * If running in PGLite, it will release a mutex to allow other queries to be executed.
+ * If running in a full pgsql server, it will do nothing.
+ * Do not call this function if you did not call `acquireDBMutex()` before.
+ */
+export function releaseDBMutex(lockName: string) {
+  if (!ENV.PGLITE) return;
+
+  if (_waitUntilFree.running.name !== lockName) {
+    console.error(
+      "releasing mutex",
+      _waitUntilFree.running,
+      "but the lock name does not match",
+      lockName,
+    );
+  }
+  _waitUntilFree.running = { name: "", date: "" };
+  db_mutex = "free";
+}
+
+/**
+ * Helper function to run a db query in a thread-safe manner.
+ * If running in PGLite, it will acquire a mutex to ensure that only one query is executed at a time.
+ * If running in a full pgsql server, it will run normally.
+ */
+export async function runPreparedQuery<T>(
+  p: Promise<T[]>,
+  name: string,
+): Promise<T[]> {
+  let result: T[];
+  try {
+    if (ENV.PGLITE) {
+      await run(() => acquireDBMutex(`pq:${name}`));
+    }
+
+    let t;
+    const timeout: Promise<T[]> = new Promise((resolve) => {
+      t = setTimeout(() => {
+        console.error("[CRITICAL ERROR] Database query timed out", name);
+        if (ENV.PGLITE) {
+          // TODO: This is a temporary fix to allow the query to continue.
+          // Only allow to continue in PGLITE.
+          // We suspect this is PGLITE specific error.
+          resolve([]);
+        }
+      }, 2500);
+    });
+    result = await Promise.race([p, timeout]);
+    clearTimeout(t);
+  } finally {
+    releaseDBMutex(`pq:${name}`);
+  }
+  return result;
+}
+
+export const getConnection = (
+  creds?: PoolConfig,
+  readonly = false,
+): pg.Pool => {
+  if (!creds) {
+    creds = {
+      host: ENV.DB_HOST,
+      user: ENV.DB_USER,
+      password: ENV.PGLITE ? undefined : ENV.DB_PW,
+      database: ENV.DB_NAME,
+      port: ENV.DB_PORT,
+    };
+  }
+  if (readonly && readonlyDBConn) return readonlyDBConn;
+
+  // TODO: make this configurable for non pglite instances
+  const MAX_CONNECTIONS = ENV.PGLITE ? 1 : 10;
+
+  const pool = new pg.Pool({ ...creds, max: MAX_CONNECTIONS });
+  pool.on("error", (err: unknown) =>
+    log.remote(
+      ComponentNames.EFFECTSTREAM_PGLITE,
+      ["query"],
+      SeverityNumber.ERROR,
+      (log) => log(err),
+    ));
+  pool.on("connect", (_client: PoolClient) => {
+    // On each new client initiated, need to register for error(this is a serious bug on pg, the client throw errors although it should not)
+    // https://github.com/brianc/node-postgres/issues/2499#issuecomment-805477725
+    _client.on("error", (err: Error) => {
+      log.remote(
+        ComponentNames.EFFECTSTREAM_PGLITE,
+        ["connect"],
+        SeverityNumber.ERROR,
+        (log) => log(err),
+      );
+    });
+  });
+
+  if (readonly) {
+    const ensureReadOnly =
+      `SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;`;
+    void pool.query(ensureReadOnly); // note: this query modifies the DB state
+    readonlyDBConn = pool;
+  }
+
+  return pool;
+};
+
+// For notifications use a direct (non-pooled) persistent connection.
+export const getPersistentConnection = (creds: PoolConfig): Client => {
+  const client = new pg.Client(creds);
+  client.connect(() => {});
+  // https://github.com/brianc/node-postgres/issues/2499#issuecomment-805477725
+  // On each new client initiated, need to register for error(this is a serious bug on pg, the client throw errors although it should not)
+  client.on("error", (err: Error) => {
+    log.remote(
+      ComponentNames.EFFECTSTREAM_PGLITE,
+      ["query"],
+      SeverityNumber.ERROR,
+      (log) => log(err),
+    );
+  });
+  return client;
+};

package/src/register-events.ts

@@ -0,0 +1,40 @@
+import type { PoolClient } from "pg";
+import {
+  getEventByTopic,
+  getTopics,
+  registerEventType,
+} from "./sql/events.queries.ts";
+
+export async function registerEventTypes(
+  pool: PoolClient,
+  eventDescriptions: { topic: string; name: string }[],
+): Promise<boolean> {
+  const allStoredTopics = await getTopics.run(undefined, pool);
+
+  // check that no events were deleted.
+  for (const storedTopic of allStoredTopics) {
+    const found = eventDescriptions.find((ed) =>
+      ed.topic === storedTopic.topic
+    );
+
+    if (found === undefined) {
+      return false;
+    }
+  }
+
+  for (const event of eventDescriptions) {
+    const registeredTopic = await getEventByTopic.run(
+      { topic: event.topic },
+      pool,
+    );
+
+    if (registeredTopic.length === 0) {
+      await registerEventType.run(
+        { name: event.name, topic: event.topic },
+        pool,
+      );
+    }
+  }
+
+  return true;
+}

package/src/scheduled-constructors.ts

@@ -0,0 +1,117 @@
+import type { StateUpdateStream } from "@effectstream/coroutine";
+import { World } from "@effectstream/coroutine";
+import {
+  newScheduledHeightData,
+  newScheduledTimestampData,
+  removeAllScheduledDataByInputData,
+  removeScheduledBlockData,
+  removeScheduledTimestampData,
+} from "./sql/rollup_inputs.queries.ts";
+import {
+  AddressType,
+  type Caip2,
+  type TxHash,
+  type UnknownFormat,
+} from "@effectstream/utils";
+import {
+  type BlockNumber,
+  strip0x,
+  type TimestampMs,
+  type WalletAddress,
+} from "@effectstream/utils";
+import { Buffer } from "node:buffer";
+
+export type UpdateAtTrigger = { blockHeight: BlockNumber } | {
+  timestamp: TimestampMs;
+};
+
+/**
+ * Create an SQL update which schedules a piece of data to be run through
+ * the STF at a future block height / timestamp.
+ *
+ * @param inputData The input to pass to the STF, generally in Paima Concise format.
+ * @param blockHeight The future block height at which to post the input to the STF.
+ * @returns
+ */
+export function* createScheduledData(
+  inputData: string,
+  trigger: UpdateAtTrigger,
+  source:
+    | {
+      primitiveName: string;
+      txHash: TxHash;
+      caip2: Caip2;
+      fromAddress: WalletAddress;
+      fromAddressType: AddressType;
+      contractAddress: undefined | WalletAddress;
+    }
+    | {
+      precompile: UnknownFormat;
+    },
+): StateUpdateStream<void> {
+  if ("blockHeight" in trigger) {
+    const sourceParams = "precompile" in source
+      ? {
+        from_address: source.precompile,
+        // TODO: Check this value. What is the precompile types?
+        from_address_type: AddressType.NONE,
+        primitive_name: null,
+        origin_tx_hash: null,
+        caip2: null,
+        origin_contract_address: null,
+      }
+      : {
+        from_address: source.fromAddress,
+        from_address_type: source.fromAddressType,
+        primitive_name: source.primitiveName,
+        origin_tx_hash: Buffer.from(strip0x(source.txHash), "hex"),
+        caip2: source.caip2,
+        origin_contract_address: source.contractAddress,
+      };
+
+    yield* World.resolve(newScheduledHeightData, {
+      future_block_height: trigger.blockHeight,
+      input_data: inputData,
+      ...sourceParams,
+    });
+  } else {
+    if (!("precompile" in source)) {
+      // in particular primitives only schedule data in the same block they trigger
+      throw new Error("primitives should not schedule timers by date");
+    }
+    yield* World.resolve(newScheduledTimestampData, {
+      future_ms_timestamp: new Date(trigger.timestamp),
+      input_data: inputData,
+      from_address: source.precompile,
+      from_address_type: AddressType.NONE,
+    });
+  }
+}
+
+// Create an SQL update which deletes an upcoming scheduled data
+// NOTE: if trigger is null, then delete ALL schedules that match inputData.
+export function* deleteScheduledData(
+  inputData: string,
+  trigger: UpdateAtTrigger | null,
+): StateUpdateStream<void> {
+  if (trigger === null) {
+    yield* World.resolve(removeAllScheduledDataByInputData, {
+      input_data: inputData,
+    });
+    return;
+  }
+
+  if ("blockHeight" in trigger) {
+    // Delete exact schedule by command and height
+    yield* World.resolve(removeScheduledBlockData, {
+      block_height: trigger.blockHeight,
+      input_data: inputData,
+    });
+  } else {
+    // Delete exact schedule by command and timestamp
+    yield* World.resolve(removeScheduledTimestampData, {
+      ms_timestamp: new Date(trigger.timestamp),
+      input_data: inputData,
+    });
+  }
+}

package/src/snapshot-handler.ts

@@ -0,0 +1,164 @@
+import { ENV } from "@effectstream/utils/node-env";
+import { sleep, until, type Operation } from "effection";
+
+export interface SnapshotRetentionConfig {
+  /** One per hour for last 24 h. Default: `true` */
+  lastDayHourly?: boolean;
+  /** One per 6 h for last 3 days. Default: `true` */
+  last3DaysSixHourly?: boolean;
+  /** Days of daily snapshots to keep. Default: `7` */
+  lastNDaysDaily?: number;
+}
+
+export interface SnapshotConfig {
+  /** Wall-clock seconds between snapshots. Default: `3600` (1 hour) */
+  intervalSeconds?: number;
+  /** Output directory. Default: `"./snapshots"` */
+  path?: string;
+  /** Time-based tiered retention (uses file mtime). */
+  retention?: SnapshotRetentionConfig;
+}
+
+/**
+ * Creates a database snapshot using `pg_dump` in custom format (`-F c`).
+ * Must be called AFTER releasing the DB mutex (pg_dump opens its own connection).
+ *
+ * @see docs/home/1000-effectstream-engine/1003-database-snapshots.md
+ */
+export async function createSnapshot(
+  config?: SnapshotConfig,
+): Promise<void> {
+  // pg_dump queries pg_catalog directly; PGlite's catalog may not match the
+  // locally-installed pg_dump version, so skip snapshots under PGlite.
+  if (ENV.PGLITE) {
+    console.log(`[Snapshot] Skipping: pg_dump is not compatible with PGlite.`);
+    return;
+  }
+
+  const snapshotDir = config?.path ?? "./snapshots";
+  const timestamp = new Date().toISOString().replace(/:/g, "-").replace(/\.\d{3}/, "");
+  const snapshotPath = `${snapshotDir}/snapshot-${timestamp}.dump`;
+
+  await Deno.mkdir(snapshotDir, { recursive: true });
+  console.log(`[Snapshot] Creating snapshot at ${snapshotPath}...`);
+
+  const cmd = new Deno.Command("pg_dump", {
+    args: [
+      "-h", ENV.DB_HOST,
+      "-p", String(ENV.DB_PORT),
+      "-U", ENV.DB_USER,
+      "-d", ENV.DB_NAME,
+      "-F", "c",
+      "-f", snapshotPath,
+    ],
+    env: { ...Deno.env.toObject(), PGPASSWORD: ENV.DB_PW ?? "" },
+    stdout: "inherit",
+    stderr: "inherit",
+  });
+
+  const status = await cmd.output();
+  if (!status.success) {
+    throw new Error(`[Snapshot] pg_dump failed with exit code ${status.code}`);
+  }
+
+  console.log(`[Snapshot] Snapshot created: ${snapshotPath}`);
+  await applyRetentionPolicy(snapshotDir, config?.retention);
+}
+
+/**
+ * Long-lived fiber that fires snapshots on a fixed cadence, independent of
+ * block production. Spawn once at startup; cadence is `intervalSeconds`.
+ * Blocking on `createSnapshot` makes overlapping dumps structurally impossible.
+ */
+export function* runSnapshotLoop(
+  snapshotConfig: SnapshotConfig,
+): Operation<void> {
+  const intervalMs = (snapshotConfig.intervalSeconds ?? 3600) * 1000;
+  console.log(`[Snapshot] Loop started (interval ${intervalMs / 1000}s, path ${snapshotConfig.path ?? "./snapshots"}, cwd ${Deno.cwd()})`);
+  while (true) {
+    console.log(`[Snapshot] Firing`);
+    try {
+      yield* until(createSnapshot(snapshotConfig));
+      console.log(`[Snapshot] Done; sleeping ${intervalMs / 1000}s`);
+    } catch (e) {
+      console.error("[Snapshot] Failed:", e);
+    }
+    yield* sleep(intervalMs);
+  }
+}
+
+/**
+ * Time-based tiered retention: keeps one file per bucket (newest wins),
+ * deletes everything beyond `lastNDaysDaily` days.
+ */
+async function applyRetentionPolicy(
+  snapshotDir: string,
+  retention?: SnapshotRetentionConfig,
+): Promise<void> {
+  const lastDayHourly      = retention?.lastDayHourly      ?? true;
+  const last3DaysSixHourly = retention?.last3DaysSixHourly ?? true;
+  const lastNDaysDaily     = retention?.lastNDaysDaily      ?? 7;
+
+  const now    = Date.now();
+  const MS_1H  = 3_600_000;
+  const MS_24H = 24 * MS_1H;
+  const MS_3D  = 3  * MS_24H;
+  const MS_ND  = lastNDaysDaily * MS_24H;
+
+  const entries: { path: string; mtime: number }[] = [];
+
+  try {
+    for await (const entry of Deno.readDir(snapshotDir)) {
+      if (!entry.isFile || !entry.name.startsWith("snapshot-") || !entry.name.endsWith(".dump")) continue;
+      const filePath = `${snapshotDir}/${entry.name}`;
+      try {
+        const stat = await Deno.stat(filePath);
+        entries.push({ path: filePath, mtime: stat.mtime?.getTime() ?? 0 });
+      } catch { /* file vanished between readDir and stat */ }
+    }
+  } catch (e) {
+    console.warn("[Snapshot] Could not read snapshot directory for retention:", e);
+    return;
+  }
+
+  if (entries.length === 0) return;
+
+  // Newest first so the first entry per bucket is always the one we keep.
+  entries.sort((a, b) => b.mtime - a.mtime);
+
+  const keepers     = new Set<string>();
+  const seenBuckets = new Set<string>();
+
+  for (const entry of entries) {
+    const age = now - entry.mtime;
+    if (age > MS_ND) continue;
+
+    let bucketKey: string;
+
+    if (age <= MS_24H) {
+      if (!lastDayHourly) { keepers.add(entry.path); continue; }
+      bucketKey = `h:${Math.floor(age / MS_1H)}`;
+    } else if (age <= MS_3D) {
+      if (!last3DaysSixHourly) { keepers.add(entry.path); continue; }
+      bucketKey = `6h:${Math.floor(age / (6 * MS_1H))}`;
+    } else {
+      bucketKey = `d:${Math.floor(age / MS_24H)}`;
+    }
+
+    if (!seenBuckets.has(bucketKey)) {
+      seenBuckets.add(bucketKey);
+      keepers.add(entry.path);
+    }
+  }
+
+  for (const entry of entries) {
+    if (!keepers.has(entry.path)) {
+      try {
+        await Deno.remove(entry.path);
+        console.log(`[Snapshot] Retention: deleted ${entry.path}`);
+      } catch (e) {
+        console.warn(`[Snapshot] Failed to delete ${entry.path}:`, e);
+      }
+    }
+  }
+}