@effectstream/db 0.100.0

package/scripts/pgtyped-update.ts

@@ -0,0 +1,119 @@
+#!/usr/bin/env bun
+/**
+ * Runs database startup and pgtyped generation concurrently
+ */
+
+import { spawn } from "node:child_process";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(import.meta.url?.startsWith("file:") ? fileURLToPath(import.meta.url) : import.meta.url.replace("file://", ""));
+
+interface ProcessInfo {
+  name: string;
+  command: string;
+  args: string[];
+  wait: boolean;
+}
+
+function createPrefix(name: string): string {
+  const colors = {
+    "db:up": "\x1b[36m", // cyan
+    "pgtyped:internal": "\x1b[33m", // yellow
+  };
+  const reset = "\x1b[0m";
+  const color = colors[name as keyof typeof colors] || "\x1b[37m"; // white
+  return `${color}[${name}]${reset} `;
+}
+
+async function runProcess(processInfo: ProcessInfo): Promise<void> {
+  const prefix = createPrefix(processInfo.name);
+
+  console.log(`${prefix}Starting...`);
+
+  const child = spawn(processInfo.command, processInfo.args, {
+    stdio: ["inherit", "pipe", "pipe"],
+    shell: processInfo.command === "npx",
+  });
+
+  const stdoutPromise = new Promise<void>((resolve) => {
+    child.stdout?.on("data", (chunk: Buffer | string) => {
+      const text = chunk.toString();
+      text.split("\n").forEach((line) => {
+        if (line.trim()) console.log(`${prefix}${line}`);
+      });
+    });
+    child.stdout?.on("end", resolve);
+    if (!child.stdout) resolve();
+  });
+
+  const stderrPromise = new Promise<void>((resolve) => {
+    child.stderr?.on("data", (chunk: Buffer | string) => {
+      const text = chunk.toString();
+      text.split("\n").forEach((line) => {
+        if (line.trim()) console.error(`${prefix}${line}`);
+      });
+    });
+    child.stderr?.on("end", resolve);
+    if (!child.stderr) resolve();
+  });
+
+  const exitCode = new Promise<number | null>((resolve) => {
+    child.on("close", (code) => resolve(code));
+  });
+
+  if (processInfo.wait) {
+    const [code] = await Promise.all([exitCode, stdoutPromise, stderrPromise]);
+    if (code === 0) {
+      console.log(`${prefix}✅ Completed successfully`);
+    } else {
+      console.error(`${prefix}❌ Failed with exit code ${code}`);
+      throw new Error(`Process ${processInfo.name} failed`);
+    }
+  }
+}
+
+async function main() {
+  const processes: ProcessInfo[] = [
+    {
+      name: "db:up",
+      command: "bun",
+      args: ["run", join(__dirname, "start-pglite.ts")],
+      wait: false,
+    },
+    {
+      name: "db:wait",
+      command: "bun",
+      args: ["run", join(__dirname, "wait-for-db.ts")],
+      wait: true,
+    },
+    {
+      name: "apply-migrations",
+      command: "bun",
+      args: ["run", join(__dirname, "apply-migrations.ts")],
+      wait: true,
+    },
+    {
+      name: "pgtyped",
+      command: "npx",
+      args: ["pgtyped", "-c", "./pgtypedconfig.json"],
+      wait: true,
+    },
+  ];
+
+  console.log("🚀 Starting concurrent processes...\n");
+
+  try {
+    for (const process of processes) {
+      await runProcess(process);
+    }
+    console.log("\n✅ All processes completed successfully");
+  } catch (error) {
+    console.error("\n❌ One or more processes failed:", error);
+    process.exit(1);
+  }
+}
+
+if (import.meta.main) {
+  await main();
+}

package/scripts/start-pglite.ts

@@ -0,0 +1,71 @@
+import { type DebugLevel, PGlite } from "@electric-sql/pglite";
+// TODO This is not working, so we load the pg_ivm extension from the node_modules folder
+// import { pg_ivm } from "@electric-sql/pglite/pg_ivm";
+import net from "node:net";
+import { statSync } from "node:fs";
+import { fromNodeSocket } from "pg-gateway/node";
+import { ENV } from "@effectstream/utils/node-env";
+import { args, cwd } from "@effectstream/utils/runtime";
+
+// TODO PORT be a ENV variable
+// Get port from arguments.
+const portArgName = "--port";
+const argv = args();
+const portArgIndex = argv.indexOf(portArgName);
+const portValue = portArgIndex !== -1 ? argv[portArgIndex + 1] : "5432";
+const port = parseInt(portValue);
+if (isNaN(port)) {
+  throw new Error(`Port argument ${portArgName} is not a number`);
+}
+
+// Resolve pg_ivm extension path via import.meta.resolve (works with bun's hoisting)
+const pgliteEntry = import.meta.resolve("@electric-sql/pglite");
+const pgliteDir = pgliteEntry.replace(/\/dist\/[^/]+$/, "");
+const pgIvmUrl = new URL(`${pgliteDir}/dist/pg_ivm.tar.gz`);
+
+const db = new PGlite(
+  ENV.PGLITE_DATA_DIR,
+  {
+    username: ENV.DB_USER,
+    database: ENV.DB_NAME,
+    extensions: {
+      pg_ivm: pgIvmUrl,
+    },
+    debug: (ENV.DEBUG_PGLITE as DebugLevel) || 0,
+  },
+);
+
+{
+  // TODO: consider switching to pglite-socket once it works
+  //       https://discord.com/channels/933657521581858818/1371976702674075780/1371992712076595250
+  const server = net.createServer(async (socket) => {
+    await fromNodeSocket(socket, {
+      serverVersion: "16.3",
+
+      auth: {
+        // No password required
+        method: "trust",
+      },
+
+      async onStartup() {
+        // Wait for PGlite to be ready before further processing
+        await db.waitReady;
+      },
+
+      // Hook into each client message
+      async onMessage(data, { isAuthenticated }) {
+        // Only forward messages to PGlite after authentication
+        if (!isAuthenticated) {
+          return;
+        }
+
+        // Forward raw message to PGlite and send response to client
+        return await db.execProtocolRaw(data);
+      },
+    });
+  });
+
+  server.listen(port, () => {
+    console.info(`database: server listening on port ${port}`);
+  });
+}

package/scripts/wait-for-db.ts

@@ -0,0 +1,43 @@
+#!/usr/bin/env bun
+import { spawn } from "node:child_process";
+import { args, exit } from "@effectstream/utils/runtime";
+
+// Get port from arguments.
+const portArgName = "--port";
+const argv = args();
+const portArgIndex = argv.indexOf(portArgName);
+const portValue = portArgIndex !== -1 ? argv[portArgIndex + 1] : "5432";
+const port = parseInt(portValue);
+if (isNaN(port)) {
+  throw new Error(`Port argument ${portArgName} is not a number`);
+}
+
+async function waitForDb() {
+  try {
+    console.log("waiting for db on port", port);
+    const child = spawn("npx", ["wait-on", `tcp:${port}`], {
+      stdio: "inherit",
+      shell: true,
+    });
+
+    const code = await new Promise<number | null>((resolve) => {
+      child.on("close", (code, _sig) => resolve(code));
+    });
+
+    if (code === 0) {
+      console.log("✅ Database is ready on port 5432");
+    } else {
+      console.error("❌ Failed to connect to database on port 5432");
+      exit(code ?? 1);
+    }
+  } catch (error) {
+    console.error("❌ Error waiting for database:", error);
+    exit(1);
+  }
+}
+
+export { waitForDb };
+
+if (import.meta.main) {
+  await waitForDb();
+}

package/src/dynamic-tables.ts

@@ -0,0 +1,110 @@
+import { until } from "effection";
+
+// This import causes a circular dependency with the sync package.
+// import type { AllSyncProtocols } from "@effectstream/sync";
+import type { PoolClient } from "pg";
+import type { VersionInfo } from "../migrations/system-version.ts";
+import { applyMigrations } from "../scripts/apply-migrations.ts";
+import { findMigrationByName } from "./sql/system.queries.ts";
+
+/**
+ * Creates dynamic tables for the given sync protocols.
+ * This is used to create the IVMs for the given sync protocols.
+ *
+ * For example for ERC20 it creates a table that holds the balance for each address.
+ * For ERC721 it creates a table that holds the token id for each address.
+ *
+ * @param dbConn - The database connection.
+ * @param syncProtocols - The sync protocols.
+ */
+export function* createDynamicTables(
+  versionInfo: VersionInfo,
+  lastBlockHeight: number,
+  dbConn: PoolClient,
+  syncProtocols: any[], // AllSyncProtocols[],
+) {
+  for (const syncProtocol of syncProtocols) {
+    for (const primitive of syncProtocol.config.primitives) {
+      yield* createDynamicTableForPrimitive(primitive, lastBlockHeight, dbConn);
+    }
+  }
+}
+
+function* createDynamicTableForPrimitive(
+  p: {
+    primitive: {
+      type: string;
+      name: string;
+    };
+  },
+  lastBlockHeight: number,
+  dbConn: PoolClient,
+) {
+  const type = p.primitive.type;
+  const name = p.primitive.name;
+
+  // const primitive = PrimitiveRegistry.getPrimitive(name);
+  const primitive = (globalThis as any).EFFECTSTREAM_REGISTRY[name];
+  if (!primitive) {
+    // This should never happen.
+    console.error(`Cannot find primitive ${name} in the registry. If this is custom primitive, ensure it's registered.`);
+    throw new Error(`Primitive ${name} not found`);
+  }
+  const sqlFunction = primitive.getDynamicTables;
+
+  // This primitive does not have dynamic tables.
+  if (!sqlFunction) return;
+  const code = sqlFunction(name);
+  if (!code) return;
+
+  const migrationName = `dynamic-tables-${type}-${name}`;
+  const [migration] = yield* until(findMigrationByName.run({
+    name: migrationName,
+    isSystemMigration: true,
+  }, dbConn));
+
+  // This particular migration has been applied, so we can skip it.
+  if (migration) return;
+
+  yield* until(applyMigrations(
+    dbConn as any, // Client,
+    lastBlockHeight,
+    migrationName,
+    code,
+    true,
+  ));
+}
+
+/**
+ * Returns the prefix for the given primitive name.
+ * This is useful to query the table given you know the primitive name.
+ * @param primitiveType - The type of the primitive.
+ * @returns The prefix for the given primitive name.
+ */
+export function getPrimitivePrefix(
+  primitiveType: string,
+): string[] {
+  const expectedPrimitive: any = Object.values(
+    (globalThis as any).EFFECTSTREAM_REGISTRY,
+  ).find((primitive: any) => primitive.internalTypeName === primitiveType);
+  return expectedPrimitive?.getViewPrefix() || [];
+}
+
+/**
+ * The intermediate prefix is used to query the intermediate table for the given primitive type.
+ * The intermediate table is used to store the intermediate data for the given primitive type.
+ *
+ * For example for ERC20 it stores the balance for each address.
+ * For ERC721 it stores the token id for each address.
+ *
+ * @param primitiveType - The type of the primitive.
+ * @returns The intermediate prefix for the given primitive type.
+ */
+export function getPrimitiveIntermediatePrefix(
+  primitiveType: string,
+): string[] {
+  const expectedPrimitive: any = Object.values(
+    (globalThis as any).EFFECTSTREAM_REGISTRY,
+  ).find((primitive: any) => primitive.internalTypeName === primitiveType);
+  return expectedPrimitive?.getIntermediatePrefix() || [];
+}

package/src/event-indexing.ts

@@ -0,0 +1,31 @@
+import type { PoolClient } from "pg";
+
+export async function createIndexesForEvents(
+  pool: PoolClient,
+  eventDescriptions: { topic: string; fieldName: string; indexed: boolean }[],
+): Promise<boolean> {
+  for (const event of eventDescriptions) {
+    const indexName = `index_${
+      event.topic.slice(0, 20)
+    }_${event.fieldName.toLowerCase()}`;
+
+    const checkQuery =
+      `SELECT * FROM pg_indexes WHERE indexname = '${indexName}';`;
+
+    const result = await pool.query(checkQuery);
+
+    const createQuery =
+      `CREATE INDEX ${indexName} ON event(topic, (data->>'${event.fieldName}'));`;
+    const deleteQuery = `DROP INDEX ${indexName};`;
+
+    if (result.rowCount === 0 && event.indexed) {
+      await pool.query(createQuery);
+    }
+
+    if (result.rowCount !== 0 && !event.indexed) {
+      await pool.query(deleteQuery);
+    }
+  }
+
+  return true;
+}

package/src/migrations.ts

@@ -0,0 +1,28 @@
+// TODO: Circular dependency.
+// import type { DBMigrations } from "@effectstream/runtime";
+
+/**
+ * This function is used by Paima Engine to apply the migration at the correct block heights.
+ * It returns the migration script for the given block height.
+ * @param startBlockHeight - The paima block height to start applying the migrations from (inclusive).
+ * @param endBlockHeight - The paima block height to stop applying the migrations at (inclusive).
+ * @returns The migration script for the given block height.
+ */
+export function getMigrationsForBlockHeight(
+  migrationTable: /*DBMigrations[]*/ any[] | undefined,
+  startBlockHeight: number,
+  endBlockHeight: number,
+): /*DBMigrations[]*/ any[] {
+  if (!migrationTable) {
+    return [];
+  }
+  const migrationsToApply = migrationTable
+    .filter((migration) => {
+      const targetBlockHeight = migration.blockHeight ?? 1;
+      return (
+        targetBlockHeight >= startBlockHeight &&
+        targetBlockHeight <= endBlockHeight
+      );
+    });
+  return migrationsToApply;
+}

package/src/mod.ts

@@ -0,0 +1,65 @@
+// https://github.com/denoland/deno/issues/26611#issuecomment-2492987153
+
+// https://github.com/adelsz/pgtyped/issues/565
+export type { Json } from "./sql/events.queries.ts";
+
+export {
+  getAchievementProgress,
+  setAchievementProgress,
+} from "./sql/achievements.queries.ts";
+export type {
+  IGetAchievementProgressParams,
+  IGetAchievementProgressResult,
+  ISetAchievementProgressParams,
+  ISetAchievementProgressResult,
+} from "./sql/achievements.queries.ts";
+// https://github.com/adelsz/pgtyped/issues/565
+export {
+  blockHeightDone,
+  getBlockByHash,
+  getBlockHeights,
+  getBlockSeeds,
+  getLatestProcessedBlockHeight,
+  saveLastBlock,
+} from "./sql/block-heights.queries.ts";
+export type {
+  IBlockHeightDoneParams,
+  IBlockHeightDoneResult,
+  IGetBlockByHashParams,
+  IGetBlockByHashResult,
+  IGetBlockHeightsParams,
+  IGetBlockHeightsResult,
+  IGetBlockSeedsParams,
+  IGetBlockSeedsResult,
+  IGetLatestProcessedBlockHeightParams,
+  IGetLatestProcessedBlockHeightResult,
+  ISaveLastBlockParams,
+  ISaveLastBlockResult,
+} from "./sql/block-heights.queries.ts";
+export * from "./sql/statistics.queries.ts";
+export * from "./sql/nonces.queries.ts";
+export * from "./sql/rollup_inputs.queries.ts";
+export * from "./sql/accounts.queries.ts";
+export * from "./sql/events.queries.ts";
+export * from "./sql/sync-protocols/page.queries.ts";
+export * from "./sql/primitives.queries.ts";
+export * from "./sql/system.queries.ts";
+export * from "./sql/tables.queries.ts";
+
+export * from "./event-indexing.ts";
+export * from "./register-events.ts";
+export * from "./pg-connection.ts";
+export * from "./scheduled-constructors.ts";
+export * from "./system.ts";
+
+export {
+  createDynamicTables,
+  getPrimitiveIntermediatePrefix,
+  getPrimitivePrefix,
+} from "./dynamic-tables.ts";
+// export { startPGlite } from "./start-pglite.ts";
+
+export { getMigrationsForBlockHeight } from "./migrations.ts";
+
+export { createSnapshot, runSnapshotLoop } from "./snapshot-handler.ts";
+export type { SnapshotConfig, SnapshotRetentionConfig } from "./snapshot-handler.ts";

package/src/pg-connection.test.ts

@@ -0,0 +1,99 @@
+import { test, expect } from "bun:test";
+import {
+  acquireDBMutex,
+  releaseDBMutex,
+  waitUntilFree,
+} from "./pg-connection.ts";
+import { run, sleep, spawn } from "effection";
+import { setEnv } from "@effectstream/utils/runtime";
+
+// Force PGLITE env var to true for testing mutex logic
+setEnv("PGLITE", "true");
+
+test("DB Mutex - acquires and releases lock", async () => {
+  await run(function* () {
+    const lockName = "test-lock";
+
+    yield* acquireDBMutex(lockName);
+    let state = waitUntilFree();
+    expect(state.db_mutex).toEqual("locked");
+    expect(state.running.name).toEqual(lockName);
+
+    releaseDBMutex(lockName);
+    state = waitUntilFree();
+    expect(state.db_mutex).toEqual("free");
+    expect(state.running.name).toEqual("");
+  });
+});
+
+test("DB Mutex - queues requests", async () => {
+  await run(function* () {
+    const lock1 = "lock-1";
+    const lock2 = "lock-2";
+
+    yield* acquireDBMutex(lock1);
+
+    // Spawn a second task that tries to acquire lock
+    let lock2Acquired = false;
+    yield* spawn(function* () {
+      yield* acquireDBMutex(lock2);
+      lock2Acquired = true;
+      releaseDBMutex(lock2);
+    });
+
+    yield* sleep(50);
+    // Lock 2 should be waiting
+    expect(lock2Acquired).toEqual(false);
+    let state = waitUntilFree();
+    expect(state.waiting.length).toEqual(1);
+    expect(state.waiting[0].name).toEqual(lock2);
+
+    // Release lock 1
+    releaseDBMutex(lock1);
+
+    yield* sleep(50);
+    // Lock 2 should now be acquired (and released immediately in this test flow, but let's check flow)
+    expect(lock2Acquired).toEqual(true);
+
+    state = waitUntilFree();
+    expect(state.db_mutex).toEqual("free");
+  });
+});
+
+test("DB Mutex - respects priority", async () => {
+  await run(function* () {
+    const mainLock = "main";
+    const lowPrio = "low";
+    const highPrio = "high";
+
+    yield* acquireDBMutex(mainLock);
+
+    const acquireOrder: string[] = [];
+
+    yield* spawn(function* () {
+      yield* acquireDBMutex(lowPrio, "low");
+      acquireOrder.push(lowPrio);
+      releaseDBMutex(lowPrio);
+    });
+
+    yield* sleep(10);
+
+    yield* spawn(function* () {
+        yield* acquireDBMutex(highPrio, "high");
+        acquireOrder.push(highPrio);
+        releaseDBMutex(highPrio);
+    });
+
+    yield* sleep(10);
+
+    let state = waitUntilFree();
+    expect(state.waiting.length).toEqual(2);
+
+    releaseDBMutex(mainLock);
+    yield* sleep(50);
+
+    // High priority should have been acquired first
+    expect(acquireOrder[0]).toEqual(highPrio);
+    expect(acquireOrder[1]).toEqual(lowPrio);
+  });
+});