@apibara/plugin-drizzle 2.1.0-beta.1 → 2.1.0-beta.11

package/dist/index.d.ts

@@ -1,17 +1,144 @@
 import * as _apibara_indexer_plugins from '@apibara/indexer/plugins';
-import { TablesRelationalConfig, ExtractTablesWithRelations } from 'drizzle-orm';
+import { DrizzleConfig, TablesRelationalConfig, ExtractTablesWithRelations } from 'drizzle-orm';
 import { PgQueryResultHKT, PgTransaction, PgDatabase } from 'drizzle-orm/pg-core';
+import { PGliteOptions, PGlite } from '@electric-sql/pglite';
+import { MigrationConfig } from 'drizzle-orm/migrator';
+import { NodePgDatabase as NodePgDatabase$1 } from 'drizzle-orm/node-postgres';
+import { PgliteDatabase as PgliteDatabase$1 } from 'drizzle-orm/pglite';
+import pg from 'pg';
+
+/**
+ * Union type of all possible drizzle database options
+ */
+type DrizzleOptions = PgliteDrizzleOptions | NodePgDrizzleOptions;
+/**
+ * Configuration options for Node-Postgres database connection
+ */
+type NodePgDrizzleOptions = {
+    /**
+     * Type of database to use -
+     * - "pglite" - PGLite database
+     * - "node-postgres" - Node-Postgres database
+     * @default "pglite"
+     */
+    type: "node-postgres";
+    /**
+     * Connection string to use for the database
+     * @default ""
+     */
+    connectionString?: string;
+    /**
+     * Pool configuration options for Node-Postgres
+     */
+    poolConfig?: pg.PoolConfig;
+    /**
+     * Additional drizzle configuration options
+     */
+    config?: Omit<DrizzleConfig, "schema">;
+};
+/**
+ * Configuration options for PGLite database connection
+ */
+type PgliteDrizzleOptions = {
+    /**
+     * Type of database to use -
+     * - "pglite" - PGLite database
+     * - "node-postgres" - Node-Postgres database
+     */
+    type?: "pglite";
+    /**
+     * Connection string to use for the database
+     * @default "memory://pglite"
+     */
+    connectionString?: string;
+    /**
+     * Pool configuration is not supported for PGLite
+     */
+    poolConfig?: never;
+    /**
+     * Additional drizzle configuration options with PGLite specific connection options
+     */
+    config?: Omit<DrizzleConfig, "schema"> & {
+        connection?: (PGliteOptions & {
+            dataDir?: string;
+        }) | string;
+    };
+};
+/**
+ * Extended PGLite database type with client information
+ */
+type PgliteDatabase<TSchema extends Record<string, unknown>> = PgliteDatabase$1<TSchema> & {
+    $client: PGlite;
+};
+/**
+ * Extended Node-Postgres database type with client information
+ */
+type NodePgDatabase<TSchema extends Record<string, unknown>> = NodePgDatabase$1<TSchema> & {
+    $client: pg.Pool;
+};
+type Database<TOptions extends DrizzleOptions, TSchema extends Record<string, unknown>> = TOptions extends PgliteDrizzleOptions ? PgliteDatabase<TSchema> : NodePgDatabase<TSchema>;
+/**
+ * Creates a new Drizzle database instance based on the provided options
+ * @param options - Configuration options for the database connection
+ * @returns A configured Drizzle database instance
+ * @throws {Error} If an invalid database type is specified
+ */
+declare function drizzle<TSchema extends Record<string, unknown>, TOptions extends DrizzleOptions>(options?: TOptions & {
+    /**
+     * Schema to use for the database
+     * @default {}
+     */
+    schema?: TSchema;
+}): Database<TOptions, TSchema>;
+/**
+ * Options for database migration
+ */
+type MigrateOptions = MigrationConfig;
+/**
+ * Performs database migration based on the provided configuration
+ * @param db - The database instance to migrate
+ * @param options - Migration configuration options
+ *
+ * @important This function runs migrations on the database instance provided to the `drizzleStorage` plugin.
+ * It automatically detects the type of database and runs the appropriate migrate function
+ * (PGLite or Node-Postgres).
+ *
+ * @example
+ * ```ts
+ * await migrate(db, { migrationsFolder: "./drizzle" });
+ * ```
+ */
+declare function migrate<TSchema extends Record<string, unknown>>(db: PgliteDatabase<TSchema> | NodePgDatabase<TSchema>, options: MigrateOptions): Promise<void>;
 
 type DrizzleStorage<TQueryResult extends PgQueryResultHKT, TFullSchema extends Record<string, unknown> = Record<string, never>, TSchema extends TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>> = {
     db: PgTransaction<TQueryResult, TFullSchema, TSchema>;
 };
 declare function useDrizzleStorage<TQueryResult extends PgQueryResultHKT, TFullSchema extends Record<string, unknown> = Record<string, never>, TSchema extends TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>>(_db?: PgDatabase<TQueryResult, TFullSchema, TSchema>): DrizzleStorage<TQueryResult, TFullSchema, TSchema>;
 interface DrizzleStorageOptions<TQueryResult extends PgQueryResultHKT, TFullSchema extends Record<string, unknown> = Record<string, never>, TSchema extends TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>> {
+    /**
+     * The Drizzle database instance.
+     */
     db: PgDatabase<TQueryResult, TFullSchema, TSchema>;
+    /**
+     * Whether to persist the indexer's state. Defaults to true.
+     */
     persistState?: boolean;
+    /**
+     * The name of the indexer. Default value is 'default'.
+     */
     indexerName?: string;
+    /**
+     * The schema of the database.
+     */
     schema?: Record<string, unknown>;
+    /**
+     * The column to use as the id. Defaults to 'id'.
+     */
     idColumn?: string;
+    /**
+     * The options for the database migration. When provided, the database will automatically run migrations before the indexer runs.
+     */
+    migrate?: MigrateOptions;
 }
 /**
  * Creates a plugin that uses Drizzle as the storage layer.
@@ -22,7 +149,8 @@ interface DrizzleStorageOptions<TQueryResult extends PgQueryResultHKT, TFullSche
  * @param options.indexerName - The name of the indexer. Defaults value is 'default'.
  * @param options.schema - The schema of the database.
  * @param options.idColumn - The column to use as the id. Defaults to 'id'.
+ * @param options.migrate - The options for the database migration. when provided, the database will automatically run migrations before the indexer runs.
  */
-declare function drizzleStorage<TFilter, TBlock, TQueryResult extends PgQueryResultHKT, TFullSchema extends Record<string, unknown> = Record<string, never>, TSchema extends TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>>({ db, persistState: enablePersistence, indexerName: identifier, schema, idColumn, }: DrizzleStorageOptions<TQueryResult, TFullSchema, TSchema>): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
+declare function drizzleStorage<TFilter, TBlock, TQueryResult extends PgQueryResultHKT, TFullSchema extends Record<string, unknown> = Record<string, never>, TSchema extends TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>>({ db, persistState: enablePersistence, indexerName: identifier, schema, idColumn, migrate: migrateOptions, }: DrizzleStorageOptions<TQueryResult, TFullSchema, TSchema>): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
 
-export { type DrizzleStorage, type DrizzleStorageOptions, drizzleStorage, useDrizzleStorage };
+export { type Database, type DrizzleOptions, type DrizzleStorage, type DrizzleStorageOptions, type MigrateOptions, type NodePgDatabase, type NodePgDrizzleOptions, type PgliteDatabase, type PgliteDrizzleOptions, drizzle, drizzleStorage, migrate, useDrizzleStorage };

package/dist/index.mjs

@@ -1,7 +1,14 @@
 import { useIndexerContext } from '@apibara/indexer';
-import { defineIndexerPlugin } from '@apibara/indexer/plugins';
+import { defineIndexerPlugin, useLogger } from '@apibara/indexer/plugins';
 import { generateIndexerId } from '@apibara/indexer/internal';
 import { useInternalContext } from '@apibara/indexer/internal/plugins';
+import { D as DRIZZLE_PROPERTY, a as DRIZZLE_STORAGE_DB_PROPERTY } from './shared/plugin-drizzle.f8d1b186.mjs';
+import { PGlite } from '@electric-sql/pglite';
+import { drizzle as drizzle$1 } from 'drizzle-orm/node-postgres';
+import { migrate as migrate$2 } from 'drizzle-orm/node-postgres/migrator';
+import { drizzle as drizzle$2 } from 'drizzle-orm/pglite';
+import { migrate as migrate$1 } from 'drizzle-orm/pglite/migrator';
+import pg from 'pg';
 import { normalizeCursor } from '@apibara/protocol';
 import { eq, and, isNull, gt, lt, sql } from 'drizzle-orm';
 import { pgTable, text, integer, primaryKey, serial, char, jsonb } from 'drizzle-orm/pg-core';
@@ -34,6 +41,50 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+function drizzle(options) {
+  const {
+    connectionString = "memory://",
+    schema,
+    type = "pglite",
+    config,
+    poolConfig
+  } = options ?? {};
+  if (connectionString.startsWith("postgres://") || type === "node-postgres") {
+    const pool = new pg.Pool({
+      connectionString,
+      ...poolConfig || {}
+    });
+    return drizzle$1(pool, { schema, ...config || {} });
+  }
+  if (type === "pglite") {
+    return drizzle$2({
+      schema,
+      connection: {
+        dataDir: connectionString || "memory://pglite"
+      },
+      ...config || {}
+    });
+  }
+  throw new Error("Invalid database type");
+}
+async function migrate(db, options) {
+  const isPglite = !!("$client" in db && db.$client instanceof PGlite);
+  try {
+    if (isPglite) {
+      await migrate$1(db, options);
+    } else {
+      await migrate$2(db, options);
+    }
+  } catch (error) {
+    throw new DrizzleStorageError(
+      "Failed to apply migrations! Please check if you have generated migrations using drizzle:generate",
+      {
+        cause: error
+      }
+    );
+  }
+}
+
 const CHECKPOINTS_TABLE_NAME = "__indexer_checkpoints";
 const FILTERS_TABLE_NAME = "__indexer_filters";
 const SCHEMA_VERSION_TABLE_NAME = "__indexer_schema_version";
@@ -211,6 +262,17 @@ async function finalizeState(props) {
     });
   }
 }
+async function resetPersistence(props) {
+  const { tx, indexerId } = props;
+  try {
+    await tx.delete(checkpoints).where(eq(checkpoints.id, indexerId));
+    await tx.delete(filters).where(eq(filters.id, indexerId));
+  } catch (error) {
+    throw new DrizzleStorageError("Failed to reset persistence state", {
+      cause: error
+    });
+  }
+}
 
 function getReorgTriggerName(table, indexerId) {
   return `${table}_reorg_${indexerId}`;
@@ -431,8 +493,37 @@ async function finalize(tx, cursor, indexerId) {
     });
   }
 }
+async function cleanupStorage(tx, tables, indexerId) {
+  try {
+    for (const table of tables) {
+      await tx.execute(
+        sql.raw(
+          `DROP TRIGGER IF EXISTS ${getReorgTriggerName(table, indexerId)} ON ${table};`
+        )
+      );
+    }
+    await tx.execute(
+      sql.raw(`
+        DELETE FROM __reorg_rollback
+        WHERE indexer_id = '${indexerId}'
+      `)
+    );
+    for (const table of tables) {
+      try {
+        await tx.execute(sql.raw(`TRUNCATE TABLE ${table} CASCADE;`));
+      } catch (error) {
+        throw new DrizzleStorageError(`Failed to truncate table ${table}`, {
+          cause: error
+        });
+      }
+    }
+  } catch (error) {
+    throw new DrizzleStorageError("Failed to clean up storage", {
+      cause: error
+    });
+  }
+}
 
-const DRIZZLE_PROPERTY = "_drizzle";
 const MAX_RETRIES = 5;
 function useDrizzleStorage(_db) {
   const context = useIndexerContext();
@@ -448,11 +539,13 @@ function drizzleStorage({
   persistState: enablePersistence = true,
   indexerName: identifier = "default",
   schema,
-  idColumn = "id"
+  idColumn = "id",
+  migrate: migrateOptions
 }) {
   return defineIndexerPlugin((indexer) => {
     let tableNames = [];
     let indexerId = "";
+    const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
     try {
       tableNames = Object.values(schema ?? db._.schema ?? {}).map(
         (table) => table.dbName
@@ -463,11 +556,33 @@ function drizzleStorage({
       });
     }
     indexer.hooks.hook("run:before", async () => {
-      const { indexerName: indexerFileName, availableIndexers } = useInternalContext();
+      const internalContext = useInternalContext();
+      const context = useIndexerContext();
+      const logger = useLogger();
+      context[DRIZZLE_STORAGE_DB_PROPERTY] = db;
+      const { indexerName: indexerFileName, availableIndexers } = internalContext;
       indexerId = generateIndexerId(indexerFileName, identifier);
+      if (alwaysReindex) {
+        logger.warn(
+          `Reindexing: Deleting all data from tables - ${tableNames.join(", ")}`
+        );
+        await withTransaction(db, async (tx) => {
+          await cleanupStorage(tx, tableNames, indexerId);
+          if (enablePersistence) {
+            await resetPersistence({ tx, indexerId });
+          }
+          logger.success("Tables have been cleaned up for reindexing");
+        });
+      }
       let retries = 0;
+      let migrationsApplied = false;
       while (retries <= MAX_RETRIES) {
         try {
+          if (migrateOptions && !migrationsApplied) {
+            await migrate(db, migrateOptions);
+            migrationsApplied = true;
+            logger.success("Migrations applied");
+          }
           await withTransaction(db, async (tx) => {
             await initializeReorgRollbackTable(tx, indexerId);
             if (enablePersistence) {
@@ -477,6 +592,9 @@ function drizzleStorage({
           break;
         } catch (error) {
           if (retries === MAX_RETRIES) {
+            if (error instanceof DrizzleStorageError) {
+              throw error;
+            }
             throw new DrizzleStorageError(
               "Initialization failed after 5 retries",
               {
@@ -598,4 +716,4 @@ function drizzleStorage({
   });
 }
 
-export { drizzleStorage, useDrizzleStorage };
+export { drizzle, drizzleStorage, migrate, useDrizzleStorage };

package/dist/shared/plugin-drizzle.e884ca32.cjs

@@ -0,0 +1,7 @@
+'use strict';
+
+const DRIZZLE_PROPERTY = "_drizzle";
+const DRIZZLE_STORAGE_DB_PROPERTY = "_drizzleStorageDB";
+
+exports.DRIZZLE_PROPERTY = DRIZZLE_PROPERTY;
+exports.DRIZZLE_STORAGE_DB_PROPERTY = DRIZZLE_STORAGE_DB_PROPERTY;

package/dist/shared/plugin-drizzle.f8d1b186.mjs

@@ -0,0 +1,4 @@
+const DRIZZLE_PROPERTY = "_drizzle";
+const DRIZZLE_STORAGE_DB_PROPERTY = "_drizzleStorageDB";
+
+export { DRIZZLE_PROPERTY as D, DRIZZLE_STORAGE_DB_PROPERTY as a };

package/dist/testing.cjs

@@ -0,0 +1,13 @@
+'use strict';
+
+const constants = require('./shared/plugin-drizzle.e884ca32.cjs');
+
+function getTestDatabase(context) {
+  const db = context[constants.DRIZZLE_STORAGE_DB_PROPERTY];
+  if (!db) {
+    throw new Error("Drizzle database not found in context");
+  }
+  return db;
+}
+
+exports.getTestDatabase = getTestDatabase;

package/dist/testing.d.cts

@@ -0,0 +1,6 @@
+import { VcrResult } from '@apibara/indexer/testing';
+import { PgDatabase, PgQueryResultHKT } from 'drizzle-orm/pg-core';
+
+declare function getTestDatabase(context: VcrResult): PgDatabase<PgQueryResultHKT>;
+
+export { getTestDatabase };

package/dist/testing.d.mts

@@ -0,0 +1,6 @@
+import { VcrResult } from '@apibara/indexer/testing';
+import { PgDatabase, PgQueryResultHKT } from 'drizzle-orm/pg-core';
+
+declare function getTestDatabase(context: VcrResult): PgDatabase<PgQueryResultHKT>;
+
+export { getTestDatabase };

package/dist/testing.d.ts

@@ -0,0 +1,6 @@
+import { VcrResult } from '@apibara/indexer/testing';
+import { PgDatabase, PgQueryResultHKT } from 'drizzle-orm/pg-core';
+
+declare function getTestDatabase(context: VcrResult): PgDatabase<PgQueryResultHKT>;
+
+export { getTestDatabase };

package/dist/testing.mjs

@@ -0,0 +1,11 @@
+import { a as DRIZZLE_STORAGE_DB_PROPERTY } from './shared/plugin-drizzle.f8d1b186.mjs';
+
+function getTestDatabase(context) {
+  const db = context[DRIZZLE_STORAGE_DB_PROPERTY];
+  if (!db) {
+    throw new Error("Drizzle database not found in context");
+  }
+  return db;
+}
+
+export { getTestDatabase };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apibara/plugin-drizzle",
-  "version": "2.1.0-beta.1",
+  "version": "2.1.0-beta.11",
   "type": "module",
   "files": [
     "dist",
@@ -15,6 +15,12 @@
       "import": "./dist/index.mjs",
       "require": "./dist/index.cjs",
       "default": "./dist/index.mjs"
+    },
+    "./testing": {
+      "types": "./dist/testing.d.ts",
+      "import": "./dist/testing.mjs",
+      "require": "./dist/testing.cjs",
+      "default": "./dist/testing.mjs"
     }
   },
   "scripts": {
@@ -39,8 +45,8 @@
     "vitest": "^1.6.0"
   },
   "dependencies": {
-    "@apibara/indexer": "2.1.0-beta.1",
-    "@apibara/protocol": "2.1.0-beta.1",
+    "@apibara/indexer": "2.1.0-beta.11",
+    "@apibara/protocol": "2.1.0-beta.11",
     "postgres-range": "^1.1.4"
   }
 }

package/src/constants.ts

@@ -0,0 +1,2 @@
+export const DRIZZLE_PROPERTY = "_drizzle";
+export const DRIZZLE_STORAGE_DB_PROPERTY = "_drizzleStorageDB";

package/src/helper.ts

@@ -0,0 +1,192 @@
+import { PGlite, type PGliteOptions } from "@electric-sql/pglite";
+import type { DrizzleConfig } from "drizzle-orm";
+import type { MigrationConfig } from "drizzle-orm/migrator";
+import {
+  type NodePgDatabase as OriginalNodePgDatabase,
+  drizzle as drizzleNode,
+} from "drizzle-orm/node-postgres";
+import { migrate as migrateNode } from "drizzle-orm/node-postgres/migrator";
+import {} from "drizzle-orm/pg-core";
+import {
+  type PgliteDatabase as OriginalPgliteDatabase,
+  drizzle as drizzlePGLite,
+} from "drizzle-orm/pglite";
+import { migrate as migratePGLite } from "drizzle-orm/pglite/migrator";
+import pg from "pg";
+import { DrizzleStorageError } from "./utils";
+
+/**
+ * Union type of all possible drizzle database options
+ */
+export type DrizzleOptions = PgliteDrizzleOptions | NodePgDrizzleOptions;
+
+/**
+ * Configuration options for Node-Postgres database connection
+ */
+export type NodePgDrizzleOptions = {
+  /**
+   * Type of database to use -
+   * - "pglite" - PGLite database
+   * - "node-postgres" - Node-Postgres database
+   * @default "pglite"
+   */
+  type: "node-postgres";
+  /**
+   * Connection string to use for the database
+   * @default ""
+   */
+  connectionString?: string;
+  /**
+   * Pool configuration options for Node-Postgres
+   */
+  poolConfig?: pg.PoolConfig;
+  /**
+   * Additional drizzle configuration options
+   */
+  config?: Omit<DrizzleConfig, "schema">;
+};
+
+/**
+ * Configuration options for PGLite database connection
+ */
+export type PgliteDrizzleOptions = {
+  /**
+   * Type of database to use -
+   * - "pglite" - PGLite database
+   * - "node-postgres" - Node-Postgres database
+   */
+  type?: "pglite";
+  /**
+   * Connection string to use for the database
+   * @default "memory://pglite"
+   */
+  connectionString?: string;
+  /**
+   * Pool configuration is not supported for PGLite
+   */
+  poolConfig?: never;
+  /**
+   * Additional drizzle configuration options with PGLite specific connection options
+   */
+  config?: Omit<DrizzleConfig, "schema"> & {
+    connection?:
+      | (PGliteOptions & {
+          dataDir?: string;
+        })
+      | string;
+  };
+};
+
+/**
+ * Extended PGLite database type with client information
+ */
+export type PgliteDatabase<TSchema extends Record<string, unknown>> =
+  OriginalPgliteDatabase<TSchema> & {
+    $client: PGlite;
+  };
+
+/**
+ * Extended Node-Postgres database type with client information
+ */
+export type NodePgDatabase<TSchema extends Record<string, unknown>> =
+  OriginalNodePgDatabase<TSchema> & {
+    $client: pg.Pool;
+  };
+
+export type Database<
+  TOptions extends DrizzleOptions,
+  TSchema extends Record<string, unknown>,
+> = TOptions extends PgliteDrizzleOptions
+  ? PgliteDatabase<TSchema>
+  : NodePgDatabase<TSchema>;
+
+/**
+ * Creates a new Drizzle database instance based on the provided options
+ * @param options - Configuration options for the database connection
+ * @returns A configured Drizzle database instance
+ * @throws {Error} If an invalid database type is specified
+ */
+export function drizzle<
+  TSchema extends Record<string, unknown>,
+  TOptions extends DrizzleOptions,
+>(
+  options?: TOptions & {
+    /**
+     * Schema to use for the database
+     * @default {}
+     */
+    schema?: TSchema;
+  },
+): Database<TOptions, TSchema> {
+  const {
+    connectionString = "memory://",
+    schema,
+    type = "pglite",
+    config,
+    poolConfig,
+  } = options ?? {};
+
+  if (connectionString.startsWith("postgres://") || type === "node-postgres") {
+    const pool = new pg.Pool({
+      connectionString,
+      ...(poolConfig || {}),
+    });
+    return drizzleNode(pool, { schema, ...(config || {}) }) as Database<
+      TOptions,
+      TSchema
+    >;
+  }
+
+  if (type === "pglite") {
+    return drizzlePGLite({
+      schema: schema as TSchema,
+      connection: {
+        dataDir: connectionString || "memory://pglite",
+      },
+      ...(config || {}),
+    }) as Database<TOptions, TSchema>;
+  }
+
+  throw new Error("Invalid database type");
+}
+
+/**
+ * Options for database migration
+ */
+export type MigrateOptions = MigrationConfig;
+
+/**
+ * Performs database migration based on the provided configuration
+ * @param db - The database instance to migrate
+ * @param options - Migration configuration options
+ *
+ * @important This function runs migrations on the database instance provided to the `drizzleStorage` plugin.
+ * It automatically detects the type of database and runs the appropriate migrate function
+ * (PGLite or Node-Postgres).
+ *
+ * @example
+ * ```ts
+ * await migrate(db, { migrationsFolder: "./drizzle" });
+ * ```
+ */
+export async function migrate<TSchema extends Record<string, unknown>>(
+  db: PgliteDatabase<TSchema> | NodePgDatabase<TSchema>,
+  options: MigrateOptions,
+) {
+  const isPglite = !!("$client" in db && db.$client instanceof PGlite);
+
+  try {
+    if (isPglite) {
+      await migratePGLite(db as PgliteDatabase<TSchema>, options);
+    } else {
+      await migrateNode(db as NodePgDatabase<TSchema>, options);
+    }
+  } catch (error) {
+    throw new DrizzleStorageError(
+      "Failed to apply migrations! Please check if you have generated migrations using drizzle:generate",
+      {
+        cause: error,
+      },
+    );
+  }
+}