@apibara/plugin-mongo 2.1.0-beta.1 → 2.1.0-beta.10

package/dist/index.cjs

@@ -39,6 +39,29 @@ async function finalize(db, session, cursor, collections) {
     );
   }
 }
+async function cleanupStorage(db, session, collections) {
+  for (const collection of collections) {
+    try {
+      await db.collection(collection).deleteMany({}, { session });
+    } catch (error) {
+      throw new Error(`Failed to clean up collection ${collection}: ${error}`);
+    }
+  }
+}
+
+class MongoStorageError extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "MongoStorageError";
+  }
+}
+async function withTransaction(client, cb) {
+  return await client.withSession(async (session) => {
+    return await session.withTransaction(async (session2) => {
+      return await cb(session2);
+    });
+  });
+}
 
 const checkpointCollectionName = "checkpoints";
 const filterCollectionName = "filters";
@@ -134,6 +157,17 @@ async function finalizeState(props) {
     { session }
   );
 }
+async function resetPersistence(props) {
+  const { db, session, indexerId } = props;
+  try {
+    await db.collection(checkpointCollectionName).deleteMany({ id: indexerId }, { session });
+    await db.collection(filterCollectionName).deleteMany({ id: indexerId }, { session });
+  } catch (error) {
+    throw new MongoStorageError("Failed to reset persistence state", {
+      cause: error
+    });
+  }
+}
 
 class MongoStorage {
   constructor(db, session, endCursor) {
@@ -305,20 +339,6 @@ class MongoCollection {
   }
 }
 
-class MongoStorageError extends Error {
-  constructor(message) {
-    super(message);
-    this.name = "MongoStorageError";
-  }
-}
-async function withTransaction(client, cb) {
-  return await client.withSession(async (session) => {
-    return await session.withTransaction(async (session2) => {
-      return await cb(session2);
-    });
-  });
-}
-
 const MONGO_PROPERTY = "_mongo";
 function useMongoStorage() {
   const context = indexer.useIndexerContext();
@@ -339,9 +359,24 @@ function mongoStorage({
 }) {
   return plugins.defineIndexerPlugin((indexer) => {
     let indexerId = "";
+    const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
     indexer.hooks.hook("run:before", async () => {
       const { indexerName } = plugins$1.useInternalContext();
       indexerId = internal.generateIndexerId(indexerName, identifier);
+      const logger = plugins.useLogger();
+      if (alwaysReindex) {
+        logger.warn(
+          `Reindexing: Deleting all data from collections - ${collections.join(", ")}`
+        );
+        await withTransaction(client, async (session) => {
+          const db = client.db(dbName, dbOptions);
+          await cleanupStorage(db, session, collections);
+          if (enablePersistence) {
+            await resetPersistence({ db, session, indexerId });
+          }
+          logger.success("All data has been cleaned up for reindexing");
+        });
+      }
       await withTransaction(client, async (session) => {
         const db = client.db(dbName, dbOptions);
         if (enablePersistence) {

package/dist/index.mjs

@@ -1,5 +1,5 @@
 import { useIndexerContext } from '@apibara/indexer';
-import { defineIndexerPlugin } from '@apibara/indexer/plugins';
+import { defineIndexerPlugin, useLogger } from '@apibara/indexer/plugins';
 import { generateIndexerId } from '@apibara/indexer/internal';
 import { useInternalContext } from '@apibara/indexer/internal/plugins';
 import { normalizeCursor } from '@apibara/protocol';
@@ -37,6 +37,29 @@ async function finalize(db, session, cursor, collections) {
     );
   }
 }
+async function cleanupStorage(db, session, collections) {
+  for (const collection of collections) {
+    try {
+      await db.collection(collection).deleteMany({}, { session });
+    } catch (error) {
+      throw new Error(`Failed to clean up collection ${collection}: ${error}`);
+    }
+  }
+}
+
+class MongoStorageError extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "MongoStorageError";
+  }
+}
+async function withTransaction(client, cb) {
+  return await client.withSession(async (session) => {
+    return await session.withTransaction(async (session2) => {
+      return await cb(session2);
+    });
+  });
+}
 
 const checkpointCollectionName = "checkpoints";
 const filterCollectionName = "filters";
@@ -132,6 +155,17 @@ async function finalizeState(props) {
     { session }
   );
 }
+async function resetPersistence(props) {
+  const { db, session, indexerId } = props;
+  try {
+    await db.collection(checkpointCollectionName).deleteMany({ id: indexerId }, { session });
+    await db.collection(filterCollectionName).deleteMany({ id: indexerId }, { session });
+  } catch (error) {
+    throw new MongoStorageError("Failed to reset persistence state", {
+      cause: error
+    });
+  }
+}
 
 class MongoStorage {
   constructor(db, session, endCursor) {
@@ -303,20 +337,6 @@ class MongoCollection {
   }
 }
 
-class MongoStorageError extends Error {
-  constructor(message) {
-    super(message);
-    this.name = "MongoStorageError";
-  }
-}
-async function withTransaction(client, cb) {
-  return await client.withSession(async (session) => {
-    return await session.withTransaction(async (session2) => {
-      return await cb(session2);
-    });
-  });
-}
-
 const MONGO_PROPERTY = "_mongo";
 function useMongoStorage() {
   const context = useIndexerContext();
@@ -337,9 +357,24 @@ function mongoStorage({
 }) {
   return defineIndexerPlugin((indexer) => {
     let indexerId = "";
+    const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
     indexer.hooks.hook("run:before", async () => {
       const { indexerName } = useInternalContext();
       indexerId = generateIndexerId(indexerName, identifier);
+      const logger = useLogger();
+      if (alwaysReindex) {
+        logger.warn(
+          `Reindexing: Deleting all data from collections - ${collections.join(", ")}`
+        );
+        await withTransaction(client, async (session) => {
+          const db = client.db(dbName, dbOptions);
+          await cleanupStorage(db, session, collections);
+          if (enablePersistence) {
+            await resetPersistence({ db, session, indexerId });
+          }
+          logger.success("All data has been cleaned up for reindexing");
+        });
+      }
       await withTransaction(client, async (session) => {
         const db = client.db(dbName, dbOptions);
         if (enablePersistence) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apibara/plugin-mongo",
-  "version": "2.1.0-beta.1",
+  "version": "2.1.0-beta.10",
   "type": "module",
   "files": [
     "dist",
@@ -35,7 +35,7 @@
     "mongodb": "^6.12.0"
   },
   "dependencies": {
-    "@apibara/indexer": "2.1.0-beta.1",
-    "@apibara/protocol": "2.1.0-beta.1"
+    "@apibara/indexer": "2.1.0-beta.10",
+    "@apibara/protocol": "2.1.0-beta.10"
   }
 }

package/src/index.ts

@@ -1,16 +1,17 @@
 import { useIndexerContext } from "@apibara/indexer";
-import { defineIndexerPlugin } from "@apibara/indexer/plugins";
+import { defineIndexerPlugin, useLogger } from "@apibara/indexer/plugins";
 import type { DbOptions, MongoClient } from "mongodb";
 
 import { generateIndexerId } from "@apibara/indexer/internal";
 import { useInternalContext } from "@apibara/indexer/internal/plugins";
-import { finalize, invalidate } from "./mongo";
+import { cleanupStorage, finalize, invalidate } from "./mongo";
 import {
   finalizeState,
   getState,
   initializePersistentState,
   invalidateState,
   persistState,
+  resetPersistence,
 } from "./persistence";
 import { MongoStorage } from "./storage";
 import { MongoStorageError, withTransaction } from "./utils";
@@ -60,10 +61,30 @@ export function mongoStorage<TFilter, TBlock>({
 }: MongoStorageOptions) {
   return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
     let indexerId = "";
+    const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
 
     indexer.hooks.hook("run:before", async () => {
       const { indexerName } = useInternalContext();
       indexerId = generateIndexerId(indexerName, identifier);
+      const logger = useLogger();
+
+      if (alwaysReindex) {
+        logger.warn(
+          `Reindexing: Deleting all data from collections - ${collections.join(", ")}`,
+        );
+
+        await withTransaction(client, async (session) => {
+          const db = client.db(dbName, dbOptions);
+
+          await cleanupStorage(db, session, collections);
+
+          if (enablePersistence) {
+            await resetPersistence({ db, session, indexerId });
+          }
+
+          logger.success("All data has been cleaned up for reindexing");
+        });
+      }
 
       await withTransaction(client, async (session) => {
         const db = client.db(dbName, dbOptions);

package/src/mongo.ts

@@ -49,3 +49,18 @@ export async function finalize(
     );
   }
 }
+
+export async function cleanupStorage(
+  db: Db,
+  session: ClientSession,
+  collections: string[],
+) {
+  for (const collection of collections) {
+    try {
+      // Delete all documents in the collection
+      await db.collection(collection).deleteMany({}, { session });
+    } catch (error) {
+      throw new Error(`Failed to clean up collection ${collection}: ${error}`);
+    }
+  }
+}

package/src/persistence.ts

@@ -1,5 +1,6 @@
 import { type Cursor, normalizeCursor } from "@apibara/protocol";
 import type { ClientSession, Db } from "mongodb";
+import { MongoStorageError } from "./utils";
 
 export type CheckpointSchema = {
   id: string;
@@ -161,3 +162,27 @@ export async function finalizeState(props: {
     { session },
   );
 }
+
+export async function resetPersistence(props: {
+  db: Db;
+  session: ClientSession;
+  indexerId: string;
+}) {
+  const { db, session, indexerId } = props;
+
+  try {
+    // Delete all checkpoints for this indexer
+    await db
+      .collection<CheckpointSchema>(checkpointCollectionName)
+      .deleteMany({ id: indexerId }, { session });
+
+    // Delete all filters for this indexer
+    await db
+      .collection<FilterSchema>(filterCollectionName)
+      .deleteMany({ id: indexerId }, { session });
+  } catch (error) {
+    throw new MongoStorageError("Failed to reset persistence state", {
+      cause: error,
+    });
+  }
+}

package/src/utils.ts

@@ -1,8 +1,8 @@
 import type { ClientSession, MongoClient } from "mongodb";
 
 export class MongoStorageError extends Error {
-  constructor(message: string) {
-    super(message);
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
     this.name = "MongoStorageError";
   }
 }