@apibara/plugin-mongo 2.1.0-beta.3 → 2.1.0-beta.30

package/dist/index.cjs

@@ -39,17 +39,42 @@ async function finalize(db, session, cursor, collections) {
     );
   }
 }
+async function cleanupStorage(db, session, collections) {
+  for (const collection of collections) {
+    try {
+      await db.collection(collection).deleteMany({}, { session });
+    } catch (error) {
+      throw new Error(`Failed to clean up collection ${collection}`, {
+        cause: error
+      });
+    }
+  }
+}
+
+class MongoStorageError extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "MongoStorageError";
+  }
+}
+async function withTransaction(client, cb) {
+  return await client.withSession(async (session) => {
+    return await session.withTransaction(
+      async (session2) => {
+        return await cb(session2);
+      },
+      {
+        retryWrites: false
+      }
+    );
+  });
+}
 
 const checkpointCollectionName = "checkpoints";
 const filterCollectionName = "filters";
 async function initializePersistentState(db, session) {
-  const checkpoint = await db.createCollection(
-    checkpointCollectionName,
-    { session }
-  );
-  const filter = await db.createCollection(filterCollectionName, {
-    session
-  });
+  const checkpoint = db.collection(checkpointCollectionName);
+  const filter = db.collection(filterCollectionName);
   await checkpoint.createIndex({ id: 1 }, { session });
   await filter.createIndex({ id: 1, fromBlock: 1 }, { session });
 }
@@ -61,7 +86,7 @@ async function persistState(props) {
       {
         $set: {
           orderKey: Number(endCursor.orderKey),
-          uniqueKey: endCursor.uniqueKey
+          uniqueKey: endCursor.uniqueKey ? endCursor.uniqueKey : null
         }
       },
       { upsert: true, session }
@@ -134,6 +159,17 @@ async function finalizeState(props) {
     { session }
   );
 }
+async function resetPersistence(props) {
+  const { db, session, indexerId } = props;
+  try {
+    await db.collection(checkpointCollectionName).deleteMany({ id: indexerId }, { session });
+    await db.collection(filterCollectionName).deleteMany({ id: indexerId }, { session });
+  } catch (error) {
+    throw new MongoStorageError("Failed to reset persistence state", {
+      cause: error
+    });
+  }
+}
 
 class MongoStorage {
   constructor(db, session, endCursor) {
@@ -305,20 +341,6 @@ class MongoCollection {
   }
 }
 
-class MongoStorageError extends Error {
-  constructor(message) {
-    super(message);
-    this.name = "MongoStorageError";
-  }
-}
-async function withTransaction(client, cb) {
-  return await client.withSession(async (session) => {
-    return await session.withTransaction(async (session2) => {
-      return await cb(session2);
-    });
-  });
-}
-
 const MONGO_PROPERTY = "_mongo";
 function useMongoStorage() {
   const context = indexer.useIndexerContext();
@@ -339,14 +361,27 @@ function mongoStorage({
 }) {
   return plugins.defineIndexerPlugin((indexer) => {
     let indexerId = "";
+    const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
+    let prevFinality;
     indexer.hooks.hook("run:before", async () => {
       const { indexerName } = plugins$1.useInternalContext();
       indexerId = internal.generateIndexerId(indexerName, identifier);
+      const logger = plugins.useLogger();
       await withTransaction(client, async (session) => {
         const db = client.db(dbName, dbOptions);
         if (enablePersistence) {
           await initializePersistentState(db, session);
         }
+        if (alwaysReindex) {
+          logger.warn(
+            `Reindexing: Deleting all data from collections - ${collections.join(", ")}`
+          );
+          await cleanupStorage(db, session, collections);
+          if (enablePersistence) {
+            await resetPersistence({ db, session, indexerId });
+          }
+          logger.success("All data has been cleaned up for reindexing");
+        }
       });
     });
     indexer.hooks.hook("connect:before", async ({ request }) => {
@@ -399,7 +434,7 @@ function mongoStorage({
       });
     });
     indexer.hooks.hook("message:finalize", async ({ message }) => {
-      const { cursor } = message.finalize;
+      const { cursor } = message;
       if (!cursor) {
         throw new MongoStorageError("finalized cursor is undefined");
       }
@@ -412,7 +447,7 @@ function mongoStorage({
       });
     });
     indexer.hooks.hook("message:invalidate", async ({ message }) => {
-      const { cursor } = message.invalidate;
+      const { cursor } = message;
       if (!cursor) {
         throw new MongoStorageError("invalidate cursor is undefined");
       }
@@ -426,16 +461,19 @@ function mongoStorage({
     });
     indexer.hooks.hook("handler:middleware", async ({ use }) => {
       use(async (context, next) => {
-        const { endCursor } = context;
+        const { endCursor, finality, cursor } = context;
         if (!endCursor) {
           throw new MongoStorageError("end cursor is undefined");
         }
         await withTransaction(client, async (session) => {
           const db = client.db(dbName, dbOptions);
           context[MONGO_PROPERTY] = new MongoStorage(db, session, endCursor);
+          if (prevFinality === "pending") {
+            await invalidate(db, session, cursor, collections);
+          }
           await next();
           delete context[MONGO_PROPERTY];
-          if (enablePersistence) {
+          if (enablePersistence && finality !== "pending") {
             await persistState({
               db,
               endCursor,
@@ -443,6 +481,7 @@ function mongoStorage({
               indexerId
             });
           }
+          prevFinality = finality;
         });
       });
     });

package/dist/index.mjs

@@ -1,5 +1,5 @@
 import { useIndexerContext } from '@apibara/indexer';
-import { defineIndexerPlugin } from '@apibara/indexer/plugins';
+import { defineIndexerPlugin, useLogger } from '@apibara/indexer/plugins';
 import { generateIndexerId } from '@apibara/indexer/internal';
 import { useInternalContext } from '@apibara/indexer/internal/plugins';
 import { normalizeCursor } from '@apibara/protocol';
@@ -37,17 +37,42 @@ async function finalize(db, session, cursor, collections) {
     );
   }
 }
+async function cleanupStorage(db, session, collections) {
+  for (const collection of collections) {
+    try {
+      await db.collection(collection).deleteMany({}, { session });
+    } catch (error) {
+      throw new Error(`Failed to clean up collection ${collection}`, {
+        cause: error
+      });
+    }
+  }
+}
+
+class MongoStorageError extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "MongoStorageError";
+  }
+}
+async function withTransaction(client, cb) {
+  return await client.withSession(async (session) => {
+    return await session.withTransaction(
+      async (session2) => {
+        return await cb(session2);
+      },
+      {
+        retryWrites: false
+      }
+    );
+  });
+}
 
 const checkpointCollectionName = "checkpoints";
 const filterCollectionName = "filters";
 async function initializePersistentState(db, session) {
-  const checkpoint = await db.createCollection(
-    checkpointCollectionName,
-    { session }
-  );
-  const filter = await db.createCollection(filterCollectionName, {
-    session
-  });
+  const checkpoint = db.collection(checkpointCollectionName);
+  const filter = db.collection(filterCollectionName);
   await checkpoint.createIndex({ id: 1 }, { session });
   await filter.createIndex({ id: 1, fromBlock: 1 }, { session });
 }
@@ -59,7 +84,7 @@ async function persistState(props) {
       {
         $set: {
           orderKey: Number(endCursor.orderKey),
-          uniqueKey: endCursor.uniqueKey
+          uniqueKey: endCursor.uniqueKey ? endCursor.uniqueKey : null
         }
       },
       { upsert: true, session }
@@ -132,6 +157,17 @@ async function finalizeState(props) {
     { session }
   );
 }
+async function resetPersistence(props) {
+  const { db, session, indexerId } = props;
+  try {
+    await db.collection(checkpointCollectionName).deleteMany({ id: indexerId }, { session });
+    await db.collection(filterCollectionName).deleteMany({ id: indexerId }, { session });
+  } catch (error) {
+    throw new MongoStorageError("Failed to reset persistence state", {
+      cause: error
+    });
+  }
+}
 
 class MongoStorage {
   constructor(db, session, endCursor) {
@@ -303,20 +339,6 @@ class MongoCollection {
   }
 }
 
-class MongoStorageError extends Error {
-  constructor(message) {
-    super(message);
-    this.name = "MongoStorageError";
-  }
-}
-async function withTransaction(client, cb) {
-  return await client.withSession(async (session) => {
-    return await session.withTransaction(async (session2) => {
-      return await cb(session2);
-    });
-  });
-}
-
 const MONGO_PROPERTY = "_mongo";
 function useMongoStorage() {
   const context = useIndexerContext();
@@ -337,14 +359,27 @@ function mongoStorage({
 }) {
   return defineIndexerPlugin((indexer) => {
     let indexerId = "";
+    const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
+    let prevFinality;
     indexer.hooks.hook("run:before", async () => {
       const { indexerName } = useInternalContext();
       indexerId = generateIndexerId(indexerName, identifier);
+      const logger = useLogger();
       await withTransaction(client, async (session) => {
         const db = client.db(dbName, dbOptions);
         if (enablePersistence) {
           await initializePersistentState(db, session);
         }
+        if (alwaysReindex) {
+          logger.warn(
+            `Reindexing: Deleting all data from collections - ${collections.join(", ")}`
+          );
+          await cleanupStorage(db, session, collections);
+          if (enablePersistence) {
+            await resetPersistence({ db, session, indexerId });
+          }
+          logger.success("All data has been cleaned up for reindexing");
+        }
       });
     });
     indexer.hooks.hook("connect:before", async ({ request }) => {
@@ -397,7 +432,7 @@ function mongoStorage({
       });
     });
     indexer.hooks.hook("message:finalize", async ({ message }) => {
-      const { cursor } = message.finalize;
+      const { cursor } = message;
       if (!cursor) {
         throw new MongoStorageError("finalized cursor is undefined");
       }
@@ -410,7 +445,7 @@ function mongoStorage({
       });
     });
     indexer.hooks.hook("message:invalidate", async ({ message }) => {
-      const { cursor } = message.invalidate;
+      const { cursor } = message;
       if (!cursor) {
         throw new MongoStorageError("invalidate cursor is undefined");
       }
@@ -424,16 +459,19 @@ function mongoStorage({
     });
     indexer.hooks.hook("handler:middleware", async ({ use }) => {
       use(async (context, next) => {
-        const { endCursor } = context;
+        const { endCursor, finality, cursor } = context;
         if (!endCursor) {
           throw new MongoStorageError("end cursor is undefined");
         }
         await withTransaction(client, async (session) => {
           const db = client.db(dbName, dbOptions);
           context[MONGO_PROPERTY] = new MongoStorage(db, session, endCursor);
+          if (prevFinality === "pending") {
+            await invalidate(db, session, cursor, collections);
+          }
           await next();
           delete context[MONGO_PROPERTY];
-          if (enablePersistence) {
+          if (enablePersistence && finality !== "pending") {
             await persistState({
               db,
               endCursor,
@@ -441,6 +479,7 @@ function mongoStorage({
               indexerId
             });
           }
+          prevFinality = finality;
         });
       });
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apibara/plugin-mongo",
-  "version": "2.1.0-beta.3",
+  "version": "2.1.0-beta.30",
   "type": "module",
   "files": [
     "dist",
@@ -35,7 +35,7 @@
     "mongodb": "^6.12.0"
   },
   "dependencies": {
-    "@apibara/indexer": "2.1.0-beta.3",
-    "@apibara/protocol": "2.1.0-beta.3"
+    "@apibara/indexer": "2.1.0-beta.30",
+    "@apibara/protocol": "2.1.0-beta.30"
   }
 }

package/src/index.ts

@@ -1,16 +1,18 @@
 import { useIndexerContext } from "@apibara/indexer";
-import { defineIndexerPlugin } from "@apibara/indexer/plugins";
+import { defineIndexerPlugin, useLogger } from "@apibara/indexer/plugins";
 import type { DbOptions, MongoClient } from "mongodb";
 
 import { generateIndexerId } from "@apibara/indexer/internal";
 import { useInternalContext } from "@apibara/indexer/internal/plugins";
-import { finalize, invalidate } from "./mongo";
+import type { Cursor, DataFinality } from "@apibara/protocol";
+import { cleanupStorage, finalize, invalidate } from "./mongo";
 import {
   finalizeState,
   getState,
   initializePersistentState,
   invalidateState,
   persistState,
+  resetPersistence,
 } from "./persistence";
 import { MongoStorage } from "./storage";
 import { MongoStorageError, withTransaction } from "./utils";
@@ -60,16 +62,33 @@ export function mongoStorage<TFilter, TBlock>({
 }: MongoStorageOptions) {
   return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
     let indexerId = "";
+    const alwaysReindex = process.env["APIBARA_ALWAYS_REINDEX"] === "true";
+    let prevFinality: DataFinality | undefined;
 
     indexer.hooks.hook("run:before", async () => {
       const { indexerName } = useInternalContext();
       indexerId = generateIndexerId(indexerName, identifier);
+      const logger = useLogger();
 
       await withTransaction(client, async (session) => {
         const db = client.db(dbName, dbOptions);
         if (enablePersistence) {
           await initializePersistentState(db, session);
         }
+
+        if (alwaysReindex) {
+          logger.warn(
+            `Reindexing: Deleting all data from collections - ${collections.join(", ")}`,
+          );
+
+          await cleanupStorage(db, session, collections);
+
+          if (enablePersistence) {
+            await resetPersistence({ db, session, indexerId });
+          }
+
+          logger.success("All data has been cleaned up for reindexing");
+        }
       });
     });
 
@@ -133,7 +152,7 @@ export function mongoStorage<TFilter, TBlock>({
     });
 
     indexer.hooks.hook("message:finalize", async ({ message }) => {
-      const { cursor } = message.finalize;
+      const { cursor } = message;
 
       if (!cursor) {
         throw new MongoStorageError("finalized cursor is undefined");
@@ -150,7 +169,7 @@ export function mongoStorage<TFilter, TBlock>({
     });
 
     indexer.hooks.hook("message:invalidate", async ({ message }) => {
-      const { cursor } = message.invalidate;
+      const { cursor } = message;
 
       if (!cursor) {
         throw new MongoStorageError("invalidate cursor is undefined");
@@ -168,7 +187,11 @@ export function mongoStorage<TFilter, TBlock>({
 
     indexer.hooks.hook("handler:middleware", async ({ use }) => {
       use(async (context, next) => {
-        const { endCursor } = context;
+        const { endCursor, finality, cursor } = context as {
+          cursor: Cursor;
+          endCursor: Cursor;
+          finality: DataFinality;
+        };
 
         if (!endCursor) {
           throw new MongoStorageError("end cursor is undefined");
@@ -178,10 +201,16 @@ export function mongoStorage<TFilter, TBlock>({
           const db = client.db(dbName, dbOptions);
           context[MONGO_PROPERTY] = new MongoStorage(db, session, endCursor);
 
+          if (prevFinality === "pending") {
+            // invalidate if previous block's finality was "pending"
+            await invalidate(db, session, cursor, collections);
+          }
+
           await next();
+
           delete context[MONGO_PROPERTY];
 
-          if (enablePersistence) {
+          if (enablePersistence && finality !== "pending") {
             await persistState({
               db,
               endCursor,
@@ -189,6 +218,8 @@ export function mongoStorage<TFilter, TBlock>({
               indexerId,
             });
           }
+
+          prevFinality = finality;
         });
       });
     });

package/src/mongo.ts

@@ -49,3 +49,20 @@ export async function finalize(
     );
   }
 }
+
+export async function cleanupStorage(
+  db: Db,
+  session: ClientSession,
+  collections: string[],
+) {
+  for (const collection of collections) {
+    try {
+      // Delete all documents in the collection
+      await db.collection(collection).deleteMany({}, { session });
+    } catch (error) {
+      throw new Error(`Failed to clean up collection ${collection}`, {
+        cause: error,
+      });
+    }
+  }
+}

package/src/persistence.ts

@@ -1,5 +1,6 @@
 import { type Cursor, normalizeCursor } from "@apibara/protocol";
 import type { ClientSession, Db } from "mongodb";
+import { MongoStorageError } from "./utils";
 
 export type CheckpointSchema = {
   id: string;
@@ -21,13 +22,8 @@ export async function initializePersistentState(
   db: Db,
   session: ClientSession,
 ) {
-  const checkpoint = await db.createCollection<CheckpointSchema>(
-    checkpointCollectionName,
-    { session },
-  );
-  const filter = await db.createCollection<FilterSchema>(filterCollectionName, {
-    session,
-  });
+  const checkpoint = db.collection<CheckpointSchema>(checkpointCollectionName);
+  const filter = db.collection<FilterSchema>(filterCollectionName);
 
   await checkpoint.createIndex({ id: 1 }, { session });
   await filter.createIndex({ id: 1, fromBlock: 1 }, { session });
@@ -48,7 +44,7 @@ export async function persistState<TFilter>(props: {
       {
         $set: {
           orderKey: Number(endCursor.orderKey),
-          uniqueKey: endCursor.uniqueKey,
+          uniqueKey: endCursor.uniqueKey ? endCursor.uniqueKey : null,
         },
       },
       { upsert: true, session },
@@ -161,3 +157,27 @@ export async function finalizeState(props: {
     { session },
   );
 }
+
+export async function resetPersistence(props: {
+  db: Db;
+  session: ClientSession;
+  indexerId: string;
+}) {
+  const { db, session, indexerId } = props;
+
+  try {
+    // Delete all checkpoints for this indexer
+    await db
+      .collection<CheckpointSchema>(checkpointCollectionName)
+      .deleteMany({ id: indexerId }, { session });
+
+    // Delete all filters for this indexer
+    await db
+      .collection<FilterSchema>(filterCollectionName)
+      .deleteMany({ id: indexerId }, { session });
+  } catch (error) {
+    throw new MongoStorageError("Failed to reset persistence state", {
+      cause: error,
+    });
+  }
+}

package/src/utils.ts

@@ -1,8 +1,8 @@
 import type { ClientSession, MongoClient } from "mongodb";
 
 export class MongoStorageError extends Error {
-  constructor(message: string) {
-    super(message);
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
     this.name = "MongoStorageError";
   }
 }
@@ -12,8 +12,13 @@ export async function withTransaction<T>(
   cb: (session: ClientSession) => Promise<T>,
 ) {
   return await client.withSession(async (session) => {
-    return await session.withTransaction(async (session) => {
-      return await cb(session);
-    });
+    return await session.withTransaction(
+      async (session) => {
+        return await cb(session);
+      },
+      {
+        retryWrites: false,
+      },
+    );
   });
 }