@apibara/plugin-sqlite 2.0.0-beta.36 → 2.0.0-beta.38

package/dist/index.cjs

@@ -3,10 +3,12 @@
 const indexer = require('@apibara/indexer');
 const plugins = require('@apibara/indexer/plugins');
 const protocol = require('@apibara/protocol');
+const internal = require('@apibara/indexer/internal');
+const plugins$1 = require('@apibara/indexer/internal/plugins');
 
 class SqliteStorageError extends Error {
-  constructor(message) {
-    super(message);
+  constructor(message, options) {
+    super(message, options);
     this.name = "SqliteStorageError";
   }
 }
@@ -38,6 +40,9 @@ function serialize(obj) {
     "	"
   );
 }
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
 
 function initializeKeyValueStore(db) {
   assertInTransaction(db);
@@ -125,32 +130,32 @@ function initializePersistentState(db) {
   db.exec(statements.createFiltersTable);
 }
 function persistState(props) {
-  const { db, endCursor, filter, indexerName } = props;
+  const { db, endCursor, filter, indexerId } = props;
   assertInTransaction(db);
   db.prepare(statements.putCheckpoint).run(
-    indexerName,
+    indexerId,
     Number(endCursor.orderKey),
     endCursor.uniqueKey
   );
   if (filter) {
     db.prepare(statements.updateFilterToBlock).run(
       Number(endCursor.orderKey),
-      indexerName
+      indexerId
     );
     db.prepare(statements.insertFilter).run(
-      indexerName,
+      indexerId,
       serialize(filter),
       Number(endCursor.orderKey)
     );
   }
 }
 function getState(props) {
-  const { db, indexerName } = props;
+  const { db, indexerId } = props;
   assertInTransaction(db);
   const storedCursor = db.prepare(
     statements.getCheckpoint
-  ).get(indexerName);
-  const storedFilter = db.prepare(statements.getFilter).get(indexerName);
+  ).get(indexerId);
+  const storedFilter = db.prepare(statements.getFilter).get(indexerId);
   let cursor;
   let filter;
   if (storedCursor?.order_key) {
@@ -165,22 +170,22 @@ function getState(props) {
   return { cursor, filter };
 }
 function finalizeState(props) {
-  const { cursor, db, indexerName } = props;
+  const { cursor, db, indexerId } = props;
   assertInTransaction(db);
   db.prepare(statements.finalizeFilter).run(
-    indexerName,
+    indexerId,
     Number(cursor.orderKey)
   );
 }
 function invalidateState(props) {
-  const { cursor, db, indexerName } = props;
+  const { cursor, db, indexerId } = props;
   assertInTransaction(db);
   db.prepare(statements.invalidateFilterDelete).run(
-    indexerName,
+    indexerId,
     Number(cursor.orderKey)
   );
   db.prepare(statements.invalidateFilterUpdate).run(
-    indexerName,
+    indexerId,
     Number(cursor.orderKey)
   );
 }
@@ -242,6 +247,7 @@ const statements = {
 };
 
 const KV_PROPERTY = "_kv_sqlite";
+const MAX_RETRIES = 5;
 function useSqliteKeyValueStore() {
   const kv = indexer.useIndexerContext()[KV_PROPERTY];
   if (!kv) {
@@ -257,25 +263,45 @@ function sqliteStorage({
   keyValueStore: enableKeyValueStore = true,
   serialize: serializeFn = serialize,
   deserialize: deserializeFn = deserialize,
-  indexerName = "default"
+  indexerName: identifier = "default"
 }) {
   return plugins.defineIndexerPlugin((indexer) => {
+    let indexerId = "";
     indexer.hooks.hook("run:before", async () => {
-      await withTransaction(database, async (db) => {
-        if (enablePersistState) {
-          initializePersistentState(db);
-        }
-        if (enableKeyValueStore) {
-          initializeKeyValueStore(db);
+      const { indexerName: indexerFileName, availableIndexers } = plugins$1.useInternalContext();
+      indexerId = internal.generateIndexerId(indexerFileName, identifier);
+      let retries = 0;
+      while (retries <= MAX_RETRIES) {
+        try {
+          await withTransaction(database, async (db) => {
+            if (enablePersistState) {
+              initializePersistentState(db);
+            }
+            if (enableKeyValueStore) {
+              initializeKeyValueStore(db);
+            }
+          });
+          break;
+        } catch (error) {
+          if (retries === MAX_RETRIES) {
+            throw new SqliteStorageError(
+              "Initialization failed after 5 retries",
+              {
+                cause: error
+              }
+            );
+          }
+          await sleep(retries * 1e3);
+          retries++;
         }
-      });
+      }
     });
     indexer.hooks.hook("connect:before", async ({ request }) => {
       if (!enablePersistState) {
         return;
       }
       return await withTransaction(database, async (db) => {
-        const { cursor, filter } = getState({ db, indexerName });
+        const { cursor, filter } = getState({ db, indexerId });
         if (cursor) {
           request.startingCursor = cursor;
         }
@@ -291,7 +317,7 @@ function sqliteStorage({
       }
       await withTransaction(database, async (db) => {
         if (enablePersistState) {
-          invalidateState({ db, cursor, indexerName });
+          invalidateState({ db, cursor, indexerId });
         }
         if (enableKeyValueStore) {
           invalidateKV(db, cursor);
@@ -307,7 +333,7 @@ function sqliteStorage({
         persistState({
           db: database,
           endCursor,
-          indexerName,
+          indexerId,
           filter: request.filter[1]
         });
       }
@@ -319,7 +345,7 @@ function sqliteStorage({
       }
       await withTransaction(database, async (db) => {
         if (enablePersistState) {
-          finalizeState({ db, cursor, indexerName });
+          finalizeState({ db, cursor, indexerId });
         }
         if (enableKeyValueStore) {
           finalizeKV(db, cursor);
@@ -333,7 +359,7 @@ function sqliteStorage({
       }
       await withTransaction(database, async (db) => {
         if (enablePersistState) {
-          invalidateState({ db, cursor, indexerName });
+          invalidateState({ db, cursor, indexerId });
         }
         if (enableKeyValueStore) {
           invalidateKV(db, cursor);
@@ -362,7 +388,7 @@ function sqliteStorage({
           }
           await next();
           if (enablePersistState) {
-            persistState({ db, endCursor: ctx.endCursor, indexerName });
+            persistState({ db, endCursor: ctx.endCursor, indexerId });
           }
           if (enableKeyValueStore) {
             delete ctx[KV_PROPERTY];

package/dist/index.d.cts

@@ -37,6 +37,6 @@ type SqliteStorageOptions = {
  * @param options.deserialize - A function to deserialize the value from the KV.
  * @param options.indexerName - The name of the indexer. Defaults value is 'default'.
  */
-declare function sqliteStorage<TFilter, TBlock>({ database, persistState: enablePersistState, keyValueStore: enableKeyValueStore, serialize: serializeFn, deserialize: deserializeFn, indexerName, }: SqliteStorageOptions): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
+declare function sqliteStorage<TFilter, TBlock>({ database, persistState: enablePersistState, keyValueStore: enableKeyValueStore, serialize: serializeFn, deserialize: deserializeFn, indexerName: identifier, }: SqliteStorageOptions): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
 
 export { KeyValueStore, type SqliteStorageOptions, sqliteStorage, useSqliteKeyValueStore };

package/dist/index.d.mts

@@ -37,6 +37,6 @@ type SqliteStorageOptions = {
  * @param options.deserialize - A function to deserialize the value from the KV.
  * @param options.indexerName - The name of the indexer. Defaults value is 'default'.
  */
-declare function sqliteStorage<TFilter, TBlock>({ database, persistState: enablePersistState, keyValueStore: enableKeyValueStore, serialize: serializeFn, deserialize: deserializeFn, indexerName, }: SqliteStorageOptions): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
+declare function sqliteStorage<TFilter, TBlock>({ database, persistState: enablePersistState, keyValueStore: enableKeyValueStore, serialize: serializeFn, deserialize: deserializeFn, indexerName: identifier, }: SqliteStorageOptions): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
 
 export { KeyValueStore, type SqliteStorageOptions, sqliteStorage, useSqliteKeyValueStore };

package/dist/index.d.ts

@@ -37,6 +37,6 @@ type SqliteStorageOptions = {
  * @param options.deserialize - A function to deserialize the value from the KV.
  * @param options.indexerName - The name of the indexer. Defaults value is 'default'.
  */
-declare function sqliteStorage<TFilter, TBlock>({ database, persistState: enablePersistState, keyValueStore: enableKeyValueStore, serialize: serializeFn, deserialize: deserializeFn, indexerName, }: SqliteStorageOptions): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
+declare function sqliteStorage<TFilter, TBlock>({ database, persistState: enablePersistState, keyValueStore: enableKeyValueStore, serialize: serializeFn, deserialize: deserializeFn, indexerName: identifier, }: SqliteStorageOptions): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
 
 export { KeyValueStore, type SqliteStorageOptions, sqliteStorage, useSqliteKeyValueStore };

package/dist/index.mjs

@@ -1,10 +1,12 @@
 import { useIndexerContext } from '@apibara/indexer';
 import { defineIndexerPlugin } from '@apibara/indexer/plugins';
 import { isCursor } from '@apibara/protocol';
+import { generateIndexerId } from '@apibara/indexer/internal';
+import { useInternalContext } from '@apibara/indexer/internal/plugins';
 
 class SqliteStorageError extends Error {
-  constructor(message) {
-    super(message);
+  constructor(message, options) {
+    super(message, options);
     this.name = "SqliteStorageError";
   }
 }
@@ -36,6 +38,9 @@ function serialize(obj) {
     "	"
   );
 }
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
 
 function initializeKeyValueStore(db) {
   assertInTransaction(db);
@@ -123,32 +128,32 @@ function initializePersistentState(db) {
   db.exec(statements.createFiltersTable);
 }
 function persistState(props) {
-  const { db, endCursor, filter, indexerName } = props;
+  const { db, endCursor, filter, indexerId } = props;
   assertInTransaction(db);
   db.prepare(statements.putCheckpoint).run(
-    indexerName,
+    indexerId,
     Number(endCursor.orderKey),
     endCursor.uniqueKey
   );
   if (filter) {
     db.prepare(statements.updateFilterToBlock).run(
       Number(endCursor.orderKey),
-      indexerName
+      indexerId
     );
     db.prepare(statements.insertFilter).run(
-      indexerName,
+      indexerId,
       serialize(filter),
       Number(endCursor.orderKey)
     );
   }
 }
 function getState(props) {
-  const { db, indexerName } = props;
+  const { db, indexerId } = props;
   assertInTransaction(db);
   const storedCursor = db.prepare(
     statements.getCheckpoint
-  ).get(indexerName);
-  const storedFilter = db.prepare(statements.getFilter).get(indexerName);
+  ).get(indexerId);
+  const storedFilter = db.prepare(statements.getFilter).get(indexerId);
   let cursor;
   let filter;
   if (storedCursor?.order_key) {
@@ -163,22 +168,22 @@ function getState(props) {
   return { cursor, filter };
 }
 function finalizeState(props) {
-  const { cursor, db, indexerName } = props;
+  const { cursor, db, indexerId } = props;
   assertInTransaction(db);
   db.prepare(statements.finalizeFilter).run(
-    indexerName,
+    indexerId,
     Number(cursor.orderKey)
   );
 }
 function invalidateState(props) {
-  const { cursor, db, indexerName } = props;
+  const { cursor, db, indexerId } = props;
   assertInTransaction(db);
   db.prepare(statements.invalidateFilterDelete).run(
-    indexerName,
+    indexerId,
     Number(cursor.orderKey)
   );
   db.prepare(statements.invalidateFilterUpdate).run(
-    indexerName,
+    indexerId,
     Number(cursor.orderKey)
   );
 }
@@ -240,6 +245,7 @@ const statements = {
 };
 
 const KV_PROPERTY = "_kv_sqlite";
+const MAX_RETRIES = 5;
 function useSqliteKeyValueStore() {
   const kv = useIndexerContext()[KV_PROPERTY];
   if (!kv) {
@@ -255,25 +261,45 @@ function sqliteStorage({
   keyValueStore: enableKeyValueStore = true,
   serialize: serializeFn = serialize,
   deserialize: deserializeFn = deserialize,
-  indexerName = "default"
+  indexerName: identifier = "default"
 }) {
   return defineIndexerPlugin((indexer) => {
+    let indexerId = "";
     indexer.hooks.hook("run:before", async () => {
-      await withTransaction(database, async (db) => {
-        if (enablePersistState) {
-          initializePersistentState(db);
-        }
-        if (enableKeyValueStore) {
-          initializeKeyValueStore(db);
+      const { indexerName: indexerFileName, availableIndexers } = useInternalContext();
+      indexerId = generateIndexerId(indexerFileName, identifier);
+      let retries = 0;
+      while (retries <= MAX_RETRIES) {
+        try {
+          await withTransaction(database, async (db) => {
+            if (enablePersistState) {
+              initializePersistentState(db);
+            }
+            if (enableKeyValueStore) {
+              initializeKeyValueStore(db);
+            }
+          });
+          break;
+        } catch (error) {
+          if (retries === MAX_RETRIES) {
+            throw new SqliteStorageError(
+              "Initialization failed after 5 retries",
+              {
+                cause: error
+              }
+            );
+          }
+          await sleep(retries * 1e3);
+          retries++;
         }
-      });
+      }
     });
     indexer.hooks.hook("connect:before", async ({ request }) => {
       if (!enablePersistState) {
         return;
       }
       return await withTransaction(database, async (db) => {
-        const { cursor, filter } = getState({ db, indexerName });
+        const { cursor, filter } = getState({ db, indexerId });
         if (cursor) {
           request.startingCursor = cursor;
         }
@@ -289,7 +315,7 @@ function sqliteStorage({
       }
       await withTransaction(database, async (db) => {
         if (enablePersistState) {
-          invalidateState({ db, cursor, indexerName });
+          invalidateState({ db, cursor, indexerId });
         }
         if (enableKeyValueStore) {
           invalidateKV(db, cursor);
@@ -305,7 +331,7 @@ function sqliteStorage({
         persistState({
           db: database,
           endCursor,
-          indexerName,
+          indexerId,
           filter: request.filter[1]
         });
       }
@@ -317,7 +343,7 @@ function sqliteStorage({
       }
       await withTransaction(database, async (db) => {
         if (enablePersistState) {
-          finalizeState({ db, cursor, indexerName });
+          finalizeState({ db, cursor, indexerId });
         }
         if (enableKeyValueStore) {
           finalizeKV(db, cursor);
@@ -331,7 +357,7 @@ function sqliteStorage({
       }
       await withTransaction(database, async (db) => {
         if (enablePersistState) {
-          invalidateState({ db, cursor, indexerName });
+          invalidateState({ db, cursor, indexerId });
         }
         if (enableKeyValueStore) {
           invalidateKV(db, cursor);
@@ -360,7 +386,7 @@ function sqliteStorage({
           }
           await next();
           if (enablePersistState) {
-            persistState({ db, endCursor: ctx.endCursor, indexerName });
+            persistState({ db, endCursor: ctx.endCursor, indexerId });
           }
           if (enableKeyValueStore) {
             delete ctx[KV_PROPERTY];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apibara/plugin-sqlite",
-  "version": "2.0.0-beta.36",
+  "version": "2.0.0-beta.38",
   "type": "module",
   "files": [
     "dist",
@@ -35,7 +35,7 @@
     "better-sqlite3": "^9.0.0"
   },
   "dependencies": {
-    "@apibara/indexer": "2.0.0-beta.37",
-    "@apibara/protocol": "2.0.0-beta.37"
+    "@apibara/indexer": "2.0.0-beta.39",
+    "@apibara/protocol": "2.0.0-beta.39"
   }
 }

package/src/index.ts

@@ -3,6 +3,8 @@ import { defineIndexerPlugin } from "@apibara/indexer/plugins";
 import { isCursor } from "@apibara/protocol";
 import type { Database as SqliteDatabase } from "better-sqlite3";
 
+import { generateIndexerId } from "@apibara/indexer/internal";
+import { useInternalContext } from "@apibara/indexer/internal/plugins";
 import {
   KeyValueStore,
   finalizeKV,
@@ -23,10 +25,12 @@ import {
   assertInTransaction,
   deserialize,
   serialize,
+  sleep,
   withTransaction,
 } from "./utils";
 
 const KV_PROPERTY = "_kv_sqlite" as const;
+const MAX_RETRIES = 5;
 
 export { KeyValueStore } from "./kv";
 
@@ -68,19 +72,44 @@ export function sqliteStorage<TFilter, TBlock>({
   keyValueStore: enableKeyValueStore = true,
   serialize: serializeFn = serialize,
   deserialize: deserializeFn = deserialize,
-  indexerName = "default",
+  indexerName: identifier = "default",
 }: SqliteStorageOptions) {
   return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
-    indexer.hooks.hook("run:before", async () => {
-      await withTransaction(database, async (db) => {
-        if (enablePersistState) {
-          initializePersistentState(db);
-        }
+    let indexerId = "";
 
-        if (enableKeyValueStore) {
-          initializeKeyValueStore(db);
+    indexer.hooks.hook("run:before", async () => {
+      const { indexerName: indexerFileName, availableIndexers } =
+        useInternalContext();
+
+      indexerId = generateIndexerId(indexerFileName, identifier);
+
+      let retries = 0;
+
+      while (retries <= MAX_RETRIES) {
+        try {
+          await withTransaction(database, async (db) => {
+            if (enablePersistState) {
+              initializePersistentState(db);
+            }
+
+            if (enableKeyValueStore) {
+              initializeKeyValueStore(db);
+            }
+          });
+          break;
+        } catch (error) {
+          if (retries === MAX_RETRIES) {
+            throw new SqliteStorageError(
+              "Initialization failed after 5 retries",
+              {
+                cause: error,
+              },
+            );
+          }
+          await sleep(retries * 1000);
+          retries++;
         }
-      });
+      }
     });
 
     indexer.hooks.hook("connect:before", async ({ request }) => {
@@ -89,7 +118,7 @@ export function sqliteStorage<TFilter, TBlock>({
       }
 
       return await withTransaction(database, async (db) => {
-        const { cursor, filter } = getState<TFilter>({ db, indexerName });
+        const { cursor, filter } = getState<TFilter>({ db, indexerId });
 
         if (cursor) {
           request.startingCursor = cursor;
@@ -111,7 +140,7 @@ export function sqliteStorage<TFilter, TBlock>({
 
       await withTransaction(database, async (db) => {
         if (enablePersistState) {
-          invalidateState({ db, cursor, indexerName });
+          invalidateState({ db, cursor, indexerId });
         }
 
         if (enableKeyValueStore) {
@@ -133,7 +162,7 @@ export function sqliteStorage<TFilter, TBlock>({
         persistState({
           db: database,
           endCursor,
-          indexerName,
+          indexerId,
           filter: request.filter[1],
         });
       }
@@ -148,7 +177,7 @@ export function sqliteStorage<TFilter, TBlock>({
 
       await withTransaction(database, async (db) => {
         if (enablePersistState) {
-          finalizeState({ db, cursor, indexerName });
+          finalizeState({ db, cursor, indexerId });
         }
 
         if (enableKeyValueStore) {
@@ -166,7 +195,7 @@ export function sqliteStorage<TFilter, TBlock>({
 
       await withTransaction(database, async (db) => {
         if (enablePersistState) {
-          invalidateState({ db, cursor, indexerName });
+          invalidateState({ db, cursor, indexerId });
         }
 
         if (enableKeyValueStore) {
@@ -201,7 +230,7 @@ export function sqliteStorage<TFilter, TBlock>({
           await next();
 
           if (enablePersistState) {
-            persistState({ db, endCursor: ctx.endCursor, indexerName });
+            persistState({ db, endCursor: ctx.endCursor, indexerId });
           }
 
           if (enableKeyValueStore) {

package/src/persistence.ts

@@ -13,14 +13,14 @@ export function persistState<TFilter>(props: {
   db: Database;
   endCursor: Cursor;
   filter?: TFilter;
-  indexerName: string;
+  indexerId: string;
 }) {
-  const { db, endCursor, filter, indexerName } = props;
+  const { db, endCursor, filter, indexerId } = props;
 
   assertInTransaction(db);
 
   db.prepare(statements.putCheckpoint).run(
-    indexerName,
+    indexerId,
     Number(endCursor.orderKey),
     endCursor.uniqueKey,
   );
@@ -28,10 +28,10 @@ export function persistState<TFilter>(props: {
   if (filter) {
     db.prepare(statements.updateFilterToBlock).run(
       Number(endCursor.orderKey),
-      indexerName,
+      indexerId,
     );
     db.prepare(statements.insertFilter).run(
-      indexerName,
+      indexerId,
       serialize(filter as Record<string, unknown>),
       Number(endCursor.orderKey),
     );
@@ -40,18 +40,18 @@ export function persistState<TFilter>(props: {
 
 export function getState<TFilter>(props: {
   db: Database;
-  indexerName: string;
+  indexerId: string;
 }) {
-  const { db, indexerName } = props;
+  const { db, indexerId } = props;
   assertInTransaction(db);
   const storedCursor = db
     .prepare<string, { order_key?: number; unique_key?: string }>(
       statements.getCheckpoint,
     )
-    .get(indexerName);
+    .get(indexerId);
   const storedFilter = db
     .prepare<string, { filter: string }>(statements.getFilter)
-    .get(indexerName);
+    .get(indexerId);
 
   let cursor: Cursor | undefined;
   let filter: TFilter | undefined;
@@ -73,12 +73,12 @@ export function getState<TFilter>(props: {
 export function finalizeState(props: {
   db: Database;
   cursor: Cursor;
-  indexerName: string;
+  indexerId: string;
 }) {
-  const { cursor, db, indexerName } = props;
+  const { cursor, db, indexerId } = props;
   assertInTransaction(db);
   db.prepare<[string, number]>(statements.finalizeFilter).run(
-    indexerName,
+    indexerId,
     Number(cursor.orderKey),
   );
 }
@@ -86,16 +86,16 @@ export function finalizeState(props: {
 export function invalidateState(props: {
   db: Database;
   cursor: Cursor;
-  indexerName: string;
+  indexerId: string;
 }) {
-  const { cursor, db, indexerName } = props;
+  const { cursor, db, indexerId } = props;
   assertInTransaction(db);
   db.prepare<[string, number]>(statements.invalidateFilterDelete).run(
-    indexerName,
+    indexerId,
     Number(cursor.orderKey),
   );
   db.prepare<[string, number]>(statements.invalidateFilterUpdate).run(
-    indexerName,
+    indexerId,
     Number(cursor.orderKey),
   );
 }

package/src/utils.ts

@@ -4,8 +4,8 @@ export type SerializeFn = <T>(value: T) => string;
 export type DeserializeFn = <T>(value: string) => T;
 
 export class SqliteStorageError extends Error {
-  constructor(message: string) {
-    super(message);
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
     this.name = "SqliteStorageError";
   }
 }
@@ -45,3 +45,7 @@ export function serialize<T>(obj: T): string {
     "\t",
   );
 }
+
+export function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}