@apibara/plugin-sqlite 2.0.0-beta.27

package/README.md

@@ -0,0 +1,7 @@
+# `@apibara/plugin-sqlite`
+
+TODO
+
+## Installation
+
+TODO

package/dist/index.cjs

@@ -0,0 +1,274 @@
+'use strict';
+
+const indexer = require('@apibara/indexer');
+const plugins = require('@apibara/indexer/plugins');
+const protocol = require('@apibara/protocol');
+
+class SqliteStorageError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "SqliteStorageError";
+  }
+}
+async function withTransaction(db, cb) {
+  db.prepare("BEGIN TRANSACTION").run();
+  try {
+    await cb(db);
+  } catch (error) {
+    db.prepare("ROLLBACK TRANSACTION").run();
+    throw error;
+  }
+  db.prepare("COMMIT TRANSACTION").run();
+}
+function assertInTransaction(db) {
+  if (!db.inTransaction) {
+    throw new SqliteStorageError("Database is not in transaction");
+  }
+}
+function deserialize(str) {
+  return JSON.parse(
+    str,
+    (_, value) => typeof value === "string" && value.match(/^\d+n$/) ? BigInt(value.slice(0, -1)) : value
+  );
+}
+function serialize(obj) {
+  return JSON.stringify(
+    obj,
+    (_, value) => typeof value === "bigint" ? `${value.toString()}n` : value,
+    "	"
+  );
+}
+
+function initializeKeyValueStore(db) {
+  assertInTransaction(db);
+  db.exec(statements$1.createTable);
+}
+class KeyValueStore {
+  constructor(db, endCursor, finality, serialize, deserialize) {
+    this.db = db;
+    this.endCursor = endCursor;
+    this.finality = finality;
+    this.serialize = serialize;
+    this.deserialize = deserialize;
+    assertInTransaction(db);
+  }
+  get(key) {
+    const row = this.db.prepare(statements$1.get).get(key);
+    return row ? this.deserialize(row.v) : void 0;
+  }
+  put(key, value) {
+    this.db.prepare(statements$1.updateToBlock).run(Number(this.endCursor.orderKey), key);
+    this.db.prepare(statements$1.insertIntoKvs).run(
+      Number(this.endCursor.orderKey),
+      key,
+      this.serialize(value)
+    );
+  }
+  del(key) {
+    this.db.prepare(statements$1.del).run(Number(this.endCursor.orderKey), key);
+  }
+}
+const statements$1 = {
+  createTable: `
+    CREATE TABLE IF NOT EXISTS kvs (
+      from_block INTEGER NOT NULL,
+      to_block INTEGER,
+      k TEXT NOT NULL,
+      v BLOB NOT NULL,
+      PRIMARY KEY (from_block, k)
+      );`,
+  get: `
+    SELECT v
+    FROM kvs
+    WHERE k = ? AND to_block IS NULL`,
+  updateToBlock: `
+    UPDATE kvs
+    SET to_block = ?
+    WHERE k = ? AND to_block IS NULL`,
+  insertIntoKvs: `
+    INSERT INTO kvs (from_block, to_block, k, v)
+    VALUES (?, NULL, ?, ?)`,
+  del: `
+    UPDATE kvs
+    SET to_block = ?
+    WHERE k = ? AND to_block IS NULL`
+};
+
+const DEFAULT_INDEXER_ID = "default";
+function initializePersistentState(db) {
+  assertInTransaction(db);
+  db.exec(statements.createCheckpointsTable);
+  db.exec(statements.createFiltersTable);
+}
+function persistState(db, endCursor, filter) {
+  assertInTransaction(db);
+  db.prepare(statements.putCheckpoint).run(
+    DEFAULT_INDEXER_ID,
+    Number(endCursor.orderKey),
+    endCursor.uniqueKey
+  );
+  if (filter) {
+    db.prepare(statements.updateFilterToBlock).run(
+      Number(endCursor.orderKey),
+      DEFAULT_INDEXER_ID
+    );
+    db.prepare(statements.insertFilter).run(
+      DEFAULT_INDEXER_ID,
+      serialize(filter),
+      Number(endCursor.orderKey)
+    );
+  }
+}
+function getState(db) {
+  assertInTransaction(db);
+  const storedCursor = db.prepare(
+    statements.getCheckpoint
+  ).get(DEFAULT_INDEXER_ID);
+  const storedFilter = db.prepare(statements.getFilter).get(DEFAULT_INDEXER_ID);
+  let cursor;
+  let filter;
+  if (storedCursor?.order_key) {
+    cursor = {
+      orderKey: BigInt(storedCursor.order_key),
+      uniqueKey: storedCursor.unique_key
+    };
+  }
+  if (storedFilter) {
+    filter = deserialize(storedFilter.filter);
+  }
+  return { cursor, filter };
+}
+const statements = {
+  createCheckpointsTable: `
+    CREATE TABLE IF NOT EXISTS checkpoints (
+      id TEXT NOT NULL PRIMARY KEY,
+      order_key INTEGER,
+      unique_key TEXT
+    );`,
+  createFiltersTable: `
+    CREATE TABLE IF NOT EXISTS filters (
+      id TEXT NOT NULL,
+      filter BLOB NOT NULL,
+      from_block INTEGER NOT NULL,
+      to_block INTEGER,
+      PRIMARY KEY (id, from_block)
+    );`,
+  getCheckpoint: `
+    SELECT *
+    FROM checkpoints
+    WHERE id = ?`,
+  putCheckpoint: `
+    INSERT INTO checkpoints (id, order_key, unique_key)
+    VALUES (?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      order_key = excluded.order_key,
+      unique_key = excluded.unique_key`,
+  delCheckpoint: `
+    DELETE FROM checkpoints
+    WHERE id = ?`,
+  getFilter: `
+    SELECT *
+    FROM filters
+    WHERE id = ? AND to_block IS NULL`,
+  updateFilterToBlock: `
+    UPDATE filters
+    SET to_block = ?
+    WHERE id = ? AND to_block IS NULL`,
+  insertFilter: `
+    INSERT INTO filters (id, filter, from_block)
+    VALUES (?, ?, ?)
+    ON CONFLICT(id, from_block) DO UPDATE SET
+      filter = excluded.filter,
+      from_block = excluded.from_block`,
+  delFilter: `
+    DELETE FROM filters
+    WHERE id = ?`
+};
+
+const KV_PROPERTY = "_kv_sqlite";
+function useSqliteKeyValueStore() {
+  const kv = indexer.useIndexerContext()[KV_PROPERTY];
+  if (!kv) {
+    throw new SqliteStorageError(
+      "SQLite key-value store is not available. Did you forget to enable it?"
+    );
+  }
+  return kv;
+}
+function sqliteStorage({
+  database,
+  persistState: enablePersistState = true,
+  keyValueStore: enableKeyValueStore = true,
+  serialize: serializeFn = serialize,
+  deserialize: deserializeFn = deserialize
+}) {
+  return plugins.defineIndexerPlugin((indexer) => {
+    indexer.hooks.hook("run:before", async () => {
+      await withTransaction(database, async (db) => {
+        if (enablePersistState) {
+          initializePersistentState(db);
+        }
+        if (enableKeyValueStore) {
+          initializeKeyValueStore(db);
+        }
+      });
+    });
+    indexer.hooks.hook("connect:before", async ({ request }) => {
+      if (!enablePersistState) {
+        return;
+      }
+      return await withTransaction(database, async (db) => {
+        const { cursor, filter } = getState(db);
+        if (cursor) {
+          request.startingCursor = cursor;
+        }
+        if (filter) {
+          request.filter[1] = filter;
+        }
+      });
+    });
+    indexer.hooks.hook("handler:middleware", ({ use }) => {
+      use(async (ctx, next) => {
+        if (!ctx.finality) {
+          throw new SqliteStorageError("finality is undefined");
+        }
+        if (!ctx.endCursor || !protocol.isCursor(ctx.endCursor)) {
+          throw new SqliteStorageError(
+            "endCursor is undefined or not a cursor"
+          );
+        }
+        await withTransaction(database, async (db) => {
+          if (enableKeyValueStore) {
+            ctx[KV_PROPERTY] = new KeyValueStore(
+              db,
+              ctx.endCursor,
+              ctx.finality,
+              serializeFn,
+              deserializeFn
+            );
+          }
+          await next();
+          if (enablePersistState) {
+            persistState(db, ctx.endCursor);
+          }
+          if (enableKeyValueStore) {
+            delete ctx[KV_PROPERTY];
+          }
+        });
+      });
+    });
+    indexer.hooks.hook("connect:factory", ({ request, endCursor }) => {
+      if (!enablePersistState) {
+        return;
+      }
+      assertInTransaction(database);
+      if (endCursor && request.filter[1]) {
+        persistState(database, endCursor, request.filter[1]);
+      }
+    });
+  });
+}
+
+exports.KeyValueStore = KeyValueStore;
+exports.sqliteStorage = sqliteStorage;
+exports.useSqliteKeyValueStore = useSqliteKeyValueStore;

package/dist/index.d.cts

@@ -0,0 +1,40 @@
+import * as _apibara_indexer_plugins from '@apibara/indexer/plugins';
+import { Database } from 'better-sqlite3';
+import { Cursor, DataFinality } from '@apibara/protocol';
+
+type SerializeFn = <T>(value: T) => string;
+type DeserializeFn = <T>(value: string) => T;
+
+declare class KeyValueStore {
+    private readonly db;
+    private readonly endCursor;
+    private readonly finality;
+    private readonly serialize;
+    private readonly deserialize;
+    constructor(db: Database, endCursor: Cursor, finality: DataFinality, serialize: SerializeFn, deserialize: DeserializeFn);
+    get<T>(key: string): T | undefined;
+    put<T>(key: string, value: T): void;
+    del(key: string): void;
+}
+
+declare function useSqliteKeyValueStore(): KeyValueStore;
+type SqliteStorageOptions = {
+    database: Database;
+    keyValueStore?: boolean;
+    persistState?: boolean;
+    serialize?: SerializeFn;
+    deserialize?: DeserializeFn;
+};
+/**
+ * Creates a plugin that uses SQLite as the storage layer.
+ *
+ * Supports storing the indexer's state and provides a simple Key-Value store.
+ * @param options.database - The SQLite database instance.
+ * @param options.persistState - Whether to persist the indexer's state. Defaults to true.
+ * @param options.keyValueStore - Whether to enable the Key-Value store. Defaults to true.
+ * @param options.serialize - A function to serialize the value to the KV.
+ * @param options.deserialize - A function to deserialize the value from the KV.
+ */
+declare function sqliteStorage<TFilter, TBlock>({ database, persistState: enablePersistState, keyValueStore: enableKeyValueStore, serialize: serializeFn, deserialize: deserializeFn, }: SqliteStorageOptions): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
+
+export { KeyValueStore, type SqliteStorageOptions, sqliteStorage, useSqliteKeyValueStore };

package/dist/index.d.mts

@@ -0,0 +1,40 @@
+import * as _apibara_indexer_plugins from '@apibara/indexer/plugins';
+import { Database } from 'better-sqlite3';
+import { Cursor, DataFinality } from '@apibara/protocol';
+
+type SerializeFn = <T>(value: T) => string;
+type DeserializeFn = <T>(value: string) => T;
+
+declare class KeyValueStore {
+    private readonly db;
+    private readonly endCursor;
+    private readonly finality;
+    private readonly serialize;
+    private readonly deserialize;
+    constructor(db: Database, endCursor: Cursor, finality: DataFinality, serialize: SerializeFn, deserialize: DeserializeFn);
+    get<T>(key: string): T | undefined;
+    put<T>(key: string, value: T): void;
+    del(key: string): void;
+}
+
+declare function useSqliteKeyValueStore(): KeyValueStore;
+type SqliteStorageOptions = {
+    database: Database;
+    keyValueStore?: boolean;
+    persistState?: boolean;
+    serialize?: SerializeFn;
+    deserialize?: DeserializeFn;
+};
+/**
+ * Creates a plugin that uses SQLite as the storage layer.
+ *
+ * Supports storing the indexer's state and provides a simple Key-Value store.
+ * @param options.database - The SQLite database instance.
+ * @param options.persistState - Whether to persist the indexer's state. Defaults to true.
+ * @param options.keyValueStore - Whether to enable the Key-Value store. Defaults to true.
+ * @param options.serialize - A function to serialize the value to the KV.
+ * @param options.deserialize - A function to deserialize the value from the KV.
+ */
+declare function sqliteStorage<TFilter, TBlock>({ database, persistState: enablePersistState, keyValueStore: enableKeyValueStore, serialize: serializeFn, deserialize: deserializeFn, }: SqliteStorageOptions): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
+
+export { KeyValueStore, type SqliteStorageOptions, sqliteStorage, useSqliteKeyValueStore };

package/dist/index.d.ts

@@ -0,0 +1,40 @@
+import * as _apibara_indexer_plugins from '@apibara/indexer/plugins';
+import { Database } from 'better-sqlite3';
+import { Cursor, DataFinality } from '@apibara/protocol';
+
+type SerializeFn = <T>(value: T) => string;
+type DeserializeFn = <T>(value: string) => T;
+
+declare class KeyValueStore {
+    private readonly db;
+    private readonly endCursor;
+    private readonly finality;
+    private readonly serialize;
+    private readonly deserialize;
+    constructor(db: Database, endCursor: Cursor, finality: DataFinality, serialize: SerializeFn, deserialize: DeserializeFn);
+    get<T>(key: string): T | undefined;
+    put<T>(key: string, value: T): void;
+    del(key: string): void;
+}
+
+declare function useSqliteKeyValueStore(): KeyValueStore;
+type SqliteStorageOptions = {
+    database: Database;
+    keyValueStore?: boolean;
+    persistState?: boolean;
+    serialize?: SerializeFn;
+    deserialize?: DeserializeFn;
+};
+/**
+ * Creates a plugin that uses SQLite as the storage layer.
+ *
+ * Supports storing the indexer's state and provides a simple Key-Value store.
+ * @param options.database - The SQLite database instance.
+ * @param options.persistState - Whether to persist the indexer's state. Defaults to true.
+ * @param options.keyValueStore - Whether to enable the Key-Value store. Defaults to true.
+ * @param options.serialize - A function to serialize the value to the KV.
+ * @param options.deserialize - A function to deserialize the value from the KV.
+ */
+declare function sqliteStorage<TFilter, TBlock>({ database, persistState: enablePersistState, keyValueStore: enableKeyValueStore, serialize: serializeFn, deserialize: deserializeFn, }: SqliteStorageOptions): _apibara_indexer_plugins.IndexerPlugin<TFilter, TBlock>;
+
+export { KeyValueStore, type SqliteStorageOptions, sqliteStorage, useSqliteKeyValueStore };

package/dist/index.mjs

@@ -0,0 +1,270 @@
+import { useIndexerContext } from '@apibara/indexer';
+import { defineIndexerPlugin } from '@apibara/indexer/plugins';
+import { isCursor } from '@apibara/protocol';
+
+class SqliteStorageError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "SqliteStorageError";
+  }
+}
+async function withTransaction(db, cb) {
+  db.prepare("BEGIN TRANSACTION").run();
+  try {
+    await cb(db);
+  } catch (error) {
+    db.prepare("ROLLBACK TRANSACTION").run();
+    throw error;
+  }
+  db.prepare("COMMIT TRANSACTION").run();
+}
+function assertInTransaction(db) {
+  if (!db.inTransaction) {
+    throw new SqliteStorageError("Database is not in transaction");
+  }
+}
+function deserialize(str) {
+  return JSON.parse(
+    str,
+    (_, value) => typeof value === "string" && value.match(/^\d+n$/) ? BigInt(value.slice(0, -1)) : value
+  );
+}
+function serialize(obj) {
+  return JSON.stringify(
+    obj,
+    (_, value) => typeof value === "bigint" ? `${value.toString()}n` : value,
+    "	"
+  );
+}
+
+function initializeKeyValueStore(db) {
+  assertInTransaction(db);
+  db.exec(statements$1.createTable);
+}
+class KeyValueStore {
+  constructor(db, endCursor, finality, serialize, deserialize) {
+    this.db = db;
+    this.endCursor = endCursor;
+    this.finality = finality;
+    this.serialize = serialize;
+    this.deserialize = deserialize;
+    assertInTransaction(db);
+  }
+  get(key) {
+    const row = this.db.prepare(statements$1.get).get(key);
+    return row ? this.deserialize(row.v) : void 0;
+  }
+  put(key, value) {
+    this.db.prepare(statements$1.updateToBlock).run(Number(this.endCursor.orderKey), key);
+    this.db.prepare(statements$1.insertIntoKvs).run(
+      Number(this.endCursor.orderKey),
+      key,
+      this.serialize(value)
+    );
+  }
+  del(key) {
+    this.db.prepare(statements$1.del).run(Number(this.endCursor.orderKey), key);
+  }
+}
+const statements$1 = {
+  createTable: `
+    CREATE TABLE IF NOT EXISTS kvs (
+      from_block INTEGER NOT NULL,
+      to_block INTEGER,
+      k TEXT NOT NULL,
+      v BLOB NOT NULL,
+      PRIMARY KEY (from_block, k)
+      );`,
+  get: `
+    SELECT v
+    FROM kvs
+    WHERE k = ? AND to_block IS NULL`,
+  updateToBlock: `
+    UPDATE kvs
+    SET to_block = ?
+    WHERE k = ? AND to_block IS NULL`,
+  insertIntoKvs: `
+    INSERT INTO kvs (from_block, to_block, k, v)
+    VALUES (?, NULL, ?, ?)`,
+  del: `
+    UPDATE kvs
+    SET to_block = ?
+    WHERE k = ? AND to_block IS NULL`
+};
+
+const DEFAULT_INDEXER_ID = "default";
+function initializePersistentState(db) {
+  assertInTransaction(db);
+  db.exec(statements.createCheckpointsTable);
+  db.exec(statements.createFiltersTable);
+}
+function persistState(db, endCursor, filter) {
+  assertInTransaction(db);
+  db.prepare(statements.putCheckpoint).run(
+    DEFAULT_INDEXER_ID,
+    Number(endCursor.orderKey),
+    endCursor.uniqueKey
+  );
+  if (filter) {
+    db.prepare(statements.updateFilterToBlock).run(
+      Number(endCursor.orderKey),
+      DEFAULT_INDEXER_ID
+    );
+    db.prepare(statements.insertFilter).run(
+      DEFAULT_INDEXER_ID,
+      serialize(filter),
+      Number(endCursor.orderKey)
+    );
+  }
+}
+function getState(db) {
+  assertInTransaction(db);
+  const storedCursor = db.prepare(
+    statements.getCheckpoint
+  ).get(DEFAULT_INDEXER_ID);
+  const storedFilter = db.prepare(statements.getFilter).get(DEFAULT_INDEXER_ID);
+  let cursor;
+  let filter;
+  if (storedCursor?.order_key) {
+    cursor = {
+      orderKey: BigInt(storedCursor.order_key),
+      uniqueKey: storedCursor.unique_key
+    };
+  }
+  if (storedFilter) {
+    filter = deserialize(storedFilter.filter);
+  }
+  return { cursor, filter };
+}
+const statements = {
+  createCheckpointsTable: `
+    CREATE TABLE IF NOT EXISTS checkpoints (
+      id TEXT NOT NULL PRIMARY KEY,
+      order_key INTEGER,
+      unique_key TEXT
+    );`,
+  createFiltersTable: `
+    CREATE TABLE IF NOT EXISTS filters (
+      id TEXT NOT NULL,
+      filter BLOB NOT NULL,
+      from_block INTEGER NOT NULL,
+      to_block INTEGER,
+      PRIMARY KEY (id, from_block)
+    );`,
+  getCheckpoint: `
+    SELECT *
+    FROM checkpoints
+    WHERE id = ?`,
+  putCheckpoint: `
+    INSERT INTO checkpoints (id, order_key, unique_key)
+    VALUES (?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      order_key = excluded.order_key,
+      unique_key = excluded.unique_key`,
+  delCheckpoint: `
+    DELETE FROM checkpoints
+    WHERE id = ?`,
+  getFilter: `
+    SELECT *
+    FROM filters
+    WHERE id = ? AND to_block IS NULL`,
+  updateFilterToBlock: `
+    UPDATE filters
+    SET to_block = ?
+    WHERE id = ? AND to_block IS NULL`,
+  insertFilter: `
+    INSERT INTO filters (id, filter, from_block)
+    VALUES (?, ?, ?)
+    ON CONFLICT(id, from_block) DO UPDATE SET
+      filter = excluded.filter,
+      from_block = excluded.from_block`,
+  delFilter: `
+    DELETE FROM filters
+    WHERE id = ?`
+};
+
+const KV_PROPERTY = "_kv_sqlite";
+function useSqliteKeyValueStore() {
+  const kv = useIndexerContext()[KV_PROPERTY];
+  if (!kv) {
+    throw new SqliteStorageError(
+      "SQLite key-value store is not available. Did you forget to enable it?"
+    );
+  }
+  return kv;
+}
+function sqliteStorage({
+  database,
+  persistState: enablePersistState = true,
+  keyValueStore: enableKeyValueStore = true,
+  serialize: serializeFn = serialize,
+  deserialize: deserializeFn = deserialize
+}) {
+  return defineIndexerPlugin((indexer) => {
+    indexer.hooks.hook("run:before", async () => {
+      await withTransaction(database, async (db) => {
+        if (enablePersistState) {
+          initializePersistentState(db);
+        }
+        if (enableKeyValueStore) {
+          initializeKeyValueStore(db);
+        }
+      });
+    });
+    indexer.hooks.hook("connect:before", async ({ request }) => {
+      if (!enablePersistState) {
+        return;
+      }
+      return await withTransaction(database, async (db) => {
+        const { cursor, filter } = getState(db);
+        if (cursor) {
+          request.startingCursor = cursor;
+        }
+        if (filter) {
+          request.filter[1] = filter;
+        }
+      });
+    });
+    indexer.hooks.hook("handler:middleware", ({ use }) => {
+      use(async (ctx, next) => {
+        if (!ctx.finality) {
+          throw new SqliteStorageError("finality is undefined");
+        }
+        if (!ctx.endCursor || !isCursor(ctx.endCursor)) {
+          throw new SqliteStorageError(
+            "endCursor is undefined or not a cursor"
+          );
+        }
+        await withTransaction(database, async (db) => {
+          if (enableKeyValueStore) {
+            ctx[KV_PROPERTY] = new KeyValueStore(
+              db,
+              ctx.endCursor,
+              ctx.finality,
+              serializeFn,
+              deserializeFn
+            );
+          }
+          await next();
+          if (enablePersistState) {
+            persistState(db, ctx.endCursor);
+          }
+          if (enableKeyValueStore) {
+            delete ctx[KV_PROPERTY];
+          }
+        });
+      });
+    });
+    indexer.hooks.hook("connect:factory", ({ request, endCursor }) => {
+      if (!enablePersistState) {
+        return;
+      }
+      assertInTransaction(database);
+      if (endCursor && request.filter[1]) {
+        persistState(database, endCursor, request.filter[1]);
+      }
+    });
+  });
+}
+
+export { KeyValueStore, sqliteStorage, useSqliteKeyValueStore };

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@apibara/plugin-sqlite",
+  "version": "2.0.0-beta.27",
+  "type": "module",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "scripts": {
+    "build": "unbuild",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check .",
+    "lint:fix": "pnpm lint --write",
+    "test": "vitest",
+    "test:ci": "vitest run"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.11",
+    "@types/node": "^20.14.0",
+    "unbuild": "^2.0.0",
+    "vitest": "^1.6.0"
+  },
+  "peerDependencies": {
+    "better-sqlite3": "^9.0.0"
+  },
+  "dependencies": {
+    "@apibara/indexer": "2.0.0-beta.28",
+    "@apibara/protocol": "2.0.0-beta.28"
+  }
+}

package/src/index.ts

@@ -0,0 +1,144 @@
+import { useIndexerContext } from "@apibara/indexer";
+import { defineIndexerPlugin } from "@apibara/indexer/plugins";
+import { isCursor } from "@apibara/protocol";
+import type { Database as SqliteDatabase } from "better-sqlite3";
+
+import { KeyValueStore, initializeKeyValueStore } from "./kv";
+import {
+  getState,
+  initializePersistentState,
+  persistState,
+} from "./persistence";
+import {
+  type DeserializeFn,
+  type SerializeFn,
+  SqliteStorageError,
+  assertInTransaction,
+  deserialize,
+  serialize,
+  withTransaction,
+} from "./utils";
+
+const KV_PROPERTY = "_kv_sqlite" as const;
+
+export { KeyValueStore } from "./kv";
+
+export function useSqliteKeyValueStore(): KeyValueStore {
+  const kv = useIndexerContext()[KV_PROPERTY] as KeyValueStore | undefined;
+  if (!kv) {
+    throw new SqliteStorageError(
+      "SQLite key-value store is not available. Did you forget to enable it?",
+    );
+  }
+
+  return kv;
+}
+
+export type SqliteStorageOptions = {
+  database: SqliteDatabase;
+  keyValueStore?: boolean;
+  persistState?: boolean;
+
+  serialize?: SerializeFn;
+  deserialize?: DeserializeFn;
+};
+
+/**
+ * Creates a plugin that uses SQLite as the storage layer.
+ *
+ * Supports storing the indexer's state and provides a simple Key-Value store.
+ * @param options.database - The SQLite database instance.
+ * @param options.persistState - Whether to persist the indexer's state. Defaults to true.
+ * @param options.keyValueStore - Whether to enable the Key-Value store. Defaults to true.
+ * @param options.serialize - A function to serialize the value to the KV.
+ * @param options.deserialize - A function to deserialize the value from the KV.
+ */
+export function sqliteStorage<TFilter, TBlock>({
+  database,
+  persistState: enablePersistState = true,
+  keyValueStore: enableKeyValueStore = true,
+  serialize: serializeFn = serialize,
+  deserialize: deserializeFn = deserialize,
+}: SqliteStorageOptions) {
+  return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
+    indexer.hooks.hook("run:before", async () => {
+      await withTransaction(database, async (db) => {
+        if (enablePersistState) {
+          initializePersistentState(db);
+        }
+
+        if (enableKeyValueStore) {
+          initializeKeyValueStore(db);
+        }
+      });
+    });
+
+    indexer.hooks.hook("connect:before", async ({ request }) => {
+      if (!enablePersistState) {
+        return;
+      }
+
+      return await withTransaction(database, async (db) => {
+        const { cursor, filter } = getState<TFilter>(db);
+
+        if (cursor) {
+          request.startingCursor = cursor;
+        }
+
+        if (filter) {
+          request.filter[1] = filter;
+        }
+      });
+    });
+
+    indexer.hooks.hook("handler:middleware", ({ use }) => {
+      use(async (ctx, next) => {
+        if (!ctx.finality) {
+          throw new SqliteStorageError("finality is undefined");
+        }
+
+        if (!ctx.endCursor || !isCursor(ctx.endCursor)) {
+          throw new SqliteStorageError(
+            "endCursor is undefined or not a cursor",
+          );
+        }
+
+        await withTransaction(database, async (db) => {
+          if (enableKeyValueStore) {
+            ctx[KV_PROPERTY] = new KeyValueStore(
+              db,
+              ctx.endCursor,
+              ctx.finality,
+              serializeFn,
+              deserializeFn,
+            );
+          }
+
+          await next();
+
+          if (enablePersistState) {
+            persistState(db, ctx.endCursor);
+          }
+
+          if (enableKeyValueStore) {
+            delete ctx[KV_PROPERTY];
+          }
+        });
+      });
+    });
+
+    indexer.hooks.hook("connect:factory", ({ request, endCursor }) => {
+      if (!enablePersistState) {
+        return;
+      }
+
+      // The connect factory hook is called while indexing a block, so the database should be in a transaction
+      // created by the middleware.
+      assertInTransaction(database);
+
+      if (endCursor && request.filter[1]) {
+        persistState(database, endCursor, request.filter[1]);
+      }
+    });
+  });
+}

package/src/kv.ts

@@ -0,0 +1,84 @@
+import type { Cursor, DataFinality } from "@apibara/protocol";
+import type { Database } from "better-sqlite3";
+
+import {
+  type DeserializeFn,
+  type SerializeFn,
+  assertInTransaction,
+} from "./utils";
+
+export function initializeKeyValueStore(db: Database) {
+  assertInTransaction(db);
+  db.exec(statements.createTable);
+}
+
+export class KeyValueStore {
+  constructor(
+    private readonly db: Database,
+    private readonly endCursor: Cursor,
+    private readonly finality: DataFinality,
+    private readonly serialize: SerializeFn,
+    private readonly deserialize: DeserializeFn,
+  ) {
+    assertInTransaction(db);
+  }
+
+  get<T>(key: string): T | undefined {
+    const row = this.db.prepare<string, KeyValueRow>(statements.get).get(key);
+
+    return row ? this.deserialize(row.v) : undefined;
+  }
+
+  put<T>(key: string, value: T) {
+    this.db
+      .prepare<[number, string], KeyValueRow>(statements.updateToBlock)
+      .run(Number(this.endCursor.orderKey), key);
+
+    this.db
+      .prepare<[number, string, string], KeyValueRow>(statements.insertIntoKvs)
+      .run(
+        Number(this.endCursor.orderKey),
+        key,
+        this.serialize(value as Record<string, unknown>),
+      );
+  }
+
+  del(key: string) {
+    this.db
+      .prepare<[number, string], KeyValueRow>(statements.del)
+      .run(Number(this.endCursor.orderKey), key);
+  }
+}
+
+type KeyValueRow = {
+  from_block: number;
+  to_block: number;
+  k: string;
+  v: string;
+};
+
+const statements = {
+  createTable: `
+    CREATE TABLE IF NOT EXISTS kvs (
+      from_block INTEGER NOT NULL,
+      to_block INTEGER,
+      k TEXT NOT NULL,
+      v BLOB NOT NULL,
+      PRIMARY KEY (from_block, k)
+      );`,
+  get: `
+    SELECT v
+    FROM kvs
+    WHERE k = ? AND to_block IS NULL`,
+  updateToBlock: `
+    UPDATE kvs
+    SET to_block = ?
+    WHERE k = ? AND to_block IS NULL`,
+  insertIntoKvs: `
+    INSERT INTO kvs (from_block, to_block, k, v)
+    VALUES (?, NULL, ?, ?)`,
+  del: `
+    UPDATE kvs
+    SET to_block = ?
+    WHERE k = ? AND to_block IS NULL`,
+};

package/src/persistence.ts

@@ -0,0 +1,113 @@
+import type { Cursor } from "@apibara/protocol";
+import type { Database } from "better-sqlite3";
+
+import { assertInTransaction, deserialize, serialize } from "./utils";
+
+const DEFAULT_INDEXER_ID = "default";
+
+export function initializePersistentState(db: Database) {
+  assertInTransaction(db);
+  db.exec(statements.createCheckpointsTable);
+  db.exec(statements.createFiltersTable);
+}
+
+export function persistState<TFilter>(
+  db: Database,
+  endCursor: Cursor,
+  filter?: TFilter,
+) {
+  assertInTransaction(db);
+
+  db.prepare(statements.putCheckpoint).run(
+    DEFAULT_INDEXER_ID,
+    Number(endCursor.orderKey),
+    endCursor.uniqueKey,
+  );
+
+  if (filter) {
+    db.prepare(statements.updateFilterToBlock).run(
+      Number(endCursor.orderKey),
+      DEFAULT_INDEXER_ID,
+    );
+    db.prepare(statements.insertFilter).run(
+      DEFAULT_INDEXER_ID,
+      serialize(filter as Record<string, unknown>),
+      Number(endCursor.orderKey),
+    );
+  }
+}
+
+export function getState<TFilter>(db: Database) {
+  assertInTransaction(db);
+  const storedCursor = db
+    .prepare<string, { order_key?: number; unique_key?: string }>(
+      statements.getCheckpoint,
+    )
+    .get(DEFAULT_INDEXER_ID);
+  const storedFilter = db
+    .prepare<string, { filter: string }>(statements.getFilter)
+    .get(DEFAULT_INDEXER_ID);
+
+  let cursor: Cursor | undefined;
+  let filter: TFilter | undefined;
+
+  if (storedCursor?.order_key) {
+    cursor = {
+      orderKey: BigInt(storedCursor.order_key),
+      uniqueKey: storedCursor.unique_key as `0x${string}`,
+    };
+  }
+
+  if (storedFilter) {
+    filter = deserialize(storedFilter.filter) as TFilter;
+  }
+
+  return { cursor, filter };
+}
+
+const statements = {
+  createCheckpointsTable: `
+    CREATE TABLE IF NOT EXISTS checkpoints (
+      id TEXT NOT NULL PRIMARY KEY,
+      order_key INTEGER,
+      unique_key TEXT
+    );`,
+  createFiltersTable: `
+    CREATE TABLE IF NOT EXISTS filters (
+      id TEXT NOT NULL,
+      filter BLOB NOT NULL,
+      from_block INTEGER NOT NULL,
+      to_block INTEGER,
+      PRIMARY KEY (id, from_block)
+    );`,
+  getCheckpoint: `
+    SELECT *
+    FROM checkpoints
+    WHERE id = ?`,
+  putCheckpoint: `
+    INSERT INTO checkpoints (id, order_key, unique_key)
+    VALUES (?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      order_key = excluded.order_key,
+      unique_key = excluded.unique_key`,
+  delCheckpoint: `
+    DELETE FROM checkpoints
+    WHERE id = ?`,
+  getFilter: `
+    SELECT *
+    FROM filters
+    WHERE id = ? AND to_block IS NULL`,
+  updateFilterToBlock: `
+    UPDATE filters
+    SET to_block = ?
+    WHERE id = ? AND to_block IS NULL`,
+  insertFilter: `
+    INSERT INTO filters (id, filter, from_block)
+    VALUES (?, ?, ?)
+    ON CONFLICT(id, from_block) DO UPDATE SET
+      filter = excluded.filter,
+      from_block = excluded.from_block`,
+  delFilter: `
+    DELETE FROM filters
+    WHERE id = ?`,
+};

package/src/utils.ts

@@ -0,0 +1,47 @@
+import type { Database } from "better-sqlite3";
+
+export type SerializeFn = <T>(value: T) => string;
+export type DeserializeFn = <T>(value: string) => T;
+
+export class SqliteStorageError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "SqliteStorageError";
+  }
+}
+
+export async function withTransaction(
+  db: Database,
+  cb: (db: Database) => Promise<void>,
+) {
+  db.prepare("BEGIN TRANSACTION").run();
+  try {
+    await cb(db);
+  } catch (error) {
+    db.prepare("ROLLBACK TRANSACTION").run();
+    throw error;
+  }
+  db.prepare("COMMIT TRANSACTION").run();
+}
+
+export function assertInTransaction(db: Database) {
+  if (!db.inTransaction) {
+    throw new SqliteStorageError("Database is not in transaction");
+  }
+}
+
+export function deserialize<T>(str: string): T {
+  return JSON.parse(str, (_, value) =>
+    typeof value === "string" && value.match(/^\d+n$/)
+      ? BigInt(value.slice(0, -1))
+      : value,
+  ) as T;
+}
+
+export function serialize<T>(obj: T): string {
+  return JSON.stringify(
+    obj,
+    (_, value) => (typeof value === "bigint" ? `${value.toString()}n` : value),
+    "\t",
+  );
+}