@apibara/indexer 2.0.0-beta.3 → 2.0.0-beta.30

package/src/plugins/logger.ts

@@ -0,0 +1,30 @@
+import { type ConsolaInstance, type ConsolaReporter, consola } from "consola";
+import { useIndexerContext } from "../context";
+import { defineIndexerPlugin } from "./config";
+
+export type { ConsolaReporter, ConsolaInstance } from "consola";
+
+export function logger<TFilter, TBlock, TTxnParams>({
+  logger,
+}: { logger?: ConsolaReporter } = {}) {
+  return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
+    indexer.hooks.hook("run:before", () => {
+      const ctx = useIndexerContext();
+
+      if (logger) {
+        ctx.logger = consola.create({ reporters: [logger] });
+      } else {
+        ctx.logger = consola.create({});
+      }
+    });
+  });
+}
+
+export function useLogger(): ConsolaInstance {
+  const ctx = useIndexerContext();
+
+  if (!ctx?.logger)
+    throw new Error("Logger plugin is not available in context");
+
+  return ctx.logger;
+}

package/src/plugins/persistence.ts

@@ -1,202 +1,39 @@
-import type { Cursor } from "@apibara/protocol";
-import type { Database as SqliteDatabase, Statement } from "better-sqlite3";
-import { deserialize, serialize } from "../vcr";
-import { defineIndexerPlugin } from "./config";
+import { type Cursor, isCursor } from "@apibara/protocol";
 
-export function sqlitePersistence<TFilter, TBlock, TTxnParams>({
-  database,
-}: { database: SqliteDatabase }) {
-  return defineIndexerPlugin<TFilter, TBlock, TTxnParams>((indexer) => {
-    let store: SqlitePersistence<TFilter>;
+import { defineIndexerPlugin } from "./config";
 
-    indexer.hooks.hook("run:before", () => {
-      SqlitePersistence.initialize(database);
-
-      store = new SqlitePersistence(database);
-    });
+/**
+ * A plugin that persists the last cursor and filter to memory.
+ */
+export function inMemoryPersistence<TFilter, TBlock>() {
+  return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
+    let lastCursor: Cursor | undefined;
+    let lastFilter: TFilter | undefined;
 
     indexer.hooks.hook("connect:before", ({ request }) => {
-      const { cursor, filter } = store.get();
-
-      if (cursor) {
-        request.startingCursor = cursor;
+      if (lastCursor) {
+        request.startingCursor = lastCursor;
       }
 
-      if (filter) {
-        request.filter[1] = filter;
-      }
-    });
-
-    indexer.hooks.hook("transaction:commit", ({ endCursor }) => {
-      if (endCursor) {
-        store.put({ cursor: endCursor });
+      if (lastFilter) {
+        request.filter[1] = lastFilter;
       }
     });
 
     indexer.hooks.hook("connect:factory", ({ request, endCursor }) => {
       if (request.filter[1]) {
-        store.put({ cursor: endCursor, filter: request.filter[1] });
+        lastCursor = endCursor;
+        lastFilter = request.filter[1];
       }
     });
-  });
-}
-
-export class SqlitePersistence<TFilter> {
-  /** Sqlite Queries Prepare Statements */
-  private _getCheckpointQuery: Statement<string, CheckpointRow>;
-  private _putCheckpointQuery: Statement<
-    [string, number, `0x${string}` | undefined]
-  >;
-  private _delCheckpointQuery: Statement<string>;
-  private _getFilterQuery: Statement<string, FilterRow>;
-  private _updateFilterToBlockQuery: Statement<[number, string]>;
-  private _insertFilterQuery: Statement<[string, string, number]>;
-  private _delFilterQuery: Statement<string>;
-
-  constructor(private _db: SqliteDatabase) {
-    this._getCheckpointQuery = this._db.prepare(statements.getCheckpoint);
-    this._putCheckpointQuery = this._db.prepare(statements.putCheckpoint);
-    this._delCheckpointQuery = this._db.prepare(statements.delCheckpoint);
-    this._getFilterQuery = this._db.prepare(statements.getFilter);
-    this._updateFilterToBlockQuery = this._db.prepare(
-      statements.updateFilterToBlock,
-    );
-    this._insertFilterQuery = this._db.prepare(statements.insertFilter);
-    this._delFilterQuery = this._db.prepare(statements.delFilter);
-  }
-
-  static initialize(db: SqliteDatabase) {
-    db.prepare(statements.createCheckpointsTable).run();
-    db.prepare(statements.createFiltersTable).run();
-  }
-
-  public get(): { cursor?: Cursor; filter?: TFilter } {
-    const cursor = this._getCheckpoint();
-    const filter = this._getFilter();
-
-    return { cursor, filter };
-  }
-
-  public put({ cursor, filter }: { cursor?: Cursor; filter?: TFilter }) {
-    if (cursor) {
-      this._putCheckpoint(cursor);
-
-      if (filter) {
-        this._putFilter(filter, cursor);
-      }
-    }
-  }
 
-  public del() {
-    this._delCheckpoint();
-    this._delFilter();
-  }
-
-  // --- CHECKPOINTS TABLE METHODS ---
-
-  private _getCheckpoint(): Cursor | undefined {
-    const row = this._getCheckpointQuery.get("default");
-
-    if (!row) return undefined;
-
-    return { orderKey: BigInt(row.order_key), uniqueKey: row.unique_key };
-  }
-
-  private _putCheckpoint(cursor: Cursor) {
-    this._putCheckpointQuery.run(
-      "default",
-      Number(cursor.orderKey),
-      cursor.uniqueKey,
-    );
-  }
-
-  private _delCheckpoint() {
-    this._delCheckpointQuery.run("default");
-  }
-
-  // --- FILTERS TABLE METHODS ---
-
-  private _getFilter(): TFilter | undefined {
-    const row = this._getFilterQuery.get("default");
-
-    if (!row) return undefined;
-
-    return deserialize(row.filter) as TFilter;
-  }
-
-  private _putFilter(filter: TFilter, endCursor: Cursor) {
-    this._updateFilterToBlockQuery.run(Number(endCursor.orderKey), "default");
-    this._insertFilterQuery.run(
-      "default",
-      serialize(filter as Record<string, unknown>),
-      Number(endCursor.orderKey),
-    );
-  }
-
-  private _delFilter() {
-    this._delFilterQuery.run("default");
-  }
+    indexer.hooks.hook("handler:middleware", ({ use }) => {
+      use(async (context, next) => {
+        await next();
+        if (context.endCursor && isCursor(context.endCursor)) {
+          lastCursor = context.endCursor;
+        }
+      });
+    });
+  });
 }
-
-const statements = {
-  beginTxn: "BEGIN TRANSACTION",
-  commitTxn: "COMMIT TRANSACTION",
-  rollbackTxn: "ROLLBACK TRANSACTION",
-  createCheckpointsTable: `
-    CREATE TABLE IF NOT EXISTS checkpoints (
-      id TEXT NOT NULL PRIMARY KEY,
-      order_key INTEGER NOT NULL,
-      unique_key TEXT
-    );`,
-  createFiltersTable: `
-    CREATE TABLE IF NOT EXISTS filters (
-      id TEXT NOT NULL,
-      filter BLOB NOT NULL,
-      from_block INTEGER NOT NULL,
-      to_block INTEGER,
-      PRIMARY KEY (id, from_block)
-    );`,
-  getCheckpoint: `
-    SELECT *
-    FROM checkpoints
-    WHERE id = ?`,
-  putCheckpoint: `
-    INSERT INTO checkpoints (id, order_key, unique_key)
-    VALUES (?, ?, ?)
-    ON CONFLICT(id) DO UPDATE SET
-      order_key = excluded.order_key,
-      unique_key = excluded.unique_key`,
-  delCheckpoint: `
-    DELETE FROM checkpoints
-    WHERE id = ?`,
-  getFilter: `
-    SELECT *
-    FROM filters
-    WHERE id = ? AND to_block IS NULL`,
-  updateFilterToBlock: `
-    UPDATE filters
-    SET to_block = ?
-    WHERE id = ? AND to_block IS NULL`,
-  insertFilter: `
-    INSERT INTO filters (id, filter, from_block)
-    VALUES (?, ?, ?)
-    ON CONFLICT(id, from_block) DO UPDATE SET
-      filter = excluded.filter,
-      from_block = excluded.from_block`,
-  delFilter: `
-    DELETE FROM filters
-    WHERE id = ?`,
-};
-
-export type CheckpointRow = {
-  id: string;
-  order_key: number;
-  unique_key?: `0x${string}`;
-};
-
-export type FilterRow = {
-  id: string;
-  filter: string;
-  from_block: number;
-  to_block?: number;
-};

package/src/testing/index.ts

@@ -1,3 +1,50 @@
-export * from "./setup";
-export * from "./vcr";
-export * from "./helper";
+import { createClient } from "@apibara/protocol";
+import ci from "ci-info";
+import { type IndexerWithStreamConfig, createIndexer } from "../indexer";
+import { logger } from "../plugins/logger";
+import type { CassetteOptions, VcrConfig } from "../vcr/config";
+import { isCassetteAvailable } from "../vcr/helper";
+import { record } from "../vcr/record";
+import { replay } from "../vcr/replay";
+
+export function createVcr() {
+  return {
+    async run<TFilter, TBlock>(
+      cassetteName: string,
+      indexerConfig: IndexerWithStreamConfig<TFilter, TBlock>,
+      range: { fromBlock: bigint; toBlock: bigint },
+    ) {
+      const vcrConfig: VcrConfig = {
+        cassetteDir: "cassettes",
+      };
+
+      const cassetteOptions: CassetteOptions = {
+        name: cassetteName,
+        startingCursor: {
+          orderKey: range.fromBlock,
+        },
+        endingCursor: {
+          orderKey: range.toBlock,
+        },
+      };
+
+      indexerConfig.plugins = [logger(), ...(indexerConfig.plugins ?? [])];
+
+      const indexer = createIndexer(indexerConfig);
+
+      if (!isCassetteAvailable(vcrConfig, cassetteName)) {
+        if (ci.isCI) {
+          throw new Error("Cannot record cassette in CI");
+        }
+
+        const client = createClient(
+          indexer.streamConfig,
+          indexer.options.streamUrl,
+        );
+        await record(vcrConfig, client, indexer, cassetteOptions);
+      } else {
+        await replay(vcrConfig, indexer, cassetteName);
+      }
+    },
+  };
+}

package/src/vcr/record.ts

@@ -1,8 +1,7 @@
 import fs from "node:fs/promises";
 import path from "node:path";
-import { type Indexer, run } from "@apibara/indexer";
 import type { Client, StreamDataResponse } from "@apibara/protocol";
-import { klona } from "klona/full";
+import { type Indexer, run } from "../indexer";
 import type { CassetteOptions, VcrConfig } from "./config";
 import { serialize } from "./helper";
 
@@ -14,10 +13,9 @@ export type CassetteDataType<TFilter, TBlock> = {
 export async function record<TFilter, TBlock, TTxnParams>(
   vcrConfig: VcrConfig,
   client: Client<TFilter, TBlock>,
-  indexerArg: Indexer<TFilter, TBlock, TTxnParams>,
+  indexer: Indexer<TFilter, TBlock>,
   cassetteOptions: CassetteOptions,
 ) {
-  const indexer = klona(indexerArg);
   const messages: StreamDataResponse<TBlock>[] = [];
 
   indexer.hooks.addHooks({
@@ -33,10 +31,14 @@ export async function record<TFilter, TBlock, TTxnParams>(
         filter: indexer.options.filter,
         messages: messages,
       };
+
+      await fs.mkdir(vcrConfig.cassetteDir, { recursive: true });
+
       const filePath = path.join(
         vcrConfig.cassetteDir,
         `${cassetteOptions.name}.json`,
       );
+
       await fs.writeFile(filePath, serialize(output), { flag: "w" });
     },
   });

package/src/vcr/replay.ts

@@ -1,34 +1,21 @@
 import assert from "node:assert";
 import fs from "node:fs";
 import path from "node:path";
-import type { Client, Cursor } from "@apibara/protocol";
+import type { Client } from "@apibara/protocol";
 import { MockClient } from "@apibara/protocol/testing";
 import { type Indexer, run } from "../indexer";
-import type { SinkData } from "../sink";
-import { vcr } from "../testing/vcr";
 import { type CassetteDataType, deserialize } from "../vcr";
 import type { VcrConfig } from "./config";
 
 export async function replay<TFilter, TBlock, TTxnParams>(
   vcrConfig: VcrConfig,
-  indexer: Indexer<TFilter, TBlock, TTxnParams>,
+  indexer: Indexer<TFilter, TBlock>,
   cassetteName: string,
-): Promise<VcrReplayResult> {
+) {
   const client = loadCassette<TFilter, TBlock>(vcrConfig, cassetteName);
-
-  const sink = vcr();
-
   await run(client, indexer);
-
-  return {
-    outputs: sink.result,
-  };
 }
 
-export type VcrReplayResult = {
-  outputs: Array<{ endCursor?: Cursor; data: SinkData[] }>;
-};
-
 export function loadCassette<TFilter, TBlock>(
   vcrConfig: VcrConfig,
   cassetteName: string,
@@ -41,11 +28,14 @@ export function loadCassette<TFilter, TBlock>(
   const { filter, messages } = cassetteData;
 
   return new MockClient<TFilter, TBlock>((request, options) => {
+    // Notice that the request filter is an array of filters,
+    // so we need to wrap the indexer filter in an array.
     assert.deepStrictEqual(
       request.filter,
-      filter,
-      "Request and Cassette filter mismatch",
+      [filter],
+      "Indexer and cassette filter mismatch. Hint: delete the cassette and run again.",
     );
+
     return messages;
   });
 }

package/src/hooks/index.ts

@@ -1,2 +0,0 @@
-export * from "./useKVStore";
-export * from "./useSink";

package/src/hooks/useKVStore.ts

@@ -1,12 +0,0 @@
-import { useIndexerContext } from "../context";
-import type { KVStore } from "../plugins/kv";
-
-export type UseKVStoreResult = InstanceType<typeof KVStore>;
-
-export function useKVStore(): UseKVStoreResult {
-  const ctx = useIndexerContext();
-
-  if (!ctx?.kv) throw new Error("KV Plugin is not available in context!");
-
-  return ctx.kv;
-}

package/src/hooks/useSink.ts

@@ -1,13 +0,0 @@
-import type { IndexerContext } from "../context";
-
-export function useSink<TTxnParams>({
-  context,
-}: {
-  context: IndexerContext<TTxnParams>;
-}) {
-  if (!context.sinkTransaction) {
-    throw new Error("Transaction context doesn't exist!");
-  }
-
-  return context.sinkTransaction;
-}

package/src/plugins/kv.test.ts

@@ -1,120 +0,0 @@
-import Database, { type Database as SqliteDatabase } from "better-sqlite3";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { KVStore } from "./kv";
-
-type ValueType = { data: bigint };
-
-type DatabaseRowType = {
-  from_block: number;
-  k: string;
-  to_block: number;
-  v: unknown;
-};
-
-describe("KVStore", () => {
-  let db: SqliteDatabase;
-  let store: KVStore;
-  const key = "test_key";
-
-  beforeAll(() => {
-    db = new Database(":memory:");
-    KVStore.initialize(db);
-    store = new KVStore(db, "finalized", { orderKey: 5_000_000n });
-  });
-
-  afterAll(async () => {
-    db.close();
-  });
-
-  it("should begin transaction", () => {
-    store.beginTransaction();
-  });
-
-  it("should put and get a value", async () => {
-    const value = { data: 0n };
-
-    store.put<ValueType>(key, value);
-    const result = store.get<ValueType>(key);
-
-    expect(result).toEqual(value);
-  });
-
-  it("should commit transaction", async () => {
-    store.commitTransaction();
-
-    const value = { data: 0n };
-
-    const result = store.get<ValueType>(key);
-
-    expect(result).toEqual(value);
-  });
-
-  it("should return undefined for non-existing key", async () => {
-    const result = store.get<ValueType>("non_existent_key");
-    expect(result).toBeUndefined();
-  });
-
-  it("should begin transaction", async () => {
-    store.beginTransaction();
-  });
-
-  it("should update an existing value", async () => {
-    store = new KVStore(db, "finalized", { orderKey: 5_000_020n });
-
-    const value = { data: 50n };
-
-    store.put<ValueType>(key, value);
-    const result = store.get<ValueType>(key);
-
-    expect(result).toEqual(value);
-  });
-
-  it("should delete a value", async () => {
-    store.del(key);
-    const result = store.get<ValueType>(key);
-
-    expect(result).toBeUndefined();
-
-    const rows = db
-      .prepare<string, DatabaseRowType>(
-        `
-      SELECT from_block, to_block, k, v
-      FROM kvs
-      WHERE k = ?
-    `,
-      )
-      .all(key);
-
-    // Check that the old is correctly marked with to_block
-    expect(rows[0].to_block).toBe(Number(5_000_020n));
-  });
-
-  it("should rollback transaction", async () => {
-    await store.rollbackTransaction();
-  });
-
-  it("should revert the changes to last commit", async () => {
-    const rows = db
-      .prepare(
-        `
-      SELECT from_block, to_block, k, v
-      FROM kvs
-      WHERE k = ?
-    `,
-      )
-      .all([key]);
-
-    expect(rows).toMatchInlineSnapshot(`
-      [
-        {
-          "from_block": 5000000,
-          "k": "test_key",
-          "to_block": null,
-          "v": "{
-      	"data": "0n"
-      }",
-        },
-      ]
-    `);
-  });
-});

package/src/plugins/kv.ts

@@ -1,132 +0,0 @@
-import assert from "node:assert";
-import type { Cursor, DataFinality } from "@apibara/protocol";
-import type { Database as SqliteDatabase, Statement } from "better-sqlite3";
-import { useIndexerContext } from "../context";
-import { deserialize, serialize } from "../vcr";
-import { defineIndexerPlugin } from "./config";
-
-export function kv<TFilter, TBlock, TTxnParams>({
-  database,
-}: { database: SqliteDatabase }) {
-  return defineIndexerPlugin<TFilter, TBlock, TTxnParams>((indexer) => {
-    indexer.hooks.hook("run:before", () => {
-      KVStore.initialize(database);
-    });
-
-    indexer.hooks.hook("handler:before", ({ finality, endCursor }) => {
-      const ctx = useIndexerContext();
-
-      assert(endCursor, new Error("endCursor cannot be undefined"));
-
-      ctx.kv = new KVStore(database, finality, endCursor);
-
-      ctx.kv.beginTransaction();
-    });
-
-    indexer.hooks.hook("handler:after", () => {
-      const ctx = useIndexerContext();
-
-      ctx.kv.commitTransaction();
-
-      ctx.kv = null;
-    });
-
-    indexer.hooks.hook("handler:exception", () => {
-      const ctx = useIndexerContext();
-
-      ctx.kv.rollbackTransaction();
-
-      ctx.kv = null;
-    });
-  });
-}
-
-export class KVStore {
-  /** Sqlite Queries Prepare Statements */
-  private _beginTxnQuery: Statement;
-  private _commitTxnQuery: Statement;
-  private _rollbackTxnQuery: Statement;
-  private _getQuery: Statement<string, { v: string }>;
-  private _updateToBlockQuery: Statement<[number, string]>;
-  private _insertIntoKvsQuery: Statement<[number, string, string]>;
-  private _delQuery: Statement<[number, string]>;
-
-  constructor(
-    private _db: SqliteDatabase,
-    private _finality: DataFinality,
-    private _endCursor: Cursor,
-  ) {
-    this._beginTxnQuery = this._db.prepare(statements.beginTxn);
-    this._commitTxnQuery = this._db.prepare(statements.commitTxn);
-    this._rollbackTxnQuery = this._db.prepare(statements.rollbackTxn);
-    this._getQuery = this._db.prepare(statements.get);
-    this._updateToBlockQuery = this._db.prepare(statements.updateToBlock);
-    this._insertIntoKvsQuery = this._db.prepare(statements.insertIntoKvs);
-    this._delQuery = this._db.prepare(statements.del);
-  }
-
-  static initialize(db: SqliteDatabase) {
-    db.prepare(statements.createTable).run();
-  }
-
-  beginTransaction() {
-    this._beginTxnQuery.run();
-  }
-
-  commitTransaction() {
-    this._commitTxnQuery.run();
-  }
-
-  rollbackTransaction() {
-    this._rollbackTxnQuery.run();
-  }
-
-  get<T>(key: string): T {
-    const row = this._getQuery.get(key);
-
-    return row ? deserialize(row.v) : undefined;
-  }
-
-  put<T>(key: string, value: T) {
-    this._updateToBlockQuery.run(Number(this._endCursor.orderKey), key);
-
-    this._insertIntoKvsQuery.run(
-      Number(this._endCursor.orderKey),
-      key,
-      serialize(value as Record<string, unknown>),
-    );
-  }
-
-  del(key: string) {
-    this._delQuery.run(Number(this._endCursor.orderKey), key);
-  }
-}
-
-const statements = {
-  beginTxn: "BEGIN TRANSACTION",
-  commitTxn: "COMMIT TRANSACTION",
-  rollbackTxn: "ROLLBACK TRANSACTION",
-  createTable: `
-    CREATE TABLE IF NOT EXISTS kvs (
-      from_block INTEGER NOT NULL,
-      to_block INTEGER,
-      k TEXT NOT NULL,
-      v BLOB NOT NULL,
-      PRIMARY KEY (from_block, k)
-      );`,
-  get: `
-    SELECT v
-    FROM kvs
-    WHERE k = ? AND to_block IS NULL`,
-  updateToBlock: `
-    UPDATE kvs
-    SET to_block = ?
-    WHERE k = ? AND to_block IS NULL`,
-  insertIntoKvs: `
-    INSERT INTO kvs (from_block, to_block, k, v)
-    VALUES (?, NULL, ?, ?)`,
-  del: `
-    UPDATE kvs
-    SET to_block = ?
-    WHERE k = ? AND to_block IS NULL`,
-};