@apibara/indexer 2.0.0-beta.9 → 2.1.0-beta.2

package/src/internal/testing.ts

@@ -0,0 +1,148 @@
+import { type Finalize, type Invalidate, isCursor } from "@apibara/protocol";
+import {
+  type MockBlock,
+  type MockFilter,
+  MockStream,
+  type MockStreamResponse,
+} from "@apibara/protocol/testing";
+
+import { useIndexerContext } from "../context";
+import { type IndexerConfig, createIndexer, defineIndexer } from "../indexer";
+import { defineIndexerPlugin } from "../plugins";
+import { type InternalContext, internalContext } from "./plugins";
+
+export type MockMessagesOptions = {
+  invalidate?: {
+    invalidateFromIndex: number;
+    invalidateTriggerIndex: number;
+  };
+  finalize?: {
+    finalizeToIndex: number;
+    finalizeTriggerIndex: number;
+  };
+};
+
+export function generateMockMessages(
+  count = 10,
+  options?: MockMessagesOptions,
+): MockStreamResponse[] {
+  const invalidateAt = options?.invalidate;
+  const finalizeAt = options?.finalize;
+  const messages: MockStreamResponse[] = [];
+
+  for (let i = 0; i < count; i++) {
+    if (invalidateAt && i === invalidateAt.invalidateTriggerIndex) {
+      messages.push({
+        _tag: "invalidate",
+        invalidate: {
+          cursor: {
+            orderKey: BigInt(5_000_000 + invalidateAt.invalidateFromIndex),
+          },
+        },
+      } as Invalidate);
+    } else if (finalizeAt && i === finalizeAt.finalizeTriggerIndex) {
+      messages.push({
+        _tag: "finalize",
+        finalize: {
+          cursor: {
+            orderKey: BigInt(5_000_000 + finalizeAt.finalizeToIndex),
+          },
+        },
+      } as Finalize);
+    } else {
+      messages.push({
+        _tag: "data",
+        data: {
+          cursor: { orderKey: BigInt(5_000_000 + i - 1) },
+          finality: "accepted",
+          data: [{ data: `${5_000_000 + i}` }],
+          endCursor: { orderKey: BigInt(5_000_000 + i) },
+          production: "backfill",
+        },
+      });
+    }
+  }
+
+  return messages;
+}
+
+type MockIndexerParams = {
+  internalContext?: InternalContext;
+  override?: Partial<IndexerConfig<MockFilter, MockBlock>>;
+};
+
+export function getMockIndexer(params?: MockIndexerParams) {
+  const { internalContext: contextParams, override } = params ?? {};
+  const { plugins, ...rest } = override ?? {};
+
+  return createIndexer(
+    defineIndexer(MockStream)({
+      streamUrl: "https://sepolia.ethereum.a5a.ch",
+      finality: "accepted",
+      filter: {},
+      async transform() {},
+      plugins: [
+        internalContext(
+          contextParams ??
+            ({
+              availableIndexers: ["testing"],
+              indexerName: "testing",
+            } as InternalContext),
+        ),
+        ...(plugins ?? []),
+      ],
+      ...(rest ?? {}),
+    }),
+  );
+}
+
+export type MockRet = {
+  data: string;
+};
+
+/**
+ * A mock sink used for testing. The indexer function can write to the output array.
+ * The indexer context is optionally written to the metadata object.
+ */
+export function mockSink<TFilter, TBlock>({
+  output,
+  metadata,
+}: { output: unknown[]; metadata?: Record<string, unknown> }) {
+  return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
+    indexer.hooks.hook("connect:before", ({ request }) => {
+      if (metadata?.lastCursor && isCursor(metadata.lastCursor)) {
+        request.startingCursor = metadata.lastCursor;
+      }
+
+      if (metadata?.lastFilter) {
+        request.filter[1] = metadata.lastFilter as TFilter;
+      }
+    });
+
+    indexer.hooks.hook("connect:factory", ({ request, endCursor }) => {
+      if (request.filter[1]) {
+        if (metadata) {
+          metadata.lastCursor = endCursor;
+          metadata.lastFilter = request.filter[1];
+        }
+      }
+    });
+
+    indexer.hooks.hook("handler:middleware", ({ use }) => {
+      use(async (context, next) => {
+        context.output = output;
+        await next();
+        context.output = null;
+
+        if (metadata) {
+          metadata.lastCursor = context.endCursor;
+        }
+      });
+    });
+  });
+}
+
+export function useMockSink(): { output: unknown[] } {
+  const context = useIndexerContext();
+  return { output: context.output };
+}

package/src/plugins/config.ts

@@ -1,11 +1,11 @@
 import type { Indexer } from "../indexer";
 
-export type IndexerPlugin<TFilter, TBlock, TTxnParams> = (
-  indexer: Indexer<TFilter, TBlock, TTxnParams>,
+export type IndexerPlugin<TFilter, TBlock> = (
+  indexer: Indexer<TFilter, TBlock>,
 ) => void;
 
-export function defineIndexerPlugin<TFilter, TBlock, TTxnParams>(
-  def: IndexerPlugin<TFilter, TBlock, TTxnParams>,
+export function defineIndexerPlugin<TFilter, TBlock>(
+  def: IndexerPlugin<TFilter, TBlock>,
 ) {
   return def;
 }

package/src/plugins/context.ts

@@ -0,0 +1,40 @@
+import { useIndexerContext } from "../context";
+import { defineIndexerPlugin } from "./config";
+
+export const INTERNAL_CONTEXT_PROPERTY = "_internal";
+
+export function internalContext<TFilter, TBlock, TTxnParams>(
+  values: Record<string, unknown>,
+) {
+  return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
+    indexer.hooks.hook("run:before", () => {
+      try {
+        const ctx = useIndexerContext();
+        ctx[INTERNAL_CONTEXT_PROPERTY] = {
+          ...(ctx[INTERNAL_CONTEXT_PROPERTY] || {}),
+          ...values,
+        };
+      } catch (error) {
+        throw new Error("Failed to set internal context", {
+          cause: error,
+        });
+      }
+    });
+  });
+}
+
+export type InternalContext = {
+  indexerName: string;
+  availableIndexers: string[];
+};
+
+export function useInternalContext(): InternalContext {
+  const ctx = useIndexerContext();
+
+  if (ctx[INTERNAL_CONTEXT_PROPERTY] === undefined) {
+    throw new Error(
+      "Internal context is not available, possibly 'internalContext' plugin is missing!",
+    );
+  }
+  return ctx[INTERNAL_CONTEXT_PROPERTY];
+}

package/src/plugins/index.ts

@@ -1 +1,8 @@
-export * from "./config";
+export { defineIndexerPlugin, type IndexerPlugin } from "./config";
+export {
+  type ConsolaInstance,
+  type ConsolaReporter,
+  logger,
+  useLogger,
+} from "./logger";
+export { inMemoryPersistence } from "./persistence";

package/src/plugins/logger.ts

@@ -0,0 +1,30 @@
+import { type ConsolaInstance, type ConsolaReporter, consola } from "consola";
+import { useIndexerContext } from "../context";
+import { defineIndexerPlugin } from "./config";
+
+export type { ConsolaReporter, ConsolaInstance } from "consola";
+
+export function logger<TFilter, TBlock, TTxnParams>({
+  logger,
+}: { logger?: ConsolaReporter } = {}) {
+  return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
+    indexer.hooks.hook("run:before", () => {
+      const ctx = useIndexerContext();
+
+      if (logger) {
+        ctx.logger = consola.create({ reporters: [logger] });
+      } else {
+        ctx.logger = consola.create({});
+      }
+    });
+  });
+}
+
+export function useLogger(): ConsolaInstance {
+  const ctx = useIndexerContext();
+
+  if (!ctx?.logger)
+    throw new Error("Logger plugin is not available in context");
+
+  return ctx.logger;
+}

package/src/plugins/persistence.ts

@@ -1,202 +1,39 @@
-import type { Cursor } from "@apibara/protocol";
-import type { Database as SqliteDatabase, Statement } from "better-sqlite3";
-import { deserialize, serialize } from "../vcr";
-import { defineIndexerPlugin } from "./config";
+import { type Cursor, isCursor } from "@apibara/protocol";
 
-export function sqlitePersistence<TFilter, TBlock, TTxnParams>({
-  database,
-}: { database: SqliteDatabase }) {
-  return defineIndexerPlugin<TFilter, TBlock, TTxnParams>((indexer) => {
-    let store: SqlitePersistence<TFilter>;
+import { defineIndexerPlugin } from "./config";
 
-    indexer.hooks.hook("run:before", () => {
-      SqlitePersistence.initialize(database);
-
-      store = new SqlitePersistence(database);
-    });
+/**
+ * A plugin that persists the last cursor and filter to memory.
+ */
+export function inMemoryPersistence<TFilter, TBlock>() {
+  return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
+    let lastCursor: Cursor | undefined;
+    let lastFilter: TFilter | undefined;
 
     indexer.hooks.hook("connect:before", ({ request }) => {
-      const { cursor, filter } = store.get();
-
-      if (cursor) {
-        request.startingCursor = cursor;
+      if (lastCursor) {
+        request.startingCursor = lastCursor;
       }
 
-      if (filter) {
-        request.filter[1] = filter;
-      }
-    });
-
-    indexer.hooks.hook("transaction:commit", ({ endCursor }) => {
-      if (endCursor) {
-        store.put({ cursor: endCursor });
+      if (lastFilter) {
+        request.filter[1] = lastFilter;
       }
     });
 
     indexer.hooks.hook("connect:factory", ({ request, endCursor }) => {
       if (request.filter[1]) {
-        store.put({ cursor: endCursor, filter: request.filter[1] });
+        lastCursor = endCursor;
+        lastFilter = request.filter[1];
       }
     });
-  });
-}
-
-export class SqlitePersistence<TFilter> {
-  /** Sqlite Queries Prepare Statements */
-  private _getCheckpointQuery: Statement<string, CheckpointRow>;
-  private _putCheckpointQuery: Statement<
-    [string, number, `0x${string}` | undefined]
-  >;
-  private _delCheckpointQuery: Statement<string>;
-  private _getFilterQuery: Statement<string, FilterRow>;
-  private _updateFilterToBlockQuery: Statement<[number, string]>;
-  private _insertFilterQuery: Statement<[string, string, number]>;
-  private _delFilterQuery: Statement<string>;
-
-  constructor(private _db: SqliteDatabase) {
-    this._getCheckpointQuery = this._db.prepare(statements.getCheckpoint);
-    this._putCheckpointQuery = this._db.prepare(statements.putCheckpoint);
-    this._delCheckpointQuery = this._db.prepare(statements.delCheckpoint);
-    this._getFilterQuery = this._db.prepare(statements.getFilter);
-    this._updateFilterToBlockQuery = this._db.prepare(
-      statements.updateFilterToBlock,
-    );
-    this._insertFilterQuery = this._db.prepare(statements.insertFilter);
-    this._delFilterQuery = this._db.prepare(statements.delFilter);
-  }
-
-  static initialize(db: SqliteDatabase) {
-    db.prepare(statements.createCheckpointsTable).run();
-    db.prepare(statements.createFiltersTable).run();
-  }
-
-  public get(): { cursor?: Cursor; filter?: TFilter } {
-    const cursor = this._getCheckpoint();
-    const filter = this._getFilter();
-
-    return { cursor, filter };
-  }
-
-  public put({ cursor, filter }: { cursor?: Cursor; filter?: TFilter }) {
-    if (cursor) {
-      this._putCheckpoint(cursor);
-
-      if (filter) {
-        this._putFilter(filter, cursor);
-      }
-    }
-  }
 
-  public del() {
-    this._delCheckpoint();
-    this._delFilter();
-  }
-
-  // --- CHECKPOINTS TABLE METHODS ---
-
-  private _getCheckpoint(): Cursor | undefined {
-    const row = this._getCheckpointQuery.get("default");
-
-    if (!row) return undefined;
-
-    return { orderKey: BigInt(row.order_key), uniqueKey: row.unique_key };
-  }
-
-  private _putCheckpoint(cursor: Cursor) {
-    this._putCheckpointQuery.run(
-      "default",
-      Number(cursor.orderKey),
-      cursor.uniqueKey,
-    );
-  }
-
-  private _delCheckpoint() {
-    this._delCheckpointQuery.run("default");
-  }
-
-  // --- FILTERS TABLE METHODS ---
-
-  private _getFilter(): TFilter | undefined {
-    const row = this._getFilterQuery.get("default");
-
-    if (!row) return undefined;
-
-    return deserialize(row.filter) as TFilter;
-  }
-
-  private _putFilter(filter: TFilter, endCursor: Cursor) {
-    this._updateFilterToBlockQuery.run(Number(endCursor.orderKey), "default");
-    this._insertFilterQuery.run(
-      "default",
-      serialize(filter as Record<string, unknown>),
-      Number(endCursor.orderKey),
-    );
-  }
-
-  private _delFilter() {
-    this._delFilterQuery.run("default");
-  }
+    indexer.hooks.hook("handler:middleware", ({ use }) => {
+      use(async (context, next) => {
+        await next();
+        if (context.endCursor && isCursor(context.endCursor)) {
+          lastCursor = context.endCursor;
+        }
+      });
+    });
+  });
 }
-
-const statements = {
-  beginTxn: "BEGIN TRANSACTION",
-  commitTxn: "COMMIT TRANSACTION",
-  rollbackTxn: "ROLLBACK TRANSACTION",
-  createCheckpointsTable: `
-    CREATE TABLE IF NOT EXISTS checkpoints (
-      id TEXT NOT NULL PRIMARY KEY,
-      order_key INTEGER NOT NULL,
-      unique_key TEXT
-    );`,
-  createFiltersTable: `
-    CREATE TABLE IF NOT EXISTS filters (
-      id TEXT NOT NULL,
-      filter BLOB NOT NULL,
-      from_block INTEGER NOT NULL,
-      to_block INTEGER,
-      PRIMARY KEY (id, from_block)
-    );`,
-  getCheckpoint: `
-    SELECT *
-    FROM checkpoints
-    WHERE id = ?`,
-  putCheckpoint: `
-    INSERT INTO checkpoints (id, order_key, unique_key)
-    VALUES (?, ?, ?)
-    ON CONFLICT(id) DO UPDATE SET
-      order_key = excluded.order_key,
-      unique_key = excluded.unique_key`,
-  delCheckpoint: `
-    DELETE FROM checkpoints
-    WHERE id = ?`,
-  getFilter: `
-    SELECT *
-    FROM filters
-    WHERE id = ? AND to_block IS NULL`,
-  updateFilterToBlock: `
-    UPDATE filters
-    SET to_block = ?
-    WHERE id = ? AND to_block IS NULL`,
-  insertFilter: `
-    INSERT INTO filters (id, filter, from_block)
-    VALUES (?, ?, ?)
-    ON CONFLICT(id, from_block) DO UPDATE SET
-      filter = excluded.filter,
-      from_block = excluded.from_block`,
-  delFilter: `
-    DELETE FROM filters
-    WHERE id = ?`,
-};
-
-export type CheckpointRow = {
-  id: string;
-  order_key: number;
-  unique_key?: `0x${string}`;
-};
-
-export type FilterRow = {
-  id: string;
-  filter: string;
-  from_block: number;
-  to_block?: number;
-};

package/src/testing/index.ts

@@ -1,3 +1,58 @@
-export * from "./setup";
-export * from "./vcr";
-export * from "./helper";
+import { createClient } from "@apibara/protocol";
+import ci from "ci-info";
+import { type IndexerWithStreamConfig, createIndexer } from "../indexer";
+import { type InternalContext, internalContext } from "../plugins/context";
+import { logger } from "../plugins/logger";
+import type { CassetteOptions, VcrConfig } from "../vcr/config";
+import { isCassetteAvailable } from "../vcr/helper";
+import { record } from "../vcr/record";
+import { replay } from "../vcr/replay";
+
+export function createVcr() {
+  return {
+    async run<TFilter, TBlock>(
+      cassetteName: string,
+      indexerConfig: IndexerWithStreamConfig<TFilter, TBlock>,
+      range: { fromBlock: bigint; toBlock: bigint },
+    ) {
+      const vcrConfig: VcrConfig = {
+        cassetteDir: "cassettes",
+      };
+
+      const cassetteOptions: CassetteOptions = {
+        name: cassetteName,
+        startingCursor: {
+          orderKey: range.fromBlock,
+        },
+        endingCursor: {
+          orderKey: range.toBlock,
+        },
+      };
+
+      indexerConfig.plugins = [
+        internalContext({
+          indexerName: cassetteName,
+          availableIndexers: [cassetteName],
+        } as InternalContext),
+        logger(),
+        ...(indexerConfig.plugins ?? []),
+      ];
+
+      const indexer = createIndexer(indexerConfig);
+
+      if (!isCassetteAvailable(vcrConfig, cassetteName)) {
+        if (ci.isCI) {
+          throw new Error("Cannot record cassette in CI");
+        }
+
+        const client = createClient(
+          indexer.streamConfig,
+          indexer.options.streamUrl,
+        );
+        await record(vcrConfig, client, indexer, cassetteOptions);
+      } else {
+        await replay(vcrConfig, indexer, cassetteName);
+      }
+    },
+  };
+}

package/src/vcr/record.ts

@@ -1,7 +1,6 @@
 import fs from "node:fs/promises";
 import path from "node:path";
 import type { Client, StreamDataResponse } from "@apibara/protocol";
-import { klona } from "klona/full";
 import { type Indexer, run } from "../indexer";
 import type { CassetteOptions, VcrConfig } from "./config";
 import { serialize } from "./helper";
@@ -14,10 +13,9 @@ export type CassetteDataType<TFilter, TBlock> = {
 export async function record<TFilter, TBlock, TTxnParams>(
   vcrConfig: VcrConfig,
   client: Client<TFilter, TBlock>,
-  indexerArg: Indexer<TFilter, TBlock, TTxnParams>,
+  indexer: Indexer<TFilter, TBlock>,
   cassetteOptions: CassetteOptions,
 ) {
-  const indexer = klona(indexerArg);
   const messages: StreamDataResponse<TBlock>[] = [];
 
   indexer.hooks.addHooks({
@@ -33,10 +31,14 @@ export async function record<TFilter, TBlock, TTxnParams>(
         filter: indexer.options.filter,
         messages: messages,
       };
+
+      await fs.mkdir(vcrConfig.cassetteDir, { recursive: true });
+
       const filePath = path.join(
         vcrConfig.cassetteDir,
         `${cassetteOptions.name}.json`,
       );
+
       await fs.writeFile(filePath, serialize(output), { flag: "w" });
     },
   });

package/src/vcr/replay.ts

@@ -1,34 +1,21 @@
 import assert from "node:assert";
 import fs from "node:fs";
 import path from "node:path";
-import type { Client, Cursor } from "@apibara/protocol";
+import type { Client } from "@apibara/protocol";
 import { MockClient } from "@apibara/protocol/testing";
 import { type Indexer, run } from "../indexer";
-import type { SinkData } from "../sink";
-import { vcr } from "../testing/vcr";
 import { type CassetteDataType, deserialize } from "../vcr";
 import type { VcrConfig } from "./config";
 
 export async function replay<TFilter, TBlock, TTxnParams>(
   vcrConfig: VcrConfig,
-  indexer: Indexer<TFilter, TBlock, TTxnParams>,
+  indexer: Indexer<TFilter, TBlock>,
   cassetteName: string,
-): Promise<VcrReplayResult> {
+) {
   const client = loadCassette<TFilter, TBlock>(vcrConfig, cassetteName);
-
-  const sink = vcr();
-
   await run(client, indexer);
-
-  return {
-    outputs: sink.result,
-  };
 }
 
-export type VcrReplayResult = {
-  outputs: Array<{ endCursor?: Cursor; data: SinkData[] }>;
-};
-
 export function loadCassette<TFilter, TBlock>(
   vcrConfig: VcrConfig,
   cassetteName: string,
@@ -41,11 +28,14 @@ export function loadCassette<TFilter, TBlock>(
   const { filter, messages } = cassetteData;
 
   return new MockClient<TFilter, TBlock>((request, options) => {
+    // Notice that the request filter is an array of filters,
+    // so we need to wrap the indexer filter in an array.
     assert.deepStrictEqual(
       request.filter,
-      filter,
-      "Request and Cassette filter mismatch",
+      [filter],
+      "Indexer and cassette filter mismatch. Hint: delete the cassette and run again.",
     );
+
     return messages;
   });
 }