@apibara/indexer 0.4.1 → 2.0.0-beta.3

package/src/indexer.ts

@@ -0,0 +1,322 @@
+import type {
+  Client,
+  Cursor,
+  DataFinality,
+  StreamConfig,
+  StreamDataOptions,
+  StreamDataRequest,
+  StreamDataResponse,
+} from "@apibara/protocol";
+import consola from "consola";
+import {
+  type Hookable,
+  type NestedHooks,
+  createDebugger,
+  createHooks,
+} from "hookable";
+
+import assert from "node:assert";
+import {
+  type IndexerContext,
+  indexerAsyncContext,
+  useIndexerContext,
+} from "./context";
+import { tracer } from "./otel";
+import type { IndexerPlugin } from "./plugins";
+import { type Sink, defaultSink } from "./sink";
+
+export interface IndexerHooks<TFilter, TBlock> {
+  "run:before": () => void;
+  "run:after": () => void;
+  "connect:before": ({
+    request,
+    options,
+  }: {
+    request: StreamDataRequest<TFilter>;
+    options: StreamDataOptions;
+  }) => void;
+  "connect:after": () => void;
+  "connect:factory": ({
+    request,
+    endCursor,
+  }: {
+    request: StreamDataRequest<TFilter>;
+    endCursor?: Cursor;
+  }) => void;
+  "handler:before": ({
+    block,
+    finality,
+    endCursor,
+  }: {
+    block: TBlock;
+    finality: DataFinality;
+    endCursor?: Cursor;
+  }) => void;
+  "handler:after": ({
+    block,
+    finality,
+    endCursor,
+  }: {
+    block: TBlock;
+    finality: DataFinality;
+    endCursor?: Cursor;
+  }) => void;
+  "transaction:commit": ({
+    finality,
+    endCursor,
+  }: {
+    finality: DataFinality;
+    endCursor?: Cursor;
+  }) => void;
+  "handler:exception": ({ error }: { error: Error }) => void;
+  message: ({ message }: { message: StreamDataResponse<TBlock> }) => void;
+}
+
+export interface IndexerConfig<TFilter, TBlock, TTxnParams> {
+  streamUrl: string;
+  filter: TFilter;
+  finality?: DataFinality;
+  startingCursor?: Cursor;
+  sink?: Sink<TTxnParams>;
+  factory?: ({
+    block,
+    context,
+  }: { block: TBlock; context: IndexerContext<TTxnParams> }) => Promise<{
+    filter?: TFilter;
+  }>;
+  transform: (args: {
+    block: TBlock;
+    cursor?: Cursor | undefined;
+    endCursor?: Cursor | undefined;
+    finality: DataFinality;
+    context: IndexerContext<TTxnParams>;
+  }) => Promise<void>;
+  hooks?: NestedHooks<IndexerHooks<TFilter, TBlock>>;
+  plugins?: ReadonlyArray<IndexerPlugin<TFilter, TBlock, TTxnParams>>;
+  debug?: boolean;
+}
+
+export interface IndexerWithStreamConfig<TFilter, TBlock, TTxnParams>
+  extends IndexerConfig<TFilter, TBlock, TTxnParams> {
+  streamConfig: StreamConfig<TFilter, TBlock>;
+}
+
+export function defineIndexer<TFilter, TBlock>(
+  streamConfig: StreamConfig<TFilter, TBlock>,
+) {
+  return <TTxnParams>(
+    config: IndexerConfig<TFilter, TBlock, TTxnParams>,
+  ): IndexerWithStreamConfig<TFilter, TBlock, TTxnParams> => ({
+    streamConfig,
+    ...config,
+  });
+}
+
+export interface Indexer<TFilter, TBlock, TTxnParams> {
+  streamConfig: StreamConfig<TFilter, TBlock>;
+  options: IndexerConfig<TFilter, TBlock, TTxnParams>;
+  hooks: Hookable<IndexerHooks<TFilter, TBlock>>;
+}
+
+export function createIndexer<TFilter, TBlock, TTxnParams>({
+  streamConfig,
+  ...options
+}: IndexerWithStreamConfig<TFilter, TBlock, TTxnParams>) {
+  const indexer: Indexer<TFilter, TBlock, TTxnParams> = {
+    options,
+    streamConfig,
+    hooks: createHooks<IndexerHooks<TFilter, TBlock>>(),
+  };
+
+  if (indexer.options.debug) {
+    createDebugger(indexer.hooks, { tag: "indexer" });
+  }
+
+  indexer.hooks.addHooks(indexer.options.hooks ?? {});
+
+  for (const plugin of indexer.options.plugins ?? []) {
+    plugin(indexer);
+  }
+
+  return indexer;
+}
+
+export async function run<TFilter, TBlock, TTxnParams>(
+  client: Client<TFilter, TBlock>,
+  indexer: Indexer<TFilter, TBlock, TTxnParams>,
+) {
+  await indexerAsyncContext.callAsync({}, async () => {
+    const context = useIndexerContext<TTxnParams>();
+
+    const sink = indexer.options.sink ?? defaultSink();
+
+    context.sink = sink as Sink<TTxnParams>;
+
+    await indexer.hooks.callHook("run:before");
+
+    // Check if the it's factory mode or not
+    const isFactoryMode = indexer.options.factory !== undefined;
+
+    // if factory mode we add a empty filter
+    const request = indexer.streamConfig.Request.make({
+      filter: isFactoryMode
+        ? [indexer.options.filter, {} as TFilter]
+        : [indexer.options.filter],
+      finality: indexer.options.finality,
+      startingCursor: indexer.options.startingCursor,
+    });
+
+    const options: StreamDataOptions = {};
+
+    await indexer.hooks.callHook("connect:before", { request, options });
+
+    // store main filter, so later it can be merged
+    let mainFilter: TFilter;
+    if (isFactoryMode) {
+      mainFilter = request.filter[1];
+    }
+
+    // create stream
+    let stream: AsyncIterator<
+      StreamDataResponse<TBlock>,
+      StreamDataResponse<TBlock>
+    > = client.streamData(request, options)[Symbol.asyncIterator]();
+
+    await indexer.hooks.callHook("connect:after");
+
+    while (true) {
+      const { value: message, done } = await stream.next();
+
+      if (done) {
+        break;
+      }
+
+      await indexer.hooks.callHook("message", { message });
+
+      switch (message._tag) {
+        case "data": {
+          await tracer.startActiveSpan("message data", async (span) => {
+            const blocks = message.data.data;
+            const { cursor, endCursor, finality } = message.data;
+
+            await sink.transaction(
+              { cursor, endCursor, finality },
+              async (txn) => {
+                // attach transaction to context
+                context.sinkTransaction = txn as TTxnParams;
+
+                let block: TBlock | null;
+
+                // when factory mode
+                if (isFactoryMode) {
+                  assert(indexer.options.factory !== undefined);
+
+                  const [factoryBlock, mainBlock] = blocks;
+
+                  block = mainBlock;
+
+                  if (factoryBlock !== null) {
+                    const { filter } = await indexer.options.factory({
+                      block: factoryBlock,
+                      context,
+                    });
+
+                    // write returned data from factory function if filter is not defined
+                    if (filter) {
+                      // when filter is defined
+                      // merge old and new filters
+                      mainFilter = indexer.streamConfig.mergeFilter(
+                        mainFilter,
+                        filter,
+                      );
+
+                      // create request with new filters
+                      const request = indexer.streamConfig.Request.make({
+                        filter: [indexer.options.filter, mainFilter],
+                        finality: indexer.options.finality,
+                        startingCursor: cursor,
+                      });
+
+                      await indexer.hooks.callHook("connect:factory", {
+                        request,
+                        endCursor,
+                      });
+
+                      // create new stream with new request
+                      stream = client
+                        .streamData(request, options)
+                        [Symbol.asyncIterator]();
+
+                      const { value: message } = await stream.next();
+
+                      assert(message._tag === "data");
+
+                      const [_factoryBlock, _block] = message.data.data;
+
+                      block = _block;
+                    }
+                  }
+                } else {
+                  // when not in factory mode
+                  block = blocks[0];
+                }
+
+                // if block is not null
+                if (block) {
+                  await tracer.startActiveSpan("handler", async (span) => {
+                    await indexer.hooks.callHook("handler:before", {
+                      block,
+                      endCursor,
+                      finality,
+                    });
+
+                    try {
+                      await indexer.options.transform({
+                        block,
+                        cursor,
+                        endCursor,
+                        finality,
+                        context,
+                      });
+                      await indexer.hooks.callHook("handler:after", {
+                        block,
+                        finality,
+                        endCursor,
+                      });
+                    } catch (error) {
+                      assert(error instanceof Error);
+                      await indexer.hooks.callHook("handler:exception", {
+                        error,
+                      });
+                      throw error;
+                    }
+
+                    span.end();
+                  });
+                }
+              },
+            );
+            await indexer.hooks.callHook("transaction:commit", {
+              finality,
+              endCursor,
+            });
+            span.end();
+          });
+          break;
+        }
+        case "invalidate": {
+          await tracer.startActiveSpan("message invalidate", async (span) => {
+            await sink.invalidate(message.invalidate.cursor);
+          });
+          break;
+        }
+        default: {
+          consola.warn("unexpected message", message);
+          throw new Error("not implemented");
+        }
+      }
+
+      await indexer.hooks.callHook("run:after");
+    }
+  });
+}

package/src/otel.ts

@@ -0,0 +1,3 @@
+import { trace } from "@opentelemetry/api";
+
+export const tracer = trace.getTracer("@apibara/indexer");

package/src/plugins/config.ts

@@ -0,0 +1,11 @@
+import type { Indexer } from "../indexer";
+
+export type IndexerPlugin<TFilter, TBlock, TTxnParams> = (
+  indexer: Indexer<TFilter, TBlock, TTxnParams>,
+) => void;
+
+export function defineIndexerPlugin<TFilter, TBlock, TTxnParams>(
+  def: IndexerPlugin<TFilter, TBlock, TTxnParams>,
+) {
+  return def;
+}

package/src/plugins/kv.test.ts

@@ -0,0 +1,120 @@
+import Database, { type Database as SqliteDatabase } from "better-sqlite3";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { KVStore } from "./kv";
+
+type ValueType = { data: bigint };
+
+type DatabaseRowType = {
+  from_block: number;
+  k: string;
+  to_block: number;
+  v: unknown;
+};
+
+describe("KVStore", () => {
+  let db: SqliteDatabase;
+  let store: KVStore;
+  const key = "test_key";
+
+  beforeAll(() => {
+    db = new Database(":memory:");
+    KVStore.initialize(db);
+    store = new KVStore(db, "finalized", { orderKey: 5_000_000n });
+  });
+
+  afterAll(async () => {
+    db.close();
+  });
+
+  it("should begin transaction", () => {
+    store.beginTransaction();
+  });
+
+  it("should put and get a value", async () => {
+    const value = { data: 0n };
+
+    store.put<ValueType>(key, value);
+    const result = store.get<ValueType>(key);
+
+    expect(result).toEqual(value);
+  });
+
+  it("should commit transaction", async () => {
+    store.commitTransaction();
+
+    const value = { data: 0n };
+
+    const result = store.get<ValueType>(key);
+
+    expect(result).toEqual(value);
+  });
+
+  it("should return undefined for non-existing key", async () => {
+    const result = store.get<ValueType>("non_existent_key");
+    expect(result).toBeUndefined();
+  });
+
+  it("should begin transaction", async () => {
+    store.beginTransaction();
+  });
+
+  it("should update an existing value", async () => {
+    store = new KVStore(db, "finalized", { orderKey: 5_000_020n });
+
+    const value = { data: 50n };
+
+    store.put<ValueType>(key, value);
+    const result = store.get<ValueType>(key);
+
+    expect(result).toEqual(value);
+  });
+
+  it("should delete a value", async () => {
+    store.del(key);
+    const result = store.get<ValueType>(key);
+
+    expect(result).toBeUndefined();
+
+    const rows = db
+      .prepare<string, DatabaseRowType>(
+        `
+      SELECT from_block, to_block, k, v
+      FROM kvs
+      WHERE k = ?
+    `,
+      )
+      .all(key);
+
+    // Check that the old is correctly marked with to_block
+    expect(rows[0].to_block).toBe(Number(5_000_020n));
+  });
+
+  it("should rollback transaction", async () => {
+    await store.rollbackTransaction();
+  });
+
+  it("should revert the changes to last commit", async () => {
+    const rows = db
+      .prepare(
+        `
+      SELECT from_block, to_block, k, v
+      FROM kvs
+      WHERE k = ?
+    `,
+      )
+      .all([key]);
+
+    expect(rows).toMatchInlineSnapshot(`
+      [
+        {
+          "from_block": 5000000,
+          "k": "test_key",
+          "to_block": null,
+          "v": "{
+      	"data": "0n"
+      }",
+        },
+      ]
+    `);
+  });
+});

package/src/plugins/kv.ts

@@ -0,0 +1,132 @@
+import assert from "node:assert";
+import type { Cursor, DataFinality } from "@apibara/protocol";
+import type { Database as SqliteDatabase, Statement } from "better-sqlite3";
+import { useIndexerContext } from "../context";
+import { deserialize, serialize } from "../vcr";
+import { defineIndexerPlugin } from "./config";
+
+export function kv<TFilter, TBlock, TTxnParams>({
+  database,
+}: { database: SqliteDatabase }) {
+  return defineIndexerPlugin<TFilter, TBlock, TTxnParams>((indexer) => {
+    indexer.hooks.hook("run:before", () => {
+      KVStore.initialize(database);
+    });
+
+    indexer.hooks.hook("handler:before", ({ finality, endCursor }) => {
+      const ctx = useIndexerContext();
+
+      assert(endCursor, new Error("endCursor cannot be undefined"));
+
+      ctx.kv = new KVStore(database, finality, endCursor);
+
+      ctx.kv.beginTransaction();
+    });
+
+    indexer.hooks.hook("handler:after", () => {
+      const ctx = useIndexerContext();
+
+      ctx.kv.commitTransaction();
+
+      ctx.kv = null;
+    });
+
+    indexer.hooks.hook("handler:exception", () => {
+      const ctx = useIndexerContext();
+
+      ctx.kv.rollbackTransaction();
+
+      ctx.kv = null;
+    });
+  });
+}
+
+export class KVStore {
+  /** Sqlite Queries Prepare Statements */
+  private _beginTxnQuery: Statement;
+  private _commitTxnQuery: Statement;
+  private _rollbackTxnQuery: Statement;
+  private _getQuery: Statement<string, { v: string }>;
+  private _updateToBlockQuery: Statement<[number, string]>;
+  private _insertIntoKvsQuery: Statement<[number, string, string]>;
+  private _delQuery: Statement<[number, string]>;
+
+  constructor(
+    private _db: SqliteDatabase,
+    private _finality: DataFinality,
+    private _endCursor: Cursor,
+  ) {
+    this._beginTxnQuery = this._db.prepare(statements.beginTxn);
+    this._commitTxnQuery = this._db.prepare(statements.commitTxn);
+    this._rollbackTxnQuery = this._db.prepare(statements.rollbackTxn);
+    this._getQuery = this._db.prepare(statements.get);
+    this._updateToBlockQuery = this._db.prepare(statements.updateToBlock);
+    this._insertIntoKvsQuery = this._db.prepare(statements.insertIntoKvs);
+    this._delQuery = this._db.prepare(statements.del);
+  }
+
+  static initialize(db: SqliteDatabase) {
+    db.prepare(statements.createTable).run();
+  }
+
+  beginTransaction() {
+    this._beginTxnQuery.run();
+  }
+
+  commitTransaction() {
+    this._commitTxnQuery.run();
+  }
+
+  rollbackTransaction() {
+    this._rollbackTxnQuery.run();
+  }
+
+  get<T>(key: string): T {
+    const row = this._getQuery.get(key);
+
+    return row ? deserialize(row.v) : undefined;
+  }
+
+  put<T>(key: string, value: T) {
+    this._updateToBlockQuery.run(Number(this._endCursor.orderKey), key);
+
+    this._insertIntoKvsQuery.run(
+      Number(this._endCursor.orderKey),
+      key,
+      serialize(value as Record<string, unknown>),
+    );
+  }
+
+  del(key: string) {
+    this._delQuery.run(Number(this._endCursor.orderKey), key);
+  }
+}
+
+const statements = {
+  beginTxn: "BEGIN TRANSACTION",
+  commitTxn: "COMMIT TRANSACTION",
+  rollbackTxn: "ROLLBACK TRANSACTION",
+  createTable: `
+    CREATE TABLE IF NOT EXISTS kvs (
+      from_block INTEGER NOT NULL,
+      to_block INTEGER,
+      k TEXT NOT NULL,
+      v BLOB NOT NULL,
+      PRIMARY KEY (from_block, k)
+      );`,
+  get: `
+    SELECT v
+    FROM kvs
+    WHERE k = ? AND to_block IS NULL`,
+  updateToBlock: `
+    UPDATE kvs
+    SET to_block = ?
+    WHERE k = ? AND to_block IS NULL`,
+  insertIntoKvs: `
+    INSERT INTO kvs (from_block, to_block, k, v)
+    VALUES (?, NULL, ?, ?)`,
+  del: `
+    UPDATE kvs
+    SET to_block = ?
+    WHERE k = ? AND to_block IS NULL`,
+};