@apibara/indexer 2.0.0-beta.0 → 2.0.0-beta.4

package/src/indexer.ts

@@ -16,12 +16,16 @@ import {
 } from "hookable";
 
 import assert from "node:assert";
-import { indexerAsyncContext } from "./context";
+import {
+  type IndexerContext,
+  indexerAsyncContext,
+  useIndexerContext,
+} from "./context";
 import { tracer } from "./otel";
 import type { IndexerPlugin } from "./plugins";
-import { type Sink, type SinkData, defaultSink } from "./sink";
+import { type Sink, defaultSink } from "./sink";
 
-export interface IndexerHooks<TFilter, TBlock, TRet> {
+export interface IndexerHooks<TFilter, TBlock> {
   "run:before": () => void;
   "run:after": () => void;
   "connect:before": ({
@@ -48,63 +52,80 @@ export interface IndexerHooks<TFilter, TBlock, TRet> {
     finality: DataFinality;
     endCursor?: Cursor;
   }) => void;
-  "handler:after": ({ output }: { output: TRet[] }) => void;
-  "handler:exception": ({ error }: { error: Error }) => void;
-  "sink:write": ({ data }: { data: TRet[] }) => void;
-  "sink:flush": ({
+  "handler:after": ({
+    block,
+    finality,
     endCursor,
+  }: {
+    block: TBlock;
+    finality: DataFinality;
+    endCursor?: Cursor;
+  }) => void;
+  "transaction:commit": ({
     finality,
-  }: { endCursor?: Cursor; finality: DataFinality }) => void;
+    endCursor,
+  }: {
+    finality: DataFinality;
+    endCursor?: Cursor;
+  }) => void;
+  "handler:exception": ({ error }: { error: Error }) => void;
   message: ({ message }: { message: StreamDataResponse<TBlock> }) => void;
 }
 
-export interface IndexerConfig<TFilter, TBlock, TRet> {
+export interface IndexerConfig<TFilter, TBlock, TTxnParams> {
   streamUrl: string;
   filter: TFilter;
   finality?: DataFinality;
   startingCursor?: Cursor;
-  factory?: (block: TBlock) => Promise<{ filter?: TFilter; data?: TRet[] }>;
+  sink?: Sink<TTxnParams>;
+  factory?: ({
+    block,
+    context,
+  }: { block: TBlock; context: IndexerContext<TTxnParams> }) => Promise<{
+    filter?: TFilter;
+  }>;
   transform: (args: {
     block: TBlock;
     cursor?: Cursor | undefined;
     endCursor?: Cursor | undefined;
     finality: DataFinality;
-  }) => Promise<TRet[]>;
-  hooks?: NestedHooks<IndexerHooks<TFilter, TBlock, TRet>>;
-  plugins?: ReadonlyArray<IndexerPlugin<TFilter, TBlock, TRet>>;
+    context: IndexerContext<TTxnParams>;
+  }) => Promise<void>;
+  hooks?: NestedHooks<IndexerHooks<TFilter, TBlock>>;
+  plugins?: ReadonlyArray<IndexerPlugin<TFilter, TBlock, TTxnParams>>;
   debug?: boolean;
 }
 
-export interface IndexerWithStreamConfig<TFilter, TBlock, TRet>
-  extends IndexerConfig<TFilter, TBlock, TRet> {
+export interface IndexerWithStreamConfig<TFilter, TBlock, TTxnParams>
+  extends IndexerConfig<TFilter, TBlock, TTxnParams> {
   streamConfig: StreamConfig<TFilter, TBlock>;
 }
 
 export function defineIndexer<TFilter, TBlock>(
   streamConfig: StreamConfig<TFilter, TBlock>,
 ) {
-  return <TRet>(
-    config: IndexerConfig<TFilter, TBlock, TRet>,
-  ): IndexerWithStreamConfig<TFilter, TBlock, TRet> => ({
+  return <TTxnParams>(
+    config: IndexerConfig<TFilter, TBlock, TTxnParams>,
+  ): IndexerWithStreamConfig<TFilter, TBlock, TTxnParams> => ({
     streamConfig,
     ...config,
   });
 }
 
-export interface Indexer<TFilter, TBlock, TRet> {
+export interface Indexer<TFilter, TBlock, TTxnParams> {
   streamConfig: StreamConfig<TFilter, TBlock>;
-  options: IndexerConfig<TFilter, TBlock, TRet>;
-  hooks: Hookable<IndexerHooks<TFilter, TBlock, TRet>>;
+  options: IndexerConfig<TFilter, TBlock, TTxnParams>;
+  hooks: Hookable<IndexerHooks<TFilter, TBlock>>;
 }
 
-export function createIndexer<TFilter, TBlock, TRet>({
+export function createIndexer<TFilter, TBlock, TTxnParams>({
   streamConfig,
   ...options
-}: IndexerWithStreamConfig<TFilter, TBlock, TRet>) {
-  const indexer: Indexer<TFilter, TBlock, TRet> = {
+}: IndexerWithStreamConfig<TFilter, TBlock, TTxnParams>) {
+  const indexer: Indexer<TFilter, TBlock, TTxnParams> = {
     options,
     streamConfig,
-    hooks: createHooks<IndexerHooks<TFilter, TBlock, TRet>>(),
+    hooks: createHooks<IndexerHooks<TFilter, TBlock>>(),
   };
 
   if (indexer.options.debug) {
@@ -120,22 +141,18 @@ export function createIndexer<TFilter, TBlock, TRet>({
   return indexer;
 }
 
-export async function run<TFilter, TBlock, TRet>(
+export async function run<TFilter, TBlock, TTxnParams>(
   client: Client<TFilter, TBlock>,
-  indexer: Indexer<TFilter, TBlock, TRet>,
-  sinkArg?: Sink,
+  indexer: Indexer<TFilter, TBlock, TTxnParams>,
 ) {
   await indexerAsyncContext.callAsync({}, async () => {
-    await indexer.hooks.callHook("run:before");
+    const context = useIndexerContext<TTxnParams>();
 
-    const sink = sinkArg ?? defaultSink();
+    const sink = indexer.options.sink ?? defaultSink();
 
-    sink.hook("write", async ({ data }) => {
-      await indexer.hooks.callHook("sink:write", { data: data as TRet[] });
-    });
-    sink.hook("flush", async ({ endCursor, finality }) => {
-      await indexer.hooks.callHook("sink:flush", { endCursor, finality });
-    });
+    context.sink = sink as Sink<TTxnParams>;
+
+    await indexer.hooks.callHook("run:before");
 
     // Check if the it's factory mode or not
     const isFactoryMode = indexer.options.factory !== undefined;
@@ -151,7 +168,6 @@ export async function run<TFilter, TBlock, TRet>(
 
     const options: StreamDataOptions = {};
 
-    // TODO persistence plugin filter
     await indexer.hooks.callHook("connect:before", { request, options });
 
     // store main filter, so later it can be merged
@@ -161,162 +177,146 @@ export async function run<TFilter, TBlock, TRet>(
     }
 
     // create stream
-    let stream = client.streamData(request, options);
+    let stream: AsyncIterator<
+      StreamDataResponse<TBlock>,
+      StreamDataResponse<TBlock>
+    > = client.streamData(request, options)[Symbol.asyncIterator]();
 
     await indexer.hooks.callHook("connect:after");
 
-    // on state ->
-    // normal: iterate as usual
-    // recover: reconnect after updating filter
-    let state: { _tag: "normal" } | { _tag: "recover"; data?: TRet[] } = {
-      _tag: "normal",
-    };
-
     while (true) {
-      for await (const message of stream) {
-        await indexer.hooks.callHook("message", { message });
-
-        switch (message._tag) {
-          case "data": {
-            await tracer.startActiveSpan("message data", async (span) => {
-              const blocks = message.data.data;
-              const { cursor, endCursor, finality } = message.data;
-
-              let block: TBlock | null;
-
-              // combine output of factory and transform function
-              const output: TRet[] = [];
-
-              // when factory mode
-              if (isFactoryMode) {
-                assert(indexer.options.factory !== undefined);
-
-                const [factoryBlock, mainBlock] = blocks;
-
-                block = mainBlock;
-
-                if (state._tag === "normal" && factoryBlock !== null) {
-                  const { data, filter } =
-                    await indexer.options.factory(factoryBlock);
-
-                  // write returned data from factory function if filter is not defined
-                  if (!filter) {
-                    output.push(...(data ?? []));
-                  } else {
-                    // when filter is defined
-                    // merge old and new filters
-                    mainFilter = indexer.streamConfig.mergeFilter(
-                      mainFilter,
-                      filter,
-                    );
-
-                    // create request with new filters
-                    const request = indexer.streamConfig.Request.make({
-                      filter: [indexer.options.filter, mainFilter],
-                      finality: indexer.options.finality,
-                      startingCursor: cursor,
-                    });
+      const { value: message, done } = await stream.next();
 
-                    await indexer.hooks.callHook("connect:factory", {
-                      request,
-                      endCursor,
+      if (done) {
+        break;
+      }
+
+      await indexer.hooks.callHook("message", { message });
+
+      switch (message._tag) {
+        case "data": {
+          await tracer.startActiveSpan("message data", async (span) => {
+            const blocks = message.data.data;
+            const { cursor, endCursor, finality } = message.data;
+
+            await sink.transaction(
+              { cursor, endCursor, finality },
+              async (txn) => {
+                // attach transaction to context
+                context.sinkTransaction = txn as TTxnParams;
+
+                let block: TBlock | null;
+
+                // when factory mode
+                if (isFactoryMode) {
+                  assert(indexer.options.factory !== undefined);
+
+                  const [factoryBlock, mainBlock] = blocks;
+
+                  block = mainBlock;
+
+                  if (factoryBlock !== null) {
+                    const { filter } = await indexer.options.factory({
+                      block: factoryBlock,
+                      context,
                     });
 
-                    // create new stream with new request
-                    stream = client.streamData(request, options);
+                    // write returned data from factory function if filter is not defined
+                    if (filter) {
+                      // when filter is defined
+                      // merge old and new filters
+                      mainFilter = indexer.streamConfig.mergeFilter(
+                        mainFilter,
+                        filter,
+                      );
 
-                    // change state to recover mode
-                    state = {
-                      _tag: "recover",
-                      data,
-                    };
+                      // create request with new filters
+                      const request = indexer.streamConfig.Request.make({
+                        filter: [indexer.options.filter, mainFilter],
+                        finality: indexer.options.finality,
+                        startingCursor: cursor,
+                      });
 
-                    return;
+                      await indexer.hooks.callHook("connect:factory", {
+                        request,
+                        endCursor,
+                      });
+
+                      // create new stream with new request
+                      stream = client
+                        .streamData(request, options)
+                        [Symbol.asyncIterator]();
+
+                      const { value: message } = await stream.next();
+
+                      assert(message._tag === "data");
+
+                      const [_factoryBlock, _block] = message.data.data;
+
+                      block = _block;
+                    }
                   }
+                } else {
+                  // when not in factory mode
+                  block = blocks[0];
                 }
-                // after restart when state in recover mode
-                else if (state._tag === "recover") {
-                  // we write data to output
-                  output.push(...(state.data ?? []));
-                  // change state back to normal to avoid infinite loop
-                  state = { _tag: "normal" };
-                }
-              } else {
-                // when not in factory mode
-                block = blocks[0];
-              }
-
-              // if block is not null
-              if (block) {
-                await tracer.startActiveSpan("handler", async (span) => {
-                  await indexer.hooks.callHook("handler:before", {
-                    block,
-                    endCursor,
-                    finality,
-                  });
 
-                  try {
-                    const transformOutput = await indexer.options.transform({
+                // if block is not null
+                if (block) {
+                  await tracer.startActiveSpan("handler", async (span) => {
+                    await indexer.hooks.callHook("handler:before", {
                       block,
-                      cursor,
                       endCursor,
                       finality,
                     });
 
-                    // write transformed data to output
-                    output.push(...transformOutput);
-
-                    await indexer.hooks.callHook("handler:after", { output });
-                  } catch (error) {
-                    assert(error instanceof Error);
-                    await indexer.hooks.callHook("handler:exception", {
-                      error,
-                    });
-                    throw error;
-                  }
-
-                  span.end();
-                });
-              }
-
-              // if output has data, write it to sink
-              if (output.length > 0) {
-                await tracer.startActiveSpan("sink write", async (span) => {
-                  await sink.write({
-                    data: output as SinkData[],
-                    cursor,
-                    endCursor,
-                    finality,
+                    try {
+                      await indexer.options.transform({
+                        block,
+                        cursor,
+                        endCursor,
+                        finality,
+                        context,
+                      });
+                      await indexer.hooks.callHook("handler:after", {
+                        block,
+                        finality,
+                        endCursor,
+                      });
+                    } catch (error) {
+                      assert(error instanceof Error);
+                      await indexer.hooks.callHook("handler:exception", {
+                        error,
+                      });
+                      throw error;
+                    }
+
+                    span.end();
                   });
-
-                  span.end();
-                });
-              }
-              span.end();
+                }
+              },
+            );
+            await indexer.hooks.callHook("transaction:commit", {
+              finality,
+              endCursor,
             });
-            break;
-          }
-          default: {
-            consola.warn("unexpected message", message);
-            throw new Error("not implemented");
-          }
+            span.end();
+          });
+          break;
         }
-
-        // if stream needs a restart
-        // break out of the current stream iterator
-        if (state._tag !== "normal") {
+        case "invalidate": {
+          await tracer.startActiveSpan("message invalidate", async (span) => {
+            await sink.invalidate(message.invalidate.cursor);
+          });
           break;
         }
-      }
-
-      // when restarting stream we continue while loop again
-      if (state._tag !== "normal") {
-        continue;
+        default: {
+          consola.warn("unexpected message", message);
+          throw new Error("not implemented");
+        }
       }
 
       await indexer.hooks.callHook("run:after");
-
-      break;
     }
   });
 }

package/src/plugins/config.ts

@@ -1,11 +1,11 @@
 import type { Indexer } from "../indexer";
 
-export type IndexerPlugin<TFilter, TBlock, TRet> = (
-  indexer: Indexer<TFilter, TBlock, TRet>,
+export type IndexerPlugin<TFilter, TBlock, TTxnParams> = (
+  indexer: Indexer<TFilter, TBlock, TTxnParams>,
 ) => void;
 
-export function defineIndexerPlugin<TFilter, TBlock, TRet>(
-  def: IndexerPlugin<TFilter, TBlock, TRet>,
+export function defineIndexerPlugin<TFilter, TBlock, TTxnParams>(
+  def: IndexerPlugin<TFilter, TBlock, TTxnParams>,
 ) {
   return def;
 }

package/src/plugins/kv.ts

@@ -5,10 +5,10 @@ import { useIndexerContext } from "../context";
 import { deserialize, serialize } from "../vcr";
 import { defineIndexerPlugin } from "./config";
 
-export function kv<TFilter, TBlock, TRet>({
+export function kv<TFilter, TBlock, TTxnParams>({
   database,
 }: { database: SqliteDatabase }) {
-  return defineIndexerPlugin<TFilter, TBlock, TRet>((indexer) => {
+  return defineIndexerPlugin<TFilter, TBlock, TTxnParams>((indexer) => {
     indexer.hooks.hook("run:before", () => {
       KVStore.initialize(database);
     });

package/src/plugins/persistence.test.ts

@@ -9,7 +9,7 @@ import { klona } from "klona/full";
 import { describe, expect, it } from "vitest";
 import { run } from "../indexer";
 import { generateMockMessages } from "../testing";
-import { type MockRet, getMockIndexer } from "../testing/indexer";
+import { getMockIndexer } from "../testing/indexer";
 import { SqlitePersistence, sqlitePersistence } from "./persistence";
 
 describe("Persistence", () => {
@@ -120,12 +120,16 @@ describe("Persistence", () => {
 
     const db = new Database(":memory:");
 
-    const persistence = sqlitePersistence<MockFilter, MockBlock, MockRet>({
-      database: db,
-    });
-
     // create mock indexer with persistence plugin
-    const indexer = klona(getMockIndexer([persistence]));
+    const indexer = klona(
+      getMockIndexer({
+        plugins: [
+          sqlitePersistence({
+            database: db,
+          }),
+        ],
+      }),
+    );
 
     await run(client, indexer);
 

package/src/plugins/persistence.ts

@@ -3,10 +3,10 @@ import type { Database as SqliteDatabase, Statement } from "better-sqlite3";
 import { deserialize, serialize } from "../vcr";
 import { defineIndexerPlugin } from "./config";
 
-export function sqlitePersistence<TFilter, TBlock, TRet>({
+export function sqlitePersistence<TFilter, TBlock, TTxnParams>({
   database,
 }: { database: SqliteDatabase }) {
-  return defineIndexerPlugin<TFilter, TBlock, TRet>((indexer) => {
+  return defineIndexerPlugin<TFilter, TBlock, TTxnParams>((indexer) => {
     let store: SqlitePersistence<TFilter>;
 
     indexer.hooks.hook("run:before", () => {
@@ -27,7 +27,7 @@ export function sqlitePersistence<TFilter, TBlock, TRet>({
       }
     });
 
-    indexer.hooks.hook("sink:flush", ({ endCursor }) => {
+    indexer.hooks.hook("transaction:commit", ({ endCursor }) => {
       if (endCursor) {
         store.put({ cursor: endCursor });
       }

package/src/sink.ts

@@ -1,36 +1,33 @@
 import type { Cursor, DataFinality } from "@apibara/protocol";
-import { Hookable } from "hookable";
+import consola from "consola";
 
 export type SinkData = Record<string, unknown>;
 
-export interface SinkEvents {
-  write({ data }: { data: SinkData[] }): void;
-  flush({
-    endCursor,
-    finality,
-  }: { endCursor?: Cursor; finality: DataFinality }): void;
-}
-
-export type SinkWriteArgs = {
-  data: SinkData[];
-  cursor?: Cursor | undefined;
-  endCursor?: Cursor | undefined;
+export type SinkCursorParams = {
+  cursor?: Cursor;
+  endCursor?: Cursor;
   finality: DataFinality;
 };
 
-export abstract class Sink extends Hookable<SinkEvents> {
-  abstract write({
-    data,
-    cursor,
-    endCursor,
-    finality,
-  }: SinkWriteArgs): Promise<void>;
+export abstract class Sink<TTxnParams = unknown> {
+  abstract transaction(
+    { cursor, endCursor, finality }: SinkCursorParams,
+    cb: (params: TTxnParams) => Promise<void>,
+  ): Promise<void>;
+
+  abstract invalidate(cursor?: Cursor): Promise<void>;
 }
 
-export class DefaultSink extends Sink {
-  async write({ data, endCursor, finality }: SinkWriteArgs) {
-    await this.callHook("write", { data });
-    await this.callHook("flush", { endCursor, finality });
+export class DefaultSink extends Sink<unknown> {
+  async transaction(
+    { cursor, endCursor, finality }: SinkCursorParams,
+    cb: (params: unknown) => Promise<void>,
+  ): Promise<void> {
+    await cb({});
+  }
+
+  async invalidate(cursor?: Cursor) {
+    consola.info(`Invalidating cursor ${cursor?.orderKey}`);
   }
 }
 

package/src/sinks/csv.test.ts

@@ -5,10 +5,11 @@ import {
   type MockFilter,
 } from "@apibara/protocol/testing";
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { useSink } from "../hooks";
 import { run } from "../indexer";
 import {} from "../plugins/persistence";
 import { generateMockMessages } from "../testing";
-import { type MockRet, getMockIndexer } from "../testing/indexer";
+import { getMockIndexer } from "../testing/indexer";
 import { csv } from "./csv";
 
 describe("Run Test", () => {
@@ -31,8 +32,19 @@ describe("Run Test", () => {
       return generateMockMessages();
     });
 
-    const sink = csv<MockRet>({ filepath: "test.csv" });
-    await run(client, getMockIndexer(), sink);
+    const sink = csv({ filepath: "test.csv" });
+    await run(
+      client,
+      getMockIndexer({
+        sink,
+        override: {
+          transform: async ({ context, endCursor, block: { data } }) => {
+            const { writer } = useSink({ context });
+            writer.insert([{ data }]);
+          },
+        },
+      }),
+    );
 
     const csvData = await fs.readFile("test.csv", "utf-8");