@apibara/indexer 2.0.0-beta.3 → 2.0.0-beta.31

package/src/indexer.ts

@@ -1,11 +1,17 @@
-import type {
-  Client,
-  Cursor,
-  DataFinality,
-  StreamConfig,
-  StreamDataOptions,
-  StreamDataRequest,
-  StreamDataResponse,
+import {
+  type Client,
+  ClientError,
+  type Cursor,
+  type DataFinality,
+  type Finalize,
+  type Heartbeat,
+  type Invalidate,
+  Status,
+  type StreamConfig,
+  type StreamDataOptions,
+  type StreamDataRequest,
+  type StreamDataResponse,
+  type SystemMessage,
 } from "@apibara/protocol";
 import consola from "consola";
 import {
@@ -16,6 +22,7 @@ import {
 } from "hookable";
 
 import assert from "node:assert";
+import { type MiddlewareFunction, type NextFunction, compose } from "./compose";
 import {
   type IndexerContext,
   indexerAsyncContext,
@@ -23,7 +30,10 @@ import {
 } from "./context";
 import { tracer } from "./otel";
 import type { IndexerPlugin } from "./plugins";
-import { type Sink, defaultSink } from "./sink";
+
+export type UseMiddlewareFunction = (
+  fn: MiddlewareFunction<IndexerContext>,
+) => void;
 
 export interface IndexerHooks<TFilter, TBlock> {
   "run:before": () => void;
@@ -35,7 +45,9 @@ export interface IndexerHooks<TFilter, TBlock> {
     request: StreamDataRequest<TFilter>;
     options: StreamDataOptions;
   }) => void;
-  "connect:after": () => void;
+  "connect:after": ({
+    request,
+  }: { request: StreamDataRequest<TFilter> }) => void;
   "connect:factory": ({
     request,
     endCursor,
@@ -43,45 +55,23 @@ export interface IndexerHooks<TFilter, TBlock> {
     request: StreamDataRequest<TFilter>;
     endCursor?: Cursor;
   }) => void;
-  "handler:before": ({
-    block,
-    finality,
-    endCursor,
-  }: {
-    block: TBlock;
-    finality: DataFinality;
-    endCursor?: Cursor;
-  }) => void;
-  "handler:after": ({
-    block,
-    finality,
-    endCursor,
-  }: {
-    block: TBlock;
-    finality: DataFinality;
-    endCursor?: Cursor;
-  }) => void;
-  "transaction:commit": ({
-    finality,
-    endCursor,
-  }: {
-    finality: DataFinality;
-    endCursor?: Cursor;
-  }) => void;
-  "handler:exception": ({ error }: { error: Error }) => void;
+  "handler:middleware": ({ use }: { use: UseMiddlewareFunction }) => void;
   message: ({ message }: { message: StreamDataResponse<TBlock> }) => void;
+  "message:invalidate": ({ message }: { message: Invalidate }) => void;
+  "message:finalize": ({ message }: { message: Finalize }) => void;
+  "message:heartbeat": ({ message }: { message: Heartbeat }) => void;
+  "message:systemMessage": ({ message }: { message: SystemMessage }) => void;
 }
 
-export interface IndexerConfig<TFilter, TBlock, TTxnParams> {
+export interface IndexerConfig<TFilter, TBlock> {
   streamUrl: string;
   filter: TFilter;
   finality?: DataFinality;
   startingCursor?: Cursor;
-  sink?: Sink<TTxnParams>;
   factory?: ({
     block,
     context,
-  }: { block: TBlock; context: IndexerContext<TTxnParams> }) => Promise<{
+  }: { block: TBlock; context: IndexerContext }) => Promise<{
     filter?: TFilter;
   }>;
   transform: (args: {
@@ -89,40 +79,40 @@ export interface IndexerConfig<TFilter, TBlock, TTxnParams> {
     cursor?: Cursor | undefined;
     endCursor?: Cursor | undefined;
     finality: DataFinality;
-    context: IndexerContext<TTxnParams>;
+    context: IndexerContext;
   }) => Promise<void>;
   hooks?: NestedHooks<IndexerHooks<TFilter, TBlock>>;
-  plugins?: ReadonlyArray<IndexerPlugin<TFilter, TBlock, TTxnParams>>;
+  plugins?: ReadonlyArray<IndexerPlugin<TFilter, TBlock>>;
   debug?: boolean;
 }
 
-export interface IndexerWithStreamConfig<TFilter, TBlock, TTxnParams>
-  extends IndexerConfig<TFilter, TBlock, TTxnParams> {
+export interface IndexerWithStreamConfig<TFilter, TBlock>
+  extends IndexerConfig<TFilter, TBlock> {
   streamConfig: StreamConfig<TFilter, TBlock>;
 }
 
 export function defineIndexer<TFilter, TBlock>(
   streamConfig: StreamConfig<TFilter, TBlock>,
 ) {
-  return <TTxnParams>(
-    config: IndexerConfig<TFilter, TBlock, TTxnParams>,
-  ): IndexerWithStreamConfig<TFilter, TBlock, TTxnParams> => ({
+  return (
+    config: IndexerConfig<TFilter, TBlock>,
+  ): IndexerWithStreamConfig<TFilter, TBlock> => ({
     streamConfig,
     ...config,
   });
 }
 
-export interface Indexer<TFilter, TBlock, TTxnParams> {
+export interface Indexer<TFilter, TBlock> {
   streamConfig: StreamConfig<TFilter, TBlock>;
-  options: IndexerConfig<TFilter, TBlock, TTxnParams>;
+  options: IndexerConfig<TFilter, TBlock>;
   hooks: Hookable<IndexerHooks<TFilter, TBlock>>;
 }
 
-export function createIndexer<TFilter, TBlock, TTxnParams>({
+export function createIndexer<TFilter, TBlock>({
   streamConfig,
   ...options
-}: IndexerWithStreamConfig<TFilter, TBlock, TTxnParams>) {
-  const indexer: Indexer<TFilter, TBlock, TTxnParams> = {
+}: IndexerWithStreamConfig<TFilter, TBlock>) {
+  const indexer: Indexer<TFilter, TBlock> = {
     options,
     streamConfig,
     hooks: createHooks<IndexerHooks<TFilter, TBlock>>(),
@@ -141,16 +131,75 @@ export function createIndexer<TFilter, TBlock, TTxnParams>({
   return indexer;
 }
 
-export async function run<TFilter, TBlock, TTxnParams>(
+export interface ReconnectOptions {
+  maxRetries?: number;
+  retryDelay?: number;
+  maxWait?: number;
+}
+
+export async function runWithReconnect<TFilter, TBlock>(
   client: Client<TFilter, TBlock>,
-  indexer: Indexer<TFilter, TBlock, TTxnParams>,
+  indexer: Indexer<TFilter, TBlock>,
+  options: ReconnectOptions = {},
 ) {
-  await indexerAsyncContext.callAsync({}, async () => {
-    const context = useIndexerContext<TTxnParams>();
+  let retryCount = 0;
+
+  const maxRetries = options.maxRetries ?? 10;
+  const retryDelay = options.retryDelay ?? 1_000;
+  const maxWait = options.maxWait ?? 30_000;
 
-    const sink = indexer.options.sink ?? defaultSink();
+  const runOptions: RunOptions = {
+    onConnect() {
+      retryCount = 0;
+    },
+  };
 
-    context.sink = sink as Sink<TTxnParams>;
+  while (true) {
+    try {
+      await run(client, indexer, runOptions);
+      return;
+    } catch (error) {
+      // Only reconnect on internal/server errors.
+      // All other errors should be rethrown.
+
+      retryCount++;
+
+      if (error instanceof ClientError) {
+        if (error.code === Status.INTERNAL) {
+          if (retryCount < maxRetries) {
+            consola.error(
+              "Internal server error, reconnecting...",
+              error.message,
+            );
+
+            // Add jitter to the retry delay to avoid all clients retrying at the same time.
+            const delay = Math.random() * (retryDelay * 0.2) + retryDelay;
+            await new Promise((resolve) =>
+              setTimeout(resolve, Math.min(retryCount * delay, maxWait)),
+            );
+
+            continue;
+          }
+        }
+      }
+
+      throw error;
+    }
+  }
+}
+
+export interface RunOptions {
+  onConnect?: () => void | Promise<void>;
+}
+
+export async function run<TFilter, TBlock>(
+  client: Client<TFilter, TBlock>,
+  indexer: Indexer<TFilter, TBlock>,
+  runOptions: RunOptions = {},
+) {
+  await indexerAsyncContext.callAsync({}, async () => {
+    const context = useIndexerContext();
+    const middleware = await registerMiddleware(indexer);
 
     await indexer.hooks.callHook("run:before");
 
@@ -182,7 +231,9 @@ export async function run<TFilter, TBlock, TTxnParams>(
       StreamDataResponse<TBlock>
     > = client.streamData(request, options)[Symbol.asyncIterator]();
 
-    await indexer.hooks.callHook("connect:after");
+    await indexer.hooks.callHook("connect:after", { request });
+
+    let onConnectCalled = false;
 
     while (true) {
       const { value: message, done } = await stream.next();
@@ -191,6 +242,13 @@ export async function run<TFilter, TBlock, TTxnParams>(
         break;
       }
 
+      if (!onConnectCalled) {
+        onConnectCalled = true;
+        if (runOptions.onConnect) {
+          await runOptions.onConnect();
+        }
+      }
+
       await indexer.hooks.callHook("message", { message });
 
       switch (message._tag) {
@@ -199,117 +257,138 @@ export async function run<TFilter, TBlock, TTxnParams>(
             const blocks = message.data.data;
             const { cursor, endCursor, finality } = message.data;
 
-            await sink.transaction(
-              { cursor, endCursor, finality },
-              async (txn) => {
-                // attach transaction to context
-                context.sinkTransaction = txn as TTxnParams;
+            context.cursor = cursor;
+            context.endCursor = endCursor;
+            context.finality = finality;
 
-                let block: TBlock | null;
+            await middleware(context, async () => {
+              let block: TBlock | null;
 
-                // when factory mode
-                if (isFactoryMode) {
-                  assert(indexer.options.factory !== undefined);
+              // when factory mode
+              if (isFactoryMode) {
+                assert(indexer.options.factory !== undefined);
 
-                  const [factoryBlock, mainBlock] = blocks;
+                const [factoryBlock, mainBlock] = blocks;
 
-                  block = mainBlock;
+                block = mainBlock;
 
-                  if (factoryBlock !== null) {
-                    const { filter } = await indexer.options.factory({
-                      block: factoryBlock,
-                      context,
-                    });
-
-                    // write returned data from factory function if filter is not defined
-                    if (filter) {
-                      // when filter is defined
-                      // merge old and new filters
-                      mainFilter = indexer.streamConfig.mergeFilter(
-                        mainFilter,
-                        filter,
-                      );
+                if (factoryBlock !== null) {
+                  const { filter } = await indexer.options.factory({
+                    block: factoryBlock,
+                    context,
+                  });
 
-                      // create request with new filters
-                      const request = indexer.streamConfig.Request.make({
-                        filter: [indexer.options.filter, mainFilter],
-                        finality: indexer.options.finality,
-                        startingCursor: cursor,
-                      });
+                  // write returned data from factory function if filter is not defined
+                  if (filter) {
+                    // when filter is defined
+                    // merge old and new filters
+                    mainFilter = indexer.streamConfig.mergeFilter(
+                      mainFilter,
+                      filter,
+                    );
+
+                    // create request with new filters
+                    const request = indexer.streamConfig.Request.make({
+                      filter: [indexer.options.filter, mainFilter],
+                      finality: indexer.options.finality,
+                      startingCursor: cursor,
+                    });
 
-                      await indexer.hooks.callHook("connect:factory", {
-                        request,
-                        endCursor,
-                      });
+                    await indexer.hooks.callHook("connect:factory", {
+                      request,
+                      endCursor,
+                    });
 
-                      // create new stream with new request
-                      stream = client
-                        .streamData(request, options)
-                        [Symbol.asyncIterator]();
+                    // create new stream with new request
+                    stream = client
+                      .streamData(request, options)
+                      [Symbol.asyncIterator]();
 
-                      const { value: message } = await stream.next();
+                    const { value: message } = await stream.next();
 
-                      assert(message._tag === "data");
+                    assert(message._tag === "data");
 
-                      const [_factoryBlock, _block] = message.data.data;
+                    const [_factoryBlock, _block] = message.data.data;
 
-                      block = _block;
-                    }
+                    block = _block;
                   }
-                } else {
-                  // when not in factory mode
-                  block = blocks[0];
                 }
-
-                // if block is not null
-                if (block) {
-                  await tracer.startActiveSpan("handler", async (span) => {
-                    await indexer.hooks.callHook("handler:before", {
-                      block,
-                      endCursor,
-                      finality,
-                    });
-
-                    try {
-                      await indexer.options.transform({
-                        block,
-                        cursor,
-                        endCursor,
-                        finality,
-                        context,
-                      });
-                      await indexer.hooks.callHook("handler:after", {
-                        block,
-                        finality,
-                        endCursor,
-                      });
-                    } catch (error) {
-                      assert(error instanceof Error);
-                      await indexer.hooks.callHook("handler:exception", {
-                        error,
-                      });
-                      throw error;
-                    }
-
-                    span.end();
+              } else {
+                // when not in factory mode
+                block = blocks[0];
+              }
+
+              // if block is not null
+              if (block) {
+                await tracer.startActiveSpan("handler", async (span) => {
+                  await indexer.options.transform({
+                    block,
+                    cursor,
+                    endCursor,
+                    finality,
+                    context,
                   });
-                }
-              },
-            );
-            await indexer.hooks.callHook("transaction:commit", {
-              finality,
-              endCursor,
+
+                  span.end();
+                });
+              }
             });
+
             span.end();
           });
+
+          context.cursor = undefined;
+          context.endCursor = undefined;
+          context.finality = undefined;
+
           break;
         }
         case "invalidate": {
           await tracer.startActiveSpan("message invalidate", async (span) => {
-            await sink.invalidate(message.invalidate.cursor);
+            await indexer.hooks.callHook("message:invalidate", { message });
+            span.end();
+          });
+          break;
+        }
+        case "finalize": {
+          await tracer.startActiveSpan("message finalize", async (span) => {
+            await indexer.hooks.callHook("message:finalize", { message });
+            span.end();
           });
           break;
         }
+        case "heartbeat": {
+          await tracer.startActiveSpan("message heartbeat", async (span) => {
+            await indexer.hooks.callHook("message:heartbeat", { message });
+            span.end();
+          });
+          break;
+        }
+        case "systemMessage": {
+          await tracer.startActiveSpan(
+            "message systemMessage",
+            async (span) => {
+              switch (message.systemMessage.output?._tag) {
+                case "stderr": {
+                  consola.warn(message.systemMessage.output.stderr);
+                  break;
+                }
+                case "stdout": {
+                  consola.info(message.systemMessage.output.stdout);
+                  break;
+                }
+                default: {
+                }
+              }
+
+              await indexer.hooks.callHook("message:systemMessage", {
+                message,
+              });
+              span.end();
+            },
+          );
+          break;
+        }
         default: {
           consola.warn("unexpected message", message);
           throw new Error("not implemented");
@@ -320,3 +399,24 @@ export async function run<TFilter, TBlock, TTxnParams>(
     }
   });
 }
+
+async function registerMiddleware<TFilter, TBlock>(
+  indexer: Indexer<TFilter, TBlock>,
+): Promise<MiddlewareFunction<IndexerContext>> {
+  const middleware: MiddlewareFunction<IndexerContext>[] = [];
+  const use = (fn: MiddlewareFunction<IndexerContext>) => {
+    middleware.push(fn);
+  };
+
+  await indexer.hooks.callHook("handler:middleware", { use });
+
+  const composed = compose(middleware);
+
+  // Return a named function to help debugging
+  return async function _composedIndexerMiddleware(
+    context: IndexerContext,
+    next?: NextFunction,
+  ) {
+    await composed(context, next);
+  };
+}

package/src/internal/testing.ts

@@ -0,0 +1,135 @@
+import { type Finalize, type Invalidate, isCursor } from "@apibara/protocol";
+import {
+  type MockBlock,
+  type MockFilter,
+  MockStream,
+  type MockStreamResponse,
+} from "@apibara/protocol/testing";
+
+import { useIndexerContext } from "../context";
+import { type IndexerConfig, createIndexer, defineIndexer } from "../indexer";
+import { type IndexerPlugin, defineIndexerPlugin } from "../plugins";
+
+export type MockMessagesOptions = {
+  invalidate?: {
+    invalidateFromIndex: number;
+    invalidateTriggerIndex: number;
+  };
+  finalize?: {
+    finalizeToIndex: number;
+    finalizeTriggerIndex: number;
+  };
+};
+
+export function generateMockMessages(
+  count = 10,
+  options?: MockMessagesOptions,
+): MockStreamResponse[] {
+  const invalidateAt = options?.invalidate;
+  const finalizeAt = options?.finalize;
+  const messages: MockStreamResponse[] = [];
+
+  for (let i = 0; i < count; i++) {
+    if (invalidateAt && i === invalidateAt.invalidateTriggerIndex) {
+      messages.push({
+        _tag: "invalidate",
+        invalidate: {
+          cursor: {
+            orderKey: BigInt(5_000_000 + invalidateAt.invalidateFromIndex),
+          },
+        },
+      } as Invalidate);
+    } else if (finalizeAt && i === finalizeAt.finalizeTriggerIndex) {
+      messages.push({
+        _tag: "finalize",
+        finalize: {
+          cursor: {
+            orderKey: BigInt(5_000_000 + finalizeAt.finalizeToIndex),
+          },
+        },
+      } as Finalize);
+    } else {
+      messages.push({
+        _tag: "data",
+        data: {
+          cursor: { orderKey: BigInt(5_000_000 + i - 1) },
+          finality: "accepted",
+          data: [{ data: `${5_000_000 + i}` }],
+          endCursor: { orderKey: BigInt(5_000_000 + i) },
+        },
+      });
+    }
+  }
+
+  return messages;
+}
+
+export function getMockIndexer({
+  plugins,
+  override,
+}: {
+  plugins?: ReadonlyArray<IndexerPlugin<MockFilter, MockBlock>>;
+  override?: Partial<IndexerConfig<MockFilter, MockBlock>>;
+} = {}) {
+  return createIndexer(
+    defineIndexer(MockStream)({
+      streamUrl: "https://sepolia.ethereum.a5a.ch",
+      finality: "accepted",
+      filter: {},
+      async transform({ block: { data }, context }) {},
+      plugins,
+      ...override,
+    }),
+  );
+}
+
+export type MockRet = {
+  data: string;
+};
+
+/**
+ * A mock sink used for testing. The indexer function can write to the output array.
+ * The indexer context is optionally written to the metadata object.
+ */
+export function mockSink<TFilter, TBlock>({
+  output,
+  metadata,
+}: { output: unknown[]; metadata?: Record<string, unknown> }) {
+  return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
+    indexer.hooks.hook("connect:before", ({ request }) => {
+      if (metadata?.lastCursor && isCursor(metadata.lastCursor)) {
+        request.startingCursor = metadata.lastCursor;
+      }
+
+      if (metadata?.lastFilter) {
+        request.filter[1] = metadata.lastFilter as TFilter;
+      }
+    });
+
+    indexer.hooks.hook("connect:factory", ({ request, endCursor }) => {
+      if (request.filter[1]) {
+        if (metadata) {
+          metadata.lastCursor = endCursor;
+          metadata.lastFilter = request.filter[1];
+        }
+      }
+    });
+
+    indexer.hooks.hook("handler:middleware", ({ use }) => {
+      use(async (context, next) => {
+        context.output = output;
+        await next();
+        context.output = null;
+
+        if (metadata) {
+          metadata.lastCursor = context.endCursor;
+        }
+      });
+    });
+  });
+}
+
+export function useMockSink(): { output: unknown[] } {
+  const context = useIndexerContext();
+  return { output: context.output };
+}

package/src/plugins/config.ts

@@ -1,11 +1,11 @@
 import type { Indexer } from "../indexer";
 
-export type IndexerPlugin<TFilter, TBlock, TTxnParams> = (
-  indexer: Indexer<TFilter, TBlock, TTxnParams>,
+export type IndexerPlugin<TFilter, TBlock> = (
+  indexer: Indexer<TFilter, TBlock>,
 ) => void;
 
-export function defineIndexerPlugin<TFilter, TBlock, TTxnParams>(
-  def: IndexerPlugin<TFilter, TBlock, TTxnParams>,
+export function defineIndexerPlugin<TFilter, TBlock>(
+  def: IndexerPlugin<TFilter, TBlock>,
 ) {
   return def;
 }

package/src/plugins/index.ts

@@ -1 +1,8 @@
-export * from "./config";
+export { defineIndexerPlugin, type IndexerPlugin } from "./config";
+export {
+  type ConsolaInstance,
+  type ConsolaReporter,
+  logger,
+  useLogger,
+} from "./logger";
+export { inMemoryPersistence } from "./persistence";