@apibara/indexer 2.0.0-beta.9 → 2.1.0-beta.1

package/src/indexer.ts

@@ -1,11 +1,17 @@
-import type {
-  Client,
-  Cursor,
-  DataFinality,
-  StreamConfig,
-  StreamDataOptions,
-  StreamDataRequest,
-  StreamDataResponse,
+import {
+  type Client,
+  ClientError,
+  type Cursor,
+  type DataFinality,
+  type Finalize,
+  type Heartbeat,
+  type Invalidate,
+  Status,
+  type StreamConfig,
+  type StreamDataOptions,
+  type StreamDataRequest,
+  type StreamDataResponse,
+  type SystemMessage,
 } from "@apibara/protocol";
 import consola from "consola";
 import {
@@ -16,6 +22,7 @@ import {
 } from "hookable";
 
 import assert from "node:assert";
+import { type MiddlewareFunction, type NextFunction, compose } from "./compose";
 import {
   type IndexerContext,
   indexerAsyncContext,
@@ -23,7 +30,10 @@ import {
 } from "./context";
 import { tracer } from "./otel";
 import type { IndexerPlugin } from "./plugins";
-import { type Sink, defaultSink } from "./sink";
+
+export type UseMiddlewareFunction = (
+  fn: MiddlewareFunction<IndexerContext>,
+) => void;
 
 export interface IndexerHooks<TFilter, TBlock> {
   "run:before": () => void;
@@ -35,7 +45,9 @@ export interface IndexerHooks<TFilter, TBlock> {
     request: StreamDataRequest<TFilter>;
     options: StreamDataOptions;
   }) => void;
-  "connect:after": () => void;
+  "connect:after": ({
+    request,
+  }: { request: StreamDataRequest<TFilter> }) => void;
   "connect:factory": ({
     request,
     endCursor,
@@ -43,45 +55,36 @@ export interface IndexerHooks<TFilter, TBlock> {
     request: StreamDataRequest<TFilter>;
     endCursor?: Cursor;
   }) => void;
-  "handler:before": ({
-    block,
-    finality,
-    endCursor,
-  }: {
-    block: TBlock;
-    finality: DataFinality;
-    endCursor?: Cursor;
-  }) => void;
-  "handler:after": ({
-    block,
-    finality,
-    endCursor,
-  }: {
-    block: TBlock;
-    finality: DataFinality;
-    endCursor?: Cursor;
-  }) => void;
-  "transaction:commit": ({
-    finality,
-    endCursor,
-  }: {
-    finality: DataFinality;
-    endCursor?: Cursor;
-  }) => void;
-  "handler:exception": ({ error }: { error: Error }) => void;
+  "handler:middleware": ({ use }: { use: UseMiddlewareFunction }) => void;
   message: ({ message }: { message: StreamDataResponse<TBlock> }) => void;
+  "message:invalidate": ({ message }: { message: Invalidate }) => void;
+  "message:finalize": ({ message }: { message: Finalize }) => void;
+  "message:heartbeat": ({ message }: { message: Heartbeat }) => void;
+  "message:systemMessage": ({ message }: { message: SystemMessage }) => void;
 }
 
-export interface IndexerConfig<TFilter, TBlock, TTxnParams> {
+export type IndexerStartingCursor =
+  | {
+      startingCursor?: never;
+      startingBlock: bigint;
+    }
+  | {
+      startingCursor: Cursor;
+      startingBlock?: never;
+    }
+  | {
+      startingCursor?: never;
+      startingBlock?: never;
+    };
+
+export type IndexerConfig<TFilter, TBlock> = {
   streamUrl: string;
   filter: TFilter;
   finality?: DataFinality;
-  startingCursor?: Cursor;
-  sink?: Sink<TTxnParams>;
   factory?: ({
     block,
     context,
-  }: { block: TBlock; context: IndexerContext<TTxnParams> }) => Promise<{
+  }: { block: TBlock; context: IndexerContext }) => Promise<{
     filter?: TFilter;
   }>;
   transform: (args: {
@@ -89,40 +92,42 @@ export interface IndexerConfig<TFilter, TBlock, TTxnParams> {
     cursor?: Cursor | undefined;
     endCursor?: Cursor | undefined;
     finality: DataFinality;
-    context: IndexerContext<TTxnParams>;
+    context: IndexerContext;
   }) => Promise<void>;
   hooks?: NestedHooks<IndexerHooks<TFilter, TBlock>>;
-  plugins?: ReadonlyArray<IndexerPlugin<TFilter, TBlock, TTxnParams>>;
+  plugins?: ReadonlyArray<IndexerPlugin<TFilter, TBlock>>;
   debug?: boolean;
-}
+} & IndexerStartingCursor;
 
-export interface IndexerWithStreamConfig<TFilter, TBlock, TTxnParams>
-  extends IndexerConfig<TFilter, TBlock, TTxnParams> {
+export type IndexerWithStreamConfig<TFilter, TBlock> = IndexerConfig<
+  TFilter,
+  TBlock
+> & {
   streamConfig: StreamConfig<TFilter, TBlock>;
-}
+};
 
 export function defineIndexer<TFilter, TBlock>(
   streamConfig: StreamConfig<TFilter, TBlock>,
 ) {
-  return <TTxnParams>(
-    config: IndexerConfig<TFilter, TBlock, TTxnParams>,
-  ): IndexerWithStreamConfig<TFilter, TBlock, TTxnParams> => ({
+  return (
+    config: IndexerConfig<TFilter, TBlock>,
+  ): IndexerWithStreamConfig<TFilter, TBlock> => ({
     streamConfig,
     ...config,
   });
 }
 
-export interface Indexer<TFilter, TBlock, TTxnParams> {
+export interface Indexer<TFilter, TBlock> {
   streamConfig: StreamConfig<TFilter, TBlock>;
-  options: IndexerConfig<TFilter, TBlock, TTxnParams>;
+  options: IndexerConfig<TFilter, TBlock>;
   hooks: Hookable<IndexerHooks<TFilter, TBlock>>;
 }
 
-export function createIndexer<TFilter, TBlock, TTxnParams>({
+export function createIndexer<TFilter, TBlock>({
   streamConfig,
   ...options
-}: IndexerWithStreamConfig<TFilter, TBlock, TTxnParams>) {
-  const indexer: Indexer<TFilter, TBlock, TTxnParams> = {
+}: IndexerWithStreamConfig<TFilter, TBlock>) {
+  const indexer: Indexer<TFilter, TBlock> = {
     options,
     streamConfig,
     hooks: createHooks<IndexerHooks<TFilter, TBlock>>(),
@@ -141,29 +146,97 @@ export function createIndexer<TFilter, TBlock, TTxnParams>({
   return indexer;
 }
 
-export async function run<TFilter, TBlock, TTxnParams>(
+export interface ReconnectOptions {
+  maxRetries?: number;
+  retryDelay?: number;
+  maxWait?: number;
+}
+
+export async function runWithReconnect<TFilter, TBlock>(
   client: Client<TFilter, TBlock>,
-  indexer: Indexer<TFilter, TBlock, TTxnParams>,
+  indexer: Indexer<TFilter, TBlock>,
+  options: ReconnectOptions = {},
 ) {
-  await indexerAsyncContext.callAsync({}, async () => {
-    const context = useIndexerContext<TTxnParams>();
+  let retryCount = 0;
+
+  const maxRetries = options.maxRetries ?? 10;
+  const retryDelay = options.retryDelay ?? 1_000;
+  const maxWait = options.maxWait ?? 30_000;
+
+  const runOptions: RunOptions = {
+    onConnect() {
+      retryCount = 0;
+    },
+  };
 
-    const sink = indexer.options.sink ?? defaultSink();
+  while (true) {
+    try {
+      await run(client, indexer, runOptions);
+      return;
+    } catch (error) {
+      // Only reconnect on internal/server errors.
+      // All other errors should be rethrown.
+
+      retryCount++;
+
+      if (error instanceof ClientError) {
+        if (error.code === Status.INTERNAL) {
+          if (retryCount < maxRetries) {
+            consola.error(
+              "Internal server error, reconnecting...",
+              error.message,
+            );
 
-    context.sink = sink as Sink<TTxnParams>;
+            // Add jitter to the retry delay to avoid all clients retrying at the same time.
+            const delay = Math.random() * (retryDelay * 0.2) + retryDelay;
+            await new Promise((resolve) =>
+              setTimeout(resolve, Math.min(retryCount * delay, maxWait)),
+            );
+
+            continue;
+          }
+        }
+      }
+
+      throw error;
+    }
+  }
+}
+
+export interface RunOptions {
+  onConnect?: () => void | Promise<void>;
+}
+
+export async function run<TFilter, TBlock>(
+  client: Client<TFilter, TBlock>,
+  indexer: Indexer<TFilter, TBlock>,
+  runOptions: RunOptions = {},
+) {
+  await indexerAsyncContext.callAsync({}, async () => {
+    const context = useIndexerContext();
+    const middleware = await registerMiddleware(indexer);
 
     await indexer.hooks.callHook("run:before");
 
-    // Check if the it's factory mode or not
     const isFactoryMode = indexer.options.factory !== undefined;
 
-    // if factory mode we add a empty filter
+    // Give priority to startingCursor over startingBlock.
+    let startingCursor: Cursor | undefined;
+    if (indexer.options.startingCursor) {
+      startingCursor = indexer.options.startingCursor;
+    } else if (indexer.options.startingBlock !== undefined) {
+      startingCursor = {
+        orderKey: indexer.options.startingBlock,
+      };
+    }
+
+    // if factory mode we add a empty filter at the end of the filter array.
     const request = indexer.streamConfig.Request.make({
       filter: isFactoryMode
         ? [indexer.options.filter, {} as TFilter]
         : [indexer.options.filter],
       finality: indexer.options.finality,
-      startingCursor: indexer.options.startingCursor,
+      startingCursor,
     });
 
     const options: StreamDataOptions = {};
@@ -176,13 +249,14 @@ export async function run<TFilter, TBlock, TTxnParams>(
       mainFilter = request.filter[1];
     }
 
-    // create stream
     let stream: AsyncIterator<
       StreamDataResponse<TBlock>,
       StreamDataResponse<TBlock>
     > = client.streamData(request, options)[Symbol.asyncIterator]();
 
-    await indexer.hooks.callHook("connect:after");
+    await indexer.hooks.callHook("connect:after", { request });
+
+    let onConnectCalled = false;
 
     while (true) {
       const { value: message, done } = await stream.next();
@@ -191,6 +265,13 @@ export async function run<TFilter, TBlock, TTxnParams>(
         break;
       }
 
+      if (!onConnectCalled) {
+        onConnectCalled = true;
+        if (runOptions.onConnect) {
+          await runOptions.onConnect();
+        }
+      }
+
       await indexer.hooks.callHook("message", { message });
 
       switch (message._tag) {
@@ -199,117 +280,138 @@ export async function run<TFilter, TBlock, TTxnParams>(
             const blocks = message.data.data;
             const { cursor, endCursor, finality } = message.data;
 
-            await sink.transaction(
-              { cursor, endCursor, finality },
-              async (txn) => {
-                // attach transaction to context
-                context.sinkTransaction = txn as TTxnParams;
-
-                let block: TBlock | null;
+            context.cursor = cursor;
+            context.endCursor = endCursor;
+            context.finality = finality;
 
-                // when factory mode
-                if (isFactoryMode) {
-                  assert(indexer.options.factory !== undefined);
+            await middleware(context, async () => {
+              let block: TBlock | null;
 
-                  const [factoryBlock, mainBlock] = blocks;
+              // when factory mode
+              if (isFactoryMode) {
+                assert(indexer.options.factory !== undefined);
 
-                  block = mainBlock;
+                const [factoryBlock, mainBlock] = blocks;
 
-                  if (factoryBlock !== null) {
-                    const { filter } = await indexer.options.factory({
-                      block: factoryBlock,
-                      context,
-                    });
+                block = mainBlock;
 
-                    // write returned data from factory function if filter is not defined
-                    if (filter) {
-                      // when filter is defined
-                      // merge old and new filters
-                      mainFilter = indexer.streamConfig.mergeFilter(
-                        mainFilter,
-                        filter,
-                      );
+                if (factoryBlock !== null) {
+                  const { filter } = await indexer.options.factory({
+                    block: factoryBlock,
+                    context,
+                  });
 
-                      // create request with new filters
-                      const request = indexer.streamConfig.Request.make({
-                        filter: [indexer.options.filter, mainFilter],
-                        finality: indexer.options.finality,
-                        startingCursor: cursor,
-                      });
+                  // write returned data from factory function if filter is not defined
+                  if (filter) {
+                    // when filter is defined
+                    // merge old and new filters
+                    mainFilter = indexer.streamConfig.mergeFilter(
+                      mainFilter,
+                      filter,
+                    );
+
+                    // create request with new filters
+                    const request = indexer.streamConfig.Request.make({
+                      filter: [indexer.options.filter, mainFilter],
+                      finality: indexer.options.finality,
+                      startingCursor: cursor,
+                    });
 
-                      await indexer.hooks.callHook("connect:factory", {
-                        request,
-                        endCursor,
-                      });
+                    await indexer.hooks.callHook("connect:factory", {
+                      request,
+                      endCursor,
+                    });
 
-                      // create new stream with new request
-                      stream = client
-                        .streamData(request, options)
-                        [Symbol.asyncIterator]();
+                    // create new stream with new request
+                    stream = client
+                      .streamData(request, options)
+                      [Symbol.asyncIterator]();
 
-                      const { value: message } = await stream.next();
+                    const { value: message } = await stream.next();
 
-                      assert(message._tag === "data");
+                    assert(message._tag === "data");
 
-                      const [_factoryBlock, _block] = message.data.data;
+                    const [_factoryBlock, _block] = message.data.data;
 
-                      block = _block;
-                    }
+                    block = _block;
                   }
-                } else {
-                  // when not in factory mode
-                  block = blocks[0];
                 }
-
-                // if block is not null
-                if (block) {
-                  await tracer.startActiveSpan("handler", async (span) => {
-                    await indexer.hooks.callHook("handler:before", {
-                      block,
-                      endCursor,
-                      finality,
-                    });
-
-                    try {
-                      await indexer.options.transform({
-                        block,
-                        cursor,
-                        endCursor,
-                        finality,
-                        context,
-                      });
-                      await indexer.hooks.callHook("handler:after", {
-                        block,
-                        finality,
-                        endCursor,
-                      });
-                    } catch (error) {
-                      assert(error instanceof Error);
-                      await indexer.hooks.callHook("handler:exception", {
-                        error,
-                      });
-                      throw error;
-                    }
-
-                    span.end();
+              } else {
+                // when not in factory mode
+                block = blocks[0];
+              }
+
+              // if block is not null
+              if (block) {
+                await tracer.startActiveSpan("handler", async (span) => {
+                  await indexer.options.transform({
+                    block,
+                    cursor,
+                    endCursor,
+                    finality,
+                    context,
                   });
-                }
-              },
-            );
-            await indexer.hooks.callHook("transaction:commit", {
-              finality,
-              endCursor,
+
+                  span.end();
+                });
+              }
             });
+
             span.end();
           });
+
+          context.cursor = undefined;
+          context.endCursor = undefined;
+          context.finality = undefined;
+
           break;
         }
         case "invalidate": {
           await tracer.startActiveSpan("message invalidate", async (span) => {
-            await sink.invalidate(message.invalidate.cursor);
+            await indexer.hooks.callHook("message:invalidate", { message });
+            span.end();
+          });
+          break;
+        }
+        case "finalize": {
+          await tracer.startActiveSpan("message finalize", async (span) => {
+            await indexer.hooks.callHook("message:finalize", { message });
+            span.end();
           });
           break;
         }
+        case "heartbeat": {
+          await tracer.startActiveSpan("message heartbeat", async (span) => {
+            await indexer.hooks.callHook("message:heartbeat", { message });
+            span.end();
+          });
+          break;
+        }
+        case "systemMessage": {
+          await tracer.startActiveSpan(
+            "message systemMessage",
+            async (span) => {
+              switch (message.systemMessage.output?._tag) {
+                case "stderr": {
+                  consola.warn(message.systemMessage.output.stderr);
+                  break;
+                }
+                case "stdout": {
+                  consola.info(message.systemMessage.output.stdout);
+                  break;
+                }
+                default: {
+                }
+              }
+
+              await indexer.hooks.callHook("message:systemMessage", {
+                message,
+              });
+              span.end();
+            },
+          );
+          break;
+        }
         default: {
           consola.warn("unexpected message", message);
           throw new Error("not implemented");
@@ -320,3 +422,24 @@ export async function run<TFilter, TBlock, TTxnParams>(
     }
   });
 }
+
+async function registerMiddleware<TFilter, TBlock>(
+  indexer: Indexer<TFilter, TBlock>,
+): Promise<MiddlewareFunction<IndexerContext>> {
+  const middleware: MiddlewareFunction<IndexerContext>[] = [];
+  const use = (fn: MiddlewareFunction<IndexerContext>) => {
+    middleware.push(fn);
+  };
+
+  await indexer.hooks.callHook("handler:middleware", { use });
+
+  const composed = compose(middleware);
+
+  // Return a named function to help debugging
+  return async function _composedIndexerMiddleware(
+    context: IndexerContext,
+    next?: NextFunction,
+  ) {
+    await composed(context, next);
+  };
+}

package/src/internal/index.ts

@@ -0,0 +1,6 @@
+export function generateIndexerId(fileBasedName: string, identifier?: string) {
+  return `indexer_${fileBasedName}_${identifier || "default"}`.replace(
+    /[^a-zA-Z0-9_]/g,
+    "_",
+  );
+}

package/src/internal/plugins.ts

@@ -0,0 +1 @@
+export * from "../plugins/context";