@apibara/indexer 2.1.0-beta.4 → 2.1.0-beta.40

package/src/indexer.ts

@@ -1,11 +1,13 @@
 import {
   type Client,
   ClientError,
+  type CreateClientOptions,
   type Cursor,
   type DataFinality,
+  type DataProduction,
   type Finalize,
-  type Heartbeat,
   type Invalidate,
+  ServerError,
   Status,
   type StreamConfig,
   type StreamDataOptions,
@@ -28,14 +30,16 @@ import {
   indexerAsyncContext,
   useIndexerContext,
 } from "./context";
-import { tracer } from "./otel";
+import { createIndexerMetrics, createTracer } from "./otel";
 import type { IndexerPlugin } from "./plugins";
+import { useInternalContext } from "./plugins/context";
 
 export type UseMiddlewareFunction = (
   fn: MiddlewareFunction<IndexerContext>,
 ) => void;
 
 export interface IndexerHooks<TFilter, TBlock> {
+  "plugins:init": () => void;
   "run:before": () => void;
   "run:after": () => void;
   "connect:before": ({
@@ -47,7 +51,9 @@ export interface IndexerHooks<TFilter, TBlock> {
   }) => void;
   "connect:after": ({
     request,
-  }: { request: StreamDataRequest<TFilter> }) => void;
+  }: {
+    request: StreamDataRequest<TFilter>;
+  }) => void;
   "connect:factory": ({
     request,
     endCursor,
@@ -59,7 +65,7 @@ export interface IndexerHooks<TFilter, TBlock> {
   message: ({ message }: { message: StreamDataResponse<TBlock> }) => void;
   "message:invalidate": ({ message }: { message: Invalidate }) => void;
   "message:finalize": ({ message }: { message: Finalize }) => void;
-  "message:heartbeat": ({ message }: { message: Heartbeat }) => void;
+  "message:heartbeat": () => void;
   "message:systemMessage": ({ message }: { message: SystemMessage }) => void;
 }
 
@@ -77,23 +83,24 @@ export type IndexerStartingCursor =
       startingBlock?: never;
     };
 
+export type HandlerArgs<TBlock> = {
+  block: TBlock;
+  cursor?: Cursor | undefined;
+  endCursor?: Cursor | undefined;
+  finality: DataFinality;
+  production: DataProduction;
+  context: IndexerContext;
+};
+
 export type IndexerConfig<TFilter, TBlock> = {
   streamUrl: string;
   filter: TFilter;
   finality?: DataFinality;
-  factory?: ({
-    block,
-    context,
-  }: { block: TBlock; context: IndexerContext }) => Promise<{
+  clientOptions?: CreateClientOptions;
+  factory?: (args: HandlerArgs<TBlock>) => Promise<{
     filter?: TFilter;
   }>;
-  transform: (args: {
-    block: TBlock;
-    cursor?: Cursor | undefined;
-    endCursor?: Cursor | undefined;
-    finality: DataFinality;
-    context: IndexerContext;
-  }) => Promise<void>;
+  transform: (args: HandlerArgs<TBlock>) => Promise<void>;
   hooks?: NestedHooks<IndexerHooks<TFilter, TBlock>>;
   plugins?: ReadonlyArray<IndexerPlugin<TFilter, TBlock>>;
   debug?: boolean;
@@ -179,13 +186,18 @@ export async function runWithReconnect<TFilter, TBlock>(
 
       retryCount++;
 
-      if (error instanceof ClientError) {
+      if (error instanceof ClientError || error instanceof ServerError) {
+        const isServerError = error instanceof ServerError;
+
         if (error.code === Status.INTERNAL) {
           if (retryCount < maxRetries) {
             consola.error(
-              "Internal server error, reconnecting...",
-              error.message,
+              `Internal ${isServerError ? "server" : "client"} error: ${
+                error.message
+              }`,
             );
+            consola.start("Reconnecting...");
+            console.log();
 
             // Add jitter to the retry delay to avoid all clients retrying at the same time.
             const delay = Math.random() * (retryDelay * 0.2) + retryDelay;
@@ -214,10 +226,22 @@ export async function run<TFilter, TBlock>(
 ) {
   await indexerAsyncContext.callAsync({}, async () => {
     const context = useIndexerContext();
+
+    if (indexer.options.debug) {
+      context.debug = true;
+    }
+
+    await indexer.hooks.callHook("plugins:init");
+
     const middleware = await registerMiddleware(indexer);
 
+    const indexerMetrics = createIndexerMetrics();
+    const tracer = createTracer();
+
     await indexer.hooks.callHook("run:before");
 
+    const { indexerName: indexerId } = useInternalContext();
+
     const isFactoryMode = indexer.options.factory !== undefined;
 
     // Give priority to startingCursor over startingBlock.
@@ -235,13 +259,13 @@ export async function run<TFilter, TBlock>(
     }
 
     // if factory mode we add a empty filter at the end of the filter array.
-    const request = indexer.streamConfig.Request.make({
+    const request = {
       filter: isFactoryMode
         ? [indexer.options.filter, {} as TFilter]
         : [indexer.options.filter],
       finality: indexer.options.finality,
       startingCursor,
-    });
+    } as StreamDataRequest<TFilter>;
 
     const options: StreamDataOptions = {};
 
@@ -282,17 +306,25 @@ export async function run<TFilter, TBlock>(
         case "data": {
           await tracer.startActiveSpan("message data", async (span) => {
             const blocks = message.data.data;
-            const { cursor, endCursor, finality } = message.data;
+            const { cursor, endCursor, finality, production } = message.data;
 
             context.cursor = cursor;
             context.endCursor = endCursor;
             context.finality = finality;
 
+            // Record current block number being processed
+            indexerMetrics.currentBlockGauge.record(
+              Number(endCursor?.orderKey),
+              {
+                indexer_id: indexerId,
+              },
+            );
+
             await middleware(context, async () => {
               let block: TBlock | null;
 
               // when factory mode
-              if (isFactoryMode) {
+              if (isFactoryMode && finality !== "pending") {
                 assert(indexer.options.factory !== undefined);
 
                 const [factoryBlock, mainBlock] = blocks;
@@ -302,6 +334,10 @@ export async function run<TFilter, TBlock>(
                 if (factoryBlock !== null) {
                   const { filter } = await indexer.options.factory({
                     block: factoryBlock,
+                    cursor,
+                    endCursor,
+                    finality,
+                    production,
                     context,
                   });
 
@@ -315,11 +351,11 @@ export async function run<TFilter, TBlock>(
                     );
 
                     // create request with new filters
-                    const request = indexer.streamConfig.Request.make({
+                    const request = {
                       filter: [indexer.options.filter, mainFilter],
                       finality: indexer.options.finality,
                       startingCursor: cursor,
-                    });
+                    } as StreamDataRequest<TFilter>;
 
                     await indexer.hooks.callHook("connect:factory", {
                       request,
@@ -353,6 +389,7 @@ export async function run<TFilter, TBlock>(
                     cursor,
                     endCursor,
                     finality,
+                    production,
                     context,
                   });
 
@@ -364,6 +401,11 @@ export async function run<TFilter, TBlock>(
             span.end();
           });
 
+          // Record processed block metric
+          indexerMetrics.processedBlockCounter.add(1, {
+            indexer_id: indexerId,
+          });
+
           context.cursor = undefined;
           context.endCursor = undefined;
           context.finality = undefined;
@@ -372,21 +414,29 @@ export async function run<TFilter, TBlock>(
         }
         case "invalidate": {
           await tracer.startActiveSpan("message invalidate", async (span) => {
-            await indexer.hooks.callHook("message:invalidate", { message });
+            // Record reorg metric
+            indexerMetrics.reorgCounter.add(1, {
+              indexer_id: indexerId,
+            });
+            await indexer.hooks.callHook("message:invalidate", {
+              message: message.invalidate,
+            });
             span.end();
           });
           break;
         }
         case "finalize": {
           await tracer.startActiveSpan("message finalize", async (span) => {
-            await indexer.hooks.callHook("message:finalize", { message });
+            await indexer.hooks.callHook("message:finalize", {
+              message: message.finalize,
+            });
             span.end();
           });
           break;
         }
         case "heartbeat": {
           await tracer.startActiveSpan("message heartbeat", async (span) => {
-            await indexer.hooks.callHook("message:heartbeat", { message });
+            await indexer.hooks.callHook("message:heartbeat");
             span.end();
           });
           break;
@@ -409,7 +459,7 @@ export async function run<TFilter, TBlock>(
               }
 
               await indexer.hooks.callHook("message:systemMessage", {
-                message,
+                message: message.systemMessage,
               });
               span.end();
             },

package/src/internal/testing.ts

@@ -5,10 +5,9 @@ import {
   MockStream,
   type MockStreamResponse,
 } from "@apibara/protocol/testing";
-
 import { useIndexerContext } from "../context";
 import { type IndexerConfig, createIndexer, defineIndexer } from "../indexer";
-import { defineIndexerPlugin } from "../plugins";
+import { defineIndexerPlugin, logger } from "../plugins";
 import { type InternalContext, internalContext } from "./plugins";
 
 export type MockMessagesOptions = {
@@ -20,6 +19,8 @@ export type MockMessagesOptions = {
     finalizeToIndex: number;
     finalizeTriggerIndex: number;
   };
+  uniqueKey?: boolean;
+  baseBlockNumber?: bigint;
 };
 
 export function generateMockMessages(
@@ -30,33 +31,50 @@ export function generateMockMessages(
   const finalizeAt = options?.finalize;
   const messages: MockStreamResponse[] = [];
 
+  const baseBlockNumber = options?.baseBlockNumber ?? BigInt(5_000_000);
+
   for (let i = 0; i < count; i++) {
+    const currentBlockNumber = baseBlockNumber + BigInt(i);
+    const uniqueKey = uniqueKeyFromOrderKey(currentBlockNumber);
     if (invalidateAt && i === invalidateAt.invalidateTriggerIndex) {
+      const invalidateToBlock =
+        baseBlockNumber + BigInt(invalidateAt.invalidateFromIndex);
       messages.push({
         _tag: "invalidate",
         invalidate: {
           cursor: {
-            orderKey: BigInt(5_000_000 + invalidateAt.invalidateFromIndex),
+            orderKey: invalidateToBlock,
+            uniqueKey: options?.uniqueKey
+              ? uniqueKeyFromOrderKey(invalidateToBlock)
+              : undefined,
           },
-        },
-      } as Invalidate);
+        } as Invalidate,
+      });
     } else if (finalizeAt && i === finalizeAt.finalizeTriggerIndex) {
+      const fianlizedToBlock =
+        baseBlockNumber + BigInt(finalizeAt.finalizeToIndex);
       messages.push({
         _tag: "finalize",
         finalize: {
           cursor: {
-            orderKey: BigInt(5_000_000 + finalizeAt.finalizeToIndex),
+            orderKey: fianlizedToBlock,
+            uniqueKey: options?.uniqueKey
+              ? uniqueKeyFromOrderKey(fianlizedToBlock)
+              : undefined,
           },
-        },
-      } as Finalize);
+        } as Finalize,
+      });
     } else {
       messages.push({
         _tag: "data",
         data: {
-          cursor: { orderKey: BigInt(5_000_000 + i - 1) },
+          cursor: { orderKey: currentBlockNumber - 1n },
           finality: "accepted",
-          data: [{ data: `${5_000_000 + i}` }],
-          endCursor: { orderKey: BigInt(5_000_000 + i) },
+          data: [{ data: `${baseBlockNumber + BigInt(i)}` }],
+          endCursor: {
+            orderKey: currentBlockNumber,
+            uniqueKey: options?.uniqueKey ? uniqueKey : undefined,
+          },
           production: "backfill",
         },
       });
@@ -66,6 +84,10 @@ export function generateMockMessages(
   return messages;
 }
 
+function uniqueKeyFromOrderKey(orderKey: bigint): `0x${string}` {
+  return `0xff00${orderKey.toString()}`;
+}
+
 type MockIndexerParams = {
   internalContext?: InternalContext;
   override?: Partial<IndexerConfig<MockFilter, MockBlock>>;
@@ -82,6 +104,7 @@ export function getMockIndexer(params?: MockIndexerParams) {
       filter: {},
       async transform() {},
       plugins: [
+        logger(),
         internalContext(
           contextParams ??
             ({

package/src/otel.ts

@@ -1,3 +1,30 @@
-import { trace } from "@opentelemetry/api";
+import { metrics, trace } from "@opentelemetry/api";
 
-export const tracer = trace.getTracer("@apibara/indexer");
+export function createTracer() {
+  return trace.getTracer("@apibara/indexer");
+}
+
+export function createIndexerMetrics() {
+  const meter = metrics.getMeter("@apibara/indexer");
+
+  const currentBlockGauge = meter.createGauge("current_block", {
+    description: "Current block number being processed",
+    unit: "{block}",
+  });
+
+  const processedBlockCounter = meter.createCounter("processed_blocks", {
+    description: "Number of blocks processed",
+    unit: "{blocks}",
+  });
+
+  const reorgCounter = meter.createCounter("reorgs", {
+    description: "Number of reorgs (invalidate messages) received",
+    unit: "{reorgs}",
+  });
+
+  return {
+    currentBlockGauge,
+    processedBlockCounter,
+    reorgCounter,
+  };
+}

package/src/plugins/context.ts

@@ -7,7 +7,7 @@ export function internalContext<TFilter, TBlock, TTxnParams>(
   values: Record<string, unknown>,
 ) {
   return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
-    indexer.hooks.hook("run:before", () => {
+    indexer.hooks.hook("plugins:init", () => {
       try {
         const ctx = useIndexerContext();
         ctx[INTERNAL_CONTEXT_PROPERTY] = {

package/src/plugins/logger.ts

@@ -1,4 +1,9 @@
-import { type ConsolaInstance, type ConsolaReporter, consola } from "consola";
+import {
+  type ConsolaInstance,
+  type ConsolaReporter,
+  LogLevels,
+  consola,
+} from "consola";
 import { useIndexerContext } from "../context";
 import { defineIndexerPlugin } from "./config";
 
@@ -8,7 +13,7 @@ export function logger<TFilter, TBlock, TTxnParams>({
   logger,
 }: { logger?: ConsolaReporter } = {}) {
   return defineIndexerPlugin<TFilter, TBlock>((indexer) => {
-    indexer.hooks.hook("run:before", () => {
+    indexer.hooks.hook("plugins:init", () => {
       const ctx = useIndexerContext();
 
       if (logger) {
@@ -16,6 +21,10 @@ export function logger<TFilter, TBlock, TTxnParams>({
       } else {
         ctx.logger = consola.create({});
       }
+
+      if (ctx.debug) {
+        ctx.logger.level = LogLevels.debug;
+      }
     });
   });
 }

package/src/testing/index.ts

@@ -1,6 +1,12 @@
-import { createClient } from "@apibara/protocol";
+import { createAuthenticatedClient } from "@apibara/protocol";
 import ci from "ci-info";
-import { type IndexerWithStreamConfig, createIndexer } from "../indexer";
+import { type NestedHooks, mergeHooks } from "hookable";
+import { useIndexerContext } from "../context";
+import {
+  type IndexerHooks,
+  type IndexerWithStreamConfig,
+  createIndexer,
+} from "../indexer";
 import { type InternalContext, internalContext } from "../plugins/context";
 import { logger } from "../plugins/logger";
 import type { CassetteOptions, VcrConfig } from "../vcr/config";
@@ -8,24 +14,36 @@ import { isCassetteAvailable } from "../vcr/helper";
 import { record } from "../vcr/record";
 import { replay } from "../vcr/replay";
 
+export type VcrResult = Record<string, unknown>;
+
 export function createVcr() {
+  let result: VcrResult;
+
   return {
     async run<TFilter, TBlock>(
       cassetteName: string,
       indexerConfig: IndexerWithStreamConfig<TFilter, TBlock>,
-      range: { fromBlock: bigint; toBlock: bigint },
+      config: {
+        range: { fromBlock: bigint; toBlock: bigint };
+        hooks?: NestedHooks<IndexerHooks<TFilter, TBlock>>;
+      },
     ) {
       const vcrConfig: VcrConfig = {
         cassetteDir: "cassettes",
       };
 
+      indexerConfig.hooks = mergeHooks(
+        indexerConfig.hooks ?? {},
+        config.hooks ?? {},
+      );
+
       const cassetteOptions: CassetteOptions = {
         name: cassetteName,
         startingCursor: {
-          orderKey: range.fromBlock,
+          orderKey: config.range.fromBlock,
         },
         endingCursor: {
-          orderKey: range.toBlock,
+          orderKey: config.range.toBlock,
         },
       };
 
@@ -40,12 +58,16 @@ export function createVcr() {
 
       const indexer = createIndexer(indexerConfig);
 
+      indexer.hooks.hook("run:after", () => {
+        result = useIndexerContext();
+      });
+
       if (!isCassetteAvailable(vcrConfig, cassetteName)) {
         if (ci.isCI) {
           throw new Error("Cannot record cassette in CI");
         }
 
-        const client = createClient(
+        const client = createAuthenticatedClient(
           indexer.streamConfig,
           indexer.options.streamUrl,
         );
@@ -53,6 +75,8 @@ export function createVcr() {
       } else {
         await replay(vcrConfig, indexer, cassetteName);
       }
+
+      return result;
     },
   };
 }

package/dist/shared/indexer.601ceab0.cjs

@@ -1,7 +0,0 @@
-'use strict';
-
-function defineIndexerPlugin(def) {
-  return def;
-}
-
-exports.defineIndexerPlugin = defineIndexerPlugin;

package/dist/shared/indexer.9b21ddd2.mjs

@@ -1,5 +0,0 @@
-function defineIndexerPlugin(def) {
-  return def;
-}
-
-export { defineIndexerPlugin as d };

package/src/compose.test.ts

@@ -1,76 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { type MiddlewareFunction, type NextFunction, compose } from "./compose";
-
-type C = {
-  bag: Record<string, unknown>;
-  finalized: boolean;
-};
-
-type MiddlewareTuple = MiddlewareFunction<C>;
-
-describe("compose", () => {
-  async function a(context: C, next: NextFunction) {
-    context.bag.log = "log";
-    await next();
-  }
-
-  async function b(context: C, next: NextFunction) {
-    await next();
-    context.bag.headers = "custom-header";
-  }
-
-  async function c(context: C, next: NextFunction) {
-    context.bag.xxx = "yyy";
-    await next();
-    context.bag.zzz = context.bag.xxx;
-  }
-
-  async function handler(context: C, next: NextFunction) {
-    context.bag.log = `${context.bag.log} message`;
-    await next();
-    context.bag.message = "new response";
-  }
-
-  const middleware: MiddlewareTuple[] = [];
-
-  middleware.push(a);
-  middleware.push(b);
-  middleware.push(c);
-  middleware.push(handler);
-
-  it("composes", async () => {
-    const context: C = {
-      bag: {},
-      finalized: false,
-    };
-
-    const composed = compose<C>(middleware);
-    await composed(context);
-
-    expect(context.bag.log).toBeDefined();
-    expect(context.bag.log).toBe("log message");
-    expect(context.bag.headers).toBe("custom-header");
-    expect(context.bag.xxx).toBe("yyy");
-    expect(context.bag.zzz).toBe("yyy");
-    expect(context.finalized).toBe(false);
-  });
-
-  it("accepts a next function", async () => {
-    const context: C = {
-      bag: {},
-      finalized: false,
-    };
-
-    const composed = compose<C>(middleware);
-    await composed(context, async () => {
-      context.finalized = true;
-    });
-
-    expect(context.bag.log).toBeDefined();
-    expect(context.bag.log).toBe("log message");
-    expect(context.bag.headers).toBe("custom-header");
-    expect(context.bag.xxx).toBe("yyy");
-    expect(context.bag.zzz).toBe("yyy");
-    expect(context.finalized).toBe(true);
-  });
-});