@apibara/indexer 2.1.0-beta.3 → 2.1.0-beta.30

package/src/internal/testing.ts

@@ -5,10 +5,9 @@ import {
   MockStream,
   type MockStreamResponse,
 } from "@apibara/protocol/testing";
-
 import { useIndexerContext } from "../context";
 import { type IndexerConfig, createIndexer, defineIndexer } from "../indexer";
-import { defineIndexerPlugin } from "../plugins";
+import { defineIndexerPlugin, logger } from "../plugins";
 import { type InternalContext, internalContext } from "./plugins";
 
 export type MockMessagesOptions = {
@@ -20,6 +19,8 @@ export type MockMessagesOptions = {
     finalizeToIndex: number;
     finalizeTriggerIndex: number;
   };
+  uniqueKey?: boolean;
+  baseBlockNumber?: bigint;
 };
 
 export function generateMockMessages(
@@ -30,33 +31,50 @@ export function generateMockMessages(
   const finalizeAt = options?.finalize;
   const messages: MockStreamResponse[] = [];
 
+  const baseBlockNumber = options?.baseBlockNumber ?? BigInt(5_000_000);
+
   for (let i = 0; i < count; i++) {
+    const currentBlockNumber = baseBlockNumber + BigInt(i);
+    const uniqueKey = uniqueKeyFromOrderKey(currentBlockNumber);
     if (invalidateAt && i === invalidateAt.invalidateTriggerIndex) {
+      const invalidateToBlock =
+        baseBlockNumber + BigInt(invalidateAt.invalidateFromIndex);
       messages.push({
         _tag: "invalidate",
         invalidate: {
           cursor: {
-            orderKey: BigInt(5_000_000 + invalidateAt.invalidateFromIndex),
+            orderKey: invalidateToBlock,
+            uniqueKey: options?.uniqueKey
+              ? uniqueKeyFromOrderKey(invalidateToBlock)
+              : undefined,
           },
-        },
-      } as Invalidate);
+        } as Invalidate,
+      });
     } else if (finalizeAt && i === finalizeAt.finalizeTriggerIndex) {
+      const fianlizedToBlock =
+        baseBlockNumber + BigInt(finalizeAt.finalizeToIndex);
       messages.push({
         _tag: "finalize",
         finalize: {
           cursor: {
-            orderKey: BigInt(5_000_000 + finalizeAt.finalizeToIndex),
+            orderKey: fianlizedToBlock,
+            uniqueKey: options?.uniqueKey
+              ? uniqueKeyFromOrderKey(fianlizedToBlock)
+              : undefined,
           },
-        },
-      } as Finalize);
+        } as Finalize,
+      });
     } else {
       messages.push({
         _tag: "data",
         data: {
-          cursor: { orderKey: BigInt(5_000_000 + i - 1) },
+          cursor: { orderKey: currentBlockNumber - 1n },
           finality: "accepted",
-          data: [{ data: `${5_000_000 + i}` }],
-          endCursor: { orderKey: BigInt(5_000_000 + i) },
+          data: [{ data: `${baseBlockNumber + BigInt(i)}` }],
+          endCursor: {
+            orderKey: currentBlockNumber,
+            uniqueKey: options?.uniqueKey ? uniqueKey : undefined,
+          },
           production: "backfill",
         },
       });
@@ -66,6 +84,10 @@ export function generateMockMessages(
   return messages;
 }
 
+function uniqueKeyFromOrderKey(orderKey: bigint): `0x${string}` {
+  return `0xff00${orderKey.toString()}`;
+}
+
 type MockIndexerParams = {
   internalContext?: InternalContext;
   override?: Partial<IndexerConfig<MockFilter, MockBlock>>;
@@ -82,6 +104,7 @@ export function getMockIndexer(params?: MockIndexerParams) {
       filter: {},
       async transform() {},
       plugins: [
+        logger(),
         internalContext(
           contextParams ??
             ({

package/src/otel.ts

@@ -1,3 +1,30 @@
-import { trace } from "@opentelemetry/api";
+import { metrics, trace } from "@opentelemetry/api";
 
-export const tracer = trace.getTracer("@apibara/indexer");
+export function createTracer() {
+  return trace.getTracer("@apibara/indexer");
+}
+
+export function createIndexerMetrics() {
+  const meter = metrics.getMeter("@apibara/indexer");
+
+  const currentBlockGauge = meter.createGauge("current_block", {
+    description: "Current block number being processed",
+    unit: "{block}",
+  });
+
+  const processedBlockCounter = meter.createCounter("processed_blocks", {
+    description: "Number of blocks processed",
+    unit: "{blocks}",
+  });
+
+  const reorgCounter = meter.createCounter("reorgs", {
+    description: "Number of reorgs (invalidate messages) received",
+    unit: "{reorgs}",
+  });
+
+  return {
+    currentBlockGauge,
+    processedBlockCounter,
+    reorgCounter,
+  };
+}

package/src/testing/index.ts

@@ -1,5 +1,6 @@
 import { createClient } from "@apibara/protocol";
 import ci from "ci-info";
+import { useIndexerContext } from "../context";
 import { type IndexerWithStreamConfig, createIndexer } from "../indexer";
 import { type InternalContext, internalContext } from "../plugins/context";
 import { logger } from "../plugins/logger";
@@ -8,7 +9,11 @@ import { isCassetteAvailable } from "../vcr/helper";
 import { record } from "../vcr/record";
 import { replay } from "../vcr/replay";
 
+export type VcrResult = Record<string, unknown>;
+
 export function createVcr() {
+  let result: VcrResult;
+
   return {
     async run<TFilter, TBlock>(
       cassetteName: string,
@@ -40,6 +45,10 @@ export function createVcr() {
 
       const indexer = createIndexer(indexerConfig);
 
+      indexer.hooks.hook("run:after", () => {
+        result = useIndexerContext();
+      });
+
       if (!isCassetteAvailable(vcrConfig, cassetteName)) {
         if (ci.isCI) {
           throw new Error("Cannot record cassette in CI");
@@ -53,6 +62,8 @@ export function createVcr() {
       } else {
         await replay(vcrConfig, indexer, cassetteName);
       }
+
+      return result;
     },
   };
 }

package/dist/shared/indexer.601ceab0.cjs

@@ -1,7 +0,0 @@
-'use strict';
-
-function defineIndexerPlugin(def) {
-  return def;
-}
-
-exports.defineIndexerPlugin = defineIndexerPlugin;

package/dist/shared/indexer.9b21ddd2.mjs

@@ -1,5 +0,0 @@
-function defineIndexerPlugin(def) {
-  return def;
-}
-
-export { defineIndexerPlugin as d };

package/src/compose.test.ts

@@ -1,76 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { type MiddlewareFunction, type NextFunction, compose } from "./compose";
-
-type C = {
-  bag: Record<string, unknown>;
-  finalized: boolean;
-};
-
-type MiddlewareTuple = MiddlewareFunction<C>;
-
-describe("compose", () => {
-  async function a(context: C, next: NextFunction) {
-    context.bag.log = "log";
-    await next();
-  }
-
-  async function b(context: C, next: NextFunction) {
-    await next();
-    context.bag.headers = "custom-header";
-  }
-
-  async function c(context: C, next: NextFunction) {
-    context.bag.xxx = "yyy";
-    await next();
-    context.bag.zzz = context.bag.xxx;
-  }
-
-  async function handler(context: C, next: NextFunction) {
-    context.bag.log = `${context.bag.log} message`;
-    await next();
-    context.bag.message = "new response";
-  }
-
-  const middleware: MiddlewareTuple[] = [];
-
-  middleware.push(a);
-  middleware.push(b);
-  middleware.push(c);
-  middleware.push(handler);
-
-  it("composes", async () => {
-    const context: C = {
-      bag: {},
-      finalized: false,
-    };
-
-    const composed = compose<C>(middleware);
-    await composed(context);
-
-    expect(context.bag.log).toBeDefined();
-    expect(context.bag.log).toBe("log message");
-    expect(context.bag.headers).toBe("custom-header");
-    expect(context.bag.xxx).toBe("yyy");
-    expect(context.bag.zzz).toBe("yyy");
-    expect(context.finalized).toBe(false);
-  });
-
-  it("accepts a next function", async () => {
-    const context: C = {
-      bag: {},
-      finalized: false,
-    };
-
-    const composed = compose<C>(middleware);
-    await composed(context, async () => {
-      context.finalized = true;
-    });
-
-    expect(context.bag.log).toBeDefined();
-    expect(context.bag.log).toBe("log message");
-    expect(context.bag.headers).toBe("custom-header");
-    expect(context.bag.xxx).toBe("yyy");
-    expect(context.bag.zzz).toBe("yyy");
-    expect(context.finalized).toBe(true);
-  });
-});

package/src/indexer.test.ts

@@ -1,430 +0,0 @@
-import type { Cursor, DataFinality } from "@apibara/protocol";
-import {
-  type MockBlock,
-  MockClient,
-  type MockFilter,
-} from "@apibara/protocol/testing";
-import { describe, expect, it } from "vitest";
-import { type IndexerContext, useMessageMetadataContext } from "./context";
-import { run } from "./indexer";
-import {
-  generateMockMessages,
-  getMockIndexer,
-  mockSink,
-  useMockSink,
-} from "./internal/testing";
-
-async function transform<TData>({
-  block: { data },
-}: {
-  block: { data?: TData };
-  cursor?: Cursor;
-  endCursor?: Cursor;
-  finality?: DataFinality;
-  context: IndexerContext;
-}) {
-  const { cursor, endCursor, finality } = useMessageMetadataContext();
-  const { output } = useMockSink();
-  output.push({
-    data,
-    cursor: cursor?.orderKey,
-    endCursor: endCursor?.orderKey,
-    finality,
-  });
-}
-
-describe("Run Test", () => {
-  it("should stream messages", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return generateMockMessages();
-    });
-
-    const output: unknown[] = [];
-
-    const indexer = getMockIndexer({
-      override: {
-        plugins: [mockSink({ output })],
-        transform,
-      },
-    });
-
-    await run(client, indexer);
-
-    expect(output).toMatchInlineSnapshot(`
-      [
-        {
-          "cursor": 4999999n,
-          "data": "5000000",
-          "endCursor": 5000000n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 5000000n,
-          "data": "5000001",
-          "endCursor": 5000001n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 5000001n,
-          "data": "5000002",
-          "endCursor": 5000002n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 5000002n,
-          "data": "5000003",
-          "endCursor": 5000003n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 5000003n,
-          "data": "5000004",
-          "endCursor": 5000004n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 5000004n,
-          "data": "5000005",
-          "endCursor": 5000005n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 5000005n,
-          "data": "5000006",
-          "endCursor": 5000006n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 5000006n,
-          "data": "5000007",
-          "endCursor": 5000007n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 5000007n,
-          "data": "5000008",
-          "endCursor": 5000008n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 5000008n,
-          "data": "5000009",
-          "endCursor": 5000009n,
-          "finality": "accepted",
-        },
-      ]
-    `);
-  });
-
-  it("factory mode: indexer should merge filters and restart when needed", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      const [_factoryFilter, mainFilter] = request.filter;
-
-      if (Object.keys(mainFilter).length === 0) {
-        expect(request.startingCursor?.orderKey).toEqual(100n);
-
-        return [
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 100n },
-              endCursor: { orderKey: 101n },
-              data: [null, null],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 101n },
-              endCursor: { orderKey: 102n },
-              data: [null, null],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 102n },
-              endCursor: { orderKey: 103n },
-              data: [{ data: "B" }, null],
-              production: "backfill",
-            },
-          },
-        ];
-      }
-
-      if (mainFilter.filter === "B") {
-        expect(request.startingCursor?.orderKey).toEqual(102n);
-
-        return [
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 102n },
-              endCursor: { orderKey: 103n },
-              data: [{ data: "B" }, { data: "103B" }],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 103n },
-              endCursor: { orderKey: 104n },
-              data: [null, { data: "104B" }],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 104n },
-              endCursor: { orderKey: 105n },
-              data: [null, { data: "105B" }],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 105n },
-              endCursor: { orderKey: 106n },
-              data: [{ data: "C" }, { data: "106B" }],
-              production: "backfill",
-            },
-          },
-        ];
-      }
-
-      if (mainFilter.filter === "BC") {
-        expect(request.startingCursor?.orderKey).toEqual(105n);
-
-        return [
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 105n },
-              endCursor: { orderKey: 106n },
-              data: [{ data: "C" }, { data: "106BC" }],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 106n },
-              endCursor: { orderKey: 107n },
-              data: [null, { data: "107BC" }],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 107n },
-              endCursor: { orderKey: 108n },
-              data: [null, { data: "108BC" }],
-              production: "backfill",
-            },
-          },
-        ];
-      }
-
-      return [];
-    });
-
-    const output: unknown[] = [];
-    const metadata: Record<string, unknown> = {};
-
-    const indexer = getMockIndexer({
-      override: {
-        plugins: [mockSink({ output, metadata })],
-        startingCursor: { orderKey: 100n },
-        factory: async ({ block }) => {
-          if (block.data === "B") {
-            return { filter: { filter: "B" } };
-          }
-
-          if (block.data === "C") {
-            return { filter: { filter: "C" } };
-          }
-
-          return {};
-        },
-        transform,
-      },
-    });
-
-    await run(client, indexer);
-
-    expect((metadata.lastCursor as Cursor).orderKey).toEqual(108n);
-    expect((metadata.lastFilter as { filter: unknown }).filter).toEqual("BC");
-
-    expect(output).toMatchInlineSnapshot(`
-      [
-        {
-          "cursor": 102n,
-          "data": "103B",
-          "endCursor": 103n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 103n,
-          "data": "104B",
-          "endCursor": 104n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 104n,
-          "data": "105B",
-          "endCursor": 105n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 105n,
-          "data": "106BC",
-          "endCursor": 106n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 106n,
-          "data": "107BC",
-          "endCursor": 107n,
-          "finality": "accepted",
-        },
-        {
-          "cursor": 107n,
-          "data": "108BC",
-          "endCursor": 108n,
-          "finality": "accepted",
-        },
-      ]
-    `);
-  });
-
-  it("factory mode: last cursor should persist when error is thrown in indexer", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      const [_factoryFilter, mainFilter] = request.filter;
-
-      if (Object.keys(mainFilter).length === 0) {
-        expect(request.startingCursor?.orderKey).toEqual(100n);
-
-        return [
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 100n },
-              endCursor: { orderKey: 101n },
-              data: [null, null],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 101n },
-              endCursor: { orderKey: 102n },
-              data: [null, null],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 102n },
-              endCursor: { orderKey: 103n },
-              data: [{ data: "B" }, null],
-              production: "backfill",
-            },
-          },
-          Error("this error should not occurr!"),
-        ];
-      }
-
-      if (mainFilter.filter === "B") {
-        expect(request.startingCursor?.orderKey).toEqual(102n);
-
-        return [
-          Error("this error should occurr!"),
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 103n },
-              endCursor: { orderKey: 104n },
-              data: [null, { data: "104B" }],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 104n },
-              endCursor: { orderKey: 105n },
-              data: [null, { data: "105B" }],
-              production: "backfill",
-            },
-          },
-          {
-            _tag: "data",
-            data: {
-              finality: "accepted",
-              cursor: { orderKey: 105n },
-              endCursor: { orderKey: 106n },
-              data: [{ data: "C" }, { data: "106B" }],
-              production: "backfill",
-            },
-          },
-        ];
-      }
-
-      return [];
-    });
-
-    const output: unknown[] = [];
-    const metadata: Record<string, unknown> = {};
-
-    const indexer = getMockIndexer({
-      override: {
-        plugins: [mockSink({ output, metadata })],
-        startingCursor: { orderKey: 100n },
-        factory: async ({ block }) => {
-          if (block.data === "B") {
-            return { filter: { filter: "B" } };
-          }
-
-          if (block.data === "C") {
-            return { filter: { filter: "C" } };
-          }
-
-          return {};
-        },
-        transform,
-      },
-    });
-
-    await expect(() => run(client, indexer)).rejects.toThrowError(
-      "this error should occurr!",
-    );
-
-    expect((metadata.lastCursor as Cursor).orderKey).toEqual(103n);
-    expect((metadata.lastFilter as { filter: unknown }).filter).toEqual("B");
-
-    expect(output).toMatchInlineSnapshot("[]");
-  });
-});