@apibara/indexer 2.0.0-beta.9 → 2.1.0-beta.2

package/src/context.ts

@@ -1,19 +1,25 @@
 import { AsyncLocalStorage } from "node:async_hooks";
+import type { Cursor, DataFinality } from "@apibara/protocol";
 import { getContext } from "unctx";
-import type { Sink } from "./sink";
 
 // biome-ignore lint/suspicious/noExplicitAny: context type
-export interface IndexerContext<TTxnParams = any> extends Record<string, any> {
-  sink?: Sink<TTxnParams>;
-  sinkTransaction?: TTxnParams;
-}
+export interface IndexerContext extends Record<string, any> {}
 
 export const indexerAsyncContext = getContext<IndexerContext>("indexer", {
   asyncContext: true,
   AsyncLocalStorage,
 });
 
-// biome-ignore lint/suspicious/noExplicitAny: <explanation>
-export function useIndexerContext<TTxnParams = any>() {
-  return indexerAsyncContext.use() as IndexerContext<TTxnParams>;
+export function useIndexerContext() {
+  return indexerAsyncContext.use() as IndexerContext;
+}
+
+export interface MessageMetadataContext extends IndexerContext {
+  cursor?: Cursor;
+  endCursor?: Cursor;
+  finality?: DataFinality;
+}
+
+export function useMessageMetadataContext(): MessageMetadataContext {
+  return useIndexerContext() as MessageMetadataContext;
 }

package/src/index.ts

@@ -1,7 +1,2 @@
 export * from "./indexer";
-export * from "./sink";
 export { useIndexerContext } from "./context";
-
-export * from "./plugins";
-export * from "./vcr";
-export * from "./hooks";

package/src/indexer.test.ts

@@ -1,15 +1,37 @@
+import type { Cursor, DataFinality } from "@apibara/protocol";
 import {
   type MockBlock,
   MockClient,
   type MockFilter,
 } from "@apibara/protocol/testing";
-import Database from "better-sqlite3";
 import { describe, expect, it } from "vitest";
-import { useSink } from "./hooks";
+import { type IndexerContext, useMessageMetadataContext } from "./context";
 import { run } from "./indexer";
-import { SqlitePersistence, sqlitePersistence } from "./plugins/persistence";
-import { generateMockMessages, vcr } from "./testing";
-import { getMockIndexer } from "./testing/indexer";
+import {
+  generateMockMessages,
+  getMockIndexer,
+  mockSink,
+  useMockSink,
+} from "./internal/testing";
+
+async function transform<TData>({
+  block: { data },
+}: {
+  block: { data?: TData };
+  cursor?: Cursor;
+  endCursor?: Cursor;
+  finality?: DataFinality;
+  context: IndexerContext;
+}) {
+  const { cursor, endCursor, finality } = useMessageMetadataContext();
+  const { output } = useMockSink();
+  output.push({
+    data,
+    cursor: cursor?.orderKey,
+    endCursor: endCursor?.orderKey,
+    finality,
+  });
+}
 
 describe("Run Test", () => {
   it("should stream messages", async () => {
@@ -17,121 +39,78 @@ describe("Run Test", () => {
       return generateMockMessages();
     });
 
-    const sink = vcr();
+    const output: unknown[] = [];
 
     const indexer = getMockIndexer({
-      sink,
       override: {
-        transform: async ({ context, endCursor, block: { data } }) => {
-          const { writer } = useSink({ context });
-          writer.insert([{ data }]);
-        },
+        plugins: [mockSink({ output })],
+        transform,
       },
     });
 
     await run(client, indexer);
 
-    expect(sink.result).toMatchInlineSnapshot(`
+    expect(output).toMatchInlineSnapshot(`
       [
         {
-          "data": [
-            {
-              "data": "5000000",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000000n,
-          },
+          "cursor": 4999999n,
+          "data": "5000000",
+          "endCursor": 5000000n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "5000001",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000001n,
-          },
+          "cursor": 5000000n,
+          "data": "5000001",
+          "endCursor": 5000001n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "5000002",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000002n,
-          },
+          "cursor": 5000001n,
+          "data": "5000002",
+          "endCursor": 5000002n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "5000003",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000003n,
-          },
+          "cursor": 5000002n,
+          "data": "5000003",
+          "endCursor": 5000003n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "5000004",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000004n,
-          },
+          "cursor": 5000003n,
+          "data": "5000004",
+          "endCursor": 5000004n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "5000005",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000005n,
-          },
+          "cursor": 5000004n,
+          "data": "5000005",
+          "endCursor": 5000005n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "5000006",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000006n,
-          },
+          "cursor": 5000005n,
+          "data": "5000006",
+          "endCursor": 5000006n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "5000007",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000007n,
-          },
+          "cursor": 5000006n,
+          "data": "5000007",
+          "endCursor": 5000007n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "5000008",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000008n,
-          },
+          "cursor": 5000007n,
+          "data": "5000008",
+          "endCursor": 5000008n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "5000009",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 5000009n,
-          },
+          "cursor": 5000008n,
+          "data": "5000009",
+          "endCursor": 5000009n,
+          "finality": "accepted",
         },
       ]
     `);
@@ -152,6 +131,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 100n },
               endCursor: { orderKey: 101n },
               data: [null, null],
+              production: "backfill",
             },
           },
           {
@@ -161,6 +141,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 101n },
               endCursor: { orderKey: 102n },
               data: [null, null],
+              production: "backfill",
             },
           },
           {
@@ -170,6 +151,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 102n },
               endCursor: { orderKey: 103n },
               data: [{ data: "B" }, null],
+              production: "backfill",
             },
           },
         ];
@@ -186,6 +168,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 102n },
               endCursor: { orderKey: 103n },
               data: [{ data: "B" }, { data: "103B" }],
+              production: "backfill",
             },
           },
           {
@@ -195,6 +178,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 103n },
               endCursor: { orderKey: 104n },
               data: [null, { data: "104B" }],
+              production: "backfill",
             },
           },
           {
@@ -204,6 +188,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 104n },
               endCursor: { orderKey: 105n },
               data: [null, { data: "105B" }],
+              production: "backfill",
             },
           },
           {
@@ -213,6 +198,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 105n },
               endCursor: { orderKey: 106n },
               data: [{ data: "C" }, { data: "106B" }],
+              production: "backfill",
             },
           },
         ];
@@ -229,6 +215,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 105n },
               endCursor: { orderKey: 106n },
               data: [{ data: "C" }, { data: "106BC" }],
+              production: "backfill",
             },
           },
           {
@@ -238,6 +225,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 106n },
               endCursor: { orderKey: 107n },
               data: [null, { data: "107BC" }],
+              production: "backfill",
             },
           },
           {
@@ -247,6 +235,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 107n },
               endCursor: { orderKey: 108n },
               data: [null, { data: "108BC" }],
+              production: "backfill",
             },
           },
         ];
@@ -255,19 +244,12 @@ describe("Run Test", () => {
       return [];
     });
 
-    const db = Database(":memory:");
+    const output: unknown[] = [];
+    const metadata: Record<string, unknown> = {};
 
-    const sink = vcr();
-
-    // create mock indexer with persistence plugin
     const indexer = getMockIndexer({
-      plugins: [
-        sqlitePersistence({
-          database: db,
-        }),
-      ],
-      sink,
       override: {
+        plugins: [mockSink({ output, metadata })],
         startingCursor: { orderKey: 100n },
         factory: async ({ block }) => {
           if (block.data === "B") {
@@ -280,88 +262,55 @@ describe("Run Test", () => {
 
           return {};
         },
-        transform: async ({ context, endCursor, block: { data } }) => {
-          const { writer } = useSink({ context });
-          writer.insert([{ data }]);
-        },
+        transform,
       },
     });
 
     await run(client, indexer);
 
-    const store = new SqlitePersistence<MockFilter>(db);
-
-    const latest = store.get();
+    expect((metadata.lastCursor as Cursor).orderKey).toEqual(108n);
+    expect((metadata.lastFilter as { filter: unknown }).filter).toEqual("BC");
 
-    expect(latest.cursor?.orderKey).toEqual(108n);
-    expect(latest.filter?.filter).toEqual("BC");
-
-    expect(sink.result).toMatchInlineSnapshot(`
+    expect(output).toMatchInlineSnapshot(`
       [
         {
-          "data": [
-            {
-              "data": "103B",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 103n,
-          },
+          "cursor": 102n,
+          "data": "103B",
+          "endCursor": 103n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "104B",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 104n,
-          },
+          "cursor": 103n,
+          "data": "104B",
+          "endCursor": 104n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "105B",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 105n,
-          },
+          "cursor": 104n,
+          "data": "105B",
+          "endCursor": 105n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "106BC",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 106n,
-          },
+          "cursor": 105n,
+          "data": "106BC",
+          "endCursor": 106n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "107BC",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 107n,
-          },
+          "cursor": 106n,
+          "data": "107BC",
+          "endCursor": 107n,
+          "finality": "accepted",
         },
         {
-          "data": [
-            {
-              "data": "108BC",
-            },
-          ],
-          "endCursor": {
-            "orderKey": 108n,
-          },
+          "cursor": 107n,
+          "data": "108BC",
+          "endCursor": 108n,
+          "finality": "accepted",
         },
       ]
     `);
-
-    db.close();
   });
 
   it("factory mode: last cursor should persist when error is thrown in indexer", async () => {
@@ -379,6 +328,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 100n },
               endCursor: { orderKey: 101n },
               data: [null, null],
+              production: "backfill",
             },
           },
           {
@@ -388,6 +338,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 101n },
               endCursor: { orderKey: 102n },
               data: [null, null],
+              production: "backfill",
             },
           },
           {
@@ -397,6 +348,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 102n },
               endCursor: { orderKey: 103n },
               data: [{ data: "B" }, null],
+              production: "backfill",
             },
           },
           Error("this error should not occurr!"),
@@ -415,6 +367,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 103n },
               endCursor: { orderKey: 104n },
               data: [null, { data: "104B" }],
+              production: "backfill",
             },
           },
           {
@@ -424,6 +377,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 104n },
               endCursor: { orderKey: 105n },
               data: [null, { data: "105B" }],
+              production: "backfill",
             },
           },
           {
@@ -433,6 +387,7 @@ describe("Run Test", () => {
               cursor: { orderKey: 105n },
               endCursor: { orderKey: 106n },
               data: [{ data: "C" }, { data: "106B" }],
+              production: "backfill",
             },
           },
         ];
@@ -441,19 +396,12 @@ describe("Run Test", () => {
       return [];
     });
 
-    const db = Database(":memory:");
+    const output: unknown[] = [];
+    const metadata: Record<string, unknown> = {};
 
-    const sink = vcr();
-
-    // create mock indexer with persistence plugin
     const indexer = getMockIndexer({
-      plugins: [
-        sqlitePersistence({
-          database: db,
-        }),
-      ],
-      sink,
       override: {
+        plugins: [mockSink({ output, metadata })],
         startingCursor: { orderKey: 100n },
         factory: async ({ block }) => {
           if (block.data === "B") {
@@ -466,10 +414,7 @@ describe("Run Test", () => {
 
           return {};
         },
-        transform: async ({ context, endCursor, block: { data } }) => {
-          const { writer } = useSink({ context });
-          writer.insert([{ data }]);
-        },
+        transform,
       },
     });
 
@@ -477,15 +422,9 @@ describe("Run Test", () => {
       "this error should occurr!",
     );
 
-    const store = new SqlitePersistence<MockFilter>(db);
-
-    const latest = store.get();
-
-    expect(latest.cursor?.orderKey).toEqual(103n);
-    expect(latest.filter?.filter).toEqual("B");
-
-    expect(sink.result).toMatchInlineSnapshot("[]");
+    expect((metadata.lastCursor as Cursor).orderKey).toEqual(103n);
+    expect((metadata.lastFilter as { filter: unknown }).filter).toEqual("B");
 
-    db.close();
+    expect(output).toMatchInlineSnapshot("[]");
   });
 });