@apibara/indexer 0.4.1 → 2.0.0-beta.3

package/README.md

@@ -1,14 +1,7 @@
 # `@apibara/indexer`
 
-This package provides type definitions and helper functions to write Apibara
-indexers.
+TODO
 
 ## Installation
 
-Apibara indexers use the Deno javascript runtime. Install the package by importing it from a CDN,
-for example:
-
-```ts
-import { Config } from "https://esm.run/@apibara/indexer";
-```
-
+TODO

package/package.json

@@ -1,62 +1,83 @@
 {
   "name": "@apibara/indexer",
-  "version": "0.4.1",
+  "version": "2.0.0-beta.3",
   "type": "module",
+  "source": "./src/index.ts",
+  "main": "./dist/index.mjs",
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
-      "import": "./dist/index.js",
-      "default": "./dist/index.js"
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs",
+      "default": "./dist/index.mjs"
     },
-    "./sink/console": {
-      "types": "./dist/sink/console.d.ts",
-      "import": "./dist/sink/console.js",
-      "default": "./dist/sink/console.js"
+    "./sinks/sqlite": {
+      "types": "./dist/sinks/sqlite.d.ts",
+      "import": "./dist/sinks/sqlite.mjs",
+      "require": "./dist/sinks/sqlite.cjs",
+      "default": "./dist/sinks/sqlite.mjs"
     },
-    "./sink/mongo": {
-      "types": "./dist/sink/mongo.d.ts",
-      "import": "./dist/sink/mongo.js",
-      "default": "./dist/sink/mongo.js"
+    "./sinks/csv": {
+      "types": "./dist/sinks/csv.d.ts",
+      "import": "./dist/sinks/csv.mjs",
+      "require": "./dist/sinks/csv.cjs",
+      "default": "./dist/sinks/csv.mjs"
     },
-    "./sink/parquet": {
-      "types": "./dist/sink/parquet.d.ts",
-      "import": "./dist/sink/parquet.js",
-      "default": "./dist/sink/parquet.js"
+    "./sinks/drizzle": {
+      "types": "./dist/sinks/drizzle/index.d.ts",
+      "import": "./dist/sinks/drizzle/index.mjs",
+      "require": "./dist/sinks/drizzle/index.cjs",
+      "default": "./dist/sinks/drizzle/index.mjs"
     },
-    "./sink/postgres": {
-      "types": "./dist/sink/postgres.d.ts",
-      "import": "./dist/sink/postgres.js",
-      "default": "./dist/sink/postgres.js"
-    },
-    "./sink/webhook": {
-      "types": "./dist/sink/webhook.d.ts",
-      "import": "./dist/sink/webhook.js",
-      "default": "./dist/sink/webhook.js"
-    },
-    "./starknet": {
-      "types": "./dist/starknet/index.d.ts",
-      "import": "./dist/starknet/index.js",
-      "default": "./dist/starknet/index.js"
+    "./testing": {
+      "types": "./dist/testing/index.d.ts",
+      "import": "./dist/testing/index.mjs",
+      "require": "./dist/testing/index.cjs",
+      "default": "./dist/testing/index.mjs"
     }
   },
+  "publishConfig": {},
+  "scripts": {
+    "build": "unbuild",
+    "lint": "biome check .",
+    "typecheck": "tsc --noEmit",
+    "lint:fix": "pnpm lint --write",
+    "test": "vitest",
+    "test:ci": "vitest run",
+    "format": "biome format . --write"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.11",
+    "@types/node": "^20.14.0",
+    "@types/pg": "^8.11.10",
+    "better-sqlite3": "^11.1.2",
+    "csv-stringify": "^6.5.0",
+    "drizzle-orm": "^0.33.0",
+    "pg": "^8.12.0",
+    "postgres-range": "^1.1.4",
+    "unbuild": "^2.0.0",
+    "vitest": "^1.6.0"
+  },
+  "dependencies": {
+    "@apibara/protocol": "2.0.0-beta.3",
+    "@opentelemetry/api": "^1.9.0",
+    "consola": "^3.2.3",
+    "hookable": "^5.5.3",
+    "klona": "^2.0.6",
+    "nice-grpc": "^2.1.8",
+    "unctx": "^2.3.1"
+  },
+  "peerDependencies": {
+    "better-sqlite3": "^11.1.2",
+    "csv-stringify": "^6.5.0",
+    "drizzle-orm": "^0.33.0",
+    "postgres-range": "^1.1.4",
+    "vitest": "^1.6.0"
+  },
   "files": [
     "dist",
     "src",
     "README.md"
   ],
-  "devDependencies": {
-    "vitest": "^0.34.3"
-  },
-  "dependencies": {
-    "starknet": "^5.24.3",
-    "zod": "^3.22.2"
-  },
-  "scripts": {
-    "build": "tsc",
-    "test": "vitest",
-    "test:typecheck": "vitest typecheck",
-    "lint": "biome check .",
-    "lint:fix": "pnpm lint --apply",
-    "format": "biome format . --write"
-  }
-}
+  "types": "./dist/index.d.ts"
+}

package/src/context.ts

@@ -0,0 +1,19 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { getContext } from "unctx";
+import type { Sink } from "./sink";
+
+// biome-ignore lint/suspicious/noExplicitAny: context type
+export interface IndexerContext<TTxnParams = any> extends Record<string, any> {
+  sink?: Sink<TTxnParams>;
+  sinkTransaction?: TTxnParams;
+}
+
+export const indexerAsyncContext = getContext<IndexerContext>("indexer", {
+  asyncContext: true,
+  AsyncLocalStorage,
+});
+
+// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+export function useIndexerContext<TTxnParams = any>() {
+  return indexerAsyncContext.use() as IndexerContext<TTxnParams>;
+}

package/src/hooks/index.ts

@@ -0,0 +1,2 @@
+export * from "./useKVStore";
+export * from "./useSink";

package/src/hooks/useKVStore.ts

@@ -0,0 +1,12 @@
+import { useIndexerContext } from "../context";
+import type { KVStore } from "../plugins/kv";
+
+export type UseKVStoreResult = InstanceType<typeof KVStore>;
+
+export function useKVStore(): UseKVStoreResult {
+  const ctx = useIndexerContext();
+
+  if (!ctx?.kv) throw new Error("KV Plugin is not available in context!");
+
+  return ctx.kv;
+}

package/src/hooks/useSink.ts

@@ -0,0 +1,13 @@
+import type { IndexerContext } from "../context";
+
+export function useSink<TTxnParams>({
+  context,
+}: {
+  context: IndexerContext<TTxnParams>;
+}) {
+  if (!context.sinkTransaction) {
+    throw new Error("Transaction context doesn't exist!");
+  }
+
+  return context.sinkTransaction;
+}

package/src/index.ts

@@ -1 +1,7 @@
-export * from "./config";
+export * from "./indexer";
+export * from "./sink";
+export { useIndexerContext } from "./context";
+
+export * from "./plugins";
+export * from "./vcr";
+export * from "./hooks";

package/src/indexer.test.ts

@@ -0,0 +1,491 @@
+import {
+  type MockBlock,
+  MockClient,
+  type MockFilter,
+} from "@apibara/protocol/testing";
+import Database from "better-sqlite3";
+import { describe, expect, it } from "vitest";
+import { useSink } from "./hooks";
+import { run } from "./indexer";
+import { SqlitePersistence, sqlitePersistence } from "./plugins/persistence";
+import { generateMockMessages, vcr } from "./testing";
+import { getMockIndexer } from "./testing/indexer";
+
+describe("Run Test", () => {
+  it("should stream messages", async () => {
+    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
+      return generateMockMessages();
+    });
+
+    const sink = vcr();
+
+    const indexer = getMockIndexer({
+      sink,
+      override: {
+        transform: async ({ context, endCursor, block: { data } }) => {
+          const { writer } = useSink({ context });
+          writer.insert([{ data }]);
+        },
+      },
+    });
+
+    await run(client, indexer);
+
+    expect(sink.result).toMatchInlineSnapshot(`
+      [
+        {
+          "data": [
+            {
+              "data": "5000000",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000000n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "5000001",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000001n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "5000002",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000002n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "5000003",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000003n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "5000004",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000004n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "5000005",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000005n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "5000006",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000006n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "5000007",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000007n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "5000008",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000008n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "5000009",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 5000009n,
+          },
+        },
+      ]
+    `);
+  });
+
+  it("factory mode: indexer should merge filters and restart when needed", async () => {
+    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
+      const [_factoryFilter, mainFilter] = request.filter;
+
+      if (Object.keys(mainFilter).length === 0) {
+        expect(request.startingCursor?.orderKey).toEqual(100n);
+
+        return [
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 100n },
+              endCursor: { orderKey: 101n },
+              data: [null, null],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 101n },
+              endCursor: { orderKey: 102n },
+              data: [null, null],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 102n },
+              endCursor: { orderKey: 103n },
+              data: [{ data: "B" }, null],
+            },
+          },
+        ];
+      }
+
+      if (mainFilter.filter === "B") {
+        expect(request.startingCursor?.orderKey).toEqual(102n);
+
+        return [
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 102n },
+              endCursor: { orderKey: 103n },
+              data: [{ data: "B" }, { data: "103B" }],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 103n },
+              endCursor: { orderKey: 104n },
+              data: [null, { data: "104B" }],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 104n },
+              endCursor: { orderKey: 105n },
+              data: [null, { data: "105B" }],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 105n },
+              endCursor: { orderKey: 106n },
+              data: [{ data: "C" }, { data: "106B" }],
+            },
+          },
+        ];
+      }
+
+      if (mainFilter.filter === "BC") {
+        expect(request.startingCursor?.orderKey).toEqual(105n);
+
+        return [
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 105n },
+              endCursor: { orderKey: 106n },
+              data: [{ data: "C" }, { data: "106BC" }],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 106n },
+              endCursor: { orderKey: 107n },
+              data: [null, { data: "107BC" }],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 107n },
+              endCursor: { orderKey: 108n },
+              data: [null, { data: "108BC" }],
+            },
+          },
+        ];
+      }
+
+      return [];
+    });
+
+    const db = Database(":memory:");
+
+    const sink = vcr();
+
+    // create mock indexer with persistence plugin
+    const indexer = getMockIndexer({
+      plugins: [
+        sqlitePersistence({
+          database: db,
+        }),
+      ],
+      sink,
+      override: {
+        startingCursor: { orderKey: 100n },
+        factory: async ({ block }) => {
+          if (block.data === "B") {
+            return { filter: { filter: "B" } };
+          }
+
+          if (block.data === "C") {
+            return { filter: { filter: "C" } };
+          }
+
+          return {};
+        },
+        transform: async ({ context, endCursor, block: { data } }) => {
+          const { writer } = useSink({ context });
+          writer.insert([{ data }]);
+        },
+      },
+    });
+
+    await run(client, indexer);
+
+    const store = new SqlitePersistence<MockFilter>(db);
+
+    const latest = store.get();
+
+    expect(latest.cursor?.orderKey).toEqual(108n);
+    expect(latest.filter?.filter).toEqual("BC");
+
+    expect(sink.result).toMatchInlineSnapshot(`
+      [
+        {
+          "data": [
+            {
+              "data": "103B",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 103n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "104B",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 104n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "105B",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 105n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "106BC",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 106n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "107BC",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 107n,
+          },
+        },
+        {
+          "data": [
+            {
+              "data": "108BC",
+            },
+          ],
+          "endCursor": {
+            "orderKey": 108n,
+          },
+        },
+      ]
+    `);
+
+    db.close();
+  });
+
+  it("factory mode: last cursor should persist when error is thrown in indexer", async () => {
+    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
+      const [_factoryFilter, mainFilter] = request.filter;
+
+      if (Object.keys(mainFilter).length === 0) {
+        expect(request.startingCursor?.orderKey).toEqual(100n);
+
+        return [
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 100n },
+              endCursor: { orderKey: 101n },
+              data: [null, null],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 101n },
+              endCursor: { orderKey: 102n },
+              data: [null, null],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 102n },
+              endCursor: { orderKey: 103n },
+              data: [{ data: "B" }, null],
+            },
+          },
+          Error("this error should not occurr!"),
+        ];
+      }
+
+      if (mainFilter.filter === "B") {
+        expect(request.startingCursor?.orderKey).toEqual(102n);
+
+        return [
+          Error("this error should occurr!"),
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 103n },
+              endCursor: { orderKey: 104n },
+              data: [null, { data: "104B" }],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 104n },
+              endCursor: { orderKey: 105n },
+              data: [null, { data: "105B" }],
+            },
+          },
+          {
+            _tag: "data",
+            data: {
+              finality: "accepted",
+              cursor: { orderKey: 105n },
+              endCursor: { orderKey: 106n },
+              data: [{ data: "C" }, { data: "106B" }],
+            },
+          },
+        ];
+      }
+
+      return [];
+    });
+
+    const db = Database(":memory:");
+
+    const sink = vcr();
+
+    // create mock indexer with persistence plugin
+    const indexer = getMockIndexer({
+      plugins: [
+        sqlitePersistence({
+          database: db,
+        }),
+      ],
+      sink,
+      override: {
+        startingCursor: { orderKey: 100n },
+        factory: async ({ block }) => {
+          if (block.data === "B") {
+            return { filter: { filter: "B" } };
+          }
+
+          if (block.data === "C") {
+            return { filter: { filter: "C" } };
+          }
+
+          return {};
+        },
+        transform: async ({ context, endCursor, block: { data } }) => {
+          const { writer } = useSink({ context });
+          writer.insert([{ data }]);
+        },
+      },
+    });
+
+    await expect(() => run(client, indexer)).rejects.toThrowError(
+      "this error should occurr!",
+    );
+
+    const store = new SqlitePersistence<MockFilter>(db);
+
+    const latest = store.get();
+
+    expect(latest.cursor?.orderKey).toEqual(103n);
+    expect(latest.filter?.filter).toEqual("B");
+
+    expect(sink.result).toMatchInlineSnapshot("[]");
+
+    db.close();
+  });
+});