@apibara/indexer 2.0.0-beta.0 → 2.0.0-beta.4

package/src/sinks/drizzle/update.ts

@@ -0,0 +1,47 @@
+import type { Cursor } from "@apibara/protocol";
+import {
+  type ExtractTablesWithRelations,
+  type SQL,
+  type TablesRelationalConfig,
+  sql,
+} from "drizzle-orm";
+import type {
+  PgQueryResultHKT,
+  PgTable,
+  PgTransaction,
+  PgUpdateBase,
+  PgUpdateSetSource,
+} from "drizzle-orm/pg-core";
+
+export class DrizzleSinkUpdate<
+  TTable extends PgTable,
+  TQueryResult extends PgQueryResultHKT,
+  TFullSchema extends Record<string, unknown> = Record<string, never>,
+  TSchema extends
+    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
+> {
+  constructor(
+    private db: PgTransaction<TQueryResult, TFullSchema, TSchema>,
+    private table: TTable,
+    private endCursor?: Cursor,
+  ) {}
+
+  set(values: PgUpdateSetSource<TTable>): PgUpdateBase<TTable, TQueryResult> {
+    const originalUpdate = this.db.update(this.table);
+    const originalSet = originalUpdate.set(values);
+    return {
+      ...originalSet,
+      where: async (where: SQL | undefined) => {
+        await this.db
+          .update(this.table)
+          .set({
+            _cursor: sql`int8range(lower(_cursor), ${Number(this.endCursor?.orderKey!)}, '[)')`,
+          } as PgUpdateSetSource<TTable>)
+          .where(sql`${where ? sql`${where} AND ` : sql``}upper_inf(_cursor)`)
+          .execute();
+
+        return originalSet.where(where);
+      },
+    } as PgUpdateBase<TTable, TQueryResult>;
+  }
+}

package/src/sinks/drizzle/utils.ts

@@ -0,0 +1,36 @@
+import type { PgTableFn } from "drizzle-orm/pg-core";
+import { pgTable as drizzlePgTable } from "drizzle-orm/pg-core";
+import range from "postgres-range";
+import { Int8Range, int8range } from "./Int8Range";
+
+export const pgTable: PgTableFn = (name, columns, extraConfig?) => {
+  return drizzlePgTable(
+    name,
+    {
+      ...columns,
+      _cursor: int8range("_cursor").notNull(),
+    },
+    extraConfig,
+  );
+};
+
+export const getDrizzleCursor = (
+  cursor_range: [bigint | undefined, bigint | undefined] | bigint | undefined,
+) => {
+  const isArray = Array.isArray(cursor_range);
+  const [lower, upper] = isArray ? cursor_range : [cursor_range, undefined];
+  let isNoUpperBound = false;
+  if (!lower) {
+    throw new Error("Lower bound cursor is required");
+  }
+  if (!upper) {
+    isNoUpperBound = true;
+  }
+  return new Int8Range(
+    new range.Range(
+      Number(lower),
+      Number(upper),
+      range.RANGE_LB_INC | (isNoUpperBound ? range.RANGE_UB_INF : 0),
+    ),
+  );
+};

package/src/sinks/sqlite.test.ts

@@ -5,6 +5,7 @@ import {
 } from "@apibara/protocol/testing";
 import Database from "better-sqlite3";
 import { describe, expect, it } from "vitest";
+import { useSink } from "../hooks";
 import { run } from "../indexer";
 import {} from "../plugins/persistence";
 import { generateMockMessages } from "../testing";
@@ -33,7 +34,18 @@ describe("Run Test", () => {
       database: db,
       tableName: "test",
     });
-    await run(client, getMockIndexer(), sink);
+    await run(
+      client,
+      getMockIndexer({
+        sink,
+        override: {
+          transform: async ({ context, endCursor, block: { data } }) => {
+            const { writer } = useSink({ context });
+            writer.insert([{ data }]);
+          },
+        },
+      }),
+    );
 
     const sinkData = db.prepare("SELECT * FROM test").all();
 

package/src/sinks/sqlite.ts

@@ -1,6 +1,6 @@
 import type { Cursor } from "@apibara/protocol";
 import type { Database as SqliteDatabase } from "better-sqlite3";
-import { Sink, type SinkData, type SinkWriteArgs } from "../sink";
+import { Sink, type SinkCursorParams, type SinkData } from "../sink";
 
 export type SqliteSinkOptions = {
   /**
@@ -24,6 +24,48 @@ export type SqliteSinkOptions = {
   onConflict?: { on: string; update: string[] };
 };
 
+type TxnContext = {
+  buffer: SinkData[];
+};
+
+type TxnParams = {
+  writer: {
+    insert: (data: SinkData[]) => void;
+  };
+};
+const transactionHelper = (context: TxnContext) => {
+  return {
+    insert: (data: SinkData[]) => {
+      context.buffer.push(...data);
+    },
+  };
+};
+
+/**
+ * A sink that writes data to a SQLite database.
+ *
+ * @example
+ *
+ * ```ts
+ * const sink = sqlite({
+ *   database: db,
+ *   tableName: "test",
+ * });
+ *
+ * ...
+ * async transform({context, endCursor}){
+ *  const { writer } = useSink(context);
+ *  const insertHelper = writer(endCursor);
+ *
+ *  insertHelper.insert([
+ *    { id: 1, name: "John" },
+ *    { id: 2, name: "Jane" },
+ *  ]);
+ * }
+ *
+ * ```
+ */
+
 export class SqliteSink extends Sink {
   private _config: Omit<SqliteSinkOptions, "database">;
   private _db: SqliteDatabase;
@@ -35,13 +77,31 @@ export class SqliteSink extends Sink {
     this._db = database;
   }
 
-  async write({ data, endCursor, finality }: SinkWriteArgs) {
-    await this.callHook("write", { data });
-
+  private async write({
+    data,
+    endCursor,
+  }: { data: SinkData[]; endCursor?: Cursor }) {
     data = this.processCursorColumn(data, endCursor);
     await this.insertJsonArray(data);
+  }
+
+  async transaction(
+    { cursor, endCursor, finality }: SinkCursorParams,
+    cb: (params: TxnParams) => Promise<void>,
+  ) {
+    const context: TxnContext = {
+      buffer: [],
+    };
+
+    const writer = transactionHelper(context);
+
+    await cb({ writer });
+    await this.write({ data: context.buffer, endCursor });
+  }
 
-    await this.callHook("flush", { endCursor, finality });
+  async invalidate(cursor?: Cursor) {
+    // TODO: Implement
+    throw new Error("Not implemented");
   }
 
   private async insertJsonArray(data: SinkData[]) {

package/src/testing/indexer.ts

@@ -3,23 +3,30 @@ import {
   type MockFilter,
   MockStream,
 } from "@apibara/protocol/testing";
-import { createIndexer, defineIndexer } from "../indexer";
+import { type IndexerConfig, createIndexer, defineIndexer } from "../indexer";
 import type { IndexerPlugin } from "../plugins";
+import type { Sink } from "../sink";
 
-export const getMockIndexer = (
-  plugins: ReadonlyArray<IndexerPlugin<MockFilter, MockBlock, MockRet>> = [],
-) =>
+export const getMockIndexer = <TTxnParams>({
+  plugins,
+  sink,
+  override,
+}: {
+  plugins?: ReadonlyArray<IndexerPlugin<MockFilter, MockBlock, TTxnParams>>;
+  sink?: Sink<TTxnParams>;
+  override?: Partial<IndexerConfig<MockFilter, MockBlock, TTxnParams>>;
+} = {}) =>
   createIndexer(
     defineIndexer(MockStream)({
       streamUrl: "https://sepolia.ethereum.a5a.ch",
       finality: "accepted",
       filter: {},
-      async transform({ block: { data } }) {
-        if (!data) return [];
-
-        return [{ data }];
+      async transform({ block: { data }, context }) {
+        // TODO
       },
+      sink,
       plugins,
+      ...override,
     }),
   );
 

package/src/testing/setup.ts

@@ -16,17 +16,17 @@ export const test = viTest.extend({
   },
 });
 
-type WithClientContext<TFilter, TBlock, TRet> = {
+type WithClientContext<TFilter, TBlock, TTxnParams> = {
   run: (
-    indexerArgs: Indexer<TFilter, TBlock, TRet>,
+    indexerArgs: Indexer<TFilter, TBlock, TTxnParams>,
   ) => Promise<VcrReplayResult>;
 };
 
-async function withClient<TFilter, TBlock, TRet>(
+async function withClient<TFilter, TBlock, TTxnParams>(
   cassetteName: string,
   range: { fromBlock: bigint; toBlock: bigint },
   callback: (
-    context: WithClientContext<TFilter, TBlock, TRet>,
+    context: WithClientContext<TFilter, TBlock, TTxnParams>,
   ) => Promise<void>,
 ) {
   const vcrConfig: VcrConfig = {
@@ -43,7 +43,7 @@ async function withClient<TFilter, TBlock, TRet>(
     },
   };
 
-  const context: WithClientContext<TFilter, TBlock, TRet> = {
+  const context: WithClientContext<TFilter, TBlock, TTxnParams> = {
     async run(indexer) {
       const client = loadCassette<TFilter, TBlock>(vcrConfig, cassetteName);
 

package/src/testing/vcr.ts

@@ -1,13 +1,51 @@
-import { Sink, type SinkWriteArgs } from "../sink";
+import type { Cursor } from "@apibara/protocol";
+import { Sink, type SinkCursorParams, type SinkData } from "../sink";
 import type { VcrReplayResult } from "../vcr";
 
+type TxnContext = {
+  buffer: SinkData[];
+};
+
+type TxnParams = {
+  writer: {
+    insert: (data: SinkData[]) => void;
+  };
+};
+
+const transactionHelper = (context: TxnContext) => {
+  return {
+    insert: (data: SinkData[]) => {
+      context.buffer.push(...data);
+    },
+  };
+};
+
 export class VcrSink extends Sink {
   public result: VcrReplayResult["outputs"] = [];
 
-  async write({ data, endCursor, finality }: SinkWriteArgs) {
-    await this.callHook("write", { data });
+  write({ data, endCursor }: { data: SinkData[]; endCursor?: Cursor }) {
+    if (data.length === 0) return;
+
     this.result.push({ data, endCursor });
-    await this.callHook("flush", { endCursor, finality });
+  }
+
+  async transaction(
+    { cursor, endCursor, finality }: SinkCursorParams,
+    cb: (params: TxnParams) => Promise<void>,
+  ) {
+    const context: TxnContext = {
+      buffer: [],
+    };
+
+    const writer = transactionHelper(context);
+
+    await cb({ writer });
+    this.write({ data: context.buffer, endCursor });
+  }
+
+  async invalidate(cursor?: Cursor) {
+    // TODO: Implement
+    throw new Error("Not implemented");
   }
 }
 

package/src/vcr/record.ts

@@ -11,10 +11,10 @@ export type CassetteDataType<TFilter, TBlock> = {
   messages: StreamDataResponse<TBlock>[];
 };
 
-export async function record<TFilter, TBlock, TRet>(
+export async function record<TFilter, TBlock, TTxnParams>(
   vcrConfig: VcrConfig,
   client: Client<TFilter, TBlock>,
-  indexerArg: Indexer<TFilter, TBlock, TRet>,
+  indexerArg: Indexer<TFilter, TBlock, TTxnParams>,
   cassetteOptions: CassetteOptions,
 ) {
   const indexer = klona(indexerArg);

package/src/vcr/replay.ts

@@ -9,16 +9,16 @@ import { vcr } from "../testing/vcr";
 import { type CassetteDataType, deserialize } from "../vcr";
 import type { VcrConfig } from "./config";
 
-export async function replay<TFilter, TBlock, TRet>(
+export async function replay<TFilter, TBlock, TTxnParams>(
   vcrConfig: VcrConfig,
-  indexer: Indexer<TFilter, TBlock, TRet>,
+  indexer: Indexer<TFilter, TBlock, TTxnParams>,
   cassetteName: string,
 ): Promise<VcrReplayResult> {
   const client = loadCassette<TFilter, TBlock>(vcrConfig, cassetteName);
 
   const sink = vcr();
 
-  await run(client, indexer, sink);
+  await run(client, indexer);
 
   return {
     outputs: sink.result,

package/.turbo/turbo-build.log

@@ -1,37 +0,0 @@
-
-> @apibara/indexer@2.0.0-beta.0 build /home/runner/work/typescript-sdk/typescript-sdk/packages/indexer
-> unbuild
-
-ℹ Building indexer
-ℹ Cleaning dist directory: ./dist
-✔ Build succeeded for indexer
-  dist/index.cjs (total size: 11.1 kB, chunk size: 1.03 kB, exports: DefaultSink, Sink, createIndexer, defaultSink, defineIndexer, defineIndexerPlugin, deserialize, isCassetteAvailable, loadCassette, record, replay, run, serialize, useIndexerContext)
-  └─ dist/shared/indexer.c8ef02ea.cjs (9.67 kB)
-  └─ dist/shared/indexer.e05aedca.cjs (431 B)
-
-  dist/sinks/sqlite.cjs (total size: 2.88 kB, chunk size: 2.45 kB, exports: SqliteSink, sqlite)
-  └─ dist/shared/indexer.e05aedca.cjs (431 B)
-
-  dist/sinks/csv.cjs (total size: 2.37 kB, chunk size: 1.94 kB, exports: CsvSink, csv)
-  └─ dist/shared/indexer.e05aedca.cjs (431 B)
-
-  dist/testing/index.cjs (total size: 11.7 kB, chunk size: 1.58 kB, exports: VcrSink, generateMockMessages, test, vcr)
-  └─ dist/shared/indexer.c8ef02ea.cjs (9.67 kB)
-  └─ dist/shared/indexer.e05aedca.cjs (431 B)
-
-  dist/index.mjs (total size: 9.91 kB, chunk size: 661 B, exports: DefaultSink, Sink, createIndexer, defaultSink, defineIndexer, defineIndexerPlugin, deserialize, isCassetteAvailable, loadCassette, record, replay, run, serialize, useIndexerContext)
-  └─ dist/shared/indexer.50aa7ab0.mjs (8.88 kB)
-  └─ dist/shared/indexer.371c0482.mjs (373 B)
-
-  dist/sinks/sqlite.mjs (total size: 2.78 kB, chunk size: 2.4 kB, exports: SqliteSink, sqlite)
-  └─ dist/shared/indexer.371c0482.mjs (373 B)
-
-  dist/sinks/csv.mjs (total size: 2.08 kB, chunk size: 1.7 kB, exports: CsvSink, csv)
-  └─ dist/shared/indexer.371c0482.mjs (373 B)
-
-  dist/testing/index.mjs (total size: 10.8 kB, chunk size: 1.55 kB, exports: VcrSink, generateMockMessages, test, vcr)
-  └─ dist/shared/indexer.50aa7ab0.mjs (8.88 kB)
-  └─ dist/shared/indexer.371c0482.mjs (373 B)
-
-Σ Total dist size (byte size): 61.4 kB
-

package/CHANGELOG.md

@@ -1,83 +0,0 @@
-# @apibara/indexer
-
-## 0.4.1
-
-### Patch Changes
-
-- [#73](https://github.com/apibara/typescript-sdk/pull/73) [`4520e1e4b5d16ea63877f2adea9f2fa7c2ec74e6`](https://github.com/apibara/typescript-sdk/commit/4520e1e4b5d16ea63877f2adea9f2fa7c2ec74e6) Thanks [@fracek](https://github.com/fracek)! - Fix execution resources types
-
-## 0.4.0
-
-### Minor Changes
-
-- [#71](https://github.com/apibara/typescript-sdk/pull/71) [`267120a54b084e5a18ff112b044776ccba97353f`](https://github.com/apibara/typescript-sdk/commit/267120a54b084e5a18ff112b044776ccba97353f) Thanks [@fracek](https://github.com/fracek)! - Update Starknet type definitions
-
-## 0.3.1
-
-### Patch Changes
-
-- [#69](https://github.com/apibara/typescript-sdk/pull/69) [`c62d304bf8e6bcd5db7583b9378349a9c13bc0bf`](https://github.com/apibara/typescript-sdk/commit/c62d304bf8e6bcd5db7583b9378349a9c13bc0bf) Thanks [@fracek](https://github.com/fracek)! - Make all fields nullable
-
-## 0.3.0
-
-### Minor Changes
-
-- [#66](https://github.com/apibara/typescript-sdk/pull/66) [`448f74c30088784726c3a8e6efa3f34966e4b424`](https://github.com/apibara/typescript-sdk/commit/448f74c30088784726c3a8e6efa3f34966e4b424) Thanks [@fracek](https://github.com/fracek)! - Add V3 transaction types
-
-## 0.2.5
-
-### Patch Changes
-
-- [#64](https://github.com/apibara/typescript-sdk/pull/64) [`fa3ba914f38c278ad8a9bc74a325914b560856a3`](https://github.com/apibara/typescript-sdk/commit/fa3ba914f38c278ad8a9bc74a325914b560856a3) Thanks [@fracek](https://github.com/fracek)! - Fix bug in lookupEventFromSelector
-
-## 0.2.4
-
-### Patch Changes
-
-- [#62](https://github.com/apibara/typescript-sdk/pull/62) [`39fe427cf763e5b7d273e0ded7be2f68adf447b9`](https://github.com/apibara/typescript-sdk/commit/39fe427cf763e5b7d273e0ded7be2f68adf447b9) Thanks [@fracek](https://github.com/fracek)! - Add method to lookup event name from selector
-
-## 0.2.3
-
-### Patch Changes
-
-- [#60](https://github.com/apibara/typescript-sdk/pull/60) [`3282cc44c917ce1764ad223282a6f5040d3b89c1`](https://github.com/apibara/typescript-sdk/commit/3282cc44c917ce1764ad223282a6f5040d3b89c1) Thanks [@fracek](https://github.com/fracek)! - Customize Contract event filter
-
-## 0.2.2
-
-### Patch Changes
-
-- [#52](https://github.com/apibara/typescript-sdk/pull/52) [`59bbe52`](https://github.com/apibara/typescript-sdk/commit/59bbe526c1fd2c7d7315a50df02c061f1d87f770) Thanks [@fracek](https://github.com/fracek)! - Add include transaction and receipt option
-
-## 0.2.1
-
-### Patch Changes
-
-- [#50](https://github.com/apibara/typescript-sdk/pull/50) [`b65086a`](https://github.com/apibara/typescript-sdk/commit/b65086a3663d40a9c27bae1e5fb7fc0cad79581f) Thanks [@fracek](https://github.com/fracek)! - Fix Starknet Block types
-
-## 0.2.0
-
-### Minor Changes
-
-- [#48](https://github.com/apibara/typescript-sdk/pull/48) [`b54a103`](https://github.com/apibara/typescript-sdk/commit/b54a103321752dccc7aba5988a2dc598b8d7dfc8) Thanks [@fracek](https://github.com/fracek)! - Improve starknet Filter type
-
-### Patch Changes
-
-- [#48](https://github.com/apibara/typescript-sdk/pull/48) [`b54a103`](https://github.com/apibara/typescript-sdk/commit/b54a103321752dccc7aba5988a2dc598b8d7dfc8) Thanks [@fracek](https://github.com/fracek)! - Fix Mongo sink type
-
-## 0.1.2
-
-### Patch Changes
-
-- [#46](https://github.com/apibara/typescript-sdk/pull/46) [`aade5c4`](https://github.com/apibara/typescript-sdk/commit/aade5c46da8daec2e3aa7749a5a7d083cca25867) Thanks [@fracek](https://github.com/fracek)! - Publish typescript types
-
-## 0.1.1
-
-### Patch Changes
-
-- [#44](https://github.com/apibara/typescript-sdk/pull/44) [`0a96377`](https://github.com/apibara/typescript-sdk/commit/0a963770459c71d21a84d56dbb9e74f4beaa7349) Thanks [@fracek](https://github.com/fracek)! - Fix esm module exports
-
-## 0.1.0
-
-### Minor Changes
-
-- [#42](https://github.com/apibara/typescript-sdk/pull/42) [`603cfa7`](https://github.com/apibara/typescript-sdk/commit/603cfa72bac2c3bc0de54a3fc046555f7165ae56) Thanks [@fracek](https://github.com/fracek)! - Initial release