@apibara/indexer 2.0.0-beta.9 → 2.1.0-beta.2

package/src/sinks/sqlite.test.ts

@@ -1,99 +0,0 @@
-import {
-  type MockBlock,
-  MockClient,
-  type MockFilter,
-} from "@apibara/protocol/testing";
-import Database from "better-sqlite3";
-import { describe, expect, it } from "vitest";
-import { useSink } from "../hooks";
-import { run } from "../indexer";
-import {} from "../plugins/persistence";
-import { generateMockMessages } from "../testing";
-import { getMockIndexer } from "../testing/indexer";
-import { sqlite } from "./sqlite";
-
-describe("Run Test", () => {
-  it("should store in sqlite db via sqlitesink", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return generateMockMessages();
-    });
-
-    const db = new Database(":memory:");
-
-    db.prepare("DROP TABLE IF EXISTS test;").run();
-
-    db.prepare(
-      `
-        CREATE TABLE IF NOT EXISTS test (
-              data BLOB,
-              _cursor BIGINT
-          );`,
-    ).run();
-
-    const sink = sqlite({
-      database: db,
-      tableName: "test",
-    });
-    await run(
-      client,
-      getMockIndexer({
-        sink,
-        override: {
-          transform: async ({ context, endCursor, block: { data } }) => {
-            const { writer } = useSink({ context });
-            writer.insert([{ data }]);
-          },
-        },
-      }),
-    );
-
-    const sinkData = db.prepare("SELECT * FROM test").all();
-
-    expect(sinkData).toMatchInlineSnapshot(`
-      [
-        {
-          "_cursor": 5000000,
-          "data": "5000000",
-        },
-        {
-          "_cursor": 5000001,
-          "data": "5000001",
-        },
-        {
-          "_cursor": 5000002,
-          "data": "5000002",
-        },
-        {
-          "_cursor": 5000003,
-          "data": "5000003",
-        },
-        {
-          "_cursor": 5000004,
-          "data": "5000004",
-        },
-        {
-          "_cursor": 5000005,
-          "data": "5000005",
-        },
-        {
-          "_cursor": 5000006,
-          "data": "5000006",
-        },
-        {
-          "_cursor": 5000007,
-          "data": "5000007",
-        },
-        {
-          "_cursor": 5000008,
-          "data": "5000008",
-        },
-        {
-          "_cursor": 5000009,
-          "data": "5000009",
-        },
-      ]
-    `);
-
-    db.close();
-  });
-});

package/src/sinks/sqlite.ts

@@ -1,170 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import type { Database as SqliteDatabase } from "better-sqlite3";
-import { Sink, type SinkCursorParams, type SinkData } from "../sink";
-
-export type SqliteSinkOptions = {
-  /**
-   * Database instance of better-sqlite3
-   */
-  database: SqliteDatabase;
-  /**
-   * The name of the table where data will be inserted.
-   */
-  tableName: string;
-  /**
-   * An optional column name used to store the cursor value. If specified,
-   * the value of this column must match the `endCursor.orderKey` for each row.
-   */
-  cursorColumn?: string;
-  /**
-   * An optional configuration to handle conflicts during data insertion.
-   * - `on`: The column name on which conflicts are detected.
-   * - `update`: An array of column names to be updated if a conflict occurs.
-   */
-  onConflict?: { on: string; update: string[] };
-};
-
-type TxnContext = {
-  buffer: SinkData[];
-};
-
-type TxnParams = {
-  writer: {
-    insert: (data: SinkData[]) => void;
-  };
-};
-const transactionHelper = (context: TxnContext) => {
-  return {
-    insert: (data: SinkData[]) => {
-      context.buffer.push(...data);
-    },
-  };
-};
-
-/**
- * A sink that writes data to a SQLite database.
- *
- * @example
- *
- * ```ts
- * const sink = sqlite({
- *   database: db,
- *   tableName: "test",
- * });
- *
- * ...
- * async transform({context, endCursor}){
- *  const { writer } = useSink(context);
- *  const insertHelper = writer(endCursor);
- *
- *  insertHelper.insert([
- *    { id: 1, name: "John" },
- *    { id: 2, name: "Jane" },
- *  ]);
- * }
- *
- * ```
- */
-
-export class SqliteSink extends Sink {
-  private _config: Omit<SqliteSinkOptions, "database">;
-  private _db: SqliteDatabase;
-
-  constructor(options: SqliteSinkOptions) {
-    super();
-    const { database, ...config } = options;
-    this._config = config;
-    this._db = database;
-  }
-
-  private async write({
-    data,
-    endCursor,
-  }: { data: SinkData[]; endCursor?: Cursor }) {
-    data = this.processCursorColumn(data, endCursor);
-    await this.insertJsonArray(data);
-  }
-
-  async transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: TxnParams) => Promise<void>,
-  ) {
-    const context: TxnContext = {
-      buffer: [],
-    };
-
-    const writer = transactionHelper(context);
-
-    await cb({ writer });
-    await this.write({ data: context.buffer, endCursor });
-  }
-
-  async invalidate(cursor?: Cursor) {
-    // TODO: Implement
-    throw new Error("Not implemented");
-  }
-
-  private async insertJsonArray(data: SinkData[]) {
-    if (data.length === 0) return;
-
-    // Get columns from the first row of the object array
-    const columns = Object.keys(data[0]);
-    const columnNames = columns.join(", ");
-    const placeholders = columns.map(() => "?").join(", ");
-
-    // Handle onConflict option
-    const conflictClause = this.buildConflictClause();
-
-    // Build the SQL insert statement with multiple rows
-    const insertSQL = `INSERT INTO ${this._config.tableName} (${columnNames}) VALUES `;
-    const valuePlaceholders = data.map(() => `(${placeholders})`).join(", ");
-    const statement = insertSQL + valuePlaceholders + conflictClause;
-
-    // Prepare and execute the SQL statement
-    const values = data.flatMap((row) => columns.map((col) => row[col]));
-
-    this._db.prepare(statement).run(values);
-  }
-
-  private processCursorColumn(
-    data: SinkData[],
-    endCursor?: Cursor,
-  ): SinkData[] {
-    const { cursorColumn } = this._config;
-
-    if (
-      cursorColumn &&
-      data.some(
-        (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey),
-      )
-    ) {
-      throw new Error(
-        `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`,
-      );
-    }
-
-    if (cursorColumn) {
-      return data;
-    }
-
-    return data.map((row) => ({
-      ...row,
-      _cursor: Number(endCursor?.orderKey),
-    }));
-  }
-
-  private buildConflictClause(): string {
-    const { on, update } = this._config.onConflict || {};
-    if (on && update && update.length > 0) {
-      const updateColumns = update
-        .map((col) => `${col}=excluded.${col}`)
-        .join(", ");
-      return ` ON CONFLICT(${on}) DO UPDATE SET ${updateColumns}`;
-    }
-    return "";
-  }
-}
-
-export const sqlite = (args: SqliteSinkOptions) => {
-  return new SqliteSink(args);
-};

package/src/testing/helper.ts

@@ -1,13 +0,0 @@
-import type { MockStreamResponse } from "@apibara/protocol/testing";
-
-export function generateMockMessages(count = 10): MockStreamResponse[] {
-  return [...Array(count)].map((_, i) => ({
-    _tag: "data",
-    data: {
-      cursor: { orderKey: BigInt(5_000_000 - 1) },
-      finality: "accepted",
-      data: [{ data: `${5_000_000 + i}` }],
-      endCursor: { orderKey: BigInt(5_000_000 + i) },
-    },
-  }));
-}

package/src/testing/indexer.ts

@@ -1,35 +0,0 @@
-import {
-  type MockBlock,
-  type MockFilter,
-  MockStream,
-} from "@apibara/protocol/testing";
-import { type IndexerConfig, createIndexer, defineIndexer } from "../indexer";
-import type { IndexerPlugin } from "../plugins";
-import type { Sink } from "../sink";
-
-export const getMockIndexer = <TTxnParams>({
-  plugins,
-  sink,
-  override,
-}: {
-  plugins?: ReadonlyArray<IndexerPlugin<MockFilter, MockBlock, TTxnParams>>;
-  sink?: Sink<TTxnParams>;
-  override?: Partial<IndexerConfig<MockFilter, MockBlock, TTxnParams>>;
-} = {}) =>
-  createIndexer(
-    defineIndexer(MockStream)({
-      streamUrl: "https://sepolia.ethereum.a5a.ch",
-      finality: "accepted",
-      filter: {},
-      async transform({ block: { data }, context }) {
-        // TODO
-      },
-      sink,
-      plugins,
-      ...override,
-    }),
-  );
-
-export type MockRet = {
-  data: string;
-};

package/src/testing/setup.ts

@@ -1,59 +0,0 @@
-import { test as viTest } from "vitest";
-import type { Indexer } from "../indexer";
-import {
-  type CassetteOptions,
-  type VcrConfig,
-  type VcrReplayResult,
-  isCassetteAvailable,
-  loadCassette,
-  record,
-  replay,
-} from "../vcr";
-
-export const test = viTest.extend({
-  vcr: {
-    withClient,
-  },
-});
-
-type WithClientContext<TFilter, TBlock, TTxnParams> = {
-  run: (
-    indexerArgs: Indexer<TFilter, TBlock, TTxnParams>,
-  ) => Promise<VcrReplayResult>;
-};
-
-async function withClient<TFilter, TBlock, TTxnParams>(
-  cassetteName: string,
-  range: { fromBlock: bigint; toBlock: bigint },
-  callback: (
-    context: WithClientContext<TFilter, TBlock, TTxnParams>,
-  ) => Promise<void>,
-) {
-  const vcrConfig: VcrConfig = {
-    cassetteDir: "cassettes",
-  };
-
-  const cassetteOptions: CassetteOptions = {
-    name: cassetteName,
-    startingCursor: {
-      orderKey: range.fromBlock,
-    },
-    endingCursor: {
-      orderKey: range.toBlock,
-    },
-  };
-
-  const context: WithClientContext<TFilter, TBlock, TTxnParams> = {
-    async run(indexer) {
-      const client = loadCassette<TFilter, TBlock>(vcrConfig, cassetteName);
-
-      if (!isCassetteAvailable(vcrConfig, cassetteName)) {
-        await record(vcrConfig, client, indexer, cassetteOptions);
-      }
-
-      return await replay(vcrConfig, indexer, cassetteName);
-    },
-  };
-
-  await callback(context);
-}

package/src/testing/vcr.ts

@@ -1,54 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import { Sink, type SinkCursorParams, type SinkData } from "../sink";
-import type { VcrReplayResult } from "../vcr";
-
-type TxnContext = {
-  buffer: SinkData[];
-};
-
-type TxnParams = {
-  writer: {
-    insert: (data: SinkData[]) => void;
-  };
-};
-
-const transactionHelper = (context: TxnContext) => {
-  return {
-    insert: (data: SinkData[]) => {
-      context.buffer.push(...data);
-    },
-  };
-};
-
-export class VcrSink extends Sink {
-  public result: VcrReplayResult["outputs"] = [];
-
-  write({ data, endCursor }: { data: SinkData[]; endCursor?: Cursor }) {
-    if (data.length === 0) return;
-
-    this.result.push({ data, endCursor });
-  }
-
-  async transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: TxnParams) => Promise<void>,
-  ) {
-    const context: TxnContext = {
-      buffer: [],
-    };
-
-    const writer = transactionHelper(context);
-
-    await cb({ writer });
-    this.write({ data: context.buffer, endCursor });
-  }
-
-  async invalidate(cursor?: Cursor) {
-    // TODO: Implement
-    throw new Error("Not implemented");
-  }
-}
-
-export function vcr() {
-  return new VcrSink();
-}