@apibara/indexer 2.0.0-beta.4 → 2.0.0-beta.41

package/src/plugins/kv.test.ts

@@ -1,120 +0,0 @@
-import Database, { type Database as SqliteDatabase } from "better-sqlite3";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { KVStore } from "./kv";
-
-type ValueType = { data: bigint };
-
-type DatabaseRowType = {
-  from_block: number;
-  k: string;
-  to_block: number;
-  v: unknown;
-};
-
-describe("KVStore", () => {
-  let db: SqliteDatabase;
-  let store: KVStore;
-  const key = "test_key";
-
-  beforeAll(() => {
-    db = new Database(":memory:");
-    KVStore.initialize(db);
-    store = new KVStore(db, "finalized", { orderKey: 5_000_000n });
-  });
-
-  afterAll(async () => {
-    db.close();
-  });
-
-  it("should begin transaction", () => {
-    store.beginTransaction();
-  });
-
-  it("should put and get a value", async () => {
-    const value = { data: 0n };
-
-    store.put<ValueType>(key, value);
-    const result = store.get<ValueType>(key);
-
-    expect(result).toEqual(value);
-  });
-
-  it("should commit transaction", async () => {
-    store.commitTransaction();
-
-    const value = { data: 0n };
-
-    const result = store.get<ValueType>(key);
-
-    expect(result).toEqual(value);
-  });
-
-  it("should return undefined for non-existing key", async () => {
-    const result = store.get<ValueType>("non_existent_key");
-    expect(result).toBeUndefined();
-  });
-
-  it("should begin transaction", async () => {
-    store.beginTransaction();
-  });
-
-  it("should update an existing value", async () => {
-    store = new KVStore(db, "finalized", { orderKey: 5_000_020n });
-
-    const value = { data: 50n };
-
-    store.put<ValueType>(key, value);
-    const result = store.get<ValueType>(key);
-
-    expect(result).toEqual(value);
-  });
-
-  it("should delete a value", async () => {
-    store.del(key);
-    const result = store.get<ValueType>(key);
-
-    expect(result).toBeUndefined();
-
-    const rows = db
-      .prepare<string, DatabaseRowType>(
-        `
-      SELECT from_block, to_block, k, v
-      FROM kvs
-      WHERE k = ?
-    `,
-      )
-      .all(key);
-
-    // Check that the old is correctly marked with to_block
-    expect(rows[0].to_block).toBe(Number(5_000_020n));
-  });
-
-  it("should rollback transaction", async () => {
-    await store.rollbackTransaction();
-  });
-
-  it("should revert the changes to last commit", async () => {
-    const rows = db
-      .prepare(
-        `
-      SELECT from_block, to_block, k, v
-      FROM kvs
-      WHERE k = ?
-    `,
-      )
-      .all([key]);
-
-    expect(rows).toMatchInlineSnapshot(`
-      [
-        {
-          "from_block": 5000000,
-          "k": "test_key",
-          "to_block": null,
-          "v": "{
-      	"data": "0n"
-      }",
-        },
-      ]
-    `);
-  });
-});

package/src/plugins/kv.ts

@@ -1,132 +0,0 @@
-import assert from "node:assert";
-import type { Cursor, DataFinality } from "@apibara/protocol";
-import type { Database as SqliteDatabase, Statement } from "better-sqlite3";
-import { useIndexerContext } from "../context";
-import { deserialize, serialize } from "../vcr";
-import { defineIndexerPlugin } from "./config";
-
-export function kv<TFilter, TBlock, TTxnParams>({
-  database,
-}: { database: SqliteDatabase }) {
-  return defineIndexerPlugin<TFilter, TBlock, TTxnParams>((indexer) => {
-    indexer.hooks.hook("run:before", () => {
-      KVStore.initialize(database);
-    });
-
-    indexer.hooks.hook("handler:before", ({ finality, endCursor }) => {
-      const ctx = useIndexerContext();
-
-      assert(endCursor, new Error("endCursor cannot be undefined"));
-
-      ctx.kv = new KVStore(database, finality, endCursor);
-
-      ctx.kv.beginTransaction();
-    });
-
-    indexer.hooks.hook("handler:after", () => {
-      const ctx = useIndexerContext();
-
-      ctx.kv.commitTransaction();
-
-      ctx.kv = null;
-    });
-
-    indexer.hooks.hook("handler:exception", () => {
-      const ctx = useIndexerContext();
-
-      ctx.kv.rollbackTransaction();
-
-      ctx.kv = null;
-    });
-  });
-}
-
-export class KVStore {
-  /** Sqlite Queries Prepare Statements */
-  private _beginTxnQuery: Statement;
-  private _commitTxnQuery: Statement;
-  private _rollbackTxnQuery: Statement;
-  private _getQuery: Statement<string, { v: string }>;
-  private _updateToBlockQuery: Statement<[number, string]>;
-  private _insertIntoKvsQuery: Statement<[number, string, string]>;
-  private _delQuery: Statement<[number, string]>;
-
-  constructor(
-    private _db: SqliteDatabase,
-    private _finality: DataFinality,
-    private _endCursor: Cursor,
-  ) {
-    this._beginTxnQuery = this._db.prepare(statements.beginTxn);
-    this._commitTxnQuery = this._db.prepare(statements.commitTxn);
-    this._rollbackTxnQuery = this._db.prepare(statements.rollbackTxn);
-    this._getQuery = this._db.prepare(statements.get);
-    this._updateToBlockQuery = this._db.prepare(statements.updateToBlock);
-    this._insertIntoKvsQuery = this._db.prepare(statements.insertIntoKvs);
-    this._delQuery = this._db.prepare(statements.del);
-  }
-
-  static initialize(db: SqliteDatabase) {
-    db.prepare(statements.createTable).run();
-  }
-
-  beginTransaction() {
-    this._beginTxnQuery.run();
-  }
-
-  commitTransaction() {
-    this._commitTxnQuery.run();
-  }
-
-  rollbackTransaction() {
-    this._rollbackTxnQuery.run();
-  }
-
-  get<T>(key: string): T {
-    const row = this._getQuery.get(key);
-
-    return row ? deserialize(row.v) : undefined;
-  }
-
-  put<T>(key: string, value: T) {
-    this._updateToBlockQuery.run(Number(this._endCursor.orderKey), key);
-
-    this._insertIntoKvsQuery.run(
-      Number(this._endCursor.orderKey),
-      key,
-      serialize(value as Record<string, unknown>),
-    );
-  }
-
-  del(key: string) {
-    this._delQuery.run(Number(this._endCursor.orderKey), key);
-  }
-}
-
-const statements = {
-  beginTxn: "BEGIN TRANSACTION",
-  commitTxn: "COMMIT TRANSACTION",
-  rollbackTxn: "ROLLBACK TRANSACTION",
-  createTable: `
-    CREATE TABLE IF NOT EXISTS kvs (
-      from_block INTEGER NOT NULL,
-      to_block INTEGER,
-      k TEXT NOT NULL,
-      v BLOB NOT NULL,
-      PRIMARY KEY (from_block, k)
-      );`,
-  get: `
-    SELECT v
-    FROM kvs
-    WHERE k = ? AND to_block IS NULL`,
-  updateToBlock: `
-    UPDATE kvs
-    SET to_block = ?
-    WHERE k = ? AND to_block IS NULL`,
-  insertIntoKvs: `
-    INSERT INTO kvs (from_block, to_block, k, v)
-    VALUES (?, NULL, ?, ?)`,
-  del: `
-    UPDATE kvs
-    SET to_block = ?
-    WHERE k = ? AND to_block IS NULL`,
-};

package/src/plugins/persistence.test.ts

@@ -1,151 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import {
-  type MockBlock,
-  MockClient,
-  type MockFilter,
-} from "@apibara/protocol/testing";
-import Database from "better-sqlite3";
-import { klona } from "klona/full";
-import { describe, expect, it } from "vitest";
-import { run } from "../indexer";
-import { generateMockMessages } from "../testing";
-import { getMockIndexer } from "../testing/indexer";
-import { SqlitePersistence, sqlitePersistence } from "./persistence";
-
-describe("Persistence", () => {
-  const initDB = () => {
-    const db = new Database(":memory:");
-    SqlitePersistence.initialize(db);
-    return db;
-  };
-
-  it("should handle storing and updating a cursor & filter", () => {
-    const db = initDB();
-    const store = new SqlitePersistence<MockFilter>(db);
-
-    // Assert there's no data
-    let latest = store.get();
-
-    expect(latest.cursor).toBeUndefined();
-    expect(latest.filter).toBeUndefined();
-
-    // Insert value
-    const cursor: Cursor = {
-      orderKey: 5_000_000n,
-    };
-    const filter: MockFilter = {
-      filter: "X",
-    };
-    store.put({ cursor, filter });
-
-    // Check that value was stored
-    latest = store.get();
-
-    expect(latest.cursor).toEqual({
-      orderKey: 5_000_000n,
-      uniqueKey: null,
-    });
-    expect(latest.filter).toEqual({
-      filter: "X",
-    });
-
-    // Update value
-    const updatedCursor: Cursor = {
-      orderKey: 5_000_010n,
-      uniqueKey: "0x1234567890",
-    };
-    const updatedFilter: MockFilter = {
-      filter: "Y",
-    };
-
-    store.put({ cursor: updatedCursor, filter: updatedFilter });
-
-    // Check that value was updated
-    latest = store.get();
-
-    expect(latest.cursor).toEqual({
-      orderKey: 5_000_010n,
-      uniqueKey: "0x1234567890",
-    });
-    expect(latest.filter).toEqual({
-      filter: "Y",
-    });
-
-    db.close();
-  });
-
-  it("should handle storing and deleting a cursor & filter", () => {
-    const db = initDB();
-    const store = new SqlitePersistence(db);
-
-    // Assert there's no data
-    let latest = store.get();
-    expect(latest.cursor).toBeUndefined();
-    expect(latest.filter).toBeUndefined();
-
-    // Insert value
-    const cursor: Cursor = {
-      orderKey: 5_000_000n,
-    };
-    const filter: MockFilter = {
-      filter: "X",
-    };
-    store.put({ cursor, filter });
-
-    // Check that value was stored
-    latest = store.get();
-    expect(latest.cursor).toEqual({
-      orderKey: 5_000_000n,
-      uniqueKey: null,
-    });
-    expect(latest.filter).toEqual({
-      filter: "X",
-    });
-
-    // Delete value
-    store.del();
-
-    // Check there's no data
-    latest = store.get();
-    expect(latest.cursor).toBeUndefined();
-    expect(latest.filter).toBeUndefined();
-
-    db.close();
-  });
-
-  it("should work with indexer and store cursor of last message", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return messages;
-    });
-
-    const db = new Database(":memory:");
-
-    // create mock indexer with persistence plugin
-    const indexer = klona(
-      getMockIndexer({
-        plugins: [
-          sqlitePersistence({
-            database: db,
-          }),
-        ],
-      }),
-    );
-
-    await run(client, indexer);
-
-    const store = new SqlitePersistence<MockFilter>(db);
-
-    const latest = store.get();
-
-    expect(latest.cursor).toMatchInlineSnapshot(`
-      {
-        "orderKey": 5000009n,
-        "uniqueKey": null,
-      }
-    `);
-
-    db.close();
-  });
-});
-
-const messages = generateMockMessages();

package/src/sink.ts

@@ -1,36 +0,0 @@
-import type { Cursor, DataFinality } from "@apibara/protocol";
-import consola from "consola";
-
-export type SinkData = Record<string, unknown>;
-
-export type SinkCursorParams = {
-  cursor?: Cursor;
-  endCursor?: Cursor;
-  finality: DataFinality;
-};
-
-export abstract class Sink<TTxnParams = unknown> {
-  abstract transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: TTxnParams) => Promise<void>,
-  ): Promise<void>;
-
-  abstract invalidate(cursor?: Cursor): Promise<void>;
-}
-
-export class DefaultSink extends Sink<unknown> {
-  async transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: unknown) => Promise<void>,
-  ): Promise<void> {
-    await cb({});
-  }
-
-  async invalidate(cursor?: Cursor) {
-    consola.info(`Invalidating cursor ${cursor?.orderKey}`);
-  }
-}
-
-export function defaultSink() {
-  return new DefaultSink();
-}

package/src/sinks/csv.test.ts

@@ -1,65 +0,0 @@
-import fs from "node:fs/promises";
-import {
-  type MockBlock,
-  MockClient,
-  type MockFilter,
-} from "@apibara/protocol/testing";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { useSink } from "../hooks";
-import { run } from "../indexer";
-import {} from "../plugins/persistence";
-import { generateMockMessages } from "../testing";
-import { getMockIndexer } from "../testing/indexer";
-import { csv } from "./csv";
-
-describe("Run Test", () => {
-  async function cleanup() {
-    try {
-      await fs.unlink("test.csv");
-    } catch {}
-  }
-
-  beforeEach(async () => {
-    await cleanup();
-  });
-
-  afterEach(async () => {
-    await cleanup();
-  });
-
-  it("should store in csv file via csv sink", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return generateMockMessages();
-    });
-
-    const sink = csv({ filepath: "test.csv" });
-    await run(
-      client,
-      getMockIndexer({
-        sink,
-        override: {
-          transform: async ({ context, endCursor, block: { data } }) => {
-            const { writer } = useSink({ context });
-            writer.insert([{ data }]);
-          },
-        },
-      }),
-    );
-
-    const csvData = await fs.readFile("test.csv", "utf-8");
-
-    expect(csvData).toMatchInlineSnapshot(`
-      "5000000,5000000
-      5000001,5000001
-      5000002,5000002
-      5000003,5000003
-      5000004,5000004
-      5000005,5000005
-      5000006,5000006
-      5000007,5000007
-      5000008,5000008
-      5000009,5000009
-      "
-    `);
-  });
-});

package/src/sinks/csv.ts

@@ -1,159 +0,0 @@
-import fs from "node:fs";
-import type { Cursor } from "@apibara/protocol";
-import { type Options, type Stringifier, stringify } from "csv-stringify";
-import { Sink, type SinkCursorParams, type SinkData } from "../sink";
-
-export type CsvArgs = {
-  /**
-   * csv-stringy options
-   * @reference https://csv.js.org/stringify/options/
-   */
-  csvOptions?: Options;
-  /**
-   * filepath for your csv file
-   */
-  filepath: string;
-};
-
-export type CsvSinkOptions = {
-  /**
-   * An optional column name used to store the cursor value. If specified,
-   * the value of this column must match the `endCursor.orderKey` for each row.
-   */
-  cursorColumn?: string;
-};
-
-type TxnContext = {
-  buffer: SinkData[];
-};
-
-type TxnParams = {
-  writer: {
-    insert: (data: SinkData[]) => void;
-  };
-};
-
-const transactionHelper = (context: TxnContext) => {
-  return {
-    insert: (data: SinkData[]) => {
-      context.buffer.push(...data);
-    },
-  };
-};
-
-/**
- * A sink that writes data to a CSV file.
- *
- * @example
- *
- * ```ts
- * const sink = csv({
- *   filepath: "./data.csv",
- *   csvOptions: {
- *     header: true,
- *   },
- * });
- *
- * ...
- * async transform({context, endCursor}){
- *  const { writer } = useSink(context);
- *  const insertHelper = writer(endCursor);
- *
- *  insertHelper.insert([
- *    { id: 1, name: "John" },
- *    { id: 2, name: "Jane" },
- *  ]);
- * }
- *
- * ```
- */
-export class CsvSink extends Sink {
-  constructor(
-    private _stringifier: Stringifier,
-    private _config: CsvSinkOptions,
-  ) {
-    super();
-  }
-
-  private async write({
-    data,
-    endCursor,
-  }: { data: SinkData[]; endCursor?: Cursor }) {
-    // adds a "_cursor" property if "cursorColumn" is not specified by user
-    data = this.processCursorColumn(data, endCursor);
-    // Insert the data into csv
-    await this.insertToCSV(data);
-  }
-
-  async transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: TxnParams) => Promise<void>,
-  ) {
-    const context: TxnContext = {
-      buffer: [],
-    };
-
-    const writer = transactionHelper(context);
-
-    await cb({ writer });
-    await this.write({ data: context.buffer, endCursor });
-  }
-
-  async invalidate(cursor?: Cursor) {
-    // TODO: Implement
-    throw new Error("Not implemented");
-  }
-
-  private async insertToCSV(data: SinkData[]) {
-    if (data.length === 0) return;
-
-    return await new Promise<void>((resolve, reject) => {
-      for (const row of data) {
-        this._stringifier.write(row, (err) => {
-          if (err) throw new Error(err.message);
-
-          // resolve when all rows are inserted into csv
-          if (row === data[data.length - 1]) {
-            resolve();
-          }
-        });
-      }
-    });
-  }
-
-  private processCursorColumn(
-    data: SinkData[],
-    endCursor?: Cursor,
-  ): SinkData[] {
-    const { cursorColumn } = this._config;
-
-    if (
-      cursorColumn &&
-      data.some(
-        (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey),
-      )
-    ) {
-      throw new Error(
-        `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`,
-      );
-    }
-
-    if (cursorColumn) {
-      return data;
-    }
-
-    return data.map((row) => ({
-      ...row,
-      _cursor: Number(endCursor?.orderKey),
-    }));
-  }
-}
-
-export const csv = (args: CsvArgs & CsvSinkOptions) => {
-  const { csvOptions, filepath, ...sinkOptions } = args;
-  const stringifier = stringify({ ...csvOptions });
-
-  const writeStream = fs.createWriteStream(filepath, { flags: "a" });
-  stringifier.pipe(writeStream);
-  return new CsvSink(stringifier, sinkOptions);
-};

package/src/sinks/drizzle/Int8Range.ts

@@ -1,52 +0,0 @@
-import { customType } from "drizzle-orm/pg-core";
-import type { Range } from "postgres-range";
-import {
-  parse as rangeParse,
-  serialize as rangeSerialize,
-} from "postgres-range";
-
-type Comparable = string | number;
-
-type RangeBound<T extends Comparable> =
-  | T
-  | {
-      value: T;
-      inclusive: boolean;
-    };
-
-export class Int8Range {
-  constructor(public readonly range: Range<number>) {}
-
-  get start(): RangeBound<number> | null {
-    return this.range.lower != null
-      ? {
-          value: this.range.lower,
-          inclusive: this.range.isLowerBoundClosed(),
-        }
-      : null;
-  }
-
-  get end(): RangeBound<number> | null {
-    return this.range.upper != null
-      ? {
-          value: this.range.upper,
-          inclusive: this.range.isUpperBoundClosed(),
-        }
-      : null;
-  }
-}
-
-export const int8range = customType<{
-  data: Int8Range;
-}>({
-  dataType: () => "int8range",
-  fromDriver: (value: unknown): Int8Range => {
-    if (typeof value !== "string") {
-      throw new Error("Expected string");
-    }
-
-    const parsed = rangeParse(value, (val) => Number.parseInt(val, 10));
-    return new Int8Range(parsed);
-  },
-  toDriver: (value: Int8Range): string => rangeSerialize(value.range),
-});