@apibara/indexer 2.0.0-beta.8 → 2.1.0-beta.1

package/dist/sinks/sqlite.d.ts

@@ -1,71 +0,0 @@
-import { Cursor } from '@apibara/protocol';
-import { Database } from 'better-sqlite3';
-import { S as Sink, a as SinkCursorParams, b as SinkData } from '../shared/indexer.b9c8f0d8.js';
-
-type SqliteSinkOptions = {
-    /**
-     * Database instance of better-sqlite3
-     */
-    database: Database;
-    /**
-     * The name of the table where data will be inserted.
-     */
-    tableName: string;
-    /**
-     * An optional column name used to store the cursor value. If specified,
-     * the value of this column must match the `endCursor.orderKey` for each row.
-     */
-    cursorColumn?: string;
-    /**
-     * An optional configuration to handle conflicts during data insertion.
-     * - `on`: The column name on which conflicts are detected.
-     * - `update`: An array of column names to be updated if a conflict occurs.
-     */
-    onConflict?: {
-        on: string;
-        update: string[];
-    };
-};
-type TxnParams = {
-    writer: {
-        insert: (data: SinkData[]) => void;
-    };
-};
-/**
- * A sink that writes data to a SQLite database.
- *
- * @example
- *
- * ```ts
- * const sink = sqlite({
- *   database: db,
- *   tableName: "test",
- * });
- *
- * ...
- * async transform({context, endCursor}){
- *  const { writer } = useSink(context);
- *  const insertHelper = writer(endCursor);
- *
- *  insertHelper.insert([
- *    { id: 1, name: "John" },
- *    { id: 2, name: "Jane" },
- *  ]);
- * }
- *
- * ```
- */
-declare class SqliteSink extends Sink {
-    private _config;
-    private _db;
-    constructor(options: SqliteSinkOptions);
-    private write;
-    transaction({ cursor, endCursor, finality }: SinkCursorParams, cb: (params: TxnParams) => Promise<void>): Promise<void>;
-    invalidate(cursor?: Cursor): Promise<void>;
-    private insertJsonArray;
-    private processCursorColumn;
-    private buildConflictClause;
-}
-declare const sqlite: (args: SqliteSinkOptions) => SqliteSink;
-
-export { SqliteSink, type SqliteSinkOptions, sqlite };

package/dist/sinks/sqlite.mjs

@@ -1,87 +0,0 @@
-import { S as Sink } from '../shared/indexer.93d6b2eb.mjs';
-import 'consola';
-
-var __defProp = Object.defineProperty;
-var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
-var __publicField = (obj, key, value) => {
-  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
-  return value;
-};
-const transactionHelper = (context) => {
-  return {
-    insert: (data) => {
-      context.buffer.push(...data);
-    }
-  };
-};
-class SqliteSink extends Sink {
-  constructor(options) {
-    super();
-    __publicField(this, "_config");
-    __publicField(this, "_db");
-    const { database, ...config } = options;
-    this._config = config;
-    this._db = database;
-  }
-  async write({
-    data,
-    endCursor
-  }) {
-    data = this.processCursorColumn(data, endCursor);
-    await this.insertJsonArray(data);
-  }
-  async transaction({ cursor, endCursor, finality }, cb) {
-    const context = {
-      buffer: []
-    };
-    const writer = transactionHelper(context);
-    await cb({ writer });
-    await this.write({ data: context.buffer, endCursor });
-  }
-  async invalidate(cursor) {
-    throw new Error("Not implemented");
-  }
-  async insertJsonArray(data) {
-    if (data.length === 0)
-      return;
-    const columns = Object.keys(data[0]);
-    const columnNames = columns.join(", ");
-    const placeholders = columns.map(() => "?").join(", ");
-    const conflictClause = this.buildConflictClause();
-    const insertSQL = `INSERT INTO ${this._config.tableName} (${columnNames}) VALUES `;
-    const valuePlaceholders = data.map(() => `(${placeholders})`).join(", ");
-    const statement = insertSQL + valuePlaceholders + conflictClause;
-    const values = data.flatMap((row) => columns.map((col) => row[col]));
-    this._db.prepare(statement).run(values);
-  }
-  processCursorColumn(data, endCursor) {
-    const { cursorColumn } = this._config;
-    if (cursorColumn && data.some(
-      (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey)
-    )) {
-      throw new Error(
-        `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`
-      );
-    }
-    if (cursorColumn) {
-      return data;
-    }
-    return data.map((row) => ({
-      ...row,
-      _cursor: Number(endCursor?.orderKey)
-    }));
-  }
-  buildConflictClause() {
-    const { on, update } = this._config.onConflict || {};
-    if (on && update && update.length > 0) {
-      const updateColumns = update.map((col) => `${col}=excluded.${col}`).join(", ");
-      return ` ON CONFLICT(${on}) DO UPDATE SET ${updateColumns}`;
-    }
-    return "";
-  }
-}
-const sqlite = (args) => {
-  return new SqliteSink(args);
-};
-
-export { SqliteSink, sqlite };

package/src/hooks/index.ts

@@ -1,2 +0,0 @@
-export * from "./useKVStore";
-export * from "./useSink";

package/src/hooks/useKVStore.ts

@@ -1,12 +0,0 @@
-import { useIndexerContext } from "../context";
-import type { KVStore } from "../plugins/kv";
-
-export type UseKVStoreResult = InstanceType<typeof KVStore>;
-
-export function useKVStore(): UseKVStoreResult {
-  const ctx = useIndexerContext();
-
-  if (!ctx?.kv) throw new Error("KV Plugin is not available in context!");
-
-  return ctx.kv;
-}

package/src/hooks/useSink.ts

@@ -1,13 +0,0 @@
-import type { IndexerContext } from "../context";
-
-export function useSink<TTxnParams>({
-  context,
-}: {
-  context: IndexerContext<TTxnParams>;
-}) {
-  if (!context.sinkTransaction) {
-    throw new Error("Transaction context doesn't exist!");
-  }
-
-  return context.sinkTransaction;
-}

package/src/plugins/kv.test.ts

@@ -1,120 +0,0 @@
-import Database, { type Database as SqliteDatabase } from "better-sqlite3";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { KVStore } from "./kv";
-
-type ValueType = { data: bigint };
-
-type DatabaseRowType = {
-  from_block: number;
-  k: string;
-  to_block: number;
-  v: unknown;
-};
-
-describe("KVStore", () => {
-  let db: SqliteDatabase;
-  let store: KVStore;
-  const key = "test_key";
-
-  beforeAll(() => {
-    db = new Database(":memory:");
-    KVStore.initialize(db);
-    store = new KVStore(db, "finalized", { orderKey: 5_000_000n });
-  });
-
-  afterAll(async () => {
-    db.close();
-  });
-
-  it("should begin transaction", () => {
-    store.beginTransaction();
-  });
-
-  it("should put and get a value", async () => {
-    const value = { data: 0n };
-
-    store.put<ValueType>(key, value);
-    const result = store.get<ValueType>(key);
-
-    expect(result).toEqual(value);
-  });
-
-  it("should commit transaction", async () => {
-    store.commitTransaction();
-
-    const value = { data: 0n };
-
-    const result = store.get<ValueType>(key);
-
-    expect(result).toEqual(value);
-  });
-
-  it("should return undefined for non-existing key", async () => {
-    const result = store.get<ValueType>("non_existent_key");
-    expect(result).toBeUndefined();
-  });
-
-  it("should begin transaction", async () => {
-    store.beginTransaction();
-  });
-
-  it("should update an existing value", async () => {
-    store = new KVStore(db, "finalized", { orderKey: 5_000_020n });
-
-    const value = { data: 50n };
-
-    store.put<ValueType>(key, value);
-    const result = store.get<ValueType>(key);
-
-    expect(result).toEqual(value);
-  });
-
-  it("should delete a value", async () => {
-    store.del(key);
-    const result = store.get<ValueType>(key);
-
-    expect(result).toBeUndefined();
-
-    const rows = db
-      .prepare<string, DatabaseRowType>(
-        `
-      SELECT from_block, to_block, k, v
-      FROM kvs
-      WHERE k = ?
-    `,
-      )
-      .all(key);
-
-    // Check that the old is correctly marked with to_block
-    expect(rows[0].to_block).toBe(Number(5_000_020n));
-  });
-
-  it("should rollback transaction", async () => {
-    await store.rollbackTransaction();
-  });
-
-  it("should revert the changes to last commit", async () => {
-    const rows = db
-      .prepare(
-        `
-      SELECT from_block, to_block, k, v
-      FROM kvs
-      WHERE k = ?
-    `,
-      )
-      .all([key]);
-
-    expect(rows).toMatchInlineSnapshot(`
-      [
-        {
-          "from_block": 5000000,
-          "k": "test_key",
-          "to_block": null,
-          "v": "{
-      	"data": "0n"
-      }",
-        },
-      ]
-    `);
-  });
-});

package/src/plugins/kv.ts

@@ -1,132 +0,0 @@
-import assert from "node:assert";
-import type { Cursor, DataFinality } from "@apibara/protocol";
-import type { Database as SqliteDatabase, Statement } from "better-sqlite3";
-import { useIndexerContext } from "../context";
-import { deserialize, serialize } from "../vcr";
-import { defineIndexerPlugin } from "./config";
-
-export function kv<TFilter, TBlock, TTxnParams>({
-  database,
-}: { database: SqliteDatabase }) {
-  return defineIndexerPlugin<TFilter, TBlock, TTxnParams>((indexer) => {
-    indexer.hooks.hook("run:before", () => {
-      KVStore.initialize(database);
-    });
-
-    indexer.hooks.hook("handler:before", ({ finality, endCursor }) => {
-      const ctx = useIndexerContext();
-
-      assert(endCursor, new Error("endCursor cannot be undefined"));
-
-      ctx.kv = new KVStore(database, finality, endCursor);
-
-      ctx.kv.beginTransaction();
-    });
-
-    indexer.hooks.hook("handler:after", () => {
-      const ctx = useIndexerContext();
-
-      ctx.kv.commitTransaction();
-
-      ctx.kv = null;
-    });
-
-    indexer.hooks.hook("handler:exception", () => {
-      const ctx = useIndexerContext();
-
-      ctx.kv.rollbackTransaction();
-
-      ctx.kv = null;
-    });
-  });
-}
-
-export class KVStore {
-  /** Sqlite Queries Prepare Statements */
-  private _beginTxnQuery: Statement;
-  private _commitTxnQuery: Statement;
-  private _rollbackTxnQuery: Statement;
-  private _getQuery: Statement<string, { v: string }>;
-  private _updateToBlockQuery: Statement<[number, string]>;
-  private _insertIntoKvsQuery: Statement<[number, string, string]>;
-  private _delQuery: Statement<[number, string]>;
-
-  constructor(
-    private _db: SqliteDatabase,
-    private _finality: DataFinality,
-    private _endCursor: Cursor,
-  ) {
-    this._beginTxnQuery = this._db.prepare(statements.beginTxn);
-    this._commitTxnQuery = this._db.prepare(statements.commitTxn);
-    this._rollbackTxnQuery = this._db.prepare(statements.rollbackTxn);
-    this._getQuery = this._db.prepare(statements.get);
-    this._updateToBlockQuery = this._db.prepare(statements.updateToBlock);
-    this._insertIntoKvsQuery = this._db.prepare(statements.insertIntoKvs);
-    this._delQuery = this._db.prepare(statements.del);
-  }
-
-  static initialize(db: SqliteDatabase) {
-    db.prepare(statements.createTable).run();
-  }
-
-  beginTransaction() {
-    this._beginTxnQuery.run();
-  }
-
-  commitTransaction() {
-    this._commitTxnQuery.run();
-  }
-
-  rollbackTransaction() {
-    this._rollbackTxnQuery.run();
-  }
-
-  get<T>(key: string): T {
-    const row = this._getQuery.get(key);
-
-    return row ? deserialize(row.v) : undefined;
-  }
-
-  put<T>(key: string, value: T) {
-    this._updateToBlockQuery.run(Number(this._endCursor.orderKey), key);
-
-    this._insertIntoKvsQuery.run(
-      Number(this._endCursor.orderKey),
-      key,
-      serialize(value as Record<string, unknown>),
-    );
-  }
-
-  del(key: string) {
-    this._delQuery.run(Number(this._endCursor.orderKey), key);
-  }
-}
-
-const statements = {
-  beginTxn: "BEGIN TRANSACTION",
-  commitTxn: "COMMIT TRANSACTION",
-  rollbackTxn: "ROLLBACK TRANSACTION",
-  createTable: `
-    CREATE TABLE IF NOT EXISTS kvs (
-      from_block INTEGER NOT NULL,
-      to_block INTEGER,
-      k TEXT NOT NULL,
-      v BLOB NOT NULL,
-      PRIMARY KEY (from_block, k)
-      );`,
-  get: `
-    SELECT v
-    FROM kvs
-    WHERE k = ? AND to_block IS NULL`,
-  updateToBlock: `
-    UPDATE kvs
-    SET to_block = ?
-    WHERE k = ? AND to_block IS NULL`,
-  insertIntoKvs: `
-    INSERT INTO kvs (from_block, to_block, k, v)
-    VALUES (?, NULL, ?, ?)`,
-  del: `
-    UPDATE kvs
-    SET to_block = ?
-    WHERE k = ? AND to_block IS NULL`,
-};

package/src/plugins/persistence.test.ts

@@ -1,151 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import {
-  type MockBlock,
-  MockClient,
-  type MockFilter,
-} from "@apibara/protocol/testing";
-import Database from "better-sqlite3";
-import { klona } from "klona/full";
-import { describe, expect, it } from "vitest";
-import { run } from "../indexer";
-import { generateMockMessages } from "../testing";
-import { getMockIndexer } from "../testing/indexer";
-import { SqlitePersistence, sqlitePersistence } from "./persistence";
-
-describe("Persistence", () => {
-  const initDB = () => {
-    const db = new Database(":memory:");
-    SqlitePersistence.initialize(db);
-    return db;
-  };
-
-  it("should handle storing and updating a cursor & filter", () => {
-    const db = initDB();
-    const store = new SqlitePersistence<MockFilter>(db);
-
-    // Assert there's no data
-    let latest = store.get();
-
-    expect(latest.cursor).toBeUndefined();
-    expect(latest.filter).toBeUndefined();
-
-    // Insert value
-    const cursor: Cursor = {
-      orderKey: 5_000_000n,
-    };
-    const filter: MockFilter = {
-      filter: "X",
-    };
-    store.put({ cursor, filter });
-
-    // Check that value was stored
-    latest = store.get();
-
-    expect(latest.cursor).toEqual({
-      orderKey: 5_000_000n,
-      uniqueKey: null,
-    });
-    expect(latest.filter).toEqual({
-      filter: "X",
-    });
-
-    // Update value
-    const updatedCursor: Cursor = {
-      orderKey: 5_000_010n,
-      uniqueKey: "0x1234567890",
-    };
-    const updatedFilter: MockFilter = {
-      filter: "Y",
-    };
-
-    store.put({ cursor: updatedCursor, filter: updatedFilter });
-
-    // Check that value was updated
-    latest = store.get();
-
-    expect(latest.cursor).toEqual({
-      orderKey: 5_000_010n,
-      uniqueKey: "0x1234567890",
-    });
-    expect(latest.filter).toEqual({
-      filter: "Y",
-    });
-
-    db.close();
-  });
-
-  it("should handle storing and deleting a cursor & filter", () => {
-    const db = initDB();
-    const store = new SqlitePersistence(db);
-
-    // Assert there's no data
-    let latest = store.get();
-    expect(latest.cursor).toBeUndefined();
-    expect(latest.filter).toBeUndefined();
-
-    // Insert value
-    const cursor: Cursor = {
-      orderKey: 5_000_000n,
-    };
-    const filter: MockFilter = {
-      filter: "X",
-    };
-    store.put({ cursor, filter });
-
-    // Check that value was stored
-    latest = store.get();
-    expect(latest.cursor).toEqual({
-      orderKey: 5_000_000n,
-      uniqueKey: null,
-    });
-    expect(latest.filter).toEqual({
-      filter: "X",
-    });
-
-    // Delete value
-    store.del();
-
-    // Check there's no data
-    latest = store.get();
-    expect(latest.cursor).toBeUndefined();
-    expect(latest.filter).toBeUndefined();
-
-    db.close();
-  });
-
-  it("should work with indexer and store cursor of last message", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return messages;
-    });
-
-    const db = new Database(":memory:");
-
-    // create mock indexer with persistence plugin
-    const indexer = klona(
-      getMockIndexer({
-        plugins: [
-          sqlitePersistence({
-            database: db,
-          }),
-        ],
-      }),
-    );
-
-    await run(client, indexer);
-
-    const store = new SqlitePersistence<MockFilter>(db);
-
-    const latest = store.get();
-
-    expect(latest.cursor).toMatchInlineSnapshot(`
-      {
-        "orderKey": 5000009n,
-        "uniqueKey": null,
-      }
-    `);
-
-    db.close();
-  });
-});
-
-const messages = generateMockMessages();

package/src/sink.ts

@@ -1,36 +0,0 @@
-import type { Cursor, DataFinality } from "@apibara/protocol";
-import consola from "consola";
-
-export type SinkData = Record<string, unknown>;
-
-export type SinkCursorParams = {
-  cursor?: Cursor;
-  endCursor?: Cursor;
-  finality: DataFinality;
-};
-
-export abstract class Sink<TTxnParams = unknown> {
-  abstract transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: TTxnParams) => Promise<void>,
-  ): Promise<void>;
-
-  abstract invalidate(cursor?: Cursor): Promise<void>;
-}
-
-export class DefaultSink extends Sink<unknown> {
-  async transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: unknown) => Promise<void>,
-  ): Promise<void> {
-    await cb({});
-  }
-
-  async invalidate(cursor?: Cursor) {
-    consola.info(`Invalidating cursor ${cursor?.orderKey}`);
-  }
-}
-
-export function defaultSink() {
-  return new DefaultSink();
-}

package/src/sinks/csv.test.ts

@@ -1,65 +0,0 @@
-import fs from "node:fs/promises";
-import {
-  type MockBlock,
-  MockClient,
-  type MockFilter,
-} from "@apibara/protocol/testing";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { useSink } from "../hooks";
-import { run } from "../indexer";
-import {} from "../plugins/persistence";
-import { generateMockMessages } from "../testing";
-import { getMockIndexer } from "../testing/indexer";
-import { csv } from "./csv";
-
-describe("Run Test", () => {
-  async function cleanup() {
-    try {
-      await fs.unlink("test.csv");
-    } catch {}
-  }
-
-  beforeEach(async () => {
-    await cleanup();
-  });
-
-  afterEach(async () => {
-    await cleanup();
-  });
-
-  it("should store in csv file via csv sink", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return generateMockMessages();
-    });
-
-    const sink = csv({ filepath: "test.csv" });
-    await run(
-      client,
-      getMockIndexer({
-        sink,
-        override: {
-          transform: async ({ context, endCursor, block: { data } }) => {
-            const { writer } = useSink({ context });
-            writer.insert([{ data }]);
-          },
-        },
-      }),
-    );
-
-    const csvData = await fs.readFile("test.csv", "utf-8");
-
-    expect(csvData).toMatchInlineSnapshot(`
-      "5000000,5000000
-      5000001,5000001
-      5000002,5000002
-      5000003,5000003
-      5000004,5000004
-      5000005,5000005
-      5000006,5000006
-      5000007,5000007
-      5000008,5000008
-      5000009,5000009
-      "
-    `);
-  });
-});