@apibara/indexer 0.4.1 → 2.0.0-beta.3

package/src/plugins/persistence.test.ts

@@ -0,0 +1,151 @@
+import type { Cursor } from "@apibara/protocol";
+import {
+  type MockBlock,
+  MockClient,
+  type MockFilter,
+} from "@apibara/protocol/testing";
+import Database from "better-sqlite3";
+import { klona } from "klona/full";
+import { describe, expect, it } from "vitest";
+import { run } from "../indexer";
+import { generateMockMessages } from "../testing";
+import { getMockIndexer } from "../testing/indexer";
+import { SqlitePersistence, sqlitePersistence } from "./persistence";
+
+describe("Persistence", () => {
+  const initDB = () => {
+    const db = new Database(":memory:");
+    SqlitePersistence.initialize(db);
+    return db;
+  };
+
+  it("should handle storing and updating a cursor & filter", () => {
+    const db = initDB();
+    const store = new SqlitePersistence<MockFilter>(db);
+
+    // Assert there's no data
+    let latest = store.get();
+
+    expect(latest.cursor).toBeUndefined();
+    expect(latest.filter).toBeUndefined();
+
+    // Insert value
+    const cursor: Cursor = {
+      orderKey: 5_000_000n,
+    };
+    const filter: MockFilter = {
+      filter: "X",
+    };
+    store.put({ cursor, filter });
+
+    // Check that value was stored
+    latest = store.get();
+
+    expect(latest.cursor).toEqual({
+      orderKey: 5_000_000n,
+      uniqueKey: null,
+    });
+    expect(latest.filter).toEqual({
+      filter: "X",
+    });
+
+    // Update value
+    const updatedCursor: Cursor = {
+      orderKey: 5_000_010n,
+      uniqueKey: "0x1234567890",
+    };
+    const updatedFilter: MockFilter = {
+      filter: "Y",
+    };
+
+    store.put({ cursor: updatedCursor, filter: updatedFilter });
+
+    // Check that value was updated
+    latest = store.get();
+
+    expect(latest.cursor).toEqual({
+      orderKey: 5_000_010n,
+      uniqueKey: "0x1234567890",
+    });
+    expect(latest.filter).toEqual({
+      filter: "Y",
+    });
+
+    db.close();
+  });
+
+  it("should handle storing and deleting a cursor & filter", () => {
+    const db = initDB();
+    const store = new SqlitePersistence(db);
+
+    // Assert there's no data
+    let latest = store.get();
+    expect(latest.cursor).toBeUndefined();
+    expect(latest.filter).toBeUndefined();
+
+    // Insert value
+    const cursor: Cursor = {
+      orderKey: 5_000_000n,
+    };
+    const filter: MockFilter = {
+      filter: "X",
+    };
+    store.put({ cursor, filter });
+
+    // Check that value was stored
+    latest = store.get();
+    expect(latest.cursor).toEqual({
+      orderKey: 5_000_000n,
+      uniqueKey: null,
+    });
+    expect(latest.filter).toEqual({
+      filter: "X",
+    });
+
+    // Delete value
+    store.del();
+
+    // Check there's no data
+    latest = store.get();
+    expect(latest.cursor).toBeUndefined();
+    expect(latest.filter).toBeUndefined();
+
+    db.close();
+  });
+
+  it("should work with indexer and store cursor of last message", async () => {
+    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
+      return messages;
+    });
+
+    const db = new Database(":memory:");
+
+    // create mock indexer with persistence plugin
+    const indexer = klona(
+      getMockIndexer({
+        plugins: [
+          sqlitePersistence({
+            database: db,
+          }),
+        ],
+      }),
+    );
+
+    await run(client, indexer);
+
+    const store = new SqlitePersistence<MockFilter>(db);
+
+    const latest = store.get();
+
+    expect(latest.cursor).toMatchInlineSnapshot(`
+      {
+        "orderKey": 5000009n,
+        "uniqueKey": null,
+      }
+    `);
+
+    db.close();
+  });
+});
+
+const messages = generateMockMessages();

package/src/plugins/persistence.ts

@@ -0,0 +1,202 @@
+import type { Cursor } from "@apibara/protocol";
+import type { Database as SqliteDatabase, Statement } from "better-sqlite3";
+import { deserialize, serialize } from "../vcr";
+import { defineIndexerPlugin } from "./config";
+
+export function sqlitePersistence<TFilter, TBlock, TTxnParams>({
+  database,
+}: { database: SqliteDatabase }) {
+  return defineIndexerPlugin<TFilter, TBlock, TTxnParams>((indexer) => {
+    let store: SqlitePersistence<TFilter>;
+
+    indexer.hooks.hook("run:before", () => {
+      SqlitePersistence.initialize(database);
+
+      store = new SqlitePersistence(database);
+    });
+
+    indexer.hooks.hook("connect:before", ({ request }) => {
+      const { cursor, filter } = store.get();
+
+      if (cursor) {
+        request.startingCursor = cursor;
+      }
+
+      if (filter) {
+        request.filter[1] = filter;
+      }
+    });
+
+    indexer.hooks.hook("transaction:commit", ({ endCursor }) => {
+      if (endCursor) {
+        store.put({ cursor: endCursor });
+      }
+    });
+
+    indexer.hooks.hook("connect:factory", ({ request, endCursor }) => {
+      if (request.filter[1]) {
+        store.put({ cursor: endCursor, filter: request.filter[1] });
+      }
+    });
+  });
+}
+
+export class SqlitePersistence<TFilter> {
+  /** Sqlite Queries Prepare Statements */
+  private _getCheckpointQuery: Statement<string, CheckpointRow>;
+  private _putCheckpointQuery: Statement<
+    [string, number, `0x${string}` | undefined]
+  >;
+  private _delCheckpointQuery: Statement<string>;
+  private _getFilterQuery: Statement<string, FilterRow>;
+  private _updateFilterToBlockQuery: Statement<[number, string]>;
+  private _insertFilterQuery: Statement<[string, string, number]>;
+  private _delFilterQuery: Statement<string>;
+
+  constructor(private _db: SqliteDatabase) {
+    this._getCheckpointQuery = this._db.prepare(statements.getCheckpoint);
+    this._putCheckpointQuery = this._db.prepare(statements.putCheckpoint);
+    this._delCheckpointQuery = this._db.prepare(statements.delCheckpoint);
+    this._getFilterQuery = this._db.prepare(statements.getFilter);
+    this._updateFilterToBlockQuery = this._db.prepare(
+      statements.updateFilterToBlock,
+    );
+    this._insertFilterQuery = this._db.prepare(statements.insertFilter);
+    this._delFilterQuery = this._db.prepare(statements.delFilter);
+  }
+
+  static initialize(db: SqliteDatabase) {
+    db.prepare(statements.createCheckpointsTable).run();
+    db.prepare(statements.createFiltersTable).run();
+  }
+
+  public get(): { cursor?: Cursor; filter?: TFilter } {
+    const cursor = this._getCheckpoint();
+    const filter = this._getFilter();
+
+    return { cursor, filter };
+  }
+
+  public put({ cursor, filter }: { cursor?: Cursor; filter?: TFilter }) {
+    if (cursor) {
+      this._putCheckpoint(cursor);
+
+      if (filter) {
+        this._putFilter(filter, cursor);
+      }
+    }
+  }
+
+  public del() {
+    this._delCheckpoint();
+    this._delFilter();
+  }
+
+  // --- CHECKPOINTS TABLE METHODS ---
+
+  private _getCheckpoint(): Cursor | undefined {
+    const row = this._getCheckpointQuery.get("default");
+
+    if (!row) return undefined;
+
+    return { orderKey: BigInt(row.order_key), uniqueKey: row.unique_key };
+  }
+
+  private _putCheckpoint(cursor: Cursor) {
+    this._putCheckpointQuery.run(
+      "default",
+      Number(cursor.orderKey),
+      cursor.uniqueKey,
+    );
+  }
+
+  private _delCheckpoint() {
+    this._delCheckpointQuery.run("default");
+  }
+
+  // --- FILTERS TABLE METHODS ---
+
+  private _getFilter(): TFilter | undefined {
+    const row = this._getFilterQuery.get("default");
+
+    if (!row) return undefined;
+
+    return deserialize(row.filter) as TFilter;
+  }
+
+  private _putFilter(filter: TFilter, endCursor: Cursor) {
+    this._updateFilterToBlockQuery.run(Number(endCursor.orderKey), "default");
+    this._insertFilterQuery.run(
+      "default",
+      serialize(filter as Record<string, unknown>),
+      Number(endCursor.orderKey),
+    );
+  }
+
+  private _delFilter() {
+    this._delFilterQuery.run("default");
+  }
+}
+
+const statements = {
+  beginTxn: "BEGIN TRANSACTION",
+  commitTxn: "COMMIT TRANSACTION",
+  rollbackTxn: "ROLLBACK TRANSACTION",
+  createCheckpointsTable: `
+    CREATE TABLE IF NOT EXISTS checkpoints (
+      id TEXT NOT NULL PRIMARY KEY,
+      order_key INTEGER NOT NULL,
+      unique_key TEXT
+    );`,
+  createFiltersTable: `
+    CREATE TABLE IF NOT EXISTS filters (
+      id TEXT NOT NULL,
+      filter BLOB NOT NULL,
+      from_block INTEGER NOT NULL,
+      to_block INTEGER,
+      PRIMARY KEY (id, from_block)
+    );`,
+  getCheckpoint: `
+    SELECT *
+    FROM checkpoints
+    WHERE id = ?`,
+  putCheckpoint: `
+    INSERT INTO checkpoints (id, order_key, unique_key)
+    VALUES (?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      order_key = excluded.order_key,
+      unique_key = excluded.unique_key`,
+  delCheckpoint: `
+    DELETE FROM checkpoints
+    WHERE id = ?`,
+  getFilter: `
+    SELECT *
+    FROM filters
+    WHERE id = ? AND to_block IS NULL`,
+  updateFilterToBlock: `
+    UPDATE filters
+    SET to_block = ?
+    WHERE id = ? AND to_block IS NULL`,
+  insertFilter: `
+    INSERT INTO filters (id, filter, from_block)
+    VALUES (?, ?, ?)
+    ON CONFLICT(id, from_block) DO UPDATE SET
+      filter = excluded.filter,
+      from_block = excluded.from_block`,
+  delFilter: `
+    DELETE FROM filters
+    WHERE id = ?`,
+};
+
+export type CheckpointRow = {
+  id: string;
+  order_key: number;
+  unique_key?: `0x${string}`;
+};
+
+export type FilterRow = {
+  id: string;
+  filter: string;
+  from_block: number;
+  to_block?: number;
+};

package/src/sink.ts

@@ -0,0 +1,36 @@
+import type { Cursor, DataFinality } from "@apibara/protocol";
+import consola from "consola";
+
+export type SinkData = Record<string, unknown>;
+
+export type SinkCursorParams = {
+  cursor?: Cursor;
+  endCursor?: Cursor;
+  finality: DataFinality;
+};
+
+export abstract class Sink<TTxnParams = unknown> {
+  abstract transaction(
+    { cursor, endCursor, finality }: SinkCursorParams,
+    cb: (params: TTxnParams) => Promise<void>,
+  ): Promise<void>;
+
+  abstract invalidate(cursor?: Cursor): Promise<void>;
+}
+
+export class DefaultSink extends Sink<unknown> {
+  async transaction(
+    { cursor, endCursor, finality }: SinkCursorParams,
+    cb: (params: unknown) => Promise<void>,
+  ): Promise<void> {
+    await cb({});
+  }
+
+  async invalidate(cursor?: Cursor) {
+    consola.info(`Invalidating cursor ${cursor?.orderKey}`);
+  }
+}
+
+export function defaultSink() {
+  return new DefaultSink();
+}

package/src/sinks/csv.test.ts

@@ -0,0 +1,65 @@
+import fs from "node:fs/promises";
+import {
+  type MockBlock,
+  MockClient,
+  type MockFilter,
+} from "@apibara/protocol/testing";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { useSink } from "../hooks";
+import { run } from "../indexer";
+import {} from "../plugins/persistence";
+import { generateMockMessages } from "../testing";
+import { getMockIndexer } from "../testing/indexer";
+import { csv } from "./csv";
+
+describe("Run Test", () => {
+  async function cleanup() {
+    try {
+      await fs.unlink("test.csv");
+    } catch {}
+  }
+
+  beforeEach(async () => {
+    await cleanup();
+  });
+
+  afterEach(async () => {
+    await cleanup();
+  });
+
+  it("should store in csv file via csv sink", async () => {
+    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
+      return generateMockMessages();
+    });
+
+    const sink = csv({ filepath: "test.csv" });
+    await run(
+      client,
+      getMockIndexer({
+        sink,
+        override: {
+          transform: async ({ context, endCursor, block: { data } }) => {
+            const { writer } = useSink({ context });
+            writer.insert([{ data }]);
+          },
+        },
+      }),
+    );
+
+    const csvData = await fs.readFile("test.csv", "utf-8");
+
+    expect(csvData).toMatchInlineSnapshot(`
+      "5000000,5000000
+      5000001,5000001
+      5000002,5000002
+      5000003,5000003
+      5000004,5000004
+      5000005,5000005
+      5000006,5000006
+      5000007,5000007
+      5000008,5000008
+      5000009,5000009
+      "
+    `);
+  });
+});

package/src/sinks/csv.ts

@@ -0,0 +1,159 @@
+import fs from "node:fs";
+import type { Cursor } from "@apibara/protocol";
+import { type Options, type Stringifier, stringify } from "csv-stringify";
+import { Sink, type SinkCursorParams, type SinkData } from "../sink";
+
+export type CsvArgs = {
+  /**
+   * csv-stringy options
+   * @reference https://csv.js.org/stringify/options/
+   */
+  csvOptions?: Options;
+  /**
+   * filepath for your csv file
+   */
+  filepath: string;
+};
+
+export type CsvSinkOptions = {
+  /**
+   * An optional column name used to store the cursor value. If specified,
+   * the value of this column must match the `endCursor.orderKey` for each row.
+   */
+  cursorColumn?: string;
+};
+
+type TxnContext = {
+  buffer: SinkData[];
+};
+
+type TxnParams = {
+  writer: {
+    insert: (data: SinkData[]) => void;
+  };
+};
+
+const transactionHelper = (context: TxnContext) => {
+  return {
+    insert: (data: SinkData[]) => {
+      context.buffer.push(...data);
+    },
+  };
+};
+
+/**
+ * A sink that writes data to a CSV file.
+ *
+ * @example
+ *
+ * ```ts
+ * const sink = csv({
+ *   filepath: "./data.csv",
+ *   csvOptions: {
+ *     header: true,
+ *   },
+ * });
+ *
+ * ...
+ * async transform({context, endCursor}){
+ *  const { writer } = useSink(context);
+ *  const insertHelper = writer(endCursor);
+ *
+ *  insertHelper.insert([
+ *    { id: 1, name: "John" },
+ *    { id: 2, name: "Jane" },
+ *  ]);
+ * }
+ *
+ * ```
+ */
+export class CsvSink extends Sink {
+  constructor(
+    private _stringifier: Stringifier,
+    private _config: CsvSinkOptions,
+  ) {
+    super();
+  }
+
+  private async write({
+    data,
+    endCursor,
+  }: { data: SinkData[]; endCursor?: Cursor }) {
+    // adds a "_cursor" property if "cursorColumn" is not specified by user
+    data = this.processCursorColumn(data, endCursor);
+    // Insert the data into csv
+    await this.insertToCSV(data);
+  }
+
+  async transaction(
+    { cursor, endCursor, finality }: SinkCursorParams,
+    cb: (params: TxnParams) => Promise<void>,
+  ) {
+    const context: TxnContext = {
+      buffer: [],
+    };
+
+    const writer = transactionHelper(context);
+
+    await cb({ writer });
+    await this.write({ data: context.buffer, endCursor });
+  }
+
+  async invalidate(cursor?: Cursor) {
+    // TODO: Implement
+    throw new Error("Not implemented");
+  }
+
+  private async insertToCSV(data: SinkData[]) {
+    if (data.length === 0) return;
+
+    return await new Promise<void>((resolve, reject) => {
+      for (const row of data) {
+        this._stringifier.write(row, (err) => {
+          if (err) throw new Error(err.message);
+
+          // resolve when all rows are inserted into csv
+          if (row === data[data.length - 1]) {
+            resolve();
+          }
+        });
+      }
+    });
+  }
+
+  private processCursorColumn(
+    data: SinkData[],
+    endCursor?: Cursor,
+  ): SinkData[] {
+    const { cursorColumn } = this._config;
+
+    if (
+      cursorColumn &&
+      data.some(
+        (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey),
+      )
+    ) {
+      throw new Error(
+        `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`,
+      );
+    }
+
+    if (cursorColumn) {
+      return data;
+    }
+
+    return data.map((row) => ({
+      ...row,
+      _cursor: Number(endCursor?.orderKey),
+    }));
+  }
+}
+
+export const csv = (args: CsvArgs & CsvSinkOptions) => {
+  const { csvOptions, filepath, ...sinkOptions } = args;
+  const stringifier = stringify({ ...csvOptions });
+
+  const writeStream = fs.createWriteStream(filepath, { flags: "a" });
+  stringifier.pipe(writeStream);
+  return new CsvSink(stringifier, sinkOptions);
+};

package/src/sinks/drizzle/Int8Range.ts

@@ -0,0 +1,52 @@
+import { customType } from "drizzle-orm/pg-core";
+import type { Range } from "postgres-range";
+import {
+  parse as rangeParse,
+  serialize as rangeSerialize,
+} from "postgres-range";
+
+type Comparable = string | number;
+
+type RangeBound<T extends Comparable> =
+  | T
+  | {
+      value: T;
+      inclusive: boolean;
+    };
+
+export class Int8Range {
+  constructor(public readonly range: Range<number>) {}
+
+  get start(): RangeBound<number> | null {
+    return this.range.lower != null
+      ? {
+          value: this.range.lower,
+          inclusive: this.range.isLowerBoundClosed(),
+        }
+      : null;
+  }
+
+  get end(): RangeBound<number> | null {
+    return this.range.upper != null
+      ? {
+          value: this.range.upper,
+          inclusive: this.range.isUpperBoundClosed(),
+        }
+      : null;
+  }
+}
+
+export const int8range = customType<{
+  data: Int8Range;
+}>({
+  dataType: () => "int8range",
+  fromDriver: (value: unknown): Int8Range => {
+    if (typeof value !== "string") {
+      throw new Error("Expected string");
+    }
+
+    const parsed = rangeParse(value, (val) => Number.parseInt(val, 10));
+    return new Int8Range(parsed);
+  },
+  toDriver: (value: Int8Range): string => rangeSerialize(value.range),
+});

package/src/sinks/drizzle/delete.ts

@@ -0,0 +1,42 @@
+import type { Cursor } from "@apibara/protocol";
+import {
+  type ExtractTablesWithRelations,
+  type SQL,
+  type TablesRelationalConfig,
+  sql,
+} from "drizzle-orm";
+import type {
+  PgQueryResultHKT,
+  PgTable,
+  PgTransaction,
+  PgUpdateBase,
+} from "drizzle-orm/pg-core";
+
+export class DrizzleSinkDelete<
+  TTable extends PgTable,
+  TQueryResult extends PgQueryResultHKT,
+  TFullSchema extends Record<string, unknown> = Record<string, never>,
+  TSchema extends
+    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
+> {
+  constructor(
+    private db: PgTransaction<TQueryResult, TFullSchema, TSchema>,
+    private table: TTable,
+    private endCursor?: Cursor,
+  ) {}
+
+  //@ts-ignore
+  where(where: SQL): Omit<
+    // for type safety used PgUpdateBase instead of PgDeleteBase
+    PgUpdateBase<TTable, TQueryResult, undefined, false, "where">,
+    "where"
+  > {
+    return this.db
+      .update(this.table)
+      .set({
+        // @ts-ignore
+        _cursor: sql`int8range(lower(_cursor), ${Number(this.endCursor?.orderKey!)}, '[)')`,
+      })
+      .where(where);
+  }
+}