@apibara/indexer 2.0.0-beta.9 → 2.1.0-beta.2

package/src/plugins/persistence.test.ts

@@ -1,151 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import {
-  type MockBlock,
-  MockClient,
-  type MockFilter,
-} from "@apibara/protocol/testing";
-import Database from "better-sqlite3";
-import { klona } from "klona/full";
-import { describe, expect, it } from "vitest";
-import { run } from "../indexer";
-import { generateMockMessages } from "../testing";
-import { getMockIndexer } from "../testing/indexer";
-import { SqlitePersistence, sqlitePersistence } from "./persistence";
-
-describe("Persistence", () => {
-  const initDB = () => {
-    const db = new Database(":memory:");
-    SqlitePersistence.initialize(db);
-    return db;
-  };
-
-  it("should handle storing and updating a cursor & filter", () => {
-    const db = initDB();
-    const store = new SqlitePersistence<MockFilter>(db);
-
-    // Assert there's no data
-    let latest = store.get();
-
-    expect(latest.cursor).toBeUndefined();
-    expect(latest.filter).toBeUndefined();
-
-    // Insert value
-    const cursor: Cursor = {
-      orderKey: 5_000_000n,
-    };
-    const filter: MockFilter = {
-      filter: "X",
-    };
-    store.put({ cursor, filter });
-
-    // Check that value was stored
-    latest = store.get();
-
-    expect(latest.cursor).toEqual({
-      orderKey: 5_000_000n,
-      uniqueKey: null,
-    });
-    expect(latest.filter).toEqual({
-      filter: "X",
-    });
-
-    // Update value
-    const updatedCursor: Cursor = {
-      orderKey: 5_000_010n,
-      uniqueKey: "0x1234567890",
-    };
-    const updatedFilter: MockFilter = {
-      filter: "Y",
-    };
-
-    store.put({ cursor: updatedCursor, filter: updatedFilter });
-
-    // Check that value was updated
-    latest = store.get();
-
-    expect(latest.cursor).toEqual({
-      orderKey: 5_000_010n,
-      uniqueKey: "0x1234567890",
-    });
-    expect(latest.filter).toEqual({
-      filter: "Y",
-    });
-
-    db.close();
-  });
-
-  it("should handle storing and deleting a cursor & filter", () => {
-    const db = initDB();
-    const store = new SqlitePersistence(db);
-
-    // Assert there's no data
-    let latest = store.get();
-    expect(latest.cursor).toBeUndefined();
-    expect(latest.filter).toBeUndefined();
-
-    // Insert value
-    const cursor: Cursor = {
-      orderKey: 5_000_000n,
-    };
-    const filter: MockFilter = {
-      filter: "X",
-    };
-    store.put({ cursor, filter });
-
-    // Check that value was stored
-    latest = store.get();
-    expect(latest.cursor).toEqual({
-      orderKey: 5_000_000n,
-      uniqueKey: null,
-    });
-    expect(latest.filter).toEqual({
-      filter: "X",
-    });
-
-    // Delete value
-    store.del();
-
-    // Check there's no data
-    latest = store.get();
-    expect(latest.cursor).toBeUndefined();
-    expect(latest.filter).toBeUndefined();
-
-    db.close();
-  });
-
-  it("should work with indexer and store cursor of last message", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return messages;
-    });
-
-    const db = new Database(":memory:");
-
-    // create mock indexer with persistence plugin
-    const indexer = klona(
-      getMockIndexer({
-        plugins: [
-          sqlitePersistence({
-            database: db,
-          }),
-        ],
-      }),
-    );
-
-    await run(client, indexer);
-
-    const store = new SqlitePersistence<MockFilter>(db);
-
-    const latest = store.get();
-
-    expect(latest.cursor).toMatchInlineSnapshot(`
-      {
-        "orderKey": 5000009n,
-        "uniqueKey": null,
-      }
-    `);
-
-    db.close();
-  });
-});
-
-const messages = generateMockMessages();

package/src/sink.ts

@@ -1,36 +0,0 @@
-import type { Cursor, DataFinality } from "@apibara/protocol";
-import consola from "consola";
-
-export type SinkData = Record<string, unknown>;
-
-export type SinkCursorParams = {
-  cursor?: Cursor;
-  endCursor?: Cursor;
-  finality: DataFinality;
-};
-
-export abstract class Sink<TTxnParams = unknown> {
-  abstract transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: TTxnParams) => Promise<void>,
-  ): Promise<void>;
-
-  abstract invalidate(cursor?: Cursor): Promise<void>;
-}
-
-export class DefaultSink extends Sink<unknown> {
-  async transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: unknown) => Promise<void>,
-  ): Promise<void> {
-    await cb({});
-  }
-
-  async invalidate(cursor?: Cursor) {
-    consola.info(`Invalidating cursor ${cursor?.orderKey}`);
-  }
-}
-
-export function defaultSink() {
-  return new DefaultSink();
-}

package/src/sinks/csv.test.ts

@@ -1,65 +0,0 @@
-import fs from "node:fs/promises";
-import {
-  type MockBlock,
-  MockClient,
-  type MockFilter,
-} from "@apibara/protocol/testing";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { useSink } from "../hooks";
-import { run } from "../indexer";
-import {} from "../plugins/persistence";
-import { generateMockMessages } from "../testing";
-import { getMockIndexer } from "../testing/indexer";
-import { csv } from "./csv";
-
-describe("Run Test", () => {
-  async function cleanup() {
-    try {
-      await fs.unlink("test.csv");
-    } catch {}
-  }
-
-  beforeEach(async () => {
-    await cleanup();
-  });
-
-  afterEach(async () => {
-    await cleanup();
-  });
-
-  it("should store in csv file via csv sink", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return generateMockMessages();
-    });
-
-    const sink = csv({ filepath: "test.csv" });
-    await run(
-      client,
-      getMockIndexer({
-        sink,
-        override: {
-          transform: async ({ context, endCursor, block: { data } }) => {
-            const { writer } = useSink({ context });
-            writer.insert([{ data }]);
-          },
-        },
-      }),
-    );
-
-    const csvData = await fs.readFile("test.csv", "utf-8");
-
-    expect(csvData).toMatchInlineSnapshot(`
-      "5000000,5000000
-      5000001,5000001
-      5000002,5000002
-      5000003,5000003
-      5000004,5000004
-      5000005,5000005
-      5000006,5000006
-      5000007,5000007
-      5000008,5000008
-      5000009,5000009
-      "
-    `);
-  });
-});

package/src/sinks/csv.ts

@@ -1,159 +0,0 @@
-import fs from "node:fs";
-import type { Cursor } from "@apibara/protocol";
-import { type Options, type Stringifier, stringify } from "csv-stringify";
-import { Sink, type SinkCursorParams, type SinkData } from "../sink";
-
-export type CsvArgs = {
-  /**
-   * csv-stringy options
-   * @reference https://csv.js.org/stringify/options/
-   */
-  csvOptions?: Options;
-  /**
-   * filepath for your csv file
-   */
-  filepath: string;
-};
-
-export type CsvSinkOptions = {
-  /**
-   * An optional column name used to store the cursor value. If specified,
-   * the value of this column must match the `endCursor.orderKey` for each row.
-   */
-  cursorColumn?: string;
-};
-
-type TxnContext = {
-  buffer: SinkData[];
-};
-
-type TxnParams = {
-  writer: {
-    insert: (data: SinkData[]) => void;
-  };
-};
-
-const transactionHelper = (context: TxnContext) => {
-  return {
-    insert: (data: SinkData[]) => {
-      context.buffer.push(...data);
-    },
-  };
-};
-
-/**
- * A sink that writes data to a CSV file.
- *
- * @example
- *
- * ```ts
- * const sink = csv({
- *   filepath: "./data.csv",
- *   csvOptions: {
- *     header: true,
- *   },
- * });
- *
- * ...
- * async transform({context, endCursor}){
- *  const { writer } = useSink(context);
- *  const insertHelper = writer(endCursor);
- *
- *  insertHelper.insert([
- *    { id: 1, name: "John" },
- *    { id: 2, name: "Jane" },
- *  ]);
- * }
- *
- * ```
- */
-export class CsvSink extends Sink {
-  constructor(
-    private _stringifier: Stringifier,
-    private _config: CsvSinkOptions,
-  ) {
-    super();
-  }
-
-  private async write({
-    data,
-    endCursor,
-  }: { data: SinkData[]; endCursor?: Cursor }) {
-    // adds a "_cursor" property if "cursorColumn" is not specified by user
-    data = this.processCursorColumn(data, endCursor);
-    // Insert the data into csv
-    await this.insertToCSV(data);
-  }
-
-  async transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: TxnParams) => Promise<void>,
-  ) {
-    const context: TxnContext = {
-      buffer: [],
-    };
-
-    const writer = transactionHelper(context);
-
-    await cb({ writer });
-    await this.write({ data: context.buffer, endCursor });
-  }
-
-  async invalidate(cursor?: Cursor) {
-    // TODO: Implement
-    throw new Error("Not implemented");
-  }
-
-  private async insertToCSV(data: SinkData[]) {
-    if (data.length === 0) return;
-
-    return await new Promise<void>((resolve, reject) => {
-      for (const row of data) {
-        this._stringifier.write(row, (err) => {
-          if (err) throw new Error(err.message);
-
-          // resolve when all rows are inserted into csv
-          if (row === data[data.length - 1]) {
-            resolve();
-          }
-        });
-      }
-    });
-  }
-
-  private processCursorColumn(
-    data: SinkData[],
-    endCursor?: Cursor,
-  ): SinkData[] {
-    const { cursorColumn } = this._config;
-
-    if (
-      cursorColumn &&
-      data.some(
-        (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey),
-      )
-    ) {
-      throw new Error(
-        `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`,
-      );
-    }
-
-    if (cursorColumn) {
-      return data;
-    }
-
-    return data.map((row) => ({
-      ...row,
-      _cursor: Number(endCursor?.orderKey),
-    }));
-  }
-}
-
-export const csv = (args: CsvArgs & CsvSinkOptions) => {
-  const { csvOptions, filepath, ...sinkOptions } = args;
-  const stringifier = stringify({ ...csvOptions });
-
-  const writeStream = fs.createWriteStream(filepath, { flags: "a" });
-  stringifier.pipe(writeStream);
-  return new CsvSink(stringifier, sinkOptions);
-};

package/src/sinks/drizzle/Int8Range.ts

@@ -1,52 +0,0 @@
-import { customType } from "drizzle-orm/pg-core";
-import type { Range } from "postgres-range";
-import {
-  parse as rangeParse,
-  serialize as rangeSerialize,
-} from "postgres-range";
-
-type Comparable = string | number;
-
-type RangeBound<T extends Comparable> =
-  | T
-  | {
-      value: T;
-      inclusive: boolean;
-    };
-
-export class Int8Range {
-  constructor(public readonly range: Range<number>) {}
-
-  get start(): RangeBound<number> | null {
-    return this.range.lower != null
-      ? {
-          value: this.range.lower,
-          inclusive: this.range.isLowerBoundClosed(),
-        }
-      : null;
-  }
-
-  get end(): RangeBound<number> | null {
-    return this.range.upper != null
-      ? {
-          value: this.range.upper,
-          inclusive: this.range.isUpperBoundClosed(),
-        }
-      : null;
-  }
-}
-
-export const int8range = customType<{
-  data: Int8Range;
-}>({
-  dataType: () => "int8range",
-  fromDriver: (value: unknown): Int8Range => {
-    if (typeof value !== "string") {
-      throw new Error("Expected string");
-    }
-
-    const parsed = rangeParse(value, (val) => Number.parseInt(val, 10));
-    return new Int8Range(parsed);
-  },
-  toDriver: (value: Int8Range): string => rangeSerialize(value.range),
-});

package/src/sinks/drizzle/delete.ts

@@ -1,42 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import {
-  type ExtractTablesWithRelations,
-  type SQL,
-  type TablesRelationalConfig,
-  sql,
-} from "drizzle-orm";
-import type {
-  PgQueryResultHKT,
-  PgTable,
-  PgTransaction,
-  PgUpdateBase,
-} from "drizzle-orm/pg-core";
-
-export class DrizzleSinkDelete<
-  TTable extends PgTable,
-  TQueryResult extends PgQueryResultHKT,
-  TFullSchema extends Record<string, unknown> = Record<string, never>,
-  TSchema extends
-    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
-> {
-  constructor(
-    private db: PgTransaction<TQueryResult, TFullSchema, TSchema>,
-    private table: TTable,
-    private endCursor?: Cursor,
-  ) {}
-
-  //@ts-ignore
-  where(where: SQL): Omit<
-    // for type safety used PgUpdateBase instead of PgDeleteBase
-    PgUpdateBase<TTable, TQueryResult, undefined, false, "where">,
-    "where"
-  > {
-    return this.db
-      .update(this.table)
-      .set({
-        // @ts-ignore
-        _cursor: sql`int8range(lower(_cursor), ${Number(this.endCursor?.orderKey!)}, '[)')`,
-      })
-      .where(where);
-  }
-}