@apibara/indexer 2.0.0-beta.9 → 2.1.0-beta.2

package/src/sinks/drizzle/drizzle.test.ts

@@ -1,239 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import {
-  type MockBlock,
-  MockClient,
-  type MockFilter,
-} from "@apibara/protocol/testing";
-import { asc, eq, sql } from "drizzle-orm";
-import { drizzle } from "drizzle-orm/node-postgres";
-import { serial, text } from "drizzle-orm/pg-core";
-import { Client } from "pg";
-import { beforeAll, beforeEach, describe, expect, it } from "vitest";
-import {
-  type Int8Range,
-  drizzle as drizzleSink,
-  getDrizzleCursor,
-  pgTable,
-} from ".";
-import { useSink } from "../../hooks";
-import { run } from "../../indexer";
-import { generateMockMessages } from "../../testing";
-import { getMockIndexer } from "../../testing/indexer";
-
-const testTable = pgTable("test_table", {
-  id: serial("id").primaryKey(),
-  data: text("data"),
-});
-
-const client = new Client({
-  connectionString: "postgres://postgres:postgres@localhost:5432/postgres",
-});
-
-await client.connect();
-
-const db = drizzle(client);
-
-describe("Drizzle Test", () => {
-  beforeAll(async () => {
-    // drop test_table if exists
-    await db.execute(sql`DROP TABLE IF EXISTS test_table`);
-    // create test_table with db
-    await db.execute(
-      sql`CREATE TABLE test_table (id SERIAL PRIMARY KEY, data TEXT, _cursor INT8RANGE)`,
-    );
-  });
-
-  beforeEach(async () => {
-    await db.delete(testTable).execute();
-  });
-
-  it("should insert data", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return generateMockMessages(5);
-    });
-
-    const sink = drizzleSink({ database: db, tables: [testTable] });
-
-    const indexer = getMockIndexer({
-      sink,
-      override: {
-        transform: async ({ context, endCursor, block: { data } }) => {
-          const { db } = useSink({ context });
-          // Insert a new row into the test_table
-          // The id is set to the current cursor's orderKey
-          // The data is set to the block data
-          await db
-            .insert(testTable)
-            .values([{ id: Number(endCursor?.orderKey), data }]);
-        },
-      },
-    });
-
-    await run(client, indexer);
-
-    const result = await db.select().from(testTable).orderBy(asc(testTable.id));
-
-    expect(result).toHaveLength(5);
-    expect(result[0].data).toBe("5000000");
-    expect(result[2].data).toBe("5000002");
-  });
-
-  it("should update data", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return generateMockMessages(5);
-    });
-
-    const sink = drizzleSink({ database: db, tables: [testTable] });
-
-    const indexer = getMockIndexer({
-      sink,
-      override: {
-        transform: async ({ context, endCursor, block: { data } }) => {
-          const { db } = useSink({ context });
-
-          // insert data for each message in db
-          await db
-            .insert(testTable)
-            .values([{ id: Number(endCursor?.orderKey), data }]);
-
-          // update data for id 5000002 when orderKey is 5000004
-          // this is to test if the update query is working
-          if (endCursor?.orderKey === 5000004n) {
-            await db
-              .update(testTable)
-              .set({ data: "0000000" })
-              .where(eq(testTable.id, 5000002));
-          }
-        },
-      },
-    });
-
-    await run(client, indexer);
-
-    const result = await db.select().from(testTable).orderBy(asc(testTable.id));
-
-    expect(result).toHaveLength(5);
-    expect(result[2].data).toBe("0000000");
-  });
-
-  it("should delete data", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return generateMockMessages(5);
-    });
-
-    const sink = drizzleSink({ database: db, tables: [testTable] });
-
-    const indexer = getMockIndexer({
-      sink,
-      override: {
-        transform: async ({ context, endCursor, block: { data } }) => {
-          const { db } = useSink({ context });
-
-          // insert data for each message in db
-          await db
-            .insert(testTable)
-            .values([{ id: Number(endCursor?.orderKey), data }]);
-
-          // delete data for id 5000002 when orderKey is 5000004
-          // this is to test if the delete query is working
-          if (endCursor?.orderKey === 5000004n) {
-            await db.delete(testTable).where(eq(testTable.id, 5000002));
-          }
-        },
-      },
-    });
-
-    await run(client, indexer);
-
-    const result = await db.select().from(testTable).orderBy(asc(testTable.id));
-
-    expect(result).toHaveLength(5);
-
-    // as when you run delete query on a data, it isnt literally deleted from the db,
-    // instead, we just update the upper bound of that row to the current cursor
-    // check if the cursor upper bound has been set correctly
-    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-    expect(((result[2] as any)._cursor as Int8Range).range.upper).toBe(5000004);
-  });
-
-  it("should select data", async () => {
-    const client = new MockClient<MockFilter, MockBlock>((request, options) => {
-      return generateMockMessages(5);
-    });
-
-    const sink = drizzleSink({ database: db, tables: [testTable] });
-
-    let result: (typeof testTable.$inferSelect)[] = [];
-
-    const indexer = getMockIndexer({
-      sink,
-      override: {
-        transform: async ({ context, endCursor, block: { data } }) => {
-          const { db } = useSink({ context });
-
-          // insert data for each message in db
-          await db
-            .insert(testTable)
-            .values([{ id: Number(endCursor?.orderKey), data }]);
-
-          // delete data for id 5000002 when orderKey is 5000004
-          // this will update the upper bound of the row with id 5000002 from infinity to 5000004
-          // so when we select all rows, row with id 5000002 will not be included
-          // as when we run select query it should only return rows with upper bound infinity
-          if (endCursor?.orderKey === 5000003n) {
-            await db.delete(testTable).where(eq(testTable.id, 5000002));
-          }
-
-          // when on last message of mock stream, select all rows from db
-          if (endCursor?.orderKey === 5000004n) {
-            result = await db
-              .select()
-              .from(testTable)
-              .orderBy(asc(testTable.id));
-          }
-        },
-      },
-    });
-
-    await run(client, indexer);
-
-    expect(result).toHaveLength(4);
-    expect(result.find((r) => r.id === 5000002)).toBeUndefined();
-    // check if all rows are still in db
-    const allRows = await db.select().from(testTable);
-    expect(allRows).toHaveLength(5);
-  });
-
-  it("should invalidate data correctly", async () => {
-    const sink = drizzleSink({ database: db, tables: [testTable] });
-
-    // Insert some test data
-    await db.insert(testTable).values(
-      // @ts-ignore
-      [
-        { id: 1, data: "data1", _cursor: getDrizzleCursor([1n, 5n]) },
-        { id: 2, data: "data2", _cursor: getDrizzleCursor([2n, 5n]) },
-        { id: 3, data: "data3", _cursor: getDrizzleCursor(3n) },
-        { id: 4, data: "data4", _cursor: getDrizzleCursor(4n) },
-        { id: 5, data: "data5", _cursor: getDrizzleCursor(5n) },
-      ],
-    );
-
-    // Create a cursor at position 3
-    const cursor: Cursor = { orderKey: 3n };
-
-    // Invalidate data
-    await sink.invalidate(cursor);
-
-    // Check the results
-    const result = await db.select().from(testTable).orderBy(asc(testTable.id));
-
-    expect(result).toHaveLength(3);
-    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-    expect(((result[0] as any)._cursor as Int8Range).range.upper).toBe(null);
-    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-    expect(((result[1] as any)._cursor as Int8Range).range.upper).toBe(null);
-    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-    expect(((result[2] as any)._cursor as Int8Range).range.upper).toBe(null);
-  });
-});

package/src/sinks/drizzle/drizzle.ts

@@ -1,115 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import {
-  type ExtractTablesWithRelations,
-  type TablesRelationalConfig,
-  gt,
-  sql,
-} from "drizzle-orm";
-import type {
-  AnyPgTable,
-  PgDatabase,
-  PgQueryResultHKT,
-  PgTableWithColumns,
-  TableConfig,
-} from "drizzle-orm/pg-core";
-import { Sink, type SinkCursorParams } from "../../sink";
-import { DrizzleSinkTransaction } from "./transaction";
-
-export type DrizzleSinkTables<
-  TTableConfig extends Record<string, TableConfig>,
-> = {
-  [K in keyof TTableConfig]: PgTableWithColumns<TTableConfig[K]>;
-};
-
-export type DrizzleSinkOptions<
-  TQueryResult extends PgQueryResultHKT,
-  TFullSchema extends Record<string, unknown> = Record<string, never>,
-  TSchema extends
-    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
-> = {
-  /**
-   * Database instance of drizzle-orm
-   */
-  database: PgDatabase<TQueryResult, TFullSchema, TSchema>;
-  tables: AnyPgTable[];
-};
-
-/**
- * A sink that writes data to a PostgreSQL database using Drizzle ORM.
- *
- * @example
- *
- * ```ts
- * const sink = drizzle({
- *   database: db,
- * });
- *
- * ...
- * async transform({context, endCursor}){
- *  const { transaction } = useSink(context);
- *  const db = transaction(endCursor);
- *
- *  db.insert(users).values([
- *    { id: 1, name: "John" },
- *    { id: 2, name: "Jane" },
- *  ]);
- * }
- *
- * ```
- */
-export class DrizzleSink<
-  TQueryResult extends PgQueryResultHKT,
-  TFullSchema extends Record<string, unknown> = Record<string, never>,
-  TSchema extends
-    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
-> extends Sink {
-  private _db: PgDatabase<TQueryResult, TFullSchema, TSchema>;
-  private _tables: AnyPgTable[];
-  constructor(options: DrizzleSinkOptions<TQueryResult, TFullSchema, TSchema>) {
-    super();
-    const { database, tables } = options;
-    this._db = database;
-    this._tables = tables;
-  }
-
-  async transaction(
-    { cursor, endCursor, finality }: SinkCursorParams,
-    cb: (params: {
-      db: DrizzleSinkTransaction<TQueryResult, TFullSchema, TSchema>;
-    }) => Promise<void>,
-  ): Promise<void> {
-    await this._db.transaction(async (db) => {
-      await cb({ db: new DrizzleSinkTransaction(db, endCursor) });
-    });
-  }
-
-  async invalidate(cursor?: Cursor) {
-    await this._db.transaction(async (db) => {
-      for (const table of this._tables) {
-        // delete all rows whose lowerbound of "_cursor" (int8range) column is greater than the invalidate cursor
-        await db
-          .delete(table)
-          .where(gt(sql`lower(_cursor)`, sql`${Number(cursor?.orderKey)}`))
-          .returning();
-        // and for rows whose upperbound of "_cursor" (int8range) column is greater than the invalidate cursor, set the upperbound to infinity
-        await db
-          .update(table)
-          .set({
-            _cursor: sql`int8range(lower(_cursor), NULL, '[)')`,
-          })
-          .where(gt(sql`upper(_cursor)`, sql`${Number(cursor?.orderKey)}`));
-      }
-    });
-  }
-}
-
-export const drizzle = <
-  TQueryResult extends PgQueryResultHKT,
-  TFullSchema extends Record<string, unknown> = Record<string, never>,
-  TSchema extends
-    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
->(
-  args: DrizzleSinkOptions<TQueryResult, TFullSchema, TSchema>,
-) => {
-  return new DrizzleSink(args);
-};

package/src/sinks/drizzle/index.ts

@@ -1,6 +0,0 @@
-export * from "./drizzle";
-export * from "./Int8Range";
-export * from "./utils";
-export * from "./transaction";
-export * from "./update";
-export * from "./delete";

package/src/sinks/drizzle/insert.ts

@@ -1,42 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import type {
-  ExtractTablesWithRelations,
-  TablesRelationalConfig,
-} from "drizzle-orm";
-import type {
-  PgInsertValue as DrizzleInsertValue,
-  PgQueryResultHKT,
-  PgTable,
-  PgTransaction,
-} from "drizzle-orm/pg-core";
-import { type PgInsertValue, getDrizzleCursor } from "./utils";
-
-export class DrizzleSinkInsert<
-  TTable extends PgTable,
-  TQueryResult extends PgQueryResultHKT,
-  TFullSchema extends Record<string, unknown> = Record<string, never>,
-  TSchema extends
-    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
-> {
-  constructor(
-    private db: PgTransaction<TQueryResult, TFullSchema, TSchema>,
-    private table: TTable,
-    private endCursor?: Cursor,
-  ) {}
-
-  values(values: PgInsertValue<TTable> | PgInsertValue<TTable>[]) {
-    const originalInsert = this.db.insert(this.table);
-    const cursoredValues = (Array.isArray(values) ? values : [values]).map(
-      (v) => {
-        return {
-          ...v,
-          _cursor: getDrizzleCursor(this.endCursor?.orderKey),
-        };
-      },
-    );
-
-    return originalInsert.values(
-      cursoredValues as DrizzleInsertValue<TTable>[],
-    );
-  }
-}

package/src/sinks/drizzle/select.ts

@@ -1,44 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import {
-  type ExtractTablesWithRelations,
-  type SQL,
-  type Subquery,
-  type TablesRelationalConfig,
-  sql,
-} from "drizzle-orm";
-import type {
-  PgQueryResultHKT,
-  PgTable,
-  PgTransaction,
-  SelectedFields,
-} from "drizzle-orm/pg-core";
-import type { PgViewBase } from "drizzle-orm/pg-core/view-base";
-
-export class DrizzleSinkSelect<
-  TSelection extends SelectedFields,
-  TQueryResult extends PgQueryResultHKT,
-  TFullSchema extends Record<string, unknown> = Record<string, never>,
-  TSchema extends
-    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
-> {
-  constructor(
-    private db: PgTransaction<TQueryResult, TFullSchema, TSchema>,
-    private fields?: TSelection,
-    private endCursor?: Cursor,
-  ) {}
-
-  from<TFrom extends PgTable | Subquery | PgViewBase | SQL>(source: TFrom) {
-    if (this.fields) {
-      const originalFrom = this.db.select(this.fields).from(source);
-      return {
-        ...originalFrom,
-        where: (where?: SQL) => {
-          const combinedWhere = sql`${where ? sql`${where} AND ` : sql``}upper_inf(_cursor)`;
-          return originalFrom.where(combinedWhere);
-        },
-      };
-    }
-
-    return this.db.select().from(source).where(sql`upper_inf(_cursor)`);
-  }
-}

package/src/sinks/drizzle/transaction.ts

@@ -1,49 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import type {
-  ExtractTablesWithRelations,
-  TablesRelationalConfig,
-} from "drizzle-orm";
-import type {
-  PgQueryResultHKT,
-  PgSelectBuilder,
-  PgTable,
-  PgTransaction,
-  SelectedFields,
-} from "drizzle-orm/pg-core";
-import { DrizzleSinkDelete } from "./delete";
-import { DrizzleSinkInsert } from "./insert";
-import { DrizzleSinkSelect } from "./select";
-import { DrizzleSinkUpdate } from "./update";
-
-export class DrizzleSinkTransaction<
-  TQueryResult extends PgQueryResultHKT,
-  TFullSchema extends Record<string, unknown> = Record<string, never>,
-  TSchema extends
-    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
-> {
-  constructor(
-    private db: PgTransaction<TQueryResult, TFullSchema, TSchema>,
-    private endCursor?: Cursor,
-  ) {}
-
-  insert<TTable extends PgTable>(table: TTable) {
-    return new DrizzleSinkInsert(this.db, table, this.endCursor);
-  }
-
-  update<TTable extends PgTable>(table: TTable) {
-    return new DrizzleSinkUpdate(this.db, table, this.endCursor);
-  }
-
-  delete<TTable extends PgTable>(table: TTable) {
-    return new DrizzleSinkDelete(this.db, table, this.endCursor);
-  }
-
-  // @ts-ignore
-  select(): PgSelectBuilder<undefined>;
-  select<TSelection extends SelectedFields>(
-    fields: TSelection,
-  ): PgSelectBuilder<TSelection>;
-  select(fields?: SelectedFields) {
-    return new DrizzleSinkSelect(this.db, fields, this.endCursor);
-  }
-}

package/src/sinks/drizzle/update.ts

@@ -1,47 +0,0 @@
-import type { Cursor } from "@apibara/protocol";
-import {
-  type ExtractTablesWithRelations,
-  type SQL,
-  type TablesRelationalConfig,
-  sql,
-} from "drizzle-orm";
-import type {
-  PgQueryResultHKT,
-  PgTable,
-  PgTransaction,
-  PgUpdateBase,
-  PgUpdateSetSource,
-} from "drizzle-orm/pg-core";
-
-export class DrizzleSinkUpdate<
-  TTable extends PgTable,
-  TQueryResult extends PgQueryResultHKT,
-  TFullSchema extends Record<string, unknown> = Record<string, never>,
-  TSchema extends
-    TablesRelationalConfig = ExtractTablesWithRelations<TFullSchema>,
-> {
-  constructor(
-    private db: PgTransaction<TQueryResult, TFullSchema, TSchema>,
-    private table: TTable,
-    private endCursor?: Cursor,
-  ) {}
-
-  set(values: PgUpdateSetSource<TTable>): PgUpdateBase<TTable, TQueryResult> {
-    const originalUpdate = this.db.update(this.table);
-    const originalSet = originalUpdate.set(values);
-    return {
-      ...originalSet,
-      where: async (where: SQL | undefined) => {
-        await this.db
-          .update(this.table)
-          .set({
-            _cursor: sql`int8range(lower(_cursor), ${Number(this.endCursor?.orderKey!)}, '[)')`,
-          } as PgUpdateSetSource<TTable>)
-          .where(sql`${where ? sql`${where} AND ` : sql``}upper_inf(_cursor)`)
-          .execute();
-
-        return originalSet.where(where);
-      },
-    } as PgUpdateBase<TTable, TQueryResult>;
-  }
-}

package/src/sinks/drizzle/utils.ts

@@ -1,99 +0,0 @@
-import type {
-  BuildColumns,
-  BuildExtraConfigColumns,
-  NotNull,
-  Placeholder,
-  SQL,
-} from "drizzle-orm";
-import type {
-  PgColumnBuilderBase,
-  PgCustomColumnBuilder,
-  PgTable,
-  PgTableExtraConfig,
-  PgTableWithColumns,
-} from "drizzle-orm/pg-core";
-import { pgTable as drizzlePgTable } from "drizzle-orm/pg-core";
-import range from "postgres-range";
-import { Int8Range, int8range } from "./Int8Range";
-
-export type CursorColumnBuilder = NotNull<
-  PgCustomColumnBuilder<{
-    name: "_cursor";
-    dataType: "custom";
-    columnType: "PgCustomColumn";
-    data: Int8Range;
-    driverParam: undefined;
-    enumValues: undefined;
-    generated: undefined;
-  }>
->;
-
-// Redefining the type of `pgTable` to include the `_cursor` column.
-export type PgTableWithCursorFn<
-  TSchema extends string | undefined = undefined,
-> = <
-  TTableName extends string,
-  TColumnsMap extends Record<string, PgColumnBuilderBase>,
->(
-  name: TTableName,
-  columns: TColumnsMap,
-  extraConfig?: (
-    self: BuildExtraConfigColumns<
-      TTableName,
-      TColumnsMap & { _cursor: CursorColumnBuilder },
-      "pg"
-    >,
-  ) => PgTableExtraConfig,
-) => PgTableWithColumns<{
-  name: TTableName;
-  schema: TSchema;
-  columns: BuildColumns<
-    TTableName,
-    TColumnsMap & { _cursor: CursorColumnBuilder },
-    "pg"
-  >;
-  dialect: "pg";
-}>;
-
-// Same as the drizzle's `PgInsertValue` type, but without the `_cursor` column.
-export type PgInsertValue<TTable extends PgTable> = Omit<
-  {
-    [Key in keyof TTable["$inferInsert"]]:
-      | TTable["$inferInsert"][Key]
-      | SQL
-      | Placeholder;
-  } & {},
-  "_cursor"
->;
-
-export const pgTable: PgTableWithCursorFn = (name, columns, extraConfig?) => {
-  return drizzlePgTable(
-    name,
-    {
-      ...columns,
-      _cursor: int8range("_cursor").notNull(),
-    },
-    extraConfig,
-  );
-};
-
-export const getDrizzleCursor = (
-  cursor_range: [bigint | undefined, bigint | undefined] | bigint | undefined,
-) => {
-  const isArray = Array.isArray(cursor_range);
-  const [lower, upper] = isArray ? cursor_range : [cursor_range, undefined];
-  let isNoUpperBound = false;
-  if (!lower) {
-    throw new Error("Lower bound cursor is required");
-  }
-  if (!upper) {
-    isNoUpperBound = true;
-  }
-  return new Int8Range(
-    new range.Range(
-      Number(lower),
-      Number(upper),
-      range.RANGE_LB_INC | (isNoUpperBound ? range.RANGE_UB_INF : 0),
-    ),
-  );
-};