@apibara/indexer 2.0.0-beta.5 → 2.0.0-beta.7

package/dist/sinks/drizzle/index.mjs

@@ -0,0 +1,196 @@
+import { sql, gt } from 'drizzle-orm';
+import { S as Sink } from '../../shared/indexer.93d6b2eb.mjs';
+import { customType, pgTable as pgTable$1 } from 'drizzle-orm/pg-core';
+import range, { parse, serialize } from 'postgres-range';
+import 'consola';
+
+class DrizzleSinkDelete {
+  constructor(db, table, endCursor) {
+    this.db = db;
+    this.table = table;
+    this.endCursor = endCursor;
+  }
+  //@ts-ignore
+  where(where) {
+    return this.db.update(this.table).set({
+      // @ts-ignore
+      _cursor: sql`int8range(lower(_cursor), ${Number(this.endCursor?.orderKey)}, '[)')`
+    }).where(where);
+  }
+}
+
+class Int8Range {
+  constructor(range) {
+    this.range = range;
+  }
+  get start() {
+    return this.range.lower != null ? {
+      value: this.range.lower,
+      inclusive: this.range.isLowerBoundClosed()
+    } : null;
+  }
+  get end() {
+    return this.range.upper != null ? {
+      value: this.range.upper,
+      inclusive: this.range.isUpperBoundClosed()
+    } : null;
+  }
+}
+const int8range = customType({
+  dataType: () => "int8range",
+  fromDriver: (value) => {
+    if (typeof value !== "string") {
+      throw new Error("Expected string");
+    }
+    const parsed = parse(value, (val) => Number.parseInt(val, 10));
+    return new Int8Range(parsed);
+  },
+  toDriver: (value) => serialize(value.range)
+});
+
+const pgTable = (name, columns, extraConfig) => {
+  return pgTable$1(
+    name,
+    {
+      ...columns,
+      _cursor: int8range("_cursor").notNull()
+    },
+    extraConfig
+  );
+};
+const getDrizzleCursor = (cursor_range) => {
+  const isArray = Array.isArray(cursor_range);
+  const [lower, upper] = isArray ? cursor_range : [cursor_range, void 0];
+  let isNoUpperBound = false;
+  if (!lower) {
+    throw new Error("Lower bound cursor is required");
+  }
+  if (!upper) {
+    isNoUpperBound = true;
+  }
+  return new Int8Range(
+    new range.Range(
+      Number(lower),
+      Number(upper),
+      range.RANGE_LB_INC | (isNoUpperBound ? range.RANGE_UB_INF : 0)
+    )
+  );
+};
+
+class DrizzleSinkInsert {
+  constructor(db, table, endCursor) {
+    this.db = db;
+    this.table = table;
+    this.endCursor = endCursor;
+  }
+  values(values) {
+    const originalInsert = this.db.insert(this.table);
+    const cursoredValues = (Array.isArray(values) ? values : [values]).map(
+      (v) => {
+        return {
+          ...v,
+          _cursor: getDrizzleCursor(this.endCursor?.orderKey)
+        };
+      }
+    );
+    return originalInsert.values(cursoredValues);
+  }
+}
+
+class DrizzleSinkSelect {
+  constructor(db, fields, endCursor) {
+    this.db = db;
+    this.fields = fields;
+    this.endCursor = endCursor;
+  }
+  from(source) {
+    if (this.fields) {
+      const originalFrom = this.db.select(this.fields).from(source);
+      return {
+        ...originalFrom,
+        where: (where) => {
+          const combinedWhere = sql`${where ? sql`${where} AND ` : sql``}upper_inf(_cursor)`;
+          return originalFrom.where(combinedWhere);
+        }
+      };
+    }
+    return this.db.select().from(source).where(sql`upper_inf(_cursor)`);
+  }
+}
+
+class DrizzleSinkUpdate {
+  constructor(db, table, endCursor) {
+    this.db = db;
+    this.table = table;
+    this.endCursor = endCursor;
+  }
+  set(values) {
+    const originalUpdate = this.db.update(this.table);
+    const originalSet = originalUpdate.set(values);
+    return {
+      ...originalSet,
+      where: async (where) => {
+        await this.db.update(this.table).set({
+          _cursor: sql`int8range(lower(_cursor), ${Number(this.endCursor?.orderKey)}, '[)')`
+        }).where(sql`${where ? sql`${where} AND ` : sql``}upper_inf(_cursor)`).execute();
+        return originalSet.where(where);
+      }
+    };
+  }
+}
+
+class DrizzleSinkTransaction {
+  constructor(db, endCursor) {
+    this.db = db;
+    this.endCursor = endCursor;
+  }
+  insert(table) {
+    return new DrizzleSinkInsert(this.db, table, this.endCursor);
+  }
+  update(table) {
+    return new DrizzleSinkUpdate(this.db, table, this.endCursor);
+  }
+  delete(table) {
+    return new DrizzleSinkDelete(this.db, table, this.endCursor);
+  }
+  select(fields) {
+    return new DrizzleSinkSelect(this.db, fields, this.endCursor);
+  }
+}
+
+var __defProp = Object.defineProperty;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __publicField = (obj, key, value) => {
+  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+  return value;
+};
+class DrizzleSink extends Sink {
+  constructor(options) {
+    super();
+    __publicField(this, "_db");
+    __publicField(this, "_tables");
+    const { database, tables } = options;
+    this._db = database;
+    this._tables = tables;
+  }
+  async transaction({ cursor, endCursor, finality }, cb) {
+    await this._db.transaction(async (db) => {
+      await cb({ db: new DrizzleSinkTransaction(db, endCursor) });
+    });
+  }
+  async invalidate(cursor) {
+    await this._db.transaction(async (db) => {
+      for (const table of this._tables) {
+        await db.delete(table).where(gt(sql`lower(_cursor)`, sql`${Number(cursor?.orderKey)}`)).returning();
+        await db.update(table).set({
+          _cursor: sql`int8range(lower(_cursor), NULL, '[)')`
+        }).where(gt(sql`upper(_cursor)`, sql`${Number(cursor?.orderKey)}`));
+      }
+    });
+  }
+}
+const drizzle = (args) => {
+  return new DrizzleSink(args);
+};
+
+export { DrizzleSink, DrizzleSinkDelete, DrizzleSinkTransaction, DrizzleSinkUpdate, Int8Range, drizzle, getDrizzleCursor, int8range, pgTable };

package/dist/sinks/sqlite.cjs

@@ -0,0 +1,90 @@
+'use strict';
+
+const sink = require('../shared/indexer.a8b7ab1f.cjs');
+require('consola');
+
+var __defProp = Object.defineProperty;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __publicField = (obj, key, value) => {
+  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+  return value;
+};
+const transactionHelper = (context) => {
+  return {
+    insert: (data) => {
+      context.buffer.push(...data);
+    }
+  };
+};
+class SqliteSink extends sink.Sink {
+  constructor(options) {
+    super();
+    __publicField(this, "_config");
+    __publicField(this, "_db");
+    const { database, ...config } = options;
+    this._config = config;
+    this._db = database;
+  }
+  async write({
+    data,
+    endCursor
+  }) {
+    data = this.processCursorColumn(data, endCursor);
+    await this.insertJsonArray(data);
+  }
+  async transaction({ cursor, endCursor, finality }, cb) {
+    const context = {
+      buffer: []
+    };
+    const writer = transactionHelper(context);
+    await cb({ writer });
+    await this.write({ data: context.buffer, endCursor });
+  }
+  async invalidate(cursor) {
+    throw new Error("Not implemented");
+  }
+  async insertJsonArray(data) {
+    if (data.length === 0)
+      return;
+    const columns = Object.keys(data[0]);
+    const columnNames = columns.join(", ");
+    const placeholders = columns.map(() => "?").join(", ");
+    const conflictClause = this.buildConflictClause();
+    const insertSQL = `INSERT INTO ${this._config.tableName} (${columnNames}) VALUES `;
+    const valuePlaceholders = data.map(() => `(${placeholders})`).join(", ");
+    const statement = insertSQL + valuePlaceholders + conflictClause;
+    const values = data.flatMap((row) => columns.map((col) => row[col]));
+    this._db.prepare(statement).run(values);
+  }
+  processCursorColumn(data, endCursor) {
+    const { cursorColumn } = this._config;
+    if (cursorColumn && data.some(
+      (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey)
+    )) {
+      throw new Error(
+        `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`
+      );
+    }
+    if (cursorColumn) {
+      return data;
+    }
+    return data.map((row) => ({
+      ...row,
+      _cursor: Number(endCursor?.orderKey)
+    }));
+  }
+  buildConflictClause() {
+    const { on, update } = this._config.onConflict || {};
+    if (on && update && update.length > 0) {
+      const updateColumns = update.map((col) => `${col}=excluded.${col}`).join(", ");
+      return ` ON CONFLICT(${on}) DO UPDATE SET ${updateColumns}`;
+    }
+    return "";
+  }
+}
+const sqlite = (args) => {
+  return new SqliteSink(args);
+};
+
+exports.SqliteSink = SqliteSink;
+exports.sqlite = sqlite;

package/dist/sinks/sqlite.d.cts

@@ -0,0 +1,71 @@
+import { Cursor } from '@apibara/protocol';
+import { Database } from 'better-sqlite3';
+import { S as Sink, a as SinkCursorParams, b as SinkData } from '../shared/indexer.b9c8f0d8.cjs';
+
+type SqliteSinkOptions = {
+    /**
+     * Database instance of better-sqlite3
+     */
+    database: Database;
+    /**
+     * The name of the table where data will be inserted.
+     */
+    tableName: string;
+    /**
+     * An optional column name used to store the cursor value. If specified,
+     * the value of this column must match the `endCursor.orderKey` for each row.
+     */
+    cursorColumn?: string;
+    /**
+     * An optional configuration to handle conflicts during data insertion.
+     * - `on`: The column name on which conflicts are detected.
+     * - `update`: An array of column names to be updated if a conflict occurs.
+     */
+    onConflict?: {
+        on: string;
+        update: string[];
+    };
+};
+type TxnParams = {
+    writer: {
+        insert: (data: SinkData[]) => void;
+    };
+};
+/**
+ * A sink that writes data to a SQLite database.
+ *
+ * @example
+ *
+ * ```ts
+ * const sink = sqlite({
+ *   database: db,
+ *   tableName: "test",
+ * });
+ *
+ * ...
+ * async transform({context, endCursor}){
+ *  const { writer } = useSink(context);
+ *  const insertHelper = writer(endCursor);
+ *
+ *  insertHelper.insert([
+ *    { id: 1, name: "John" },
+ *    { id: 2, name: "Jane" },
+ *  ]);
+ * }
+ *
+ * ```
+ */
+declare class SqliteSink extends Sink {
+    private _config;
+    private _db;
+    constructor(options: SqliteSinkOptions);
+    private write;
+    transaction({ cursor, endCursor, finality }: SinkCursorParams, cb: (params: TxnParams) => Promise<void>): Promise<void>;
+    invalidate(cursor?: Cursor): Promise<void>;
+    private insertJsonArray;
+    private processCursorColumn;
+    private buildConflictClause;
+}
+declare const sqlite: (args: SqliteSinkOptions) => SqliteSink;
+
+export { SqliteSink, type SqliteSinkOptions, sqlite };

package/dist/sinks/sqlite.d.mts

@@ -0,0 +1,71 @@
+import { Cursor } from '@apibara/protocol';
+import { Database } from 'better-sqlite3';
+import { S as Sink, a as SinkCursorParams, b as SinkData } from '../shared/indexer.b9c8f0d8.mjs';
+
+type SqliteSinkOptions = {
+    /**
+     * Database instance of better-sqlite3
+     */
+    database: Database;
+    /**
+     * The name of the table where data will be inserted.
+     */
+    tableName: string;
+    /**
+     * An optional column name used to store the cursor value. If specified,
+     * the value of this column must match the `endCursor.orderKey` for each row.
+     */
+    cursorColumn?: string;
+    /**
+     * An optional configuration to handle conflicts during data insertion.
+     * - `on`: The column name on which conflicts are detected.
+     * - `update`: An array of column names to be updated if a conflict occurs.
+     */
+    onConflict?: {
+        on: string;
+        update: string[];
+    };
+};
+type TxnParams = {
+    writer: {
+        insert: (data: SinkData[]) => void;
+    };
+};
+/**
+ * A sink that writes data to a SQLite database.
+ *
+ * @example
+ *
+ * ```ts
+ * const sink = sqlite({
+ *   database: db,
+ *   tableName: "test",
+ * });
+ *
+ * ...
+ * async transform({context, endCursor}){
+ *  const { writer } = useSink(context);
+ *  const insertHelper = writer(endCursor);
+ *
+ *  insertHelper.insert([
+ *    { id: 1, name: "John" },
+ *    { id: 2, name: "Jane" },
+ *  ]);
+ * }
+ *
+ * ```
+ */
+declare class SqliteSink extends Sink {
+    private _config;
+    private _db;
+    constructor(options: SqliteSinkOptions);
+    private write;
+    transaction({ cursor, endCursor, finality }: SinkCursorParams, cb: (params: TxnParams) => Promise<void>): Promise<void>;
+    invalidate(cursor?: Cursor): Promise<void>;
+    private insertJsonArray;
+    private processCursorColumn;
+    private buildConflictClause;
+}
+declare const sqlite: (args: SqliteSinkOptions) => SqliteSink;
+
+export { SqliteSink, type SqliteSinkOptions, sqlite };

package/dist/sinks/sqlite.d.ts

@@ -0,0 +1,71 @@
+import { Cursor } from '@apibara/protocol';
+import { Database } from 'better-sqlite3';
+import { S as Sink, a as SinkCursorParams, b as SinkData } from '../shared/indexer.b9c8f0d8.js';
+
+type SqliteSinkOptions = {
+    /**
+     * Database instance of better-sqlite3
+     */
+    database: Database;
+    /**
+     * The name of the table where data will be inserted.
+     */
+    tableName: string;
+    /**
+     * An optional column name used to store the cursor value. If specified,
+     * the value of this column must match the `endCursor.orderKey` for each row.
+     */
+    cursorColumn?: string;
+    /**
+     * An optional configuration to handle conflicts during data insertion.
+     * - `on`: The column name on which conflicts are detected.
+     * - `update`: An array of column names to be updated if a conflict occurs.
+     */
+    onConflict?: {
+        on: string;
+        update: string[];
+    };
+};
+type TxnParams = {
+    writer: {
+        insert: (data: SinkData[]) => void;
+    };
+};
+/**
+ * A sink that writes data to a SQLite database.
+ *
+ * @example
+ *
+ * ```ts
+ * const sink = sqlite({
+ *   database: db,
+ *   tableName: "test",
+ * });
+ *
+ * ...
+ * async transform({context, endCursor}){
+ *  const { writer } = useSink(context);
+ *  const insertHelper = writer(endCursor);
+ *
+ *  insertHelper.insert([
+ *    { id: 1, name: "John" },
+ *    { id: 2, name: "Jane" },
+ *  ]);
+ * }
+ *
+ * ```
+ */
+declare class SqliteSink extends Sink {
+    private _config;
+    private _db;
+    constructor(options: SqliteSinkOptions);
+    private write;
+    transaction({ cursor, endCursor, finality }: SinkCursorParams, cb: (params: TxnParams) => Promise<void>): Promise<void>;
+    invalidate(cursor?: Cursor): Promise<void>;
+    private insertJsonArray;
+    private processCursorColumn;
+    private buildConflictClause;
+}
+declare const sqlite: (args: SqliteSinkOptions) => SqliteSink;
+
+export { SqliteSink, type SqliteSinkOptions, sqlite };

package/dist/sinks/sqlite.mjs

@@ -0,0 +1,87 @@
+import { S as Sink } from '../shared/indexer.93d6b2eb.mjs';
+import 'consola';
+
+var __defProp = Object.defineProperty;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __publicField = (obj, key, value) => {
+  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+  return value;
+};
+const transactionHelper = (context) => {
+  return {
+    insert: (data) => {
+      context.buffer.push(...data);
+    }
+  };
+};
+class SqliteSink extends Sink {
+  constructor(options) {
+    super();
+    __publicField(this, "_config");
+    __publicField(this, "_db");
+    const { database, ...config } = options;
+    this._config = config;
+    this._db = database;
+  }
+  async write({
+    data,
+    endCursor
+  }) {
+    data = this.processCursorColumn(data, endCursor);
+    await this.insertJsonArray(data);
+  }
+  async transaction({ cursor, endCursor, finality }, cb) {
+    const context = {
+      buffer: []
+    };
+    const writer = transactionHelper(context);
+    await cb({ writer });
+    await this.write({ data: context.buffer, endCursor });
+  }
+  async invalidate(cursor) {
+    throw new Error("Not implemented");
+  }
+  async insertJsonArray(data) {
+    if (data.length === 0)
+      return;
+    const columns = Object.keys(data[0]);
+    const columnNames = columns.join(", ");
+    const placeholders = columns.map(() => "?").join(", ");
+    const conflictClause = this.buildConflictClause();
+    const insertSQL = `INSERT INTO ${this._config.tableName} (${columnNames}) VALUES `;
+    const valuePlaceholders = data.map(() => `(${placeholders})`).join(", ");
+    const statement = insertSQL + valuePlaceholders + conflictClause;
+    const values = data.flatMap((row) => columns.map((col) => row[col]));
+    this._db.prepare(statement).run(values);
+  }
+  processCursorColumn(data, endCursor) {
+    const { cursorColumn } = this._config;
+    if (cursorColumn && data.some(
+      (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey)
+    )) {
+      throw new Error(
+        `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`
+      );
+    }
+    if (cursorColumn) {
+      return data;
+    }
+    return data.map((row) => ({
+      ...row,
+      _cursor: Number(endCursor?.orderKey)
+    }));
+  }
+  buildConflictClause() {
+    const { on, update } = this._config.onConflict || {};
+    if (on && update && update.length > 0) {
+      const updateColumns = update.map((col) => `${col}=excluded.${col}`).join(", ");
+      return ` ON CONFLICT(${on}) DO UPDATE SET ${updateColumns}`;
+    }
+    return "";
+  }
+}
+const sqlite = (args) => {
+  return new SqliteSink(args);
+};
+
+export { SqliteSink, sqlite };

package/dist/testing/index.cjs

@@ -0,0 +1,64 @@
+'use strict';
+
+const vitest = require('vitest');
+const helper = require('../shared/indexer.318d3617.cjs');
+const vcr_index = require('../shared/indexer.541d43eb.cjs');
+require('node:async_hooks');
+require('unctx');
+require('@opentelemetry/api');
+require('node:fs');
+require('node:path');
+require('node:fs/promises');
+require('klona/full');
+require('consola');
+require('hookable');
+require('node:assert');
+require('../shared/indexer.a8b7ab1f.cjs');
+require('@apibara/protocol/testing');
+
+const test = vitest.test.extend({
+  vcr: {
+    withClient
+  }
+});
+async function withClient(cassetteName, range, callback) {
+  const vcrConfig = {
+    cassetteDir: "cassettes"
+  };
+  const cassetteOptions = {
+    name: cassetteName,
+    startingCursor: {
+      orderKey: range.fromBlock
+    },
+    endingCursor: {
+      orderKey: range.toBlock
+    }
+  };
+  const context = {
+    async run(indexer) {
+      const client = vcr_index.loadCassette(vcrConfig, cassetteName);
+      if (!helper.isCassetteAvailable(vcrConfig, cassetteName)) {
+        await vcr_index.record(vcrConfig, client, indexer, cassetteOptions);
+      }
+      return await vcr_index.replay(vcrConfig, indexer, cassetteName);
+    }
+  };
+  await callback(context);
+}
+
+function generateMockMessages(count = 10) {
+  return [...Array(count)].map((_, i) => ({
+    _tag: "data",
+    data: {
+      cursor: { orderKey: BigInt(5e6 - 1) },
+      finality: "accepted",
+      data: [{ data: `${5e6 + i}` }],
+      endCursor: { orderKey: BigInt(5e6 + i) }
+    }
+  }));
+}
+
+exports.VcrSink = vcr_index.VcrSink;
+exports.vcr = vcr_index.vcr;
+exports.generateMockMessages = generateMockMessages;
+exports.test = test;

package/dist/testing/index.d.cts

@@ -0,0 +1,40 @@
+import * as vitest from 'vitest';
+import { f as Indexer } from '../shared/indexer.c7ed6b83.cjs';
+import { a as VcrReplayResult } from '../shared/indexer.500fd281.cjs';
+import { Cursor } from '@apibara/protocol';
+import { S as Sink, b as SinkData, a as SinkCursorParams } from '../shared/indexer.b9c8f0d8.cjs';
+import { MockStreamResponse } from '@apibara/protocol/testing';
+import 'hookable';
+
+declare const test: vitest.TestAPI<{
+    vcr: {
+        withClient: typeof withClient;
+    };
+}>;
+type WithClientContext<TFilter, TBlock, TTxnParams> = {
+    run: (indexerArgs: Indexer<TFilter, TBlock, TTxnParams>) => Promise<VcrReplayResult>;
+};
+declare function withClient<TFilter, TBlock, TTxnParams>(cassetteName: string, range: {
+    fromBlock: bigint;
+    toBlock: bigint;
+}, callback: (context: WithClientContext<TFilter, TBlock, TTxnParams>) => Promise<void>): Promise<void>;
+
+type TxnParams = {
+    writer: {
+        insert: (data: SinkData[]) => void;
+    };
+};
+declare class VcrSink extends Sink {
+    result: VcrReplayResult["outputs"];
+    write({ data, endCursor }: {
+        data: SinkData[];
+        endCursor?: Cursor;
+    }): void;
+    transaction({ cursor, endCursor, finality }: SinkCursorParams, cb: (params: TxnParams) => Promise<void>): Promise<void>;
+    invalidate(cursor?: Cursor): Promise<void>;
+}
+declare function vcr(): VcrSink;
+
+declare function generateMockMessages(count?: number): MockStreamResponse[];
+
+export { VcrSink, generateMockMessages, test, vcr };