@apibara/indexer 2.0.0-beta.0 → 2.0.0-beta.3

package/dist/shared/indexer.c8ef02ea.cjs

@@ -1,289 +0,0 @@
-'use strict';
-
-const fs$1 = require('node:fs/promises');
-const path = require('node:path');
-const hookable = require('hookable');
-const full = require('klona/full');
-const assert = require('node:assert');
-const fs = require('node:fs');
-const testing = require('@apibara/protocol/testing');
-const consola = require('consola');
-const node_async_hooks = require('node:async_hooks');
-const unctx = require('unctx');
-const api = require('@opentelemetry/api');
-const sink = require('./indexer.e05aedca.cjs');
-
-function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
-
-const fs__default$1 = /*#__PURE__*/_interopDefaultCompat(fs$1);
-const path__default = /*#__PURE__*/_interopDefaultCompat(path);
-const assert__default = /*#__PURE__*/_interopDefaultCompat(assert);
-const fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
-const consola__default = /*#__PURE__*/_interopDefaultCompat(consola);
-
-const indexerAsyncContext = unctx.getContext("indexer", {
-  asyncContext: true,
-  AsyncLocalStorage: node_async_hooks.AsyncLocalStorage
-});
-function useIndexerContext() {
-  return indexerAsyncContext.use();
-}
-
-const tracer = api.trace.getTracer("@apibara/indexer");
-
-function defineIndexer(streamConfig) {
-  return (config) => ({
-    streamConfig,
-    ...config
-  });
-}
-function createIndexer({
-  streamConfig,
-  ...options
-}) {
-  const indexer = {
-    options,
-    streamConfig,
-    hooks: hookable.createHooks()
-  };
-  if (indexer.options.debug) {
-    hookable.createDebugger(indexer.hooks, { tag: "indexer" });
-  }
-  indexer.hooks.addHooks(indexer.options.hooks ?? {});
-  for (const plugin of indexer.options.plugins ?? []) {
-    plugin(indexer);
-  }
-  return indexer;
-}
-async function run(client, indexer, sinkArg) {
-  await indexerAsyncContext.callAsync({}, async () => {
-    await indexer.hooks.callHook("run:before");
-    const sink$1 = sinkArg ?? sink.defaultSink();
-    sink$1.hook("write", async ({ data }) => {
-      await indexer.hooks.callHook("sink:write", { data });
-    });
-    sink$1.hook("flush", async ({ endCursor, finality }) => {
-      await indexer.hooks.callHook("sink:flush", { endCursor, finality });
-    });
-    const isFactoryMode = indexer.options.factory !== void 0;
-    const request = indexer.streamConfig.Request.make({
-      filter: isFactoryMode ? [indexer.options.filter, {}] : [indexer.options.filter],
-      finality: indexer.options.finality,
-      startingCursor: indexer.options.startingCursor
-    });
-    const options = {};
-    await indexer.hooks.callHook("connect:before", { request, options });
-    let mainFilter;
-    if (isFactoryMode) {
-      mainFilter = request.filter[1];
-    }
-    let stream = client.streamData(request, options);
-    await indexer.hooks.callHook("connect:after");
-    let state = {
-      _tag: "normal"
-    };
-    while (true) {
-      for await (const message of stream) {
-        await indexer.hooks.callHook("message", { message });
-        switch (message._tag) {
-          case "data": {
-            await tracer.startActiveSpan("message data", async (span) => {
-              const blocks = message.data.data;
-              const { cursor, endCursor, finality } = message.data;
-              let block;
-              const output = [];
-              if (isFactoryMode) {
-                assert__default(indexer.options.factory !== void 0);
-                const [factoryBlock, mainBlock] = blocks;
-                block = mainBlock;
-                if (state._tag === "normal" && factoryBlock !== null) {
-                  const { data, filter } = await indexer.options.factory(factoryBlock);
-                  if (!filter) {
-                    output.push(...data ?? []);
-                  } else {
-                    mainFilter = indexer.streamConfig.mergeFilter(
-                      mainFilter,
-                      filter
-                    );
-                    const request2 = indexer.streamConfig.Request.make({
-                      filter: [indexer.options.filter, mainFilter],
-                      finality: indexer.options.finality,
-                      startingCursor: cursor
-                    });
-                    await indexer.hooks.callHook("connect:factory", {
-                      request: request2,
-                      endCursor
-                    });
-                    stream = client.streamData(request2, options);
-                    state = {
-                      _tag: "recover",
-                      data
-                    };
-                    return;
-                  }
-                } else if (state._tag === "recover") {
-                  output.push(...state.data ?? []);
-                  state = { _tag: "normal" };
-                }
-              } else {
-                block = blocks[0];
-              }
-              if (block) {
-                await tracer.startActiveSpan("handler", async (span2) => {
-                  await indexer.hooks.callHook("handler:before", {
-                    block,
-                    endCursor,
-                    finality
-                  });
-                  try {
-                    const transformOutput = await indexer.options.transform({
-                      block,
-                      cursor,
-                      endCursor,
-                      finality
-                    });
-                    output.push(...transformOutput);
-                    await indexer.hooks.callHook("handler:after", { output });
-                  } catch (error) {
-                    assert__default(error instanceof Error);
-                    await indexer.hooks.callHook("handler:exception", {
-                      error
-                    });
-                    throw error;
-                  }
-                  span2.end();
-                });
-              }
-              if (output.length > 0) {
-                await tracer.startActiveSpan("sink write", async (span2) => {
-                  await sink$1.write({
-                    data: output,
-                    cursor,
-                    endCursor,
-                    finality
-                  });
-                  span2.end();
-                });
-              }
-              span.end();
-            });
-            break;
-          }
-          default: {
-            consola__default.warn("unexpected message", message);
-            throw new Error("not implemented");
-          }
-        }
-        if (state._tag !== "normal") {
-          break;
-        }
-      }
-      if (state._tag !== "normal") {
-        continue;
-      }
-      await indexer.hooks.callHook("run:after");
-      break;
-    }
-  });
-}
-
-function deserialize(str) {
-  return JSON.parse(
-    str,
-    (_, value) => typeof value === "string" && value.match(/^\d+n$/) ? BigInt(value.slice(0, -1)) : value
-  );
-}
-function serialize(obj) {
-  return JSON.stringify(
-    obj,
-    (_, value) => typeof value === "bigint" ? `${value.toString()}n` : value,
-    "	"
-  );
-}
-function isCassetteAvailable(vcrConfig, cassetteName) {
-  const filePath = path__default.join(vcrConfig.cassetteDir, `${cassetteName}.json`);
-  return fs__default.existsSync(filePath);
-}
-
-async function record(vcrConfig, client, indexerArg, cassetteOptions) {
-  const indexer = full.klona(indexerArg);
-  const messages = [];
-  indexer.hooks.addHooks({
-    "connect:before"({ options, request }) {
-      request.startingCursor = cassetteOptions.startingCursor;
-      options.endingCursor = cassetteOptions.endingCursor;
-    },
-    message({ message }) {
-      messages.push(message);
-    },
-    async "run:after"() {
-      const output = {
-        filter: indexer.options.filter,
-        messages
-      };
-      const filePath = path__default.join(
-        vcrConfig.cassetteDir,
-        `${cassetteOptions.name}.json`
-      );
-      await fs__default$1.writeFile(filePath, serialize(output), { flag: "w" });
-    }
-  });
-  await run(client, indexer);
-}
-
-var __defProp = Object.defineProperty;
-var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
-var __publicField = (obj, key, value) => {
-  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
-  return value;
-};
-class VcrSink extends sink.Sink {
-  constructor() {
-    super(...arguments);
-    __publicField(this, "result", []);
-  }
-  async write({ data, endCursor, finality }) {
-    await this.callHook("write", { data });
-    this.result.push({ data, endCursor });
-    await this.callHook("flush", { endCursor, finality });
-  }
-}
-function vcr() {
-  return new VcrSink();
-}
-
-async function replay(vcrConfig, indexer, cassetteName) {
-  const client = loadCassette(vcrConfig, cassetteName);
-  const sink = vcr();
-  await run(client, indexer, sink);
-  return {
-    outputs: sink.result
-  };
-}
-function loadCassette(vcrConfig, cassetteName) {
-  const filePath = path__default.join(vcrConfig.cassetteDir, `${cassetteName}.json`);
-  const data = fs__default.readFileSync(filePath, "utf8");
-  const cassetteData = deserialize(data);
-  const { filter, messages } = cassetteData;
-  return new testing.MockClient((request, options) => {
-    assert__default.deepStrictEqual(
-      request.filter,
-      filter,
-      "Request and Cassette filter mismatch"
-    );
-    return messages;
-  });
-}
-
-exports.VcrSink = VcrSink;
-exports.createIndexer = createIndexer;
-exports.defineIndexer = defineIndexer;
-exports.deserialize = deserialize;
-exports.isCassetteAvailable = isCassetteAvailable;
-exports.loadCassette = loadCassette;
-exports.record = record;
-exports.replay = replay;
-exports.run = run;
-exports.serialize = serialize;
-exports.useIndexerContext = useIndexerContext;
-exports.vcr = vcr;

package/dist/shared/indexer.e05aedca.cjs

@@ -1,19 +0,0 @@
-'use strict';
-
-const hookable = require('hookable');
-
-class Sink extends hookable.Hookable {
-}
-class DefaultSink extends Sink {
-  async write({ data, endCursor, finality }) {
-    await this.callHook("write", { data });
-    await this.callHook("flush", { endCursor, finality });
-  }
-}
-function defaultSink() {
-  return new DefaultSink();
-}
-
-exports.DefaultSink = DefaultSink;
-exports.Sink = Sink;
-exports.defaultSink = defaultSink;

package/dist/shared/indexer.f7dd57e5.d.mts

@@ -1,91 +0,0 @@
-import { StreamDataRequest, StreamDataOptions, Cursor, DataFinality, StreamDataResponse, StreamConfig, Client } from '@apibara/protocol';
-import { NestedHooks, Hookable } from 'hookable';
-import { c as Sink, S as SinkData } from './indexer.7c118fb5.mjs';
-
-type IndexerPlugin<TFilter, TBlock, TRet> = (indexer: Indexer<TFilter, TBlock, TRet>) => void;
-declare function defineIndexerPlugin<TFilter, TBlock, TRet>(def: IndexerPlugin<TFilter, TBlock, TRet>): IndexerPlugin<TFilter, TBlock, TRet>;
-
-interface IndexerHooks<TFilter, TBlock, TRet> {
-    "run:before": () => void;
-    "run:after": () => void;
-    "connect:before": ({ request, options, }: {
-        request: StreamDataRequest<TFilter>;
-        options: StreamDataOptions;
-    }) => void;
-    "connect:after": () => void;
-    "connect:factory": ({ request, endCursor, }: {
-        request: StreamDataRequest<TFilter>;
-        endCursor?: Cursor;
-    }) => void;
-    "handler:before": ({ block, finality, endCursor, }: {
-        block: TBlock;
-        finality: DataFinality;
-        endCursor?: Cursor;
-    }) => void;
-    "handler:after": ({ output }: {
-        output: TRet[];
-    }) => void;
-    "handler:exception": ({ error }: {
-        error: Error;
-    }) => void;
-    "sink:write": ({ data }: {
-        data: TRet[];
-    }) => void;
-    "sink:flush": ({ endCursor, finality, }: {
-        endCursor?: Cursor;
-        finality: DataFinality;
-    }) => void;
-    message: ({ message }: {
-        message: StreamDataResponse<TBlock>;
-    }) => void;
-}
-interface IndexerConfig<TFilter, TBlock, TRet> {
-    streamUrl: string;
-    filter: TFilter;
-    finality?: DataFinality;
-    startingCursor?: Cursor;
-    factory?: (block: TBlock) => Promise<{
-        filter?: TFilter;
-        data?: TRet[];
-    }>;
-    transform: (args: {
-        block: TBlock;
-        cursor?: Cursor | undefined;
-        endCursor?: Cursor | undefined;
-        finality: DataFinality;
-    }) => Promise<TRet[]>;
-    hooks?: NestedHooks<IndexerHooks<TFilter, TBlock, TRet>>;
-    plugins?: ReadonlyArray<IndexerPlugin<TFilter, TBlock, TRet>>;
-    debug?: boolean;
-}
-interface IndexerWithStreamConfig<TFilter, TBlock, TRet> extends IndexerConfig<TFilter, TBlock, TRet> {
-    streamConfig: StreamConfig<TFilter, TBlock>;
-}
-declare function defineIndexer<TFilter, TBlock>(streamConfig: StreamConfig<TFilter, TBlock>): <TRet>(config: IndexerConfig<TFilter, TBlock, TRet>) => IndexerWithStreamConfig<TFilter, TBlock, TRet>;
-interface Indexer<TFilter, TBlock, TRet> {
-    streamConfig: StreamConfig<TFilter, TBlock>;
-    options: IndexerConfig<TFilter, TBlock, TRet>;
-    hooks: Hookable<IndexerHooks<TFilter, TBlock, TRet>>;
-}
-declare function createIndexer<TFilter, TBlock, TRet>({ streamConfig, ...options }: IndexerWithStreamConfig<TFilter, TBlock, TRet>): Indexer<TFilter, TBlock, TRet>;
-declare function run<TFilter, TBlock, TRet>(client: Client<TFilter, TBlock>, indexer: Indexer<TFilter, TBlock, TRet>, sinkArg?: Sink): Promise<void>;
-
-type VcrConfig = {
-    cassetteDir: string;
-};
-type CassetteOptions = {
-    name: string;
-    startingCursor: Cursor;
-    endingCursor: Cursor;
-};
-
-declare function replay<TFilter, TBlock, TRet>(vcrConfig: VcrConfig, indexer: Indexer<TFilter, TBlock, TRet>, cassetteName: string): Promise<VcrReplayResult>;
-type VcrReplayResult = {
-    outputs: Array<{
-        endCursor?: Cursor;
-        data: SinkData[];
-    }>;
-};
-declare function loadCassette<TFilter, TBlock>(vcrConfig: VcrConfig, cassetteName: string): Client<TFilter, TBlock>;
-
-export { type CassetteOptions as C, type Indexer as I, type VcrConfig as V, type IndexerHooks as a, type IndexerConfig as b, type IndexerWithStreamConfig as c, defineIndexer as d, createIndexer as e, type IndexerPlugin as f, defineIndexerPlugin as g, replay as h, type VcrReplayResult as i, loadCassette as l, run as r };

package/dist/sinks/csv.cjs

@@ -1,66 +0,0 @@
-'use strict';
-
-const fs = require('node:fs');
-const csvStringify = require('csv-stringify');
-const sink = require('../shared/indexer.e05aedca.cjs');
-require('hookable');
-
-function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
-
-const fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
-
-class CsvSink extends sink.Sink {
-  constructor(_stringifier, _config) {
-    super();
-    this._stringifier = _stringifier;
-    this._config = _config;
-  }
-  async write({ data, endCursor, finality }) {
-    await this.callHook("write", { data });
-    data = this.processCursorColumn(data, endCursor);
-    await this.insertToCSV(data);
-    await this.callHook("flush", { endCursor, finality });
-  }
-  async insertToCSV(data) {
-    if (data.length === 0)
-      return;
-    return await new Promise((resolve, reject) => {
-      for (const row of data) {
-        this._stringifier.write(row, (err) => {
-          if (err)
-            throw new Error(err.message);
-          if (row === data[data.length - 1]) {
-            resolve();
-          }
-        });
-      }
-    });
-  }
-  processCursorColumn(data, endCursor) {
-    const { cursorColumn } = this._config;
-    if (cursorColumn && data.some(
-      (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey)
-    )) {
-      throw new Error(
-        `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`
-      );
-    }
-    if (cursorColumn) {
-      return data;
-    }
-    return data.map((row) => ({
-      ...row,
-      _cursor: Number(endCursor?.orderKey)
-    }));
-  }
-}
-const csv = (args) => {
-  const { csvOptions, filepath, ...sinkOptions } = args;
-  const stringifier = csvStringify.stringify({ ...csvOptions });
-  const writeStream = fs__default.createWriteStream(filepath, { flags: "a" });
-  stringifier.pipe(writeStream);
-  return new CsvSink(stringifier, sinkOptions);
-};
-
-exports.CsvSink = CsvSink;
-exports.csv = csv;

package/dist/sinks/csv.d.cts

@@ -1,34 +0,0 @@
-import { Options, Stringifier } from 'csv-stringify';
-import { c as Sink, b as SinkWriteArgs } from '../shared/indexer.7c118fb5.cjs';
-import '@apibara/protocol';
-import 'hookable';
-
-type CsvArgs = {
-    /**
-     * csv-stringy options
-     * @reference https://csv.js.org/stringify/options/
-     */
-    csvOptions?: Options;
-    /**
-     * filepath for your csv file
-     */
-    filepath: string;
-};
-type CsvSinkOptions = {
-    /**
-     * An optional column name used to store the cursor value. If specified,
-     * the value of this column must match the `endCursor.orderKey` for each row.
-     */
-    cursorColumn?: string;
-};
-declare class CsvSink extends Sink {
-    private _stringifier;
-    private _config;
-    constructor(_stringifier: Stringifier, _config: CsvSinkOptions);
-    write({ data, endCursor, finality }: SinkWriteArgs): Promise<void>;
-    private insertToCSV;
-    private processCursorColumn;
-}
-declare const csv: <TData extends Record<string, unknown>>(args: CsvArgs & CsvSinkOptions) => CsvSink;
-
-export { type CsvArgs, CsvSink, type CsvSinkOptions, csv };

package/dist/sinks/csv.d.mts

@@ -1,34 +0,0 @@
-import { Options, Stringifier } from 'csv-stringify';
-import { c as Sink, b as SinkWriteArgs } from '../shared/indexer.7c118fb5.mjs';
-import '@apibara/protocol';
-import 'hookable';
-
-type CsvArgs = {
-    /**
-     * csv-stringy options
-     * @reference https://csv.js.org/stringify/options/
-     */
-    csvOptions?: Options;
-    /**
-     * filepath for your csv file
-     */
-    filepath: string;
-};
-type CsvSinkOptions = {
-    /**
-     * An optional column name used to store the cursor value. If specified,
-     * the value of this column must match the `endCursor.orderKey` for each row.
-     */
-    cursorColumn?: string;
-};
-declare class CsvSink extends Sink {
-    private _stringifier;
-    private _config;
-    constructor(_stringifier: Stringifier, _config: CsvSinkOptions);
-    write({ data, endCursor, finality }: SinkWriteArgs): Promise<void>;
-    private insertToCSV;
-    private processCursorColumn;
-}
-declare const csv: <TData extends Record<string, unknown>>(args: CsvArgs & CsvSinkOptions) => CsvSink;
-
-export { type CsvArgs, CsvSink, type CsvSinkOptions, csv };

package/dist/sinks/csv.d.ts

@@ -1,34 +0,0 @@
-import { Options, Stringifier } from 'csv-stringify';
-import { c as Sink, b as SinkWriteArgs } from '../shared/indexer.7c118fb5.js';
-import '@apibara/protocol';
-import 'hookable';
-
-type CsvArgs = {
-    /**
-     * csv-stringy options
-     * @reference https://csv.js.org/stringify/options/
-     */
-    csvOptions?: Options;
-    /**
-     * filepath for your csv file
-     */
-    filepath: string;
-};
-type CsvSinkOptions = {
-    /**
-     * An optional column name used to store the cursor value. If specified,
-     * the value of this column must match the `endCursor.orderKey` for each row.
-     */
-    cursorColumn?: string;
-};
-declare class CsvSink extends Sink {
-    private _stringifier;
-    private _config;
-    constructor(_stringifier: Stringifier, _config: CsvSinkOptions);
-    write({ data, endCursor, finality }: SinkWriteArgs): Promise<void>;
-    private insertToCSV;
-    private processCursorColumn;
-}
-declare const csv: <TData extends Record<string, unknown>>(args: CsvArgs & CsvSinkOptions) => CsvSink;
-
-export { type CsvArgs, CsvSink, type CsvSinkOptions, csv };

package/dist/sinks/csv.mjs

@@ -1,59 +0,0 @@
-import fs from 'node:fs';
-import { stringify } from 'csv-stringify';
-import { S as Sink } from '../shared/indexer.371c0482.mjs';
-import 'hookable';
-
-class CsvSink extends Sink {
-  constructor(_stringifier, _config) {
-    super();
-    this._stringifier = _stringifier;
-    this._config = _config;
-  }
-  async write({ data, endCursor, finality }) {
-    await this.callHook("write", { data });
-    data = this.processCursorColumn(data, endCursor);
-    await this.insertToCSV(data);
-    await this.callHook("flush", { endCursor, finality });
-  }
-  async insertToCSV(data) {
-    if (data.length === 0)
-      return;
-    return await new Promise((resolve, reject) => {
-      for (const row of data) {
-        this._stringifier.write(row, (err) => {
-          if (err)
-            throw new Error(err.message);
-          if (row === data[data.length - 1]) {
-            resolve();
-          }
-        });
-      }
-    });
-  }
-  processCursorColumn(data, endCursor) {
-    const { cursorColumn } = this._config;
-    if (cursorColumn && data.some(
-      (row) => Number(row[cursorColumn]) !== Number(endCursor?.orderKey)
-    )) {
-      throw new Error(
-        `Mismatch of ${cursorColumn} and Cursor ${Number(endCursor?.orderKey)}`
-      );
-    }
-    if (cursorColumn) {
-      return data;
-    }
-    return data.map((row) => ({
-      ...row,
-      _cursor: Number(endCursor?.orderKey)
-    }));
-  }
-}
-const csv = (args) => {
-  const { csvOptions, filepath, ...sinkOptions } = args;
-  const stringifier = stringify({ ...csvOptions });
-  const writeStream = fs.createWriteStream(filepath, { flags: "a" });
-  stringifier.pipe(writeStream);
-  return new CsvSink(stringifier, sinkOptions);
-};
-
-export { CsvSink, csv };