@rosen-bridge/abstract-extractor 2.1.0 → 2.1.1

package/lib/ergo/initializable/AbstractInitializable.ts

@@ -1,228 +0,0 @@
-import { AbstractLogger } from '@rosen-bridge/abstract-logger';
-import { groupBy, sortBy } from 'lodash-es';
-import { BlockInfo, ErgoNetworkType } from '@rosen-bridge/scanner-interfaces';
-
-import { AbstractBoxData, ExtendedTransaction } from '../interfaces';
-import { API_LIMIT, RETRIAL_COUNT } from '../../constants';
-import { AbstractErgoExtractor } from '../AbstractErgoExtractor';
-import { AbstractInitializableErgoExtractorAction } from './AbstractInitializableAction';
-import { ExplorerNetwork } from '../network/ExplorerNetwork';
-import { NodeNetwork } from '../network/NodeNetwork';
-import { AbstractErgoExtractorEntity } from '../AbstractErgoExtractorEntity';
-
-export abstract class AbstractInitializableErgoExtractor<
-  ExtractedData extends AbstractBoxData,
-  ExtractorEntity extends AbstractErgoExtractorEntity,
-> extends AbstractErgoExtractor<ExtractedData, ExtractorEntity> {
-  protected initialize: boolean;
-  private address: string;
-  protected abstract actions: AbstractInitializableErgoExtractorAction<
-    ExtractedData,
-    ExtractorEntity
-  >;
-
-  private network: ExplorerNetwork | NodeNetwork;
-
-  constructor(
-    type: ErgoNetworkType,
-    url: string,
-    address: string,
-    logger?: AbstractLogger,
-    initialize = true,
-  ) {
-    super(logger);
-    this.address = address;
-    this.initialize = initialize;
-    if (type == ErgoNetworkType.Explorer) {
-      this.network = new ExplorerNetwork(url);
-      this.initializeBoxes = (initialBlock: BlockInfo) => {
-        return this.initializeWithExplorer(initialBlock);
-      };
-    } else if (type == ErgoNetworkType.Node) {
-      this.network = new NodeNetwork(url);
-      this.initializeBoxes = (initialBlock: BlockInfo) => {
-        return this.initializeWithNode(initialBlock);
-      };
-    } else throw new Error(`Network type ${type} is not supported`);
-  }
-
-  /**
-   * Initialize extractor using Explorer network
-   * @param initialBlock
-   */
-  private initializeWithExplorer = async (initialBlock: BlockInfo) => {
-    const explorerNetwork = this.network as ExplorerNetwork;
-    let fromHeight = 0,
-      toHeight = initialBlock.height;
-    await this.initWithRetrial(async () => {
-      while (fromHeight < toHeight) {
-        let txs: Array<ExtendedTransaction>;
-        // eslint-disable-next-line no-constant-condition
-        while (true) {
-          txs = await explorerNetwork.getAddressTransactionsWithHeight(
-            this.address,
-            fromHeight,
-            toHeight,
-          );
-          this.logger.debug(
-            `Found ${txs.length} transactions for the address from height ${fromHeight} to height ${toHeight}`,
-          );
-          if (txs.length < API_LIMIT || fromHeight === toHeight) {
-            break; // Exit loop if we have fewer transactions than the limit or if the range is reduced to a single height
-          }
-          toHeight = Math.floor((toHeight - fromHeight) / 2) + fromHeight;
-          this.logger.debug(
-            `Limiting the query height range to [${fromHeight}, ${toHeight}]`,
-          );
-        }
-        if (txs.length < API_LIMIT) {
-          if (txs.length > 0) await this.processTransactionBatch(txs);
-        } else {
-          this.logger.debug(
-            `Block at height ${fromHeight} has more than (or equal) ${API_LIMIT} relevant txs, processing all txs in the block`,
-          );
-          const blockId = await explorerNetwork.getBlockIdAtHeight(fromHeight);
-          const blockTxs = await explorerNetwork.getBlockTxs(blockId);
-          this.logger.debug(
-            `Found ${blockTxs.length} transactions at height ${fromHeight}`,
-          );
-          await this.processTransactions(blockTxs, {
-            hash: blockId,
-            height: fromHeight,
-          });
-        }
-        fromHeight = toHeight + 1;
-        toHeight = initialBlock.height;
-      }
-    });
-  };
-
-  /**
-   * Get the total tx count from Node network
-   * @returns total tx count of the address
-   */
-  private getTotalTxCount = async () => {
-    const response = await (
-      this.network as NodeNetwork
-    ).getAddressTransactionsWithOffsetLimit(this.address, 0, 0);
-    return response.total;
-  };
-
-  /**
-   * Initialize extractor using Node network
-   * @param initialBlock
-   */
-  private initializeWithNode = async (initialBlock: BlockInfo) => {
-    const txCountBeforeInit = await this.getTotalTxCount();
-    let offset = 0,
-      total = 1,
-      round = 1;
-    await this.initWithRetrial(async () => {
-      // Repeat the whole process twice to cover all spent boxes
-      // After round 1 all boxes have been saved and processed once
-      // After round 2 spending information of all stored boxes are updated successfully
-      while (round <= 2) {
-        this.logger.debug(`Starting round ${round} of initialization`);
-        while (offset < total) {
-          const response = await (
-            this.network as NodeNetwork
-          ).getAddressTransactionsWithOffsetLimit(
-            this.address,
-            offset,
-            API_LIMIT,
-          );
-          total = response.total;
-          const txs = response.items.filter(
-            (tx) => tx.inclusionHeight <= initialBlock.height,
-          );
-          this.logger.debug(
-            `Found ${txs.length} transactions below the initial height with offset ${offset} and total number of transactions ${total}`,
-          );
-          if (txs.length > 0) await this.processTransactionBatch(txs);
-          offset += API_LIMIT;
-        }
-        round++;
-        offset = 0; // next round initial offset
-      }
-    });
-    const txCountAfterInit = await this.getTotalTxCount();
-    if (txCountAfterInit != txCountBeforeInit) {
-      throw Error(
-        'Total transaction count changed during initialization phase, the stored data is not valid',
-      );
-    }
-  };
-
-  /**
-   * Process a batch of transactions
-   * group txs into blocks and process them using `processTransactions`
-   * @param txs
-   */
-  private processTransactionBatch = async (txs: Array<ExtendedTransaction>) => {
-    txs = sortBy(txs, (tx) => tx.inclusionHeight);
-    const groupedTxs = groupBy(txs, (tx) => tx.blockId);
-    this.logger.debug(
-      `The transaction batch grouped to ${
-        Object.keys(groupedTxs).length
-      } blocks`,
-    );
-    for (const blockId in groupedTxs) {
-      const blockTxs = groupedTxs[blockId];
-      const block = { hash: blockId, height: blockTxs[0].inclusionHeight };
-      this.logger.debug(
-        `Processing transactions at height ${blockTxs[0].inclusionHeight}`,
-      );
-      const success = await this.processTransactions(blockTxs, block);
-      if (!success)
-        throw Error(
-          `Processing transactions failed at height ${blockTxs[0].inclusionHeight}`,
-        );
-    }
-  };
-
-  /**
-   * Initialize the extractor with retrial on any unexpected problem
-   * its the common part of initialize with Node and Explorer network
-   * @param job
-   */
-  private initWithRetrial = async (job: () => Promise<void>) => {
-    let trial = 1;
-    if (this.initialize) {
-      this.logger.debug(
-        `Initializing ${this.getId()} started, removing all existing data`,
-      );
-      await this.actions.removeAllData(this.getId());
-      while (trial <= RETRIAL_COUNT) {
-        try {
-          await job();
-          break;
-        } catch (e) {
-          this.logger.warn(
-            `Initialization for ${this.getId()} failed with error :${e}`,
-          );
-          if (trial == RETRIAL_COUNT)
-            throw Error(
-              `Initialization for ${this.getId()} failed after ${RETRIAL_COUNT} retrial`,
-            );
-          trial += 1;
-          this.logger.info(
-            `Trying again to initialize ${this.getId()} with trial step ${trial}`,
-          );
-        }
-      }
-      this.logger.info(
-        `Initialization completed successfully for ${this.getId()}`,
-      );
-    } else {
-      this.logger.info(`Initialization for ${this.getId()} is turned off`);
-    }
-  };
-
-  /**
-   * initialize extractor database with data created below the initial height
-   * ignore initialization if this feature is off
-   * try to get data multiple times to pass accidental network problems
-   * @param initialBlock
-   */
-  initializeBoxes: (initialBlock: BlockInfo) => Promise<void>;
-}

package/lib/ergo/initializable/AbstractInitializableAction.ts

@@ -1,33 +0,0 @@
-import {
-  DataSource,
-  EntityTarget,
-  FindOptionsWhere,
-} from '@rosen-bridge/extended-typeorm';
-import { AbstractLogger } from '@rosen-bridge/abstract-logger';
-
-import { AbstractErgoExtractorAction } from '../AbstractErgoExtractorAction';
-import { AbstractBoxData } from '../interfaces';
-import { AbstractErgoExtractorEntity } from '../AbstractErgoExtractorEntity';
-
-export abstract class AbstractInitializableErgoExtractorAction<
-  ExtractedData extends AbstractBoxData,
-  ExtractorEntity extends AbstractErgoExtractorEntity,
-> extends AbstractErgoExtractorAction<ExtractedData, ExtractorEntity> {
-  constructor(
-    dataSource: DataSource,
-    repo: EntityTarget<ExtractorEntity>,
-    logger?: AbstractLogger,
-  ) {
-    super(dataSource, repo, logger);
-  }
-
-  /**
-   * remove all existing data for the extractor
-   * @param extractorId
-   */
-  removeAllData = async (extractorId: string) => {
-    await this.repository.delete({
-      extractor: extractorId,
-    } as FindOptionsWhere<ExtractorEntity>);
-  };
-}

package/lib/ergo/initializable/index.ts

@@ -1,2 +0,0 @@
-export * from './AbstractInitializable';
-export * from './AbstractInitializableAction';

package/lib/ergo/interfaces.ts

@@ -1,51 +0,0 @@
-import { OutputBox, Transaction } from '@rosen-bridge/scanner-interfaces';
-
-export interface ErgoBox extends OutputBox {
-  blockId: string;
-  inclusionHeight: number;
-  spentBlockId?: string;
-  spentHeight?: number;
-  spentTransactionId?: string;
-  spentIndex?: number;
-}
-
-export interface ExtendedTransaction extends Transaction {
-  inclusionHeight: number;
-  blockId: string;
-}
-
-export interface SpendInfo {
-  boxId: string;
-  txId: string;
-  index: number;
-  extras?: { [key: string]: string };
-}
-
-export interface AbstractBoxData {
-  boxId: string;
-  serialized: string;
-}
-
-export enum CallbackType {
-  Insert = 'insert',
-  Update = 'update',
-  Spend = 'spend',
-  Delete = 'delete',
-}
-
-export interface BoxInfo {
-  boxId: string;
-}
-
-export type CallbackDataMap<ExtractedData extends AbstractBoxData> = {
-  [CallbackType.Update]: BoxInfo[];
-  [CallbackType.Insert]: ExtractedData[];
-  [CallbackType.Delete]: ExtractedData[];
-  [CallbackType.Spend]: BoxInfo[];
-};
-
-export type CallbackMap<ExtractedData extends AbstractBoxData> = {
-  [K in CallbackType]: (data: CallbackDataMap<ExtractedData>[K]) => void;
-};
-
-export type TxExtra = { [key: string]: string };

package/lib/ergo/network/AbstractNetwork.ts

@@ -1,29 +0,0 @@
-import { ErgoBox } from '../interfaces';
-
-export abstract class AbstractNetwork {
-  /**
-   * return related boxes by specified address with limit offset
-   * @param address
-   * @param offset
-   * @param limit
-   * @returns related boxes
-   */
-  abstract getBoxesByAddress: (
-    address: string,
-    offset: number,
-    limit: number,
-  ) => Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }>;
-
-  /**
-   * return related boxes by specified token with limit offset
-   * @param tokenId
-   * @param offset
-   * @param limit
-   * @returns related boxes
-   */
-  abstract getBoxesByTokenId: (
-    tokenId: string,
-    offset: number,
-    limit: number,
-  ) => Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }>;
-}

package/lib/ergo/network/ExplorerNetwork.ts

@@ -1,237 +0,0 @@
-import ergoExplorerClientFactory from '@rosen-clients/ergo-explorer';
-import { AbstractNetwork } from './AbstractNetwork';
-import { V1 } from '@rosen-clients/ergo-explorer';
-import { BlockInfo, Transaction } from '@rosen-bridge/scanner-interfaces';
-
-import { ErgoBox, ExtendedTransaction } from '../interfaces';
-import { mapValues, pick } from 'lodash-es';
-import { API_LIMIT } from '../../constants';
-
-export class ExplorerNetwork extends AbstractNetwork {
-  private api;
-
-  constructor(url: string) {
-    super();
-    this.api = ergoExplorerClientFactory(url);
-  }
-
-  /**
-   * return spending information of a specified box by having spendTxId
-   * @param boxId
-   * @param spendTxId
-   */
-  getSpendingInfo = async (
-    boxId: string,
-    spendTxId: string,
-  ): Promise<BlockInfo & { spendIndex: number }> => {
-    const tx = await this.api.v1.getApiV1TransactionsP1(spendTxId);
-    const spendIndex = tx.inputs?.findIndex((box) => box.boxId === boxId);
-    if (spendIndex == undefined)
-      throw Error(
-        `Impossible behavior, the box [${boxId}] should have been spent in tx [${spendTxId}]`,
-      );
-    return {
-      hash: tx.blockId,
-      height: tx.inclusionHeight,
-      spendIndex,
-    };
-  };
-
-  /**
-   * convert explorer api boxes to ErgoBox interface
-   * @param box
-   * @returns ErgoBox
-   */
-  private convertBox = async (box: V1.OutputInfo): Promise<ErgoBox> => {
-    const spendInfo = box.spentTransactionId
-      ? await this.getSpendingInfo(box.boxId, box.spentTransactionId)
-      : undefined;
-    return {
-      blockId: box.blockId,
-      boxId: box.boxId,
-      creationHeight: box.creationHeight,
-      inclusionHeight: box.settlementHeight,
-      ergoTree: box.ergoTree,
-      index: box.index,
-      transactionId: box.transactionId,
-      value: box.value,
-      additionalRegisters: mapValues(
-        box.additionalRegisters,
-        'serializedValue',
-      ),
-      assets:
-        box.assets?.map((asset) => pick(asset, ['tokenId', 'amount'])) ?? [],
-      spentHeight: spendInfo?.height,
-      spentBlockId: spendInfo?.hash,
-      spentTransactionId: box.spentTransactionId,
-      spentIndex: spendInfo?.spendIndex,
-    };
-  };
-
-  /**
-   * convert explorer transaction to extractor transaction type
-   * @param tx
-   */
-  private convertTransaction = (
-    tx: V1.TransactionInfo,
-  ): ExtendedTransaction => {
-    return {
-      id: tx.id,
-      inclusionHeight: tx.inclusionHeight,
-      blockId: tx.blockId,
-      dataInputs:
-        tx.dataInputs?.map((dataInput) => ({
-          boxId: dataInput.boxId,
-        })) ?? [],
-      // TODO: Add input extension to explorer local/ergo/rosen-bridge/scanner/-/issues/156
-      inputs: tx.inputs?.map((input) => ({ boxId: input.boxId })) ?? [],
-      outputs:
-        tx.outputs?.map((output) => ({
-          boxId: output.boxId,
-          transactionId: output.transactionId,
-          additionalRegisters: mapValues(
-            output.additionalRegisters,
-            'serializedValue',
-          ),
-          assets:
-            output.assets?.map((asset) => pick(asset, ['tokenId', 'amount'])) ??
-            [],
-          ergoTree: output.ergoTree,
-          creationHeight: output.creationHeight,
-          index: output.index,
-          value: output.value,
-        })) ?? [],
-    };
-  };
-
-  /**
-   * convert explorer block transaction to transaction type
-   * @param tx
-   */
-  private convertBlockTransaction = (tx: V1.TransactionInfo1): Transaction => {
-    return {
-      id: tx.id,
-      dataInputs:
-        tx.dataInputs?.map((dataInput) => ({
-          boxId: dataInput.id,
-        })) ?? [],
-      // TODO: Add input extension local/ergo/rosen-bridge/scanner/-/issues/156
-      inputs: tx.inputs?.map((input) => ({ boxId: input.id })) ?? [],
-      outputs:
-        tx.outputs?.map((output) => ({
-          boxId: output.id,
-          transactionId: output.txId,
-          additionalRegisters: output.additionalRegisters,
-          assets:
-            output.assets?.map((asset) => pick(asset, ['tokenId', 'amount'])) ??
-            [],
-          ergoTree: output.ergoTree,
-          creationHeight: output.creationHeight,
-          index: output.index,
-          value: output.value,
-        })) ?? [],
-    };
-  };
-
-  /**
-   * use explorer api to return related transactions of the specified address in the height range
-   * @param tokenId
-   * @param offset
-   * @param limit
-   * @returns related transactions
-   */
-  getAddressTransactionsWithHeight = async (
-    address: string,
-    fromHeight: number,
-    toHeight: number,
-  ): Promise<Array<ExtendedTransaction>> => {
-    const txs = await this.api.v1.getApiV1AddressesP1Transactions(address, {
-      fromHeight,
-      toHeight,
-      limit: API_LIMIT,
-    });
-    if (!txs.items)
-      throw new Error(
-        'Explorer AddressTransactions api expected to have items',
-      );
-    return txs.items.map((tx) => this.convertTransaction(tx));
-  };
-
-  /**
-   * use explorer api to get the block id at the specified height
-   * @param height
-   * @returns block id
-   */
-  getBlockIdAtHeight = async (height: number): Promise<string> => {
-    const id = await this.api.v0.getApiV0BlocksAtP1(height);
-    return id[0];
-  };
-
-  /**
-   * use explorer api to return all transactions in a block
-   * @param blockId
-   * @returns converted transactions
-   */
-  getBlockTxs = async (blockId: string): Promise<Array<Transaction>> => {
-    const block = await this.api.v1.getApiV1BlocksP1(blockId);
-    if (!block.block.blockTransactions) {
-      throw new Error(
-        `Expected explorer block api to include block transactions for block ${blockId}`,
-      );
-    }
-    return block.block.blockTransactions.map((tx) =>
-      this.convertBlockTransaction(tx),
-    );
-  };
-
-  /**
-   * use explorer api to return related boxes by specified address
-   * @param address
-   * @param offset
-   * @param limit
-   * @returns related boxes
-   */
-  getBoxesByAddress = async (
-    address: string,
-    offset: number,
-    limit: number,
-  ): Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }> => {
-    const boxes = await this.api.v1.getApiV1BoxesUnspentByaddressP1(address, {
-      offset: offset,
-      limit: limit,
-      sortDirection: 'desc',
-    });
-    if (!boxes.items)
-      throw new Error('Explorer BoxesByAddress api expected to have items');
-    const resultBoxes: Array<ErgoBox> = [];
-    for (const box of boxes.items) {
-      resultBoxes.push(await this.convertBox(box));
-    }
-    return { boxes: resultBoxes, hasNextBatch: boxes.total > offset + limit };
-  };
-
-  /**
-   * use explorer api to return related boxes by specified token id
-   * @param tokenId
-   * @param offset
-   * @param limit
-   * @returns related boxes
-   */
-  getBoxesByTokenId = async (
-    tokenId: string,
-    offset: number,
-    limit: number,
-  ): Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }> => {
-    const boxes = await this.api.v1.getApiV1BoxesBytokenidP1(tokenId, {
-      offset: offset,
-      limit: limit,
-    });
-    if (!boxes.items)
-      throw new Error('Explorer BoxesByTokeId api expected to have items');
-    const resultBoxes: Array<ErgoBox> = [];
-    for (const box of boxes.items) {
-      resultBoxes.push(await this.convertBox(box));
-    }
-    return { boxes: resultBoxes, hasNextBatch: boxes.total > offset + limit };
-  };
-}