@rosen-bridge/abstract-extractor 0.1.0-2afdd7

package/lib/ergo/initializable/AbstractInitializable.ts

@@ -0,0 +1,148 @@
+import { AbstractLogger } from '@rosen-bridge/abstract-logger';
+import JsonBigInt from '@rosen-bridge/json-bigint';
+
+import { ErgoBox, ErgoExtractedData } from '../interfaces';
+import { API_LIMIT, RETRIAL_COUNT } from '../../constants';
+import { AbstractErgoExtractor } from '../AbstractErgoExtractor';
+import { AbstractInitializableErgoExtractorAction } from './AbstractInitializableAction';
+import { BlockInfo } from '../../interfaces';
+
+export abstract class AbstractInitializableErgoExtractor<
+  ExtractedData extends ErgoExtractedData
+> extends AbstractErgoExtractor<ExtractedData> {
+  protected initialize: boolean;
+  protected abstract actions: AbstractInitializableErgoExtractorAction<ExtractedData>;
+
+  /**
+   * create an initializable ergo extractor
+   * @param initialize ignore the initialization step if its false
+   * @param logger
+   */
+  constructor(initialize = true, logger?: AbstractLogger) {
+    super(logger);
+    this.initialize = initialize;
+  }
+
+  /**
+   * return init required boxes with offset limit
+   * @param offset
+   * @param limit
+   * @return boxes in batch
+   */
+  abstract getBoxesWithOffsetLimit: (
+    offset: number,
+    limit: number
+  ) => Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }>;
+
+  /**
+   * return block information of specified tx
+   * @param txId
+   * @return block info
+   */
+  abstract getTxBlock: (txId: string) => Promise<BlockInfo>;
+
+  /**
+   * return all related data below the initial height (including the init height)
+   * @param initialHeight
+   * @return extracted data in batch
+   */
+  fetchDataWithOffsetLimit = async (
+    initialHeight: number,
+    offset: number,
+    limit: number
+  ): Promise<{
+    extractedBoxes: Array<ExtractedData>;
+    hasNextBatch: boolean;
+  }> => {
+    const extractedBoxes: Array<ExtractedData> = [];
+    const apiOutput = await this.getBoxesWithOffsetLimit(offset, limit);
+
+    const filteredBoxes = apiOutput.boxes.filter(
+      (box: ErgoBox) => box.creationHeight <= initialHeight && this.hasData(box)
+    );
+    for (const box of filteredBoxes) {
+      const data = this.extractBoxData(box, box.blockId, box.creationHeight);
+      if (!data) continue;
+      let spendBlock, spendHeight;
+      if (box.spentTransactionId) {
+        const block = await this.getTxBlock(box.spentTransactionId);
+        if (block.height <= initialHeight) {
+          this.logger.debug(
+            `Box with id ${box.boxId} spent at block ${JsonBigInt.stringify(
+              block
+            )} bellow the initial height`
+          );
+          spendBlock = block.hash;
+          spendHeight = block.height;
+        }
+      }
+      const extractedData = {
+        ...data,
+        spendBlock,
+        spendHeight,
+      } as ExtractedData;
+      this.logger.debug(
+        `Extracted data ${JsonBigInt.stringify(extractedData)} from box ${
+          box.boxId
+        } in initialization phase`
+      );
+      extractedBoxes.push(extractedData);
+    }
+    return { extractedBoxes, hasNextBatch: apiOutput.hasNextBatch };
+  };
+
+  /**
+   * initialize extractor database with data created below the initial height
+   * ignore initialization if this feature is off
+   * try to get data multiple times to pass accidental network problems
+   * @param initialBlock
+   */
+  initializeBoxes = async (initialBlock: BlockInfo) => {
+    let trial = 1;
+    if (this.initialize) {
+      this.logger.debug(
+        `Initializing ${this.getId()} started, removing all existing data`
+      );
+      await this.actions.removeAllData(this.getId());
+
+      let hasNextBatch = true;
+      let offset = 0;
+      while (hasNextBatch) {
+        try {
+          const data = await this.fetchDataWithOffsetLimit(
+            initialBlock.height,
+            offset,
+            API_LIMIT
+          );
+          this.logger.info(
+            `Inserting ${
+              data.extractedBoxes.length
+            } new extracted data in ${this.getId()} initialization`
+          );
+          const insertSuccess = await this.actions.insertBoxes(
+            data.extractedBoxes,
+            this.getId()
+          );
+          if (!insertSuccess) throw new Error('Could not store extracted data');
+
+          hasNextBatch = data.hasNextBatch;
+          offset += API_LIMIT;
+        } catch (e) {
+          this.logger.warn(
+            `Initialization for ${this.getId()} failed with error :${e}`
+          );
+          if (trial == RETRIAL_COUNT)
+            throw Error(
+              `Initialization for ${this.getId()} failed after ${RETRIAL_COUNT} retrial`
+            );
+          trial += 1;
+          this.logger.info(
+            `Trying again to initialize ${this.getId()} with trial step ${trial}`
+          );
+        }
+      }
+    } else {
+      this.logger.info(`Initialization for ${this.getId()} is turned off`);
+    }
+  };
+}

package/lib/ergo/initializable/AbstractInitializableAction.ts

@@ -0,0 +1,11 @@
+import { AbstractErgoExtractorAction } from '../AbstractErgoExtractorAction';
+
+export abstract class AbstractInitializableErgoExtractorAction<
+  ExtractedData
+> extends AbstractErgoExtractorAction<ExtractedData> {
+  /**
+   * remove all existing data for the extractor
+   * @param extractorId
+   */
+  abstract removeAllData: (extractorId: string) => Promise<void>;
+}

package/lib/ergo/initializable/InitializableByAddress.ts

@@ -0,0 +1,53 @@
+import { AbstractLogger } from '@rosen-bridge/abstract-logger';
+
+import { ErgoExtractedData, ErgoNetworkType, ErgoBox } from '../interfaces';
+import { AbstractInitializableErgoExtractor } from './AbstractInitializable';
+import { BlockInfo } from '../../interfaces';
+import { ExplorerNetwork } from '../network/ExplorerNetwork';
+import { NodeNetwork } from '../network/NodeNetwork';
+import { AbstractNetwork } from '../network/AbstractNetwork';
+
+export abstract class AbstractInitializableByAddressErgoExtractor<
+  ExtractedData extends ErgoExtractedData
+> extends AbstractInitializableErgoExtractor<ExtractedData> {
+  private address: string;
+  private network: AbstractNetwork;
+
+  constructor(
+    type: ErgoNetworkType,
+    url: string,
+    address: string,
+    initialize?: boolean,
+    logger?: AbstractLogger
+  ) {
+    super(initialize, logger);
+    this.address = address;
+    if (type == ErgoNetworkType.Explorer) {
+      this.network = new ExplorerNetwork(url);
+    } else if (type == ErgoNetworkType.Node) {
+      this.network = new NodeNetwork(url);
+    } else throw new Error(`Network type ${type} is not supported`);
+  }
+
+  /**
+   * return boxes by token id from the specified network source
+   * @param offset
+   * @param limit
+   * @return boxes in batch
+   */
+  getBoxesWithOffsetLimit = (
+    offset: number,
+    limit: number
+  ): Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }> => {
+    return this.network.getBoxesByAddress(this.address, offset, limit);
+  };
+
+  /**
+   * return block information from the specified network source
+   * @param txId
+   * @return block info
+   */
+  getTxBlock = (txId: string): Promise<BlockInfo> => {
+    return this.network.getTxBlock(txId);
+  };
+}

package/lib/ergo/initializable/InitializableByToken.ts

@@ -0,0 +1,53 @@
+import { AbstractLogger } from '@rosen-bridge/abstract-logger';
+
+import { ErgoExtractedData, ErgoNetworkType, ErgoBox } from '../interfaces';
+import { AbstractInitializableErgoExtractor } from './AbstractInitializable';
+import { BlockInfo } from '../../interfaces';
+import { ExplorerNetwork } from '../network/ExplorerNetwork';
+import { NodeNetwork } from '../network/NodeNetwork';
+import { AbstractNetwork } from '../network/AbstractNetwork';
+
+export abstract class AbstractInitializableByTokenErgoExtractor<
+  ExtractedData extends ErgoExtractedData
+> extends AbstractInitializableErgoExtractor<ExtractedData> {
+  private tokenId: string;
+  private network: AbstractNetwork;
+
+  constructor(
+    type: ErgoNetworkType,
+    url: string,
+    tokenId: string,
+    initialize?: boolean,
+    logger?: AbstractLogger
+  ) {
+    super(initialize, logger);
+    this.tokenId = tokenId;
+    if (type == ErgoNetworkType.Explorer) {
+      this.network = new ExplorerNetwork(url);
+    } else if (type == ErgoNetworkType.Node) {
+      this.network = new NodeNetwork(url);
+    } else throw new Error(`Network type ${type} is not supported`);
+  }
+
+  /**
+   * return boxes by token id from the specified network source
+   * @param offset
+   * @param limit
+   * @return boxes in batch
+   */
+  getBoxesWithOffsetLimit = (
+    offset: number,
+    limit: number
+  ): Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }> => {
+    return this.network.getBoxesByTokenId(this.tokenId, offset, limit);
+  };
+
+  /**
+   * return block information from the specified network source
+   * @param txId
+   * @return block info
+   */
+  getTxBlock = (txId: string): Promise<BlockInfo> => {
+    return this.network.getTxBlock(txId);
+  };
+}

package/lib/ergo/initializable/index.ts

@@ -0,0 +1,4 @@
+export * from './AbstractInitializable';
+export * from './AbstractInitializableAction';
+export * from './InitializableByAddress';
+export * from './InitializableByToken';

package/lib/ergo/interfaces.ts

@@ -0,0 +1,64 @@
+export enum ErgoNetworkType {
+  Explorer = 'explorer',
+  Node = 'node',
+}
+
+export type InputBox = {
+  boxId: string;
+};
+
+export type DataInput = {
+  boxId: string;
+};
+
+export type Asset = {
+  tokenId: string;
+  amount: bigint;
+};
+
+export type AdditionalRegisters = {
+  R4?: string;
+  R5?: string;
+  R6?: string;
+  R7?: string;
+  R8?: string;
+  R9?: string;
+};
+
+export type OutputBox = {
+  boxId: string;
+  value: bigint;
+  ergoTree: string;
+  creationHeight: number;
+  assets?: Array<Asset>;
+  additionalRegisters?: AdditionalRegisters;
+  transactionId: string;
+  index: bigint | number;
+};
+
+export interface ErgoBox extends OutputBox {
+  blockId: string;
+  spentTransactionId?: string;
+}
+
+export type Transaction = {
+  id: string;
+  inputs: Array<InputBox>;
+  dataInputs: Array<DataInput>;
+  outputs: Array<OutputBox>;
+  size?: bigint;
+};
+
+export interface SpendInfo {
+  boxId: string;
+  txId: string;
+  index: number;
+}
+
+export interface ErgoExtractedData {
+  boxId: string;
+  height: number;
+  blockId: string;
+  spendBlock?: string;
+  spendHeight?: number;
+}

package/lib/ergo/network/AbstractNetwork.ts

@@ -0,0 +1,35 @@
+import { BlockInfo } from '../../interfaces';
+import { ErgoBox } from '../interfaces';
+export abstract class AbstractNetwork {
+  /**
+   * return block information of specified tx
+   * @param txId
+   */
+  abstract getTxBlock: (txId: string) => Promise<BlockInfo>;
+
+  /**
+   * return related boxes by specified address with limit offset
+   * @param address
+   * @param offset
+   * @param limit
+   * @returns related boxes
+   */
+  abstract getBoxesByAddress: (
+    address: string,
+    offset: number,
+    limit: number
+  ) => Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }>;
+
+  /**
+   * return related boxes by specified token with limit offset
+   * @param tokenId
+   * @param offset
+   * @param limit
+   * @returns related boxes
+   */
+  abstract getBoxesByTokenId: (
+    tokenId: string,
+    offset: number,
+    limit: number
+  ) => Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }>;
+}

package/lib/ergo/network/ExplorerNetwork.ts

@@ -0,0 +1,68 @@
+import ergoExplorerClientFactory from '@rosen-clients/ergo-explorer';
+
+import { BlockInfo } from '../../interfaces';
+import { ErgoBox } from '../interfaces';
+import { AbstractNetwork } from './AbstractNetwork';
+
+export class ExplorerNetwork extends AbstractNetwork {
+  private api;
+
+  constructor(url: string) {
+    super();
+    this.api = ergoExplorerClientFactory(url);
+  }
+
+  /**
+   * return block information of specified tx
+   * @param txId
+   */
+  getTxBlock = async (txId: string): Promise<BlockInfo> => {
+    const tx = await this.api.v1.getApiV1TransactionsP1(txId);
+    return {
+      hash: tx.blockId,
+      height: tx.inclusionHeight,
+    };
+  };
+
+  /**
+   * use explorer api to return related boxes by specified address
+   * @param address
+   * @param offset
+   * @param limit
+   * @returns related boxes
+   */
+  getBoxesByAddress = async (
+    address: string,
+    offset: number,
+    limit: number
+  ): Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }> => {
+    const boxes = await this.api.v1.getApiV1BoxesByaddressP1(address, {
+      offset: offset,
+      limit: limit,
+    });
+    if (!boxes.items)
+      throw new Error('Explorer BoxesByAddress api expected to have items');
+    return { boxes: boxes.items, hasNextBatch: boxes.total > offset + limit };
+  };
+
+  /**
+   * use explorer api to return related boxes by specified token id
+   * @param tokenId
+   * @param offset
+   * @param limit
+   * @returns related boxes
+   */
+  getBoxesByTokenId = async (
+    tokenId: string,
+    offset: number,
+    limit: number
+  ): Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }> => {
+    const boxes = await this.api.v1.getApiV1BoxesBytokenidP1(tokenId, {
+      offset: offset,
+      limit: limit,
+    });
+    if (!boxes.items)
+      throw new Error('Explorer BoxesByTokeId api expected to have items');
+    return { boxes: boxes.items, hasNextBatch: boxes.total > offset + limit };
+  };
+}

package/lib/ergo/network/NodeNetwork.ts

@@ -0,0 +1,93 @@
+import ergoNodeClientFactory, {
+  IndexedErgoBox,
+} from '@rosen-clients/ergo-node';
+
+import { BlockInfo } from '../../interfaces';
+import { ErgoBox } from '../interfaces';
+import { AbstractNetwork } from './AbstractNetwork';
+
+export class NodeNetwork extends AbstractNetwork {
+  private api;
+
+  constructor(url: string) {
+    super();
+    this.api = ergoNodeClientFactory(url);
+  }
+
+  /**
+   * covert node api boxes to ErgoBox interface
+   * @param box
+   * @returns ErgoBox
+   */
+  convertToErgoBox = async (box: IndexedErgoBox): Promise<ErgoBox> => ({
+    transactionId: box.transactionId || '',
+    index: box.index || 0,
+    value: box.value || 0n,
+    ergoTree: box.ergoTree || '',
+    creationHeight: box.creationHeight || 0,
+    assets: box.assets || [],
+    additionalRegisters: box.additionalRegisters,
+    boxId: box.boxId || '',
+    blockId: (await this.getTxBlock(box.transactionId!)).hash,
+  });
+
+  /**
+   * return block information of specified tx
+   * @param txId
+   */
+  getTxBlock = async (txId: string): Promise<BlockInfo> => {
+    const tx = await this.api.getTxById(txId);
+    return {
+      hash: tx.blockId,
+      height: tx.inclusionHeight,
+    };
+  };
+
+  /**
+   * use node api to return related boxes by specified address
+   * @param address
+   * @param offset
+   * @param limit
+   * @returns related boxes
+   */
+  getBoxesByAddress = async (
+    address: string,
+    offset: number,
+    limit: number
+  ): Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }> => {
+    const boxes = await this.api.getBoxesByAddress(address, {
+      offset: offset,
+      limit: limit,
+    });
+    if (!boxes.items)
+      throw new Error('Ergo node BoxesByAddress api expected to have items');
+    const ergoBoxes = await Promise.all(
+      boxes.items.map(async (box) => await this.convertToErgoBox(box))
+    );
+    return { boxes: ergoBoxes, hasNextBatch: boxes.items.length > 0 };
+  };
+
+  /**
+   * use node api to return related boxes by specified token id
+   * @param tokenId
+   * @param offset
+   * @param limit
+   * @returns related boxes
+   */
+  getBoxesByTokenId = async (
+    tokenId: string,
+    offset: number,
+    limit: number
+  ): Promise<{ boxes: ErgoBox[]; hasNextBatch: boolean }> => {
+    const boxes = await this.api.getBoxesByTokenId(tokenId, {
+      offset: offset,
+      limit: limit,
+    });
+    if (!boxes.items)
+      throw new Error('Ergo node BoxesByTokenId api expected to have items');
+    const ergoBoxes = await Promise.all(
+      boxes.items.map(async (box) => await this.convertToErgoBox(box))
+    );
+    return { boxes: ergoBoxes, hasNextBatch: boxes.items.length > 0 };
+  };
+}

package/lib/ergo/utils.ts

@@ -0,0 +1,15 @@
+import { intersection } from 'lodash-es';
+import { OutputBox } from './interfaces';
+
+/**
+ * Check box to have specified tokens
+ * @param box
+ * @return true if box has the required token and false otherwise
+ */
+export const boxHasToken = (box: OutputBox, tokenIds: string[]) => {
+  if (!box.assets) return false;
+  const boxTokens = box.assets.map((token) => token.tokenId);
+  const requiredTokens = intersection(tokenIds, boxTokens);
+  if (requiredTokens.length == tokenIds.length) return true;
+  return false;
+};

package/lib/index.ts

@@ -0,0 +1,3 @@
+export * from './interfaces';
+export * from './ergo';
+export * from './AbstractExtractor';

package/lib/interfaces.ts

@@ -0,0 +1,12 @@
+export interface Block {
+  parentHash: string;
+  hash: string;
+  height: number;
+  timestamp: number;
+  extra?: string;
+}
+
+export interface BlockInfo {
+  height: number;
+  hash: string;
+}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@rosen-bridge/abstract-extractor",
+  "version": "0.1.0-2afdd7",
+  "description": "Rosen Bridge extractor interfaces to work with scanner",
+  "repository": "",
+  "license": "GPL-3.0",
+  "author": "Rosen Team",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc --build tsconfig.build.json",
+    "coverage": "npm run test -- --coverage",
+    "lint": "eslint --fix . && npm run prettify",
+    "prettify": "prettier --write . --ignore-path ./.gitignore",
+    "release": "npm run test -- --run && npm run build && npm publish --access public",
+    "test": "NODE_OPTIONS=--loader=extensionless vitest",
+    "type-check": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "@types/lodash-es": "^4.17.12",
+    "@types/node": "^20.11.9",
+    "@typescript-eslint/eslint-plugin": "^6.19.1",
+    "@typescript-eslint/parser": "^6.19.1",
+    "@vitest/coverage-istanbul": "^1.2.2",
+    "eslint": "^8.56.0",
+    "eslint-config-prettier": "^9.1.0",
+    "extensionless": "^1.9.6",
+    "prettier": "^3.2.4",
+    "typescript": "^5.3.3",
+    "vitest": "^1.2.2"
+  },
+  "engines": {
+    "node": ">=20.11.0"
+  },
+  "dependencies": {
+    "@rosen-bridge/abstract-logger": "^1.0.0",
+    "@rosen-clients/ergo-explorer": "^1.1.1",
+    "@rosen-clients/ergo-node": "^1.1.1",
+    "lodash-es": "^4.17.21"
+  }
+}

package/tests/AbstractExtractor.mock.ts

@@ -0,0 +1,26 @@
+import { V1 } from '@rosen-clients/ergo-explorer';
+import {
+  AbstractErgoExtractor,
+  BlockInfo,
+  OutputBox,
+  ErgoExtractedData,
+  AbstractErgoExtractorAction,
+} from '../lib';
+
+export class MockedErgoExtractor extends AbstractErgoExtractor<ErgoExtractedData> {
+  actions: AbstractErgoExtractorAction<ErgoExtractedData>;
+
+  getId = () => 'Test';
+
+  initializeBoxes: (initialBlock: BlockInfo) => Promise<void>;
+
+  hasData = (box: V1.OutputInfo | OutputBox) => false;
+
+  extractBoxData = (
+    box: V1.OutputInfo | OutputBox,
+    blockId: string,
+    height: number
+  ): Omit<ErgoExtractedData, 'spendBlock' | 'spendHeight'> | undefined => {
+    return undefined;
+  };
+}