@rosen-bridge/abstract-extractor 2.0.3 → 2.1.1

package/package.json

@@ -1,48 +1,50 @@
 {
   "name": "@rosen-bridge/abstract-extractor",
-  "version": "2.0.3",
+  "version": "2.1.1",
   "description": "Rosen Bridge extractor interfaces to work with scanner",
-  "repository": "",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/rosen-bridge/scanner.git"
+  },
   "license": "MIT",
   "author": "Rosen Team",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "CHANGELOG.md"
+  ],
   "scripts": {
     "build": "tsc --build tsconfig.build.json",
     "coverage": "npm run test -- --coverage",
     "lint": "eslint --fix . && npm run prettify",
     "prettify": "prettier --write . --ignore-path ./.gitignore",
     "release": "npm run test -- --run && npm run build && npm publish --access public",
-    "test": "NODE_OPTIONS=--loader=extensionless vitest",
+    "test": "NODE_OPTIONS='--import tsx' vitest",
     "type-check": "tsc --noEmit"
   },
   "devDependencies": {
     "@types/lodash-es": "^4.17.12",
-    "@types/node": "^20.11.9",
+    "@types/node": "^22.18.0",
     "@types/uuid": "^10.0.0",
-    "@typescript-eslint/eslint-plugin": "^6.19.1",
-    "@typescript-eslint/parser": "^6.19.1",
-    "@vitest/coverage-istanbul": "^1.2.2",
+    "@vitest/coverage-istanbul": "^3.1.4",
     "await-semaphore": "^0.1.3",
-    "eslint": "^8.56.0",
-    "eslint-config-prettier": "^9.1.0",
-    "extensionless": "^1.9.6",
-    "prettier": "^3.2.4",
+    "tsx": "^4.19.4",
     "typescript": "^5.3.3",
-    "vitest": "^1.2.2"
-  },
-  "engines": {
-    "node": ">=20.11.0"
+    "vitest": "^3.1.4"
   },
   "dependencies": {
-    "@rosen-bridge/abstract-logger": "^2.0.1",
-    "@rosen-bridge/json-bigint": "^0.1.0",
-    "@rosen-bridge/scanner-interfaces": "^0.1.1",
-    "@rosen-clients/ergo-explorer": "^1.1.5",
-    "@rosen-clients/ergo-node": "^1.2.3",
+    "@rosen-bridge/abstract-logger": "^3.0.0",
+    "@rosen-bridge/extended-typeorm": "^1.0.0",
+    "@rosen-bridge/json-bigint": "^1.0.0",
+    "@rosen-bridge/scanner-interfaces": "^0.2.0",
+    "@rosen-clients/ergo-explorer": "^2.0.0",
+    "@rosen-clients/ergo-node": "^3.0.0",
     "lodash-es": "^4.17.21",
-    "typeorm": "^0.3.20",
     "uuid": "^9.0.0"
+  },
+  "engines": {
+    "node": ">=22.18.0"
   }
 }

package/.eslintignore

@@ -1 +0,0 @@
-dist

package/lib/abstractExtractor.ts

@@ -1,31 +0,0 @@
-import { BlockInfo, Block } from '@rosen-bridge/scanner-interfaces';
-
-export abstract class AbstractExtractor<TransactionType> {
-  /**
-   * process a list of transactions in a block and store required information
-   * @param txs list of transactions in the block
-   * @param block
-   * @return true if the process is completed successfully and false otherwise
-   */
-  abstract processTransactions: (
-    txs: Array<TransactionType>,
-    block: Block
-  ) => Promise<boolean>;
-
-  /**
-   * return extractor id. This id must be unique over all extractors.
-   */
-  abstract getId: () => string;
-
-  /**
-   * fork one block and remove all stored information for this block
-   * @param hash block hash
-   */
-  abstract forkBlock: (hash: string) => Promise<void>;
-
-  /**
-   * initialize extractor database with data created below the initial height
-   * @param initialBlock
-   */
-  abstract initializeBoxes: (initialBlock: BlockInfo) => Promise<void>;
-}

package/lib/constants.ts

@@ -1,3 +0,0 @@
-export const API_LIMIT = 100;
-export const RETRIAL_COUNT = 10;
-export const DB_CHUNK_SIZE = 100;

package/lib/ergo/AbstractErgoExtractor.ts

@@ -1,227 +0,0 @@
-import { AbstractLogger, DummyLogger } from '@rosen-bridge/abstract-logger';
-import JsonBigInt from '@rosen-bridge/json-bigint';
-import { Mutex } from 'await-semaphore';
-import { v4 as uuidv4 } from 'uuid';
-import {
-  Transaction,
-  OutputBox,
-  InputExtension,
-  BlockInfo,
-} from '@rosen-bridge/scanner-interfaces';
-
-import { AbstractExtractor } from '../abstractExtractor';
-import { AbstractErgoExtractorAction } from './AbstractErgoExtractorAction';
-import {
-  AbstractBoxData,
-  SpendInfo,
-  CallbackType,
-  CallbackMap,
-  CallbackDataMap,
-  TxExtra,
-} from './interfaces';
-import { AbstractErgoExtractorEntity } from './AbstractErgoExtractorEntity';
-
-export abstract class AbstractErgoExtractor<
-  ExtractedData extends AbstractBoxData,
-  ExtractorEntity extends AbstractErgoExtractorEntity
-> extends AbstractExtractor<Transaction> {
-  protected abstract actions: AbstractErgoExtractorAction<
-    ExtractedData,
-    ExtractorEntity
-  >;
-  protected logger: AbstractLogger;
-  protected callbacks: {
-    [K in CallbackType]: Map<string, CallbackMap<ExtractedData>[K]>;
-  } = {
-    [CallbackType.Update]: new Map(),
-    [CallbackType.Insert]: new Map(),
-    [CallbackType.Delete]: new Map(),
-    [CallbackType.Spend]: new Map(),
-  };
-  private callbackMutex = new Mutex();
-
-  constructor(logger = new DummyLogger()) {
-    super();
-    this.logger = logger;
-  }
-
-  /**
-   * hook a new callback on a callback type
-   * @param type
-   * @param id
-   * @param callback
-   * @returns callback registered id
-   */
-  hook = async <T extends CallbackType>(
-    type: T,
-    callback: CallbackMap<ExtractedData>[T]
-  ): Promise<string> => {
-    const release = await this.callbackMutex.acquire();
-    const callbackMap = this.callbacks[type];
-    const id = uuidv4();
-    callbackMap.set(id, callback);
-    release();
-    return id;
-  };
-
-  /**
-   * unhook a callback on a type
-   * returns false if there is no registered callback with the id
-   * @param type
-   * @param id
-   * @returns success status
-   */
-  unhook = async (type: CallbackType, id: string): Promise<boolean> => {
-    const release = await this.callbackMutex.acquire();
-    const callbackMap = this.callbacks[type];
-    if (!callbackMap.has(id)) {
-      this.logger.warn(
-        `Callback with Id [${id}] is not registered for type [${type}].`
-      );
-      return false;
-    }
-    callbackMap.delete(id);
-    release();
-    return true;
-  };
-
-  /**
-   * trigger all callbacks registered on a specific type with the provided data
-   * @param type
-   * @param data
-   */
-  protected triggerCallbacks<T extends CallbackType>(
-    type: T,
-    data: CallbackDataMap<ExtractedData>[T]
-  ): void {
-    const callbackMap = this.callbacks[type];
-    callbackMap.forEach((callback) => {
-      callback(data);
-    });
-  }
-
-  /**
-   * extract box data to proper format (not including spending information)
-   * @param box
-   * @param inputExtensions all input box extensions in transaction
-   * @return extracted data in proper format
-   */
-  abstract extractBoxData: (
-    box: OutputBox,
-    inputExtensions: InputExtension[],
-    txExtra?: TxExtra
-  ) => ExtractedData | undefined;
-
-  /**
-   * check proper data format in the box
-   * @param box
-   * @return true if the box has the required data and false otherwise
-   */
-  abstract hasData: (box: OutputBox) => boolean;
-
-  /**
-   * create spend info array for the transaction
-   * @param tx
-   * @returns spend info array of the transaction
-   */
-  getTransactionSpendInfo = (tx: Transaction) => {
-    let boxIndex = 1;
-    const spendInfoArray = [];
-    for (const input of tx.inputs) {
-      spendInfoArray.push({ txId: tx.id, boxId: input.boxId, index: boxIndex });
-      boxIndex += 1;
-    }
-    return spendInfoArray;
-  };
-
-  /**
-   * extract transaction extra information
-   * override this function if there is extra needed information
-   * @param tx
-   * @returns
-   */
-  getTransactionExtraData = (
-    tx: Transaction // eslint-disable-line @typescript-eslint/no-unused-vars
-  ): TxExtra => {
-    return {};
-  };
-
-  /**
-   * process a list of transactions in a block and store required information
-   * @param txs list of transactions in the block
-   * @param block
-   * @return true if the process is completed successfully and false otherwise
-   */
-  processTransactions = async (
-    txs: Transaction[],
-    block: BlockInfo
-  ): Promise<boolean> => {
-    try {
-      const boxes: Array<ExtractedData> = [];
-      const spentInfos: Array<SpendInfo> = [];
-      for (const tx of txs) {
-        const inputExtensions = tx.inputs.map((input) => input.extension || {});
-        for (const output of tx.outputs) {
-          if (!this.hasData(output)) {
-            continue;
-          }
-          this.logger.debug(`Trying to extract data from box ${output.boxId}`);
-          const extractedData = this.extractBoxData(
-            output,
-            inputExtensions,
-            this.getTransactionExtraData(tx)
-          );
-          if (extractedData) {
-            this.logger.debug(
-              `Extracted data ${JsonBigInt.stringify(extractedData)} from box ${
-                output.boxId
-              }`
-            );
-            boxes.push(extractedData);
-          }
-        }
-        spentInfos.push(...this.getTransactionSpendInfo(tx));
-      }
-
-      if (boxes.length > 0) {
-        if (!(await this.actions.storeBoxes(boxes, block, this.getId()))) {
-          this.logger.warn(
-            `Data insertion failed for ${this.getId()} at the block ${
-              block.height
-            }`
-          );
-          return false;
-        }
-        this.triggerCallbacks(CallbackType.Insert, boxes);
-      }
-      const spentData = await this.actions.spendBoxes(
-        spentInfos,
-        block,
-        this.getId()
-      );
-      if (spentData.length > 0) {
-        this.triggerCallbacks(CallbackType.Spend, spentData);
-      }
-    } catch (e) {
-      this.logger.error(
-        `Processing transactions failed for ${this.getId()} at the block ${
-          block.height
-        } with error: ${e}`
-      );
-      return false;
-    }
-    return true;
-  };
-
-  /**
-   * fork one block and remove all stored information for this block
-   * @param hash block hash
-   */
-  forkBlock = async (hash: string): Promise<void> => {
-    const result = await this.actions.deleteBlockBoxes(hash, this.getId());
-    if (result.deletedData.length > 0)
-      this.triggerCallbacks(CallbackType.Delete, result.deletedData);
-    if (result.updatedData.length > 0)
-      this.triggerCallbacks(CallbackType.Update, result.updatedData);
-  };
-}

package/lib/ergo/AbstractErgoExtractorAction.ts

@@ -1,324 +0,0 @@
-import {
-  DataSource,
-  In,
-  Repository,
-  Not,
-  EntityTarget,
-  FindOptionsWhere,
-  QueryRunner,
-} from 'typeorm';
-import { chunk, difference, pick } from 'lodash-es';
-import { AbstractLogger, DummyLogger } from '@rosen-bridge/abstract-logger';
-import JsonBigInt from '@rosen-bridge/json-bigint';
-import { BlockInfo } from '@rosen-bridge/scanner-interfaces';
-
-import { AbstractBoxData, BoxInfo, SpendInfo } from './interfaces';
-import { DB_CHUNK_SIZE } from '../constants';
-import { AbstractErgoExtractorEntity } from './AbstractErgoExtractorEntity';
-import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
-
-export abstract class AbstractErgoExtractorAction<
-  ExtractedData extends AbstractBoxData,
-  ExtractorEntity extends AbstractErgoExtractorEntity
-> {
-  private readonly datasource: DataSource;
-  readonly logger: AbstractLogger;
-  protected readonly repository: Repository<ExtractorEntity>;
-  private repo: EntityTarget<ExtractorEntity>;
-
-  constructor(
-    dataSource: DataSource,
-    repo: EntityTarget<ExtractorEntity>,
-    logger?: AbstractLogger
-  ) {
-    this.datasource = dataSource;
-    this.logger = logger ? logger : new DummyLogger();
-    this.repository = this.datasource.getRepository(repo);
-    this.repo = repo;
-  }
-
-  /**
-   * create the database entity from extracted data and block information
-   */
-  protected createEntity = (
-    data: ExtractedData[], // eslint-disable-line @typescript-eslint/no-unused-vars
-    block: BlockInfo, // eslint-disable-line @typescript-eslint/no-unused-vars
-    extractor: string // eslint-disable-line @typescript-eslint/no-unused-vars
-  ): Array<Omit<ExtractorEntity, 'id'>> => {
-    throw Error(
-      'You must implement `createEntity` or override `insertEntities` and `updateEntities`'
-    );
-  };
-
-  /**
-   * convert the database entity back to raw data
-   */
-  protected convertEntityToData = (
-    entities: ExtractorEntity[] // eslint-disable-line @typescript-eslint/no-unused-vars
-  ): ExtractedData[] => {
-    throw Error(
-      'You must implement `convertEntityToData` or override `deleteBlockEntities`'
-    );
-  };
-
-  /**
-   * insert entities extracted from a block to database
-   * @param queryRunner
-   * @param boxesToInsert
-   * @param block
-   * @param extractor
-   */
-  protected insertEntities = async (
-    queryRunner: QueryRunner,
-    boxesToInsert: Array<ExtractedData>,
-    block: BlockInfo,
-    extractor: string
-  ) => {
-    const repository = queryRunner.manager.getRepository(this.repo);
-    await repository.insert(
-      this.createEntity(
-        boxesToInsert,
-        block,
-        extractor
-      ) as unknown as QueryDeepPartialEntity<ExtractorEntity>[]
-    );
-  };
-
-  /**
-   * update entities related to a box
-   * @param queryRunner
-   * @param updateBox
-   * @param block
-   * @param extractor
-   */
-  protected updateEntity = async (
-    queryRunner: QueryRunner,
-    updateBox: ExtractedData, // eslint-disable-line @typescript-eslint/no-unused-vars
-    block: BlockInfo, // eslint-disable-line @typescript-eslint/no-unused-vars
-    extractor: string
-  ) => {
-    const repository = queryRunner.manager.getRepository(this.repo);
-    const box = this.createEntity([updateBox], block, extractor)[0];
-    await repository.update(
-      {
-        boxId: box.boxId,
-        extractor: extractor,
-      } as FindOptionsWhere<ExtractorEntity>,
-      box as unknown as QueryDeepPartialEntity<ExtractorEntity>
-    );
-  };
-
-  /**
-   * delete all data extracted from a block
-   * @param queryRunner
-   * @param extractor
-   * @param block
-   * @returns
-   */
-  protected deleteBlockEntities = async (
-    queryRunner: QueryRunner,
-    extractor: string,
-    block: string
-  ): Promise<ExtractedData[]> => {
-    const repository = queryRunner.manager.getRepository(this.repo);
-    const deletedData = await repository.find({
-      where: {
-        extractor: extractor,
-        block: block,
-      } as FindOptionsWhere<ExtractorEntity>,
-    });
-    await repository.delete({
-      extractor: extractor,
-      block: block,
-    } as unknown as FindOptionsWhere<ExtractorEntity>);
-    return this.convertEntityToData(deletedData);
-  };
-
-  /**
-   * delete all data extracted from a block
-   * @param queryRunner
-   * @param extractor
-   * @param block
-   * @returns
-   */
-  protected updateBlockEntities = async (
-    queryRunner: QueryRunner, // eslint-disable-line @typescript-eslint/no-unused-vars
-    extractor: string,
-    block: string
-  ): Promise<ExtractorEntity[]> => {
-    const repository = this.datasource.getRepository(
-      this.repo as EntityTarget<new () => ExtractorEntity>
-    );
-    const updatedData = await repository.find({
-      where: {
-        extractor: extractor,
-        spendBlock: block,
-        block: Not(block),
-      } as unknown as FindOptionsWhere<ExtractorEntity>,
-    });
-    await repository.update(
-      {
-        spendBlock: block,
-        extractor: extractor,
-      } as unknown as FindOptionsWhere<ExtractorEntity>,
-      {
-        spendBlock: null,
-        spendHeight: 0,
-      }
-    );
-    return updatedData as unknown as ExtractorEntity[];
-  };
-
-  /**
-   * insert all extracted box data in an atomic transaction
-   * update the data if a box with the same id is already stored in db
-   * @param boxes
-   * @param block
-   * @param extractor
-   * @return inserted items and updated box ids
-   * returns undefined in case of any problem
-   */
-  storeBoxes = async (
-    boxes: Array<ExtractedData>,
-    block: BlockInfo,
-    extractor: string
-  ): Promise<boolean> => {
-    let boxesToInsert: ExtractedData[] = [],
-      boxesToUpdate: ExtractedData[] = [];
-    const queryRunner = this.datasource.createQueryRunner();
-    await queryRunner.connect();
-    await queryRunner.startTransaction();
-    try {
-      const repository = queryRunner.manager.getRepository(this.repo);
-      const dbBoxIds = (
-        await repository.findBy({
-          boxId: In(boxes.map((item) => item.boxId)),
-          extractor: extractor,
-        } as FindOptionsWhere<ExtractorEntity>)
-      ).map((box) => box.boxId);
-      if (dbBoxIds.length > 0)
-        this.logger.debug(`Found stored boxes with same boxId`, dbBoxIds);
-
-      boxesToUpdate = boxes.filter((box) => dbBoxIds.includes(box.boxId));
-      boxesToInsert = difference(boxes, boxesToUpdate);
-
-      if (boxesToInsert.length > 0) {
-        this.logger.debug(`Inserting boxes`);
-        await this.insertEntities(queryRunner, boxesToInsert, block, extractor);
-      }
-      if (boxesToUpdate.length > 0)
-        this.logger.info(
-          `Updating boxes with following Ids in the database: [${boxesToUpdate
-            .map((col) => col.boxId)
-            .join(', ')}]`
-        );
-      for (const box of boxesToUpdate) {
-        this.logger.debug(
-          `Updating boxes in database [${JsonBigInt.stringify(box)}]`
-        );
-        await this.updateEntity(queryRunner, box, block, extractor);
-      }
-      await queryRunner.commitTransaction();
-      return true;
-    } catch (e) {
-      this.logger.error(`An error occurred during store boxes action: ${e}`);
-      await queryRunner.rollbackTransaction();
-      return false;
-    } finally {
-      await queryRunner.release();
-    }
-  };
-
-  /**
-   * update spending information of stored boxes
-   * chunk spendInfos to prevent large database queries
-   * Note: It only updates the spendHeight and spendBlock fields. If updating
-   * anything else is required, override this implementation to include the
-   * additional fields.
-   * @param spendInfos
-   * @param block
-   * @param extractor
-   * @returns spent box ids
-   */
-  spendBoxes = async (
-    spendInfos: Array<SpendInfo>,
-    block: BlockInfo,
-    extractor: string
-  ): Promise<BoxInfo[]> => {
-    const spentData = [];
-    const spendInfoChunks = chunk(spendInfos, DB_CHUNK_SIZE);
-    for (const spendInfoChunk of spendInfoChunks) {
-      const boxIds = spendInfoChunk.map((info) => info.boxId);
-      const updateResult = await this.repository.update(
-        {
-          boxId: In(boxIds),
-          extractor: extractor,
-        } as FindOptionsWhere<ExtractorEntity>,
-        {
-          spendBlock: block.hash,
-          spendHeight: block.height,
-        } as unknown as QueryDeepPartialEntity<ExtractorEntity>
-      );
-
-      if (updateResult.affected && updateResult.affected > 0) {
-        const spentRows = await this.repository.findBy({
-          boxId: In(boxIds),
-          spendBlock: block.hash,
-        } as FindOptionsWhere<ExtractorEntity>);
-        spentData.push(...spentRows);
-        for (const row of spentRows) {
-          this.logger.debug(
-            `Spent box with boxId [${row.boxId}] at height ${block.height}`
-          );
-        }
-      }
-    }
-    return spentData.map((data) => pick(data, 'boxId'));
-  };
-
-  /**
-   * delete extracted data from a specific block
-   * if a box is spend in this block mark it as unspent
-   * if a box is created in this block remove it from database
-   * @param block
-   * @param extractor
-   * @return deleted items and updated box ids
-   */
-  deleteBlockBoxes = async (
-    block: string,
-    extractor: string
-  ): Promise<{ deletedData: ExtractedData[]; updatedData: BoxInfo[] }> => {
-    const queryRunner = this.datasource.createQueryRunner();
-    await queryRunner.connect();
-    await queryRunner.startTransaction();
-    try {
-      this.logger.info(
-        `Deleting boxes in block ${block} and extractor ${extractor}`
-      );
-      const updatedData = await this.updateBlockEntities(
-        queryRunner,
-        extractor,
-        block
-      );
-      const deletedData = await this.deleteBlockEntities(
-        queryRunner,
-        extractor,
-        block
-      );
-      await queryRunner.commitTransaction();
-      return {
-        deletedData,
-        updatedData: updatedData.map((data) => pick(data, 'boxId')),
-      };
-    } catch (error) {
-      await queryRunner.rollbackTransaction();
-      this.logger.error(
-        `An error occurred while deleting data extracted from block ${block}`,
-        error
-      );
-      throw error;
-    } finally {
-      await queryRunner.release();
-    }
-  };
-}

package/lib/ergo/AbstractErgoExtractorEntity.ts

@@ -1,28 +0,0 @@
-import { Column, PrimaryGeneratedColumn, Unique } from 'typeorm';
-
-@Unique(['boxId', 'extractor'])
-export abstract class AbstractErgoExtractorEntity {
-  @PrimaryGeneratedColumn()
-  id: number;
-
-  @Column({ type: 'varchar' })
-  boxId: string;
-
-  @Column({ type: 'varchar' })
-  block: string;
-
-  @Column({ type: 'int' })
-  height: number;
-
-  @Column({ nullable: true, type: 'varchar' })
-  spendBlock?: string | null;
-
-  @Column({ nullable: true, type: 'int' })
-  spendHeight?: number | null;
-
-  @Column({ type: 'varchar' })
-  extractor: string;
-
-  @Column({ type: 'varchar' })
-  serialized: string;
-}

package/lib/ergo/index.ts

@@ -1,9 +0,0 @@
-export * from './AbstractErgoExtractor';
-export * from './AbstractErgoExtractorAction';
-export * from './interfaces';
-export * from './network/ExplorerNetwork';
-export * from './network/NodeNetwork';
-export * from './network/AbstractNetwork';
-export * from './initializable';
-export * from './utils';
-export * from './AbstractErgoExtractorEntity';