@rosen-bridge/abstract-extractor 1.0.1 → 2.0.0-181e9e8f

package/lib/{AbstractExtractor.ts → abstractExtractor.ts}

@@ -1,4 +1,4 @@
-import { BlockInfo, Block } from './interfaces';
+import { BlockInfo, Block } from '@rosen-bridge/scanner-interfaces';
 
 export abstract class AbstractExtractor<TransactionType> {
   /**

package/lib/ergo/AbstractErgoExtractor.ts

@@ -2,18 +2,22 @@ import { AbstractLogger, DummyLogger } from '@rosen-bridge/abstract-logger';
 import JsonBigInt from '@rosen-bridge/json-bigint';
 import { Mutex } from 'await-semaphore';
 import { v4 as uuidv4 } from 'uuid';
-
-import { AbstractExtractor } from '../AbstractExtractor';
-import { AbstractErgoExtractorAction } from './AbstractErgoExtractorAction';
-import { BlockInfo } from '../interfaces';
 import {
   Transaction,
   OutputBox,
+  InputExtension,
+  BlockInfo,
+} from '@rosen-bridge/scanner-interfaces';
+
+import { AbstractExtractor } from '../abstractExtractor';
+import { AbstractErgoExtractorAction } from './AbstractErgoExtractorAction';
+import {
   AbstractBoxData,
   SpendInfo,
   CallbackType,
   CallbackMap,
   CallbackDataMap,
+  TxExtra,
 } from './interfaces';
 import { AbstractErgoExtractorEntity } from './AbstractErgoExtractorEntity';
 
@@ -99,9 +103,14 @@ export abstract class AbstractErgoExtractor<
   /**
    * extract box data to proper format (not including spending information)
    * @param box
+   * @param inputExtensions all input box extensions in transaction
    * @return extracted data in proper format
    */
-  abstract extractBoxData: (box: OutputBox) => ExtractedData | undefined;
+  abstract extractBoxData: (
+    box: OutputBox,
+    inputExtensions: InputExtension[],
+    txExtra?: TxExtra
+  ) => ExtractedData | undefined;
 
   /**
    * check proper data format in the box
@@ -110,6 +119,31 @@ export abstract class AbstractErgoExtractor<
    */
   abstract hasData: (box: OutputBox) => boolean;
 
+  /**
+   * create spend info array for the transaction
+   * @param tx
+   * @returns spend info array of the transaction
+   */
+  getTransactionSpendInfo = (tx: Transaction) => {
+    let boxIndex = 1;
+    const spendInfoArray = [];
+    for (const input of tx.inputs) {
+      spendInfoArray.push({ txId: tx.id, boxId: input.boxId, index: boxIndex });
+      boxIndex += 1;
+    }
+    return spendInfoArray;
+  };
+
+  /**
+   * extract transaction extra information
+   * override this function if there is extra needed information
+   * @param tx
+   * @returns
+   */
+  getTransactionExtraData = (tx: Transaction): TxExtra => {
+    return {};
+  };
+
   /**
    * process a list of transactions in a block and store required information
    * @param txs list of transactions in the block
@@ -124,12 +158,17 @@ export abstract class AbstractErgoExtractor<
       const boxes: Array<ExtractedData> = [];
       const spentInfos: Array<SpendInfo> = [];
       for (const tx of txs) {
+        const inputExtensions = tx.inputs.map((input) => input.extension || {});
         for (const output of tx.outputs) {
           if (!this.hasData(output)) {
             continue;
           }
           this.logger.debug(`Trying to extract data from box ${output.boxId}`);
-          const extractedData = this.extractBoxData(output);
+          const extractedData = this.extractBoxData(
+            output,
+            inputExtensions,
+            this.getTransactionExtraData(tx)
+          );
           if (extractedData) {
             this.logger.debug(
               `Extracted data ${JsonBigInt.stringify(extractedData)} from box ${
@@ -139,11 +178,7 @@ export abstract class AbstractErgoExtractor<
             boxes.push(extractedData);
           }
         }
-        let boxIndex = 1;
-        for (const input of tx.inputs) {
-          spentInfos.push({ txId: tx.id, boxId: input.boxId, index: boxIndex });
-          boxIndex += 1;
-        }
+        spentInfos.push(...this.getTransactionSpendInfo(tx));
       }
 
       if (boxes.length > 0) {

package/lib/ergo/AbstractErgoExtractorAction.ts

@@ -5,12 +5,13 @@ import {
   Not,
   EntityTarget,
   FindOptionsWhere,
+  QueryRunner,
 } from 'typeorm';
 import { chunk, difference, pick } from 'lodash-es';
 import { AbstractLogger, DummyLogger } from '@rosen-bridge/abstract-logger';
 import JsonBigInt from '@rosen-bridge/json-bigint';
+import { BlockInfo } from '@rosen-bridge/scanner-interfaces';
 
-import { BlockInfo } from '../interfaces';
 import { AbstractBoxData, BoxInfo, SpendInfo } from './interfaces';
 import { DB_CHUNK_SIZE } from '../constants';
 import { AbstractErgoExtractorEntity } from './AbstractErgoExtractorEntity';
@@ -38,18 +39,122 @@ export abstract class AbstractErgoExtractorAction<
   /**
    * create the database entity from extracted data and block information
    */
-  protected abstract createEntity: (
+  protected createEntity = (
     data: ExtractedData[],
     block: BlockInfo,
     extractor: string
-  ) => Array<Omit<ExtractorEntity, 'id'>>;
+  ): Array<Omit<ExtractorEntity, 'id'>> => {
+    throw Error(
+      'You must implement `createEntity` or override `insertEntities` and `updateEntities`'
+    );
+  };
 
   /**
    * convert the database entity back to raw data
    */
-  protected abstract convertEntityToData: (
+  protected convertEntityToData = (
     entities: ExtractorEntity[]
-  ) => ExtractedData[];
+  ): ExtractedData[] => {
+    throw Error(
+      'You must implement `convertEntityToData` or override `deleteBlockEntities`'
+    );
+  };
+
+  /**
+   * insert entities extracted from a block to database
+   * @param queryRunner
+   * @param boxesToInsert
+   * @param block
+   * @param extractor
+   */
+  protected insertEntities = async (
+    queryRunner: QueryRunner,
+    boxesToInsert: Array<ExtractedData>,
+    block: BlockInfo,
+    extractor: string
+  ) => {
+    const repository = queryRunner.manager.getRepository(this.repo);
+    await repository.insert(
+      this.createEntity(boxesToInsert, block, extractor) as any
+    );
+  };
+
+  /**
+   * update entities related to a box
+   * @param queryRunner
+   * @param updateBox
+   * @param block
+   * @param extractor
+   */
+  protected updateEntity = async (
+    queryRunner: QueryRunner,
+    updateBox: ExtractedData,
+    block: BlockInfo,
+    extractor: string
+  ) => {
+    const repository = queryRunner.manager.getRepository(this.repo);
+    const box = this.createEntity([updateBox], block, extractor)[0];
+    await repository.update(
+      {
+        boxId: box.boxId,
+        extractor: extractor,
+      } as FindOptionsWhere<ExtractorEntity>,
+      box as any
+    );
+  };
+
+  /**
+   * delete all data extracted from a block
+   * @param queryRunner
+   * @param extractor
+   * @param block
+   * @returns
+   */
+  protected deleteBlockEntities = async (
+    queryRunner: QueryRunner,
+    extractor: string,
+    block: string
+  ): Promise<ExtractedData[]> => {
+    const repository = queryRunner.manager.getRepository(this.repo);
+    const deletedData = await repository.find({
+      where: { extractor: extractor, block: block } as any,
+    });
+    await repository.delete({
+      extractor: extractor,
+      block: block,
+    } as any);
+    return this.convertEntityToData(deletedData);
+  };
+
+  /**
+   * delete all data extracted from a block
+   * @param queryRunner
+   * @param extractor
+   * @param block
+   * @returns
+   */
+  protected updateBlockEntities = async (
+    queryRunner: QueryRunner,
+    extractor: string,
+    block: string
+  ): Promise<ExtractorEntity[]> => {
+    const repository = queryRunner.manager.getRepository(this.repo);
+    const updatedData = await repository.find({
+      where: {
+        extractor: extractor,
+        spendBlock: block,
+        block: Not(block),
+      } as any,
+    });
+    await repository.update(
+      {
+        spendBlock: block,
+        extractor: extractor,
+      } as FindOptionsWhere<ExtractorEntity>,
+      { spendBlock: null, spendHeight: 0 } as any
+    );
+    return updatedData;
+  };
 
   /**
    * insert all extracted box data in an atomic transaction
@@ -65,7 +170,6 @@ export abstract class AbstractErgoExtractorAction<
     block: BlockInfo,
     extractor: string
   ): Promise<boolean> => {
-    let success = true;
     let boxesToInsert: ExtractedData[] = [],
       boxesToUpdate: ExtractedData[] = [];
     const queryRunner = this.datasource.createQueryRunner();
@@ -87,9 +191,7 @@ export abstract class AbstractErgoExtractorAction<
 
       if (boxesToInsert.length > 0) {
         this.logger.debug(`Inserting boxes`);
-        await repository.insert(
-          this.createEntity(boxesToInsert, block, extractor) as any
-        );
+        await this.insertEntities(queryRunner, boxesToInsert, block, extractor);
       }
       if (boxesToUpdate.length > 0)
         this.logger.info(
@@ -97,27 +199,21 @@ export abstract class AbstractErgoExtractorAction<
             .map((col) => col.boxId)
             .join(', ')}]`
         );
-      for (const box of this.createEntity(boxesToUpdate, block, extractor)) {
+      for (const box of boxesToUpdate) {
         this.logger.debug(
           `Updating boxes in database [${JsonBigInt.stringify(box)}]`
         );
-        await repository.update(
-          {
-            boxId: box.boxId,
-            extractor: extractor,
-          } as FindOptionsWhere<ExtractorEntity>,
-          box as any
-        );
+        await this.updateEntity(queryRunner, box, block, extractor);
       }
       await queryRunner.commitTransaction();
+      return true;
     } catch (e) {
       this.logger.error(`An error occurred during store boxes action: ${e}`);
       await queryRunner.rollbackTransaction();
-      success = false;
+      return false;
     } finally {
       await queryRunner.release();
     }
-    return success;
   };
 
   /**
@@ -176,33 +272,37 @@ export abstract class AbstractErgoExtractorAction<
     block: string,
     extractor: string
   ): Promise<{ deletedData: ExtractedData[]; updatedData: BoxInfo[] }> => {
-    this.logger.info(
-      `Deleting boxes in block ${block} and extractor ${extractor}`
-    );
-    const deletedData = await this.repository.find({
-      where: { extractor: extractor, block: block } as any,
-    });
-    const updatedData = await this.repository.find({
-      where: {
-        extractor: extractor,
-        spendBlock: block,
-        block: Not(block),
-      } as any,
-    });
-    await this.repository.delete({
-      extractor: extractor,
-      block: block,
-    } as any);
-    await this.repository.update(
-      {
-        spendBlock: block,
-        extractor: extractor,
-      } as FindOptionsWhere<ExtractorEntity>,
-      { spendBlock: null, spendHeight: 0 } as any
-    );
-    return {
-      deletedData: this.convertEntityToData(deletedData),
-      updatedData: updatedData.map((data) => pick(data, 'boxId')),
-    };
+    const queryRunner = this.datasource.createQueryRunner();
+    await queryRunner.connect();
+    await queryRunner.startTransaction();
+    try {
+      this.logger.info(
+        `Deleting boxes in block ${block} and extractor ${extractor}`
+      );
+      const updatedData = await this.updateBlockEntities(
+        queryRunner,
+        extractor,
+        block
+      );
+      const deletedData = await this.deleteBlockEntities(
+        queryRunner,
+        extractor,
+        block
+      );
+      await queryRunner.commitTransaction();
+      return {
+        deletedData,
+        updatedData: updatedData.map((data) => pick(data, 'boxId')),
+      };
+    } catch (error) {
+      await queryRunner.rollbackTransaction();
+      this.logger.error(
+        `An error occurred while deleting data extracted from block ${block}`,
+        error
+      );
+      throw error;
+    } finally {
+      await queryRunner.release();
+    }
   };
 }

package/lib/ergo/initializable/AbstractInitializable.ts

@@ -1,15 +1,11 @@
 import { AbstractLogger } from '@rosen-bridge/abstract-logger';
 import { groupBy, sortBy } from 'lodash-es';
+import { BlockInfo, ErgoNetworkType } from '@rosen-bridge/scanner-interfaces';
 
-import {
-  AbstractBoxData,
-  ErgoNetworkType,
-  ExtendedTransaction,
-} from '../interfaces';
+import { AbstractBoxData, ExtendedTransaction } from '../interfaces';
 import { API_LIMIT, RETRIAL_COUNT } from '../../constants';
 import { AbstractErgoExtractor } from '../AbstractErgoExtractor';
 import { AbstractInitializableErgoExtractorAction } from './AbstractInitializableAction';
-import { BlockInfo } from '../../interfaces';
 import { ExplorerNetwork } from '../network/ExplorerNetwork';
 import { NodeNetwork } from '../network/NodeNetwork';
 import { AbstractErgoExtractorEntity } from '../AbstractErgoExtractorEntity';

package/lib/ergo/interfaces.ts

@@ -1,40 +1,4 @@
-export enum ErgoNetworkType {
-  Explorer = 'explorer',
-  Node = 'node',
-}
-
-export type InputBox = {
-  boxId: string;
-};
-
-export type DataInput = {
-  boxId: string;
-};
-
-export type Asset = {
-  tokenId: string;
-  amount: bigint;
-};
-
-export type AdditionalRegisters = {
-  R4?: string;
-  R5?: string;
-  R6?: string;
-  R7?: string;
-  R8?: string;
-  R9?: string;
-};
-
-export type OutputBox = {
-  boxId: string;
-  value: bigint;
-  ergoTree: string;
-  creationHeight: number;
-  assets?: Array<Asset>;
-  additionalRegisters?: AdditionalRegisters;
-  transactionId: string;
-  index: bigint | number;
-};
+import { OutputBox, Transaction } from '@rosen-bridge/scanner-interfaces';
 
 export interface ErgoBox extends OutputBox {
   blockId: string;
@@ -45,14 +9,6 @@ export interface ErgoBox extends OutputBox {
   spentIndex?: number;
 }
 
-export type Transaction = {
-  id: string;
-  inputs: Array<InputBox>;
-  dataInputs: Array<DataInput>;
-  outputs: Array<OutputBox>;
-  size?: bigint;
-};
-
 export interface ExtendedTransaction extends Transaction {
   inclusionHeight: number;
   blockId: string;
@@ -62,7 +18,7 @@ export interface SpendInfo {
   boxId: string;
   txId: string;
   index: number;
-  extras?: string[];
+  extras?: { [key: string]: string };
 }
 
 export interface AbstractBoxData {
@@ -91,3 +47,5 @@ export type CallbackDataMap<ExtractedData extends AbstractBoxData> = {
 export type CallbackMap<ExtractedData extends AbstractBoxData> = {
   [K in CallbackType]: (data: CallbackDataMap<ExtractedData>[K]) => void;
 };
+
+export type TxExtra = { [key: string]: string };

package/lib/ergo/network/ExplorerNetwork.ts

@@ -1,9 +1,9 @@
 import ergoExplorerClientFactory from '@rosen-clients/ergo-explorer';
 import { AbstractNetwork } from './AbstractNetwork';
 import { V1 } from '@rosen-clients/ergo-explorer';
+import { BlockInfo, Transaction } from '@rosen-bridge/scanner-interfaces';
 
-import { BlockInfo } from '../../interfaces';
-import { ErgoBox, ExtendedTransaction, Transaction } from '../interfaces';
+import { ErgoBox, ExtendedTransaction } from '../interfaces';
 import { mapValues, pick } from 'lodash-es';
 import { API_LIMIT } from '../../constants';
 
@@ -59,7 +59,8 @@ export class ExplorerNetwork extends AbstractNetwork {
         box.additionalRegisters,
         'serializedValue'
       ),
-      assets: box.assets?.map((asset) => pick(asset, ['tokenId', 'amount'])),
+      assets:
+        box.assets?.map((asset) => pick(asset, ['tokenId', 'amount'])) ?? [],
       spentHeight: spendInfo?.height,
       spentBlockId: spendInfo?.hash,
       spentTransactionId: box.spentTransactionId,
@@ -82,6 +83,7 @@ export class ExplorerNetwork extends AbstractNetwork {
         tx.dataInputs?.map((dataInput) => ({
           boxId: dataInput.boxId,
         })) ?? [],
+      // TODO: Add input extension to explorer local/ergo/rosen-bridge/scanner/-/issues/156
       inputs: tx.inputs?.map((input) => ({ boxId: input.boxId })) ?? [],
       outputs:
         tx.outputs?.map((output) => ({
@@ -91,9 +93,9 @@ export class ExplorerNetwork extends AbstractNetwork {
             output.additionalRegisters,
             'serializedValue'
           ),
-          assets: output.assets?.map((asset) =>
-            pick(asset, ['tokenId', 'amount'])
-          ),
+          assets:
+            output.assets?.map((asset) => pick(asset, ['tokenId', 'amount'])) ??
+            [],
           ergoTree: output.ergoTree,
           creationHeight: output.creationHeight,
           index: output.index,
@@ -113,15 +115,16 @@ export class ExplorerNetwork extends AbstractNetwork {
         tx.dataInputs?.map((dataInput) => ({
           boxId: dataInput.id,
         })) ?? [],
+      // TODO: Add input extension local/ergo/rosen-bridge/scanner/-/issues/156
       inputs: tx.inputs?.map((input) => ({ boxId: input.id })) ?? [],
       outputs:
         tx.outputs?.map((output) => ({
           boxId: output.id,
           transactionId: output.txId,
           additionalRegisters: output.additionalRegisters,
-          assets: output.assets?.map((asset) =>
-            pick(asset, ['tokenId', 'amount'])
-          ),
+          assets:
+            output.assets?.map((asset) => pick(asset, ['tokenId', 'amount'])) ??
+            [],
           ergoTree: output.ergoTree,
           creationHeight: output.creationHeight,
           index: output.index,

package/lib/ergo/network/NodeNetwork.ts

@@ -2,8 +2,8 @@ import ergoNodeClientFactory, {
   IndexedErgoBox,
   IndexedErgoTransaction,
 } from '@rosen-clients/ergo-node';
+import { BlockInfo } from '@rosen-bridge/scanner-interfaces';
 
-import { BlockInfo } from '../../interfaces';
 import { ErgoBox, ExtendedTransaction } from '../interfaces';
 import { AbstractNetwork } from './AbstractNetwork';
 
@@ -64,7 +64,8 @@ export class NodeNetwork extends AbstractNetwork {
         additionalRegisters: output.additionalRegisters,
         boxId: output.boxId || '',
       })),
-      inputs: tx.inputs.map((input) => ({ boxId: input.boxId })) ?? [],
+      // TODO: Add input extension local/ergo/rosen-bridge/scanner/-/issues/156
+      inputs: tx.inputs.map((input) => ({ boxId: input.boxId! })) ?? [],
       dataInputs: tx.dataInputs,
     };
   };

package/lib/ergo/utils.ts

@@ -1,5 +1,5 @@
 import { intersection } from 'lodash-es';
-import { OutputBox } from './interfaces';
+import { OutputBox } from '@rosen-bridge/scanner-interfaces';
 
 /**
  * Check box to have specified tokens

package/lib/index.ts

@@ -1,4 +1,3 @@
-export * from './interfaces';
 export * from './ergo';
-export * from './AbstractExtractor';
 export * from './constants';
+export * from './abstractExtractor';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rosen-bridge/abstract-extractor",
-  "version": "1.0.1",
+  "version": "2.0.0-181e9e8f",
   "description": "Rosen Bridge extractor interfaces to work with scanner",
   "repository": "",
   "license": "GPL-3.0",
@@ -38,8 +38,9 @@
   "dependencies": {
     "@rosen-bridge/abstract-logger": "^2.0.1",
     "@rosen-bridge/json-bigint": "^0.1.0",
+    "@rosen-bridge/scanner-interfaces": "^0.1.0",
     "@rosen-clients/ergo-explorer": "^1.1.2",
-    "@rosen-clients/ergo-node": "^1.1.1",
+    "@rosen-clients/ergo-node": "^1.2.0",
     "lodash-es": "^4.17.21",
     "typeorm": "^0.3.20",
     "uuid": "^9.0.0"

package/tests/AbstractErgoExtractor.mock.ts

@@ -1,8 +1,8 @@
 import { V1 } from '@rosen-clients/ergo-explorer';
+import { BlockInfo, OutputBox } from '@rosen-bridge/scanner-interfaces';
+
 import {
   AbstractErgoExtractor,
-  BlockInfo,
-  OutputBox,
   AbstractBoxData,
   AbstractErgoExtractorAction,
   AbstractErgoExtractorEntity,

package/tests/AbstractErgoExtractor.spec.ts

@@ -1,8 +1,8 @@
 import { V1 } from '@rosen-clients/ergo-explorer';
 import { describe, it, expect, vitest } from 'vitest';
+import { OutputBox } from '@rosen-bridge/scanner-interfaces';
 
 import {
-  OutputBox,
   AbstractBoxData,
   AbstractErgoExtractorAction,
   CallbackType,
@@ -22,7 +22,7 @@ describe('AbstractErgoExtractor', () => {
      * - spy `extractBoxData` and `storeBoxes`
      * - run test (call `processTransactions`)
      * @expected
-     * - to call `extractBoxData` for the specific box
+     * - to call `extractBoxData` for the specific box and all input extensions
      * - to insert the extracted box to database
      * - to return true when total procedure is successful
      * - to trigger `INSERT` callbacks with correct data
@@ -49,7 +49,11 @@ describe('AbstractErgoExtractor', () => {
       const result = await extractor.processTransactions([tx], block);
 
       expect(extractSpy).toBeCalledTimes(1);
-      expect(extractSpy).toBeCalledWith(tx.outputs[0]);
+      expect(extractSpy).toBeCalledWith(
+        tx.outputs[0],
+        [tx.inputs[0].extension, {}],
+        {}
+      );
       expect(storeSpy).toBeCalledWith([extractedData], block, 'Test');
       expect(result).toEqual(true);
       expect(triggerCallbacks).toBeCalledWith(CallbackType.Insert, [

package/tests/AbstractErgoExtractorAction.mock.ts

@@ -1,10 +1,10 @@
 import { DataSource } from 'typeorm';
 import { pick } from 'lodash-es';
+import { BlockInfo } from '@rosen-bridge/scanner-interfaces';
 
 import {
   AbstractErgoExtractorAction,
   AbstractErgoExtractorEntity,
-  BlockInfo,
   AbstractBoxData,
 } from '../lib';
 import { TestEntity } from './testUtils';

package/tests/initializable/AbstractInitializable.mock.ts

@@ -1,14 +1,11 @@
 import { V1 } from '@rosen-clients/ergo-explorer';
+import { BlockInfo, OutputBox } from '@rosen-bridge/scanner-interfaces';
+
 import {
   AbstractInitializableErgoExtractor,
   AbstractInitializableErgoExtractorAction,
 } from '../../lib/ergo/initializable';
-import {
-  OutputBox,
-  AbstractBoxData,
-  BlockInfo,
-  AbstractErgoExtractorEntity,
-} from '../../lib';
+import { AbstractBoxData, AbstractErgoExtractorEntity } from '../../lib';
 import { ergoBoxes } from './testData';
 
 export class MockedInitializableErgoExtractor extends AbstractInitializableErgoExtractor<

package/tests/initializable/AbstractInitializable.spec.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it, vi, vitest } from 'vitest';
+import { ErgoNetworkType } from '@rosen-bridge/scanner-interfaces';
 
 import {
-  ErgoNetworkType,
   NodeNetwork,
   AbstractBoxData,
   ExplorerNetwork,

package/tests/initializable/AbstractInitializableAction.mock.ts

@@ -1,9 +1,9 @@
 import { DataSource } from 'typeorm';
 import { pick } from 'lodash-es';
+import { BlockInfo } from '@rosen-bridge/scanner-interfaces';
 
 import {
   AbstractErgoExtractorEntity,
-  BlockInfo,
   AbstractBoxData,
   AbstractInitializableErgoExtractorAction,
 } from '../../lib';