@rosen-bridge/abstract-extractor 0.3.0 → 1.0.0

package/lib/ergo/AbstractErgoExtractorAction.ts

@@ -1,39 +1,208 @@
+import {
+  DataSource,
+  In,
+  Repository,
+  Not,
+  EntityTarget,
+  FindOptionsWhere,
+} from 'typeorm';
+import { chunk, difference, pick } from 'lodash-es';
+import { AbstractLogger, DummyLogger } from '@rosen-bridge/abstract-logger';
+import JsonBigInt from '@rosen-bridge/json-bigint';
+
 import { BlockInfo } from '../interfaces';
-import { SpendInfo } from './interfaces';
+import { AbstractBoxData, BoxInfo, SpendInfo } from './interfaces';
+import { DB_CHUNK_SIZE } from '../constants';
+import { AbstractErgoExtractorEntity } from './AbstractErgoExtractorEntity';
+
+export abstract class AbstractErgoExtractorAction<
+  ExtractedData extends AbstractBoxData,
+  ExtractorEntity extends AbstractErgoExtractorEntity
+> {
+  private readonly datasource: DataSource;
+  readonly logger: AbstractLogger;
+  protected readonly repository: Repository<ExtractorEntity>;
+  private repo: EntityTarget<ExtractorEntity>;
+
+  constructor(
+    dataSource: DataSource,
+    repo: EntityTarget<ExtractorEntity>,
+    logger?: AbstractLogger
+  ) {
+    this.datasource = dataSource;
+    this.logger = logger ? logger : new DummyLogger();
+    this.repository = this.datasource.getRepository(repo);
+    this.repo = repo;
+  }
 
-export abstract class AbstractErgoExtractorAction<ExtractedData> {
   /**
-   * insert all extracted box data in an atomic transaction
-   * @param data
-   * @param extractorId
-   * @return process success
+   * create the database entity from extracted data and block information
    */
-  abstract insertBoxes: (
+  protected abstract createEntity: (
     data: ExtractedData[],
-    extractorId: string
-  ) => Promise<boolean>;
+    block: BlockInfo,
+    extractor: string
+  ) => Array<Omit<ExtractorEntity, 'id'>>;
+
+  /**
+   * convert the database entity back to raw data
+   */
+  protected abstract convertEntityToData: (
+    entities: ExtractorEntity[]
+  ) => ExtractedData[];
+
+  /**
+   * insert all extracted box data in an atomic transaction
+   * update the data if a box with the same id is already stored in db
+   * @param boxes
+   * @param block
+   * @param extractor
+   * @return inserted items and updated box ids
+   * returns undefined in case of any problem
+   */
+  storeBoxes = async (
+    boxes: Array<ExtractedData>,
+    block: BlockInfo,
+    extractor: string
+  ): Promise<boolean> => {
+    let success = true;
+    let boxesToInsert: ExtractedData[] = [],
+      boxesToUpdate: ExtractedData[] = [];
+    const queryRunner = this.datasource.createQueryRunner();
+    await queryRunner.connect();
+    await queryRunner.startTransaction();
+    try {
+      const repository = queryRunner.manager.getRepository(this.repo);
+      const dbBoxIds = (
+        await repository.findBy({
+          boxId: In(boxes.map((item) => item.boxId)),
+          extractor: extractor,
+        } as FindOptionsWhere<ExtractorEntity>)
+      ).map((box) => box.boxId);
+      if (dbBoxIds.length > 0)
+        this.logger.debug(`Found stored boxes with same boxId`, dbBoxIds);
+
+      boxesToUpdate = boxes.filter((box) => dbBoxIds.includes(box.boxId));
+      boxesToInsert = difference(boxes, boxesToUpdate);
+
+      if (boxesToInsert.length > 0) {
+        this.logger.debug(`Inserting boxes`);
+        await repository.insert(
+          this.createEntity(boxesToInsert, block, extractor) as any
+        );
+      }
+      if (boxesToUpdate.length > 0)
+        this.logger.info(
+          `Updating boxes with following Ids in the database: [${boxesToUpdate
+            .map((col) => col.boxId)
+            .join(', ')}]`
+        );
+      for (const box of this.createEntity(boxesToUpdate, block, extractor)) {
+        this.logger.debug(
+          `Updating boxes in database [${JsonBigInt.stringify(box)}]`
+        );
+        await repository.update(
+          {
+            boxId: box.boxId,
+            extractor: extractor,
+          } as FindOptionsWhere<ExtractorEntity>,
+          box as any
+        );
+      }
+      await queryRunner.commitTransaction();
+    } catch (e) {
+      this.logger.error(`An error occurred during store boxes action: ${e}`);
+      await queryRunner.rollbackTransaction();
+      success = false;
+    } finally {
+      await queryRunner.release();
+    }
+    return success;
+  };
 
   /**
    * update spending information of stored boxes
+   * chunk spendInfos to prevent large database queries
+   * Note: It only updates the spendHeight and spendBlock fields. If updating
+   * anything else is required, override this implementation to include the
+   * additional fields.
    * @param spendInfos
    * @param block
-   * @param extractorId
+   * @param extractor
+   * @returns spent box ids
    */
-  abstract spendBoxes: (
-    spendInfos: SpendInfo[],
+  spendBoxes = async (
+    spendInfos: Array<SpendInfo>,
     block: BlockInfo,
-    extractorId: string
-  ) => Promise<void>;
+    extractor: string
+  ): Promise<BoxInfo[]> => {
+    const spentData = [];
+    const spendInfoChunks = chunk(spendInfos, DB_CHUNK_SIZE);
+    for (const spendInfoChunk of spendInfoChunks) {
+      const boxIds = spendInfoChunk.map((info) => info.boxId);
+      const updateResult = await this.repository.update(
+        {
+          boxId: In(boxIds),
+          extractor: extractor,
+        } as FindOptionsWhere<ExtractorEntity>,
+        { spendBlock: block.hash, spendHeight: block.height } as any
+      );
+
+      if (updateResult.affected && updateResult.affected > 0) {
+        const spentRows = await this.repository.findBy({
+          boxId: In(boxIds),
+          spendBlock: block.hash,
+        } as FindOptionsWhere<ExtractorEntity>);
+        spentData.push(...spentRows);
+        for (const row of spentRows) {
+          this.logger.debug(
+            `Spent box with boxId [${row.boxId}] at height ${block.height}`
+          );
+        }
+      }
+    }
+    return spentData.map((data) => pick(data, 'boxId'));
+  };
 
   /**
    * delete extracted data from a specific block
    * if a box is spend in this block mark it as unspent
    * if a box is created in this block remove it from database
    * @param block
-   * @param extractorId
+   * @param extractor
+   * @return deleted items and updated box ids
    */
-  abstract deleteBlockBoxes: (
+  deleteBlockBoxes = async (
     block: string,
-    extractorId: string
-  ) => Promise<void>;
+    extractor: string
+  ): Promise<{ deletedData: ExtractedData[]; updatedData: BoxInfo[] }> => {
+    this.logger.info(
+      `Deleting boxes in block ${block} and extractor ${extractor}`
+    );
+    const deletedData = await this.repository.find({
+      where: { extractor: extractor, block: block } as any,
+    });
+    const updatedData = await this.repository.find({
+      where: {
+        extractor: extractor,
+        spendBlock: block,
+        block: Not(block),
+      } as any,
+    });
+    await this.repository.delete({
+      extractor: extractor,
+      block: block,
+    } as any);
+    await this.repository.update(
+      {
+        spendBlock: block,
+        extractor: extractor,
+      } as FindOptionsWhere<ExtractorEntity>,
+      { spendBlock: null, spendHeight: 0 } as any
+    );
+    return {
+      deletedData: this.convertEntityToData(deletedData),
+      updatedData: updatedData.map((data) => pick(data, 'boxId')),
+    };
+  };
 }

package/lib/ergo/AbstractErgoExtractorEntity.ts

@@ -0,0 +1,28 @@
+import { Column, PrimaryGeneratedColumn, Unique } from 'typeorm';
+
+@Unique(['boxId', 'extractor'])
+export abstract class AbstractErgoExtractorEntity {
+  @PrimaryGeneratedColumn()
+  id: number;
+
+  @Column({ type: 'varchar' })
+  boxId: string;
+
+  @Column({ type: 'varchar' })
+  block: string;
+
+  @Column({ type: 'int' })
+  height: number;
+
+  @Column({ nullable: true, type: 'varchar' })
+  spendBlock?: string | null;
+
+  @Column({ nullable: true, type: 'int' })
+  spendHeight?: number | null;
+
+  @Column({ type: 'varchar' })
+  extractor: string;
+
+  @Column({ type: 'varchar' })
+  serialized: string;
+}

package/lib/ergo/index.ts

@@ -6,3 +6,4 @@ export * from './network/NodeNetwork';
 export * from './network/AbstractNetwork';
 export * from './initializable';
 export * from './utils';
+export * from './AbstractErgoExtractorEntity';

package/lib/ergo/initializable/AbstractInitializable.ts

@@ -1,6 +1,8 @@
 import { AbstractLogger } from '@rosen-bridge/abstract-logger';
+import { groupBy, sortBy } from 'lodash-es';
+
 import {
-  ErgoExtractedData,
+  AbstractBoxData,
   ErgoNetworkType,
   ExtendedTransaction,
 } from '../interfaces';
@@ -10,14 +12,18 @@ import { AbstractInitializableErgoExtractorAction } from './AbstractInitializabl
 import { BlockInfo } from '../../interfaces';
 import { ExplorerNetwork } from '../network/ExplorerNetwork';
 import { NodeNetwork } from '../network/NodeNetwork';
-import { groupBy, sortBy } from 'lodash-es';
+import { AbstractErgoExtractorEntity } from '../AbstractErgoExtractorEntity';
 
 export abstract class AbstractInitializableErgoExtractor<
-  ExtractedData extends ErgoExtractedData
-> extends AbstractErgoExtractor<ExtractedData> {
+  ExtractedData extends AbstractBoxData,
+  ExtractorEntity extends AbstractErgoExtractorEntity
+> extends AbstractErgoExtractor<ExtractedData, ExtractorEntity> {
   protected initialize: boolean;
   private address: string;
-  protected abstract actions: AbstractInitializableErgoExtractorAction<ExtractedData>;
+  protected abstract actions: AbstractInitializableErgoExtractorAction<
+    ExtractedData,
+    ExtractorEntity
+  >;
 
   private network: ExplorerNetwork | NodeNetwork;
 
@@ -112,14 +118,15 @@ export abstract class AbstractInitializableErgoExtractor<
    */
   private initializeWithNode = async (initialBlock: BlockInfo) => {
     const txCountBeforeInit = await this.getTotalTxCount();
+    let offset = 0,
+      total = 1,
+      round = 1;
     await this.initWithRetrial(async () => {
       // Repeat the whole process twice to cover all spent boxes
       // After round 1 all boxes have been saved and processed once
       // After round 2 spending information of all stored boxes are updated successfully
-      for (let round = 0; round <= 1; round++) {
+      while (round <= 2) {
         this.logger.debug(`Starting round ${round} of initialization`);
-        let offset = 0,
-          total = 1;
         while (offset < total) {
           const response = await (
             this.network as NodeNetwork
@@ -138,6 +145,8 @@ export abstract class AbstractInitializableErgoExtractor<
           if (txs.length > 0) await this.processTransactionBatch(txs);
           offset += API_LIMIT;
         }
+        round++;
+        offset = 0; // next round initial offset
       }
     });
     const txCountAfterInit = await this.getTotalTxCount();
@@ -167,7 +176,11 @@ export abstract class AbstractInitializableErgoExtractor<
       this.logger.debug(
         `Processing transactions at height ${blockTxs[0].inclusionHeight}`
       );
-      await this.processTransactions(blockTxs, block);
+      const success = await this.processTransactions(blockTxs, block);
+      if (!success)
+        throw Error(
+          `Processing transactions failed at height ${blockTxs[0].inclusionHeight}`
+        );
     }
   };
 

package/lib/ergo/initializable/AbstractInitializableAction.ts

@@ -1,11 +1,27 @@
+import { DataSource, EntityTarget } from 'typeorm';
+import { AbstractLogger } from '@rosen-bridge/abstract-logger';
+
 import { AbstractErgoExtractorAction } from '../AbstractErgoExtractorAction';
+import { AbstractBoxData } from '../interfaces';
+import { AbstractErgoExtractorEntity } from '../AbstractErgoExtractorEntity';
 
 export abstract class AbstractInitializableErgoExtractorAction<
-  ExtractedData
-> extends AbstractErgoExtractorAction<ExtractedData> {
+  ExtractedData extends AbstractBoxData,
+  ExtractorEntity extends AbstractErgoExtractorEntity
+> extends AbstractErgoExtractorAction<ExtractedData, ExtractorEntity> {
+  constructor(
+    dataSource: DataSource,
+    repo: EntityTarget<ExtractorEntity>,
+    logger?: AbstractLogger
+  ) {
+    super(dataSource, repo, logger);
+  }
+
   /**
    * remove all existing data for the extractor
    * @param extractorId
    */
-  abstract removeAllData: (extractorId: string) => Promise<void>;
+  removeAllData = async (extractorId: string) => {
+    await this.repository.delete({ extractor: extractorId } as any);
+  };
 }

package/lib/ergo/interfaces.ts

@@ -62,14 +62,32 @@ export interface SpendInfo {
   boxId: string;
   txId: string;
   index: number;
+  extras?: string[];
 }
 
-export interface ErgoExtractedData {
+export interface AbstractBoxData {
+  boxId: string;
+  serialized: string;
+}
+
+export enum CallbackType {
+  Insert = 'insert',
+  Update = 'update',
+  Spend = 'spend',
+  Delete = 'delete',
+}
+
+export interface BoxInfo {
   boxId: string;
-  height: number;
-  blockId: string;
-  spendBlock?: string;
-  spendHeight?: number;
-  spendTxId?: string;
-  spendIndex?: number;
 }
+
+export type CallbackDataMap<ExtractedData extends AbstractBoxData> = {
+  [CallbackType.Update]: BoxInfo[];
+  [CallbackType.Insert]: ExtractedData[];
+  [CallbackType.Delete]: ExtractedData[];
+  [CallbackType.Spend]: BoxInfo[];
+};
+
+export type CallbackMap<ExtractedData extends AbstractBoxData> = {
+  [K in CallbackType]: (data: CallbackDataMap<ExtractedData>[K]) => void;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rosen-bridge/abstract-extractor",
-  "version": "0.3.0",
+  "version": "1.0.0",
   "description": "Rosen Bridge extractor interfaces to work with scanner",
   "repository": "",
   "license": "GPL-3.0",
@@ -20,9 +20,11 @@
   "devDependencies": {
     "@types/lodash-es": "^4.17.12",
     "@types/node": "^20.11.9",
+    "@types/uuid": "^10.0.0",
     "@typescript-eslint/eslint-plugin": "^6.19.1",
     "@typescript-eslint/parser": "^6.19.1",
     "@vitest/coverage-istanbul": "^1.2.2",
+    "await-semaphore": "^0.1.3",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",
     "extensionless": "^1.9.6",
@@ -38,6 +40,8 @@
     "@rosen-bridge/json-bigint": "^0.1.0",
     "@rosen-clients/ergo-explorer": "^1.1.2",
     "@rosen-clients/ergo-node": "^1.1.1",
-    "lodash-es": "^4.17.21"
+    "lodash-es": "^4.17.21",
+    "typeorm": "^0.3.20",
+    "uuid": "^9.0.0"
   }
 }

package/tests/{AbstractExtractor.mock.ts → AbstractErgoExtractor.mock.ts}

@@ -3,12 +3,19 @@ import {
   AbstractErgoExtractor,
   BlockInfo,
   OutputBox,
-  ErgoExtractedData,
+  AbstractBoxData,
   AbstractErgoExtractorAction,
+  AbstractErgoExtractorEntity,
 } from '../lib';
 
-export class MockedErgoExtractor extends AbstractErgoExtractor<ErgoExtractedData> {
-  actions: AbstractErgoExtractorAction<ErgoExtractedData>;
+export class MockedErgoExtractor extends AbstractErgoExtractor<
+  AbstractBoxData,
+  AbstractErgoExtractorEntity
+> {
+  actions: AbstractErgoExtractorAction<
+    AbstractBoxData,
+    AbstractErgoExtractorEntity
+  >;
 
   getId = () => 'Test';
 
@@ -17,10 +24,8 @@ export class MockedErgoExtractor extends AbstractErgoExtractor<ErgoExtractedData
   hasData = (box: V1.OutputInfo | OutputBox) => false;
 
   extractBoxData = (
-    box: V1.OutputInfo | OutputBox,
-    blockId: string,
-    height: number
-  ): Omit<ErgoExtractedData, 'spendBlock' | 'spendHeight'> | undefined => {
+    box: V1.OutputInfo | OutputBox
+  ): AbstractBoxData | undefined => {
     return undefined;
   };
 }