@rosen-bridge/abstract-extractor 0.3.1 → 1.0.0

package/lib/ergo/AbstractErgoExtractorEntity.ts

@@ -0,0 +1,28 @@
+import { Column, PrimaryGeneratedColumn, Unique } from 'typeorm';
+
+@Unique(['boxId', 'extractor'])
+export abstract class AbstractErgoExtractorEntity {
+  @PrimaryGeneratedColumn()
+  id: number;
+
+  @Column({ type: 'varchar' })
+  boxId: string;
+
+  @Column({ type: 'varchar' })
+  block: string;
+
+  @Column({ type: 'int' })
+  height: number;
+
+  @Column({ nullable: true, type: 'varchar' })
+  spendBlock?: string | null;
+
+  @Column({ nullable: true, type: 'int' })
+  spendHeight?: number | null;
+
+  @Column({ type: 'varchar' })
+  extractor: string;
+
+  @Column({ type: 'varchar' })
+  serialized: string;
+}

package/lib/ergo/index.ts

@@ -6,3 +6,4 @@ export * from './network/NodeNetwork';
 export * from './network/AbstractNetwork';
 export * from './initializable';
 export * from './utils';
+export * from './AbstractErgoExtractorEntity';

package/lib/ergo/initializable/AbstractInitializable.ts

@@ -1,6 +1,8 @@
 import { AbstractLogger } from '@rosen-bridge/abstract-logger';
+import { groupBy, sortBy } from 'lodash-es';
+
 import {
-  ErgoExtractedData,
+  AbstractBoxData,
   ErgoNetworkType,
   ExtendedTransaction,
 } from '../interfaces';
@@ -10,14 +12,18 @@ import { AbstractInitializableErgoExtractorAction } from './AbstractInitializabl
 import { BlockInfo } from '../../interfaces';
 import { ExplorerNetwork } from '../network/ExplorerNetwork';
 import { NodeNetwork } from '../network/NodeNetwork';
-import { groupBy, sortBy } from 'lodash-es';
+import { AbstractErgoExtractorEntity } from '../AbstractErgoExtractorEntity';
 
 export abstract class AbstractInitializableErgoExtractor<
-  ExtractedData extends ErgoExtractedData
-> extends AbstractErgoExtractor<ExtractedData> {
+  ExtractedData extends AbstractBoxData,
+  ExtractorEntity extends AbstractErgoExtractorEntity
+> extends AbstractErgoExtractor<ExtractedData, ExtractorEntity> {
   protected initialize: boolean;
   private address: string;
-  protected abstract actions: AbstractInitializableErgoExtractorAction<ExtractedData>;
+  protected abstract actions: AbstractInitializableErgoExtractorAction<
+    ExtractedData,
+    ExtractorEntity
+  >;
 
   private network: ExplorerNetwork | NodeNetwork;
 
@@ -112,14 +118,15 @@ export abstract class AbstractInitializableErgoExtractor<
    */
   private initializeWithNode = async (initialBlock: BlockInfo) => {
     const txCountBeforeInit = await this.getTotalTxCount();
+    let offset = 0,
+      total = 1,
+      round = 1;
     await this.initWithRetrial(async () => {
       // Repeat the whole process twice to cover all spent boxes
       // After round 1 all boxes have been saved and processed once
       // After round 2 spending information of all stored boxes are updated successfully
-      for (let round = 0; round <= 1; round++) {
+      while (round <= 2) {
         this.logger.debug(`Starting round ${round} of initialization`);
-        let offset = 0,
-          total = 1;
         while (offset < total) {
           const response = await (
             this.network as NodeNetwork
@@ -138,6 +145,8 @@ export abstract class AbstractInitializableErgoExtractor<
           if (txs.length > 0) await this.processTransactionBatch(txs);
           offset += API_LIMIT;
         }
+        round++;
+        offset = 0; // next round initial offset
       }
     });
     const txCountAfterInit = await this.getTotalTxCount();
@@ -167,7 +176,11 @@ export abstract class AbstractInitializableErgoExtractor<
       this.logger.debug(
         `Processing transactions at height ${blockTxs[0].inclusionHeight}`
       );
-      await this.processTransactions(blockTxs, block);
+      const success = await this.processTransactions(blockTxs, block);
+      if (!success)
+        throw Error(
+          `Processing transactions failed at height ${blockTxs[0].inclusionHeight}`
+        );
     }
   };
 

package/lib/ergo/initializable/AbstractInitializableAction.ts

@@ -1,11 +1,27 @@
+import { DataSource, EntityTarget } from 'typeorm';
+import { AbstractLogger } from '@rosen-bridge/abstract-logger';
+
 import { AbstractErgoExtractorAction } from '../AbstractErgoExtractorAction';
+import { AbstractBoxData } from '../interfaces';
+import { AbstractErgoExtractorEntity } from '../AbstractErgoExtractorEntity';
 
 export abstract class AbstractInitializableErgoExtractorAction<
-  ExtractedData
-> extends AbstractErgoExtractorAction<ExtractedData> {
+  ExtractedData extends AbstractBoxData,
+  ExtractorEntity extends AbstractErgoExtractorEntity
+> extends AbstractErgoExtractorAction<ExtractedData, ExtractorEntity> {
+  constructor(
+    dataSource: DataSource,
+    repo: EntityTarget<ExtractorEntity>,
+    logger?: AbstractLogger
+  ) {
+    super(dataSource, repo, logger);
+  }
+
   /**
    * remove all existing data for the extractor
    * @param extractorId
    */
-  abstract removeAllData: (extractorId: string) => Promise<void>;
+  removeAllData = async (extractorId: string) => {
+    await this.repository.delete({ extractor: extractorId } as any);
+  };
 }

package/lib/ergo/interfaces.ts

@@ -65,6 +65,29 @@ export interface SpendInfo {
   extras?: string[];
 }
 
-export interface ErgoExtractedData {
+export interface AbstractBoxData {
   boxId: string;
+  serialized: string;
 }
+
+export enum CallbackType {
+  Insert = 'insert',
+  Update = 'update',
+  Spend = 'spend',
+  Delete = 'delete',
+}
+
+export interface BoxInfo {
+  boxId: string;
+}
+
+export type CallbackDataMap<ExtractedData extends AbstractBoxData> = {
+  [CallbackType.Update]: BoxInfo[];
+  [CallbackType.Insert]: ExtractedData[];
+  [CallbackType.Delete]: ExtractedData[];
+  [CallbackType.Spend]: BoxInfo[];
+};
+
+export type CallbackMap<ExtractedData extends AbstractBoxData> = {
+  [K in CallbackType]: (data: CallbackDataMap<ExtractedData>[K]) => void;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rosen-bridge/abstract-extractor",
-  "version": "0.3.1",
+  "version": "1.0.0",
   "description": "Rosen Bridge extractor interfaces to work with scanner",
   "repository": "",
   "license": "GPL-3.0",
@@ -20,9 +20,11 @@
   "devDependencies": {
     "@types/lodash-es": "^4.17.12",
     "@types/node": "^20.11.9",
+    "@types/uuid": "^10.0.0",
     "@typescript-eslint/eslint-plugin": "^6.19.1",
     "@typescript-eslint/parser": "^6.19.1",
     "@vitest/coverage-istanbul": "^1.2.2",
+    "await-semaphore": "^0.1.3",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",
     "extensionless": "^1.9.6",
@@ -38,6 +40,8 @@
     "@rosen-bridge/json-bigint": "^0.1.0",
     "@rosen-clients/ergo-explorer": "^1.1.2",
     "@rosen-clients/ergo-node": "^1.1.1",
-    "lodash-es": "^4.17.21"
+    "lodash-es": "^4.17.21",
+    "typeorm": "^0.3.20",
+    "uuid": "^9.0.0"
   }
 }

package/tests/{AbstractExtractor.mock.ts → AbstractErgoExtractor.mock.ts}

@@ -3,12 +3,19 @@ import {
   AbstractErgoExtractor,
   BlockInfo,
   OutputBox,
-  ErgoExtractedData,
+  AbstractBoxData,
   AbstractErgoExtractorAction,
+  AbstractErgoExtractorEntity,
 } from '../lib';
 
-export class MockedErgoExtractor extends AbstractErgoExtractor<ErgoExtractedData> {
-  actions: AbstractErgoExtractorAction<ErgoExtractedData>;
+export class MockedErgoExtractor extends AbstractErgoExtractor<
+  AbstractBoxData,
+  AbstractErgoExtractorEntity
+> {
+  actions: AbstractErgoExtractorAction<
+    AbstractBoxData,
+    AbstractErgoExtractorEntity
+  >;
 
   getId = () => 'Test';
 
@@ -18,7 +25,7 @@ export class MockedErgoExtractor extends AbstractErgoExtractor<ErgoExtractedData
 
   extractBoxData = (
     box: V1.OutputInfo | OutputBox
-  ): ErgoExtractedData | undefined => {
+  ): AbstractBoxData | undefined => {
     return undefined;
   };
 }

package/tests/AbstractErgoExtractor.spec.ts

@@ -0,0 +1,281 @@
+import { V1 } from '@rosen-clients/ergo-explorer';
+import { describe, it, expect, vitest } from 'vitest';
+
+import {
+  OutputBox,
+  AbstractBoxData,
+  AbstractErgoExtractorAction,
+  CallbackType,
+  AbstractErgoExtractorEntity,
+} from '../lib';
+import { block, extractedData, tx } from './testData';
+import { MockedErgoExtractor } from './AbstractErgoExtractor.mock';
+
+describe('AbstractErgoExtractor', () => {
+  describe('processTransactions', () => {
+    /**
+     * @target processTransactions should initialize extractor with specified id and insert status into db
+     * @dependencies
+     * @scenario
+     * - mock extractor
+     * - mock `hasData` to return true for one box
+     * - spy `extractBoxData` and `storeBoxes`
+     * - run test (call `processTransactions`)
+     * @expected
+     * - to call `extractBoxData` for the specific box
+     * - to insert the extracted box to database
+     * - to return true when total procedure is successful
+     * - to trigger `INSERT` callbacks with correct data
+     */
+    it('should process boxes with data and insert data into database', async () => {
+      const extractor = new MockedErgoExtractor();
+      const triggerCallbacks = vitest.fn();
+      extractor['triggerCallbacks'] = triggerCallbacks;
+      extractor.hasData = (box: V1.OutputInfo | OutputBox) => {
+        if (box.boxId == tx.outputs[0].boxId) return true;
+        return false;
+      };
+      const extractSpy = vitest.fn().mockReturnValue(extractedData);
+      extractor.extractBoxData = extractSpy;
+      const storeSpy = vitest.fn().mockResolvedValue(true);
+      const spendSpy = vitest.fn().mockResolvedValue([]);
+      extractor['actions'] = {
+        storeBoxes: storeSpy,
+        spendBoxes: spendSpy,
+      } as unknown as AbstractErgoExtractorAction<
+        AbstractBoxData,
+        AbstractErgoExtractorEntity
+      >;
+      const result = await extractor.processTransactions([tx], block);
+
+      expect(extractSpy).toBeCalledTimes(1);
+      expect(extractSpy).toBeCalledWith(tx.outputs[0]);
+      expect(storeSpy).toBeCalledWith([extractedData], block, 'Test');
+      expect(result).toEqual(true);
+      expect(triggerCallbacks).toBeCalledWith(CallbackType.Insert, [
+        extractedData,
+      ]);
+    });
+
+    /**
+     * @target processTransactions should extract spending information of all input boxes
+     * @dependencies
+     * @scenario
+     * - mock extractor (hasData returns false as default)
+     * - spy `extractBoxData`, `storeBoxes` and `spendBoxes`
+     * - run test (call `processTransactions`)
+     * @expected
+     * - not to call `extractBoxData` and `storeBoxes` when there is not any box with data
+     * - to extractor spend info of input boxes and call `spendBoxes`
+     * - to return true when total procedure is successful
+     * - to trigger `SPEND` callbacks with correct data
+     */
+    it('should extract spending information of all input boxes', async () => {
+      const extractor = new MockedErgoExtractor();
+      const triggerCallbacks = vitest.fn();
+      extractor['triggerCallbacks'] = triggerCallbacks;
+      const extractSpy = vitest.fn();
+      extractor.extractBoxData = extractSpy;
+      const storeSpy = vitest.fn().mockResolvedValue(true);
+      const spendSpy = vitest
+        .fn()
+        .mockResolvedValue([
+          { boxId: tx.inputs[0].boxId },
+          { boxId: tx.inputs[1].boxId },
+        ]);
+      extractor['actions'] = {
+        storeBoxes: storeSpy,
+        spendBoxes: spendSpy,
+      } as unknown as AbstractErgoExtractorAction<
+        AbstractBoxData,
+        AbstractErgoExtractorEntity
+      >;
+      const result = await extractor.processTransactions([tx], block);
+
+      expect(extractSpy).not.toBeCalled();
+      expect(storeSpy).not.toBeCalled();
+      expect(spendSpy).toBeCalledWith(
+        [
+          { boxId: tx.inputs[0].boxId, txId: tx.id, index: 1 },
+          { boxId: tx.inputs[1].boxId, txId: tx.id, index: 2 },
+        ],
+        block,
+        'Test'
+      );
+      expect(result).toEqual(true);
+      expect(triggerCallbacks).toBeCalledWith(CallbackType.Spend, [
+        { boxId: tx.inputs[0].boxId },
+        { boxId: tx.inputs[1].boxId },
+      ]);
+    });
+
+    /**
+     * @target processTransactions should return false if data insertion fails
+     * @dependencies
+     * @scenario
+     * - mock extractor
+     * - mock `hasData` to return true for one box
+     * - spy `extractBoxData` and `storeBoxes`
+     * - run test (call `processTransactions`)
+     * @expected
+     * - to return false when `insertBoxes` returns false
+     * - not to call `spendBoxes` if data insertion fails
+     */
+    it('should return false if data insertion fails', async () => {
+      const extractor = new MockedErgoExtractor();
+      extractor.hasData = (box: V1.OutputInfo | OutputBox) => {
+        if (box.boxId == tx.outputs[0].boxId) return true;
+        return false;
+      };
+      const extractSpy = vitest.fn().mockReturnValue(extractedData);
+      extractor.extractBoxData = extractSpy;
+      const storeSpy = vitest.fn().mockResolvedValue(false);
+      const spendSpy = vitest.fn();
+      extractor['actions'] = {
+        storeBoxes: storeSpy,
+        spendBoxes: spendSpy,
+      } as unknown as AbstractErgoExtractorAction<
+        AbstractBoxData,
+        AbstractErgoExtractorEntity
+      >;
+      const result = await extractor.processTransactions([tx], block);
+
+      expect(result).toEqual(false);
+      expect(spendSpy).not.toBeCalled();
+    });
+  });
+
+  describe('forkBlock', () => {
+    /**
+     * @target forkBlock should remove all data extracted from the specified block
+     * @dependencies
+     * @scenario
+     * - mock extractor
+     * - spy `deleteBlockBoxes`
+     * - run test (call `forkBlock`)
+     * @expected
+     * - to call `deleteBlockBoxes` for the specific box
+     * - to trigger `DELETE` callbacks for the deleted box
+     * - to trigger `UPDATE` callbacks for the spent box
+     */
+    it('should remove all data extracted from the specified block', async () => {
+      const extractor = new MockedErgoExtractor();
+      const removeSpy = vitest.fn().mockResolvedValue({
+        deletedData: [{ boxId: 'box1' }],
+        updatedData: [{ boxId: 'box2' }],
+      });
+      extractor['actions'] = {
+        deleteBlockBoxes: removeSpy,
+      } as unknown as AbstractErgoExtractorAction<
+        AbstractBoxData,
+        AbstractErgoExtractorEntity
+      >;
+      const triggerCallbackSpy = vitest.fn().mockClear();
+      extractor['triggerCallbacks'] = triggerCallbackSpy;
+      await extractor.forkBlock(block.hash);
+      expect(removeSpy).toBeCalledWith(block.hash, 'Test');
+      expect(triggerCallbackSpy).toBeCalledWith(CallbackType.Delete, [
+        { boxId: 'box1' },
+      ]);
+      expect(triggerCallbackSpy).toBeCalledWith(CallbackType.Update, [
+        { boxId: 'box2' },
+      ]);
+    });
+  });
+
+  describe('hook', () => {
+    /**
+     * @target hook should hook a new callback on insert with the new id
+     * @dependencies
+     * @scenario
+     * - mock extractor
+     * - mock a callback for insert
+     * - run test (call `hook`)
+     * @expected
+     * - hook the callback with the specified id
+     * - return true
+     */
+    it('should hook a new callback on insert with the new id', async () => {
+      const extractor = new MockedErgoExtractor();
+      const insertCallback = vitest.fn();
+      const id = await extractor.hook(CallbackType.Insert, insertCallback);
+      expect(extractor['callbacks'][CallbackType.Insert]).toEqual(
+        new Map().set(id, insertCallback)
+      );
+    });
+  });
+
+  describe('unhook', () => {
+    /**
+     * @target unhook should unhook the callback on insert with the specified id
+     * @dependencies
+     * @scenario
+     * - mock extractor
+     * - mock a callback for insert
+     * - hook the callback
+     * - run test (call `unhook`)
+     * @expected
+     * - unhook the callback with the specified id
+     * - return true
+     */
+    it('should unhook the callback on insert with the specified id', async () => {
+      const extractor = new MockedErgoExtractor();
+      const insertCallback = vitest.fn();
+      const id = await extractor.hook(CallbackType.Insert, insertCallback);
+      const result = await extractor.unhook(CallbackType.Insert, id);
+      expect(result).toBeTruthy();
+      expect(extractor['callbacks'][CallbackType.Insert].get(id)).toEqual(
+        undefined
+      );
+    });
+
+    /**
+     * @target unhook should not unhook callbacks with the same id on other types
+     * @dependencies
+     * @scenario
+     * - mock extractor
+     * - mock two callbacks for insert
+     * - hook the first callback
+     * - run test (call `unhook` with the same id for second callback)
+     * @expected
+     * - not to change hooked callbacks when the callback with the id
+     * doesn't exists on the specified type
+     * - return false
+     */
+    it('should not unhook callbacks with the same id on other types', async () => {
+      const extractor = new MockedErgoExtractor();
+      const insertCallback = vitest.fn();
+      const id = await extractor.hook(CallbackType.Insert, insertCallback);
+      const result = await extractor.unhook(CallbackType.Update, id);
+      expect(result).toBeFalsy();
+      expect(extractor['callbacks'][CallbackType.Insert].get(id)).toEqual(
+        insertCallback
+      );
+      expect(extractor['callbacks'][CallbackType.Update].get(id)).toEqual(
+        undefined
+      );
+    });
+  });
+
+  describe('triggerCallbacks', () => {
+    /**
+     * @target triggerCallbacks should trigger all callbacks hooked on a type
+     * @dependencies
+     * @scenario
+     * - mock extractor
+     * - mock a callback for insert
+     * - hook the callback
+     * - run test (call `triggerCallbacks` for insert type)
+     * @expected
+     * - trigger all callbacks with the specified id
+     */
+    it('should trigger all callbacks hooked on a type', async () => {
+      const extractor = new MockedErgoExtractor();
+      const insertCallback = vitest.fn();
+      await extractor.hook(CallbackType.Insert, insertCallback);
+      const insertedData = [{ boxId: 'boxId', serialized: 'serialized' }];
+      await extractor['triggerCallbacks'](CallbackType.Insert, insertedData);
+      expect(insertCallback).toBeCalledWith(insertedData);
+    });
+  });
+});

package/tests/AbstractErgoExtractorAction.mock.ts

@@ -0,0 +1,45 @@
+import { DataSource } from 'typeorm';
+import { pick } from 'lodash-es';
+
+import {
+  AbstractErgoExtractorAction,
+  AbstractErgoExtractorEntity,
+  BlockInfo,
+  AbstractBoxData,
+} from '../lib';
+import { TestEntity } from './testUtils';
+
+export class TestErgoExtractorAction extends AbstractErgoExtractorAction<
+  AbstractBoxData,
+  AbstractErgoExtractorEntity
+> {
+  constructor(dataSource: DataSource) {
+    super(dataSource, TestEntity);
+  }
+
+  /**
+   * create the test database entity from data and block information
+   */
+  createEntity = (
+    boxes: AbstractBoxData[],
+    block: BlockInfo,
+    extractor: string
+  ): Omit<AbstractErgoExtractorEntity, 'id'>[] => {
+    return boxes.map((box) => ({
+      boxId: box.boxId,
+      block: block.hash,
+      height: block.height,
+      serialized: box.serialized,
+      extractor: extractor,
+    }));
+  };
+
+  /**
+   * convert the database entity back to raw data
+   */
+  convertEntityToData = (
+    entities: AbstractErgoExtractorEntity[]
+  ): AbstractBoxData[] => {
+    return entities.map((data) => pick(data, ['boxId', 'serialized']));
+  };
+}