@rosen-bridge/abstract-extractor 0.2.0 → 0.3.1

package/tests/initializable/AbstractInitializable.mock.ts

@@ -21,11 +21,7 @@ export class MockedInitializableErgoExtractor extends AbstractInitializableErgoE
     return Promise.resolve({ boxes: ergoBoxes, hasNextBatch: true });
   };
 
-  extractBoxData = (
-    box: V1.OutputInfo | OutputBox,
-    blockId: string,
-    height: number
-  ) => {
+  extractBoxData = (box: V1.OutputInfo | OutputBox) => {
     return undefined;
   };
 }

package/tests/initializable/AbstractInitializable.spec.ts

@@ -1,247 +1,355 @@
-import { describe, it, expect, vitest } from 'vitest';
+import { describe, expect, it, vi, vitest } from 'vitest';
 
+import {
+  ErgoNetworkType,
+  NodeNetwork,
+  ErgoExtractedData,
+  ExplorerNetwork,
+} from '../../lib';
 import { MockedInitializableErgoExtractor } from './AbstractInitializable.mock';
-import { ergoBoxes, extractedData } from './testData';
-import { OutputBox, ErgoExtractedData } from '../../lib';
+import { transactionBatch } from './testData';
+import { RETRIAL_COUNT } from '../../lib/constants';
 import { AbstractInitializableErgoExtractorAction } from '../../lib/ergo/initializable';
 
-describe('AbstractInitializableErgoExtractorAction', () => {
-  describe('fetchDataWithOffsetLimit', () => {
+describe('AbstractInitializableErgoExtractor', () => {
+  describe('getTotalTxCount', () => {
     /**
-     * @target fetchDataWithOffsetLimit should extract unspent box data in the initial block
+     * @target getTotalTxCount should return the total tx count
      * @dependencies
+     * - explorer network
      * @scenario
      * - mock extractor
-     * - mock `hasData` to return true for an unspent box in the init block
-     * - spy `extractData`
-     * - run test (call `fetchDataWithOffsetLimit`)
+     * - mock `getAddressTransactionsWithOffsetLimit` in node network
+     * - run test (call `getTotalTxCount`)
      * @expected
-     * - to call `extractData` with unspent box information
-     * - return one extracted element (the unspent box) and api total
+     * - return one total tx count
      */
-    it('should extract unspent box data in the initial block', async () => {
-      const extractor = new MockedInitializableErgoExtractor();
-      const unspentBox = ergoBoxes[0];
-      extractor.hasData = (box: OutputBox) => {
-        if (box.boxId == unspentBox.boxId) return true;
-        return false;
-      };
-      const extractSpy = vitest.fn().mockReturnValue(extractedData);
-      extractor.extractBoxData = extractSpy;
-      const txBlockSpy = vitest.fn();
-      extractor.getTxBlock = txBlockSpy;
-      const data = await extractor.fetchDataWithOffsetLimit(1252681, 0, 2);
-      expect(extractSpy).toHaveBeenCalledWith(
-        unspentBox,
-        unspentBox.blockId,
-        unspentBox.inclusionHeight
+    it('should return the total tx count', async () => {
+      const extractor = new MockedInitializableErgoExtractor(
+        ErgoNetworkType.Node,
+        'node_url',
+        'address'
       );
-      expect(txBlockSpy).not.toHaveBeenCalled();
-      expect(data).toEqual({
-        extractedBoxes: [
-          { ...extractedData, spendBlock: undefined, spendHeight: undefined },
-        ],
-        hasNextBatch: true,
+      (
+        extractor['network'] as NodeNetwork
+      ).getAddressTransactionsWithOffsetLimit = vitest.fn().mockResolvedValue({
+        items: [],
+        total: 196704,
       });
+      const totalTxCount = await extractor['getTotalTxCount']();
+      expect(totalTxCount).toEqual(196704);
     });
+  });
 
+  describe('processTransactionBatch', () => {
     /**
-     * @target fetchDataWithOffsetLimit should filter boxes created below the initial block
+     * @target processTransactionBatch should group transactions and process them in correct order
      * @dependencies
      * @scenario
      * - mock extractor
-     * - mock `hasData` to return true for all boxes
-     * - spy `extractData`
-     * - run test (call `fetchDataWithOffsetLimit`)
+     * - spy `processTransactions`
+     * - run test (call `processTransactionBatch`)
      * @expected
-     * - to call `extractData` with spent box information
-     * - return one extracted element created bellow the initial block
+     * - to group txs in two blocks
+     * - to process each blocks transaction separately
      */
-    it('should filter boxes created below the initial block', async () => {
-      const extractor = new MockedInitializableErgoExtractor();
-      const spentBox = ergoBoxes[1];
-      extractor.hasData = (box: OutputBox) => true;
-      const extractSpy = vitest.fn().mockReturnValue(extractedData);
-      extractor.extractBoxData = extractSpy;
-      const data = await extractor.fetchDataWithOffsetLimit(1252678, 0, 2);
-      expect(extractSpy).toHaveBeenCalledWith(
-        spentBox,
-        spentBox.blockId,
-        spentBox.inclusionHeight
+    it('should group transactions and process them in correct order', async () => {
+      const extractor = new MockedInitializableErgoExtractor(
+        ErgoNetworkType.Node,
+        'node_url',
+        'address'
       );
-      expect(data).toEqual({
-        extractedBoxes: [
-          { ...extractedData, spendBlock: undefined, spendHeight: undefined },
-        ],
-        hasNextBatch: true,
+      const processSpy = vitest.fn();
+      extractor.processTransactions = processSpy;
+      await extractor['processTransactionBatch'](transactionBatch);
+      expect(processSpy).toBeCalledTimes(2);
+      expect(processSpy).toHaveBeenCalledWith([transactionBatch[2]], {
+        hash: '92eaff11a4a1c29b8654473258f9d837e9ce99fc0a8a4f27c484d5871afcde64',
+        height: 1320698,
       });
+      expect(processSpy).toHaveBeenLastCalledWith(
+        transactionBatch.slice(0, 2),
+        {
+          hash: 'b861e2134821fcf1fcdad7e6edd56c4e4495b42b2fd72762a4c79ed1db78b44b',
+          height: 1320705,
+        }
+      );
     });
+  });
 
+  describe('initWithRetrial', () => {
     /**
-     * @target fetchDataWithOffsetLimit should extract spent box data created and spent below the initial block
+     * @target initWithRetrial should not run the init job when its not set
      * @dependencies
+     * - database
      * @scenario
-     * - mock extractor
-     * - mock `hasData` to return true for an spent box created below the init block
-     * - spy `extractData`
-     * - run test (call `fetchDataWithOffsetLimit`)
+     * - mock extractor with false initialization
+     * - mock `removeAllData` in database actions
+     * - mock init job
+     * - run test (call `initWithRetrial`)
      * @expected
-     * - to call `extractData` with spent box information
-     * - return one extracted element with spending information
+     * - not to remove database old data
+     * - not to run the init job
      */
-    it('should extract spent box data created and spent below the initial block', async () => {
-      const extractor = new MockedInitializableErgoExtractor();
-      const spentBox = ergoBoxes[1];
-      extractor.hasData = (box: OutputBox) => {
-        if (box.boxId == spentBox.boxId) return true;
-        return false;
-      };
-      const extractSpy = vitest.fn().mockReturnValue(extractedData);
-      extractor.extractBoxData = extractSpy;
-      const data = await extractor.fetchDataWithOffsetLimit(1252681, 0, 2);
-      expect(extractSpy).toHaveBeenCalledWith(
-        spentBox,
-        spentBox.blockId,
-        spentBox.inclusionHeight
+    it('should not run the init job when its not set', async () => {
+      const extractor = new MockedInitializableErgoExtractor(
+        ErgoNetworkType.Node,
+        'node_url',
+        'address',
+        undefined,
+        false
       );
-      expect(data).toEqual({
-        extractedBoxes: [
-          {
-            ...extractedData,
-            spendBlock:
-              '9239ebca7b3b10701895e491a2213da4e07a37abc413d05434a5fab04993a19d',
-            spendHeight: 1252680,
-            spendTxId:
-              'b08fa920a6907d0e76eb0ad754439f1a26fa112b39f4de489620e4e6241930b7',
-            spendIndex: 1,
-          },
-        ],
-        hasNextBatch: true,
-      });
+      const removeSpy = vitest.fn();
+      extractor['actions'] = {
+        removeAllData: removeSpy,
+      } as unknown as AbstractInitializableErgoExtractorAction<ErgoExtractedData>;
+      const initSpy = vitest.fn();
+      await extractor['initWithRetrial'](initSpy);
+      expect(removeSpy).not.toHaveBeenCalled();
+      expect(initSpy).not.toHaveBeenCalled();
     });
 
     /**
-     * @target fetchDataWithOffsetLimit should extract spent box data created below the initial block but spent later
+     * @target initWithRetrial should call init job once
      * @dependencies
+     * - database
      * @scenario
      * - mock extractor
-     * - mock `hasData` to return true for an spent box created below the init block
-     * - spy `extractData`
-     * - run test (call `fetchDataWithOffsetLimit`)
+     * - mock `removeAllData` in database actions
+     * - mock init job
+     * - run test (call `initWithRetrial`)
      * @expected
-     * - to call `extractData` with spent box information
-     * - return one extracted element without spending information
+     * - to remove database old data once
+     * - call init job once
      */
-    it('should extract spent box data created below the initial block but spent later', async () => {
-      const extractor = new MockedInitializableErgoExtractor();
-      const spentBox = ergoBoxes[1];
-      extractor.hasData = (box: OutputBox) => {
-        if (box.boxId == spentBox.boxId) return true;
-        return false;
-      };
-      const extractSpy = vitest.fn().mockReturnValue(extractedData);
-      extractor.extractBoxData = extractSpy;
-      const data = await extractor.fetchDataWithOffsetLimit(1252678, 0, 2);
-      expect(extractSpy).toHaveBeenCalledWith(
-        spentBox,
-        spentBox.blockId,
-        spentBox.inclusionHeight
+    it('should call init job once', async () => {
+      const extractor = new MockedInitializableErgoExtractor(
+        ErgoNetworkType.Node,
+        'node_url',
+        'address'
       );
-      expect(data).toEqual({
-        extractedBoxes: [
-          { ...extractedData, spendBlock: undefined, spendHeight: undefined },
-        ],
-        hasNextBatch: true,
-      });
+      const removeSpy = vitest.fn();
+      extractor['actions'] = {
+        removeAllData: removeSpy,
+      } as unknown as AbstractInitializableErgoExtractorAction<ErgoExtractedData>;
+      const initSpy = vitest.fn();
+      await extractor['initWithRetrial'](initSpy);
+      expect(removeSpy).toHaveBeenCalledOnce();
+      expect(initSpy).toHaveBeenCalledOnce();
     });
-  });
 
-  describe('initializeBoxes', () => {
     /**
-     * @target initializeBoxes should remove all data and reinsert the newly extracted data
+     * @target initWithRetrial should call init job RETRIAL_COUNT number of times
      * @dependencies
+     * - database
      * @scenario
      * - mock extractor
-     * - spy database functions
-     * - mock `fetchDataWithOffsetLimit` to return one extracted data
-     * - run test (call `initializeBoxes`)
+     * - mock `removeAllData` in database actions
+     * - mock init job to throw error
+     * - run test (call `initWithRetrial`)
      * @expected
-     * - to call removeAllData once
-     * - to call insertBoxes once with extracted data
+     * - to remove database old data once
+     * - call init job RETRIAL_COUNT number of times
+     * - to throw error after trials
      */
-    it('should remove all data and reinsert the newly extracted data', async () => {
-      const extractor = new MockedInitializableErgoExtractor();
+    it('should call init job RETRIAL_COUNT number of times', async () => {
+      const extractor = new MockedInitializableErgoExtractor(
+        ErgoNetworkType.Node,
+        'node_url',
+        'address'
+      );
       const removeSpy = vitest.fn();
-      const insertSpy = vitest.fn().mockResolvedValue(true);
       extractor['actions'] = {
         removeAllData: removeSpy,
-        insertBoxes: insertSpy,
       } as unknown as AbstractInitializableErgoExtractorAction<ErgoExtractedData>;
-      extractor.fetchDataWithOffsetLimit = vitest.fn().mockResolvedValue({
-        extractedBoxes: [extractedData],
-        hasNextBatch: false,
-      });
+      const initSpy = vitest.fn().mockRejectedValue(0);
+      await expect(
+        async () => await extractor['initWithRetrial'](initSpy)
+      ).rejects.toThrowError();
+      expect(removeSpy).toHaveBeenCalledOnce();
+      expect(initSpy).toHaveBeenCalledTimes(RETRIAL_COUNT);
+    });
+  });
 
-      await extractor.initializeBoxes({ height: 100, hash: 'hash' });
-      expect(removeSpy).toBeCalledTimes(1);
-      expect(insertSpy).toBeCalledWith([extractedData], 'Test');
+  describe('initializeWithNode', () => {
+    /**
+     * @target initializeWithNode should process all transactions bellow the init height twice
+     * @dependencies
+     * - node network
+     * @scenario
+     * - mock extractor
+     * - mock `getAddressTransactionsWithOffsetLimit` in node network
+     * - mock `initWithRetrial` to run the job once
+     * - mock `getTotalTxCount` to return 3
+     * - run test (call `initializeWithNode`)
+     * @expected
+     * - to filter the transactions and process the txs bellow the init height
+     * - to process all filtered transactions twice
+     */
+    it('should process all transactions bellow the init height twice', async () => {
+      const extractor = new MockedInitializableErgoExtractor(
+        ErgoNetworkType.Node,
+        'node_url',
+        'address'
+      );
+      (
+        extractor['network'] as NodeNetwork
+      ).getAddressTransactionsWithOffsetLimit = vitest.fn().mockResolvedValue({
+        items: transactionBatch,
+        total: 3,
+      });
+      extractor['initWithRetrial'] = async (job: () => Promise<void>) => job();
+      extractor['getTotalTxCount'] = async () => 3;
+      const processSpy = vitest.fn();
+      extractor['processTransactionBatch'] = processSpy;
+      await extractor['initializeWithNode']({ hash: 'hash', height: 1320698 });
+      expect(processSpy).toBeCalledTimes(2);
+      expect(processSpy).toBeCalledWith([transactionBatch[2]]);
     });
 
     /**
-     * @target initializeBoxes should iterate on api when data count is more than api limit
+     * @target initializeWithNode should throw error when total transaction count changes
      * @dependencies
+     * - node network
      * @scenario
      * - mock extractor
-     * - spy database functions
-     * - mock `fetchDataWithOffsetLimit` to return total number 101 (greater than API_LIMIT)
-     * - run test (call `initializeBoxes`)
+     * - mock `getAddressTransactionsWithOffsetLimit` in node network
+     * - mock `initWithRetrial` to run the job once
+     * - mock `getTotalTxCount` to return 3 then 4
+     * - run test (call `initializeWithNode`)
      * @expected
-     * - to call removeAllData once
-     * - to call insertBoxes twice
+     * - to throw error when the number of transactions change during initialization
      */
-    it('should iterate on api when data count is more than api limit', async () => {
-      const extractor = new MockedInitializableErgoExtractor();
-      const removeSpy = vitest.fn();
-      const insertSpy = vitest.fn().mockResolvedValue(true);
-      extractor['actions'] = {
-        removeAllData: removeSpy,
-        insertBoxes: insertSpy,
-      } as unknown as AbstractInitializableErgoExtractorAction<ErgoExtractedData>;
-      extractor.fetchDataWithOffsetLimit = vitest
+    it('should throw error when total transaction count changes', async () => {
+      const extractor = new MockedInitializableErgoExtractor(
+        ErgoNetworkType.Node,
+        'node_url',
+        'address'
+      );
+      (
+        extractor['network'] as NodeNetwork
+      ).getAddressTransactionsWithOffsetLimit = vitest.fn().mockResolvedValue({
+        items: transactionBatch,
+        total: 3,
+      });
+      extractor['initWithRetrial'] = async (job: () => Promise<void>) => job();
+      extractor['getTotalTxCount'] = vitest
         .fn()
-        .mockResolvedValueOnce({
-          extractedBoxes: [extractedData],
-          hasNextBatch: true,
-        })
-        .mockResolvedValueOnce({ extractedBoxes: [], hasNextBatch: false });
-
-      await extractor.initializeBoxes({ height: 100, hash: 'hash' });
-      expect(removeSpy).toBeCalledTimes(1);
-      expect(insertSpy).toBeCalledTimes(2);
+        .mockResolvedValueOnce(3)
+        .mockResolvedValue(4);
+      extractor['processTransactionBatch'] = vitest.fn();
+      expect(() =>
+        extractor['initializeWithNode']({ hash: 'hash', height: 1320698 })
+      ).rejects.toThrowError();
     });
+  });
 
+  describe('initializeWithExplorer', () => {
     /**
-     * @target initializeBoxes should not run initialization when initialize flag is false
+     * @target initializeWithExplorer should limit the height and process
+     * transactions when they are less than the API_LIMIT
      * @dependencies
+     * - explorer network
      * @scenario
      * - mock extractor
-     * - spy database functions
-     * - run test (call `initializeBoxes`)
+     * - mock `initWithRetrial` to run the job once
+     * - mock `getAddressTransactionsWithHeight` in explorer network
+     *    return API_LIMIT transactions in first call (to limit the height range)
+     *    return 0 transactions in second call (to pass a range without process)
+     *    return transactionBatch in last call (to process the transactions when they are less than API_LIMIT)
+     * - run test (call `initializeWithExplorer`)
      * @expected
-     * - not to call any functions
+     * - to limit the height when the transaction count is equal to API_LIMIT
+     * - to cover all height ranges
+     * - to process transactions when they are less than API_LIMIT
      */
-    it('should not run initialization when initialize flag is false', async () => {
-      const extractor = new MockedInitializableErgoExtractor(false);
-      const removeSpy = vitest.fn();
-      const insertSpy = vitest.fn();
-      extractor['actions'] = {
-        removeAllData: removeSpy,
-        insertBoxes: insertSpy,
-      } as unknown as AbstractInitializableErgoExtractorAction<ErgoExtractedData>;
+    it(`should limit the height and process transactions when they are less than
+      the API_LIMIT`, async () => {
+      const extractor = new MockedInitializableErgoExtractor(
+        ErgoNetworkType.Explorer,
+        'explorer_url',
+        'address'
+      );
+      extractor['initWithRetrial'] = async (job: () => Promise<void>) => job();
+      const addressTxSpy = vitest
+        .fn()
+        .mockResolvedValueOnce(new Array(100).fill(transactionBatch[0]))
+        .mockResolvedValueOnce([])
+        .mockResolvedValueOnce(transactionBatch);
+      (
+        extractor['network'] as ExplorerNetwork
+      ).getAddressTransactionsWithHeight = addressTxSpy;
+      const processSpy = vitest.fn();
+      extractor['processTransactionBatch'] = processSpy;
+      await extractor['initializeWithExplorer']({
+        hash: 'hash',
+        height: 1320800,
+      });
+      expect(addressTxSpy).toHaveBeenCalledWith('address', 0, 1320800);
+      expect(addressTxSpy).toHaveBeenCalledWith('address', 0, 660400);
+      expect(addressTxSpy).toHaveBeenLastCalledWith('address', 660401, 1320800);
+      expect(processSpy).toHaveBeenCalledTimes(1);
+      expect(processSpy).toHaveBeenCalledWith(transactionBatch);
+    });
 
-      await extractor.initializeBoxes({ height: 100, hash: 'hash' });
-      expect(removeSpy).not.toBeCalled();
-      expect(insertSpy).not.toBeCalled();
+    /**
+     * @target initializeWithExplorer should process all block transactions when
+     * the number of transactions in the block is more than API_LIMIT
+     * @dependencies
+     * - explorer network
+     * @scenario
+     * - mock extractor
+     * - mock `initWithRetrial` to run the job once
+     * - mock `getAddressTransactionsWithHeight` to return 100 transactions in one block
+     * - spy all other functions to check the calls
+     * - run test (call `initializeWithExplorer`)
+     * @expected
+     * - to process all transactions in block 1320000
+     * - not to stuck at a large block and complete the process
+     */
+    it(`should process all block transactions when the number of transactions in
+      the block is more than API_LIMIT`, async () => {
+      // mock extractor
+      const extractor = new MockedInitializableErgoExtractor(
+        ErgoNetworkType.Explorer,
+        'explorer_url',
+        'address'
+      );
+      // mock `initWithRetrial` to run the job once
+      extractor['initWithRetrial'] = async (job: () => Promise<void>) => job();
+      // mock `getAddressTransactionsWithHeight` to return 100 transactions in one block
+      const exNetwork = extractor['network'] as ExplorerNetwork;
+      const addressTxSpy = vitest
+        .fn()
+        .mockImplementation(
+          (address: string, fromHeight: number, toHeight: number) => {
+            if (fromHeight <= 1320000 && toHeight >= 1320000)
+              return new Array(100).fill(transactionBatch[0]);
+            return [];
+          }
+        );
+      exNetwork.getAddressTransactionsWithHeight = addressTxSpy;
+      // spy all other functions to check the calls
+      const blockIdSpy = vi.fn().mockResolvedValue('blockId');
+      const blockTxsSpy = vi.fn().mockResolvedValue([]);
+      exNetwork.getBlockIdAtHeight = blockIdSpy;
+      exNetwork.getBlockTxs = blockTxsSpy;
+      const processSpy = vitest.fn();
+      const processBatchSpy = vitest.fn();
+      extractor.processTransactions = processSpy;
+      extractor['processTransactionBatch'] = processBatchSpy;
+      // run test (call `initializeWithExplorer`)
+      await extractor['initializeWithExplorer']({
+        hash: 'hash',
+        height: 1320800,
+      });
+      expect(blockIdSpy).toHaveBeenCalledWith(1320000);
+      expect(blockTxsSpy).toHaveBeenCalledWith('blockId');
+      expect(processSpy).toHaveBeenCalledWith([], {
+        hash: 'blockId',
+        height: 1320000,
+      });
+      expect(processBatchSpy).not.toBeCalled();
     });
   });
 });