@subql/node-ethereum 0.4.1-8 → 0.4.1-storeCache-0.0.3

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js

@@ -10,23 +10,23 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkerBlockDispatcherService = void 0;
-const assert_1 = __importDefault(require("assert"));
 const path_1 = __importDefault(require("path"));
 const common_1 = require("@nestjs/common");
 const event_emitter_1 = require("@nestjs/event-emitter");
-const schedule_1 = require("@nestjs/schedule");
 const node_core_1 = require("@subql/node-core");
-const chalk_1 = __importDefault(require("chalk"));
-const lodash_1 = require("lodash");
-const project_service_1 = require("../project.service");
-const base_block_dispatcher_1 = require("./base-block-dispatcher");
+const SubqueryProject_1 = require("../../configure/SubqueryProject");
+const dynamic_ds_service_1 = require("../dynamic-ds.service");
+const unfinalizedBlocks_service_1 = require("../unfinalizedBlocks.service");
 const logger = (0, node_core_1.getLogger)('WorkerBlockDispatcherService');
-async function createIndexerWorker() {
+async function createIndexerWorker(store, dynamicDsService, unfinalizedBlocksService) {
     const indexerWorker = node_core_1.Worker.create(path_1.default.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), [
         'initWorker',
         'processBlock',
@@ -34,142 +34,56 @@ async function createIndexerWorker() {
         'numFetchedBlocks',
         'numFetchingBlocks',
         'getStatus',
-        'reloadDynamicDs',
-    ]);
+        'getMemoryLeft',
+        'waitForWorkerBatchSize',
+    ], {
+        storeCount: store.count.bind(store),
+        storeGet: store.get.bind(store),
+        storeGetByField: store.getByField.bind(store),
+        storeGetOneByField: store.getOneByField.bind(store),
+        storeSet: store.set.bind(store),
+        storeBulkCreate: store.bulkCreate.bind(store),
+        storeBulkUpdate: store.bulkUpdate.bind(store),
+        storeRemove: store.remove.bind(store),
+        dynamicDsCreateDynamicDatasource: dynamicDsService.createDynamicDatasource.bind(dynamicDsService),
+        dynamicDsGetDynamicDatasources: dynamicDsService.getDynamicDatasources.bind(dynamicDsService),
+        unfinalizedBlocksProcess: unfinalizedBlocksService.processUnfinalizedBlocks.bind(unfinalizedBlocksService),
+    });
     await indexerWorker.initWorker();
     return indexerWorker;
 }
-let WorkerBlockDispatcherService = class WorkerBlockDispatcherService extends base_block_dispatcher_1.BaseBlockDispatcher {
-    constructor(nodeConfig, eventEmitter, projectService) {
-        const numWorkers = nodeConfig.workers;
-        super(nodeConfig, eventEmitter, projectService, new node_core_1.AutoQueue(numWorkers * nodeConfig.batchSize * 2));
-        this.taskCounter = 0;
-        this.isShutdown = false;
-        this.numWorkers = numWorkers;
-    }
-    async init(onDynamicDsCreated) {
-        if (this.nodeConfig.unfinalizedBlocks) {
-            throw new Error('Sorry, best block feature is not supported with workers yet.');
-        }
-        this.workers = await Promise.all(new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()));
-        this.onDynamicDsCreated = onDynamicDsCreated;
-        const blockAmount = await this.projectService.getProcessedBlockCount();
-        this.setProcessedBlockCount(blockAmount !== null && blockAmount !== void 0 ? blockAmount : 0);
-    }
-    async onApplicationShutdown() {
-        this.isShutdown = true;
-        // Stop processing blocks
-        this.queue.abort();
-        // Stop all workers
-        if (this.workers) {
-            await Promise.all(this.workers.map((w) => w.terminate()));
-        }
-    }
-    enqueueBlocks(heights, latestBufferHeight) {
-        if (!heights.length)
-            return;
-        logger.info(`Enqueing blocks [${heights[0]}...${(0, lodash_1.last)(heights)}], total ${heights.length} blocks`);
-        // eslint-disable-next-line no-constant-condition
-        if (true) {
-            /*
-             * Load balancing:
-             * worker1: 1,2,3
-             * worker2: 4,5,6
-             */
-            const workerIdx = this.getNextWorkerIndex();
-            heights.map((height) => this.enqueueBlock(height, workerIdx));
-        }
-        else {
-            /*
-             * Load balancing:
-             * worker1: 1,3,5
-             * worker2: 2,4,6
-             */
-            heights.map((height) => this.enqueueBlock(height, this.getNextWorkerIndex()));
-        }
-        this.latestBufferedHeight = latestBufferHeight !== null && latestBufferHeight !== void 0 ? latestBufferHeight : (0, lodash_1.last)(heights);
-    }
-    enqueueBlock(height, workerIdx) {
-        if (this.isShutdown)
-            return;
-        const worker = this.workers[workerIdx];
-        (0, assert_1.default)(worker, `Worker ${workerIdx} not found`);
-        // Used to compare before and after as a way to check if queue was flushed
-        const bufferedHeight = this.latestBufferedHeight;
-        const pendingBlock = worker.fetchBlock(height);
-        const processBlock = async () => {
-            var _a;
-            try {
-                const start = new Date();
-                const result = await pendingBlock;
-                const end = new Date();
-                if (bufferedHeight > this.latestBufferedHeight) {
-                    logger.debug(`Queue was reset for new DS, discarding fetched blocks`);
-                    return;
-                }
-                const waitTime = end.getTime() - start.getTime();
-                if (waitTime > 1000) {
-                    logger.info(`Waiting to fetch block ${height}: ${chalk_1.default.red(`${waitTime}ms`)}`);
-                }
-                else if (waitTime > 200) {
-                    logger.info(`Waiting to fetch block ${height}: ${chalk_1.default.yellow(`${waitTime}ms`)}`);
-                }
-                // logger.info(
-                //   `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,
-                // );
-                this.preProcessBlock(height);
-                const { dynamicDsCreated, operationHash, reindexBlockHeight } = await worker.processBlock(height);
-                await this.postProcessBlock(height, {
-                    dynamicDsCreated,
-                    operationHash: Buffer.from(operationHash, 'base64'),
-                    reindexBlockHeight,
-                });
-                if (dynamicDsCreated) {
-                    // Ensure all workers are aware of all dynamic ds
-                    await Promise.all(this.workers.map((w) => w.reloadDynamicDs()));
-                }
-            }
-            catch (e) {
-                logger.error(e, `failed to index block at height ${height} ${e.handler ? `${e.handler}(${(_a = e.stack) !== null && _a !== void 0 ? _a : ''})` : ''}`);
-                process.exit(1);
-            }
-        };
-        void this.queue.put(processBlock);
-    }
-    async sampleWorkerStatus() {
-        for (const worker of this.workers) {
-            const status = await worker.getStatus();
-            logger.info(JSON.stringify(status));
-        }
-    }
-    // Getter doesn't seem to cary from abstract class
-    get latestBufferedHeight() {
-        return this._latestBufferedHeight;
-    }
-    set latestBufferedHeight(height) {
-        super.latestBufferedHeight = height;
-        // There is only a single queue with workers so we treat them as the same
-        this.eventEmitter.emit(node_core_1.IndexerEvent.BlockQueueSize, {
-            value: this.queueSize,
-        });
+let WorkerBlockDispatcherService = class WorkerBlockDispatcherService extends node_core_1.WorkerBlockDispatcher {
+    constructor(nodeConfig, eventEmitter, projectService, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService, unfinalizedBlocksSevice) {
+        super(nodeConfig, eventEmitter, projectService, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService, () => createIndexerWorker(storeService.getStore(), dynamicDsService, unfinalizedBlocksSevice));
     }
-    getNextWorkerIndex() {
-        const index = this.taskCounter % this.numWorkers;
-        this.taskCounter++;
-        return index;
+    async fetchBlock(worker, height) {
+        const start = new Date();
+        await worker.fetchBlock(height);
+        const end = new Date();
+        // const waitTime = end.getTime() - start.getTime();
+        // if (waitTime > 1000) {
+        //   logger.info(
+        //     `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,
+        //   );
+        // } else if (waitTime > 200) {
+        //   logger.info(
+        //     `Waiting to fetch block ${height}: ${chalk.yellow(`${waitTime}ms`)}`,
+        //   );
+        // }
     }
 };
-__decorate([
-    (0, schedule_1.Interval)(15000),
-    __metadata("design:type", Function),
-    __metadata("design:paramtypes", []),
-    __metadata("design:returntype", Promise)
-], WorkerBlockDispatcherService.prototype, "sampleWorkerStatus", null);
 WorkerBlockDispatcherService = __decorate([
     (0, common_1.Injectable)(),
+    __param(2, (0, common_1.Inject)('IProjectService')),
+    __param(7, (0, common_1.Inject)('ISubqueryProject')),
     __metadata("design:paramtypes", [node_core_1.NodeConfig,
-        event_emitter_1.EventEmitter2,
-        project_service_1.ProjectService])
+        event_emitter_1.EventEmitter2, Object, node_core_1.SmartBatchService,
+        node_core_1.StoreService,
+        node_core_1.StoreCacheService,
+        node_core_1.PoiService,
+        SubqueryProject_1.SubqueryProject,
+        dynamic_ds_service_1.DynamicDsService,
+        unfinalizedBlocks_service_1.UnfinalizedBlocksService])
 ], WorkerBlockDispatcherService);
 exports.WorkerBlockDispatcherService = WorkerBlockDispatcherService;
 //# sourceMappingURL=worker-block-dispatcher.service.js.map

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAAmE;AACnE,yDAAsD;AACtD,+CAA4C;AAC5C,gDAM0B;AAC1B,kDAA0B;AAC1B,mCAA8B;AAC9B,wDAAoD;AAUpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAmBzD,KAAK,UAAU,mBAAmB;IAChC,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CACjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,iBAAiB;KAClB,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,2CAAoC;IAS5C,YACE,UAAsB,EACtB,YAA2B,EAC3B,cAA8B;QAE9B,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;QACtC,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,qBAAS,CAAC,UAAU,GAAG,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACrD,CAAC;QAdI,gBAAW,GAAG,CAAC,CAAC;QAChB,eAAU,GAAG,KAAK,CAAC;QAczB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;SACH;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,yBAAyB;QACzB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAEnB,mBAAmB;QACnB,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SAC3D;IACH,CAAC;IAED,aAAa,CAAC,OAAiB,EAAE,kBAA2B;QAC1D,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QAC5B,MAAM,CAAC,IAAI,CACT,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,YAC/C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,iDAAiD;QACjD,IAAI,IAAI,EAAE;YACR;;;;eAIG;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;SAC/D;aAAM;YACL;;;;eAIG;YACH,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CACrD,CAAC;SACH;QAED,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;IAClE,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,SAAiB;QACpD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,IAAA,gBAAM,EAAC,MAAM,EAAE,UAAU,SAAS,YAAY,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC;QACjD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;;YAC9B,IAAI;gBACF,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,IAAI,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE;oBAC9C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO;iBACR;gBAED,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjD,IAAI,QAAQ,GAAG,IAAI,EAAE;oBACnB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,CAAC,EAAE,CAClE,CAAC;iBACH;qBAAM,IAAI,QAAQ,GAAG,GAAG,EAAE;oBACzB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,MAAM,CAC/C,GAAG,QAAQ,IAAI,CAChB,EAAE,CACJ,CAAC;iBACH;gBAED,eAAe;gBACf,gKAAgK;gBAChK,KAAK;gBAEL,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAE7B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,kBAAkB,EAAE,GAC3D,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE;oBAClC,gBAAgB;oBAChB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;oBACnD,kBAAkB;iBACnB,CAAC,CAAC;gBAEH,IAAI,gBAAgB,EAAE;oBACpB,iDAAiD;oBACjD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;iBACjE;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC,CAAC;QAEF,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAGK,AAAN,KAAK,CAAC,kBAAkB;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,oBAAoB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAc;QACrC,KAAK,CAAC,oBAAoB,GAAG,MAAM,CAAC;QACpC,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;YAClD,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAA;AA3BO;IADL,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sEAMf;AA/JU,4BAA4B;IADxC,IAAA,mBAAU,GAAE;qCAYG,sBAAU;QACR,6BAAa;QACX,gCAAc;GAbrB,4BAA4B,CAqLxC;AArLY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  getLogger,\n  NodeConfig,\n  IndexerEvent,\n  Worker,\n  AutoQueue,\n} from '@subql/node-core';\nimport chalk from 'chalk';\nimport { last } from 'lodash';\nimport { ProjectService } from '../project.service';\nimport {\n  FetchBlock,\n  ProcessBlock,\n  InitWorker,\n  NumFetchedBlocks,\n  NumFetchingBlocks,\n  GetWorkerStatus,\n  ReloadDynamicDs,\n} from '../worker/worker';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IIndexerWorker = {\n  processBlock: ProcessBlock;\n  fetchBlock: FetchBlock;\n  numFetchedBlocks: NumFetchedBlocks;\n  numFetchingBlocks: NumFetchingBlocks;\n  getStatus: GetWorkerStatus;\n  reloadDynamicDs: ReloadDynamicDs;\n};\n\ntype IInitIndexerWorker = IIndexerWorker & {\n  initWorker: InitWorker;\n};\n\ntype IndexerWorker = IIndexerWorker & {\n  terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(): Promise<IndexerWorker> {\n  const indexerWorker = Worker.create<IInitIndexerWorker>(\n    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n    [\n      'initWorker',\n      'processBlock',\n      'fetchBlock',\n      'numFetchedBlocks',\n      'numFetchingBlocks',\n      'getStatus',\n      'reloadDynamicDs',\n    ],\n  );\n\n  await indexerWorker.initWorker();\n\n  return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n  extends BaseBlockDispatcher<AutoQueue<void>>\n  implements OnApplicationShutdown\n{\n  private workers: IndexerWorker[];\n  private numWorkers: number;\n\n  private taskCounter = 0;\n  private isShutdown = false;\n\n  constructor(\n    nodeConfig: NodeConfig,\n    eventEmitter: EventEmitter2,\n    projectService: ProjectService,\n  ) {\n    const numWorkers = nodeConfig.workers;\n    super(\n      nodeConfig,\n      eventEmitter,\n      projectService,\n      new AutoQueue(numWorkers * nodeConfig.batchSize * 2),\n    );\n    this.numWorkers = numWorkers;\n  }\n\n  async init(\n    onDynamicDsCreated: (height: number) => Promise<void>,\n  ): Promise<void> {\n    if (this.nodeConfig.unfinalizedBlocks) {\n      throw new Error(\n        'Sorry, best block feature is not supported with workers yet.',\n      );\n    }\n\n    this.workers = await Promise.all(\n      new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()),\n    );\n\n    this.onDynamicDsCreated = onDynamicDsCreated;\n\n    const blockAmount = await this.projectService.getProcessedBlockCount();\n    this.setProcessedBlockCount(blockAmount ?? 0);\n  }\n\n  async onApplicationShutdown(): Promise<void> {\n    this.isShutdown = true;\n    // Stop processing blocks\n    this.queue.abort();\n\n    // Stop all workers\n    if (this.workers) {\n      await Promise.all(this.workers.map((w) => w.terminate()));\n    }\n  }\n\n  enqueueBlocks(heights: number[], latestBufferHeight?: number): void {\n    if (!heights.length) return;\n    logger.info(\n      `Enqueing blocks [${heights[0]}...${last(heights)}], total ${\n        heights.length\n      } blocks`,\n    );\n\n    // eslint-disable-next-line no-constant-condition\n    if (true) {\n      /*\n       * Load balancing:\n       * worker1: 1,2,3\n       * worker2: 4,5,6\n       */\n      const workerIdx = this.getNextWorkerIndex();\n      heights.map((height) => this.enqueueBlock(height, workerIdx));\n    } else {\n      /*\n       * Load balancing:\n       * worker1: 1,3,5\n       * worker2: 2,4,6\n       */\n      heights.map((height) =>\n        this.enqueueBlock(height, this.getNextWorkerIndex()),\n      );\n    }\n\n    this.latestBufferedHeight = latestBufferHeight ?? last(heights);\n  }\n\n  private enqueueBlock(height: number, workerIdx: number) {\n    if (this.isShutdown) return;\n    const worker = this.workers[workerIdx];\n\n    assert(worker, `Worker ${workerIdx} not found`);\n\n    // Used to compare before and after as a way to check if queue was flushed\n    const bufferedHeight = this.latestBufferedHeight;\n    const pendingBlock = worker.fetchBlock(height);\n\n    const processBlock = async () => {\n      try {\n        const start = new Date();\n        const result = await pendingBlock;\n        const end = new Date();\n\n        if (bufferedHeight > this.latestBufferedHeight) {\n          logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n          return;\n        }\n\n        const waitTime = end.getTime() - start.getTime();\n        if (waitTime > 1000) {\n          logger.info(\n            `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n          );\n        } else if (waitTime > 200) {\n          logger.info(\n            `Waiting to fetch block ${height}: ${chalk.yellow(\n              `${waitTime}ms`,\n            )}`,\n          );\n        }\n\n        // logger.info(\n        //   `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,\n        // );\n\n        this.preProcessBlock(height);\n\n        const { dynamicDsCreated, operationHash, reindexBlockHeight } =\n          await worker.processBlock(height);\n\n        await this.postProcessBlock(height, {\n          dynamicDsCreated,\n          operationHash: Buffer.from(operationHash, 'base64'),\n          reindexBlockHeight,\n        });\n\n        if (dynamicDsCreated) {\n          // Ensure all workers are aware of all dynamic ds\n          await Promise.all(this.workers.map((w) => w.reloadDynamicDs()));\n        }\n      } catch (e) {\n        logger.error(\n          e,\n          `failed to index block at height ${height} ${\n            e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n          }`,\n        );\n        process.exit(1);\n      }\n    };\n\n    void this.queue.put(processBlock);\n  }\n\n  @Interval(15000)\n  async sampleWorkerStatus(): Promise<void> {\n    for (const worker of this.workers) {\n      const status = await worker.getStatus();\n      logger.info(JSON.stringify(status));\n    }\n  }\n\n  // Getter doesn't seem to cary from abstract class\n  get latestBufferedHeight(): number {\n    return this._latestBufferedHeight;\n  }\n\n  set latestBufferedHeight(height: number) {\n    super.latestBufferedHeight = height;\n    // There is only a single queue with workers so we treat them as the same\n    this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n      value: this.queueSize,\n    });\n  }\n\n  private getNextWorkerIndex(): number {\n    const index = this.taskCounter % this.numWorkers;\n\n    this.taskCounter++;\n\n    return index;\n  }\n}\n"]}
+{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,gDAAwB;AACxB,2CAA2E;AAC3E,yDAAsD;AACtD,gDAa0B;AAG1B,qEAGyC;AACzC,8DAAyD;AACzD,4EAGsC;AAItC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAMzD,KAAK,UAAU,mBAAmB,CAChC,KAAY,EACZ,gBAAmD,EACnD,wBAAmD;IAEnD,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CAIjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,eAAe;QACf,wBAAwB;KACzB,EACD;QACE,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;QACnC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7C,kBAAkB,EAAE,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC;QACnD,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7C,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7C,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QACrC,gCAAgC,EAC9B,gBAAgB,CAAC,uBAAuB,CAAC,IAAI,CAAC,gBAAgB,CAAC;QACjE,8BAA8B,EAC5B,gBAAgB,CAAC,qBAAqB,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAC/D,wBAAwB,EACtB,wBAAwB,CAAC,wBAAwB,CAAC,IAAI,CACpD,wBAAwB,CACzB;KACJ,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,iCAAoD;IAG5D,YACE,UAAsB,EACtB,YAA2B,EACA,cAA+B,EAC1D,iBAAoC,EACpC,YAA0B,EAC1B,iBAAoC,EACpC,UAAsB,EACM,OAAwB,EACpD,gBAAkC,EAClC,uBAAiD;QAEjD,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,GAAG,EAAE,CACH,mBAAmB,CACjB,YAAY,CAAC,QAAQ,EAAE,EACvB,gBAAgB,EAChB,uBAAuB,CACxB,CACJ,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,UAAU,CACxB,MAAqB,EACrB,MAAc;QAEd,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,oDAAoD;QACpD,yBAAyB;QACzB,iBAAiB;QACjB,yEAAyE;QACzE,OAAO;QACP,+BAA+B;QAC/B,iBAAiB;QACjB,4EAA4E;QAC5E,OAAO;QACP,IAAI;IACN,CAAC;CACF,CAAA;AAtDY,4BAA4B;IADxC,IAAA,mBAAU,GAAE;IAQR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;IAKzB,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAPf,sBAAU;QACR,6BAAa,UAER,6BAAiB;QACtB,wBAAY;QACP,6BAAiB;QACxB,sBAAU;QACe,iCAAe;QAClC,qCAAgB;QACT,oDAAwB;GAdxC,4BAA4B,CAsDxC;AAtDY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport path from 'path';\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  getLogger,\n  NodeConfig,\n  Worker,\n  SmartBatchService,\n  StoreService,\n  PoiService,\n  StoreCacheService,\n  IProjectService,\n  IDynamicDsService,\n  HostStore,\n  HostDynamicDS,\n  WorkerBlockDispatcher,\n} from '@subql/node-core';\nimport { Store } from '@subql/types';\nimport chalk from 'chalk';\nimport {\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../../configure/SubqueryProject';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport {\n  IUnfinalizedBlocksService,\n  UnfinalizedBlocksService,\n} from '../unfinalizedBlocks.service';\nimport { IIndexerWorker, IInitIndexerWorker } from '../worker/worker';\nimport { HostUnfinalizedBlocks } from '../worker/worker.unfinalizedBlocks.service';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IndexerWorker = IIndexerWorker & {\n  terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(\n  store: Store,\n  dynamicDsService: IDynamicDsService<SubqlProjectDs>,\n  unfinalizedBlocksService: IUnfinalizedBlocksService,\n): Promise<IndexerWorker> {\n  const indexerWorker = Worker.create<\n    IInitIndexerWorker,\n    HostDynamicDS<SubqlProjectDs> & HostStore & HostUnfinalizedBlocks\n  >(\n    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n    [\n      'initWorker',\n      'processBlock',\n      'fetchBlock',\n      'numFetchedBlocks',\n      'numFetchingBlocks',\n      'getStatus',\n      'getMemoryLeft',\n      'waitForWorkerBatchSize',\n    ],\n    {\n      storeCount: store.count.bind(store),\n      storeGet: store.get.bind(store),\n      storeGetByField: store.getByField.bind(store),\n      storeGetOneByField: store.getOneByField.bind(store),\n      storeSet: store.set.bind(store),\n      storeBulkCreate: store.bulkCreate.bind(store),\n      storeBulkUpdate: store.bulkUpdate.bind(store),\n      storeRemove: store.remove.bind(store),\n      dynamicDsCreateDynamicDatasource:\n        dynamicDsService.createDynamicDatasource.bind(dynamicDsService),\n      dynamicDsGetDynamicDatasources:\n        dynamicDsService.getDynamicDatasources.bind(dynamicDsService),\n      unfinalizedBlocksProcess:\n        unfinalizedBlocksService.processUnfinalizedBlocks.bind(\n          unfinalizedBlocksService,\n        ),\n    },\n  );\n\n  await indexerWorker.initWorker();\n\n  return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n  extends WorkerBlockDispatcher<SubqlProjectDs, IndexerWorker>\n  implements OnApplicationShutdown\n{\n  constructor(\n    nodeConfig: NodeConfig,\n    eventEmitter: EventEmitter2,\n    @Inject('IProjectService') projectService: IProjectService,\n    smartBatchService: SmartBatchService,\n    storeService: StoreService,\n    storeCacheService: StoreCacheService,\n    poiService: PoiService,\n    @Inject('ISubqueryProject') project: SubqueryProject,\n    dynamicDsService: DynamicDsService,\n    unfinalizedBlocksSevice: UnfinalizedBlocksService,\n  ) {\n    super(\n      nodeConfig,\n      eventEmitter,\n      projectService,\n      smartBatchService,\n      storeService,\n      storeCacheService,\n      poiService,\n      project,\n      dynamicDsService,\n      () =>\n        createIndexerWorker(\n          storeService.getStore(),\n          dynamicDsService,\n          unfinalizedBlocksSevice,\n        ),\n    );\n  }\n\n  protected async fetchBlock(\n    worker: IndexerWorker,\n    height: number,\n  ): Promise<void> {\n    const start = new Date();\n    await worker.fetchBlock(height);\n    const end = new Date();\n\n    // const waitTime = end.getTime() - start.getTime();\n    // if (waitTime > 1000) {\n    //   logger.info(\n    //     `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n    //   );\n    // } else if (waitTime > 200) {\n    //   logger.info(\n    //     `Waiting to fetch block ${height}: ${chalk.yellow(`${waitTime}ms`)}`,\n    //   );\n    // }\n  }\n}\n"]}

package/dist/indexer/dynamic-ds.service.d.ts

@@ -1,28 +1,9 @@
-import { MetadataRepo } from '@subql/node-core';
-import { Transaction } from 'sequelize/types';
+import { DatasourceParams, DynamicDsService as BaseDynamicDsService } from '@subql/node-core';
 import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';
 import { DsProcessorService } from './ds-processor.service';
-interface DatasourceParams {
-    templateName: string;
-    args?: Record<string, unknown>;
-    startBlock: number;
-}
-export declare class DynamicDsService {
+export declare class DynamicDsService extends BaseDynamicDsService<SubqlProjectDs> {
     private readonly dsProcessorService;
     private readonly project;
-    private metaDataRepo;
-    private tempDsRecords;
     constructor(dsProcessorService: DsProcessorService, project: SubqueryProject);
-    init(metaDataRepo: MetadataRepo): void;
-    private _datasources;
-    resetDynamicDatasource(targetHeight: number, tx: Transaction): Promise<void>;
-    createDynamicDatasource(params: DatasourceParams, tx: Transaction): Promise<SubqlProjectDs>;
-    getDynamicDatasources(): Promise<SubqlProjectDs[]>;
-    reloadDynamicDatasources(): Promise<void>;
-    private loadDynamicDatasources;
-    deleteTempDsRecords(blockHeight: number): void;
-    private getDynamicDatasourceParams;
-    private saveDynamicDatasourceParams;
-    private getDatasource;
+    protected getDatasource(params: DatasourceParams): Promise<SubqlProjectDs>;
 }
-export {};

package/dist/indexer/dynamic-ds.service.js

@@ -13,12 +13,8 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 var __param = (this && this.__param) || function (paramIndex, decorator) {
     return function (target, key) { decorator(target, key, paramIndex); }
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DynamicDsService = void 0;
-const assert_1 = __importDefault(require("assert"));
 const common_1 = require("@nestjs/common");
 const common_ethereum_1 = require("@subql/common-ethereum");
 const node_core_1 = require("@subql/node-core");
@@ -27,98 +23,15 @@ const class_validator_1 = require("class-validator");
 const lodash_1 = require("lodash");
 const SubqueryProject_1 = require("../configure/SubqueryProject");
 const ds_processor_service_1 = require("./ds-processor.service");
-const logger = (0, node_core_1.getLogger)('dynamic-ds');
-const METADATA_KEY = 'dynamicDatasources';
-const TEMP_DS_PREFIX = 'ds_';
-let DynamicDsService = class DynamicDsService {
+let DynamicDsService = class DynamicDsService extends node_core_1.DynamicDsService {
     constructor(dsProcessorService, project) {
+        super();
         this.dsProcessorService = dsProcessorService;
         this.project = project;
     }
-    init(metaDataRepo) {
-        this.metaDataRepo = metaDataRepo;
-    }
-    async resetDynamicDatasource(targetHeight, tx) {
-        const dynamicDs = await this.getDynamicDatasourceParams();
-        if (dynamicDs.length !== 0) {
-            const filteredDs = dynamicDs.filter((ds) => ds.startBlock <= targetHeight);
-            const dsRecords = JSON.stringify(filteredDs);
-            await this.metaDataRepo.upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx });
-        }
-    }
-    async createDynamicDatasource(params, tx) {
-        try {
-            const ds = await this.getDatasource(params);
-            await this.saveDynamicDatasourceParams(params, tx);
-            logger.info(`Created new dynamic datasource from template: "${params.templateName}"`);
-            if (!this._datasources)
-                this._datasources = [];
-            this._datasources.push(ds);
-            return ds;
-        }
-        catch (e) {
-            logger.error(e.message);
-            process.exit(1);
-        }
-    }
-    async getDynamicDatasources() {
-        // Workers should not cache this result in order to keep in sync
-        if (!this._datasources) {
-            this._datasources = await this.loadDynamicDatasources();
-        }
-        return this._datasources;
-    }
-    // This is used to sync between worker threads
-    async reloadDynamicDatasources() {
-        this._datasources = await this.loadDynamicDatasources();
-    }
-    async loadDynamicDatasources() {
-        try {
-            const params = await this.getDynamicDatasourceParams();
-            const dataSources = await Promise.all(params.map((params) => this.getDatasource(params)));
-            logger.info(`Loaded ${dataSources.length} dynamic datasources`);
-            return dataSources;
-        }
-        catch (e) {
-            logger.error(`Unable to get dynamic datasources:\n${e.message}`);
-            process.exit(1);
-        }
-    }
-    deleteTempDsRecords(blockHeight) {
-        // Main thread will not have tempDsRecords with workers
-        if (this.tempDsRecords) {
-            delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];
-        }
-    }
-    async getDynamicDatasourceParams(blockHeight) {
-        var _a, _b, _c;
-        (0, assert_1.default)(this.metaDataRepo, `Model _metadata does not exist`);
-        const record = await this.metaDataRepo.findByPk(METADATA_KEY);
-        let results = [];
-        const metaResults = JSON.parse((_a = record === null || record === void 0 ? void 0 : record.value) !== null && _a !== void 0 ? _a : '[]');
-        if (metaResults.length) {
-            results = [...metaResults];
-        }
-        if (blockHeight !== undefined) {
-            const tempResults = JSON.parse((_c = (_b = this.tempDsRecords) === null || _b === void 0 ? void 0 : _b[TEMP_DS_PREFIX + blockHeight]) !== null && _c !== void 0 ? _c : '[]');
-            if (tempResults.length) {
-                results = (0, lodash_1.unionWith)(results, tempResults, lodash_1.isEqual);
-            }
-        }
-        return results;
-    }
-    async saveDynamicDatasourceParams(dsParams, tx) {
-        const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);
-        (0, assert_1.default)(this.metaDataRepo, `Model _metadata does not exist`);
-        const dsRecords = JSON.stringify([...existing, dsParams]);
-        await this.metaDataRepo
-            .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })
-            .then(() => {
-            this.tempDsRecords = Object.assign(Object.assign({}, this.tempDsRecords), { [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords });
-        });
-    }
     async getDatasource(params) {
-        const template = (0, lodash_1.cloneDeep)(this.project.templates.find((t) => t.name === params.templateName));
+        var _a;
+        const template = (0, lodash_1.cloneDeep)((_a = this.project.templates) === null || _a === void 0 ? void 0 : _a.find((t) => t.name === params.templateName));
         if (!template) {
             throw new Error(`Unable to find matching template in project for name: "${params.templateName}"`);
         }

package/dist/indexer/dynamic-ds.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAE5B,2CAAoD;AACpD,4DAIgC;AAChC,gDAA2D;AAC3D,yDAAiD;AACjD,qDAA+C;AAC/C,mCAAuD;AAEvD,kEAA+E;AAC/E,iEAA4D;AAE5D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,MAAM,YAAY,GAAG,oBAAoB,CAAC;AAC1C,MAAM,cAAc,GAAG,KAAK,CAAC;AAStB,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAI3B,YACmB,kBAAsC,EACV,OAAwB;QADpD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACV,YAAO,GAAP,OAAO,CAAiB;IACpE,CAAC;IAEJ,IAAI,CAAC,YAA0B;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAID,KAAK,CAAC,sBAAsB,CAC1B,YAAoB,EACpB,EAAe;QAEf,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YAC1B,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,YAAY,CACtC,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAC5B,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EACvC,EAAE,WAAW,EAAE,EAAE,EAAE,CACpB,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB,CAC3B,MAAwB,EACxB,EAAe;QAEf,IAAI;YACF,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE5C,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAEnD,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;YAC/C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE3B,OAAO,EAAE,CAAC;SACX;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,gEAAgE;QAChE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;SACzD;QAED,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,IAAI;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;YAEvD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CACnD,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,MAAM,sBAAsB,CAAC,CAAC;YAEhE,OAAO,WAAW,CAAC;SACpB;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,mBAAmB,CAAC,WAAmB;QACrC,uDAAuD;QACvD,IAAI,IAAI,CAAC,aAAa,EAAE;YACtB,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC;SACzD;IACH,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,WAAoB;;QAEpB,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE9D,IAAI,OAAO,GAAuB,EAAE,CAAC;QAErC,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAgB,mCAAI,IAAI,CAClC,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,EAAE;YACtB,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;SAC5B;QAED,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAA,MAAA,IAAI,CAAC,aAAa,0CAAG,cAAc,GAAG,WAAW,CAAC,mCAAI,IAAI,CAC3D,CAAC;YACF,IAAI,WAAW,CAAC,MAAM,EAAE;gBACtB,OAAO,GAAG,IAAA,kBAAS,EAAC,OAAO,EAAE,WAAW,EAAE,gBAAO,CAAC,CAAC;aACpD;SACF;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAA0B,EAC1B,EAAe;QAEf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE5E,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY;aACpB,MAAM,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;aACpE,IAAI,CAAC,GAAG,EAAE;YACT,IAAI,CAAC,aAAa,mCACb,IAAI,CAAC,aAAa,KACrB,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAClD,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAwB;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACnE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,KAAK,CAAC,OAAO,mCACR,KAAK,CAAC,OAAO,GACb,MAAM,CAAC,IAAI,CACf,CAAC;gBAEF,MAAM,QAAQ,GAAG,IAAA,gCAAY,EAC3B,qDAAmC,EACnC,KAAK,CACN,CAAC;gBAEF,MAAM,MAAM,GAAG,IAAA,8BAAY,EAAC,QAAQ,EAAE;oBACpC,SAAS,EAAE,IAAI;oBACf,oBAAoB,EAAE,KAAK;iBAC5B,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,EAAE;oBACjB,MAAM,IAAI,KAAK,CACb,0BAA0B,MAAM;yBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;yBACxB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;iBACH;aACF;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AA/LY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;IAOR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCADU,yCAAkB;QACD,iCAAe;GAN5D,gBAAgB,CA+L5B;AA/LY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  EthereumRuntimeDataSourceV0_3_0Impl,\n  isCustomDs,\n  isRuntimeDs,\n} from '@subql/common-ethereum';\nimport { getLogger, MetadataRepo } from '@subql/node-core';\nimport { plainToClass } from 'class-transformer';\nimport { validateSync } from 'class-validator';\nimport { cloneDeep, isEqual, unionWith } from 'lodash';\nimport { Transaction } from 'sequelize/types';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\nconst logger = getLogger('dynamic-ds');\n\nconst METADATA_KEY = 'dynamicDatasources';\nconst TEMP_DS_PREFIX = 'ds_';\n\ninterface DatasourceParams {\n  templateName: string;\n  args?: Record<string, unknown>;\n  startBlock: number;\n}\n\n@Injectable()\nexport class DynamicDsService {\n  private metaDataRepo: MetadataRepo;\n  private tempDsRecords: Record<string, string>;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n  ) {}\n\n  init(metaDataRepo: MetadataRepo): void {\n    this.metaDataRepo = metaDataRepo;\n  }\n\n  private _datasources: SubqlProjectDs[];\n\n  async resetDynamicDatasource(\n    targetHeight: number,\n    tx: Transaction,\n  ): Promise<void> {\n    const dynamicDs = await this.getDynamicDatasourceParams();\n    if (dynamicDs.length !== 0) {\n      const filteredDs = dynamicDs.filter(\n        (ds) => ds.startBlock <= targetHeight,\n      );\n      const dsRecords = JSON.stringify(filteredDs);\n      await this.metaDataRepo.upsert(\n        { key: METADATA_KEY, value: dsRecords },\n        { transaction: tx },\n      );\n    }\n  }\n\n  async createDynamicDatasource(\n    params: DatasourceParams,\n    tx: Transaction,\n  ): Promise<SubqlProjectDs> {\n    try {\n      const ds = await this.getDatasource(params);\n\n      await this.saveDynamicDatasourceParams(params, tx);\n\n      logger.info(\n        `Created new dynamic datasource from template: \"${params.templateName}\"`,\n      );\n\n      if (!this._datasources) this._datasources = [];\n      this._datasources.push(ds);\n\n      return ds;\n    } catch (e) {\n      logger.error(e.message);\n      process.exit(1);\n    }\n  }\n\n  async getDynamicDatasources(): Promise<SubqlProjectDs[]> {\n    // Workers should not cache this result in order to keep in sync\n    if (!this._datasources) {\n      this._datasources = await this.loadDynamicDatasources();\n    }\n\n    return this._datasources;\n  }\n\n  // This is used to sync between worker threads\n  async reloadDynamicDatasources(): Promise<void> {\n    this._datasources = await this.loadDynamicDatasources();\n  }\n\n  private async loadDynamicDatasources(): Promise<SubqlProjectDs[]> {\n    try {\n      const params = await this.getDynamicDatasourceParams();\n\n      const dataSources = await Promise.all(\n        params.map((params) => this.getDatasource(params)),\n      );\n\n      logger.info(`Loaded ${dataSources.length} dynamic datasources`);\n\n      return dataSources;\n    } catch (e) {\n      logger.error(`Unable to get dynamic datasources:\\n${e.message}`);\n      process.exit(1);\n    }\n  }\n\n  deleteTempDsRecords(blockHeight: number): void {\n    // Main thread will not have tempDsRecords with workers\n    if (this.tempDsRecords) {\n      delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];\n    }\n  }\n\n  private async getDynamicDatasourceParams(\n    blockHeight?: number,\n  ): Promise<DatasourceParams[]> {\n    assert(this.metaDataRepo, `Model _metadata does not exist`);\n    const record = await this.metaDataRepo.findByPk(METADATA_KEY);\n\n    let results: DatasourceParams[] = [];\n\n    const metaResults: DatasourceParams[] = JSON.parse(\n      (record?.value as string) ?? '[]',\n    );\n    if (metaResults.length) {\n      results = [...metaResults];\n    }\n\n    if (blockHeight !== undefined) {\n      const tempResults: DatasourceParams[] = JSON.parse(\n        this.tempDsRecords?.[TEMP_DS_PREFIX + blockHeight] ?? '[]',\n      );\n      if (tempResults.length) {\n        results = unionWith(results, tempResults, isEqual);\n      }\n    }\n\n    return results;\n  }\n\n  private async saveDynamicDatasourceParams(\n    dsParams: DatasourceParams,\n    tx: Transaction,\n  ): Promise<void> {\n    const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);\n\n    assert(this.metaDataRepo, `Model _metadata does not exist`);\n    const dsRecords = JSON.stringify([...existing, dsParams]);\n    await this.metaDataRepo\n      .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })\n      .then(() => {\n        this.tempDsRecords = {\n          ...this.tempDsRecords,\n          [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords,\n        };\n      });\n  }\n\n  private async getDatasource(\n    params: DatasourceParams,\n  ): Promise<SubqlProjectDs> {\n    const template = cloneDeep(\n      this.project.templates.find((t) => t.name === params.templateName),\n    );\n\n    if (!template) {\n      throw new Error(\n        `Unable to find matching template in project for name: \"${params.templateName}\"`,\n      );\n    }\n\n    const dsObj = {\n      ...template,\n      startBlock: params.startBlock,\n    } as SubqlProjectDs;\n    delete dsObj.name;\n    try {\n      if (isCustomDs(dsObj)) {\n        dsObj.processor.options = {\n          ...dsObj.processor.options,\n          ...params.args,\n        };\n        await this.dsProcessorService.validateCustomDs([dsObj]);\n      } else if (isRuntimeDs(dsObj)) {\n        dsObj.options = {\n          ...dsObj.options,\n          ...params.args,\n        };\n\n        const parsedDs = plainToClass(\n          EthereumRuntimeDataSourceV0_3_0Impl,\n          dsObj,\n        );\n\n        const errors = validateSync(parsedDs, {\n          whitelist: true,\n          forbidNonWhitelisted: false,\n        });\n        if (errors.length) {\n          throw new Error(\n            `Dynamic ds is invalid\\n${errors\n              .map((e) => e.toString())\n              .join('\\n')}`,\n          );\n        }\n      }\n      return dsObj;\n    } catch (e) {\n      throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n    }\n  }\n}\n"]}
+{"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,4DAIgC;AAChC,gDAG0B;AAC1B,yDAAiD;AACjD,qDAA+C;AAC/C,mCAAmC;AACnC,kEAA+E;AAC/E,iEAA4D;AAGrD,IAAM,gBAAgB,GAAtB,MAAM,gBAAiB,SAAQ,4BAAoC;IACxE,YACmB,kBAAsC,EACV,OAAwB;QAErE,KAAK,EAAE,CAAC;QAHS,uBAAkB,GAAlB,kBAAkB,CAAoB;QACV,YAAO,GAAP,OAAO,CAAiB;IAGvE,CAAC;IAES,KAAK,CAAC,aAAa,CAC3B,MAAwB;;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,MAAA,IAAI,CAAC,OAAO,CAAC,SAAS,0CAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,KAAK,CAAC,OAAO,mCACR,KAAK,CAAC,OAAO,GACb,MAAM,CAAC,IAAI,CACf,CAAC;gBAEF,MAAM,QAAQ,GAAG,IAAA,gCAAY,EAC3B,qDAAmC,EACnC,KAAK,CACN,CAAC;gBAEF,MAAM,MAAM,GAAG,IAAA,8BAAY,EAAC,QAAQ,EAAE;oBACpC,SAAS,EAAE,IAAI;oBACf,oBAAoB,EAAE,KAAK;iBAC5B,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,EAAE;oBACjB,MAAM,IAAI,KAAK,CACb,0BAA0B,MAAM;yBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;yBACxB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;iBACH;aACF;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AA7DY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;IAIR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCADU,yCAAkB;QACD,iCAAe;GAH5D,gBAAgB,CA6D5B;AA7DY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  EthereumRuntimeDataSourceV0_3_0Impl,\n  isCustomDs,\n  isRuntimeDs,\n} from '@subql/common-ethereum';\nimport {\n  DatasourceParams,\n  DynamicDsService as BaseDynamicDsService,\n} from '@subql/node-core';\nimport { plainToClass } from 'class-transformer';\nimport { validateSync } from 'class-validator';\nimport { cloneDeep } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\n@Injectable()\nexport class DynamicDsService extends BaseDynamicDsService<SubqlProjectDs> {\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n  ) {\n    super();\n  }\n\n  protected async getDatasource(\n    params: DatasourceParams,\n  ): Promise<SubqlProjectDs> {\n    const template = cloneDeep(\n      this.project.templates?.find((t) => t.name === params.templateName),\n    );\n\n    if (!template) {\n      throw new Error(\n        `Unable to find matching template in project for name: \"${params.templateName}\"`,\n      );\n    }\n\n    const dsObj = {\n      ...template,\n      startBlock: params.startBlock,\n    } as SubqlProjectDs;\n    delete dsObj.name;\n    try {\n      if (isCustomDs(dsObj)) {\n        dsObj.processor.options = {\n          ...dsObj.processor.options,\n          ...params.args,\n        };\n        await this.dsProcessorService.validateCustomDs([dsObj]);\n      } else if (isRuntimeDs(dsObj)) {\n        dsObj.options = {\n          ...dsObj.options,\n          ...params.args,\n        };\n\n        const parsedDs = plainToClass(\n          EthereumRuntimeDataSourceV0_3_0Impl,\n          dsObj,\n        );\n\n        const errors = validateSync(parsedDs, {\n          whitelist: true,\n          forbidNonWhitelisted: false,\n        });\n        if (errors.length) {\n          throw new Error(\n            `Dynamic ds is invalid\\n${errors\n              .map((e) => e.toString())\n              .join('\\n')}`,\n          );\n        }\n      }\n      return dsObj;\n    } catch (e) {\n      throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n    }\n  }\n}\n"]}

package/dist/indexer/fetch.module.js

@@ -28,6 +28,7 @@ FetchModule = __decorate([
     (0, common_1.Module)({
         providers: [
             node_core_1.StoreService,
+            node_core_1.StoreCacheService,
             {
                 provide: node_core_1.ApiService,
                 useFactory: async (project, eventEmitter) => {
@@ -38,17 +39,31 @@ FetchModule = __decorate([
                 inject: ['ISubqueryProject', event_emitter_1.EventEmitter2],
             },
             indexer_manager_1.IndexerManager,
+            {
+                provide: node_core_1.SmartBatchService,
+                useFactory: (nodeConfig) => {
+                    return new node_core_1.SmartBatchService(nodeConfig.batchSize);
+                },
+                inject: [node_core_1.NodeConfig],
+            },
             {
                 provide: 'IBlockDispatcher',
-                useFactory: (nodeConfig, eventEmitter, projectService, apiService, indexerManager) => nodeConfig.workers !== undefined
-                    ? new blockDispatcher_1.WorkerBlockDispatcherService(nodeConfig, eventEmitter, projectService)
-                    : new blockDispatcher_1.BlockDispatcherService(apiService, nodeConfig, indexerManager, eventEmitter, projectService),
+                useFactory: (nodeConfig, eventEmitter, projectService, apiService, indexerManager, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService, unfinalizedBlocks) => nodeConfig.workers !== undefined
+                    ? new blockDispatcher_1.WorkerBlockDispatcherService(nodeConfig, eventEmitter, projectService, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService, unfinalizedBlocks)
+                    : new blockDispatcher_1.BlockDispatcherService(apiService, nodeConfig, indexerManager, eventEmitter, projectService, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService),
                 inject: [
                     node_core_1.NodeConfig,
                     event_emitter_1.EventEmitter2,
-                    project_service_1.ProjectService,
+                    'IProjectService',
                     node_core_1.ApiService,
                     indexer_manager_1.IndexerManager,
+                    node_core_1.SmartBatchService,
+                    node_core_1.StoreService,
+                    node_core_1.StoreCacheService,
+                    node_core_1.PoiService,
+                    'ISubqueryProject',
+                    dynamic_ds_service_1.DynamicDsService,
+                    unfinalizedBlocks_service_1.UnfinalizedBlocksService,
                 ],
             },
             fetch_service_1.FetchService,
@@ -67,10 +82,13 @@ FetchModule = __decorate([
             dynamic_ds_service_1.DynamicDsService,
             node_core_1.PoiService,
             node_core_1.MmrService,
-            project_service_1.ProjectService,
+            {
+                useClass: project_service_1.ProjectService,
+                provide: 'IProjectService',
+            },
             unfinalizedBlocks_service_1.UnfinalizedBlocksService,
         ],
-        exports: [node_core_1.StoreService, node_core_1.MmrService],
+        exports: [node_core_1.StoreService, node_core_1.MmrService, node_core_1.StoreCacheService],
     })
 ], FetchModule);
 exports.FetchModule = FetchModule;

package/dist/indexer/fetch.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.module.js","sourceRoot":"","sources":["../../src/indexer/fetch.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAAsD;AACtD,gDAO0B;AAE1B,2EAAsE;AACtE,uDAG2B;AAC3B,6DAAyD;AACzD,iEAA4D;AAC5D,6DAAwD;AACxD,mDAA+C;AAC/C,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAqEhE,IAAM,WAAW,GAAjB,MAAM,WAAW;CAAG,CAAA;AAAd,WAAW;IAnEvB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,yCAAkB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;oBACjE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,6BAAa,CAAC;aAC5C;YACD,gCAAc;YACd;gBACE,OAAO,EAAE,kBAAkB;gBAC3B,UAAU,EAAE,CACV,UAAsB,EACtB,YAA2B,EAC3B,cAA8B,EAC9B,UAAsB,EACtB,cAA8B,EAC9B,EAAE,CACF,UAAU,CAAC,OAAO,KAAK,SAAS;oBAC9B,CAAC,CAAC,IAAI,8CAA4B,CAC9B,UAAU,EACV,YAAY,EACZ,cAAc,CACf;oBACH,CAAC,CAAC,IAAI,wCAAsB,CACxB,UAAU,EACV,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,CACf;gBACP,MAAM,EAAE;oBACN,sBAAU;oBACV,6BAAa;oBACb,gCAAc;oBACd,sBAAU;oBACV,gCAAc;iBACf;aACF;YACD,4BAAY;YACZ,4BAAgB;YAChB;gBACE,OAAO,EAAE,sCAAiB;gBAC1B,UAAU,EAAE,KAAK,EAAE,OAAwB,EAAE,UAAsB,EAAE,EAAE;oBACrE,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;oBACrE,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC;oBAC/B,OAAO,iBAAiB,CAAC;gBAC3B,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,sBAAU,CAAC;aACzC;YACD,gCAAc;YACd,yCAAkB;YAClB,qCAAgB;YAChB,sBAAU;YACV,sBAAU;YACV,gCAAc;YACd,oDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,wBAAY,EAAE,sBAAU,CAAC;KACpC,CAAC;GACW,WAAW,CAAG;AAAd,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  BenchmarkService,\n  MmrService,\n  StoreService,\n  PoiService,\n  ApiService,\n  NodeConfig,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum/api.service.ethereum';\nimport {\n  BlockDispatcherService,\n  WorkerBlockDispatcherService,\n} from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { FetchService } from './fetch.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    StoreService,\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new EthereumApiService(project, eventEmitter);\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', EventEmitter2],\n    },\n    IndexerManager,\n    {\n      provide: 'IBlockDispatcher',\n      useFactory: (\n        nodeConfig: NodeConfig,\n        eventEmitter: EventEmitter2,\n        projectService: ProjectService,\n        apiService: ApiService,\n        indexerManager: IndexerManager,\n      ) =>\n        nodeConfig.workers !== undefined\n          ? new WorkerBlockDispatcherService(\n              nodeConfig,\n              eventEmitter,\n              projectService,\n            )\n          : new BlockDispatcherService(\n              apiService,\n              nodeConfig,\n              indexerManager,\n              eventEmitter,\n              projectService,\n            ),\n      inject: [\n        NodeConfig,\n        EventEmitter2,\n        ProjectService,\n        ApiService,\n        IndexerManager,\n      ],\n    },\n    FetchService,\n    BenchmarkService,\n    {\n      provide: DictionaryService,\n      useFactory: async (project: SubqueryProject, nodeConfig: NodeConfig) => {\n        const dictionaryService = new DictionaryService(project, nodeConfig);\n        await dictionaryService.init();\n        return dictionaryService;\n      },\n      inject: ['ISubqueryProject', NodeConfig],\n    },\n    SandboxService,\n    DsProcessorService,\n    DynamicDsService,\n    PoiService,\n    MmrService,\n    ProjectService,\n    UnfinalizedBlocksService,\n  ],\n  exports: [StoreService, MmrService],\n})\nexport class FetchModule {}\n"]}
+{"version":3,"file":"fetch.module.js","sourceRoot":"","sources":["../../src/indexer/fetch.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAAsD;AACtD,gDAS0B;AAE1B,2EAAsE;AACtE,uDAG2B;AAC3B,6DAAyD;AACzD,iEAA4D;AAC5D,6DAAwD;AACxD,mDAA+C;AAC/C,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AA2GhE,IAAM,WAAW,GAAjB,MAAM,WAAW;CAAG,CAAA;AAAd,WAAW;IAzGvB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ,6BAAiB;YACjB;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,yCAAkB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;oBACjE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,6BAAa,CAAC;aAC5C;YACD,gCAAc;YACd;gBACE,OAAO,EAAE,6BAAiB;gBAC1B,UAAU,EAAE,CAAC,UAAsB,EAAE,EAAE;oBACrC,OAAO,IAAI,6BAAiB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;gBACrD,CAAC;gBACD,MAAM,EAAE,CAAC,sBAAU,CAAC;aACrB;YACD;gBACE,OAAO,EAAE,kBAAkB;gBAC3B,UAAU,EAAE,CACV,UAAsB,EACtB,YAA2B,EAC3B,cAA8B,EAC9B,UAAsB,EACtB,cAA8B,EAC9B,iBAAoC,EACpC,YAA0B,EAC1B,iBAAoC,EACpC,UAAsB,EACtB,OAAwB,EACxB,gBAAkC,EAClC,iBAA2C,EAC3C,EAAE,CACF,UAAU,CAAC,OAAO,KAAK,SAAS;oBAC9B,CAAC,CAAC,IAAI,8CAA4B,CAC9B,UAAU,EACV,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,iBAAiB,CAClB;oBACH,CAAC,CAAC,IAAI,wCAAsB,CACxB,UAAU,EACV,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,CACjB;gBACP,MAAM,EAAE;oBACN,sBAAU;oBACV,6BAAa;oBACb,iBAAiB;oBACjB,sBAAU;oBACV,gCAAc;oBACd,6BAAiB;oBACjB,wBAAY;oBACZ,6BAAiB;oBACjB,sBAAU;oBACV,kBAAkB;oBAClB,qCAAgB;oBAChB,oDAAwB;iBACzB;aACF;YACD,4BAAY;YACZ,4BAAgB;YAChB;gBACE,OAAO,EAAE,sCAAiB;gBAC1B,UAAU,EAAE,KAAK,EAAE,OAAwB,EAAE,UAAsB,EAAE,EAAE;oBACrE,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;oBACrE,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC;oBAC/B,OAAO,iBAAiB,CAAC;gBAC3B,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,sBAAU,CAAC;aACzC;YACD,gCAAc;YACd,yCAAkB;YAClB,qCAAgB;YAChB,sBAAU;YACV,sBAAU;YACV;gBACE,QAAQ,EAAE,gCAAc;gBACxB,OAAO,EAAE,iBAAiB;aAC3B;YACD,oDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,wBAAY,EAAE,sBAAU,EAAE,6BAAiB,CAAC;KACvD,CAAC;GACW,WAAW,CAAG;AAAd,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  BenchmarkService,\n  MmrService,\n  StoreService,\n  PoiService,\n  ApiService,\n  NodeConfig,\n  SmartBatchService,\n  StoreCacheService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum/api.service.ethereum';\nimport {\n  BlockDispatcherService,\n  WorkerBlockDispatcherService,\n} from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { FetchService } from './fetch.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    StoreService,\n    StoreCacheService,\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new EthereumApiService(project, eventEmitter);\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', EventEmitter2],\n    },\n    IndexerManager,\n    {\n      provide: SmartBatchService,\n      useFactory: (nodeConfig: NodeConfig) => {\n        return new SmartBatchService(nodeConfig.batchSize);\n      },\n      inject: [NodeConfig],\n    },\n    {\n      provide: 'IBlockDispatcher',\n      useFactory: (\n        nodeConfig: NodeConfig,\n        eventEmitter: EventEmitter2,\n        projectService: ProjectService,\n        apiService: ApiService,\n        indexerManager: IndexerManager,\n        smartBatchService: SmartBatchService,\n        storeService: StoreService,\n        storeCacheService: StoreCacheService,\n        poiService: PoiService,\n        project: SubqueryProject,\n        dynamicDsService: DynamicDsService,\n        unfinalizedBlocks: UnfinalizedBlocksService,\n      ) =>\n        nodeConfig.workers !== undefined\n          ? new WorkerBlockDispatcherService(\n              nodeConfig,\n              eventEmitter,\n              projectService,\n              smartBatchService,\n              storeService,\n              storeCacheService,\n              poiService,\n              project,\n              dynamicDsService,\n              unfinalizedBlocks,\n            )\n          : new BlockDispatcherService(\n              apiService,\n              nodeConfig,\n              indexerManager,\n              eventEmitter,\n              projectService,\n              smartBatchService,\n              storeService,\n              storeCacheService,\n              poiService,\n              project,\n              dynamicDsService,\n            ),\n      inject: [\n        NodeConfig,\n        EventEmitter2,\n        'IProjectService',\n        ApiService,\n        IndexerManager,\n        SmartBatchService,\n        StoreService,\n        StoreCacheService,\n        PoiService,\n        'ISubqueryProject',\n        DynamicDsService,\n        UnfinalizedBlocksService,\n      ],\n    },\n    FetchService,\n    BenchmarkService,\n    {\n      provide: DictionaryService,\n      useFactory: async (project: SubqueryProject, nodeConfig: NodeConfig) => {\n        const dictionaryService = new DictionaryService(project, nodeConfig);\n        await dictionaryService.init();\n        return dictionaryService;\n      },\n      inject: ['ISubqueryProject', NodeConfig],\n    },\n    SandboxService,\n    DsProcessorService,\n    DynamicDsService,\n    PoiService,\n    MmrService,\n    {\n      useClass: ProjectService,\n      provide: 'IProjectService',\n    },\n    UnfinalizedBlocksService,\n  ],\n  exports: [StoreService, MmrService, StoreCacheService],\n})\nexport class FetchModule {}\n"]}

package/dist/indexer/fetch.service.d.ts

@@ -2,9 +2,9 @@ import { OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import { SchedulerRegistry } from '@nestjs/schedule';
 import { ApiService, NodeConfig } from '@subql/node-core';
-import { DictionaryQueryEntry, ApiWrapper } from '@subql/types-ethereum';
+import { ApiWrapper, DictionaryQueryEntry } from '@subql/types-ethereum';
 import { SubqueryProject } from '../configure/SubqueryProject';
-import { IBlockDispatcher } from './blockDispatcher';
+import { IEthereumBlockDispatcher } from './blockDispatcher';
 import { DictionaryService } from './dictionary.service';
 import { DynamicDsService } from './dynamic-ds.service';
 import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
@@ -26,7 +26,7 @@ export declare class FetchService implements OnApplicationShutdown {
     private dictionaryGenesisMatches;
     private evmChainId;
     private bypassBlocks;
-    constructor(apiService: ApiService, nodeConfig: NodeConfig, project: SubqueryProject, blockDispatcher: IBlockDispatcher, dictionaryService: DictionaryService, dynamicDsService: DynamicDsService, unfinalizedBlocksService: UnfinalizedBlocksService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry);
+    constructor(apiService: ApiService, nodeConfig: NodeConfig, project: SubqueryProject, blockDispatcher: IEthereumBlockDispatcher, dictionaryService: DictionaryService, dynamicDsService: DynamicDsService, unfinalizedBlocksService: UnfinalizedBlocksService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry);
     onApplicationShutdown(): void;
     get api(): ApiWrapper;
     syncDynamicDatascourcesFromMeta(): Promise<void>;