@subql/node-ethereum 0.1.1 → 0.2.1

package/dist/indexer/blockDispatcher/base-block-dispatcher.d.ts

@@ -0,0 +1,40 @@
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { IQueue, NodeConfig } from '@subql/node-core';
+import { ProjectService } from '../project.service';
+export declare type ProcessBlockResponse = {
+    dynamicDsCreated: boolean;
+    operationHash: Uint8Array;
+    reindexBlockHeight: number;
+};
+export interface IBlockDispatcher {
+    init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
+    enqueueBlocks(heights: number[]): void;
+    queueSize: number;
+    freeSize: number;
+    latestBufferedHeight: number | undefined;
+    flushQueue(height: number): void;
+    rewind(height: number): Promise<void>;
+}
+export declare abstract class BaseBlockDispatcher<Q extends IQueue> implements IBlockDispatcher {
+    protected nodeConfig: NodeConfig;
+    protected eventEmitter: EventEmitter2;
+    protected projectService: ProjectService;
+    protected queue: Q;
+    protected _latestBufferedHeight: number;
+    protected _processedBlockCount: number;
+    protected latestProcessedHeight: number;
+    protected currentProcessingHeight: number;
+    protected onDynamicDsCreated: (height: number) => Promise<void>;
+    constructor(nodeConfig: NodeConfig, eventEmitter: EventEmitter2, projectService: ProjectService, queue: Q);
+    abstract enqueueBlocks(heights: number[]): void;
+    abstract init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
+    get queueSize(): number;
+    get freeSize(): number;
+    get latestBufferedHeight(): number;
+    set latestBufferedHeight(height: number);
+    protected setProcessedBlockCount(processedBlockCount: number): void;
+    rewind(lastCorrectHeight: number): Promise<void>;
+    flushQueue(height: number): void;
+    protected preProcessBlock(height: number): void;
+    protected postProcessBlock(height: number, processBlockResponse: ProcessBlockResponse): Promise<void>;
+}

package/dist/indexer/blockDispatcher/base-block-dispatcher.js

@@ -0,0 +1,97 @@
+"use strict";
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseBlockDispatcher = void 0;
+const assert_1 = __importDefault(require("assert"));
+const util_1 = require("@polkadot/util");
+const node_core_1 = require("@subql/node-core");
+const logger = (0, node_core_1.getLogger)('BaseBlockDispatcherService');
+const NULL_MERKEL_ROOT = (0, util_1.hexToU8a)('0x00');
+function isNullMerkelRoot(operationHash) {
+    return (0, util_1.u8aEq)(operationHash, NULL_MERKEL_ROOT);
+}
+class BaseBlockDispatcher {
+    constructor(nodeConfig, eventEmitter, projectService, queue) {
+        this.nodeConfig = nodeConfig;
+        this.eventEmitter = eventEmitter;
+        this.projectService = projectService;
+        this.queue = queue;
+    }
+    get queueSize() {
+        return this.queue.size;
+    }
+    get freeSize() {
+        return this.queue.freeSpace;
+    }
+    get latestBufferedHeight() {
+        return this._latestBufferedHeight;
+    }
+    set latestBufferedHeight(height) {
+        this.eventEmitter.emit(node_core_1.IndexerEvent.BlocknumberQueueSize, {
+            value: this.queueSize,
+        });
+        this._latestBufferedHeight = height;
+    }
+    setProcessedBlockCount(processedBlockCount) {
+        this._processedBlockCount = processedBlockCount;
+        this.eventEmitter.emit(node_core_1.IndexerEvent.BlockProcessedCount, {
+            processedBlockCount,
+            timestamp: Date.now(),
+        });
+    }
+    //  Compare it with current indexing number, if last corrected is already indexed
+    //  rewind, also flush queued blocks, drop current indexing transaction, set last processed to correct block too
+    //  if rollback is greater than current index flush queue only
+    async rewind(lastCorrectHeight) {
+        if (lastCorrectHeight <= this.currentProcessingHeight) {
+            logger.info(`Found last verified block at height ${lastCorrectHeight}, rewinding...`);
+            await this.projectService.reindex(lastCorrectHeight);
+            this.latestProcessedHeight = lastCorrectHeight;
+            logger.info(`Successful rewind to block ${lastCorrectHeight}!`);
+        }
+        this.flushQueue(lastCorrectHeight);
+        logger.info(`Queued blocks flushed!`); //Also last buffered height reset, next fetching should start after lastCorrectHeight
+    }
+    flushQueue(height) {
+        this.latestBufferedHeight = height;
+        this.queue.flush();
+    }
+    // Is called directly before a block is processed
+    preProcessBlock(height) {
+        this.currentProcessingHeight = height;
+        this.eventEmitter.emit(node_core_1.IndexerEvent.BlockProcessing, {
+            height,
+            timestamp: Date.now(),
+        });
+    }
+    // Is called directly after a block is processed
+    async postProcessBlock(height, processBlockResponse) {
+        const { dynamicDsCreated, operationHash, reindexBlockHeight } = processBlockResponse;
+        if (reindexBlockHeight !== null && reindexBlockHeight !== undefined) {
+            await this.rewind(reindexBlockHeight);
+            this.latestProcessedHeight = reindexBlockHeight;
+        }
+        else {
+            if (this.nodeConfig.proofOfIndex && !isNullMerkelRoot(operationHash)) {
+                if (!this.projectService.blockOffset) {
+                    // Which means during project init, it has not found offset and set value
+                    await this.projectService.upsertMetadataBlockOffset(height - 1);
+                }
+                void this.projectService.setBlockOffset(height - 1);
+            }
+            if (dynamicDsCreated) {
+                await this.onDynamicDsCreated(height);
+            }
+            (0, assert_1.default)(!this.latestProcessedHeight || height > this.latestProcessedHeight, `Block processed out of order. Height: ${height}. Latest: ${this.latestProcessedHeight}`);
+            // In memory _processedBlockCount increase, db metadata increase BlockCount in indexer.manager
+            this.setProcessedBlockCount(this._processedBlockCount + 1);
+            this.latestProcessedHeight = height;
+        }
+    }
+}
+exports.BaseBlockDispatcher = BaseBlockDispatcher;
+//# sourceMappingURL=base-block-dispatcher.js.map

package/dist/indexer/blockDispatcher/base-block-dispatcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-block-dispatcher.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/base-block-dispatcher.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;AAEtC,oDAA4B;AAE5B,yCAAiD;AACjD,gDAA+E;AAG/E,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,4BAA4B,CAAC,CAAC;AAsBvD,MAAM,gBAAgB,GAAG,IAAA,eAAQ,EAAC,MAAM,CAAC,CAAC;AAE1C,SAAS,gBAAgB,CAAC,aAAyB;IACjD,OAAO,IAAA,YAAK,EAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC;AAChD,CAAC;AAED,MAAsB,mBAAmB;IASvC,YACY,UAAsB,EACtB,YAA2B,EAC3B,cAA8B,EAC9B,KAAQ;QAHR,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAe;QAC3B,mBAAc,GAAd,cAAc,CAAgB;QAC9B,UAAK,GAAL,KAAK,CAAG;IACjB,CAAC;IAOJ,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,IAAI,oBAAoB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAc;QACrC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,oBAAoB,EAAE;YACxD,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC;IACtC,CAAC;IAES,sBAAsB,CAAC,mBAA2B;QAC1D,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAChD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,mBAAmB,EAAE;YACvD,mBAAmB;YACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,gHAAgH;IAChH,8DAA8D;IAC9D,KAAK,CAAC,MAAM,CAAC,iBAAyB;QACpC,IAAI,iBAAiB,IAAI,IAAI,CAAC,uBAAuB,EAAE;YACrD,MAAM,CAAC,IAAI,CACT,uCAAuC,iBAAiB,gBAAgB,CACzE,CAAC;YACF,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACrD,IAAI,CAAC,qBAAqB,GAAG,iBAAiB,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,8BAA8B,iBAAiB,GAAG,CAAC,CAAC;SACjE;QACD,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,qFAAqF;IAC9H,CAAC;IAED,UAAU,CAAC,MAAc;QACvB,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC;QACnC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,iDAAiD;IACvC,eAAe,CAAC,MAAc;QACtC,IAAI,CAAC,uBAAuB,GAAG,MAAM,CAAC;QACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;YACnD,MAAM;YACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;IACL,CAAC;IAED,gDAAgD;IACtC,KAAK,CAAC,gBAAgB,CAC9B,MAAc,EACd,oBAA0C;QAE1C,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,kBAAkB,EAAE,GAC3D,oBAAoB,CAAC;QACvB,IAAI,kBAAkB,KAAK,IAAI,IAAI,kBAAkB,KAAK,SAAS,EAAE;YACnE,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;YACtC,IAAI,CAAC,qBAAqB,GAAG,kBAAkB,CAAC;SACjD;aAAM;YACL,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,EAAE;gBACpE,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE;oBACpC,yEAAyE;oBACzE,MAAM,IAAI,CAAC,cAAc,CAAC,yBAAyB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;iBACjE;gBACD,KAAK,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACrD;YACD,IAAI,gBAAgB,EAAE;gBACpB,MAAM,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;aACvC;YACD,IAAA,gBAAM,EACJ,CAAC,IAAI,CAAC,qBAAqB,IAAI,MAAM,GAAG,IAAI,CAAC,qBAAqB,EAClE,yCAAyC,MAAM,aAAa,IAAI,CAAC,qBAAqB,EAAE,CACzF,CAAC;YACF,8FAA8F;YAC9F,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC,CAAC;YAC3D,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC;SACrC;IACH,CAAC;CACF;AA5GD,kDA4GC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { hexToU8a, u8aEq } from '@polkadot/util';\nimport { getLogger, IndexerEvent, IQueue, NodeConfig } from '@subql/node-core';\nimport { ProjectService } from '../project.service';\n\nconst logger = getLogger('BaseBlockDispatcherService');\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  operationHash: Uint8Array;\n  reindexBlockHeight: number;\n};\n\nexport interface IBlockDispatcher {\n  init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;\n\n  enqueueBlocks(heights: number[]): void;\n\n  queueSize: number;\n  freeSize: number;\n  latestBufferedHeight: number | undefined;\n\n  // Remove all enqueued blocks, used when a dynamic ds is created\n  flushQueue(height: number): void;\n  rewind(height: number): Promise<void>;\n}\n\nconst NULL_MERKEL_ROOT = hexToU8a('0x00');\n\nfunction isNullMerkelRoot(operationHash: Uint8Array): boolean {\n  return u8aEq(operationHash, NULL_MERKEL_ROOT);\n}\n\nexport abstract class BaseBlockDispatcher<Q extends IQueue>\n  implements IBlockDispatcher\n{\n  protected _latestBufferedHeight: number;\n  protected _processedBlockCount: number;\n  protected latestProcessedHeight: number;\n  protected currentProcessingHeight: number;\n  protected onDynamicDsCreated: (height: number) => Promise<void>;\n\n  constructor(\n    protected nodeConfig: NodeConfig,\n    protected eventEmitter: EventEmitter2,\n    protected projectService: ProjectService,\n    protected queue: Q,\n  ) {}\n\n  abstract enqueueBlocks(heights: number[]): void;\n  abstract init(\n    onDynamicDsCreated: (height: number) => Promise<void>,\n  ): Promise<void>;\n\n  get queueSize(): number {\n    return this.queue.size;\n  }\n\n  get freeSize(): number {\n    return this.queue.freeSpace;\n  }\n\n  get latestBufferedHeight(): number {\n    return this._latestBufferedHeight;\n  }\n\n  set latestBufferedHeight(height: number) {\n    this.eventEmitter.emit(IndexerEvent.BlocknumberQueueSize, {\n      value: this.queueSize,\n    });\n    this._latestBufferedHeight = height;\n  }\n\n  protected setProcessedBlockCount(processedBlockCount: number): void {\n    this._processedBlockCount = processedBlockCount;\n    this.eventEmitter.emit(IndexerEvent.BlockProcessedCount, {\n      processedBlockCount,\n      timestamp: Date.now(),\n    });\n  }\n\n  //  Compare it with current indexing number, if last corrected is already indexed\n  //  rewind, also flush queued blocks, drop current indexing transaction, set last processed to correct block too\n  //  if rollback is greater than current index flush queue only\n  async rewind(lastCorrectHeight: number): Promise<void> {\n    if (lastCorrectHeight <= this.currentProcessingHeight) {\n      logger.info(\n        `Found last verified block at height ${lastCorrectHeight}, rewinding...`,\n      );\n      await this.projectService.reindex(lastCorrectHeight);\n      this.latestProcessedHeight = lastCorrectHeight;\n      logger.info(`Successful rewind to block ${lastCorrectHeight}!`);\n    }\n    this.flushQueue(lastCorrectHeight);\n    logger.info(`Queued blocks flushed!`); //Also last buffered height reset, next fetching should start after lastCorrectHeight\n  }\n\n  flushQueue(height: number): void {\n    this.latestBufferedHeight = height;\n    this.queue.flush();\n  }\n\n  // Is called directly before a block is processed\n  protected preProcessBlock(height: number): void {\n    this.currentProcessingHeight = height;\n    this.eventEmitter.emit(IndexerEvent.BlockProcessing, {\n      height,\n      timestamp: Date.now(),\n    });\n  }\n\n  // Is called directly after a block is processed\n  protected async postProcessBlock(\n    height: number,\n    processBlockResponse: ProcessBlockResponse,\n  ): Promise<void> {\n    const { dynamicDsCreated, operationHash, reindexBlockHeight } =\n      processBlockResponse;\n    if (reindexBlockHeight !== null && reindexBlockHeight !== undefined) {\n      await this.rewind(reindexBlockHeight);\n      this.latestProcessedHeight = reindexBlockHeight;\n    } else {\n      if (this.nodeConfig.proofOfIndex && !isNullMerkelRoot(operationHash)) {\n        if (!this.projectService.blockOffset) {\n          // Which means during project init, it has not found offset and set value\n          await this.projectService.upsertMetadataBlockOffset(height - 1);\n        }\n        void this.projectService.setBlockOffset(height - 1);\n      }\n      if (dynamicDsCreated) {\n        await this.onDynamicDsCreated(height);\n      }\n      assert(\n        !this.latestProcessedHeight || height > this.latestProcessedHeight,\n        `Block processed out of order. Height: ${height}. Latest: ${this.latestProcessedHeight}`,\n      );\n      // In memory _processedBlockCount increase, db metadata increase BlockCount in indexer.manager\n      this.setProcessedBlockCount(this._processedBlockCount + 1);\n      this.latestProcessedHeight = height;\n    }\n  }\n}\n"]}

package/dist/indexer/blockDispatcher/block-dispatcher.service.d.ts

@@ -0,0 +1,23 @@
+import { OnApplicationShutdown } from '@nestjs/common';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { ApiService, NodeConfig, Queue } from '@subql/node-core';
+import { IndexerManager } from '../indexer.manager';
+import { ProjectService } from '../project.service';
+import { BaseBlockDispatcher } from './base-block-dispatcher';
+/**
+ * @description Intended to behave the same as WorkerBlockDispatcherService but doesn't use worker threads or any parallel processing
+ */
+export declare class BlockDispatcherService extends BaseBlockDispatcher<Queue<number>> implements OnApplicationShutdown {
+    private apiService;
+    private indexerManager;
+    private processQueue;
+    private fetching;
+    private isShutdown;
+    private fetchBlocksBatches;
+    constructor(apiService: ApiService, nodeConfig: NodeConfig, indexerManager: IndexerManager, eventEmitter: EventEmitter2, projectService: ProjectService);
+    init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
+    onApplicationShutdown(): void;
+    enqueueBlocks(heights: number[]): void;
+    flushQueue(height: number): void;
+    private fetchBlocksFromQueue;
+}

package/dist/indexer/blockDispatcher/block-dispatcher.service.js

@@ -0,0 +1,134 @@
+"use strict";
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BlockDispatcherService = void 0;
+const common_1 = require("@nestjs/common");
+const event_emitter_1 = require("@nestjs/event-emitter");
+const node_core_1 = require("@subql/node-core");
+const lodash_1 = require("lodash");
+const indexer_manager_1 = require("../indexer.manager");
+const project_service_1 = require("../project.service");
+const base_block_dispatcher_1 = require("./base-block-dispatcher");
+const logger = (0, node_core_1.getLogger)('BlockDispatcherService');
+/**
+ * @description Intended to behave the same as WorkerBlockDispatcherService but doesn't use worker threads or any parallel processing
+ */
+let BlockDispatcherService = class BlockDispatcherService extends base_block_dispatcher_1.BaseBlockDispatcher {
+    constructor(apiService, nodeConfig, indexerManager, eventEmitter, projectService) {
+        super(nodeConfig, eventEmitter, projectService, new node_core_1.Queue(nodeConfig.batchSize * 3));
+        this.apiService = apiService;
+        this.indexerManager = indexerManager;
+        this.fetching = false;
+        this.isShutdown = false;
+        this.processQueue = new node_core_1.AutoQueue(nodeConfig.batchSize * 3);
+        const fetchBlocks = this.apiService.api.fetchBlocks.bind(this.apiService.api);
+        if (this.nodeConfig.profiler) {
+            this.fetchBlocksBatches = (0, node_core_1.profilerWrap)(fetchBlocks, 'EthereumUtil', 'fetchBlocksBatches');
+        }
+        else {
+            this.fetchBlocksBatches = fetchBlocks;
+        }
+    }
+    // eslint-disable-next-line @typescript-eslint/require-await
+    async init(onDynamicDsCreated) {
+        this.onDynamicDsCreated = onDynamicDsCreated;
+        const blockAmount = await this.projectService.getProcessedBlockCount();
+        this.setProcessedBlockCount(blockAmount !== null && blockAmount !== void 0 ? blockAmount : 0);
+    }
+    onApplicationShutdown() {
+        this.isShutdown = true;
+        this.processQueue.abort();
+    }
+    enqueueBlocks(heights) {
+        if (!heights.length)
+            return;
+        logger.info(`Enqueing blocks ${heights[0]}...${(0, lodash_1.last)(heights)}, total ${heights.length} blocks`);
+        this.queue.putMany(heights);
+        this.latestBufferedHeight = (0, lodash_1.last)(heights);
+        void this.fetchBlocksFromQueue().catch((e) => {
+            logger.error(e, 'Failed to fetch blocks from queue');
+            if (!this.isShutdown) {
+                process.exit(1);
+            }
+        });
+    }
+    flushQueue(height) {
+        super.flushQueue(height);
+        this.processQueue.flush();
+    }
+    async fetchBlocksFromQueue() {
+        if (this.fetching || this.isShutdown)
+            return;
+        // Process queue is full, no point in fetching more blocks
+        // if (this.processQueue.freeSpace < this.nodeConfig.batchSize) return;
+        this.fetching = true;
+        try {
+            while (!this.isShutdown) {
+                const blockNums = this.queue.takeMany(Math.min(this.nodeConfig.batchSize, this.processQueue.freeSpace));
+                // Used to compare before and after as a way to check if queue was flushed
+                const bufferedHeight = this._latestBufferedHeight;
+                // Queue is empty
+                if (!blockNums.length) {
+                    // The process queue might be full so no block nums were taken, wait and try again
+                    if (this.queue.size) {
+                        await (0, node_core_1.delay)(1);
+                        continue;
+                    }
+                    break;
+                }
+                logger.info(`fetch block [${blockNums[0]},${blockNums[blockNums.length - 1]}], total ${blockNums.length} blocks`);
+                const blocks = await this.fetchBlocksBatches(blockNums);
+                if (bufferedHeight > this._latestBufferedHeight) {
+                    logger.debug(`Queue was reset for new DS, discarding fetched blocks`);
+                    continue;
+                }
+                const blockTasks = blocks.map((block) => async () => {
+                    var _a;
+                    const height = block.block.block.header.number.toNumber();
+                    try {
+                        this.preProcessBlock(height);
+                        const processBlockResponse = await this.indexerManager.indexBlock(block);
+                        await this.postProcessBlock(height, processBlockResponse);
+                    }
+                    catch (e) {
+                        if (this.isShutdown) {
+                            return;
+                        }
+                        logger.error(e, `failed to index block at height ${height} ${e.handler ? `${e.handler}(${(_a = e.stack) !== null && _a !== void 0 ? _a : ''})` : ''}`);
+                        throw e;
+                    }
+                });
+                // There can be enough of a delay after fetching blocks that shutdown could now be true
+                if (this.isShutdown)
+                    break;
+                this.processQueue.putMany(blockTasks);
+                this.eventEmitter.emit(node_core_1.IndexerEvent.BlockQueueSize, {
+                    value: this.processQueue.size,
+                });
+            }
+        }
+        finally {
+            this.fetching = false;
+        }
+    }
+};
+BlockDispatcherService = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [node_core_1.ApiService,
+        node_core_1.NodeConfig,
+        indexer_manager_1.IndexerManager,
+        event_emitter_1.EventEmitter2,
+        project_service_1.ProjectService])
+], BlockDispatcherService);
+exports.BlockDispatcherService = BlockDispatcherService;
+//# sourceMappingURL=block-dispatcher.service.js.map

package/dist/indexer/blockDispatcher/block-dispatcher.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAAmE;AACnE,yDAAsD;AACtD,gDAS0B;AAC1B,mCAA8B;AAC9B,wDAAoD;AACpD,wDAAoD;AACpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,wBAAwB,CAAC,CAAC;AAEnD;;GAEG;AAEI,IAAM,sBAAsB,GAA5B,MAAM,sBACX,SAAQ,2CAAkC;IAS1C,YACU,UAAsB,EAC9B,UAAsB,EACd,cAA8B,EACtC,YAA2B,EAC3B,cAA8B;QAE9B,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,iBAAK,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACpC,CAAC;QAXM,eAAU,GAAV,UAAU,CAAY;QAEtB,mBAAc,GAAd,cAAc,CAAgB;QAPhC,aAAQ,GAAG,KAAK,CAAC;QACjB,eAAU,GAAG,KAAK,CAAC;QAgBzB,IAAI,CAAC,YAAY,GAAG,IAAI,qBAAS,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QAE5D,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CACtD,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC5B,IAAI,CAAC,kBAAkB,GAAG,IAAA,wBAAY,EACpC,WAAW,EACX,cAAc,EACd,oBAAoB,CACrB,CAAC;SACH;aAAM;YACL,IAAI,CAAC,kBAAkB,GAAG,WAAW,CAAC;SACvC;IACH,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,aAAa,CAAC,OAAiB;QAC7B,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QAE5B,MAAM,CAAC,IAAI,CACT,mBAAmB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,WAC9C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,oBAAoB,GAAG,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;QAE1C,KAAK,IAAI,CAAC,oBAAoB,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3C,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,mCAAmC,CAAC,CAAC;YACrD,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,UAAU,CAAC,MAAc;QACvB,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,oBAAoB;QAChC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7C,0DAA0D;QAC1D,uEAAuE;QAEvE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,IAAI;YACF,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE;gBACvB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CACnC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CACjE,CAAC;gBACF,0EAA0E;gBAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;gBAElD,iBAAiB;gBACjB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;oBACrB,kFAAkF;oBAClF,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;wBACnB,MAAM,IAAA,iBAAK,EAAC,CAAC,CAAC,CAAC;wBACf,SAAS;qBACV;oBACD,MAAM;iBACP;gBAED,MAAM,CAAC,IAAI,CACT,gBAAgB,SAAS,CAAC,CAAC,CAAC,IAC1B,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAChC,YAAY,SAAS,CAAC,MAAM,SAAS,CACtC,CAAC;gBAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;gBAExD,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE;oBAC/C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,SAAS;iBACV;gBACD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE;;oBAClD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;oBAC1D,IAAI;wBACF,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;wBAE7B,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAC/D,KAAK,CACN,CAAC;wBAEF,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;qBAC3D;oBAAC,OAAO,CAAC,EAAE;wBACV,IAAI,IAAI,CAAC,UAAU,EAAE;4BACnB,OAAO;yBACR;wBACD,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;wBACF,MAAM,CAAC,CAAC;qBACT;gBACH,CAAC,CAAC,CAAC;gBAEH,uFAAuF;gBACvF,IAAI,IAAI,CAAC,UAAU;oBAAE,MAAM;gBAE3B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBAEtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;oBAClD,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI;iBAC9B,CAAC,CAAC;aACJ;SACF;gBAAS;YACR,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;SACvB;IACH,CAAC;CACF,CAAA;AAxJY,sBAAsB;IADlC,IAAA,mBAAU,GAAE;qCAYW,sBAAU;QAClB,sBAAU;QACE,gCAAc;QACxB,6BAAa;QACX,gCAAc;GAfrB,sBAAsB,CAwJlC;AAxJY,wDAAsB","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  getLogger,\n  NodeConfig,\n  IndexerEvent,\n  delay,\n  profilerWrap,\n  AutoQueue,\n  Queue,\n} from '@subql/node-core';\nimport { last } from 'lodash';\nimport { IndexerManager } from '../indexer.manager';\nimport { ProjectService } from '../project.service';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('BlockDispatcherService');\n\n/**\n * @description Intended to behave the same as WorkerBlockDispatcherService but doesn't use worker threads or any parallel processing\n */\n@Injectable()\nexport class BlockDispatcherService\n  extends BaseBlockDispatcher<Queue<number>>\n  implements OnApplicationShutdown\n{\n  private processQueue: AutoQueue<void>;\n\n  private fetching = false;\n  private isShutdown = false;\n  private fetchBlocksBatches: ApiService['api']['fetchBlocks'];\n\n  constructor(\n    private apiService: ApiService,\n    nodeConfig: NodeConfig,\n    private indexerManager: IndexerManager,\n    eventEmitter: EventEmitter2,\n    projectService: ProjectService,\n  ) {\n    super(\n      nodeConfig,\n      eventEmitter,\n      projectService,\n      new Queue(nodeConfig.batchSize * 3),\n    );\n    this.processQueue = new AutoQueue(nodeConfig.batchSize * 3);\n\n    const fetchBlocks = this.apiService.api.fetchBlocks.bind(\n      this.apiService.api,\n    );\n    if (this.nodeConfig.profiler) {\n      this.fetchBlocksBatches = profilerWrap(\n        fetchBlocks,\n        'EthereumUtil',\n        'fetchBlocksBatches',\n      );\n    } else {\n      this.fetchBlocksBatches = fetchBlocks;\n    }\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  async init(\n    onDynamicDsCreated: (height: number) => Promise<void>,\n  ): Promise<void> {\n    this.onDynamicDsCreated = onDynamicDsCreated;\n    const blockAmount = await this.projectService.getProcessedBlockCount();\n    this.setProcessedBlockCount(blockAmount ?? 0);\n  }\n\n  onApplicationShutdown(): void {\n    this.isShutdown = true;\n    this.processQueue.abort();\n  }\n\n  enqueueBlocks(heights: number[]): void {\n    if (!heights.length) return;\n\n    logger.info(\n      `Enqueing blocks ${heights[0]}...${last(heights)}, total ${\n        heights.length\n      } blocks`,\n    );\n\n    this.queue.putMany(heights);\n    this.latestBufferedHeight = last(heights);\n\n    void this.fetchBlocksFromQueue().catch((e) => {\n      logger.error(e, 'Failed to fetch blocks from queue');\n      if (!this.isShutdown) {\n        process.exit(1);\n      }\n    });\n  }\n\n  flushQueue(height: number): void {\n    super.flushQueue(height);\n    this.processQueue.flush();\n  }\n\n  private async fetchBlocksFromQueue(): Promise<void> {\n    if (this.fetching || this.isShutdown) return;\n    // Process queue is full, no point in fetching more blocks\n    // if (this.processQueue.freeSpace < this.nodeConfig.batchSize) return;\n\n    this.fetching = true;\n\n    try {\n      while (!this.isShutdown) {\n        const blockNums = this.queue.takeMany(\n          Math.min(this.nodeConfig.batchSize, this.processQueue.freeSpace),\n        );\n        // Used to compare before and after as a way to check if queue was flushed\n        const bufferedHeight = this._latestBufferedHeight;\n\n        // Queue is empty\n        if (!blockNums.length) {\n          // The process queue might be full so no block nums were taken, wait and try again\n          if (this.queue.size) {\n            await delay(1);\n            continue;\n          }\n          break;\n        }\n\n        logger.info(\n          `fetch block [${blockNums[0]},${\n            blockNums[blockNums.length - 1]\n          }], total ${blockNums.length} blocks`,\n        );\n\n        const blocks = await this.fetchBlocksBatches(blockNums);\n\n        if (bufferedHeight > this._latestBufferedHeight) {\n          logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n          continue;\n        }\n        const blockTasks = blocks.map((block) => async () => {\n          const height = block.block.block.header.number.toNumber();\n          try {\n            this.preProcessBlock(height);\n\n            const processBlockResponse = await this.indexerManager.indexBlock(\n              block,\n            );\n\n            await this.postProcessBlock(height, processBlockResponse);\n          } catch (e) {\n            if (this.isShutdown) {\n              return;\n            }\n            logger.error(\n              e,\n              `failed to index block at height ${height} ${\n                e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n              }`,\n            );\n            throw e;\n          }\n        });\n\n        // There can be enough of a delay after fetching blocks that shutdown could now be true\n        if (this.isShutdown) break;\n\n        this.processQueue.putMany(blockTasks);\n\n        this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n          value: this.processQueue.size,\n        });\n      }\n    } finally {\n      this.fetching = false;\n    }\n  }\n}\n"]}

package/dist/indexer/blockDispatcher/index.d.ts

@@ -0,0 +1,4 @@
+import { IBlockDispatcher } from './base-block-dispatcher';
+import { BlockDispatcherService } from './block-dispatcher.service';
+import { WorkerBlockDispatcherService } from './worker-block-dispatcher.service';
+export { IBlockDispatcher, BlockDispatcherService, WorkerBlockDispatcherService, };

package/dist/indexer/blockDispatcher/index.js

@@ -0,0 +1,10 @@
+"use strict";
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerBlockDispatcherService = exports.BlockDispatcherService = void 0;
+const block_dispatcher_service_1 = require("./block-dispatcher.service");
+Object.defineProperty(exports, "BlockDispatcherService", { enumerable: true, get: function () { return block_dispatcher_service_1.BlockDispatcherService; } });
+const worker_block_dispatcher_service_1 = require("./worker-block-dispatcher.service");
+Object.defineProperty(exports, "WorkerBlockDispatcherService", { enumerable: true, get: function () { return worker_block_dispatcher_service_1.WorkerBlockDispatcherService; } });
+//# sourceMappingURL=index.js.map

package/dist/indexer/blockDispatcher/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/index.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAGtC,yEAAoE;AAKlE,uGALO,iDAAsB,OAKP;AAJxB,uFAAiF;AAK/E,6GALO,8DAA4B,OAKP","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { IBlockDispatcher } from './base-block-dispatcher';\nimport { BlockDispatcherService } from './block-dispatcher.service';\nimport { WorkerBlockDispatcherService } from './worker-block-dispatcher.service';\n\nexport {\n  IBlockDispatcher,\n  BlockDispatcherService,\n  WorkerBlockDispatcherService,\n};\n"]}

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.d.ts

@@ -0,0 +1,20 @@
+import { OnApplicationShutdown } from '@nestjs/common';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { NodeConfig, AutoQueue } from '@subql/node-core';
+import { ProjectService } from '../project.service';
+import { BaseBlockDispatcher } from './base-block-dispatcher';
+export declare class WorkerBlockDispatcherService extends BaseBlockDispatcher<AutoQueue<void>> implements OnApplicationShutdown {
+    private workers;
+    private numWorkers;
+    private taskCounter;
+    private isShutdown;
+    constructor(nodeConfig: NodeConfig, eventEmitter: EventEmitter2, projectService: ProjectService);
+    init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
+    onApplicationShutdown(): Promise<void>;
+    enqueueBlocks(heights: number[]): void;
+    private enqueueBlock;
+    sampleWorkerStatus(): Promise<void>;
+    get latestBufferedHeight(): number;
+    set latestBufferedHeight(height: number);
+    private getNextWorkerIndex;
+}

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js

@@ -0,0 +1,170 @@
+"use strict";
+// Copyright 2020-2021 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerBlockDispatcherService = void 0;
+const assert_1 = __importDefault(require("assert"));
+const path_1 = __importDefault(require("path"));
+const common_1 = require("@nestjs/common");
+const event_emitter_1 = require("@nestjs/event-emitter");
+const schedule_1 = require("@nestjs/schedule");
+const node_core_1 = require("@subql/node-core");
+const chalk_1 = __importDefault(require("chalk"));
+const lodash_1 = require("lodash");
+const project_service_1 = require("../project.service");
+const base_block_dispatcher_1 = require("./base-block-dispatcher");
+const logger = (0, node_core_1.getLogger)('WorkerBlockDispatcherService');
+async function createIndexerWorker() {
+    const indexerWorker = node_core_1.Worker.create(path_1.default.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), [
+        'initWorker',
+        'processBlock',
+        'fetchBlock',
+        'numFetchedBlocks',
+        'numFetchingBlocks',
+        'getStatus',
+    ]);
+    await indexerWorker.initWorker();
+    return indexerWorker;
+}
+let WorkerBlockDispatcherService = class WorkerBlockDispatcherService extends base_block_dispatcher_1.BaseBlockDispatcher {
+    constructor(nodeConfig, eventEmitter, projectService) {
+        const numWorkers = nodeConfig.workers;
+        super(nodeConfig, eventEmitter, projectService, new node_core_1.AutoQueue(numWorkers * nodeConfig.batchSize * 2));
+        this.taskCounter = 0;
+        this.isShutdown = false;
+        this.numWorkers = numWorkers;
+    }
+    async init(onDynamicDsCreated) {
+        if (this.nodeConfig.unfinalizedBlocks) {
+            throw new Error('Sorry, best block feature is not supported with workers yet.');
+        }
+        this.workers = await Promise.all(new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()));
+        this.onDynamicDsCreated = onDynamicDsCreated;
+        const blockAmount = await this.projectService.getProcessedBlockCount();
+        this.setProcessedBlockCount(blockAmount !== null && blockAmount !== void 0 ? blockAmount : 0);
+    }
+    async onApplicationShutdown() {
+        this.isShutdown = true;
+        // Stop processing blocks
+        this.queue.abort();
+        // Stop all workers
+        if (this.workers) {
+            await Promise.all(this.workers.map((w) => w.terminate()));
+        }
+    }
+    enqueueBlocks(heights) {
+        if (!heights.length)
+            return;
+        logger.info(`Enqueing blocks [${heights[0]}...${(0, lodash_1.last)(heights)}], total ${heights.length} blocks`);
+        // eslint-disable-next-line no-constant-condition
+        if (true) {
+            /*
+             * Load balancing:
+             * worker1: 1,2,3
+             * worker2: 4,5,6
+             */
+            const workerIdx = this.getNextWorkerIndex();
+            heights.map((height) => this.enqueueBlock(height, workerIdx));
+        }
+        else {
+            /*
+             * Load balancing:
+             * worker1: 1,3,5
+             * worker2: 2,4,6
+             */
+            heights.map((height) => this.enqueueBlock(height, this.getNextWorkerIndex()));
+        }
+        this.latestBufferedHeight = (0, lodash_1.last)(heights);
+    }
+    enqueueBlock(height, workerIdx) {
+        if (this.isShutdown)
+            return;
+        const worker = this.workers[workerIdx];
+        (0, assert_1.default)(worker, `Worker ${workerIdx} not found`);
+        // Used to compare before and after as a way to check if queue was flushed
+        const bufferedHeight = this.latestBufferedHeight;
+        const pendingBlock = worker.fetchBlock(height);
+        const processBlock = async () => {
+            var _a;
+            try {
+                const start = new Date();
+                const result = await pendingBlock;
+                const end = new Date();
+                if (bufferedHeight > this.latestBufferedHeight) {
+                    logger.debug(`Queue was reset for new DS, discarding fetched blocks`);
+                    return;
+                }
+                const waitTime = end.getTime() - start.getTime();
+                if (waitTime > 1000) {
+                    logger.info(`Waiting to fetch block ${height}: ${chalk_1.default.red(`${waitTime}ms`)}`);
+                }
+                else if (waitTime > 200) {
+                    logger.info(`Waiting to fetch block ${height}: ${chalk_1.default.yellow(`${waitTime}ms`)}`);
+                }
+                // logger.info(
+                //   `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,
+                // );
+                this.preProcessBlock(height);
+                const { dynamicDsCreated, operationHash, reindexBlockHeight } = await worker.processBlock(height);
+                await this.postProcessBlock(height, {
+                    dynamicDsCreated,
+                    operationHash: Buffer.from(operationHash, 'base64'),
+                    reindexBlockHeight,
+                });
+            }
+            catch (e) {
+                logger.error(e, `failed to index block at height ${height} ${e.handler ? `${e.handler}(${(_a = e.stack) !== null && _a !== void 0 ? _a : ''})` : ''}`);
+                throw e;
+            }
+        };
+        void this.queue.put(processBlock);
+    }
+    async sampleWorkerStatus() {
+        for (const worker of this.workers) {
+            const status = await worker.getStatus();
+            logger.info(JSON.stringify(status));
+        }
+    }
+    // Getter doesn't seem to cary from abstract class
+    get latestBufferedHeight() {
+        return this._latestBufferedHeight;
+    }
+    set latestBufferedHeight(height) {
+        super.latestBufferedHeight = height;
+        // There is only a single queue with workers so we treat them as the same
+        this.eventEmitter.emit(node_core_1.IndexerEvent.BlockQueueSize, {
+            value: this.queueSize,
+        });
+    }
+    getNextWorkerIndex() {
+        const index = this.taskCounter % this.numWorkers;
+        this.taskCounter++;
+        return index;
+    }
+};
+__decorate([
+    (0, schedule_1.Interval)(15000),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], WorkerBlockDispatcherService.prototype, "sampleWorkerStatus", null);
+WorkerBlockDispatcherService = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [node_core_1.NodeConfig,
+        event_emitter_1.EventEmitter2,
+        project_service_1.ProjectService])
+], WorkerBlockDispatcherService);
+exports.WorkerBlockDispatcherService = WorkerBlockDispatcherService;
+//# sourceMappingURL=worker-block-dispatcher.service.js.map

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAAmE;AACnE,yDAAsD;AACtD,+CAA4C;AAC5C,gDAM0B;AAC1B,kDAA0B;AAC1B,mCAA8B;AAC9B,wDAAoD;AASpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAkBzD,KAAK,UAAU,mBAAmB;IAChC,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CACjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;KACZ,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,2CAAoC;IAS5C,YACE,UAAsB,EACtB,YAA2B,EAC3B,cAA8B;QAE9B,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;QACtC,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,qBAAS,CAAC,UAAU,GAAG,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACrD,CAAC;QAdI,gBAAW,GAAG,CAAC,CAAC;QAChB,eAAU,GAAG,KAAK,CAAC;QAczB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;SACH;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,yBAAyB;QACzB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAEnB,mBAAmB;QACnB,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SAC3D;IACH,CAAC;IAED,aAAa,CAAC,OAAiB;QAC7B,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QAC5B,MAAM,CAAC,IAAI,CACT,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,YAC/C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,iDAAiD;QACjD,IAAI,IAAI,EAAE;YACR;;;;eAIG;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;SAC/D;aAAM;YACL;;;;eAIG;YACH,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CACrD,CAAC;SACH;QAED,IAAI,CAAC,oBAAoB,GAAG,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,SAAiB;QACpD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,IAAA,gBAAM,EAAC,MAAM,EAAE,UAAU,SAAS,YAAY,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC;QACjD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;;YAC9B,IAAI;gBACF,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,IAAI,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE;oBAC9C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO;iBACR;gBAED,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjD,IAAI,QAAQ,GAAG,IAAI,EAAE;oBACnB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,CAAC,EAAE,CAClE,CAAC;iBACH;qBAAM,IAAI,QAAQ,GAAG,GAAG,EAAE;oBACzB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,MAAM,CAC/C,GAAG,QAAQ,IAAI,CAChB,EAAE,CACJ,CAAC;iBACH;gBAED,eAAe;gBACf,gKAAgK;gBAChK,KAAK;gBAEL,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAE7B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,kBAAkB,EAAE,GAC3D,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE;oBAClC,gBAAgB;oBAChB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;oBACnD,kBAAkB;iBACnB,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;gBACF,MAAM,CAAC,CAAC;aACT;QACH,CAAC,CAAC;QAEF,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAGK,AAAN,KAAK,CAAC,kBAAkB;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,oBAAoB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAc;QACrC,KAAK,CAAC,oBAAoB,GAAG,MAAM,CAAC;QACpC,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;YAClD,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAA;AA3BO;IADL,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sEAMf;AA1JU,4BAA4B;IADxC,IAAA,mBAAU,GAAE;qCAYG,sBAAU;QACR,6BAAa;QACX,gCAAc;GAbrB,4BAA4B,CAgLxC;AAhLY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  getLogger,\n  NodeConfig,\n  IndexerEvent,\n  Worker,\n  AutoQueue,\n} from '@subql/node-core';\nimport chalk from 'chalk';\nimport { last } from 'lodash';\nimport { ProjectService } from '../project.service';\nimport {\n  FetchBlock,\n  ProcessBlock,\n  InitWorker,\n  NumFetchedBlocks,\n  NumFetchingBlocks,\n  GetWorkerStatus,\n} from '../worker/worker';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IIndexerWorker = {\n  processBlock: ProcessBlock;\n  fetchBlock: FetchBlock;\n  numFetchedBlocks: NumFetchedBlocks;\n  numFetchingBlocks: NumFetchingBlocks;\n  getStatus: GetWorkerStatus;\n};\n\ntype IInitIndexerWorker = IIndexerWorker & {\n  initWorker: InitWorker;\n};\n\ntype IndexerWorker = IIndexerWorker & {\n  terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(): Promise<IndexerWorker> {\n  const indexerWorker = Worker.create<IInitIndexerWorker>(\n    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n    [\n      'initWorker',\n      'processBlock',\n      'fetchBlock',\n      'numFetchedBlocks',\n      'numFetchingBlocks',\n      'getStatus',\n    ],\n  );\n\n  await indexerWorker.initWorker();\n\n  return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n  extends BaseBlockDispatcher<AutoQueue<void>>\n  implements OnApplicationShutdown\n{\n  private workers: IndexerWorker[];\n  private numWorkers: number;\n\n  private taskCounter = 0;\n  private isShutdown = false;\n\n  constructor(\n    nodeConfig: NodeConfig,\n    eventEmitter: EventEmitter2,\n    projectService: ProjectService,\n  ) {\n    const numWorkers = nodeConfig.workers;\n    super(\n      nodeConfig,\n      eventEmitter,\n      projectService,\n      new AutoQueue(numWorkers * nodeConfig.batchSize * 2),\n    );\n    this.numWorkers = numWorkers;\n  }\n\n  async init(\n    onDynamicDsCreated: (height: number) => Promise<void>,\n  ): Promise<void> {\n    if (this.nodeConfig.unfinalizedBlocks) {\n      throw new Error(\n        'Sorry, best block feature is not supported with workers yet.',\n      );\n    }\n\n    this.workers = await Promise.all(\n      new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()),\n    );\n\n    this.onDynamicDsCreated = onDynamicDsCreated;\n\n    const blockAmount = await this.projectService.getProcessedBlockCount();\n    this.setProcessedBlockCount(blockAmount ?? 0);\n  }\n\n  async onApplicationShutdown(): Promise<void> {\n    this.isShutdown = true;\n    // Stop processing blocks\n    this.queue.abort();\n\n    // Stop all workers\n    if (this.workers) {\n      await Promise.all(this.workers.map((w) => w.terminate()));\n    }\n  }\n\n  enqueueBlocks(heights: number[]): void {\n    if (!heights.length) return;\n    logger.info(\n      `Enqueing blocks [${heights[0]}...${last(heights)}], total ${\n        heights.length\n      } blocks`,\n    );\n\n    // eslint-disable-next-line no-constant-condition\n    if (true) {\n      /*\n       * Load balancing:\n       * worker1: 1,2,3\n       * worker2: 4,5,6\n       */\n      const workerIdx = this.getNextWorkerIndex();\n      heights.map((height) => this.enqueueBlock(height, workerIdx));\n    } else {\n      /*\n       * Load balancing:\n       * worker1: 1,3,5\n       * worker2: 2,4,6\n       */\n      heights.map((height) =>\n        this.enqueueBlock(height, this.getNextWorkerIndex()),\n      );\n    }\n\n    this.latestBufferedHeight = last(heights);\n  }\n\n  private enqueueBlock(height: number, workerIdx: number) {\n    if (this.isShutdown) return;\n    const worker = this.workers[workerIdx];\n\n    assert(worker, `Worker ${workerIdx} not found`);\n\n    // Used to compare before and after as a way to check if queue was flushed\n    const bufferedHeight = this.latestBufferedHeight;\n    const pendingBlock = worker.fetchBlock(height);\n\n    const processBlock = async () => {\n      try {\n        const start = new Date();\n        const result = await pendingBlock;\n        const end = new Date();\n\n        if (bufferedHeight > this.latestBufferedHeight) {\n          logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n          return;\n        }\n\n        const waitTime = end.getTime() - start.getTime();\n        if (waitTime > 1000) {\n          logger.info(\n            `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n          );\n        } else if (waitTime > 200) {\n          logger.info(\n            `Waiting to fetch block ${height}: ${chalk.yellow(\n              `${waitTime}ms`,\n            )}`,\n          );\n        }\n\n        // logger.info(\n        //   `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,\n        // );\n\n        this.preProcessBlock(height);\n\n        const { dynamicDsCreated, operationHash, reindexBlockHeight } =\n          await worker.processBlock(height);\n\n        await this.postProcessBlock(height, {\n          dynamicDsCreated,\n          operationHash: Buffer.from(operationHash, 'base64'),\n          reindexBlockHeight,\n        });\n      } catch (e) {\n        logger.error(\n          e,\n          `failed to index block at height ${height} ${\n            e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n          }`,\n        );\n        throw e;\n      }\n    };\n\n    void this.queue.put(processBlock);\n  }\n\n  @Interval(15000)\n  async sampleWorkerStatus(): Promise<void> {\n    for (const worker of this.workers) {\n      const status = await worker.getStatus();\n      logger.info(JSON.stringify(status));\n    }\n  }\n\n  // Getter doesn't seem to cary from abstract class\n  get latestBufferedHeight(): number {\n    return this._latestBufferedHeight;\n  }\n\n  set latestBufferedHeight(height: number) {\n    super.latestBufferedHeight = height;\n    // There is only a single queue with workers so we treat them as the same\n    this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n      value: this.queueSize,\n    });\n  }\n\n  private getNextWorkerIndex(): number {\n    const index = this.taskCounter % this.numWorkers;\n\n    this.taskCounter++;\n\n    return index;\n  }\n}\n"]}

package/dist/indexer/dynamic-ds.service.d.ts

@@ -15,6 +15,7 @@ export declare class DynamicDsService {
     constructor(dsProcessorService: DsProcessorService, project: SubqueryProject);
     init(metaDataRepo: MetadataRepo): void;
     private _datasources;
+    resetDynamicDatasource(targetHeight: number, tx: Transaction): Promise<void>;
     createDynamicDatasource(params: DatasourceParams, tx: Transaction): Promise<SubqlProjectDs>;
     getDynamicDatasources(): Promise<SubqlProjectDs[]>;
     deleteTempDsRecords(blockHeight: number): void;

package/dist/indexer/dynamic-ds.service.js

@@ -33,6 +33,14 @@ let DynamicDsService = class DynamicDsService {
     init(metaDataRepo) {
         this.metaDataRepo = metaDataRepo;
     }
+    async resetDynamicDatasource(targetHeight, tx) {
+        const dynamicDs = await this.getDynamicDatasourceParams();
+        if (dynamicDs.length !== 0) {
+            const filteredDs = dynamicDs.filter((ds) => ds.startBlock <= targetHeight);
+            const dsRecords = JSON.stringify(filteredDs);
+            await this.metaDataRepo.upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx });
+        }
+    }
     async createDynamicDatasource(params, tx) {
         try {
             const ds = await this.getDatasource(params);
@@ -65,22 +73,21 @@ let DynamicDsService = class DynamicDsService {
         delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];
     }
     async getDynamicDatasourceParams(blockHeight) {
-        var _a;
+        var _a, _b, _c;
         (0, assert_1.default)(this.metaDataRepo, `Model _metadata does not exist`);
         const record = await this.metaDataRepo.findByPk(METADATA_KEY);
-        let results = record === null || record === void 0 ? void 0 : record.value;
-        if (!results || typeof results !== 'string') {
-            if (blockHeight !== undefined) {
-                results = (_a = this.tempDsRecords) === null || _a === void 0 ? void 0 : _a[TEMP_DS_PREFIX + blockHeight];
-                if (!results || typeof results !== 'string') {
-                    return [];
-                }
-            }
-            else {
-                return [];
+        let results = [];
+        const metaResults = JSON.parse((_a = record === null || record === void 0 ? void 0 : record.value) !== null && _a !== void 0 ? _a : '[]');
+        if (metaResults.length) {
+            results = [...metaResults];
+        }
+        if (blockHeight !== undefined) {
+            const tempResults = JSON.parse((_c = (_b = this.tempDsRecords) === null || _b === void 0 ? void 0 : _b[TEMP_DS_PREFIX + blockHeight]) !== null && _c !== void 0 ? _c : '[]');
+            if (tempResults.length) {
+                results = (0, lodash_1.unionWith)(results, tempResults, lodash_1.isEqual);
             }
         }
-        return JSON.parse(results);
+        return results;
     }
     async saveDynamicDatasourceParams(dsParams, tx) {
         const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);