@subql/node-ethereum 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -94,7 +94,7 @@ let BlockDispatcherService = class BlockDispatcherService extends base_block_dis
|
|
|
94
94
|
}
|
|
95
95
|
const blockTasks = blocks.map((block) => async () => {
|
|
96
96
|
var _a;
|
|
97
|
-
const height = block.
|
|
97
|
+
const height = block.blockHeight;
|
|
98
98
|
try {
|
|
99
99
|
this.preProcessBlock(height);
|
|
100
100
|
const processBlockResponse = await this.indexerManager.indexBlock(block);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAAmE;AACnE,yDAAsD;AACtD,gDAS0B;AAC1B,mCAA8B;
|
|
1
|
+
{"version":3,"file":"block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAAmE;AACnE,yDAAsD;AACtD,gDAS0B;AAC1B,mCAA8B;AAE9B,wDAAoD;AACpD,wDAAoD;AACpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,wBAAwB,CAAC,CAAC;AAEnD;;GAEG;AAEI,IAAM,sBAAsB,GAA5B,MAAM,sBACX,SAAQ,2CAAkC;IAS1C,YACU,UAAsB,EAC9B,UAAsB,EACd,cAA8B,EACtC,YAA2B,EAC3B,cAA8B;QAE9B,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,iBAAK,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACpC,CAAC;QAXM,eAAU,GAAV,UAAU,CAAY;QAEtB,mBAAc,GAAd,cAAc,CAAgB;QAPhC,aAAQ,GAAG,KAAK,CAAC;QACjB,eAAU,GAAG,KAAK,CAAC;QAgBzB,IAAI,CAAC,YAAY,GAAG,IAAI,qBAAS,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QAE5D,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CACtD,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC5B,IAAI,CAAC,kBAAkB,GAAG,IAAA,wBAAY,EACpC,WAAW,EACX,cAAc,EACd,oBAAoB,CACrB,CAAC;SACH;aAAM;YACL,IAAI,CAAC,kBAAkB,GAAG,WAAW,CAAC;SACvC;IACH,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,aAAa,CAAC,OAAiB;QAC7B,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QAE5B,MAAM,CAAC,IAAI,CACT,mBAAmB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,WAC9C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,oBAAoB,GAAG,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;QAE1C,KAAK,IAAI,CAAC,oBAAoB,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3C,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,mCAAmC,CAAC,CAAC;YACrD,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,UAAU,CAAC,MAAc;QACvB,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,oBAAoB;QAChC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7C,0DAA0D;QAC1D,uEAAuE;QAEvE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,IAAI;YACF,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE;gBACvB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CACnC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CACjE,CAAC;gBACF,0EAA0E;gBAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;gBAElD,iBAAiB;gBACjB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;oBACrB,kFAAkF;oBAClF,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;wBACnB,MAAM,IAAA,iBAAK,EAAC,CAAC,CAAC,CAAC;wBACf,SAAS;qBACV;oBACD,MAAM;iBACP;gBAED,MAAM,CAAC,IAAI,CACT,gBAAgB,SAAS,CAAC,CAAC,CAAC,IAC1B,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAChC,YAAY,SAAS,CAAC,MAAM,SAAS,CACtC,CAAC;gBAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;gBAExD,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE;oBAC/C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,SAAS;iBACV;gBACD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE;;oBAClD,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC;oBACjC,IAAI;wBACF,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;wBAE7B,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAC/D,KAAK,CACN,CAAC;wBAEF,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;qBAC3D;oBAAC,OAAO,CAAC,EAAE;wBACV,IAAI,IAAI,CAAC,UAAU,EAAE;4BACnB,OAAO;yBACR;wBACD,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;wBACF,MAAM,CAAC,CAAC;qBACT;gBACH,CAAC,CAAC,CAAC;gBAEH,uFAAuF;gBACvF,IAAI,IAAI,CAAC,UAAU;oBAAE,MAAM;gBAE3B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBAEtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;oBAClD,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI;iBAC9B,CAAC,CAAC;aACJ;SACF;gBAAS;YACR,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;SACvB;IACH,CAAC;CACF,CAAA;AAxJY,sBAAsB;IADlC,IAAA,mBAAU,GAAE;qCAYW,sBAAU;QAClB,sBAAU;QACE,gCAAc;QACxB,6BAAa;QACX,gCAAc;GAfrB,sBAAsB,CAwJlC;AAxJY,wDAAsB","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n ApiService,\n getLogger,\n NodeConfig,\n IndexerEvent,\n delay,\n profilerWrap,\n AutoQueue,\n Queue,\n} from '@subql/node-core';\nimport { last } from 'lodash';\nimport { EthereumApiService } from '../../ethereum';\nimport { IndexerManager } from '../indexer.manager';\nimport { ProjectService } from '../project.service';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('BlockDispatcherService');\n\n/**\n * @description Intended to behave the same as WorkerBlockDispatcherService but doesn't use worker threads or any parallel processing\n */\n@Injectable()\nexport class BlockDispatcherService\n extends BaseBlockDispatcher<Queue<number>>\n implements OnApplicationShutdown\n{\n private processQueue: AutoQueue<void>;\n\n private fetching = false;\n private isShutdown = false;\n private fetchBlocksBatches: EthereumApiService['api']['fetchBlocks'];\n\n constructor(\n private apiService: ApiService,\n nodeConfig: NodeConfig,\n private indexerManager: IndexerManager,\n eventEmitter: EventEmitter2,\n projectService: ProjectService,\n ) {\n super(\n nodeConfig,\n eventEmitter,\n projectService,\n new Queue(nodeConfig.batchSize * 3),\n );\n this.processQueue = new AutoQueue(nodeConfig.batchSize * 3);\n\n const fetchBlocks = this.apiService.api.fetchBlocks.bind(\n this.apiService.api,\n );\n if (this.nodeConfig.profiler) {\n this.fetchBlocksBatches = profilerWrap(\n fetchBlocks,\n 'EthereumUtil',\n 'fetchBlocksBatches',\n );\n } else {\n this.fetchBlocksBatches = fetchBlocks;\n }\n }\n\n // eslint-disable-next-line @typescript-eslint/require-await\n async init(\n onDynamicDsCreated: (height: number) => Promise<void>,\n ): Promise<void> {\n this.onDynamicDsCreated = onDynamicDsCreated;\n const blockAmount = await this.projectService.getProcessedBlockCount();\n this.setProcessedBlockCount(blockAmount ?? 0);\n }\n\n onApplicationShutdown(): void {\n this.isShutdown = true;\n this.processQueue.abort();\n }\n\n enqueueBlocks(heights: number[]): void {\n if (!heights.length) return;\n\n logger.info(\n `Enqueing blocks ${heights[0]}...${last(heights)}, total ${\n heights.length\n } blocks`,\n );\n\n this.queue.putMany(heights);\n this.latestBufferedHeight = last(heights);\n\n void this.fetchBlocksFromQueue().catch((e) => {\n logger.error(e, 'Failed to fetch blocks from queue');\n if (!this.isShutdown) {\n process.exit(1);\n }\n });\n }\n\n flushQueue(height: number): void {\n super.flushQueue(height);\n this.processQueue.flush();\n }\n\n private async fetchBlocksFromQueue(): Promise<void> {\n if (this.fetching || this.isShutdown) return;\n // Process queue is full, no point in fetching more blocks\n // if (this.processQueue.freeSpace < this.nodeConfig.batchSize) return;\n\n this.fetching = true;\n\n try {\n while (!this.isShutdown) {\n const blockNums = this.queue.takeMany(\n Math.min(this.nodeConfig.batchSize, this.processQueue.freeSpace),\n );\n // Used to compare before and after as a way to check if queue was flushed\n const bufferedHeight = this._latestBufferedHeight;\n\n // Queue is empty\n if (!blockNums.length) {\n // The process queue might be full so no block nums were taken, wait and try again\n if (this.queue.size) {\n await delay(1);\n continue;\n }\n break;\n }\n\n logger.info(\n `fetch block [${blockNums[0]},${\n blockNums[blockNums.length - 1]\n }], total ${blockNums.length} blocks`,\n );\n\n const blocks = await this.fetchBlocksBatches(blockNums);\n\n if (bufferedHeight > this._latestBufferedHeight) {\n logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n continue;\n }\n const blockTasks = blocks.map((block) => async () => {\n const height = block.blockHeight;\n try {\n this.preProcessBlock(height);\n\n const processBlockResponse = await this.indexerManager.indexBlock(\n block,\n );\n\n await this.postProcessBlock(height, processBlockResponse);\n } catch (e) {\n if (this.isShutdown) {\n return;\n }\n logger.error(\n e,\n `failed to index block at height ${height} ${\n e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n }`,\n );\n throw e;\n }\n });\n\n // There can be enough of a delay after fetching blocks that shutdown could now be true\n if (this.isShutdown) break;\n\n this.processQueue.putMany(blockTasks);\n\n this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n value: this.processQueue.size,\n });\n }\n } finally {\n this.fetching = false;\n }\n }\n}\n"]}
|