@subql/node-ethereum 0.4.1-11 → 0.4.1-12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -66,8 +66,10 @@ let WorkerBlockDispatcherService = class WorkerBlockDispatcherService extends ba
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
enqueueBlocks(heights, latestBufferHeight) {
|
|
69
|
-
if (!heights.length)
|
|
69
|
+
if (!!latestBufferHeight && !heights.length) {
|
|
70
|
+
this.latestBufferedHeight = latestBufferHeight;
|
|
70
71
|
return;
|
|
72
|
+
}
|
|
71
73
|
logger.info(`Enqueing blocks [${heights[0]}...${(0, lodash_1.last)(heights)}], total ${heights.length} blocks`);
|
|
72
74
|
// eslint-disable-next-line no-constant-condition
|
|
73
75
|
if (true) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAAmE;AACnE,yDAAsD;AACtD,+CAA4C;AAC5C,gDAM0B;AAC1B,kDAA0B;AAC1B,mCAA8B;AAC9B,wDAAoD;AAUpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAmBzD,KAAK,UAAU,mBAAmB;IAChC,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CACjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,iBAAiB;KAClB,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,2CAAoC;IAS5C,YACE,UAAsB,EACtB,YAA2B,EAC3B,cAA8B;QAE9B,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;QACtC,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,qBAAS,CAAC,UAAU,GAAG,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACrD,CAAC;QAdI,gBAAW,GAAG,CAAC,CAAC;QAChB,eAAU,GAAG,KAAK,CAAC;QAczB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;SACH;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,yBAAyB;QACzB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAEnB,mBAAmB;QACnB,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SAC3D;IACH,CAAC;IAED,aAAa,CAAC,OAAiB,EAAE,kBAA2B;QAC1D,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QAC5B,MAAM,CAAC,IAAI,CACT,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,YAC/C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,iDAAiD;QACjD,IAAI,IAAI,EAAE;YACR;;;;eAIG;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;SAC/D;aAAM;YACL;;;;eAIG;YACH,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CACrD,CAAC;SACH;QAED,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;IAClE,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,SAAiB;QACpD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,IAAA,gBAAM,EAAC,MAAM,EAAE,UAAU,SAAS,YAAY,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAEjD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;;YAC9B,IAAI;gBACF,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,IAAI,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE;oBAC9C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO;iBACR;gBAED,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjD,IAAI,QAAQ,GAAG,IAAI,EAAE;oBACnB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,CAAC,EAAE,CAClE,CAAC;iBACH;qBAAM,IAAI,QAAQ,GAAG,GAAG,EAAE;oBACzB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,MAAM,CAC/C,GAAG,QAAQ,IAAI,CAChB,EAAE,CACJ,CAAC;iBACH;gBAED,eAAe;gBACf,gKAAgK;gBAChK,KAAK;gBAEL,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAE7B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,kBAAkB,EAAE,GAC3D,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE;oBAClC,gBAAgB;oBAChB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;oBACnD,kBAAkB;iBACnB,CAAC,CAAC;gBAEH,IAAI,gBAAgB,EAAE;oBACpB,iDAAiD;oBACjD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;iBACjE;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC,CAAC;QAEF,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAGK,AAAN,KAAK,CAAC,kBAAkB;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,oBAAoB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAc;QACrC,KAAK,CAAC,oBAAoB,GAAG,MAAM,CAAC;QACpC,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;YAClD,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAA;AA3BO;IADL,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sEAMf;AAhKU,4BAA4B;IADxC,IAAA,mBAAU,GAAE;qCAYG,sBAAU;QACR,6BAAa;QACX,gCAAc;GAbrB,4BAA4B,CAsLxC;AAtLY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n getLogger,\n NodeConfig,\n IndexerEvent,\n Worker,\n AutoQueue,\n} from '@subql/node-core';\nimport chalk from 'chalk';\nimport { last } from 'lodash';\nimport { ProjectService } from '../project.service';\nimport {\n FetchBlock,\n ProcessBlock,\n InitWorker,\n NumFetchedBlocks,\n NumFetchingBlocks,\n GetWorkerStatus,\n ReloadDynamicDs,\n} from '../worker/worker';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IIndexerWorker = {\n processBlock: ProcessBlock;\n fetchBlock: FetchBlock;\n numFetchedBlocks: NumFetchedBlocks;\n numFetchingBlocks: NumFetchingBlocks;\n getStatus: GetWorkerStatus;\n reloadDynamicDs: ReloadDynamicDs;\n};\n\ntype IInitIndexerWorker = IIndexerWorker & {\n initWorker: InitWorker;\n};\n\ntype IndexerWorker = IIndexerWorker & {\n terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(): Promise<IndexerWorker> {\n const indexerWorker = Worker.create<IInitIndexerWorker>(\n path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n [\n 'initWorker',\n 'processBlock',\n 'fetchBlock',\n 'numFetchedBlocks',\n 'numFetchingBlocks',\n 'getStatus',\n 'reloadDynamicDs',\n ],\n );\n\n await indexerWorker.initWorker();\n\n return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n extends BaseBlockDispatcher<AutoQueue<void>>\n implements OnApplicationShutdown\n{\n private workers: IndexerWorker[];\n private numWorkers: number;\n\n private taskCounter = 0;\n private isShutdown = false;\n\n constructor(\n nodeConfig: NodeConfig,\n eventEmitter: EventEmitter2,\n projectService: ProjectService,\n ) {\n const numWorkers = nodeConfig.workers;\n super(\n nodeConfig,\n eventEmitter,\n projectService,\n new AutoQueue(numWorkers * nodeConfig.batchSize * 2),\n );\n this.numWorkers = numWorkers;\n }\n\n async init(\n onDynamicDsCreated: (height: number) => Promise<void>,\n ): Promise<void> {\n if (this.nodeConfig.unfinalizedBlocks) {\n throw new Error(\n 'Sorry, best block feature is not supported with workers yet.',\n );\n }\n\n this.workers = await Promise.all(\n new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()),\n );\n\n this.onDynamicDsCreated = onDynamicDsCreated;\n\n const blockAmount = await this.projectService.getProcessedBlockCount();\n this.setProcessedBlockCount(blockAmount ?? 0);\n }\n\n async onApplicationShutdown(): Promise<void> {\n this.isShutdown = true;\n // Stop processing blocks\n this.queue.abort();\n\n // Stop all workers\n if (this.workers) {\n await Promise.all(this.workers.map((w) => w.terminate()));\n }\n }\n\n enqueueBlocks(heights: number[], latestBufferHeight?: number): void {\n if (!heights.length) return;\n logger.info(\n `Enqueing blocks [${heights[0]}...${last(heights)}], total ${\n heights.length\n } blocks`,\n );\n\n // eslint-disable-next-line no-constant-condition\n if (true) {\n /*\n * Load balancing:\n * worker1: 1,2,3\n * worker2: 4,5,6\n */\n const workerIdx = this.getNextWorkerIndex();\n heights.map((height) => this.enqueueBlock(height, workerIdx));\n } else {\n /*\n * Load balancing:\n * worker1: 1,3,5\n * worker2: 2,4,6\n */\n heights.map((height) =>\n this.enqueueBlock(height, this.getNextWorkerIndex()),\n );\n }\n\n this.latestBufferedHeight = latestBufferHeight ?? last(heights);\n }\n\n private enqueueBlock(height: number, workerIdx: number) {\n if (this.isShutdown) return;\n const worker = this.workers[workerIdx];\n\n assert(worker, `Worker ${workerIdx} not found`);\n\n // Used to compare before and after as a way to check if queue was flushed\n const bufferedHeight = this.latestBufferedHeight;\n\n const pendingBlock = worker.fetchBlock(height);\n\n const processBlock = async () => {\n try {\n const start = new Date();\n const result = await pendingBlock;\n const end = new Date();\n\n if (bufferedHeight > this.latestBufferedHeight) {\n logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n return;\n }\n\n const waitTime = end.getTime() - start.getTime();\n if (waitTime > 1000) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n );\n } else if (waitTime > 200) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.yellow(\n `${waitTime}ms`,\n )}`,\n );\n }\n\n // logger.info(\n // `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,\n // );\n\n this.preProcessBlock(height);\n\n const { dynamicDsCreated, operationHash, reindexBlockHeight } =\n await worker.processBlock(height);\n\n await this.postProcessBlock(height, {\n dynamicDsCreated,\n operationHash: Buffer.from(operationHash, 'base64'),\n reindexBlockHeight,\n });\n\n if (dynamicDsCreated) {\n // Ensure all workers are aware of all dynamic ds\n await Promise.all(this.workers.map((w) => w.reloadDynamicDs()));\n }\n } catch (e) {\n logger.error(\n e,\n `failed to index block at height ${height} ${\n e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n }`,\n );\n process.exit(1);\n }\n };\n\n void this.queue.put(processBlock);\n }\n\n @Interval(15000)\n async sampleWorkerStatus(): Promise<void> {\n for (const worker of this.workers) {\n const status = await worker.getStatus();\n logger.info(JSON.stringify(status));\n }\n }\n\n // Getter doesn't seem to cary from abstract class\n get latestBufferedHeight(): number {\n return this._latestBufferedHeight;\n }\n\n set latestBufferedHeight(height: number) {\n super.latestBufferedHeight = height;\n // There is only a single queue with workers so we treat them as the same\n this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n value: this.queueSize,\n });\n }\n\n private getNextWorkerIndex(): number {\n const index = this.taskCounter % this.numWorkers;\n\n this.taskCounter++;\n\n return index;\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAAmE;AACnE,yDAAsD;AACtD,+CAA4C;AAC5C,gDAM0B;AAC1B,kDAA0B;AAC1B,mCAA8B;AAC9B,wDAAoD;AAUpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAmBzD,KAAK,UAAU,mBAAmB;IAChC,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CACjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,iBAAiB;KAClB,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,2CAAoC;IAS5C,YACE,UAAsB,EACtB,YAA2B,EAC3B,cAA8B;QAE9B,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;QACtC,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,qBAAS,CAAC,UAAU,GAAG,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACrD,CAAC;QAdI,gBAAW,GAAG,CAAC,CAAC;QAChB,eAAU,GAAG,KAAK,CAAC;QAczB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;SACH;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,yBAAyB;QACzB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAEnB,mBAAmB;QACnB,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SAC3D;IACH,CAAC;IAED,aAAa,CAAC,OAAiB,EAAE,kBAA2B;QAC1D,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAC3C,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,CAAC;YAC/C,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CACT,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,YAC/C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,iDAAiD;QACjD,IAAI,IAAI,EAAE;YACR;;;;eAIG;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;SAC/D;aAAM;YACL;;;;eAIG;YACH,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CACrD,CAAC;SACH;QAED,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;IAClE,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,SAAiB;QACpD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,IAAA,gBAAM,EAAC,MAAM,EAAE,UAAU,SAAS,YAAY,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAEjD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;;YAC9B,IAAI;gBACF,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,IAAI,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE;oBAC9C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO;iBACR;gBAED,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjD,IAAI,QAAQ,GAAG,IAAI,EAAE;oBACnB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,CAAC,EAAE,CAClE,CAAC;iBACH;qBAAM,IAAI,QAAQ,GAAG,GAAG,EAAE;oBACzB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,MAAM,CAC/C,GAAG,QAAQ,IAAI,CAChB,EAAE,CACJ,CAAC;iBACH;gBAED,eAAe;gBACf,gKAAgK;gBAChK,KAAK;gBAEL,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAE7B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,kBAAkB,EAAE,GAC3D,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE;oBAClC,gBAAgB;oBAChB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;oBACnD,kBAAkB;iBACnB,CAAC,CAAC;gBAEH,IAAI,gBAAgB,EAAE;oBACpB,iDAAiD;oBACjD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;iBACjE;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC,CAAC;QAEF,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAGK,AAAN,KAAK,CAAC,kBAAkB;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,oBAAoB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAc;QACrC,KAAK,CAAC,oBAAoB,GAAG,MAAM,CAAC;QACpC,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;YAClD,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAA;AA3BO;IADL,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sEAMf;AAnKU,4BAA4B;IADxC,IAAA,mBAAU,GAAE;qCAYG,sBAAU;QACR,6BAAa;QACX,gCAAc;GAbrB,4BAA4B,CAyLxC;AAzLY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n getLogger,\n NodeConfig,\n IndexerEvent,\n Worker,\n AutoQueue,\n} from '@subql/node-core';\nimport chalk from 'chalk';\nimport { last } from 'lodash';\nimport { ProjectService } from '../project.service';\nimport {\n FetchBlock,\n ProcessBlock,\n InitWorker,\n NumFetchedBlocks,\n NumFetchingBlocks,\n GetWorkerStatus,\n ReloadDynamicDs,\n} from '../worker/worker';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IIndexerWorker = {\n processBlock: ProcessBlock;\n fetchBlock: FetchBlock;\n numFetchedBlocks: NumFetchedBlocks;\n numFetchingBlocks: NumFetchingBlocks;\n getStatus: GetWorkerStatus;\n reloadDynamicDs: ReloadDynamicDs;\n};\n\ntype IInitIndexerWorker = IIndexerWorker & {\n initWorker: InitWorker;\n};\n\ntype IndexerWorker = IIndexerWorker & {\n terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(): Promise<IndexerWorker> {\n const indexerWorker = Worker.create<IInitIndexerWorker>(\n path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n [\n 'initWorker',\n 'processBlock',\n 'fetchBlock',\n 'numFetchedBlocks',\n 'numFetchingBlocks',\n 'getStatus',\n 'reloadDynamicDs',\n ],\n );\n\n await indexerWorker.initWorker();\n\n return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n extends BaseBlockDispatcher<AutoQueue<void>>\n implements OnApplicationShutdown\n{\n private workers: IndexerWorker[];\n private numWorkers: number;\n\n private taskCounter = 0;\n private isShutdown = false;\n\n constructor(\n nodeConfig: NodeConfig,\n eventEmitter: EventEmitter2,\n projectService: ProjectService,\n ) {\n const numWorkers = nodeConfig.workers;\n super(\n nodeConfig,\n eventEmitter,\n projectService,\n new AutoQueue(numWorkers * nodeConfig.batchSize * 2),\n );\n this.numWorkers = numWorkers;\n }\n\n async init(\n onDynamicDsCreated: (height: number) => Promise<void>,\n ): Promise<void> {\n if (this.nodeConfig.unfinalizedBlocks) {\n throw new Error(\n 'Sorry, best block feature is not supported with workers yet.',\n );\n }\n\n this.workers = await Promise.all(\n new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()),\n );\n\n this.onDynamicDsCreated = onDynamicDsCreated;\n\n const blockAmount = await this.projectService.getProcessedBlockCount();\n this.setProcessedBlockCount(blockAmount ?? 0);\n }\n\n async onApplicationShutdown(): Promise<void> {\n this.isShutdown = true;\n // Stop processing blocks\n this.queue.abort();\n\n // Stop all workers\n if (this.workers) {\n await Promise.all(this.workers.map((w) => w.terminate()));\n }\n }\n\n enqueueBlocks(heights: number[], latestBufferHeight?: number): void {\n if (!!latestBufferHeight && !heights.length) {\n this.latestBufferedHeight = latestBufferHeight;\n return;\n }\n logger.info(\n `Enqueing blocks [${heights[0]}...${last(heights)}], total ${\n heights.length\n } blocks`,\n );\n\n // eslint-disable-next-line no-constant-condition\n if (true) {\n /*\n * Load balancing:\n * worker1: 1,2,3\n * worker2: 4,5,6\n */\n const workerIdx = this.getNextWorkerIndex();\n heights.map((height) => this.enqueueBlock(height, workerIdx));\n } else {\n /*\n * Load balancing:\n * worker1: 1,3,5\n * worker2: 2,4,6\n */\n heights.map((height) =>\n this.enqueueBlock(height, this.getNextWorkerIndex()),\n );\n }\n\n this.latestBufferedHeight = latestBufferHeight ?? last(heights);\n }\n\n private enqueueBlock(height: number, workerIdx: number) {\n if (this.isShutdown) return;\n const worker = this.workers[workerIdx];\n\n assert(worker, `Worker ${workerIdx} not found`);\n\n // Used to compare before and after as a way to check if queue was flushed\n const bufferedHeight = this.latestBufferedHeight;\n\n const pendingBlock = worker.fetchBlock(height);\n\n const processBlock = async () => {\n try {\n const start = new Date();\n const result = await pendingBlock;\n const end = new Date();\n\n if (bufferedHeight > this.latestBufferedHeight) {\n logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n return;\n }\n\n const waitTime = end.getTime() - start.getTime();\n if (waitTime > 1000) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n );\n } else if (waitTime > 200) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.yellow(\n `${waitTime}ms`,\n )}`,\n );\n }\n\n // logger.info(\n // `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,\n // );\n\n this.preProcessBlock(height);\n\n const { dynamicDsCreated, operationHash, reindexBlockHeight } =\n await worker.processBlock(height);\n\n await this.postProcessBlock(height, {\n dynamicDsCreated,\n operationHash: Buffer.from(operationHash, 'base64'),\n reindexBlockHeight,\n });\n\n if (dynamicDsCreated) {\n // Ensure all workers are aware of all dynamic ds\n await Promise.all(this.workers.map((w) => w.reloadDynamicDs()));\n }\n } catch (e) {\n logger.error(\n e,\n `failed to index block at height ${height} ${\n e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n }`,\n );\n process.exit(1);\n }\n };\n\n void this.queue.put(processBlock);\n }\n\n @Interval(15000)\n async sampleWorkerStatus(): Promise<void> {\n for (const worker of this.workers) {\n const status = await worker.getStatus();\n logger.info(JSON.stringify(status));\n }\n }\n\n // Getter doesn't seem to cary from abstract class\n get latestBufferedHeight(): number {\n return this._latestBufferedHeight;\n }\n\n set latestBufferedHeight(height: number) {\n super.latestBufferedHeight = height;\n // There is only a single queue with workers so we treat them as the same\n this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n value: this.queueSize,\n });\n }\n\n private getNextWorkerIndex(): number {\n const index = this.taskCounter % this.numWorkers;\n\n this.taskCounter++;\n\n return index;\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@subql/node-ethereum",
|
|
3
|
-
"version": "0.4.1-
|
|
3
|
+
"version": "0.4.1-12",
|
|
4
4
|
"description": "",
|
|
5
5
|
"author": "Ian He",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -74,5 +74,5 @@
|
|
|
74
74
|
"/dist",
|
|
75
75
|
"/bin"
|
|
76
76
|
],
|
|
77
|
-
"stableVersion": "0.4.1-
|
|
77
|
+
"stableVersion": "0.4.1-11"
|
|
78
78
|
}
|