@subql/node-ethereum 0.4.1-10 → 0.4.1-12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js +3 -1
- package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js.map +1 -1
- package/dist/indexer/fetch.service.js +2 -2
- package/dist/indexer/fetch.service.js.map +1 -1
- package/package.json +2 -2
|
@@ -66,8 +66,10 @@ let WorkerBlockDispatcherService = class WorkerBlockDispatcherService extends ba
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
enqueueBlocks(heights, latestBufferHeight) {
|
|
69
|
-
if (!heights.length)
|
|
69
|
+
if (!!latestBufferHeight && !heights.length) {
|
|
70
|
+
this.latestBufferedHeight = latestBufferHeight;
|
|
70
71
|
return;
|
|
72
|
+
}
|
|
71
73
|
logger.info(`Enqueing blocks [${heights[0]}...${(0, lodash_1.last)(heights)}], total ${heights.length} blocks`);
|
|
72
74
|
// eslint-disable-next-line no-constant-condition
|
|
73
75
|
if (true) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAAmE;AACnE,yDAAsD;AACtD,+CAA4C;AAC5C,gDAM0B;AAC1B,kDAA0B;AAC1B,mCAA8B;AAC9B,wDAAoD;AAUpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAmBzD,KAAK,UAAU,mBAAmB;IAChC,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CACjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,iBAAiB;KAClB,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,2CAAoC;IAS5C,YACE,UAAsB,EACtB,YAA2B,EAC3B,cAA8B;QAE9B,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;QACtC,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,qBAAS,CAAC,UAAU,GAAG,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACrD,CAAC;QAdI,gBAAW,GAAG,CAAC,CAAC;QAChB,eAAU,GAAG,KAAK,CAAC;QAczB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;SACH;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,yBAAyB;QACzB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAEnB,mBAAmB;QACnB,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SAC3D;IACH,CAAC;IAED,aAAa,CAAC,OAAiB,EAAE,kBAA2B;QAC1D,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QAC5B,MAAM,CAAC,IAAI,CACT,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,YAC/C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,iDAAiD;QACjD,IAAI,IAAI,EAAE;YACR;;;;eAIG;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;SAC/D;aAAM;YACL;;;;eAIG;YACH,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CACrD,CAAC;SACH;QAED,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;IAClE,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,SAAiB;QACpD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,IAAA,gBAAM,EAAC,MAAM,EAAE,UAAU,SAAS,YAAY,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAEjD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;;YAC9B,IAAI;gBACF,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,IAAI,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE;oBAC9C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO;iBACR;gBAED,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjD,IAAI,QAAQ,GAAG,IAAI,EAAE;oBACnB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,CAAC,EAAE,CAClE,CAAC;iBACH;qBAAM,IAAI,QAAQ,GAAG,GAAG,EAAE;oBACzB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,MAAM,CAC/C,GAAG,QAAQ,IAAI,CAChB,EAAE,CACJ,CAAC;iBACH;gBAED,eAAe;gBACf,gKAAgK;gBAChK,KAAK;gBAEL,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAE7B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,kBAAkB,EAAE,GAC3D,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE;oBAClC,gBAAgB;oBAChB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;oBACnD,kBAAkB;iBACnB,CAAC,CAAC;gBAEH,IAAI,gBAAgB,EAAE;oBACpB,iDAAiD;oBACjD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;iBACjE;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC,CAAC;QAEF,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAGK,AAAN,KAAK,CAAC,kBAAkB;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,oBAAoB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAc;QACrC,KAAK,CAAC,oBAAoB,GAAG,MAAM,CAAC;QACpC,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;YAClD,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAA;AA3BO;IADL,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sEAMf;AAhKU,4BAA4B;IADxC,IAAA,mBAAU,GAAE;qCAYG,sBAAU;QACR,6BAAa;QACX,gCAAc;GAbrB,4BAA4B,CAsLxC;AAtLY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n getLogger,\n NodeConfig,\n IndexerEvent,\n Worker,\n AutoQueue,\n} from '@subql/node-core';\nimport chalk from 'chalk';\nimport { last } from 'lodash';\nimport { ProjectService } from '../project.service';\nimport {\n FetchBlock,\n ProcessBlock,\n InitWorker,\n NumFetchedBlocks,\n NumFetchingBlocks,\n GetWorkerStatus,\n ReloadDynamicDs,\n} from '../worker/worker';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IIndexerWorker = {\n processBlock: ProcessBlock;\n fetchBlock: FetchBlock;\n numFetchedBlocks: NumFetchedBlocks;\n numFetchingBlocks: NumFetchingBlocks;\n getStatus: GetWorkerStatus;\n reloadDynamicDs: ReloadDynamicDs;\n};\n\ntype IInitIndexerWorker = IIndexerWorker & {\n initWorker: InitWorker;\n};\n\ntype IndexerWorker = IIndexerWorker & {\n terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(): Promise<IndexerWorker> {\n const indexerWorker = Worker.create<IInitIndexerWorker>(\n path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n [\n 'initWorker',\n 'processBlock',\n 'fetchBlock',\n 'numFetchedBlocks',\n 'numFetchingBlocks',\n 'getStatus',\n 'reloadDynamicDs',\n ],\n );\n\n await indexerWorker.initWorker();\n\n return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n extends BaseBlockDispatcher<AutoQueue<void>>\n implements OnApplicationShutdown\n{\n private workers: IndexerWorker[];\n private numWorkers: number;\n\n private taskCounter = 0;\n private isShutdown = false;\n\n constructor(\n nodeConfig: NodeConfig,\n eventEmitter: EventEmitter2,\n projectService: ProjectService,\n ) {\n const numWorkers = nodeConfig.workers;\n super(\n nodeConfig,\n eventEmitter,\n projectService,\n new AutoQueue(numWorkers * nodeConfig.batchSize * 2),\n );\n this.numWorkers = numWorkers;\n }\n\n async init(\n onDynamicDsCreated: (height: number) => Promise<void>,\n ): Promise<void> {\n if (this.nodeConfig.unfinalizedBlocks) {\n throw new Error(\n 'Sorry, best block feature is not supported with workers yet.',\n );\n }\n\n this.workers = await Promise.all(\n new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()),\n );\n\n this.onDynamicDsCreated = onDynamicDsCreated;\n\n const blockAmount = await this.projectService.getProcessedBlockCount();\n this.setProcessedBlockCount(blockAmount ?? 0);\n }\n\n async onApplicationShutdown(): Promise<void> {\n this.isShutdown = true;\n // Stop processing blocks\n this.queue.abort();\n\n // Stop all workers\n if (this.workers) {\n await Promise.all(this.workers.map((w) => w.terminate()));\n }\n }\n\n enqueueBlocks(heights: number[], latestBufferHeight?: number): void {\n if (!heights.length) return;\n logger.info(\n `Enqueing blocks [${heights[0]}...${last(heights)}], total ${\n heights.length\n } blocks`,\n );\n\n // eslint-disable-next-line no-constant-condition\n if (true) {\n /*\n * Load balancing:\n * worker1: 1,2,3\n * worker2: 4,5,6\n */\n const workerIdx = this.getNextWorkerIndex();\n heights.map((height) => this.enqueueBlock(height, workerIdx));\n } else {\n /*\n * Load balancing:\n * worker1: 1,3,5\n * worker2: 2,4,6\n */\n heights.map((height) =>\n this.enqueueBlock(height, this.getNextWorkerIndex()),\n );\n }\n\n this.latestBufferedHeight = latestBufferHeight ?? last(heights);\n }\n\n private enqueueBlock(height: number, workerIdx: number) {\n if (this.isShutdown) return;\n const worker = this.workers[workerIdx];\n\n assert(worker, `Worker ${workerIdx} not found`);\n\n // Used to compare before and after as a way to check if queue was flushed\n const bufferedHeight = this.latestBufferedHeight;\n\n const pendingBlock = worker.fetchBlock(height);\n\n const processBlock = async () => {\n try {\n const start = new Date();\n const result = await pendingBlock;\n const end = new Date();\n\n if (bufferedHeight > this.latestBufferedHeight) {\n logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n return;\n }\n\n const waitTime = end.getTime() - start.getTime();\n if (waitTime > 1000) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n );\n } else if (waitTime > 200) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.yellow(\n `${waitTime}ms`,\n )}`,\n );\n }\n\n // logger.info(\n // `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,\n // );\n\n this.preProcessBlock(height);\n\n const { dynamicDsCreated, operationHash, reindexBlockHeight } =\n await worker.processBlock(height);\n\n await this.postProcessBlock(height, {\n dynamicDsCreated,\n operationHash: Buffer.from(operationHash, 'base64'),\n reindexBlockHeight,\n });\n\n if (dynamicDsCreated) {\n // Ensure all workers are aware of all dynamic ds\n await Promise.all(this.workers.map((w) => w.reloadDynamicDs()));\n }\n } catch (e) {\n logger.error(\n e,\n `failed to index block at height ${height} ${\n e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n }`,\n );\n process.exit(1);\n }\n };\n\n void this.queue.put(processBlock);\n }\n\n @Interval(15000)\n async sampleWorkerStatus(): Promise<void> {\n for (const worker of this.workers) {\n const status = await worker.getStatus();\n logger.info(JSON.stringify(status));\n }\n }\n\n // Getter doesn't seem to cary from abstract class\n get latestBufferedHeight(): number {\n return this._latestBufferedHeight;\n }\n\n set latestBufferedHeight(height: number) {\n super.latestBufferedHeight = height;\n // There is only a single queue with workers so we treat them as the same\n this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n value: this.queueSize,\n });\n }\n\n private getNextWorkerIndex(): number {\n const index = this.taskCounter % this.numWorkers;\n\n this.taskCounter++;\n\n return index;\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAAmE;AACnE,yDAAsD;AACtD,+CAA4C;AAC5C,gDAM0B;AAC1B,kDAA0B;AAC1B,mCAA8B;AAC9B,wDAAoD;AAUpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAmBzD,KAAK,UAAU,mBAAmB;IAChC,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CACjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,iBAAiB;KAClB,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,2CAAoC;IAS5C,YACE,UAAsB,EACtB,YAA2B,EAC3B,cAA8B;QAE9B,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;QACtC,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,qBAAS,CAAC,UAAU,GAAG,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACrD,CAAC;QAdI,gBAAW,GAAG,CAAC,CAAC;QAChB,eAAU,GAAG,KAAK,CAAC;QAczB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;SACH;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,yBAAyB;QACzB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAEnB,mBAAmB;QACnB,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SAC3D;IACH,CAAC;IAED,aAAa,CAAC,OAAiB,EAAE,kBAA2B;QAC1D,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAC3C,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,CAAC;YAC/C,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CACT,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,YAC/C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,iDAAiD;QACjD,IAAI,IAAI,EAAE;YACR;;;;eAIG;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;SAC/D;aAAM;YACL;;;;eAIG;YACH,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CACrD,CAAC;SACH;QAED,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;IAClE,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,SAAiB;QACpD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,IAAA,gBAAM,EAAC,MAAM,EAAE,UAAU,SAAS,YAAY,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAEjD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;;YAC9B,IAAI;gBACF,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,IAAI,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE;oBAC9C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO;iBACR;gBAED,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjD,IAAI,QAAQ,GAAG,IAAI,EAAE;oBACnB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,CAAC,EAAE,CAClE,CAAC;iBACH;qBAAM,IAAI,QAAQ,GAAG,GAAG,EAAE;oBACzB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,MAAM,CAC/C,GAAG,QAAQ,IAAI,CAChB,EAAE,CACJ,CAAC;iBACH;gBAED,eAAe;gBACf,gKAAgK;gBAChK,KAAK;gBAEL,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAE7B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,kBAAkB,EAAE,GAC3D,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE;oBAClC,gBAAgB;oBAChB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;oBACnD,kBAAkB;iBACnB,CAAC,CAAC;gBAEH,IAAI,gBAAgB,EAAE;oBACpB,iDAAiD;oBACjD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;iBACjE;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC,CAAC;QAEF,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAGK,AAAN,KAAK,CAAC,kBAAkB;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,oBAAoB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAc;QACrC,KAAK,CAAC,oBAAoB,GAAG,MAAM,CAAC;QACpC,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;YAClD,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAA;AA3BO;IADL,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sEAMf;AAnKU,4BAA4B;IADxC,IAAA,mBAAU,GAAE;qCAYG,sBAAU;QACR,6BAAa;QACX,gCAAc;GAbrB,4BAA4B,CAyLxC;AAzLY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n getLogger,\n NodeConfig,\n IndexerEvent,\n Worker,\n AutoQueue,\n} from '@subql/node-core';\nimport chalk from 'chalk';\nimport { last } from 'lodash';\nimport { ProjectService } from '../project.service';\nimport {\n FetchBlock,\n ProcessBlock,\n InitWorker,\n NumFetchedBlocks,\n NumFetchingBlocks,\n GetWorkerStatus,\n ReloadDynamicDs,\n} from '../worker/worker';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IIndexerWorker = {\n processBlock: ProcessBlock;\n fetchBlock: FetchBlock;\n numFetchedBlocks: NumFetchedBlocks;\n numFetchingBlocks: NumFetchingBlocks;\n getStatus: GetWorkerStatus;\n reloadDynamicDs: ReloadDynamicDs;\n};\n\ntype IInitIndexerWorker = IIndexerWorker & {\n initWorker: InitWorker;\n};\n\ntype IndexerWorker = IIndexerWorker & {\n terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(): Promise<IndexerWorker> {\n const indexerWorker = Worker.create<IInitIndexerWorker>(\n path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n [\n 'initWorker',\n 'processBlock',\n 'fetchBlock',\n 'numFetchedBlocks',\n 'numFetchingBlocks',\n 'getStatus',\n 'reloadDynamicDs',\n ],\n );\n\n await indexerWorker.initWorker();\n\n return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n extends BaseBlockDispatcher<AutoQueue<void>>\n implements OnApplicationShutdown\n{\n private workers: IndexerWorker[];\n private numWorkers: number;\n\n private taskCounter = 0;\n private isShutdown = false;\n\n constructor(\n nodeConfig: NodeConfig,\n eventEmitter: EventEmitter2,\n projectService: ProjectService,\n ) {\n const numWorkers = nodeConfig.workers;\n super(\n nodeConfig,\n eventEmitter,\n projectService,\n new AutoQueue(numWorkers * nodeConfig.batchSize * 2),\n );\n this.numWorkers = numWorkers;\n }\n\n async init(\n onDynamicDsCreated: (height: number) => Promise<void>,\n ): Promise<void> {\n if (this.nodeConfig.unfinalizedBlocks) {\n throw new Error(\n 'Sorry, best block feature is not supported with workers yet.',\n );\n }\n\n this.workers = await Promise.all(\n new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()),\n );\n\n this.onDynamicDsCreated = onDynamicDsCreated;\n\n const blockAmount = await this.projectService.getProcessedBlockCount();\n this.setProcessedBlockCount(blockAmount ?? 0);\n }\n\n async onApplicationShutdown(): Promise<void> {\n this.isShutdown = true;\n // Stop processing blocks\n this.queue.abort();\n\n // Stop all workers\n if (this.workers) {\n await Promise.all(this.workers.map((w) => w.terminate()));\n }\n }\n\n enqueueBlocks(heights: number[], latestBufferHeight?: number): void {\n if (!!latestBufferHeight && !heights.length) {\n this.latestBufferedHeight = latestBufferHeight;\n return;\n }\n logger.info(\n `Enqueing blocks [${heights[0]}...${last(heights)}], total ${\n heights.length\n } blocks`,\n );\n\n // eslint-disable-next-line no-constant-condition\n if (true) {\n /*\n * Load balancing:\n * worker1: 1,2,3\n * worker2: 4,5,6\n */\n const workerIdx = this.getNextWorkerIndex();\n heights.map((height) => this.enqueueBlock(height, workerIdx));\n } else {\n /*\n * Load balancing:\n * worker1: 1,3,5\n * worker2: 2,4,6\n */\n heights.map((height) =>\n this.enqueueBlock(height, this.getNextWorkerIndex()),\n );\n }\n\n this.latestBufferedHeight = latestBufferHeight ?? last(heights);\n }\n\n private enqueueBlock(height: number, workerIdx: number) {\n if (this.isShutdown) return;\n const worker = this.workers[workerIdx];\n\n assert(worker, `Worker ${workerIdx} not found`);\n\n // Used to compare before and after as a way to check if queue was flushed\n const bufferedHeight = this.latestBufferedHeight;\n\n const pendingBlock = worker.fetchBlock(height);\n\n const processBlock = async () => {\n try {\n const start = new Date();\n const result = await pendingBlock;\n const end = new Date();\n\n if (bufferedHeight > this.latestBufferedHeight) {\n logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n return;\n }\n\n const waitTime = end.getTime() - start.getTime();\n if (waitTime > 1000) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n );\n } else if (waitTime > 200) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.yellow(\n `${waitTime}ms`,\n )}`,\n );\n }\n\n // logger.info(\n // `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,\n // );\n\n this.preProcessBlock(height);\n\n const { dynamicDsCreated, operationHash, reindexBlockHeight } =\n await worker.processBlock(height);\n\n await this.postProcessBlock(height, {\n dynamicDsCreated,\n operationHash: Buffer.from(operationHash, 'base64'),\n reindexBlockHeight,\n });\n\n if (dynamicDsCreated) {\n // Ensure all workers are aware of all dynamic ds\n await Promise.all(this.workers.map((w) => w.reloadDynamicDs()));\n }\n } catch (e) {\n logger.error(\n e,\n `failed to index block at height ${height} ${\n e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n }`,\n );\n process.exit(1);\n }\n };\n\n void this.queue.put(processBlock);\n }\n\n @Interval(15000)\n async sampleWorkerStatus(): Promise<void> {\n for (const worker of this.workers) {\n const status = await worker.getStatus();\n logger.info(JSON.stringify(status));\n }\n }\n\n // Getter doesn't seem to cary from abstract class\n get latestBufferedHeight(): number {\n return this._latestBufferedHeight;\n }\n\n set latestBufferedHeight(height: number) {\n super.latestBufferedHeight = height;\n // There is only a single queue with workers so we treat them as the same\n this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n value: this.queueSize,\n });\n }\n\n private getNextWorkerIndex(): number {\n const index = this.taskCounter % this.numWorkers;\n\n this.taskCounter++;\n\n return index;\n }\n}\n"]}
|
|
@@ -457,13 +457,13 @@ __decorate([
|
|
|
457
457
|
__metadata("design:returntype", void 0)
|
|
458
458
|
], FetchService.prototype, "checkBatchScale", null);
|
|
459
459
|
__decorate([
|
|
460
|
-
(0, schedule_1.Interval)(BLOCK_TIME_VARIANCE
|
|
460
|
+
(0, schedule_1.Interval)(BLOCK_TIME_VARIANCE),
|
|
461
461
|
__metadata("design:type", Function),
|
|
462
462
|
__metadata("design:paramtypes", []),
|
|
463
463
|
__metadata("design:returntype", Promise)
|
|
464
464
|
], FetchService.prototype, "getFinalizedBlockHead", null);
|
|
465
465
|
__decorate([
|
|
466
|
-
(0, schedule_1.Interval)(BLOCK_TIME_VARIANCE
|
|
466
|
+
(0, schedule_1.Interval)(BLOCK_TIME_VARIANCE),
|
|
467
467
|
__metadata("design:type", Function),
|
|
468
468
|
__metadata("design:paramtypes", []),
|
|
469
469
|
__metadata("design:returntype", Promise)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.service.js","sourceRoot":"","sources":["../../src/indexer/fetch.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAA2E;AAC3E,yDAAsD;AACtD,+CAA+D;AAC/D,4DAIgC;AAChC,gDAU0B;AAU1B,mCAAsE;AACtE,kEAA+E;AAC/E,+DAA0D;AAC1D,4CAAkE;AAElE,6DAAyD;AACzD,6DAAwD;AACxD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,OAAO,CAAC,CAAC;AAClC,IAAI,mBAAmB,GAAG,IAAI,CAAC;AAC/B,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,SAAS,uBAAuB,CAC9B,MAAyB,EACzB,SAA0E;IAE1E,MAAM,UAAU,GAA+B,EAAE,CAAC;IAElD,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QAC5B,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE5E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,mBAAmB,EAAE;YACrE,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,eAAe;aACzB,CAAC,CAAC;SACJ;KACF;SAAM;QACL,IAAI,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,OAAO,EAAE;YACtB,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE;gBACtC,qBAAqB;aACtB,CAAC,CAAC;SACJ;KACF;IACD,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE;gBACV,SAAS;aACV;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK;gBACL,KAAK,EAAE,IAAA,qBAAY,EAAC,KAAK,CAAC;gBAC1B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;SACJ;KACF;IACD,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiC;IAEjC,MAAM,UAAU,GAA+B,EAAE,CAAC;IAClD,IAAI,MAAM,CAAC,IAAI,EAAE;QACf,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;SACjC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,EAAE,EAAE;QACb,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;SAC/B,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE;QACnB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,IAAA,0BAAiB,EAAC,MAAM,CAAC,QAAQ,CAAC;YACzC,OAAO,EAAE,SAAS;SACnB,CAAC,CAAC;KACJ;IACD,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,UAAU;KACX,CAAC;AACJ,CAAC;AAGM,IAAM,YAAY,GAAlB,MAAM,YAAY;IAUvB,YACU,UAAsB,EACtB,UAAsB,EACM,OAAwB,EACxB,eAAiC,EAC7D,iBAAoC,EACpC,gBAAkC,EAClC,wBAAkD,EAClD,YAA2B,EAC3B,iBAAoC;QARpC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACxB,oBAAe,GAAf,eAAe,CAAkB;QAC7D,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,iBAAY,GAAZ,YAAY,CAAe;QAC3B,sBAAiB,GAAjB,iBAAiB,CAAmB;QAhBtC,eAAU,GAAG,KAAK,CAAC;QAGnB,6BAAwB,GAAG,IAAI,CAAC;QAEhC,iBAAY,GAAa,EAAE,CAAC;QAalC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,IAAI,CAAC,yBAAyB;YAC5B,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;IACxD,CAAC;IAED,2BAA2B,CAAC,UAAkB;;QAC5C,MAAM,YAAY,GAA2B,EAAE,CAAC;QAMhD,MAAM,eAAe,GAA4B,MAAM,CAAC,MAAM,CAC5D,IAAA,gBAAO,EAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CACzD,CAAC,GAAG,CAAC,CAAC,OAAyB,EAAE,EAAE;YAClC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,uCACK,GAAG,KACN,cAAc,EAAE,OAAO,IACvB;QACJ,CAAC,CAAC,CAAC;QAEH,wDAAwD;QACxD,kDAAkD;QAClD,MAAM,UAAU,GAA4B,IAAI,CAAC,OAAO,CAAC,WAAW;aACjE,MAAM,CAAC,eAAe,CAAC;aACvB,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,UAAU,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAE/C,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE;YAC3B,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,kCAAkC;gBAClC,IAAI,CAAC,OAAO,CAAC,MAAM;oBAAE,OAAO,EAAE,CAAC;gBAE/B,QAAQ,OAAO,CAAC,IAAI,EAAE;oBACpB,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,EAAE,CAAC;oBACZ,KAAK,qCAAmB,CAAC,IAAI,CAAC,CAAC;wBAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAmC,CAAC;wBAC3D,IACE,MAAM,CAAC,IAAI,KAAK,SAAS;4BACzB,MAAM,CAAC,EAAE,KAAK,SAAS;4BACvB,MAAM,CAAC,QAAQ,EACf;4BACA,YAAY,CAAC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;yBACnD;6BAAM;4BACL,OAAO,EAAE,CAAC;yBACX;wBACD,MAAM;qBACP;oBACD,KAAK,qCAAmB,CAAC,KAAK,CAAC,CAAC;wBAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,MAA2B,CAAC;wBACnD,IAAI,EAAE,CAAC,cAAc,EAAE;4BACrB,YAAY,CAAC,IAAI,CACf,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,CACnD,CAAC;yBACH;6BAAM,IAAI,CAAA,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,KAAI,MAAM,CAAC,MAAM,EAAE;4BAC/C,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;yBAChE;6BAAM;4BACL,OAAO,EAAE,CAAC;yBACX;wBACD,MAAM;qBACP;oBACD,QAAQ;iBACT;aACF;SACF;QAED,OAAO,IAAA,eAAM,EACX,YAAY,EACZ,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAC9B,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACxC,EAAE,CACN,CAAC;IACJ,CAAC;IAED,gBAAgB;QACd,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAC5C,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,EAC/D,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAC5C,CAAC;IACJ,CAAC;IAED,IAAY,aAAa;;QACvB,OAAO,CACL,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU;YACjC,IAAI,CAAC,wBAAwB;YAC7B,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,CAChD,MAAA,IAAI,CAAC,eAAe,CAAC,oBAAoB,mCACvC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CACnE,CAAC,MAAM,CACT,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;;QAC5B,IAAI,CAAA,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,0CAAE,YAAY,MAAK,SAAS,EAAE;YACpD,IAAI,CAAC,YAAY,GAAG,IAAA,iCAAqB,EACvC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAClC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,WAAW,CAAC,CAAC;SACvC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE;YACZ,MAAM,cAAc,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC;YAEjE,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;YAEpE,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAChC,oBAAoB,EACpB,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,gBAAgB,EAAE,EAAE,mBAAmB,CAAC,CACrE,CAAC;SACH;QACD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAE7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;YACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE;YACnC,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;SAChE;QAED,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9B,4DAA4D;QAC5D,iHAAiH;QACjH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,CAAC;QAE5D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAEzD,IAAI,YAAY,EAAE;YAChB,IAAI,CAAC,iBAAiB,CAAC,wBAAwB,CAC7C,QAAQ,CAAC,SAAS,CAAC,WAAW,CAC/B,CAAC;SACH;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,wBAAwB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAGD,eAAe;QACb,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE;YACvC,MAAM,KAAK,GAAG,IAAA,4BAAgB,EAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAErE,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,EAAE;gBACjC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;aAC7B;SACF;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC9D,OAAO;SACR;QACD,IAAI;YACF,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,aAAa,sBAAsB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC3D,sBAAsB,CACvB,CAAC;YACF,IAAI,IAAI,CAAC,qBAAqB,KAAK,sBAAsB,EAAE;gBACzD,IAAI,CAAC,qBAAqB,GAAG,sBAAsB,CAAC;gBACpD,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;gBACtE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;wBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;4BAC/C,MAAM,EAAE,IAAI,CAAC,qBAAqB;yBACnC,CAAC,CAAC;qBACJ;iBACF;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;SAC7D;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,OAAO;SACR;QACD,IAAI;YACF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC9D,MAAM,CAAC,KAAK,CAAC,QAAQ,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,IAAI,CAAC,gBAAgB,KAAK,iBAAiB,EAAE;gBAC/C,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,SAAS,EAAE;oBAC7C,MAAM,EAAE,IAAI,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;gBAEH,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACrC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;wBAC/C,MAAM,EAAE,IAAI,CAAC,gBAAgB;qBAC9B,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,sCAAsC,CAAC,CAAC;SACxD;IACH,CAAC;IACO,KAAK,CAAC,SAAS,CAAC,eAAuB;QAC7C,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC;IAED,UAAU;QACR,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;YACzC,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;gBAClB,SAAS;aACV;YACD,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IACE,OAAO,CAAC,IAAI,KAAK,qCAAmB,CAAC,KAAK;oBAC1C,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,MAAM,CAAC,MAAM,EACrB;oBACA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iBACrC;aACF;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,eAAe,CAAC,WAAmB,EAAE,SAAiB;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,WAAW,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;YAC5C,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;gBACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB;SACF;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,uBAAuB,CAAC,gBAAwB;QAC9C,OAAO,IAAI,CAAC,eAAe,CACzB,gBAAgB,EAChB,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACxD,gBAAgB,CACnB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,eAAuB;QAC/C,IAAI,gBAAwB,CAAC;QAC7B,IAAI,eAAuB,CAAC;QAC5B,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,CACxB,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC7D,CAAC;QAEF,MAAM,mBAAmB,GAAG,GAAW,EAAE;YACvC,OAAO,IAAI,CAAC,eAAe,CAAC,oBAAoB;gBAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,GAAG,CAAC;gBAC/C,CAAC,CAAC,eAAe,CAAC;QACtB,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,GAAG,mBAAmB,EAAE,EAAE;YAC9D,MAAM,CAAC,IAAI,CACT,2BACE,IAAI,CAAC,iBAAiB,CAAC,WACzB,8BAA8B,mBAAmB,EAAE,+BAA+B,CACnF,CAAC;SACH;QAED,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE;YACvB,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;YAEzC,eAAe,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAC3D,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CAC5D,CAAC;YACF,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB;gBACpD,CAAC,CAAC,IAAI,CAAC,gBAAgB;gBACvB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC;YAE/B,IACE,IAAI,CAAC,eAAe,CAAC,QAAQ,GAAG,eAAe;gBAC/C,gBAAgB,GAAG,YAAY,EAC/B;gBACA,MAAM,IAAA,iBAAK,EAAC,CAAC,CAAC,CAAC;gBACf,SAAS;aACV;YAED,IACE,IAAI,CAAC,aAAa;gBAClB,gBAAgB,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,EACtD;gBACA,MAAM,aAAa,GAAG,gBAAgB,GAAG,yBAAyB,CAAC;gBACnE,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CACvC,gBAAgB,EAChB,aAAa,CACd,CAAC;gBAEF,IAAI;oBACF,MAAM,UAAU,GACd,MAAM,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAClD,gBAAgB,EAChB,aAAa,EACb,eAAe,CAChB,CAAC;oBAEJ,IAAI,gBAAgB,KAAK,mBAAmB,EAAE,EAAE;wBAC9C,MAAM,CAAC,KAAK,CACV,gEAAgE,CACjE,CAAC;wBACF,SAAS;qBACV;oBAED,IACE,UAAU;wBACV,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,gBAAgB,CAAC,EACvD;wBACA,IAAI,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;wBAEjC,WAAW,GAAG,WAAW;6BACtB,MAAM,CAAC,YAAY,CAAC;6BACpB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBACzB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;4BAC5B,iFAAiF;4BACjF,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,EAAE,EACF,IAAI,CAAC,GAAG,CACN,aAAa,GAAG,CAAC,EACjB,UAAU,CAAC,SAAS,CAAC,mBAAmB,CACzC,CACF,CAAC;yBACH;6BAAM;4BACL,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,WAAW,CAAC,MAAM,EAClB,IAAI,CAAC,eAAe,CAAC,QAAQ,CAC9B,CAAC;4BACF,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;4BAC3D,MAAM,kBAAkB,GACtB,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;4BAE3C,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;yBACH;wBACD,SAAS,CAAC,4BAA4B;qBACvC;oBACD,iCAAiC;iBAClC;gBAAC,OAAO,CAAC,EAAE;oBACV,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACvD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;iBACrD;aACF;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CACvC,gBAAgB,EAChB,eAAe,CAChB,CAAC;YAEF,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE;gBACnE,MAAM,eAAe,GAAG,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,CAAC,CAAC;gBACvE,MAAM,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBACpE,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;aACH;iBAAM;gBACL,MAAM,eAAe,GAAG,IAAA,cAAK,EAAC,gBAAgB,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;gBAC/D,MAAM,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBACpE,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;aACH;SACF;IACH,CAAC;IAEO,qBAAqB,CAC3B,kBAA4B,EAC5B,cAAwB;QAExB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,EAAE,GAAG,cAAc,CAAC,CAAC;IAC5D,CAAC;IACO,kBAAkB,CAAC,kBAA4B;QACrD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,EAAE;YACpD,OAAO,kBAAkB,CAAC;SAC3B;QAED,MAAM,YAAY,GAAG,IAAA,8BAAkB,EACrC,IAAI,CAAC,YAAY,EACjB,kBAAkB,CACnB,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAC7C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,CAC3C,CAAC;QACF,IAAI,cAAc,CAAC,MAAM,EAAE;YACzB,MAAM,CAAC,IAAI,CAAC,qBAAqB,cAAc,EAAE,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,YAAY,GAAG,IAAA,gBAAO,EAAC,IAAI,CAAC,YAAY,EAAE,GAAG,cAAc,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC;IACtB,CAAC;IAEO,kBAAkB,CACxB,gBAAwB,EACxB,eAAuB;QAEvB,IAAI,cAAc,GAAG,gBAAgB,GAAG,eAAe,GAAG,CAAC,CAAC;QAE5D,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE;YAC/C,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;gBACrC,IAAI,cAAc,IAAI,IAAI,CAAC,gBAAgB,EAAE;oBAC3C,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC;iBACxC;aACF;iBAAM;gBACL,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;aAC7C;SACF;QACD,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,oBAAoB,CAC1B,UAAmC,EACnC,gBAAyB;QAEzB,IAAI,UAAU,KAAK,SAAS,EAAE;YAC5B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;YAE3C,IACE,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE;gBAClD,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EACpD;gBACA,MAAM,CAAC,KAAK,CACV,+KAA+K,CAChL,CAAC;gBACF,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;gBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;oBACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;iBAClC,CAAC,CAAC;gBACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;gBACpD,OAAO,KAAK,CAAC;aACd;YAED,IAAI,gBAAgB,KAAK,SAAS,EAAE;gBAClC,IAAI,QAAQ,CAAC,mBAAmB,GAAG,gBAAgB,EAAE;oBACnD,MAAM,CAAC,IAAI,CACT,kEAAkE,CACnE,CAAC;oBACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;oBACpD,OAAO,KAAK,CAAC;iBACd;aACF;YACD,OAAO,IAAI,CAAC;SACb;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACrC,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAAC,WAAmB;QAClD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;CACF,CAAA;AAvUC;IAAC,IAAA,mBAAQ,EAAC,qBAAqB,CAAC;;;;mDAS/B;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,GAAG,IAAI,CAAC;;;;yDA0BpC;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,GAAG,IAAI,CAAC;;;;oDAwBpC;AAtPU,YAAY;IADxB,IAAA,mBAAU,GAAE;IAcR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;IAC1B,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHP,sBAAU;QACV,sBAAU;QACe,iCAAe,UAEjC,sCAAiB;QAClB,qCAAgB;QACR,oDAAwB;QACpC,6BAAa;QACR,4BAAiB;GAnBnC,YAAY,CA8fxB;AA9fY,oCAAY","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval, SchedulerRegistry } from '@nestjs/schedule';\nimport {\n isCustomDs,\n EthereumHandlerKind,\n SubqlHandlerFilter,\n} from '@subql/common-ethereum';\nimport {\n ApiService,\n Dictionary,\n checkMemoryUsage,\n delay,\n getLogger,\n IndexerEvent,\n NodeConfig,\n transformBypassBlocks,\n cleanedBatchBlocks,\n} from '@subql/node-core';\nimport {\n DictionaryQueryEntry,\n ApiWrapper,\n EthereumLogFilter,\n EthereumTransactionFilter,\n SubqlEthereumProcessorOptions,\n DictionaryQueryCondition,\n} from '@subql/types-ethereum';\nimport { MetaData } from '@subql/utils';\nimport { range, sortBy, uniqBy, without, groupBy, add } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { calcInterval } from '../ethereum/utils.ethereum';\nimport { eventToTopic, functionToSighash } from '../utils/string';\nimport { IBlockDispatcher } from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('fetch');\nlet BLOCK_TIME_VARIANCE = 5000;\nconst DICTIONARY_MAX_QUERY_SIZE = 10000;\nconst CHECK_MEMORY_INTERVAL = 60000;\nconst MINIMUM_BATCH_SIZE = 5;\nconst INTERVAL_PERCENT = 0.9;\nconst QUERY_ADDRESS_LIMIT = 50;\n\nfunction eventFilterToQueryEntry(\n filter: EthereumLogFilter,\n dsOptions: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[],\n): DictionaryQueryEntry {\n const conditions: DictionaryQueryCondition[] = [];\n\n if (Array.isArray(dsOptions)) {\n const addresses = dsOptions.map((option) => option.address).filter(Boolean);\n\n if (addresses.length !== 0 && addresses.length <= QUERY_ADDRESS_LIMIT) {\n conditions.push({\n field: 'address',\n value: addresses,\n matcher: 'inInsensitive',\n });\n }\n } else {\n if (dsOptions?.address) {\n conditions.push({\n field: 'address',\n value: dsOptions.address.toLowerCase(),\n // matcher: 'equals',\n });\n }\n }\n if (filter.topics) {\n for (let i = 0; i < Math.min(filter.topics.length, 4); i++) {\n const topic = filter.topics[i];\n if (!topic) {\n continue;\n }\n const field = `topics${i}`;\n conditions.push({\n field,\n value: eventToTopic(topic),\n matcher: 'equalTo',\n });\n }\n }\n return {\n entity: 'evmLogs',\n conditions,\n };\n}\n\nfunction callFilterToQueryEntry(\n filter: EthereumTransactionFilter,\n): DictionaryQueryEntry {\n const conditions: DictionaryQueryCondition[] = [];\n if (filter.from) {\n conditions.push({\n field: 'from',\n value: filter.from.toLowerCase(),\n });\n }\n if (filter.to) {\n conditions.push({\n field: 'to',\n value: filter.to.toLowerCase(),\n });\n }\n if (filter.function) {\n conditions.push({\n field: 'func',\n value: functionToSighash(filter.function),\n matcher: 'equalTo',\n });\n }\n return {\n entity: 'evmTransactions',\n conditions,\n };\n}\n\n@Injectable()\nexport class FetchService implements OnApplicationShutdown {\n private latestBestHeight: number;\n private latestFinalizedHeight: number;\n private isShutdown = false;\n private batchSizeScale: number;\n private templateDynamicDatasouces: SubqlProjectDs[];\n private dictionaryGenesisMatches = true;\n private evmChainId: string;\n private bypassBlocks: number[] = [];\n\n constructor(\n private apiService: ApiService,\n private nodeConfig: NodeConfig,\n @Inject('ISubqueryProject') private project: SubqueryProject,\n @Inject('IBlockDispatcher') private blockDispatcher: IBlockDispatcher,\n private dictionaryService: DictionaryService,\n private dynamicDsService: DynamicDsService,\n private unfinalizedBlocksService: UnfinalizedBlocksService,\n private eventEmitter: EventEmitter2,\n private schedulerRegistry: SchedulerRegistry,\n ) {\n this.batchSizeScale = 1;\n }\n\n onApplicationShutdown(): void {\n this.isShutdown = true;\n }\n\n get api(): ApiWrapper {\n return this.apiService.api;\n }\n\n async syncDynamicDatascourcesFromMeta(): Promise<void> {\n this.templateDynamicDatasouces =\n await this.dynamicDsService.getDynamicDatasources();\n }\n\n buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[] {\n const queryEntries: DictionaryQueryEntry[] = [];\n\n type GroupedSubqlProjectDs = SubqlProjectDs & {\n groupedOptions?: SubqlEthereumProcessorOptions[];\n };\n\n const groupdDynamicDs: GroupedSubqlProjectDs[] = Object.values(\n groupBy(this.templateDynamicDatasouces, (ds) => ds.name),\n ).map((grouped: SubqlProjectDs[]) => {\n const options = grouped.map((ds) => ds.options);\n const ref = grouped[0];\n\n return {\n ...ref,\n groupedOptions: options,\n };\n });\n\n // Only run the ds that is equal or less than startBlock\n // sort array from lowest ds.startBlock to highest\n const filteredDs: GroupedSubqlProjectDs[] = this.project.dataSources\n .concat(groupdDynamicDs)\n .filter((ds) => ds.startBlock <= startBlock)\n .sort((a, b) => a.startBlock - b.startBlock);\n\n for (const ds of filteredDs) {\n for (const handler of ds.mapping.handlers) {\n // No filters, cant use dictionary\n if (!handler.filter) return [];\n\n switch (handler.kind) {\n case EthereumHandlerKind.Block:\n return [];\n case EthereumHandlerKind.Call: {\n const filter = handler.filter as EthereumTransactionFilter;\n if (\n filter.from !== undefined ||\n filter.to !== undefined ||\n filter.function\n ) {\n queryEntries.push(callFilterToQueryEntry(filter));\n } else {\n return [];\n }\n break;\n }\n case EthereumHandlerKind.Event: {\n const filter = handler.filter as EthereumLogFilter;\n if (ds.groupedOptions) {\n queryEntries.push(\n eventFilterToQueryEntry(filter, ds.groupedOptions),\n );\n } else if (ds.options?.address || filter.topics) {\n queryEntries.push(eventFilterToQueryEntry(filter, ds.options));\n } else {\n return [];\n }\n break;\n }\n default:\n }\n }\n }\n\n return uniqBy(\n queryEntries,\n (item) =>\n `${item.entity}|${JSON.stringify(\n sortBy(item.conditions, (c) => c.field),\n )}`,\n );\n }\n\n updateDictionary(): void {\n this.dictionaryService.buildDictionaryEntryMap<SubqlProjectDs>(\n this.project.dataSources.concat(this.templateDynamicDatasouces),\n this.buildDictionaryQueryEntries.bind(this),\n );\n }\n\n private get useDictionary(): boolean {\n return (\n !!this.project.network.dictionary &&\n this.dictionaryGenesisMatches &&\n !!this.dictionaryService.getDictionaryQueryEntries(\n this.blockDispatcher.latestBufferedHeight ??\n Math.min(...this.project.dataSources.map((ds) => ds.startBlock)),\n ).length\n );\n }\n\n async init(startHeight: number): Promise<void> {\n if (this.project.network?.bypassBlocks !== undefined) {\n this.bypassBlocks = transformBypassBlocks(\n this.project.network.bypassBlocks,\n ).filter((blk) => blk >= startHeight);\n }\n if (this.api) {\n const CHAIN_INTERVAL = calcInterval(this.api) * INTERVAL_PERCENT;\n\n BLOCK_TIME_VARIANCE = Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);\n\n this.schedulerRegistry.addInterval(\n 'getLatestBlockHead',\n setInterval(() => void this.getBestBlockHead(), BLOCK_TIME_VARIANCE),\n );\n }\n await this.syncDynamicDatascourcesFromMeta();\n\n this.updateDictionary();\n this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n value: Number(this.useDictionary),\n });\n\n if (this.project.network.dictionary) {\n this.evmChainId = await this.dictionaryService.getEvmChainId();\n }\n\n await this.getFinalizedBlockHead();\n await this.getBestBlockHead();\n\n // Call metadata here, other network should align with this\n // For substrate, we might use the specVersion metadata in future if we have same error handling as in node-core\n const metadata = await this.dictionaryService.getMetadata();\n\n const validChecker = this.dictionaryValidation(metadata);\n\n if (validChecker) {\n this.dictionaryService.setDictionaryStartHeight(\n metadata._metadata.startHeight,\n );\n }\n\n await this.blockDispatcher.init(this.resetForNewDs.bind(this));\n void this.startLoop(startHeight);\n }\n\n getUseDictionary(): boolean {\n return this.useDictionary;\n }\n\n getLatestFinalizedHeight(): number {\n return this.latestFinalizedHeight;\n }\n\n @Interval(CHECK_MEMORY_INTERVAL)\n checkBatchScale(): void {\n if (this.nodeConfig['scale-batch-size']) {\n const scale = checkMemoryUsage(this.batchSizeScale, this.nodeConfig);\n\n if (this.batchSizeScale !== scale) {\n this.batchSizeScale = scale;\n }\n }\n }\n\n @Interval(BLOCK_TIME_VARIANCE * 1000)\n async getFinalizedBlockHead(): Promise<void> {\n if (!this.api) {\n logger.debug(`Skip fetch finalized block until API is ready`);\n return;\n }\n try {\n const currentFinalizedHeight = await this.api.getFinalizedBlockHeight();\n logger.debug(`finalized:${currentFinalizedHeight.toString()}`);\n const finalizedHeader = await this.api.getBlockByHeightOrHash(\n currentFinalizedHeight,\n );\n if (this.latestFinalizedHeight !== currentFinalizedHeight) {\n this.latestFinalizedHeight = currentFinalizedHeight;\n this.unfinalizedBlocksService.registerFinalizedBlock(finalizedHeader);\n if (!this.nodeConfig.unfinalizedBlocks) {\n if (!this.nodeConfig.unfinalizedBlocks) {\n this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n height: this.latestFinalizedHeight,\n });\n }\n }\n }\n } catch (e) {\n logger.warn(e, `Having a problem when get finalized block`);\n }\n }\n\n @Interval(BLOCK_TIME_VARIANCE * 1000)\n async getBestBlockHead(): Promise<void> {\n if (!this.api) {\n logger.debug(`Skip fetch best block until API is ready`);\n return;\n }\n try {\n const currentBestHeight = await this.api.getBestBlockHeight();\n logger.debug(`best:${currentBestHeight.toString()}`);\n if (this.latestBestHeight !== currentBestHeight) {\n this.latestBestHeight = currentBestHeight;\n this.eventEmitter.emit(IndexerEvent.BlockBest, {\n height: this.latestBestHeight,\n });\n\n if (this.nodeConfig.unfinalizedBlocks) {\n this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n height: this.latestBestHeight,\n });\n }\n }\n } catch (e) {\n logger.warn(e, `Having a problem when get best block`);\n }\n }\n private async startLoop(initBlockHeight: number): Promise<void> {\n await this.fillNextBlockBuffer(initBlockHeight);\n }\n\n getModulos(): number[] {\n const modulos: number[] = [];\n for (const ds of this.project.dataSources) {\n if (isCustomDs(ds)) {\n continue;\n }\n for (const handler of ds.mapping.handlers) {\n if (\n handler.kind === EthereumHandlerKind.Block &&\n handler.filter &&\n handler.filter.modulo\n ) {\n modulos.push(handler.filter.modulo);\n }\n }\n }\n return modulos;\n }\n\n getModuloBlocks(startHeight: number, endHeight: number): number[] {\n const modulos = this.getModulos();\n const moduloBlocks: number[] = [];\n for (let i = startHeight; i < endHeight; i++) {\n if (modulos.find((m) => i % m === 0)) {\n moduloBlocks.push(i);\n }\n }\n return moduloBlocks;\n }\n\n getEnqueuedModuloBlocks(startBlockHeight: number): number[] {\n return this.getModuloBlocks(\n startBlockHeight,\n this.nodeConfig.batchSize * Math.max(...this.getModulos()) +\n startBlockHeight,\n ).slice(0, this.nodeConfig.batchSize);\n }\n\n async fillNextBlockBuffer(initBlockHeight: number): Promise<void> {\n let startBlockHeight: number;\n let scaledBatchSize: number;\n const handlers = [].concat(\n ...this.project.dataSources.map((ds) => ds.mapping.handlers),\n );\n\n const getStartBlockHeight = (): number => {\n return this.blockDispatcher.latestBufferedHeight\n ? this.blockDispatcher.latestBufferedHeight + 1\n : initBlockHeight;\n };\n\n if (this.dictionaryService.startHeight > getStartBlockHeight()) {\n logger.warn(\n `Dictionary start height ${\n this.dictionaryService.startHeight\n } is beyond indexing height ${getStartBlockHeight()}, skipping dictionary for now`,\n );\n }\n\n while (!this.isShutdown) {\n startBlockHeight = getStartBlockHeight();\n\n scaledBatchSize = Math.max(\n Math.round(this.batchSizeScale * this.nodeConfig.batchSize),\n Math.min(MINIMUM_BATCH_SIZE, this.nodeConfig.batchSize * 3),\n );\n const latestHeight = this.nodeConfig.unfinalizedBlocks\n ? this.latestBestHeight\n : this.latestFinalizedHeight;\n\n if (\n this.blockDispatcher.freeSize < scaledBatchSize ||\n startBlockHeight > latestHeight\n ) {\n await delay(1);\n continue;\n }\n\n if (\n this.useDictionary &&\n startBlockHeight >= this.dictionaryService.startHeight\n ) {\n const queryEndBlock = startBlockHeight + DICTIONARY_MAX_QUERY_SIZE;\n const moduloBlocks = this.getModuloBlocks(\n startBlockHeight,\n queryEndBlock,\n );\n\n try {\n const dictionary =\n await this.dictionaryService.scopedDictionaryEntries(\n startBlockHeight,\n queryEndBlock,\n scaledBatchSize,\n );\n\n if (startBlockHeight !== getStartBlockHeight()) {\n logger.debug(\n `Queue was reset for new DS, discarding dictionary query result`,\n );\n continue;\n }\n\n if (\n dictionary &&\n this.dictionaryValidation(dictionary, startBlockHeight)\n ) {\n let { batchBlocks } = dictionary;\n\n batchBlocks = batchBlocks\n .concat(moduloBlocks)\n .sort((a, b) => a - b);\n if (batchBlocks.length === 0) {\n // There we're no blocks in this query range, we can set a new height we're up to\n this.blockDispatcher.enqueueBlocks(\n [],\n Math.min(\n queryEndBlock - 1,\n dictionary._metadata.lastProcessedHeight,\n ),\n );\n } else {\n const maxBlockSize = Math.min(\n batchBlocks.length,\n this.blockDispatcher.freeSize,\n );\n const enqueuingBlocks = batchBlocks.slice(0, maxBlockSize);\n const cleanedBatchBlocks =\n this.filteredBlockBatch(enqueuingBlocks);\n\n this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n }\n continue; // skip nextBlockRange() way\n }\n // else use this.nextBlockRange()\n } catch (e) {\n logger.debug(`Fetch dictionary stopped: ${e.message}`);\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n }\n }\n\n const endHeight = this.nextEndBlockHeight(\n startBlockHeight,\n scaledBatchSize,\n );\n\n if (handlers.length && this.getModulos().length === handlers.length) {\n const enqueuingBlocks = this.getEnqueuedModuloBlocks(startBlockHeight);\n const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);\n this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n } else {\n const enqueuingBlocks = range(startBlockHeight, endHeight + 1);\n const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);\n this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n }\n }\n }\n\n private getLatestBufferHeight(\n cleanedBatchBlocks: number[],\n rawBatchBlocks: number[],\n ): number {\n return Math.max(...cleanedBatchBlocks, ...rawBatchBlocks);\n }\n private filteredBlockBatch(currentBatchBlocks: number[]): number[] {\n if (!this.bypassBlocks.length || !currentBatchBlocks) {\n return currentBatchBlocks;\n }\n\n const cleanedBatch = cleanedBatchBlocks(\n this.bypassBlocks,\n currentBatchBlocks,\n );\n\n const pollutedBlocks = this.bypassBlocks.filter(\n (b) => b < Math.max(...currentBatchBlocks),\n );\n if (pollutedBlocks.length) {\n logger.info(`Bypassing blocks: ${pollutedBlocks}`);\n }\n this.bypassBlocks = without(this.bypassBlocks, ...pollutedBlocks);\n return cleanedBatch;\n }\n\n private nextEndBlockHeight(\n startBlockHeight: number,\n scaledBatchSize: number,\n ): number {\n let endBlockHeight = startBlockHeight + scaledBatchSize - 1;\n\n if (endBlockHeight > this.latestFinalizedHeight) {\n if (this.nodeConfig.unfinalizedBlocks) {\n if (endBlockHeight >= this.latestBestHeight) {\n endBlockHeight = this.latestBestHeight;\n }\n } else {\n endBlockHeight = this.latestFinalizedHeight;\n }\n }\n return endBlockHeight;\n }\n\n private dictionaryValidation(\n dictionary: { _metadata: MetaData },\n startBlockHeight?: number,\n ): boolean {\n if (dictionary !== undefined) {\n const { _metadata: metaData } = dictionary;\n\n if (\n metaData.genesisHash !== this.api.getGenesisHash() &&\n this.evmChainId !== this.api.getChainId().toString()\n ) {\n logger.error(\n 'The dictionary that you have specified does not match the chain you are indexing, it will be ignored. Please update your project manifest to reference the correct dictionary',\n );\n this.dictionaryGenesisMatches = false;\n this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n value: Number(this.useDictionary),\n });\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n return false;\n }\n\n if (startBlockHeight !== undefined) {\n if (metaData.lastProcessedHeight < startBlockHeight) {\n logger.warn(\n `Dictionary indexed block is behind current indexing block height`,\n );\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n return false;\n }\n }\n return true;\n }\n return false;\n }\n\n async resetForNewDs(blockHeight: number): Promise<void> {\n await this.syncDynamicDatascourcesFromMeta();\n this.dynamicDsService.deleteTempDsRecords(blockHeight);\n this.updateDictionary();\n this.blockDispatcher.flushQueue(blockHeight);\n }\n\n async resetForIncorrectBestBlock(blockHeight: number): Promise<void> {\n await this.syncDynamicDatascourcesFromMeta();\n this.updateDictionary();\n this.blockDispatcher.flushQueue(blockHeight);\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"fetch.service.js","sourceRoot":"","sources":["../../src/indexer/fetch.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAA2E;AAC3E,yDAAsD;AACtD,+CAA+D;AAC/D,4DAIgC;AAChC,gDAU0B;AAU1B,mCAAsE;AACtE,kEAA+E;AAC/E,+DAA0D;AAC1D,4CAAkE;AAElE,6DAAyD;AACzD,6DAAwD;AACxD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,OAAO,CAAC,CAAC;AAClC,IAAI,mBAAmB,GAAG,IAAI,CAAC;AAC/B,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,SAAS,uBAAuB,CAC9B,MAAyB,EACzB,SAA0E;IAE1E,MAAM,UAAU,GAA+B,EAAE,CAAC;IAElD,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QAC5B,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE5E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,mBAAmB,EAAE;YACrE,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,eAAe;aACzB,CAAC,CAAC;SACJ;KACF;SAAM;QACL,IAAI,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,OAAO,EAAE;YACtB,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE;gBACtC,qBAAqB;aACtB,CAAC,CAAC;SACJ;KACF;IACD,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE;gBACV,SAAS;aACV;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK;gBACL,KAAK,EAAE,IAAA,qBAAY,EAAC,KAAK,CAAC;gBAC1B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;SACJ;KACF;IACD,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiC;IAEjC,MAAM,UAAU,GAA+B,EAAE,CAAC;IAClD,IAAI,MAAM,CAAC,IAAI,EAAE;QACf,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;SACjC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,EAAE,EAAE;QACb,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;SAC/B,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE;QACnB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,IAAA,0BAAiB,EAAC,MAAM,CAAC,QAAQ,CAAC;YACzC,OAAO,EAAE,SAAS;SACnB,CAAC,CAAC;KACJ;IACD,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,UAAU;KACX,CAAC;AACJ,CAAC;AAGM,IAAM,YAAY,GAAlB,MAAM,YAAY;IAUvB,YACU,UAAsB,EACtB,UAAsB,EACM,OAAwB,EACxB,eAAiC,EAC7D,iBAAoC,EACpC,gBAAkC,EAClC,wBAAkD,EAClD,YAA2B,EAC3B,iBAAoC;QARpC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACxB,oBAAe,GAAf,eAAe,CAAkB;QAC7D,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,iBAAY,GAAZ,YAAY,CAAe;QAC3B,sBAAiB,GAAjB,iBAAiB,CAAmB;QAhBtC,eAAU,GAAG,KAAK,CAAC;QAGnB,6BAAwB,GAAG,IAAI,CAAC;QAEhC,iBAAY,GAAa,EAAE,CAAC;QAalC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,IAAI,CAAC,yBAAyB;YAC5B,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;IACxD,CAAC;IAED,2BAA2B,CAAC,UAAkB;;QAC5C,MAAM,YAAY,GAA2B,EAAE,CAAC;QAMhD,MAAM,eAAe,GAA4B,MAAM,CAAC,MAAM,CAC5D,IAAA,gBAAO,EAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CACzD,CAAC,GAAG,CAAC,CAAC,OAAyB,EAAE,EAAE;YAClC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,uCACK,GAAG,KACN,cAAc,EAAE,OAAO,IACvB;QACJ,CAAC,CAAC,CAAC;QAEH,wDAAwD;QACxD,kDAAkD;QAClD,MAAM,UAAU,GAA4B,IAAI,CAAC,OAAO,CAAC,WAAW;aACjE,MAAM,CAAC,eAAe,CAAC;aACvB,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,UAAU,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAE/C,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE;YAC3B,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,kCAAkC;gBAClC,IAAI,CAAC,OAAO,CAAC,MAAM;oBAAE,OAAO,EAAE,CAAC;gBAE/B,QAAQ,OAAO,CAAC,IAAI,EAAE;oBACpB,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,EAAE,CAAC;oBACZ,KAAK,qCAAmB,CAAC,IAAI,CAAC,CAAC;wBAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAmC,CAAC;wBAC3D,IACE,MAAM,CAAC,IAAI,KAAK,SAAS;4BACzB,MAAM,CAAC,EAAE,KAAK,SAAS;4BACvB,MAAM,CAAC,QAAQ,EACf;4BACA,YAAY,CAAC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;yBACnD;6BAAM;4BACL,OAAO,EAAE,CAAC;yBACX;wBACD,MAAM;qBACP;oBACD,KAAK,qCAAmB,CAAC,KAAK,CAAC,CAAC;wBAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,MAA2B,CAAC;wBACnD,IAAI,EAAE,CAAC,cAAc,EAAE;4BACrB,YAAY,CAAC,IAAI,CACf,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,CACnD,CAAC;yBACH;6BAAM,IAAI,CAAA,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,KAAI,MAAM,CAAC,MAAM,EAAE;4BAC/C,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;yBAChE;6BAAM;4BACL,OAAO,EAAE,CAAC;yBACX;wBACD,MAAM;qBACP;oBACD,QAAQ;iBACT;aACF;SACF;QAED,OAAO,IAAA,eAAM,EACX,YAAY,EACZ,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAC9B,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACxC,EAAE,CACN,CAAC;IACJ,CAAC;IAED,gBAAgB;QACd,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAC5C,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,EAC/D,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAC5C,CAAC;IACJ,CAAC;IAED,IAAY,aAAa;;QACvB,OAAO,CACL,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU;YACjC,IAAI,CAAC,wBAAwB;YAC7B,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,CAChD,MAAA,IAAI,CAAC,eAAe,CAAC,oBAAoB,mCACvC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CACnE,CAAC,MAAM,CACT,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;;QAC5B,IAAI,CAAA,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,0CAAE,YAAY,MAAK,SAAS,EAAE;YACpD,IAAI,CAAC,YAAY,GAAG,IAAA,iCAAqB,EACvC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAClC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,WAAW,CAAC,CAAC;SACvC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE;YACZ,MAAM,cAAc,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC;YAEjE,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;YAEpE,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAChC,oBAAoB,EACpB,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,gBAAgB,EAAE,EAAE,mBAAmB,CAAC,CACrE,CAAC;SACH;QACD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAE7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;YACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE;YACnC,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;SAChE;QAED,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9B,4DAA4D;QAC5D,iHAAiH;QACjH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,CAAC;QAE5D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAEzD,IAAI,YAAY,EAAE;YAChB,IAAI,CAAC,iBAAiB,CAAC,wBAAwB,CAC7C,QAAQ,CAAC,SAAS,CAAC,WAAW,CAC/B,CAAC;SACH;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,wBAAwB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAGD,eAAe;QACb,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE;YACvC,MAAM,KAAK,GAAG,IAAA,4BAAgB,EAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAErE,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,EAAE;gBACjC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;aAC7B;SACF;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC9D,OAAO;SACR;QACD,IAAI;YACF,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,aAAa,sBAAsB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC3D,sBAAsB,CACvB,CAAC;YACF,IAAI,IAAI,CAAC,qBAAqB,KAAK,sBAAsB,EAAE;gBACzD,IAAI,CAAC,qBAAqB,GAAG,sBAAsB,CAAC;gBACpD,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;gBACtE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;wBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;4BAC/C,MAAM,EAAE,IAAI,CAAC,qBAAqB;yBACnC,CAAC,CAAC;qBACJ;iBACF;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;SAC7D;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,OAAO;SACR;QACD,IAAI;YACF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC9D,MAAM,CAAC,KAAK,CAAC,QAAQ,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,IAAI,CAAC,gBAAgB,KAAK,iBAAiB,EAAE;gBAC/C,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,SAAS,EAAE;oBAC7C,MAAM,EAAE,IAAI,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;gBAEH,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACrC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;wBAC/C,MAAM,EAAE,IAAI,CAAC,gBAAgB;qBAC9B,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,sCAAsC,CAAC,CAAC;SACxD;IACH,CAAC;IACO,KAAK,CAAC,SAAS,CAAC,eAAuB;QAC7C,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC;IAED,UAAU;QACR,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;YACzC,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;gBAClB,SAAS;aACV;YACD,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IACE,OAAO,CAAC,IAAI,KAAK,qCAAmB,CAAC,KAAK;oBAC1C,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,MAAM,CAAC,MAAM,EACrB;oBACA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iBACrC;aACF;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,eAAe,CAAC,WAAmB,EAAE,SAAiB;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,WAAW,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;YAC5C,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;gBACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB;SACF;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,uBAAuB,CAAC,gBAAwB;QAC9C,OAAO,IAAI,CAAC,eAAe,CACzB,gBAAgB,EAChB,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACxD,gBAAgB,CACnB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,eAAuB;QAC/C,IAAI,gBAAwB,CAAC;QAC7B,IAAI,eAAuB,CAAC;QAC5B,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,CACxB,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC7D,CAAC;QAEF,MAAM,mBAAmB,GAAG,GAAW,EAAE;YACvC,OAAO,IAAI,CAAC,eAAe,CAAC,oBAAoB;gBAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,GAAG,CAAC;gBAC/C,CAAC,CAAC,eAAe,CAAC;QACtB,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,GAAG,mBAAmB,EAAE,EAAE;YAC9D,MAAM,CAAC,IAAI,CACT,2BACE,IAAI,CAAC,iBAAiB,CAAC,WACzB,8BAA8B,mBAAmB,EAAE,+BAA+B,CACnF,CAAC;SACH;QAED,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE;YACvB,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;YAEzC,eAAe,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAC3D,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CAC5D,CAAC;YACF,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB;gBACpD,CAAC,CAAC,IAAI,CAAC,gBAAgB;gBACvB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC;YAE/B,IACE,IAAI,CAAC,eAAe,CAAC,QAAQ,GAAG,eAAe;gBAC/C,gBAAgB,GAAG,YAAY,EAC/B;gBACA,MAAM,IAAA,iBAAK,EAAC,CAAC,CAAC,CAAC;gBACf,SAAS;aACV;YAED,IACE,IAAI,CAAC,aAAa;gBAClB,gBAAgB,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,EACtD;gBACA,MAAM,aAAa,GAAG,gBAAgB,GAAG,yBAAyB,CAAC;gBACnE,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CACvC,gBAAgB,EAChB,aAAa,CACd,CAAC;gBAEF,IAAI;oBACF,MAAM,UAAU,GACd,MAAM,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAClD,gBAAgB,EAChB,aAAa,EACb,eAAe,CAChB,CAAC;oBAEJ,IAAI,gBAAgB,KAAK,mBAAmB,EAAE,EAAE;wBAC9C,MAAM,CAAC,KAAK,CACV,gEAAgE,CACjE,CAAC;wBACF,SAAS;qBACV;oBAED,IACE,UAAU;wBACV,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,gBAAgB,CAAC,EACvD;wBACA,IAAI,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;wBAEjC,WAAW,GAAG,WAAW;6BACtB,MAAM,CAAC,YAAY,CAAC;6BACpB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBACzB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;4BAC5B,iFAAiF;4BACjF,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,EAAE,EACF,IAAI,CAAC,GAAG,CACN,aAAa,GAAG,CAAC,EACjB,UAAU,CAAC,SAAS,CAAC,mBAAmB,CACzC,CACF,CAAC;yBACH;6BAAM;4BACL,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,WAAW,CAAC,MAAM,EAClB,IAAI,CAAC,eAAe,CAAC,QAAQ,CAC9B,CAAC;4BACF,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;4BAC3D,MAAM,kBAAkB,GACtB,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;4BAE3C,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;yBACH;wBACD,SAAS,CAAC,4BAA4B;qBACvC;oBACD,iCAAiC;iBAClC;gBAAC,OAAO,CAAC,EAAE;oBACV,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACvD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;iBACrD;aACF;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CACvC,gBAAgB,EAChB,eAAe,CAChB,CAAC;YAEF,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE;gBACnE,MAAM,eAAe,GAAG,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,CAAC,CAAC;gBACvE,MAAM,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBACpE,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;aACH;iBAAM;gBACL,MAAM,eAAe,GAAG,IAAA,cAAK,EAAC,gBAAgB,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;gBAC/D,MAAM,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBACpE,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;aACH;SACF;IACH,CAAC;IAEO,qBAAqB,CAC3B,kBAA4B,EAC5B,cAAwB;QAExB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,EAAE,GAAG,cAAc,CAAC,CAAC;IAC5D,CAAC;IACO,kBAAkB,CAAC,kBAA4B;QACrD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,EAAE;YACpD,OAAO,kBAAkB,CAAC;SAC3B;QAED,MAAM,YAAY,GAAG,IAAA,8BAAkB,EACrC,IAAI,CAAC,YAAY,EACjB,kBAAkB,CACnB,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAC7C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,CAC3C,CAAC;QACF,IAAI,cAAc,CAAC,MAAM,EAAE;YACzB,MAAM,CAAC,IAAI,CAAC,qBAAqB,cAAc,EAAE,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,YAAY,GAAG,IAAA,gBAAO,EAAC,IAAI,CAAC,YAAY,EAAE,GAAG,cAAc,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC;IACtB,CAAC;IAEO,kBAAkB,CACxB,gBAAwB,EACxB,eAAuB;QAEvB,IAAI,cAAc,GAAG,gBAAgB,GAAG,eAAe,GAAG,CAAC,CAAC;QAE5D,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE;YAC/C,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;gBACrC,IAAI,cAAc,IAAI,IAAI,CAAC,gBAAgB,EAAE;oBAC3C,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC;iBACxC;aACF;iBAAM;gBACL,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;aAC7C;SACF;QACD,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,oBAAoB,CAC1B,UAAmC,EACnC,gBAAyB;QAEzB,IAAI,UAAU,KAAK,SAAS,EAAE;YAC5B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;YAE3C,IACE,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE;gBAClD,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EACpD;gBACA,MAAM,CAAC,KAAK,CACV,+KAA+K,CAChL,CAAC;gBACF,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;gBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;oBACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;iBAClC,CAAC,CAAC;gBACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;gBACpD,OAAO,KAAK,CAAC;aACd;YAED,IAAI,gBAAgB,KAAK,SAAS,EAAE;gBAClC,IAAI,QAAQ,CAAC,mBAAmB,GAAG,gBAAgB,EAAE;oBACnD,MAAM,CAAC,IAAI,CACT,kEAAkE,CACnE,CAAC;oBACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;oBACpD,OAAO,KAAK,CAAC;iBACd;aACF;YACD,OAAO,IAAI,CAAC;SACb;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACrC,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAAC,WAAmB;QAClD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;CACF,CAAA;AAvUC;IAAC,IAAA,mBAAQ,EAAC,qBAAqB,CAAC;;;;mDAS/B;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,CAAC;;;;yDA0B7B;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,CAAC;;;;oDAwB7B;AAtPU,YAAY;IADxB,IAAA,mBAAU,GAAE;IAcR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;IAC1B,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHP,sBAAU;QACV,sBAAU;QACe,iCAAe,UAEjC,sCAAiB;QAClB,qCAAgB;QACR,oDAAwB;QACpC,6BAAa;QACR,4BAAiB;GAnBnC,YAAY,CA8fxB;AA9fY,oCAAY","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval, SchedulerRegistry } from '@nestjs/schedule';\nimport {\n isCustomDs,\n EthereumHandlerKind,\n SubqlHandlerFilter,\n} from '@subql/common-ethereum';\nimport {\n ApiService,\n Dictionary,\n checkMemoryUsage,\n delay,\n getLogger,\n IndexerEvent,\n NodeConfig,\n transformBypassBlocks,\n cleanedBatchBlocks,\n} from '@subql/node-core';\nimport {\n DictionaryQueryEntry,\n ApiWrapper,\n EthereumLogFilter,\n EthereumTransactionFilter,\n SubqlEthereumProcessorOptions,\n DictionaryQueryCondition,\n} from '@subql/types-ethereum';\nimport { MetaData } from '@subql/utils';\nimport { range, sortBy, uniqBy, without, groupBy, add } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { calcInterval } from '../ethereum/utils.ethereum';\nimport { eventToTopic, functionToSighash } from '../utils/string';\nimport { IBlockDispatcher } from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('fetch');\nlet BLOCK_TIME_VARIANCE = 5000;\nconst DICTIONARY_MAX_QUERY_SIZE = 10000;\nconst CHECK_MEMORY_INTERVAL = 60000;\nconst MINIMUM_BATCH_SIZE = 5;\nconst INTERVAL_PERCENT = 0.9;\nconst QUERY_ADDRESS_LIMIT = 50;\n\nfunction eventFilterToQueryEntry(\n filter: EthereumLogFilter,\n dsOptions: SubqlEthereumProcessorOptions | SubqlEthereumProcessorOptions[],\n): DictionaryQueryEntry {\n const conditions: DictionaryQueryCondition[] = [];\n\n if (Array.isArray(dsOptions)) {\n const addresses = dsOptions.map((option) => option.address).filter(Boolean);\n\n if (addresses.length !== 0 && addresses.length <= QUERY_ADDRESS_LIMIT) {\n conditions.push({\n field: 'address',\n value: addresses,\n matcher: 'inInsensitive',\n });\n }\n } else {\n if (dsOptions?.address) {\n conditions.push({\n field: 'address',\n value: dsOptions.address.toLowerCase(),\n // matcher: 'equals',\n });\n }\n }\n if (filter.topics) {\n for (let i = 0; i < Math.min(filter.topics.length, 4); i++) {\n const topic = filter.topics[i];\n if (!topic) {\n continue;\n }\n const field = `topics${i}`;\n conditions.push({\n field,\n value: eventToTopic(topic),\n matcher: 'equalTo',\n });\n }\n }\n return {\n entity: 'evmLogs',\n conditions,\n };\n}\n\nfunction callFilterToQueryEntry(\n filter: EthereumTransactionFilter,\n): DictionaryQueryEntry {\n const conditions: DictionaryQueryCondition[] = [];\n if (filter.from) {\n conditions.push({\n field: 'from',\n value: filter.from.toLowerCase(),\n });\n }\n if (filter.to) {\n conditions.push({\n field: 'to',\n value: filter.to.toLowerCase(),\n });\n }\n if (filter.function) {\n conditions.push({\n field: 'func',\n value: functionToSighash(filter.function),\n matcher: 'equalTo',\n });\n }\n return {\n entity: 'evmTransactions',\n conditions,\n };\n}\n\n@Injectable()\nexport class FetchService implements OnApplicationShutdown {\n private latestBestHeight: number;\n private latestFinalizedHeight: number;\n private isShutdown = false;\n private batchSizeScale: number;\n private templateDynamicDatasouces: SubqlProjectDs[];\n private dictionaryGenesisMatches = true;\n private evmChainId: string;\n private bypassBlocks: number[] = [];\n\n constructor(\n private apiService: ApiService,\n private nodeConfig: NodeConfig,\n @Inject('ISubqueryProject') private project: SubqueryProject,\n @Inject('IBlockDispatcher') private blockDispatcher: IBlockDispatcher,\n private dictionaryService: DictionaryService,\n private dynamicDsService: DynamicDsService,\n private unfinalizedBlocksService: UnfinalizedBlocksService,\n private eventEmitter: EventEmitter2,\n private schedulerRegistry: SchedulerRegistry,\n ) {\n this.batchSizeScale = 1;\n }\n\n onApplicationShutdown(): void {\n this.isShutdown = true;\n }\n\n get api(): ApiWrapper {\n return this.apiService.api;\n }\n\n async syncDynamicDatascourcesFromMeta(): Promise<void> {\n this.templateDynamicDatasouces =\n await this.dynamicDsService.getDynamicDatasources();\n }\n\n buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[] {\n const queryEntries: DictionaryQueryEntry[] = [];\n\n type GroupedSubqlProjectDs = SubqlProjectDs & {\n groupedOptions?: SubqlEthereumProcessorOptions[];\n };\n\n const groupdDynamicDs: GroupedSubqlProjectDs[] = Object.values(\n groupBy(this.templateDynamicDatasouces, (ds) => ds.name),\n ).map((grouped: SubqlProjectDs[]) => {\n const options = grouped.map((ds) => ds.options);\n const ref = grouped[0];\n\n return {\n ...ref,\n groupedOptions: options,\n };\n });\n\n // Only run the ds that is equal or less than startBlock\n // sort array from lowest ds.startBlock to highest\n const filteredDs: GroupedSubqlProjectDs[] = this.project.dataSources\n .concat(groupdDynamicDs)\n .filter((ds) => ds.startBlock <= startBlock)\n .sort((a, b) => a.startBlock - b.startBlock);\n\n for (const ds of filteredDs) {\n for (const handler of ds.mapping.handlers) {\n // No filters, cant use dictionary\n if (!handler.filter) return [];\n\n switch (handler.kind) {\n case EthereumHandlerKind.Block:\n return [];\n case EthereumHandlerKind.Call: {\n const filter = handler.filter as EthereumTransactionFilter;\n if (\n filter.from !== undefined ||\n filter.to !== undefined ||\n filter.function\n ) {\n queryEntries.push(callFilterToQueryEntry(filter));\n } else {\n return [];\n }\n break;\n }\n case EthereumHandlerKind.Event: {\n const filter = handler.filter as EthereumLogFilter;\n if (ds.groupedOptions) {\n queryEntries.push(\n eventFilterToQueryEntry(filter, ds.groupedOptions),\n );\n } else if (ds.options?.address || filter.topics) {\n queryEntries.push(eventFilterToQueryEntry(filter, ds.options));\n } else {\n return [];\n }\n break;\n }\n default:\n }\n }\n }\n\n return uniqBy(\n queryEntries,\n (item) =>\n `${item.entity}|${JSON.stringify(\n sortBy(item.conditions, (c) => c.field),\n )}`,\n );\n }\n\n updateDictionary(): void {\n this.dictionaryService.buildDictionaryEntryMap<SubqlProjectDs>(\n this.project.dataSources.concat(this.templateDynamicDatasouces),\n this.buildDictionaryQueryEntries.bind(this),\n );\n }\n\n private get useDictionary(): boolean {\n return (\n !!this.project.network.dictionary &&\n this.dictionaryGenesisMatches &&\n !!this.dictionaryService.getDictionaryQueryEntries(\n this.blockDispatcher.latestBufferedHeight ??\n Math.min(...this.project.dataSources.map((ds) => ds.startBlock)),\n ).length\n );\n }\n\n async init(startHeight: number): Promise<void> {\n if (this.project.network?.bypassBlocks !== undefined) {\n this.bypassBlocks = transformBypassBlocks(\n this.project.network.bypassBlocks,\n ).filter((blk) => blk >= startHeight);\n }\n if (this.api) {\n const CHAIN_INTERVAL = calcInterval(this.api) * INTERVAL_PERCENT;\n\n BLOCK_TIME_VARIANCE = Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);\n\n this.schedulerRegistry.addInterval(\n 'getLatestBlockHead',\n setInterval(() => void this.getBestBlockHead(), BLOCK_TIME_VARIANCE),\n );\n }\n await this.syncDynamicDatascourcesFromMeta();\n\n this.updateDictionary();\n this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n value: Number(this.useDictionary),\n });\n\n if (this.project.network.dictionary) {\n this.evmChainId = await this.dictionaryService.getEvmChainId();\n }\n\n await this.getFinalizedBlockHead();\n await this.getBestBlockHead();\n\n // Call metadata here, other network should align with this\n // For substrate, we might use the specVersion metadata in future if we have same error handling as in node-core\n const metadata = await this.dictionaryService.getMetadata();\n\n const validChecker = this.dictionaryValidation(metadata);\n\n if (validChecker) {\n this.dictionaryService.setDictionaryStartHeight(\n metadata._metadata.startHeight,\n );\n }\n\n await this.blockDispatcher.init(this.resetForNewDs.bind(this));\n void this.startLoop(startHeight);\n }\n\n getUseDictionary(): boolean {\n return this.useDictionary;\n }\n\n getLatestFinalizedHeight(): number {\n return this.latestFinalizedHeight;\n }\n\n @Interval(CHECK_MEMORY_INTERVAL)\n checkBatchScale(): void {\n if (this.nodeConfig['scale-batch-size']) {\n const scale = checkMemoryUsage(this.batchSizeScale, this.nodeConfig);\n\n if (this.batchSizeScale !== scale) {\n this.batchSizeScale = scale;\n }\n }\n }\n\n @Interval(BLOCK_TIME_VARIANCE)\n async getFinalizedBlockHead(): Promise<void> {\n if (!this.api) {\n logger.debug(`Skip fetch finalized block until API is ready`);\n return;\n }\n try {\n const currentFinalizedHeight = await this.api.getFinalizedBlockHeight();\n logger.debug(`finalized:${currentFinalizedHeight.toString()}`);\n const finalizedHeader = await this.api.getBlockByHeightOrHash(\n currentFinalizedHeight,\n );\n if (this.latestFinalizedHeight !== currentFinalizedHeight) {\n this.latestFinalizedHeight = currentFinalizedHeight;\n this.unfinalizedBlocksService.registerFinalizedBlock(finalizedHeader);\n if (!this.nodeConfig.unfinalizedBlocks) {\n if (!this.nodeConfig.unfinalizedBlocks) {\n this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n height: this.latestFinalizedHeight,\n });\n }\n }\n }\n } catch (e) {\n logger.warn(e, `Having a problem when get finalized block`);\n }\n }\n\n @Interval(BLOCK_TIME_VARIANCE)\n async getBestBlockHead(): Promise<void> {\n if (!this.api) {\n logger.debug(`Skip fetch best block until API is ready`);\n return;\n }\n try {\n const currentBestHeight = await this.api.getBestBlockHeight();\n logger.debug(`best:${currentBestHeight.toString()}`);\n if (this.latestBestHeight !== currentBestHeight) {\n this.latestBestHeight = currentBestHeight;\n this.eventEmitter.emit(IndexerEvent.BlockBest, {\n height: this.latestBestHeight,\n });\n\n if (this.nodeConfig.unfinalizedBlocks) {\n this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n height: this.latestBestHeight,\n });\n }\n }\n } catch (e) {\n logger.warn(e, `Having a problem when get best block`);\n }\n }\n private async startLoop(initBlockHeight: number): Promise<void> {\n await this.fillNextBlockBuffer(initBlockHeight);\n }\n\n getModulos(): number[] {\n const modulos: number[] = [];\n for (const ds of this.project.dataSources) {\n if (isCustomDs(ds)) {\n continue;\n }\n for (const handler of ds.mapping.handlers) {\n if (\n handler.kind === EthereumHandlerKind.Block &&\n handler.filter &&\n handler.filter.modulo\n ) {\n modulos.push(handler.filter.modulo);\n }\n }\n }\n return modulos;\n }\n\n getModuloBlocks(startHeight: number, endHeight: number): number[] {\n const modulos = this.getModulos();\n const moduloBlocks: number[] = [];\n for (let i = startHeight; i < endHeight; i++) {\n if (modulos.find((m) => i % m === 0)) {\n moduloBlocks.push(i);\n }\n }\n return moduloBlocks;\n }\n\n getEnqueuedModuloBlocks(startBlockHeight: number): number[] {\n return this.getModuloBlocks(\n startBlockHeight,\n this.nodeConfig.batchSize * Math.max(...this.getModulos()) +\n startBlockHeight,\n ).slice(0, this.nodeConfig.batchSize);\n }\n\n async fillNextBlockBuffer(initBlockHeight: number): Promise<void> {\n let startBlockHeight: number;\n let scaledBatchSize: number;\n const handlers = [].concat(\n ...this.project.dataSources.map((ds) => ds.mapping.handlers),\n );\n\n const getStartBlockHeight = (): number => {\n return this.blockDispatcher.latestBufferedHeight\n ? this.blockDispatcher.latestBufferedHeight + 1\n : initBlockHeight;\n };\n\n if (this.dictionaryService.startHeight > getStartBlockHeight()) {\n logger.warn(\n `Dictionary start height ${\n this.dictionaryService.startHeight\n } is beyond indexing height ${getStartBlockHeight()}, skipping dictionary for now`,\n );\n }\n\n while (!this.isShutdown) {\n startBlockHeight = getStartBlockHeight();\n\n scaledBatchSize = Math.max(\n Math.round(this.batchSizeScale * this.nodeConfig.batchSize),\n Math.min(MINIMUM_BATCH_SIZE, this.nodeConfig.batchSize * 3),\n );\n const latestHeight = this.nodeConfig.unfinalizedBlocks\n ? this.latestBestHeight\n : this.latestFinalizedHeight;\n\n if (\n this.blockDispatcher.freeSize < scaledBatchSize ||\n startBlockHeight > latestHeight\n ) {\n await delay(1);\n continue;\n }\n\n if (\n this.useDictionary &&\n startBlockHeight >= this.dictionaryService.startHeight\n ) {\n const queryEndBlock = startBlockHeight + DICTIONARY_MAX_QUERY_SIZE;\n const moduloBlocks = this.getModuloBlocks(\n startBlockHeight,\n queryEndBlock,\n );\n\n try {\n const dictionary =\n await this.dictionaryService.scopedDictionaryEntries(\n startBlockHeight,\n queryEndBlock,\n scaledBatchSize,\n );\n\n if (startBlockHeight !== getStartBlockHeight()) {\n logger.debug(\n `Queue was reset for new DS, discarding dictionary query result`,\n );\n continue;\n }\n\n if (\n dictionary &&\n this.dictionaryValidation(dictionary, startBlockHeight)\n ) {\n let { batchBlocks } = dictionary;\n\n batchBlocks = batchBlocks\n .concat(moduloBlocks)\n .sort((a, b) => a - b);\n if (batchBlocks.length === 0) {\n // There we're no blocks in this query range, we can set a new height we're up to\n this.blockDispatcher.enqueueBlocks(\n [],\n Math.min(\n queryEndBlock - 1,\n dictionary._metadata.lastProcessedHeight,\n ),\n );\n } else {\n const maxBlockSize = Math.min(\n batchBlocks.length,\n this.blockDispatcher.freeSize,\n );\n const enqueuingBlocks = batchBlocks.slice(0, maxBlockSize);\n const cleanedBatchBlocks =\n this.filteredBlockBatch(enqueuingBlocks);\n\n this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n }\n continue; // skip nextBlockRange() way\n }\n // else use this.nextBlockRange()\n } catch (e) {\n logger.debug(`Fetch dictionary stopped: ${e.message}`);\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n }\n }\n\n const endHeight = this.nextEndBlockHeight(\n startBlockHeight,\n scaledBatchSize,\n );\n\n if (handlers.length && this.getModulos().length === handlers.length) {\n const enqueuingBlocks = this.getEnqueuedModuloBlocks(startBlockHeight);\n const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);\n this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n } else {\n const enqueuingBlocks = range(startBlockHeight, endHeight + 1);\n const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);\n this.blockDispatcher.enqueueBlocks(\n cleanedBatchBlocks,\n this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n );\n }\n }\n }\n\n private getLatestBufferHeight(\n cleanedBatchBlocks: number[],\n rawBatchBlocks: number[],\n ): number {\n return Math.max(...cleanedBatchBlocks, ...rawBatchBlocks);\n }\n private filteredBlockBatch(currentBatchBlocks: number[]): number[] {\n if (!this.bypassBlocks.length || !currentBatchBlocks) {\n return currentBatchBlocks;\n }\n\n const cleanedBatch = cleanedBatchBlocks(\n this.bypassBlocks,\n currentBatchBlocks,\n );\n\n const pollutedBlocks = this.bypassBlocks.filter(\n (b) => b < Math.max(...currentBatchBlocks),\n );\n if (pollutedBlocks.length) {\n logger.info(`Bypassing blocks: ${pollutedBlocks}`);\n }\n this.bypassBlocks = without(this.bypassBlocks, ...pollutedBlocks);\n return cleanedBatch;\n }\n\n private nextEndBlockHeight(\n startBlockHeight: number,\n scaledBatchSize: number,\n ): number {\n let endBlockHeight = startBlockHeight + scaledBatchSize - 1;\n\n if (endBlockHeight > this.latestFinalizedHeight) {\n if (this.nodeConfig.unfinalizedBlocks) {\n if (endBlockHeight >= this.latestBestHeight) {\n endBlockHeight = this.latestBestHeight;\n }\n } else {\n endBlockHeight = this.latestFinalizedHeight;\n }\n }\n return endBlockHeight;\n }\n\n private dictionaryValidation(\n dictionary: { _metadata: MetaData },\n startBlockHeight?: number,\n ): boolean {\n if (dictionary !== undefined) {\n const { _metadata: metaData } = dictionary;\n\n if (\n metaData.genesisHash !== this.api.getGenesisHash() &&\n this.evmChainId !== this.api.getChainId().toString()\n ) {\n logger.error(\n 'The dictionary that you have specified does not match the chain you are indexing, it will be ignored. Please update your project manifest to reference the correct dictionary',\n );\n this.dictionaryGenesisMatches = false;\n this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n value: Number(this.useDictionary),\n });\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n return false;\n }\n\n if (startBlockHeight !== undefined) {\n if (metaData.lastProcessedHeight < startBlockHeight) {\n logger.warn(\n `Dictionary indexed block is behind current indexing block height`,\n );\n this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n return false;\n }\n }\n return true;\n }\n return false;\n }\n\n async resetForNewDs(blockHeight: number): Promise<void> {\n await this.syncDynamicDatascourcesFromMeta();\n this.dynamicDsService.deleteTempDsRecords(blockHeight);\n this.updateDictionary();\n this.blockDispatcher.flushQueue(blockHeight);\n }\n\n async resetForIncorrectBestBlock(blockHeight: number): Promise<void> {\n await this.syncDynamicDatascourcesFromMeta();\n this.updateDictionary();\n this.blockDispatcher.flushQueue(blockHeight);\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@subql/node-ethereum",
|
|
3
|
-
"version": "0.4.1-
|
|
3
|
+
"version": "0.4.1-12",
|
|
4
4
|
"description": "",
|
|
5
5
|
"author": "Ian He",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -74,5 +74,5 @@
|
|
|
74
74
|
"/dist",
|
|
75
75
|
"/bin"
|
|
76
76
|
],
|
|
77
|
-
"stableVersion": "0.4.1-
|
|
77
|
+
"stableVersion": "0.4.1-11"
|
|
78
78
|
}
|