npm - @subql/node-ethereum - Versions diffs - 0.4.1-6 → 0.4.1-7 - Mend

@subql/node-ethereum 0.4.1-6 → 0.4.1-7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (12) hide show

package/dist/.tsbuildinfo +1 -1
package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js +5 -0
package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js.map +1 -1
package/dist/indexer/dynamic-ds.service.d.ts +2 -0
package/dist/indexer/dynamic-ds.service.js +22 -10
package/dist/indexer/dynamic-ds.service.js.map +1 -1
package/dist/indexer/project.service.js +1 -1
package/dist/indexer/project.service.js.map +1 -1
package/dist/indexer/worker/worker.d.ts +2 -0
package/dist/indexer/worker/worker.js +14 -1
package/dist/indexer/worker/worker.js.map +1 -1
package/package.json +2 -2

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js CHANGED Viewed

@@ -34,6 +34,7 @@ async function createIndexerWorker() {
         'numFetchedBlocks',
         'numFetchingBlocks',
         'getStatus',
+        'reloadDynamicDs',
     ]);
     await indexerWorker.initWorker();
     return indexerWorker;
@@ -123,6 +124,10 @@ let WorkerBlockDispatcherService = class WorkerBlockDispatcherService extends ba
                     operationHash: Buffer.from(operationHash, 'base64'),
                     reindexBlockHeight,
                 });
+                if (dynamicDsCreated) {
+                    // Ensure all workers are aware of all dynamic ds
+                    await Promise.all(this.workers.map((w) => w.reloadDynamicDs()));
+                }
             }
             catch (e) {
                 logger.error(e, `failed to index block at height ${height} ${e.handler ? `${e.handler}(${(_a = e.stack) !== null && _a !== void 0 ? _a : ''})` : ''}`);

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAAmE;AACnE,yDAAsD;AACtD,+CAA4C;AAC5C,gDAM0B;AAC1B,kDAA0B;AAC1B,mCAA8B;AAC9B,wDAAoD;AASpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAkBzD,KAAK,UAAU,mBAAmB;IAChC,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CACjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;KACZ,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,2CAAoC;IAS5C,YACE,UAAsB,EACtB,YAA2B,EAC3B,cAA8B;QAE9B,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;QACtC,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,qBAAS,CAAC,UAAU,GAAG,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACrD,CAAC;QAdI,gBAAW,GAAG,CAAC,CAAC;QAChB,eAAU,GAAG,KAAK,CAAC;QAczB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;SACH;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,yBAAyB;QACzB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAEnB,mBAAmB;QACnB,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SAC3D;IACH,CAAC;IAED,aAAa,CAAC,OAAiB,EAAE,kBAA2B;QAC1D,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QAC5B,MAAM,CAAC,IAAI,CACT,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,YAC/C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,iDAAiD;QACjD,IAAI,IAAI,EAAE;YACR;;;;eAIG;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;SAC/D;aAAM;YACL;;;;eAIG;YACH,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CACrD,CAAC;SACH;QAED,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;IAClE,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,SAAiB;QACpD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,IAAA,gBAAM,EAAC,MAAM,EAAE,UAAU,SAAS,YAAY,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC;QACjD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;;YAC9B,IAAI;gBACF,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,IAAI,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE;oBAC9C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO;iBACR;gBAED,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjD,IAAI,QAAQ,GAAG,IAAI,EAAE;oBACnB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,CAAC,EAAE,CAClE,CAAC;iBACH;qBAAM,IAAI,QAAQ,GAAG,GAAG,EAAE;oBACzB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,MAAM,CAC/C,GAAG,QAAQ,IAAI,CAChB,EAAE,CACJ,CAAC;iBACH;gBAED,eAAe;gBACf,gKAAgK;gBAChK,KAAK;gBAEL,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAE7B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,kBAAkB,EAAE,GAC3D,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE;oBAClC,gBAAgB;oBAChB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;oBACnD,kBAAkB;iBACnB,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC,CAAC;QAEF,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAGK,AAAN,KAAK,CAAC,kBAAkB;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,oBAAoB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAc;QACrC,KAAK,CAAC,oBAAoB,GAAG,MAAM,CAAC;QACpC,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;YAClD,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAA;AA3BO;IADL,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sEAMf;AA1JU,4BAA4B;IADxC,IAAA,mBAAU,GAAE;qCAYG,sBAAU;QACR,6BAAa;QACX,gCAAc;GAbrB,4BAA4B,CAgLxC;AAhLY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n getLogger,\n NodeConfig,\n IndexerEvent,\n Worker,\n AutoQueue,\n} from '@subql/node-core';\nimport chalk from 'chalk';\nimport { last } from 'lodash';\nimport { ProjectService } from '../project.service';\nimport {\n FetchBlock,\n ProcessBlock,\n InitWorker,\n NumFetchedBlocks,\n NumFetchingBlocks,\n GetWorkerStatus,\n} from '../worker/worker';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IIndexerWorker = {\n processBlock: ProcessBlock;\n fetchBlock: FetchBlock;\n numFetchedBlocks: NumFetchedBlocks;\n numFetchingBlocks: NumFetchingBlocks;\n getStatus: GetWorkerStatus;\n};\n\ntype IInitIndexerWorker = IIndexerWorker & {\n initWorker: InitWorker;\n};\n\ntype IndexerWorker = IIndexerWorker & {\n terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(): Promise<IndexerWorker> {\n const indexerWorker = Worker.create<IInitIndexerWorker>(\n path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n [\n 'initWorker',\n 'processBlock',\n 'fetchBlock',\n 'numFetchedBlocks',\n 'numFetchingBlocks',\n 'getStatus',\n ],\n );\n\n await indexerWorker.initWorker();\n\n return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n extends BaseBlockDispatcher<AutoQueue<void>>\n implements OnApplicationShutdown\n{\n private workers: IndexerWorker[];\n private numWorkers: number;\n\n private taskCounter = 0;\n private isShutdown = false;\n\n constructor(\n nodeConfig: NodeConfig,\n eventEmitter: EventEmitter2,\n projectService: ProjectService,\n ) {\n const numWorkers = nodeConfig.workers;\n super(\n nodeConfig,\n eventEmitter,\n projectService,\n new AutoQueue(numWorkers * nodeConfig.batchSize * 2),\n );\n this.numWorkers = numWorkers;\n }\n\n async init(\n onDynamicDsCreated: (height: number) => Promise<void>,\n ): Promise<void> {\n if (this.nodeConfig.unfinalizedBlocks) {\n throw new Error(\n 'Sorry, best block feature is not supported with workers yet.',\n );\n }\n\n this.workers = await Promise.all(\n new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()),\n );\n\n this.onDynamicDsCreated = onDynamicDsCreated;\n\n const blockAmount = await this.projectService.getProcessedBlockCount();\n this.setProcessedBlockCount(blockAmount ?? 0);\n }\n\n async onApplicationShutdown(): Promise<void> {\n this.isShutdown = true;\n // Stop processing blocks\n this.queue.abort();\n\n // Stop all workers\n if (this.workers) {\n await Promise.all(this.workers.map((w) => w.terminate()));\n }\n }\n\n enqueueBlocks(heights: number[], latestBufferHeight?: number): void {\n if (!heights.length) return;\n logger.info(\n `Enqueing blocks [${heights[0]}...${last(heights)}], total ${\n heights.length\n } blocks`,\n );\n\n // eslint-disable-next-line no-constant-condition\n if (true) {\n /\n Load balancing:\n * worker1: 1,2,3\n * worker2: 4,5,6\n /\n const workerIdx = this.getNextWorkerIndex();\n heights.map((height) => this.enqueueBlock(height, workerIdx));\n } else {\n /\n * Load balancing:\n * worker1: 1,3,5\n * worker2: 2,4,6\n */\n heights.map((height) =>\n this.enqueueBlock(height, this.getNextWorkerIndex()),\n );\n }\n\n this.latestBufferedHeight = latestBufferHeight ?? last(heights);\n }\n\n private enqueueBlock(height: number, workerIdx: number) {\n if (this.isShutdown) return;\n const worker = this.workers[workerIdx];\n\n assert(worker, `Worker ${workerIdx} not found`);\n\n // Used to compare before and after as a way to check if queue was flushed\n const bufferedHeight = this.latestBufferedHeight;\n const pendingBlock = worker.fetchBlock(height);\n\n const processBlock = async () => {\n try {\n const start = new Date();\n const result = await pendingBlock;\n const end = new Date();\n\n if (bufferedHeight > this.latestBufferedHeight) {\n logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n return;\n }\n\n const waitTime = end.getTime() - start.getTime();\n if (waitTime > 1000) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n );\n } else if (waitTime > 200) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.yellow(\n `${waitTime}ms`,\n )}`,\n );\n }\n\n // logger.info(\n // `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,\n // );\n\n this.preProcessBlock(height);\n\n const { dynamicDsCreated, operationHash, reindexBlockHeight } =\n await worker.processBlock(height);\n\n await this.postProcessBlock(height, {\n dynamicDsCreated,\n operationHash: Buffer.from(operationHash, 'base64'),\n reindexBlockHeight,\n });\n } catch (e) {\n logger.error(\n e,\n `failed to index block at height ${height} ${\n e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n }`,\n );\n process.exit(1);\n }\n };\n\n void this.queue.put(processBlock);\n }\n\n @Interval(15000)\n async sampleWorkerStatus(): Promise<void> {\n for (const worker of this.workers) {\n const status = await worker.getStatus();\n logger.info(JSON.stringify(status));\n }\n }\n\n // Getter doesn't seem to cary from abstract class\n get latestBufferedHeight(): number {\n return this._latestBufferedHeight;\n }\n\n set latestBufferedHeight(height: number) {\n super.latestBufferedHeight = height;\n // There is only a single queue with workers so we treat them as the same\n this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n value: this.queueSize,\n });\n }\n\n private getNextWorkerIndex(): number {\n const index = this.taskCounter % this.numWorkers;\n\n this.taskCounter++;\n\n return index;\n }\n}\n"]}
1	+ {"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAAmE;AACnE,yDAAsD;AACtD,+CAA4C;AAC5C,gDAM0B;AAC1B,kDAA0B;AAC1B,mCAA8B;AAC9B,wDAAoD;AAUpD,mEAA8D;AAE9D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAmBzD,KAAK,UAAU,mBAAmB;IAChC,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CACjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,iBAAiB;KAClB,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGM,IAAM,4BAA4B,GAAlC,MAAM,4BACX,SAAQ,2CAAoC;IAS5C,YACE,UAAsB,EACtB,YAA2B,EAC3B,cAA8B;QAE9B,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;QACtC,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,IAAI,qBAAS,CAAC,UAAU,GAAG,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CACrD,CAAC;QAdI,gBAAW,GAAG,CAAC,CAAC;QAChB,eAAU,GAAG,KAAK,CAAC;QAczB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,IAAI,CACR,kBAAqD;QAErD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;SACH;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;QACvE,IAAI,CAAC,sBAAsB,CAAC,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,yBAAyB;QACzB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAEnB,mBAAmB;QACnB,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;SAC3D;IACH,CAAC;IAED,aAAa,CAAC,OAAiB,EAAE,kBAA2B;QAC1D,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QAC5B,MAAM,CAAC,IAAI,CACT,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,IAAA,aAAI,EAAC,OAAO,CAAC,YAC/C,OAAO,CAAC,MACV,SAAS,CACV,CAAC;QAEF,iDAAiD;QACjD,IAAI,IAAI,EAAE;YACR;;;;eAIG;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;SAC/D;aAAM;YACL;;;;eAIG;YACH,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CACrD,CAAC;SACH;QAED,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAA,aAAI,EAAC,OAAO,CAAC,CAAC;IAClE,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,SAAiB;QACpD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,IAAA,gBAAM,EAAC,MAAM,EAAE,UAAU,SAAS,YAAY,CAAC,CAAC;QAEhD,0EAA0E;QAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC;QACjD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;;YAC9B,IAAI;gBACF,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;gBAEvB,IAAI,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE;oBAC9C,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBACtE,OAAO;iBACR;gBAED,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;gBACjD,IAAI,QAAQ,GAAG,IAAI,EAAE;oBACnB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,CAAC,EAAE,CAClE,CAAC;iBACH;qBAAM,IAAI,QAAQ,GAAG,GAAG,EAAE;oBACzB,MAAM,CAAC,IAAI,CACT,0BAA0B,MAAM,KAAK,eAAK,CAAC,MAAM,CAC/C,GAAG,QAAQ,IAAI,CAChB,EAAE,CACJ,CAAC;iBACH;gBAED,eAAe;gBACf,gKAAgK;gBAChK,KAAK;gBAEL,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAE7B,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,kBAAkB,EAAE,GAC3D,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE;oBAClC,gBAAgB;oBAChB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;oBACnD,kBAAkB;iBACnB,CAAC,CAAC;gBAEH,IAAI,gBAAgB,EAAE;oBACpB,iDAAiD;oBACjD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;iBACjE;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CACV,CAAC,EACD,mCAAmC,MAAM,IACvC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACjD,EAAE,CACH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC,CAAC;QAEF,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAGK,AAAN,KAAK,CAAC,kBAAkB;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,oBAAoB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAc;QACrC,KAAK,CAAC,oBAAoB,GAAG,MAAM,CAAC;QACpC,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,EAAE;YAClD,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,OAAO,KAAK,CAAC;IACf,CAAC;CACF,CAAA;AA3BO;IADL,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sEAMf;AA/JU,4BAA4B;IADxC,IAAA,mBAAU,GAAE;qCAYG,sBAAU;QACR,6BAAa;QACX,gCAAc;GAbrB,4BAA4B,CAqLxC;AArLY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n getLogger,\n NodeConfig,\n IndexerEvent,\n Worker,\n AutoQueue,\n} from '@subql/node-core';\nimport chalk from 'chalk';\nimport { last } from 'lodash';\nimport { ProjectService } from '../project.service';\nimport {\n FetchBlock,\n ProcessBlock,\n InitWorker,\n NumFetchedBlocks,\n NumFetchingBlocks,\n GetWorkerStatus,\n ReloadDynamicDs,\n} from '../worker/worker';\nimport { BaseBlockDispatcher } from './base-block-dispatcher';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IIndexerWorker = {\n processBlock: ProcessBlock;\n fetchBlock: FetchBlock;\n numFetchedBlocks: NumFetchedBlocks;\n numFetchingBlocks: NumFetchingBlocks;\n getStatus: GetWorkerStatus;\n reloadDynamicDs: ReloadDynamicDs;\n};\n\ntype IInitIndexerWorker = IIndexerWorker & {\n initWorker: InitWorker;\n};\n\ntype IndexerWorker = IIndexerWorker & {\n terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(): Promise<IndexerWorker> {\n const indexerWorker = Worker.create<IInitIndexerWorker>(\n path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n [\n 'initWorker',\n 'processBlock',\n 'fetchBlock',\n 'numFetchedBlocks',\n 'numFetchingBlocks',\n 'getStatus',\n 'reloadDynamicDs',\n ],\n );\n\n await indexerWorker.initWorker();\n\n return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n extends BaseBlockDispatcher<AutoQueue<void>>\n implements OnApplicationShutdown\n{\n private workers: IndexerWorker[];\n private numWorkers: number;\n\n private taskCounter = 0;\n private isShutdown = false;\n\n constructor(\n nodeConfig: NodeConfig,\n eventEmitter: EventEmitter2,\n projectService: ProjectService,\n ) {\n const numWorkers = nodeConfig.workers;\n super(\n nodeConfig,\n eventEmitter,\n projectService,\n new AutoQueue(numWorkers * nodeConfig.batchSize * 2),\n );\n this.numWorkers = numWorkers;\n }\n\n async init(\n onDynamicDsCreated: (height: number) => Promise<void>,\n ): Promise<void> {\n if (this.nodeConfig.unfinalizedBlocks) {\n throw new Error(\n 'Sorry, best block feature is not supported with workers yet.',\n );\n }\n\n this.workers = await Promise.all(\n new Array(this.numWorkers).fill(0).map(() => createIndexerWorker()),\n );\n\n this.onDynamicDsCreated = onDynamicDsCreated;\n\n const blockAmount = await this.projectService.getProcessedBlockCount();\n this.setProcessedBlockCount(blockAmount ?? 0);\n }\n\n async onApplicationShutdown(): Promise<void> {\n this.isShutdown = true;\n // Stop processing blocks\n this.queue.abort();\n\n // Stop all workers\n if (this.workers) {\n await Promise.all(this.workers.map((w) => w.terminate()));\n }\n }\n\n enqueueBlocks(heights: number[], latestBufferHeight?: number): void {\n if (!heights.length) return;\n logger.info(\n `Enqueing blocks [${heights[0]}...${last(heights)}], total ${\n heights.length\n } blocks`,\n );\n\n // eslint-disable-next-line no-constant-condition\n if (true) {\n /\n Load balancing:\n * worker1: 1,2,3\n * worker2: 4,5,6\n /\n const workerIdx = this.getNextWorkerIndex();\n heights.map((height) => this.enqueueBlock(height, workerIdx));\n } else {\n /\n * Load balancing:\n * worker1: 1,3,5\n * worker2: 2,4,6\n */\n heights.map((height) =>\n this.enqueueBlock(height, this.getNextWorkerIndex()),\n );\n }\n\n this.latestBufferedHeight = latestBufferHeight ?? last(heights);\n }\n\n private enqueueBlock(height: number, workerIdx: number) {\n if (this.isShutdown) return;\n const worker = this.workers[workerIdx];\n\n assert(worker, `Worker ${workerIdx} not found`);\n\n // Used to compare before and after as a way to check if queue was flushed\n const bufferedHeight = this.latestBufferedHeight;\n const pendingBlock = worker.fetchBlock(height);\n\n const processBlock = async () => {\n try {\n const start = new Date();\n const result = await pendingBlock;\n const end = new Date();\n\n if (bufferedHeight > this.latestBufferedHeight) {\n logger.debug(`Queue was reset for new DS, discarding fetched blocks`);\n return;\n }\n\n const waitTime = end.getTime() - start.getTime();\n if (waitTime > 1000) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n );\n } else if (waitTime > 200) {\n logger.info(\n `Waiting to fetch block ${height}: ${chalk.yellow(\n `${waitTime}ms`,\n )}`,\n );\n }\n\n // logger.info(\n // `worker ${workerIdx} processing block ${height}, fetched blocks: ${await worker.numFetchedBlocks()}, fetching blocks: ${await worker.numFetchingBlocks()}`,\n // );\n\n this.preProcessBlock(height);\n\n const { dynamicDsCreated, operationHash, reindexBlockHeight } =\n await worker.processBlock(height);\n\n await this.postProcessBlock(height, {\n dynamicDsCreated,\n operationHash: Buffer.from(operationHash, 'base64'),\n reindexBlockHeight,\n });\n\n if (dynamicDsCreated) {\n // Ensure all workers are aware of all dynamic ds\n await Promise.all(this.workers.map((w) => w.reloadDynamicDs()));\n }\n } catch (e) {\n logger.error(\n e,\n `failed to index block at height ${height} ${\n e.handler ? `${e.handler}(${e.stack ?? ''})` : ''\n }`,\n );\n process.exit(1);\n }\n };\n\n void this.queue.put(processBlock);\n }\n\n @Interval(15000)\n async sampleWorkerStatus(): Promise<void> {\n for (const worker of this.workers) {\n const status = await worker.getStatus();\n logger.info(JSON.stringify(status));\n }\n }\n\n // Getter doesn't seem to cary from abstract class\n get latestBufferedHeight(): number {\n return this._latestBufferedHeight;\n }\n\n set latestBufferedHeight(height: number) {\n super.latestBufferedHeight = height;\n // There is only a single queue with workers so we treat them as the same\n this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {\n value: this.queueSize,\n });\n }\n\n private getNextWorkerIndex(): number {\n const index = this.taskCounter % this.numWorkers;\n\n this.taskCounter++;\n\n return index;\n }\n}\n"]}

package/dist/indexer/dynamic-ds.service.d.ts CHANGED Viewed

@@ -18,6 +18,8 @@ export declare class DynamicDsService {
     resetDynamicDatasource(targetHeight: number, tx: Transaction): Promise<void>;
     createDynamicDatasource(params: DatasourceParams, tx: Transaction): Promise<SubqlProjectDs>;
     getDynamicDatasources(): Promise<SubqlProjectDs[]>;
+    reloadDynamicDatasources(): Promise<void>;
+    private loadDynamicDatasources;
     deleteTempDsRecords(blockHeight: number): void;
     private getDynamicDatasourceParams;
     private saveDynamicDatasourceParams;

package/dist/indexer/dynamic-ds.service.js CHANGED Viewed

@@ -62,20 +62,33 @@ let DynamicDsService = class DynamicDsService {
         }
     }
     async getDynamicDatasources() {
+        // Workers should not cache this result in order to keep in sync
         if (!this._datasources) {
-            try {
-                const params = await this.getDynamicDatasourceParams();
-                this._datasources = await Promise.all(params.map((params) => this.getDatasource(params)));
-            }
-            catch (e) {
-                logger.error(`Unable to get dynamic datasources:\n${e.message}`);
-                process.exit(1);
-            }
+            this._datasources = await this.loadDynamicDatasources();
         }
         return this._datasources;
     }
+    // This is used to sync between worker threads
+    async reloadDynamicDatasources() {
+        this._datasources = await this.loadDynamicDatasources();
+    }
+    async loadDynamicDatasources() {
+        try {
+            const params = await this.getDynamicDatasourceParams();
+            const dataSources = await Promise.all(params.map((params) => this.getDatasource(params)));
+            logger.info(`Loaded ${dataSources.length} dynamic datasources`);
+            return dataSources;
+        }
+        catch (e) {
+            logger.error(`Unable to get dynamic datasources:\n${e.message}`);
+            process.exit(1);
+        }
+    }
     deleteTempDsRecords(blockHeight) {
-        delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];
+        // Main thread will not have tempDsRecords with workers
+        if (this.tempDsRecords) {
+            delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];
+        }
     }
     async getDynamicDatasourceParams(blockHeight) {
         var _a, _b, _c;
@@ -109,7 +122,6 @@ let DynamicDsService = class DynamicDsService {
         if (!template) {
             throw new Error(`Unable to find matching template in project for name: "${params.templateName}"`);
         }
-        logger.info(`Initialised dynamic datasource from template: "${params.templateName}"`);
         const dsObj = Object.assign(Object.assign({}, template), { startBlock: params.startBlock });
         delete dsObj.name;
         try {

package/dist/indexer/dynamic-ds.service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,2CAAoD;AACpD,4DAIgC;AAChC,gDAA2D;AAC3D,yDAAiD;AACjD,qDAA+C;AAC/C,mCAAuD;AAEvD,kEAA+E;AAC/E,iEAA4D;AAE5D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,MAAM,YAAY,GAAG,oBAAoB,CAAC;AAC1C,MAAM,cAAc,GAAG,KAAK,CAAC;AAStB,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAI3B,YACmB,kBAAsC,EACV,OAAwB;QADpD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACV,YAAO,GAAP,OAAO,CAAiB;IACpE,CAAC;IAEJ,IAAI,CAAC,YAA0B;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAID,KAAK,CAAC,sBAAsB,CAC1B,YAAoB,EACpB,EAAe;QAEf,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YAC1B,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,YAAY,CACtC,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAC5B,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EACvC,EAAE,WAAW,EAAE,EAAE,EAAE,CACpB,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB,CAC3B,MAAwB,EACxB,EAAe;QAEf,IAAI;YACF,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE5C,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAEnD,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;YAC/C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE3B,OAAO,EAAE,CAAC;SACX;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAEvD,IAAI,CAAC,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CACnD,CAAC;aACH;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,mBAAmB,CAAC,WAAmB;QACrC,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,WAAoB;;QAEpB,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE9D,IAAI,OAAO,GAAuB,EAAE,CAAC;QAErC,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAgB,mCAAI,IAAI,CAClC,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,EAAE;YACtB,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;SAC5B;QAED,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAA,MAAA,IAAI,CAAC,aAAa,0CAAG,cAAc,GAAG,WAAW,CAAC,mCAAI,IAAI,CAC3D,CAAC;YACF,IAAI,WAAW,CAAC,MAAM,EAAE;gBACtB,OAAO,GAAG,IAAA,kBAAS,EAAC,OAAO,EAAE,WAAW,EAAE,gBAAO,CAAC,CAAC;aACpD;SACF;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAA0B,EAC1B,EAAe;QAEf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE5E,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY;aACpB,MAAM,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;aACpE,IAAI,CAAC,GAAG,EAAE;YACT,IAAI,CAAC,aAAa,mCACb,IAAI,CAAC,aAAa,KACrB,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAClD,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAwB;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACnE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;QAEF,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,KAAK,CAAC,OAAO,mCACR,KAAK,CAAC,OAAO,GACb,MAAM,CAAC,IAAI,CACf,CAAC;gBAEF,MAAM,QAAQ,GAAG,IAAA,gCAAY,EAC3B,qDAAmC,EACnC,KAAK,CACN,CAAC;gBAEF,MAAM,MAAM,GAAG,IAAA,8BAAY,EAAC,QAAQ,EAAE;oBACpC,SAAS,EAAE,IAAI;oBACf,oBAAoB,EAAE,KAAK;iBAC5B,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,EAAE;oBACjB,MAAM,IAAI,KAAK,CACb,0BAA0B,MAAM;yBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;yBACxB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;iBACH;aACF;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AAlLY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;IAOR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCADU,yCAAkB;QACD,iCAAe;GAN5D,gBAAgB,CAkL5B;AAlLY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n EthereumRuntimeDataSourceV0_3_0Impl,\n isCustomDs,\n isRuntimeDs,\n} from '@subql/common-ethereum';\nimport { getLogger, MetadataRepo } from '@subql/node-core';\nimport { plainToClass } from 'class-transformer';\nimport { validateSync } from 'class-validator';\nimport { cloneDeep, isEqual, unionWith } from 'lodash';\nimport { Transaction } from 'sequelize/types';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\nconst logger = getLogger('dynamic-ds');\n\nconst METADATA_KEY = 'dynamicDatasources';\nconst TEMP_DS_PREFIX = 'ds_';\n\ninterface DatasourceParams {\n templateName: string;\n args?: Record<string, unknown>;\n startBlock: number;\n}\n\n@Injectable()\nexport class DynamicDsService {\n private metaDataRepo: MetadataRepo;\n private tempDsRecords: Record<string, string>;\n\n constructor(\n private readonly dsProcessorService: DsProcessorService,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n ) {}\n\n init(metaDataRepo: MetadataRepo): void {\n this.metaDataRepo = metaDataRepo;\n }\n\n private _datasources: SubqlProjectDs[];\n\n async resetDynamicDatasource(\n targetHeight: number,\n tx: Transaction,\n ): Promise<void> {\n const dynamicDs = await this.getDynamicDatasourceParams();\n if (dynamicDs.length !== 0) {\n const filteredDs = dynamicDs.filter(\n (ds) => ds.startBlock <= targetHeight,\n );\n const dsRecords = JSON.stringify(filteredDs);\n await this.metaDataRepo.upsert(\n { key: METADATA_KEY, value: dsRecords },\n { transaction: tx },\n );\n }\n }\n\n async createDynamicDatasource(\n params: DatasourceParams,\n tx: Transaction,\n ): Promise<SubqlProjectDs> {\n try {\n const ds = await this.getDatasource(params);\n\n await this.saveDynamicDatasourceParams(params, tx);\n\n logger.info(\n `Created new dynamic datasource from template: \"${params.templateName}\"`,\n );\n\n if (!this._datasources) this._datasources = [];\n this._datasources.push(ds);\n\n return ds;\n } catch (e) {\n logger.error(e.message);\n process.exit(1);\n }\n }\n\n async getDynamicDatasources(): Promise<SubqlProjectDs[]> {\n if (!this._datasources) {\n try {\n const params = await this.getDynamicDatasourceParams();\n\n this._datasources = await Promise.all(\n params.map((params) => this.getDatasource(params)),\n );\n } catch (e) {\n logger.error(`Unable to get dynamic datasources:\\n${e.message}`);\n process.exit(1);\n }\n }\n\n return this._datasources;\n }\n\n deleteTempDsRecords(blockHeight: number): void {\n delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];\n }\n\n private async getDynamicDatasourceParams(\n blockHeight?: number,\n ): Promise<DatasourceParams[]> {\n assert(this.metaDataRepo, `Model _metadata does not exist`);\n const record = await this.metaDataRepo.findByPk(METADATA_KEY);\n\n let results: DatasourceParams[] = [];\n\n const metaResults: DatasourceParams[] = JSON.parse(\n (record?.value as string) ?? '[]',\n );\n if (metaResults.length) {\n results = [...metaResults];\n }\n\n if (blockHeight !== undefined) {\n const tempResults: DatasourceParams[] = JSON.parse(\n this.tempDsRecords?.[TEMP_DS_PREFIX + blockHeight] ?? '[]',\n );\n if (tempResults.length) {\n results = unionWith(results, tempResults, isEqual);\n }\n }\n\n return results;\n }\n\n private async saveDynamicDatasourceParams(\n dsParams: DatasourceParams,\n tx: Transaction,\n ): Promise<void> {\n const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);\n\n assert(this.metaDataRepo, `Model _metadata does not exist`);\n const dsRecords = JSON.stringify([...existing, dsParams]);\n await this.metaDataRepo\n .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })\n .then(() => {\n this.tempDsRecords = {\n ...this.tempDsRecords,\n [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords,\n };\n });\n }\n\n private async getDatasource(\n params: DatasourceParams,\n ): Promise<SubqlProjectDs> {\n const template = cloneDeep(\n this.project.templates.find((t) => t.name === params.templateName),\n );\n\n if (!template) {\n throw new Error(\n `Unable to find matching template in project for name: \"${params.templateName}\"`,\n );\n }\n\n logger.info(\n `Initialised dynamic datasource from template: \"${params.templateName}\"`,\n );\n\n const dsObj = {\n ...template,\n startBlock: params.startBlock,\n } as SubqlProjectDs;\n delete dsObj.name;\n try {\n if (isCustomDs(dsObj)) {\n dsObj.processor.options = {\n ...dsObj.processor.options,\n ...params.args,\n };\n await this.dsProcessorService.validateCustomDs([dsObj]);\n } else if (isRuntimeDs(dsObj)) {\n dsObj.options = {\n ...dsObj.options,\n ...params.args,\n };\n\n const parsedDs = plainToClass(\n EthereumRuntimeDataSourceV0_3_0Impl,\n dsObj,\n );\n\n const errors = validateSync(parsedDs, {\n whitelist: true,\n forbidNonWhitelisted: false,\n });\n if (errors.length) {\n throw new Error(\n `Dynamic ds is invalid\\n${errors\n .map((e) => e.toString())\n .join('\\n')}`,\n );\n }\n }\n return dsObj;\n } catch (e) {\n throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n }\n }\n}\n"]}
1	+ {"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAE5B,2CAAoD;AACpD,4DAIgC;AAChC,gDAA2D;AAC3D,yDAAiD;AACjD,qDAA+C;AAC/C,mCAAuD;AAEvD,kEAA+E;AAC/E,iEAA4D;AAE5D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,MAAM,YAAY,GAAG,oBAAoB,CAAC;AAC1C,MAAM,cAAc,GAAG,KAAK,CAAC;AAStB,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAI3B,YACmB,kBAAsC,EACV,OAAwB;QADpD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACV,YAAO,GAAP,OAAO,CAAiB;IACpE,CAAC;IAEJ,IAAI,CAAC,YAA0B;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAID,KAAK,CAAC,sBAAsB,CAC1B,YAAoB,EACpB,EAAe;QAEf,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YAC1B,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,YAAY,CACtC,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAC5B,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EACvC,EAAE,WAAW,EAAE,EAAE,EAAE,CACpB,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB,CAC3B,MAAwB,EACxB,EAAe;QAEf,IAAI;YACF,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE5C,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAEnD,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;YAC/C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE3B,OAAO,EAAE,CAAC;SACX;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,gEAAgE;QAChE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;SACzD;QAED,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,8CAA8C;IAC9C,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,IAAI;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;YAEvD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CACnD,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,MAAM,sBAAsB,CAAC,CAAC;YAEhE,OAAO,WAAW,CAAC;SACpB;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,mBAAmB,CAAC,WAAmB;QACrC,uDAAuD;QACvD,IAAI,IAAI,CAAC,aAAa,EAAE;YACtB,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC;SACzD;IACH,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,WAAoB;;QAEpB,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE9D,IAAI,OAAO,GAAuB,EAAE,CAAC;QAErC,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAgB,mCAAI,IAAI,CAClC,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,EAAE;YACtB,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;SAC5B;QAED,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAA,MAAA,IAAI,CAAC,aAAa,0CAAG,cAAc,GAAG,WAAW,CAAC,mCAAI,IAAI,CAC3D,CAAC;YACF,IAAI,WAAW,CAAC,MAAM,EAAE;gBACtB,OAAO,GAAG,IAAA,kBAAS,EAAC,OAAO,EAAE,WAAW,EAAE,gBAAO,CAAC,CAAC;aACpD;SACF;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAA0B,EAC1B,EAAe;QAEf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE5E,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY;aACpB,MAAM,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;aACpE,IAAI,CAAC,GAAG,EAAE;YACT,IAAI,CAAC,aAAa,mCACb,IAAI,CAAC,aAAa,KACrB,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAClD,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAwB;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACnE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,KAAK,CAAC,OAAO,mCACR,KAAK,CAAC,OAAO,GACb,MAAM,CAAC,IAAI,CACf,CAAC;gBAEF,MAAM,QAAQ,GAAG,IAAA,gCAAY,EAC3B,qDAAmC,EACnC,KAAK,CACN,CAAC;gBAEF,MAAM,MAAM,GAAG,IAAA,8BAAY,EAAC,QAAQ,EAAE;oBACpC,SAAS,EAAE,IAAI;oBACf,oBAAoB,EAAE,KAAK;iBAC5B,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,EAAE;oBACjB,MAAM,IAAI,KAAK,CACb,0BAA0B,MAAM;yBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;yBACxB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;iBACH;aACF;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AA/LY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;IAOR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCADU,yCAAkB;QACD,iCAAe;GAN5D,gBAAgB,CA+L5B;AA/LY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n EthereumRuntimeDataSourceV0_3_0Impl,\n isCustomDs,\n isRuntimeDs,\n} from '@subql/common-ethereum';\nimport { getLogger, MetadataRepo } from '@subql/node-core';\nimport { plainToClass } from 'class-transformer';\nimport { validateSync } from 'class-validator';\nimport { cloneDeep, isEqual, unionWith } from 'lodash';\nimport { Transaction } from 'sequelize/types';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\nconst logger = getLogger('dynamic-ds');\n\nconst METADATA_KEY = 'dynamicDatasources';\nconst TEMP_DS_PREFIX = 'ds_';\n\ninterface DatasourceParams {\n templateName: string;\n args?: Record<string, unknown>;\n startBlock: number;\n}\n\n@Injectable()\nexport class DynamicDsService {\n private metaDataRepo: MetadataRepo;\n private tempDsRecords: Record<string, string>;\n\n constructor(\n private readonly dsProcessorService: DsProcessorService,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n ) {}\n\n init(metaDataRepo: MetadataRepo): void {\n this.metaDataRepo = metaDataRepo;\n }\n\n private _datasources: SubqlProjectDs[];\n\n async resetDynamicDatasource(\n targetHeight: number,\n tx: Transaction,\n ): Promise<void> {\n const dynamicDs = await this.getDynamicDatasourceParams();\n if (dynamicDs.length !== 0) {\n const filteredDs = dynamicDs.filter(\n (ds) => ds.startBlock <= targetHeight,\n );\n const dsRecords = JSON.stringify(filteredDs);\n await this.metaDataRepo.upsert(\n { key: METADATA_KEY, value: dsRecords },\n { transaction: tx },\n );\n }\n }\n\n async createDynamicDatasource(\n params: DatasourceParams,\n tx: Transaction,\n ): Promise<SubqlProjectDs> {\n try {\n const ds = await this.getDatasource(params);\n\n await this.saveDynamicDatasourceParams(params, tx);\n\n logger.info(\n `Created new dynamic datasource from template: \"${params.templateName}\"`,\n );\n\n if (!this._datasources) this._datasources = [];\n this._datasources.push(ds);\n\n return ds;\n } catch (e) {\n logger.error(e.message);\n process.exit(1);\n }\n }\n\n async getDynamicDatasources(): Promise<SubqlProjectDs[]> {\n // Workers should not cache this result in order to keep in sync\n if (!this._datasources) {\n this._datasources = await this.loadDynamicDatasources();\n }\n\n return this._datasources;\n }\n\n // This is used to sync between worker threads\n async reloadDynamicDatasources(): Promise<void> {\n this._datasources = await this.loadDynamicDatasources();\n }\n\n private async loadDynamicDatasources(): Promise<SubqlProjectDs[]> {\n try {\n const params = await this.getDynamicDatasourceParams();\n\n const dataSources = await Promise.all(\n params.map((params) => this.getDatasource(params)),\n );\n\n logger.info(`Loaded ${dataSources.length} dynamic datasources`);\n\n return dataSources;\n } catch (e) {\n logger.error(`Unable to get dynamic datasources:\\n${e.message}`);\n process.exit(1);\n }\n }\n\n deleteTempDsRecords(blockHeight: number): void {\n // Main thread will not have tempDsRecords with workers\n if (this.tempDsRecords) {\n delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];\n }\n }\n\n private async getDynamicDatasourceParams(\n blockHeight?: number,\n ): Promise<DatasourceParams[]> {\n assert(this.metaDataRepo, `Model _metadata does not exist`);\n const record = await this.metaDataRepo.findByPk(METADATA_KEY);\n\n let results: DatasourceParams[] = [];\n\n const metaResults: DatasourceParams[] = JSON.parse(\n (record?.value as string) ?? '[]',\n );\n if (metaResults.length) {\n results = [...metaResults];\n }\n\n if (blockHeight !== undefined) {\n const tempResults: DatasourceParams[] = JSON.parse(\n this.tempDsRecords?.[TEMP_DS_PREFIX + blockHeight] ?? '[]',\n );\n if (tempResults.length) {\n results = unionWith(results, tempResults, isEqual);\n }\n }\n\n return results;\n }\n\n private async saveDynamicDatasourceParams(\n dsParams: DatasourceParams,\n tx: Transaction,\n ): Promise<void> {\n const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);\n\n assert(this.metaDataRepo, `Model _metadata does not exist`);\n const dsRecords = JSON.stringify([...existing, dsParams]);\n await this.metaDataRepo\n .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })\n .then(() => {\n this.tempDsRecords = {\n ...this.tempDsRecords,\n [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords,\n };\n });\n }\n\n private async getDatasource(\n params: DatasourceParams,\n ): Promise<SubqlProjectDs> {\n const template = cloneDeep(\n this.project.templates.find((t) => t.name === params.templateName),\n );\n\n if (!template) {\n throw new Error(\n `Unable to find matching template in project for name: \"${params.templateName}\"`,\n );\n }\n\n const dsObj = {\n ...template,\n startBlock: params.startBlock,\n } as SubqlProjectDs;\n delete dsObj.name;\n try {\n if (isCustomDs(dsObj)) {\n dsObj.processor.options = {\n ...dsObj.processor.options,\n ...params.args,\n };\n await this.dsProcessorService.validateCustomDs([dsObj]);\n } else if (isRuntimeDs(dsObj)) {\n dsObj.options = {\n ...dsObj.options,\n ...params.args,\n };\n\n const parsedDs = plainToClass(\n EthereumRuntimeDataSourceV0_3_0Impl,\n dsObj,\n );\n\n const errors = validateSync(parsedDs, {\n whitelist: true,\n forbidNonWhitelisted: false,\n });\n if (errors.length) {\n throw new Error(\n `Dynamic ds is invalid\\n${errors\n .map((e) => e.toString())\n .join('\\n')}`,\n );\n }\n }\n return dsObj;\n } catch (e) {\n throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n }\n }\n}\n"]}

package/dist/indexer/project.service.js CHANGED Viewed

@@ -86,10 +86,10 @@ let ProjectService = class ProjectService {
             this._startHeight = await this.getStartHeight();
         }
         else {
+            this._schema = await this.getExistingProjectSchema();
             this.metadataRepo = await (0, node_core_1.MetadataFactory)(this.sequelize, this.schema, this.nodeConfig.multiChain, this.project.network.chainId);
             this.dynamicDsService.init(this.metadataRepo);
             await this.sequelize.sync();
-            this._schema = await this.getExistingProjectSchema();
             (0, assert_1.default)(this._schema, 'Schema should be created in main thread');
             await this.initDbSchema();
             if (this.nodeConfig.proofOfIndex) {

package/dist/indexer/project.service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAAoD;AACpD,yDAAsD;AACtD,gDAY0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAqE;AACrE,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AAExD,2EAIqC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAMzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B,EAC3B,uBAAiD;QAVxC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACQ,YAAO,GAAP,OAAO,CAAiB;QACpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,4BAAuB,GAAvB,uBAAuB,CAA0B;IACxD,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAChD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;SACjD;aAAM;YACL,IAAI,CAAC,YAAY,GAAG,MAAM,IAAA,2BAAe,EACvC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,CAAC,UAAU,EAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAC7B,CAAC;YAEF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACrD,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;SACF;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CACzD,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CACxB,CAAC;QAEF,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;SACjC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5D,CAAC;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,YAAY,GAAG,MAAM,IAAA,2BAAe,EACxC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,CAAC,UAAU,EAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAC7B,CAAC;QAEF,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,aAAa;YACb,SAAS;YACT,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;YACtB,mBAAmB;YACnB,cAAc;SACN,CAAC;QAEX,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC;YACzC,KAAK,EAAE;gBACL,GAAG,EAAE,IAAI;aACV;SACF,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YAC5C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3B,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAiE,CAAC,CAAC;QAEtE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,YAAY;oBACjB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI;iBACrC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,mBAAmB;oBACxB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO;iBACxC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,aAAa;oBAClB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;iBACtC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,oBAAoB;oBACzB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO;iBACzC,CAAC;aACH,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;SACvE;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;SACjE;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SACrE;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,MAAM,YAAY,CAAC,MAAM,CAAC;gBACxB,GAAG,EAAE,oBAAoB;gBACzB,KAAK,EAAE,cAAc;aACtB,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SACtE;QAED,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE;YAC/B,MAAM,YAAY,CAAC,MAAM,CAAC;gBACxB,GAAG,EAAE,mBAAmB;gBACxB,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,MAAM,YAAY,CAAC,MAAM,CAAC;gBACxB,GAAG,EAAE,aAAa;gBAClB,KAAK,EAAE,IAAI,CAAC,4BAA4B,EAAE;aAC3C,CAAC,CAAC;SACJ;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAAc;QAC5C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC7B,GAAG,EAAE,aAAa;YAClB,KAAK,EAAE,MAAM;SACd,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,4BAA4B;QAChC,MAAM,GAAG,GAAG,MAAM,IAAA,2BAAe,EAC/B,IAAI,CAAC,YAAY,EACjB,2DAA+B,CAChC,CAAC;QACF,IAAI,GAAG,EAAE;YACP,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAe,CAAC;SACtC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,8BAA8B;QAClC,OAAO,IAAA,2BAAe,EACpB,IAAI,CAAC,YAAY,EACjB,iEAAqC,CACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY;YACjB,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU;aACnB,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;aACxC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACP,CAAC;IACD,KAAK,CAAC,sBAAsB;QAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,qBAAqB,EAAE;SACtC,CAAC,CAAC;QAEH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;CACF,CAAA;AAhWY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAaR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALU,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;QACF,oDAAwB;GAjBhD,cAAc,CAgW1B;AAhWY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n ApiService,\n MetadataFactory,\n MetadataRepo,\n NodeConfig,\n IndexerEvent,\n StoreService,\n PoiService,\n MmrService,\n getLogger,\n getExistingProjectSchema,\n getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n generateTimestampReferenceForBlockFilters,\n SubqlProjectDs,\n SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema, initHotSchemaReload } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { BestBlocks } from './types';\nimport {\n METADATA_LAST_FINALIZED_PROCESSED_KEY,\n METADATA_UNFINALIZED_BLOCKS_KEY,\n UnfinalizedBlocksService,\n} from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService {\n private _schema: string;\n private metadataRepo: MetadataRepo;\n private _startHeight: number;\n private _blockOffset: number;\n\n constructor(\n private readonly dsProcessorService: DsProcessorService,\n private readonly apiService: ApiService,\n private readonly poiService: PoiService,\n protected readonly mmrService: MmrService,\n private readonly sequelize: Sequelize,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n private readonly storeService: StoreService,\n private readonly nodeConfig: NodeConfig,\n private readonly dynamicDsService: DynamicDsService,\n private eventEmitter: EventEmitter2,\n private unfinalizedBlockService: UnfinalizedBlocksService,\n ) {}\n\n get schema(): string {\n return this._schema;\n }\n\n get dataSources(): SubqlProjectDs[] {\n return this.project.dataSources;\n }\n\n get blockOffset(): number {\n return this._blockOffset;\n }\n\n get startHeight(): number {\n return this._startHeight;\n }\n\n get isHistorical(): boolean {\n return this.storeService.historical;\n }\n\n // eslint-disable-next-line @typescript-eslint/require-await\n private async getExistingProjectSchema(): Promise<string> {\n return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n }\n\n async init(): Promise<void> {\n // Used to load assets into DS-processor, has to be done in any thread\n await this.dsProcessorService.validateProjectCustomDatasources();\n // Do extra work on main thread to setup stuff\n this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n this.project.dataSources,\n this.apiService.api,\n );\n if (isMainThread) {\n this._schema = await this.ensureProject();\n await this.initDbSchema();\n this.metadataRepo = await this.ensureMetadata();\n this.dynamicDsService.init(this.metadataRepo);\n\n await this.initHotSchemaReload();\n\n if (this.nodeConfig.proofOfIndex) {\n const blockOffset = await this.getMetadataBlockOffset();\n void this.setBlockOffset(Number(blockOffset));\n await this.poiService.init(this.schema);\n }\n\n this._startHeight = await this.getStartHeight();\n } else {\n this.metadataRepo = await MetadataFactory(\n this.sequelize,\n this.schema,\n this.nodeConfig.multiChain,\n this.project.network.chainId,\n );\n\n this.dynamicDsService.init(this.metadataRepo);\n\n await this.sequelize.sync();\n\n this._schema = await this.getExistingProjectSchema();\n assert(this._schema, 'Schema should be created in main thread');\n await this.initDbSchema();\n\n if (this.nodeConfig.proofOfIndex) {\n await this.poiService.init(this.schema);\n }\n }\n\n if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n logger.error(\n 'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n );\n process.exit(1);\n }\n\n const reindexedTo = await this.unfinalizedBlockService.init(\n this.metadataRepo,\n this.reindex.bind(this),\n );\n\n if (reindexedTo !== undefined) {\n this._startHeight = reindexedTo;\n }\n }\n\n private async ensureProject(): Promise<string> {\n let schema = await this.getExistingProjectSchema();\n if (!schema) {\n schema = await this.createProjectSchema();\n }\n this.eventEmitter.emit(IndexerEvent.Ready, {\n value: true,\n });\n\n return schema;\n }\n\n private async createProjectSchema(): Promise<string> {\n let schema: string;\n if (this.nodeConfig.localMode) {\n // create tables in default schema if local mode is enabled\n schema = DEFAULT_DB_SCHEMA;\n } else {\n schema = this.nodeConfig.dbSchema;\n const schemas = await this.sequelize.showAllSchemas(undefined);\n if (!(schemas as unknown as string[]).includes(schema)) {\n await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n }\n }\n\n return schema;\n }\n\n private async initHotSchemaReload(): Promise<void> {\n await initHotSchemaReload(this.schema, this.storeService);\n }\n private async initDbSchema(): Promise<void> {\n await initDbSchema(this.project, this.schema, this.storeService);\n }\n\n private async ensureMetadata(): Promise<MetadataRepo> {\n const metadataRepo = await MetadataFactory(\n this.sequelize,\n this.schema,\n this.nodeConfig.multiChain,\n this.project.network.chainId,\n );\n\n this.eventEmitter.emit(\n IndexerEvent.NetworkMetadata,\n this.apiService.networkMeta,\n );\n\n const keys = [\n 'lastProcessedHeight',\n 'blockOffset',\n 'indexerNodeVersion',\n 'chain',\n 'specName',\n 'genesisHash',\n 'startHeight',\n 'chainId',\n 'processedBlockCount',\n 'lastFinalizedVerifiedHeight',\n 'schemaMigrationCount',\n 'unfinalizedBlocks',\n 'bypassBlocks',\n ] as const;\n\n const entries = await metadataRepo.findAll({\n where: {\n key: keys,\n },\n });\n\n const keyValue = entries.reduce((arr, curr) => {\n arr[curr.key] = curr.value;\n return arr;\n }, {} as { [key in typeof keys[number]]: string \| boolean \| number });\n\n const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n if (this.project.runner) {\n await Promise.all([\n metadataRepo.upsert({\n key: 'runnerNode',\n value: this.project.runner.node.name,\n }),\n metadataRepo.upsert({\n key: 'runnerNodeVersion',\n value: this.project.runner.node.version,\n }),\n metadataRepo.upsert({\n key: 'runnerQuery',\n value: this.project.runner.query.name,\n }),\n metadataRepo.upsert({\n key: 'runnerQueryVersion',\n value: this.project.runner.query.version,\n }),\n ]);\n }\n if (!keyValue.genesisHash) {\n await metadataRepo.upsert({ key: 'genesisHash', value: genesisHash });\n } else {\n // Check if the configured genesisHash matches the currently stored genesisHash\n assert(\n // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n (this.project.network.genesisHash ?? genesisHash) ===\n keyValue.genesisHash,\n 'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n );\n }\n if (keyValue.chain !== chain) {\n await metadataRepo.upsert({ key: 'chain', value: chain });\n }\n\n if (keyValue.specName !== specName) {\n await metadataRepo.upsert({ key: 'specName', value: specName });\n }\n\n // If project was created before this feature, don't add the key. If it is project created after, add this key.\n if (!keyValue.processedBlockCount && !keyValue.lastProcessedHeight) {\n await metadataRepo.upsert({ key: 'processedBlockCount', value: 0 });\n }\n\n if (keyValue.indexerNodeVersion !== packageVersion) {\n await metadataRepo.upsert({\n key: 'indexerNodeVersion',\n value: packageVersion,\n });\n }\n if (!keyValue.schemaMigrationCount) {\n await metadataRepo.upsert({ key: 'schemaMigrationCount', value: 0 });\n }\n\n if (!keyValue.unfinalizedBlocks) {\n await metadataRepo.upsert({\n key: 'unfinalizedBlocks',\n value: '{}',\n });\n }\n if (!keyValue.startHeight) {\n await metadataRepo.upsert({\n key: 'startHeight',\n value: this.getStartBlockFromDataSources(),\n });\n }\n\n return metadataRepo;\n }\n\n async upsertMetadataBlockOffset(height: number): Promise<void> {\n await this.metadataRepo.upsert({\n key: 'blockOffset',\n value: height,\n });\n }\n\n async getMetadataUnfinalizedBlocks(): Promise<BestBlocks \| undefined> {\n const val = await getMetaDataInfo<string>(\n this.metadataRepo,\n METADATA_UNFINALIZED_BLOCKS_KEY,\n );\n if (val) {\n return JSON.parse(val) as BestBlocks;\n }\n return undefined;\n }\n\n async getLastFinalizedVerifiedHeight(): Promise<number \| undefined> {\n return getMetaDataInfo(\n this.metadataRepo,\n METADATA_LAST_FINALIZED_PROCESSED_KEY,\n );\n }\n\n async getMetadataBlockOffset(): Promise<number \| undefined> {\n return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n }\n\n async getLastProcessedHeight(): Promise<number \| undefined> {\n return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n }\n\n private async getStartHeight(): Promise<number> {\n let startHeight: number;\n const lastProcessedHeight = await this.getLastProcessedHeight();\n if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n startHeight = Number(lastProcessedHeight) + 1;\n } else {\n startHeight = this.getStartBlockFromDataSources();\n }\n return startHeight;\n }\n\n async setBlockOffset(offset: number): Promise<void> {\n if (\n this._blockOffset \|\|\n offset === null \|\|\n offset === undefined \|\|\n isNaN(offset)\n ) {\n return;\n }\n logger.info(`set blockOffset to ${offset}`);\n this._blockOffset = offset;\n return this.mmrService\n .syncFileBaseFromPoi(this.schema, offset)\n .catch((err) => {\n logger.error(err, 'failed to sync poi to mmr');\n process.exit(1);\n });\n }\n async getProcessedBlockCount(): Promise<number> {\n const res = await this.metadataRepo.findOne({\n where: { key: 'processedBlockCount' },\n });\n\n return res?.value as number \| undefined;\n }\n\n private getStartBlockFromDataSources() {\n const startBlocksList = this.project.dataSources.map(\n (item) => item.startBlock ?? 1,\n );\n if (startBlocksList.length === 0) {\n logger.error(\n `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n );\n process.exit(1);\n } else {\n return Math.min(...startBlocksList);\n }\n }\n\n async reindex(targetBlockHeight: number): Promise<void> {\n const lastProcessedHeight = await this.getLastProcessedHeight();\n\n return reindex(\n this.getStartBlockFromDataSources(),\n await this.getMetadataBlockOffset(),\n targetBlockHeight,\n lastProcessedHeight,\n this.storeService,\n this.dynamicDsService,\n this.mmrService,\n this.sequelize,\n /* Not providing force clean service, it should never be needed */\n );\n }\n}\n"]}
1	+ {"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAAoD;AACpD,yDAAsD;AACtD,gDAY0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAqE;AACrE,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AAExD,2EAIqC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAMzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B,EAC3B,uBAAiD;QAVxC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACQ,YAAO,GAAP,OAAO,CAAiB;QACpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,4BAAuB,GAAvB,uBAAuB,CAA0B;IACxD,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAChD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;SACjD;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACrD,IAAI,CAAC,YAAY,GAAG,MAAM,IAAA,2BAAe,EACvC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,CAAC,UAAU,EAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAC7B,CAAC;YAEF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;SACF;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CACzD,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CACxB,CAAC;QAEF,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;SACjC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5D,CAAC;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,YAAY,GAAG,MAAM,IAAA,2BAAe,EACxC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,CAAC,UAAU,EAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAC7B,CAAC;QAEF,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,aAAa;YACb,SAAS;YACT,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;YACtB,mBAAmB;YACnB,cAAc;SACN,CAAC;QAEX,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC;YACzC,KAAK,EAAE;gBACL,GAAG,EAAE,IAAI;aACV;SACF,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YAC5C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3B,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAiE,CAAC,CAAC;QAEtE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,YAAY;oBACjB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI;iBACrC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,mBAAmB;oBACxB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO;iBACxC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,aAAa;oBAClB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;iBACtC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,oBAAoB;oBACzB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO;iBACzC,CAAC;aACH,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;SACvE;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;SACjE;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SACrE;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,MAAM,YAAY,CAAC,MAAM,CAAC;gBACxB,GAAG,EAAE,oBAAoB;gBACzB,KAAK,EAAE,cAAc;aACtB,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SACtE;QAED,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE;YAC/B,MAAM,YAAY,CAAC,MAAM,CAAC;gBACxB,GAAG,EAAE,mBAAmB;gBACxB,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,MAAM,YAAY,CAAC,MAAM,CAAC;gBACxB,GAAG,EAAE,aAAa;gBAClB,KAAK,EAAE,IAAI,CAAC,4BAA4B,EAAE;aAC3C,CAAC,CAAC;SACJ;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAAc;QAC5C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC7B,GAAG,EAAE,aAAa;YAClB,KAAK,EAAE,MAAM;SACd,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,4BAA4B;QAChC,MAAM,GAAG,GAAG,MAAM,IAAA,2BAAe,EAC/B,IAAI,CAAC,YAAY,EACjB,2DAA+B,CAChC,CAAC;QACF,IAAI,GAAG,EAAE;YACP,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAe,CAAC;SACtC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,8BAA8B;QAClC,OAAO,IAAA,2BAAe,EACpB,IAAI,CAAC,YAAY,EACjB,iEAAqC,CACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY;YACjB,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU;aACnB,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;aACxC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACP,CAAC;IACD,KAAK,CAAC,sBAAsB;QAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,qBAAqB,EAAE;SACtC,CAAC,CAAC;QAEH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;CACF,CAAA;AAhWY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAaR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALU,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;QACF,oDAAwB;GAjBhD,cAAc,CAgW1B;AAhWY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n ApiService,\n MetadataFactory,\n MetadataRepo,\n NodeConfig,\n IndexerEvent,\n StoreService,\n PoiService,\n MmrService,\n getLogger,\n getExistingProjectSchema,\n getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n generateTimestampReferenceForBlockFilters,\n SubqlProjectDs,\n SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema, initHotSchemaReload } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { BestBlocks } from './types';\nimport {\n METADATA_LAST_FINALIZED_PROCESSED_KEY,\n METADATA_UNFINALIZED_BLOCKS_KEY,\n UnfinalizedBlocksService,\n} from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService {\n private _schema: string;\n private metadataRepo: MetadataRepo;\n private _startHeight: number;\n private _blockOffset: number;\n\n constructor(\n private readonly dsProcessorService: DsProcessorService,\n private readonly apiService: ApiService,\n private readonly poiService: PoiService,\n protected readonly mmrService: MmrService,\n private readonly sequelize: Sequelize,\n @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n private readonly storeService: StoreService,\n private readonly nodeConfig: NodeConfig,\n private readonly dynamicDsService: DynamicDsService,\n private eventEmitter: EventEmitter2,\n private unfinalizedBlockService: UnfinalizedBlocksService,\n ) {}\n\n get schema(): string {\n return this._schema;\n }\n\n get dataSources(): SubqlProjectDs[] {\n return this.project.dataSources;\n }\n\n get blockOffset(): number {\n return this._blockOffset;\n }\n\n get startHeight(): number {\n return this._startHeight;\n }\n\n get isHistorical(): boolean {\n return this.storeService.historical;\n }\n\n // eslint-disable-next-line @typescript-eslint/require-await\n private async getExistingProjectSchema(): Promise<string> {\n return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n }\n\n async init(): Promise<void> {\n // Used to load assets into DS-processor, has to be done in any thread\n await this.dsProcessorService.validateProjectCustomDatasources();\n // Do extra work on main thread to setup stuff\n this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n this.project.dataSources,\n this.apiService.api,\n );\n if (isMainThread) {\n this._schema = await this.ensureProject();\n await this.initDbSchema();\n this.metadataRepo = await this.ensureMetadata();\n this.dynamicDsService.init(this.metadataRepo);\n\n await this.initHotSchemaReload();\n\n if (this.nodeConfig.proofOfIndex) {\n const blockOffset = await this.getMetadataBlockOffset();\n void this.setBlockOffset(Number(blockOffset));\n await this.poiService.init(this.schema);\n }\n\n this._startHeight = await this.getStartHeight();\n } else {\n this._schema = await this.getExistingProjectSchema();\n this.metadataRepo = await MetadataFactory(\n this.sequelize,\n this.schema,\n this.nodeConfig.multiChain,\n this.project.network.chainId,\n );\n\n this.dynamicDsService.init(this.metadataRepo);\n\n await this.sequelize.sync();\n\n assert(this._schema, 'Schema should be created in main thread');\n await this.initDbSchema();\n\n if (this.nodeConfig.proofOfIndex) {\n await this.poiService.init(this.schema);\n }\n }\n\n if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n logger.error(\n 'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n );\n process.exit(1);\n }\n\n const reindexedTo = await this.unfinalizedBlockService.init(\n this.metadataRepo,\n this.reindex.bind(this),\n );\n\n if (reindexedTo !== undefined) {\n this._startHeight = reindexedTo;\n }\n }\n\n private async ensureProject(): Promise<string> {\n let schema = await this.getExistingProjectSchema();\n if (!schema) {\n schema = await this.createProjectSchema();\n }\n this.eventEmitter.emit(IndexerEvent.Ready, {\n value: true,\n });\n\n return schema;\n }\n\n private async createProjectSchema(): Promise<string> {\n let schema: string;\n if (this.nodeConfig.localMode) {\n // create tables in default schema if local mode is enabled\n schema = DEFAULT_DB_SCHEMA;\n } else {\n schema = this.nodeConfig.dbSchema;\n const schemas = await this.sequelize.showAllSchemas(undefined);\n if (!(schemas as unknown as string[]).includes(schema)) {\n await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n }\n }\n\n return schema;\n }\n\n private async initHotSchemaReload(): Promise<void> {\n await initHotSchemaReload(this.schema, this.storeService);\n }\n private async initDbSchema(): Promise<void> {\n await initDbSchema(this.project, this.schema, this.storeService);\n }\n\n private async ensureMetadata(): Promise<MetadataRepo> {\n const metadataRepo = await MetadataFactory(\n this.sequelize,\n this.schema,\n this.nodeConfig.multiChain,\n this.project.network.chainId,\n );\n\n this.eventEmitter.emit(\n IndexerEvent.NetworkMetadata,\n this.apiService.networkMeta,\n );\n\n const keys = [\n 'lastProcessedHeight',\n 'blockOffset',\n 'indexerNodeVersion',\n 'chain',\n 'specName',\n 'genesisHash',\n 'startHeight',\n 'chainId',\n 'processedBlockCount',\n 'lastFinalizedVerifiedHeight',\n 'schemaMigrationCount',\n 'unfinalizedBlocks',\n 'bypassBlocks',\n ] as const;\n\n const entries = await metadataRepo.findAll({\n where: {\n key: keys,\n },\n });\n\n const keyValue = entries.reduce((arr, curr) => {\n arr[curr.key] = curr.value;\n return arr;\n }, {} as { [key in typeof keys[number]]: string \| boolean \| number });\n\n const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n if (this.project.runner) {\n await Promise.all([\n metadataRepo.upsert({\n key: 'runnerNode',\n value: this.project.runner.node.name,\n }),\n metadataRepo.upsert({\n key: 'runnerNodeVersion',\n value: this.project.runner.node.version,\n }),\n metadataRepo.upsert({\n key: 'runnerQuery',\n value: this.project.runner.query.name,\n }),\n metadataRepo.upsert({\n key: 'runnerQueryVersion',\n value: this.project.runner.query.version,\n }),\n ]);\n }\n if (!keyValue.genesisHash) {\n await metadataRepo.upsert({ key: 'genesisHash', value: genesisHash });\n } else {\n // Check if the configured genesisHash matches the currently stored genesisHash\n assert(\n // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n (this.project.network.genesisHash ?? genesisHash) ===\n keyValue.genesisHash,\n 'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n );\n }\n if (keyValue.chain !== chain) {\n await metadataRepo.upsert({ key: 'chain', value: chain });\n }\n\n if (keyValue.specName !== specName) {\n await metadataRepo.upsert({ key: 'specName', value: specName });\n }\n\n // If project was created before this feature, don't add the key. If it is project created after, add this key.\n if (!keyValue.processedBlockCount && !keyValue.lastProcessedHeight) {\n await metadataRepo.upsert({ key: 'processedBlockCount', value: 0 });\n }\n\n if (keyValue.indexerNodeVersion !== packageVersion) {\n await metadataRepo.upsert({\n key: 'indexerNodeVersion',\n value: packageVersion,\n });\n }\n if (!keyValue.schemaMigrationCount) {\n await metadataRepo.upsert({ key: 'schemaMigrationCount', value: 0 });\n }\n\n if (!keyValue.unfinalizedBlocks) {\n await metadataRepo.upsert({\n key: 'unfinalizedBlocks',\n value: '{}',\n });\n }\n if (!keyValue.startHeight) {\n await metadataRepo.upsert({\n key: 'startHeight',\n value: this.getStartBlockFromDataSources(),\n });\n }\n\n return metadataRepo;\n }\n\n async upsertMetadataBlockOffset(height: number): Promise<void> {\n await this.metadataRepo.upsert({\n key: 'blockOffset',\n value: height,\n });\n }\n\n async getMetadataUnfinalizedBlocks(): Promise<BestBlocks \| undefined> {\n const val = await getMetaDataInfo<string>(\n this.metadataRepo,\n METADATA_UNFINALIZED_BLOCKS_KEY,\n );\n if (val) {\n return JSON.parse(val) as BestBlocks;\n }\n return undefined;\n }\n\n async getLastFinalizedVerifiedHeight(): Promise<number \| undefined> {\n return getMetaDataInfo(\n this.metadataRepo,\n METADATA_LAST_FINALIZED_PROCESSED_KEY,\n );\n }\n\n async getMetadataBlockOffset(): Promise<number \| undefined> {\n return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n }\n\n async getLastProcessedHeight(): Promise<number \| undefined> {\n return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n }\n\n private async getStartHeight(): Promise<number> {\n let startHeight: number;\n const lastProcessedHeight = await this.getLastProcessedHeight();\n if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n startHeight = Number(lastProcessedHeight) + 1;\n } else {\n startHeight = this.getStartBlockFromDataSources();\n }\n return startHeight;\n }\n\n async setBlockOffset(offset: number): Promise<void> {\n if (\n this._blockOffset \|\|\n offset === null \|\|\n offset === undefined \|\|\n isNaN(offset)\n ) {\n return;\n }\n logger.info(`set blockOffset to ${offset}`);\n this._blockOffset = offset;\n return this.mmrService\n .syncFileBaseFromPoi(this.schema, offset)\n .catch((err) => {\n logger.error(err, 'failed to sync poi to mmr');\n process.exit(1);\n });\n }\n async getProcessedBlockCount(): Promise<number> {\n const res = await this.metadataRepo.findOne({\n where: { key: 'processedBlockCount' },\n });\n\n return res?.value as number \| undefined;\n }\n\n private getStartBlockFromDataSources() {\n const startBlocksList = this.project.dataSources.map(\n (item) => item.startBlock ?? 1,\n );\n if (startBlocksList.length === 0) {\n logger.error(\n `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n );\n process.exit(1);\n } else {\n return Math.min(...startBlocksList);\n }\n }\n\n async reindex(targetBlockHeight: number): Promise<void> {\n const lastProcessedHeight = await this.getLastProcessedHeight();\n\n return reindex(\n this.getStartBlockFromDataSources(),\n await this.getMetadataBlockOffset(),\n targetBlockHeight,\n lastProcessedHeight,\n this.storeService,\n this.dynamicDsService,\n this.mmrService,\n this.sequelize,\n /* Not providing force clean service, it should never be needed */\n );\n }\n}\n"]}

package/dist/indexer/worker/worker.d.ts CHANGED Viewed

@@ -5,10 +5,12 @@ declare function processBlock(height: number): Promise<ProcessBlockResponse>;
 declare function numFetchedBlocks(): Promise<number>;
 declare function numFetchingBlocks(): Promise<number>;
 declare function getStatus(): Promise<WorkerStatusResponse>;
+declare function reloadDynamicDs(): Promise<void>;
 export declare type InitWorker = typeof initWorker;
 export declare type FetchBlock = typeof fetchBlock;
 export declare type ProcessBlock = typeof processBlock;
 export declare type NumFetchedBlocks = typeof numFetchedBlocks;
 export declare type NumFetchingBlocks = typeof numFetchingBlocks;
 export declare type GetWorkerStatus = typeof getStatus;
+export declare type ReloadDynamicDs = typeof reloadDynamicDs;
 export {};

package/dist/indexer/worker/worker.js CHANGED Viewed

@@ -17,11 +17,13 @@ const assert_1 = __importDefault(require("assert"));
 const node_worker_threads_1 = require("node:worker_threads");
 const core_1 = require("@nestjs/core");
 const node_core_1 = require("@subql/node-core");
+const dynamic_ds_service_1 = require("../dynamic-ds.service");
 const indexer_manager_1 = require("../indexer.manager");
 const worker_module_1 = require("./worker.module");
 const worker_service_1 = require("./worker.service");
 let app;
 let workerService;
+let dynamicDsService;
 const logger = (0, node_core_1.getLogger)(`worker #${node_worker_threads_1.threadId}`);
 async function initWorker() {
     try {
@@ -37,6 +39,7 @@ async function initWorker() {
         // Initialise async services, we do this here rather than in factories so we can capture one off events
         await indexerManager.start();
         workerService = app.get(worker_service_1.WorkerService);
+        dynamicDsService = app.get(dynamic_ds_service_1.DynamicDsService);
     }
     catch (e) {
         console.log('Failed to start worker', e);
@@ -50,7 +53,13 @@ async function fetchBlock(height) {
 }
 async function processBlock(height) {
     (0, assert_1.default)(workerService, 'Not initialised');
-    return workerService.processBlock(height);
+    const res = await workerService.processBlock(height);
+    // Clean up the temp ds records for worker thread instance
+    if (res.dynamicDsCreated) {
+        const dynamicDsService = app.get(dynamic_ds_service_1.DynamicDsService);
+        dynamicDsService.deleteTempDsRecords(height);
+    }
+    return res;
 }
 // eslint-disable-next-line @typescript-eslint/require-await
 async function numFetchedBlocks() {
@@ -69,6 +78,9 @@ async function getStatus() {
         isIndexing: workerService.isIndexing,
     };
 }
+async function reloadDynamicDs() {
+    return dynamicDsService.reloadDynamicDatasources();
+}
 // Register these functions to be exposed to worker host
 (0, node_core_1.registerWorker)({
     initWorker,
@@ -77,6 +89,7 @@ async function getStatus() {
     numFetchedBlocks,
     numFetchingBlocks,
     getStatus,
+    reloadDynamicDs,
 });
 process.on('uncaughtException', (e) => {
     logger.error(e, 'Uncaught Exception');

package/dist/indexer/worker/worker.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAE/C,uCAA2C;AAC3C,gDAAyE;AACzE,wDAAoD;AACpD,mDAA+C;AAC/C,qDAK0B;AAE1B,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;~~AAEjC~~,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;~~KACxC~~;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,~~OAAO~~,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;~~AAC5C~~,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,wDAAwD;AACxD,IAAA,0BAAc,EAAC;IACb,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;~~CACV~~,CAAC,CAAC;~~AAUH~~,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n argv.debug,\n argv.outputFmt as 'json' \| 'colored',\n argv.logLevel as string \| undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport { registerWorker, getLogger, NestLogger } from '@subql/node-core';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n FetchBlockResponse,\n ProcessBlockResponse,\n WorkerService,\n WorkerStatusResponse,\n} from './worker.service';\n\nlet app: INestApplication;\nlet workerService: WorkerService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n try {\n if (app) {\n logger.warn('Worker already initialised');\n return;\n }\n\n app = await NestFactory.create(WorkerModule, {\n logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n });\n\n await app.init();\n\n const indexerManager = app.get(IndexerManager);\n // Initialise async services, we do this here rather than in factories so we can capture one off events\n await indexerManager.start();\n\n workerService = app.get(WorkerService);\n } catch (e) {\n console.log('Failed to start worker', e);\n logger.error(e, 'Failed to start worker');\n throw e;\n }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n assert(workerService, 'Not initialised');\n\n return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n assert(workerService, 'Not initialised');\n\n ~~return~~ workerService.processBlock(height);\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n return {\n threadId,\n fetchedBlocks: workerService.numFetchedBlocks,\n toFetchBlocks: workerService.numFetchingBlocks,\n isIndexing: workerService.isIndexing,\n };\n}\n\n// Register these functions to be exposed to worker host\nregisterWorker({\n initWorker,\n fetchBlock,\n processBlock,\n numFetchedBlocks,\n numFetchingBlocks,\n getStatus,\n});\n\n// Export types to be used on the parent\nexport type InitWorker = typeof initWorker;\nexport type FetchBlock = typeof fetchBlock;\nexport type ProcessBlock = typeof processBlock;\nexport type NumFetchedBlocks = typeof numFetchedBlocks;\nexport type NumFetchingBlocks = typeof numFetchingBlocks;\nexport type GetWorkerStatus = typeof getStatus;\n\nprocess.on('uncaughtException', (e) => {\n logger.error(e, 'Uncaught Exception');\n throw e;\n});\n"]}
1	+ {"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAE/C,uCAA2C;AAC3C,gDAAyE;AACzE,8DAAyD;AACzD,wDAAoD;AACpD,mDAA+C;AAC/C,qDAK0B;AAE1B,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;AACjC,IAAI,gBAAkC,CAAC;AAEvC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;QACvC,gBAAgB,GAAG,GAAG,CAAC,GAAG,CAAC,qCAAgB,CAAC,CAAC;KAC9C;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAErD,0DAA0D;IAC1D,IAAI,GAAG,CAAC,gBAAgB,EAAE;QACxB,MAAM,gBAAgB,GAAG,GAAG,CAAC,GAAG,CAAC,qCAAgB,CAAC,CAAC;QACnD,gBAAgB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;KAC9C;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,eAAe;IAC5B,OAAO,gBAAgB,CAAC,wBAAwB,EAAE,CAAC;AACrD,CAAC;AAED,wDAAwD;AACxD,IAAA,0BAAc,EAAC;IACb,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,eAAe;CAChB,CAAC,CAAC;AAWH,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n argv.debug,\n argv.outputFmt as 'json' \| 'colored',\n argv.logLevel as string \| undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport { registerWorker, getLogger, NestLogger } from '@subql/node-core';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n FetchBlockResponse,\n ProcessBlockResponse,\n WorkerService,\n WorkerStatusResponse,\n} from './worker.service';\n\nlet app: INestApplication;\nlet workerService: WorkerService;\nlet dynamicDsService: DynamicDsService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n try {\n if (app) {\n logger.warn('Worker already initialised');\n return;\n }\n\n app = await NestFactory.create(WorkerModule, {\n logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n });\n\n await app.init();\n\n const indexerManager = app.get(IndexerManager);\n // Initialise async services, we do this here rather than in factories so we can capture one off events\n await indexerManager.start();\n\n workerService = app.get(WorkerService);\n dynamicDsService = app.get(DynamicDsService);\n } catch (e) {\n console.log('Failed to start worker', e);\n logger.error(e, 'Failed to start worker');\n throw e;\n }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n assert(workerService, 'Not initialised');\n\n return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n assert(workerService, 'Not initialised');\n\n const res = await workerService.processBlock(height);\n\n // Clean up the temp ds records for worker thread instance\n if (res.dynamicDsCreated) {\n const dynamicDsService = app.get(DynamicDsService);\n dynamicDsService.deleteTempDsRecords(height);\n }\n\n return res;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n return {\n threadId,\n fetchedBlocks: workerService.numFetchedBlocks,\n toFetchBlocks: workerService.numFetchingBlocks,\n isIndexing: workerService.isIndexing,\n };\n}\n\nasync function reloadDynamicDs(): Promise<void> {\n return dynamicDsService.reloadDynamicDatasources();\n}\n\n// Register these functions to be exposed to worker host\nregisterWorker({\n initWorker,\n fetchBlock,\n processBlock,\n numFetchedBlocks,\n numFetchingBlocks,\n getStatus,\n reloadDynamicDs,\n});\n\n// Export types to be used on the parent\nexport type InitWorker = typeof initWorker;\nexport type FetchBlock = typeof fetchBlock;\nexport type ProcessBlock = typeof processBlock;\nexport type NumFetchedBlocks = typeof numFetchedBlocks;\nexport type NumFetchingBlocks = typeof numFetchingBlocks;\nexport type GetWorkerStatus = typeof getStatus;\nexport type ReloadDynamicDs = typeof reloadDynamicDs;\n\nprocess.on('uncaughtException', (e) => {\n logger.error(e, 'Uncaught Exception');\n throw e;\n});\n"]}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@subql/node-ethereum",
-  "version": "0.4.1-6",
+  "version": "0.4.1-7",
   "description": "",
   "author": "Ian He",
   "license": "Apache-2.0",
@@ -74,5 +74,5 @@
     "/dist",
     "/bin"
   ],
-  "stableVersion": "0.4.1-5"
+  "stableVersion": "0.4.1-6"
 }