@subql/node-ethereum 1.10.1-3 → 2.0.0

package/dist/indexer/worker/worker.service.js

@@ -10,17 +10,22 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkerService = void 0;
 const node_worker_threads_1 = require("node:worker_threads");
 const common_1 = require("@nestjs/common");
 const node_core_1 = require("@subql/node-core");
+const ethereum_1 = require("../../ethereum");
 const indexer_manager_1 = require("../indexer.manager");
 const logger = (0, node_core_1.getLogger)(`Worker Service #${node_worker_threads_1.threadId}`);
 let WorkerService = class WorkerService {
-    constructor(apiService, indexerManager, nodeConfig) {
+    constructor(apiService, indexerManager, projectService, nodeConfig) {
         this.apiService = apiService;
         this.indexerManager = indexerManager;
+        this.projectService = projectService;
         this.fetchedBlocks = {};
         this._isIndexing = false;
         this.queue = new node_core_1.AutoQueue(undefined, nodeConfig.batchSize);
@@ -30,16 +35,23 @@ let WorkerService = class WorkerService {
             return await this.queue.put(async () => {
                 // If a dynamic ds is created we might be asked to fetch blocks again, use existing result
                 if (!this.fetchedBlocks[height]) {
-                    const [block] = await this.apiService.api.fetchBlocks([height]);
+                    if (node_core_1.memoryLock.isLocked()) {
+                        const start = Date.now();
+                        await node_core_1.memoryLock.waitForUnlock();
+                        const end = Date.now();
+                        logger.debug(`memory lock wait time: ${end - start}ms`);
+                    }
+                    const [block] = await this.apiService.fetchBlocks([height]);
                     this.fetchedBlocks[height] = block;
                 }
-                const block = this.fetchedBlocks[height];
+                // const block = this.fetchedBlocks[height];
                 // Return info to get the runtime version, this lets the worker thread know
                 return undefined;
             });
         }
         catch (e) {
-            logger.error(e, `Failed to fetch block ${height}`);
+            logger.error(/*e, */ `Failed to fetch block ${height}`);
+            throw e;
         }
     }
     async processBlock(height) {
@@ -50,14 +62,15 @@ let WorkerService = class WorkerService {
                 throw new Error(`Block ${height} has not been fetched`);
             }
             delete this.fetchedBlocks[height];
-            const response = await this.indexerManager.indexBlock(block);
-            this._isIndexing = false;
-            return Object.assign(Object.assign({}, response), { operationHash: Buffer.from(response.operationHash).toString('base64') });
+            return await this.indexerManager.indexBlock(block, await this.projectService.getAllDataSources(height));
         }
         catch (e) {
             logger.error(e, `Failed to index block ${height}: ${e.stack}`);
             throw e;
         }
+        finally {
+            this._isIndexing = false;
+        }
     }
     get numFetchedBlocks() {
         return Object.keys(this.fetchedBlocks).length;
@@ -71,9 +84,9 @@ let WorkerService = class WorkerService {
 };
 WorkerService = __decorate([
     (0, common_1.Injectable)(),
-    __metadata("design:paramtypes", [node_core_1.ApiService,
-        indexer_manager_1.IndexerManager,
-        node_core_1.NodeConfig])
+    __param(2, (0, common_1.Inject)('IProjectService')),
+    __metadata("design:paramtypes", [ethereum_1.EthereumApiService,
+        indexer_manager_1.IndexerManager, Object, node_core_1.NodeConfig])
 ], WorkerService);
 exports.WorkerService = WorkerService;
 //# sourceMappingURL=worker.service.js.map

package/dist/indexer/worker/worker.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAA4C;AAC5C,gDAAgF;AAEhF,wDAAoD;AAgBpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGjD,IAAM,aAAa,GAAnB,MAAM,aAAa;IAMxB,YACU,UAAsB,EACtB,cAA8B,EACtC,UAAsB;QAFd,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAPhC,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAS1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAChE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAEzC,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,EAAE,CAAC,CAAC;SACpD;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAE7D,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,uCACK,QAAQ,KACX,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IACrE;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AApEY,aAAa;IADzB,IAAA,mBAAU,GAAE;qCAQW,sBAAU;QACN,gCAAc;QAC1B,sBAAU;GATb,aAAa,CAoEzB;AApEY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Injectable } from '@nestjs/common';\nimport { ApiService, NodeConfig, getLogger, AutoQueue } from '@subql/node-core';\nimport { EthereumBlockWrapper, BlockWrapper } from '@subql/types-ethereum';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  operationHash: string; // Base64 encoded u8a array\n};\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: ApiService,\n    private indexerManager: IndexerManager,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          const [block] = await this.apiService.api.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        const block = this.fetchedBlocks[height];\n\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(e, `Failed to fetch block ${height}`);\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      const response = await this.indexerManager.indexBlock(block);\n\n      this._isIndexing = false;\n      return {\n        ...response,\n        operationHash: Buffer.from(response.operationHash).toString('base64'),\n      };\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}
+{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAAoD;AACpD,gDAM0B;AAG1B,6CAAoD;AACpD,wDAAoD;AAiBpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGjD,IAAM,aAAa,GAAnB,MAAM,aAAa;IAMxB,YACU,UAA8B,EAC9B,cAA8B,EAE9B,cAA+C,EACvD,UAAsB;QAJd,eAAU,GAAV,UAAU,CAAoB;QAC9B,mBAAc,GAAd,cAAc,CAAgB;QAE9B,mBAAc,GAAd,cAAc,CAAiC;QATjD,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAW1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,IAAI,sBAAU,CAAC,QAAQ,EAAE,EAAE;wBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACzB,MAAM,sBAAU,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACvB,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC;qBACzD;oBAED,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAC5D,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,4CAA4C;gBAC5C,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,yBAAyB,MAAM,EAAE,CAAC,CAAC;YACxD,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CACzC,KAAK,EACL,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CACpD,CAAC;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC1B;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AA5EY,aAAa;IADzB,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCAFN,6BAAkB;QACd,gCAAc,UAG1B,sBAAU;GAXb,aAAa,CA4EzB;AA5EY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  NodeConfig,\n  getLogger,\n  AutoQueue,\n  memoryLock,\n  IProjectService,\n} from '@subql/node-core';\nimport { BlockWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { EthereumApiService } from '../../ethereum';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  blockHash: string;\n  reindexBlockHeight: number;\n};\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: EthereumApiService,\n    private indexerManager: IndexerManager,\n    @Inject('IProjectService')\n    private projectService: IProjectService<SubqlProjectDs>,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          if (memoryLock.isLocked()) {\n            const start = Date.now();\n            await memoryLock.waitForUnlock();\n            const end = Date.now();\n            logger.debug(`memory lock wait time: ${end - start}ms`);\n          }\n\n          const [block] = await this.apiService.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        // const block = this.fetchedBlocks[height];\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(/*e, */ `Failed to fetch block ${height}`);\n      throw e;\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      return await this.indexerManager.indexBlock(\n        block,\n        await this.projectService.getAllDataSources(height),\n      );\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    } finally {\n      this._isIndexing = false;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}

package/dist/indexer/worker/worker.unfinalizedBlocks.service.d.ts

@@ -0,0 +1,11 @@
+import { EthereumBlock } from '@subql/types-ethereum';
+import { IUnfinalizedBlocksService } from '../unfinalizedBlocks.service';
+export type HostUnfinalizedBlocks = {
+    unfinalizedBlocksProcess: (block: EthereumBlock) => Promise<number | null>;
+};
+export declare const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[];
+export declare class WorkerUnfinalizedBlocksService implements IUnfinalizedBlocksService {
+    private host;
+    constructor(host: HostUnfinalizedBlocks);
+    processUnfinalizedBlocks(block: EthereumBlock): Promise<number | null>;
+}

package/dist/indexer/worker/worker.unfinalizedBlocks.service.js

@@ -0,0 +1,32 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerUnfinalizedBlocksService = exports.hostUnfinalizedBlocksKeys = void 0;
+const common_1 = require("@nestjs/common");
+exports.hostUnfinalizedBlocksKeys = [
+    'unfinalizedBlocksProcess',
+];
+let WorkerUnfinalizedBlocksService = class WorkerUnfinalizedBlocksService {
+    constructor(host) {
+        this.host = host;
+    }
+    async processUnfinalizedBlocks(block) {
+        return this.host.unfinalizedBlocksProcess(block);
+    }
+};
+WorkerUnfinalizedBlocksService = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [Object])
+], WorkerUnfinalizedBlocksService);
+exports.WorkerUnfinalizedBlocksService = WorkerUnfinalizedBlocksService;
+//# sourceMappingURL=worker.unfinalizedBlocks.service.js.map

package/dist/indexer/worker/worker.unfinalizedBlocks.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAQ/B,QAAA,yBAAyB,GAAoC;IACxE,0BAA0B;CAC3B,CAAC;AAGK,IAAM,8BAA8B,GAApC,MAAM,8BAA8B;IAGzC,YAAoB,IAA2B;QAA3B,SAAI,GAAJ,IAAI,CAAuB;IAAG,CAAC;IAEnD,KAAK,CAAC,wBAAwB,CAAC,KAAoB;QACjD,OAAO,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;CACF,CAAA;AARY,8BAA8B;IAD1C,IAAA,mBAAU,GAAE;;GACA,8BAA8B,CAQ1C;AARY,wEAA8B","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { EthereumBlock } from '@subql/types-ethereum';\nimport { IUnfinalizedBlocksService } from '../unfinalizedBlocks.service';\n\nexport type HostUnfinalizedBlocks = {\n  unfinalizedBlocksProcess: (block: EthereumBlock) => Promise<number | null>;\n};\n\nexport const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[] = [\n  'unfinalizedBlocksProcess',\n];\n\n@Injectable()\nexport class WorkerUnfinalizedBlocksService\n  implements IUnfinalizedBlocksService\n{\n  constructor(private host: HostUnfinalizedBlocks) {}\n\n  async processUnfinalizedBlocks(block: EthereumBlock): Promise<number | null> {\n    return this.host.unfinalizedBlocksProcess(block);\n  }\n}\n"]}

package/dist/init.js

@@ -8,7 +8,6 @@ const common_1 = require("@subql/common");
 const node_core_1 = require("@subql/node-core");
 const app_module_1 = require("./app.module");
 const fetch_service_1 = require("./indexer/fetch.service");
-const project_service_1 = require("./indexer/project.service");
 const yargs_1 = require("./yargs");
 const pjson = require('../package.json');
 const { argv } = yargs_1.yargsOptions;
@@ -35,13 +34,14 @@ async function bootstrap() {
             logger: debug ? new node_core_1.NestLogger() : false,
         });
         await app.init();
-        const projectService = app.get(project_service_1.ProjectService);
+        const projectService = app.get('IProjectService');
         const fetchService = app.get(fetch_service_1.FetchService);
         const apiService = app.get(node_core_1.ApiService);
         // Initialise async services, we do this here rather than in factories, so we can capture one off events
         await apiService.init();
         await projectService.init();
         await fetchService.init(projectService.startHeight);
+        app.enableShutdownHooks();
         await app.listen(port);
         logger.info(`Node started on port: ${port}`);
     }

package/dist/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,0CAAkD;AAClD,gDAAqE;AACrE,6CAAyC;AACzC,2DAAuD;AACvD,+DAA2D;AAC3D,mCAAuC;AACvC,MAAM,KAAK,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEhC,KAAK,UAAU,SAAS;;IAC7B,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,eAAe,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAEzB,MAAM,QAAQ,GAAG,CAAC,CAAM,EAAE,EAAE;QAC1B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAA,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mCAAI,CAAC,MAAM,IAAA,0BAAiB,EAAC,YAAY,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,CAAC,KAAK,CACV,wDAAwD,IAAI,KAC1D,IAAI,GAAG,EACT,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,IAAI,CAAC,MAAM,EAAE;QACf,MAAM,CAAC,IAAI,CACT,mHAAmH,CACpH,CAAC;KACH;IAED,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,sBAAS,EAAE;YAC9C,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,sBAAU,EAAE,CAAC,CAAC,CAAC,KAAK;SACzC,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,4BAAY,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAU,CAAC,CAAC;QAEvC,wGAAwG;QACxG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;KAC9C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC;AA/CD,8BA+CC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { findAvailablePort } from '@subql/common';\nimport { getLogger, ApiService, NestLogger } from '@subql/node-core';\nimport { AppModule } from './app.module';\nimport { FetchService } from './indexer/fetch.service';\nimport { ProjectService } from './indexer/project.service';\nimport { yargsOptions } from './yargs';\nconst pjson = require('../package.json');\n\nconst { argv } = yargsOptions;\n\nconst DEFAULT_PORT = 3000;\nconst logger = getLogger('subql-node');\n\nexport async function bootstrap() {\n  logger.info(`Current ${pjson.name} version is ${pjson.version}`);\n  const debug = argv.debug;\n\n  const validate = (x: any) => {\n    const p = parseInt(x);\n    return isNaN(p) ? null : p;\n  };\n\n  const port = validate(argv.port) ?? (await findAvailablePort(DEFAULT_PORT));\n  if (!port) {\n    logger.error(\n      `Unable to find available port (tried ports in range (${port}..${\n        port + 10\n      })). Try setting a free port manually by setting the --port flag`,\n    );\n    process.exit(1);\n  }\n\n  if (argv.unsafe) {\n    logger.warn(\n      'UNSAFE MODE IS ENABLED. This is not recommended for most projects and will not be supported by our hosted service',\n    );\n  }\n\n  try {\n    const app = await NestFactory.create(AppModule, {\n      logger: debug ? new NestLogger() : false,\n    });\n    await app.init();\n\n    const projectService = app.get(ProjectService);\n    const fetchService = app.get(FetchService);\n    const apiService = app.get(ApiService);\n\n    // Initialise async services, we do this here rather than in factories, so we can capture one off events\n    await apiService.init();\n    await projectService.init();\n    await fetchService.init(projectService.startHeight);\n\n    await app.listen(port);\n\n    logger.info(`Node started on port: ${port}`);\n  } catch (e) {\n    logger.error(e, 'Node failed to start');\n    process.exit(1);\n  }\n}\n"]}
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,0CAAkD;AAClD,gDAAqE;AACrE,6CAAyC;AACzC,2DAAuD;AAEvD,mCAAuC;AACvC,MAAM,KAAK,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEhC,KAAK,UAAU,SAAS;;IAC7B,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,eAAe,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAEzB,MAAM,QAAQ,GAAG,CAAC,CAAM,EAAE,EAAE;QAC1B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAA,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mCAAI,CAAC,MAAM,IAAA,0BAAiB,EAAC,YAAY,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,CAAC,KAAK,CACV,wDAAwD,IAAI,KAC1D,IAAI,GAAG,EACT,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,IAAI,CAAC,MAAM,EAAE;QACf,MAAM,CAAC,IAAI,CACT,mHAAmH,CACpH,CAAC;KACH;IAED,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,sBAAS,EAAE;YAC9C,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,sBAAU,EAAE,CAAC,CAAC,CAAC,KAAK;SACzC,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAmB,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAClE,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,4BAAY,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAU,CAAC,CAAC;QAEvC,wGAAwG;QACxG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEpD,GAAG,CAAC,mBAAmB,EAAE,CAAC;QAE1B,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;KAC9C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC;AAjDD,8BAiDC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { findAvailablePort } from '@subql/common';\nimport { getLogger, ApiService, NestLogger } from '@subql/node-core';\nimport { AppModule } from './app.module';\nimport { FetchService } from './indexer/fetch.service';\nimport { ProjectService } from './indexer/project.service';\nimport { yargsOptions } from './yargs';\nconst pjson = require('../package.json');\n\nconst { argv } = yargsOptions;\n\nconst DEFAULT_PORT = 3000;\nconst logger = getLogger('subql-node');\n\nexport async function bootstrap(): Promise<void> {\n  logger.info(`Current ${pjson.name} version is ${pjson.version}`);\n  const debug = argv.debug;\n\n  const validate = (x: any) => {\n    const p = parseInt(x);\n    return isNaN(p) ? null : p;\n  };\n\n  const port = validate(argv.port) ?? (await findAvailablePort(DEFAULT_PORT));\n  if (!port) {\n    logger.error(\n      `Unable to find available port (tried ports in range (${port}..${\n        port + 10\n      })). Try setting a free port manually by setting the --port flag`,\n    );\n    process.exit(1);\n  }\n\n  if (argv.unsafe) {\n    logger.warn(\n      'UNSAFE MODE IS ENABLED. This is not recommended for most projects and will not be supported by our hosted service',\n    );\n  }\n\n  try {\n    const app = await NestFactory.create(AppModule, {\n      logger: debug ? new NestLogger() : false,\n    });\n    await app.init();\n\n    const projectService: ProjectService = app.get('IProjectService');\n    const fetchService = app.get(FetchService);\n    const apiService = app.get(ApiService);\n\n    // Initialise async services, we do this here rather than in factories, so we can capture one off events\n    await apiService.init();\n    await projectService.init();\n    await fetchService.init(projectService.startHeight);\n\n    app.enableShutdownHooks();\n\n    await app.listen(port);\n\n    logger.info(`Node started on port: ${port}`);\n  } catch (e) {\n    logger.error(e, 'Node failed to start');\n    process.exit(1);\n  }\n}\n"]}

package/dist/main.js

@@ -5,10 +5,15 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const logger_1 = require("@subql/node-core/logger");
 const yargs_1 = require("./yargs");
 const { argv } = yargs_1.yargsOptions;
+BigInt.prototype.toJSON = function () {
+    return this.toString();
+};
 // initLogger is imported from true path, to make sure getLogger (or other logger values that relies on logger) isn't initialised
 (0, logger_1.initLogger)(argv.debug, argv.outputFmt, argv.logLevel);
 // Lazy import, to allow logger to be initialised before bootstrap()
 // As bootstrap runs services that requires logger
 const { bootstrap } = require('./init');
-void bootstrap();
+if (!(argv._[0] === 'test')) {
+    void bootstrap();
+}
 //# sourceMappingURL=main.js.map

package/dist/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;AAEtC,oDAAqD;AACrD,mCAAuC;AAEvC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,iIAAiI;AACjI,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oEAAoE;AACpE,kDAAkD;AAClD,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACxC,KAAK,SAAS,EAAE,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { initLogger } from '@subql/node-core/logger';\nimport { yargsOptions } from './yargs';\n\nconst { argv } = yargsOptions;\n\n// initLogger is imported from true path, to make sure getLogger (or other logger values that relies on logger) isn't initialised\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\n// Lazy import, to allow logger to be initialised before bootstrap()\n// As bootstrap runs services that requires logger\nconst { bootstrap } = require('./init');\nvoid bootstrap();\n"]}
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;AAEtC,oDAAqD;AACrD,mCAAuC;AAEvC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE7B,MAAM,CAAC,SAAiB,CAAC,MAAM,GAAG;IACjC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;AACzB,CAAC,CAAC;AAEF,iIAAiI;AACjI,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oEAAoE;AACpE,kDAAkD;AAClD,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACxC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,EAAE;IAC3B,KAAK,SAAS,EAAE,CAAC;CAClB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { initLogger } from '@subql/node-core/logger';\nimport { yargsOptions } from './yargs';\n\nconst { argv } = yargsOptions;\n\n(BigInt.prototype as any).toJSON = function () {\n  return this.toString();\n};\n\n// initLogger is imported from true path, to make sure getLogger (or other logger values that relies on logger) isn't initialised\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\n// Lazy import, to allow logger to be initialised before bootstrap()\n// As bootstrap runs services that requires logger\nconst { bootstrap } = require('./init');\nif (!(argv._[0] === 'test')) {\n  void bootstrap();\n}\n"]}

package/dist/meta/meta.module.js

@@ -67,6 +67,14 @@ MetaModule = __decorate([
                 name: 'subql_indexer_processed_block_count',
                 help: 'The number of processed block',
             }),
+            (0, nestjs_prometheus_1.makeGaugeProvider)({
+                name: 'subql_indexer_store_cache_threshold',
+                help: 'Store cache will flush once cache record size excess this threshold',
+            }),
+            (0, nestjs_prometheus_1.makeGaugeProvider)({
+                name: 'subql_indexer_store_cache_records_size',
+                help: 'Number of records waiting to flush in store cache',
+            }),
             meta_service_1.MetaService,
             node_core_1.HealthService,
             node_core_1.ReadyService,

package/dist/meta/meta.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta.module.js","sourceRoot":"","sources":["../../src/meta/meta.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,mEAGqC;AACrC,0DAAsD;AACtD,uDAAmD;AACnD,iDAA6C;AAwDtC,IAAM,UAAU,GAAhB,MAAM,UAAU;CAAG,CAAA;AAAb,UAAU;IAtDtB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,oCAAgB,CAAC,QAAQ,EAAE,EAAE,0BAAW,CAAC;QACnD,WAAW,EAAE,CAAC,gCAAc,EAAE,4BAAgB,EAAE,2BAAe,CAAC;QAChE,SAAS,EAAE;YACT,+BAAmB;YACnB,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,6BAA6B;gBACnC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,4CAA4C;aACnD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,uCAAuC;gBAC7C,IAAI,EAAE,qCAAqC;aAC5C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,mCAAmC;gBACzC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,iCAAiC;gBACvC,IAAI,EAAE,8BAA8B;aACrC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,wCAAwC;aAC/C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,+CAA+C;aACtD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,sDAAsD;aAC7D,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,+BAA+B;aACtC,CAAC;YACF,0BAAW;YACX,yBAAa;YACb,wBAAY;SACb;KACF,CAAC;GACW,UAAU,CAAG;AAAb,gCAAU","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  MetricEventListener,\n  ReadyController,\n  ReadyService,\n  HealthController,\n  HealthService,\n} from '@subql/node-core';\nimport {\n  makeGaugeProvider,\n  PrometheusModule,\n} from '@willsoto/nestjs-prometheus';\nimport { FetchModule } from '../indexer/fetch.module';\nimport { MetaController } from './meta.controller';\nimport { MetaService } from './meta.service';\n\n@Module({\n  imports: [PrometheusModule.register(), FetchModule],\n  controllers: [MetaController, HealthController, ReadyController],\n  providers: [\n    MetricEventListener,\n    makeGaugeProvider({\n      name: 'subql_indexer_api_connected',\n      help: 'The indexer api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_injected_api_connected',\n      help: 'The indexer injected api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processing_block_height',\n      help: 'The current processing block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_height',\n      help: 'The last processed block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_target_block_height',\n      help: 'The latest finalized block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_best_block_height',\n      help: 'The latest best block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_block_queue_size',\n      help: 'The size of fetched block queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_blocknumber_queue_size',\n      help: 'The size of fetched block number queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_using_dictionary',\n      help: 'The status of indexer is using the dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_skip_dictionary_count',\n      help: 'The number of times indexer been skip use dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_count',\n      help: 'The number of processed block',\n    }),\n    MetaService,\n    HealthService,\n    ReadyService,\n  ],\n})\nexport class MetaModule {}\n"]}
+{"version":3,"file":"meta.module.js","sourceRoot":"","sources":["../../src/meta/meta.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,mEAGqC;AACrC,0DAAsD;AACtD,uDAAmD;AACnD,iDAA6C;AAgEtC,IAAM,UAAU,GAAhB,MAAM,UAAU;CAAG,CAAA;AAAb,UAAU;IA9DtB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,oCAAgB,CAAC,QAAQ,EAAE,EAAE,0BAAW,CAAC;QACnD,WAAW,EAAE,CAAC,gCAAc,EAAE,4BAAgB,EAAE,2BAAe,CAAC;QAChE,SAAS,EAAE;YACT,+BAAmB;YACnB,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,6BAA6B;gBACnC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,4CAA4C;aACnD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,uCAAuC;gBAC7C,IAAI,EAAE,qCAAqC;aAC5C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,mCAAmC;gBACzC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,iCAAiC;gBACvC,IAAI,EAAE,8BAA8B;aACrC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,wCAAwC;aAC/C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,+CAA+C;aACtD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,sDAAsD;aAC7D,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,+BAA+B;aACtC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,qEAAqE;aAC5E,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,wCAAwC;gBAC9C,IAAI,EAAE,mDAAmD;aAC1D,CAAC;YACF,0BAAW;YACX,yBAAa;YACb,wBAAY;SACb;KACF,CAAC;GACW,UAAU,CAAG;AAAb,gCAAU","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  MetricEventListener,\n  ReadyController,\n  ReadyService,\n  HealthController,\n  HealthService,\n} from '@subql/node-core';\nimport {\n  makeGaugeProvider,\n  PrometheusModule,\n} from '@willsoto/nestjs-prometheus';\nimport { FetchModule } from '../indexer/fetch.module';\nimport { MetaController } from './meta.controller';\nimport { MetaService } from './meta.service';\n\n@Module({\n  imports: [PrometheusModule.register(), FetchModule],\n  controllers: [MetaController, HealthController, ReadyController],\n  providers: [\n    MetricEventListener,\n    makeGaugeProvider({\n      name: 'subql_indexer_api_connected',\n      help: 'The indexer api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_injected_api_connected',\n      help: 'The indexer injected api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processing_block_height',\n      help: 'The current processing block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_height',\n      help: 'The last processed block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_target_block_height',\n      help: 'The latest finalized block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_best_block_height',\n      help: 'The latest best block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_block_queue_size',\n      help: 'The size of fetched block queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_blocknumber_queue_size',\n      help: 'The size of fetched block number queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_using_dictionary',\n      help: 'The status of indexer is using the dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_skip_dictionary_count',\n      help: 'The number of times indexer been skip use dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_count',\n      help: 'The number of processed block',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_threshold',\n      help: 'Store cache will flush once cache record size excess this threshold',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_records_size',\n      help: 'Number of records waiting to flush in store cache',\n    }),\n    MetaService,\n    HealthService,\n    ReadyService,\n  ],\n})\nexport class MetaModule {}\n"]}

package/dist/meta/meta.service.d.ts

@@ -1,6 +1,7 @@
-import { BestBlockPayload, EventPayload, NetworkMetadataPayload, ProcessBlockPayload, ProcessedBlockCountPayload, TargetBlockPayload, StoreService } from '@subql/node-core';
+import { BestBlockPayload, EventPayload, NetworkMetadataPayload, ProcessBlockPayload, ProcessedBlockCountPayload, TargetBlockPayload, StoreService, NodeConfig } from '@subql/node-core';
 export declare class MetaService {
     private storeService;
+    private nodeConfig;
     private currentProcessingHeight;
     private currentProcessingTimestamp;
     private bestHeight;
@@ -12,7 +13,16 @@ export declare class MetaService {
     private lastProcessedHeight;
     private lastProcessedTimestamp;
     private processedBlockCount;
-    constructor(storeService: StoreService);
+    private accEnqueueBlocks;
+    private accFetchBlocks;
+    private currentFilteringBlockNum;
+    private accRpcCalls;
+    private lastReportedFilteringBlockNum;
+    private lastReportedEnqueueBlocks;
+    private lastReportedFetchBlocks;
+    private lastReportedRpcCalls;
+    private lastStatsReportedTs;
+    constructor(storeService: StoreService, nodeConfig: NodeConfig);
     getMeta(): {
         chain: string;
         specName: string;
@@ -31,7 +41,7 @@ export declare class MetaService {
         injectedApiConnected: boolean;
         usingDictionary: boolean;
     };
-    getTargetHeight(): Promise<void>;
+    getTargetHeight(): void;
     handleProcessingBlock(blockPayload: ProcessBlockPayload): void;
     handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void;
     handleTargetBlock(blockPayload: TargetBlockPayload): void;
@@ -39,4 +49,9 @@ export declare class MetaService {
     handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void;
     handleApiConnected({ value }: EventPayload<number>): void;
     handleUsingDictionary({ value }: EventPayload<number>): void;
+    handleEnqueueBlocks(size: number): void;
+    handleFilteringBlocks(height: number): void;
+    handleFetchBlock(): void;
+    handleRpcCall(): void;
+    blockFilteringSpeed(): void;
 }

package/dist/meta/meta.service.js

@@ -20,15 +20,25 @@ const UPDATE_HEIGHT_INTERVAL = 60000;
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { version: polkadotSdkVersion } = require('@polkadot/api/package.json');
 const { version: packageVersion } = require('../../package.json');
+const logger = (0, node_core_1.getLogger)('profiler');
 let MetaService = class MetaService {
-    constructor(storeService) {
+    constructor(storeService, nodeConfig) {
         this.storeService = storeService;
+        this.nodeConfig = nodeConfig;
+        this.accEnqueueBlocks = 0;
+        this.accFetchBlocks = 0;
+        this.currentFilteringBlockNum = 0;
+        this.accRpcCalls = 0;
+        this.lastReportedFilteringBlockNum = 0;
+        this.lastReportedEnqueueBlocks = 0;
+        this.lastReportedFetchBlocks = 0;
+        this.lastReportedRpcCalls = 0;
     }
     getMeta() {
         return Object.assign({ currentProcessingHeight: this.currentProcessingHeight, currentProcessingTimestamp: this.currentProcessingTimestamp, targetHeight: this.targetHeight, bestHeight: this.bestHeight, indexerNodeVersion: packageVersion, lastProcessedHeight: this.lastProcessedHeight, lastProcessedTimestamp: this.lastProcessedTimestamp, uptime: process.uptime(), polkadotSdkVersion, processedBlockCount: this.processedBlockCount, apiConnected: this.apiConnected, injectedApiConnected: this.injectedApiConnected, usingDictionary: this.usingDictionary }, this.networkMeta);
     }
-    async getTargetHeight() {
-        await this.storeService.setMetadata('targetHeight', this.targetHeight);
+    getTargetHeight() {
+        this.storeService.storeCache.metadata.set('targetHeight', this.targetHeight);
     }
     handleProcessingBlock(blockPayload) {
         this.currentProcessingHeight = blockPayload.height;
@@ -53,12 +63,55 @@ let MetaService = class MetaService {
     handleUsingDictionary({ value }) {
         this.usingDictionary = !!value;
     }
+    handleEnqueueBlocks(size) {
+        this.accEnqueueBlocks += size;
+        if (!this.lastStatsReportedTs) {
+            this.lastStatsReportedTs = new Date();
+        }
+    }
+    handleFilteringBlocks(height) {
+        this.currentFilteringBlockNum = height;
+        if (!this.lastStatsReportedTs) {
+            this.lastReportedFilteringBlockNum = height;
+        }
+    }
+    handleFetchBlock() {
+        this.accFetchBlocks++;
+        if (!this.lastStatsReportedTs) {
+            this.lastStatsReportedTs = new Date();
+        }
+    }
+    handleRpcCall() {
+        this.accRpcCalls++;
+        if (!this.lastStatsReportedTs) {
+            this.lastStatsReportedTs = new Date();
+        }
+    }
+    blockFilteringSpeed() {
+        if (!this.nodeConfig.profiler) {
+            return;
+        }
+        const count = this.accEnqueueBlocks - this.lastReportedEnqueueBlocks;
+        this.lastReportedEnqueueBlocks = this.accEnqueueBlocks;
+        const filteringCount = this.currentFilteringBlockNum - this.lastReportedFilteringBlockNum;
+        const now = new Date();
+        const timepass = now.getTime() - this.lastStatsReportedTs.getTime();
+        this.lastStatsReportedTs = now;
+        this.lastReportedFilteringBlockNum = this.currentFilteringBlockNum;
+        const rpcCalls = this.accRpcCalls - this.lastReportedRpcCalls;
+        this.lastReportedRpcCalls = this.accRpcCalls;
+        const fetchCount = this.accFetchBlocks - this.lastReportedFetchBlocks;
+        this.lastReportedFetchBlocks = this.accFetchBlocks;
+        logger.info(`actual block filtering: ${(count / (timepass / 1000)).toFixed(2)}/sec, \
+seeming speed: ${(filteringCount / (timepass / 1000)).toFixed(2)}/sec, rpcCalls: ${(rpcCalls / (timepass / 1000)).toFixed(2)}/sec \
+fetch speed: ${(fetchCount / (timepass / 1000)).toFixed(2)}/sec`);
+    }
 };
 __decorate([
     (0, schedule_1.Interval)(UPDATE_HEIGHT_INTERVAL),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", []),
-    __metadata("design:returntype", Promise)
+    __metadata("design:returntype", void 0)
 ], MetaService.prototype, "getTargetHeight", null);
 __decorate([
     (0, event_emitter_1.OnEvent)(node_core_1.IndexerEvent.BlockProcessing),
@@ -102,9 +155,40 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", void 0)
 ], MetaService.prototype, "handleUsingDictionary", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('enqueueBlocks'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Number]),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleEnqueueBlocks", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('filteringBlocks'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Number]),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleFilteringBlocks", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('fetchBlock'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleFetchBlock", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('rpcCall'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleRpcCall", null);
+__decorate([
+    (0, schedule_1.Interval)(10000),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "blockFilteringSpeed", null);
 MetaService = __decorate([
     (0, common_1.Injectable)(),
-    __metadata("design:paramtypes", [node_core_1.StoreService])
+    __metadata("design:paramtypes", [node_core_1.StoreService,
+        node_core_1.NodeConfig])
 ], MetaService);
 exports.MetaService = MetaService;
 //# sourceMappingURL=meta.service.js.map

package/dist/meta/meta.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta.service.js","sourceRoot":"","sources":["../../src/meta/meta.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,yDAAgD;AAChD,+CAA4C;AAC5C,gDAS0B;AAE1B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC9E,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAG3D,IAAM,WAAW,GAAjB,MAAM,WAAW;IAatB,YAAoB,YAA0B;QAA1B,iBAAY,GAAZ,YAAY,CAAc;IAAG,CAAC;IAElD,OAAO;QACL,uBACE,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,EACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B,EAC3D,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,UAAU,EAAE,IAAI,CAAC,UAAU,EAC3B,kBAAkB,EAAE,cAAc,EAClC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,EACnD,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EACxB,kBAAkB,EAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAC/C,eAAe,EAAE,IAAI,CAAC,eAAe,IAClC,IAAI,CAAC,WAAW,EACnB;IACJ,CAAC;IAGK,AAAN,KAAK,CAAC,eAAe;QACnB,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACzE,CAAC;IAGD,qBAAqB,CAAC,YAAiC;QACrD,IAAI,CAAC,uBAAuB,GAAG,YAAY,CAAC,MAAM,CAAC;QACnD,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,oBAAoB,CAAC,YAAwC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,YAAY,CAAC,mBAAmB,CAAC;QAC5D,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,iBAAiB,CAAC,YAAgC;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,CAAC;IAGD,eAAe,CAAC,YAA8B;QAC5C,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACxC,CAAC;IAGD,qBAAqB,CAAC,WAAmC;QACvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAGD,kBAAkB,CAAC,EAAE,KAAK,EAAwB;QAChD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC;IAC9B,CAAC;IAGD,qBAAqB,CAAC,EAAE,KAAK,EAAwB;QACnD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;CACF,CAAA;AAxCO;IADL,IAAA,mBAAQ,EAAC,sBAAsB,CAAC;;;;kDAGhC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAIrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,mBAAmB,CAAC;;;;uDAIzC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,WAAW,CAAC;;;;oDAGjC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,SAAS,CAAC;;;;kDAG/B;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,YAAY,CAAC;;;;qDAGlC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AA1EU,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAcuB,wBAAY;GAbnC,WAAW,CA2EvB;AA3EY,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { OnEvent } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  BestBlockPayload,\n  EventPayload,\n  IndexerEvent,\n  NetworkMetadataPayload,\n  ProcessBlockPayload,\n  ProcessedBlockCountPayload,\n  TargetBlockPayload,\n  StoreService,\n} from '@subql/node-core';\n\nconst UPDATE_HEIGHT_INTERVAL = 60000;\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: polkadotSdkVersion } = require('@polkadot/api/package.json');\nconst { version: packageVersion } = require('../../package.json');\n\n@Injectable()\nexport class MetaService {\n  private currentProcessingHeight: number;\n  private currentProcessingTimestamp: number;\n  private bestHeight: number;\n  private targetHeight: number;\n  private networkMeta: NetworkMetadataPayload;\n  private apiConnected: boolean;\n  private usingDictionary: boolean;\n  private injectedApiConnected: boolean;\n  private lastProcessedHeight: number;\n  private lastProcessedTimestamp: number;\n  private processedBlockCount: number;\n\n  constructor(private storeService: StoreService) {}\n\n  getMeta() {\n    return {\n      currentProcessingHeight: this.currentProcessingHeight,\n      currentProcessingTimestamp: this.currentProcessingTimestamp,\n      targetHeight: this.targetHeight,\n      bestHeight: this.bestHeight,\n      indexerNodeVersion: packageVersion,\n      lastProcessedHeight: this.lastProcessedHeight,\n      lastProcessedTimestamp: this.lastProcessedTimestamp,\n      uptime: process.uptime(),\n      polkadotSdkVersion,\n      processedBlockCount: this.processedBlockCount,\n      apiConnected: this.apiConnected,\n      injectedApiConnected: this.injectedApiConnected,\n      usingDictionary: this.usingDictionary,\n      ...this.networkMeta,\n    };\n  }\n\n  @Interval(UPDATE_HEIGHT_INTERVAL)\n  async getTargetHeight(): Promise<void> {\n    await this.storeService.setMetadata('targetHeight', this.targetHeight);\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessing)\n  handleProcessingBlock(blockPayload: ProcessBlockPayload): void {\n    this.currentProcessingHeight = blockPayload.height;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessedCount)\n  handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void {\n    this.processedBlockCount = blockPayload.processedBlockCount;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockTarget)\n  handleTargetBlock(blockPayload: TargetBlockPayload): void {\n    this.targetHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.BlockBest)\n  handleBestBlock(blockPayload: BestBlockPayload): void {\n    this.bestHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.NetworkMetadata)\n  handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void {\n    this.networkMeta = networkMeta;\n  }\n\n  @OnEvent(IndexerEvent.ApiConnected)\n  handleApiConnected({ value }: EventPayload<number>): void {\n    this.apiConnected = !!value;\n  }\n\n  @OnEvent(IndexerEvent.UsingDictionary)\n  handleUsingDictionary({ value }: EventPayload<number>): void {\n    this.usingDictionary = !!value;\n  }\n}\n"]}
+{"version":3,"file":"meta.service.js","sourceRoot":"","sources":["../../src/meta/meta.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,yDAAgD;AAChD,+CAA4C;AAC5C,gDAW0B;AAE1B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC9E,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,UAAU,CAAC,CAAC;AAG9B,IAAM,WAAW,GAAjB,MAAM,WAAW;IAsBtB,YACU,YAA0B,EAC1B,UAAsB;QADtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QAZxB,qBAAgB,GAAG,CAAC,CAAC;QACrB,mBAAc,GAAG,CAAC,CAAC;QACnB,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gBAAW,GAAG,CAAC,CAAC;QAChB,kCAA6B,GAAG,CAAC,CAAC;QAClC,8BAAyB,GAAG,CAAC,CAAC;QAC9B,4BAAuB,GAAG,CAAC,CAAC;QAC5B,yBAAoB,GAAG,CAAC,CAAC;IAM9B,CAAC;IAEJ,OAAO;QACL,uBACE,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,EACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B,EAC3D,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,UAAU,EAAE,IAAI,CAAC,UAAU,EAC3B,kBAAkB,EAAE,cAAc,EAClC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,EACnD,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EACxB,kBAAkB,EAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAC/C,eAAe,EAAE,IAAI,CAAC,eAAe,IAClC,IAAI,CAAC,WAAW,EACnB;IACJ,CAAC;IAGD,eAAe;QACb,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACvC,cAAc,EACd,IAAI,CAAC,YAAY,CAClB,CAAC;IACJ,CAAC;IAGD,qBAAqB,CAAC,YAAiC;QACrD,IAAI,CAAC,uBAAuB,GAAG,YAAY,CAAC,MAAM,CAAC;QACnD,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,oBAAoB,CAAC,YAAwC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,YAAY,CAAC,mBAAmB,CAAC;QAC5D,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,iBAAiB,CAAC,YAAgC;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,CAAC;IAGD,eAAe,CAAC,YAA8B;QAC5C,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACxC,CAAC;IAGD,qBAAqB,CAAC,WAAmC;QACvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAGD,kBAAkB,CAAC,EAAE,KAAK,EAAwB;QAChD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC;IAC9B,CAAC;IAGD,qBAAqB,CAAC,EAAE,KAAK,EAAwB;QACnD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;IAGD,mBAAmB,CAAC,IAAY;QAC9B,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,qBAAqB,CAAC,MAAc;QAClC,IAAI,CAAC,wBAAwB,GAAG,MAAM,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,6BAA6B,GAAG,MAAM,CAAC;SAC7C;IACH,CAAC;IAGD,gBAAgB;QACd,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,aAAa;QACX,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,mBAAmB;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC7B,OAAO;SACR;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,yBAAyB,CAAC;QACrE,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvD,MAAM,cAAc,GAClB,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,6BAA6B,CAAC;QACrE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;QAC/B,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC,wBAAwB,CAAC;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAC9D,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,uBAAuB,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,cAAc,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,KAAK,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACxE,CAAC,CACF;iBACY,CAAC,cAAc,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACvD,CAAC,CACF,mBAAmB,CAAC,QAAQ,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;eAClD,CAAC,UAAU,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;CACF,CAAA;AAtGC;IAAC,IAAA,mBAAQ,EAAC,sBAAsB,CAAC;;;;kDAMhC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAIrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,mBAAmB,CAAC;;;;uDAIzC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,WAAW,CAAC;;;;oDAGjC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,SAAS,CAAC;;;;kDAG/B;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,YAAY,CAAC;;;;qDAGlC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,eAAe,CAAC;;;;sDAMxB;AAED;IAAC,IAAA,uBAAO,EAAC,iBAAiB,CAAC;;;;wDAM1B;AAED;IAAC,IAAA,uBAAO,EAAC,YAAY,CAAC;;;;mDAMrB;AAED;IAAC,IAAA,uBAAO,EAAC,SAAS,CAAC;;;;gDAMlB;AAED;IAAC,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sDAwBf;AAnJU,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAwBa,wBAAY;QACd,sBAAU;GAxBrB,WAAW,CAoJvB;AApJY,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { OnEvent } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  BestBlockPayload,\n  EventPayload,\n  IndexerEvent,\n  NetworkMetadataPayload,\n  ProcessBlockPayload,\n  ProcessedBlockCountPayload,\n  TargetBlockPayload,\n  StoreService,\n  getLogger,\n  NodeConfig,\n} from '@subql/node-core';\n\nconst UPDATE_HEIGHT_INTERVAL = 60000;\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: polkadotSdkVersion } = require('@polkadot/api/package.json');\nconst { version: packageVersion } = require('../../package.json');\nconst logger = getLogger('profiler');\n\n@Injectable()\nexport class MetaService {\n  private currentProcessingHeight: number;\n  private currentProcessingTimestamp: number;\n  private bestHeight: number;\n  private targetHeight: number;\n  private networkMeta: NetworkMetadataPayload;\n  private apiConnected: boolean;\n  private usingDictionary: boolean;\n  private injectedApiConnected: boolean;\n  private lastProcessedHeight: number;\n  private lastProcessedTimestamp: number;\n  private processedBlockCount: number;\n  private accEnqueueBlocks = 0;\n  private accFetchBlocks = 0;\n  private currentFilteringBlockNum = 0;\n  private accRpcCalls = 0;\n  private lastReportedFilteringBlockNum = 0;\n  private lastReportedEnqueueBlocks = 0;\n  private lastReportedFetchBlocks = 0;\n  private lastReportedRpcCalls = 0;\n  private lastStatsReportedTs: Date;\n\n  constructor(\n    private storeService: StoreService,\n    private nodeConfig: NodeConfig,\n  ) {}\n\n  getMeta() {\n    return {\n      currentProcessingHeight: this.currentProcessingHeight,\n      currentProcessingTimestamp: this.currentProcessingTimestamp,\n      targetHeight: this.targetHeight,\n      bestHeight: this.bestHeight,\n      indexerNodeVersion: packageVersion,\n      lastProcessedHeight: this.lastProcessedHeight,\n      lastProcessedTimestamp: this.lastProcessedTimestamp,\n      uptime: process.uptime(),\n      polkadotSdkVersion,\n      processedBlockCount: this.processedBlockCount,\n      apiConnected: this.apiConnected,\n      injectedApiConnected: this.injectedApiConnected,\n      usingDictionary: this.usingDictionary,\n      ...this.networkMeta,\n    };\n  }\n\n  @Interval(UPDATE_HEIGHT_INTERVAL)\n  getTargetHeight(): void {\n    this.storeService.storeCache.metadata.set(\n      'targetHeight',\n      this.targetHeight,\n    );\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessing)\n  handleProcessingBlock(blockPayload: ProcessBlockPayload): void {\n    this.currentProcessingHeight = blockPayload.height;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessedCount)\n  handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void {\n    this.processedBlockCount = blockPayload.processedBlockCount;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockTarget)\n  handleTargetBlock(blockPayload: TargetBlockPayload): void {\n    this.targetHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.BlockBest)\n  handleBestBlock(blockPayload: BestBlockPayload): void {\n    this.bestHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.NetworkMetadata)\n  handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void {\n    this.networkMeta = networkMeta;\n  }\n\n  @OnEvent(IndexerEvent.ApiConnected)\n  handleApiConnected({ value }: EventPayload<number>): void {\n    this.apiConnected = !!value;\n  }\n\n  @OnEvent(IndexerEvent.UsingDictionary)\n  handleUsingDictionary({ value }: EventPayload<number>): void {\n    this.usingDictionary = !!value;\n  }\n\n  @OnEvent('enqueueBlocks')\n  handleEnqueueBlocks(size: number): void {\n    this.accEnqueueBlocks += size;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('filteringBlocks')\n  handleFilteringBlocks(height: number): void {\n    this.currentFilteringBlockNum = height;\n    if (!this.lastStatsReportedTs) {\n      this.lastReportedFilteringBlockNum = height;\n    }\n  }\n\n  @OnEvent('fetchBlock')\n  handleFetchBlock(): void {\n    this.accFetchBlocks++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('rpcCall')\n  handleRpcCall(): void {\n    this.accRpcCalls++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @Interval(10000)\n  blockFilteringSpeed(): void {\n    if (!this.nodeConfig.profiler) {\n      return;\n    }\n    const count = this.accEnqueueBlocks - this.lastReportedEnqueueBlocks;\n    this.lastReportedEnqueueBlocks = this.accEnqueueBlocks;\n    const filteringCount =\n      this.currentFilteringBlockNum - this.lastReportedFilteringBlockNum;\n    const now = new Date();\n    const timepass = now.getTime() - this.lastStatsReportedTs.getTime();\n    this.lastStatsReportedTs = now;\n    this.lastReportedFilteringBlockNum = this.currentFilteringBlockNum;\n    const rpcCalls = this.accRpcCalls - this.lastReportedRpcCalls;\n    this.lastReportedRpcCalls = this.accRpcCalls;\n    const fetchCount = this.accFetchBlocks - this.lastReportedFetchBlocks;\n    this.lastReportedFetchBlocks = this.accFetchBlocks;\n    logger.info(`actual block filtering: ${(count / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, \\\nseeming speed: ${(filteringCount / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, rpcCalls: ${(rpcCalls / (timepass / 1000)).toFixed(2)}/sec \\\nfetch speed: ${(fetchCount / (timepass / 1000)).toFixed(2)}/sec`);\n  }\n}\n"]}

package/dist/subcommands/forceClean.service.d.ts

@@ -1,8 +1,10 @@
 import { NodeConfig } from '@subql/node-core';
 import { Sequelize } from 'sequelize';
+import { SubqueryProject } from '../configure/SubqueryProject';
 export declare class ForceCleanService {
     private readonly sequelize;
     private readonly nodeConfig;
-    constructor(sequelize: Sequelize, nodeConfig: NodeConfig);
+    protected project: SubqueryProject;
+    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, project: SubqueryProject);
     forceClean(): Promise<void>;
 }

package/dist/subcommands/forceClean.service.js

@@ -10,6 +10,9 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -18,12 +21,15 @@ exports.ForceCleanService = void 0;
 const fs_1 = __importDefault(require("fs"));
 const common_1 = require("@nestjs/common");
 const node_core_1 = require("@subql/node-core");
+const utils_1 = require("@subql/utils");
 const sequelize_1 = require("sequelize");
+const SubqueryProject_1 = require("../configure/SubqueryProject");
 const logger = (0, node_core_1.getLogger)('Force-clean');
 let ForceCleanService = class ForceCleanService {
-    constructor(sequelize, nodeConfig) {
+    constructor(sequelize, nodeConfig, project) {
         this.sequelize = sequelize;
         this.nodeConfig = nodeConfig;
+        this.project = project;
     }
     async forceClean() {
         const schema = await (0, node_core_1.getExistingProjectSchema)(this.nodeConfig, this.sequelize);
@@ -31,19 +37,35 @@ let ForceCleanService = class ForceCleanService {
             logger.error('Unable to locate schema');
             throw new Error('Schema does not exist.');
         }
+        const modelsRelation = (0, utils_1.getAllEntitiesRelations)(this.project.schema);
         try {
             // drop existing project schema and metadata table
             await this.sequelize.dropSchema(`"${schema}"`, {
                 logging: false,
                 benchmark: false,
             });
+            // TODO, remove this soon, once original enum are cleaned
+            // Deprecate, now enums are moved under schema, drop schema will remove project enums
+            await Promise.all(modelsRelation.enums.map(async (e) => {
+                const enumTypeNameDeprecated = `${schema}_enum_${(0, node_core_1.enumNameToHash)(e.name)}`;
+                const resultsDeprecated = await (0, node_core_1.getEnumDeprecated)(this.sequelize, enumTypeNameDeprecated);
+                if (resultsDeprecated.length !== 0) {
+                    await this.sequelize.query(`
+            DROP TYPE "${enumTypeNameDeprecated}";
+          `);
+                }
+            }));
             // remove schema from subquery table (might not exist)
-            await this.sequelize.query(` DELETE
+            const checker = await this.sequelize.query(`
+              SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'public' AND  TABLE_NAME = 'subqueries'`);
+            if (checker[1].rowCount > 0) {
+                await this.sequelize.query(` DELETE
                   FROM public.subqueries
                   WHERE name = :name`, {
-                replacements: { name: this.nodeConfig.subqueryName },
-                type: sequelize_1.QueryTypes.DELETE,
-            });
+                    replacements: { name: this.nodeConfig.subqueryName },
+                    type: sequelize_1.QueryTypes.DELETE,
+                });
+            }
             logger.info('force cleaned schema and tables');
             if (fs_1.default.existsSync(this.nodeConfig.mmrPath)) {
                 await fs_1.default.promises.unlink(this.nodeConfig.mmrPath);
@@ -58,8 +80,10 @@ let ForceCleanService = class ForceCleanService {
 };
 ForceCleanService = __decorate([
     (0, common_1.Injectable)(),
+    __param(2, (0, common_1.Inject)('ISubqueryProject')),
     __metadata("design:paramtypes", [sequelize_1.Sequelize,
-        node_core_1.NodeConfig])
+        node_core_1.NodeConfig,
+        SubqueryProject_1.SubqueryProject])
 ], ForceCleanService);
 exports.ForceCleanService = ForceCleanService;
 //# sourceMappingURL=forceClean.service.js.map

package/dist/subcommands/forceClean.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"forceClean.service.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,2CAA4C;AAC5C,gDAI0B;AAC1B,yCAAkD;AAElD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AAGjC,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAC5B,YACmB,SAAoB,EACpB,UAAsB;QADtB,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;IACtC,CAAC;IAEJ,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,IAAA,oCAAwB,EAC3C,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QAED,IAAI;YACF,kDAAkD;YAClD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,MAAM,GAAG,EAAE;gBAC7C,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,sDAAsD;YACtD,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxB;;qCAE6B,EAC7B;gBACE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBACpD,IAAI,EAAE,sBAAU,CAAC,MAAM;aACxB,CACF,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAE/C,IAAI,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC1C,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;aAC7C;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;SACX;IACH,CAAC;CACF,CAAA;AA7CY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAGmB,qBAAS;QACR,sBAAU;GAH9B,iBAAiB,CA6C7B;AA7CY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport { Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  NodeConfig,\n  getExistingProjectSchema,\n} from '@subql/node-core';\nimport { QueryTypes, Sequelize } from 'sequelize';\n\nconst logger = getLogger('Force-clean');\n\n@Injectable()\nexport class ForceCleanService {\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n  ) {}\n\n  async forceClean(): Promise<void> {\n    const schema = await getExistingProjectSchema(\n      this.nodeConfig,\n      this.sequelize,\n    );\n    if (!schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n\n    try {\n      // drop existing project schema and metadata table\n      await this.sequelize.dropSchema(`\"${schema}\"`, {\n        logging: false,\n        benchmark: false,\n      });\n\n      // remove schema from subquery table (might not exist)\n      await this.sequelize.query(\n        ` DELETE\n                  FROM public.subqueries\n                  WHERE name = :name`,\n        {\n          replacements: { name: this.nodeConfig.subqueryName },\n          type: QueryTypes.DELETE,\n        },\n      );\n\n      logger.info('force cleaned schema and tables');\n\n      if (fs.existsSync(this.nodeConfig.mmrPath)) {\n        await fs.promises.unlink(this.nodeConfig.mmrPath);\n        logger.info('force cleaned file based mmr');\n      }\n    } catch (err) {\n      logger.error(err, 'failed to force clean');\n      throw err;\n    }\n  }\n}\n"]}
+{"version":3,"file":"forceClean.service.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,2CAAoD;AACpD,gDAM0B;AAC1B,wCAAuD;AACvD,yCAAkD;AAClD,kEAA+D;AAE/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AAGjC,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAC5B,YACmB,SAAoB,EACpB,UAAsB,EACD,OAAwB;QAF7C,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACD,YAAO,GAAP,OAAO,CAAiB;IAC7D,CAAC;IAEJ,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,IAAA,oCAAwB,EAC3C,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEpE,IAAI;YACF,kDAAkD;YAClD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,MAAM,GAAG,EAAE;gBAC7C,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,yDAAyD;YACzD,qFAAqF;YACrF,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,sBAAsB,GAAG,GAAG,MAAM,SAAS,IAAA,0BAAc,EAC7D,CAAC,CAAC,IAAI,CACP,EAAE,CAAC;gBACJ,MAAM,iBAAiB,GAAG,MAAM,IAAA,6BAAiB,EAC/C,IAAI,CAAC,SAAS,EACd,sBAAsB,CACvB,CAAC;gBACF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAClC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;yBACd,sBAAsB;WACpC,CAAC,CAAC;iBACF;YACH,CAAC,CAAC,CACH,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxC;mHAC2G,CAC5G,CAAC;YAEF,IAAK,OAAO,CAAC,CAAC,CAAS,CAAC,QAAQ,GAAG,CAAC,EAAE;gBACpC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxB;;qCAE2B,EAC3B;oBACE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;oBACpD,IAAI,EAAE,sBAAU,CAAC,MAAM;iBACxB,CACF,CAAC;aACH;YAED,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAE/C,IAAI,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC1C,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;aAC7C;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;SACX;IACH,CAAC;CACF,CAAA;AAzEY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFC,qBAAS;QACR,sBAAU;QACQ,iCAAe;GAJrD,iBAAiB,CAyE7B;AAzEY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  NodeConfig,\n  getExistingProjectSchema,\n  enumNameToHash,\n  getEnumDeprecated,\n} from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport { QueryTypes, Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nconst logger = getLogger('Force-clean');\n\n@Injectable()\nexport class ForceCleanService {\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') protected project: SubqueryProject,\n  ) {}\n\n  async forceClean(): Promise<void> {\n    const schema = await getExistingProjectSchema(\n      this.nodeConfig,\n      this.sequelize,\n    );\n    if (!schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    const modelsRelation = getAllEntitiesRelations(this.project.schema);\n\n    try {\n      // drop existing project schema and metadata table\n      await this.sequelize.dropSchema(`\"${schema}\"`, {\n        logging: false,\n        benchmark: false,\n      });\n\n      // TODO, remove this soon, once original enum are cleaned\n      // Deprecate, now enums are moved under schema, drop schema will remove project enums\n      await Promise.all(\n        modelsRelation.enums.map(async (e) => {\n          const enumTypeNameDeprecated = `${schema}_enum_${enumNameToHash(\n            e.name,\n          )}`;\n          const resultsDeprecated = await getEnumDeprecated(\n            this.sequelize,\n            enumTypeNameDeprecated,\n          );\n          if (resultsDeprecated.length !== 0) {\n            await this.sequelize.query(`\n            DROP TYPE \"${enumTypeNameDeprecated}\";\n          `);\n          }\n        }),\n      );\n\n      // remove schema from subquery table (might not exist)\n      const checker = await this.sequelize.query(\n        `\n              SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'public' AND  TABLE_NAME = 'subqueries'`,\n      );\n\n      if ((checker[1] as any).rowCount > 0) {\n        await this.sequelize.query(\n          ` DELETE\n                  FROM public.subqueries\n                  WHERE name = :name`,\n          {\n            replacements: { name: this.nodeConfig.subqueryName },\n            type: QueryTypes.DELETE,\n          },\n        );\n      }\n\n      logger.info('force cleaned schema and tables');\n\n      if (fs.existsSync(this.nodeConfig.mmrPath)) {\n        await fs.promises.unlink(this.nodeConfig.mmrPath);\n        logger.info('force cleaned file based mmr');\n      }\n    } catch (err) {\n      logger.error(err, 'failed to force clean');\n      throw err;\n    }\n  }\n}\n"]}

package/dist/subcommands/reindex.init.js

@@ -13,7 +13,12 @@ async function reindexInit(targetHeight) {
         const app = await core_1.NestFactory.create(reindex_module_1.ReindexModule);
         await app.init();
         const reindexService = app.get(reindex_service_1.ReindexService);
-        await reindexService.reindex(targetHeight);
+        await reindexService.init();
+        const actualReindexHeight = await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight);
+        if (actualReindexHeight !== targetHeight) {
+            logger.info(`Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindexHeight}, will index to ${actualReindexHeight}`);
+        }
+        await reindexService.reindex(actualReindexHeight);
     }
     catch (e) {
         logger.error(e, 'Reindex failed to execute');