@subql/node-ethereum 0.4.1-8 → 0.4.1-storeCache-0.0.3

package/dist/indexer/worker/worker.js

@@ -15,15 +15,16 @@ const { argv } = yargs_1.yargsOptions;
 (0, logger_1.initLogger)(argv.debug, argv.outputFmt, argv.logLevel);
 const assert_1 = __importDefault(require("assert"));
 const node_worker_threads_1 = require("node:worker_threads");
+const v8_1 = require("v8");
 const core_1 = require("@nestjs/core");
 const node_core_1 = require("@subql/node-core");
 const dynamic_ds_service_1 = require("../dynamic-ds.service");
 const indexer_manager_1 = require("../indexer.manager");
 const worker_module_1 = require("./worker.module");
 const worker_service_1 = require("./worker.service");
+const worker_unfinalizedBlocks_service_1 = require("./worker.unfinalizedBlocks.service");
 let app;
 let workerService;
-let dynamicDsService;
 const logger = (0, node_core_1.getLogger)(`worker #${node_worker_threads_1.threadId}`);
 async function initWorker() {
     try {
@@ -39,7 +40,6 @@ async function initWorker() {
         // Initialise async services, we do this here rather than in factories so we can capture one off events
         await indexerManager.start();
         workerService = app.get(worker_service_1.WorkerService);
-        dynamicDsService = app.get(dynamic_ds_service_1.DynamicDsService);
     }
     catch (e) {
         console.log('Failed to start worker', e);
@@ -78,19 +78,26 @@ async function getStatus() {
         isIndexing: workerService.isIndexing,
     };
 }
-async function reloadDynamicDs() {
-    return dynamicDsService.reloadDynamicDatasources();
+// eslint-disable-next-line @typescript-eslint/require-await
+async function getMemoryLeft() {
+    const totalHeap = (0, v8_1.getHeapStatistics)().heap_size_limit;
+    const heapUsed = process.memoryUsage().heapUsed;
+    return totalHeap - heapUsed;
+}
+async function waitForWorkerBatchSize(heapSizeInBytes) {
+    await (0, node_core_1.waitForBatchSize)(heapSizeInBytes);
 }
 // Register these functions to be exposed to worker host
-(0, node_core_1.registerWorker)({
+global.host = node_core_1.WorkerHost.create([...node_core_1.hostStoreKeys, ...node_core_1.hostDynamicDsKeys, ...worker_unfinalizedBlocks_service_1.hostUnfinalizedBlocksKeys], {
     initWorker,
     fetchBlock,
     processBlock,
     numFetchedBlocks,
     numFetchingBlocks,
     getStatus,
-    reloadDynamicDs,
-});
+    getMemoryLeft,
+    waitForWorkerBatchSize,
+}, logger);
 process.on('uncaughtException', (e) => {
     logger.error(e, 'Uncaught Exception');
     throw e;

package/dist/indexer/worker/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAE/C,uCAA2C;AAC3C,gDAAyE;AACzE,8DAAyD;AACzD,wDAAoD;AACpD,mDAA+C;AAC/C,qDAK0B;AAE1B,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;AACjC,IAAI,gBAAkC,CAAC;AAEvC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;QACvC,gBAAgB,GAAG,GAAG,CAAC,GAAG,CAAC,qCAAgB,CAAC,CAAC;KAC9C;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAErD,0DAA0D;IAC1D,IAAI,GAAG,CAAC,gBAAgB,EAAE;QACxB,MAAM,gBAAgB,GAAG,GAAG,CAAC,GAAG,CAAC,qCAAgB,CAAC,CAAC;QACnD,gBAAgB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;KAC9C;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,eAAe;IAC5B,OAAO,gBAAgB,CAAC,wBAAwB,EAAE,CAAC;AACrD,CAAC;AAED,wDAAwD;AACxD,IAAA,0BAAc,EAAC;IACb,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,eAAe;CAChB,CAAC,CAAC;AAWH,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport { registerWorker, getLogger, NestLogger } from '@subql/node-core';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n  FetchBlockResponse,\n  ProcessBlockResponse,\n  WorkerService,\n  WorkerStatusResponse,\n} from './worker.service';\n\nlet app: INestApplication;\nlet workerService: WorkerService;\nlet dynamicDsService: DynamicDsService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n  try {\n    if (app) {\n      logger.warn('Worker already initialised');\n      return;\n    }\n\n    app = await NestFactory.create(WorkerModule, {\n      logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n    });\n\n    await app.init();\n\n    const indexerManager = app.get(IndexerManager);\n    // Initialise async services, we do this here rather than in factories so we can capture one off events\n    await indexerManager.start();\n\n    workerService = app.get(WorkerService);\n    dynamicDsService = app.get(DynamicDsService);\n  } catch (e) {\n    console.log('Failed to start worker', e);\n    logger.error(e, 'Failed to start worker');\n    throw e;\n  }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n  assert(workerService, 'Not initialised');\n\n  return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n  assert(workerService, 'Not initialised');\n\n  const res = await workerService.processBlock(height);\n\n  // Clean up the temp ds records for worker thread instance\n  if (res.dynamicDsCreated) {\n    const dynamicDsService = app.get(DynamicDsService);\n    dynamicDsService.deleteTempDsRecords(height);\n  }\n\n  return res;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n  return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n  return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n  return {\n    threadId,\n    fetchedBlocks: workerService.numFetchedBlocks,\n    toFetchBlocks: workerService.numFetchingBlocks,\n    isIndexing: workerService.isIndexing,\n  };\n}\n\nasync function reloadDynamicDs(): Promise<void> {\n  return dynamicDsService.reloadDynamicDatasources();\n}\n\n// Register these functions to be exposed to worker host\nregisterWorker({\n  initWorker,\n  fetchBlock,\n  processBlock,\n  numFetchedBlocks,\n  numFetchingBlocks,\n  getStatus,\n  reloadDynamicDs,\n});\n\n// Export types to be used on the parent\nexport type InitWorker = typeof initWorker;\nexport type FetchBlock = typeof fetchBlock;\nexport type ProcessBlock = typeof processBlock;\nexport type NumFetchedBlocks = typeof numFetchedBlocks;\nexport type NumFetchingBlocks = typeof numFetchingBlocks;\nexport type GetWorkerStatus = typeof getStatus;\nexport type ReloadDynamicDs = typeof reloadDynamicDs;\n\nprocess.on('uncaughtException', (e) => {\n  logger.error(e, 'Uncaught Exception');\n  throw e;\n});\n"]}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAC/C,2BAAuC;AAEvC,uCAA2C;AAC3C,gDAS0B;AAE1B,8DAAyD;AACzD,wDAAoD;AACpD,mDAA+C;AAC/C,qDAK0B;AAC1B,yFAG4C;AAC5C,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;AAEjC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;KACxC;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAErD,0DAA0D;IAC1D,IAAI,GAAG,CAAC,gBAAgB,EAAE;QACxB,MAAM,gBAAgB,GAAG,GAAG,CAAC,GAAG,CAAC,qCAAgB,CAAC,CAAC;QACnD,gBAAgB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;KAC9C;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,aAAa;IAC1B,MAAM,SAAS,GAAG,IAAA,sBAAiB,GAAE,CAAC,eAAe,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;IAEhD,OAAO,SAAS,GAAG,QAAQ,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,eAAuB;IAC3D,MAAM,IAAA,4BAAgB,EAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,wDAAwD;AACvD,MAAc,CAAC,IAAI,GAAG,sBAAU,CAAC,MAAM,CAItC,CAAC,GAAG,yBAAa,EAAE,GAAG,6BAAiB,EAAE,GAAG,4DAAyB,CAAC,EACtE;IACE,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,aAAa;IACb,sBAAsB;CACvB,EACD,MAAM,CACP,CAAC;AA0BF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { getHeapStatistics } from 'v8';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport {\n  waitForBatchSize,\n  WorkerHost,\n  getLogger,\n  NestLogger,\n  hostStoreKeys,\n  HostStore,\n  hostDynamicDsKeys,\n  HostDynamicDS,\n} from '@subql/node-core';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n  FetchBlockResponse,\n  ProcessBlockResponse,\n  WorkerService,\n  WorkerStatusResponse,\n} from './worker.service';\nimport {\n  HostUnfinalizedBlocks,\n  hostUnfinalizedBlocksKeys,\n} from './worker.unfinalizedBlocks.service';\nlet app: INestApplication;\nlet workerService: WorkerService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n  try {\n    if (app) {\n      logger.warn('Worker already initialised');\n      return;\n    }\n\n    app = await NestFactory.create(WorkerModule, {\n      logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n    });\n\n    await app.init();\n\n    const indexerManager = app.get(IndexerManager);\n    // Initialise async services, we do this here rather than in factories so we can capture one off events\n    await indexerManager.start();\n\n    workerService = app.get(WorkerService);\n  } catch (e) {\n    console.log('Failed to start worker', e);\n    logger.error(e, 'Failed to start worker');\n    throw e;\n  }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n  assert(workerService, 'Not initialised');\n  return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n  assert(workerService, 'Not initialised');\n\n  const res = await workerService.processBlock(height);\n\n  // Clean up the temp ds records for worker thread instance\n  if (res.dynamicDsCreated) {\n    const dynamicDsService = app.get(DynamicDsService);\n    dynamicDsService.deleteTempDsRecords(height);\n  }\n\n  return res;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n  return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n  return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n  return {\n    threadId,\n    fetchedBlocks: workerService.numFetchedBlocks,\n    toFetchBlocks: workerService.numFetchingBlocks,\n    isIndexing: workerService.isIndexing,\n  };\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getMemoryLeft(): Promise<number> {\n  const totalHeap = getHeapStatistics().heap_size_limit;\n  const heapUsed = process.memoryUsage().heapUsed;\n\n  return totalHeap - heapUsed;\n}\n\nasync function waitForWorkerBatchSize(heapSizeInBytes: number): Promise<void> {\n  await waitForBatchSize(heapSizeInBytes);\n}\n\n// Register these functions to be exposed to worker host\n(global as any).host = WorkerHost.create<\n  HostStore & HostDynamicDS<SubqlProjectDs> & HostUnfinalizedBlocks,\n  IInitIndexerWorker\n>(\n  [...hostStoreKeys, ...hostDynamicDsKeys, ...hostUnfinalizedBlocksKeys],\n  {\n    initWorker,\n    fetchBlock,\n    processBlock,\n    numFetchedBlocks,\n    numFetchingBlocks,\n    getStatus,\n    getMemoryLeft,\n    waitForWorkerBatchSize,\n  },\n  logger,\n);\n\n// Export types to be used on the parent\ntype InitWorker = typeof initWorker;\ntype FetchBlock = typeof fetchBlock;\ntype ProcessBlock = typeof processBlock;\ntype NumFetchedBlocks = typeof numFetchedBlocks;\ntype NumFetchingBlocks = typeof numFetchingBlocks;\ntype GetWorkerStatus = typeof getStatus;\ntype GetMemoryLeft = typeof getMemoryLeft;\ntype WaitForWorkerBatchSize = typeof waitForWorkerBatchSize;\n\nexport type IIndexerWorker = {\n  processBlock: ProcessBlock;\n  fetchBlock: FetchBlock;\n  numFetchedBlocks: NumFetchedBlocks;\n  numFetchingBlocks: NumFetchingBlocks;\n  getStatus: GetWorkerStatus;\n  getMemoryLeft: GetMemoryLeft;\n  waitForWorkerBatchSize: WaitForWorkerBatchSize;\n};\n\nexport type IInitIndexerWorker = IIndexerWorker & {\n  initWorker: InitWorker;\n};\n\nprocess.on('uncaughtException', (e) => {\n  logger.error(e, 'Uncaught Exception');\n  throw e;\n});\n"]}

package/dist/indexer/worker/worker.service.d.ts

@@ -1,11 +1,11 @@
-import { ApiService, NodeConfig } from '@subql/node-core';
+import { NodeConfig, ApiService } from '@subql/node-core';
 import { IndexerManager } from '../indexer.manager';
 export declare type FetchBlockResponse = {
     parentHash: string;
 } | undefined;
 export declare type ProcessBlockResponse = {
     dynamicDsCreated: boolean;
-    operationHash: string;
+    blockHash: string;
     reindexBlockHeight: number;
 };
 export declare type WorkerStatusResponse = {

package/dist/indexer/worker/worker.service.js

@@ -30,10 +30,16 @@ let WorkerService = class WorkerService {
             return await this.queue.put(async () => {
                 // If a dynamic ds is created we might be asked to fetch blocks again, use existing result
                 if (!this.fetchedBlocks[height]) {
+                    if (node_core_1.memoryLock.isLocked()) {
+                        const start = Date.now();
+                        await node_core_1.memoryLock.waitForUnlock();
+                        const end = Date.now();
+                        logger.debug(`memory lock wait time: ${end - start}ms`);
+                    }
                     const [block] = await this.apiService.api.fetchBlocks([height]);
                     this.fetchedBlocks[height] = block;
                 }
-                const block = this.fetchedBlocks[height];
+                // const block = this.fetchedBlocks[height];
                 // Return info to get the runtime version, this lets the worker thread know
                 return undefined;
             });
@@ -50,14 +56,15 @@ let WorkerService = class WorkerService {
                 throw new Error(`Block ${height} has not been fetched`);
             }
             delete this.fetchedBlocks[height];
-            const response = await this.indexerManager.indexBlock(block);
-            this._isIndexing = false;
-            return Object.assign(Object.assign({}, response), { operationHash: Buffer.from(response.operationHash).toString('base64') });
+            return await this.indexerManager.indexBlock(block);
         }
         catch (e) {
             logger.error(e, `Failed to index block ${height}: ${e.stack}`);
             throw e;
         }
+        finally {
+            this._isIndexing = false;
+        }
     }
     get numFetchedBlocks() {
         return Object.keys(this.fetchedBlocks).length;

package/dist/indexer/worker/worker.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAA4C;AAC5C,gDAAgF;AAEhF,wDAAoD;AAiBpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGjD,IAAM,aAAa,GAAnB,MAAM,aAAa;IAMxB,YACU,UAAsB,EACtB,cAA8B,EACtC,UAAsB;QAFd,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAPhC,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAS1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAChE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAEzC,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,EAAE,CAAC,CAAC;SACpD;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAE7D,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,uCACK,QAAQ,KACX,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IACrE;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AApEY,aAAa;IADzB,IAAA,mBAAU,GAAE;qCAQW,sBAAU;QACN,gCAAc;QAC1B,sBAAU;GATb,aAAa,CAoEzB;AApEY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Injectable } from '@nestjs/common';\nimport { ApiService, NodeConfig, getLogger, AutoQueue } from '@subql/node-core';\nimport { EthereumBlockWrapper, BlockWrapper } from '@subql/types-ethereum';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  operationHash: string; // Base64 encoded u8a array\n  reindexBlockHeight: number;\n};\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: ApiService,\n    private indexerManager: IndexerManager,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          const [block] = await this.apiService.api.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        const block = this.fetchedBlocks[height];\n\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(e, `Failed to fetch block ${height}`);\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      const response = await this.indexerManager.indexBlock(block);\n\n      this._isIndexing = false;\n      return {\n        ...response,\n        operationHash: Buffer.from(response.operationHash).toString('base64'),\n      };\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}
+{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAA4C;AAC5C,gDAM0B;AAE1B,wDAAoD;AAiBpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGjD,IAAM,aAAa,GAAnB,MAAM,aAAa;IAMxB,YACU,UAAsB,EACtB,cAA8B,EACtC,UAAsB;QAFd,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAPhC,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAS1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,IAAI,sBAAU,CAAC,QAAQ,EAAE,EAAE;wBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACzB,MAAM,sBAAU,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACvB,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC;qBACzD;oBAED,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAChE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,4CAA4C;gBAE5C,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,EAAE,CAAC,CAAC;SACpD;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;SACpD;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC1B;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AAvEY,aAAa;IADzB,IAAA,mBAAU,GAAE;qCAQW,sBAAU;QACN,gCAAc;QAC1B,sBAAU;GATb,aAAa,CAuEzB;AAvEY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Injectable } from '@nestjs/common';\nimport {\n  NodeConfig,\n  getLogger,\n  AutoQueue,\n  memoryLock,\n  ApiService,\n} from '@subql/node-core';\nimport { BlockWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  blockHash: string;\n  reindexBlockHeight: number;\n};\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: ApiService,\n    private indexerManager: IndexerManager,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          if (memoryLock.isLocked()) {\n            const start = Date.now();\n            await memoryLock.waitForUnlock();\n            const end = Date.now();\n            logger.debug(`memory lock wait time: ${end - start}ms`);\n          }\n\n          const [block] = await this.apiService.api.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        // const block = this.fetchedBlocks[height];\n\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(e, `Failed to fetch block ${height}`);\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      return await this.indexerManager.indexBlock(block);\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    } finally {\n      this._isIndexing = false;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}

package/dist/indexer/worker/worker.unfinalizedBlocks.service.d.ts

@@ -0,0 +1,11 @@
+import { EthereumBlock } from '@subql/types-ethereum';
+import { IUnfinalizedBlocksService } from '../unfinalizedBlocks.service';
+export declare type HostUnfinalizedBlocks = {
+    unfinalizedBlocksProcess: (block: EthereumBlock) => Promise<number | null>;
+};
+export declare const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[];
+export declare class WorkerUnfinalizedBlocksService implements IUnfinalizedBlocksService {
+    private host;
+    constructor(host: HostUnfinalizedBlocks);
+    processUnfinalizedBlocks(block: EthereumBlock): Promise<number | null>;
+}

package/dist/indexer/worker/worker.unfinalizedBlocks.service.js

@@ -0,0 +1,32 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerUnfinalizedBlocksService = exports.hostUnfinalizedBlocksKeys = void 0;
+const common_1 = require("@nestjs/common");
+exports.hostUnfinalizedBlocksKeys = [
+    'unfinalizedBlocksProcess',
+];
+let WorkerUnfinalizedBlocksService = class WorkerUnfinalizedBlocksService {
+    constructor(host) {
+        this.host = host;
+    }
+    async processUnfinalizedBlocks(block) {
+        return this.host.unfinalizedBlocksProcess(block);
+    }
+};
+WorkerUnfinalizedBlocksService = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [Object])
+], WorkerUnfinalizedBlocksService);
+exports.WorkerUnfinalizedBlocksService = WorkerUnfinalizedBlocksService;
+//# sourceMappingURL=worker.unfinalizedBlocks.service.js.map

package/dist/indexer/worker/worker.unfinalizedBlocks.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAQ/B,QAAA,yBAAyB,GAAoC;IACxE,0BAA0B;CAC3B,CAAC;AAGK,IAAM,8BAA8B,GAApC,MAAM,8BAA8B;IAGzC,YAAoB,IAA2B;QAA3B,SAAI,GAAJ,IAAI,CAAuB;IAAG,CAAC;IAEnD,KAAK,CAAC,wBAAwB,CAAC,KAAoB;QACjD,OAAO,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;CACF,CAAA;AARY,8BAA8B;IAD1C,IAAA,mBAAU,GAAE;;GACA,8BAA8B,CAQ1C;AARY,wEAA8B","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { EthereumBlock } from '@subql/types-ethereum';\nimport { IUnfinalizedBlocksService } from '../unfinalizedBlocks.service';\n\nexport type HostUnfinalizedBlocks = {\n  unfinalizedBlocksProcess: (block: EthereumBlock) => Promise<number | null>;\n};\n\nexport const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[] = [\n  'unfinalizedBlocksProcess',\n];\n\n@Injectable()\nexport class WorkerUnfinalizedBlocksService\n  implements IUnfinalizedBlocksService\n{\n  constructor(private host: HostUnfinalizedBlocks) {}\n\n  async processUnfinalizedBlocks(block: EthereumBlock): Promise<number | null> {\n    return this.host.unfinalizedBlocksProcess(block);\n  }\n}\n"]}

package/dist/init.js

@@ -8,7 +8,6 @@ const common_1 = require("@subql/common");
 const node_core_1 = require("@subql/node-core");
 const app_module_1 = require("./app.module");
 const fetch_service_1 = require("./indexer/fetch.service");
-const project_service_1 = require("./indexer/project.service");
 const yargs_1 = require("./yargs");
 const pjson = require('../package.json');
 const { argv } = yargs_1.yargsOptions;
@@ -35,13 +34,14 @@ async function bootstrap() {
             logger: debug ? new node_core_1.NestLogger() : false,
         });
         await app.init();
-        const projectService = app.get(project_service_1.ProjectService);
+        const projectService = app.get('IProjectService');
         const fetchService = app.get(fetch_service_1.FetchService);
         const apiService = app.get(node_core_1.ApiService);
         // Initialise async services, we do this here rather than in factories, so we can capture one off events
         await apiService.init();
         await projectService.init();
         await fetchService.init(projectService.startHeight);
+        app.enableShutdownHooks();
         await app.listen(port);
         logger.info(`Node started on port: ${port}`);
     }

package/dist/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,0CAAkD;AAClD,gDAAqE;AACrE,6CAAyC;AACzC,2DAAuD;AACvD,+DAA2D;AAC3D,mCAAuC;AACvC,MAAM,KAAK,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEhC,KAAK,UAAU,SAAS;;IAC7B,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,eAAe,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAEzB,MAAM,QAAQ,GAAG,CAAC,CAAM,EAAE,EAAE;QAC1B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAA,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mCAAI,CAAC,MAAM,IAAA,0BAAiB,EAAC,YAAY,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,CAAC,KAAK,CACV,wDAAwD,IAAI,KAC1D,IAAI,GAAG,EACT,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,IAAI,CAAC,MAAM,EAAE;QACf,MAAM,CAAC,IAAI,CACT,mHAAmH,CACpH,CAAC;KACH;IAED,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,sBAAS,EAAE;YAC9C,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,sBAAU,EAAE,CAAC,CAAC,CAAC,KAAK;SACzC,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,4BAAY,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAU,CAAC,CAAC;QAEvC,wGAAwG;QACxG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;KAC9C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC;AA/CD,8BA+CC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { findAvailablePort } from '@subql/common';\nimport { getLogger, ApiService, NestLogger } from '@subql/node-core';\nimport { AppModule } from './app.module';\nimport { FetchService } from './indexer/fetch.service';\nimport { ProjectService } from './indexer/project.service';\nimport { yargsOptions } from './yargs';\nconst pjson = require('../package.json');\n\nconst { argv } = yargsOptions;\n\nconst DEFAULT_PORT = 3000;\nconst logger = getLogger('subql-node');\n\nexport async function bootstrap() {\n  logger.info(`Current ${pjson.name} version is ${pjson.version}`);\n  const debug = argv.debug;\n\n  const validate = (x: any) => {\n    const p = parseInt(x);\n    return isNaN(p) ? null : p;\n  };\n\n  const port = validate(argv.port) ?? (await findAvailablePort(DEFAULT_PORT));\n  if (!port) {\n    logger.error(\n      `Unable to find available port (tried ports in range (${port}..${\n        port + 10\n      })). Try setting a free port manually by setting the --port flag`,\n    );\n    process.exit(1);\n  }\n\n  if (argv.unsafe) {\n    logger.warn(\n      'UNSAFE MODE IS ENABLED. This is not recommended for most projects and will not be supported by our hosted service',\n    );\n  }\n\n  try {\n    const app = await NestFactory.create(AppModule, {\n      logger: debug ? new NestLogger() : false,\n    });\n    await app.init();\n\n    const projectService = app.get(ProjectService);\n    const fetchService = app.get(FetchService);\n    const apiService = app.get(ApiService);\n\n    // Initialise async services, we do this here rather than in factories, so we can capture one off events\n    await apiService.init();\n    await projectService.init();\n    await fetchService.init(projectService.startHeight);\n\n    await app.listen(port);\n\n    logger.info(`Node started on port: ${port}`);\n  } catch (e) {\n    logger.error(e, 'Node failed to start');\n    process.exit(1);\n  }\n}\n"]}
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,0CAAkD;AAClD,gDAAqE;AACrE,6CAAyC;AACzC,2DAAuD;AAEvD,mCAAuC;AACvC,MAAM,KAAK,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEhC,KAAK,UAAU,SAAS;;IAC7B,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,eAAe,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAEzB,MAAM,QAAQ,GAAG,CAAC,CAAM,EAAE,EAAE;QAC1B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAA,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mCAAI,CAAC,MAAM,IAAA,0BAAiB,EAAC,YAAY,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,CAAC,KAAK,CACV,wDAAwD,IAAI,KAC1D,IAAI,GAAG,EACT,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,IAAI,CAAC,MAAM,EAAE;QACf,MAAM,CAAC,IAAI,CACT,mHAAmH,CACpH,CAAC;KACH;IAED,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,sBAAS,EAAE;YAC9C,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,sBAAU,EAAE,CAAC,CAAC,CAAC,KAAK;SACzC,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAmB,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAClE,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,4BAAY,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAU,CAAC,CAAC;QAEvC,wGAAwG;QACxG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEpD,GAAG,CAAC,mBAAmB,EAAE,CAAC;QAE1B,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;KAC9C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC;AAjDD,8BAiDC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { findAvailablePort } from '@subql/common';\nimport { getLogger, ApiService, NestLogger } from '@subql/node-core';\nimport { AppModule } from './app.module';\nimport { FetchService } from './indexer/fetch.service';\nimport { ProjectService } from './indexer/project.service';\nimport { yargsOptions } from './yargs';\nconst pjson = require('../package.json');\n\nconst { argv } = yargsOptions;\n\nconst DEFAULT_PORT = 3000;\nconst logger = getLogger('subql-node');\n\nexport async function bootstrap(): Promise<void> {\n  logger.info(`Current ${pjson.name} version is ${pjson.version}`);\n  const debug = argv.debug;\n\n  const validate = (x: any) => {\n    const p = parseInt(x);\n    return isNaN(p) ? null : p;\n  };\n\n  const port = validate(argv.port) ?? (await findAvailablePort(DEFAULT_PORT));\n  if (!port) {\n    logger.error(\n      `Unable to find available port (tried ports in range (${port}..${\n        port + 10\n      })). Try setting a free port manually by setting the --port flag`,\n    );\n    process.exit(1);\n  }\n\n  if (argv.unsafe) {\n    logger.warn(\n      'UNSAFE MODE IS ENABLED. This is not recommended for most projects and will not be supported by our hosted service',\n    );\n  }\n\n  try {\n    const app = await NestFactory.create(AppModule, {\n      logger: debug ? new NestLogger() : false,\n    });\n    await app.init();\n\n    const projectService: ProjectService = app.get('IProjectService');\n    const fetchService = app.get(FetchService);\n    const apiService = app.get(ApiService);\n\n    // Initialise async services, we do this here rather than in factories, so we can capture one off events\n    await apiService.init();\n    await projectService.init();\n    await fetchService.init(projectService.startHeight);\n\n    app.enableShutdownHooks();\n\n    await app.listen(port);\n\n    logger.info(`Node started on port: ${port}`);\n  } catch (e) {\n    logger.error(e, 'Node failed to start');\n    process.exit(1);\n  }\n}\n"]}

package/dist/meta/meta.module.js

@@ -67,6 +67,14 @@ MetaModule = __decorate([
                 name: 'subql_indexer_processed_block_count',
                 help: 'The number of processed block',
             }),
+            (0, nestjs_prometheus_1.makeGaugeProvider)({
+                name: 'subql_indexer_store_cache_threshold',
+                help: 'Store cache will flush once cache record size excess this threshold',
+            }),
+            (0, nestjs_prometheus_1.makeGaugeProvider)({
+                name: 'subql_indexer_store_cache_records_size',
+                help: 'Number of records waiting to flush in store cache',
+            }),
             meta_service_1.MetaService,
             node_core_1.HealthService,
             node_core_1.ReadyService,

package/dist/meta/meta.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta.module.js","sourceRoot":"","sources":["../../src/meta/meta.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,mEAGqC;AACrC,0DAAsD;AACtD,uDAAmD;AACnD,iDAA6C;AAwDtC,IAAM,UAAU,GAAhB,MAAM,UAAU;CAAG,CAAA;AAAb,UAAU;IAtDtB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,oCAAgB,CAAC,QAAQ,EAAE,EAAE,0BAAW,CAAC;QACnD,WAAW,EAAE,CAAC,gCAAc,EAAE,4BAAgB,EAAE,2BAAe,CAAC;QAChE,SAAS,EAAE;YACT,+BAAmB;YACnB,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,6BAA6B;gBACnC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,4CAA4C;aACnD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,uCAAuC;gBAC7C,IAAI,EAAE,qCAAqC;aAC5C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,mCAAmC;gBACzC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,iCAAiC;gBACvC,IAAI,EAAE,8BAA8B;aACrC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,wCAAwC;aAC/C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,+CAA+C;aACtD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,sDAAsD;aAC7D,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,+BAA+B;aACtC,CAAC;YACF,0BAAW;YACX,yBAAa;YACb,wBAAY;SACb;KACF,CAAC;GACW,UAAU,CAAG;AAAb,gCAAU","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  MetricEventListener,\n  ReadyController,\n  ReadyService,\n  HealthController,\n  HealthService,\n} from '@subql/node-core';\nimport {\n  makeGaugeProvider,\n  PrometheusModule,\n} from '@willsoto/nestjs-prometheus';\nimport { FetchModule } from '../indexer/fetch.module';\nimport { MetaController } from './meta.controller';\nimport { MetaService } from './meta.service';\n\n@Module({\n  imports: [PrometheusModule.register(), FetchModule],\n  controllers: [MetaController, HealthController, ReadyController],\n  providers: [\n    MetricEventListener,\n    makeGaugeProvider({\n      name: 'subql_indexer_api_connected',\n      help: 'The indexer api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_injected_api_connected',\n      help: 'The indexer injected api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processing_block_height',\n      help: 'The current processing block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_height',\n      help: 'The last processed block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_target_block_height',\n      help: 'The latest finalized block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_best_block_height',\n      help: 'The latest best block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_block_queue_size',\n      help: 'The size of fetched block queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_blocknumber_queue_size',\n      help: 'The size of fetched block number queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_using_dictionary',\n      help: 'The status of indexer is using the dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_skip_dictionary_count',\n      help: 'The number of times indexer been skip use dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_count',\n      help: 'The number of processed block',\n    }),\n    MetaService,\n    HealthService,\n    ReadyService,\n  ],\n})\nexport class MetaModule {}\n"]}
+{"version":3,"file":"meta.module.js","sourceRoot":"","sources":["../../src/meta/meta.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,mEAGqC;AACrC,0DAAsD;AACtD,uDAAmD;AACnD,iDAA6C;AAgEtC,IAAM,UAAU,GAAhB,MAAM,UAAU;CAAG,CAAA;AAAb,UAAU;IA9DtB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,oCAAgB,CAAC,QAAQ,EAAE,EAAE,0BAAW,CAAC;QACnD,WAAW,EAAE,CAAC,gCAAc,EAAE,4BAAgB,EAAE,2BAAe,CAAC;QAChE,SAAS,EAAE;YACT,+BAAmB;YACnB,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,6BAA6B;gBACnC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,4CAA4C;aACnD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,uCAAuC;gBAC7C,IAAI,EAAE,qCAAqC;aAC5C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,mCAAmC;gBACzC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,iCAAiC;gBACvC,IAAI,EAAE,8BAA8B;aACrC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,wCAAwC;aAC/C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,+CAA+C;aACtD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,sDAAsD;aAC7D,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,+BAA+B;aACtC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,qEAAqE;aAC5E,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,wCAAwC;gBAC9C,IAAI,EAAE,mDAAmD;aAC1D,CAAC;YACF,0BAAW;YACX,yBAAa;YACb,wBAAY;SACb;KACF,CAAC;GACW,UAAU,CAAG;AAAb,gCAAU","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  MetricEventListener,\n  ReadyController,\n  ReadyService,\n  HealthController,\n  HealthService,\n} from '@subql/node-core';\nimport {\n  makeGaugeProvider,\n  PrometheusModule,\n} from '@willsoto/nestjs-prometheus';\nimport { FetchModule } from '../indexer/fetch.module';\nimport { MetaController } from './meta.controller';\nimport { MetaService } from './meta.service';\n\n@Module({\n  imports: [PrometheusModule.register(), FetchModule],\n  controllers: [MetaController, HealthController, ReadyController],\n  providers: [\n    MetricEventListener,\n    makeGaugeProvider({\n      name: 'subql_indexer_api_connected',\n      help: 'The indexer api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_injected_api_connected',\n      help: 'The indexer injected api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processing_block_height',\n      help: 'The current processing block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_height',\n      help: 'The last processed block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_target_block_height',\n      help: 'The latest finalized block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_best_block_height',\n      help: 'The latest best block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_block_queue_size',\n      help: 'The size of fetched block queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_blocknumber_queue_size',\n      help: 'The size of fetched block number queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_using_dictionary',\n      help: 'The status of indexer is using the dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_skip_dictionary_count',\n      help: 'The number of times indexer been skip use dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_count',\n      help: 'The number of processed block',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_threshold',\n      help: 'Store cache will flush once cache record size excess this threshold',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_records_size',\n      help: 'Number of records waiting to flush in store cache',\n    }),\n    MetaService,\n    HealthService,\n    ReadyService,\n  ],\n})\nexport class MetaModule {}\n"]}

package/dist/meta/meta.service.d.ts

@@ -1,6 +1,7 @@
-import { BestBlockPayload, EventPayload, NetworkMetadataPayload, ProcessBlockPayload, ProcessedBlockCountPayload, TargetBlockPayload, StoreService } from '@subql/node-core';
+import { BestBlockPayload, EventPayload, NetworkMetadataPayload, ProcessBlockPayload, ProcessedBlockCountPayload, TargetBlockPayload, StoreService, NodeConfig } from '@subql/node-core';
 export declare class MetaService {
     private storeService;
+    private nodeConfig;
     private currentProcessingHeight;
     private currentProcessingTimestamp;
     private bestHeight;
@@ -12,7 +13,16 @@ export declare class MetaService {
     private lastProcessedHeight;
     private lastProcessedTimestamp;
     private processedBlockCount;
-    constructor(storeService: StoreService);
+    private accEnqueueBlocks;
+    private accFetchBlocks;
+    private currentFilteringBlockNum;
+    private accRpcCalls;
+    private lastReportedFilteringBlockNum;
+    private lastReportedEnqueueBlocks;
+    private lastReportedFetchBlocks;
+    private lastReportedRpcCalls;
+    private lastStatsReportedTs;
+    constructor(storeService: StoreService, nodeConfig: NodeConfig);
     getMeta(): {
         chain: string;
         specName: string;
@@ -31,7 +41,7 @@ export declare class MetaService {
         injectedApiConnected: boolean;
         usingDictionary: boolean;
     };
-    getTargetHeight(): Promise<void>;
+    getTargetHeight(): void;
     handleProcessingBlock(blockPayload: ProcessBlockPayload): void;
     handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void;
     handleTargetBlock(blockPayload: TargetBlockPayload): void;
@@ -39,4 +49,9 @@ export declare class MetaService {
     handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void;
     handleApiConnected({ value }: EventPayload<number>): void;
     handleUsingDictionary({ value }: EventPayload<number>): void;
+    handleEnqueueBlocks(size: number): void;
+    handleFilteringBlocks(height: number): void;
+    handleFetchBlock(): void;
+    handleRpcCall(): void;
+    blockFilteringSpeed(): void;
 }

package/dist/meta/meta.service.js

@@ -20,15 +20,25 @@ const UPDATE_HEIGHT_INTERVAL = 60000;
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { version: polkadotSdkVersion } = require('@polkadot/api/package.json');
 const { version: packageVersion } = require('../../package.json');
+const logger = (0, node_core_1.getLogger)('profiler');
 let MetaService = class MetaService {
-    constructor(storeService) {
+    constructor(storeService, nodeConfig) {
         this.storeService = storeService;
+        this.nodeConfig = nodeConfig;
+        this.accEnqueueBlocks = 0;
+        this.accFetchBlocks = 0;
+        this.currentFilteringBlockNum = 0;
+        this.accRpcCalls = 0;
+        this.lastReportedFilteringBlockNum = 0;
+        this.lastReportedEnqueueBlocks = 0;
+        this.lastReportedFetchBlocks = 0;
+        this.lastReportedRpcCalls = 0;
     }
     getMeta() {
         return Object.assign({ currentProcessingHeight: this.currentProcessingHeight, currentProcessingTimestamp: this.currentProcessingTimestamp, targetHeight: this.targetHeight, bestHeight: this.bestHeight, indexerNodeVersion: packageVersion, lastProcessedHeight: this.lastProcessedHeight, lastProcessedTimestamp: this.lastProcessedTimestamp, uptime: process.uptime(), polkadotSdkVersion, processedBlockCount: this.processedBlockCount, apiConnected: this.apiConnected, injectedApiConnected: this.injectedApiConnected, usingDictionary: this.usingDictionary }, this.networkMeta);
     }
-    async getTargetHeight() {
-        await this.storeService.setMetadata('targetHeight', this.targetHeight);
+    getTargetHeight() {
+        this.storeService.storeCache.metadata.set('targetHeight', this.targetHeight);
     }
     handleProcessingBlock(blockPayload) {
         this.currentProcessingHeight = blockPayload.height;
@@ -53,12 +63,55 @@ let MetaService = class MetaService {
     handleUsingDictionary({ value }) {
         this.usingDictionary = !!value;
     }
+    handleEnqueueBlocks(size) {
+        this.accEnqueueBlocks += size;
+        if (!this.lastStatsReportedTs) {
+            this.lastStatsReportedTs = new Date();
+        }
+    }
+    handleFilteringBlocks(height) {
+        this.currentFilteringBlockNum = height;
+        if (!this.lastStatsReportedTs) {
+            this.lastReportedFilteringBlockNum = height;
+        }
+    }
+    handleFetchBlock() {
+        this.accFetchBlocks++;
+        if (!this.lastStatsReportedTs) {
+            this.lastStatsReportedTs = new Date();
+        }
+    }
+    handleRpcCall() {
+        this.accRpcCalls++;
+        if (!this.lastStatsReportedTs) {
+            this.lastStatsReportedTs = new Date();
+        }
+    }
+    blockFilteringSpeed() {
+        if (!this.nodeConfig.profiler) {
+            return;
+        }
+        const count = this.accEnqueueBlocks - this.lastReportedEnqueueBlocks;
+        this.lastReportedEnqueueBlocks = this.accEnqueueBlocks;
+        const filteringCount = this.currentFilteringBlockNum - this.lastReportedFilteringBlockNum;
+        const now = new Date();
+        const timepass = now.getTime() - this.lastStatsReportedTs.getTime();
+        this.lastStatsReportedTs = now;
+        this.lastReportedFilteringBlockNum = this.currentFilteringBlockNum;
+        const rpcCalls = this.accRpcCalls - this.lastReportedRpcCalls;
+        this.lastReportedRpcCalls = this.accRpcCalls;
+        const fetchCount = this.accFetchBlocks - this.lastReportedFetchBlocks;
+        this.lastReportedFetchBlocks = this.accFetchBlocks;
+        logger.info(`actual block filtering: ${(count / (timepass / 1000)).toFixed(2)}/sec, \
+seeming speed: ${(filteringCount / (timepass / 1000)).toFixed(2)}/sec, rpcCalls: ${(rpcCalls / (timepass / 1000)).toFixed(2)}/sec \
+fetch speed: ${(fetchCount / (timepass / 1000)).toFixed(2)}/sec`);
+    }
 };
 __decorate([
     (0, schedule_1.Interval)(UPDATE_HEIGHT_INTERVAL),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", []),
-    __metadata("design:returntype", Promise)
+    __metadata("design:returntype", void 0)
 ], MetaService.prototype, "getTargetHeight", null);
 __decorate([
     (0, event_emitter_1.OnEvent)(node_core_1.IndexerEvent.BlockProcessing),
@@ -102,9 +155,40 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", void 0)
 ], MetaService.prototype, "handleUsingDictionary", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('enqueueBlocks'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Number]),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleEnqueueBlocks", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('filteringBlocks'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Number]),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleFilteringBlocks", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('fetchBlock'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleFetchBlock", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('rpcCall'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleRpcCall", null);
+__decorate([
+    (0, schedule_1.Interval)(10000),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "blockFilteringSpeed", null);
 MetaService = __decorate([
     (0, common_1.Injectable)(),
-    __metadata("design:paramtypes", [node_core_1.StoreService])
+    __metadata("design:paramtypes", [node_core_1.StoreService,
+        node_core_1.NodeConfig])
 ], MetaService);
 exports.MetaService = MetaService;
 //# sourceMappingURL=meta.service.js.map

package/dist/meta/meta.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta.service.js","sourceRoot":"","sources":["../../src/meta/meta.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,yDAAgD;AAChD,+CAA4C;AAC5C,gDAS0B;AAE1B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC9E,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAG3D,IAAM,WAAW,GAAjB,MAAM,WAAW;IAatB,YAAoB,YAA0B;QAA1B,iBAAY,GAAZ,YAAY,CAAc;IAAG,CAAC;IAElD,OAAO;QACL,uBACE,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,EACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B,EAC3D,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,UAAU,EAAE,IAAI,CAAC,UAAU,EAC3B,kBAAkB,EAAE,cAAc,EAClC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,EACnD,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EACxB,kBAAkB,EAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAC/C,eAAe,EAAE,IAAI,CAAC,eAAe,IAClC,IAAI,CAAC,WAAW,EACnB;IACJ,CAAC;IAGK,AAAN,KAAK,CAAC,eAAe;QACnB,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACzE,CAAC;IAGD,qBAAqB,CAAC,YAAiC;QACrD,IAAI,CAAC,uBAAuB,GAAG,YAAY,CAAC,MAAM,CAAC;QACnD,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,oBAAoB,CAAC,YAAwC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,YAAY,CAAC,mBAAmB,CAAC;QAC5D,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,iBAAiB,CAAC,YAAgC;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,CAAC;IAGD,eAAe,CAAC,YAA8B;QAC5C,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACxC,CAAC;IAGD,qBAAqB,CAAC,WAAmC;QACvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAGD,kBAAkB,CAAC,EAAE,KAAK,EAAwB;QAChD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC;IAC9B,CAAC;IAGD,qBAAqB,CAAC,EAAE,KAAK,EAAwB;QACnD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;CACF,CAAA;AAxCO;IADL,IAAA,mBAAQ,EAAC,sBAAsB,CAAC;;;;kDAGhC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAIrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,mBAAmB,CAAC;;;;uDAIzC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,WAAW,CAAC;;;;oDAGjC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,SAAS,CAAC;;;;kDAG/B;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,YAAY,CAAC;;;;qDAGlC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AA1EU,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAcuB,wBAAY;GAbnC,WAAW,CA2EvB;AA3EY,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { OnEvent } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  BestBlockPayload,\n  EventPayload,\n  IndexerEvent,\n  NetworkMetadataPayload,\n  ProcessBlockPayload,\n  ProcessedBlockCountPayload,\n  TargetBlockPayload,\n  StoreService,\n} from '@subql/node-core';\n\nconst UPDATE_HEIGHT_INTERVAL = 60000;\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: polkadotSdkVersion } = require('@polkadot/api/package.json');\nconst { version: packageVersion } = require('../../package.json');\n\n@Injectable()\nexport class MetaService {\n  private currentProcessingHeight: number;\n  private currentProcessingTimestamp: number;\n  private bestHeight: number;\n  private targetHeight: number;\n  private networkMeta: NetworkMetadataPayload;\n  private apiConnected: boolean;\n  private usingDictionary: boolean;\n  private injectedApiConnected: boolean;\n  private lastProcessedHeight: number;\n  private lastProcessedTimestamp: number;\n  private processedBlockCount: number;\n\n  constructor(private storeService: StoreService) {}\n\n  getMeta() {\n    return {\n      currentProcessingHeight: this.currentProcessingHeight,\n      currentProcessingTimestamp: this.currentProcessingTimestamp,\n      targetHeight: this.targetHeight,\n      bestHeight: this.bestHeight,\n      indexerNodeVersion: packageVersion,\n      lastProcessedHeight: this.lastProcessedHeight,\n      lastProcessedTimestamp: this.lastProcessedTimestamp,\n      uptime: process.uptime(),\n      polkadotSdkVersion,\n      processedBlockCount: this.processedBlockCount,\n      apiConnected: this.apiConnected,\n      injectedApiConnected: this.injectedApiConnected,\n      usingDictionary: this.usingDictionary,\n      ...this.networkMeta,\n    };\n  }\n\n  @Interval(UPDATE_HEIGHT_INTERVAL)\n  async getTargetHeight(): Promise<void> {\n    await this.storeService.setMetadata('targetHeight', this.targetHeight);\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessing)\n  handleProcessingBlock(blockPayload: ProcessBlockPayload): void {\n    this.currentProcessingHeight = blockPayload.height;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessedCount)\n  handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void {\n    this.processedBlockCount = blockPayload.processedBlockCount;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockTarget)\n  handleTargetBlock(blockPayload: TargetBlockPayload): void {\n    this.targetHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.BlockBest)\n  handleBestBlock(blockPayload: BestBlockPayload): void {\n    this.bestHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.NetworkMetadata)\n  handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void {\n    this.networkMeta = networkMeta;\n  }\n\n  @OnEvent(IndexerEvent.ApiConnected)\n  handleApiConnected({ value }: EventPayload<number>): void {\n    this.apiConnected = !!value;\n  }\n\n  @OnEvent(IndexerEvent.UsingDictionary)\n  handleUsingDictionary({ value }: EventPayload<number>): void {\n    this.usingDictionary = !!value;\n  }\n}\n"]}
+{"version":3,"file":"meta.service.js","sourceRoot":"","sources":["../../src/meta/meta.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,yDAAgD;AAChD,+CAA4C;AAC5C,gDAW0B;AAE1B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC9E,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,UAAU,CAAC,CAAC;AAG9B,IAAM,WAAW,GAAjB,MAAM,WAAW;IAsBtB,YACU,YAA0B,EAC1B,UAAsB;QADtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QAZxB,qBAAgB,GAAG,CAAC,CAAC;QACrB,mBAAc,GAAG,CAAC,CAAC;QACnB,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gBAAW,GAAG,CAAC,CAAC;QAChB,kCAA6B,GAAG,CAAC,CAAC;QAClC,8BAAyB,GAAG,CAAC,CAAC;QAC9B,4BAAuB,GAAG,CAAC,CAAC;QAC5B,yBAAoB,GAAG,CAAC,CAAC;IAM9B,CAAC;IAEJ,OAAO;QACL,uBACE,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,EACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B,EAC3D,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,UAAU,EAAE,IAAI,CAAC,UAAU,EAC3B,kBAAkB,EAAE,cAAc,EAClC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,EACnD,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EACxB,kBAAkB,EAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAC/C,eAAe,EAAE,IAAI,CAAC,eAAe,IAClC,IAAI,CAAC,WAAW,EACnB;IACJ,CAAC;IAGD,eAAe;QACb,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACvC,cAAc,EACd,IAAI,CAAC,YAAY,CAClB,CAAC;IACJ,CAAC;IAGD,qBAAqB,CAAC,YAAiC;QACrD,IAAI,CAAC,uBAAuB,GAAG,YAAY,CAAC,MAAM,CAAC;QACnD,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,oBAAoB,CAAC,YAAwC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,YAAY,CAAC,mBAAmB,CAAC;QAC5D,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,iBAAiB,CAAC,YAAgC;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,CAAC;IAGD,eAAe,CAAC,YAA8B;QAC5C,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACxC,CAAC;IAGD,qBAAqB,CAAC,WAAmC;QACvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAGD,kBAAkB,CAAC,EAAE,KAAK,EAAwB;QAChD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC;IAC9B,CAAC;IAGD,qBAAqB,CAAC,EAAE,KAAK,EAAwB;QACnD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;IAGD,mBAAmB,CAAC,IAAY;QAC9B,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,qBAAqB,CAAC,MAAc;QAClC,IAAI,CAAC,wBAAwB,GAAG,MAAM,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,6BAA6B,GAAG,MAAM,CAAC;SAC7C;IACH,CAAC;IAGD,gBAAgB;QACd,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,aAAa;QACX,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,mBAAmB;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC7B,OAAO;SACR;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,yBAAyB,CAAC;QACrE,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvD,MAAM,cAAc,GAClB,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,6BAA6B,CAAC;QACrE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;QAC/B,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC,wBAAwB,CAAC;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAC9D,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,uBAAuB,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,cAAc,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,KAAK,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACxE,CAAC,CACF;iBACY,CAAC,cAAc,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACvD,CAAC,CACF,mBAAmB,CAAC,QAAQ,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;eAClD,CAAC,UAAU,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;CACF,CAAA;AAtGC;IAAC,IAAA,mBAAQ,EAAC,sBAAsB,CAAC;;;;kDAMhC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAIrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,mBAAmB,CAAC;;;;uDAIzC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,WAAW,CAAC;;;;oDAGjC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,SAAS,CAAC;;;;kDAG/B;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,YAAY,CAAC;;;;qDAGlC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,eAAe,CAAC;;;;sDAMxB;AAED;IAAC,IAAA,uBAAO,EAAC,iBAAiB,CAAC;;;;wDAM1B;AAED;IAAC,IAAA,uBAAO,EAAC,YAAY,CAAC;;;;mDAMrB;AAED;IAAC,IAAA,uBAAO,EAAC,SAAS,CAAC;;;;gDAMlB;AAED;IAAC,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sDAwBf;AAnJU,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAwBa,wBAAY;QACd,sBAAU;GAxBrB,WAAW,CAoJvB;AApJY,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { OnEvent } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  BestBlockPayload,\n  EventPayload,\n  IndexerEvent,\n  NetworkMetadataPayload,\n  ProcessBlockPayload,\n  ProcessedBlockCountPayload,\n  TargetBlockPayload,\n  StoreService,\n  getLogger,\n  NodeConfig,\n} from '@subql/node-core';\n\nconst UPDATE_HEIGHT_INTERVAL = 60000;\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: polkadotSdkVersion } = require('@polkadot/api/package.json');\nconst { version: packageVersion } = require('../../package.json');\nconst logger = getLogger('profiler');\n\n@Injectable()\nexport class MetaService {\n  private currentProcessingHeight: number;\n  private currentProcessingTimestamp: number;\n  private bestHeight: number;\n  private targetHeight: number;\n  private networkMeta: NetworkMetadataPayload;\n  private apiConnected: boolean;\n  private usingDictionary: boolean;\n  private injectedApiConnected: boolean;\n  private lastProcessedHeight: number;\n  private lastProcessedTimestamp: number;\n  private processedBlockCount: number;\n  private accEnqueueBlocks = 0;\n  private accFetchBlocks = 0;\n  private currentFilteringBlockNum = 0;\n  private accRpcCalls = 0;\n  private lastReportedFilteringBlockNum = 0;\n  private lastReportedEnqueueBlocks = 0;\n  private lastReportedFetchBlocks = 0;\n  private lastReportedRpcCalls = 0;\n  private lastStatsReportedTs: Date;\n\n  constructor(\n    private storeService: StoreService,\n    private nodeConfig: NodeConfig,\n  ) {}\n\n  getMeta() {\n    return {\n      currentProcessingHeight: this.currentProcessingHeight,\n      currentProcessingTimestamp: this.currentProcessingTimestamp,\n      targetHeight: this.targetHeight,\n      bestHeight: this.bestHeight,\n      indexerNodeVersion: packageVersion,\n      lastProcessedHeight: this.lastProcessedHeight,\n      lastProcessedTimestamp: this.lastProcessedTimestamp,\n      uptime: process.uptime(),\n      polkadotSdkVersion,\n      processedBlockCount: this.processedBlockCount,\n      apiConnected: this.apiConnected,\n      injectedApiConnected: this.injectedApiConnected,\n      usingDictionary: this.usingDictionary,\n      ...this.networkMeta,\n    };\n  }\n\n  @Interval(UPDATE_HEIGHT_INTERVAL)\n  getTargetHeight(): void {\n    this.storeService.storeCache.metadata.set(\n      'targetHeight',\n      this.targetHeight,\n    );\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessing)\n  handleProcessingBlock(blockPayload: ProcessBlockPayload): void {\n    this.currentProcessingHeight = blockPayload.height;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessedCount)\n  handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void {\n    this.processedBlockCount = blockPayload.processedBlockCount;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockTarget)\n  handleTargetBlock(blockPayload: TargetBlockPayload): void {\n    this.targetHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.BlockBest)\n  handleBestBlock(blockPayload: BestBlockPayload): void {\n    this.bestHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.NetworkMetadata)\n  handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void {\n    this.networkMeta = networkMeta;\n  }\n\n  @OnEvent(IndexerEvent.ApiConnected)\n  handleApiConnected({ value }: EventPayload<number>): void {\n    this.apiConnected = !!value;\n  }\n\n  @OnEvent(IndexerEvent.UsingDictionary)\n  handleUsingDictionary({ value }: EventPayload<number>): void {\n    this.usingDictionary = !!value;\n  }\n\n  @OnEvent('enqueueBlocks')\n  handleEnqueueBlocks(size: number): void {\n    this.accEnqueueBlocks += size;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('filteringBlocks')\n  handleFilteringBlocks(height: number): void {\n    this.currentFilteringBlockNum = height;\n    if (!this.lastStatsReportedTs) {\n      this.lastReportedFilteringBlockNum = height;\n    }\n  }\n\n  @OnEvent('fetchBlock')\n  handleFetchBlock(): void {\n    this.accFetchBlocks++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('rpcCall')\n  handleRpcCall(): void {\n    this.accRpcCalls++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @Interval(10000)\n  blockFilteringSpeed(): void {\n    if (!this.nodeConfig.profiler) {\n      return;\n    }\n    const count = this.accEnqueueBlocks - this.lastReportedEnqueueBlocks;\n    this.lastReportedEnqueueBlocks = this.accEnqueueBlocks;\n    const filteringCount =\n      this.currentFilteringBlockNum - this.lastReportedFilteringBlockNum;\n    const now = new Date();\n    const timepass = now.getTime() - this.lastStatsReportedTs.getTime();\n    this.lastStatsReportedTs = now;\n    this.lastReportedFilteringBlockNum = this.currentFilteringBlockNum;\n    const rpcCalls = this.accRpcCalls - this.lastReportedRpcCalls;\n    this.lastReportedRpcCalls = this.accRpcCalls;\n    const fetchCount = this.accFetchBlocks - this.lastReportedFetchBlocks;\n    this.lastReportedFetchBlocks = this.accFetchBlocks;\n    logger.info(`actual block filtering: ${(count / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, \\\nseeming speed: ${(filteringCount / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, rpcCalls: ${(rpcCalls / (timepass / 1000)).toFixed(2)}/sec \\\nfetch speed: ${(fetchCount / (timepass / 1000)).toFixed(2)}/sec`);\n  }\n}\n"]}

package/dist/subcommands/forceClean.service.js

@@ -44,12 +44,16 @@ let ForceCleanService = class ForceCleanService {
                 logging: false,
                 benchmark: false,
             });
-            // drop all related enums
+            // TODO, remove this soon, once original enum are cleaned
+            // Deprecate, now enums are moved under schema, drop schema will remove project enums
             await Promise.all(modelsRelation.enums.map(async (e) => {
-                const enumTypeName = `${schema}_enum_${(0, node_core_1.enumNameToHash)(e.name)}`;
-                await this.sequelize.query(`
-            DROP TYPE "${enumTypeName}";
+                const enumTypeNameDeprecated = `${schema}_enum_${(0, node_core_1.enumNameToHash)(e.name)}`;
+                const resultsDeprecated = await (0, node_core_1.getEnumDeprecated)(this.sequelize, enumTypeNameDeprecated);
+                if (resultsDeprecated.length !== 0) {
+                    await this.sequelize.query(`
+            DROP TYPE "${enumTypeNameDeprecated}";
           `);
+                }
             }));
             // remove schema from subquery table (might not exist)
             const checker = await this.sequelize.query(`

package/dist/subcommands/forceClean.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"forceClean.service.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,2CAAoD;AACpD,gDAK0B;AAC1B,wCAAuD;AACvD,yCAAkD;AAClD,kEAA+D;AAE/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AAGjC,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAC5B,YACmB,SAAoB,EACpB,UAAsB,EACD,OAAwB;QAF7C,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACD,YAAO,GAAP,OAAO,CAAiB;IAC7D,CAAC;IAEJ,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,IAAA,oCAAwB,EAC3C,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEpE,IAAI;YACF,kDAAkD;YAClD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,MAAM,GAAG,EAAE;gBAC7C,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,yBAAyB;YACzB,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,YAAY,GAAG,GAAG,MAAM,SAAS,IAAA,0BAAc,EAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;yBACZ,YAAY;WAC1B,CAAC,CAAC;YACL,CAAC,CAAC,CACH,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxC;mHAC2G,CAC5G,CAAC;YAEF,IAAK,OAAO,CAAC,CAAC,CAAS,CAAC,QAAQ,GAAG,CAAC,EAAE;gBACpC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxB;;qCAE2B,EAC3B;oBACE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;oBACpD,IAAI,EAAE,sBAAU,CAAC,MAAM;iBACxB,CACF,CAAC;aACH;YAED,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAE/C,IAAI,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC1C,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;aAC7C;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;SACX;IACH,CAAC;CACF,CAAA;AAhEY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFC,qBAAS;QACR,sBAAU;QACQ,iCAAe;GAJrD,iBAAiB,CAgE7B;AAhEY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  NodeConfig,\n  getExistingProjectSchema,\n  enumNameToHash,\n} from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport { QueryTypes, Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nconst logger = getLogger('Force-clean');\n\n@Injectable()\nexport class ForceCleanService {\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') protected project: SubqueryProject,\n  ) {}\n\n  async forceClean(): Promise<void> {\n    const schema = await getExistingProjectSchema(\n      this.nodeConfig,\n      this.sequelize,\n    );\n    if (!schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    const modelsRelation = getAllEntitiesRelations(this.project.schema);\n\n    try {\n      // drop existing project schema and metadata table\n      await this.sequelize.dropSchema(`\"${schema}\"`, {\n        logging: false,\n        benchmark: false,\n      });\n\n      // drop all related enums\n      await Promise.all(\n        modelsRelation.enums.map(async (e) => {\n          const enumTypeName = `${schema}_enum_${enumNameToHash(e.name)}`;\n          await this.sequelize.query(`\n            DROP TYPE \"${enumTypeName}\";\n          `);\n        }),\n      );\n\n      // remove schema from subquery table (might not exist)\n      const checker = await this.sequelize.query(\n        `\n              SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'public' AND  TABLE_NAME = 'subqueries'`,\n      );\n\n      if ((checker[1] as any).rowCount > 0) {\n        await this.sequelize.query(\n          ` DELETE\n                  FROM public.subqueries\n                  WHERE name = :name`,\n          {\n            replacements: { name: this.nodeConfig.subqueryName },\n            type: QueryTypes.DELETE,\n          },\n        );\n      }\n\n      logger.info('force cleaned schema and tables');\n\n      if (fs.existsSync(this.nodeConfig.mmrPath)) {\n        await fs.promises.unlink(this.nodeConfig.mmrPath);\n        logger.info('force cleaned file based mmr');\n      }\n    } catch (err) {\n      logger.error(err, 'failed to force clean');\n      throw err;\n    }\n  }\n}\n"]}
+{"version":3,"file":"forceClean.service.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,2CAAoD;AACpD,gDAM0B;AAC1B,wCAAuD;AACvD,yCAAkD;AAClD,kEAA+D;AAE/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AAGjC,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAC5B,YACmB,SAAoB,EACpB,UAAsB,EACD,OAAwB;QAF7C,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACD,YAAO,GAAP,OAAO,CAAiB;IAC7D,CAAC;IAEJ,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,IAAA,oCAAwB,EAC3C,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEpE,IAAI;YACF,kDAAkD;YAClD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,MAAM,GAAG,EAAE;gBAC7C,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,yDAAyD;YACzD,qFAAqF;YACrF,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,sBAAsB,GAAG,GAAG,MAAM,SAAS,IAAA,0BAAc,EAC7D,CAAC,CAAC,IAAI,CACP,EAAE,CAAC;gBACJ,MAAM,iBAAiB,GAAG,MAAM,IAAA,6BAAiB,EAC/C,IAAI,CAAC,SAAS,EACd,sBAAsB,CACvB,CAAC;gBACF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAClC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;yBACd,sBAAsB;WACpC,CAAC,CAAC;iBACF;YACH,CAAC,CAAC,CACH,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxC;mHAC2G,CAC5G,CAAC;YAEF,IAAK,OAAO,CAAC,CAAC,CAAS,CAAC,QAAQ,GAAG,CAAC,EAAE;gBACpC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxB;;qCAE2B,EAC3B;oBACE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;oBACpD,IAAI,EAAE,sBAAU,CAAC,MAAM;iBACxB,CACF,CAAC;aACH;YAED,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAE/C,IAAI,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC1C,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;aAC7C;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;SACX;IACH,CAAC;CACF,CAAA;AAzEY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFC,qBAAS;QACR,sBAAU;QACQ,iCAAe;GAJrD,iBAAiB,CAyE7B;AAzEY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  NodeConfig,\n  getExistingProjectSchema,\n  enumNameToHash,\n  getEnumDeprecated,\n} from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport { QueryTypes, Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nconst logger = getLogger('Force-clean');\n\n@Injectable()\nexport class ForceCleanService {\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') protected project: SubqueryProject,\n  ) {}\n\n  async forceClean(): Promise<void> {\n    const schema = await getExistingProjectSchema(\n      this.nodeConfig,\n      this.sequelize,\n    );\n    if (!schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    const modelsRelation = getAllEntitiesRelations(this.project.schema);\n\n    try {\n      // drop existing project schema and metadata table\n      await this.sequelize.dropSchema(`\"${schema}\"`, {\n        logging: false,\n        benchmark: false,\n      });\n\n      // TODO, remove this soon, once original enum are cleaned\n      // Deprecate, now enums are moved under schema, drop schema will remove project enums\n      await Promise.all(\n        modelsRelation.enums.map(async (e) => {\n          const enumTypeNameDeprecated = `${schema}_enum_${enumNameToHash(\n            e.name,\n          )}`;\n          const resultsDeprecated = await getEnumDeprecated(\n            this.sequelize,\n            enumTypeNameDeprecated,\n          );\n          if (resultsDeprecated.length !== 0) {\n            await this.sequelize.query(`\n            DROP TYPE \"${enumTypeNameDeprecated}\";\n          `);\n          }\n        }),\n      );\n\n      // remove schema from subquery table (might not exist)\n      const checker = await this.sequelize.query(\n        `\n              SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'public' AND  TABLE_NAME = 'subqueries'`,\n      );\n\n      if ((checker[1] as any).rowCount > 0) {\n        await this.sequelize.query(\n          ` DELETE\n                  FROM public.subqueries\n                  WHERE name = :name`,\n          {\n            replacements: { name: this.nodeConfig.subqueryName },\n            type: QueryTypes.DELETE,\n          },\n        );\n      }\n\n      logger.info('force cleaned schema and tables');\n\n      if (fs.existsSync(this.nodeConfig.mmrPath)) {\n        await fs.promises.unlink(this.nodeConfig.mmrPath);\n        logger.info('force cleaned file based mmr');\n      }\n    } catch (err) {\n      logger.error(err, 'failed to force clean');\n      throw err;\n    }\n  }\n}\n"]}

package/dist/subcommands/reindex.init.js

@@ -14,7 +14,11 @@ async function reindexInit(targetHeight) {
         await app.init();
         const reindexService = app.get(reindex_service_1.ReindexService);
         await reindexService.init();
-        await reindexService.reindex(targetHeight);
+        const actualReindexHeight = await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight);
+        if (actualReindexHeight !== targetHeight) {
+            logger.info(`Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindexHeight}, will index to ${actualReindexHeight}`);
+        }
+        await reindexService.reindex(actualReindexHeight);
     }
     catch (e) {
         logger.error(e, 'Reindex failed to execute');

package/dist/subcommands/reindex.init.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.init.js","sourceRoot":"","sources":["../../src/subcommands/reindex.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAA6C;AAC7C,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AACjC,KAAK,UAAU,WAAW,CAAC,YAAoB;IACpD,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAE/C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;KAC5C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAdD,kCAcC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger } from '@subql/node-core';\nimport { ReindexModule } from './reindex.module';\nimport { ReindexService } from './reindex.service';\n\nconst logger = getLogger('CLI-Reindex');\nexport async function reindexInit(targetHeight: number): Promise<void> {\n  try {\n    const app = await NestFactory.create(ReindexModule);\n\n    await app.init();\n    const reindexService = app.get(ReindexService);\n\n    await reindexService.init();\n    await reindexService.reindex(targetHeight);\n  } catch (e) {\n    logger.error(e, 'Reindex failed to execute');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}
+{"version":3,"file":"reindex.init.js","sourceRoot":"","sources":["../../src/subcommands/reindex.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAA6C;AAC7C,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AACjC,KAAK,UAAU,WAAW,CAAC,YAAoB;IACpD,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAE/C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,mBAAmB,GACvB,MAAM,cAAc,CAAC,oCAAoC,CAAC,YAAY,CAAC,CAAC;QAC1E,IAAI,mBAAmB,KAAK,YAAY,EAAE;YACxC,MAAM,CAAC,IAAI,CACT,6BAA6B,YAAY,qCAAqC,mBAAmB,mBAAmB,mBAAmB,EAAE,CAC1I,CAAC;SACH;QACD,MAAM,cAAc,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;KACnD;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AArBD,kCAqBC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger } from '@subql/node-core';\nimport { ReindexModule } from './reindex.module';\nimport { ReindexService } from './reindex.service';\n\nconst logger = getLogger('CLI-Reindex');\nexport async function reindexInit(targetHeight: number): Promise<void> {\n  try {\n    const app = await NestFactory.create(ReindexModule);\n\n    await app.init();\n    const reindexService = app.get(ReindexService);\n\n    await reindexService.init();\n    const actualReindexHeight =\n      await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight);\n    if (actualReindexHeight !== targetHeight) {\n      logger.info(\n        `Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindexHeight}, will index to ${actualReindexHeight}`,\n      );\n    }\n    await reindexService.reindex(actualReindexHeight);\n  } catch (e) {\n    logger.error(e, 'Reindex failed to execute');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}