@subql/node-ethereum 0.4.1-9 → 0.4.1-storeCache-0.0.4

package/dist/init.js

@@ -8,7 +8,6 @@ const common_1 = require("@subql/common");
 const node_core_1 = require("@subql/node-core");
 const app_module_1 = require("./app.module");
 const fetch_service_1 = require("./indexer/fetch.service");
-const project_service_1 = require("./indexer/project.service");
 const yargs_1 = require("./yargs");
 const pjson = require('../package.json');
 const { argv } = yargs_1.yargsOptions;
@@ -35,13 +34,14 @@ async function bootstrap() {
             logger: debug ? new node_core_1.NestLogger() : false,
         });
         await app.init();
-        const projectService = app.get(project_service_1.ProjectService);
+        const projectService = app.get('IProjectService');
         const fetchService = app.get(fetch_service_1.FetchService);
         const apiService = app.get(node_core_1.ApiService);
         // Initialise async services, we do this here rather than in factories, so we can capture one off events
         await apiService.init();
         await projectService.init();
         await fetchService.init(projectService.startHeight);
+        app.enableShutdownHooks();
         await app.listen(port);
         logger.info(`Node started on port: ${port}`);
     }

package/dist/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,0CAAkD;AAClD,gDAAqE;AACrE,6CAAyC;AACzC,2DAAuD;AACvD,+DAA2D;AAC3D,mCAAuC;AACvC,MAAM,KAAK,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEhC,KAAK,UAAU,SAAS;;IAC7B,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,eAAe,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAEzB,MAAM,QAAQ,GAAG,CAAC,CAAM,EAAE,EAAE;QAC1B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAA,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mCAAI,CAAC,MAAM,IAAA,0BAAiB,EAAC,YAAY,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,CAAC,KAAK,CACV,wDAAwD,IAAI,KAC1D,IAAI,GAAG,EACT,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,IAAI,CAAC,MAAM,EAAE;QACf,MAAM,CAAC,IAAI,CACT,mHAAmH,CACpH,CAAC;KACH;IAED,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,sBAAS,EAAE;YAC9C,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,sBAAU,EAAE,CAAC,CAAC,CAAC,KAAK;SACzC,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,4BAAY,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAU,CAAC,CAAC;QAEvC,wGAAwG;QACxG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;KAC9C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC;AA/CD,8BA+CC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { findAvailablePort } from '@subql/common';\nimport { getLogger, ApiService, NestLogger } from '@subql/node-core';\nimport { AppModule } from './app.module';\nimport { FetchService } from './indexer/fetch.service';\nimport { ProjectService } from './indexer/project.service';\nimport { yargsOptions } from './yargs';\nconst pjson = require('../package.json');\n\nconst { argv } = yargsOptions;\n\nconst DEFAULT_PORT = 3000;\nconst logger = getLogger('subql-node');\n\nexport async function bootstrap() {\n  logger.info(`Current ${pjson.name} version is ${pjson.version}`);\n  const debug = argv.debug;\n\n  const validate = (x: any) => {\n    const p = parseInt(x);\n    return isNaN(p) ? null : p;\n  };\n\n  const port = validate(argv.port) ?? (await findAvailablePort(DEFAULT_PORT));\n  if (!port) {\n    logger.error(\n      `Unable to find available port (tried ports in range (${port}..${\n        port + 10\n      })). Try setting a free port manually by setting the --port flag`,\n    );\n    process.exit(1);\n  }\n\n  if (argv.unsafe) {\n    logger.warn(\n      'UNSAFE MODE IS ENABLED. This is not recommended for most projects and will not be supported by our hosted service',\n    );\n  }\n\n  try {\n    const app = await NestFactory.create(AppModule, {\n      logger: debug ? new NestLogger() : false,\n    });\n    await app.init();\n\n    const projectService = app.get(ProjectService);\n    const fetchService = app.get(FetchService);\n    const apiService = app.get(ApiService);\n\n    // Initialise async services, we do this here rather than in factories, so we can capture one off events\n    await apiService.init();\n    await projectService.init();\n    await fetchService.init(projectService.startHeight);\n\n    await app.listen(port);\n\n    logger.info(`Node started on port: ${port}`);\n  } catch (e) {\n    logger.error(e, 'Node failed to start');\n    process.exit(1);\n  }\n}\n"]}
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,0CAAkD;AAClD,gDAAqE;AACrE,6CAAyC;AACzC,2DAAuD;AAEvD,mCAAuC;AACvC,MAAM,KAAK,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEhC,KAAK,UAAU,SAAS;;IAC7B,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,eAAe,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAEzB,MAAM,QAAQ,GAAG,CAAC,CAAM,EAAE,EAAE;QAC1B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAA,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mCAAI,CAAC,MAAM,IAAA,0BAAiB,EAAC,YAAY,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,CAAC,KAAK,CACV,wDAAwD,IAAI,KAC1D,IAAI,GAAG,EACT,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,IAAI,CAAC,MAAM,EAAE;QACf,MAAM,CAAC,IAAI,CACT,mHAAmH,CACpH,CAAC;KACH;IAED,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,sBAAS,EAAE;YAC9C,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,sBAAU,EAAE,CAAC,CAAC,CAAC,KAAK;SACzC,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAmB,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAClE,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,4BAAY,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAU,CAAC,CAAC;QAEvC,wGAAwG;QACxG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEpD,GAAG,CAAC,mBAAmB,EAAE,CAAC;QAE1B,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;KAC9C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;AACH,CAAC;AAjDD,8BAiDC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { findAvailablePort } from '@subql/common';\nimport { getLogger, ApiService, NestLogger } from '@subql/node-core';\nimport { AppModule } from './app.module';\nimport { FetchService } from './indexer/fetch.service';\nimport { ProjectService } from './indexer/project.service';\nimport { yargsOptions } from './yargs';\nconst pjson = require('../package.json');\n\nconst { argv } = yargsOptions;\n\nconst DEFAULT_PORT = 3000;\nconst logger = getLogger('subql-node');\n\nexport async function bootstrap(): Promise<void> {\n  logger.info(`Current ${pjson.name} version is ${pjson.version}`);\n  const debug = argv.debug;\n\n  const validate = (x: any) => {\n    const p = parseInt(x);\n    return isNaN(p) ? null : p;\n  };\n\n  const port = validate(argv.port) ?? (await findAvailablePort(DEFAULT_PORT));\n  if (!port) {\n    logger.error(\n      `Unable to find available port (tried ports in range (${port}..${\n        port + 10\n      })). Try setting a free port manually by setting the --port flag`,\n    );\n    process.exit(1);\n  }\n\n  if (argv.unsafe) {\n    logger.warn(\n      'UNSAFE MODE IS ENABLED. This is not recommended for most projects and will not be supported by our hosted service',\n    );\n  }\n\n  try {\n    const app = await NestFactory.create(AppModule, {\n      logger: debug ? new NestLogger() : false,\n    });\n    await app.init();\n\n    const projectService: ProjectService = app.get('IProjectService');\n    const fetchService = app.get(FetchService);\n    const apiService = app.get(ApiService);\n\n    // Initialise async services, we do this here rather than in factories, so we can capture one off events\n    await apiService.init();\n    await projectService.init();\n    await fetchService.init(projectService.startHeight);\n\n    app.enableShutdownHooks();\n\n    await app.listen(port);\n\n    logger.info(`Node started on port: ${port}`);\n  } catch (e) {\n    logger.error(e, 'Node failed to start');\n    process.exit(1);\n  }\n}\n"]}

package/dist/meta/meta.module.js

@@ -67,6 +67,14 @@ MetaModule = __decorate([
                 name: 'subql_indexer_processed_block_count',
                 help: 'The number of processed block',
             }),
+            (0, nestjs_prometheus_1.makeGaugeProvider)({
+                name: 'subql_indexer_store_cache_threshold',
+                help: 'Store cache will flush once cache record size excess this threshold',
+            }),
+            (0, nestjs_prometheus_1.makeGaugeProvider)({
+                name: 'subql_indexer_store_cache_records_size',
+                help: 'Number of records waiting to flush in store cache',
+            }),
             meta_service_1.MetaService,
             node_core_1.HealthService,
             node_core_1.ReadyService,

package/dist/meta/meta.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta.module.js","sourceRoot":"","sources":["../../src/meta/meta.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,mEAGqC;AACrC,0DAAsD;AACtD,uDAAmD;AACnD,iDAA6C;AAwDtC,IAAM,UAAU,GAAhB,MAAM,UAAU;CAAG,CAAA;AAAb,UAAU;IAtDtB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,oCAAgB,CAAC,QAAQ,EAAE,EAAE,0BAAW,CAAC;QACnD,WAAW,EAAE,CAAC,gCAAc,EAAE,4BAAgB,EAAE,2BAAe,CAAC;QAChE,SAAS,EAAE;YACT,+BAAmB;YACnB,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,6BAA6B;gBACnC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,4CAA4C;aACnD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,uCAAuC;gBAC7C,IAAI,EAAE,qCAAqC;aAC5C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,mCAAmC;gBACzC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,iCAAiC;gBACvC,IAAI,EAAE,8BAA8B;aACrC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,wCAAwC;aAC/C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,+CAA+C;aACtD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,sDAAsD;aAC7D,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,+BAA+B;aACtC,CAAC;YACF,0BAAW;YACX,yBAAa;YACb,wBAAY;SACb;KACF,CAAC;GACW,UAAU,CAAG;AAAb,gCAAU","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  MetricEventListener,\n  ReadyController,\n  ReadyService,\n  HealthController,\n  HealthService,\n} from '@subql/node-core';\nimport {\n  makeGaugeProvider,\n  PrometheusModule,\n} from '@willsoto/nestjs-prometheus';\nimport { FetchModule } from '../indexer/fetch.module';\nimport { MetaController } from './meta.controller';\nimport { MetaService } from './meta.service';\n\n@Module({\n  imports: [PrometheusModule.register(), FetchModule],\n  controllers: [MetaController, HealthController, ReadyController],\n  providers: [\n    MetricEventListener,\n    makeGaugeProvider({\n      name: 'subql_indexer_api_connected',\n      help: 'The indexer api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_injected_api_connected',\n      help: 'The indexer injected api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processing_block_height',\n      help: 'The current processing block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_height',\n      help: 'The last processed block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_target_block_height',\n      help: 'The latest finalized block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_best_block_height',\n      help: 'The latest best block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_block_queue_size',\n      help: 'The size of fetched block queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_blocknumber_queue_size',\n      help: 'The size of fetched block number queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_using_dictionary',\n      help: 'The status of indexer is using the dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_skip_dictionary_count',\n      help: 'The number of times indexer been skip use dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_count',\n      help: 'The number of processed block',\n    }),\n    MetaService,\n    HealthService,\n    ReadyService,\n  ],\n})\nexport class MetaModule {}\n"]}
+{"version":3,"file":"meta.module.js","sourceRoot":"","sources":["../../src/meta/meta.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,mEAGqC;AACrC,0DAAsD;AACtD,uDAAmD;AACnD,iDAA6C;AAgEtC,IAAM,UAAU,GAAhB,MAAM,UAAU;CAAG,CAAA;AAAb,UAAU;IA9DtB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,oCAAgB,CAAC,QAAQ,EAAE,EAAE,0BAAW,CAAC;QACnD,WAAW,EAAE,CAAC,gCAAc,EAAE,4BAAgB,EAAE,2BAAe,CAAC;QAChE,SAAS,EAAE;YACT,+BAAmB;YACnB,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,6BAA6B;gBACnC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,4CAA4C;aACnD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,uCAAuC;gBAC7C,IAAI,EAAE,qCAAqC;aAC5C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,mCAAmC;gBACzC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,iCAAiC;gBACvC,IAAI,EAAE,8BAA8B;aACrC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,wCAAwC;aAC/C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,+CAA+C;aACtD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,sDAAsD;aAC7D,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,+BAA+B;aACtC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,qEAAqE;aAC5E,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,wCAAwC;gBAC9C,IAAI,EAAE,mDAAmD;aAC1D,CAAC;YACF,0BAAW;YACX,yBAAa;YACb,wBAAY;SACb;KACF,CAAC;GACW,UAAU,CAAG;AAAb,gCAAU","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  MetricEventListener,\n  ReadyController,\n  ReadyService,\n  HealthController,\n  HealthService,\n} from '@subql/node-core';\nimport {\n  makeGaugeProvider,\n  PrometheusModule,\n} from '@willsoto/nestjs-prometheus';\nimport { FetchModule } from '../indexer/fetch.module';\nimport { MetaController } from './meta.controller';\nimport { MetaService } from './meta.service';\n\n@Module({\n  imports: [PrometheusModule.register(), FetchModule],\n  controllers: [MetaController, HealthController, ReadyController],\n  providers: [\n    MetricEventListener,\n    makeGaugeProvider({\n      name: 'subql_indexer_api_connected',\n      help: 'The indexer api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_injected_api_connected',\n      help: 'The indexer injected api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processing_block_height',\n      help: 'The current processing block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_height',\n      help: 'The last processed block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_target_block_height',\n      help: 'The latest finalized block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_best_block_height',\n      help: 'The latest best block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_block_queue_size',\n      help: 'The size of fetched block queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_blocknumber_queue_size',\n      help: 'The size of fetched block number queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_using_dictionary',\n      help: 'The status of indexer is using the dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_skip_dictionary_count',\n      help: 'The number of times indexer been skip use dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_count',\n      help: 'The number of processed block',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_threshold',\n      help: 'Store cache will flush once cache record size excess this threshold',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_records_size',\n      help: 'Number of records waiting to flush in store cache',\n    }),\n    MetaService,\n    HealthService,\n    ReadyService,\n  ],\n})\nexport class MetaModule {}\n"]}

package/dist/meta/meta.service.d.ts

@@ -1,6 +1,7 @@
-import { BestBlockPayload, EventPayload, NetworkMetadataPayload, ProcessBlockPayload, ProcessedBlockCountPayload, TargetBlockPayload, StoreService } from '@subql/node-core';
+import { BestBlockPayload, EventPayload, NetworkMetadataPayload, ProcessBlockPayload, ProcessedBlockCountPayload, TargetBlockPayload, StoreService, NodeConfig } from '@subql/node-core';
 export declare class MetaService {
     private storeService;
+    private nodeConfig;
     private currentProcessingHeight;
     private currentProcessingTimestamp;
     private bestHeight;
@@ -12,7 +13,16 @@ export declare class MetaService {
     private lastProcessedHeight;
     private lastProcessedTimestamp;
     private processedBlockCount;
-    constructor(storeService: StoreService);
+    private accEnqueueBlocks;
+    private accFetchBlocks;
+    private currentFilteringBlockNum;
+    private accRpcCalls;
+    private lastReportedFilteringBlockNum;
+    private lastReportedEnqueueBlocks;
+    private lastReportedFetchBlocks;
+    private lastReportedRpcCalls;
+    private lastStatsReportedTs;
+    constructor(storeService: StoreService, nodeConfig: NodeConfig);
     getMeta(): {
         chain: string;
         specName: string;
@@ -31,7 +41,7 @@ export declare class MetaService {
         injectedApiConnected: boolean;
         usingDictionary: boolean;
     };
-    getTargetHeight(): Promise<void>;
+    getTargetHeight(): void;
     handleProcessingBlock(blockPayload: ProcessBlockPayload): void;
     handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void;
     handleTargetBlock(blockPayload: TargetBlockPayload): void;
@@ -39,4 +49,9 @@ export declare class MetaService {
     handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void;
     handleApiConnected({ value }: EventPayload<number>): void;
     handleUsingDictionary({ value }: EventPayload<number>): void;
+    handleEnqueueBlocks(size: number): void;
+    handleFilteringBlocks(height: number): void;
+    handleFetchBlock(): void;
+    handleRpcCall(): void;
+    blockFilteringSpeed(): void;
 }

package/dist/meta/meta.service.js

@@ -20,15 +20,25 @@ const UPDATE_HEIGHT_INTERVAL = 60000;
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { version: polkadotSdkVersion } = require('@polkadot/api/package.json');
 const { version: packageVersion } = require('../../package.json');
+const logger = (0, node_core_1.getLogger)('profiler');
 let MetaService = class MetaService {
-    constructor(storeService) {
+    constructor(storeService, nodeConfig) {
         this.storeService = storeService;
+        this.nodeConfig = nodeConfig;
+        this.accEnqueueBlocks = 0;
+        this.accFetchBlocks = 0;
+        this.currentFilteringBlockNum = 0;
+        this.accRpcCalls = 0;
+        this.lastReportedFilteringBlockNum = 0;
+        this.lastReportedEnqueueBlocks = 0;
+        this.lastReportedFetchBlocks = 0;
+        this.lastReportedRpcCalls = 0;
     }
     getMeta() {
         return Object.assign({ currentProcessingHeight: this.currentProcessingHeight, currentProcessingTimestamp: this.currentProcessingTimestamp, targetHeight: this.targetHeight, bestHeight: this.bestHeight, indexerNodeVersion: packageVersion, lastProcessedHeight: this.lastProcessedHeight, lastProcessedTimestamp: this.lastProcessedTimestamp, uptime: process.uptime(), polkadotSdkVersion, processedBlockCount: this.processedBlockCount, apiConnected: this.apiConnected, injectedApiConnected: this.injectedApiConnected, usingDictionary: this.usingDictionary }, this.networkMeta);
     }
-    async getTargetHeight() {
-        await this.storeService.setMetadata('targetHeight', this.targetHeight);
+    getTargetHeight() {
+        this.storeService.storeCache.metadata.set('targetHeight', this.targetHeight);
     }
     handleProcessingBlock(blockPayload) {
         this.currentProcessingHeight = blockPayload.height;
@@ -53,12 +63,55 @@ let MetaService = class MetaService {
     handleUsingDictionary({ value }) {
         this.usingDictionary = !!value;
     }
+    handleEnqueueBlocks(size) {
+        this.accEnqueueBlocks += size;
+        if (!this.lastStatsReportedTs) {
+            this.lastStatsReportedTs = new Date();
+        }
+    }
+    handleFilteringBlocks(height) {
+        this.currentFilteringBlockNum = height;
+        if (!this.lastStatsReportedTs) {
+            this.lastReportedFilteringBlockNum = height;
+        }
+    }
+    handleFetchBlock() {
+        this.accFetchBlocks++;
+        if (!this.lastStatsReportedTs) {
+            this.lastStatsReportedTs = new Date();
+        }
+    }
+    handleRpcCall() {
+        this.accRpcCalls++;
+        if (!this.lastStatsReportedTs) {
+            this.lastStatsReportedTs = new Date();
+        }
+    }
+    blockFilteringSpeed() {
+        if (!this.nodeConfig.profiler) {
+            return;
+        }
+        const count = this.accEnqueueBlocks - this.lastReportedEnqueueBlocks;
+        this.lastReportedEnqueueBlocks = this.accEnqueueBlocks;
+        const filteringCount = this.currentFilteringBlockNum - this.lastReportedFilteringBlockNum;
+        const now = new Date();
+        const timepass = now.getTime() - this.lastStatsReportedTs.getTime();
+        this.lastStatsReportedTs = now;
+        this.lastReportedFilteringBlockNum = this.currentFilteringBlockNum;
+        const rpcCalls = this.accRpcCalls - this.lastReportedRpcCalls;
+        this.lastReportedRpcCalls = this.accRpcCalls;
+        const fetchCount = this.accFetchBlocks - this.lastReportedFetchBlocks;
+        this.lastReportedFetchBlocks = this.accFetchBlocks;
+        logger.info(`actual block filtering: ${(count / (timepass / 1000)).toFixed(2)}/sec, \
+seeming speed: ${(filteringCount / (timepass / 1000)).toFixed(2)}/sec, rpcCalls: ${(rpcCalls / (timepass / 1000)).toFixed(2)}/sec \
+fetch speed: ${(fetchCount / (timepass / 1000)).toFixed(2)}/sec`);
+    }
 };
 __decorate([
     (0, schedule_1.Interval)(UPDATE_HEIGHT_INTERVAL),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", []),
-    __metadata("design:returntype", Promise)
+    __metadata("design:returntype", void 0)
 ], MetaService.prototype, "getTargetHeight", null);
 __decorate([
     (0, event_emitter_1.OnEvent)(node_core_1.IndexerEvent.BlockProcessing),
@@ -102,9 +155,40 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", void 0)
 ], MetaService.prototype, "handleUsingDictionary", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('enqueueBlocks'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Number]),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleEnqueueBlocks", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('filteringBlocks'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Number]),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleFilteringBlocks", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('fetchBlock'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleFetchBlock", null);
+__decorate([
+    (0, event_emitter_1.OnEvent)('rpcCall'),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "handleRpcCall", null);
+__decorate([
+    (0, schedule_1.Interval)(10000),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", void 0)
+], MetaService.prototype, "blockFilteringSpeed", null);
 MetaService = __decorate([
     (0, common_1.Injectable)(),
-    __metadata("design:paramtypes", [node_core_1.StoreService])
+    __metadata("design:paramtypes", [node_core_1.StoreService,
+        node_core_1.NodeConfig])
 ], MetaService);
 exports.MetaService = MetaService;
 //# sourceMappingURL=meta.service.js.map

package/dist/meta/meta.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta.service.js","sourceRoot":"","sources":["../../src/meta/meta.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,yDAAgD;AAChD,+CAA4C;AAC5C,gDAS0B;AAE1B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC9E,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAG3D,IAAM,WAAW,GAAjB,MAAM,WAAW;IAatB,YAAoB,YAA0B;QAA1B,iBAAY,GAAZ,YAAY,CAAc;IAAG,CAAC;IAElD,OAAO;QACL,uBACE,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,EACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B,EAC3D,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,UAAU,EAAE,IAAI,CAAC,UAAU,EAC3B,kBAAkB,EAAE,cAAc,EAClC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,EACnD,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EACxB,kBAAkB,EAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAC/C,eAAe,EAAE,IAAI,CAAC,eAAe,IAClC,IAAI,CAAC,WAAW,EACnB;IACJ,CAAC;IAGK,AAAN,KAAK,CAAC,eAAe;QACnB,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACzE,CAAC;IAGD,qBAAqB,CAAC,YAAiC;QACrD,IAAI,CAAC,uBAAuB,GAAG,YAAY,CAAC,MAAM,CAAC;QACnD,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,oBAAoB,CAAC,YAAwC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,YAAY,CAAC,mBAAmB,CAAC;QAC5D,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,iBAAiB,CAAC,YAAgC;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,CAAC;IAGD,eAAe,CAAC,YAA8B;QAC5C,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACxC,CAAC;IAGD,qBAAqB,CAAC,WAAmC;QACvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAGD,kBAAkB,CAAC,EAAE,KAAK,EAAwB;QAChD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC;IAC9B,CAAC;IAGD,qBAAqB,CAAC,EAAE,KAAK,EAAwB;QACnD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;CACF,CAAA;AAxCO;IADL,IAAA,mBAAQ,EAAC,sBAAsB,CAAC;;;;kDAGhC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAIrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,mBAAmB,CAAC;;;;uDAIzC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,WAAW,CAAC;;;;oDAGjC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,SAAS,CAAC;;;;kDAG/B;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,YAAY,CAAC;;;;qDAGlC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AA1EU,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAcuB,wBAAY;GAbnC,WAAW,CA2EvB;AA3EY,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { OnEvent } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  BestBlockPayload,\n  EventPayload,\n  IndexerEvent,\n  NetworkMetadataPayload,\n  ProcessBlockPayload,\n  ProcessedBlockCountPayload,\n  TargetBlockPayload,\n  StoreService,\n} from '@subql/node-core';\n\nconst UPDATE_HEIGHT_INTERVAL = 60000;\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: polkadotSdkVersion } = require('@polkadot/api/package.json');\nconst { version: packageVersion } = require('../../package.json');\n\n@Injectable()\nexport class MetaService {\n  private currentProcessingHeight: number;\n  private currentProcessingTimestamp: number;\n  private bestHeight: number;\n  private targetHeight: number;\n  private networkMeta: NetworkMetadataPayload;\n  private apiConnected: boolean;\n  private usingDictionary: boolean;\n  private injectedApiConnected: boolean;\n  private lastProcessedHeight: number;\n  private lastProcessedTimestamp: number;\n  private processedBlockCount: number;\n\n  constructor(private storeService: StoreService) {}\n\n  getMeta() {\n    return {\n      currentProcessingHeight: this.currentProcessingHeight,\n      currentProcessingTimestamp: this.currentProcessingTimestamp,\n      targetHeight: this.targetHeight,\n      bestHeight: this.bestHeight,\n      indexerNodeVersion: packageVersion,\n      lastProcessedHeight: this.lastProcessedHeight,\n      lastProcessedTimestamp: this.lastProcessedTimestamp,\n      uptime: process.uptime(),\n      polkadotSdkVersion,\n      processedBlockCount: this.processedBlockCount,\n      apiConnected: this.apiConnected,\n      injectedApiConnected: this.injectedApiConnected,\n      usingDictionary: this.usingDictionary,\n      ...this.networkMeta,\n    };\n  }\n\n  @Interval(UPDATE_HEIGHT_INTERVAL)\n  async getTargetHeight(): Promise<void> {\n    await this.storeService.setMetadata('targetHeight', this.targetHeight);\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessing)\n  handleProcessingBlock(blockPayload: ProcessBlockPayload): void {\n    this.currentProcessingHeight = blockPayload.height;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessedCount)\n  handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void {\n    this.processedBlockCount = blockPayload.processedBlockCount;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockTarget)\n  handleTargetBlock(blockPayload: TargetBlockPayload): void {\n    this.targetHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.BlockBest)\n  handleBestBlock(blockPayload: BestBlockPayload): void {\n    this.bestHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.NetworkMetadata)\n  handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void {\n    this.networkMeta = networkMeta;\n  }\n\n  @OnEvent(IndexerEvent.ApiConnected)\n  handleApiConnected({ value }: EventPayload<number>): void {\n    this.apiConnected = !!value;\n  }\n\n  @OnEvent(IndexerEvent.UsingDictionary)\n  handleUsingDictionary({ value }: EventPayload<number>): void {\n    this.usingDictionary = !!value;\n  }\n}\n"]}
+{"version":3,"file":"meta.service.js","sourceRoot":"","sources":["../../src/meta/meta.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,yDAAgD;AAChD,+CAA4C;AAC5C,gDAW0B;AAE1B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC9E,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,UAAU,CAAC,CAAC;AAG9B,IAAM,WAAW,GAAjB,MAAM,WAAW;IAsBtB,YACU,YAA0B,EAC1B,UAAsB;QADtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QAZxB,qBAAgB,GAAG,CAAC,CAAC;QACrB,mBAAc,GAAG,CAAC,CAAC;QACnB,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gBAAW,GAAG,CAAC,CAAC;QAChB,kCAA6B,GAAG,CAAC,CAAC;QAClC,8BAAyB,GAAG,CAAC,CAAC;QAC9B,4BAAuB,GAAG,CAAC,CAAC;QAC5B,yBAAoB,GAAG,CAAC,CAAC;IAM9B,CAAC;IAEJ,OAAO;QACL,uBACE,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,EACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B,EAC3D,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,UAAU,EAAE,IAAI,CAAC,UAAU,EAC3B,kBAAkB,EAAE,cAAc,EAClC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,EACnD,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EACxB,kBAAkB,EAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAC/C,eAAe,EAAE,IAAI,CAAC,eAAe,IAClC,IAAI,CAAC,WAAW,EACnB;IACJ,CAAC;IAGD,eAAe;QACb,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACvC,cAAc,EACd,IAAI,CAAC,YAAY,CAClB,CAAC;IACJ,CAAC;IAGD,qBAAqB,CAAC,YAAiC;QACrD,IAAI,CAAC,uBAAuB,GAAG,YAAY,CAAC,MAAM,CAAC;QACnD,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,oBAAoB,CAAC,YAAwC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,YAAY,CAAC,mBAAmB,CAAC;QAC5D,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,iBAAiB,CAAC,YAAgC;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,CAAC;IAGD,eAAe,CAAC,YAA8B;QAC5C,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACxC,CAAC;IAGD,qBAAqB,CAAC,WAAmC;QACvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAGD,kBAAkB,CAAC,EAAE,KAAK,EAAwB;QAChD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC;IAC9B,CAAC;IAGD,qBAAqB,CAAC,EAAE,KAAK,EAAwB;QACnD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;IAGD,mBAAmB,CAAC,IAAY;QAC9B,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,qBAAqB,CAAC,MAAc;QAClC,IAAI,CAAC,wBAAwB,GAAG,MAAM,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,6BAA6B,GAAG,MAAM,CAAC;SAC7C;IACH,CAAC;IAGD,gBAAgB;QACd,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,aAAa;QACX,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,mBAAmB;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC7B,OAAO;SACR;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,yBAAyB,CAAC;QACrE,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvD,MAAM,cAAc,GAClB,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,6BAA6B,CAAC;QACrE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;QAC/B,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC,wBAAwB,CAAC;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAC9D,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,uBAAuB,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,cAAc,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,KAAK,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACxE,CAAC,CACF;iBACY,CAAC,cAAc,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACvD,CAAC,CACF,mBAAmB,CAAC,QAAQ,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;eAClD,CAAC,UAAU,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;CACF,CAAA;AAtGC;IAAC,IAAA,mBAAQ,EAAC,sBAAsB,CAAC;;;;kDAMhC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAIrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,mBAAmB,CAAC;;;;uDAIzC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,WAAW,CAAC;;;;oDAGjC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,SAAS,CAAC;;;;kDAG/B;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,YAAY,CAAC;;;;qDAGlC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,eAAe,CAAC;;;;sDAMxB;AAED;IAAC,IAAA,uBAAO,EAAC,iBAAiB,CAAC;;;;wDAM1B;AAED;IAAC,IAAA,uBAAO,EAAC,YAAY,CAAC;;;;mDAMrB;AAED;IAAC,IAAA,uBAAO,EAAC,SAAS,CAAC;;;;gDAMlB;AAED;IAAC,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sDAwBf;AAnJU,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAwBa,wBAAY;QACd,sBAAU;GAxBrB,WAAW,CAoJvB;AApJY,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { OnEvent } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  BestBlockPayload,\n  EventPayload,\n  IndexerEvent,\n  NetworkMetadataPayload,\n  ProcessBlockPayload,\n  ProcessedBlockCountPayload,\n  TargetBlockPayload,\n  StoreService,\n  getLogger,\n  NodeConfig,\n} from '@subql/node-core';\n\nconst UPDATE_HEIGHT_INTERVAL = 60000;\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: polkadotSdkVersion } = require('@polkadot/api/package.json');\nconst { version: packageVersion } = require('../../package.json');\nconst logger = getLogger('profiler');\n\n@Injectable()\nexport class MetaService {\n  private currentProcessingHeight: number;\n  private currentProcessingTimestamp: number;\n  private bestHeight: number;\n  private targetHeight: number;\n  private networkMeta: NetworkMetadataPayload;\n  private apiConnected: boolean;\n  private usingDictionary: boolean;\n  private injectedApiConnected: boolean;\n  private lastProcessedHeight: number;\n  private lastProcessedTimestamp: number;\n  private processedBlockCount: number;\n  private accEnqueueBlocks = 0;\n  private accFetchBlocks = 0;\n  private currentFilteringBlockNum = 0;\n  private accRpcCalls = 0;\n  private lastReportedFilteringBlockNum = 0;\n  private lastReportedEnqueueBlocks = 0;\n  private lastReportedFetchBlocks = 0;\n  private lastReportedRpcCalls = 0;\n  private lastStatsReportedTs: Date;\n\n  constructor(\n    private storeService: StoreService,\n    private nodeConfig: NodeConfig,\n  ) {}\n\n  getMeta() {\n    return {\n      currentProcessingHeight: this.currentProcessingHeight,\n      currentProcessingTimestamp: this.currentProcessingTimestamp,\n      targetHeight: this.targetHeight,\n      bestHeight: this.bestHeight,\n      indexerNodeVersion: packageVersion,\n      lastProcessedHeight: this.lastProcessedHeight,\n      lastProcessedTimestamp: this.lastProcessedTimestamp,\n      uptime: process.uptime(),\n      polkadotSdkVersion,\n      processedBlockCount: this.processedBlockCount,\n      apiConnected: this.apiConnected,\n      injectedApiConnected: this.injectedApiConnected,\n      usingDictionary: this.usingDictionary,\n      ...this.networkMeta,\n    };\n  }\n\n  @Interval(UPDATE_HEIGHT_INTERVAL)\n  getTargetHeight(): void {\n    this.storeService.storeCache.metadata.set(\n      'targetHeight',\n      this.targetHeight,\n    );\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessing)\n  handleProcessingBlock(blockPayload: ProcessBlockPayload): void {\n    this.currentProcessingHeight = blockPayload.height;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessedCount)\n  handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void {\n    this.processedBlockCount = blockPayload.processedBlockCount;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockTarget)\n  handleTargetBlock(blockPayload: TargetBlockPayload): void {\n    this.targetHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.BlockBest)\n  handleBestBlock(blockPayload: BestBlockPayload): void {\n    this.bestHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.NetworkMetadata)\n  handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void {\n    this.networkMeta = networkMeta;\n  }\n\n  @OnEvent(IndexerEvent.ApiConnected)\n  handleApiConnected({ value }: EventPayload<number>): void {\n    this.apiConnected = !!value;\n  }\n\n  @OnEvent(IndexerEvent.UsingDictionary)\n  handleUsingDictionary({ value }: EventPayload<number>): void {\n    this.usingDictionary = !!value;\n  }\n\n  @OnEvent('enqueueBlocks')\n  handleEnqueueBlocks(size: number): void {\n    this.accEnqueueBlocks += size;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('filteringBlocks')\n  handleFilteringBlocks(height: number): void {\n    this.currentFilteringBlockNum = height;\n    if (!this.lastStatsReportedTs) {\n      this.lastReportedFilteringBlockNum = height;\n    }\n  }\n\n  @OnEvent('fetchBlock')\n  handleFetchBlock(): void {\n    this.accFetchBlocks++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('rpcCall')\n  handleRpcCall(): void {\n    this.accRpcCalls++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @Interval(10000)\n  blockFilteringSpeed(): void {\n    if (!this.nodeConfig.profiler) {\n      return;\n    }\n    const count = this.accEnqueueBlocks - this.lastReportedEnqueueBlocks;\n    this.lastReportedEnqueueBlocks = this.accEnqueueBlocks;\n    const filteringCount =\n      this.currentFilteringBlockNum - this.lastReportedFilteringBlockNum;\n    const now = new Date();\n    const timepass = now.getTime() - this.lastStatsReportedTs.getTime();\n    this.lastStatsReportedTs = now;\n    this.lastReportedFilteringBlockNum = this.currentFilteringBlockNum;\n    const rpcCalls = this.accRpcCalls - this.lastReportedRpcCalls;\n    this.lastReportedRpcCalls = this.accRpcCalls;\n    const fetchCount = this.accFetchBlocks - this.lastReportedFetchBlocks;\n    this.lastReportedFetchBlocks = this.accFetchBlocks;\n    logger.info(`actual block filtering: ${(count / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, \\\nseeming speed: ${(filteringCount / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, rpcCalls: ${(rpcCalls / (timepass / 1000)).toFixed(2)}/sec \\\nfetch speed: ${(fetchCount / (timepass / 1000)).toFixed(2)}/sec`);\n  }\n}\n"]}

package/dist/subcommands/forceClean.service.js

@@ -44,12 +44,16 @@ let ForceCleanService = class ForceCleanService {
                 logging: false,
                 benchmark: false,
             });
-            // drop all related enums
+            // TODO, remove this soon, once original enum are cleaned
+            // Deprecate, now enums are moved under schema, drop schema will remove project enums
             await Promise.all(modelsRelation.enums.map(async (e) => {
-                const enumTypeName = `${schema}_enum_${(0, node_core_1.enumNameToHash)(e.name)}`;
-                await this.sequelize.query(`
-            DROP TYPE "${enumTypeName}";
+                const enumTypeNameDeprecated = `${schema}_enum_${(0, node_core_1.enumNameToHash)(e.name)}`;
+                const resultsDeprecated = await (0, node_core_1.getEnumDeprecated)(this.sequelize, enumTypeNameDeprecated);
+                if (resultsDeprecated.length !== 0) {
+                    await this.sequelize.query(`
+            DROP TYPE "${enumTypeNameDeprecated}";
           `);
+                }
             }));
             // remove schema from subquery table (might not exist)
             const checker = await this.sequelize.query(`

package/dist/subcommands/forceClean.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"forceClean.service.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,2CAAoD;AACpD,gDAK0B;AAC1B,wCAAuD;AACvD,yCAAkD;AAClD,kEAA+D;AAE/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AAGjC,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAC5B,YACmB,SAAoB,EACpB,UAAsB,EACD,OAAwB;QAF7C,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACD,YAAO,GAAP,OAAO,CAAiB;IAC7D,CAAC;IAEJ,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,IAAA,oCAAwB,EAC3C,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEpE,IAAI;YACF,kDAAkD;YAClD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,MAAM,GAAG,EAAE;gBAC7C,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,yBAAyB;YACzB,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,YAAY,GAAG,GAAG,MAAM,SAAS,IAAA,0BAAc,EAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;yBACZ,YAAY;WAC1B,CAAC,CAAC;YACL,CAAC,CAAC,CACH,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxC;mHAC2G,CAC5G,CAAC;YAEF,IAAK,OAAO,CAAC,CAAC,CAAS,CAAC,QAAQ,GAAG,CAAC,EAAE;gBACpC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxB;;qCAE2B,EAC3B;oBACE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;oBACpD,IAAI,EAAE,sBAAU,CAAC,MAAM;iBACxB,CACF,CAAC;aACH;YAED,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAE/C,IAAI,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC1C,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;aAC7C;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;SACX;IACH,CAAC;CACF,CAAA;AAhEY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFC,qBAAS;QACR,sBAAU;QACQ,iCAAe;GAJrD,iBAAiB,CAgE7B;AAhEY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  NodeConfig,\n  getExistingProjectSchema,\n  enumNameToHash,\n} from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport { QueryTypes, Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nconst logger = getLogger('Force-clean');\n\n@Injectable()\nexport class ForceCleanService {\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') protected project: SubqueryProject,\n  ) {}\n\n  async forceClean(): Promise<void> {\n    const schema = await getExistingProjectSchema(\n      this.nodeConfig,\n      this.sequelize,\n    );\n    if (!schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    const modelsRelation = getAllEntitiesRelations(this.project.schema);\n\n    try {\n      // drop existing project schema and metadata table\n      await this.sequelize.dropSchema(`\"${schema}\"`, {\n        logging: false,\n        benchmark: false,\n      });\n\n      // drop all related enums\n      await Promise.all(\n        modelsRelation.enums.map(async (e) => {\n          const enumTypeName = `${schema}_enum_${enumNameToHash(e.name)}`;\n          await this.sequelize.query(`\n            DROP TYPE \"${enumTypeName}\";\n          `);\n        }),\n      );\n\n      // remove schema from subquery table (might not exist)\n      const checker = await this.sequelize.query(\n        `\n              SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'public' AND  TABLE_NAME = 'subqueries'`,\n      );\n\n      if ((checker[1] as any).rowCount > 0) {\n        await this.sequelize.query(\n          ` DELETE\n                  FROM public.subqueries\n                  WHERE name = :name`,\n          {\n            replacements: { name: this.nodeConfig.subqueryName },\n            type: QueryTypes.DELETE,\n          },\n        );\n      }\n\n      logger.info('force cleaned schema and tables');\n\n      if (fs.existsSync(this.nodeConfig.mmrPath)) {\n        await fs.promises.unlink(this.nodeConfig.mmrPath);\n        logger.info('force cleaned file based mmr');\n      }\n    } catch (err) {\n      logger.error(err, 'failed to force clean');\n      throw err;\n    }\n  }\n}\n"]}
+{"version":3,"file":"forceClean.service.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,2CAAoD;AACpD,gDAM0B;AAC1B,wCAAuD;AACvD,yCAAkD;AAClD,kEAA+D;AAE/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AAGjC,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAC5B,YACmB,SAAoB,EACpB,UAAsB,EACD,OAAwB;QAF7C,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACD,YAAO,GAAP,OAAO,CAAiB;IAC7D,CAAC;IAEJ,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,IAAA,oCAAwB,EAC3C,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEpE,IAAI;YACF,kDAAkD;YAClD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,MAAM,GAAG,EAAE;gBAC7C,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,yDAAyD;YACzD,qFAAqF;YACrF,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,sBAAsB,GAAG,GAAG,MAAM,SAAS,IAAA,0BAAc,EAC7D,CAAC,CAAC,IAAI,CACP,EAAE,CAAC;gBACJ,MAAM,iBAAiB,GAAG,MAAM,IAAA,6BAAiB,EAC/C,IAAI,CAAC,SAAS,EACd,sBAAsB,CACvB,CAAC;gBACF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAClC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;yBACd,sBAAsB;WACpC,CAAC,CAAC;iBACF;YACH,CAAC,CAAC,CACH,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxC;mHAC2G,CAC5G,CAAC;YAEF,IAAK,OAAO,CAAC,CAAC,CAAS,CAAC,QAAQ,GAAG,CAAC,EAAE;gBACpC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxB;;qCAE2B,EAC3B;oBACE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;oBACpD,IAAI,EAAE,sBAAU,CAAC,MAAM;iBACxB,CACF,CAAC;aACH;YAED,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAE/C,IAAI,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC1C,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;aAC7C;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;SACX;IACH,CAAC;CACF,CAAA;AAzEY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFC,qBAAS;QACR,sBAAU;QACQ,iCAAe;GAJrD,iBAAiB,CAyE7B;AAzEY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  NodeConfig,\n  getExistingProjectSchema,\n  enumNameToHash,\n  getEnumDeprecated,\n} from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport { QueryTypes, Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nconst logger = getLogger('Force-clean');\n\n@Injectable()\nexport class ForceCleanService {\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') protected project: SubqueryProject,\n  ) {}\n\n  async forceClean(): Promise<void> {\n    const schema = await getExistingProjectSchema(\n      this.nodeConfig,\n      this.sequelize,\n    );\n    if (!schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    const modelsRelation = getAllEntitiesRelations(this.project.schema);\n\n    try {\n      // drop existing project schema and metadata table\n      await this.sequelize.dropSchema(`\"${schema}\"`, {\n        logging: false,\n        benchmark: false,\n      });\n\n      // TODO, remove this soon, once original enum are cleaned\n      // Deprecate, now enums are moved under schema, drop schema will remove project enums\n      await Promise.all(\n        modelsRelation.enums.map(async (e) => {\n          const enumTypeNameDeprecated = `${schema}_enum_${enumNameToHash(\n            e.name,\n          )}`;\n          const resultsDeprecated = await getEnumDeprecated(\n            this.sequelize,\n            enumTypeNameDeprecated,\n          );\n          if (resultsDeprecated.length !== 0) {\n            await this.sequelize.query(`\n            DROP TYPE \"${enumTypeNameDeprecated}\";\n          `);\n          }\n        }),\n      );\n\n      // remove schema from subquery table (might not exist)\n      const checker = await this.sequelize.query(\n        `\n              SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'public' AND  TABLE_NAME = 'subqueries'`,\n      );\n\n      if ((checker[1] as any).rowCount > 0) {\n        await this.sequelize.query(\n          ` DELETE\n                  FROM public.subqueries\n                  WHERE name = :name`,\n          {\n            replacements: { name: this.nodeConfig.subqueryName },\n            type: QueryTypes.DELETE,\n          },\n        );\n      }\n\n      logger.info('force cleaned schema and tables');\n\n      if (fs.existsSync(this.nodeConfig.mmrPath)) {\n        await fs.promises.unlink(this.nodeConfig.mmrPath);\n        logger.info('force cleaned file based mmr');\n      }\n    } catch (err) {\n      logger.error(err, 'failed to force clean');\n      throw err;\n    }\n  }\n}\n"]}

package/dist/subcommands/reindex.init.js

@@ -14,7 +14,11 @@ async function reindexInit(targetHeight) {
         await app.init();
         const reindexService = app.get(reindex_service_1.ReindexService);
         await reindexService.init();
-        await reindexService.reindex(targetHeight);
+        const actualReindexHeight = await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight);
+        if (actualReindexHeight !== targetHeight) {
+            logger.info(`Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindexHeight}, will index to ${actualReindexHeight}`);
+        }
+        await reindexService.reindex(actualReindexHeight);
     }
     catch (e) {
         logger.error(e, 'Reindex failed to execute');

package/dist/subcommands/reindex.init.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.init.js","sourceRoot":"","sources":["../../src/subcommands/reindex.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAA6C;AAC7C,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AACjC,KAAK,UAAU,WAAW,CAAC,YAAoB;IACpD,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAE/C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;KAC5C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAdD,kCAcC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger } from '@subql/node-core';\nimport { ReindexModule } from './reindex.module';\nimport { ReindexService } from './reindex.service';\n\nconst logger = getLogger('CLI-Reindex');\nexport async function reindexInit(targetHeight: number): Promise<void> {\n  try {\n    const app = await NestFactory.create(ReindexModule);\n\n    await app.init();\n    const reindexService = app.get(ReindexService);\n\n    await reindexService.init();\n    await reindexService.reindex(targetHeight);\n  } catch (e) {\n    logger.error(e, 'Reindex failed to execute');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}
+{"version":3,"file":"reindex.init.js","sourceRoot":"","sources":["../../src/subcommands/reindex.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAA6C;AAC7C,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AACjC,KAAK,UAAU,WAAW,CAAC,YAAoB;IACpD,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAE/C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,mBAAmB,GACvB,MAAM,cAAc,CAAC,oCAAoC,CAAC,YAAY,CAAC,CAAC;QAC1E,IAAI,mBAAmB,KAAK,YAAY,EAAE;YACxC,MAAM,CAAC,IAAI,CACT,6BAA6B,YAAY,qCAAqC,mBAAmB,mBAAmB,mBAAmB,EAAE,CAC1I,CAAC;SACH;QACD,MAAM,cAAc,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;KACnD;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AArBD,kCAqBC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger } from '@subql/node-core';\nimport { ReindexModule } from './reindex.module';\nimport { ReindexService } from './reindex.service';\n\nconst logger = getLogger('CLI-Reindex');\nexport async function reindexInit(targetHeight: number): Promise<void> {\n  try {\n    const app = await NestFactory.create(ReindexModule);\n\n    await app.init();\n    const reindexService = app.get(ReindexService);\n\n    await reindexService.init();\n    const actualReindexHeight =\n      await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight);\n    if (actualReindexHeight !== targetHeight) {\n      logger.info(\n        `Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindexHeight}, will index to ${actualReindexHeight}`,\n      );\n    }\n    await reindexService.reindex(actualReindexHeight);\n  } catch (e) {\n    logger.error(e, 'Reindex failed to execute');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}

package/dist/subcommands/reindex.module.js

@@ -14,6 +14,7 @@ const node_core_1 = require("@subql/node-core");
 const configure_module_1 = require("../configure/configure.module");
 const ds_processor_service_1 = require("../indexer/ds-processor.service");
 const dynamic_ds_service_1 = require("../indexer/dynamic-ds.service");
+const unfinalizedBlocks_service_1 = require("../indexer/unfinalizedBlocks.service");
 const forceClean_service_1 = require("./forceClean.service");
 const reindex_service_1 = require("./reindex.service");
 let ReindexFeatureModule = class ReindexFeatureModule {
@@ -21,12 +22,19 @@ let ReindexFeatureModule = class ReindexFeatureModule {
 ReindexFeatureModule = __decorate([
     (0, common_1.Module)({
         providers: [
+            node_core_1.StoreCacheService,
             node_core_1.StoreService,
             reindex_service_1.ReindexService,
             node_core_1.MmrService,
             forceClean_service_1.ForceCleanService,
+            unfinalizedBlocks_service_1.UnfinalizedBlocksService,
             dynamic_ds_service_1.DynamicDsService,
             ds_processor_service_1.DsProcessorService,
+            {
+                // Used to work with DI for unfinalizedBlocksService but not used with reindex
+                provide: node_core_1.ApiService,
+                useFactory: () => undefined,
+            },
         ],
         controllers: [],
     })

package/dist/subcommands/reindex.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAK0B;AAC1B,oEAAgE;AAChE,0EAAqE;AACrE,sEAAiE;AACjE,6DAAyD;AACzD,uDAAmD;AAa5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAXhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ,gCAAc;YACd,sBAAU;YACV,sCAAiB;YACjB,qCAAgB;YAChB,yCAAkB;SACnB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAU1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IARzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;SACrB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  DbModule,\n  MmrService,\n  NodeConfig,\n  StoreService,\n} from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { DsProcessorService } from '../indexer/ds-processor.service';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [\n    StoreService,\n    ReindexService,\n    MmrService,\n    ForceCleanService,\n    DynamicDsService,\n    DsProcessorService,\n  ],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}
+{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,oEAAgE;AAChE,0EAAqE;AACrE,sEAAiE;AACjE,oFAAgF;AAChF,6DAAyD;AACzD,uDAAmD;AAoB5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAlBhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,6BAAiB;YACjB,wBAAY;YACZ,gCAAc;YACd,sBAAU;YACV,sCAAiB;YACjB,oDAAwB;YACxB,qCAAgB;YAChB,yCAAkB;YAClB;gBACE,8EAA8E;gBAC9E,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS;aAC5B;SACF;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAU1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IARzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;SACrB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  ApiService,\n  DbModule,\n  MmrService,\n  StoreCacheService,\n  StoreService,\n} from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { DsProcessorService } from '../indexer/ds-processor.service';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [\n    StoreCacheService,\n    StoreService,\n    ReindexService,\n    MmrService,\n    ForceCleanService,\n    UnfinalizedBlocksService,\n    DynamicDsService,\n    DsProcessorService,\n    {\n      // Used to work with DI for unfinalizedBlocksService but not used with reindex\n      provide: ApiService,\n      useFactory: () => undefined,\n    },\n  ],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}

package/dist/subcommands/reindex.service.d.ts

@@ -2,6 +2,7 @@ import { MmrService, NodeConfig, StoreService } from '@subql/node-core';
 import { Sequelize } from 'sequelize';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { DynamicDsService } from '../indexer/dynamic-ds.service';
+import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
 import { ForceCleanService } from './forceClean.service';
 export declare class ReindexService {
     private readonly sequelize;
@@ -10,11 +11,13 @@ export declare class ReindexService {
     private readonly mmrService;
     private readonly project;
     private readonly forceCleanService;
+    private readonly unfinalizedBlocksService;
     private readonly dynamicDsService;
     private schema;
     private metadataRepo;
-    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, storeService: StoreService, mmrService: MmrService, project: SubqueryProject, forceCleanService: ForceCleanService, dynamicDsService: DynamicDsService);
+    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, storeService: StoreService, mmrService: MmrService, project: SubqueryProject, forceCleanService: ForceCleanService, unfinalizedBlocksService: UnfinalizedBlocksService, dynamicDsService: DynamicDsService);
     init(): Promise<void>;
+    getTargetHeightWithUnfinalizedBlocks(inputHeight: number): Promise<number>;
     private getExistingProjectSchema;
     private getLastProcessedHeight;
     private getMetadataBlockOffset;

package/dist/subcommands/reindex.service.js

@@ -20,18 +20,20 @@ const node_core_1 = require("@subql/node-core");
 const sequelize_1 = require("sequelize");
 const SubqueryProject_1 = require("../configure/SubqueryProject");
 const dynamic_ds_service_1 = require("../indexer/dynamic-ds.service");
+const unfinalizedBlocks_service_1 = require("../indexer/unfinalizedBlocks.service");
 const project_1 = require("../utils/project");
 const reindex_1 = require("../utils/reindex");
 const forceClean_service_1 = require("./forceClean.service");
 const logger = (0, node_core_1.getLogger)('Reindex');
 let ReindexService = class ReindexService {
-    constructor(sequelize, nodeConfig, storeService, mmrService, project, forceCleanService, dynamicDsService) {
+    constructor(sequelize, nodeConfig, storeService, mmrService, project, forceCleanService, unfinalizedBlocksService, dynamicDsService) {
         this.sequelize = sequelize;
         this.nodeConfig = nodeConfig;
         this.storeService = storeService;
         this.mmrService = mmrService;
         this.project = project;
         this.forceCleanService = forceCleanService;
+        this.unfinalizedBlocksService = unfinalizedBlocksService;
         this.dynamicDsService = dynamicDsService;
     }
     async init() {
@@ -41,24 +43,31 @@ let ReindexService = class ReindexService {
             throw new Error('Schema does not exist.');
         }
         await this.initDbSchema();
-        // Should we use an optional arg for multiChain?
-        this.metadataRepo = await (0, node_core_1.MetadataFactory)(this.sequelize, this.schema, this.nodeConfig.multiChain, this.project.network.chainId);
+        this.metadataRepo = this.storeService.storeCache.metadata;
         this.dynamicDsService.init(this.metadataRepo);
     }
+    async getTargetHeightWithUnfinalizedBlocks(inputHeight) {
+        // Why does this happen?
+        this.unfinalizedBlocksService.metadataRepo = this.metadataRepo;
+        const unfinalizedBlocks = await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();
+        const bestBlocks = unfinalizedBlocks.filter(([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight);
+        if (bestBlocks.length === 0) {
+            return inputHeight;
+        }
+        const [firstBestBlock] = bestBlocks[0];
+        return Math.min(inputHeight, firstBestBlock);
+    }
     async getExistingProjectSchema() {
         return (0, node_core_1.getExistingProjectSchema)(this.nodeConfig, this.sequelize);
     }
     async getLastProcessedHeight() {
-        return (0, node_core_1.getMetaDataInfo)(this.metadataRepo, 'lastProcessedHeight');
+        return this.metadataRepo.find('lastProcessedHeight');
     }
     async getMetadataBlockOffset() {
-        return (0, node_core_1.getMetaDataInfo)(this.metadataRepo, 'blockOffset');
+        return this.metadataRepo.find('blockOffset');
     }
     async getMetadataSpecName() {
-        const res = await this.metadataRepo.findOne({
-            where: { key: 'specName' },
-        });
-        return res === null || res === void 0 ? void 0 : res.value;
+        return this.metadataRepo.find('specName');
     }
     async initDbSchema() {
         await (0, project_1.initDbSchema)(this.project, this.schema, this.storeService);
@@ -80,7 +89,8 @@ let ReindexService = class ReindexService {
             this.getStartBlockFromDataSources(),
             this.getLastProcessedHeight(),
         ]);
-        return (0, reindex_1.reindex)(startHeight, await this.getMetadataBlockOffset(), targetBlockHeight, lastProcessedHeight, this.storeService, this.dynamicDsService, this.mmrService, this.sequelize, this.forceCleanService);
+        await (0, reindex_1.reindex)(startHeight, await this.getMetadataBlockOffset(), targetBlockHeight, lastProcessedHeight, this.storeService, this.unfinalizedBlocksService, this.dynamicDsService, this.mmrService, this.sequelize, this.forceCleanService);
+        await this.storeService.storeCache.flushCache();
     }
 };
 ReindexService = __decorate([
@@ -92,6 +102,7 @@ ReindexService = __decorate([
         node_core_1.MmrService,
         SubqueryProject_1.SubqueryProject,
         forceClean_service_1.ForceCleanService,
+        unfinalizedBlocks_service_1.UnfinalizedBlocksService,
         dynamic_ds_service_1.DynamicDsService])
 ], ReindexService);
 exports.ReindexService = ReindexService;

package/dist/subcommands/reindex.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAS0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAwB,EACpD,iBAAoC,EACpC,gBAAkC;QANlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACpD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,gDAAgD;QAChD,IAAI,CAAC,YAAY,GAAG,MAAM,IAAA,2BAAe,EACvC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,CAAC,UAAU,EAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAC7B,CAAC;QACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE;SAC3B,CAAC,CAAC;QACH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,4BAA4B;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,OAAO,IAAA,iBAAO,EACZ,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;IACJ,CAAC;CACF,CAAA;AAxFY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAJC,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACe,iCAAe;QACjC,sCAAiB;QAClB,qCAAgB;GAX1C,cAAc,CAwF1B;AAxFY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MetadataFactory,\n  MetadataRepo,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: MetadataRepo;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n    // Should we use an optional arg for multiChain?\n    this.metadataRepo = await MetadataFactory(\n      this.sequelize,\n      this.schema,\n      this.nodeConfig.multiChain,\n      this.project.network.chainId,\n    );\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    const res = await this.metadataRepo.findOne({\n      where: { key: 'specName' },\n    });\n    return res?.value as string | undefined;\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  private async getStartBlockFromDataSources() {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    return reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n  }\n}\n"]}
+{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAO0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,oFAAgF;AAChF,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAwB,EACpD,iBAAoC,EACpC,wBAAkD,EAClD,gBAAkC;QAPlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACpD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAE1D,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,oCAAoC,CACxC,WAAmB;QAEnB,wBAAwB;QACvB,IAAI,CAAC,wBAAgC,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACxE,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,wBAAwB,CAAC,4BAA4B,EAAE,CAAC;QACrE,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CACzC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAC9D,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,OAAO,WAAW,CAAC;SACpB;QACD,MAAM,CAAC,cAAc,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,4BAA4B;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,MAAM,IAAA,iBAAO,EACX,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,wBAAwB,EAC7B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;QACF,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;IAClD,CAAC;CACF,CAAA;AArGY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAJC,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACe,iCAAe;QACjC,sCAAiB;QACV,oDAAwB;QAChC,qCAAgB;GAZ1C,cAAc,CAqG1B;AArGY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  CacheMetadataModel,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: CacheMetadataModel;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly unfinalizedBlocksService: UnfinalizedBlocksService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n\n    this.metadataRepo = this.storeService.storeCache.metadata;\n\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  async getTargetHeightWithUnfinalizedBlocks(\n    inputHeight: number,\n  ): Promise<number> {\n    // Why does this happen?\n    (this.unfinalizedBlocksService as any).metadataRepo = this.metadataRepo;\n    const unfinalizedBlocks =\n      await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();\n    const bestBlocks = unfinalizedBlocks.filter(\n      ([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight,\n    );\n    if (bestBlocks.length === 0) {\n      return inputHeight;\n    }\n    const [firstBestBlock] = bestBlocks[0];\n    return Math.min(inputHeight, firstBestBlock);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.metadataRepo.find('lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.metadataRepo.find('blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    return this.metadataRepo.find('specName');\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  private async getStartBlockFromDataSources() {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    await reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlocksService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n    await this.storeService.storeCache.flushCache();\n  }\n}\n"]}