@subql/node-ethereum 1.10.1-2 → 2.0.0

package/dist/subcommands/reindex.init.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.init.js","sourceRoot":"","sources":["../../src/subcommands/reindex.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAA6C;AAC7C,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AACjC,KAAK,UAAU,WAAW,CAAC,YAAoB;IACpD,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;KAC5C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAZD,kCAYC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger } from '@subql/node-core';\nimport { ReindexModule } from './reindex.module';\nimport { ReindexService } from './reindex.service';\n\nconst logger = getLogger('CLI-Reindex');\nexport async function reindexInit(targetHeight: number): Promise<void> {\n  try {\n    const app = await NestFactory.create(ReindexModule);\n\n    await app.init();\n    const reindexService = app.get(ReindexService);\n    await reindexService.reindex(targetHeight);\n  } catch (e) {\n    logger.error(e, 'Reindex failed to execute');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}
+{"version":3,"file":"reindex.init.js","sourceRoot":"","sources":["../../src/subcommands/reindex.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAA6C;AAC7C,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AACjC,KAAK,UAAU,WAAW,CAAC,YAAoB;IACpD,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAE/C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,mBAAmB,GACvB,MAAM,cAAc,CAAC,oCAAoC,CAAC,YAAY,CAAC,CAAC;QAC1E,IAAI,mBAAmB,KAAK,YAAY,EAAE;YACxC,MAAM,CAAC,IAAI,CACT,6BAA6B,YAAY,qCAAqC,mBAAmB,mBAAmB,mBAAmB,EAAE,CAC1I,CAAC;SACH;QACD,MAAM,cAAc,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;KACnD;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AArBD,kCAqBC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger } from '@subql/node-core';\nimport { ReindexModule } from './reindex.module';\nimport { ReindexService } from './reindex.service';\n\nconst logger = getLogger('CLI-Reindex');\nexport async function reindexInit(targetHeight: number): Promise<void> {\n  try {\n    const app = await NestFactory.create(ReindexModule);\n\n    await app.init();\n    const reindexService = app.get(ReindexService);\n\n    await reindexService.init();\n    const actualReindexHeight =\n      await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight);\n    if (actualReindexHeight !== targetHeight) {\n      logger.info(\n        `Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindexHeight}, will index to ${actualReindexHeight}`,\n      );\n    }\n    await reindexService.reindex(actualReindexHeight);\n  } catch (e) {\n    logger.error(e, 'Reindex failed to execute');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}

package/dist/subcommands/reindex.module.js

@@ -10,15 +10,33 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ReindexModule = exports.ReindexFeatureModule = void 0;
 const common_1 = require("@nestjs/common");
+const event_emitter_1 = require("@nestjs/event-emitter");
 const node_core_1 = require("@subql/node-core");
 const configure_module_1 = require("../configure/configure.module");
+const ds_processor_service_1 = require("../indexer/ds-processor.service");
+const dynamic_ds_service_1 = require("../indexer/dynamic-ds.service");
+const unfinalizedBlocks_service_1 = require("../indexer/unfinalizedBlocks.service");
 const forceClean_service_1 = require("./forceClean.service");
 const reindex_service_1 = require("./reindex.service");
 let ReindexFeatureModule = class ReindexFeatureModule {
 };
 ReindexFeatureModule = __decorate([
     (0, common_1.Module)({
-        providers: [node_core_1.StoreService, reindex_service_1.ReindexService, node_core_1.MmrService, forceClean_service_1.ForceCleanService],
+        providers: [
+            node_core_1.StoreCacheService,
+            node_core_1.StoreService,
+            reindex_service_1.ReindexService,
+            node_core_1.MmrService,
+            forceClean_service_1.ForceCleanService,
+            unfinalizedBlocks_service_1.UnfinalizedBlocksService,
+            dynamic_ds_service_1.DynamicDsService,
+            ds_processor_service_1.DsProcessorService,
+            {
+                // Used to work with DI for unfinalizedBlocksService but not used with reindex
+                provide: node_core_1.ApiService,
+                useFactory: () => undefined,
+            },
+        ],
         controllers: [],
     })
 ], ReindexFeatureModule);
@@ -31,6 +49,7 @@ ReindexModule = __decorate([
             node_core_1.DbModule.forRoot(),
             configure_module_1.ConfigureModule.register(),
             ReindexFeatureModule,
+            event_emitter_1.EventEmitterModule.forRoot(),
         ],
         controllers: [],
     })

package/dist/subcommands/reindex.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAAsE;AACtE,oEAAgE;AAChE,6DAAyD;AACzD,uDAAmD;AAM5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAJhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE,CAAC,wBAAY,EAAE,gCAAc,EAAE,sBAAU,EAAE,sCAAiB,CAAC;QACxE,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAU1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IARzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;SACrB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { DbModule, MmrService, StoreService } from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [StoreService, ReindexService, MmrService, ForceCleanService],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}
+{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAA2D;AAC3D,gDAM0B;AAC1B,oEAAgE;AAChE,0EAAqE;AACrE,sEAAiE;AACjE,oFAAgF;AAChF,6DAAyD;AACzD,uDAAmD;AAoB5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAlBhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,6BAAiB;YACjB,wBAAY;YACZ,gCAAc;YACd,sBAAU;YACV,sCAAiB;YACjB,oDAAwB;YACxB,qCAAgB;YAChB,yCAAkB;YAClB;gBACE,8EAA8E;gBAC9E,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS;aAC5B;SACF;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAW1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IATzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;YACpB,kCAAkB,CAAC,OAAO,EAAE;SAC7B;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitterModule } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  DbModule,\n  MmrService,\n  StoreCacheService,\n  StoreService,\n} from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { DsProcessorService } from '../indexer/ds-processor.service';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [\n    StoreCacheService,\n    StoreService,\n    ReindexService,\n    MmrService,\n    ForceCleanService,\n    UnfinalizedBlocksService,\n    DynamicDsService,\n    DsProcessorService,\n    {\n      // Used to work with DI for unfinalizedBlocksService but not used with reindex\n      provide: ApiService,\n      useFactory: () => undefined,\n    },\n  ],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n    EventEmitterModule.forRoot(),\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}

package/dist/subcommands/reindex.service.d.ts

@@ -1,6 +1,8 @@
 import { MmrService, NodeConfig, StoreService } from '@subql/node-core';
 import { Sequelize } from 'sequelize';
 import { SubqueryProject } from '../configure/SubqueryProject';
+import { DynamicDsService } from '../indexer/dynamic-ds.service';
+import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
 import { ForceCleanService } from './forceClean.service';
 export declare class ReindexService {
     private readonly sequelize;
@@ -9,11 +11,13 @@ export declare class ReindexService {
     private readonly mmrService;
     private readonly project;
     private readonly forceCleanService;
+    private readonly unfinalizedBlocksService;
+    private readonly dynamicDsService;
     private schema;
     private metadataRepo;
-    private specName;
-    private startHeight;
-    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, storeService: StoreService, mmrService: MmrService, project: SubqueryProject, forceCleanService: ForceCleanService);
+    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, storeService: StoreService, mmrService: MmrService, project: SubqueryProject, forceCleanService: ForceCleanService, unfinalizedBlocksService: UnfinalizedBlocksService, dynamicDsService: DynamicDsService);
+    init(): Promise<void>;
+    getTargetHeightWithUnfinalizedBlocks(inputHeight: number): Promise<number>;
     private getExistingProjectSchema;
     private getLastProcessedHeight;
     private getMetadataBlockOffset;

package/dist/subcommands/reindex.service.js

@@ -10,44 +10,69 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ReindexService = void 0;
 const common_1 = require("@nestjs/common");
 const node_core_1 = require("@subql/node-core");
 const sequelize_1 = require("sequelize");
 const SubqueryProject_1 = require("../configure/SubqueryProject");
+const dynamic_ds_service_1 = require("../indexer/dynamic-ds.service");
+const unfinalizedBlocks_service_1 = require("../indexer/unfinalizedBlocks.service");
 const project_1 = require("../utils/project");
+const reindex_1 = require("../utils/reindex");
 const forceClean_service_1 = require("./forceClean.service");
 const logger = (0, node_core_1.getLogger)('Reindex');
 let ReindexService = class ReindexService {
-    constructor(sequelize, nodeConfig, storeService, mmrService, project, forceCleanService) {
+    constructor(sequelize, nodeConfig, storeService, mmrService, project, forceCleanService, unfinalizedBlocksService, dynamicDsService) {
         this.sequelize = sequelize;
         this.nodeConfig = nodeConfig;
         this.storeService = storeService;
         this.mmrService = mmrService;
         this.project = project;
         this.forceCleanService = forceCleanService;
+        this.unfinalizedBlocksService = unfinalizedBlocksService;
+        this.dynamicDsService = dynamicDsService;
+    }
+    async init() {
+        this.schema = await this.getExistingProjectSchema();
+        if (!this.schema) {
+            logger.error('Unable to locate schema');
+            throw new Error('Schema does not exist.');
+        }
+        await this.initDbSchema();
+        this.metadataRepo = this.storeService.storeCache.metadata;
+        this.dynamicDsService.init(this.metadataRepo);
+    }
+    async getTargetHeightWithUnfinalizedBlocks(inputHeight) {
+        // Why does this happen?
+        this.unfinalizedBlocksService.metadataRepo = this.metadataRepo;
+        const unfinalizedBlocks = await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();
+        const bestBlocks = unfinalizedBlocks.filter(([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight);
+        if (bestBlocks.length === 0) {
+            return inputHeight;
+        }
+        const [firstBestBlock] = bestBlocks[0];
+        return Math.min(inputHeight, firstBestBlock);
     }
     async getExistingProjectSchema() {
         return (0, node_core_1.getExistingProjectSchema)(this.nodeConfig, this.sequelize);
     }
     async getLastProcessedHeight() {
-        return (0, node_core_1.getMetaDataInfo)(this.metadataRepo, 'lastProcessedHeight');
+        return this.metadataRepo.find('lastProcessedHeight');
     }
     async getMetadataBlockOffset() {
-        return (0, node_core_1.getMetaDataInfo)(this.metadataRepo, 'blockOffset');
+        return this.metadataRepo.find('blockOffset');
     }
     async getMetadataSpecName() {
-        const res = await this.metadataRepo.findOne({
-            where: { key: 'specName' },
-        });
-        return res === null || res === void 0 ? void 0 : res.value;
+        return this.metadataRepo.find('specName');
     }
     async initDbSchema() {
         await (0, project_1.initDbSchema)(this.project, this.schema, this.storeService);
     }
-    // eslint-disable-next-line @typescript-eslint/require-await
-    async getStartBlockFromDataSources() {
+    getStartBlockFromDataSources() {
         const datasources = this.project.dataSources;
         const startBlocksList = datasources.map((item) => { var _a; return (_a = item.startBlock) !== null && _a !== void 0 ? _a : 1; });
         if (startBlocksList.length === 0) {
@@ -59,56 +84,25 @@ let ReindexService = class ReindexService {
         }
     }
     async reindex(targetBlockHeight) {
-        this.schema = await this.getExistingProjectSchema();
-        if (!this.schema) {
-            logger.error('Unable to locate schema');
-            throw new Error('Schema does not exist.');
-        }
-        await this.initDbSchema();
-        this.metadataRepo = (0, node_core_1.MetadataFactory)(this.sequelize, this.schema);
-        this.startHeight = await this.getStartBlockFromDataSources();
-        const lastProcessedHeight = await this.getLastProcessedHeight();
-        if (!this.storeService.historical) {
-            logger.warn('Unable to reindex, historical state not enabled');
-            return;
-        }
-        if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) {
-            logger.warn(`Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`);
-            return;
-        }
-        // if startHeight is greater than the targetHeight, just force clean
-        if (targetBlockHeight < this.startHeight) {
-            logger.info(`targetHeight: ${targetBlockHeight} is less than startHeight: ${this.startHeight}. Hence executing force-clean`);
-            await this.forceCleanService.forceClean();
-        }
-        else {
-            logger.info(`Reindexing to block: ${targetBlockHeight}`);
-            const transaction = await this.sequelize.transaction();
-            try {
-                await this.storeService.rewind(targetBlockHeight, transaction);
-                const blockOffset = await this.getMetadataBlockOffset();
-                if (blockOffset) {
-                    await this.mmrService.deleteMmrNode(targetBlockHeight + 1, blockOffset);
-                }
-                await transaction.commit();
-                logger.info('Reindex Success');
-            }
-            catch (err) {
-                logger.error(err, 'Reindexing failed');
-                await transaction.rollback();
-                throw err;
-            }
-        }
+        const [startHeight, lastProcessedHeight] = await Promise.all([
+            this.getStartBlockFromDataSources(),
+            this.getLastProcessedHeight(),
+        ]);
+        await (0, reindex_1.reindex)(startHeight, await this.getMetadataBlockOffset(), targetBlockHeight, lastProcessedHeight, this.storeService, this.unfinalizedBlocksService, this.dynamicDsService, this.mmrService, this.sequelize, this.forceCleanService);
+        await this.storeService.storeCache.flushCache(true, true);
     }
 };
 ReindexService = __decorate([
     (0, common_1.Injectable)(),
+    __param(4, (0, common_1.Inject)('ISubqueryProject')),
     __metadata("design:paramtypes", [sequelize_1.Sequelize,
         node_core_1.NodeConfig,
         node_core_1.StoreService,
         node_core_1.MmrService,
         SubqueryProject_1.SubqueryProject,
-        forceClean_service_1.ForceCleanService])
+        forceClean_service_1.ForceCleanService,
+        unfinalizedBlocks_service_1.UnfinalizedBlocksService,
+        dynamic_ds_service_1.DynamicDsService])
 ], ReindexService);
 exports.ReindexService = ReindexService;
 //# sourceMappingURL=reindex.service.js.map

package/dist/subcommands/reindex.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,gDAS0B;AAC1B,yCAAsC;AACtC,kEAA+E;AAC/E,8CAAgD;AAEhD,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAKzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACtB,OAAwB,EACxB,iBAAoC;QALpC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,YAAO,GAAP,OAAO,CAAiB;QACxB,sBAAiB,GAAjB,iBAAiB,CAAmB;IACpD,CAAC;IAEI,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE;SAC3B,CAAC,CAAC;QACH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,4BAA4B;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,YAAY,GAAG,IAAA,2BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjE,IAAI,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;QAE7D,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE;YACjC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC/D,OAAO;SACR;QACD,IAAI,CAAC,mBAAmB,IAAI,mBAAmB,GAAG,iBAAiB,EAAE;YACnE,MAAM,CAAC,IAAI,CACT,gCAAgC,iBAAiB,6BAA6B,mBAAmB,4BAA4B,CAC9H,CAAC;YACF,OAAO;SACR;QAED,oEAAoE;QACpE,IAAI,iBAAiB,GAAG,IAAI,CAAC,WAAW,EAAE;YACxC,MAAM,CAAC,IAAI,CACT,iBAAiB,iBAAiB,8BAA8B,IAAI,CAAC,WAAW,+BAA+B,CAChH,CAAC;YACF,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,CAAC;SAC3C;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,wBAAwB,iBAAiB,EAAE,CAAC,CAAC;YACzD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;YACvD,IAAI;gBACF,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;gBAE/D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,IAAI,WAAW,EAAE;oBACf,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CACjC,iBAAiB,GAAG,CAAC,EACrB,WAAW,CACZ,CAAC;iBACH;gBACD,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;gBAC3B,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;aAChC;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;gBACvC,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;gBAC7B,MAAM,GAAG,CAAC;aACX;SACF;IACH,CAAC;CACF,CAAA;AA1GY,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAOmB,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACb,iCAAe;QACL,sCAAiB;GAX5C,cAAc,CA0G1B;AA1GY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MetadataFactory,\n  MetadataRepo,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { initDbSchema } from '../utils/project';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: MetadataRepo;\n  private specName: string;\n  private startHeight: number;\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n  ) {}\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    const res = await this.metadataRepo.findOne({\n      where: { key: 'specName' },\n    });\n    return res?.value as string | undefined;\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  private async getStartBlockFromDataSources() {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n\n    this.metadataRepo = MetadataFactory(this.sequelize, this.schema);\n\n    this.startHeight = await this.getStartBlockFromDataSources();\n\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n\n    if (!this.storeService.historical) {\n      logger.warn('Unable to reindex, historical state not enabled');\n      return;\n    }\n    if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) {\n      logger.warn(\n        `Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`,\n      );\n      return;\n    }\n\n    // if startHeight is greater than the targetHeight, just force clean\n    if (targetBlockHeight < this.startHeight) {\n      logger.info(\n        `targetHeight: ${targetBlockHeight} is less than startHeight: ${this.startHeight}. Hence executing force-clean`,\n      );\n      await this.forceCleanService.forceClean();\n    } else {\n      logger.info(`Reindexing to block: ${targetBlockHeight}`);\n      const transaction = await this.sequelize.transaction();\n      try {\n        await this.storeService.rewind(targetBlockHeight, transaction);\n\n        const blockOffset = await this.getMetadataBlockOffset();\n        if (blockOffset) {\n          await this.mmrService.deleteMmrNode(\n            targetBlockHeight + 1,\n            blockOffset,\n          );\n        }\n        await transaction.commit();\n        logger.info('Reindex Success');\n      } catch (err) {\n        logger.error(err, 'Reindexing failed');\n        await transaction.rollback();\n        throw err;\n      }\n    }\n  }\n}\n"]}
+{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAO0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,oFAAgF;AAChF,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAwB,EACpD,iBAAoC,EACpC,wBAAkD,EAClD,gBAAkC;QAPlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACpD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAE1D,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,oCAAoC,CACxC,WAAmB;QAEnB,wBAAwB;QACvB,IAAI,CAAC,wBAAgC,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACxE,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,wBAAwB,CAAC,4BAA4B,EAAE,CAAC;QACrE,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CACzC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAC9D,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,OAAO,WAAW,CAAC;SACpB;QACD,MAAM,CAAC,cAAc,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,4BAA4B;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,MAAM,IAAA,iBAAO,EACX,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,wBAAwB,EAC7B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;QACF,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;CACF,CAAA;AApGY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAJC,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACe,iCAAe;QACjC,sCAAiB;QACV,oDAAwB;QAChC,qCAAgB;GAZ1C,cAAc,CAoG1B;AApGY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  CacheMetadataModel,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: CacheMetadataModel;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly unfinalizedBlocksService: UnfinalizedBlocksService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n\n    this.metadataRepo = this.storeService.storeCache.metadata;\n\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  async getTargetHeightWithUnfinalizedBlocks(\n    inputHeight: number,\n  ): Promise<number> {\n    // Why does this happen?\n    (this.unfinalizedBlocksService as any).metadataRepo = this.metadataRepo;\n    const unfinalizedBlocks =\n      await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();\n    const bestBlocks = unfinalizedBlocks.filter(\n      ([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight,\n    );\n    if (bestBlocks.length === 0) {\n      return inputHeight;\n    }\n    const [firstBestBlock] = bestBlocks[0];\n    return Math.min(inputHeight, firstBestBlock);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.metadataRepo.find('lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.metadataRepo.find('blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    return this.metadataRepo.find('specName');\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private getStartBlockFromDataSources(): number {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    await reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlocksService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n    await this.storeService.storeCache.flushCache(true, true);\n  }\n}\n"]}

package/dist/subcommands/testing.init.d.ts

@@ -0,0 +1 @@
+export declare function testingInit(): Promise<void>;

package/dist/subcommands/testing.init.js

@@ -0,0 +1,34 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.testingInit = void 0;
+const core_1 = require("@nestjs/core");
+const node_core_1 = require("@subql/node-core");
+const project_service_1 = require("../indexer/project.service");
+const testing_module_1 = require("./testing.module");
+const testing_service_1 = require("./testing.service");
+const logger = (0, node_core_1.getLogger)('Testing');
+async function testingInit() {
+    try {
+        const app = await core_1.NestFactory.create(testing_module_1.TestingModule, {
+            logger: new node_core_1.NestLogger(),
+        });
+        await app.init();
+        const projectService = app.get(project_service_1.ProjectService);
+        const apiService = app.get(node_core_1.ApiService);
+        // Initialise async services, we do this here rather than in factories, so we can capture one off events
+        await apiService.init();
+        await projectService.init();
+        const testingService = app.get(testing_service_1.TestingService);
+        await testingService.init();
+        await testingService.run();
+    }
+    catch (e) {
+        logger.error(e, 'Testing failed');
+        process.exit(1);
+    }
+    process.exit(0);
+}
+exports.testingInit = testingInit;
+//# sourceMappingURL=testing.init.js.map

package/dist/subcommands/testing.init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing.init.js","sourceRoot":"","sources":["../../src/subcommands/testing.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAAqE;AACrE,gEAA4D;AAC5D,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAC7B,KAAK,UAAU,WAAW;IAC/B,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,EAAE;YAClD,MAAM,EAAE,IAAI,sBAAU,EAAE;SACzB,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAU,CAAC,CAAC;QAEvC,wGAAwG;QACxG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAE5B,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,cAAc,CAAC,GAAG,EAAE,CAAC;KAC5B;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAClC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAtBD,kCAsBC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger, NestLogger, ApiService } from '@subql/node-core';\nimport { ProjectService } from '../indexer/project.service';\nimport { TestingModule } from './testing.module';\nimport { TestingService } from './testing.service';\n\nconst logger = getLogger('Testing');\nexport async function testingInit(): Promise<void> {\n  try {\n    const app = await NestFactory.create(TestingModule, {\n      logger: new NestLogger(),\n    });\n\n    await app.init();\n    const projectService = app.get(ProjectService);\n    const apiService = app.get(ApiService);\n\n    // Initialise async services, we do this here rather than in factories, so we can capture one off events\n    await apiService.init();\n    await projectService.init();\n\n    const testingService = app.get(TestingService);\n    await testingService.init();\n    await testingService.run();\n  } catch (e) {\n    logger.error(e, 'Testing failed');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}

package/dist/subcommands/testing.module.d.ts

@@ -0,0 +1,4 @@
+export declare class TestingFeatureModule {
+}
+export declare class TestingModule {
+}

package/dist/subcommands/testing.module.js

@@ -0,0 +1,77 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestingModule = exports.TestingFeatureModule = void 0;
+const common_1 = require("@nestjs/common");
+const event_emitter_1 = require("@nestjs/event-emitter");
+const schedule_1 = require("@nestjs/schedule");
+const node_core_1 = require("@subql/node-core");
+const configure_module_1 = require("../configure/configure.module");
+const ethereum_1 = require("../ethereum");
+const ds_processor_service_1 = require("../indexer/ds-processor.service");
+const dynamic_ds_service_1 = require("../indexer/dynamic-ds.service");
+const fetch_module_1 = require("../indexer/fetch.module");
+const indexer_manager_1 = require("../indexer/indexer.manager");
+const project_service_1 = require("../indexer/project.service");
+const sandbox_service_1 = require("../indexer/sandbox.service");
+const unfinalizedBlocks_service_1 = require("../indexer/unfinalizedBlocks.service");
+const meta_module_1 = require("../meta/meta.module");
+const testing_service_1 = require("./testing.service");
+let TestingFeatureModule = class TestingFeatureModule {
+};
+TestingFeatureModule = __decorate([
+    (0, common_1.Module)({
+        providers: [
+            node_core_1.StoreService,
+            testing_service_1.TestingService,
+            event_emitter_1.EventEmitter2,
+            node_core_1.PoiService,
+            sandbox_service_1.SandboxService,
+            ds_processor_service_1.DsProcessorService,
+            dynamic_ds_service_1.DynamicDsService,
+            unfinalizedBlocks_service_1.UnfinalizedBlocksService,
+            project_service_1.ProjectService,
+            node_core_1.ConnectionPoolService,
+            {
+                provide: 'IProjectService',
+                useClass: project_service_1.ProjectService,
+            },
+            {
+                provide: node_core_1.ApiService,
+                useFactory: async (project, connectionPoolService, eventEmitter) => {
+                    const apiService = new ethereum_1.EthereumApiService(project, connectionPoolService, eventEmitter);
+                    await apiService.init();
+                    return apiService;
+                },
+                inject: ['ISubqueryProject', node_core_1.ConnectionPoolService, event_emitter_1.EventEmitter2],
+            },
+            indexer_manager_1.IndexerManager,
+        ],
+        imports: [meta_module_1.MetaModule, fetch_module_1.FetchModule],
+        controllers: [],
+    })
+], TestingFeatureModule);
+exports.TestingFeatureModule = TestingFeatureModule;
+let TestingModule = class TestingModule {
+};
+TestingModule = __decorate([
+    (0, common_1.Module)({
+        imports: [
+            node_core_1.DbModule.forRoot(),
+            configure_module_1.ConfigureModule.register(),
+            event_emitter_1.EventEmitterModule.forRoot(),
+            schedule_1.ScheduleModule.forRoot(),
+            TestingFeatureModule,
+        ],
+        controllers: [],
+    })
+], TestingModule);
+exports.TestingModule = TestingModule;
+//# sourceMappingURL=testing.module.js.map

package/dist/subcommands/testing.module.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing.module.js","sourceRoot":"","sources":["../../src/subcommands/testing.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAA0E;AAC1E,+CAAkD;AAClD,gDAM0B;AAC1B,oEAAgE;AAEhE,0CAAiD;AAEjD,0EAAqE;AACrE,sEAAiE;AACjE,0DAAsD;AACtD,gEAA4D;AAC5D,gEAA4D;AAC5D,gEAA4D;AAC5D,oFAAgF;AAChF,qDAAiD;AACjD,uDAAmD;AAyC5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAvChC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ,gCAAc;YACd,6BAAa;YACb,sBAAU;YACV,gCAAc;YACd,yCAAkB;YAClB,qCAAgB;YAChB,oDAAwB;YACxB,gCAAc;YACd,iCAAqB;YACrB;gBACE,OAAO,EAAE,iBAAiB;gBAC1B,QAAQ,EAAE,gCAAc;aACzB;YACD;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,qBAAmE,EACnE,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,6BAAkB,CACvC,OAAO,EACP,qBAAqB,EACrB,YAAY,CACb,CAAC;oBACF,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,iCAAqB,EAAE,6BAAa,CAAC;aACnE;YACD,gCAAc;SACf;QAED,OAAO,EAAE,CAAC,wBAAU,EAAE,0BAAW,CAAC;QAClC,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAY1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IAVzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,kCAAkB,CAAC,OAAO,EAAE;YAC5B,yBAAc,CAAC,OAAO,EAAE;YACxB,oBAAoB;SACrB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2, EventEmitterModule } from '@nestjs/event-emitter';\nimport { ScheduleModule } from '@nestjs/schedule';\nimport {\n  ApiService,\n  ConnectionPoolService,\n  DbModule,\n  PoiService,\n  StoreService,\n} from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum';\nimport { EthereumApiConnection } from '../ethereum/api.connection';\nimport { DsProcessorService } from '../indexer/ds-processor.service';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { FetchModule } from '../indexer/fetch.module';\nimport { IndexerManager } from '../indexer/indexer.manager';\nimport { ProjectService } from '../indexer/project.service';\nimport { SandboxService } from '../indexer/sandbox.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { MetaModule } from '../meta/meta.module';\nimport { TestingService } from './testing.service';\n\n@Module({\n  providers: [\n    StoreService,\n    TestingService,\n    EventEmitter2,\n    PoiService,\n    SandboxService,\n    DsProcessorService,\n    DynamicDsService,\n    UnfinalizedBlocksService,\n    ProjectService,\n    ConnectionPoolService,\n    {\n      provide: 'IProjectService',\n      useClass: ProjectService,\n    },\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        connectionPoolService: ConnectionPoolService<EthereumApiConnection>,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new EthereumApiService(\n          project,\n          connectionPoolService,\n          eventEmitter,\n        );\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2],\n    },\n    IndexerManager,\n  ],\n\n  imports: [MetaModule, FetchModule],\n  controllers: [],\n})\nexport class TestingFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    EventEmitterModule.forRoot(),\n    ScheduleModule.forRoot(),\n    TestingFeatureModule,\n  ],\n  controllers: [],\n})\nexport class TestingModule {}\n"]}

package/dist/subcommands/testing.service.d.ts

@@ -0,0 +1,9 @@
+import { ApiService, NodeConfig, StoreService, TestingService as BaseTestingService } from '@subql/node-core';
+import { EthereumBlockWrapper } from '@subql/types-ethereum';
+import { Sequelize } from 'sequelize';
+import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';
+import { IndexerManager } from '../indexer/indexer.manager';
+export declare class TestingService extends BaseTestingService<EthereumBlockWrapper, SubqlProjectDs> {
+    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, storeService: StoreService, project: SubqueryProject, apiService: ApiService, indexerManager: IndexerManager);
+    indexBlock(block: EthereumBlockWrapper, handler: string): Promise<void>;
+}

package/dist/subcommands/testing.service.js

@@ -0,0 +1,43 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestingService = void 0;
+const common_1 = require("@nestjs/common");
+const node_core_1 = require("@subql/node-core");
+const sequelize_1 = require("sequelize");
+const SubqueryProject_1 = require("../configure/SubqueryProject");
+const indexer_manager_1 = require("../indexer/indexer.manager");
+const logger = (0, node_core_1.getLogger)('subql-testing');
+let TestingService = class TestingService extends node_core_1.TestingService {
+    constructor(sequelize, nodeConfig, storeService, project, apiService, indexerManager) {
+        super(sequelize, nodeConfig, storeService, project, apiService, indexerManager);
+    }
+    async indexBlock(block, handler) {
+        await this.indexerManager.indexBlock(block, this.getDsWithHandler(handler));
+    }
+};
+TestingService = __decorate([
+    (0, common_1.Injectable)(),
+    __param(3, (0, common_1.Inject)('ISubqueryProject')),
+    __metadata("design:paramtypes", [sequelize_1.Sequelize,
+        node_core_1.NodeConfig,
+        node_core_1.StoreService,
+        SubqueryProject_1.SubqueryProject,
+        node_core_1.ApiService,
+        indexer_manager_1.IndexerManager])
+], TestingService);
+exports.TestingService = TestingService;
+//# sourceMappingURL=testing.service.js.map

package/dist/subcommands/testing.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing.service.js","sourceRoot":"","sources":["../../src/subcommands/testing.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAM0B;AAE1B,yCAAsC;AACtC,kEAA+E;AAC/E,gEAA4D;AAE5D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,eAAe,CAAC,CAAC;AAGnC,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,0BAGnC;IACC,YACE,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EACE,OAAwB,EACpD,UAAsB,EACtB,cAA8B;QAE9B,KAAK,CACH,SAAS,EACT,UAAU,EACV,YAAY,EACZ,OAAO,EACP,UAAU,EACV,cAAc,CACf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAA2B,EAC3B,OAAe;QAEf,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9E,CAAC;CACF,CAAA;AA5BY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IASR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHhB,qBAAS;QACR,sBAAU;QACR,wBAAY;QACW,iCAAe;QACxC,sBAAU;QACN,gCAAc;GAVrB,cAAc,CA4B1B;AA5BY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  ApiService,\n  NodeConfig,\n  StoreService,\n  getLogger,\n  TestingService as BaseTestingService,\n} from '@subql/node-core';\nimport { EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { Sequelize } from 'sequelize';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { IndexerManager } from '../indexer/indexer.manager';\n\nconst logger = getLogger('subql-testing');\n\n@Injectable()\nexport class TestingService extends BaseTestingService<\n  EthereumBlockWrapper,\n  SubqlProjectDs\n> {\n  constructor(\n    sequelize: Sequelize,\n    nodeConfig: NodeConfig,\n    storeService: StoreService,\n    @Inject('ISubqueryProject') project: SubqueryProject,\n    apiService: ApiService,\n    indexerManager: IndexerManager,\n  ) {\n    super(\n      sequelize,\n      nodeConfig,\n      storeService,\n      project,\n      apiService,\n      indexerManager,\n    );\n  }\n\n  async indexBlock(\n    block: EthereumBlockWrapper,\n    handler: string,\n  ): Promise<void> {\n    await this.indexerManager.indexBlock(block, this.getDsWithHandler(handler));\n  }\n}\n"]}

package/dist/utils/project.d.ts

@@ -11,3 +11,6 @@ export declare function getChainTypes(reader: Reader, root: string, file: string
 export declare function loadDataSourceScript(reader: Reader, file?: string): Promise<string>;
 export declare function getProjectRoot(reader: Reader): Promise<string>;
 export declare function initDbSchema(project: SubqueryProject, schema: string, storeService: StoreService): Promise<void>;
+export declare function initHotSchemaReload(schema: string, storeService: StoreService): Promise<void>;
+export declare function retryOnFailEth<T>(request: () => Promise<T>, errors?: string[]): Promise<T>;
+export declare function onlyHasLogDataSources(dataSources: SubqlProjectDs[]): boolean;

package/dist/utils/project.js

@@ -5,12 +5,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.initDbSchema = exports.getProjectRoot = exports.loadDataSourceScript = exports.getChainTypes = exports.updateDataSourcesV0_2_0 = exports.isCustomHandler = exports.isBaseHandler = exports.getProjectEntry = exports.prepareProjectDir = void 0;
+exports.onlyHasLogDataSources = exports.retryOnFailEth = exports.initHotSchemaReload = exports.initDbSchema = exports.getProjectRoot = exports.loadDataSourceScript = exports.getChainTypes = exports.updateDataSourcesV0_2_0 = exports.isCustomHandler = exports.isBaseHandler = exports.getProjectEntry = exports.prepareProjectDir = void 0;
 const fs_1 = __importDefault(require("fs"));
 const os_1 = __importDefault(require("os"));
 const path_1 = __importDefault(require("path"));
 const common_1 = require("@subql/common");
 const common_ethereum_1 = require("@subql/common-ethereum");
+const node_core_1 = require("@subql/node-core");
 const utils_1 = require("@subql/utils");
 const js_yaml_1 = __importDefault(require("js-yaml"));
 const tar_1 = __importDefault(require("tar"));
@@ -62,6 +63,27 @@ exports.isCustomHandler = isCustomHandler;
 async function updateDataSourcesV0_2_0(_dataSources, reader, root) {
     // force convert to updated ds
     return Promise.all(_dataSources.map(async (dataSource) => {
+        if (dataSource.kind === 'flare/Runtime') {
+            dataSource.kind = common_ethereum_1.EthereumDatasourceKind.Runtime;
+        }
+        dataSource.mapping.handlers = dataSource.mapping.handlers.map((handler) => {
+            switch (handler.kind) {
+                case 'flare/BlockHandler': {
+                    handler.kind = common_ethereum_1.EthereumHandlerKind.Block;
+                    break;
+                }
+                case 'flare/TransactionHandler': {
+                    handler.kind = common_ethereum_1.EthereumHandlerKind.Call;
+                    break;
+                }
+                case 'flare/LogHandler': {
+                    handler.kind = common_ethereum_1.EthereumHandlerKind.Event;
+                    break;
+                }
+                default:
+            }
+            return handler;
+        });
         const entryScript = await loadDataSourceScript(reader, dataSource.mapping.file);
         const file = await updateDataSourcesEntry(reader, dataSource.mapping.file, root, entryScript);
         if (dataSource.assets) {
@@ -188,4 +210,26 @@ async function initDbSchema(project, schema, storeService) {
     await storeService.init(modelsRelation, schema);
 }
 exports.initDbSchema = initDbSchema;
+async function initHotSchemaReload(schema, storeService) {
+    await storeService.initHotSchemaReloadQueries(schema);
+}
+exports.initHotSchemaReload = initHotSchemaReload;
+const handledErrors = ['timeout'];
+// eslint-disable-next-line @typescript-eslint/require-await
+async function retryOnFailEth(request, errors = handledErrors) {
+    return (0, node_core_1.retryOnFail)(request, (e) => !!errors.find((t) => t === (e === null || e === void 0 ? void 0 : e.reason)));
+}
+exports.retryOnFailEth = retryOnFailEth;
+function onlyHasLogDataSources(dataSources) {
+    for (const ds of dataSources) {
+        for (const handler of ds.mapping.handlers) {
+            if (handler.kind !== common_ethereum_1.SubqlEthereumHandlerKind.EthEvent &&
+                handler.kind !== common_ethereum_1.SubqlEthereumHandlerKind.FlareEvent) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+exports.onlyHasLogDataSources = onlyHasLogDataSources;
 //# sourceMappingURL=project.js.map