@subql/node-ethereum 0.4.1-8 → 0.4.1-storeCache-0.0.3

package/dist/subcommands/reindex.module.js

@@ -14,6 +14,7 @@ const node_core_1 = require("@subql/node-core");
 const configure_module_1 = require("../configure/configure.module");
 const ds_processor_service_1 = require("../indexer/ds-processor.service");
 const dynamic_ds_service_1 = require("../indexer/dynamic-ds.service");
+const unfinalizedBlocks_service_1 = require("../indexer/unfinalizedBlocks.service");
 const forceClean_service_1 = require("./forceClean.service");
 const reindex_service_1 = require("./reindex.service");
 let ReindexFeatureModule = class ReindexFeatureModule {
@@ -21,12 +22,19 @@ let ReindexFeatureModule = class ReindexFeatureModule {
 ReindexFeatureModule = __decorate([
     (0, common_1.Module)({
         providers: [
+            node_core_1.StoreCacheService,
             node_core_1.StoreService,
             reindex_service_1.ReindexService,
             node_core_1.MmrService,
             forceClean_service_1.ForceCleanService,
+            unfinalizedBlocks_service_1.UnfinalizedBlocksService,
             dynamic_ds_service_1.DynamicDsService,
             ds_processor_service_1.DsProcessorService,
+            {
+                // Used to work with DI for unfinalizedBlocksService but not used with reindex
+                provide: node_core_1.ApiService,
+                useFactory: () => undefined,
+            },
         ],
         controllers: [],
     })

package/dist/subcommands/reindex.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAK0B;AAC1B,oEAAgE;AAChE,0EAAqE;AACrE,sEAAiE;AACjE,6DAAyD;AACzD,uDAAmD;AAa5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAXhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ,gCAAc;YACd,sBAAU;YACV,sCAAiB;YACjB,qCAAgB;YAChB,yCAAkB;SACnB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAU1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IARzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;SACrB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  DbModule,\n  MmrService,\n  NodeConfig,\n  StoreService,\n} from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { DsProcessorService } from '../indexer/ds-processor.service';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [\n    StoreService,\n    ReindexService,\n    MmrService,\n    ForceCleanService,\n    DynamicDsService,\n    DsProcessorService,\n  ],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}
+{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,oEAAgE;AAChE,0EAAqE;AACrE,sEAAiE;AACjE,oFAAgF;AAChF,6DAAyD;AACzD,uDAAmD;AAoB5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAlBhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,6BAAiB;YACjB,wBAAY;YACZ,gCAAc;YACd,sBAAU;YACV,sCAAiB;YACjB,oDAAwB;YACxB,qCAAgB;YAChB,yCAAkB;YAClB;gBACE,8EAA8E;gBAC9E,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS;aAC5B;SACF;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAU1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IARzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;SACrB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  ApiService,\n  DbModule,\n  MmrService,\n  StoreCacheService,\n  StoreService,\n} from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { DsProcessorService } from '../indexer/ds-processor.service';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [\n    StoreCacheService,\n    StoreService,\n    ReindexService,\n    MmrService,\n    ForceCleanService,\n    UnfinalizedBlocksService,\n    DynamicDsService,\n    DsProcessorService,\n    {\n      // Used to work with DI for unfinalizedBlocksService but not used with reindex\n      provide: ApiService,\n      useFactory: () => undefined,\n    },\n  ],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}

package/dist/subcommands/reindex.service.d.ts

@@ -2,6 +2,7 @@ import { MmrService, NodeConfig, StoreService } from '@subql/node-core';
 import { Sequelize } from 'sequelize';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { DynamicDsService } from '../indexer/dynamic-ds.service';
+import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
 import { ForceCleanService } from './forceClean.service';
 export declare class ReindexService {
     private readonly sequelize;
@@ -10,11 +11,13 @@ export declare class ReindexService {
     private readonly mmrService;
     private readonly project;
     private readonly forceCleanService;
+    private readonly unfinalizedBlocksService;
     private readonly dynamicDsService;
     private schema;
     private metadataRepo;
-    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, storeService: StoreService, mmrService: MmrService, project: SubqueryProject, forceCleanService: ForceCleanService, dynamicDsService: DynamicDsService);
+    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, storeService: StoreService, mmrService: MmrService, project: SubqueryProject, forceCleanService: ForceCleanService, unfinalizedBlocksService: UnfinalizedBlocksService, dynamicDsService: DynamicDsService);
     init(): Promise<void>;
+    getTargetHeightWithUnfinalizedBlocks(inputHeight: number): Promise<number>;
     private getExistingProjectSchema;
     private getLastProcessedHeight;
     private getMetadataBlockOffset;

package/dist/subcommands/reindex.service.js

@@ -20,18 +20,20 @@ const node_core_1 = require("@subql/node-core");
 const sequelize_1 = require("sequelize");
 const SubqueryProject_1 = require("../configure/SubqueryProject");
 const dynamic_ds_service_1 = require("../indexer/dynamic-ds.service");
+const unfinalizedBlocks_service_1 = require("../indexer/unfinalizedBlocks.service");
 const project_1 = require("../utils/project");
 const reindex_1 = require("../utils/reindex");
 const forceClean_service_1 = require("./forceClean.service");
 const logger = (0, node_core_1.getLogger)('Reindex');
 let ReindexService = class ReindexService {
-    constructor(sequelize, nodeConfig, storeService, mmrService, project, forceCleanService, dynamicDsService) {
+    constructor(sequelize, nodeConfig, storeService, mmrService, project, forceCleanService, unfinalizedBlocksService, dynamicDsService) {
         this.sequelize = sequelize;
         this.nodeConfig = nodeConfig;
         this.storeService = storeService;
         this.mmrService = mmrService;
         this.project = project;
         this.forceCleanService = forceCleanService;
+        this.unfinalizedBlocksService = unfinalizedBlocksService;
         this.dynamicDsService = dynamicDsService;
     }
     async init() {
@@ -41,24 +43,31 @@ let ReindexService = class ReindexService {
             throw new Error('Schema does not exist.');
         }
         await this.initDbSchema();
-        // Should we use an optional arg for multiChain?
-        this.metadataRepo = await (0, node_core_1.MetadataFactory)(this.sequelize, this.schema, this.nodeConfig.multiChain, this.project.network.chainId);
+        this.metadataRepo = this.storeService.storeCache.metadata;
         this.dynamicDsService.init(this.metadataRepo);
     }
+    async getTargetHeightWithUnfinalizedBlocks(inputHeight) {
+        // Why does this happen?
+        this.unfinalizedBlocksService.metadataRepo = this.metadataRepo;
+        const unfinalizedBlocks = await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();
+        const bestBlocks = unfinalizedBlocks.filter(([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight);
+        if (bestBlocks.length === 0) {
+            return inputHeight;
+        }
+        const [firstBestBlock] = bestBlocks[0];
+        return Math.min(inputHeight, firstBestBlock);
+    }
     async getExistingProjectSchema() {
         return (0, node_core_1.getExistingProjectSchema)(this.nodeConfig, this.sequelize);
     }
     async getLastProcessedHeight() {
-        return (0, node_core_1.getMetaDataInfo)(this.metadataRepo, 'lastProcessedHeight');
+        return this.metadataRepo.find('lastProcessedHeight');
     }
     async getMetadataBlockOffset() {
-        return (0, node_core_1.getMetaDataInfo)(this.metadataRepo, 'blockOffset');
+        return this.metadataRepo.find('blockOffset');
     }
     async getMetadataSpecName() {
-        const res = await this.metadataRepo.findOne({
-            where: { key: 'specName' },
-        });
-        return res === null || res === void 0 ? void 0 : res.value;
+        return this.metadataRepo.find('specName');
     }
     async initDbSchema() {
         await (0, project_1.initDbSchema)(this.project, this.schema, this.storeService);
@@ -80,7 +89,8 @@ let ReindexService = class ReindexService {
             this.getStartBlockFromDataSources(),
             this.getLastProcessedHeight(),
         ]);
-        return (0, reindex_1.reindex)(startHeight, await this.getMetadataBlockOffset(), targetBlockHeight, lastProcessedHeight, this.storeService, this.dynamicDsService, this.mmrService, this.sequelize, this.forceCleanService);
+        await (0, reindex_1.reindex)(startHeight, await this.getMetadataBlockOffset(), targetBlockHeight, lastProcessedHeight, this.storeService, this.unfinalizedBlocksService, this.dynamicDsService, this.mmrService, this.sequelize, this.forceCleanService);
+        await this.storeService.storeCache.flushCache();
     }
 };
 ReindexService = __decorate([
@@ -92,6 +102,7 @@ ReindexService = __decorate([
         node_core_1.MmrService,
         SubqueryProject_1.SubqueryProject,
         forceClean_service_1.ForceCleanService,
+        unfinalizedBlocks_service_1.UnfinalizedBlocksService,
         dynamic_ds_service_1.DynamicDsService])
 ], ReindexService);
 exports.ReindexService = ReindexService;

package/dist/subcommands/reindex.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAS0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAwB,EACpD,iBAAoC,EACpC,gBAAkC;QANlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACpD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,gDAAgD;QAChD,IAAI,CAAC,YAAY,GAAG,MAAM,IAAA,2BAAe,EACvC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAU,CAAC,UAAU,EAC1B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAC7B,CAAC;QACF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE;SAC3B,CAAC,CAAC;QACH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,4BAA4B;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,OAAO,IAAA,iBAAO,EACZ,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;IACJ,CAAC;CACF,CAAA;AAxFY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAJC,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACe,iCAAe;QACjC,sCAAiB;QAClB,qCAAgB;GAX1C,cAAc,CAwF1B;AAxFY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MetadataFactory,\n  MetadataRepo,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: MetadataRepo;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n    // Should we use an optional arg for multiChain?\n    this.metadataRepo = await MetadataFactory(\n      this.sequelize,\n      this.schema,\n      this.nodeConfig.multiChain,\n      this.project.network.chainId,\n    );\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    const res = await this.metadataRepo.findOne({\n      where: { key: 'specName' },\n    });\n    return res?.value as string | undefined;\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  private async getStartBlockFromDataSources() {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    return reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n  }\n}\n"]}
+{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAO0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,oFAAgF;AAChF,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAwB,EACpD,iBAAoC,EACpC,wBAAkD,EAClD,gBAAkC;QAPlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACpD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAE1D,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,oCAAoC,CACxC,WAAmB;QAEnB,wBAAwB;QACvB,IAAI,CAAC,wBAAgC,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACxE,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,wBAAwB,CAAC,4BAA4B,EAAE,CAAC;QACrE,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CACzC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAC9D,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,OAAO,WAAW,CAAC;SACpB;QACD,MAAM,CAAC,cAAc,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,4BAA4B;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,MAAM,IAAA,iBAAO,EACX,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,wBAAwB,EAC7B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;QACF,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;IAClD,CAAC;CACF,CAAA;AArGY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAJC,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACe,iCAAe;QACjC,sCAAiB;QACV,oDAAwB;QAChC,qCAAgB;GAZ1C,cAAc,CAqG1B;AArGY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  CacheMetadataModel,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: CacheMetadataModel;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly unfinalizedBlocksService: UnfinalizedBlocksService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n\n    this.metadataRepo = this.storeService.storeCache.metadata;\n\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  async getTargetHeightWithUnfinalizedBlocks(\n    inputHeight: number,\n  ): Promise<number> {\n    // Why does this happen?\n    (this.unfinalizedBlocksService as any).metadataRepo = this.metadataRepo;\n    const unfinalizedBlocks =\n      await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();\n    const bestBlocks = unfinalizedBlocks.filter(\n      ([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight,\n    );\n    if (bestBlocks.length === 0) {\n      return inputHeight;\n    }\n    const [firstBestBlock] = bestBlocks[0];\n    return Math.min(inputHeight, firstBestBlock);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.metadataRepo.find('lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.metadataRepo.find('blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    return this.metadataRepo.find('specName');\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  private async getStartBlockFromDataSources() {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    await reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlocksService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n    await this.storeService.storeCache.flushCache();\n  }\n}\n"]}

package/dist/utils/project.d.ts

@@ -13,3 +13,4 @@ export declare function getProjectRoot(reader: Reader): Promise<string>;
 export declare function initDbSchema(project: SubqueryProject, schema: string, storeService: StoreService): Promise<void>;
 export declare function initHotSchemaReload(schema: string, storeService: StoreService): Promise<void>;
 export declare function retryOnFailEth<T>(request: () => Promise<T>, errors?: string[]): Promise<T>;
+export declare function onlyHasLogDataSources(dataSources: SubqlProjectDs[]): boolean;

package/dist/utils/project.js

@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.retryOnFailEth = exports.initHotSchemaReload = exports.initDbSchema = exports.getProjectRoot = exports.loadDataSourceScript = exports.getChainTypes = exports.updateDataSourcesV0_2_0 = exports.isCustomHandler = exports.isBaseHandler = exports.getProjectEntry = exports.prepareProjectDir = void 0;
+exports.onlyHasLogDataSources = exports.retryOnFailEth = exports.initHotSchemaReload = exports.initDbSchema = exports.getProjectRoot = exports.loadDataSourceScript = exports.getChainTypes = exports.updateDataSourcesV0_2_0 = exports.isCustomHandler = exports.isBaseHandler = exports.getProjectEntry = exports.prepareProjectDir = void 0;
 const fs_1 = __importDefault(require("fs"));
 const os_1 = __importDefault(require("os"));
 const path_1 = __importDefault(require("path"));
@@ -220,4 +220,16 @@ async function retryOnFailEth(request, errors = handledErrors) {
     return (0, node_core_1.retryOnFail)(request, (e) => !!errors.find((t) => t === (e === null || e === void 0 ? void 0 : e.reason)));
 }
 exports.retryOnFailEth = retryOnFailEth;
+function onlyHasLogDataSources(dataSources) {
+    for (const ds of dataSources) {
+        for (const handler of ds.mapping.handlers) {
+            if (handler.kind !== common_ethereum_1.SubqlEthereumHandlerKind.EthEvent &&
+                handler.kind !== common_ethereum_1.SubqlEthereumHandlerKind.FlareEvent) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+exports.onlyHasLogDataSources = onlyHasLogDataSources;
 //# sourceMappingURL=project.js.map

package/dist/utils/project.js.map

@@ -1 +1 @@
-{"version":3,"file":"project.js","sourceRoot":"","sources":["../../src/utils/project.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;AAEtC,4CAAoB;AACpB,4CAAoB;AACpB,gDAAwB;AACxB,0CAA8E;AAC9E,4DAagC;AAChC,gDAA6D;AAC7D,wCAAuD;AACvD,sDAA2B;AAC3B,8CAAsB;AAGf,KAAK,UAAU,iBAAiB,CAAC,WAAmB;IACzD,MAAM,KAAK,GAAG,YAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;QAClB,MAAM,GAAG,GAAG,cAAI,CAAC,GAAG,CAAC;QACrB,MAAM,MAAM,GAAG,YAAE,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,YAAE,CAAC,WAAW,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;QACnD,oDAAoD;QACpD,MAAM,aAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,OAAO,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;KACpC;SAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;QAC9B,OAAO,WAAW,CAAC;KACpB;AACH,CAAC;AAZD,8CAYC;AAED,gDAAgD;AAChD,MAAM,iBAAiB,GAA2B,EAAE,CAAC;AAErD,SAAgB,eAAe,CAAC,IAAY;IAC1C,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAChD,IAAI;QACF,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE;YAC/B,MAAM,OAAO,GAAG,YAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;gBACb,OAAO,QAAQ,CAAC;aACjB;YACD,iBAAiB,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACpD,CAAC,CAAC,GAAG,CAAC,IAAI;gBACV,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;SACrB;QAED,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;KACnC;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAI,KAAK,CAAC,8CAA8C,IAAI,EAAE,CAAC,CAAC;KACvE;AACH,CAAC;AAlBD,0CAkBC;AAED,SAAgB,aAAa,CAC3B,OAAqB;IAErB,OAAO,MAAM,CAAC,MAAM,CAAS,qCAAmB,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC;AAJD,sCAIC;AAED,SAAgB,eAAe,CAC7B,OAAqB;IAErB,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC;AAJD,0CAIC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,YAAkE,EAClE,MAAc,EACd,IAAY;IAEZ,8BAA8B;IAC9B,OAAO,OAAO,CAAC,GAAG,CAChB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;QACpC,IAAI,UAAU,CAAC,IAAI,KAAK,eAAe,EAAE;YACvC,UAAU,CAAC,IAAI,GAAG,wCAAsB,CAAC,OAAO,CAAC;SAClD;QACD,UAAU,CAAC,OAAO,CAAC,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAC3D,CAAC,OAAO,EAAE,EAAE;YACV,QAAQ,OAAO,CAAC,IAAI,EAAE;gBACpB,KAAK,oBAAoB,CAAC,CAAC;oBACzB,OAAO,CAAC,IAAI,GAAG,qCAAmB,CAAC,KAAK,CAAC;oBACzC,MAAM;iBACP;gBACD,KAAK,0BAA0B,CAAC,CAAC;oBAC/B,OAAO,CAAC,IAAI,GAAG,qCAAmB,CAAC,IAAI,CAAC;oBACxC,MAAM;iBACP;gBACD,KAAK,kBAAkB,CAAC,CAAC;oBACvB,OAAO,CAAC,IAAI,GAAG,qCAAmB,CAAC,KAAK,CAAC;oBACzC,MAAM;iBACP;gBACD,QAAQ;aACT;YAED,OAAO,OAAO,CAAC;QACjB,CAAC,CACF,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAC5C,MAAM,EACN,UAAU,CAAC,OAAO,CAAC,IAAI,CACxB,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,sBAAsB,CACvC,MAAM,EACN,UAAU,CAAC,OAAO,CAAC,IAAI,EACvB,IAAI,EACJ,WAAW,CACZ,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,EAAE;YACrB,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;gBACzD,IAAI,MAAM,YAAY,oBAAW,EAAE;oBACjC,KAAK,CAAC,IAAI,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;iBAC7C;qBAAM;oBACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC7C,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAC7B,IAAI,EACJ,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAClC,CAAC;oBACF,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAa,CAAC,CAAC;oBACvD,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;iBACzB;aACF;SACF;QACD,IAAI,IAAA,4BAAU,EAAC,UAAU,CAAC,EAAE;YAC1B,IAAI,UAAU,CAAC,SAAS,EAAE;gBACxB,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,MAAM,eAAe,CAC/C,MAAM,EACN,IAAI,EACJ,UAAU,CAAC,SAAS,CAAC,IAAI,CAC1B,CAAC;aACH;YACD,IAAI,UAAU,CAAC,MAAM,EAAE;gBACrB,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE;oBACzC,IAAI,MAAM,YAAY,oBAAW,EAAE;wBACjC,KAAK,CAAC,IAAI,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;qBAC7C;yBAAM;wBACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAC7C,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAC7B,IAAI,EACJ,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAClC,CAAC;wBACF,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAa,CAAC,CAAC;wBACvD,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;qBACzB;iBACF;aACF;YACD,uCACK,UAAU,KACb,OAAO,kCAAO,UAAU,CAAC,OAAO,KAAE,WAAW,EAAE,IAAI,OACnD;SACH;aAAM;YACL,uCACK,UAAU,KACb,OAAO,kCAAO,UAAU,CAAC,OAAO,KAAE,WAAW,EAAE,IAAI,OACnD;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AA7FD,0DA6FC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,IAAY,EACZ,IAAY,EACZ,MAAc;IAEd,IAAI,MAAM,YAAY,oBAAW;QAAE,OAAO,IAAI,CAAC;SAC1C,IAAI,MAAM,YAAY,mBAAU,IAAI,MAAM,YAAY,qBAAY,EAAE;QACvE,MAAM,UAAU,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3E,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChD,OAAO,UAAU,CAAC;KACnB;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,MAAc,EACd,IAAY,EACZ,IAAY;IAEZ,IAAI,MAAM,YAAY,oBAAW,EAAE;QACjC,OAAO,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;KACjC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,UAAU,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3E,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAC7C,OAAO,UAAU,CAAC;KACnB;AACH,CAAC;AAEM,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,IAAY,EACZ,IAAY;IAEZ,8DAA8D;IAC9D,IAAI,MAAM,YAAY,oBAAW,EAAE;QACjC,OAAO,IAAA,gCAAc,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC;KACnC;SAAM;QACL,oGAAoG;QACpG,0FAA0F;QAC1F,0FAA0F;QAC1F,6FAA6F;QAC7F,qEAAqE;QACrE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,GAAY,CAAC;QACjB,IAAI;YACF,GAAG,GAAG,iBAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,OAAO,IAAA,iCAAe,EAAC,GAAG,CAAC,CAAC;SAC7B;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,cAAc,GAAG,GAAG,cAAI,CAAC,OAAO,CACpC,IAAI,EACJ,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAC5B,KAAK,CAAC;YACP,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YACjD,GAAG,GAAG,IAAA,sCAAoB,EAAC,cAAc,CAAC,CAAC,CAAC,mDAAmD;YAC/F,OAAO,IAAA,iCAAe,EAAC,GAAG,CAAC,CAAC;SAC7B;KACF;AACH,CAAC;AA7BD,sCA6BC;AAEM,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,IAAa;IAEb,IAAI,KAAa,CAAC;IAClB,6BAA6B;IAC7B,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC5E,IAAI,GAAG,CAAC,IAAI,EAAE;YACZ,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;SAChE;aAAM;YACL,KAAK,GAAG,QAAQ,CAAC;SAClB;KACF;IACD,eAAe;IACf,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,2BAA2B,CAAC,CAAC;KACjE;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AArBD,oDAqBC;AAED,KAAK,UAAU,WAAW;IACxB,MAAM,GAAG,GAAG,cAAI,CAAC,GAAG,CAAC;IACrB,MAAM,MAAM,GAAG,YAAE,CAAC,MAAM,EAAE,CAAC;IAC3B,OAAO,YAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;AAChD,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,MAAc;IACjD,IAAI,MAAM,YAAY,oBAAW;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC;IACtD,IAAI,MAAM,YAAY,mBAAU,IAAI,MAAM,YAAY,qBAAY,EAAE;QAClE,OAAO,WAAW,EAAE,CAAC;KACtB;AACH,CAAC;AALD,wCAKC;AAEM,KAAK,UAAU,YAAY,CAChC,OAAwB,EACxB,MAAc,EACd,YAA0B;IAE1B,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAPD,oCAOC;AAEM,KAAK,UAAU,mBAAmB,CACvC,MAAc,EACd,YAA0B;IAE1B,MAAM,YAAY,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AALD,kDAKC;AAED,MAAM,aAAa,GAAG,CAAC,SAAS,CAAC,CAAC;AAClC,4DAA4D;AACrD,KAAK,UAAU,cAAc,CAClC,OAAyB,EACzB,MAAM,GAAG,aAAa;IAEtB,OAAO,IAAA,uBAAW,EAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAK,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,MAAM,CAAA,CAAC,CAAC,CAAC;AAC5E,CAAC;AALD,wCAKC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport os from 'os';\nimport path from 'path';\nimport { GithubReader, IPFSReader, LocalReader, Reader } from '@subql/common';\nimport {\n  ChainTypes,\n  CustomDatasourceV0_2_0,\n  isCustomDs,\n  loadChainTypes,\n  loadChainTypesFromJs,\n  parseChainTypes,\n  RuntimeDataSourceV0_2_0,\n  SubqlRuntimeHandler,\n  SubqlCustomHandler,\n  SubqlHandler,\n  EthereumHandlerKind,\n  EthereumDatasourceKind,\n} from '@subql/common-ethereum';\nimport { retryOnFail, StoreService } from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport yaml from 'js-yaml';\nimport tar from 'tar';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\n\nexport async function prepareProjectDir(projectPath: string): Promise<string> {\n  const stats = fs.statSync(projectPath);\n  if (stats.isFile()) {\n    const sep = path.sep;\n    const tmpDir = os.tmpdir();\n    const tempPath = fs.mkdtempSync(`${tmpDir}${sep}`);\n    // Will promote errors if incorrect format/extension\n    await tar.x({ file: projectPath, cwd: tempPath });\n    return tempPath.concat('/package');\n  } else if (stats.isDirectory()) {\n    return projectPath;\n  }\n}\n\n// We cache this to avoid repeated reads from fs\nconst projectEntryCache: Record<string, string> = {};\n\nexport function getProjectEntry(root: string): string {\n  const pkgPath = path.join(root, 'package.json');\n  try {\n    if (!projectEntryCache[pkgPath]) {\n      const content = fs.readFileSync(pkgPath).toString();\n      const pkg = JSON.parse(content);\n      if (!pkg.main) {\n        return './dist';\n      }\n      projectEntryCache[pkgPath] = pkg.main.startsWith('./')\n        ? pkg.main\n        : `./${pkg.main}`;\n    }\n\n    return projectEntryCache[pkgPath];\n  } catch (err) {\n    throw new Error(`can not find package.json within directory ${root}`);\n  }\n}\n\nexport function isBaseHandler(\n  handler: SubqlHandler,\n): handler is SubqlRuntimeHandler {\n  return Object.values<string>(EthereumHandlerKind).includes(handler.kind);\n}\n\nexport function isCustomHandler(\n  handler: SubqlHandler,\n): handler is SubqlCustomHandler {\n  return !isBaseHandler(handler);\n}\n\nexport async function updateDataSourcesV0_2_0(\n  _dataSources: (RuntimeDataSourceV0_2_0 | CustomDatasourceV0_2_0)[],\n  reader: Reader,\n  root: string,\n): Promise<SubqlProjectDs[]> {\n  // force convert to updated ds\n  return Promise.all(\n    _dataSources.map(async (dataSource) => {\n      if (dataSource.kind === 'flare/Runtime') {\n        dataSource.kind = EthereumDatasourceKind.Runtime;\n      }\n      dataSource.mapping.handlers = dataSource.mapping.handlers.map(\n        (handler) => {\n          switch (handler.kind) {\n            case 'flare/BlockHandler': {\n              handler.kind = EthereumHandlerKind.Block;\n              break;\n            }\n            case 'flare/TransactionHandler': {\n              handler.kind = EthereumHandlerKind.Call;\n              break;\n            }\n            case 'flare/LogHandler': {\n              handler.kind = EthereumHandlerKind.Event;\n              break;\n            }\n            default:\n          }\n\n          return handler;\n        },\n      );\n\n      const entryScript = await loadDataSourceScript(\n        reader,\n        dataSource.mapping.file,\n      );\n      const file = await updateDataSourcesEntry(\n        reader,\n        dataSource.mapping.file,\n        root,\n        entryScript,\n      );\n      if (dataSource.assets) {\n        for (const [, asset] of Object.entries(dataSource.assets)) {\n          if (reader instanceof LocalReader) {\n            asset.file = path.resolve(root, asset.file);\n          } else {\n            const res = await reader.getFile(asset.file);\n            const outputPath = path.resolve(\n              root,\n              asset.file.replace('ipfs://', ''),\n            );\n            await fs.promises.writeFile(outputPath, res as string);\n            asset.file = outputPath;\n          }\n        }\n      }\n      if (isCustomDs(dataSource)) {\n        if (dataSource.processor) {\n          dataSource.processor.file = await updateProcessor(\n            reader,\n            root,\n            dataSource.processor.file,\n          );\n        }\n        if (dataSource.assets) {\n          for (const [, asset] of dataSource.assets) {\n            if (reader instanceof LocalReader) {\n              asset.file = path.resolve(root, asset.file);\n            } else {\n              const res = await reader.getFile(asset.file);\n              const outputPath = path.resolve(\n                root,\n                asset.file.replace('ipfs://', ''),\n              );\n              await fs.promises.writeFile(outputPath, res as string);\n              asset.file = outputPath;\n            }\n          }\n        }\n        return {\n          ...dataSource,\n          mapping: { ...dataSource.mapping, entryScript, file },\n        };\n      } else {\n        return {\n          ...dataSource,\n          mapping: { ...dataSource.mapping, entryScript, file },\n        };\n      }\n    }),\n  );\n}\n\nasync function updateDataSourcesEntry(\n  reader: Reader,\n  file: string,\n  root: string,\n  script: string,\n): Promise<string> {\n  if (reader instanceof LocalReader) return file;\n  else if (reader instanceof IPFSReader || reader instanceof GithubReader) {\n    const outputPath = `${path.resolve(root, file.replace('ipfs://', ''))}.js`;\n    await fs.promises.writeFile(outputPath, script);\n    return outputPath;\n  }\n}\n\nasync function updateProcessor(\n  reader: Reader,\n  root: string,\n  file: string,\n): Promise<string> {\n  if (reader instanceof LocalReader) {\n    return path.resolve(root, file);\n  } else {\n    const res = await reader.getFile(file);\n    const outputPath = `${path.resolve(root, file.replace('ipfs://', ''))}.js`;\n    await fs.promises.writeFile(outputPath, res);\n    return outputPath;\n  }\n}\n\nexport async function getChainTypes(\n  reader: Reader,\n  root: string,\n  file: string,\n): Promise<ChainTypes> {\n  // If the project is load from local, we will direct load them\n  if (reader instanceof LocalReader) {\n    return loadChainTypes(file, root);\n  } else {\n    // If it is stored in ipfs or other resources, we will use the corresponding reader to read the file\n    // Because ipfs not provide extension of the file, it is difficult to determine its format\n    // We will use yaml.load to try to load the script and parse them to supported chain types\n    // if it failed, we will give it another another attempt, and assume the script written in js\n    // we will download it to a temp folder, and load them within sandbox\n    const res = await reader.getFile(file);\n    let raw: unknown;\n    try {\n      raw = yaml.load(res);\n      return parseChainTypes(raw);\n    } catch (e) {\n      const chainTypesPath = `${path.resolve(\n        root,\n        file.replace('ipfs://', ''),\n      )}.js`;\n      await fs.promises.writeFile(chainTypesPath, res);\n      raw = loadChainTypesFromJs(chainTypesPath); //root not required, as it been packed in single js\n      return parseChainTypes(raw);\n    }\n  }\n}\n\nexport async function loadDataSourceScript(\n  reader: Reader,\n  file?: string,\n): Promise<string> {\n  let entry: string;\n  //For RuntimeDataSourceV0_0_1\n  if (!file) {\n    const pkg = await reader.getPkg();\n    if (pkg === undefined) throw new Error('Project package.json is not found');\n    if (pkg.main) {\n      entry = pkg.main.startsWith('./') ? pkg.main : `./${pkg.main}`;\n    } else {\n      entry = './dist';\n    }\n  }\n  //Else get file\n  const entryScript = await reader.getFile(file ? file : entry);\n  if (entryScript === undefined) {\n    throw new Error(`Entry file ${entry} for datasource not exist`);\n  }\n  return entryScript;\n}\n\nasync function makeTempDir(): Promise<string> {\n  const sep = path.sep;\n  const tmpDir = os.tmpdir();\n  return fs.promises.mkdtemp(`${tmpDir}${sep}`);\n}\n\nexport async function getProjectRoot(reader: Reader): Promise<string> {\n  if (reader instanceof LocalReader) return reader.root;\n  if (reader instanceof IPFSReader || reader instanceof GithubReader) {\n    return makeTempDir();\n  }\n}\n\nexport async function initDbSchema(\n  project: SubqueryProject,\n  schema: string,\n  storeService: StoreService,\n): Promise<void> {\n  const modelsRelation = getAllEntitiesRelations(project.schema);\n  await storeService.init(modelsRelation, schema);\n}\n\nexport async function initHotSchemaReload(\n  schema: string,\n  storeService: StoreService,\n): Promise<void> {\n  await storeService.initHotSchemaReloadQueries(schema);\n}\n\nconst handledErrors = ['timeout'];\n// eslint-disable-next-line @typescript-eslint/require-await\nexport async function retryOnFailEth<T>(\n  request: () => Promise<T>,\n  errors = handledErrors,\n): Promise<T> {\n  return retryOnFail(request, (e) => !!errors.find((t) => t === e?.reason));\n}\n"]}
+{"version":3,"file":"project.js","sourceRoot":"","sources":["../../src/utils/project.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;AAEtC,4CAAoB;AACpB,4CAAoB;AACpB,gDAAwB;AACxB,0CAA8E;AAC9E,4DAcgC;AAChC,gDAA6D;AAC7D,wCAAuD;AACvD,sDAA2B;AAC3B,8CAAsB;AAGf,KAAK,UAAU,iBAAiB,CAAC,WAAmB;IACzD,MAAM,KAAK,GAAG,YAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;QAClB,MAAM,GAAG,GAAG,cAAI,CAAC,GAAG,CAAC;QACrB,MAAM,MAAM,GAAG,YAAE,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,YAAE,CAAC,WAAW,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;QACnD,oDAAoD;QACpD,MAAM,aAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,OAAO,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;KACpC;SAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;QAC9B,OAAO,WAAW,CAAC;KACpB;AACH,CAAC;AAZD,8CAYC;AAED,gDAAgD;AAChD,MAAM,iBAAiB,GAA2B,EAAE,CAAC;AAErD,SAAgB,eAAe,CAAC,IAAY;IAC1C,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAChD,IAAI;QACF,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE;YAC/B,MAAM,OAAO,GAAG,YAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;gBACb,OAAO,QAAQ,CAAC;aACjB;YACD,iBAAiB,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACpD,CAAC,CAAC,GAAG,CAAC,IAAI;gBACV,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;SACrB;QAED,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;KACnC;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAI,KAAK,CAAC,8CAA8C,IAAI,EAAE,CAAC,CAAC;KACvE;AACH,CAAC;AAlBD,0CAkBC;AAED,SAAgB,aAAa,CAC3B,OAAqB;IAErB,OAAO,MAAM,CAAC,MAAM,CAAS,qCAAmB,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC;AAJD,sCAIC;AAED,SAAgB,eAAe,CAC7B,OAAqB;IAErB,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC;AAJD,0CAIC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,YAAkE,EAClE,MAAc,EACd,IAAY;IAEZ,8BAA8B;IAC9B,OAAO,OAAO,CAAC,GAAG,CAChB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;QACpC,IAAI,UAAU,CAAC,IAAI,KAAK,eAAe,EAAE;YACvC,UAAU,CAAC,IAAI,GAAG,wCAAsB,CAAC,OAAO,CAAC;SAClD;QACD,UAAU,CAAC,OAAO,CAAC,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAC3D,CAAC,OAAO,EAAE,EAAE;YACV,QAAQ,OAAO,CAAC,IAAI,EAAE;gBACpB,KAAK,oBAAoB,CAAC,CAAC;oBACzB,OAAO,CAAC,IAAI,GAAG,qCAAmB,CAAC,KAAK,CAAC;oBACzC,MAAM;iBACP;gBACD,KAAK,0BAA0B,CAAC,CAAC;oBAC/B,OAAO,CAAC,IAAI,GAAG,qCAAmB,CAAC,IAAI,CAAC;oBACxC,MAAM;iBACP;gBACD,KAAK,kBAAkB,CAAC,CAAC;oBACvB,OAAO,CAAC,IAAI,GAAG,qCAAmB,CAAC,KAAK,CAAC;oBACzC,MAAM;iBACP;gBACD,QAAQ;aACT;YAED,OAAO,OAAO,CAAC;QACjB,CAAC,CACF,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAC5C,MAAM,EACN,UAAU,CAAC,OAAO,CAAC,IAAI,CACxB,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,sBAAsB,CACvC,MAAM,EACN,UAAU,CAAC,OAAO,CAAC,IAAI,EACvB,IAAI,EACJ,WAAW,CACZ,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,EAAE;YACrB,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;gBACzD,IAAI,MAAM,YAAY,oBAAW,EAAE;oBACjC,KAAK,CAAC,IAAI,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;iBAC7C;qBAAM;oBACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC7C,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAC7B,IAAI,EACJ,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAClC,CAAC;oBACF,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAa,CAAC,CAAC;oBACvD,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;iBACzB;aACF;SACF;QACD,IAAI,IAAA,4BAAU,EAAC,UAAU,CAAC,EAAE;YAC1B,IAAI,UAAU,CAAC,SAAS,EAAE;gBACxB,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,MAAM,eAAe,CAC/C,MAAM,EACN,IAAI,EACJ,UAAU,CAAC,SAAS,CAAC,IAAI,CAC1B,CAAC;aACH;YACD,IAAI,UAAU,CAAC,MAAM,EAAE;gBACrB,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE;oBACzC,IAAI,MAAM,YAAY,oBAAW,EAAE;wBACjC,KAAK,CAAC,IAAI,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;qBAC7C;yBAAM;wBACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAC7C,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAC7B,IAAI,EACJ,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAClC,CAAC;wBACF,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAa,CAAC,CAAC;wBACvD,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;qBACzB;iBACF;aACF;YACD,uCACK,UAAU,KACb,OAAO,kCAAO,UAAU,CAAC,OAAO,KAAE,WAAW,EAAE,IAAI,OACnD;SACH;aAAM;YACL,uCACK,UAAU,KACb,OAAO,kCAAO,UAAU,CAAC,OAAO,KAAE,WAAW,EAAE,IAAI,OACnD;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AA7FD,0DA6FC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,IAAY,EACZ,IAAY,EACZ,MAAc;IAEd,IAAI,MAAM,YAAY,oBAAW;QAAE,OAAO,IAAI,CAAC;SAC1C,IAAI,MAAM,YAAY,mBAAU,IAAI,MAAM,YAAY,qBAAY,EAAE;QACvE,MAAM,UAAU,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3E,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChD,OAAO,UAAU,CAAC;KACnB;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,MAAc,EACd,IAAY,EACZ,IAAY;IAEZ,IAAI,MAAM,YAAY,oBAAW,EAAE;QACjC,OAAO,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;KACjC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,UAAU,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3E,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAC7C,OAAO,UAAU,CAAC;KACnB;AACH,CAAC;AAEM,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,IAAY,EACZ,IAAY;IAEZ,8DAA8D;IAC9D,IAAI,MAAM,YAAY,oBAAW,EAAE;QACjC,OAAO,IAAA,gCAAc,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC;KACnC;SAAM;QACL,oGAAoG;QACpG,0FAA0F;QAC1F,0FAA0F;QAC1F,6FAA6F;QAC7F,qEAAqE;QACrE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,GAAY,CAAC;QACjB,IAAI;YACF,GAAG,GAAG,iBAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,OAAO,IAAA,iCAAe,EAAC,GAAG,CAAC,CAAC;SAC7B;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,cAAc,GAAG,GAAG,cAAI,CAAC,OAAO,CACpC,IAAI,EACJ,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAC5B,KAAK,CAAC;YACP,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YACjD,GAAG,GAAG,IAAA,sCAAoB,EAAC,cAAc,CAAC,CAAC,CAAC,mDAAmD;YAC/F,OAAO,IAAA,iCAAe,EAAC,GAAG,CAAC,CAAC;SAC7B;KACF;AACH,CAAC;AA7BD,sCA6BC;AAEM,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,IAAa;IAEb,IAAI,KAAa,CAAC;IAClB,6BAA6B;IAC7B,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC5E,IAAI,GAAG,CAAC,IAAI,EAAE;YACZ,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;SAChE;aAAM;YACL,KAAK,GAAG,QAAQ,CAAC;SAClB;KACF;IACD,eAAe;IACf,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,2BAA2B,CAAC,CAAC;KACjE;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AArBD,oDAqBC;AAED,KAAK,UAAU,WAAW;IACxB,MAAM,GAAG,GAAG,cAAI,CAAC,GAAG,CAAC;IACrB,MAAM,MAAM,GAAG,YAAE,CAAC,MAAM,EAAE,CAAC;IAC3B,OAAO,YAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;AAChD,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,MAAc;IACjD,IAAI,MAAM,YAAY,oBAAW;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC;IACtD,IAAI,MAAM,YAAY,mBAAU,IAAI,MAAM,YAAY,qBAAY,EAAE;QAClE,OAAO,WAAW,EAAE,CAAC;KACtB;AACH,CAAC;AALD,wCAKC;AAEM,KAAK,UAAU,YAAY,CAChC,OAAwB,EACxB,MAAc,EACd,YAA0B;IAE1B,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAPD,oCAOC;AAEM,KAAK,UAAU,mBAAmB,CACvC,MAAc,EACd,YAA0B;IAE1B,MAAM,YAAY,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AALD,kDAKC;AAED,MAAM,aAAa,GAAG,CAAC,SAAS,CAAC,CAAC;AAClC,4DAA4D;AACrD,KAAK,UAAU,cAAc,CAClC,OAAyB,EACzB,MAAM,GAAG,aAAa;IAEtB,OAAO,IAAA,uBAAW,EAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAK,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,MAAM,CAAA,CAAC,CAAC,CAAC;AAC5E,CAAC;AALD,wCAKC;AAED,SAAgB,qBAAqB,CAAC,WAA6B;IACjE,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;QAC5B,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;YACzC,IACE,OAAO,CAAC,IAAI,KAAK,0CAAwB,CAAC,QAAQ;gBAClD,OAAO,CAAC,IAAI,KAAK,0CAAwB,CAAC,UAAU,EACpD;gBACA,OAAO,KAAK,CAAC;aACd;SACF;KACF;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAbD,sDAaC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport os from 'os';\nimport path from 'path';\nimport { GithubReader, IPFSReader, LocalReader, Reader } from '@subql/common';\nimport {\n  ChainTypes,\n  CustomDatasourceV0_2_0,\n  isCustomDs,\n  parseChainTypes,\n  RuntimeDataSourceV0_2_0,\n  SubqlRuntimeHandler,\n  SubqlCustomHandler,\n  SubqlHandler,\n  EthereumHandlerKind,\n  EthereumDatasourceKind,\n  SubqlEthereumHandlerKind,\n  loadChainTypesFromJs,\n  loadChainTypes,\n} from '@subql/common-ethereum';\nimport { retryOnFail, StoreService } from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport yaml from 'js-yaml';\nimport tar from 'tar';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\n\nexport async function prepareProjectDir(projectPath: string): Promise<string> {\n  const stats = fs.statSync(projectPath);\n  if (stats.isFile()) {\n    const sep = path.sep;\n    const tmpDir = os.tmpdir();\n    const tempPath = fs.mkdtempSync(`${tmpDir}${sep}`);\n    // Will promote errors if incorrect format/extension\n    await tar.x({ file: projectPath, cwd: tempPath });\n    return tempPath.concat('/package');\n  } else if (stats.isDirectory()) {\n    return projectPath;\n  }\n}\n\n// We cache this to avoid repeated reads from fs\nconst projectEntryCache: Record<string, string> = {};\n\nexport function getProjectEntry(root: string): string {\n  const pkgPath = path.join(root, 'package.json');\n  try {\n    if (!projectEntryCache[pkgPath]) {\n      const content = fs.readFileSync(pkgPath).toString();\n      const pkg = JSON.parse(content);\n      if (!pkg.main) {\n        return './dist';\n      }\n      projectEntryCache[pkgPath] = pkg.main.startsWith('./')\n        ? pkg.main\n        : `./${pkg.main}`;\n    }\n\n    return projectEntryCache[pkgPath];\n  } catch (err) {\n    throw new Error(`can not find package.json within directory ${root}`);\n  }\n}\n\nexport function isBaseHandler(\n  handler: SubqlHandler,\n): handler is SubqlRuntimeHandler {\n  return Object.values<string>(EthereumHandlerKind).includes(handler.kind);\n}\n\nexport function isCustomHandler(\n  handler: SubqlHandler,\n): handler is SubqlCustomHandler {\n  return !isBaseHandler(handler);\n}\n\nexport async function updateDataSourcesV0_2_0(\n  _dataSources: (RuntimeDataSourceV0_2_0 | CustomDatasourceV0_2_0)[],\n  reader: Reader,\n  root: string,\n): Promise<SubqlProjectDs[]> {\n  // force convert to updated ds\n  return Promise.all(\n    _dataSources.map(async (dataSource) => {\n      if (dataSource.kind === 'flare/Runtime') {\n        dataSource.kind = EthereumDatasourceKind.Runtime;\n      }\n      dataSource.mapping.handlers = dataSource.mapping.handlers.map(\n        (handler) => {\n          switch (handler.kind) {\n            case 'flare/BlockHandler': {\n              handler.kind = EthereumHandlerKind.Block;\n              break;\n            }\n            case 'flare/TransactionHandler': {\n              handler.kind = EthereumHandlerKind.Call;\n              break;\n            }\n            case 'flare/LogHandler': {\n              handler.kind = EthereumHandlerKind.Event;\n              break;\n            }\n            default:\n          }\n\n          return handler;\n        },\n      );\n\n      const entryScript = await loadDataSourceScript(\n        reader,\n        dataSource.mapping.file,\n      );\n      const file = await updateDataSourcesEntry(\n        reader,\n        dataSource.mapping.file,\n        root,\n        entryScript,\n      );\n      if (dataSource.assets) {\n        for (const [, asset] of Object.entries(dataSource.assets)) {\n          if (reader instanceof LocalReader) {\n            asset.file = path.resolve(root, asset.file);\n          } else {\n            const res = await reader.getFile(asset.file);\n            const outputPath = path.resolve(\n              root,\n              asset.file.replace('ipfs://', ''),\n            );\n            await fs.promises.writeFile(outputPath, res as string);\n            asset.file = outputPath;\n          }\n        }\n      }\n      if (isCustomDs(dataSource)) {\n        if (dataSource.processor) {\n          dataSource.processor.file = await updateProcessor(\n            reader,\n            root,\n            dataSource.processor.file,\n          );\n        }\n        if (dataSource.assets) {\n          for (const [, asset] of dataSource.assets) {\n            if (reader instanceof LocalReader) {\n              asset.file = path.resolve(root, asset.file);\n            } else {\n              const res = await reader.getFile(asset.file);\n              const outputPath = path.resolve(\n                root,\n                asset.file.replace('ipfs://', ''),\n              );\n              await fs.promises.writeFile(outputPath, res as string);\n              asset.file = outputPath;\n            }\n          }\n        }\n        return {\n          ...dataSource,\n          mapping: { ...dataSource.mapping, entryScript, file },\n        };\n      } else {\n        return {\n          ...dataSource,\n          mapping: { ...dataSource.mapping, entryScript, file },\n        };\n      }\n    }),\n  );\n}\n\nasync function updateDataSourcesEntry(\n  reader: Reader,\n  file: string,\n  root: string,\n  script: string,\n): Promise<string> {\n  if (reader instanceof LocalReader) return file;\n  else if (reader instanceof IPFSReader || reader instanceof GithubReader) {\n    const outputPath = `${path.resolve(root, file.replace('ipfs://', ''))}.js`;\n    await fs.promises.writeFile(outputPath, script);\n    return outputPath;\n  }\n}\n\nasync function updateProcessor(\n  reader: Reader,\n  root: string,\n  file: string,\n): Promise<string> {\n  if (reader instanceof LocalReader) {\n    return path.resolve(root, file);\n  } else {\n    const res = await reader.getFile(file);\n    const outputPath = `${path.resolve(root, file.replace('ipfs://', ''))}.js`;\n    await fs.promises.writeFile(outputPath, res);\n    return outputPath;\n  }\n}\n\nexport async function getChainTypes(\n  reader: Reader,\n  root: string,\n  file: string,\n): Promise<ChainTypes> {\n  // If the project is load from local, we will direct load them\n  if (reader instanceof LocalReader) {\n    return loadChainTypes(file, root);\n  } else {\n    // If it is stored in ipfs or other resources, we will use the corresponding reader to read the file\n    // Because ipfs not provide extension of the file, it is difficult to determine its format\n    // We will use yaml.load to try to load the script and parse them to supported chain types\n    // if it failed, we will give it another another attempt, and assume the script written in js\n    // we will download it to a temp folder, and load them within sandbox\n    const res = await reader.getFile(file);\n    let raw: unknown;\n    try {\n      raw = yaml.load(res);\n      return parseChainTypes(raw);\n    } catch (e) {\n      const chainTypesPath = `${path.resolve(\n        root,\n        file.replace('ipfs://', ''),\n      )}.js`;\n      await fs.promises.writeFile(chainTypesPath, res);\n      raw = loadChainTypesFromJs(chainTypesPath); //root not required, as it been packed in single js\n      return parseChainTypes(raw);\n    }\n  }\n}\n\nexport async function loadDataSourceScript(\n  reader: Reader,\n  file?: string,\n): Promise<string> {\n  let entry: string;\n  //For RuntimeDataSourceV0_0_1\n  if (!file) {\n    const pkg = await reader.getPkg();\n    if (pkg === undefined) throw new Error('Project package.json is not found');\n    if (pkg.main) {\n      entry = pkg.main.startsWith('./') ? pkg.main : `./${pkg.main}`;\n    } else {\n      entry = './dist';\n    }\n  }\n  //Else get file\n  const entryScript = await reader.getFile(file ? file : entry);\n  if (entryScript === undefined) {\n    throw new Error(`Entry file ${entry} for datasource not exist`);\n  }\n  return entryScript;\n}\n\nasync function makeTempDir(): Promise<string> {\n  const sep = path.sep;\n  const tmpDir = os.tmpdir();\n  return fs.promises.mkdtemp(`${tmpDir}${sep}`);\n}\n\nexport async function getProjectRoot(reader: Reader): Promise<string> {\n  if (reader instanceof LocalReader) return reader.root;\n  if (reader instanceof IPFSReader || reader instanceof GithubReader) {\n    return makeTempDir();\n  }\n}\n\nexport async function initDbSchema(\n  project: SubqueryProject,\n  schema: string,\n  storeService: StoreService,\n): Promise<void> {\n  const modelsRelation = getAllEntitiesRelations(project.schema);\n  await storeService.init(modelsRelation, schema);\n}\n\nexport async function initHotSchemaReload(\n  schema: string,\n  storeService: StoreService,\n): Promise<void> {\n  await storeService.initHotSchemaReloadQueries(schema);\n}\n\nconst handledErrors = ['timeout'];\n// eslint-disable-next-line @typescript-eslint/require-await\nexport async function retryOnFailEth<T>(\n  request: () => Promise<T>,\n  errors = handledErrors,\n): Promise<T> {\n  return retryOnFail(request, (e) => !!errors.find((t) => t === e?.reason));\n}\n\nexport function onlyHasLogDataSources(dataSources: SubqlProjectDs[]): boolean {\n  for (const ds of dataSources) {\n    for (const handler of ds.mapping.handlers) {\n      if (\n        handler.kind !== SubqlEthereumHandlerKind.EthEvent &&\n        handler.kind !== SubqlEthereumHandlerKind.FlareEvent\n      ) {\n        return false;\n      }\n    }\n  }\n\n  return true;\n}\n"]}

package/dist/utils/reindex.d.ts

@@ -1,5 +1,6 @@
 import { MmrService, StoreService } from '@subql/node-core';
 import { Sequelize } from 'sequelize';
 import { DynamicDsService } from '../indexer/dynamic-ds.service';
+import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
 import { ForceCleanService } from '../subcommands/forceClean.service';
-export declare function reindex(startHeight: number, blockOffset: number | undefined, targetBlockHeight: number, lastProcessedHeight: number, storeService: StoreService, dynamicDsService: DynamicDsService, mmrService: MmrService, sequelize: Sequelize, forceCleanService?: ForceCleanService): Promise<void>;
+export declare function reindex(startHeight: number, blockOffset: number | undefined, targetBlockHeight: number, lastProcessedHeight: number, storeService: StoreService, unfinalizedBlockService: UnfinalizedBlocksService, dynamicDsService: DynamicDsService, mmrService: MmrService, sequelize: Sequelize, forceCleanService?: ForceCleanService): Promise<void>;

package/dist/utils/reindex.js

@@ -5,7 +5,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.reindex = void 0;
 const node_core_1 = require("@subql/node-core");
 const logger = (0, node_core_1.getLogger)('Reindex');
-async function reindex(startHeight, blockOffset, targetBlockHeight, lastProcessedHeight, storeService, dynamicDsService, mmrService, sequelize, forceCleanService) {
+async function reindex(startHeight, blockOffset, targetBlockHeight, lastProcessedHeight, storeService, unfinalizedBlockService, dynamicDsService, mmrService, sequelize, forceCleanService) {
     if (!storeService.historical) {
         logger.warn('Unable to reindex, historical state not enabled');
         return;
@@ -29,11 +29,15 @@ async function reindex(startHeight, blockOffset, targetBlockHeight, lastProcesse
         try {
             await Promise.all([
                 storeService.rewind(targetBlockHeight, transaction),
-                dynamicDsService.resetDynamicDatasource(targetBlockHeight, transaction),
+                unfinalizedBlockService.resetUnfinalizedBlocks(),
+                unfinalizedBlockService.resetLastFinalizedVerifiedHeight(),
+                dynamicDsService.resetDynamicDatasource(targetBlockHeight),
             ]);
             if (blockOffset) {
                 await mmrService.deleteMmrNode(targetBlockHeight + 1, blockOffset);
             }
+            // Flush metadata changes from above Promise.all
+            await storeService.storeCache.metadata.flush(transaction);
             await transaction.commit();
             logger.info('Reindex Success');
         }

package/dist/utils/reindex.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.js","sourceRoot":"","sources":["../../src/utils/reindex.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,gDAAuE;AAKvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAE7B,KAAK,UAAU,OAAO,CAC3B,WAAmB,EACnB,WAA+B,EAC/B,iBAAyB,EACzB,mBAA2B,EAC3B,YAA0B,EAC1B,gBAAkC,EAClC,UAAsB,EACtB,SAAoB,EACpB,iBAAqC;IAErC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE;QAC5B,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAC/D,OAAO;KACR;IACD,IAAI,CAAC,mBAAmB,IAAI,mBAAmB,GAAG,iBAAiB,EAAE;QACnE,MAAM,CAAC,IAAI,CACT,gCAAgC,iBAAiB,6BAA6B,mBAAmB,4BAA4B,CAC9H,CAAC;QACF,OAAO;KACR;IAED,oEAAoE;IACpE,IAAI,iBAAiB,GAAG,WAAW,EAAE;QACnC,MAAM,CAAC,IAAI,CACT,iBAAiB,iBAAiB,8BAA8B,WAAW,+BAA+B,CAC3G,CAAC;QACF,IAAI,CAAC,iBAAiB,EAAE;YACtB,MAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,MAAM,iBAAiB,CAAC,UAAU,EAAE,CAAC;KACtC;SAAM;QACL,MAAM,CAAC,IAAI,CAAC,wBAAwB,iBAAiB,EAAE,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI;YACF,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,YAAY,CAAC,MAAM,CAAC,iBAAiB,EAAE,WAAW,CAAC;gBACnD,gBAAgB,CAAC,sBAAsB,CAAC,iBAAiB,EAAE,WAAW,CAAC;aACxE,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE;gBACf,MAAM,UAAU,CAAC,aAAa,CAAC,iBAAiB,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;aACpE;YACD,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;YACvC,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;YAC7B,MAAM,GAAG,CAAC;SACX;KACF;AACH,CAAC;AApDD,0BAoDC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { getLogger, MmrService, StoreService } from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { ForceCleanService } from '../subcommands/forceClean.service';\n\nconst logger = getLogger('Reindex');\n\nexport async function reindex(\n  startHeight: number,\n  blockOffset: number | undefined,\n  targetBlockHeight: number,\n  lastProcessedHeight: number,\n  storeService: StoreService,\n  dynamicDsService: DynamicDsService,\n  mmrService: MmrService,\n  sequelize: Sequelize,\n  forceCleanService?: ForceCleanService,\n): Promise<void> {\n  if (!storeService.historical) {\n    logger.warn('Unable to reindex, historical state not enabled');\n    return;\n  }\n  if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) {\n    logger.warn(\n      `Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`,\n    );\n    return;\n  }\n\n  // if startHeight is greater than the targetHeight, just force clean\n  if (targetBlockHeight < startHeight) {\n    logger.info(\n      `targetHeight: ${targetBlockHeight} is less than startHeight: ${startHeight}. Hence executing force-clean`,\n    );\n    if (!forceCleanService) {\n      logger.error('ForceCleanService not provided, cannot force clean');\n      process.exit(1);\n    }\n    await forceCleanService.forceClean();\n  } else {\n    logger.info(`Reindexing to block: ${targetBlockHeight}`);\n    const transaction = await sequelize.transaction();\n    try {\n      await Promise.all([\n        storeService.rewind(targetBlockHeight, transaction),\n        dynamicDsService.resetDynamicDatasource(targetBlockHeight, transaction),\n      ]);\n\n      if (blockOffset) {\n        await mmrService.deleteMmrNode(targetBlockHeight + 1, blockOffset);\n      }\n      await transaction.commit();\n      logger.info('Reindex Success');\n    } catch (err) {\n      logger.error(err, 'Reindexing failed');\n      await transaction.rollback();\n      throw err;\n    }\n  }\n}\n"]}
+{"version":3,"file":"reindex.js","sourceRoot":"","sources":["../../src/utils/reindex.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,gDAAuE;AAMvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAE7B,KAAK,UAAU,OAAO,CAC3B,WAAmB,EACnB,WAA+B,EAC/B,iBAAyB,EACzB,mBAA2B,EAC3B,YAA0B,EAC1B,uBAAiD,EACjD,gBAAkC,EAClC,UAAsB,EACtB,SAAoB,EACpB,iBAAqC;IAErC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE;QAC5B,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAC/D,OAAO;KACR;IACD,IAAI,CAAC,mBAAmB,IAAI,mBAAmB,GAAG,iBAAiB,EAAE;QACnE,MAAM,CAAC,IAAI,CACT,gCAAgC,iBAAiB,6BAA6B,mBAAmB,4BAA4B,CAC9H,CAAC;QACF,OAAO;KACR;IAED,oEAAoE;IACpE,IAAI,iBAAiB,GAAG,WAAW,EAAE;QACnC,MAAM,CAAC,IAAI,CACT,iBAAiB,iBAAiB,8BAA8B,WAAW,+BAA+B,CAC3G,CAAC;QACF,IAAI,CAAC,iBAAiB,EAAE;YACtB,MAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,MAAM,iBAAiB,CAAC,UAAU,EAAE,CAAC;KACtC;SAAM;QACL,MAAM,CAAC,IAAI,CAAC,wBAAwB,iBAAiB,EAAE,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI;YACF,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,YAAY,CAAC,MAAM,CAAC,iBAAiB,EAAE,WAAW,CAAC;gBACnD,uBAAuB,CAAC,sBAAsB,EAAE;gBAChD,uBAAuB,CAAC,gCAAgC,EAAE;gBAC1D,gBAAgB,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;aAC3D,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE;gBACf,MAAM,UAAU,CAAC,aAAa,CAAC,iBAAiB,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;aACpE;YAED,gDAAgD;YAChD,MAAM,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAE1D,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;YACvC,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;YAC7B,MAAM,GAAG,CAAC;SACX;KACF;AACH,CAAC;AA3DD,0BA2DC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { getLogger, MmrService, StoreService } from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { ForceCleanService } from '../subcommands/forceClean.service';\n\nconst logger = getLogger('Reindex');\n\nexport async function reindex(\n  startHeight: number,\n  blockOffset: number | undefined,\n  targetBlockHeight: number,\n  lastProcessedHeight: number,\n  storeService: StoreService,\n  unfinalizedBlockService: UnfinalizedBlocksService,\n  dynamicDsService: DynamicDsService,\n  mmrService: MmrService,\n  sequelize: Sequelize,\n  forceCleanService?: ForceCleanService,\n): Promise<void> {\n  if (!storeService.historical) {\n    logger.warn('Unable to reindex, historical state not enabled');\n    return;\n  }\n  if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) {\n    logger.warn(\n      `Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`,\n    );\n    return;\n  }\n\n  // if startHeight is greater than the targetHeight, just force clean\n  if (targetBlockHeight < startHeight) {\n    logger.info(\n      `targetHeight: ${targetBlockHeight} is less than startHeight: ${startHeight}. Hence executing force-clean`,\n    );\n    if (!forceCleanService) {\n      logger.error('ForceCleanService not provided, cannot force clean');\n      process.exit(1);\n    }\n    await forceCleanService.forceClean();\n  } else {\n    logger.info(`Reindexing to block: ${targetBlockHeight}`);\n    const transaction = await sequelize.transaction();\n    try {\n      await Promise.all([\n        storeService.rewind(targetBlockHeight, transaction),\n        unfinalizedBlockService.resetUnfinalizedBlocks(),\n        unfinalizedBlockService.resetLastFinalizedVerifiedHeight(),\n        dynamicDsService.resetDynamicDatasource(targetBlockHeight),\n      ]);\n\n      if (blockOffset) {\n        await mmrService.deleteMmrNode(targetBlockHeight + 1, blockOffset);\n      }\n\n      // Flush metadata changes from above Promise.all\n      await storeService.storeCache.metadata.flush(transaction);\n\n      await transaction.commit();\n      logger.info('Reindex Success');\n    } catch (err) {\n      logger.error(err, 'Reindexing failed');\n      await transaction.rollback();\n      throw err;\n    }\n  }\n}\n"]}

package/dist/utils/string.js

@@ -17,16 +17,24 @@ function hexStringEq(a, b) {
     return stringNormalizedEq((0, bytes_1.hexStripZeros)(a), (0, bytes_1.hexStripZeros)(b));
 }
 exports.hexStringEq = hexStringEq;
+const eventTopicsCache = {};
+const functionSighashCache = {};
 function eventToTopic(input) {
     if ((0, bytes_1.isHexString)(input))
         return input;
-    return (0, hash_1.id)(abi_1.EventFragment.fromString(input).format());
+    if (!eventTopicsCache[input]) {
+        eventTopicsCache[input] = (0, hash_1.id)(abi_1.EventFragment.fromString(input).format());
+    }
+    return eventTopicsCache[input];
 }
 exports.eventToTopic = eventToTopic;
 function functionToSighash(input) {
     if ((0, bytes_1.isHexString)(input))
         return input;
-    return (0, bytes_1.hexDataSlice)((0, hash_1.id)(abi_1.FunctionFragment.fromString(input).format()), 0, 4);
+    if (!functionSighashCache[input]) {
+        functionSighashCache[input] = (0, bytes_1.hexDataSlice)((0, hash_1.id)(abi_1.FunctionFragment.fromString(input).format()), 0, 4);
+    }
+    return functionSighashCache[input];
 }
 exports.functionToSighash = functionToSighash;
 //# sourceMappingURL=string.js.map

package/dist/utils/string.js.map

@@ -1 +1 @@
-{"version":3,"file":"string.js","sourceRoot":"","sources":["../../src/utils/string.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,4CAAqE;AACrE,gDAAgF;AAChF,8CAAyC;AAEzC,SAAgB,kBAAkB,CAAC,CAAS,EAAE,CAAS;IACrD,OAAO,CAAC,CAAC,WAAW,EAAE,MAAK,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,WAAW,EAAE,CAAA,CAAC;AAC9C,CAAC;AAFD,gDAEC;AAED,SAAgB,WAAW,CAAC,CAAS,EAAE,CAAS;IAC9C,IAAI,CAAC,IAAA,mBAAW,EAAC,CAAC,CAAC,IAAI,CAAC,IAAA,mBAAW,EAAC,CAAC,CAAC,EAAE;QACtC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;KAC/C;IACD,OAAO,kBAAkB,CAAC,IAAA,qBAAa,EAAC,CAAC,CAAC,EAAE,IAAA,qBAAa,EAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AALD,kCAKC;AAED,SAAgB,YAAY,CAAC,KAAa;IACxC,IAAI,IAAA,mBAAW,EAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,OAAO,IAAA,SAAE,EAAC,mBAAa,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;AACtD,CAAC;AAJD,oCAIC;AAED,SAAgB,iBAAiB,CAAC,KAAa;IAC7C,IAAI,IAAA,mBAAW,EAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,OAAO,IAAA,oBAAY,EAAC,IAAA,SAAE,EAAC,sBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC7E,CAAC;AAJD,8CAIC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { EventFragment, FunctionFragment } from '@ethersproject/abi';\nimport { isHexString, hexStripZeros, hexDataSlice } from '@ethersproject/bytes';\nimport { id } from '@ethersproject/hash';\n\nexport function stringNormalizedEq(a: string, b: string): boolean {\n  return a.toLowerCase() === b?.toLowerCase();\n}\n\nexport function hexStringEq(a: string, b: string): boolean {\n  if (!isHexString(a) || !isHexString(b)) {\n    throw new Error('Inputs are not hex strings');\n  }\n  return stringNormalizedEq(hexStripZeros(a), hexStripZeros(b));\n}\n\nexport function eventToTopic(input: string): string {\n  if (isHexString(input)) return input;\n\n  return id(EventFragment.fromString(input).format());\n}\n\nexport function functionToSighash(input: string): string {\n  if (isHexString(input)) return input;\n\n  return hexDataSlice(id(FunctionFragment.fromString(input).format()), 0, 4);\n}\n"]}
+{"version":3,"file":"string.js","sourceRoot":"","sources":["../../src/utils/string.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,4CAAqE;AACrE,gDAAgF;AAChF,8CAAyC;AAEzC,SAAgB,kBAAkB,CAAC,CAAS,EAAE,CAAS;IACrD,OAAO,CAAC,CAAC,WAAW,EAAE,MAAK,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,WAAW,EAAE,CAAA,CAAC;AAC9C,CAAC;AAFD,gDAEC;AAED,SAAgB,WAAW,CAAC,CAAS,EAAE,CAAS;IAC9C,IAAI,CAAC,IAAA,mBAAW,EAAC,CAAC,CAAC,IAAI,CAAC,IAAA,mBAAW,EAAC,CAAC,CAAC,EAAE;QACtC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;KAC/C;IACD,OAAO,kBAAkB,CAAC,IAAA,qBAAa,EAAC,CAAC,CAAC,EAAE,IAAA,qBAAa,EAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AALD,kCAKC;AAED,MAAM,gBAAgB,GAA2B,EAAE,CAAC;AACpD,MAAM,oBAAoB,GAA2B,EAAE,CAAC;AAExD,SAAgB,YAAY,CAAC,KAAa;IACxC,IAAI,IAAA,mBAAW,EAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE;QAC5B,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAA,SAAE,EAAC,mBAAa,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;KACxE;IAED,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AARD,oCAQC;AAED,SAAgB,iBAAiB,CAAC,KAAa;IAC7C,IAAI,IAAA,mBAAW,EAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE;QAChC,oBAAoB,CAAC,KAAK,CAAC,GAAG,IAAA,oBAAY,EACxC,IAAA,SAAE,EAAC,sBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,EAC/C,CAAC,EACD,CAAC,CACF,CAAC;KACH;IAED,OAAO,oBAAoB,CAAC,KAAK,CAAC,CAAC;AACrC,CAAC;AAZD,8CAYC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { EventFragment, FunctionFragment } from '@ethersproject/abi';\nimport { isHexString, hexStripZeros, hexDataSlice } from '@ethersproject/bytes';\nimport { id } from '@ethersproject/hash';\n\nexport function stringNormalizedEq(a: string, b: string): boolean {\n  return a.toLowerCase() === b?.toLowerCase();\n}\n\nexport function hexStringEq(a: string, b: string): boolean {\n  if (!isHexString(a) || !isHexString(b)) {\n    throw new Error('Inputs are not hex strings');\n  }\n  return stringNormalizedEq(hexStripZeros(a), hexStripZeros(b));\n}\n\nconst eventTopicsCache: Record<string, string> = {};\nconst functionSighashCache: Record<string, string> = {};\n\nexport function eventToTopic(input: string): string {\n  if (isHexString(input)) return input;\n\n  if (!eventTopicsCache[input]) {\n    eventTopicsCache[input] = id(EventFragment.fromString(input).format());\n  }\n\n  return eventTopicsCache[input];\n}\n\nexport function functionToSighash(input: string): string {\n  if (isHexString(input)) return input;\n\n  if (!functionSighashCache[input]) {\n    functionSighashCache[input] = hexDataSlice(\n      id(FunctionFragment.fromString(input).format()),\n      0,\n      4,\n    );\n  }\n\n  return functionSighashCache[input];\n}\n"]}