@subql/node-ethereum 0.1.1 → 0.2.0

package/dist/indexer/project.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAA4C;AAC5C,yDAAsD;AACtD,gDAY0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAgD;AAChD,iEAA4D;AAC5D,6DAAwD;AAExD,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAMzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACpB,OAAwB,EACxB,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B;QATlB,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACpB,YAAO,GAAP,OAAO,CAAiB;QACxB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;IAClC,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAChD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;SACjD;aAAM;YACL,IAAI,CAAC,YAAY,GAAG,IAAA,2BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACrD,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,YAAY,GAAG,IAAA,2BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAElE,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,SAAS;YACT,qBAAqB;SACb,CAAC;QAEX,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC;YACzC,KAAK,EAAE;gBACL,GAAG,EAAE,IAAI;aACV;SACF,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YAC5C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3B,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAiE,CAAC,CAAC;QAEtE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,YAAY;oBACjB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI;iBACrC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,mBAAmB;oBACxB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO;iBACxC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,aAAa;oBAClB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;iBACtC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,oBAAoB;oBACzB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO;iBACzC,CAAC;aACH,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;SACvE;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QAED,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;SACjE;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SACrE;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,MAAM,YAAY,CAAC,MAAM,CAAC;gBACxB,GAAG,EAAE,oBAAoB;gBACzB,KAAK,EAAE,cAAc;aACtB,CAAC,CAAC;SACJ;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAAc;QAC5C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC7B,GAAG,EAAE,aAAa;YAClB,KAAK,EAAE,MAAM;SACd,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY;YACjB,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU;aACnB,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;aACxC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACP,CAAC;IACD,KAAK,CAAC,sBAAsB;QAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,qBAAqB,EAAE;SACtC,CAAC,CAAC;QAEH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;CACF,CAAA;AAtQY,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAQ4B,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACX,iCAAe;QACV,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;GAhB1B,cAAc,CAsQ1B;AAtQY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  MetadataFactory,\n  MetadataRepo,\n  NodeConfig,\n  IndexerEvent,\n  StoreService,\n  PoiService,\n  MmrService,\n  getLogger,\n  getExistingProjectSchema,\n  getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n  generateTimestampReferenceForBlockFilters,\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema } from '../utils/project';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService {\n  private _schema: string;\n  private metadataRepo: MetadataRepo;\n  private _startHeight: number;\n  private _blockOffset: number;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly apiService: ApiService,\n    private readonly poiService: PoiService,\n    protected readonly mmrService: MmrService,\n    private readonly sequelize: Sequelize,\n    private readonly project: SubqueryProject,\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly dynamicDsService: DynamicDsService,\n    private eventEmitter: EventEmitter2,\n  ) {}\n\n  get schema(): string {\n    return this._schema;\n  }\n\n  get dataSources(): SubqlProjectDs[] {\n    return this.project.dataSources;\n  }\n\n  get blockOffset(): number {\n    return this._blockOffset;\n  }\n\n  get startHeight(): number {\n    return this._startHeight;\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  async init(): Promise<void> {\n    // Used to load assets into DS-processor, has to be done in any thread\n    await this.dsProcessorService.validateProjectCustomDatasources();\n    // Do extra work on main thread to setup stuff\n    this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n      this.project.dataSources,\n      this.apiService.api,\n    );\n    if (isMainThread) {\n      this._schema = await this.ensureProject();\n      await this.initDbSchema();\n      this.metadataRepo = await this.ensureMetadata();\n      this.dynamicDsService.init(this.metadataRepo);\n\n      if (this.nodeConfig.proofOfIndex) {\n        const blockOffset = await this.getMetadataBlockOffset();\n        void this.setBlockOffset(Number(blockOffset));\n        await this.poiService.init(this.schema);\n      }\n\n      this._startHeight = await this.getStartHeight();\n    } else {\n      this.metadataRepo = MetadataFactory(this.sequelize, this.schema);\n\n      this.dynamicDsService.init(this.metadataRepo);\n\n      await this.sequelize.sync();\n\n      this._schema = await this.getExistingProjectSchema();\n      assert(this._schema, 'Schema should be created in main thread');\n      await this.initDbSchema();\n\n      if (this.nodeConfig.proofOfIndex) {\n        await this.poiService.init(this.schema);\n      }\n    }\n  }\n\n  private async ensureProject(): Promise<string> {\n    let schema = await this.getExistingProjectSchema();\n    if (!schema) {\n      schema = await this.createProjectSchema();\n    }\n    this.eventEmitter.emit(IndexerEvent.Ready, {\n      value: true,\n    });\n\n    return schema;\n  }\n\n  private async createProjectSchema(): Promise<string> {\n    let schema: string;\n    if (this.nodeConfig.localMode) {\n      // create tables in default schema if local mode is enabled\n      schema = DEFAULT_DB_SCHEMA;\n    } else {\n      schema = this.nodeConfig.dbSchema;\n      const schemas = await this.sequelize.showAllSchemas(undefined);\n      if (!(schemas as unknown as string[]).includes(schema)) {\n        await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n      }\n    }\n\n    return schema;\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private async ensureMetadata(): Promise<MetadataRepo> {\n    const metadataRepo = MetadataFactory(this.sequelize, this.schema);\n\n    this.eventEmitter.emit(\n      IndexerEvent.NetworkMetadata,\n      this.apiService.networkMeta,\n    );\n\n    const keys = [\n      'lastProcessedHeight',\n      'blockOffset',\n      'indexerNodeVersion',\n      'chain',\n      'specName',\n      'genesisHash',\n      'chainId',\n      'processedBlockCount',\n    ] as const;\n\n    const entries = await metadataRepo.findAll({\n      where: {\n        key: keys,\n      },\n    });\n\n    const keyValue = entries.reduce((arr, curr) => {\n      arr[curr.key] = curr.value;\n      return arr;\n    }, {} as { [key in typeof keys[number]]: string | boolean | number });\n\n    const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n    if (this.project.runner) {\n      await Promise.all([\n        metadataRepo.upsert({\n          key: 'runnerNode',\n          value: this.project.runner.node.name,\n        }),\n        metadataRepo.upsert({\n          key: 'runnerNodeVersion',\n          value: this.project.runner.node.version,\n        }),\n        metadataRepo.upsert({\n          key: 'runnerQuery',\n          value: this.project.runner.query.name,\n        }),\n        metadataRepo.upsert({\n          key: 'runnerQueryVersion',\n          value: this.project.runner.query.version,\n        }),\n      ]);\n    }\n    if (!keyValue.genesisHash) {\n      await metadataRepo.upsert({ key: 'genesisHash', value: genesisHash });\n    } else {\n      // Check if the configured genesisHash matches the currently stored genesisHash\n      assert(\n        // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n        (this.project.network.genesisHash ?? genesisHash) ===\n          keyValue.genesisHash,\n        'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n      );\n    }\n\n    if (keyValue.chain !== chain) {\n      await metadataRepo.upsert({ key: 'chain', value: chain });\n    }\n\n    if (keyValue.specName !== specName) {\n      await metadataRepo.upsert({ key: 'specName', value: specName });\n    }\n\n    // If project was created before this feature, don't add the key. If it is project created after, add this key.\n    if (!keyValue.processedBlockCount && !keyValue.lastProcessedHeight) {\n      await metadataRepo.upsert({ key: 'processedBlockCount', value: 0 });\n    }\n\n    if (keyValue.indexerNodeVersion !== packageVersion) {\n      await metadataRepo.upsert({\n        key: 'indexerNodeVersion',\n        value: packageVersion,\n      });\n    }\n\n    return metadataRepo;\n  }\n\n  async upsertMetadataBlockOffset(height: number): Promise<void> {\n    await this.metadataRepo.upsert({\n      key: 'blockOffset',\n      value: height,\n    });\n  }\n\n  async getMetadataBlockOffset(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n  }\n\n  async getLastProcessedHeight(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n  }\n\n  private async getStartHeight(): Promise<number> {\n    let startHeight: number;\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n    if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n      startHeight = Number(lastProcessedHeight) + 1;\n    } else {\n      startHeight = this.getStartBlockFromDataSources();\n    }\n\n    return startHeight;\n  }\n\n  async setBlockOffset(offset: number): Promise<void> {\n    if (\n      this._blockOffset ||\n      offset === null ||\n      offset === undefined ||\n      isNaN(offset)\n    ) {\n      return;\n    }\n    logger.info(`set blockOffset to ${offset}`);\n    this._blockOffset = offset;\n    return this.mmrService\n      .syncFileBaseFromPoi(this.schema, offset)\n      .catch((err) => {\n        logger.error(err, 'failed to sync poi to mmr');\n        process.exit(1);\n      });\n  }\n  async getProcessedBlockCount(): Promise<number> {\n    const res = await this.metadataRepo.findOne({\n      where: { key: 'processedBlockCount' },\n    });\n\n    return res?.value as number | undefined;\n  }\n\n  private getStartBlockFromDataSources() {\n    const startBlocksList = this.project.dataSources.map(\n      (item) => item.startBlock ?? 1,\n    );\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n}\n"]}
+{"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAA4C;AAC5C,yDAAsD;AACtD,gDAY0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAgD;AAChD,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AAExD,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAMzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACpB,OAAwB,EACxB,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B;QATlB,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACpB,YAAO,GAAP,OAAO,CAAiB;QACxB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;IAClC,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAChD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;SACjD;aAAM;YACL,IAAI,CAAC,YAAY,GAAG,IAAA,2BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE9C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACrD,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACzC;SACF;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,YAAY,GAAG,IAAA,2BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAElE,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,SAAS;YACT,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;SACd,CAAC;QAEX,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC;YACzC,KAAK,EAAE;gBACL,GAAG,EAAE,IAAI;aACV;SACF,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YAC5C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3B,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAiE,CAAC,CAAC;QAEtE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,YAAY;oBACjB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI;iBACrC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,mBAAmB;oBACxB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO;iBACxC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,aAAa;oBAClB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;iBACtC,CAAC;gBACF,YAAY,CAAC,MAAM,CAAC;oBAClB,GAAG,EAAE,oBAAoB;oBACzB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO;iBACzC,CAAC;aACH,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;SACvE;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QAED,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;SACjE;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SACrE;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,MAAM,YAAY,CAAC,MAAM,CAAC;gBACxB,GAAG,EAAE,oBAAoB;gBACzB,KAAK,EAAE,cAAc;aACtB,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,MAAM,YAAY,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;SACtE;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAAc;QAC5C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC7B,GAAG,EAAE,aAAa;YAClB,KAAK,EAAE,MAAM;SACd,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY;YACjB,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU;aACnB,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;aACxC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACP,CAAC;IACD,KAAK,CAAC,sBAAsB;QAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,qBAAqB,EAAE;SACtC,CAAC,CAAC;QAEH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;CAEF,CAAA;AAvSY,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAQ4B,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACX,iCAAe;QACV,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;GAhB1B,cAAc,CAuS1B;AAvSY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  MetadataFactory,\n  MetadataRepo,\n  NodeConfig,\n  IndexerEvent,\n  StoreService,\n  PoiService,\n  MmrService,\n  getLogger,\n  getExistingProjectSchema,\n  getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n  generateTimestampReferenceForBlockFilters,\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService {\n  private _schema: string;\n  private metadataRepo: MetadataRepo;\n  private _startHeight: number;\n  private _blockOffset: number;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly apiService: ApiService,\n    private readonly poiService: PoiService,\n    protected readonly mmrService: MmrService,\n    private readonly sequelize: Sequelize,\n    private readonly project: SubqueryProject,\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly dynamicDsService: DynamicDsService,\n    private eventEmitter: EventEmitter2,\n  ) {}\n\n  get schema(): string {\n    return this._schema;\n  }\n\n  get dataSources(): SubqlProjectDs[] {\n    return this.project.dataSources;\n  }\n\n  get blockOffset(): number {\n    return this._blockOffset;\n  }\n\n  get startHeight(): number {\n    return this._startHeight;\n  }\n\n  get isHistorical(): boolean {\n    return this.storeService.historical;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  async init(): Promise<void> {\n    // Used to load assets into DS-processor, has to be done in any thread\n    await this.dsProcessorService.validateProjectCustomDatasources();\n    // Do extra work on main thread to setup stuff\n    this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n      this.project.dataSources,\n      this.apiService.api,\n    );\n    if (isMainThread) {\n      this._schema = await this.ensureProject();\n      await this.initDbSchema();\n      this.metadataRepo = await this.ensureMetadata();\n      this.dynamicDsService.init(this.metadataRepo);\n\n      if (this.nodeConfig.proofOfIndex) {\n        const blockOffset = await this.getMetadataBlockOffset();\n        void this.setBlockOffset(Number(blockOffset));\n        await this.poiService.init(this.schema);\n      }\n\n      this._startHeight = await this.getStartHeight();\n    } else {\n      this.metadataRepo = MetadataFactory(this.sequelize, this.schema);\n\n      this.dynamicDsService.init(this.metadataRepo);\n\n      await this.sequelize.sync();\n\n      this._schema = await this.getExistingProjectSchema();\n      assert(this._schema, 'Schema should be created in main thread');\n      await this.initDbSchema();\n\n      if (this.nodeConfig.proofOfIndex) {\n        await this.poiService.init(this.schema);\n      }\n    }\n\n    if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n      logger.error(\n        'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n      );\n      process.exit(1);\n    }\n  }\n\n  private async ensureProject(): Promise<string> {\n    let schema = await this.getExistingProjectSchema();\n    if (!schema) {\n      schema = await this.createProjectSchema();\n    }\n    this.eventEmitter.emit(IndexerEvent.Ready, {\n      value: true,\n    });\n\n    return schema;\n  }\n\n  private async createProjectSchema(): Promise<string> {\n    let schema: string;\n    if (this.nodeConfig.localMode) {\n      // create tables in default schema if local mode is enabled\n      schema = DEFAULT_DB_SCHEMA;\n    } else {\n      schema = this.nodeConfig.dbSchema;\n      const schemas = await this.sequelize.showAllSchemas(undefined);\n      if (!(schemas as unknown as string[]).includes(schema)) {\n        await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n      }\n    }\n\n    return schema;\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private async ensureMetadata(): Promise<MetadataRepo> {\n    const metadataRepo = MetadataFactory(this.sequelize, this.schema);\n\n    this.eventEmitter.emit(\n      IndexerEvent.NetworkMetadata,\n      this.apiService.networkMeta,\n    );\n\n    const keys = [\n      'lastProcessedHeight',\n      'blockOffset',\n      'indexerNodeVersion',\n      'chain',\n      'specName',\n      'genesisHash',\n      'chainId',\n      'processedBlockCount',\n      'lastFinalizedVerifiedHeight',\n      'schemaMigrationCount',\n    ] as const;\n\n    const entries = await metadataRepo.findAll({\n      where: {\n        key: keys,\n      },\n    });\n\n    const keyValue = entries.reduce((arr, curr) => {\n      arr[curr.key] = curr.value;\n      return arr;\n    }, {} as { [key in typeof keys[number]]: string | boolean | number });\n\n    const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n    if (this.project.runner) {\n      await Promise.all([\n        metadataRepo.upsert({\n          key: 'runnerNode',\n          value: this.project.runner.node.name,\n        }),\n        metadataRepo.upsert({\n          key: 'runnerNodeVersion',\n          value: this.project.runner.node.version,\n        }),\n        metadataRepo.upsert({\n          key: 'runnerQuery',\n          value: this.project.runner.query.name,\n        }),\n        metadataRepo.upsert({\n          key: 'runnerQueryVersion',\n          value: this.project.runner.query.version,\n        }),\n      ]);\n    }\n    if (!keyValue.genesisHash) {\n      await metadataRepo.upsert({ key: 'genesisHash', value: genesisHash });\n    } else {\n      // Check if the configured genesisHash matches the currently stored genesisHash\n      assert(\n        // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n        (this.project.network.genesisHash ?? genesisHash) ===\n          keyValue.genesisHash,\n        'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n      );\n    }\n\n    if (keyValue.chain !== chain) {\n      await metadataRepo.upsert({ key: 'chain', value: chain });\n    }\n\n    if (keyValue.specName !== specName) {\n      await metadataRepo.upsert({ key: 'specName', value: specName });\n    }\n\n    // If project was created before this feature, don't add the key. If it is project created after, add this key.\n    if (!keyValue.processedBlockCount && !keyValue.lastProcessedHeight) {\n      await metadataRepo.upsert({ key: 'processedBlockCount', value: 0 });\n    }\n\n    if (keyValue.indexerNodeVersion !== packageVersion) {\n      await metadataRepo.upsert({\n        key: 'indexerNodeVersion',\n        value: packageVersion,\n      });\n    }\n    if (!keyValue.schemaMigrationCount) {\n      await metadataRepo.upsert({ key: 'schemaMigrationCount', value: 0 });\n    }\n\n    return metadataRepo;\n  }\n\n  async upsertMetadataBlockOffset(height: number): Promise<void> {\n    await this.metadataRepo.upsert({\n      key: 'blockOffset',\n      value: height,\n    });\n  }\n\n  async getMetadataBlockOffset(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n  }\n\n  async getLastProcessedHeight(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n  }\n\n  private async getStartHeight(): Promise<number> {\n    let startHeight: number;\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n    if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n      startHeight = Number(lastProcessedHeight) + 1;\n    } else {\n      startHeight = this.getStartBlockFromDataSources();\n    }\n    return startHeight;\n  }\n\n  async setBlockOffset(offset: number): Promise<void> {\n    if (\n      this._blockOffset ||\n      offset === null ||\n      offset === undefined ||\n      isNaN(offset)\n    ) {\n      return;\n    }\n    logger.info(`set blockOffset to ${offset}`);\n    this._blockOffset = offset;\n    return this.mmrService\n      .syncFileBaseFromPoi(this.schema, offset)\n      .catch((err) => {\n        logger.error(err, 'failed to sync poi to mmr');\n        process.exit(1);\n      });\n  }\n  async getProcessedBlockCount(): Promise<number> {\n    const res = await this.metadataRepo.findOne({\n      where: { key: 'processedBlockCount' },\n    });\n\n    return res?.value as number | undefined;\n  }\n\n  private getStartBlockFromDataSources() {\n    const startBlocksList = this.project.dataSources.map(\n      (item) => item.startBlock ?? 1,\n    );\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n\n    return reindex(\n      this.getStartBlockFromDataSources(),\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      /* Not providing force clean service, it should never be needed */\n    );\n  }\n  n;\n}\n"]}

package/dist/indexer/worker/worker.service.d.ts

@@ -6,6 +6,7 @@ export declare type FetchBlockResponse = {
 export declare type ProcessBlockResponse = {
     dynamicDsCreated: boolean;
     operationHash: string;
+    reindexBlockHeight: number;
 };
 export declare type WorkerStatusResponse = {
     threadId: number;

package/dist/indexer/worker/worker.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAA4C;AAC5C,gDAAgF;AAEhF,wDAAoD;AAgBpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGjD,IAAM,aAAa,GAAnB,MAAM,aAAa;IAMxB,YACU,UAAsB,EACtB,cAA8B,EACtC,UAAsB;QAFd,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAPhC,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAS1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAChE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAEzC,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,EAAE,CAAC,CAAC;SACpD;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAE7D,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,uCACK,QAAQ,KACX,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IACrE;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AApEY,aAAa;IADzB,IAAA,mBAAU,GAAE;qCAQW,sBAAU;QACN,gCAAc;QAC1B,sBAAU;GATb,aAAa,CAoEzB;AApEY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Injectable } from '@nestjs/common';\nimport { ApiService, NodeConfig, getLogger, AutoQueue } from '@subql/node-core';\nimport { EthereumBlockWrapper, BlockWrapper } from '@subql/types-ethereum';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  operationHash: string; // Base64 encoded u8a array\n};\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: ApiService,\n    private indexerManager: IndexerManager,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          const [block] = await this.apiService.api.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        const block = this.fetchedBlocks[height];\n\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(e, `Failed to fetch block ${height}`);\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      const response = await this.indexerManager.indexBlock(block);\n\n      this._isIndexing = false;\n      return {\n        ...response,\n        operationHash: Buffer.from(response.operationHash).toString('base64'),\n      };\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}
+{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAA4C;AAC5C,gDAAgF;AAEhF,wDAAoD;AAiBpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGjD,IAAM,aAAa,GAAnB,MAAM,aAAa;IAMxB,YACU,UAAsB,EACtB,cAA8B,EACtC,UAAsB;QAFd,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAPhC,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAS1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAChE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAEzC,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,EAAE,CAAC,CAAC;SACpD;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAE7D,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,uCACK,QAAQ,KACX,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IACrE;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AApEY,aAAa;IADzB,IAAA,mBAAU,GAAE;qCAQW,sBAAU;QACN,gCAAc;QAC1B,sBAAU;GATb,aAAa,CAoEzB;AApEY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Injectable } from '@nestjs/common';\nimport { ApiService, NodeConfig, getLogger, AutoQueue } from '@subql/node-core';\nimport { EthereumBlockWrapper, BlockWrapper } from '@subql/types-ethereum';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  operationHash: string; // Base64 encoded u8a array\n  reindexBlockHeight: number;\n};\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: ApiService,\n    private indexerManager: IndexerManager,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          const [block] = await this.apiService.api.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        const block = this.fetchedBlocks[height];\n\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(e, `Failed to fetch block ${height}`);\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      const response = await this.indexerManager.indexBlock(block);\n\n      this._isIndexing = false;\n      return {\n        ...response,\n        operationHash: Buffer.from(response.operationHash).toString('base64'),\n      };\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}

package/dist/main.js

@@ -5,6 +5,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const logger_1 = require("@subql/node-core/logger");
 const yargs_1 = require("./yargs");
 const { argv } = yargs_1.yargsOptions;
+BigInt.prototype.toJSON = function () {
+    return this.toString();
+};
 // initLogger is imported from true path, to make sure getLogger (or other logger values that relies on logger) isn't initialised
 (0, logger_1.initLogger)(argv.debug, argv.outputFmt, argv.logLevel);
 // Lazy import, to allow logger to be initialised before bootstrap()

package/dist/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;AAEtC,oDAAqD;AACrD,mCAAuC;AAEvC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,iIAAiI;AACjI,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oEAAoE;AACpE,kDAAkD;AAClD,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACxC,KAAK,SAAS,EAAE,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { initLogger } from '@subql/node-core/logger';\nimport { yargsOptions } from './yargs';\n\nconst { argv } = yargsOptions;\n\n// initLogger is imported from true path, to make sure getLogger (or other logger values that relies on logger) isn't initialised\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\n// Lazy import, to allow logger to be initialised before bootstrap()\n// As bootstrap runs services that requires logger\nconst { bootstrap } = require('./init');\nvoid bootstrap();\n"]}
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;AAEtC,oDAAqD;AACrD,mCAAuC;AAEvC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE7B,MAAM,CAAC,SAAiB,CAAC,MAAM,GAAG;IACjC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;AACzB,CAAC,CAAC;AAEF,iIAAiI;AACjI,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oEAAoE;AACpE,kDAAkD;AAClD,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACxC,KAAK,SAAS,EAAE,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { initLogger } from '@subql/node-core/logger';\nimport { yargsOptions } from './yargs';\n\nconst { argv } = yargsOptions;\n\n(BigInt.prototype as any).toJSON = function () {\n  return this.toString();\n};\n\n// initLogger is imported from true path, to make sure getLogger (or other logger values that relies on logger) isn't initialised\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\n// Lazy import, to allow logger to be initialised before bootstrap()\n// As bootstrap runs services that requires logger\nconst { bootstrap } = require('./init');\nvoid bootstrap();\n"]}

package/dist/subcommands/reindex.init.js

@@ -13,6 +13,7 @@ async function reindexInit(targetHeight) {
         const app = await core_1.NestFactory.create(reindex_module_1.ReindexModule);
         await app.init();
         const reindexService = app.get(reindex_service_1.ReindexService);
+        await reindexService.init();
         await reindexService.reindex(targetHeight);
     }
     catch (e) {

package/dist/subcommands/reindex.init.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.init.js","sourceRoot":"","sources":["../../src/subcommands/reindex.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAA6C;AAC7C,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AACjC,KAAK,UAAU,WAAW,CAAC,YAAoB;IACpD,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;KAC5C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAZD,kCAYC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger } from '@subql/node-core';\nimport { ReindexModule } from './reindex.module';\nimport { ReindexService } from './reindex.service';\n\nconst logger = getLogger('CLI-Reindex');\nexport async function reindexInit(targetHeight: number): Promise<void> {\n  try {\n    const app = await NestFactory.create(ReindexModule);\n\n    await app.init();\n    const reindexService = app.get(ReindexService);\n    await reindexService.reindex(targetHeight);\n  } catch (e) {\n    logger.error(e, 'Reindex failed to execute');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}
+{"version":3,"file":"reindex.init.js","sourceRoot":"","sources":["../../src/subcommands/reindex.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAA6C;AAC7C,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AACjC,KAAK,UAAU,WAAW,CAAC,YAAoB;IACpD,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,CAAC,CAAC;QAEpD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAE/C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;KAC5C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAdD,kCAcC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger } from '@subql/node-core';\nimport { ReindexModule } from './reindex.module';\nimport { ReindexService } from './reindex.service';\n\nconst logger = getLogger('CLI-Reindex');\nexport async function reindexInit(targetHeight: number): Promise<void> {\n  try {\n    const app = await NestFactory.create(ReindexModule);\n\n    await app.init();\n    const reindexService = app.get(ReindexService);\n\n    await reindexService.init();\n    await reindexService.reindex(targetHeight);\n  } catch (e) {\n    logger.error(e, 'Reindex failed to execute');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}

package/dist/subcommands/reindex.module.js

@@ -12,13 +12,22 @@ exports.ReindexModule = exports.ReindexFeatureModule = void 0;
 const common_1 = require("@nestjs/common");
 const node_core_1 = require("@subql/node-core");
 const configure_module_1 = require("../configure/configure.module");
+const ds_processor_service_1 = require("../indexer/ds-processor.service");
+const dynamic_ds_service_1 = require("../indexer/dynamic-ds.service");
 const forceClean_service_1 = require("./forceClean.service");
 const reindex_service_1 = require("./reindex.service");
 let ReindexFeatureModule = class ReindexFeatureModule {
 };
 ReindexFeatureModule = __decorate([
     (0, common_1.Module)({
-        providers: [node_core_1.StoreService, reindex_service_1.ReindexService, node_core_1.MmrService, forceClean_service_1.ForceCleanService],
+        providers: [
+            node_core_1.StoreService,
+            reindex_service_1.ReindexService,
+            node_core_1.MmrService,
+            forceClean_service_1.ForceCleanService,
+            dynamic_ds_service_1.DynamicDsService,
+            ds_processor_service_1.DsProcessorService,
+        ],
         controllers: [],
     })
 ], ReindexFeatureModule);

package/dist/subcommands/reindex.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAAsE;AACtE,oEAAgE;AAChE,6DAAyD;AACzD,uDAAmD;AAM5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAJhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE,CAAC,wBAAY,EAAE,gCAAc,EAAE,sBAAU,EAAE,sCAAiB,CAAC;QACxE,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAU1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IARzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;SACrB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { DbModule, MmrService, StoreService } from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [StoreService, ReindexService, MmrService, ForceCleanService],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}
+{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAAsE;AACtE,oEAAgE;AAChE,0EAAqE;AACrE,sEAAiE;AACjE,6DAAyD;AACzD,uDAAmD;AAa5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAXhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ,gCAAc;YACd,sBAAU;YACV,sCAAiB;YACjB,qCAAgB;YAChB,yCAAkB;SACnB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAU1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IARzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;SACrB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { DbModule, MmrService, StoreService } from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { DsProcessorService } from '../indexer/ds-processor.service';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [\n    StoreService,\n    ReindexService,\n    MmrService,\n    ForceCleanService,\n    DynamicDsService,\n    DsProcessorService,\n  ],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}

package/dist/subcommands/reindex.service.d.ts

@@ -1,6 +1,7 @@
 import { MmrService, NodeConfig, StoreService } from '@subql/node-core';
 import { Sequelize } from 'sequelize';
 import { SubqueryProject } from '../configure/SubqueryProject';
+import { DynamicDsService } from '../indexer/dynamic-ds.service';
 import { ForceCleanService } from './forceClean.service';
 export declare class ReindexService {
     private readonly sequelize;
@@ -9,11 +10,11 @@ export declare class ReindexService {
     private readonly mmrService;
     private readonly project;
     private readonly forceCleanService;
+    private readonly dynamicDsService;
     private schema;
     private metadataRepo;
-    private specName;
-    private startHeight;
-    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, storeService: StoreService, mmrService: MmrService, project: SubqueryProject, forceCleanService: ForceCleanService);
+    constructor(sequelize: Sequelize, nodeConfig: NodeConfig, storeService: StoreService, mmrService: MmrService, project: SubqueryProject, forceCleanService: ForceCleanService, dynamicDsService: DynamicDsService);
+    init(): Promise<void>;
     private getExistingProjectSchema;
     private getLastProcessedHeight;
     private getMetadataBlockOffset;

package/dist/subcommands/reindex.service.js

@@ -16,17 +16,30 @@ const common_1 = require("@nestjs/common");
 const node_core_1 = require("@subql/node-core");
 const sequelize_1 = require("sequelize");
 const SubqueryProject_1 = require("../configure/SubqueryProject");
+const dynamic_ds_service_1 = require("../indexer/dynamic-ds.service");
 const project_1 = require("../utils/project");
+const reindex_1 = require("../utils/reindex");
 const forceClean_service_1 = require("./forceClean.service");
 const logger = (0, node_core_1.getLogger)('Reindex');
 let ReindexService = class ReindexService {
-    constructor(sequelize, nodeConfig, storeService, mmrService, project, forceCleanService) {
+    constructor(sequelize, nodeConfig, storeService, mmrService, project, forceCleanService, dynamicDsService) {
         this.sequelize = sequelize;
         this.nodeConfig = nodeConfig;
         this.storeService = storeService;
         this.mmrService = mmrService;
         this.project = project;
         this.forceCleanService = forceCleanService;
+        this.dynamicDsService = dynamicDsService;
+    }
+    async init() {
+        this.schema = await this.getExistingProjectSchema();
+        if (!this.schema) {
+            logger.error('Unable to locate schema');
+            throw new Error('Schema does not exist.');
+        }
+        await this.initDbSchema();
+        this.metadataRepo = (0, node_core_1.MetadataFactory)(this.sequelize, this.schema);
+        this.dynamicDsService.init(this.metadataRepo);
     }
     async getExistingProjectSchema() {
         return (0, node_core_1.getExistingProjectSchema)(this.nodeConfig, this.sequelize);
@@ -59,46 +72,11 @@ let ReindexService = class ReindexService {
         }
     }
     async reindex(targetBlockHeight) {
-        this.schema = await this.getExistingProjectSchema();
-        if (!this.schema) {
-            logger.error('Unable to locate schema');
-            throw new Error('Schema does not exist.');
-        }
-        await this.initDbSchema();
-        this.metadataRepo = (0, node_core_1.MetadataFactory)(this.sequelize, this.schema);
-        this.startHeight = await this.getStartBlockFromDataSources();
-        const lastProcessedHeight = await this.getLastProcessedHeight();
-        if (!this.storeService.historical) {
-            logger.warn('Unable to reindex, historical state not enabled');
-            return;
-        }
-        if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) {
-            logger.warn(`Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`);
-            return;
-        }
-        // if startHeight is greater than the targetHeight, just force clean
-        if (targetBlockHeight < this.startHeight) {
-            logger.info(`targetHeight: ${targetBlockHeight} is less than startHeight: ${this.startHeight}. Hence executing force-clean`);
-            await this.forceCleanService.forceClean();
-        }
-        else {
-            logger.info(`Reindexing to block: ${targetBlockHeight}`);
-            const transaction = await this.sequelize.transaction();
-            try {
-                await this.storeService.rewind(targetBlockHeight, transaction);
-                const blockOffset = await this.getMetadataBlockOffset();
-                if (blockOffset) {
-                    await this.mmrService.deleteMmrNode(targetBlockHeight + 1, blockOffset);
-                }
-                await transaction.commit();
-                logger.info('Reindex Success');
-            }
-            catch (err) {
-                logger.error(err, 'Reindexing failed');
-                await transaction.rollback();
-                throw err;
-            }
-        }
+        const [startHeight, lastProcessedHeight] = await Promise.all([
+            this.getStartBlockFromDataSources(),
+            this.getLastProcessedHeight(),
+        ]);
+        return (0, reindex_1.reindex)(startHeight, await this.getMetadataBlockOffset(), targetBlockHeight, lastProcessedHeight, this.storeService, this.dynamicDsService, this.mmrService, this.sequelize, this.forceCleanService);
     }
 };
 ReindexService = __decorate([
@@ -108,7 +86,8 @@ ReindexService = __decorate([
         node_core_1.StoreService,
         node_core_1.MmrService,
         SubqueryProject_1.SubqueryProject,
-        forceClean_service_1.ForceCleanService])
+        forceClean_service_1.ForceCleanService,
+        dynamic_ds_service_1.DynamicDsService])
 ], ReindexService);
 exports.ReindexService = ReindexService;
 //# sourceMappingURL=reindex.service.js.map

package/dist/subcommands/reindex.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,gDAS0B;AAC1B,yCAAsC;AACtC,kEAA+E;AAC/E,8CAAgD;AAEhD,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAKzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACtB,OAAwB,EACxB,iBAAoC;QALpC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,YAAO,GAAP,OAAO,CAAiB;QACxB,sBAAiB,GAAjB,iBAAiB,CAAmB;IACpD,CAAC;IAEI,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE;SAC3B,CAAC,CAAC;QACH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,4BAA4B;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,YAAY,GAAG,IAAA,2BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjE,IAAI,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;QAE7D,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE;YACjC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC/D,OAAO;SACR;QACD,IAAI,CAAC,mBAAmB,IAAI,mBAAmB,GAAG,iBAAiB,EAAE;YACnE,MAAM,CAAC,IAAI,CACT,gCAAgC,iBAAiB,6BAA6B,mBAAmB,4BAA4B,CAC9H,CAAC;YACF,OAAO;SACR;QAED,oEAAoE;QACpE,IAAI,iBAAiB,GAAG,IAAI,CAAC,WAAW,EAAE;YACxC,MAAM,CAAC,IAAI,CACT,iBAAiB,iBAAiB,8BAA8B,IAAI,CAAC,WAAW,+BAA+B,CAChH,CAAC;YACF,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,CAAC;SAC3C;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,wBAAwB,iBAAiB,EAAE,CAAC,CAAC;YACzD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;YACvD,IAAI;gBACF,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;gBAE/D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,IAAI,WAAW,EAAE;oBACf,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CACjC,iBAAiB,GAAG,CAAC,EACrB,WAAW,CACZ,CAAC;iBACH;gBACD,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;gBAC3B,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;aAChC;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;gBACvC,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;gBAC7B,MAAM,GAAG,CAAC;aACX;SACF;IACH,CAAC;CACF,CAAA;AA1GY,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAOmB,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACb,iCAAe;QACL,sCAAiB;GAX5C,cAAc,CA0G1B;AA1GY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MetadataFactory,\n  MetadataRepo,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { initDbSchema } from '../utils/project';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: MetadataRepo;\n  private specName: string;\n  private startHeight: number;\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n  ) {}\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    const res = await this.metadataRepo.findOne({\n      where: { key: 'specName' },\n    });\n    return res?.value as string | undefined;\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  private async getStartBlockFromDataSources() {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n\n    this.metadataRepo = MetadataFactory(this.sequelize, this.schema);\n\n    this.startHeight = await this.getStartBlockFromDataSources();\n\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n\n    if (!this.storeService.historical) {\n      logger.warn('Unable to reindex, historical state not enabled');\n      return;\n    }\n    if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) {\n      logger.warn(\n        `Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`,\n      );\n      return;\n    }\n\n    // if startHeight is greater than the targetHeight, just force clean\n    if (targetBlockHeight < this.startHeight) {\n      logger.info(\n        `targetHeight: ${targetBlockHeight} is less than startHeight: ${this.startHeight}. Hence executing force-clean`,\n      );\n      await this.forceCleanService.forceClean();\n    } else {\n      logger.info(`Reindexing to block: ${targetBlockHeight}`);\n      const transaction = await this.sequelize.transaction();\n      try {\n        await this.storeService.rewind(targetBlockHeight, transaction);\n\n        const blockOffset = await this.getMetadataBlockOffset();\n        if (blockOffset) {\n          await this.mmrService.deleteMmrNode(\n            targetBlockHeight + 1,\n            blockOffset,\n          );\n        }\n        await transaction.commit();\n        logger.info('Reindex Success');\n      } catch (err) {\n        logger.error(err, 'Reindexing failed');\n        await transaction.rollback();\n        throw err;\n      }\n    }\n  }\n}\n"]}
+{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,gDAS0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACtB,OAAwB,EACxB,iBAAoC,EACpC,gBAAkC;QANlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,YAAO,GAAP,OAAO,CAAiB;QACxB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,IAAA,2BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACjE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAA,2BAAe,EAAC,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;YAC1C,KAAK,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE;SAC3B,CAAC,CAAC;QACH,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAA2B,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAED,4DAA4D;IACpD,KAAK,CAAC,4BAA4B;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,OAAO,IAAA,iBAAO,EACZ,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;IACJ,CAAC;CACF,CAAA;AAlFY,cAAc;IAD1B,IAAA,mBAAU,GAAE;qCAMmB,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACb,iCAAe;QACL,sCAAiB;QAClB,qCAAgB;GAX1C,cAAc,CAkF1B;AAlFY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MetadataFactory,\n  MetadataRepo,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  getMetaDataInfo,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: MetadataRepo;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n    this.metadataRepo = MetadataFactory(this.sequelize, this.schema);\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return getMetaDataInfo(this.metadataRepo, 'blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    const res = await this.metadataRepo.findOne({\n      where: { key: 'specName' },\n    });\n    return res?.value as string | undefined;\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  private async getStartBlockFromDataSources() {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    return reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n  }\n}\n"]}

package/dist/utils/project.js

@@ -63,6 +63,27 @@ exports.isCustomHandler = isCustomHandler;
 async function updateDataSourcesV0_2_0(_dataSources, reader, root) {
     // force convert to updated ds
     return Promise.all(_dataSources.map(async (dataSource) => {
+        if (dataSource.kind === 'flare/Runtime') {
+            dataSource.kind = common_ethereum_1.EthereumDatasourceKind.Runtime;
+        }
+        dataSource.mapping.handlers = dataSource.mapping.handlers.map((handler) => {
+            switch (handler.kind) {
+                case 'flare/BlockHandler': {
+                    handler.kind = common_ethereum_1.EthereumHandlerKind.Block;
+                    break;
+                }
+                case 'flare/TransactionHandler': {
+                    handler.kind = common_ethereum_1.EthereumHandlerKind.Call;
+                    break;
+                }
+                case 'flare/LogHandler': {
+                    handler.kind = common_ethereum_1.EthereumHandlerKind.Event;
+                    break;
+                }
+                default:
+            }
+            return handler;
+        });
         const entryScript = await loadDataSourceScript(reader, dataSource.mapping.file);
         const file = await updateDataSourcesEntry(reader, dataSource.mapping.file, root, entryScript);
         if (dataSource.assets) {

package/dist/utils/project.js.map

@@ -1 +1 @@
-{"version":3,"file":"project.js","sourceRoot":"","sources":["../../src/utils/project.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;AAEtC,4CAAoB;AACpB,4CAAoB;AACpB,gDAAwB;AACxB,0CAA8E;AAC9E,4DAYgC;AAChC,gDAA6D;AAC7D,wCAAuD;AACvD,sDAA2B;AAC3B,8CAAsB;AAGf,KAAK,UAAU,iBAAiB,CAAC,WAAmB;IACzD,MAAM,KAAK,GAAG,YAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;QAClB,MAAM,GAAG,GAAG,cAAI,CAAC,GAAG,CAAC;QACrB,MAAM,MAAM,GAAG,YAAE,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,YAAE,CAAC,WAAW,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;QACnD,oDAAoD;QACpD,MAAM,aAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,OAAO,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;KACpC;SAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;QAC9B,OAAO,WAAW,CAAC;KACpB;AACH,CAAC;AAZD,8CAYC;AAED,gDAAgD;AAChD,MAAM,iBAAiB,GAA2B,EAAE,CAAC;AAErD,SAAgB,eAAe,CAAC,IAAY;IAC1C,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAChD,IAAI;QACF,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE;YAC/B,MAAM,OAAO,GAAG,YAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;gBACb,OAAO,QAAQ,CAAC;aACjB;YACD,iBAAiB,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACpD,CAAC,CAAC,GAAG,CAAC,IAAI;gBACV,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;SACrB;QAED,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;KACnC;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAI,KAAK,CAAC,8CAA8C,IAAI,EAAE,CAAC,CAAC;KACvE;AACH,CAAC;AAlBD,0CAkBC;AAED,SAAgB,aAAa,CAC3B,OAAqB;IAErB,OAAO,MAAM,CAAC,MAAM,CAAS,qCAAmB,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC;AAJD,sCAIC;AAED,SAAgB,eAAe,CAC7B,OAAqB;IAErB,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC;AAJD,0CAIC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,YAAkE,EAClE,MAAc,EACd,IAAY;IAEZ,8BAA8B;IAC9B,OAAO,OAAO,CAAC,GAAG,CAChB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;QACpC,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAC5C,MAAM,EACN,UAAU,CAAC,OAAO,CAAC,IAAI,CACxB,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,sBAAsB,CACvC,MAAM,EACN,UAAU,CAAC,OAAO,CAAC,IAAI,EACvB,IAAI,EACJ,WAAW,CACZ,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,EAAE;YACrB,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;gBACzD,IAAI,MAAM,YAAY,oBAAW,EAAE;oBACjC,KAAK,CAAC,IAAI,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;iBAC7C;qBAAM;oBACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC7C,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAC7B,IAAI,EACJ,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAClC,CAAC;oBACF,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAa,CAAC,CAAC;oBACvD,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;iBACzB;aACF;SACF;QACD,IAAI,IAAA,4BAAU,EAAC,UAAU,CAAC,EAAE;YAC1B,IAAI,UAAU,CAAC,SAAS,EAAE;gBACxB,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,MAAM,eAAe,CAC/C,MAAM,EACN,IAAI,EACJ,UAAU,CAAC,SAAS,CAAC,IAAI,CAC1B,CAAC;aACH;YACD,IAAI,UAAU,CAAC,MAAM,EAAE;gBACrB,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE;oBACzC,IAAI,MAAM,YAAY,oBAAW,EAAE;wBACjC,KAAK,CAAC,IAAI,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;qBAC7C;yBAAM;wBACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAC7C,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAC7B,IAAI,EACJ,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAClC,CAAC;wBACF,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAa,CAAC,CAAC;wBACvD,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;qBACzB;iBACF;aACF;YACD,uCACK,UAAU,KACb,OAAO,kCAAO,UAAU,CAAC,OAAO,KAAE,WAAW,EAAE,IAAI,OACnD;SACH;aAAM;YACL,uCACK,UAAU,KACb,OAAO,kCAAO,UAAU,CAAC,OAAO,KAAE,WAAW,EAAE,IAAI,OACnD;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AApED,0DAoEC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,IAAY,EACZ,IAAY,EACZ,MAAc;IAEd,IAAI,MAAM,YAAY,oBAAW;QAAE,OAAO,IAAI,CAAC;SAC1C,IAAI,MAAM,YAAY,mBAAU,IAAI,MAAM,YAAY,qBAAY,EAAE;QACvE,MAAM,UAAU,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3E,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChD,OAAO,UAAU,CAAC;KACnB;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,MAAc,EACd,IAAY,EACZ,IAAY;IAEZ,IAAI,MAAM,YAAY,oBAAW,EAAE;QACjC,OAAO,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;KACjC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,UAAU,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3E,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAC7C,OAAO,UAAU,CAAC;KACnB;AACH,CAAC;AAEM,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,IAAY,EACZ,IAAY;IAEZ,8DAA8D;IAC9D,IAAI,MAAM,YAAY,oBAAW,EAAE;QACjC,OAAO,IAAA,gCAAc,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC;KACnC;SAAM;QACL,oGAAoG;QACpG,0FAA0F;QAC1F,0FAA0F;QAC1F,6FAA6F;QAC7F,qEAAqE;QACrE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,GAAY,CAAC;QACjB,IAAI;YACF,GAAG,GAAG,iBAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,OAAO,IAAA,iCAAe,EAAC,GAAG,CAAC,CAAC;SAC7B;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,cAAc,GAAG,GAAG,cAAI,CAAC,OAAO,CACpC,IAAI,EACJ,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAC5B,KAAK,CAAC;YACP,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YACjD,GAAG,GAAG,IAAA,sCAAoB,EAAC,cAAc,CAAC,CAAC,CAAC,mDAAmD;YAC/F,OAAO,IAAA,iCAAe,EAAC,GAAG,CAAC,CAAC;SAC7B;KACF;AACH,CAAC;AA7BD,sCA6BC;AAEM,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,IAAa;IAEb,IAAI,KAAa,CAAC;IAClB,6BAA6B;IAC7B,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC5E,IAAI,GAAG,CAAC,IAAI,EAAE;YACZ,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;SAChE;aAAM;YACL,KAAK,GAAG,QAAQ,CAAC;SAClB;KACF;IACD,eAAe;IACf,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,2BAA2B,CAAC,CAAC;KACjE;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AArBD,oDAqBC;AAED,KAAK,UAAU,WAAW;IACxB,MAAM,GAAG,GAAG,cAAI,CAAC,GAAG,CAAC;IACrB,MAAM,MAAM,GAAG,YAAE,CAAC,MAAM,EAAE,CAAC;IAC3B,OAAO,YAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;AAChD,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,MAAc;IACjD,IAAI,MAAM,YAAY,oBAAW;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC;IACtD,IAAI,MAAM,YAAY,mBAAU,IAAI,MAAM,YAAY,qBAAY,EAAE;QAClE,OAAO,WAAW,EAAE,CAAC;KACtB;AACH,CAAC;AALD,wCAKC;AAEM,KAAK,UAAU,YAAY,CAChC,OAAwB,EACxB,MAAc,EACd,YAA0B;IAE1B,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAPD,oCAOC;AAED,MAAM,aAAa,GAAG,CAAC,SAAS,CAAC,CAAC;AAClC,4DAA4D;AACrD,KAAK,UAAU,cAAc,CAClC,OAAyB,EACzB,MAAM,GAAG,aAAa;IAEtB,OAAO,IAAA,uBAAW,EAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAK,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,MAAM,CAAA,CAAC,CAAC,CAAC;AAC5E,CAAC;AALD,wCAKC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport os from 'os';\nimport path from 'path';\nimport { GithubReader, IPFSReader, LocalReader, Reader } from '@subql/common';\nimport {\n  ChainTypes,\n  CustomDatasourceV0_2_0,\n  isCustomDs,\n  loadChainTypes,\n  loadChainTypesFromJs,\n  parseChainTypes,\n  RuntimeDataSourceV0_2_0,\n  SubqlRuntimeHandler,\n  SubqlCustomHandler,\n  SubqlHandler,\n  EthereumHandlerKind,\n} from '@subql/common-ethereum';\nimport { retryOnFail, StoreService } from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport yaml from 'js-yaml';\nimport tar from 'tar';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\n\nexport async function prepareProjectDir(projectPath: string): Promise<string> {\n  const stats = fs.statSync(projectPath);\n  if (stats.isFile()) {\n    const sep = path.sep;\n    const tmpDir = os.tmpdir();\n    const tempPath = fs.mkdtempSync(`${tmpDir}${sep}`);\n    // Will promote errors if incorrect format/extension\n    await tar.x({ file: projectPath, cwd: tempPath });\n    return tempPath.concat('/package');\n  } else if (stats.isDirectory()) {\n    return projectPath;\n  }\n}\n\n// We cache this to avoid repeated reads from fs\nconst projectEntryCache: Record<string, string> = {};\n\nexport function getProjectEntry(root: string): string {\n  const pkgPath = path.join(root, 'package.json');\n  try {\n    if (!projectEntryCache[pkgPath]) {\n      const content = fs.readFileSync(pkgPath).toString();\n      const pkg = JSON.parse(content);\n      if (!pkg.main) {\n        return './dist';\n      }\n      projectEntryCache[pkgPath] = pkg.main.startsWith('./')\n        ? pkg.main\n        : `./${pkg.main}`;\n    }\n\n    return projectEntryCache[pkgPath];\n  } catch (err) {\n    throw new Error(`can not find package.json within directory ${root}`);\n  }\n}\n\nexport function isBaseHandler(\n  handler: SubqlHandler,\n): handler is SubqlRuntimeHandler {\n  return Object.values<string>(EthereumHandlerKind).includes(handler.kind);\n}\n\nexport function isCustomHandler(\n  handler: SubqlHandler,\n): handler is SubqlCustomHandler {\n  return !isBaseHandler(handler);\n}\n\nexport async function updateDataSourcesV0_2_0(\n  _dataSources: (RuntimeDataSourceV0_2_0 | CustomDatasourceV0_2_0)[],\n  reader: Reader,\n  root: string,\n): Promise<SubqlProjectDs[]> {\n  // force convert to updated ds\n  return Promise.all(\n    _dataSources.map(async (dataSource) => {\n      const entryScript = await loadDataSourceScript(\n        reader,\n        dataSource.mapping.file,\n      );\n      const file = await updateDataSourcesEntry(\n        reader,\n        dataSource.mapping.file,\n        root,\n        entryScript,\n      );\n      if (dataSource.assets) {\n        for (const [, asset] of Object.entries(dataSource.assets)) {\n          if (reader instanceof LocalReader) {\n            asset.file = path.resolve(root, asset.file);\n          } else {\n            const res = await reader.getFile(asset.file);\n            const outputPath = path.resolve(\n              root,\n              asset.file.replace('ipfs://', ''),\n            );\n            await fs.promises.writeFile(outputPath, res as string);\n            asset.file = outputPath;\n          }\n        }\n      }\n      if (isCustomDs(dataSource)) {\n        if (dataSource.processor) {\n          dataSource.processor.file = await updateProcessor(\n            reader,\n            root,\n            dataSource.processor.file,\n          );\n        }\n        if (dataSource.assets) {\n          for (const [, asset] of dataSource.assets) {\n            if (reader instanceof LocalReader) {\n              asset.file = path.resolve(root, asset.file);\n            } else {\n              const res = await reader.getFile(asset.file);\n              const outputPath = path.resolve(\n                root,\n                asset.file.replace('ipfs://', ''),\n              );\n              await fs.promises.writeFile(outputPath, res as string);\n              asset.file = outputPath;\n            }\n          }\n        }\n        return {\n          ...dataSource,\n          mapping: { ...dataSource.mapping, entryScript, file },\n        };\n      } else {\n        return {\n          ...dataSource,\n          mapping: { ...dataSource.mapping, entryScript, file },\n        };\n      }\n    }),\n  );\n}\n\nasync function updateDataSourcesEntry(\n  reader: Reader,\n  file: string,\n  root: string,\n  script: string,\n): Promise<string> {\n  if (reader instanceof LocalReader) return file;\n  else if (reader instanceof IPFSReader || reader instanceof GithubReader) {\n    const outputPath = `${path.resolve(root, file.replace('ipfs://', ''))}.js`;\n    await fs.promises.writeFile(outputPath, script);\n    return outputPath;\n  }\n}\n\nasync function updateProcessor(\n  reader: Reader,\n  root: string,\n  file: string,\n): Promise<string> {\n  if (reader instanceof LocalReader) {\n    return path.resolve(root, file);\n  } else {\n    const res = await reader.getFile(file);\n    const outputPath = `${path.resolve(root, file.replace('ipfs://', ''))}.js`;\n    await fs.promises.writeFile(outputPath, res);\n    return outputPath;\n  }\n}\n\nexport async function getChainTypes(\n  reader: Reader,\n  root: string,\n  file: string,\n): Promise<ChainTypes> {\n  // If the project is load from local, we will direct load them\n  if (reader instanceof LocalReader) {\n    return loadChainTypes(file, root);\n  } else {\n    // If it is stored in ipfs or other resources, we will use the corresponding reader to read the file\n    // Because ipfs not provide extension of the file, it is difficult to determine its format\n    // We will use yaml.load to try to load the script and parse them to supported chain types\n    // if it failed, we will give it another another attempt, and assume the script written in js\n    // we will download it to a temp folder, and load them within sandbox\n    const res = await reader.getFile(file);\n    let raw: unknown;\n    try {\n      raw = yaml.load(res);\n      return parseChainTypes(raw);\n    } catch (e) {\n      const chainTypesPath = `${path.resolve(\n        root,\n        file.replace('ipfs://', ''),\n      )}.js`;\n      await fs.promises.writeFile(chainTypesPath, res);\n      raw = loadChainTypesFromJs(chainTypesPath); //root not required, as it been packed in single js\n      return parseChainTypes(raw);\n    }\n  }\n}\n\nexport async function loadDataSourceScript(\n  reader: Reader,\n  file?: string,\n): Promise<string> {\n  let entry: string;\n  //For RuntimeDataSourceV0_0_1\n  if (!file) {\n    const pkg = await reader.getPkg();\n    if (pkg === undefined) throw new Error('Project package.json is not found');\n    if (pkg.main) {\n      entry = pkg.main.startsWith('./') ? pkg.main : `./${pkg.main}`;\n    } else {\n      entry = './dist';\n    }\n  }\n  //Else get file\n  const entryScript = await reader.getFile(file ? file : entry);\n  if (entryScript === undefined) {\n    throw new Error(`Entry file ${entry} for datasource not exist`);\n  }\n  return entryScript;\n}\n\nasync function makeTempDir(): Promise<string> {\n  const sep = path.sep;\n  const tmpDir = os.tmpdir();\n  return fs.promises.mkdtemp(`${tmpDir}${sep}`);\n}\n\nexport async function getProjectRoot(reader: Reader): Promise<string> {\n  if (reader instanceof LocalReader) return reader.root;\n  if (reader instanceof IPFSReader || reader instanceof GithubReader) {\n    return makeTempDir();\n  }\n}\n\nexport async function initDbSchema(\n  project: SubqueryProject,\n  schema: string,\n  storeService: StoreService,\n): Promise<void> {\n  const modelsRelation = getAllEntitiesRelations(project.schema);\n  await storeService.init(modelsRelation, schema);\n}\n\nconst handledErrors = ['timeout'];\n// eslint-disable-next-line @typescript-eslint/require-await\nexport async function retryOnFailEth<T>(\n  request: () => Promise<T>,\n  errors = handledErrors,\n): Promise<T> {\n  return retryOnFail(request, (e) => !!errors.find((t) => t === e?.reason));\n}\n"]}
+{"version":3,"file":"project.js","sourceRoot":"","sources":["../../src/utils/project.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;AAEtC,4CAAoB;AACpB,4CAAoB;AACpB,gDAAwB;AACxB,0CAA8E;AAC9E,4DAagC;AAChC,gDAA6D;AAC7D,wCAAuD;AACvD,sDAA2B;AAC3B,8CAAsB;AAGf,KAAK,UAAU,iBAAiB,CAAC,WAAmB;IACzD,MAAM,KAAK,GAAG,YAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;QAClB,MAAM,GAAG,GAAG,cAAI,CAAC,GAAG,CAAC;QACrB,MAAM,MAAM,GAAG,YAAE,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,YAAE,CAAC,WAAW,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;QACnD,oDAAoD;QACpD,MAAM,aAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,OAAO,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;KACpC;SAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;QAC9B,OAAO,WAAW,CAAC;KACpB;AACH,CAAC;AAZD,8CAYC;AAED,gDAAgD;AAChD,MAAM,iBAAiB,GAA2B,EAAE,CAAC;AAErD,SAAgB,eAAe,CAAC,IAAY;IAC1C,MAAM,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAChD,IAAI;QACF,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE;YAC/B,MAAM,OAAO,GAAG,YAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;gBACb,OAAO,QAAQ,CAAC;aACjB;YACD,iBAAiB,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACpD,CAAC,CAAC,GAAG,CAAC,IAAI;gBACV,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;SACrB;QAED,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;KACnC;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAI,KAAK,CAAC,8CAA8C,IAAI,EAAE,CAAC,CAAC;KACvE;AACH,CAAC;AAlBD,0CAkBC;AAED,SAAgB,aAAa,CAC3B,OAAqB;IAErB,OAAO,MAAM,CAAC,MAAM,CAAS,qCAAmB,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC3E,CAAC;AAJD,sCAIC;AAED,SAAgB,eAAe,CAC7B,OAAqB;IAErB,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC;AAJD,0CAIC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,YAAkE,EAClE,MAAc,EACd,IAAY;IAEZ,8BAA8B;IAC9B,OAAO,OAAO,CAAC,GAAG,CAChB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;QACpC,IAAI,UAAU,CAAC,IAAI,KAAK,eAAe,EAAE;YACvC,UAAU,CAAC,IAAI,GAAG,wCAAsB,CAAC,OAAO,CAAC;SAClD;QACD,UAAU,CAAC,OAAO,CAAC,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAC3D,CAAC,OAAO,EAAE,EAAE;YACV,QAAQ,OAAO,CAAC,IAAI,EAAE;gBACpB,KAAK,oBAAoB,CAAC,CAAC;oBACzB,OAAO,CAAC,IAAI,GAAG,qCAAmB,CAAC,KAAK,CAAC;oBACzC,MAAM;iBACP;gBACD,KAAK,0BAA0B,CAAC,CAAC;oBAC/B,OAAO,CAAC,IAAI,GAAG,qCAAmB,CAAC,IAAI,CAAC;oBACxC,MAAM;iBACP;gBACD,KAAK,kBAAkB,CAAC,CAAC;oBACvB,OAAO,CAAC,IAAI,GAAG,qCAAmB,CAAC,KAAK,CAAC;oBACzC,MAAM;iBACP;gBACD,QAAQ;aACT;YAED,OAAO,OAAO,CAAC;QACjB,CAAC,CACF,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAC5C,MAAM,EACN,UAAU,CAAC,OAAO,CAAC,IAAI,CACxB,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,sBAAsB,CACvC,MAAM,EACN,UAAU,CAAC,OAAO,CAAC,IAAI,EACvB,IAAI,EACJ,WAAW,CACZ,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,EAAE;YACrB,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;gBACzD,IAAI,MAAM,YAAY,oBAAW,EAAE;oBACjC,KAAK,CAAC,IAAI,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;iBAC7C;qBAAM;oBACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC7C,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAC7B,IAAI,EACJ,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAClC,CAAC;oBACF,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAa,CAAC,CAAC;oBACvD,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;iBACzB;aACF;SACF;QACD,IAAI,IAAA,4BAAU,EAAC,UAAU,CAAC,EAAE;YAC1B,IAAI,UAAU,CAAC,SAAS,EAAE;gBACxB,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,MAAM,eAAe,CAC/C,MAAM,EACN,IAAI,EACJ,UAAU,CAAC,SAAS,CAAC,IAAI,CAC1B,CAAC;aACH;YACD,IAAI,UAAU,CAAC,MAAM,EAAE;gBACrB,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE;oBACzC,IAAI,MAAM,YAAY,oBAAW,EAAE;wBACjC,KAAK,CAAC,IAAI,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;qBAC7C;yBAAM;wBACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAC7C,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAC7B,IAAI,EACJ,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAClC,CAAC;wBACF,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAa,CAAC,CAAC;wBACvD,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;qBACzB;iBACF;aACF;YACD,uCACK,UAAU,KACb,OAAO,kCAAO,UAAU,CAAC,OAAO,KAAE,WAAW,EAAE,IAAI,OACnD;SACH;aAAM;YACL,uCACK,UAAU,KACb,OAAO,kCAAO,UAAU,CAAC,OAAO,KAAE,WAAW,EAAE,IAAI,OACnD;SACH;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AA7FD,0DA6FC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,IAAY,EACZ,IAAY,EACZ,MAAc;IAEd,IAAI,MAAM,YAAY,oBAAW;QAAE,OAAO,IAAI,CAAC;SAC1C,IAAI,MAAM,YAAY,mBAAU,IAAI,MAAM,YAAY,qBAAY,EAAE;QACvE,MAAM,UAAU,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3E,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAChD,OAAO,UAAU,CAAC;KACnB;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,MAAc,EACd,IAAY,EACZ,IAAY;IAEZ,IAAI,MAAM,YAAY,oBAAW,EAAE;QACjC,OAAO,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;KACjC;SAAM;QACL,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,UAAU,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3E,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAC7C,OAAO,UAAU,CAAC;KACnB;AACH,CAAC;AAEM,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,IAAY,EACZ,IAAY;IAEZ,8DAA8D;IAC9D,IAAI,MAAM,YAAY,oBAAW,EAAE;QACjC,OAAO,IAAA,gCAAc,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC;KACnC;SAAM;QACL,oGAAoG;QACpG,0FAA0F;QAC1F,0FAA0F;QAC1F,6FAA6F;QAC7F,qEAAqE;QACrE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,GAAY,CAAC;QACjB,IAAI;YACF,GAAG,GAAG,iBAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,OAAO,IAAA,iCAAe,EAAC,GAAG,CAAC,CAAC;SAC7B;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,cAAc,GAAG,GAAG,cAAI,CAAC,OAAO,CACpC,IAAI,EACJ,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAC5B,KAAK,CAAC;YACP,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YACjD,GAAG,GAAG,IAAA,sCAAoB,EAAC,cAAc,CAAC,CAAC,CAAC,mDAAmD;YAC/F,OAAO,IAAA,iCAAe,EAAC,GAAG,CAAC,CAAC;SAC7B;KACF;AACH,CAAC;AA7BD,sCA6BC;AAEM,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,IAAa;IAEb,IAAI,KAAa,CAAC;IAClB,6BAA6B;IAC7B,IAAI,CAAC,IAAI,EAAE;QACT,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC5E,IAAI,GAAG,CAAC,IAAI,EAAE;YACZ,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;SAChE;aAAM;YACL,KAAK,GAAG,QAAQ,CAAC;SAClB;KACF;IACD,eAAe;IACf,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,2BAA2B,CAAC,CAAC;KACjE;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AArBD,oDAqBC;AAED,KAAK,UAAU,WAAW;IACxB,MAAM,GAAG,GAAG,cAAI,CAAC,GAAG,CAAC;IACrB,MAAM,MAAM,GAAG,YAAE,CAAC,MAAM,EAAE,CAAC;IAC3B,OAAO,YAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC;AAChD,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,MAAc;IACjD,IAAI,MAAM,YAAY,oBAAW;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC;IACtD,IAAI,MAAM,YAAY,mBAAU,IAAI,MAAM,YAAY,qBAAY,EAAE;QAClE,OAAO,WAAW,EAAE,CAAC;KACtB;AACH,CAAC;AALD,wCAKC;AAEM,KAAK,UAAU,YAAY,CAChC,OAAwB,EACxB,MAAc,EACd,YAA0B;IAE1B,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,YAAY,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAPD,oCAOC;AAED,MAAM,aAAa,GAAG,CAAC,SAAS,CAAC,CAAC;AAClC,4DAA4D;AACrD,KAAK,UAAU,cAAc,CAClC,OAAyB,EACzB,MAAM,GAAG,aAAa;IAEtB,OAAO,IAAA,uBAAW,EAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAK,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,MAAM,CAAA,CAAC,CAAC,CAAC;AAC5E,CAAC;AALD,wCAKC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport os from 'os';\nimport path from 'path';\nimport { GithubReader, IPFSReader, LocalReader, Reader } from '@subql/common';\nimport {\n  ChainTypes,\n  CustomDatasourceV0_2_0,\n  isCustomDs,\n  loadChainTypes,\n  loadChainTypesFromJs,\n  parseChainTypes,\n  RuntimeDataSourceV0_2_0,\n  SubqlRuntimeHandler,\n  SubqlCustomHandler,\n  SubqlHandler,\n  EthereumHandlerKind,\n  EthereumDatasourceKind,\n} from '@subql/common-ethereum';\nimport { retryOnFail, StoreService } from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport yaml from 'js-yaml';\nimport tar from 'tar';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\n\nexport async function prepareProjectDir(projectPath: string): Promise<string> {\n  const stats = fs.statSync(projectPath);\n  if (stats.isFile()) {\n    const sep = path.sep;\n    const tmpDir = os.tmpdir();\n    const tempPath = fs.mkdtempSync(`${tmpDir}${sep}`);\n    // Will promote errors if incorrect format/extension\n    await tar.x({ file: projectPath, cwd: tempPath });\n    return tempPath.concat('/package');\n  } else if (stats.isDirectory()) {\n    return projectPath;\n  }\n}\n\n// We cache this to avoid repeated reads from fs\nconst projectEntryCache: Record<string, string> = {};\n\nexport function getProjectEntry(root: string): string {\n  const pkgPath = path.join(root, 'package.json');\n  try {\n    if (!projectEntryCache[pkgPath]) {\n      const content = fs.readFileSync(pkgPath).toString();\n      const pkg = JSON.parse(content);\n      if (!pkg.main) {\n        return './dist';\n      }\n      projectEntryCache[pkgPath] = pkg.main.startsWith('./')\n        ? pkg.main\n        : `./${pkg.main}`;\n    }\n\n    return projectEntryCache[pkgPath];\n  } catch (err) {\n    throw new Error(`can not find package.json within directory ${root}`);\n  }\n}\n\nexport function isBaseHandler(\n  handler: SubqlHandler,\n): handler is SubqlRuntimeHandler {\n  return Object.values<string>(EthereumHandlerKind).includes(handler.kind);\n}\n\nexport function isCustomHandler(\n  handler: SubqlHandler,\n): handler is SubqlCustomHandler {\n  return !isBaseHandler(handler);\n}\n\nexport async function updateDataSourcesV0_2_0(\n  _dataSources: (RuntimeDataSourceV0_2_0 | CustomDatasourceV0_2_0)[],\n  reader: Reader,\n  root: string,\n): Promise<SubqlProjectDs[]> {\n  // force convert to updated ds\n  return Promise.all(\n    _dataSources.map(async (dataSource) => {\n      if (dataSource.kind === 'flare/Runtime') {\n        dataSource.kind = EthereumDatasourceKind.Runtime;\n      }\n      dataSource.mapping.handlers = dataSource.mapping.handlers.map(\n        (handler) => {\n          switch (handler.kind) {\n            case 'flare/BlockHandler': {\n              handler.kind = EthereumHandlerKind.Block;\n              break;\n            }\n            case 'flare/TransactionHandler': {\n              handler.kind = EthereumHandlerKind.Call;\n              break;\n            }\n            case 'flare/LogHandler': {\n              handler.kind = EthereumHandlerKind.Event;\n              break;\n            }\n            default:\n          }\n\n          return handler;\n        },\n      );\n\n      const entryScript = await loadDataSourceScript(\n        reader,\n        dataSource.mapping.file,\n      );\n      const file = await updateDataSourcesEntry(\n        reader,\n        dataSource.mapping.file,\n        root,\n        entryScript,\n      );\n      if (dataSource.assets) {\n        for (const [, asset] of Object.entries(dataSource.assets)) {\n          if (reader instanceof LocalReader) {\n            asset.file = path.resolve(root, asset.file);\n          } else {\n            const res = await reader.getFile(asset.file);\n            const outputPath = path.resolve(\n              root,\n              asset.file.replace('ipfs://', ''),\n            );\n            await fs.promises.writeFile(outputPath, res as string);\n            asset.file = outputPath;\n          }\n        }\n      }\n      if (isCustomDs(dataSource)) {\n        if (dataSource.processor) {\n          dataSource.processor.file = await updateProcessor(\n            reader,\n            root,\n            dataSource.processor.file,\n          );\n        }\n        if (dataSource.assets) {\n          for (const [, asset] of dataSource.assets) {\n            if (reader instanceof LocalReader) {\n              asset.file = path.resolve(root, asset.file);\n            } else {\n              const res = await reader.getFile(asset.file);\n              const outputPath = path.resolve(\n                root,\n                asset.file.replace('ipfs://', ''),\n              );\n              await fs.promises.writeFile(outputPath, res as string);\n              asset.file = outputPath;\n            }\n          }\n        }\n        return {\n          ...dataSource,\n          mapping: { ...dataSource.mapping, entryScript, file },\n        };\n      } else {\n        return {\n          ...dataSource,\n          mapping: { ...dataSource.mapping, entryScript, file },\n        };\n      }\n    }),\n  );\n}\n\nasync function updateDataSourcesEntry(\n  reader: Reader,\n  file: string,\n  root: string,\n  script: string,\n): Promise<string> {\n  if (reader instanceof LocalReader) return file;\n  else if (reader instanceof IPFSReader || reader instanceof GithubReader) {\n    const outputPath = `${path.resolve(root, file.replace('ipfs://', ''))}.js`;\n    await fs.promises.writeFile(outputPath, script);\n    return outputPath;\n  }\n}\n\nasync function updateProcessor(\n  reader: Reader,\n  root: string,\n  file: string,\n): Promise<string> {\n  if (reader instanceof LocalReader) {\n    return path.resolve(root, file);\n  } else {\n    const res = await reader.getFile(file);\n    const outputPath = `${path.resolve(root, file.replace('ipfs://', ''))}.js`;\n    await fs.promises.writeFile(outputPath, res);\n    return outputPath;\n  }\n}\n\nexport async function getChainTypes(\n  reader: Reader,\n  root: string,\n  file: string,\n): Promise<ChainTypes> {\n  // If the project is load from local, we will direct load them\n  if (reader instanceof LocalReader) {\n    return loadChainTypes(file, root);\n  } else {\n    // If it is stored in ipfs or other resources, we will use the corresponding reader to read the file\n    // Because ipfs not provide extension of the file, it is difficult to determine its format\n    // We will use yaml.load to try to load the script and parse them to supported chain types\n    // if it failed, we will give it another another attempt, and assume the script written in js\n    // we will download it to a temp folder, and load them within sandbox\n    const res = await reader.getFile(file);\n    let raw: unknown;\n    try {\n      raw = yaml.load(res);\n      return parseChainTypes(raw);\n    } catch (e) {\n      const chainTypesPath = `${path.resolve(\n        root,\n        file.replace('ipfs://', ''),\n      )}.js`;\n      await fs.promises.writeFile(chainTypesPath, res);\n      raw = loadChainTypesFromJs(chainTypesPath); //root not required, as it been packed in single js\n      return parseChainTypes(raw);\n    }\n  }\n}\n\nexport async function loadDataSourceScript(\n  reader: Reader,\n  file?: string,\n): Promise<string> {\n  let entry: string;\n  //For RuntimeDataSourceV0_0_1\n  if (!file) {\n    const pkg = await reader.getPkg();\n    if (pkg === undefined) throw new Error('Project package.json is not found');\n    if (pkg.main) {\n      entry = pkg.main.startsWith('./') ? pkg.main : `./${pkg.main}`;\n    } else {\n      entry = './dist';\n    }\n  }\n  //Else get file\n  const entryScript = await reader.getFile(file ? file : entry);\n  if (entryScript === undefined) {\n    throw new Error(`Entry file ${entry} for datasource not exist`);\n  }\n  return entryScript;\n}\n\nasync function makeTempDir(): Promise<string> {\n  const sep = path.sep;\n  const tmpDir = os.tmpdir();\n  return fs.promises.mkdtemp(`${tmpDir}${sep}`);\n}\n\nexport async function getProjectRoot(reader: Reader): Promise<string> {\n  if (reader instanceof LocalReader) return reader.root;\n  if (reader instanceof IPFSReader || reader instanceof GithubReader) {\n    return makeTempDir();\n  }\n}\n\nexport async function initDbSchema(\n  project: SubqueryProject,\n  schema: string,\n  storeService: StoreService,\n): Promise<void> {\n  const modelsRelation = getAllEntitiesRelations(project.schema);\n  await storeService.init(modelsRelation, schema);\n}\n\nconst handledErrors = ['timeout'];\n// eslint-disable-next-line @typescript-eslint/require-await\nexport async function retryOnFailEth<T>(\n  request: () => Promise<T>,\n  errors = handledErrors,\n): Promise<T> {\n  return retryOnFail(request, (e) => !!errors.find((t) => t === e?.reason));\n}\n"]}

package/dist/utils/reindex.d.ts

@@ -0,0 +1,5 @@
+import { MmrService, StoreService } from '@subql/node-core';
+import { Sequelize } from 'sequelize';
+import { DynamicDsService } from '../indexer/dynamic-ds.service';
+import { ForceCleanService } from '../subcommands/forceClean.service';
+export declare function reindex(startHeight: number, blockOffset: number | undefined, targetBlockHeight: number, lastProcessedHeight: number, storeService: StoreService, dynamicDsService: DynamicDsService, mmrService: MmrService, sequelize: Sequelize, forceCleanService?: ForceCleanService): Promise<void>;

package/dist/utils/reindex.js

@@ -0,0 +1,48 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.reindex = void 0;
+const node_core_1 = require("@subql/node-core");
+const logger = (0, node_core_1.getLogger)('Reindex');
+async function reindex(startHeight, blockOffset, targetBlockHeight, lastProcessedHeight, storeService, dynamicDsService, mmrService, sequelize, forceCleanService) {
+    if (!storeService.historical) {
+        logger.warn('Unable to reindex, historical state not enabled');
+        return;
+    }
+    if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) {
+        logger.warn(`Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`);
+        return;
+    }
+    // if startHeight is greater than the targetHeight, just force clean
+    if (targetBlockHeight < startHeight) {
+        logger.info(`targetHeight: ${targetBlockHeight} is less than startHeight: ${startHeight}. Hence executing force-clean`);
+        if (!forceCleanService) {
+            logger.error('ForceCleanService not provided, cannot force clean');
+            process.exit(1);
+        }
+        await forceCleanService.forceClean();
+    }
+    else {
+        logger.info(`Reindexing to block: ${targetBlockHeight}`);
+        const transaction = await sequelize.transaction();
+        try {
+            await Promise.all([
+                storeService.rewind(targetBlockHeight, transaction),
+                dynamicDsService.resetDynamicDatasource(targetBlockHeight, transaction),
+            ]);
+            if (blockOffset) {
+                await mmrService.deleteMmrNode(targetBlockHeight + 1, blockOffset);
+            }
+            await transaction.commit();
+            logger.info('Reindex Success');
+        }
+        catch (err) {
+            logger.error(err, 'Reindexing failed');
+            await transaction.rollback();
+            throw err;
+        }
+    }
+}
+exports.reindex = reindex;
+//# sourceMappingURL=reindex.js.map

package/dist/utils/reindex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reindex.js","sourceRoot":"","sources":["../../src/utils/reindex.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,gDAAuE;AAKvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAE7B,KAAK,UAAU,OAAO,CAC3B,WAAmB,EACnB,WAA+B,EAC/B,iBAAyB,EACzB,mBAA2B,EAC3B,YAA0B,EAC1B,gBAAkC,EAClC,UAAsB,EACtB,SAAoB,EACpB,iBAAqC;IAErC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE;QAC5B,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAC/D,OAAO;KACR;IACD,IAAI,CAAC,mBAAmB,IAAI,mBAAmB,GAAG,iBAAiB,EAAE;QACnE,MAAM,CAAC,IAAI,CACT,gCAAgC,iBAAiB,6BAA6B,mBAAmB,4BAA4B,CAC9H,CAAC;QACF,OAAO;KACR;IAED,oEAAoE;IACpE,IAAI,iBAAiB,GAAG,WAAW,EAAE;QACnC,MAAM,CAAC,IAAI,CACT,iBAAiB,iBAAiB,8BAA8B,WAAW,+BAA+B,CAC3G,CAAC;QACF,IAAI,CAAC,iBAAiB,EAAE;YACtB,MAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,MAAM,iBAAiB,CAAC,UAAU,EAAE,CAAC;KACtC;SAAM;QACL,MAAM,CAAC,IAAI,CAAC,wBAAwB,iBAAiB,EAAE,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI;YACF,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,YAAY,CAAC,MAAM,CAAC,iBAAiB,EAAE,WAAW,CAAC;gBACnD,gBAAgB,CAAC,sBAAsB,CAAC,iBAAiB,EAAE,WAAW,CAAC;aACxE,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE;gBACf,MAAM,UAAU,CAAC,aAAa,CAAC,iBAAiB,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;aACpE;YACD,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;YACvC,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAC;YAC7B,MAAM,GAAG,CAAC;SACX;KACF;AACH,CAAC;AApDD,0BAoDC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { getLogger, MmrService, StoreService } from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { ForceCleanService } from '../subcommands/forceClean.service';\n\nconst logger = getLogger('Reindex');\n\nexport async function reindex(\n  startHeight: number,\n  blockOffset: number | undefined,\n  targetBlockHeight: number,\n  lastProcessedHeight: number,\n  storeService: StoreService,\n  dynamicDsService: DynamicDsService,\n  mmrService: MmrService,\n  sequelize: Sequelize,\n  forceCleanService?: ForceCleanService,\n): Promise<void> {\n  if (!storeService.historical) {\n    logger.warn('Unable to reindex, historical state not enabled');\n    return;\n  }\n  if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) {\n    logger.warn(\n      `Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`,\n    );\n    return;\n  }\n\n  // if startHeight is greater than the targetHeight, just force clean\n  if (targetBlockHeight < startHeight) {\n    logger.info(\n      `targetHeight: ${targetBlockHeight} is less than startHeight: ${startHeight}. Hence executing force-clean`,\n    );\n    if (!forceCleanService) {\n      logger.error('ForceCleanService not provided, cannot force clean');\n      process.exit(1);\n    }\n    await forceCleanService.forceClean();\n  } else {\n    logger.info(`Reindexing to block: ${targetBlockHeight}`);\n    const transaction = await sequelize.transaction();\n    try {\n      await Promise.all([\n        storeService.rewind(targetBlockHeight, transaction),\n        dynamicDsService.resetDynamicDatasource(targetBlockHeight, transaction),\n      ]);\n\n      if (blockOffset) {\n        await mmrService.deleteMmrNode(targetBlockHeight + 1, blockOffset);\n      }\n      await transaction.commit();\n      logger.info('Reindex Success');\n    } catch (err) {\n      logger.error(err, 'Reindexing failed');\n      await transaction.rollback();\n      throw err;\n    }\n  }\n}\n"]}

package/dist/yargs.d.ts

@@ -1,6 +1,6 @@
 export declare const yargsOptions: import("yargs").Argv<import("yargs").Omit<{
     targetHeight: number;
-}, "subquery" | "subquery-name" | "config" | "local" | "db-schema" | "unsafe" | "subscription" | "batch-size" | "scale-batch-size" | "timeout" | "debug" | "profiler" | "network-endpoint" | "output-fmt" | "log-level" | "migrate" | "timestamp-field" | "network-dictionary" | "dictionary-timeout" | "mmr-path" | "proof-of-index" | "ipfs" | "port" | "disable-historical" | "workers" | "query-limit"> & import("yargs").InferredOptionTypes<{
+}, "subquery" | "subquery-name" | "config" | "local" | "db-schema" | "unsafe" | "subscription" | "batch-size" | "scale-batch-size" | "timeout" | "debug" | "profiler" | "network-endpoint" | "output-fmt" | "log-level" | "migrate" | "timestamp-field" | "network-dictionary" | "dictionary-timeout" | "mmr-path" | "proof-of-index" | "ipfs" | "port" | "disable-historical" | "workers" | "query-limit" | "unfinalized-blocks"> & import("yargs").InferredOptionTypes<{
     subquery: {
         alias: string;
         demandOption: true;
@@ -151,4 +151,10 @@ export declare const yargsOptions: import("yargs").Argv<import("yargs").Omit<{
         type: "number";
         default: number;
     };
+    'unfinalized-blocks': {
+        demandOption: false;
+        default: boolean;
+        describe: string;
+        type: "boolean";
+    };
 }>>;

package/dist/yargs.js

@@ -189,5 +189,11 @@ exports.yargsOptions = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv)
         type: 'number',
         default: 100,
     },
+    'unfinalized-blocks': {
+        demandOption: false,
+        default: false,
+        describe: 'Enable to fetch and index unfinalized blocks',
+        type: 'boolean',
+    },
 });
 //# sourceMappingURL=yargs.js.map