@subql/node-ethereum 1.0.3-2 → 1.0.3-4

package/dist/indexer/unfinalizedBlocks.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../src/indexer/unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAGtC,2CAA4C;AAC5C,gDAK0B;AAE1B,mCAA8B;AAG9B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,CAAC,CAAC;AAEjC,QAAA,+BAA+B,GAAG,mBAAmB,CAAC;AACtD,QAAA,qCAAqC,GAChD,6BAA6B,CAAC;AAEhC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAUlC,IAAa,wBAAwB,GAArC,MAAa,wBAAwB;IAKnC,YACmB,UAAsB,EACtB,UAAsB,EACtB,UAA6B;QAF7B,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAmB;IAC7C,CAAC;IAEJ,KAAK,CAAC,IAAI,CACR,OAAgD;QAEhD,MAAM,CAAC,IAAI,CACT,yBACE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAClD,EAAE,CACH,CAAC;QACF,qBAAqB;QACrB,IAAI,CAAC,iBAAiB,GAAG,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACnE,IAAI,CAAC,sBAAsB,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAC1E,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC1D,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CACzC,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;YACvE,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC7C,6CAA6C;YAE7C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE/D,IAAI,YAAY,KAAK,IAAI,EAAE;gBACzB,MAAM,CAAC,IAAI,CACT,yGAAyG,YAAY,GAAG,CACzH,CAAC;gBACF,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,8BAA8B,YAAY,GAAG,CAAC,CAAC;gBAC3D,OAAO,YAAY,CAAC;aACrB;iBAAM;gBACL,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC9B,IAAI,CAAC,gCAAgC,EAAE,CAAC;aACzC;SACF;IACH,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,KAAgC;QAEhC,IAAI,KAAK,EAAE;YACT,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;SACzD;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC,YAAY,EAAE;YACjB,iDAAiD;YACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;SAC7B;aAAM;YACL,uDAAuD;YACvD,OAAO,IAAI,CAAC,4BAA4B,CAAC,YAAY,CAAC,CAAC;SACxD;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAY,GAAG;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,IAAY,oBAAoB;QAC9B,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,sBAAsB,CAAC,MAAa;QAClC,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE;YACtE,OAAO;SACR;QACD,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;IAChC,CAAC;IAEO,wBAAwB,CAAC,WAAmB,EAAE,IAAY;QAChE,IAAI,WAAW,IAAI,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAErD,eAAe;QACf,IACE,IAAI,CAAC,iBAAiB,CAAC,MAAM;YAC7B,IAAA,aAAI,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,WAAW,EACnD;YACA,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAEO,oBAAoB;QAC1B,IACE,IAAI,CAAC,sBAAsB,KAAK,SAAS;YACzC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,EACvD;YACA,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChD,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,CAAC;IAC1D,CAAC;IAED,qEAAqE;IAC7D,eAAe,CAAC,WAAmB;QACzC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACpD,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,WAAW,CACnC,CAAC;IACJ,CAAC;IAED,yCAAyC;IACjC,gBAAgB,CACtB,WAAmB;QAEnB,oDAAoD;QACpD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,yBAAyB;aACjE,OAAO,EAAE,CAAC,6CAA6C;aACvD,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAAC,CAAC;QACvE,IAAI,MAAM,EAAE;YACV,MAAM,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC;YACvC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,CAAC;SACvD;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,SAAS;QACrB,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAC/C,IAAI,CAAC,oBAAoB,CAC1B,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,mBAAmB,EAAE;YACxB,OAAO;SACR;QAED,4CAA4C;QAC5C,IAAI,mBAAmB,CAAC,WAAW,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACjE,IAAI,mBAAmB,CAAC,IAAI,KAAK,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE;gBAC1D,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CACzK,CAAC;gBACF,OAAO,IAAI,CAAC,eAAe,CAAC;aAC7B;SACF;aAAM;YACL,2CAA2C;YAC3C,IAAI,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC;YAClC;;;;eAIG;YACH,IACE,MAAM,CAAC,MAAM,GAAG,mBAAmB,CAAC,WAAW;gBAC/C,qBAAqB,EACrB;gBACA,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC5C,mBAAmB,CAAC,WAAW,CAChC,CAAC;aACH;iBAAM;gBACL,OAAO,mBAAmB,CAAC,WAAW,KAAK,MAAM,CAAC,MAAM,EAAE;oBACxD,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;iBACnE;aACF;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,mBAAmB,CAAC,IAAI,EAAE;gBAC5C,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAC3J,CAAC;gBACF,OAAO,MAAM,CAAC;aACf;SACF;QAED,OAAO;IACT,CAAC;IAEO,KAAK,CAAC,4BAA4B,CACxC,YAAmB;QAEnB,MAAM,oBAAoB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACxD,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CACpB,MAAM,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,oBAAoB,CACvD,CAAC;QAEF,IAAI,cAAc,GAAG,YAAY,CAAC;QAElC,kEAAkE;QAClE,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,oBAAoB,CAAC,OAAO,EAAE,EAAE;YAC1D,IAAI,IAAI,KAAK,cAAc,CAAC,IAAI,IAAI,IAAI,KAAK,cAAc,CAAC,UAAU,EAAE;gBACtE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;aACtB;YAED,qBAAqB;YACrB,cAAc,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CACpD,cAAc,CAAC,UAAU,CAC1B,CAAC;SACH;QAED,OAAO,IAAI,CAAC,sBAAsB,CAAC;IACrC,CAAC;IAEO,qBAAqB,CAAC,iBAAoC;QAChE,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,uCAA+B,EAC/B,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAClC,CAAC;IACJ,CAAC;IAEO,+BAA+B,CAAC,MAAc;QACpD,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,MAAM,CACP,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,uCAA+B,EAAE,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED,gCAAgC;QAC9B,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,IAAI,CACL,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,4BAA4B;QAChC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAC7C,uCAA+B,CAChC,CAAC;QACF,IAAI,GAAG,EAAE;YACP,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAsB,CAAC;SAC7C;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,8BAA8B;QAClC,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,6CAAqC,CAAC,CAAC;IAC9E,CAAC;CACF,CAAA;AAtPY,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAOoB,sBAAU;QACV,sBAAU;QACV,6BAAiB;GARrC,wBAAwB,CAsPpC;AAtPY,4DAAwB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Block } from '@ethersproject/abstract-provider';\nimport { Injectable } from '@nestjs/common';\nimport {\n  ApiService,\n  getLogger,\n  NodeConfig,\n  StoreCacheService,\n} from '@subql/node-core';\nimport { EthereumBlock } from '@subql/types-ethereum';\nimport { last } from 'lodash';\nimport { EthereumApi } from '../ethereum';\n\nconst logger = getLogger('UnfinalizedBlocks');\n\nexport const METADATA_UNFINALIZED_BLOCKS_KEY = 'unfinalizedBlocks';\nexport const METADATA_LAST_FINALIZED_PROCESSED_KEY =\n  'lastFinalizedVerifiedHeight';\n\nconst UNFINALIZED_THRESHOLD = 200;\n\ntype UnfinalizedBlocks = [blockHeight: number, blockHash: string][];\nexport interface IUnfinalizedBlocksService {\n  processUnfinalizedBlocks(\n    block: EthereumBlock | undefined,\n  ): Promise<number | null>;\n}\n\n@Injectable()\nexport class UnfinalizedBlocksService implements IUnfinalizedBlocksService {\n  private unfinalizedBlocks: UnfinalizedBlocks;\n  private finalizedHeader: Block;\n  private lastCheckedBlockHeight: number;\n\n  constructor(\n    private readonly apiService: ApiService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeCache: StoreCacheService,\n  ) {}\n\n  async init(\n    reindex: (targetHeight: number) => Promise<void>,\n  ): Promise<number | undefined> {\n    logger.info(\n      `Unfinalized blocks is ${\n        this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'\n      }`,\n    );\n    // unfinalized blocks\n    this.unfinalizedBlocks = await this.getMetadataUnfinalizedBlocks();\n    this.lastCheckedBlockHeight = await this.getLastFinalizedVerifiedHeight();\n    this.finalizedHeader = await this.api.getBlockByHeightOrHash(\n      await this.api.getFinalizedBlockHeight(),\n    );\n\n    if (!this.nodeConfig.unfinalizedBlocks && this.unfinalizedBlocks.length) {\n      logger.info('Processing unfinalized blocks');\n      // Validate any previously unfinalized blocks\n\n      const rewindHeight = await this.processUnfinalizedBlocks(null);\n\n      if (rewindHeight !== null) {\n        logger.info(\n          `Found un-finalized blocks from previous indexing but unverified, rolling back to last finalized block ${rewindHeight} `,\n        );\n        await reindex(rewindHeight);\n        logger.info(`Successful rewind to block ${rewindHeight}!`);\n        return rewindHeight;\n      } else {\n        this.resetUnfinalizedBlocks();\n        this.resetLastFinalizedVerifiedHeight();\n      }\n    }\n  }\n\n  async processUnfinalizedBlocks(\n    block: EthereumBlock | undefined,\n  ): Promise<number | null> {\n    if (block) {\n      this.registerUnfinalizedBlock(block.number, block.hash);\n    }\n\n    const forkedHeader = await this.hasForked();\n    if (!forkedHeader) {\n      // Remove blocks that are now confirmed finalized\n      this.deleteFinalizedBlock();\n    } else {\n      // Get the last unfinalized block that is now finalized\n      return this.getLastCorrectFinalizedBlock(forkedHeader);\n    }\n\n    return null;\n  }\n\n  private get api(): EthereumApi {\n    return this.apiService.api;\n  }\n\n  private get finalizedBlockNumber(): number {\n    return this.finalizedHeader.number;\n  }\n\n  registerFinalizedBlock(header: Block): void {\n    if (this.finalizedHeader && this.finalizedBlockNumber >= header.number) {\n      return;\n    }\n    this.finalizedHeader = header;\n  }\n\n  private registerUnfinalizedBlock(blockNumber: number, hash: string): void {\n    if (blockNumber <= this.finalizedBlockNumber) return;\n\n    // Ensure order\n    if (\n      this.unfinalizedBlocks.length &&\n      last(this.unfinalizedBlocks)[0] + 1 !== blockNumber\n    ) {\n      logger.error('Unfinalized block is not sequential');\n      process.exit(1);\n    }\n\n    this.unfinalizedBlocks.push([blockNumber, hash]);\n    this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n  }\n\n  private deleteFinalizedBlock(): void {\n    if (\n      this.lastCheckedBlockHeight !== undefined &&\n      this.lastCheckedBlockHeight < this.finalizedBlockNumber\n    ) {\n      this.removeFinalized(this.finalizedBlockNumber);\n      this.saveLastFinalizedVerifiedHeight(this.finalizedBlockNumber);\n      this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n    }\n    this.lastCheckedBlockHeight = this.finalizedBlockNumber;\n  }\n\n  // remove any records less and equal than input finalized blockHeight\n  private removeFinalized(blockHeight: number): void {\n    this.unfinalizedBlocks = this.unfinalizedBlocks.filter(\n      ([height]) => height > blockHeight,\n    );\n  }\n\n  // find closest record from block heights\n  private getClosestRecord(\n    blockHeight: number,\n  ): { blockHeight: number; hash: string } | undefined {\n    // Have the block in the best block, can be verified\n    const record = [...this.unfinalizedBlocks] // Copy so we can reverse\n      .reverse() // Reverse the list to find the largest block\n      .find(([bestBlockHeight]) => Number(bestBlockHeight) <= blockHeight);\n    if (record) {\n      const [bestBlockHeight, hash] = record;\n      return { blockHeight: Number(bestBlockHeight), hash };\n    }\n    return undefined;\n  }\n\n  // check unfinalized blocks for a fork, returns the header where a fork happened\n  private async hasForked(): Promise<Block | undefined> {\n    const lastVerifiableBlock = this.getClosestRecord(\n      this.finalizedBlockNumber,\n    );\n\n    // No unfinalized blocks\n    if (!lastVerifiableBlock) {\n      return;\n    }\n\n    // Unfinalized blocks beyond finalized block\n    if (lastVerifiableBlock.blockHeight === this.finalizedBlockNumber) {\n      if (lastVerifiableBlock.hash !== this.finalizedHeader.hash) {\n        logger.warn(\n          `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${this.finalizedHeader.hash}`,\n        );\n        return this.finalizedHeader;\n      }\n    } else {\n      // Unfinalized blocks below finalized block\n      let header = this.finalizedHeader;\n      /*\n       * Iterate back through parent hashes until we get the header with the matching height\n       * We use headers here rather than getBlockHash because of potential caching issues on the rpc\n       * If we're off by a large number of blocks we can optimise by getting the block hash directly\n       */\n      if (\n        header.number - lastVerifiableBlock.blockHeight >\n        UNFINALIZED_THRESHOLD\n      ) {\n        header = await this.api.getBlockByHeightOrHash(\n          lastVerifiableBlock.blockHeight,\n        );\n      } else {\n        while (lastVerifiableBlock.blockHeight !== header.number) {\n          header = await this.api.getBlockByHeightOrHash(header.parentHash);\n        }\n      }\n\n      if (header.hash !== lastVerifiableBlock.hash) {\n        logger.warn(\n          `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${header.hash}`,\n        );\n        return header;\n      }\n    }\n\n    return;\n  }\n\n  private async getLastCorrectFinalizedBlock(\n    forkedHeader: Block,\n  ): Promise<number | undefined> {\n    const bestVerifiableBlocks = this.unfinalizedBlocks.filter(\n      ([bestBlockHeight]) =>\n        Number(bestBlockHeight) <= this.finalizedBlockNumber,\n    );\n\n    let checkingHeader = forkedHeader;\n\n    // Work backwards through the blocks until we find a matching hash\n    for (const [block, hash] of bestVerifiableBlocks.reverse()) {\n      if (hash === checkingHeader.hash || hash === checkingHeader.parentHash) {\n        return Number(block);\n      }\n\n      // Get the new parent\n      checkingHeader = await this.api.getBlockByHeightOrHash(\n        checkingHeader.parentHash,\n      );\n    }\n\n    return this.lastCheckedBlockHeight;\n  }\n\n  private saveUnfinalizedBlocks(unfinalizedBlocks: UnfinalizedBlocks): void {\n    return this.storeCache.metadata.set(\n      METADATA_UNFINALIZED_BLOCKS_KEY,\n      JSON.stringify(unfinalizedBlocks),\n    );\n  }\n\n  private saveLastFinalizedVerifiedHeight(height: number): void {\n    return this.storeCache.metadata.set(\n      METADATA_LAST_FINALIZED_PROCESSED_KEY,\n      height,\n    );\n  }\n\n  resetUnfinalizedBlocks(): void {\n    this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]');\n    this.unfinalizedBlocks = [];\n  }\n\n  resetLastFinalizedVerifiedHeight(): void {\n    return this.storeCache.metadata.set(\n      METADATA_LAST_FINALIZED_PROCESSED_KEY,\n      null,\n    );\n  }\n\n  //string should be jsonb object\n  async getMetadataUnfinalizedBlocks(): Promise<UnfinalizedBlocks> {\n    const val = await this.storeCache.metadata.find(\n      METADATA_UNFINALIZED_BLOCKS_KEY,\n    );\n    if (val) {\n      return JSON.parse(val) as UnfinalizedBlocks;\n    }\n    return [];\n  }\n\n  async getLastFinalizedVerifiedHeight(): Promise<number | undefined> {\n    return this.storeCache.metadata.find(METADATA_LAST_FINALIZED_PROCESSED_KEY);\n  }\n}\n"]}
+{"version":3,"file":"unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../src/indexer/unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAGtC,2CAA4C;AAC5C,gDAK0B;AAE1B,mCAA8B;AAG9B,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,CAAC,CAAC;AAEjC,QAAA,+BAA+B,GAAG,mBAAmB,CAAC;AACtD,QAAA,qCAAqC,GAChD,6BAA6B,CAAC;AAEhC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAU3B,IAAM,wBAAwB,GAA9B,MAAM,wBAAwB;IAKnC,YACmB,UAAsB,EACtB,UAAsB,EACtB,UAA6B;QAF7B,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAmB;IAC7C,CAAC;IAEJ,KAAK,CAAC,IAAI,CACR,OAAgD;QAEhD,MAAM,CAAC,IAAI,CACT,yBACE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAClD,EAAE,CACH,CAAC;QACF,qBAAqB;QACrB,IAAI,CAAC,iBAAiB,GAAG,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;QACnE,IAAI,CAAC,sBAAsB,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAC1E,IAAI,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC1D,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CACzC,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;YACvE,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC7C,6CAA6C;YAE7C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE/D,IAAI,YAAY,KAAK,IAAI,EAAE;gBACzB,MAAM,CAAC,IAAI,CACT,yGAAyG,YAAY,GAAG,CACzH,CAAC;gBACF,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,8BAA8B,YAAY,GAAG,CAAC,CAAC;gBAC3D,OAAO,YAAY,CAAC;aACrB;iBAAM;gBACL,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC9B,IAAI,CAAC,gCAAgC,EAAE,CAAC;aACzC;SACF;IACH,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,KAAgC;QAEhC,IAAI,KAAK,EAAE;YACT,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;SACzD;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC,YAAY,EAAE;YACjB,iDAAiD;YACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;SAC7B;aAAM;YACL,uDAAuD;YACvD,OAAO,IAAI,CAAC,4BAA4B,CAAC,YAAY,CAAC,CAAC;SACxD;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAY,GAAG;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,IAAY,oBAAoB;QAC9B,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,sBAAsB,CAAC,MAAa;QAClC,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE;YACtE,OAAO;SACR;QACD,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;IAChC,CAAC;IAEO,wBAAwB,CAAC,WAAmB,EAAE,IAAY;QAChE,IAAI,WAAW,IAAI,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAErD,eAAe;QACf,IACE,IAAI,CAAC,iBAAiB,CAAC,MAAM;YAC7B,IAAA,aAAI,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,WAAW,EACnD;YACA,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAEO,oBAAoB;QAC1B,IACE,IAAI,CAAC,sBAAsB,KAAK,SAAS;YACzC,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,EACvD;YACA,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChD,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAChE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,oBAAoB,CAAC;IAC1D,CAAC;IAED,qEAAqE;IAC7D,eAAe,CAAC,WAAmB;QACzC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACpD,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,WAAW,CACnC,CAAC;IACJ,CAAC;IAED,yCAAyC;IACjC,gBAAgB,CACtB,WAAmB;QAEnB,oDAAoD;QACpD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,yBAAyB;aACjE,OAAO,EAAE,CAAC,6CAA6C;aACvD,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAAC,CAAC;QACvE,IAAI,MAAM,EAAE;YACV,MAAM,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC;YACvC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,CAAC;SACvD;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,SAAS;QACrB,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAC/C,IAAI,CAAC,oBAAoB,CAC1B,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,mBAAmB,EAAE;YACxB,OAAO;SACR;QAED,4CAA4C;QAC5C,IAAI,mBAAmB,CAAC,WAAW,KAAK,IAAI,CAAC,oBAAoB,EAAE;YACjE,IAAI,mBAAmB,CAAC,IAAI,KAAK,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE;gBAC1D,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CACzK,CAAC;gBACF,OAAO,IAAI,CAAC,eAAe,CAAC;aAC7B;SACF;aAAM;YACL,2CAA2C;YAC3C,IAAI,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC;YAClC;;;;eAIG;YACH,IACE,MAAM,CAAC,MAAM,GAAG,mBAAmB,CAAC,WAAW;gBAC/C,qBAAqB,EACrB;gBACA,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC5C,mBAAmB,CAAC,WAAW,CAChC,CAAC;aACH;iBAAM;gBACL,OAAO,mBAAmB,CAAC,WAAW,KAAK,MAAM,CAAC,MAAM,EAAE;oBACxD,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;iBACnE;aACF;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,mBAAmB,CAAC,IAAI,EAAE;gBAC5C,MAAM,CAAC,IAAI,CACT,oDAAoD,mBAAmB,CAAC,WAAW,cAAc,mBAAmB,CAAC,IAAI,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAC3J,CAAC;gBACF,OAAO,MAAM,CAAC;aACf;SACF;QAED,OAAO;IACT,CAAC;IAEO,KAAK,CAAC,4BAA4B,CACxC,YAAmB;QAEnB,MAAM,oBAAoB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACxD,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CACpB,MAAM,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,oBAAoB,CACvD,CAAC;QAEF,IAAI,cAAc,GAAG,YAAY,CAAC;QAElC,kEAAkE;QAClE,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,oBAAoB,CAAC,OAAO,EAAE,EAAE;YAC1D,IAAI,IAAI,KAAK,cAAc,CAAC,IAAI,IAAI,IAAI,KAAK,cAAc,CAAC,UAAU,EAAE;gBACtE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;aACtB;YAED,qBAAqB;YACrB,cAAc,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CACpD,cAAc,CAAC,UAAU,CAC1B,CAAC;SACH;QAED,OAAO,IAAI,CAAC,sBAAsB,CAAC;IACrC,CAAC;IAEO,qBAAqB,CAAC,iBAAoC;QAChE,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,uCAA+B,EAC/B,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAClC,CAAC;IACJ,CAAC;IAEO,+BAA+B,CAAC,MAAc;QACpD,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,MAAM,CACP,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,uCAA+B,EAAE,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED,gCAAgC;QAC9B,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACjC,6CAAqC,EACrC,IAAI,CACL,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,4BAA4B;QAChC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAC7C,uCAA+B,CAChC,CAAC;QACF,IAAI,GAAG,EAAE;YACP,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAsB,CAAC;SAC7C;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,8BAA8B;QAClC,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,6CAAqC,CAAC,CAAC;IAC9E,CAAC;CACF,CAAA;AAtPY,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAOoB,sBAAU;QACV,sBAAU;QACV,6BAAiB;GARrC,wBAAwB,CAsPpC;AAtPY,4DAAwB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Block } from '@ethersproject/abstract-provider';\nimport { Injectable } from '@nestjs/common';\nimport {\n  ApiService,\n  getLogger,\n  NodeConfig,\n  StoreCacheService,\n} from '@subql/node-core';\nimport { EthereumBlock } from '@subql/types-ethereum';\nimport { last } from 'lodash';\nimport { EthereumApi } from '../ethereum';\n\nconst logger = getLogger('UnfinalizedBlocks');\n\nexport const METADATA_UNFINALIZED_BLOCKS_KEY = 'unfinalizedBlocks';\nexport const METADATA_LAST_FINALIZED_PROCESSED_KEY =\n  'lastFinalizedVerifiedHeight';\n\nconst UNFINALIZED_THRESHOLD = 200;\n\ntype UnfinalizedBlocks = [blockHeight: number, blockHash: string][];\nexport interface IUnfinalizedBlocksService {\n  processUnfinalizedBlocks(\n    block: EthereumBlock | undefined,\n  ): Promise<number | null>;\n}\n\n@Injectable()\nexport class UnfinalizedBlocksService implements IUnfinalizedBlocksService {\n  private unfinalizedBlocks: UnfinalizedBlocks;\n  private finalizedHeader: Block;\n  private lastCheckedBlockHeight: number;\n\n  constructor(\n    private readonly apiService: ApiService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeCache: StoreCacheService,\n  ) {}\n\n  async init(\n    reindex: (targetHeight: number) => Promise<void>,\n  ): Promise<number | undefined> {\n    logger.info(\n      `Unfinalized blocks is ${\n        this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'\n      }`,\n    );\n    // unfinalized blocks\n    this.unfinalizedBlocks = await this.getMetadataUnfinalizedBlocks();\n    this.lastCheckedBlockHeight = await this.getLastFinalizedVerifiedHeight();\n    this.finalizedHeader = await this.api.getBlockByHeightOrHash(\n      await this.api.getFinalizedBlockHeight(),\n    );\n\n    if (!this.nodeConfig.unfinalizedBlocks && this.unfinalizedBlocks.length) {\n      logger.info('Processing unfinalized blocks');\n      // Validate any previously unfinalized blocks\n\n      const rewindHeight = await this.processUnfinalizedBlocks(null);\n\n      if (rewindHeight !== null) {\n        logger.info(\n          `Found un-finalized blocks from previous indexing but unverified, rolling back to last finalized block ${rewindHeight} `,\n        );\n        await reindex(rewindHeight);\n        logger.info(`Successful rewind to block ${rewindHeight}!`);\n        return rewindHeight;\n      } else {\n        this.resetUnfinalizedBlocks();\n        this.resetLastFinalizedVerifiedHeight();\n      }\n    }\n  }\n\n  async processUnfinalizedBlocks(\n    block: EthereumBlock | undefined,\n  ): Promise<number | null> {\n    if (block) {\n      this.registerUnfinalizedBlock(block.number, block.hash);\n    }\n\n    const forkedHeader = await this.hasForked();\n    if (!forkedHeader) {\n      // Remove blocks that are now confirmed finalized\n      this.deleteFinalizedBlock();\n    } else {\n      // Get the last unfinalized block that is now finalized\n      return this.getLastCorrectFinalizedBlock(forkedHeader);\n    }\n\n    return null;\n  }\n\n  private get api(): EthereumApi {\n    return this.apiService.api;\n  }\n\n  private get finalizedBlockNumber(): number {\n    return this.finalizedHeader.number;\n  }\n\n  registerFinalizedBlock(header: Block): void {\n    if (this.finalizedHeader && this.finalizedBlockNumber >= header.number) {\n      return;\n    }\n    this.finalizedHeader = header;\n  }\n\n  private registerUnfinalizedBlock(blockNumber: number, hash: string): void {\n    if (blockNumber <= this.finalizedBlockNumber) return;\n\n    // Ensure order\n    if (\n      this.unfinalizedBlocks.length &&\n      last(this.unfinalizedBlocks)[0] + 1 !== blockNumber\n    ) {\n      logger.error('Unfinalized block is not sequential');\n      process.exit(1);\n    }\n\n    this.unfinalizedBlocks.push([blockNumber, hash]);\n    this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n  }\n\n  private deleteFinalizedBlock(): void {\n    if (\n      this.lastCheckedBlockHeight !== undefined &&\n      this.lastCheckedBlockHeight < this.finalizedBlockNumber\n    ) {\n      this.removeFinalized(this.finalizedBlockNumber);\n      this.saveLastFinalizedVerifiedHeight(this.finalizedBlockNumber);\n      this.saveUnfinalizedBlocks(this.unfinalizedBlocks);\n    }\n    this.lastCheckedBlockHeight = this.finalizedBlockNumber;\n  }\n\n  // remove any records less and equal than input finalized blockHeight\n  private removeFinalized(blockHeight: number): void {\n    this.unfinalizedBlocks = this.unfinalizedBlocks.filter(\n      ([height]) => height > blockHeight,\n    );\n  }\n\n  // find closest record from block heights\n  private getClosestRecord(\n    blockHeight: number,\n  ): { blockHeight: number; hash: string } | undefined {\n    // Have the block in the best block, can be verified\n    const record = [...this.unfinalizedBlocks] // Copy so we can reverse\n      .reverse() // Reverse the list to find the largest block\n      .find(([bestBlockHeight]) => Number(bestBlockHeight) <= blockHeight);\n    if (record) {\n      const [bestBlockHeight, hash] = record;\n      return { blockHeight: Number(bestBlockHeight), hash };\n    }\n    return undefined;\n  }\n\n  // check unfinalized blocks for a fork, returns the header where a fork happened\n  private async hasForked(): Promise<Block | undefined> {\n    const lastVerifiableBlock = this.getClosestRecord(\n      this.finalizedBlockNumber,\n    );\n\n    // No unfinalized blocks\n    if (!lastVerifiableBlock) {\n      return;\n    }\n\n    // Unfinalized blocks beyond finalized block\n    if (lastVerifiableBlock.blockHeight === this.finalizedBlockNumber) {\n      if (lastVerifiableBlock.hash !== this.finalizedHeader.hash) {\n        logger.warn(\n          `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${this.finalizedHeader.hash}`,\n        );\n        return this.finalizedHeader;\n      }\n    } else {\n      // Unfinalized blocks below finalized block\n      let header = this.finalizedHeader;\n      /*\n       * Iterate back through parent hashes until we get the header with the matching height\n       * We use headers here rather than getBlockHash because of potential caching issues on the rpc\n       * If we're off by a large number of blocks we can optimise by getting the block hash directly\n       */\n      if (\n        header.number - lastVerifiableBlock.blockHeight >\n        UNFINALIZED_THRESHOLD\n      ) {\n        header = await this.api.getBlockByHeightOrHash(\n          lastVerifiableBlock.blockHeight,\n        );\n      } else {\n        while (lastVerifiableBlock.blockHeight !== header.number) {\n          header = await this.api.getBlockByHeightOrHash(header.parentHash);\n        }\n      }\n\n      if (header.hash !== lastVerifiableBlock.hash) {\n        logger.warn(\n          `Block fork found, enqueued un-finalized block at ${lastVerifiableBlock.blockHeight} with hash ${lastVerifiableBlock.hash}, actual hash is ${header.hash}`,\n        );\n        return header;\n      }\n    }\n\n    return;\n  }\n\n  private async getLastCorrectFinalizedBlock(\n    forkedHeader: Block,\n  ): Promise<number | undefined> {\n    const bestVerifiableBlocks = this.unfinalizedBlocks.filter(\n      ([bestBlockHeight]) =>\n        Number(bestBlockHeight) <= this.finalizedBlockNumber,\n    );\n\n    let checkingHeader = forkedHeader;\n\n    // Work backwards through the blocks until we find a matching hash\n    for (const [block, hash] of bestVerifiableBlocks.reverse()) {\n      if (hash === checkingHeader.hash || hash === checkingHeader.parentHash) {\n        return Number(block);\n      }\n\n      // Get the new parent\n      checkingHeader = await this.api.getBlockByHeightOrHash(\n        checkingHeader.parentHash,\n      );\n    }\n\n    return this.lastCheckedBlockHeight;\n  }\n\n  private saveUnfinalizedBlocks(unfinalizedBlocks: UnfinalizedBlocks): void {\n    return this.storeCache.metadata.set(\n      METADATA_UNFINALIZED_BLOCKS_KEY,\n      JSON.stringify(unfinalizedBlocks),\n    );\n  }\n\n  private saveLastFinalizedVerifiedHeight(height: number): void {\n    return this.storeCache.metadata.set(\n      METADATA_LAST_FINALIZED_PROCESSED_KEY,\n      height,\n    );\n  }\n\n  resetUnfinalizedBlocks(): void {\n    this.storeCache.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]');\n    this.unfinalizedBlocks = [];\n  }\n\n  resetLastFinalizedVerifiedHeight(): void {\n    return this.storeCache.metadata.set(\n      METADATA_LAST_FINALIZED_PROCESSED_KEY,\n      null,\n    );\n  }\n\n  //string should be jsonb object\n  async getMetadataUnfinalizedBlocks(): Promise<UnfinalizedBlocks> {\n    const val = await this.storeCache.metadata.find(\n      METADATA_UNFINALIZED_BLOCKS_KEY,\n    );\n    if (val) {\n      return JSON.parse(val) as UnfinalizedBlocks;\n    }\n    return [];\n  }\n\n  async getLastFinalizedVerifiedHeight(): Promise<number | undefined> {\n    return this.storeCache.metadata.find(METADATA_LAST_FINALIZED_PROCESSED_KEY);\n  }\n}\n"]}

package/dist/indexer/worker/worker.d.ts

@@ -7,15 +7,15 @@ declare function numFetchingBlocks(): Promise<number>;
 declare function getStatus(): Promise<WorkerStatusResponse>;
 declare function getMemoryLeft(): Promise<number>;
 declare function waitForWorkerBatchSize(heapSizeInBytes: number): Promise<void>;
-declare type InitWorker = typeof initWorker;
-declare type FetchBlock = typeof fetchBlock;
-declare type ProcessBlock = typeof processBlock;
-declare type NumFetchedBlocks = typeof numFetchedBlocks;
-declare type NumFetchingBlocks = typeof numFetchingBlocks;
-declare type GetWorkerStatus = typeof getStatus;
-declare type GetMemoryLeft = typeof getMemoryLeft;
-declare type WaitForWorkerBatchSize = typeof waitForWorkerBatchSize;
-export declare type IIndexerWorker = {
+type InitWorker = typeof initWorker;
+type FetchBlock = typeof fetchBlock;
+type ProcessBlock = typeof processBlock;
+type NumFetchedBlocks = typeof numFetchedBlocks;
+type NumFetchingBlocks = typeof numFetchingBlocks;
+type GetWorkerStatus = typeof getStatus;
+type GetMemoryLeft = typeof getMemoryLeft;
+type WaitForWorkerBatchSize = typeof waitForWorkerBatchSize;
+export type IIndexerWorker = {
     processBlock: ProcessBlock;
     fetchBlock: FetchBlock;
     numFetchedBlocks: NumFetchedBlocks;
@@ -24,7 +24,7 @@ export declare type IIndexerWorker = {
     getMemoryLeft: GetMemoryLeft;
     waitForWorkerBatchSize: WaitForWorkerBatchSize;
 };
-export declare type IInitIndexerWorker = IIndexerWorker & {
+export type IInitIndexerWorker = IIndexerWorker & {
     initWorker: InitWorker;
 };
 export {};

package/dist/indexer/worker/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAC/C,2BAAuC;AAEvC,uCAA2C;AAC3C,gDAS0B;AAG1B,wDAAoD;AACpD,mDAA+C;AAC/C,qDAK0B;AAC1B,yFAG4C;AAC5C,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;AAEjC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;KACxC;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,OAAO,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,aAAa;IAC1B,MAAM,SAAS,GAAG,IAAA,sBAAiB,GAAE,CAAC,eAAe,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;IAEhD,OAAO,SAAS,GAAG,QAAQ,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,eAAuB;IAC3D,MAAM,IAAA,4BAAgB,EAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,wDAAwD;AACvD,MAAc,CAAC,IAAI,GAAG,sBAAU,CAAC,MAAM,CAItC,CAAC,GAAG,yBAAa,EAAE,GAAG,6BAAiB,EAAE,GAAG,4DAAyB,CAAC,EACtE;IACE,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,aAAa;IACb,sBAAsB;CACvB,EACD,MAAM,CACP,CAAC;AA0BF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { getHeapStatistics } from 'v8';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport {\n  waitForBatchSize,\n  WorkerHost,\n  getLogger,\n  NestLogger,\n  hostStoreKeys,\n  HostStore,\n  hostDynamicDsKeys,\n  HostDynamicDS,\n} from '@subql/node-core';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n  FetchBlockResponse,\n  ProcessBlockResponse,\n  WorkerService,\n  WorkerStatusResponse,\n} from './worker.service';\nimport {\n  HostUnfinalizedBlocks,\n  hostUnfinalizedBlocksKeys,\n} from './worker.unfinalizedBlocks.service';\nlet app: INestApplication;\nlet workerService: WorkerService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n  try {\n    if (app) {\n      logger.warn('Worker already initialised');\n      return;\n    }\n\n    app = await NestFactory.create(WorkerModule, {\n      logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n    });\n\n    await app.init();\n\n    const indexerManager = app.get(IndexerManager);\n    // Initialise async services, we do this here rather than in factories so we can capture one off events\n    await indexerManager.start();\n\n    workerService = app.get(WorkerService);\n  } catch (e) {\n    console.log('Failed to start worker', e);\n    logger.error(e, 'Failed to start worker');\n    throw e;\n  }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n  assert(workerService, 'Not initialised');\n  return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n  assert(workerService, 'Not initialised');\n\n  return workerService.processBlock(height);\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n  return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n  return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n  return {\n    threadId,\n    fetchedBlocks: workerService.numFetchedBlocks,\n    toFetchBlocks: workerService.numFetchingBlocks,\n    isIndexing: workerService.isIndexing,\n  };\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getMemoryLeft(): Promise<number> {\n  const totalHeap = getHeapStatistics().heap_size_limit;\n  const heapUsed = process.memoryUsage().heapUsed;\n\n  return totalHeap - heapUsed;\n}\n\nasync function waitForWorkerBatchSize(heapSizeInBytes: number): Promise<void> {\n  await waitForBatchSize(heapSizeInBytes);\n}\n\n// Register these functions to be exposed to worker host\n(global as any).host = WorkerHost.create<\n  HostStore & HostDynamicDS<SubqlProjectDs> & HostUnfinalizedBlocks,\n  IInitIndexerWorker\n>(\n  [...hostStoreKeys, ...hostDynamicDsKeys, ...hostUnfinalizedBlocksKeys],\n  {\n    initWorker,\n    fetchBlock,\n    processBlock,\n    numFetchedBlocks,\n    numFetchingBlocks,\n    getStatus,\n    getMemoryLeft,\n    waitForWorkerBatchSize,\n  },\n  logger,\n);\n\n// Export types to be used on the parent\ntype InitWorker = typeof initWorker;\ntype FetchBlock = typeof fetchBlock;\ntype ProcessBlock = typeof processBlock;\ntype NumFetchedBlocks = typeof numFetchedBlocks;\ntype NumFetchingBlocks = typeof numFetchingBlocks;\ntype GetWorkerStatus = typeof getStatus;\ntype GetMemoryLeft = typeof getMemoryLeft;\ntype WaitForWorkerBatchSize = typeof waitForWorkerBatchSize;\n\nexport type IIndexerWorker = {\n  processBlock: ProcessBlock;\n  fetchBlock: FetchBlock;\n  numFetchedBlocks: NumFetchedBlocks;\n  numFetchingBlocks: NumFetchingBlocks;\n  getStatus: GetWorkerStatus;\n  getMemoryLeft: GetMemoryLeft;\n  waitForWorkerBatchSize: WaitForWorkerBatchSize;\n};\n\nexport type IInitIndexerWorker = IIndexerWorker & {\n  initWorker: InitWorker;\n};\n\nprocess.on('uncaughtException', (e) => {\n  logger.error(e, 'Uncaught Exception');\n  throw e;\n});\n"]}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAC/C,2BAAuC;AAEvC,uCAA2C;AAC3C,gDAS0B;AAE1B,wDAAoD;AACpD,mDAA+C;AAC/C,qDAK0B;AAC1B,yFAG4C;AAC5C,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;AAEjC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;KACxC;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,OAAO,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,aAAa;IAC1B,MAAM,SAAS,GAAG,IAAA,sBAAiB,GAAE,CAAC,eAAe,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;IAEhD,OAAO,SAAS,GAAG,QAAQ,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,eAAuB;IAC3D,MAAM,IAAA,4BAAgB,EAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,wDAAwD;AACvD,MAAc,CAAC,IAAI,GAAG,sBAAU,CAAC,MAAM,CAItC,CAAC,GAAG,yBAAa,EAAE,GAAG,6BAAiB,EAAE,GAAG,4DAAyB,CAAC,EACtE;IACE,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,aAAa;IACb,sBAAsB;CACvB,EACD,MAAM,CACP,CAAC;AA0BF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { getHeapStatistics } from 'v8';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport {\n  waitForBatchSize,\n  WorkerHost,\n  getLogger,\n  NestLogger,\n  hostStoreKeys,\n  HostStore,\n  hostDynamicDsKeys,\n  HostDynamicDS,\n} from '@subql/node-core';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n  FetchBlockResponse,\n  ProcessBlockResponse,\n  WorkerService,\n  WorkerStatusResponse,\n} from './worker.service';\nimport {\n  HostUnfinalizedBlocks,\n  hostUnfinalizedBlocksKeys,\n} from './worker.unfinalizedBlocks.service';\nlet app: INestApplication;\nlet workerService: WorkerService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n  try {\n    if (app) {\n      logger.warn('Worker already initialised');\n      return;\n    }\n\n    app = await NestFactory.create(WorkerModule, {\n      logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n    });\n\n    await app.init();\n\n    const indexerManager = app.get(IndexerManager);\n    // Initialise async services, we do this here rather than in factories so we can capture one off events\n    await indexerManager.start();\n\n    workerService = app.get(WorkerService);\n  } catch (e) {\n    console.log('Failed to start worker', e);\n    logger.error(e, 'Failed to start worker');\n    throw e;\n  }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n  assert(workerService, 'Not initialised');\n  return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n  assert(workerService, 'Not initialised');\n\n  return workerService.processBlock(height);\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n  return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n  return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n  return {\n    threadId,\n    fetchedBlocks: workerService.numFetchedBlocks,\n    toFetchBlocks: workerService.numFetchingBlocks,\n    isIndexing: workerService.isIndexing,\n  };\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getMemoryLeft(): Promise<number> {\n  const totalHeap = getHeapStatistics().heap_size_limit;\n  const heapUsed = process.memoryUsage().heapUsed;\n\n  return totalHeap - heapUsed;\n}\n\nasync function waitForWorkerBatchSize(heapSizeInBytes: number): Promise<void> {\n  await waitForBatchSize(heapSizeInBytes);\n}\n\n// Register these functions to be exposed to worker host\n(global as any).host = WorkerHost.create<\n  HostStore & HostDynamicDS<SubqlProjectDs> & HostUnfinalizedBlocks,\n  IInitIndexerWorker\n>(\n  [...hostStoreKeys, ...hostDynamicDsKeys, ...hostUnfinalizedBlocksKeys],\n  {\n    initWorker,\n    fetchBlock,\n    processBlock,\n    numFetchedBlocks,\n    numFetchingBlocks,\n    getStatus,\n    getMemoryLeft,\n    waitForWorkerBatchSize,\n  },\n  logger,\n);\n\n// Export types to be used on the parent\ntype InitWorker = typeof initWorker;\ntype FetchBlock = typeof fetchBlock;\ntype ProcessBlock = typeof processBlock;\ntype NumFetchedBlocks = typeof numFetchedBlocks;\ntype NumFetchingBlocks = typeof numFetchingBlocks;\ntype GetWorkerStatus = typeof getStatus;\ntype GetMemoryLeft = typeof getMemoryLeft;\ntype WaitForWorkerBatchSize = typeof waitForWorkerBatchSize;\n\nexport type IIndexerWorker = {\n  processBlock: ProcessBlock;\n  fetchBlock: FetchBlock;\n  numFetchedBlocks: NumFetchedBlocks;\n  numFetchingBlocks: NumFetchingBlocks;\n  getStatus: GetWorkerStatus;\n  getMemoryLeft: GetMemoryLeft;\n  waitForWorkerBatchSize: WaitForWorkerBatchSize;\n};\n\nexport type IInitIndexerWorker = IIndexerWorker & {\n  initWorker: InitWorker;\n};\n\nprocess.on('uncaughtException', (e) => {\n  logger.error(e, 'Uncaught Exception');\n  throw e;\n});\n"]}

package/dist/indexer/worker/worker.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.module.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAA2D;AAC3D,+CAAkD;AAClD,gDAA4C;AAC5C,uEAAmE;AACnE,sDAAkD;AAYlD,IAAa,YAAY,GAAzB,MAAa,YAAY;CAAG,CAAA;AAAf,YAAY;IAVxB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAkB,CAAC,OAAO,EAAE;YAC5B,kCAAe,CAAC,QAAQ,EAAE;YAC1B,yBAAc,CAAC,OAAO,EAAE;YACxB,8BAAa;SACd;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,YAAY,CAAG;AAAf,oCAAY","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitterModule } from '@nestjs/event-emitter';\nimport { ScheduleModule } from '@nestjs/schedule';\nimport { DbModule } from '@subql/node-core';\nimport { ConfigureModule } from '../../configure/configure.module';\nimport { IndexerModule } from '../indexer.module';\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    EventEmitterModule.forRoot(),\n    ConfigureModule.register(),\n    ScheduleModule.forRoot(),\n    IndexerModule,\n  ],\n  controllers: [],\n})\nexport class WorkerModule {}\n"]}
+{"version":3,"file":"worker.module.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAA2D;AAC3D,+CAAkD;AAClD,gDAA4C;AAC5C,uEAAmE;AACnE,sDAAkD;AAY3C,IAAM,YAAY,GAAlB,MAAM,YAAY;CAAG,CAAA;AAAf,YAAY;IAVxB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAkB,CAAC,OAAO,EAAE;YAC5B,kCAAe,CAAC,QAAQ,EAAE;YAC1B,yBAAc,CAAC,OAAO,EAAE;YACxB,8BAAa;SACd;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,YAAY,CAAG;AAAf,oCAAY","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitterModule } from '@nestjs/event-emitter';\nimport { ScheduleModule } from '@nestjs/schedule';\nimport { DbModule } from '@subql/node-core';\nimport { ConfigureModule } from '../../configure/configure.module';\nimport { IndexerModule } from '../indexer.module';\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    EventEmitterModule.forRoot(),\n    ConfigureModule.register(),\n    ScheduleModule.forRoot(),\n    IndexerModule,\n  ],\n  controllers: [],\n})\nexport class WorkerModule {}\n"]}

package/dist/indexer/worker/worker.service.d.ts

@@ -1,15 +1,16 @@
-import { NodeConfig } from '@subql/node-core';
+import { NodeConfig, IProjectService } from '@subql/node-core';
+import { SubqlProjectDs } from '../../configure/SubqueryProject';
 import { EthereumApiService } from '../../ethereum';
 import { IndexerManager } from '../indexer.manager';
-export declare type FetchBlockResponse = {
+export type FetchBlockResponse = {
     parentHash: string;
 } | undefined;
-export declare type ProcessBlockResponse = {
+export type ProcessBlockResponse = {
     dynamicDsCreated: boolean;
     blockHash: string;
     reindexBlockHeight: number;
 };
-export declare type WorkerStatusResponse = {
+export type WorkerStatusResponse = {
     threadId: number;
     isIndexing: boolean;
     fetchedBlocks: number;
@@ -18,10 +19,11 @@ export declare type WorkerStatusResponse = {
 export declare class WorkerService {
     private apiService;
     private indexerManager;
+    private projectService;
     private fetchedBlocks;
     private _isIndexing;
     private queue;
-    constructor(apiService: EthereumApiService, indexerManager: IndexerManager, nodeConfig: NodeConfig);
+    constructor(apiService: EthereumApiService, indexerManager: IndexerManager, projectService: IProjectService<SubqlProjectDs>, nodeConfig: NodeConfig);
     fetchBlock(height: number): Promise<FetchBlockResponse>;
     processBlock(height: number): Promise<ProcessBlockResponse>;
     get numFetchedBlocks(): number;

package/dist/indexer/worker/worker.service.js

@@ -10,6 +10,9 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkerService = void 0;
 const node_worker_threads_1 = require("node:worker_threads");
@@ -19,9 +22,10 @@ const ethereum_1 = require("../../ethereum");
 const indexer_manager_1 = require("../indexer.manager");
 const logger = (0, node_core_1.getLogger)(`Worker Service #${node_worker_threads_1.threadId}`);
 let WorkerService = class WorkerService {
-    constructor(apiService, indexerManager, nodeConfig) {
+    constructor(apiService, indexerManager, projectService, nodeConfig) {
         this.apiService = apiService;
         this.indexerManager = indexerManager;
+        this.projectService = projectService;
         this.fetchedBlocks = {};
         this._isIndexing = false;
         this.queue = new node_core_1.AutoQueue(undefined, nodeConfig.batchSize);
@@ -58,7 +62,7 @@ let WorkerService = class WorkerService {
                 throw new Error(`Block ${height} has not been fetched`);
             }
             delete this.fetchedBlocks[height];
-            return await this.indexerManager.indexBlock(block);
+            return await this.indexerManager.indexBlock(block, await this.projectService.getAllDataSources(height));
         }
         catch (e) {
             logger.error(e, `Failed to index block ${height}: ${e.stack}`);
@@ -80,9 +84,9 @@ let WorkerService = class WorkerService {
 };
 WorkerService = __decorate([
     (0, common_1.Injectable)(),
+    __param(2, (0, common_1.Inject)('IProjectService')),
     __metadata("design:paramtypes", [ethereum_1.EthereumApiService,
-        indexer_manager_1.IndexerManager,
-        node_core_1.NodeConfig])
+        indexer_manager_1.IndexerManager, Object, node_core_1.NodeConfig])
 ], WorkerService);
 exports.WorkerService = WorkerService;
 //# sourceMappingURL=worker.service.js.map

package/dist/indexer/worker/worker.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAA4C;AAC5C,gDAAgF;AAEhF,6CAAoD;AACpD,wDAAoD;AAiBpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGxD,IAAa,aAAa,GAA1B,MAAa,aAAa;IAMxB,YACU,UAA8B,EAC9B,cAA8B,EACtC,UAAsB;QAFd,eAAU,GAAV,UAAU,CAAoB;QAC9B,mBAAc,GAAd,cAAc,CAAgB;QAPhC,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAS1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,IAAI,sBAAU,CAAC,QAAQ,EAAE,EAAE;wBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACzB,MAAM,sBAAU,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACvB,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC;qBACzD;oBAED,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAC5D,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,4CAA4C;gBAE5C,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,yBAAyB,MAAM,EAAE,CAAC,CAAC;YACxD,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;SACpD;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC1B;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AAxEY,aAAa;IADzB,IAAA,mBAAU,GAAE;qCAQW,6BAAkB;QACd,gCAAc;QAC1B,sBAAU;GATb,aAAa,CAwEzB;AAxEY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Injectable } from '@nestjs/common';\nimport { NodeConfig, getLogger, AutoQueue, memoryLock } from '@subql/node-core';\nimport { BlockWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { EthereumApiService } from '../../ethereum';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  blockHash: string;\n  reindexBlockHeight: number;\n};\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: EthereumApiService,\n    private indexerManager: IndexerManager,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          if (memoryLock.isLocked()) {\n            const start = Date.now();\n            await memoryLock.waitForUnlock();\n            const end = Date.now();\n            logger.debug(`memory lock wait time: ${end - start}ms`);\n          }\n\n          const [block] = await this.apiService.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        // const block = this.fetchedBlocks[height];\n\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(/*e, */ `Failed to fetch block ${height}`);\n      throw e;\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      return await this.indexerManager.indexBlock(block);\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    } finally {\n      this._isIndexing = false;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}
+{"version":3,"file":"worker.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,6DAA+C;AAC/C,2CAAoD;AACpD,gDAM0B;AAG1B,6CAAoD;AACpD,wDAAoD;AAiBpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,mBAAmB,8BAAQ,EAAE,CAAC,CAAC;AAGjD,IAAM,aAAa,GAAnB,MAAM,aAAa;IAMxB,YACU,UAA8B,EAC9B,cAA8B,EAE9B,cAA+C,EACvD,UAAsB;QAJd,eAAU,GAAV,UAAU,CAAoB;QAC9B,mBAAc,GAAd,cAAc,CAAgB;QAE9B,mBAAc,GAAd,cAAc,CAAiC;QATjD,kBAAa,GAAiC,EAAE,CAAC;QACjD,gBAAW,GAAG,KAAK,CAAC;QAW1B,IAAI,CAAC,KAAK,GAAG,IAAI,qBAAS,CAAC,SAAS,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC7B,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrC,0FAA0F;gBAC1F,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;oBAC/B,IAAI,sBAAU,CAAC,QAAQ,EAAE,EAAE;wBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACzB,MAAM,sBAAU,CAAC,aAAa,EAAE,CAAC;wBACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACvB,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC;qBACzD;oBAED,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBAC5D,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;iBACpC;gBAED,4CAA4C;gBAC5C,2EAA2E;gBAC3E,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;SACJ;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,yBAAyB,MAAM,EAAE,CAAC,CAAC;YACxD,MAAM,CAAC,CAAC;SACT;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI;YACF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAyB,CAAC;YAEjE,IAAI,CAAC,KAAK,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,uBAAuB,CAAC,CAAC;aACzD;YAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAElC,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CACzC,KAAK,EACL,MAAM,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,CACpD,CAAC;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;SACT;gBAAS;YACR,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC1B;IACH,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AA5EY,aAAa;IADzB,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCAFN,6BAAkB;QACd,gCAAc,UAG1B,sBAAU;GAXb,aAAa,CA4EzB;AA5EY,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { threadId } from 'node:worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  NodeConfig,\n  getLogger,\n  AutoQueue,\n  memoryLock,\n  IProjectService,\n} from '@subql/node-core';\nimport { BlockWrapper, EthereumBlockWrapper } from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { EthereumApiService } from '../../ethereum';\nimport { IndexerManager } from '../indexer.manager';\n\nexport type FetchBlockResponse = { parentHash: string } | undefined;\n\nexport type ProcessBlockResponse = {\n  dynamicDsCreated: boolean;\n  blockHash: string;\n  reindexBlockHeight: number;\n};\n\nexport type WorkerStatusResponse = {\n  threadId: number;\n  isIndexing: boolean;\n  fetchedBlocks: number;\n  toFetchBlocks: number;\n};\n\nconst logger = getLogger(`Worker Service #${threadId}`);\n\n@Injectable()\nexport class WorkerService {\n  private fetchedBlocks: Record<string, BlockWrapper> = {};\n  private _isIndexing = false;\n\n  private queue: AutoQueue<FetchBlockResponse>;\n\n  constructor(\n    private apiService: EthereumApiService,\n    private indexerManager: IndexerManager,\n    @Inject('IProjectService')\n    private projectService: IProjectService<SubqlProjectDs>,\n    nodeConfig: NodeConfig,\n  ) {\n    this.queue = new AutoQueue(undefined, nodeConfig.batchSize);\n  }\n\n  async fetchBlock(height: number): Promise<FetchBlockResponse> {\n    try {\n      return await this.queue.put(async () => {\n        // If a dynamic ds is created we might be asked to fetch blocks again, use existing result\n        if (!this.fetchedBlocks[height]) {\n          if (memoryLock.isLocked()) {\n            const start = Date.now();\n            await memoryLock.waitForUnlock();\n            const end = Date.now();\n            logger.debug(`memory lock wait time: ${end - start}ms`);\n          }\n\n          const [block] = await this.apiService.fetchBlocks([height]);\n          this.fetchedBlocks[height] = block;\n        }\n\n        // const block = this.fetchedBlocks[height];\n        // Return info to get the runtime version, this lets the worker thread know\n        return undefined;\n      });\n    } catch (e) {\n      logger.error(/*e, */ `Failed to fetch block ${height}`);\n      throw e;\n    }\n  }\n\n  async processBlock(height: number): Promise<ProcessBlockResponse> {\n    try {\n      this._isIndexing = true;\n      const block = this.fetchedBlocks[height] as EthereumBlockWrapper;\n\n      if (!block) {\n        throw new Error(`Block ${height} has not been fetched`);\n      }\n\n      delete this.fetchedBlocks[height];\n\n      return await this.indexerManager.indexBlock(\n        block,\n        await this.projectService.getAllDataSources(height),\n      );\n    } catch (e) {\n      logger.error(e, `Failed to index block ${height}: ${e.stack}`);\n      throw e;\n    } finally {\n      this._isIndexing = false;\n    }\n  }\n\n  get numFetchedBlocks(): number {\n    return Object.keys(this.fetchedBlocks).length;\n  }\n\n  get numFetchingBlocks(): number {\n    return this.queue.size;\n  }\n\n  get isIndexing(): boolean {\n    return this._isIndexing;\n  }\n}\n"]}

package/dist/indexer/worker/worker.unfinalizedBlocks.service.d.ts

@@ -1,6 +1,6 @@
 import { EthereumBlock } from '@subql/types-ethereum';
 import { IUnfinalizedBlocksService } from '../unfinalizedBlocks.service';
-export declare type HostUnfinalizedBlocks = {
+export type HostUnfinalizedBlocks = {
     unfinalizedBlocksProcess: (block: EthereumBlock) => Promise<number | null>;
 };
 export declare const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[];

package/dist/indexer/worker/worker.unfinalizedBlocks.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAQ/B,QAAA,yBAAyB,GAAoC;IACxE,0BAA0B;CAC3B,CAAC;AAGF,IAAa,8BAA8B,GAA3C,MAAa,8BAA8B;IAGzC,YAAoB,IAA2B;QAA3B,SAAI,GAAJ,IAAI,CAAuB;IAAG,CAAC;IAEnD,KAAK,CAAC,wBAAwB,CAAC,KAAoB;QACjD,OAAO,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;CACF,CAAA;AARY,8BAA8B;IAD1C,IAAA,mBAAU,GAAE;;GACA,8BAA8B,CAQ1C;AARY,wEAA8B","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { EthereumBlock } from '@subql/types-ethereum';\nimport { IUnfinalizedBlocksService } from '../unfinalizedBlocks.service';\n\nexport type HostUnfinalizedBlocks = {\n  unfinalizedBlocksProcess: (block: EthereumBlock) => Promise<number | null>;\n};\n\nexport const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[] = [\n  'unfinalizedBlocksProcess',\n];\n\n@Injectable()\nexport class WorkerUnfinalizedBlocksService\n  implements IUnfinalizedBlocksService\n{\n  constructor(private host: HostUnfinalizedBlocks) {}\n\n  async processUnfinalizedBlocks(block: EthereumBlock): Promise<number | null> {\n    return this.host.unfinalizedBlocksProcess(block);\n  }\n}\n"]}
+{"version":3,"file":"worker.unfinalizedBlocks.service.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.unfinalizedBlocks.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAQ/B,QAAA,yBAAyB,GAAoC;IACxE,0BAA0B;CAC3B,CAAC;AAGK,IAAM,8BAA8B,GAApC,MAAM,8BAA8B;IAGzC,YAAoB,IAA2B;QAA3B,SAAI,GAAJ,IAAI,CAAuB;IAAG,CAAC;IAEnD,KAAK,CAAC,wBAAwB,CAAC,KAAoB;QACjD,OAAO,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;CACF,CAAA;AARY,8BAA8B;IAD1C,IAAA,mBAAU,GAAE;;GACA,8BAA8B,CAQ1C;AARY,wEAA8B","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { EthereumBlock } from '@subql/types-ethereum';\nimport { IUnfinalizedBlocksService } from '../unfinalizedBlocks.service';\n\nexport type HostUnfinalizedBlocks = {\n  unfinalizedBlocksProcess: (block: EthereumBlock) => Promise<number | null>;\n};\n\nexport const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[] = [\n  'unfinalizedBlocksProcess',\n];\n\n@Injectable()\nexport class WorkerUnfinalizedBlocksService\n  implements IUnfinalizedBlocksService\n{\n  constructor(private host: HostUnfinalizedBlocks) {}\n\n  async processUnfinalizedBlocks(block: EthereumBlock): Promise<number | null> {\n    return this.host.unfinalizedBlocksProcess(block);\n  }\n}\n"]}

package/dist/main.js

@@ -13,5 +13,7 @@ BigInt.prototype.toJSON = function () {
 // Lazy import, to allow logger to be initialised before bootstrap()
 // As bootstrap runs services that requires logger
 const { bootstrap } = require('./init');
-void bootstrap();
+if (!(argv._[0] === 'test')) {
+    void bootstrap();
+}
 //# sourceMappingURL=main.js.map

package/dist/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;AAEtC,oDAAqD;AACrD,mCAAuC;AAEvC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE7B,MAAM,CAAC,SAAiB,CAAC,MAAM,GAAG;IACjC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;AACzB,CAAC,CAAC;AAEF,iIAAiI;AACjI,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oEAAoE;AACpE,kDAAkD;AAClD,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACxC,KAAK,SAAS,EAAE,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { initLogger } from '@subql/node-core/logger';\nimport { yargsOptions } from './yargs';\n\nconst { argv } = yargsOptions;\n\n(BigInt.prototype as any).toJSON = function () {\n  return this.toString();\n};\n\n// initLogger is imported from true path, to make sure getLogger (or other logger values that relies on logger) isn't initialised\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\n// Lazy import, to allow logger to be initialised before bootstrap()\n// As bootstrap runs services that requires logger\nconst { bootstrap } = require('./init');\nvoid bootstrap();\n"]}
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;AAEtC,oDAAqD;AACrD,mCAAuC;AAEvC,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE7B,MAAM,CAAC,SAAiB,CAAC,MAAM,GAAG;IACjC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;AACzB,CAAC,CAAC;AAEF,iIAAiI;AACjI,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oEAAoE;AACpE,kDAAkD;AAClD,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACxC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,EAAE;IAC3B,KAAK,SAAS,EAAE,CAAC;CAClB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { initLogger } from '@subql/node-core/logger';\nimport { yargsOptions } from './yargs';\n\nconst { argv } = yargsOptions;\n\n(BigInt.prototype as any).toJSON = function () {\n  return this.toString();\n};\n\n// initLogger is imported from true path, to make sure getLogger (or other logger values that relies on logger) isn't initialised\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\n// Lazy import, to allow logger to be initialised before bootstrap()\n// As bootstrap runs services that requires logger\nconst { bootstrap } = require('./init');\nif (!(argv._[0] === 'test')) {\n  void bootstrap();\n}\n"]}

package/dist/meta/meta.controller.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta.controller.js","sourceRoot":"","sources":["../../src/meta/meta.controller.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAAiD;AACjD,iDAA6C;AAG7C,IAAa,cAAc,GAA3B,MAAa,cAAc;IACzB,YAAoB,WAAwB;QAAxB,gBAAW,GAAX,WAAW,CAAa;IAAG,CAAC;IAGhD,OAAO;QACL,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;IACpC,CAAC;CACF,CAAA;AAHC;IADC,IAAA,YAAG,GAAE;;;;6CAGL;AANU,cAAc;IAD1B,IAAA,mBAAU,EAAC,MAAM,CAAC;qCAEgB,0BAAW;GADjC,cAAc,CAO1B;AAPY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Controller, Get } from '@nestjs/common';\nimport { MetaService } from './meta.service';\n\n@Controller('meta')\nexport class MetaController {\n  constructor(private metaService: MetaService) {}\n\n  @Get()\n  getMeta() {\n    return this.metaService.getMeta();\n  }\n}\n"]}
+{"version":3,"file":"meta.controller.js","sourceRoot":"","sources":["../../src/meta/meta.controller.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAAiD;AACjD,iDAA6C;AAGtC,IAAM,cAAc,GAApB,MAAM,cAAc;IACzB,YAAoB,WAAwB;QAAxB,gBAAW,GAAX,WAAW,CAAa;IAAG,CAAC;IAGhD,OAAO;QACL,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;IACpC,CAAC;CACF,CAAA;AAJC;IAAC,IAAA,YAAG,GAAE;;;;6CAGL;AANU,cAAc;IAD1B,IAAA,mBAAU,EAAC,MAAM,CAAC;qCAEgB,0BAAW;GADjC,cAAc,CAO1B;AAPY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Controller, Get } from '@nestjs/common';\nimport { MetaService } from './meta.service';\n\n@Controller('meta')\nexport class MetaController {\n  constructor(private metaService: MetaService) {}\n\n  @Get()\n  getMeta() {\n    return this.metaService.getMeta();\n  }\n}\n"]}

package/dist/meta/meta.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta.module.js","sourceRoot":"","sources":["../../src/meta/meta.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,mEAGqC;AACrC,0DAAsD;AACtD,uDAAmD;AACnD,iDAA6C;AAgE7C,IAAa,UAAU,GAAvB,MAAa,UAAU;CAAG,CAAA;AAAb,UAAU;IA9DtB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,oCAAgB,CAAC,QAAQ,EAAE,EAAE,0BAAW,CAAC;QACnD,WAAW,EAAE,CAAC,gCAAc,EAAE,4BAAgB,EAAE,2BAAe,CAAC;QAChE,SAAS,EAAE;YACT,+BAAmB;YACnB,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,6BAA6B;gBACnC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,4CAA4C;aACnD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,uCAAuC;gBAC7C,IAAI,EAAE,qCAAqC;aAC5C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,mCAAmC;gBACzC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,iCAAiC;gBACvC,IAAI,EAAE,8BAA8B;aACrC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,wCAAwC;aAC/C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,+CAA+C;aACtD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,sDAAsD;aAC7D,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,+BAA+B;aACtC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,qEAAqE;aAC5E,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,wCAAwC;gBAC9C,IAAI,EAAE,mDAAmD;aAC1D,CAAC;YACF,0BAAW;YACX,yBAAa;YACb,wBAAY;SACb;KACF,CAAC;GACW,UAAU,CAAG;AAAb,gCAAU","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  MetricEventListener,\n  ReadyController,\n  ReadyService,\n  HealthController,\n  HealthService,\n} from '@subql/node-core';\nimport {\n  makeGaugeProvider,\n  PrometheusModule,\n} from '@willsoto/nestjs-prometheus';\nimport { FetchModule } from '../indexer/fetch.module';\nimport { MetaController } from './meta.controller';\nimport { MetaService } from './meta.service';\n\n@Module({\n  imports: [PrometheusModule.register(), FetchModule],\n  controllers: [MetaController, HealthController, ReadyController],\n  providers: [\n    MetricEventListener,\n    makeGaugeProvider({\n      name: 'subql_indexer_api_connected',\n      help: 'The indexer api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_injected_api_connected',\n      help: 'The indexer injected api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processing_block_height',\n      help: 'The current processing block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_height',\n      help: 'The last processed block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_target_block_height',\n      help: 'The latest finalized block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_best_block_height',\n      help: 'The latest best block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_block_queue_size',\n      help: 'The size of fetched block queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_blocknumber_queue_size',\n      help: 'The size of fetched block number queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_using_dictionary',\n      help: 'The status of indexer is using the dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_skip_dictionary_count',\n      help: 'The number of times indexer been skip use dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_count',\n      help: 'The number of processed block',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_threshold',\n      help: 'Store cache will flush once cache record size excess this threshold',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_records_size',\n      help: 'Number of records waiting to flush in store cache',\n    }),\n    MetaService,\n    HealthService,\n    ReadyService,\n  ],\n})\nexport class MetaModule {}\n"]}
+{"version":3,"file":"meta.module.js","sourceRoot":"","sources":["../../src/meta/meta.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAM0B;AAC1B,mEAGqC;AACrC,0DAAsD;AACtD,uDAAmD;AACnD,iDAA6C;AAgEtC,IAAM,UAAU,GAAhB,MAAM,UAAU;CAAG,CAAA;AAAb,UAAU;IA9DtB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,oCAAgB,CAAC,QAAQ,EAAE,EAAE,0BAAW,CAAC;QACnD,WAAW,EAAE,CAAC,gCAAc,EAAE,4BAAgB,EAAE,2BAAe,CAAC;QAChE,SAAS,EAAE;YACT,+BAAmB;YACnB,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,6BAA6B;gBACnC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,4CAA4C;aACnD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,uCAAuC;gBAC7C,IAAI,EAAE,qCAAqC;aAC5C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,mCAAmC;gBACzC,IAAI,EAAE,mCAAmC;aAC1C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,iCAAiC;gBACvC,IAAI,EAAE,8BAA8B;aACrC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,iCAAiC;aACxC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,sCAAsC;gBAC5C,IAAI,EAAE,wCAAwC;aAC/C,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,gCAAgC;gBACtC,IAAI,EAAE,+CAA+C;aACtD,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,sDAAsD;aAC7D,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,+BAA+B;aACtC,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,qCAAqC;gBAC3C,IAAI,EAAE,qEAAqE;aAC5E,CAAC;YACF,IAAA,qCAAiB,EAAC;gBAChB,IAAI,EAAE,wCAAwC;gBAC9C,IAAI,EAAE,mDAAmD;aAC1D,CAAC;YACF,0BAAW;YACX,yBAAa;YACb,wBAAY;SACb;KACF,CAAC;GACW,UAAU,CAAG;AAAb,gCAAU","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport {\n  MetricEventListener,\n  ReadyController,\n  ReadyService,\n  HealthController,\n  HealthService,\n} from '@subql/node-core';\nimport {\n  makeGaugeProvider,\n  PrometheusModule,\n} from '@willsoto/nestjs-prometheus';\nimport { FetchModule } from '../indexer/fetch.module';\nimport { MetaController } from './meta.controller';\nimport { MetaService } from './meta.service';\n\n@Module({\n  imports: [PrometheusModule.register(), FetchModule],\n  controllers: [MetaController, HealthController, ReadyController],\n  providers: [\n    MetricEventListener,\n    makeGaugeProvider({\n      name: 'subql_indexer_api_connected',\n      help: 'The indexer api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_injected_api_connected',\n      help: 'The indexer injected api connection status',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processing_block_height',\n      help: 'The current processing block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_height',\n      help: 'The last processed block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_target_block_height',\n      help: 'The latest finalized block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_best_block_height',\n      help: 'The latest best block height',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_block_queue_size',\n      help: 'The size of fetched block queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_blocknumber_queue_size',\n      help: 'The size of fetched block number queue',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_using_dictionary',\n      help: 'The status of indexer is using the dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_skip_dictionary_count',\n      help: 'The number of times indexer been skip use dictionary',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_processed_block_count',\n      help: 'The number of processed block',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_threshold',\n      help: 'Store cache will flush once cache record size excess this threshold',\n    }),\n    makeGaugeProvider({\n      name: 'subql_indexer_store_cache_records_size',\n      help: 'Number of records waiting to flush in store cache',\n    }),\n    MetaService,\n    HealthService,\n    ReadyService,\n  ],\n})\nexport class MetaModule {}\n"]}

package/dist/meta/meta.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta.service.js","sourceRoot":"","sources":["../../src/meta/meta.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,yDAAgD;AAChD,+CAA4C;AAC5C,gDAW0B;AAE1B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC9E,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,UAAU,CAAC,CAAC;AAGrC,IAAa,WAAW,GAAxB,MAAa,WAAW;IAsBtB,YACU,YAA0B,EAC1B,UAAsB;QADtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QAZxB,qBAAgB,GAAG,CAAC,CAAC;QACrB,mBAAc,GAAG,CAAC,CAAC;QACnB,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gBAAW,GAAG,CAAC,CAAC;QAChB,kCAA6B,GAAG,CAAC,CAAC;QAClC,8BAAyB,GAAG,CAAC,CAAC;QAC9B,4BAAuB,GAAG,CAAC,CAAC;QAC5B,yBAAoB,GAAG,CAAC,CAAC;IAM9B,CAAC;IAEJ,OAAO;QACL,uBACE,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,EACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B,EAC3D,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,UAAU,EAAE,IAAI,CAAC,UAAU,EAC3B,kBAAkB,EAAE,cAAc,EAClC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,EACnD,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EACxB,kBAAkB,EAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAC/C,eAAe,EAAE,IAAI,CAAC,eAAe,IAClC,IAAI,CAAC,WAAW,EACnB;IACJ,CAAC;IAGD,eAAe;QACb,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACvC,cAAc,EACd,IAAI,CAAC,YAAY,CAClB,CAAC;IACJ,CAAC;IAGD,qBAAqB,CAAC,YAAiC;QACrD,IAAI,CAAC,uBAAuB,GAAG,YAAY,CAAC,MAAM,CAAC;QACnD,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,oBAAoB,CAAC,YAAwC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,YAAY,CAAC,mBAAmB,CAAC;QAC5D,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,iBAAiB,CAAC,YAAgC;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,CAAC;IAGD,eAAe,CAAC,YAA8B;QAC5C,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACxC,CAAC;IAGD,qBAAqB,CAAC,WAAmC;QACvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAGD,kBAAkB,CAAC,EAAE,KAAK,EAAwB;QAChD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC;IAC9B,CAAC;IAGD,qBAAqB,CAAC,EAAE,KAAK,EAAwB;QACnD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;IAGD,mBAAmB,CAAC,IAAY;QAC9B,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,qBAAqB,CAAC,MAAc;QAClC,IAAI,CAAC,wBAAwB,GAAG,MAAM,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,6BAA6B,GAAG,MAAM,CAAC;SAC7C;IACH,CAAC;IAGD,gBAAgB;QACd,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,aAAa;QACX,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,mBAAmB;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC7B,OAAO;SACR;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,yBAAyB,CAAC;QACrE,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvD,MAAM,cAAc,GAClB,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,6BAA6B,CAAC;QACrE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;QAC/B,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC,wBAAwB,CAAC;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAC9D,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,uBAAuB,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,cAAc,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,KAAK,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACxE,CAAC,CACF;iBACY,CAAC,cAAc,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACvD,CAAC,CACF,mBAAmB,CAAC,QAAQ,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;eAClD,CAAC,UAAU,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;CACF,CAAA;AArGC;IADC,IAAA,mBAAQ,EAAC,sBAAsB,CAAC;;;;kDAMhC;AAGD;IADC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAIrC;AAGD;IADC,IAAA,uBAAO,EAAC,wBAAY,CAAC,mBAAmB,CAAC;;;;uDAIzC;AAGD;IADC,IAAA,uBAAO,EAAC,wBAAY,CAAC,WAAW,CAAC;;;;oDAGjC;AAGD;IADC,IAAA,uBAAO,EAAC,wBAAY,CAAC,SAAS,CAAC;;;;kDAG/B;AAGD;IADC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAGD;IADC,IAAA,uBAAO,EAAC,wBAAY,CAAC,YAAY,CAAC;;;;qDAGlC;AAGD;IADC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAGD;IADC,IAAA,uBAAO,EAAC,eAAe,CAAC;;;;sDAMxB;AAGD;IADC,IAAA,uBAAO,EAAC,iBAAiB,CAAC;;;;wDAM1B;AAGD;IADC,IAAA,uBAAO,EAAC,YAAY,CAAC;;;;mDAMrB;AAGD;IADC,IAAA,uBAAO,EAAC,SAAS,CAAC;;;;gDAMlB;AAGD;IADC,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sDAwBf;AAnJU,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAwBa,wBAAY;QACd,sBAAU;GAxBrB,WAAW,CAoJvB;AApJY,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { OnEvent } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  BestBlockPayload,\n  EventPayload,\n  IndexerEvent,\n  NetworkMetadataPayload,\n  ProcessBlockPayload,\n  ProcessedBlockCountPayload,\n  TargetBlockPayload,\n  StoreService,\n  getLogger,\n  NodeConfig,\n} from '@subql/node-core';\n\nconst UPDATE_HEIGHT_INTERVAL = 60000;\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: polkadotSdkVersion } = require('@polkadot/api/package.json');\nconst { version: packageVersion } = require('../../package.json');\nconst logger = getLogger('profiler');\n\n@Injectable()\nexport class MetaService {\n  private currentProcessingHeight: number;\n  private currentProcessingTimestamp: number;\n  private bestHeight: number;\n  private targetHeight: number;\n  private networkMeta: NetworkMetadataPayload;\n  private apiConnected: boolean;\n  private usingDictionary: boolean;\n  private injectedApiConnected: boolean;\n  private lastProcessedHeight: number;\n  private lastProcessedTimestamp: number;\n  private processedBlockCount: number;\n  private accEnqueueBlocks = 0;\n  private accFetchBlocks = 0;\n  private currentFilteringBlockNum = 0;\n  private accRpcCalls = 0;\n  private lastReportedFilteringBlockNum = 0;\n  private lastReportedEnqueueBlocks = 0;\n  private lastReportedFetchBlocks = 0;\n  private lastReportedRpcCalls = 0;\n  private lastStatsReportedTs: Date;\n\n  constructor(\n    private storeService: StoreService,\n    private nodeConfig: NodeConfig,\n  ) {}\n\n  getMeta() {\n    return {\n      currentProcessingHeight: this.currentProcessingHeight,\n      currentProcessingTimestamp: this.currentProcessingTimestamp,\n      targetHeight: this.targetHeight,\n      bestHeight: this.bestHeight,\n      indexerNodeVersion: packageVersion,\n      lastProcessedHeight: this.lastProcessedHeight,\n      lastProcessedTimestamp: this.lastProcessedTimestamp,\n      uptime: process.uptime(),\n      polkadotSdkVersion,\n      processedBlockCount: this.processedBlockCount,\n      apiConnected: this.apiConnected,\n      injectedApiConnected: this.injectedApiConnected,\n      usingDictionary: this.usingDictionary,\n      ...this.networkMeta,\n    };\n  }\n\n  @Interval(UPDATE_HEIGHT_INTERVAL)\n  getTargetHeight(): void {\n    this.storeService.storeCache.metadata.set(\n      'targetHeight',\n      this.targetHeight,\n    );\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessing)\n  handleProcessingBlock(blockPayload: ProcessBlockPayload): void {\n    this.currentProcessingHeight = blockPayload.height;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessedCount)\n  handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void {\n    this.processedBlockCount = blockPayload.processedBlockCount;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockTarget)\n  handleTargetBlock(blockPayload: TargetBlockPayload): void {\n    this.targetHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.BlockBest)\n  handleBestBlock(blockPayload: BestBlockPayload): void {\n    this.bestHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.NetworkMetadata)\n  handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void {\n    this.networkMeta = networkMeta;\n  }\n\n  @OnEvent(IndexerEvent.ApiConnected)\n  handleApiConnected({ value }: EventPayload<number>): void {\n    this.apiConnected = !!value;\n  }\n\n  @OnEvent(IndexerEvent.UsingDictionary)\n  handleUsingDictionary({ value }: EventPayload<number>): void {\n    this.usingDictionary = !!value;\n  }\n\n  @OnEvent('enqueueBlocks')\n  handleEnqueueBlocks(size: number): void {\n    this.accEnqueueBlocks += size;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('filteringBlocks')\n  handleFilteringBlocks(height: number): void {\n    this.currentFilteringBlockNum = height;\n    if (!this.lastStatsReportedTs) {\n      this.lastReportedFilteringBlockNum = height;\n    }\n  }\n\n  @OnEvent('fetchBlock')\n  handleFetchBlock(): void {\n    this.accFetchBlocks++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('rpcCall')\n  handleRpcCall(): void {\n    this.accRpcCalls++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @Interval(10000)\n  blockFilteringSpeed(): void {\n    if (!this.nodeConfig.profiler) {\n      return;\n    }\n    const count = this.accEnqueueBlocks - this.lastReportedEnqueueBlocks;\n    this.lastReportedEnqueueBlocks = this.accEnqueueBlocks;\n    const filteringCount =\n      this.currentFilteringBlockNum - this.lastReportedFilteringBlockNum;\n    const now = new Date();\n    const timepass = now.getTime() - this.lastStatsReportedTs.getTime();\n    this.lastStatsReportedTs = now;\n    this.lastReportedFilteringBlockNum = this.currentFilteringBlockNum;\n    const rpcCalls = this.accRpcCalls - this.lastReportedRpcCalls;\n    this.lastReportedRpcCalls = this.accRpcCalls;\n    const fetchCount = this.accFetchBlocks - this.lastReportedFetchBlocks;\n    this.lastReportedFetchBlocks = this.accFetchBlocks;\n    logger.info(`actual block filtering: ${(count / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, \\\nseeming speed: ${(filteringCount / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, rpcCalls: ${(rpcCalls / (timepass / 1000)).toFixed(2)}/sec \\\nfetch speed: ${(fetchCount / (timepass / 1000)).toFixed(2)}/sec`);\n  }\n}\n"]}
+{"version":3,"file":"meta.service.js","sourceRoot":"","sources":["../../src/meta/meta.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAA4C;AAC5C,yDAAgD;AAChD,+CAA4C;AAC5C,gDAW0B;AAE1B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC9E,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,UAAU,CAAC,CAAC;AAG9B,IAAM,WAAW,GAAjB,MAAM,WAAW;IAsBtB,YACU,YAA0B,EAC1B,UAAsB;QADtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QAZxB,qBAAgB,GAAG,CAAC,CAAC;QACrB,mBAAc,GAAG,CAAC,CAAC;QACnB,6BAAwB,GAAG,CAAC,CAAC;QAC7B,gBAAW,GAAG,CAAC,CAAC;QAChB,kCAA6B,GAAG,CAAC,CAAC;QAClC,8BAAyB,GAAG,CAAC,CAAC;QAC9B,4BAAuB,GAAG,CAAC,CAAC;QAC5B,yBAAoB,GAAG,CAAC,CAAC;IAM9B,CAAC;IAEJ,OAAO;QACL,uBACE,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,EACrD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B,EAC3D,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,UAAU,EAAE,IAAI,CAAC,UAAU,EAC3B,kBAAkB,EAAE,cAAc,EAClC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,sBAAsB,EAAE,IAAI,CAAC,sBAAsB,EACnD,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EACxB,kBAAkB,EAClB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAC7C,YAAY,EAAE,IAAI,CAAC,YAAY,EAC/B,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,EAC/C,eAAe,EAAE,IAAI,CAAC,eAAe,IAClC,IAAI,CAAC,WAAW,EACnB;IACJ,CAAC;IAGD,eAAe;QACb,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CACvC,cAAc,EACd,IAAI,CAAC,YAAY,CAClB,CAAC;IACJ,CAAC;IAGD,qBAAqB,CAAC,YAAiC;QACrD,IAAI,CAAC,uBAAuB,GAAG,YAAY,CAAC,MAAM,CAAC;QACnD,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,oBAAoB,CAAC,YAAwC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,YAAY,CAAC,mBAAmB,CAAC;QAC5D,IAAI,CAAC,0BAA0B,GAAG,YAAY,CAAC,SAAS,CAAC;IAC3D,CAAC;IAGD,iBAAiB,CAAC,YAAgC;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,CAAC;IAGD,eAAe,CAAC,YAA8B;QAC5C,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACxC,CAAC;IAGD,qBAAqB,CAAC,WAAmC;QACvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAGD,kBAAkB,CAAC,EAAE,KAAK,EAAwB;QAChD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC;IAC9B,CAAC;IAGD,qBAAqB,CAAC,EAAE,KAAK,EAAwB;QACnD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;IAGD,mBAAmB,CAAC,IAAY;QAC9B,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,qBAAqB,CAAC,MAAc;QAClC,IAAI,CAAC,wBAAwB,GAAG,MAAM,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,6BAA6B,GAAG,MAAM,CAAC;SAC7C;IACH,CAAC;IAGD,gBAAgB;QACd,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,aAAa;QACX,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC;SACvC;IACH,CAAC;IAGD,mBAAmB;QACjB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE;YAC7B,OAAO;SACR;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,yBAAyB,CAAC;QACrE,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACvD,MAAM,cAAc,GAClB,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,6BAA6B,CAAC;QACrE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;QACpE,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;QAC/B,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC,wBAAwB,CAAC;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAC9D,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,uBAAuB,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,cAAc,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,KAAK,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACxE,CAAC,CACF;iBACY,CAAC,cAAc,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CACvD,CAAC,CACF,mBAAmB,CAAC,QAAQ,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;eAClD,CAAC,UAAU,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;CACF,CAAA;AAtGC;IAAC,IAAA,mBAAQ,EAAC,sBAAsB,CAAC;;;;kDAMhC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAIrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,mBAAmB,CAAC;;;;uDAIzC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,WAAW,CAAC;;;;oDAGjC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,SAAS,CAAC;;;;kDAG/B;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,YAAY,CAAC;;;;qDAGlC;AAED;IAAC,IAAA,uBAAO,EAAC,wBAAY,CAAC,eAAe,CAAC;;;;wDAGrC;AAED;IAAC,IAAA,uBAAO,EAAC,eAAe,CAAC;;;;sDAMxB;AAED;IAAC,IAAA,uBAAO,EAAC,iBAAiB,CAAC;;;;wDAM1B;AAED;IAAC,IAAA,uBAAO,EAAC,YAAY,CAAC;;;;mDAMrB;AAED;IAAC,IAAA,uBAAO,EAAC,SAAS,CAAC;;;;gDAMlB;AAED;IAAC,IAAA,mBAAQ,EAAC,KAAK,CAAC;;;;sDAwBf;AAnJU,WAAW;IADvB,IAAA,mBAAU,GAAE;qCAwBa,wBAAY;QACd,sBAAU;GAxBrB,WAAW,CAoJvB;AApJY,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable } from '@nestjs/common';\nimport { OnEvent } from '@nestjs/event-emitter';\nimport { Interval } from '@nestjs/schedule';\nimport {\n  BestBlockPayload,\n  EventPayload,\n  IndexerEvent,\n  NetworkMetadataPayload,\n  ProcessBlockPayload,\n  ProcessedBlockCountPayload,\n  TargetBlockPayload,\n  StoreService,\n  getLogger,\n  NodeConfig,\n} from '@subql/node-core';\n\nconst UPDATE_HEIGHT_INTERVAL = 60000;\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: polkadotSdkVersion } = require('@polkadot/api/package.json');\nconst { version: packageVersion } = require('../../package.json');\nconst logger = getLogger('profiler');\n\n@Injectable()\nexport class MetaService {\n  private currentProcessingHeight: number;\n  private currentProcessingTimestamp: number;\n  private bestHeight: number;\n  private targetHeight: number;\n  private networkMeta: NetworkMetadataPayload;\n  private apiConnected: boolean;\n  private usingDictionary: boolean;\n  private injectedApiConnected: boolean;\n  private lastProcessedHeight: number;\n  private lastProcessedTimestamp: number;\n  private processedBlockCount: number;\n  private accEnqueueBlocks = 0;\n  private accFetchBlocks = 0;\n  private currentFilteringBlockNum = 0;\n  private accRpcCalls = 0;\n  private lastReportedFilteringBlockNum = 0;\n  private lastReportedEnqueueBlocks = 0;\n  private lastReportedFetchBlocks = 0;\n  private lastReportedRpcCalls = 0;\n  private lastStatsReportedTs: Date;\n\n  constructor(\n    private storeService: StoreService,\n    private nodeConfig: NodeConfig,\n  ) {}\n\n  getMeta() {\n    return {\n      currentProcessingHeight: this.currentProcessingHeight,\n      currentProcessingTimestamp: this.currentProcessingTimestamp,\n      targetHeight: this.targetHeight,\n      bestHeight: this.bestHeight,\n      indexerNodeVersion: packageVersion,\n      lastProcessedHeight: this.lastProcessedHeight,\n      lastProcessedTimestamp: this.lastProcessedTimestamp,\n      uptime: process.uptime(),\n      polkadotSdkVersion,\n      processedBlockCount: this.processedBlockCount,\n      apiConnected: this.apiConnected,\n      injectedApiConnected: this.injectedApiConnected,\n      usingDictionary: this.usingDictionary,\n      ...this.networkMeta,\n    };\n  }\n\n  @Interval(UPDATE_HEIGHT_INTERVAL)\n  getTargetHeight(): void {\n    this.storeService.storeCache.metadata.set(\n      'targetHeight',\n      this.targetHeight,\n    );\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessing)\n  handleProcessingBlock(blockPayload: ProcessBlockPayload): void {\n    this.currentProcessingHeight = blockPayload.height;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockProcessedCount)\n  handleProcessedBlock(blockPayload: ProcessedBlockCountPayload): void {\n    this.processedBlockCount = blockPayload.processedBlockCount;\n    this.currentProcessingTimestamp = blockPayload.timestamp;\n  }\n\n  @OnEvent(IndexerEvent.BlockTarget)\n  handleTargetBlock(blockPayload: TargetBlockPayload): void {\n    this.targetHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.BlockBest)\n  handleBestBlock(blockPayload: BestBlockPayload): void {\n    this.bestHeight = blockPayload.height;\n  }\n\n  @OnEvent(IndexerEvent.NetworkMetadata)\n  handleNetworkMetadata(networkMeta: NetworkMetadataPayload): void {\n    this.networkMeta = networkMeta;\n  }\n\n  @OnEvent(IndexerEvent.ApiConnected)\n  handleApiConnected({ value }: EventPayload<number>): void {\n    this.apiConnected = !!value;\n  }\n\n  @OnEvent(IndexerEvent.UsingDictionary)\n  handleUsingDictionary({ value }: EventPayload<number>): void {\n    this.usingDictionary = !!value;\n  }\n\n  @OnEvent('enqueueBlocks')\n  handleEnqueueBlocks(size: number): void {\n    this.accEnqueueBlocks += size;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('filteringBlocks')\n  handleFilteringBlocks(height: number): void {\n    this.currentFilteringBlockNum = height;\n    if (!this.lastStatsReportedTs) {\n      this.lastReportedFilteringBlockNum = height;\n    }\n  }\n\n  @OnEvent('fetchBlock')\n  handleFetchBlock(): void {\n    this.accFetchBlocks++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @OnEvent('rpcCall')\n  handleRpcCall(): void {\n    this.accRpcCalls++;\n    if (!this.lastStatsReportedTs) {\n      this.lastStatsReportedTs = new Date();\n    }\n  }\n\n  @Interval(10000)\n  blockFilteringSpeed(): void {\n    if (!this.nodeConfig.profiler) {\n      return;\n    }\n    const count = this.accEnqueueBlocks - this.lastReportedEnqueueBlocks;\n    this.lastReportedEnqueueBlocks = this.accEnqueueBlocks;\n    const filteringCount =\n      this.currentFilteringBlockNum - this.lastReportedFilteringBlockNum;\n    const now = new Date();\n    const timepass = now.getTime() - this.lastStatsReportedTs.getTime();\n    this.lastStatsReportedTs = now;\n    this.lastReportedFilteringBlockNum = this.currentFilteringBlockNum;\n    const rpcCalls = this.accRpcCalls - this.lastReportedRpcCalls;\n    this.lastReportedRpcCalls = this.accRpcCalls;\n    const fetchCount = this.accFetchBlocks - this.lastReportedFetchBlocks;\n    this.lastReportedFetchBlocks = this.accFetchBlocks;\n    logger.info(`actual block filtering: ${(count / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, \\\nseeming speed: ${(filteringCount / (timepass / 1000)).toFixed(\n      2,\n    )}/sec, rpcCalls: ${(rpcCalls / (timepass / 1000)).toFixed(2)}/sec \\\nfetch speed: ${(fetchCount / (timepass / 1000)).toFixed(2)}/sec`);\n  }\n}\n"]}

package/dist/subcommands/forceClean.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"forceClean.module.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAA4C;AAC5C,oEAAgE;AAChE,6DAAyD;AAMzD,IAAa,uBAAuB,GAApC,MAAa,uBAAuB;CAAG,CAAA;AAA1B,uBAAuB;IAJnC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE,CAAC,sCAAiB,CAAC;QAC9B,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,uBAAuB,CAAG;AAA1B,0DAAuB;AAUpC,IAAa,gBAAgB,GAA7B,MAAa,gBAAgB;CAAG,CAAA;AAAnB,gBAAgB;IAR5B,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,uBAAuB;SACxB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,gBAAgB,CAAG;AAAnB,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { DbModule } from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { ForceCleanService } from './forceClean.service';\n\n@Module({\n  providers: [ForceCleanService],\n  controllers: [],\n})\nexport class ForceCleanFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ForceCleanFeatureModule,\n  ],\n  controllers: [],\n})\nexport class ForceCleanModule {}\n"]}
+{"version":3,"file":"forceClean.module.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,gDAA4C;AAC5C,oEAAgE;AAChE,6DAAyD;AAMlD,IAAM,uBAAuB,GAA7B,MAAM,uBAAuB;CAAG,CAAA;AAA1B,uBAAuB;IAJnC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE,CAAC,sCAAiB,CAAC;QAC9B,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,uBAAuB,CAAG;AAA1B,0DAAuB;AAU7B,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;CAAG,CAAA;AAAnB,gBAAgB;IAR5B,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,uBAAuB;SACxB;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,gBAAgB,CAAG;AAAnB,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { DbModule } from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { ForceCleanService } from './forceClean.service';\n\n@Module({\n  providers: [ForceCleanService],\n  controllers: [],\n})\nexport class ForceCleanFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ForceCleanFeatureModule,\n  ],\n  controllers: [],\n})\nexport class ForceCleanModule {}\n"]}

package/dist/subcommands/forceClean.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"forceClean.service.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,2CAAoD;AACpD,gDAM0B;AAC1B,wCAAuD;AACvD,yCAAkD;AAClD,kEAA+D;AAE/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AAGxC,IAAa,iBAAiB,GAA9B,MAAa,iBAAiB;IAC5B,YACmB,SAAoB,EACpB,UAAsB,EACD,OAAwB;QAF7C,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACD,YAAO,GAAP,OAAO,CAAiB;IAC7D,CAAC;IAEJ,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,IAAA,oCAAwB,EAC3C,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEpE,IAAI;YACF,kDAAkD;YAClD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,MAAM,GAAG,EAAE;gBAC7C,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,yDAAyD;YACzD,qFAAqF;YACrF,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,sBAAsB,GAAG,GAAG,MAAM,SAAS,IAAA,0BAAc,EAC7D,CAAC,CAAC,IAAI,CACP,EAAE,CAAC;gBACJ,MAAM,iBAAiB,GAAG,MAAM,IAAA,6BAAiB,EAC/C,IAAI,CAAC,SAAS,EACd,sBAAsB,CACvB,CAAC;gBACF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAClC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;yBACd,sBAAsB;WACpC,CAAC,CAAC;iBACF;YACH,CAAC,CAAC,CACH,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxC;mHAC2G,CAC5G,CAAC;YAEF,IAAK,OAAO,CAAC,CAAC,CAAS,CAAC,QAAQ,GAAG,CAAC,EAAE;gBACpC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxB;;qCAE2B,EAC3B;oBACE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;oBACpD,IAAI,EAAE,sBAAU,CAAC,MAAM;iBACxB,CACF,CAAC;aACH;YAED,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAE/C,IAAI,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC1C,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;aAC7C;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;SACX;IACH,CAAC;CACF,CAAA;AAzEY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFC,qBAAS;QACR,sBAAU;QACQ,iCAAe;GAJrD,iBAAiB,CAyE7B;AAzEY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  NodeConfig,\n  getExistingProjectSchema,\n  enumNameToHash,\n  getEnumDeprecated,\n} from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport { QueryTypes, Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nconst logger = getLogger('Force-clean');\n\n@Injectable()\nexport class ForceCleanService {\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') protected project: SubqueryProject,\n  ) {}\n\n  async forceClean(): Promise<void> {\n    const schema = await getExistingProjectSchema(\n      this.nodeConfig,\n      this.sequelize,\n    );\n    if (!schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    const modelsRelation = getAllEntitiesRelations(this.project.schema);\n\n    try {\n      // drop existing project schema and metadata table\n      await this.sequelize.dropSchema(`\"${schema}\"`, {\n        logging: false,\n        benchmark: false,\n      });\n\n      // TODO, remove this soon, once original enum are cleaned\n      // Deprecate, now enums are moved under schema, drop schema will remove project enums\n      await Promise.all(\n        modelsRelation.enums.map(async (e) => {\n          const enumTypeNameDeprecated = `${schema}_enum_${enumNameToHash(\n            e.name,\n          )}`;\n          const resultsDeprecated = await getEnumDeprecated(\n            this.sequelize,\n            enumTypeNameDeprecated,\n          );\n          if (resultsDeprecated.length !== 0) {\n            await this.sequelize.query(`\n            DROP TYPE \"${enumTypeNameDeprecated}\";\n          `);\n          }\n        }),\n      );\n\n      // remove schema from subquery table (might not exist)\n      const checker = await this.sequelize.query(\n        `\n              SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'public' AND  TABLE_NAME = 'subqueries'`,\n      );\n\n      if ((checker[1] as any).rowCount > 0) {\n        await this.sequelize.query(\n          ` DELETE\n                  FROM public.subqueries\n                  WHERE name = :name`,\n          {\n            replacements: { name: this.nodeConfig.subqueryName },\n            type: QueryTypes.DELETE,\n          },\n        );\n      }\n\n      logger.info('force cleaned schema and tables');\n\n      if (fs.existsSync(this.nodeConfig.mmrPath)) {\n        await fs.promises.unlink(this.nodeConfig.mmrPath);\n        logger.info('force cleaned file based mmr');\n      }\n    } catch (err) {\n      logger.error(err, 'failed to force clean');\n      throw err;\n    }\n  }\n}\n"]}
+{"version":3,"file":"forceClean.service.js","sourceRoot":"","sources":["../../src/subcommands/forceClean.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,2CAAoD;AACpD,gDAM0B;AAC1B,wCAAuD;AACvD,yCAAkD;AAClD,kEAA+D;AAE/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,aAAa,CAAC,CAAC;AAGjC,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAC5B,YACmB,SAAoB,EACpB,UAAsB,EACD,OAAwB;QAF7C,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACD,YAAO,GAAP,OAAO,CAAiB;IAC7D,CAAC;IAEJ,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,IAAA,oCAAwB,EAC3C,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,cAAc,GAAG,IAAA,+BAAuB,EAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEpE,IAAI;YACF,kDAAkD;YAClD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,MAAM,GAAG,EAAE;gBAC7C,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,yDAAyD;YACzD,qFAAqF;YACrF,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,sBAAsB,GAAG,GAAG,MAAM,SAAS,IAAA,0BAAc,EAC7D,CAAC,CAAC,IAAI,CACP,EAAE,CAAC;gBACJ,MAAM,iBAAiB,GAAG,MAAM,IAAA,6BAAiB,EAC/C,IAAI,CAAC,SAAS,EACd,sBAAsB,CACvB,CAAC;gBACF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;oBAClC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;yBACd,sBAAsB;WACpC,CAAC,CAAC;iBACF;YACH,CAAC,CAAC,CACH,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxC;mHAC2G,CAC5G,CAAC;YAEF,IAAK,OAAO,CAAC,CAAC,CAAS,CAAC,QAAQ,GAAG,CAAC,EAAE;gBACpC,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CACxB;;qCAE2B,EAC3B;oBACE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;oBACpD,IAAI,EAAE,sBAAU,CAAC,MAAM;iBACxB,CACF,CAAC;aACH;YAED,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAE/C,IAAI,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;gBAC1C,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;aAC7C;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;SACX;IACH,CAAC;CACF,CAAA;AAzEY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAFC,qBAAS;QACR,sBAAU;QACQ,iCAAe;GAJrD,iBAAiB,CAyE7B;AAzEY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  NodeConfig,\n  getExistingProjectSchema,\n  enumNameToHash,\n  getEnumDeprecated,\n} from '@subql/node-core';\nimport { getAllEntitiesRelations } from '@subql/utils';\nimport { QueryTypes, Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nconst logger = getLogger('Force-clean');\n\n@Injectable()\nexport class ForceCleanService {\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') protected project: SubqueryProject,\n  ) {}\n\n  async forceClean(): Promise<void> {\n    const schema = await getExistingProjectSchema(\n      this.nodeConfig,\n      this.sequelize,\n    );\n    if (!schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    const modelsRelation = getAllEntitiesRelations(this.project.schema);\n\n    try {\n      // drop existing project schema and metadata table\n      await this.sequelize.dropSchema(`\"${schema}\"`, {\n        logging: false,\n        benchmark: false,\n      });\n\n      // TODO, remove this soon, once original enum are cleaned\n      // Deprecate, now enums are moved under schema, drop schema will remove project enums\n      await Promise.all(\n        modelsRelation.enums.map(async (e) => {\n          const enumTypeNameDeprecated = `${schema}_enum_${enumNameToHash(\n            e.name,\n          )}`;\n          const resultsDeprecated = await getEnumDeprecated(\n            this.sequelize,\n            enumTypeNameDeprecated,\n          );\n          if (resultsDeprecated.length !== 0) {\n            await this.sequelize.query(`\n            DROP TYPE \"${enumTypeNameDeprecated}\";\n          `);\n          }\n        }),\n      );\n\n      // remove schema from subquery table (might not exist)\n      const checker = await this.sequelize.query(\n        `\n              SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'public' AND  TABLE_NAME = 'subqueries'`,\n      );\n\n      if ((checker[1] as any).rowCount > 0) {\n        await this.sequelize.query(\n          ` DELETE\n                  FROM public.subqueries\n                  WHERE name = :name`,\n          {\n            replacements: { name: this.nodeConfig.subqueryName },\n            type: QueryTypes.DELETE,\n          },\n        );\n      }\n\n      logger.info('force cleaned schema and tables');\n\n      if (fs.existsSync(this.nodeConfig.mmrPath)) {\n        await fs.promises.unlink(this.nodeConfig.mmrPath);\n        logger.info('force cleaned file based mmr');\n      }\n    } catch (err) {\n      logger.error(err, 'failed to force clean');\n      throw err;\n    }\n  }\n}\n"]}

package/dist/subcommands/reindex.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAA2D;AAC3D,gDAM0B;AAC1B,oEAAgE;AAChE,0EAAqE;AACrE,sEAAiE;AACjE,oFAAgF;AAChF,6DAAyD;AACzD,uDAAmD;AAoBnD,IAAa,oBAAoB,GAAjC,MAAa,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAlBhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,6BAAiB;YACjB,wBAAY;YACZ,gCAAc;YACd,sBAAU;YACV,sCAAiB;YACjB,oDAAwB;YACxB,qCAAgB;YAChB,yCAAkB;YAClB;gBACE,8EAA8E;gBAC9E,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS;aAC5B;SACF;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAWjC,IAAa,aAAa,GAA1B,MAAa,aAAa;CAAG,CAAA;AAAhB,aAAa;IATzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;YACpB,kCAAkB,CAAC,OAAO,EAAE;SAC7B;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitterModule } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  DbModule,\n  MmrService,\n  StoreCacheService,\n  StoreService,\n} from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { DsProcessorService } from '../indexer/ds-processor.service';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [\n    StoreCacheService,\n    StoreService,\n    ReindexService,\n    MmrService,\n    ForceCleanService,\n    UnfinalizedBlocksService,\n    DynamicDsService,\n    DsProcessorService,\n    {\n      // Used to work with DI for unfinalizedBlocksService but not used with reindex\n      provide: ApiService,\n      useFactory: () => undefined,\n    },\n  ],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n    EventEmitterModule.forRoot(),\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}
+{"version":3,"file":"reindex.module.js","sourceRoot":"","sources":["../../src/subcommands/reindex.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAA2D;AAC3D,gDAM0B;AAC1B,oEAAgE;AAChE,0EAAqE;AACrE,sEAAiE;AACjE,oFAAgF;AAChF,6DAAyD;AACzD,uDAAmD;AAoB5C,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;CAAG,CAAA;AAAvB,oBAAoB;IAlBhC,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,6BAAiB;YACjB,wBAAY;YACZ,gCAAc;YACd,sBAAU;YACV,sCAAiB;YACjB,oDAAwB;YACxB,qCAAgB;YAChB,yCAAkB;YAClB;gBACE,8EAA8E;gBAC9E,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS;aAC5B;SACF;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,oBAAoB,CAAG;AAAvB,oDAAoB;AAW1B,IAAM,aAAa,GAAnB,MAAM,aAAa;CAAG,CAAA;AAAhB,aAAa;IATzB,IAAA,eAAM,EAAC;QACN,OAAO,EAAE;YACP,oBAAQ,CAAC,OAAO,EAAE;YAClB,kCAAe,CAAC,QAAQ,EAAE;YAC1B,oBAAoB;YACpB,kCAAkB,CAAC,OAAO,EAAE;SAC7B;QACD,WAAW,EAAE,EAAE;KAChB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitterModule } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  DbModule,\n  MmrService,\n  StoreCacheService,\n  StoreService,\n} from '@subql/node-core';\nimport { ConfigureModule } from '../configure/configure.module';\nimport { DsProcessorService } from '../indexer/ds-processor.service';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { ForceCleanService } from './forceClean.service';\nimport { ReindexService } from './reindex.service';\n\n@Module({\n  providers: [\n    StoreCacheService,\n    StoreService,\n    ReindexService,\n    MmrService,\n    ForceCleanService,\n    UnfinalizedBlocksService,\n    DynamicDsService,\n    DsProcessorService,\n    {\n      // Used to work with DI for unfinalizedBlocksService but not used with reindex\n      provide: ApiService,\n      useFactory: () => undefined,\n    },\n  ],\n  controllers: [],\n})\nexport class ReindexFeatureModule {}\n\n@Module({\n  imports: [\n    DbModule.forRoot(),\n    ConfigureModule.register(),\n    ReindexFeatureModule,\n    EventEmitterModule.forRoot(),\n  ],\n  controllers: [],\n})\nexport class ReindexModule {}\n"]}

package/dist/subcommands/reindex.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAO0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,oFAAgF;AAChF,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAGpC,IAAa,cAAc,GAA3B,MAAa,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAwB,EACpD,iBAAoC,EACpC,wBAAkD,EAClD,gBAAkC;QAPlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACpD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAE1D,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,oCAAoC,CACxC,WAAmB;QAEnB,wBAAwB;QACvB,IAAI,CAAC,wBAAgC,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACxE,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,wBAAwB,CAAC,4BAA4B,EAAE,CAAC;QACrE,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CACzC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAC9D,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,OAAO,WAAW,CAAC;SACpB;QACD,MAAM,CAAC,cAAc,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,4BAA4B;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,MAAM,IAAA,iBAAO,EACX,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,wBAAwB,EAC7B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;QACF,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;IAClD,CAAC;CACF,CAAA;AApGY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAJC,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACe,iCAAe;QACjC,sCAAiB;QACV,oDAAwB;QAChC,qCAAgB;GAZ1C,cAAc,CAoG1B;AApGY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  CacheMetadataModel,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: CacheMetadataModel;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly unfinalizedBlocksService: UnfinalizedBlocksService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n\n    this.metadataRepo = this.storeService.storeCache.metadata;\n\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  async getTargetHeightWithUnfinalizedBlocks(\n    inputHeight: number,\n  ): Promise<number> {\n    // Why does this happen?\n    (this.unfinalizedBlocksService as any).metadataRepo = this.metadataRepo;\n    const unfinalizedBlocks =\n      await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();\n    const bestBlocks = unfinalizedBlocks.filter(\n      ([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight,\n    );\n    if (bestBlocks.length === 0) {\n      return inputHeight;\n    }\n    const [firstBestBlock] = bestBlocks[0];\n    return Math.min(inputHeight, firstBestBlock);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.metadataRepo.find('lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.metadataRepo.find('blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    return this.metadataRepo.find('specName');\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private getStartBlockFromDataSources(): number {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    await reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlocksService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n    await this.storeService.storeCache.flushCache();\n  }\n}\n"]}
+{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAO0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,oFAAgF;AAChF,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAwB,EACpD,iBAAoC,EACpC,wBAAkD,EAClD,gBAAkC;QAPlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACpD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAE1D,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,oCAAoC,CACxC,WAAmB;QAEnB,wBAAwB;QACvB,IAAI,CAAC,wBAAgC,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACxE,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,wBAAwB,CAAC,4BAA4B,EAAE,CAAC;QACrE,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CACzC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAC9D,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,OAAO,WAAW,CAAC;SACpB;QACD,MAAM,CAAC,cAAc,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,4BAA4B;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,MAAM,IAAA,iBAAO,EACX,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,wBAAwB,EAC7B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;QACF,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;IAClD,CAAC;CACF,CAAA;AApGY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAJC,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACe,iCAAe;QACjC,sCAAiB;QACV,oDAAwB;QAChC,qCAAgB;GAZ1C,cAAc,CAoG1B;AApGY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  CacheMetadataModel,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: CacheMetadataModel;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly unfinalizedBlocksService: UnfinalizedBlocksService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n\n    this.metadataRepo = this.storeService.storeCache.metadata;\n\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  async getTargetHeightWithUnfinalizedBlocks(\n    inputHeight: number,\n  ): Promise<number> {\n    // Why does this happen?\n    (this.unfinalizedBlocksService as any).metadataRepo = this.metadataRepo;\n    const unfinalizedBlocks =\n      await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();\n    const bestBlocks = unfinalizedBlocks.filter(\n      ([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight,\n    );\n    if (bestBlocks.length === 0) {\n      return inputHeight;\n    }\n    const [firstBestBlock] = bestBlocks[0];\n    return Math.min(inputHeight, firstBestBlock);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.metadataRepo.find('lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.metadataRepo.find('blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    return this.metadataRepo.find('specName');\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private getStartBlockFromDataSources(): number {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    await reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlocksService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n    await this.storeService.storeCache.flushCache();\n  }\n}\n"]}

package/dist/subcommands/testing.init.d.ts

@@ -0,0 +1 @@
+export declare function testingInit(): Promise<void>;

package/dist/subcommands/testing.init.js

@@ -0,0 +1,34 @@
+"use strict";
+// Copyright 2020-2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.testingInit = void 0;
+const core_1 = require("@nestjs/core");
+const node_core_1 = require("@subql/node-core");
+const project_service_1 = require("../indexer/project.service");
+const testing_module_1 = require("./testing.module");
+const testing_service_1 = require("./testing.service");
+const logger = (0, node_core_1.getLogger)('Testing');
+async function testingInit() {
+    try {
+        const app = await core_1.NestFactory.create(testing_module_1.TestingModule, {
+            logger: new node_core_1.NestLogger(),
+        });
+        await app.init();
+        const projectService = app.get(project_service_1.ProjectService);
+        const apiService = app.get(node_core_1.ApiService);
+        // Initialise async services, we do this here rather than in factories, so we can capture one off events
+        await apiService.init();
+        await projectService.init();
+        const testingService = app.get(testing_service_1.TestingService);
+        await testingService.init();
+        await testingService.run();
+    }
+    catch (e) {
+        logger.error(e, 'Testing failed');
+        process.exit(1);
+    }
+    process.exit(0);
+}
+exports.testingInit = testingInit;
+//# sourceMappingURL=testing.init.js.map

package/dist/subcommands/testing.init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing.init.js","sourceRoot":"","sources":["../../src/subcommands/testing.init.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;AAEtC,uCAA2C;AAC3C,gDAAqE;AACrE,gEAA4D;AAC5D,qDAAiD;AACjD,uDAAmD;AAEnD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAC7B,KAAK,UAAU,WAAW;IAC/B,IAAI;QACF,MAAM,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,8BAAa,EAAE;YAClD,MAAM,EAAE,IAAI,sBAAU,EAAE;SACzB,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAU,CAAC,CAAC;QAEvC,wGAAwG;QACxG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAE5B,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,cAAc,CAAC,GAAG,EAAE,CAAC;KAC5B;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAClC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAtBD,kCAsBC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { NestFactory } from '@nestjs/core';\nimport { getLogger, NestLogger, ApiService } from '@subql/node-core';\nimport { ProjectService } from '../indexer/project.service';\nimport { TestingModule } from './testing.module';\nimport { TestingService } from './testing.service';\n\nconst logger = getLogger('Testing');\nexport async function testingInit(): Promise<void> {\n  try {\n    const app = await NestFactory.create(TestingModule, {\n      logger: new NestLogger(),\n    });\n\n    await app.init();\n    const projectService = app.get(ProjectService);\n    const apiService = app.get(ApiService);\n\n    // Initialise async services, we do this here rather than in factories, so we can capture one off events\n    await apiService.init();\n    await projectService.init();\n\n    const testingService = app.get(TestingService);\n    await testingService.init();\n    await testingService.run();\n  } catch (e) {\n    logger.error(e, 'Testing failed');\n    process.exit(1);\n  }\n  process.exit(0);\n}\n"]}

package/dist/subcommands/testing.module.d.ts

@@ -0,0 +1,4 @@
+export declare class TestingFeatureModule {
+}
+export declare class TestingModule {
+}