@subql/node-ethereum 0.3.1-0 → 0.3.1-1

package/dist/indexer/ds-processor.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"ds-processor.service.js","sourceRoot":"","sources":["../../src/indexer/ds-processor.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,gDAAwB;AACxB,2CAA4C;AAC5C,4DAMgC;AAChC,gDAAkE;AAOlE,6BAA+B;AAC/B,kEAA+D;AAQ/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,4GAA4G;IAC5G,OAAO,SAAS,CAAC,WAAW,KAAK,SAAS,CAAC;AAC7C,CAAC;AAZD,kFAYC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,OAAO,SAAS,CAAC,WAAW,KAAK,OAAO,CAAC;AAC3C,CAAC;AAXD,kFAWC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,IAAI,mCAAmC,CAAC,SAAS,CAAC,EAAE;QAClD,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,mCAAmC,CAAC,SAAS,CAAC,EAAE;QACnD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;KACrD;IAED,uCACK,SAAS,KACZ,WAAW,EAAE,OAAO,EACpB,eAAe,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1B,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EACnE,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CACtB,SAAS;aACN,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC;aAC/D,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IACzB;AACJ,CAAC;AA5BD,kFA4BC;AAED,MAAa,eAAgB,SAAQ,mBAAO;IAC1C,YAAY,MAA6B,EAAE,UAAsB;QAC/D,KAAK,CACH,MAAM,EACN,IAAI,cAAQ,CACV,6BAA6B,MAAM,CAAC,KAAK,aAAa,EACtD,cAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CACrC,EACD,UAAU,CACX,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACF;AAhBD,0CAgBC;AAGM,IAAM,kBAAkB,GAAxB,MAAM,kBAAkB;IAI7B,YACU,OAAwB,EACf,UAAsB;QAD/B,YAAO,GAAP,OAAO,CAAiB;QACf,eAAU,GAAV,UAAU,CAAY;QALjC,mBAAc,GAElB,EAAE,CAAC;IAIJ,CAAC;IAEJ,KAAK,CAAC,gBAAgB,CACpB,WAA4C;QAE5C,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YAC1C,oEAAoE;YACpE,IAAI,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,EAAE;gBAC9B,MAAM,IAAI,KAAK,CACb,YAAY,EAAE,CAAC,IAAI,6BAA6B,SAAS,CAAC,IAAI,GAAG,CAClE,CAAC;aACH;YAED,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,SAAS,CAAC,iBAAiB,CAAC,EAAE;oBAClD,MAAM,IAAI,KAAK,CACb,WAAW,OAAO,CAAC,IAAI,eAAe,MAAM,CAAC,IAAI,CAC/C,SAAS,CAAC,iBAAiB,CAC5B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACf,CAAC;iBACH;aACF;YAED,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAClC,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,eAAe,CACvD,OAAO,CAAC,MAAM,CACf,CACF,CAAC;YAEF,8CAA8C;YAC9C,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;SAClD;IACH,CAAC;IAED,KAAK,CAAC,gCAAgC;QACpC,MAAM,IAAI,CAAC,gBAAgB,CACxB,IAAI,CAAC,OAAO,CAAC,WAAyC,CAAC,MAAM,CAC5D,4BAAU,CACX,CACF,CAAC;IACJ,CAAC;IAED,cAAc,CACZ,EAAyC;QAEzC,IAAI,CAAC,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;SAC5D;QACD,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;YAC3C,MAAM,OAAO,GAAG,IAAI,eAAe,CACjC;gBACE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvB,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI;gBACxB,MAAM,EACJ,IAAI,CAAC,yDAAyD;aACjE,EACD,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,IAAI;gBACF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAK,CAAC;aACnE;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,MAAM,CAAC,CAAC;aACT;SACF;QACD,OAAO,IAAI,CAAC,cAAc,CACxB,EAAE,CAAC,SAAS,CAAC,IAAI,CACiC,CAAC;IACvD,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,SAAS,CACb,EAAiC;QAEjC,IAAI,CAAC,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;SAC5D;QAED,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YACd,OAAO,EAAE,CAAC;SACX;QAED,MAAM,GAAG,GAA2B,EAAE,CAAC;QAEvC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE;YACxC,8DAA8D;YAC9D,IAAI;gBACF,GAAG,CAAC,IAAI,CAAC,GAAG,YAAE,CAAC,YAAY,CAAC,IAAI,EAAE;oBAChC,QAAQ,EAAE,MAAM;iBACjB,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;gBACxD,MAAM,CAAC,CAAC;aACT;SACF;QAED,OAAO,GAAG,CAAC;IACb,CAAC;CACF,CAAA;AA1GY,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;qCAMQ,iCAAe;QACH,sBAAU;GAN9B,kBAAkB,CA0G9B;AA1GY,gDAAkB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport path from 'path';\nimport { Injectable } from '@nestjs/common';\nimport {\n  EthereumHandlerKind,\n  isCustomDs,\n  SubqlEthereumCustomDataSource,\n  SubqlEthereumDataSource,\n  SubqlDatasourceProcessor,\n} from '@subql/common-ethereum';\nimport { getLogger, NodeConfig, Sandbox } from '@subql/node-core';\nimport {\n  SecondLayerHandlerProcessor_0_0_0,\n  SecondLayerHandlerProcessor_1_0_0,\n  SubqlCustomDatasource,\n} from '@subql/types-ethereum';\n\nimport { VMScript } from 'vm2';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nexport interface DsPluginSandboxOption {\n  root: string;\n  entry: string;\n  script: string;\n}\n\nconst logger = getLogger('ds-sandbox');\n\nexport function isSecondLayerHandlerProcessor_0_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlCustomDatasource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_0_0_0<K, F, E, DS> {\n  // Exisiting datasource processors had no concept of specVersion, therefore undefined is equivalent to 0.0.0\n  return processor.specVersion === undefined;\n}\n\nexport function isSecondLayerHandlerProcessor_1_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlEthereumCustomDataSource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  return processor.specVersion === '1.0.0';\n}\n\nexport function asSecondLayerHandlerProcessor_1_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlEthereumCustomDataSource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  if (isSecondLayerHandlerProcessor_1_0_0(processor)) {\n    return processor;\n  }\n\n  if (!isSecondLayerHandlerProcessor_0_0_0(processor)) {\n    throw new Error('Unsupported ds processor version');\n  }\n\n  return {\n    ...processor,\n    specVersion: '1.0.0',\n    filterProcessor: (params) =>\n      processor.filterProcessor(params.filter, params.input, params.ds),\n    transformer: (params) =>\n      processor\n        .transformer(params.input, params.ds, params.api, params.assets)\n        .then((res) => [res]),\n  };\n}\n\nexport class DsPluginSandbox extends Sandbox {\n  constructor(option: DsPluginSandboxOption, nodeConfig: NodeConfig) {\n    super(\n      option,\n      new VMScript(\n        `module.exports = require('${option.entry}').default;`,\n        path.join(option.root, 'ds_sandbox'),\n      ),\n      nodeConfig,\n    );\n    this.freeze(logger, 'logger');\n  }\n\n  getDsPlugin<D extends string>(): SubqlDatasourceProcessor<D, unknown> {\n    return this.run(this.script);\n  }\n}\n\n@Injectable()\nexport class DsProcessorService {\n  private processorCache: {\n    [entry: string]: SubqlDatasourceProcessor<string, unknown>;\n  } = {};\n  constructor(\n    private project: SubqueryProject,\n    private readonly nodeConfig: NodeConfig,\n  ) {}\n\n  async validateCustomDs(\n    datasources: SubqlEthereumCustomDataSource[],\n  ): Promise<void> {\n    for (const ds of datasources) {\n      const processor = this.getDsProcessor(ds);\n      /* Standard validation applicable to all custom ds and processors */\n      if (ds.kind !== processor.kind) {\n        throw new Error(\n          `ds kind (${ds.kind}) doesnt match processor (${processor.kind})`,\n        );\n      }\n\n      for (const handler of ds.mapping.handlers) {\n        if (!(handler.kind in processor.handlerProcessors)) {\n          throw new Error(\n            `ds kind ${handler.kind} not one of ${Object.keys(\n              processor.handlerProcessors,\n            ).join(', ')}`,\n          );\n        }\n      }\n\n      ds.mapping.handlers.map((handler) =>\n        processor.handlerProcessors[handler.kind].filterValidator(\n          handler.filter,\n        ),\n      );\n\n      /* Additional processor specific validation */\n      processor.validate(ds, await this.getAssets(ds));\n    }\n  }\n\n  async validateProjectCustomDatasources(): Promise<void> {\n    await this.validateCustomDs(\n      (this.project.dataSources as SubqlEthereumDataSource[]).filter(\n        isCustomDs,\n      ),\n    );\n  }\n\n  getDsProcessor<D extends string>(\n    ds: SubqlEthereumCustomDataSource<string>,\n  ): SubqlDatasourceProcessor<D, unknown> {\n    if (!isCustomDs(ds)) {\n      throw new Error(`data source is not a custom data source`);\n    }\n    if (!this.processorCache[ds.processor.file]) {\n      const sandbox = new DsPluginSandbox(\n        {\n          root: this.project.root,\n          entry: ds.processor.file,\n          script:\n            null /* TODO get working with Readers, same as with sandbox */,\n        },\n        this.nodeConfig,\n      );\n      try {\n        this.processorCache[ds.processor.file] = sandbox.getDsPlugin<D>();\n      } catch (e) {\n        logger.error(`not supported ds @${ds.kind}`);\n        throw e;\n      }\n    }\n    return this.processorCache[\n      ds.processor.file\n    ] as unknown as SubqlDatasourceProcessor<D, unknown>;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  async getAssets(\n    ds: SubqlEthereumCustomDataSource,\n  ): Promise<Record<string, string>> {\n    if (!isCustomDs(ds)) {\n      throw new Error(`data source is not a custom data source`);\n    }\n\n    if (!ds.assets) {\n      return {};\n    }\n\n    const res: Record<string, string> = {};\n\n    for (const [name, { file }] of ds.assets) {\n      // TODO update with https://github.com/subquery/subql/pull/511\n      try {\n        res[name] = fs.readFileSync(file, {\n          encoding: 'utf8',\n        });\n      } catch (e) {\n        logger.error(`Failed to load datasource asset ${file}`);\n        throw e;\n      }\n    }\n\n    return res;\n  }\n}\n"]}
+{"version":3,"file":"ds-processor.service.js","sourceRoot":"","sources":["../../src/indexer/ds-processor.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,gDAAwB;AACxB,2CAAoD;AACpD,4DAMgC;AAChC,gDAAkE;AAOlE,6BAA+B;AAC/B,kEAA+D;AAQ/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,4GAA4G;IAC5G,OAAO,SAAS,CAAC,WAAW,KAAK,SAAS,CAAC;AAC7C,CAAC;AAZD,kFAYC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,OAAO,SAAS,CAAC,WAAW,KAAK,OAAO,CAAC;AAC3C,CAAC;AAXD,kFAWC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,IAAI,mCAAmC,CAAC,SAAS,CAAC,EAAE;QAClD,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,mCAAmC,CAAC,SAAS,CAAC,EAAE;QACnD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;KACrD;IAED,uCACK,SAAS,KACZ,WAAW,EAAE,OAAO,EACpB,eAAe,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1B,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EACnE,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CACtB,SAAS;aACN,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC;aAC/D,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IACzB;AACJ,CAAC;AA5BD,kFA4BC;AAED,MAAa,eAAgB,SAAQ,mBAAO;IAC1C,YAAY,MAA6B,EAAE,UAAsB;QAC/D,KAAK,CACH,MAAM,EACN,IAAI,cAAQ,CACV,6BAA6B,MAAM,CAAC,KAAK,aAAa,EACtD,cAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CACrC,EACD,UAAU,CACX,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACF;AAhBD,0CAgBC;AAGM,IAAM,kBAAkB,GAAxB,MAAM,kBAAkB;IAI7B,YAC+C,OAAwB,EACpD,UAAsB;QADM,YAAO,GAAP,OAAO,CAAiB;QACpD,eAAU,GAAV,UAAU,CAAY;QALjC,mBAAc,GAElB,EAAE,CAAC;IAIJ,CAAC;IAEJ,KAAK,CAAC,gBAAgB,CACpB,WAA4C;QAE5C,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YAC1C,oEAAoE;YACpE,IAAI,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,EAAE;gBAC9B,MAAM,IAAI,KAAK,CACb,YAAY,EAAE,CAAC,IAAI,6BAA6B,SAAS,CAAC,IAAI,GAAG,CAClE,CAAC;aACH;YAED,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,SAAS,CAAC,iBAAiB,CAAC,EAAE;oBAClD,MAAM,IAAI,KAAK,CACb,WAAW,OAAO,CAAC,IAAI,eAAe,MAAM,CAAC,IAAI,CAC/C,SAAS,CAAC,iBAAiB,CAC5B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACf,CAAC;iBACH;aACF;YAED,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAClC,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,eAAe,CACvD,OAAO,CAAC,MAAM,CACf,CACF,CAAC;YAEF,8CAA8C;YAC9C,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;SAClD;IACH,CAAC;IAED,KAAK,CAAC,gCAAgC;QACpC,MAAM,IAAI,CAAC,gBAAgB,CACxB,IAAI,CAAC,OAAO,CAAC,WAAyC,CAAC,MAAM,CAC5D,4BAAU,CACX,CACF,CAAC;IACJ,CAAC;IAED,cAAc,CACZ,EAAyC;QAEzC,IAAI,CAAC,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;SAC5D;QACD,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;YAC3C,MAAM,OAAO,GAAG,IAAI,eAAe,CACjC;gBACE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvB,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI;gBACxB,MAAM,EACJ,IAAI,CAAC,yDAAyD;aACjE,EACD,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,IAAI;gBACF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAK,CAAC;aACnE;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,MAAM,CAAC,CAAC;aACT;SACF;QACD,OAAO,IAAI,CAAC,cAAc,CACxB,EAAE,CAAC,SAAS,CAAC,IAAI,CACiC,CAAC;IACvD,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,SAAS,CACb,EAAiC;QAEjC,IAAI,CAAC,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;SAC5D;QAED,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YACd,OAAO,EAAE,CAAC;SACX;QAED,MAAM,GAAG,GAA2B,EAAE,CAAC;QAEvC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE;YACxC,8DAA8D;YAC9D,IAAI;gBACF,GAAG,CAAC,IAAI,CAAC,GAAG,YAAE,CAAC,YAAY,CAAC,IAAI,EAAE;oBAChC,QAAQ,EAAE,MAAM;iBACjB,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;gBACxD,MAAM,CAAC,CAAC;aACT;SACF;QAED,OAAO,GAAG,CAAC;IACb,CAAC;CACF,CAAA;AA1GY,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;IAMR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAA2B,iCAAe;QACxC,sBAAU;GAN9B,kBAAkB,CA0G9B;AA1GY,gDAAkB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport path from 'path';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  EthereumHandlerKind,\n  isCustomDs,\n  SubqlEthereumCustomDataSource,\n  SubqlEthereumDataSource,\n  SubqlDatasourceProcessor,\n} from '@subql/common-ethereum';\nimport { getLogger, NodeConfig, Sandbox } from '@subql/node-core';\nimport {\n  SecondLayerHandlerProcessor_0_0_0,\n  SecondLayerHandlerProcessor_1_0_0,\n  SubqlCustomDatasource,\n} from '@subql/types-ethereum';\n\nimport { VMScript } from 'vm2';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nexport interface DsPluginSandboxOption {\n  root: string;\n  entry: string;\n  script: string;\n}\n\nconst logger = getLogger('ds-sandbox');\n\nexport function isSecondLayerHandlerProcessor_0_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlCustomDatasource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_0_0_0<K, F, E, DS> {\n  // Exisiting datasource processors had no concept of specVersion, therefore undefined is equivalent to 0.0.0\n  return processor.specVersion === undefined;\n}\n\nexport function isSecondLayerHandlerProcessor_1_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlEthereumCustomDataSource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  return processor.specVersion === '1.0.0';\n}\n\nexport function asSecondLayerHandlerProcessor_1_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlEthereumCustomDataSource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  if (isSecondLayerHandlerProcessor_1_0_0(processor)) {\n    return processor;\n  }\n\n  if (!isSecondLayerHandlerProcessor_0_0_0(processor)) {\n    throw new Error('Unsupported ds processor version');\n  }\n\n  return {\n    ...processor,\n    specVersion: '1.0.0',\n    filterProcessor: (params) =>\n      processor.filterProcessor(params.filter, params.input, params.ds),\n    transformer: (params) =>\n      processor\n        .transformer(params.input, params.ds, params.api, params.assets)\n        .then((res) => [res]),\n  };\n}\n\nexport class DsPluginSandbox extends Sandbox {\n  constructor(option: DsPluginSandboxOption, nodeConfig: NodeConfig) {\n    super(\n      option,\n      new VMScript(\n        `module.exports = require('${option.entry}').default;`,\n        path.join(option.root, 'ds_sandbox'),\n      ),\n      nodeConfig,\n    );\n    this.freeze(logger, 'logger');\n  }\n\n  getDsPlugin<D extends string>(): SubqlDatasourceProcessor<D, unknown> {\n    return this.run(this.script);\n  }\n}\n\n@Injectable()\nexport class DsProcessorService {\n  private processorCache: {\n    [entry: string]: SubqlDatasourceProcessor<string, unknown>;\n  } = {};\n  constructor(\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly nodeConfig: NodeConfig,\n  ) {}\n\n  async validateCustomDs(\n    datasources: SubqlEthereumCustomDataSource[],\n  ): Promise<void> {\n    for (const ds of datasources) {\n      const processor = this.getDsProcessor(ds);\n      /* Standard validation applicable to all custom ds and processors */\n      if (ds.kind !== processor.kind) {\n        throw new Error(\n          `ds kind (${ds.kind}) doesnt match processor (${processor.kind})`,\n        );\n      }\n\n      for (const handler of ds.mapping.handlers) {\n        if (!(handler.kind in processor.handlerProcessors)) {\n          throw new Error(\n            `ds kind ${handler.kind} not one of ${Object.keys(\n              processor.handlerProcessors,\n            ).join(', ')}`,\n          );\n        }\n      }\n\n      ds.mapping.handlers.map((handler) =>\n        processor.handlerProcessors[handler.kind].filterValidator(\n          handler.filter,\n        ),\n      );\n\n      /* Additional processor specific validation */\n      processor.validate(ds, await this.getAssets(ds));\n    }\n  }\n\n  async validateProjectCustomDatasources(): Promise<void> {\n    await this.validateCustomDs(\n      (this.project.dataSources as SubqlEthereumDataSource[]).filter(\n        isCustomDs,\n      ),\n    );\n  }\n\n  getDsProcessor<D extends string>(\n    ds: SubqlEthereumCustomDataSource<string>,\n  ): SubqlDatasourceProcessor<D, unknown> {\n    if (!isCustomDs(ds)) {\n      throw new Error(`data source is not a custom data source`);\n    }\n    if (!this.processorCache[ds.processor.file]) {\n      const sandbox = new DsPluginSandbox(\n        {\n          root: this.project.root,\n          entry: ds.processor.file,\n          script:\n            null /* TODO get working with Readers, same as with sandbox */,\n        },\n        this.nodeConfig,\n      );\n      try {\n        this.processorCache[ds.processor.file] = sandbox.getDsPlugin<D>();\n      } catch (e) {\n        logger.error(`not supported ds @${ds.kind}`);\n        throw e;\n      }\n    }\n    return this.processorCache[\n      ds.processor.file\n    ] as unknown as SubqlDatasourceProcessor<D, unknown>;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  async getAssets(\n    ds: SubqlEthereumCustomDataSource,\n  ): Promise<Record<string, string>> {\n    if (!isCustomDs(ds)) {\n      throw new Error(`data source is not a custom data source`);\n    }\n\n    if (!ds.assets) {\n      return {};\n    }\n\n    const res: Record<string, string> = {};\n\n    for (const [name, { file }] of ds.assets) {\n      // TODO update with https://github.com/subquery/subql/pull/511\n      try {\n        res[name] = fs.readFileSync(file, {\n          encoding: 'utf8',\n        });\n      } catch (e) {\n        logger.error(`Failed to load datasource asset ${file}`);\n        throw e;\n      }\n    }\n\n    return res;\n  }\n}\n"]}

package/dist/indexer/dynamic-ds.service.js

@@ -10,6 +10,9 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -124,6 +127,7 @@ let DynamicDsService = class DynamicDsService {
 };
 DynamicDsService = __decorate([
     (0, common_1.Injectable)(),
+    __param(1, (0, common_1.Inject)('ISubqueryProject')),
     __metadata("design:paramtypes", [ds_processor_service_1.DsProcessorService,
         SubqueryProject_1.SubqueryProject])
 ], DynamicDsService);

package/dist/indexer/dynamic-ds.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,2CAA4C;AAC5C,4DAAiE;AACjE,gDAA2D;AAC3D,mCAAuD;AAEvD,kEAA+E;AAC/E,iEAA4D;AAE5D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,MAAM,YAAY,GAAG,oBAAoB,CAAC;AAC1C,MAAM,cAAc,GAAG,KAAK,CAAC;AAStB,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAI3B,YACmB,kBAAsC,EACtC,OAAwB;QADxB,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,YAAO,GAAP,OAAO,CAAiB;IACxC,CAAC;IAEJ,IAAI,CAAC,YAA0B;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAID,KAAK,CAAC,sBAAsB,CAAC,YAAoB,EAAE,EAAe;QAChE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YAC1B,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,YAAY,CACtC,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAC5B,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EACvC,EAAE,WAAW,EAAE,EAAE,EAAE,CACpB,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB,CAC3B,MAAwB,EACxB,EAAe;QAEf,IAAI;YACF,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE5C,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAEnD,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;YAC/C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE3B,OAAO,EAAE,CAAC;SACX;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAEvD,IAAI,CAAC,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CACnD,CAAC;aACH;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,mBAAmB,CAAC,WAAmB;QACrC,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,WAAoB;;QAEpB,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE9D,IAAI,OAAO,GAAuB,EAAE,CAAC;QAErC,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAgB,mCAAI,IAAI,CAClC,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,EAAE;YACtB,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;SAC5B;QAED,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAA,MAAA,IAAI,CAAC,aAAa,0CAAG,cAAc,GAAG,WAAW,CAAC,mCAAI,IAAI,CAC3D,CAAC;YACF,IAAI,WAAW,CAAC,MAAM,EAAE;gBACtB,OAAO,GAAG,IAAA,kBAAS,EAAC,OAAO,EAAE,WAAW,EAAE,gBAAO,CAAC,CAAC;aACpD;SACF;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAA0B,EAC1B,EAAe;QAEf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE5E,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY;aACpB,MAAM,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;aACpE,IAAI,CAAC,GAAG,EAAE;YACT,IAAI,CAAC,aAAa,mCACb,IAAI,CAAC,aAAa,KACrB,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAClD,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAwB;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACnE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;QAEF,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,2CAA2C;aAC5C;YAED,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AA5JY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;qCAM4B,yCAAkB;QAC7B,iCAAe;GANhC,gBAAgB,CA4J5B;AA5JY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { Injectable } from '@nestjs/common';\nimport { isCustomDs, isRuntimeDs } from '@subql/common-ethereum';\nimport { getLogger, MetadataRepo } from '@subql/node-core';\nimport { cloneDeep, isEqual, unionWith } from 'lodash';\nimport { Transaction } from 'sequelize/types';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\nconst logger = getLogger('dynamic-ds');\n\nconst METADATA_KEY = 'dynamicDatasources';\nconst TEMP_DS_PREFIX = 'ds_';\n\ninterface DatasourceParams {\n  templateName: string;\n  args?: Record<string, unknown>;\n  startBlock: number;\n}\n\n@Injectable()\nexport class DynamicDsService {\n  private metaDataRepo: MetadataRepo;\n  private tempDsRecords: Record<string, string>;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly project: SubqueryProject,\n  ) {}\n\n  init(metaDataRepo: MetadataRepo): void {\n    this.metaDataRepo = metaDataRepo;\n  }\n\n  private _datasources: SubqlProjectDs[];\n\n  async resetDynamicDatasource(targetHeight: number, tx: Transaction) {\n    const dynamicDs = await this.getDynamicDatasourceParams();\n    if (dynamicDs.length !== 0) {\n      const filteredDs = dynamicDs.filter(\n        (ds) => ds.startBlock <= targetHeight,\n      );\n      const dsRecords = JSON.stringify(filteredDs);\n      await this.metaDataRepo.upsert(\n        { key: METADATA_KEY, value: dsRecords },\n        { transaction: tx },\n      );\n    }\n  }\n\n  async createDynamicDatasource(\n    params: DatasourceParams,\n    tx: Transaction,\n  ): Promise<SubqlProjectDs> {\n    try {\n      const ds = await this.getDatasource(params);\n\n      await this.saveDynamicDatasourceParams(params, tx);\n\n      logger.info(\n        `Created new dynamic datasource from template: \"${params.templateName}\"`,\n      );\n\n      if (!this._datasources) this._datasources = [];\n      this._datasources.push(ds);\n\n      return ds;\n    } catch (e) {\n      logger.error(e.message);\n      process.exit(1);\n    }\n  }\n\n  async getDynamicDatasources(): Promise<SubqlProjectDs[]> {\n    if (!this._datasources) {\n      try {\n        const params = await this.getDynamicDatasourceParams();\n\n        this._datasources = await Promise.all(\n          params.map((params) => this.getDatasource(params)),\n        );\n      } catch (e) {\n        logger.error(`Unable to get dynamic datasources:\\n${e.message}`);\n        process.exit(1);\n      }\n    }\n\n    return this._datasources;\n  }\n\n  deleteTempDsRecords(blockHeight: number) {\n    delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];\n  }\n\n  private async getDynamicDatasourceParams(\n    blockHeight?: number,\n  ): Promise<DatasourceParams[]> {\n    assert(this.metaDataRepo, `Model _metadata does not exist`);\n    const record = await this.metaDataRepo.findByPk(METADATA_KEY);\n\n    let results: DatasourceParams[] = [];\n\n    const metaResults: DatasourceParams[] = JSON.parse(\n      (record?.value as string) ?? '[]',\n    );\n    if (metaResults.length) {\n      results = [...metaResults];\n    }\n\n    if (blockHeight !== undefined) {\n      const tempResults: DatasourceParams[] = JSON.parse(\n        this.tempDsRecords?.[TEMP_DS_PREFIX + blockHeight] ?? '[]',\n      );\n      if (tempResults.length) {\n        results = unionWith(results, tempResults, isEqual);\n      }\n    }\n\n    return results;\n  }\n\n  private async saveDynamicDatasourceParams(\n    dsParams: DatasourceParams,\n    tx: Transaction,\n  ): Promise<void> {\n    const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);\n\n    assert(this.metaDataRepo, `Model _metadata does not exist`);\n    const dsRecords = JSON.stringify([...existing, dsParams]);\n    await this.metaDataRepo\n      .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })\n      .then(() => {\n        this.tempDsRecords = {\n          ...this.tempDsRecords,\n          [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords,\n        };\n      });\n  }\n\n  private async getDatasource(\n    params: DatasourceParams,\n  ): Promise<SubqlProjectDs> {\n    const template = cloneDeep(\n      this.project.templates.find((t) => t.name === params.templateName),\n    );\n\n    if (!template) {\n      throw new Error(\n        `Unable to find matching template in project for name: \"${params.templateName}\"`,\n      );\n    }\n\n    logger.info(\n      `Initialised dynamic datasource from template: \"${params.templateName}\"`,\n    );\n\n    const dsObj = {\n      ...template,\n      startBlock: params.startBlock,\n    } as SubqlProjectDs;\n    delete dsObj.name;\n    try {\n      if (isCustomDs(dsObj)) {\n        dsObj.processor.options = {\n          ...dsObj.processor.options,\n          ...params.args,\n        };\n        await this.dsProcessorService.validateCustomDs([dsObj]);\n      } else if (isRuntimeDs(dsObj)) {\n        // XXX add any modifications to the ds here\n      }\n\n      return dsObj;\n    } catch (e) {\n      throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n    }\n  }\n}\n"]}
+{"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,2CAAoD;AACpD,4DAAiE;AACjE,gDAA2D;AAC3D,mCAAuD;AAEvD,kEAA+E;AAC/E,iEAA4D;AAE5D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,MAAM,YAAY,GAAG,oBAAoB,CAAC;AAC1C,MAAM,cAAc,GAAG,KAAK,CAAC;AAStB,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAI3B,YACmB,kBAAsC,EACV,OAAwB;QADpD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACV,YAAO,GAAP,OAAO,CAAiB;IACpE,CAAC;IAEJ,IAAI,CAAC,YAA0B;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAID,KAAK,CAAC,sBAAsB,CAAC,YAAoB,EAAE,EAAe;QAChE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC1D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;YAC1B,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,YAAY,CACtC,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAC5B,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EACvC,EAAE,WAAW,EAAE,EAAE,EAAE,CACpB,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,uBAAuB,CAC3B,MAAwB,EACxB,EAAe;QAEf,IAAI;YACF,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE5C,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAEnD,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;YAC/C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE3B,OAAO,EAAE,CAAC;SACX;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAEvD,IAAI,CAAC,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CACnD,CAAC;aACH;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,mBAAmB,CAAC,WAAmB;QACrC,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,WAAoB;;QAEpB,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE9D,IAAI,OAAO,GAAuB,EAAE,CAAC;QAErC,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAgB,mCAAI,IAAI,CAClC,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,EAAE;YACtB,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;SAC5B;QAED,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,MAAM,WAAW,GAAuB,IAAI,CAAC,KAAK,CAChD,MAAA,MAAA,IAAI,CAAC,aAAa,0CAAG,cAAc,GAAG,WAAW,CAAC,mCAAI,IAAI,CAC3D,CAAC;YACF,IAAI,WAAW,CAAC,MAAM,EAAE;gBACtB,OAAO,GAAG,IAAA,kBAAS,EAAC,OAAO,EAAE,WAAW,EAAE,gBAAO,CAAC,CAAC;aACpD;SACF;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAA0B,EAC1B,EAAe;QAEf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE5E,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY;aACpB,MAAM,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;aACpE,IAAI,CAAC,GAAG,EAAE;YACT,IAAI,CAAC,aAAa,mCACb,IAAI,CAAC,aAAa,KACrB,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAClD,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAwB;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACnE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;QAEF,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,2CAA2C;aAC5C;YAED,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AA5JY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;IAOR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCADU,yCAAkB;QACD,iCAAe;GAN5D,gBAAgB,CA4J5B;AA5JY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { isCustomDs, isRuntimeDs } from '@subql/common-ethereum';\nimport { getLogger, MetadataRepo } from '@subql/node-core';\nimport { cloneDeep, isEqual, unionWith } from 'lodash';\nimport { Transaction } from 'sequelize/types';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\nconst logger = getLogger('dynamic-ds');\n\nconst METADATA_KEY = 'dynamicDatasources';\nconst TEMP_DS_PREFIX = 'ds_';\n\ninterface DatasourceParams {\n  templateName: string;\n  args?: Record<string, unknown>;\n  startBlock: number;\n}\n\n@Injectable()\nexport class DynamicDsService {\n  private metaDataRepo: MetadataRepo;\n  private tempDsRecords: Record<string, string>;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n  ) {}\n\n  init(metaDataRepo: MetadataRepo): void {\n    this.metaDataRepo = metaDataRepo;\n  }\n\n  private _datasources: SubqlProjectDs[];\n\n  async resetDynamicDatasource(targetHeight: number, tx: Transaction) {\n    const dynamicDs = await this.getDynamicDatasourceParams();\n    if (dynamicDs.length !== 0) {\n      const filteredDs = dynamicDs.filter(\n        (ds) => ds.startBlock <= targetHeight,\n      );\n      const dsRecords = JSON.stringify(filteredDs);\n      await this.metaDataRepo.upsert(\n        { key: METADATA_KEY, value: dsRecords },\n        { transaction: tx },\n      );\n    }\n  }\n\n  async createDynamicDatasource(\n    params: DatasourceParams,\n    tx: Transaction,\n  ): Promise<SubqlProjectDs> {\n    try {\n      const ds = await this.getDatasource(params);\n\n      await this.saveDynamicDatasourceParams(params, tx);\n\n      logger.info(\n        `Created new dynamic datasource from template: \"${params.templateName}\"`,\n      );\n\n      if (!this._datasources) this._datasources = [];\n      this._datasources.push(ds);\n\n      return ds;\n    } catch (e) {\n      logger.error(e.message);\n      process.exit(1);\n    }\n  }\n\n  async getDynamicDatasources(): Promise<SubqlProjectDs[]> {\n    if (!this._datasources) {\n      try {\n        const params = await this.getDynamicDatasourceParams();\n\n        this._datasources = await Promise.all(\n          params.map((params) => this.getDatasource(params)),\n        );\n      } catch (e) {\n        logger.error(`Unable to get dynamic datasources:\\n${e.message}`);\n        process.exit(1);\n      }\n    }\n\n    return this._datasources;\n  }\n\n  deleteTempDsRecords(blockHeight: number) {\n    delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];\n  }\n\n  private async getDynamicDatasourceParams(\n    blockHeight?: number,\n  ): Promise<DatasourceParams[]> {\n    assert(this.metaDataRepo, `Model _metadata does not exist`);\n    const record = await this.metaDataRepo.findByPk(METADATA_KEY);\n\n    let results: DatasourceParams[] = [];\n\n    const metaResults: DatasourceParams[] = JSON.parse(\n      (record?.value as string) ?? '[]',\n    );\n    if (metaResults.length) {\n      results = [...metaResults];\n    }\n\n    if (blockHeight !== undefined) {\n      const tempResults: DatasourceParams[] = JSON.parse(\n        this.tempDsRecords?.[TEMP_DS_PREFIX + blockHeight] ?? '[]',\n      );\n      if (tempResults.length) {\n        results = unionWith(results, tempResults, isEqual);\n      }\n    }\n\n    return results;\n  }\n\n  private async saveDynamicDatasourceParams(\n    dsParams: DatasourceParams,\n    tx: Transaction,\n  ): Promise<void> {\n    const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);\n\n    assert(this.metaDataRepo, `Model _metadata does not exist`);\n    const dsRecords = JSON.stringify([...existing, dsParams]);\n    await this.metaDataRepo\n      .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })\n      .then(() => {\n        this.tempDsRecords = {\n          ...this.tempDsRecords,\n          [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords,\n        };\n      });\n  }\n\n  private async getDatasource(\n    params: DatasourceParams,\n  ): Promise<SubqlProjectDs> {\n    const template = cloneDeep(\n      this.project.templates.find((t) => t.name === params.templateName),\n    );\n\n    if (!template) {\n      throw new Error(\n        `Unable to find matching template in project for name: \"${params.templateName}\"`,\n      );\n    }\n\n    logger.info(\n      `Initialised dynamic datasource from template: \"${params.templateName}\"`,\n    );\n\n    const dsObj = {\n      ...template,\n      startBlock: params.startBlock,\n    } as SubqlProjectDs;\n    delete dsObj.name;\n    try {\n      if (isCustomDs(dsObj)) {\n        dsObj.processor.options = {\n          ...dsObj.processor.options,\n          ...params.args,\n        };\n        await this.dsProcessorService.validateCustomDs([dsObj]);\n      } else if (isRuntimeDs(dsObj)) {\n        // XXX add any modifications to the ds here\n      }\n\n      return dsObj;\n    } catch (e) {\n      throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n    }\n  }\n}\n"]}

package/dist/indexer/fetch.module.js

@@ -12,7 +12,6 @@ exports.FetchModule = void 0;
 const common_1 = require("@nestjs/common");
 const event_emitter_1 = require("@nestjs/event-emitter");
 const node_core_1 = require("@subql/node-core");
-const SubqueryProject_1 = require("../configure/SubqueryProject");
 const api_service_ethereum_1 = require("../ethereum/api.service.ethereum");
 const blockDispatcher_1 = require("./blockDispatcher");
 const dictionary_service_1 = require("./dictionary.service");
@@ -31,12 +30,12 @@ FetchModule = __decorate([
             node_core_1.StoreService,
             {
                 provide: node_core_1.ApiService,
-                useFactory: async (project) => {
-                    const apiService = new api_service_ethereum_1.EthereumApiService(project);
+                useFactory: async (project, eventEmitter) => {
+                    const apiService = new api_service_ethereum_1.EthereumApiService(project, eventEmitter);
                     await apiService.init();
                     return apiService;
                 },
-                inject: [SubqueryProject_1.SubqueryProject],
+                inject: ['ISubqueryProject', event_emitter_1.EventEmitter2],
             },
             indexer_manager_1.IndexerManager,
             {
@@ -54,7 +53,15 @@ FetchModule = __decorate([
             },
             fetch_service_1.FetchService,
             node_core_1.BenchmarkService,
-            dictionary_service_1.DictionaryService,
+            {
+                provide: dictionary_service_1.DictionaryService,
+                useFactory: async (project, nodeConfig) => {
+                    const dictionaryService = new dictionary_service_1.DictionaryService(project, nodeConfig);
+                    await dictionaryService.init();
+                    return dictionaryService;
+                },
+                inject: ['ISubqueryProject', node_core_1.NodeConfig],
+            },
             sandbox_service_1.SandboxService,
             ds_processor_service_1.DsProcessorService,
             dynamic_ds_service_1.DynamicDsService,

package/dist/indexer/fetch.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.module.js","sourceRoot":"","sources":["../../src/indexer/fetch.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAAsD;AACtD,gDAO0B;AAC1B,kEAA+D;AAC/D,2EAAsE;AACtE,uDAG2B;AAC3B,6DAAyD;AACzD,iEAA4D;AAC5D,6DAAwD;AACxD,mDAA+C;AAC/C,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AA2DhE,IAAM,WAAW,GAAjB,MAAM,WAAW;CAAG,CAAA;AAAd,WAAW;IAzDvB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EAAE,OAAwB,EAAE,EAAE;oBAC7C,MAAM,UAAU,GAAG,IAAI,yCAAkB,CAAC,OAAO,CAAC,CAAC;oBAEnD,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,iCAAe,CAAC;aAC1B;YACD,gCAAc;YACd;gBACE,OAAO,EAAE,kBAAkB;gBAC3B,UAAU,EAAE,CACV,UAAsB,EACtB,YAA2B,EAC3B,cAA8B,EAC9B,UAAsB,EACtB,cAA8B,EAC9B,EAAE,CACF,UAAU,CAAC,OAAO,KAAK,SAAS;oBAC9B,CAAC,CAAC,IAAI,8CAA4B,CAC9B,UAAU,EACV,YAAY,EACZ,cAAc,CACf;oBACH,CAAC,CAAC,IAAI,wCAAsB,CACxB,UAAU,EACV,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,CACf;gBACP,MAAM,EAAE;oBACN,sBAAU;oBACV,6BAAa;oBACb,gCAAc;oBACd,sBAAU;oBACV,gCAAc;iBACf;aACF;YACD,4BAAY;YACZ,4BAAgB;YAChB,sCAAiB;YACjB,gCAAc;YACd,yCAAkB;YAClB,qCAAgB;YAChB,sBAAU;YACV,sBAAU;YACV,gCAAc;YACd,oDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,wBAAY,EAAE,sBAAU,CAAC;KACpC,CAAC;GACW,WAAW,CAAG;AAAd,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  BenchmarkService,\n  MmrService,\n  StoreService,\n  PoiService,\n  ApiService,\n  NodeConfig,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum/api.service.ethereum';\nimport {\n  BlockDispatcherService,\n  WorkerBlockDispatcherService,\n} from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { FetchService } from './fetch.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    StoreService,\n    {\n      provide: ApiService,\n      useFactory: async (project: SubqueryProject) => {\n        const apiService = new EthereumApiService(project);\n\n        await apiService.init();\n        return apiService;\n      },\n      inject: [SubqueryProject],\n    },\n    IndexerManager,\n    {\n      provide: 'IBlockDispatcher',\n      useFactory: (\n        nodeConfig: NodeConfig,\n        eventEmitter: EventEmitter2,\n        projectService: ProjectService,\n        apiService: ApiService,\n        indexerManager: IndexerManager,\n      ) =>\n        nodeConfig.workers !== undefined\n          ? new WorkerBlockDispatcherService(\n              nodeConfig,\n              eventEmitter,\n              projectService,\n            )\n          : new BlockDispatcherService(\n              apiService,\n              nodeConfig,\n              indexerManager,\n              eventEmitter,\n              projectService,\n            ),\n      inject: [\n        NodeConfig,\n        EventEmitter2,\n        ProjectService,\n        ApiService,\n        IndexerManager,\n      ],\n    },\n    FetchService,\n    BenchmarkService,\n    DictionaryService,\n    SandboxService,\n    DsProcessorService,\n    DynamicDsService,\n    PoiService,\n    MmrService,\n    ProjectService,\n    UnfinalizedBlocksService,\n  ],\n  exports: [StoreService, MmrService],\n})\nexport class FetchModule {}\n"]}
+{"version":3,"file":"fetch.module.js","sourceRoot":"","sources":["../../src/indexer/fetch.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAAsD;AACtD,gDAO0B;AAE1B,2EAAsE;AACtE,uDAG2B;AAC3B,6DAAyD;AACzD,iEAA4D;AAC5D,6DAAwD;AACxD,mDAA+C;AAC/C,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAqEhE,IAAM,WAAW,GAAjB,MAAM,WAAW;CAAG,CAAA;AAAd,WAAW;IAnEvB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,yCAAkB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;oBACjE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,6BAAa,CAAC;aAC5C;YACD,gCAAc;YACd;gBACE,OAAO,EAAE,kBAAkB;gBAC3B,UAAU,EAAE,CACV,UAAsB,EACtB,YAA2B,EAC3B,cAA8B,EAC9B,UAAsB,EACtB,cAA8B,EAC9B,EAAE,CACF,UAAU,CAAC,OAAO,KAAK,SAAS;oBAC9B,CAAC,CAAC,IAAI,8CAA4B,CAC9B,UAAU,EACV,YAAY,EACZ,cAAc,CACf;oBACH,CAAC,CAAC,IAAI,wCAAsB,CACxB,UAAU,EACV,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,CACf;gBACP,MAAM,EAAE;oBACN,sBAAU;oBACV,6BAAa;oBACb,gCAAc;oBACd,sBAAU;oBACV,gCAAc;iBACf;aACF;YACD,4BAAY;YACZ,4BAAgB;YAChB;gBACE,OAAO,EAAE,sCAAiB;gBAC1B,UAAU,EAAE,KAAK,EAAE,OAAwB,EAAE,UAAsB,EAAE,EAAE;oBACrE,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;oBACrE,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC;oBAC/B,OAAO,iBAAiB,CAAC;gBAC3B,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,sBAAU,CAAC;aACzC;YACD,gCAAc;YACd,yCAAkB;YAClB,qCAAgB;YAChB,sBAAU;YACV,sBAAU;YACV,gCAAc;YACd,oDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,wBAAY,EAAE,sBAAU,CAAC;KACpC,CAAC;GACW,WAAW,CAAG;AAAd,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  BenchmarkService,\n  MmrService,\n  StoreService,\n  PoiService,\n  ApiService,\n  NodeConfig,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum/api.service.ethereum';\nimport {\n  BlockDispatcherService,\n  WorkerBlockDispatcherService,\n} from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { FetchService } from './fetch.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    StoreService,\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new EthereumApiService(project, eventEmitter);\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', EventEmitter2],\n    },\n    IndexerManager,\n    {\n      provide: 'IBlockDispatcher',\n      useFactory: (\n        nodeConfig: NodeConfig,\n        eventEmitter: EventEmitter2,\n        projectService: ProjectService,\n        apiService: ApiService,\n        indexerManager: IndexerManager,\n      ) =>\n        nodeConfig.workers !== undefined\n          ? new WorkerBlockDispatcherService(\n              nodeConfig,\n              eventEmitter,\n              projectService,\n            )\n          : new BlockDispatcherService(\n              apiService,\n              nodeConfig,\n              indexerManager,\n              eventEmitter,\n              projectService,\n            ),\n      inject: [\n        NodeConfig,\n        EventEmitter2,\n        ProjectService,\n        ApiService,\n        IndexerManager,\n      ],\n    },\n    FetchService,\n    BenchmarkService,\n    {\n      provide: DictionaryService,\n      useFactory: async (project: SubqueryProject, nodeConfig: NodeConfig) => {\n        const dictionaryService = new DictionaryService(project, nodeConfig);\n        await dictionaryService.init();\n        return dictionaryService;\n      },\n      inject: ['ISubqueryProject', NodeConfig],\n    },\n    SandboxService,\n    DsProcessorService,\n    DynamicDsService,\n    PoiService,\n    MmrService,\n    ProjectService,\n    UnfinalizedBlocksService,\n  ],\n  exports: [StoreService, MmrService],\n})\nexport class FetchModule {}\n"]}

package/dist/indexer/fetch.service.d.ts

@@ -21,11 +21,11 @@ export declare class FetchService implements OnApplicationShutdown {
     private latestBestHeight;
     private latestFinalizedHeight;
     private isShutdown;
-    private dictionaryQueryEntries?;
     private batchSizeScale;
     private templateDynamicDatasouces;
     private dictionaryGenesisMatches;
     private evmChainId;
+    private bypassBlocks;
     constructor(apiService: ApiService, nodeConfig: NodeConfig, project: SubqueryProject, blockDispatcher: IBlockDispatcher, dictionaryService: DictionaryService, dynamicDsService: DynamicDsService, unfinalizedBlocksService: UnfinalizedBlocksService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry);
     onApplicationShutdown(): void;
     get api(): ApiWrapper;
@@ -34,6 +34,8 @@ export declare class FetchService implements OnApplicationShutdown {
     updateDictionary(): void;
     private get useDictionary();
     init(startHeight: number): Promise<void>;
+    getUseDictionary(): boolean;
+    getLatestFinalizedHeight(): number;
     checkBatchScale(): void;
     getFinalizedBlockHead(): Promise<void>;
     getBestBlockHead(): Promise<void>;
@@ -42,6 +44,8 @@ export declare class FetchService implements OnApplicationShutdown {
     getModuloBlocks(startHeight: number, endHeight: number): number[];
     getEnqueuedModuloBlocks(startBlockHeight: number): number[];
     fillNextBlockBuffer(initBlockHeight: number): Promise<void>;
+    private getLatestBufferHeight;
+    private filteredBlockBatch;
     private nextEndBlockHeight;
     private dictionaryValidation;
     resetForNewDs(blockHeight: number): Promise<void>;

package/dist/indexer/fetch.service.js

@@ -94,6 +94,7 @@ let FetchService = class FetchService {
         this.schedulerRegistry = schedulerRegistry;
         this.isShutdown = false;
         this.dictionaryGenesisMatches = true;
+        this.bypassBlocks = [];
         this.batchSizeScale = 1;
     }
     onApplicationShutdown() {
@@ -164,6 +165,10 @@ let FetchService = class FetchService {
             !!this.dictionaryService.getDictionaryQueryEntries((_a = this.blockDispatcher.latestBufferedHeight) !== null && _a !== void 0 ? _a : Math.min(...this.project.dataSources.map((ds) => ds.startBlock))).length);
     }
     async init(startHeight) {
+        var _a;
+        if (((_a = this.project.network) === null || _a === void 0 ? void 0 : _a.bypassBlocks) !== undefined) {
+            this.bypassBlocks = (0, node_core_1.transformBypassBlocks)(this.project.network.bypassBlocks).filter((blk) => blk >= startHeight);
+        }
         if (this.api) {
             const CHAIN_INTERVAL = (0, utils_ethereum_1.calcInterval)(this.api) * INTERVAL_PERCENT;
             BLOCK_TIME_VARIANCE = Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);
@@ -179,9 +184,22 @@ let FetchService = class FetchService {
         }
         await this.getFinalizedBlockHead();
         await this.getBestBlockHead();
+        //  Call metadata here, other network should align with this
+        //  For substrate, we might use the specVersion metadata in future if we have same error handling as in node-core
+        const metadata = await this.dictionaryService.getMetadata();
+        const validChecker = this.dictionaryValidation(metadata);
+        if (validChecker) {
+            this.dictionaryService.setDictionaryStartHeight(metadata._metadata.startHeight);
+        }
         await this.blockDispatcher.init(this.resetForNewDs.bind(this));
         void this.startLoop(startHeight);
     }
+    getUseDictionary() {
+        return this.useDictionary;
+    }
+    getLatestFinalizedHeight() {
+        return this.latestFinalizedHeight;
+    }
     checkBatchScale() {
         if (this.nodeConfig['scale-batch-size']) {
             const scale = (0, node_core_1.checkMemoryUsage)(this.batchSizeScale, this.nodeConfig);
@@ -281,6 +299,9 @@ let FetchService = class FetchService {
                 ? this.blockDispatcher.latestBufferedHeight + 1
                 : initBlockHeight;
         };
+        if (this.dictionaryService.startHeight > getStartBlockHeight()) {
+            logger.warn(`Dictionary start height ${this.dictionaryService.startHeight} is beyond indexing height ${getStartBlockHeight()}, skipping dictionary for now`);
+        }
         while (!this.isShutdown) {
             startBlockHeight = getStartBlockHeight();
             scaledBatchSize = Math.max(Math.round(this.batchSizeScale * this.nodeConfig.batchSize), Math.min(MINIMUM_BATCH_SIZE, this.nodeConfig.batchSize * 3));
@@ -292,7 +313,8 @@ let FetchService = class FetchService {
                 await (0, node_core_1.delay)(1);
                 continue;
             }
-            if (this.useDictionary) {
+            if (this.useDictionary &&
+                startBlockHeight >= this.dictionaryService.startHeight) {
                 const queryEndBlock = startBlockHeight + DICTIONARY_MAX_QUERY_SIZE;
                 const moduloBlocks = this.getModuloBlocks(startBlockHeight, queryEndBlock);
                 try {
@@ -309,12 +331,13 @@ let FetchService = class FetchService {
                             .sort((a, b) => a - b);
                         if (batchBlocks.length === 0) {
                             // There we're no blocks in this query range, we can set a new height we're up to
-                            this.blockDispatcher.latestBufferedHeight = Math.min(queryEndBlock - 1, dictionary._metadata.lastProcessedHeight);
+                            this.blockDispatcher.enqueueBlocks([], Math.min(queryEndBlock - 1, dictionary._metadata.lastProcessedHeight));
                         }
                         else {
                             const maxBlockSize = Math.min(batchBlocks.length, this.blockDispatcher.freeSize);
-                            batchBlocks = batchBlocks.slice(0, maxBlockSize);
-                            this.blockDispatcher.enqueueBlocks(batchBlocks);
+                            const enqueuingBlocks = batchBlocks.slice(0, maxBlockSize);
+                            const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);
+                            this.blockDispatcher.enqueueBlocks(cleanedBatchBlocks, this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks));
                         }
                         continue; // skip nextBlockRange() way
                     }
@@ -327,13 +350,32 @@ let FetchService = class FetchService {
             }
             const endHeight = this.nextEndBlockHeight(startBlockHeight, scaledBatchSize);
             if (handlers.length && this.getModulos().length === handlers.length) {
-                this.blockDispatcher.enqueueBlocks(this.getEnqueuedModuloBlocks(startBlockHeight));
+                const enqueuingBlocks = this.getEnqueuedModuloBlocks(startBlockHeight);
+                const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);
+                this.blockDispatcher.enqueueBlocks(cleanedBatchBlocks, this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks));
             }
             else {
-                this.blockDispatcher.enqueueBlocks((0, lodash_1.range)(startBlockHeight, endHeight + 1));
+                const enqueuingBlocks = (0, lodash_1.range)(startBlockHeight, endHeight + 1);
+                const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);
+                this.blockDispatcher.enqueueBlocks(cleanedBatchBlocks, this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks));
             }
         }
     }
+    getLatestBufferHeight(cleanedBatchBlocks, rawBatchBlocks) {
+        return Math.max(...cleanedBatchBlocks, ...rawBatchBlocks);
+    }
+    filteredBlockBatch(currentBatchBlocks) {
+        if (!this.bypassBlocks.length || !currentBatchBlocks) {
+            return currentBatchBlocks;
+        }
+        const cleanedBatch = (0, node_core_1.cleanedBatchBlocks)(this.bypassBlocks, currentBatchBlocks);
+        const pollutedBlocks = this.bypassBlocks.filter((b) => b < Math.max(...currentBatchBlocks));
+        if (pollutedBlocks.length) {
+            logger.info(`Bypassing blocks: ${pollutedBlocks}`);
+        }
+        this.bypassBlocks = (0, lodash_1.without)(this.bypassBlocks, ...pollutedBlocks);
+        return cleanedBatch;
+    }
     nextEndBlockHeight(startBlockHeight, scaledBatchSize) {
         let endBlockHeight = startBlockHeight + scaledBatchSize - 1;
         if (endBlockHeight > this.latestFinalizedHeight) {
@@ -348,23 +390,29 @@ let FetchService = class FetchService {
         }
         return endBlockHeight;
     }
-    dictionaryValidation({ _metadata: metaData }, startBlockHeight) {
-        if (metaData.genesisHash !== this.api.getGenesisHash() &&
-            this.evmChainId !== this.api.getChainId().toString()) {
-            logger.error('The dictionary that you have specified does not match the chain you are indexing, it will be ignored. Please update your project manifest to reference the correct dictionary');
-            this.dictionaryGenesisMatches = false;
-            this.eventEmitter.emit(node_core_1.IndexerEvent.UsingDictionary, {
-                value: Number(this.useDictionary),
-            });
-            this.eventEmitter.emit(node_core_1.IndexerEvent.SkipDictionary);
-            return false;
-        }
-        if (metaData.lastProcessedHeight < startBlockHeight) {
-            logger.warn(`Dictionary indexed block is behind current indexing block height`);
-            this.eventEmitter.emit(node_core_1.IndexerEvent.SkipDictionary);
-            return false;
+    dictionaryValidation(dictionary, startBlockHeight) {
+        if (dictionary !== undefined) {
+            const { _metadata: metaData } = dictionary;
+            if (metaData.genesisHash !== this.api.getGenesisHash() &&
+                this.evmChainId !== this.api.getChainId().toString()) {
+                logger.error('The dictionary that you have specified does not match the chain you are indexing, it will be ignored. Please update your project manifest to reference the correct dictionary');
+                this.dictionaryGenesisMatches = false;
+                this.eventEmitter.emit(node_core_1.IndexerEvent.UsingDictionary, {
+                    value: Number(this.useDictionary),
+                });
+                this.eventEmitter.emit(node_core_1.IndexerEvent.SkipDictionary);
+                return false;
+            }
+            if (startBlockHeight !== undefined) {
+                if (metaData.lastProcessedHeight < startBlockHeight) {
+                    logger.warn(`Dictionary indexed block is behind current indexing block height`);
+                    this.eventEmitter.emit(node_core_1.IndexerEvent.SkipDictionary);
+                    return false;
+                }
+            }
+            return true;
         }
-        return true;
+        return false;
     }
     async resetForNewDs(blockHeight) {
         await this.syncDynamicDatascourcesFromMeta();
@@ -398,6 +446,7 @@ __decorate([
 ], FetchService.prototype, "getBestBlockHead", null);
 FetchService = __decorate([
     (0, common_1.Injectable)(),
+    __param(2, (0, common_1.Inject)('ISubqueryProject')),
     __param(3, (0, common_1.Inject)('IBlockDispatcher')),
     __metadata("design:paramtypes", [node_core_1.ApiService,
         node_core_1.NodeConfig,

package/dist/indexer/fetch.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.service.js","sourceRoot":"","sources":["../../src/indexer/fetch.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAA2E;AAC3E,yDAAsD;AACtD,+CAA+D;AAC/D,4DAIgC;AAChC,gDAQ0B;AAO1B,mCAA+C;AAC/C,kEAA+E;AAC/E,+DAA0D;AAC1D,4CAAkE;AAElE,6DAAyD;AACzD,6DAAwD;AACxD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,OAAO,CAAC,CAAC;AAClC,IAAI,mBAAmB,GAAG,IAAI,CAAC;AAC/B,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,SAAS,uBAAuB,CAC9B,MAAyB;IAEzB,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE;gBACV,SAAS;aACV;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAA,qBAAY,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACxD;KACF;IACD,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiC;IAEjC,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,IAAI,MAAM,CAAC,IAAI,EAAE;QACf,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;SACjC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,EAAE,EAAE;QACb,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;SAC/B,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE;QACnB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,IAAA,0BAAiB,EAAC,MAAM,CAAC,QAAQ,CAAC;SAC1C,CAAC,CAAC;KACJ;IACD,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,UAAU;KACX,CAAC;AACJ,CAAC;AAGM,IAAM,YAAY,GAAlB,MAAM,YAAY;IAUvB,YACU,UAAsB,EACtB,UAAsB,EACtB,OAAwB,EACI,eAAiC,EAC7D,iBAAoC,EACpC,gBAAkC,EAClC,wBAAkD,EAClD,YAA2B,EAC3B,iBAAoC;QARpC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,YAAO,GAAP,OAAO,CAAiB;QACI,oBAAe,GAAf,eAAe,CAAkB;QAC7D,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,iBAAY,GAAZ,YAAY,CAAe;QAC3B,sBAAiB,GAAjB,iBAAiB,CAAmB;QAhBtC,eAAU,GAAG,KAAK,CAAC;QAInB,6BAAwB,GAAG,IAAI,CAAC;QActC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,IAAI,CAAC,yBAAyB;YAC5B,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;IACxD,CAAC;IAED,2BAA2B,CAAC,UAAkB;QAC5C,MAAM,YAAY,GAA2B,EAAE,CAAC;QAEhD,wDAAwD;QACxD,kDAAkD;QAClD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW;aACxC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC;aACtC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,UAAU,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAE/C,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE;YAC3B,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IAAI,UAAgC,CAAC;gBACrC,UAAU,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC9B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;gBACzC,IAAI,CAAC,UAAU,CAAC,MAAM;oBAAE,OAAO,EAAE,CAAC;gBAClC,QAAQ,OAAO,CAAC,IAAI,EAAE;oBACpB,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,EAAE,CAAC;oBACZ,KAAK,qCAAmB,CAAC,IAAI,CAAC,CAAC;wBAC7B,KAAK,MAAM,MAAM,IAAI,UAAyC,EAAE;4BAC9D,IACE,MAAM,CAAC,IAAI,KAAK,SAAS;gCACzB,MAAM,CAAC,EAAE,KAAK,SAAS;gCACvB,MAAM,CAAC,QAAQ,EACf;gCACA,YAAY,CAAC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;6BACnD;iCAAM;gCACL,OAAO,EAAE,CAAC;6BACX;yBACF;wBACD,MAAM;qBACP;oBACD,KAAK,qCAAmB,CAAC,KAAK,CAAC,CAAC;wBAC9B,KAAK,MAAM,MAAM,IAAI,UAAiC,EAAE;4BACtD,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE;gCACnC,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC;6BACpD;iCAAM;gCACL,OAAO,EAAE,CAAC;6BACX;yBACF;wBACD,MAAM;qBACP;oBACD,QAAQ;iBACT;aACF;SACF;QAED,OAAO,IAAA,eAAM,EACX,YAAY,EACZ,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAC9B,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACxC,EAAE,CACN,CAAC;IACJ,CAAC;IAED,gBAAgB;QACd,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAC5C,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,EAC/D,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAC5C,CAAC;IACJ,CAAC;IAED,IAAY,aAAa;;QACvB,OAAO,CACL,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU;YACjC,IAAI,CAAC,wBAAwB;YAC7B,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,CAChD,MAAA,IAAI,CAAC,eAAe,CAAC,oBAAoB,mCACvC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CACnE,CAAC,MAAM,CACT,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;QAC5B,IAAI,IAAI,CAAC,GAAG,EAAE;YACZ,MAAM,cAAc,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC;YAEjE,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;YAEpE,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAChC,oBAAoB,EACpB,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,gBAAgB,EAAE,EAAE,mBAAmB,CAAC,CACrE,CAAC;SACH;QACD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAE7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;YACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE;YACnC,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;SAChE;QAED,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9B,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAE/D,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAGD,eAAe;QACb,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE;YACvC,MAAM,KAAK,GAAG,IAAA,4BAAgB,EAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAErE,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,EAAE;gBACjC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;aAC7B;SACF;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC9D,OAAO;SACR;QACD,IAAI;YACF,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,aAAa,sBAAsB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC3D,sBAAsB,CACvB,CAAC;YACF,IAAI,IAAI,CAAC,qBAAqB,KAAK,sBAAsB,EAAE;gBACzD,IAAI,CAAC,qBAAqB,GAAG,sBAAsB,CAAC;gBACpD,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;gBACtE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;wBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;4BAC/C,MAAM,EAAE,IAAI,CAAC,qBAAqB;yBACnC,CAAC,CAAC;qBACJ;iBACF;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;SAC7D;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,OAAO;SACR;QACD,IAAI;YACF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC9D,MAAM,CAAC,KAAK,CAAC,QAAQ,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,IAAI,CAAC,gBAAgB,KAAK,iBAAiB,EAAE;gBAC/C,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,SAAS,EAAE;oBAC7C,MAAM,EAAE,IAAI,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;gBAEH,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACrC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;wBAC/C,MAAM,EAAE,IAAI,CAAC,gBAAgB;qBAC9B,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,sCAAsC,CAAC,CAAC;SACxD;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,eAAuB;QAC7C,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC;IAED,UAAU;QACR,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;YACzC,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;gBAClB,SAAS;aACV;YACD,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IACE,OAAO,CAAC,IAAI,KAAK,qCAAmB,CAAC,KAAK;oBAC1C,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,MAAM,CAAC,MAAM,EACrB;oBACA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iBACrC;aACF;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,eAAe,CAAC,WAAmB,EAAE,SAAiB;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,WAAW,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;YAC5C,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;gBACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB;SACF;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,uBAAuB,CAAC,gBAAwB;QAC9C,OAAO,IAAI,CAAC,eAAe,CACzB,gBAAgB,EAChB,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACxD,gBAAgB,CACnB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,eAAuB;QAC/C,IAAI,gBAAwB,CAAC;QAC7B,IAAI,eAAuB,CAAC;QAC5B,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,CACxB,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC7D,CAAC;QAEF,MAAM,mBAAmB,GAAG,GAAW,EAAE;YACvC,OAAO,IAAI,CAAC,eAAe,CAAC,oBAAoB;gBAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,GAAG,CAAC;gBAC/C,CAAC,CAAC,eAAe,CAAC;QACtB,CAAC,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE;YACvB,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;YAEzC,eAAe,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAC3D,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CAC5D,CAAC;YACF,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB;gBACpD,CAAC,CAAC,IAAI,CAAC,gBAAgB;gBACvB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC;YAE/B,IACE,IAAI,CAAC,eAAe,CAAC,QAAQ,GAAG,eAAe;gBAC/C,gBAAgB,GAAG,YAAY,EAC/B;gBACA,MAAM,IAAA,iBAAK,EAAC,CAAC,CAAC,CAAC;gBACf,SAAS;aACV;YAED,IAAI,IAAI,CAAC,aAAa,EAAE;gBACtB,MAAM,aAAa,GAAG,gBAAgB,GAAG,yBAAyB,CAAC;gBACnE,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CACvC,gBAAgB,EAChB,aAAa,CACd,CAAC;gBAEF,IAAI;oBACF,MAAM,UAAU,GACd,MAAM,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAClD,gBAAgB,EAChB,aAAa,EACb,eAAe,CAChB,CAAC;oBAEJ,IAAI,gBAAgB,KAAK,mBAAmB,EAAE,EAAE;wBAC9C,MAAM,CAAC,KAAK,CACV,gEAAgE,CACjE,CAAC;wBACF,SAAS;qBACV;oBAED,IACE,UAAU;wBACV,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,gBAAgB,CAAC,EACvD;wBACA,IAAI,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;wBAEjC,WAAW,GAAG,WAAW;6BACtB,MAAM,CAAC,YAAY,CAAC;6BACpB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBACzB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;4BAC5B,iFAAiF;4BACjF,IAAI,CAAC,eAAe,CAAC,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAClD,aAAa,GAAG,CAAC,EACjB,UAAU,CAAC,SAAS,CAAC,mBAAmB,CACzC,CAAC;yBACH;6BAAM;4BACL,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,WAAW,CAAC,MAAM,EAClB,IAAI,CAAC,eAAe,CAAC,QAAQ,CAC9B,CAAC;4BACF,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;4BACjD,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;yBACjD;wBACD,SAAS,CAAC,4BAA4B;qBACvC;oBACD,iCAAiC;iBAClC;gBAAC,OAAO,CAAC,EAAE;oBACV,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACvD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;iBACrD;aACF;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CACvC,gBAAgB,EAChB,eAAe,CAChB,CAAC;YAEF,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE;gBACnE,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,CAAC,CAC/C,CAAC;aACH;iBAAM;gBACL,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,IAAA,cAAK,EAAC,gBAAgB,EAAE,SAAS,GAAG,CAAC,CAAC,CACvC,CAAC;aACH;SACF;IACH,CAAC;IAEO,kBAAkB,CACxB,gBAAwB,EACxB,eAAuB;QAEvB,IAAI,cAAc,GAAG,gBAAgB,GAAG,eAAe,GAAG,CAAC,CAAC;QAE5D,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE;YAC/C,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;gBACrC,IAAI,cAAc,IAAI,IAAI,CAAC,gBAAgB,EAAE;oBAC3C,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC;iBACxC;aACF;iBAAM;gBACL,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;aAC7C;SACF;QACD,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,oBAAoB,CAC1B,EAAE,SAAS,EAAE,QAAQ,EAAc,EACnC,gBAAwB;QAExB,IACE,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE;YAClD,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EACpD;YACA,MAAM,CAAC,KAAK,CACV,+KAA+K,CAChL,CAAC;YACF,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;YACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;gBACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;aAClC,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;SACd;QACD,IAAI,QAAQ,CAAC,mBAAmB,GAAG,gBAAgB,EAAE;YACnD,MAAM,CAAC,IAAI,CACT,kEAAkE,CACnE,CAAC;YACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;SACd;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACrC,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAAC,WAAmB;QAClD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;CACF,CAAA;AA5QC;IAAC,IAAA,mBAAQ,EAAC,qBAAqB,CAAC;;;;mDAS/B;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,GAAG,IAAI,CAAC;;;;yDA0BpC;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,GAAG,IAAI,CAAC;;;;oDAwBpC;AA7MU,YAAY;IADxB,IAAA,mBAAU,GAAE;IAeR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHP,sBAAU;QACV,sBAAU;QACb,iCAAe,UAEL,sCAAiB;QAClB,qCAAgB;QACR,oDAAwB;QACpC,6BAAa;QACR,4BAAiB;GAnBnC,YAAY,CA0ZxB;AA1ZY,oCAAY","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval, SchedulerRegistry } from '@nestjs/schedule';\nimport {\n  isCustomDs,\n  EthereumHandlerKind,\n  SubqlHandlerFilter,\n} from '@subql/common-ethereum';\nimport {\n  ApiService,\n  Dictionary,\n  checkMemoryUsage,\n  delay,\n  getLogger,\n  IndexerEvent,\n  NodeConfig,\n} from '@subql/node-core';\nimport {\n  DictionaryQueryEntry,\n  ApiWrapper,\n  EthereumLogFilter,\n  EthereumTransactionFilter,\n} from '@subql/types-ethereum';\nimport { range, sortBy, uniqBy } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { calcInterval } from '../ethereum/utils.ethereum';\nimport { eventToTopic, functionToSighash } from '../utils/string';\nimport { IBlockDispatcher } from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('fetch');\nlet BLOCK_TIME_VARIANCE = 5000;\nconst DICTIONARY_MAX_QUERY_SIZE = 10000;\nconst CHECK_MEMORY_INTERVAL = 60000;\nconst MINIMUM_BATCH_SIZE = 5;\nconst INTERVAL_PERCENT = 0.9;\n\nfunction eventFilterToQueryEntry(\n  filter: EthereumLogFilter,\n): DictionaryQueryEntry {\n  const conditions = [];\n  if (filter.address) {\n    conditions.push({\n      field: 'address',\n      value: filter.address.toLowerCase(),\n    });\n  }\n  if (filter.topics) {\n    for (let i = 0; i < Math.min(filter.topics.length, 4); i++) {\n      const topic = filter.topics[i];\n      if (!topic) {\n        continue;\n      }\n      const field = `topics${i}`;\n      conditions.push({ field, value: eventToTopic(topic) });\n    }\n  }\n  return {\n    entity: 'evmLogs',\n    conditions,\n  };\n}\n\nfunction callFilterToQueryEntry(\n  filter: EthereumTransactionFilter,\n): DictionaryQueryEntry {\n  const conditions = [];\n  if (filter.from) {\n    conditions.push({\n      field: 'from',\n      value: filter.from.toLowerCase(),\n    });\n  }\n  if (filter.to) {\n    conditions.push({\n      field: 'to',\n      value: filter.to.toLowerCase(),\n    });\n  }\n  if (filter.function) {\n    conditions.push({\n      field: 'func',\n      value: functionToSighash(filter.function),\n    });\n  }\n  return {\n    entity: 'evmTransactions',\n    conditions,\n  };\n}\n\n@Injectable()\nexport class FetchService implements OnApplicationShutdown {\n  private latestBestHeight: number;\n  private latestFinalizedHeight: number;\n  private isShutdown = false;\n  private dictionaryQueryEntries?: DictionaryQueryEntry[];\n  private batchSizeScale: number;\n  private templateDynamicDatasouces: SubqlProjectDs[];\n  private dictionaryGenesisMatches = true;\n  private evmChainId: string;\n\n  constructor(\n    private apiService: ApiService,\n    private nodeConfig: NodeConfig,\n    private project: SubqueryProject,\n    @Inject('IBlockDispatcher') private blockDispatcher: IBlockDispatcher,\n    private dictionaryService: DictionaryService,\n    private dynamicDsService: DynamicDsService,\n    private unfinalizedBlocksService: UnfinalizedBlocksService,\n    private eventEmitter: EventEmitter2,\n    private schedulerRegistry: SchedulerRegistry,\n  ) {\n    this.batchSizeScale = 1;\n  }\n\n  onApplicationShutdown(): void {\n    this.isShutdown = true;\n  }\n\n  get api(): ApiWrapper {\n    return this.apiService.api;\n  }\n\n  async syncDynamicDatascourcesFromMeta(): Promise<void> {\n    this.templateDynamicDatasouces =\n      await this.dynamicDsService.getDynamicDatasources();\n  }\n\n  buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[] {\n    const queryEntries: DictionaryQueryEntry[] = [];\n\n    // Only run the ds that is equal or less than startBlock\n    // sort array from lowest ds.startBlock to highest\n    const filteredDs = this.project.dataSources\n      .concat(this.templateDynamicDatasouces)\n      .filter((ds) => ds.startBlock <= startBlock)\n      .sort((a, b) => a.startBlock - b.startBlock);\n\n    for (const ds of filteredDs) {\n      for (const handler of ds.mapping.handlers) {\n        let filterList: SubqlHandlerFilter[];\n        filterList = [handler.filter];\n        filterList = filterList.filter((f) => f);\n        if (!filterList.length) return [];\n        switch (handler.kind) {\n          case EthereumHandlerKind.Block:\n            return [];\n          case EthereumHandlerKind.Call: {\n            for (const filter of filterList as EthereumTransactionFilter[]) {\n              if (\n                filter.from !== undefined ||\n                filter.to !== undefined ||\n                filter.function\n              ) {\n                queryEntries.push(callFilterToQueryEntry(filter));\n              } else {\n                return [];\n              }\n            }\n            break;\n          }\n          case EthereumHandlerKind.Event: {\n            for (const filter of filterList as EthereumLogFilter[]) {\n              if (filter.address || filter.topics) {\n                queryEntries.push(eventFilterToQueryEntry(filter));\n              } else {\n                return [];\n              }\n            }\n            break;\n          }\n          default:\n        }\n      }\n    }\n\n    return uniqBy(\n      queryEntries,\n      (item) =>\n        `${item.entity}|${JSON.stringify(\n          sortBy(item.conditions, (c) => c.field),\n        )}`,\n    );\n  }\n\n  updateDictionary(): void {\n    this.dictionaryService.buildDictionaryEntryMap<SubqlProjectDs>(\n      this.project.dataSources.concat(this.templateDynamicDatasouces),\n      this.buildDictionaryQueryEntries.bind(this),\n    );\n  }\n\n  private get useDictionary(): boolean {\n    return (\n      !!this.project.network.dictionary &&\n      this.dictionaryGenesisMatches &&\n      !!this.dictionaryService.getDictionaryQueryEntries(\n        this.blockDispatcher.latestBufferedHeight ??\n          Math.min(...this.project.dataSources.map((ds) => ds.startBlock)),\n      ).length\n    );\n  }\n\n  async init(startHeight: number): Promise<void> {\n    if (this.api) {\n      const CHAIN_INTERVAL = calcInterval(this.api) * INTERVAL_PERCENT;\n\n      BLOCK_TIME_VARIANCE = Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);\n\n      this.schedulerRegistry.addInterval(\n        'getLatestBlockHead',\n        setInterval(() => void this.getBestBlockHead(), BLOCK_TIME_VARIANCE),\n      );\n    }\n    await this.syncDynamicDatascourcesFromMeta();\n\n    this.updateDictionary();\n    this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n      value: Number(this.useDictionary),\n    });\n\n    if (this.project.network.dictionary) {\n      this.evmChainId = await this.dictionaryService.getEvmChainId();\n    }\n\n    await this.getFinalizedBlockHead();\n    await this.getBestBlockHead();\n\n    await this.blockDispatcher.init(this.resetForNewDs.bind(this));\n\n    void this.startLoop(startHeight);\n  }\n\n  @Interval(CHECK_MEMORY_INTERVAL)\n  checkBatchScale(): void {\n    if (this.nodeConfig['scale-batch-size']) {\n      const scale = checkMemoryUsage(this.batchSizeScale, this.nodeConfig);\n\n      if (this.batchSizeScale !== scale) {\n        this.batchSizeScale = scale;\n      }\n    }\n  }\n\n  @Interval(BLOCK_TIME_VARIANCE * 1000)\n  async getFinalizedBlockHead(): Promise<void> {\n    if (!this.api) {\n      logger.debug(`Skip fetch finalized block until API is ready`);\n      return;\n    }\n    try {\n      const currentFinalizedHeight = await this.api.getFinalizedBlockHeight();\n      logger.debug(`finalized:${currentFinalizedHeight.toString()}`);\n      const finalizedHeader = await this.api.getBlockByHeightOrHash(\n        currentFinalizedHeight,\n      );\n      if (this.latestFinalizedHeight !== currentFinalizedHeight) {\n        this.latestFinalizedHeight = currentFinalizedHeight;\n        this.unfinalizedBlocksService.registerFinalizedBlock(finalizedHeader);\n        if (!this.nodeConfig.unfinalizedBlocks) {\n          if (!this.nodeConfig.unfinalizedBlocks) {\n            this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n              height: this.latestFinalizedHeight,\n            });\n          }\n        }\n      }\n    } catch (e) {\n      logger.warn(e, `Having a problem when get finalized block`);\n    }\n  }\n\n  @Interval(BLOCK_TIME_VARIANCE * 1000)\n  async getBestBlockHead(): Promise<void> {\n    if (!this.api) {\n      logger.debug(`Skip fetch best block until API is ready`);\n      return;\n    }\n    try {\n      const currentBestHeight = await this.api.getBestBlockHeight();\n      logger.debug(`best:${currentBestHeight.toString()}`);\n      if (this.latestBestHeight !== currentBestHeight) {\n        this.latestBestHeight = currentBestHeight;\n        this.eventEmitter.emit(IndexerEvent.BlockBest, {\n          height: this.latestBestHeight,\n        });\n\n        if (this.nodeConfig.unfinalizedBlocks) {\n          this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n            height: this.latestBestHeight,\n          });\n        }\n      }\n    } catch (e) {\n      logger.warn(e, `Having a problem when get best block`);\n    }\n  }\n\n  private async startLoop(initBlockHeight: number): Promise<void> {\n    await this.fillNextBlockBuffer(initBlockHeight);\n  }\n\n  getModulos(): number[] {\n    const modulos: number[] = [];\n    for (const ds of this.project.dataSources) {\n      if (isCustomDs(ds)) {\n        continue;\n      }\n      for (const handler of ds.mapping.handlers) {\n        if (\n          handler.kind === EthereumHandlerKind.Block &&\n          handler.filter &&\n          handler.filter.modulo\n        ) {\n          modulos.push(handler.filter.modulo);\n        }\n      }\n    }\n    return modulos;\n  }\n\n  getModuloBlocks(startHeight: number, endHeight: number): number[] {\n    const modulos = this.getModulos();\n    const moduloBlocks: number[] = [];\n    for (let i = startHeight; i < endHeight; i++) {\n      if (modulos.find((m) => i % m === 0)) {\n        moduloBlocks.push(i);\n      }\n    }\n    return moduloBlocks;\n  }\n\n  getEnqueuedModuloBlocks(startBlockHeight: number): number[] {\n    return this.getModuloBlocks(\n      startBlockHeight,\n      this.nodeConfig.batchSize * Math.max(...this.getModulos()) +\n        startBlockHeight,\n    ).slice(0, this.nodeConfig.batchSize);\n  }\n\n  async fillNextBlockBuffer(initBlockHeight: number): Promise<void> {\n    let startBlockHeight: number;\n    let scaledBatchSize: number;\n    const handlers = [].concat(\n      ...this.project.dataSources.map((ds) => ds.mapping.handlers),\n    );\n\n    const getStartBlockHeight = (): number => {\n      return this.blockDispatcher.latestBufferedHeight\n        ? this.blockDispatcher.latestBufferedHeight + 1\n        : initBlockHeight;\n    };\n\n    while (!this.isShutdown) {\n      startBlockHeight = getStartBlockHeight();\n\n      scaledBatchSize = Math.max(\n        Math.round(this.batchSizeScale * this.nodeConfig.batchSize),\n        Math.min(MINIMUM_BATCH_SIZE, this.nodeConfig.batchSize * 3),\n      );\n      const latestHeight = this.nodeConfig.unfinalizedBlocks\n        ? this.latestBestHeight\n        : this.latestFinalizedHeight;\n\n      if (\n        this.blockDispatcher.freeSize < scaledBatchSize ||\n        startBlockHeight > latestHeight\n      ) {\n        await delay(1);\n        continue;\n      }\n\n      if (this.useDictionary) {\n        const queryEndBlock = startBlockHeight + DICTIONARY_MAX_QUERY_SIZE;\n        const moduloBlocks = this.getModuloBlocks(\n          startBlockHeight,\n          queryEndBlock,\n        );\n\n        try {\n          const dictionary =\n            await this.dictionaryService.scopedDictionaryEntries(\n              startBlockHeight,\n              queryEndBlock,\n              scaledBatchSize,\n            );\n\n          if (startBlockHeight !== getStartBlockHeight()) {\n            logger.debug(\n              `Queue was reset for new DS, discarding dictionary query result`,\n            );\n            continue;\n          }\n\n          if (\n            dictionary &&\n            this.dictionaryValidation(dictionary, startBlockHeight)\n          ) {\n            let { batchBlocks } = dictionary;\n\n            batchBlocks = batchBlocks\n              .concat(moduloBlocks)\n              .sort((a, b) => a - b);\n            if (batchBlocks.length === 0) {\n              // There we're no blocks in this query range, we can set a new height we're up to\n              this.blockDispatcher.latestBufferedHeight = Math.min(\n                queryEndBlock - 1,\n                dictionary._metadata.lastProcessedHeight,\n              );\n            } else {\n              const maxBlockSize = Math.min(\n                batchBlocks.length,\n                this.blockDispatcher.freeSize,\n              );\n              batchBlocks = batchBlocks.slice(0, maxBlockSize);\n              this.blockDispatcher.enqueueBlocks(batchBlocks);\n            }\n            continue; // skip nextBlockRange() way\n          }\n          // else use this.nextBlockRange()\n        } catch (e) {\n          logger.debug(`Fetch dictionary stopped: ${e.message}`);\n          this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n        }\n      }\n\n      const endHeight = this.nextEndBlockHeight(\n        startBlockHeight,\n        scaledBatchSize,\n      );\n\n      if (handlers.length && this.getModulos().length === handlers.length) {\n        this.blockDispatcher.enqueueBlocks(\n          this.getEnqueuedModuloBlocks(startBlockHeight),\n        );\n      } else {\n        this.blockDispatcher.enqueueBlocks(\n          range(startBlockHeight, endHeight + 1),\n        );\n      }\n    }\n  }\n\n  private nextEndBlockHeight(\n    startBlockHeight: number,\n    scaledBatchSize: number,\n  ): number {\n    let endBlockHeight = startBlockHeight + scaledBatchSize - 1;\n\n    if (endBlockHeight > this.latestFinalizedHeight) {\n      if (this.nodeConfig.unfinalizedBlocks) {\n        if (endBlockHeight >= this.latestBestHeight) {\n          endBlockHeight = this.latestBestHeight;\n        }\n      } else {\n        endBlockHeight = this.latestFinalizedHeight;\n      }\n    }\n    return endBlockHeight;\n  }\n\n  private dictionaryValidation(\n    { _metadata: metaData }: Dictionary,\n    startBlockHeight: number,\n  ): boolean {\n    if (\n      metaData.genesisHash !== this.api.getGenesisHash() &&\n      this.evmChainId !== this.api.getChainId().toString()\n    ) {\n      logger.error(\n        'The dictionary that you have specified does not match the chain you are indexing, it will be ignored. Please update your project manifest to reference the correct dictionary',\n      );\n      this.dictionaryGenesisMatches = false;\n      this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n        value: Number(this.useDictionary),\n      });\n      this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n      return false;\n    }\n    if (metaData.lastProcessedHeight < startBlockHeight) {\n      logger.warn(\n        `Dictionary indexed block is behind current indexing block height`,\n      );\n      this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n      return false;\n    }\n    return true;\n  }\n\n  async resetForNewDs(blockHeight: number): Promise<void> {\n    await this.syncDynamicDatascourcesFromMeta();\n    this.dynamicDsService.deleteTempDsRecords(blockHeight);\n    this.updateDictionary();\n    this.blockDispatcher.flushQueue(blockHeight);\n  }\n\n  async resetForIncorrectBestBlock(blockHeight: number): Promise<void> {\n    await this.syncDynamicDatascourcesFromMeta();\n    this.updateDictionary();\n    this.blockDispatcher.flushQueue(blockHeight);\n  }\n}\n"]}
+{"version":3,"file":"fetch.service.js","sourceRoot":"","sources":["../../src/indexer/fetch.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAA2E;AAC3E,yDAAsD;AACtD,+CAA+D;AAC/D,4DAIgC;AAChC,gDAU0B;AAQ1B,mCAAwD;AACxD,kEAA+E;AAC/E,+DAA0D;AAC1D,4CAAkE;AAElE,6DAAyD;AACzD,6DAAwD;AACxD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,OAAO,CAAC,CAAC;AAClC,IAAI,mBAAmB,GAAG,IAAI,CAAC;AAC/B,MAAM,yBAAyB,GAAG,KAAK,CAAC;AACxC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,SAAS,uBAAuB,CAC9B,MAAyB;IAEzB,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE;gBACV,SAAS;aACV;YACD,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAA,qBAAY,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACxD;KACF;IACD,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiC;IAEjC,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,IAAI,MAAM,CAAC,IAAI,EAAE;QACf,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;SACjC,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,EAAE,EAAE;QACb,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;SAC/B,CAAC,CAAC;KACJ;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE;QACnB,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,IAAA,0BAAiB,EAAC,MAAM,CAAC,QAAQ,CAAC;SAC1C,CAAC,CAAC;KACJ;IACD,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,UAAU;KACX,CAAC;AACJ,CAAC;AAGM,IAAM,YAAY,GAAlB,MAAM,YAAY;IAUvB,YACU,UAAsB,EACtB,UAAsB,EACM,OAAwB,EACxB,eAAiC,EAC7D,iBAAoC,EACpC,gBAAkC,EAClC,wBAAkD,EAClD,YAA2B,EAC3B,iBAAoC;QARpC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACxB,oBAAe,GAAf,eAAe,CAAkB;QAC7D,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,iBAAY,GAAZ,YAAY,CAAe;QAC3B,sBAAiB,GAAjB,iBAAiB,CAAmB;QAhBtC,eAAU,GAAG,KAAK,CAAC;QAGnB,6BAAwB,GAAG,IAAI,CAAC;QAEhC,iBAAY,GAAa,EAAE,CAAC;QAalC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,IAAI,CAAC,yBAAyB;YAC5B,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;IACxD,CAAC;IAED,2BAA2B,CAAC,UAAkB;QAC5C,MAAM,YAAY,GAA2B,EAAE,CAAC;QAEhD,wDAAwD;QACxD,kDAAkD;QAClD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW;aACxC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC;aACtC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,UAAU,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAE/C,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE;YAC3B,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IAAI,UAAgC,CAAC;gBACrC,UAAU,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC9B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;gBACzC,IAAI,CAAC,UAAU,CAAC,MAAM;oBAAE,OAAO,EAAE,CAAC;gBAClC,QAAQ,OAAO,CAAC,IAAI,EAAE;oBACpB,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,EAAE,CAAC;oBACZ,KAAK,qCAAmB,CAAC,IAAI,CAAC,CAAC;wBAC7B,KAAK,MAAM,MAAM,IAAI,UAAyC,EAAE;4BAC9D,IACE,MAAM,CAAC,IAAI,KAAK,SAAS;gCACzB,MAAM,CAAC,EAAE,KAAK,SAAS;gCACvB,MAAM,CAAC,QAAQ,EACf;gCACA,YAAY,CAAC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;6BACnD;iCAAM;gCACL,OAAO,EAAE,CAAC;6BACX;yBACF;wBACD,MAAM;qBACP;oBACD,KAAK,qCAAmB,CAAC,KAAK,CAAC,CAAC;wBAC9B,KAAK,MAAM,MAAM,IAAI,UAAiC,EAAE;4BACtD,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,EAAE;gCACnC,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC;6BACpD;iCAAM;gCACL,OAAO,EAAE,CAAC;6BACX;yBACF;wBACD,MAAM;qBACP;oBACD,QAAQ;iBACT;aACF;SACF;QAED,OAAO,IAAA,eAAM,EACX,YAAY,EACZ,CAAC,IAAI,EAAE,EAAE,CACP,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAC9B,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACxC,EAAE,CACN,CAAC;IACJ,CAAC;IAED,gBAAgB;QACd,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAC5C,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,EAC/D,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAC5C,CAAC;IACJ,CAAC;IAED,IAAY,aAAa;;QACvB,OAAO,CACL,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU;YACjC,IAAI,CAAC,wBAAwB;YAC7B,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,CAChD,MAAA,IAAI,CAAC,eAAe,CAAC,oBAAoB,mCACvC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CACnE,CAAC,MAAM,CACT,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;;QAC5B,IAAI,CAAA,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,0CAAE,YAAY,MAAK,SAAS,EAAE;YACpD,IAAI,CAAC,YAAY,GAAG,IAAA,iCAAqB,EACvC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAClC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,WAAW,CAAC,CAAC;SACvC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE;YACZ,MAAM,cAAc,GAAG,IAAA,6BAAY,EAAC,IAAI,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC;YAEjE,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;YAEpE,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAChC,oBAAoB,EACpB,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,gBAAgB,EAAE,EAAE,mBAAmB,CAAC,CACrE,CAAC;SACH;QACD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAE7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;YACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE;YACnC,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;SAChE;QAED,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9B,4DAA4D;QAC5D,iHAAiH;QACjH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,CAAC;QAE5D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAEzD,IAAI,YAAY,EAAE;YAChB,IAAI,CAAC,iBAAiB,CAAC,wBAAwB,CAC7C,QAAQ,CAAC,SAAS,CAAC,WAAW,CAC/B,CAAC;SACH;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,wBAAwB;QACtB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAGD,eAAe;QACb,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE;YACvC,MAAM,KAAK,GAAG,IAAA,4BAAgB,EAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAErE,IAAI,IAAI,CAAC,cAAc,KAAK,KAAK,EAAE;gBACjC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;aAC7B;SACF;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC9D,OAAO;SACR;QACD,IAAI;YACF,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,aAAa,sBAAsB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAC3D,sBAAsB,CACvB,CAAC;YACF,IAAI,IAAI,CAAC,qBAAqB,KAAK,sBAAsB,EAAE;gBACzD,IAAI,CAAC,qBAAqB,GAAG,sBAAsB,CAAC;gBACpD,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;gBACtE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;wBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;4BAC/C,MAAM,EAAE,IAAI,CAAC,qBAAqB;yBACnC,CAAC,CAAC;qBACJ;iBACF;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,2CAA2C,CAAC,CAAC;SAC7D;IACH,CAAC;IAGK,AAAN,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,OAAO;SACR;QACD,IAAI;YACF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC9D,MAAM,CAAC,KAAK,CAAC,QAAQ,iBAAiB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,IAAI,CAAC,gBAAgB,KAAK,iBAAiB,EAAE;gBAC/C,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC;gBAC1C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,SAAS,EAAE;oBAC7C,MAAM,EAAE,IAAI,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;gBAEH,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;oBACrC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,WAAW,EAAE;wBAC/C,MAAM,EAAE,IAAI,CAAC,gBAAgB;qBAC9B,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,sCAAsC,CAAC,CAAC;SACxD;IACH,CAAC;IACO,KAAK,CAAC,SAAS,CAAC,eAAuB;QAC7C,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAClD,CAAC;IAED,UAAU;QACR,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;YACzC,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;gBAClB,SAAS;aACV;YACD,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IACE,OAAO,CAAC,IAAI,KAAK,qCAAmB,CAAC,KAAK;oBAC1C,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,MAAM,CAAC,MAAM,EACrB;oBACA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;iBACrC;aACF;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,eAAe,CAAC,WAAmB,EAAE,SAAiB;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,WAAW,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;YAC5C,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;gBACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB;SACF;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,uBAAuB,CAAC,gBAAwB;QAC9C,OAAO,IAAI,CAAC,eAAe,CACzB,gBAAgB,EAChB,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACxD,gBAAgB,CACnB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,eAAuB;QAC/C,IAAI,gBAAwB,CAAC;QAC7B,IAAI,eAAuB,CAAC;QAC5B,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,CACxB,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC7D,CAAC;QAEF,MAAM,mBAAmB,GAAG,GAAW,EAAE;YACvC,OAAO,IAAI,CAAC,eAAe,CAAC,oBAAoB;gBAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,GAAG,CAAC;gBAC/C,CAAC,CAAC,eAAe,CAAC;QACtB,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,GAAG,mBAAmB,EAAE,EAAE;YAC9D,MAAM,CAAC,IAAI,CACT,2BACE,IAAI,CAAC,iBAAiB,CAAC,WACzB,8BAA8B,mBAAmB,EAAE,+BAA+B,CACnF,CAAC;SACH;QAED,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE;YACvB,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;YAEzC,eAAe,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAC3D,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC,CAC5D,CAAC;YACF,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB;gBACpD,CAAC,CAAC,IAAI,CAAC,gBAAgB;gBACvB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC;YAE/B,IACE,IAAI,CAAC,eAAe,CAAC,QAAQ,GAAG,eAAe;gBAC/C,gBAAgB,GAAG,YAAY,EAC/B;gBACA,MAAM,IAAA,iBAAK,EAAC,CAAC,CAAC,CAAC;gBACf,SAAS;aACV;YAED,IACE,IAAI,CAAC,aAAa;gBAClB,gBAAgB,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,EACtD;gBACA,MAAM,aAAa,GAAG,gBAAgB,GAAG,yBAAyB,CAAC;gBACnE,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CACvC,gBAAgB,EAChB,aAAa,CACd,CAAC;gBAEF,IAAI;oBACF,MAAM,UAAU,GACd,MAAM,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,CAClD,gBAAgB,EAChB,aAAa,EACb,eAAe,CAChB,CAAC;oBAEJ,IAAI,gBAAgB,KAAK,mBAAmB,EAAE,EAAE;wBAC9C,MAAM,CAAC,KAAK,CACV,gEAAgE,CACjE,CAAC;wBACF,SAAS;qBACV;oBAED,IACE,UAAU;wBACV,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,gBAAgB,CAAC,EACvD;wBACA,IAAI,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;wBAEjC,WAAW,GAAG,WAAW;6BACtB,MAAM,CAAC,YAAY,CAAC;6BACpB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBACzB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;4BAC5B,iFAAiF;4BACjF,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,EAAE,EACF,IAAI,CAAC,GAAG,CACN,aAAa,GAAG,CAAC,EACjB,UAAU,CAAC,SAAS,CAAC,mBAAmB,CACzC,CACF,CAAC;yBACH;6BAAM;4BACL,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,WAAW,CAAC,MAAM,EAClB,IAAI,CAAC,eAAe,CAAC,QAAQ,CAC9B,CAAC;4BACF,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;4BAC3D,MAAM,kBAAkB,GACtB,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;4BAE3C,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;yBACH;wBACD,SAAS,CAAC,4BAA4B;qBACvC;oBACD,iCAAiC;iBAClC;gBAAC,OAAO,CAAC,EAAE;oBACV,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACvD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;iBACrD;aACF;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CACvC,gBAAgB,EAChB,eAAe,CAChB,CAAC;YAEF,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE;gBACnE,MAAM,eAAe,GAAG,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,CAAC,CAAC;gBACvE,MAAM,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBACpE,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;aACH;iBAAM;gBACL,MAAM,eAAe,GAAG,IAAA,cAAK,EAAC,gBAAgB,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;gBAC/D,MAAM,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;gBACpE,IAAI,CAAC,eAAe,CAAC,aAAa,CAChC,kBAAkB,EAClB,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAChE,CAAC;aACH;SACF;IACH,CAAC;IAEO,qBAAqB,CAC3B,kBAA4B,EAC5B,cAAwB;QAExB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,EAAE,GAAG,cAAc,CAAC,CAAC;IAC5D,CAAC;IACO,kBAAkB,CAAC,kBAA4B;QACrD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,kBAAkB,EAAE;YACpD,OAAO,kBAAkB,CAAC;SAC3B;QAED,MAAM,YAAY,GAAG,IAAA,8BAAkB,EACrC,IAAI,CAAC,YAAY,EACjB,kBAAkB,CACnB,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAC7C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,CAC3C,CAAC;QACF,IAAI,cAAc,CAAC,MAAM,EAAE;YACzB,MAAM,CAAC,IAAI,CAAC,qBAAqB,cAAc,EAAE,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,YAAY,GAAG,IAAA,gBAAO,EAAC,IAAI,CAAC,YAAY,EAAE,GAAG,cAAc,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC;IACtB,CAAC;IAEO,kBAAkB,CACxB,gBAAwB,EACxB,eAAuB;QAEvB,IAAI,cAAc,GAAG,gBAAgB,GAAG,eAAe,GAAG,CAAC,CAAC;QAE5D,IAAI,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE;YAC/C,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;gBACrC,IAAI,cAAc,IAAI,IAAI,CAAC,gBAAgB,EAAE;oBAC3C,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC;iBACxC;aACF;iBAAM;gBACL,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;aAC7C;SACF;QACD,OAAO,cAAc,CAAC;IACxB,CAAC;IAEO,oBAAoB,CAC1B,UAAmC,EACnC,gBAAyB;QAEzB,IAAI,UAAU,KAAK,SAAS,EAAE;YAC5B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;YAE3C,IACE,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE;gBAClD,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EACpD;gBACA,MAAM,CAAC,KAAK,CACV,+KAA+K,CAChL,CAAC;gBACF,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;gBACtC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,eAAe,EAAE;oBACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;iBAClC,CAAC,CAAC;gBACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;gBACpD,OAAO,KAAK,CAAC;aACd;YAED,IAAI,gBAAgB,KAAK,SAAS,EAAE;gBAClC,IAAI,QAAQ,CAAC,mBAAmB,GAAG,gBAAgB,EAAE;oBACnD,MAAM,CAAC,IAAI,CACT,kEAAkE,CACnE,CAAC;oBACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,cAAc,CAAC,CAAC;oBACpD,OAAO,KAAK,CAAC;iBACd;aACF;YACD,OAAO,IAAI,CAAC;SACb;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACrC,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAAC,WAAmB;QAClD,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAC;QAC7C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;CACF,CAAA;AAvUC;IAAC,IAAA,mBAAQ,EAAC,qBAAqB,CAAC;;;;mDAS/B;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,GAAG,IAAI,CAAC;;;;yDA0BpC;AAGK;IADL,IAAA,mBAAQ,EAAC,mBAAmB,GAAG,IAAI,CAAC;;;;oDAwBpC;AArOU,YAAY;IADxB,IAAA,mBAAU,GAAE;IAcR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;IAC1B,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAHP,sBAAU;QACV,sBAAU;QACe,iCAAe,UAEjC,sCAAiB;QAClB,qCAAgB;QACR,oDAAwB;QACpC,6BAAa;QACR,4BAAiB;GAnBnC,YAAY,CA6exB;AA7eY,oCAAY","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { Interval, SchedulerRegistry } from '@nestjs/schedule';\nimport {\n  isCustomDs,\n  EthereumHandlerKind,\n  SubqlHandlerFilter,\n} from '@subql/common-ethereum';\nimport {\n  ApiService,\n  Dictionary,\n  checkMemoryUsage,\n  delay,\n  getLogger,\n  IndexerEvent,\n  NodeConfig,\n  transformBypassBlocks,\n  cleanedBatchBlocks,\n} from '@subql/node-core';\nimport {\n  DictionaryQueryEntry,\n  ApiWrapper,\n  EthereumLogFilter,\n  EthereumTransactionFilter,\n} from '@subql/types-ethereum';\nimport { MetaData } from '@subql/utils';\nimport { range, sortBy, uniqBy, without } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { calcInterval } from '../ethereum/utils.ethereum';\nimport { eventToTopic, functionToSighash } from '../utils/string';\nimport { IBlockDispatcher } from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('fetch');\nlet BLOCK_TIME_VARIANCE = 5000;\nconst DICTIONARY_MAX_QUERY_SIZE = 10000;\nconst CHECK_MEMORY_INTERVAL = 60000;\nconst MINIMUM_BATCH_SIZE = 5;\nconst INTERVAL_PERCENT = 0.9;\n\nfunction eventFilterToQueryEntry(\n  filter: EthereumLogFilter,\n): DictionaryQueryEntry {\n  const conditions = [];\n  if (filter.address) {\n    conditions.push({\n      field: 'address',\n      value: filter.address.toLowerCase(),\n    });\n  }\n  if (filter.topics) {\n    for (let i = 0; i < Math.min(filter.topics.length, 4); i++) {\n      const topic = filter.topics[i];\n      if (!topic) {\n        continue;\n      }\n      const field = `topics${i}`;\n      conditions.push({ field, value: eventToTopic(topic) });\n    }\n  }\n  return {\n    entity: 'evmLogs',\n    conditions,\n  };\n}\n\nfunction callFilterToQueryEntry(\n  filter: EthereumTransactionFilter,\n): DictionaryQueryEntry {\n  const conditions = [];\n  if (filter.from) {\n    conditions.push({\n      field: 'from',\n      value: filter.from.toLowerCase(),\n    });\n  }\n  if (filter.to) {\n    conditions.push({\n      field: 'to',\n      value: filter.to.toLowerCase(),\n    });\n  }\n  if (filter.function) {\n    conditions.push({\n      field: 'func',\n      value: functionToSighash(filter.function),\n    });\n  }\n  return {\n    entity: 'evmTransactions',\n    conditions,\n  };\n}\n\n@Injectable()\nexport class FetchService implements OnApplicationShutdown {\n  private latestBestHeight: number;\n  private latestFinalizedHeight: number;\n  private isShutdown = false;\n  private batchSizeScale: number;\n  private templateDynamicDatasouces: SubqlProjectDs[];\n  private dictionaryGenesisMatches = true;\n  private evmChainId: string;\n  private bypassBlocks: number[] = [];\n\n  constructor(\n    private apiService: ApiService,\n    private nodeConfig: NodeConfig,\n    @Inject('ISubqueryProject') private project: SubqueryProject,\n    @Inject('IBlockDispatcher') private blockDispatcher: IBlockDispatcher,\n    private dictionaryService: DictionaryService,\n    private dynamicDsService: DynamicDsService,\n    private unfinalizedBlocksService: UnfinalizedBlocksService,\n    private eventEmitter: EventEmitter2,\n    private schedulerRegistry: SchedulerRegistry,\n  ) {\n    this.batchSizeScale = 1;\n  }\n\n  onApplicationShutdown(): void {\n    this.isShutdown = true;\n  }\n\n  get api(): ApiWrapper {\n    return this.apiService.api;\n  }\n\n  async syncDynamicDatascourcesFromMeta(): Promise<void> {\n    this.templateDynamicDatasouces =\n      await this.dynamicDsService.getDynamicDatasources();\n  }\n\n  buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[] {\n    const queryEntries: DictionaryQueryEntry[] = [];\n\n    // Only run the ds that is equal or less than startBlock\n    // sort array from lowest ds.startBlock to highest\n    const filteredDs = this.project.dataSources\n      .concat(this.templateDynamicDatasouces)\n      .filter((ds) => ds.startBlock <= startBlock)\n      .sort((a, b) => a.startBlock - b.startBlock);\n\n    for (const ds of filteredDs) {\n      for (const handler of ds.mapping.handlers) {\n        let filterList: SubqlHandlerFilter[];\n        filterList = [handler.filter];\n        filterList = filterList.filter((f) => f);\n        if (!filterList.length) return [];\n        switch (handler.kind) {\n          case EthereumHandlerKind.Block:\n            return [];\n          case EthereumHandlerKind.Call: {\n            for (const filter of filterList as EthereumTransactionFilter[]) {\n              if (\n                filter.from !== undefined ||\n                filter.to !== undefined ||\n                filter.function\n              ) {\n                queryEntries.push(callFilterToQueryEntry(filter));\n              } else {\n                return [];\n              }\n            }\n            break;\n          }\n          case EthereumHandlerKind.Event: {\n            for (const filter of filterList as EthereumLogFilter[]) {\n              if (filter.address || filter.topics) {\n                queryEntries.push(eventFilterToQueryEntry(filter));\n              } else {\n                return [];\n              }\n            }\n            break;\n          }\n          default:\n        }\n      }\n    }\n\n    return uniqBy(\n      queryEntries,\n      (item) =>\n        `${item.entity}|${JSON.stringify(\n          sortBy(item.conditions, (c) => c.field),\n        )}`,\n    );\n  }\n\n  updateDictionary(): void {\n    this.dictionaryService.buildDictionaryEntryMap<SubqlProjectDs>(\n      this.project.dataSources.concat(this.templateDynamicDatasouces),\n      this.buildDictionaryQueryEntries.bind(this),\n    );\n  }\n\n  private get useDictionary(): boolean {\n    return (\n      !!this.project.network.dictionary &&\n      this.dictionaryGenesisMatches &&\n      !!this.dictionaryService.getDictionaryQueryEntries(\n        this.blockDispatcher.latestBufferedHeight ??\n          Math.min(...this.project.dataSources.map((ds) => ds.startBlock)),\n      ).length\n    );\n  }\n\n  async init(startHeight: number): Promise<void> {\n    if (this.project.network?.bypassBlocks !== undefined) {\n      this.bypassBlocks = transformBypassBlocks(\n        this.project.network.bypassBlocks,\n      ).filter((blk) => blk >= startHeight);\n    }\n    if (this.api) {\n      const CHAIN_INTERVAL = calcInterval(this.api) * INTERVAL_PERCENT;\n\n      BLOCK_TIME_VARIANCE = Math.min(BLOCK_TIME_VARIANCE, CHAIN_INTERVAL);\n\n      this.schedulerRegistry.addInterval(\n        'getLatestBlockHead',\n        setInterval(() => void this.getBestBlockHead(), BLOCK_TIME_VARIANCE),\n      );\n    }\n    await this.syncDynamicDatascourcesFromMeta();\n\n    this.updateDictionary();\n    this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n      value: Number(this.useDictionary),\n    });\n\n    if (this.project.network.dictionary) {\n      this.evmChainId = await this.dictionaryService.getEvmChainId();\n    }\n\n    await this.getFinalizedBlockHead();\n    await this.getBestBlockHead();\n\n    //  Call metadata here, other network should align with this\n    //  For substrate, we might use the specVersion metadata in future if we have same error handling as in node-core\n    const metadata = await this.dictionaryService.getMetadata();\n\n    const validChecker = this.dictionaryValidation(metadata);\n\n    if (validChecker) {\n      this.dictionaryService.setDictionaryStartHeight(\n        metadata._metadata.startHeight,\n      );\n    }\n\n    await this.blockDispatcher.init(this.resetForNewDs.bind(this));\n    void this.startLoop(startHeight);\n  }\n\n  getUseDictionary(): boolean {\n    return this.useDictionary;\n  }\n\n  getLatestFinalizedHeight(): number {\n    return this.latestFinalizedHeight;\n  }\n\n  @Interval(CHECK_MEMORY_INTERVAL)\n  checkBatchScale(): void {\n    if (this.nodeConfig['scale-batch-size']) {\n      const scale = checkMemoryUsage(this.batchSizeScale, this.nodeConfig);\n\n      if (this.batchSizeScale !== scale) {\n        this.batchSizeScale = scale;\n      }\n    }\n  }\n\n  @Interval(BLOCK_TIME_VARIANCE * 1000)\n  async getFinalizedBlockHead(): Promise<void> {\n    if (!this.api) {\n      logger.debug(`Skip fetch finalized block until API is ready`);\n      return;\n    }\n    try {\n      const currentFinalizedHeight = await this.api.getFinalizedBlockHeight();\n      logger.debug(`finalized:${currentFinalizedHeight.toString()}`);\n      const finalizedHeader = await this.api.getBlockByHeightOrHash(\n        currentFinalizedHeight,\n      );\n      if (this.latestFinalizedHeight !== currentFinalizedHeight) {\n        this.latestFinalizedHeight = currentFinalizedHeight;\n        this.unfinalizedBlocksService.registerFinalizedBlock(finalizedHeader);\n        if (!this.nodeConfig.unfinalizedBlocks) {\n          if (!this.nodeConfig.unfinalizedBlocks) {\n            this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n              height: this.latestFinalizedHeight,\n            });\n          }\n        }\n      }\n    } catch (e) {\n      logger.warn(e, `Having a problem when get finalized block`);\n    }\n  }\n\n  @Interval(BLOCK_TIME_VARIANCE * 1000)\n  async getBestBlockHead(): Promise<void> {\n    if (!this.api) {\n      logger.debug(`Skip fetch best block until API is ready`);\n      return;\n    }\n    try {\n      const currentBestHeight = await this.api.getBestBlockHeight();\n      logger.debug(`best:${currentBestHeight.toString()}`);\n      if (this.latestBestHeight !== currentBestHeight) {\n        this.latestBestHeight = currentBestHeight;\n        this.eventEmitter.emit(IndexerEvent.BlockBest, {\n          height: this.latestBestHeight,\n        });\n\n        if (this.nodeConfig.unfinalizedBlocks) {\n          this.eventEmitter.emit(IndexerEvent.BlockTarget, {\n            height: this.latestBestHeight,\n          });\n        }\n      }\n    } catch (e) {\n      logger.warn(e, `Having a problem when get best block`);\n    }\n  }\n  private async startLoop(initBlockHeight: number): Promise<void> {\n    await this.fillNextBlockBuffer(initBlockHeight);\n  }\n\n  getModulos(): number[] {\n    const modulos: number[] = [];\n    for (const ds of this.project.dataSources) {\n      if (isCustomDs(ds)) {\n        continue;\n      }\n      for (const handler of ds.mapping.handlers) {\n        if (\n          handler.kind === EthereumHandlerKind.Block &&\n          handler.filter &&\n          handler.filter.modulo\n        ) {\n          modulos.push(handler.filter.modulo);\n        }\n      }\n    }\n    return modulos;\n  }\n\n  getModuloBlocks(startHeight: number, endHeight: number): number[] {\n    const modulos = this.getModulos();\n    const moduloBlocks: number[] = [];\n    for (let i = startHeight; i < endHeight; i++) {\n      if (modulos.find((m) => i % m === 0)) {\n        moduloBlocks.push(i);\n      }\n    }\n    return moduloBlocks;\n  }\n\n  getEnqueuedModuloBlocks(startBlockHeight: number): number[] {\n    return this.getModuloBlocks(\n      startBlockHeight,\n      this.nodeConfig.batchSize * Math.max(...this.getModulos()) +\n        startBlockHeight,\n    ).slice(0, this.nodeConfig.batchSize);\n  }\n\n  async fillNextBlockBuffer(initBlockHeight: number): Promise<void> {\n    let startBlockHeight: number;\n    let scaledBatchSize: number;\n    const handlers = [].concat(\n      ...this.project.dataSources.map((ds) => ds.mapping.handlers),\n    );\n\n    const getStartBlockHeight = (): number => {\n      return this.blockDispatcher.latestBufferedHeight\n        ? this.blockDispatcher.latestBufferedHeight + 1\n        : initBlockHeight;\n    };\n\n    if (this.dictionaryService.startHeight > getStartBlockHeight()) {\n      logger.warn(\n        `Dictionary start height ${\n          this.dictionaryService.startHeight\n        } is beyond indexing height ${getStartBlockHeight()}, skipping dictionary for now`,\n      );\n    }\n\n    while (!this.isShutdown) {\n      startBlockHeight = getStartBlockHeight();\n\n      scaledBatchSize = Math.max(\n        Math.round(this.batchSizeScale * this.nodeConfig.batchSize),\n        Math.min(MINIMUM_BATCH_SIZE, this.nodeConfig.batchSize * 3),\n      );\n      const latestHeight = this.nodeConfig.unfinalizedBlocks\n        ? this.latestBestHeight\n        : this.latestFinalizedHeight;\n\n      if (\n        this.blockDispatcher.freeSize < scaledBatchSize ||\n        startBlockHeight > latestHeight\n      ) {\n        await delay(1);\n        continue;\n      }\n\n      if (\n        this.useDictionary &&\n        startBlockHeight >= this.dictionaryService.startHeight\n      ) {\n        const queryEndBlock = startBlockHeight + DICTIONARY_MAX_QUERY_SIZE;\n        const moduloBlocks = this.getModuloBlocks(\n          startBlockHeight,\n          queryEndBlock,\n        );\n\n        try {\n          const dictionary =\n            await this.dictionaryService.scopedDictionaryEntries(\n              startBlockHeight,\n              queryEndBlock,\n              scaledBatchSize,\n            );\n\n          if (startBlockHeight !== getStartBlockHeight()) {\n            logger.debug(\n              `Queue was reset for new DS, discarding dictionary query result`,\n            );\n            continue;\n          }\n\n          if (\n            dictionary &&\n            this.dictionaryValidation(dictionary, startBlockHeight)\n          ) {\n            let { batchBlocks } = dictionary;\n\n            batchBlocks = batchBlocks\n              .concat(moduloBlocks)\n              .sort((a, b) => a - b);\n            if (batchBlocks.length === 0) {\n              // There we're no blocks in this query range, we can set a new height we're up to\n              this.blockDispatcher.enqueueBlocks(\n                [],\n                Math.min(\n                  queryEndBlock - 1,\n                  dictionary._metadata.lastProcessedHeight,\n                ),\n              );\n            } else {\n              const maxBlockSize = Math.min(\n                batchBlocks.length,\n                this.blockDispatcher.freeSize,\n              );\n              const enqueuingBlocks = batchBlocks.slice(0, maxBlockSize);\n              const cleanedBatchBlocks =\n                this.filteredBlockBatch(enqueuingBlocks);\n\n              this.blockDispatcher.enqueueBlocks(\n                cleanedBatchBlocks,\n                this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n              );\n            }\n            continue; // skip nextBlockRange() way\n          }\n          // else use this.nextBlockRange()\n        } catch (e) {\n          logger.debug(`Fetch dictionary stopped: ${e.message}`);\n          this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n        }\n      }\n\n      const endHeight = this.nextEndBlockHeight(\n        startBlockHeight,\n        scaledBatchSize,\n      );\n\n      if (handlers.length && this.getModulos().length === handlers.length) {\n        const enqueuingBlocks = this.getEnqueuedModuloBlocks(startBlockHeight);\n        const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);\n        this.blockDispatcher.enqueueBlocks(\n          cleanedBatchBlocks,\n          this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n        );\n      } else {\n        const enqueuingBlocks = range(startBlockHeight, endHeight + 1);\n        const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks);\n        this.blockDispatcher.enqueueBlocks(\n          cleanedBatchBlocks,\n          this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks),\n        );\n      }\n    }\n  }\n\n  private getLatestBufferHeight(\n    cleanedBatchBlocks: number[],\n    rawBatchBlocks: number[],\n  ): number {\n    return Math.max(...cleanedBatchBlocks, ...rawBatchBlocks);\n  }\n  private filteredBlockBatch(currentBatchBlocks: number[]): number[] {\n    if (!this.bypassBlocks.length || !currentBatchBlocks) {\n      return currentBatchBlocks;\n    }\n\n    const cleanedBatch = cleanedBatchBlocks(\n      this.bypassBlocks,\n      currentBatchBlocks,\n    );\n\n    const pollutedBlocks = this.bypassBlocks.filter(\n      (b) => b < Math.max(...currentBatchBlocks),\n    );\n    if (pollutedBlocks.length) {\n      logger.info(`Bypassing blocks: ${pollutedBlocks}`);\n    }\n    this.bypassBlocks = without(this.bypassBlocks, ...pollutedBlocks);\n    return cleanedBatch;\n  }\n\n  private nextEndBlockHeight(\n    startBlockHeight: number,\n    scaledBatchSize: number,\n  ): number {\n    let endBlockHeight = startBlockHeight + scaledBatchSize - 1;\n\n    if (endBlockHeight > this.latestFinalizedHeight) {\n      if (this.nodeConfig.unfinalizedBlocks) {\n        if (endBlockHeight >= this.latestBestHeight) {\n          endBlockHeight = this.latestBestHeight;\n        }\n      } else {\n        endBlockHeight = this.latestFinalizedHeight;\n      }\n    }\n    return endBlockHeight;\n  }\n\n  private dictionaryValidation(\n    dictionary: { _metadata: MetaData },\n    startBlockHeight?: number,\n  ): boolean {\n    if (dictionary !== undefined) {\n      const { _metadata: metaData } = dictionary;\n\n      if (\n        metaData.genesisHash !== this.api.getGenesisHash() &&\n        this.evmChainId !== this.api.getChainId().toString()\n      ) {\n        logger.error(\n          'The dictionary that you have specified does not match the chain you are indexing, it will be ignored. Please update your project manifest to reference the correct dictionary',\n        );\n        this.dictionaryGenesisMatches = false;\n        this.eventEmitter.emit(IndexerEvent.UsingDictionary, {\n          value: Number(this.useDictionary),\n        });\n        this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n        return false;\n      }\n\n      if (startBlockHeight !== undefined) {\n        if (metaData.lastProcessedHeight < startBlockHeight) {\n          logger.warn(\n            `Dictionary indexed block is behind current indexing block height`,\n          );\n          this.eventEmitter.emit(IndexerEvent.SkipDictionary);\n          return false;\n        }\n      }\n      return true;\n    }\n    return false;\n  }\n\n  async resetForNewDs(blockHeight: number): Promise<void> {\n    await this.syncDynamicDatascourcesFromMeta();\n    this.dynamicDsService.deleteTempDsRecords(blockHeight);\n    this.updateDictionary();\n    this.blockDispatcher.flushQueue(blockHeight);\n  }\n\n  async resetForIncorrectBestBlock(blockHeight: number): Promise<void> {\n    await this.syncDynamicDatascourcesFromMeta();\n    this.updateDictionary();\n    this.blockDispatcher.flushQueue(blockHeight);\n  }\n}\n"]}

package/dist/indexer/indexer.manager.d.ts

@@ -12,7 +12,7 @@ export declare class IndexerManager {
     private apiService;
     private poiService;
     private sequelize;
-    private project;
+    private readonly project;
     private nodeConfig;
     private sandboxService;
     private dynamicDsService;

package/dist/indexer/indexer.manager.js

@@ -10,6 +10,9 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.IndexerManager = void 0;
 const common_1 = require("@nestjs/common");
@@ -250,6 +253,7 @@ __decorate([
 ], IndexerManager.prototype, "indexBlock", null);
 IndexerManager = __decorate([
     (0, common_1.Injectable)(),
+    __param(4, (0, common_1.Inject)('ISubqueryProject')),
     __metadata("design:paramtypes", [node_core_1.StoreService,
         node_core_1.ApiService,
         node_core_1.PoiService,