@subql/node-ethereum 1.10.1-2 → 2.0.0

package/dist/indexer/dictionary.service.js

@@ -10,20 +10,40 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DictionaryService = void 0;
+const core_1 = require("@apollo/client/core");
 const common_1 = require("@nestjs/common");
 const node_core_1 = require("@subql/node-core");
 const SubqueryProject_1 = require("../configure/SubqueryProject");
+const logger = (0, node_core_1.getLogger)('dictionary');
 let DictionaryService = class DictionaryService extends node_core_1.DictionaryService {
     constructor(project, nodeConfig) {
-        super(project.network.dictionary, nodeConfig);
+        super(project.network.dictionary, project.network.chainId, nodeConfig);
         this.project = project;
     }
+    async getEvmChainId() {
+        const query = `query{chainAlias(id: "evmChainId"){value}}`;
+        try {
+            const resp = await (0, node_core_1.timeout)(this.client.query({
+                query: (0, core_1.gql)(query),
+            }), this.nodeConfig.dictionaryTimeout);
+            return resp.data.chainAlias.value;
+        }
+        catch (e) {
+            logger.debug(e, `failed to fetch evm chainId from dictionary`);
+            return undefined;
+        }
+    }
 };
 DictionaryService = __decorate([
     (0, common_1.Injectable)(),
-    __metadata("design:paramtypes", [SubqueryProject_1.SubqueryProject, node_core_1.NodeConfig])
+    __param(0, (0, common_1.Inject)('ISubqueryProject')),
+    __metadata("design:paramtypes", [SubqueryProject_1.SubqueryProject,
+        node_core_1.NodeConfig])
 ], DictionaryService);
 exports.DictionaryService = DictionaryService;
 //# sourceMappingURL=dictionary.service.js.map

package/dist/indexer/dictionary.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"dictionary.service.js","sourceRoot":"","sources":["../../src/indexer/dictionary.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;AAEtC,2CAAmE;AACnE,gDAG0B;AAC1B,kEAA+D;AAGxD,IAAM,iBAAiB,GAAvB,MAAM,iBACX,SAAQ,6BAAqB;IAG7B,YAAsB,OAAwB,EAAE,UAAsB;QACpE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAD1B,YAAO,GAAP,OAAO,CAAiB;IAE9C,CAAC;CACF,CAAA;AAPY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;qCAKoB,iCAAe,EAAc,sBAAU;GAJ3D,iBAAiB,CAO7B;AAPY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport {\n  NodeConfig,\n  DictionaryService as CoreDictionaryService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\n@Injectable()\nexport class DictionaryService\n  extends CoreDictionaryService\n  implements OnApplicationShutdown\n{\n  constructor(protected project: SubqueryProject, nodeConfig: NodeConfig) {\n    super(project.network.dictionary, nodeConfig);\n  }\n}\n"]}
+{"version":3,"file":"dictionary.service.js","sourceRoot":"","sources":["../../src/indexer/dictionary.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,8CAA0C;AAC1C,2CAA2E;AAC3E,gDAK0B;AAC1B,kEAA+D;AAE/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAGhC,IAAM,iBAAiB,GAAvB,MAAM,iBACX,SAAQ,6BAAqB;IAG7B,YACwC,OAAwB,EAC9D,UAAsB;QAEtB,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAHjC,YAAO,GAAP,OAAO,CAAiB;IAIhE,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,MAAM,KAAK,GAAG,4CAA4C,CAAC;QAE3D,IAAI;YACF,MAAM,IAAI,GAAG,MAAM,IAAA,mBAAO,EACxB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBAChB,KAAK,EAAE,IAAA,UAAG,EAAC,KAAK,CAAC;aAClB,CAAC,EACF,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAClC,CAAC;YACF,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;SACnC;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,6CAA6C,CAAC,CAAC;YAC/D,OAAO,SAAS,CAAC;SAClB;IACH,CAAC;CACF,CAAA;AA3BY,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAMR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAAoB,iCAAe;QAClD,sBAAU;GANb,iBAAiB,CA2B7B;AA3BY,8CAAiB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { gql } from '@apollo/client/core';\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport {\n  NodeConfig,\n  DictionaryService as CoreDictionaryService,\n  timeout,\n  getLogger,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nconst logger = getLogger('dictionary');\n\n@Injectable()\nexport class DictionaryService\n  extends CoreDictionaryService\n  implements OnApplicationShutdown\n{\n  constructor(\n    @Inject('ISubqueryProject') protected project: SubqueryProject,\n    nodeConfig: NodeConfig,\n  ) {\n    super(project.network.dictionary, project.network.chainId, nodeConfig);\n  }\n\n  async getEvmChainId(): Promise<string> {\n    const query = `query{chainAlias(id: \"evmChainId\"){value}}`;\n\n    try {\n      const resp = await timeout(\n        this.client.query({\n          query: gql(query),\n        }),\n        this.nodeConfig.dictionaryTimeout,\n      );\n      return resp.data.chainAlias.value;\n    } catch (e) {\n      logger.debug(e, `failed to fetch evm chainId from dictionary`);\n      return undefined;\n    }\n  }\n}\n"]}

package/dist/indexer/ds-processor.service.d.ts

@@ -15,7 +15,7 @@ export declare class DsPluginSandbox extends Sandbox {
     getDsPlugin<D extends string>(): SubqlDatasourceProcessor<D, unknown>;
 }
 export declare class DsProcessorService {
-    private project;
+    private readonly project;
     private readonly nodeConfig;
     private processorCache;
     constructor(project: SubqueryProject, nodeConfig: NodeConfig);

package/dist/indexer/ds-processor.service.js

@@ -10,6 +10,9 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -126,6 +129,7 @@ let DsProcessorService = class DsProcessorService {
 };
 DsProcessorService = __decorate([
     (0, common_1.Injectable)(),
+    __param(0, (0, common_1.Inject)('ISubqueryProject')),
     __metadata("design:paramtypes", [SubqueryProject_1.SubqueryProject,
         node_core_1.NodeConfig])
 ], DsProcessorService);

package/dist/indexer/ds-processor.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"ds-processor.service.js","sourceRoot":"","sources":["../../src/indexer/ds-processor.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,gDAAwB;AACxB,2CAA4C;AAC5C,4DAMgC;AAChC,gDAAkE;AAOlE,6BAA+B;AAC/B,kEAA+D;AAQ/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,4GAA4G;IAC5G,OAAO,SAAS,CAAC,WAAW,KAAK,SAAS,CAAC;AAC7C,CAAC;AAZD,kFAYC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,OAAO,SAAS,CAAC,WAAW,KAAK,OAAO,CAAC;AAC3C,CAAC;AAXD,kFAWC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,IAAI,mCAAmC,CAAC,SAAS,CAAC,EAAE;QAClD,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,mCAAmC,CAAC,SAAS,CAAC,EAAE;QACnD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;KACrD;IAED,uCACK,SAAS,KACZ,WAAW,EAAE,OAAO,EACpB,eAAe,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1B,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EACnE,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CACtB,SAAS;aACN,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC;aAC/D,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IACzB;AACJ,CAAC;AA5BD,kFA4BC;AAED,MAAa,eAAgB,SAAQ,mBAAO;IAC1C,YAAY,MAA6B,EAAE,UAAsB;QAC/D,KAAK,CACH,MAAM,EACN,IAAI,cAAQ,CACV,6BAA6B,MAAM,CAAC,KAAK,aAAa,EACtD,cAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CACrC,EACD,UAAU,CACX,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACF;AAhBD,0CAgBC;AAGM,IAAM,kBAAkB,GAAxB,MAAM,kBAAkB;IAI7B,YACU,OAAwB,EACf,UAAsB;QAD/B,YAAO,GAAP,OAAO,CAAiB;QACf,eAAU,GAAV,UAAU,CAAY;QALjC,mBAAc,GAElB,EAAE,CAAC;IAIJ,CAAC;IAEJ,KAAK,CAAC,gBAAgB,CACpB,WAA4C;QAE5C,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YAC1C,oEAAoE;YACpE,IAAI,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,EAAE;gBAC9B,MAAM,IAAI,KAAK,CACb,YAAY,EAAE,CAAC,IAAI,6BAA6B,SAAS,CAAC,IAAI,GAAG,CAClE,CAAC;aACH;YAED,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,SAAS,CAAC,iBAAiB,CAAC,EAAE;oBAClD,MAAM,IAAI,KAAK,CACb,WAAW,OAAO,CAAC,IAAI,eAAe,MAAM,CAAC,IAAI,CAC/C,SAAS,CAAC,iBAAiB,CAC5B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACf,CAAC;iBACH;aACF;YAED,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAClC,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,eAAe,CACvD,OAAO,CAAC,MAAM,CACf,CACF,CAAC;YAEF,8CAA8C;YAC9C,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;SAClD;IACH,CAAC;IAED,KAAK,CAAC,gCAAgC;QACpC,MAAM,IAAI,CAAC,gBAAgB,CACxB,IAAI,CAAC,OAAO,CAAC,WAAyC,CAAC,MAAM,CAC5D,4BAAU,CACX,CACF,CAAC;IACJ,CAAC;IAED,cAAc,CACZ,EAAyC;QAEzC,IAAI,CAAC,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;SAC5D;QACD,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;YAC3C,MAAM,OAAO,GAAG,IAAI,eAAe,CACjC;gBACE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvB,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI;gBACxB,MAAM,EACJ,IAAI,CAAC,yDAAyD;aACjE,EACD,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,IAAI;gBACF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAK,CAAC;aACnE;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,MAAM,CAAC,CAAC;aACT;SACF;QACD,OAAO,IAAI,CAAC,cAAc,CACxB,EAAE,CAAC,SAAS,CAAC,IAAI,CACiC,CAAC;IACvD,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,SAAS,CACb,EAAiC;QAEjC,IAAI,CAAC,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;SAC5D;QAED,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YACd,OAAO,EAAE,CAAC;SACX;QAED,MAAM,GAAG,GAA2B,EAAE,CAAC;QAEvC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE;YACxC,8DAA8D;YAC9D,IAAI;gBACF,GAAG,CAAC,IAAI,CAAC,GAAG,YAAE,CAAC,YAAY,CAAC,IAAI,EAAE;oBAChC,QAAQ,EAAE,MAAM;iBACjB,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;gBACxD,MAAM,CAAC,CAAC;aACT;SACF;QAED,OAAO,GAAG,CAAC;IACb,CAAC;CACF,CAAA;AA1GY,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;qCAMQ,iCAAe;QACH,sBAAU;GAN9B,kBAAkB,CA0G9B;AA1GY,gDAAkB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport path from 'path';\nimport { Injectable } from '@nestjs/common';\nimport {\n  EthereumHandlerKind,\n  isCustomDs,\n  SubqlEthereumCustomDataSource,\n  SubqlEthereumDataSource,\n  SubqlDatasourceProcessor,\n} from '@subql/common-ethereum';\nimport { getLogger, NodeConfig, Sandbox } from '@subql/node-core';\nimport {\n  SecondLayerHandlerProcessor_0_0_0,\n  SecondLayerHandlerProcessor_1_0_0,\n  SubqlCustomDatasource,\n} from '@subql/types-ethereum';\n\nimport { VMScript } from 'vm2';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nexport interface DsPluginSandboxOption {\n  root: string;\n  entry: string;\n  script: string;\n}\n\nconst logger = getLogger('ds-sandbox');\n\nexport function isSecondLayerHandlerProcessor_0_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlCustomDatasource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_0_0_0<K, F, E, DS> {\n  // Exisiting datasource processors had no concept of specVersion, therefore undefined is equivalent to 0.0.0\n  return processor.specVersion === undefined;\n}\n\nexport function isSecondLayerHandlerProcessor_1_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlEthereumCustomDataSource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  return processor.specVersion === '1.0.0';\n}\n\nexport function asSecondLayerHandlerProcessor_1_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlEthereumCustomDataSource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  if (isSecondLayerHandlerProcessor_1_0_0(processor)) {\n    return processor;\n  }\n\n  if (!isSecondLayerHandlerProcessor_0_0_0(processor)) {\n    throw new Error('Unsupported ds processor version');\n  }\n\n  return {\n    ...processor,\n    specVersion: '1.0.0',\n    filterProcessor: (params) =>\n      processor.filterProcessor(params.filter, params.input, params.ds),\n    transformer: (params) =>\n      processor\n        .transformer(params.input, params.ds, params.api, params.assets)\n        .then((res) => [res]),\n  };\n}\n\nexport class DsPluginSandbox extends Sandbox {\n  constructor(option: DsPluginSandboxOption, nodeConfig: NodeConfig) {\n    super(\n      option,\n      new VMScript(\n        `module.exports = require('${option.entry}').default;`,\n        path.join(option.root, 'ds_sandbox'),\n      ),\n      nodeConfig,\n    );\n    this.freeze(logger, 'logger');\n  }\n\n  getDsPlugin<D extends string>(): SubqlDatasourceProcessor<D, unknown> {\n    return this.run(this.script);\n  }\n}\n\n@Injectable()\nexport class DsProcessorService {\n  private processorCache: {\n    [entry: string]: SubqlDatasourceProcessor<string, unknown>;\n  } = {};\n  constructor(\n    private project: SubqueryProject,\n    private readonly nodeConfig: NodeConfig,\n  ) {}\n\n  async validateCustomDs(\n    datasources: SubqlEthereumCustomDataSource[],\n  ): Promise<void> {\n    for (const ds of datasources) {\n      const processor = this.getDsProcessor(ds);\n      /* Standard validation applicable to all custom ds and processors */\n      if (ds.kind !== processor.kind) {\n        throw new Error(\n          `ds kind (${ds.kind}) doesnt match processor (${processor.kind})`,\n        );\n      }\n\n      for (const handler of ds.mapping.handlers) {\n        if (!(handler.kind in processor.handlerProcessors)) {\n          throw new Error(\n            `ds kind ${handler.kind} not one of ${Object.keys(\n              processor.handlerProcessors,\n            ).join(', ')}`,\n          );\n        }\n      }\n\n      ds.mapping.handlers.map((handler) =>\n        processor.handlerProcessors[handler.kind].filterValidator(\n          handler.filter,\n        ),\n      );\n\n      /* Additional processor specific validation */\n      processor.validate(ds, await this.getAssets(ds));\n    }\n  }\n\n  async validateProjectCustomDatasources(): Promise<void> {\n    await this.validateCustomDs(\n      (this.project.dataSources as SubqlEthereumDataSource[]).filter(\n        isCustomDs,\n      ),\n    );\n  }\n\n  getDsProcessor<D extends string>(\n    ds: SubqlEthereumCustomDataSource<string>,\n  ): SubqlDatasourceProcessor<D, unknown> {\n    if (!isCustomDs(ds)) {\n      throw new Error(`data source is not a custom data source`);\n    }\n    if (!this.processorCache[ds.processor.file]) {\n      const sandbox = new DsPluginSandbox(\n        {\n          root: this.project.root,\n          entry: ds.processor.file,\n          script:\n            null /* TODO get working with Readers, same as with sandbox */,\n        },\n        this.nodeConfig,\n      );\n      try {\n        this.processorCache[ds.processor.file] = sandbox.getDsPlugin<D>();\n      } catch (e) {\n        logger.error(`not supported ds @${ds.kind}`);\n        throw e;\n      }\n    }\n    return this.processorCache[\n      ds.processor.file\n    ] as unknown as SubqlDatasourceProcessor<D, unknown>;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  async getAssets(\n    ds: SubqlEthereumCustomDataSource,\n  ): Promise<Record<string, string>> {\n    if (!isCustomDs(ds)) {\n      throw new Error(`data source is not a custom data source`);\n    }\n\n    if (!ds.assets) {\n      return {};\n    }\n\n    const res: Record<string, string> = {};\n\n    for (const [name, { file }] of ds.assets) {\n      // TODO update with https://github.com/subquery/subql/pull/511\n      try {\n        res[name] = fs.readFileSync(file, {\n          encoding: 'utf8',\n        });\n      } catch (e) {\n        logger.error(`Failed to load datasource asset ${file}`);\n        throw e;\n      }\n    }\n\n    return res;\n  }\n}\n"]}
+{"version":3,"file":"ds-processor.service.js","sourceRoot":"","sources":["../../src/indexer/ds-processor.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,4CAAoB;AACpB,gDAAwB;AACxB,2CAAoD;AACpD,4DAMgC;AAChC,gDAAkE;AAOlE,6BAA+B;AAC/B,kEAA+D;AAQ/D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,4GAA4G;IAC5G,OAAO,SAAS,CAAC,WAAW,KAAK,SAAS,CAAC;AAC7C,CAAC;AAZD,kFAYC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,OAAO,SAAS,CAAC,WAAW,KAAK,OAAO,CAAC;AAC3C,CAAC;AAXD,kFAWC;AAED,SAAgB,mCAAmC,CAMjD,SAEkD;IAElD,IAAI,mCAAmC,CAAC,SAAS,CAAC,EAAE;QAClD,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,mCAAmC,CAAC,SAAS,CAAC,EAAE;QACnD,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;KACrD;IAED,uCACK,SAAS,KACZ,WAAW,EAAE,OAAO,EACpB,eAAe,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1B,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EACnE,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CACtB,SAAS;aACN,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC;aAC/D,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IACzB;AACJ,CAAC;AA5BD,kFA4BC;AAED,MAAa,eAAgB,SAAQ,mBAAO;IAC1C,YAAY,MAA6B,EAAE,UAAsB;QAC/D,KAAK,CACH,MAAM,EACN,IAAI,cAAQ,CACV,6BAA6B,MAAM,CAAC,KAAK,aAAa,EACtD,cAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CACrC,EACD,UAAU,CACX,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACF;AAhBD,0CAgBC;AAGM,IAAM,kBAAkB,GAAxB,MAAM,kBAAkB;IAI7B,YAC+C,OAAwB,EACpD,UAAsB;QADM,YAAO,GAAP,OAAO,CAAiB;QACpD,eAAU,GAAV,UAAU,CAAY;QALjC,mBAAc,GAElB,EAAE,CAAC;IAIJ,CAAC;IAEJ,KAAK,CAAC,gBAAgB,CACpB,WAA4C;QAE5C,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YAC1C,oEAAoE;YACpE,IAAI,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,EAAE;gBAC9B,MAAM,IAAI,KAAK,CACb,YAAY,EAAE,CAAC,IAAI,6BAA6B,SAAS,CAAC,IAAI,GAAG,CAClE,CAAC;aACH;YAED,KAAK,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACzC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,SAAS,CAAC,iBAAiB,CAAC,EAAE;oBAClD,MAAM,IAAI,KAAK,CACb,WAAW,OAAO,CAAC,IAAI,eAAe,MAAM,CAAC,IAAI,CAC/C,SAAS,CAAC,iBAAiB,CAC5B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACf,CAAC;iBACH;aACF;YAED,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAClC,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,eAAe,CACvD,OAAO,CAAC,MAAM,CACf,CACF,CAAC;YAEF,8CAA8C;YAC9C,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;SAClD;IACH,CAAC;IAED,KAAK,CAAC,gCAAgC;QACpC,MAAM,IAAI,CAAC,gBAAgB,CACxB,IAAI,CAAC,OAAO,CAAC,WAAyC,CAAC,MAAM,CAC5D,4BAAU,CACX,CACF,CAAC;IACJ,CAAC;IAED,cAAc,CACZ,EAAyC;QAEzC,IAAI,CAAC,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;SAC5D;QACD,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;YAC3C,MAAM,OAAO,GAAG,IAAI,eAAe,CACjC;gBACE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvB,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI;gBACxB,MAAM,EACJ,IAAI,CAAC,yDAAyD;aACjE,EACD,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,IAAI;gBACF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAK,CAAC;aACnE;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,MAAM,CAAC,CAAC;aACT;SACF;QACD,OAAO,IAAI,CAAC,cAAc,CACxB,EAAE,CAAC,SAAS,CAAC,IAAI,CACiC,CAAC;IACvD,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,SAAS,CACb,EAAiC;QAEjC,IAAI,CAAC,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;SAC5D;QAED,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YACd,OAAO,EAAE,CAAC;SACX;QAED,MAAM,GAAG,GAA2B,EAAE,CAAC;QAEvC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE;YACxC,8DAA8D;YAC9D,IAAI;gBACF,GAAG,CAAC,IAAI,CAAC,GAAG,YAAE,CAAC,YAAY,CAAC,IAAI,EAAE;oBAChC,QAAQ,EAAE,MAAM;iBACjB,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;gBACxD,MAAM,CAAC,CAAC;aACT;SACF;QAED,OAAO,GAAG,CAAC;IACb,CAAC;CACF,CAAA;AA1GY,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;IAMR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAA2B,iCAAe;QACxC,sBAAU;GAN9B,kBAAkB,CA0G9B;AA1GY,gDAAkB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport path from 'path';\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  EthereumHandlerKind,\n  isCustomDs,\n  SubqlEthereumCustomDataSource,\n  SubqlEthereumDataSource,\n  SubqlDatasourceProcessor,\n} from '@subql/common-ethereum';\nimport { getLogger, NodeConfig, Sandbox } from '@subql/node-core';\nimport {\n  SecondLayerHandlerProcessor_0_0_0,\n  SecondLayerHandlerProcessor_1_0_0,\n  SubqlCustomDatasource,\n} from '@subql/types-ethereum';\n\nimport { VMScript } from 'vm2';\nimport { SubqueryProject } from '../configure/SubqueryProject';\n\nexport interface DsPluginSandboxOption {\n  root: string;\n  entry: string;\n  script: string;\n}\n\nconst logger = getLogger('ds-sandbox');\n\nexport function isSecondLayerHandlerProcessor_0_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlCustomDatasource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_0_0_0<K, F, E, DS> {\n  // Exisiting datasource processors had no concept of specVersion, therefore undefined is equivalent to 0.0.0\n  return processor.specVersion === undefined;\n}\n\nexport function isSecondLayerHandlerProcessor_1_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlEthereumCustomDataSource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): processor is SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  return processor.specVersion === '1.0.0';\n}\n\nexport function asSecondLayerHandlerProcessor_1_0_0<\n  K extends EthereumHandlerKind,\n  F,\n  E,\n  DS extends SubqlEthereumCustomDataSource = SubqlEthereumCustomDataSource,\n>(\n  processor:\n    | SecondLayerHandlerProcessor_0_0_0<K, F, E, DS>\n    | SecondLayerHandlerProcessor_1_0_0<K, F, E, DS>,\n): SecondLayerHandlerProcessor_1_0_0<K, F, E, DS> {\n  if (isSecondLayerHandlerProcessor_1_0_0(processor)) {\n    return processor;\n  }\n\n  if (!isSecondLayerHandlerProcessor_0_0_0(processor)) {\n    throw new Error('Unsupported ds processor version');\n  }\n\n  return {\n    ...processor,\n    specVersion: '1.0.0',\n    filterProcessor: (params) =>\n      processor.filterProcessor(params.filter, params.input, params.ds),\n    transformer: (params) =>\n      processor\n        .transformer(params.input, params.ds, params.api, params.assets)\n        .then((res) => [res]),\n  };\n}\n\nexport class DsPluginSandbox extends Sandbox {\n  constructor(option: DsPluginSandboxOption, nodeConfig: NodeConfig) {\n    super(\n      option,\n      new VMScript(\n        `module.exports = require('${option.entry}').default;`,\n        path.join(option.root, 'ds_sandbox'),\n      ),\n      nodeConfig,\n    );\n    this.freeze(logger, 'logger');\n  }\n\n  getDsPlugin<D extends string>(): SubqlDatasourceProcessor<D, unknown> {\n    return this.run(this.script);\n  }\n}\n\n@Injectable()\nexport class DsProcessorService {\n  private processorCache: {\n    [entry: string]: SubqlDatasourceProcessor<string, unknown>;\n  } = {};\n  constructor(\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly nodeConfig: NodeConfig,\n  ) {}\n\n  async validateCustomDs(\n    datasources: SubqlEthereumCustomDataSource[],\n  ): Promise<void> {\n    for (const ds of datasources) {\n      const processor = this.getDsProcessor(ds);\n      /* Standard validation applicable to all custom ds and processors */\n      if (ds.kind !== processor.kind) {\n        throw new Error(\n          `ds kind (${ds.kind}) doesnt match processor (${processor.kind})`,\n        );\n      }\n\n      for (const handler of ds.mapping.handlers) {\n        if (!(handler.kind in processor.handlerProcessors)) {\n          throw new Error(\n            `ds kind ${handler.kind} not one of ${Object.keys(\n              processor.handlerProcessors,\n            ).join(', ')}`,\n          );\n        }\n      }\n\n      ds.mapping.handlers.map((handler) =>\n        processor.handlerProcessors[handler.kind].filterValidator(\n          handler.filter,\n        ),\n      );\n\n      /* Additional processor specific validation */\n      processor.validate(ds, await this.getAssets(ds));\n    }\n  }\n\n  async validateProjectCustomDatasources(): Promise<void> {\n    await this.validateCustomDs(\n      (this.project.dataSources as SubqlEthereumDataSource[]).filter(\n        isCustomDs,\n      ),\n    );\n  }\n\n  getDsProcessor<D extends string>(\n    ds: SubqlEthereumCustomDataSource<string>,\n  ): SubqlDatasourceProcessor<D, unknown> {\n    if (!isCustomDs(ds)) {\n      throw new Error(`data source is not a custom data source`);\n    }\n    if (!this.processorCache[ds.processor.file]) {\n      const sandbox = new DsPluginSandbox(\n        {\n          root: this.project.root,\n          entry: ds.processor.file,\n          script:\n            null /* TODO get working with Readers, same as with sandbox */,\n        },\n        this.nodeConfig,\n      );\n      try {\n        this.processorCache[ds.processor.file] = sandbox.getDsPlugin<D>();\n      } catch (e) {\n        logger.error(`not supported ds @${ds.kind}`);\n        throw e;\n      }\n    }\n    return this.processorCache[\n      ds.processor.file\n    ] as unknown as SubqlDatasourceProcessor<D, unknown>;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  async getAssets(\n    ds: SubqlEthereumCustomDataSource,\n  ): Promise<Record<string, string>> {\n    if (!isCustomDs(ds)) {\n      throw new Error(`data source is not a custom data source`);\n    }\n\n    if (!ds.assets) {\n      return {};\n    }\n\n    const res: Record<string, string> = {};\n\n    for (const [name, { file }] of ds.assets) {\n      // TODO update with https://github.com/subquery/subql/pull/511\n      try {\n        res[name] = fs.readFileSync(file, {\n          encoding: 'utf8',\n        });\n      } catch (e) {\n        logger.error(`Failed to load datasource asset ${file}`);\n        throw e;\n      }\n    }\n\n    return res;\n  }\n}\n"]}

package/dist/indexer/dynamic-ds.service.d.ts

@@ -1,25 +1,9 @@
-import { MetadataRepo } from '@subql/node-core';
-import { Transaction } from 'sequelize/types';
+import { DatasourceParams, DynamicDsService as BaseDynamicDsService } from '@subql/node-core';
 import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';
 import { DsProcessorService } from './ds-processor.service';
-interface DatasourceParams {
-    templateName: string;
-    args?: Record<string, unknown>;
-    startBlock: number;
-}
-export declare class DynamicDsService {
+export declare class DynamicDsService extends BaseDynamicDsService<SubqlProjectDs> {
     private readonly dsProcessorService;
     private readonly project;
-    private metaDataRepo;
-    private tempDsRecords;
     constructor(dsProcessorService: DsProcessorService, project: SubqueryProject);
-    init(metaDataRepo: MetadataRepo): void;
-    private _datasources;
-    createDynamicDatasource(params: DatasourceParams, tx: Transaction): Promise<SubqlProjectDs>;
-    getDynamicDatasources(): Promise<SubqlProjectDs[]>;
-    deleteTempDsRecords(blockHeight: number): void;
-    private getDynamicDatasourceParams;
-    private saveDynamicDatasourceParams;
-    private getDatasource;
+    protected getDatasource(params: DatasourceParams): Promise<SubqlProjectDs>;
 }
-export {};

package/dist/indexer/dynamic-ds.service.js

@@ -10,94 +10,31 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DynamicDsService = void 0;
-const assert_1 = __importDefault(require("assert"));
 const common_1 = require("@nestjs/common");
 const common_ethereum_1 = require("@subql/common-ethereum");
 const node_core_1 = require("@subql/node-core");
+const class_transformer_1 = require("class-transformer");
+const class_validator_1 = require("class-validator");
 const lodash_1 = require("lodash");
 const SubqueryProject_1 = require("../configure/SubqueryProject");
 const ds_processor_service_1 = require("./ds-processor.service");
-const logger = (0, node_core_1.getLogger)('dynamic-ds');
-const METADATA_KEY = 'dynamicDatasources';
-const TEMP_DS_PREFIX = 'ds_';
-let DynamicDsService = class DynamicDsService {
+let DynamicDsService = class DynamicDsService extends node_core_1.DynamicDsService {
     constructor(dsProcessorService, project) {
+        super();
         this.dsProcessorService = dsProcessorService;
         this.project = project;
     }
-    init(metaDataRepo) {
-        this.metaDataRepo = metaDataRepo;
-    }
-    async createDynamicDatasource(params, tx) {
-        try {
-            const ds = await this.getDatasource(params);
-            await this.saveDynamicDatasourceParams(params, tx);
-            logger.info(`Created new dynamic datasource from template: "${params.templateName}"`);
-            if (!this._datasources)
-                this._datasources = [];
-            this._datasources.push(ds);
-            return ds;
-        }
-        catch (e) {
-            logger.error(e.message);
-            process.exit(1);
-        }
-    }
-    async getDynamicDatasources() {
-        if (!this._datasources) {
-            try {
-                const params = await this.getDynamicDatasourceParams();
-                this._datasources = await Promise.all(params.map((params) => this.getDatasource(params)));
-            }
-            catch (e) {
-                logger.error(`Unable to get dynamic datasources:\n${e.message}`);
-                process.exit(1);
-            }
-        }
-        return this._datasources;
-    }
-    deleteTempDsRecords(blockHeight) {
-        delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];
-    }
-    async getDynamicDatasourceParams(blockHeight) {
-        var _a;
-        (0, assert_1.default)(this.metaDataRepo, `Model _metadata does not exist`);
-        const record = await this.metaDataRepo.findByPk(METADATA_KEY);
-        let results = record === null || record === void 0 ? void 0 : record.value;
-        if (!results || typeof results !== 'string') {
-            if (blockHeight !== undefined) {
-                results = (_a = this.tempDsRecords) === null || _a === void 0 ? void 0 : _a[TEMP_DS_PREFIX + blockHeight];
-                if (!results || typeof results !== 'string') {
-                    return [];
-                }
-            }
-            else {
-                return [];
-            }
-        }
-        return JSON.parse(results);
-    }
-    async saveDynamicDatasourceParams(dsParams, tx) {
-        const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);
-        (0, assert_1.default)(this.metaDataRepo, `Model _metadata does not exist`);
-        const dsRecords = JSON.stringify([...existing, dsParams]);
-        await this.metaDataRepo
-            .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })
-            .then(() => {
-            this.tempDsRecords = Object.assign(Object.assign({}, this.tempDsRecords), { [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords });
-        });
-    }
     async getDatasource(params) {
-        const template = (0, lodash_1.cloneDeep)(this.project.templates.find((t) => t.name === params.templateName));
+        var _a;
+        const template = (0, lodash_1.cloneDeep)((_a = this.project.templates) === null || _a === void 0 ? void 0 : _a.find((t) => t.name === params.templateName));
         if (!template) {
             throw new Error(`Unable to find matching template in project for name: "${params.templateName}"`);
         }
-        logger.info(`Initialised dynamic datasource from template: "${params.templateName}"`);
         const dsObj = Object.assign(Object.assign({}, template), { startBlock: params.startBlock });
         delete dsObj.name;
         try {
@@ -106,7 +43,17 @@ let DynamicDsService = class DynamicDsService {
                 await this.dsProcessorService.validateCustomDs([dsObj]);
             }
             else if ((0, common_ethereum_1.isRuntimeDs)(dsObj)) {
-                // XXX add any modifications to the ds here
+                dsObj.options = Object.assign(Object.assign({}, dsObj.options), params.args);
+                const parsedDs = (0, class_transformer_1.plainToClass)(common_ethereum_1.EthereumRuntimeDataSourceV0_3_0Impl, dsObj);
+                const errors = (0, class_validator_1.validateSync)(parsedDs, {
+                    whitelist: true,
+                    forbidNonWhitelisted: false,
+                });
+                if (errors.length) {
+                    throw new Error(`Dynamic ds is invalid\n${errors
+                        .map((e) => e.toString())
+                        .join('\n')}`);
+                }
             }
             return dsObj;
         }
@@ -117,6 +64,7 @@ let DynamicDsService = class DynamicDsService {
 };
 DynamicDsService = __decorate([
     (0, common_1.Injectable)(),
+    __param(1, (0, common_1.Inject)('ISubqueryProject')),
     __metadata("design:paramtypes", [ds_processor_service_1.DsProcessorService,
         SubqueryProject_1.SubqueryProject])
 ], DynamicDsService);

package/dist/indexer/dynamic-ds.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,2CAA4C;AAC5C,4DAAiE;AACjE,gDAA2D;AAC3D,mCAAmC;AAEnC,kEAA+E;AAC/E,iEAA4D;AAE5D,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,YAAY,CAAC,CAAC;AAEvC,MAAM,YAAY,GAAG,oBAAoB,CAAC;AAC1C,MAAM,cAAc,GAAG,KAAK,CAAC;AAStB,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAI3B,YACmB,kBAAsC,EACtC,OAAwB;QADxB,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,YAAO,GAAP,OAAO,CAAiB;IACxC,CAAC;IAEJ,IAAI,CAAC,YAA0B;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAID,KAAK,CAAC,uBAAuB,CAC3B,MAAwB,EACxB,EAAe;QAEf,IAAI;YACF,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAE5C,MAAM,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAEnD,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;YAC/C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE3B,OAAO,EAAE,CAAC;SACX;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAEvD,IAAI,CAAC,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CACnD,CAAC;aACH;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,mBAAmB,CAAC,WAAmB;QACrC,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,WAAoB;;QAEpB,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC9D,IAAI,OAAO,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAK,CAAC;QAE5B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE;YAC3C,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC7B,OAAO,GAAG,MAAA,IAAI,CAAC,aAAa,0CAAG,cAAc,GAAG,WAAW,CAAC,CAAC;gBAC7D,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE;oBAC3C,OAAO,EAAE,CAAC;iBACX;aACF;iBAAM;gBACL,OAAO,EAAE,CAAC;aACX;SACF;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAA0B,EAC1B,EAAe;QAEf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAE5E,IAAA,gBAAM,EAAC,IAAI,CAAC,YAAY,EAAE,gCAAgC,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY;aACpB,MAAM,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;aACpE,IAAI,CAAC,GAAG,EAAE;YACT,IAAI,CAAC,aAAa,mCACb,IAAI,CAAC,aAAa,GAClB,EAAE,CAAC,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CACzD,CAAC;QACJ,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAwB;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACnE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,CAAC,IAAI,CACT,kDAAkD,MAAM,CAAC,YAAY,GAAG,CACzE,CAAC;QAEF,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,2CAA2C;aAC5C;YAED,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AAxIY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;qCAM4B,yCAAkB;QAC7B,iCAAe;GANhC,gBAAgB,CAwI5B;AAxIY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { Injectable } from '@nestjs/common';\nimport { isCustomDs, isRuntimeDs } from '@subql/common-ethereum';\nimport { getLogger, MetadataRepo } from '@subql/node-core';\nimport { cloneDeep } from 'lodash';\nimport { Transaction } from 'sequelize/types';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\nconst logger = getLogger('dynamic-ds');\n\nconst METADATA_KEY = 'dynamicDatasources';\nconst TEMP_DS_PREFIX = 'ds_';\n\ninterface DatasourceParams {\n  templateName: string;\n  args?: Record<string, unknown>;\n  startBlock: number;\n}\n\n@Injectable()\nexport class DynamicDsService {\n  private metaDataRepo: MetadataRepo;\n  private tempDsRecords: Record<string, string>;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly project: SubqueryProject,\n  ) {}\n\n  init(metaDataRepo: MetadataRepo): void {\n    this.metaDataRepo = metaDataRepo;\n  }\n\n  private _datasources: SubqlProjectDs[];\n\n  async createDynamicDatasource(\n    params: DatasourceParams,\n    tx: Transaction,\n  ): Promise<SubqlProjectDs> {\n    try {\n      const ds = await this.getDatasource(params);\n\n      await this.saveDynamicDatasourceParams(params, tx);\n\n      logger.info(\n        `Created new dynamic datasource from template: \"${params.templateName}\"`,\n      );\n\n      if (!this._datasources) this._datasources = [];\n      this._datasources.push(ds);\n\n      return ds;\n    } catch (e) {\n      logger.error(e.message);\n      process.exit(1);\n    }\n  }\n\n  async getDynamicDatasources(): Promise<SubqlProjectDs[]> {\n    if (!this._datasources) {\n      try {\n        const params = await this.getDynamicDatasourceParams();\n\n        this._datasources = await Promise.all(\n          params.map((params) => this.getDatasource(params)),\n        );\n      } catch (e) {\n        logger.error(`Unable to get dynamic datasources:\\n${e.message}`);\n        process.exit(1);\n      }\n    }\n\n    return this._datasources;\n  }\n\n  deleteTempDsRecords(blockHeight: number) {\n    delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];\n  }\n\n  private async getDynamicDatasourceParams(\n    blockHeight?: number,\n  ): Promise<DatasourceParams[]> {\n    assert(this.metaDataRepo, `Model _metadata does not exist`);\n    const record = await this.metaDataRepo.findByPk(METADATA_KEY);\n    let results = record?.value;\n\n    if (!results || typeof results !== 'string') {\n      if (blockHeight !== undefined) {\n        results = this.tempDsRecords?.[TEMP_DS_PREFIX + blockHeight];\n        if (!results || typeof results !== 'string') {\n          return [];\n        }\n      } else {\n        return [];\n      }\n    }\n\n    return JSON.parse(results);\n  }\n\n  private async saveDynamicDatasourceParams(\n    dsParams: DatasourceParams,\n    tx: Transaction,\n  ): Promise<void> {\n    const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);\n\n    assert(this.metaDataRepo, `Model _metadata does not exist`);\n    const dsRecords = JSON.stringify([...existing, dsParams]);\n    await this.metaDataRepo\n      .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })\n      .then(() => {\n        this.tempDsRecords = {\n          ...this.tempDsRecords,\n          ...{ [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords },\n        };\n      });\n  }\n\n  private async getDatasource(\n    params: DatasourceParams,\n  ): Promise<SubqlProjectDs> {\n    const template = cloneDeep(\n      this.project.templates.find((t) => t.name === params.templateName),\n    );\n\n    if (!template) {\n      throw new Error(\n        `Unable to find matching template in project for name: \"${params.templateName}\"`,\n      );\n    }\n\n    logger.info(\n      `Initialised dynamic datasource from template: \"${params.templateName}\"`,\n    );\n\n    const dsObj = {\n      ...template,\n      startBlock: params.startBlock,\n    } as SubqlProjectDs;\n    delete dsObj.name;\n    try {\n      if (isCustomDs(dsObj)) {\n        dsObj.processor.options = {\n          ...dsObj.processor.options,\n          ...params.args,\n        };\n        await this.dsProcessorService.validateCustomDs([dsObj]);\n      } else if (isRuntimeDs(dsObj)) {\n        // XXX add any modifications to the ds here\n      }\n\n      return dsObj;\n    } catch (e) {\n      throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n    }\n  }\n}\n"]}
+{"version":3,"file":"dynamic-ds.service.js","sourceRoot":"","sources":["../../src/indexer/dynamic-ds.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,4DAIgC;AAChC,gDAG0B;AAC1B,yDAAiD;AACjD,qDAA+C;AAC/C,mCAAmC;AACnC,kEAA+E;AAC/E,iEAA4D;AAGrD,IAAM,gBAAgB,GAAtB,MAAM,gBAAiB,SAAQ,4BAAoC;IACxE,YACmB,kBAAsC,EACV,OAAwB;QAErE,KAAK,EAAE,CAAC;QAHS,uBAAkB,GAAlB,kBAAkB,CAAoB;QACV,YAAO,GAAP,OAAO,CAAiB;IAGvE,CAAC;IAES,KAAK,CAAC,aAAa,CAC3B,MAAwB;;QAExB,MAAM,QAAQ,GAAG,IAAA,kBAAS,EACxB,MAAA,IAAI,CAAC,OAAO,CAAC,SAAS,0CAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,YAAY,CAAC,CACpE,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,KAAK,CACb,0DAA0D,MAAM,CAAC,YAAY,GAAG,CACjF,CAAC;SACH;QAED,MAAM,KAAK,GAAG,gCACT,QAAQ,KACX,UAAU,EAAE,MAAM,CAAC,UAAU,GACZ,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC;QAClB,IAAI;YACF,IAAI,IAAA,4BAAU,EAAC,KAAK,CAAC,EAAE;gBACrB,KAAK,CAAC,SAAS,CAAC,OAAO,mCAClB,KAAK,CAAC,SAAS,CAAC,OAAO,GACvB,MAAM,CAAC,IAAI,CACf,CAAC;gBACF,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACzD;iBAAM,IAAI,IAAA,6BAAW,EAAC,KAAK,CAAC,EAAE;gBAC7B,KAAK,CAAC,OAAO,mCACR,KAAK,CAAC,OAAO,GACb,MAAM,CAAC,IAAI,CACf,CAAC;gBAEF,MAAM,QAAQ,GAAG,IAAA,gCAAY,EAC3B,qDAAmC,EACnC,KAAK,CACN,CAAC;gBAEF,MAAM,MAAM,GAAG,IAAA,8BAAY,EAAC,QAAQ,EAAE;oBACpC,SAAS,EAAE,IAAI;oBACf,oBAAoB,EAAE,KAAK;iBAC5B,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,EAAE;oBACjB,MAAM,IAAI,KAAK,CACb,0BAA0B,MAAM;yBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;yBACxB,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;iBACH;aACF;YACD,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxE;IACH,CAAC;CACF,CAAA;AA7DY,gBAAgB;IAD5B,IAAA,mBAAU,GAAE;IAIR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCADU,yCAAkB;QACD,iCAAe;GAH5D,gBAAgB,CA6D5B;AA7DY,4CAAgB","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  EthereumRuntimeDataSourceV0_3_0Impl,\n  isCustomDs,\n  isRuntimeDs,\n} from '@subql/common-ethereum';\nimport {\n  DatasourceParams,\n  DynamicDsService as BaseDynamicDsService,\n} from '@subql/node-core';\nimport { plainToClass } from 'class-transformer';\nimport { validateSync } from 'class-validator';\nimport { cloneDeep } from 'lodash';\nimport { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';\nimport { DsProcessorService } from './ds-processor.service';\n\n@Injectable()\nexport class DynamicDsService extends BaseDynamicDsService<SubqlProjectDs> {\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n  ) {\n    super();\n  }\n\n  protected async getDatasource(\n    params: DatasourceParams,\n  ): Promise<SubqlProjectDs> {\n    const template = cloneDeep(\n      this.project.templates?.find((t) => t.name === params.templateName),\n    );\n\n    if (!template) {\n      throw new Error(\n        `Unable to find matching template in project for name: \"${params.templateName}\"`,\n      );\n    }\n\n    const dsObj = {\n      ...template,\n      startBlock: params.startBlock,\n    } as SubqlProjectDs;\n    delete dsObj.name;\n    try {\n      if (isCustomDs(dsObj)) {\n        dsObj.processor.options = {\n          ...dsObj.processor.options,\n          ...params.args,\n        };\n        await this.dsProcessorService.validateCustomDs([dsObj]);\n      } else if (isRuntimeDs(dsObj)) {\n        dsObj.options = {\n          ...dsObj.options,\n          ...params.args,\n        };\n\n        const parsedDs = plainToClass(\n          EthereumRuntimeDataSourceV0_3_0Impl,\n          dsObj,\n        );\n\n        const errors = validateSync(parsedDs, {\n          whitelist: true,\n          forbidNonWhitelisted: false,\n        });\n        if (errors.length) {\n          throw new Error(\n            `Dynamic ds is invalid\\n${errors\n              .map((e) => e.toString())\n              .join('\\n')}`,\n          );\n        }\n      }\n      return dsObj;\n    } catch (e) {\n      throw new Error(`Unable to create dynamic datasource.\\n ${e.message}`);\n    }\n  }\n}\n"]}

package/dist/indexer/fetch.module.js

@@ -12,8 +12,8 @@ exports.FetchModule = void 0;
 const common_1 = require("@nestjs/common");
 const event_emitter_1 = require("@nestjs/event-emitter");
 const node_core_1 = require("@subql/node-core");
-const SubqueryProject_1 = require("../configure/SubqueryProject");
 const api_service_ethereum_1 = require("../ethereum/api.service.ethereum");
+const blockDispatcher_1 = require("./blockDispatcher");
 const dictionary_service_1 = require("./dictionary.service");
 const ds_processor_service_1 = require("./ds-processor.service");
 const dynamic_ds_service_1 = require("./dynamic-ds.service");
@@ -21,47 +21,75 @@ const fetch_service_1 = require("./fetch.service");
 const indexer_manager_1 = require("./indexer.manager");
 const project_service_1 = require("./project.service");
 const sandbox_service_1 = require("./sandbox.service");
-const block_dispatcher_service_1 = require("./worker/block-dispatcher.service");
+const unfinalizedBlocks_service_1 = require("./unfinalizedBlocks.service");
 let FetchModule = class FetchModule {
 };
 FetchModule = __decorate([
     (0, common_1.Module)({
         providers: [
             node_core_1.StoreService,
+            node_core_1.StoreCacheService,
+            node_core_1.ConnectionPoolService,
             {
                 provide: node_core_1.ApiService,
-                useFactory: async (project) => {
-                    const apiService = new api_service_ethereum_1.EthereumApiService(project);
+                useFactory: async (project, connectionPoolService, eventEmitter) => {
+                    const apiService = new api_service_ethereum_1.EthereumApiService(project, connectionPoolService, eventEmitter);
                     await apiService.init();
                     return apiService;
                 },
-                inject: [SubqueryProject_1.SubqueryProject],
+                inject: ['ISubqueryProject', node_core_1.ConnectionPoolService, event_emitter_1.EventEmitter2],
             },
             indexer_manager_1.IndexerManager,
+            {
+                provide: node_core_1.SmartBatchService,
+                useFactory: (nodeConfig) => {
+                    return new node_core_1.SmartBatchService(nodeConfig.batchSize);
+                },
+                inject: [node_core_1.NodeConfig],
+            },
             {
                 provide: 'IBlockDispatcher',
-                useFactory: (nodeConfig, eventEmitter, projectService, apiService, indexerManager) => nodeConfig.workers !== undefined
-                    ? new block_dispatcher_service_1.WorkerBlockDispatcherService(nodeConfig, eventEmitter, projectService)
-                    : new block_dispatcher_service_1.BlockDispatcherService(apiService, nodeConfig, indexerManager, eventEmitter, projectService),
+                useFactory: (nodeConfig, eventEmitter, projectService, apiService, indexerManager, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService, unfinalizedBlocks) => nodeConfig.workers !== undefined
+                    ? new blockDispatcher_1.WorkerBlockDispatcherService(nodeConfig, eventEmitter, projectService, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService, unfinalizedBlocks)
+                    : new blockDispatcher_1.BlockDispatcherService(apiService, nodeConfig, indexerManager, eventEmitter, projectService, smartBatchService, storeService, storeCacheService, poiService, project, dynamicDsService),
                 inject: [
                     node_core_1.NodeConfig,
                     event_emitter_1.EventEmitter2,
-                    project_service_1.ProjectService,
+                    'IProjectService',
                     node_core_1.ApiService,
                     indexer_manager_1.IndexerManager,
+                    node_core_1.SmartBatchService,
+                    node_core_1.StoreService,
+                    node_core_1.StoreCacheService,
+                    node_core_1.PoiService,
+                    'ISubqueryProject',
+                    dynamic_ds_service_1.DynamicDsService,
+                    unfinalizedBlocks_service_1.UnfinalizedBlocksService,
                 ],
             },
             fetch_service_1.FetchService,
             node_core_1.BenchmarkService,
-            dictionary_service_1.DictionaryService,
+            {
+                provide: dictionary_service_1.DictionaryService,
+                useFactory: async (project, nodeConfig) => {
+                    const dictionaryService = new dictionary_service_1.DictionaryService(project, nodeConfig);
+                    await dictionaryService.init();
+                    return dictionaryService;
+                },
+                inject: ['ISubqueryProject', node_core_1.NodeConfig],
+            },
             sandbox_service_1.SandboxService,
             ds_processor_service_1.DsProcessorService,
             dynamic_ds_service_1.DynamicDsService,
             node_core_1.PoiService,
             node_core_1.MmrService,
-            project_service_1.ProjectService,
+            {
+                useClass: project_service_1.ProjectService,
+                provide: 'IProjectService',
+            },
+            unfinalizedBlocks_service_1.UnfinalizedBlocksService,
         ],
-        exports: [node_core_1.StoreService, node_core_1.MmrService],
+        exports: [node_core_1.StoreService, node_core_1.MmrService, node_core_1.StoreCacheService],
     })
 ], FetchModule);
 exports.FetchModule = FetchModule;

package/dist/indexer/fetch.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.module.js","sourceRoot":"","sources":["../../src/indexer/fetch.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAAsD;AACtD,gDAO0B;AAC1B,kEAA+D;AAC/D,2EAAsE;AACtE,6DAAyD;AACzD,iEAA4D;AAC5D,6DAAwD;AACxD,mDAA+C;AAC/C,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,gFAG2C;AA0DpC,IAAM,WAAW,GAAjB,MAAM,WAAW;CAAG,CAAA;AAAd,WAAW;IAxDvB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EAAE,OAAwB,EAAE,EAAE;oBAC7C,MAAM,UAAU,GAAG,IAAI,yCAAkB,CAAC,OAAO,CAAC,CAAC;oBAEnD,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,iCAAe,CAAC;aAC1B;YACD,gCAAc;YACd;gBACE,OAAO,EAAE,kBAAkB;gBAC3B,UAAU,EAAE,CACV,UAAsB,EACtB,YAA2B,EAC3B,cAA8B,EAC9B,UAAsB,EACtB,cAA8B,EAC9B,EAAE,CACF,UAAU,CAAC,OAAO,KAAK,SAAS;oBAC9B,CAAC,CAAC,IAAI,uDAA4B,CAC9B,UAAU,EACV,YAAY,EACZ,cAAc,CACf;oBACH,CAAC,CAAC,IAAI,iDAAsB,CACxB,UAAU,EACV,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,CACf;gBACP,MAAM,EAAE;oBACN,sBAAU;oBACV,6BAAa;oBACb,gCAAc;oBACd,sBAAU;oBACV,gCAAc;iBACf;aACF;YACD,4BAAY;YACZ,4BAAgB;YAChB,sCAAiB;YACjB,gCAAc;YACd,yCAAkB;YAClB,qCAAgB;YAChB,sBAAU;YACV,sBAAU;YACV,gCAAc;SACf;QACD,OAAO,EAAE,CAAC,wBAAY,EAAE,sBAAU,CAAC;KACpC,CAAC;GACW,WAAW,CAAG;AAAd,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  BenchmarkService,\n  MmrService,\n  StoreService,\n  PoiService,\n  ApiService,\n  NodeConfig,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum/api.service.ethereum';\nimport { DictionaryService } from './dictionary.service';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { FetchService } from './fetch.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport {\n  BlockDispatcherService,\n  WorkerBlockDispatcherService,\n} from './worker/block-dispatcher.service';\n\n@Module({\n  providers: [\n    StoreService,\n    {\n      provide: ApiService,\n      useFactory: async (project: SubqueryProject) => {\n        const apiService = new EthereumApiService(project);\n\n        await apiService.init();\n        return apiService;\n      },\n      inject: [SubqueryProject],\n    },\n    IndexerManager,\n    {\n      provide: 'IBlockDispatcher',\n      useFactory: (\n        nodeConfig: NodeConfig,\n        eventEmitter: EventEmitter2,\n        projectService: ProjectService,\n        apiService: ApiService,\n        indexerManager: IndexerManager,\n      ) =>\n        nodeConfig.workers !== undefined\n          ? new WorkerBlockDispatcherService(\n              nodeConfig,\n              eventEmitter,\n              projectService,\n            )\n          : new BlockDispatcherService(\n              apiService,\n              nodeConfig,\n              indexerManager,\n              eventEmitter,\n              projectService,\n            ),\n      inject: [\n        NodeConfig,\n        EventEmitter2,\n        ProjectService,\n        ApiService,\n        IndexerManager,\n      ],\n    },\n    FetchService,\n    BenchmarkService,\n    DictionaryService,\n    SandboxService,\n    DsProcessorService,\n    DynamicDsService,\n    PoiService,\n    MmrService,\n    ProjectService,\n  ],\n  exports: [StoreService, MmrService],\n})\nexport class FetchModule {}\n"]}
+{"version":3,"file":"fetch.module.js","sourceRoot":"","sources":["../../src/indexer/fetch.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,2CAAwC;AACxC,yDAAsD;AACtD,gDAU0B;AAG1B,2EAAsE;AACtE,uDAG2B;AAC3B,6DAAyD;AACzD,iEAA4D;AAC5D,6DAAwD;AACxD,mDAA+C;AAC/C,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAiHhE,IAAM,WAAW,GAAjB,MAAM,WAAW;CAAG,CAAA;AAAd,WAAW;IA/GvB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,wBAAY;YACZ,6BAAiB;YACjB,iCAAqB;YACrB;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,qBAAmE,EACnE,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,yCAAkB,CACvC,OAAO,EACP,qBAAqB,EACrB,YAAY,CACb,CAAC;oBACF,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,iCAAqB,EAAE,6BAAa,CAAC;aACnE;YACD,gCAAc;YACd;gBACE,OAAO,EAAE,6BAAiB;gBAC1B,UAAU,EAAE,CAAC,UAAsB,EAAE,EAAE;oBACrC,OAAO,IAAI,6BAAiB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;gBACrD,CAAC;gBACD,MAAM,EAAE,CAAC,sBAAU,CAAC;aACrB;YACD;gBACE,OAAO,EAAE,kBAAkB;gBAC3B,UAAU,EAAE,CACV,UAAsB,EACtB,YAA2B,EAC3B,cAA8B,EAC9B,UAA8B,EAC9B,cAA8B,EAC9B,iBAAoC,EACpC,YAA0B,EAC1B,iBAAoC,EACpC,UAAsB,EACtB,OAAwB,EACxB,gBAAkC,EAClC,iBAA2C,EAC3C,EAAE,CACF,UAAU,CAAC,OAAO,KAAK,SAAS;oBAC9B,CAAC,CAAC,IAAI,8CAA4B,CAC9B,UAAU,EACV,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,iBAAiB,CAClB;oBACH,CAAC,CAAC,IAAI,wCAAsB,CACxB,UAAU,EACV,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,CACjB;gBACP,MAAM,EAAE;oBACN,sBAAU;oBACV,6BAAa;oBACb,iBAAiB;oBACjB,sBAAU;oBACV,gCAAc;oBACd,6BAAiB;oBACjB,wBAAY;oBACZ,6BAAiB;oBACjB,sBAAU;oBACV,kBAAkB;oBAClB,qCAAgB;oBAChB,oDAAwB;iBACzB;aACF;YACD,4BAAY;YACZ,4BAAgB;YAChB;gBACE,OAAO,EAAE,sCAAiB;gBAC1B,UAAU,EAAE,KAAK,EAAE,OAAwB,EAAE,UAAsB,EAAE,EAAE;oBACrE,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;oBACrE,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC;oBAC/B,OAAO,iBAAiB,CAAC;gBAC3B,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,sBAAU,CAAC;aACzC;YACD,gCAAc;YACd,yCAAkB;YAClB,qCAAgB;YAChB,sBAAU;YACV,sBAAU;YACV;gBACE,QAAQ,EAAE,gCAAc;gBACxB,OAAO,EAAE,iBAAiB;aAC3B;YACD,oDAAwB;SACzB;QACD,OAAO,EAAE,CAAC,wBAAY,EAAE,sBAAU,EAAE,6BAAiB,CAAC;KACvD,CAAC;GACW,WAAW,CAAG;AAAd,kCAAW","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  BenchmarkService,\n  MmrService,\n  StoreService,\n  PoiService,\n  ApiService,\n  NodeConfig,\n  ConnectionPoolService,\n  SmartBatchService,\n  StoreCacheService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiConnection } from '../ethereum/api.connection';\nimport { EthereumApiService } from '../ethereum/api.service.ethereum';\nimport {\n  BlockDispatcherService,\n  WorkerBlockDispatcherService,\n} from './blockDispatcher';\nimport { DictionaryService } from './dictionary.service';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { FetchService } from './fetch.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    StoreService,\n    StoreCacheService,\n    ConnectionPoolService,\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        connectionPoolService: ConnectionPoolService<EthereumApiConnection>,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new EthereumApiService(\n          project,\n          connectionPoolService,\n          eventEmitter,\n        );\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2],\n    },\n    IndexerManager,\n    {\n      provide: SmartBatchService,\n      useFactory: (nodeConfig: NodeConfig) => {\n        return new SmartBatchService(nodeConfig.batchSize);\n      },\n      inject: [NodeConfig],\n    },\n    {\n      provide: 'IBlockDispatcher',\n      useFactory: (\n        nodeConfig: NodeConfig,\n        eventEmitter: EventEmitter2,\n        projectService: ProjectService,\n        apiService: EthereumApiService,\n        indexerManager: IndexerManager,\n        smartBatchService: SmartBatchService,\n        storeService: StoreService,\n        storeCacheService: StoreCacheService,\n        poiService: PoiService,\n        project: SubqueryProject,\n        dynamicDsService: DynamicDsService,\n        unfinalizedBlocks: UnfinalizedBlocksService,\n      ) =>\n        nodeConfig.workers !== undefined\n          ? new WorkerBlockDispatcherService(\n              nodeConfig,\n              eventEmitter,\n              projectService,\n              smartBatchService,\n              storeService,\n              storeCacheService,\n              poiService,\n              project,\n              dynamicDsService,\n              unfinalizedBlocks,\n            )\n          : new BlockDispatcherService(\n              apiService,\n              nodeConfig,\n              indexerManager,\n              eventEmitter,\n              projectService,\n              smartBatchService,\n              storeService,\n              storeCacheService,\n              poiService,\n              project,\n              dynamicDsService,\n            ),\n      inject: [\n        NodeConfig,\n        EventEmitter2,\n        'IProjectService',\n        ApiService,\n        IndexerManager,\n        SmartBatchService,\n        StoreService,\n        StoreCacheService,\n        PoiService,\n        'ISubqueryProject',\n        DynamicDsService,\n        UnfinalizedBlocksService,\n      ],\n    },\n    FetchService,\n    BenchmarkService,\n    {\n      provide: DictionaryService,\n      useFactory: async (project: SubqueryProject, nodeConfig: NodeConfig) => {\n        const dictionaryService = new DictionaryService(project, nodeConfig);\n        await dictionaryService.init();\n        return dictionaryService;\n      },\n      inject: ['ISubqueryProject', NodeConfig],\n    },\n    SandboxService,\n    DsProcessorService,\n    DynamicDsService,\n    PoiService,\n    MmrService,\n    {\n      useClass: ProjectService,\n      provide: 'IProjectService',\n    },\n    UnfinalizedBlocksService,\n  ],\n  exports: [StoreService, MmrService, StoreCacheService],\n})\nexport class FetchModule {}\n"]}

package/dist/indexer/fetch.service.d.ts

@@ -2,11 +2,12 @@ import { OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import { SchedulerRegistry } from '@nestjs/schedule';
 import { ApiService, NodeConfig } from '@subql/node-core';
-import { DictionaryQueryEntry, ApiWrapper } from '@subql/types-ethereum';
+import { ApiWrapper, DictionaryQueryEntry } from '@subql/types-ethereum';
 import { SubqueryProject } from '../configure/SubqueryProject';
+import { IEthereumBlockDispatcher } from './blockDispatcher';
 import { DictionaryService } from './dictionary.service';
 import { DynamicDsService } from './dynamic-ds.service';
-import { IBlockDispatcher } from './worker/block-dispatcher.service';
+import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
 export declare class FetchService implements OnApplicationShutdown {
     private apiService;
     private nodeConfig;
@@ -14,31 +15,39 @@ export declare class FetchService implements OnApplicationShutdown {
     private blockDispatcher;
     private dictionaryService;
     private dynamicDsService;
+    private unfinalizedBlocksService;
     private eventEmitter;
     private schedulerRegistry;
     private latestBestHeight;
     private latestFinalizedHeight;
     private isShutdown;
-    private useDictionary;
-    private dictionaryQueryEntries?;
     private batchSizeScale;
     private templateDynamicDatasouces;
-    constructor(apiService: ApiService, nodeConfig: NodeConfig, project: SubqueryProject, blockDispatcher: IBlockDispatcher, dictionaryService: DictionaryService, dynamicDsService: DynamicDsService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry);
+    private dictionaryMetaValid;
+    private evmChainId;
+    private bypassBlocks;
+    constructor(apiService: ApiService, nodeConfig: NodeConfig, project: SubqueryProject, blockDispatcher: IEthereumBlockDispatcher, dictionaryService: DictionaryService, dynamicDsService: DynamicDsService, unfinalizedBlocksService: UnfinalizedBlocksService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry);
     onApplicationShutdown(): void;
     get api(): ApiWrapper;
     syncDynamicDatascourcesFromMeta(): Promise<void>;
-    getDictionaryQueryEntries(): DictionaryQueryEntry[];
+    buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[];
     updateDictionary(): void;
+    private get useDictionary();
     init(startHeight: number): Promise<void>;
+    getUseDictionary(): boolean;
+    getLatestFinalizedHeight(): number;
     checkBatchScale(): void;
     getFinalizedBlockHead(): Promise<void>;
     getBestBlockHead(): Promise<void>;
-    startLoop(initBlockHeight: number): Promise<void>;
+    private startLoop;
     getModulos(): number[];
     getModuloBlocks(startHeight: number, endHeight: number): number[];
     getEnqueuedModuloBlocks(startBlockHeight: number): number[];
     fillNextBlockBuffer(initBlockHeight: number): Promise<void>;
+    private getLatestBufferHeight;
+    private filteredBlockBatch;
     private nextEndBlockHeight;
     private dictionaryValidation;
     resetForNewDs(blockHeight: number): Promise<void>;
+    resetForIncorrectBestBlock(blockHeight: number): Promise<void>;
 }