@subql/node-ethereum 1.0.3-4 → 1.0.3-6

package/dist/configure/configure.module.js

@@ -22,9 +22,7 @@ const lodash_1 = require("lodash");
 const yargs_1 = require("../yargs");
 const SubqueryProject_1 = require("./SubqueryProject");
 const logger = (0, node_core_1.getLogger)('configure');
-const YargsNameMapping = {
-    local: 'localMode',
-};
+const YargsNameMapping = {};
 function yargsToIConfig(yargs) {
     return Object.entries(yargs).reduce((acc, [key, value]) => {
         var _a;

package/dist/configure/configure.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"configure.module.js","sourceRoot":"","sources":["../../src/configure/configure.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAA+D;AAC/D,0CAAsE;AAEtE,gDAM0B;AAC1B,mCAAwD;AACxD,oCAAwC;AACxC,uDAAoD;AAEpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,CAAC,CAAC;AAEtC,MAAM,gBAAgB,GAA2B;IAC/C,KAAK,EAAE,WAAW;CACnB,CAAC;AAIF,SAAS,cAAc,CAAC,KAAW;IACjC,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;;QACxD,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC;QAE1C,IAAI,GAAG,KAAK,kBAAkB,EAAE;YAC9B,IAAI;gBACF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAe,CAAC,CAAC;aACrC;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;aAClE;SACF;QACD,GAAG,CAAC,MAAA,gBAAgB,CAAC,GAAG,CAAC,mCAAI,IAAA,kBAAS,EAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;QACrD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAS,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAwB;;IACnD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,mBAAU,CAAC,CAAC;IACpD,OAAO,gCACF,MAAM,KACT,YAAY,EACV,MAAA,MAAM,CAAC,YAAY,mCACnB,CAAC,SAAS;YACR,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,mBAAU,EAAE,EAAE,CAAC;YACzC,CAAC,CAAC,IAAA,aAAI,EACF,IAAA,kCAAyB,EAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAI,CAAC,GAAG,CAAC,CAChE,CAAC,GACI,CAAC;AACjB,CAAC;AAED,kDAAkD;AAClD,SAAgB,iBAAiB,CAAC,IAAY;IAC5C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;QACrB,OAAO,KAAK,CAAC;KACd;SAAM;QACL,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,qCAAqC,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,sBAAsB;QAC7D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;QAC3E,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,CAAC,KAAK,CACV,wBAAwB,IAAI,gDAAgD,CAC7E,CAAC;SACH;QACD,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,CAAC,KAAK,CACV,wBAAwB,IAAI;kHAC8E,CAC3G,CAAC;SACH;QACD,OAAO,KAAK,IAAI,KAAK,CAAC;KACvB;AACH,CAAC;AArBD,8CAqBC;AAED,SAAS,gBAAgB;IACvB,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;IAC9B,IAAI,IAAI,CAAC,eAAe,CAAC,EAAE;QACzB,MAAM,CAAC,IAAI,CACT,iFAAiF,CAClF,CAAC;KACH;IACD,IAAI,IAAI,CAAC,KAAK,EAAE;QACd,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;KAC/D;AACH,CAAC;AAIM,IAAM,eAAe,uBAArB,MAAM,eAAe;IAC1B,MAAM,CAAC,kBAAkB,CAAC,MAAkB;QAC1C,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;YACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,IAAI,MAAM,CAAC,KAAK,EAAE;YAChB,IAAA,oBAAQ,EAAC,OAAO,CAAC,CAAC;SACnB;QAED,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;YACzB,MAAM,CAAC,GAAG,MAAM,iCAAe,CAAC,MAAM,CACpC,MAAM,CAAC,QAAQ,EACf,IAAA,eAAM,EACJ;gBACE,QAAQ,EAAE,MAAM,CAAC,gBAAgB;gBACjC,UAAU,EAAE,MAAM,CAAC,iBAAiB;aACrC,EACD,cAAK,CACN,EACD;gBACE,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CACF,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACd,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,iDAAiD,CAAC,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,OAAO;YACL,MAAM,EAAE,iBAAe;YACvB,SAAS,EAAE;gBACT;oBACE,OAAO,EAAE,sBAAU;oBACnB,QAAQ,EAAE,MAAM;iBACjB;gBACD;oBACE,OAAO,EAAE,iCAAe;oBACxB,UAAU,EAAE,OAAO;iBACpB;aACF;YACD,OAAO,EAAE,CAAC,sBAAU,EAAE,iCAAe,CAAC;SACvC,CAAC;IACJ,CAAC;IACD,MAAM,CAAC,QAAQ;QACb,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;QAC9B,IAAI,MAAkB,CAAC;QACvB,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,MAAM,GAAG,sBAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;SACjE;aAAM;YACL,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAClB,MAAM,CAAC,KAAK,CACV,oEAAoE,CACrE,CAAC;gBACF,oBAAY,CAAC,QAAQ,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,gBAAgB,EAAE,CAAC;YACnB,IAAA,gBAAM,EAAC,IAAI,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAC;YAClD,MAAM,GAAG,IAAI,sBAAU,CAAC,mBAAmB,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACpE;QAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;YACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,IAAI,MAAM,CAAC,KAAK,EAAE;YAChB,IAAA,oBAAQ,EAAC,OAAO,CAAC,CAAC;SACnB;QAED,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;YACzB,MAAM,CAAC,GAAG,MAAM,iCAAe,CAAC,MAAM,CACpC,IAAI,CAAC,QAAQ,EACb,IAAA,eAAM,EACJ;gBACE,QAAQ,EAAE,MAAM,CAAC,gBAAgB;gBACjC,UAAU,EAAE,MAAM,CAAC,iBAAiB;aACrC,EACD,cAAK,CACN,EACD;gBACE,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CACF,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACd,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,iDAAiD,CAAC,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,OAAO;YACL,MAAM,EAAE,iBAAe;YACvB,SAAS,EAAE;gBACT;oBACE,OAAO,EAAE,sBAAU;oBACnB,QAAQ,EAAE,MAAM;iBACjB;gBACD;oBACE,OAAO,EAAE,kBAAkB;oBAC3B,UAAU,EAAE,OAAO;iBACpB;aACF;YACD,OAAO,EAAE,CAAC,sBAAU,EAAE,kBAAkB,CAAC;SAC1C,CAAC;IACJ,CAAC;CACF,CAAA;AA3GY,eAAe;IAF3B,IAAA,eAAM,GAAE;IACR,IAAA,eAAM,EAAC,EAAE,CAAC;GACE,eAAe,CA2G3B;AA3GY,0CAAe","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { DynamicModule, Global, Module } from '@nestjs/common';\nimport { getProjectRootAndManifest, IPFS_REGEX } from '@subql/common';\nimport { EthereumProjectNetworkConfig } from '@subql/common-ethereum';\nimport {\n  IConfig,\n  MinConfig,\n  NodeConfig,\n  getLogger,\n  setLevel,\n} from '@subql/node-core';\nimport { camelCase, last, omitBy, isNil } from 'lodash';\nimport { yargsOptions } from '../yargs';\nimport { SubqueryProject } from './SubqueryProject';\n\nconst logger = getLogger('configure');\n\nconst YargsNameMapping: Record<string, string> = {\n  local: 'localMode',\n};\n\ntype Args = typeof yargsOptions.argv['argv'];\n\nfunction yargsToIConfig(yargs: Args): Partial<IConfig> {\n  return Object.entries(yargs).reduce((acc, [key, value]) => {\n    if (['_', '$0'].includes(key)) return acc;\n\n    if (key === 'network-registry') {\n      try {\n        value = JSON.parse(value as string);\n      } catch (e) {\n        throw new Error('Argument `network-registry` is not valid JSON');\n      }\n    }\n    acc[YargsNameMapping[key] ?? camelCase(key)] = value;\n    return acc;\n  }, {} as any);\n}\n\nfunction defaultSubqueryName(config: Partial<IConfig>): MinConfig {\n  const ipfsMatch = config.subquery.match(IPFS_REGEX);\n  return {\n    ...config,\n    subqueryName:\n      config.subqueryName ??\n      (ipfsMatch\n        ? config.subquery.replace(IPFS_REGEX, '')\n        : last(\n            getProjectRootAndManifest(config.subquery).root.split(path.sep),\n          )),\n  } as MinConfig;\n}\n\n// Check if a subquery name is a valid schema name\nexport function validDbSchemaName(name: string): boolean {\n  if (name.length === 0) {\n    return false;\n  } else {\n    name = name.toLowerCase();\n    const regexp = new RegExp('^[a-zA-Z_][a-zA-Z0-9_\\\\-\\\\/]{0,62}$');\n    const flag0 = !name.startsWith('pg_'); // Reserved identifier\n    const flag1 = regexp.test(name); // <= Valid characters, less than 63 bytes\n    if (!flag0) {\n      logger.error(\n        `Invalid schema name '${name}', schema name must not be prefixed with 'pg_'`,\n      );\n    }\n    if (!flag1) {\n      logger.error(\n        `Invalid schema name '${name}', schema name must start with a letter or underscore, \n         be less than 63 bytes and must contain only valid alphanumeric characters (can include characters '_-/')`,\n      );\n    }\n    return flag0 && flag1;\n  }\n}\n\nfunction warnDeprecations() {\n  const { argv } = yargsOptions;\n  if (argv['subquery-name']) {\n    logger.warn(\n      'Note that argument --subquery-name has been deprecated in favour of --db-schema',\n    );\n  }\n  if (argv.local) {\n    logger.warn('Note that argument --local has been deprecated');\n  }\n}\n\n@Global()\n@Module({})\nexport class ConfigureModule {\n  static registerWithConfig(config: NodeConfig): DynamicModule {\n    if (!validDbSchemaName(config.dbSchema)) {\n      process.exit(1);\n    }\n\n    if (config.debug) {\n      setLevel('debug');\n    }\n\n    const project = async () => {\n      const p = await SubqueryProject.create(\n        config.subquery,\n        omitBy<EthereumProjectNetworkConfig>(\n          {\n            endpoint: config.networkEndpoints,\n            dictionary: config.networkDictionary,\n          },\n          isNil,\n        ),\n        {\n          ipfs: config.ipfs,\n        },\n      ).catch((err) => {\n        logger.error(err, 'Create Subquery project from given path failed!');\n        process.exit(1);\n      });\n      return p;\n    };\n\n    return {\n      module: ConfigureModule,\n      providers: [\n        {\n          provide: NodeConfig,\n          useValue: config,\n        },\n        {\n          provide: SubqueryProject,\n          useFactory: project,\n        },\n      ],\n      exports: [NodeConfig, SubqueryProject],\n    };\n  }\n  static register(): DynamicModule {\n    const { argv } = yargsOptions;\n    let config: NodeConfig;\n    if (argv.config) {\n      config = NodeConfig.fromFile(argv.config, yargsToIConfig(argv));\n    } else {\n      if (!argv.subquery) {\n        logger.error(\n          'Subquery path is missing neither in cli options nor in config file',\n        );\n        yargsOptions.showHelp();\n        process.exit(1);\n      }\n\n      warnDeprecations();\n      assert(argv.subquery, 'subquery path is missing');\n      config = new NodeConfig(defaultSubqueryName(yargsToIConfig(argv)));\n    }\n\n    if (!validDbSchemaName(config.dbSchema)) {\n      process.exit(1);\n    }\n\n    if (config.debug) {\n      setLevel('debug');\n    }\n\n    const project = async () => {\n      const p = await SubqueryProject.create(\n        argv.subquery,\n        omitBy<Partial<EthereumProjectNetworkConfig>>(\n          {\n            endpoint: config.networkEndpoints,\n            dictionary: config.networkDictionary,\n          },\n          isNil,\n        ),\n        {\n          ipfs: config.ipfs,\n        },\n      ).catch((err) => {\n        logger.error(err, 'Create Subquery project from given path failed!');\n        process.exit(1);\n      });\n      return p;\n    };\n\n    return {\n      module: ConfigureModule,\n      providers: [\n        {\n          provide: NodeConfig,\n          useValue: config,\n        },\n        {\n          provide: 'ISubqueryProject',\n          useFactory: project,\n        },\n      ],\n      exports: [NodeConfig, 'ISubqueryProject'],\n    };\n  }\n}\n"]}
+{"version":3,"file":"configure.module.js","sourceRoot":"","sources":["../../src/configure/configure.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,gDAAwB;AACxB,2CAA+D;AAC/D,0CAAsE;AAEtE,gDAM0B;AAC1B,mCAAwD;AACxD,oCAAwC;AACxC,uDAAoD;AAEpD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,CAAC,CAAC;AAEtC,MAAM,gBAAgB,GAA2B,EAAE,CAAC;AAIpD,SAAS,cAAc,CAAC,KAAW;IACjC,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;;QACxD,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC;QAE1C,IAAI,GAAG,KAAK,kBAAkB,EAAE;YAC9B,IAAI;gBACF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAe,CAAC,CAAC;aACrC;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;aAClE;SACF;QACD,GAAG,CAAC,MAAA,gBAAgB,CAAC,GAAG,CAAC,mCAAI,IAAA,kBAAS,EAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;QACrD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAS,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAwB;;IACnD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,mBAAU,CAAC,CAAC;IACpD,OAAO,gCACF,MAAM,KACT,YAAY,EACV,MAAA,MAAM,CAAC,YAAY,mCACnB,CAAC,SAAS;YACR,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,mBAAU,EAAE,EAAE,CAAC;YACzC,CAAC,CAAC,IAAA,aAAI,EACF,IAAA,kCAAyB,EAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAI,CAAC,GAAG,CAAC,CAChE,CAAC,GACI,CAAC;AACjB,CAAC;AAED,kDAAkD;AAClD,SAAgB,iBAAiB,CAAC,IAAY;IAC5C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;QACrB,OAAO,KAAK,CAAC;KACd;SAAM;QACL,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,qCAAqC,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,sBAAsB;QAC7D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;QAC3E,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,CAAC,KAAK,CACV,wBAAwB,IAAI,gDAAgD,CAC7E,CAAC;SACH;QACD,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,CAAC,KAAK,CACV,wBAAwB,IAAI;kHAC8E,CAC3G,CAAC;SACH;QACD,OAAO,KAAK,IAAI,KAAK,CAAC;KACvB;AACH,CAAC;AArBD,8CAqBC;AAED,SAAS,gBAAgB;IACvB,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;IAC9B,IAAI,IAAI,CAAC,eAAe,CAAC,EAAE;QACzB,MAAM,CAAC,IAAI,CACT,iFAAiF,CAClF,CAAC;KACH;IACD,IAAI,IAAI,CAAC,KAAK,EAAE;QACd,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;KAC/D;AACH,CAAC;AAIM,IAAM,eAAe,uBAArB,MAAM,eAAe;IAC1B,MAAM,CAAC,kBAAkB,CAAC,MAAkB;QAC1C,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;YACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,IAAI,MAAM,CAAC,KAAK,EAAE;YAChB,IAAA,oBAAQ,EAAC,OAAO,CAAC,CAAC;SACnB;QAED,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;YACzB,MAAM,CAAC,GAAG,MAAM,iCAAe,CAAC,MAAM,CACpC,MAAM,CAAC,QAAQ,EACf,IAAA,eAAM,EACJ;gBACE,QAAQ,EAAE,MAAM,CAAC,gBAAgB;gBACjC,UAAU,EAAE,MAAM,CAAC,iBAAiB;aACrC,EACD,cAAK,CACN,EACD;gBACE,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CACF,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACd,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,iDAAiD,CAAC,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,OAAO;YACL,MAAM,EAAE,iBAAe;YACvB,SAAS,EAAE;gBACT;oBACE,OAAO,EAAE,sBAAU;oBACnB,QAAQ,EAAE,MAAM;iBACjB;gBACD;oBACE,OAAO,EAAE,iCAAe;oBACxB,UAAU,EAAE,OAAO;iBACpB;aACF;YACD,OAAO,EAAE,CAAC,sBAAU,EAAE,iCAAe,CAAC;SACvC,CAAC;IACJ,CAAC;IACD,MAAM,CAAC,QAAQ;QACb,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;QAC9B,IAAI,MAAkB,CAAC;QACvB,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,MAAM,GAAG,sBAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;SACjE;aAAM;YACL,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAClB,MAAM,CAAC,KAAK,CACV,oEAAoE,CACrE,CAAC;gBACF,oBAAY,CAAC,QAAQ,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,gBAAgB,EAAE,CAAC;YACnB,IAAA,gBAAM,EAAC,IAAI,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAC;YAClD,MAAM,GAAG,IAAI,sBAAU,CAAC,mBAAmB,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACpE;QAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;YACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QAED,IAAI,MAAM,CAAC,KAAK,EAAE;YAChB,IAAA,oBAAQ,EAAC,OAAO,CAAC,CAAC;SACnB;QAED,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;YACzB,MAAM,CAAC,GAAG,MAAM,iCAAe,CAAC,MAAM,CACpC,IAAI,CAAC,QAAQ,EACb,IAAA,eAAM,EACJ;gBACE,QAAQ,EAAE,MAAM,CAAC,gBAAgB;gBACjC,UAAU,EAAE,MAAM,CAAC,iBAAiB;aACrC,EACD,cAAK,CACN,EACD;gBACE,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CACF,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACd,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,iDAAiD,CAAC,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,OAAO;YACL,MAAM,EAAE,iBAAe;YACvB,SAAS,EAAE;gBACT;oBACE,OAAO,EAAE,sBAAU;oBACnB,QAAQ,EAAE,MAAM;iBACjB;gBACD;oBACE,OAAO,EAAE,kBAAkB;oBAC3B,UAAU,EAAE,OAAO;iBACpB;aACF;YACD,OAAO,EAAE,CAAC,sBAAU,EAAE,kBAAkB,CAAC;SAC1C,CAAC;IACJ,CAAC;CACF,CAAA;AA3GY,eAAe;IAF3B,IAAA,eAAM,GAAE;IACR,IAAA,eAAM,EAAC,EAAE,CAAC;GACE,eAAe,CA2G3B;AA3GY,0CAAe","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport path from 'path';\nimport { DynamicModule, Global, Module } from '@nestjs/common';\nimport { getProjectRootAndManifest, IPFS_REGEX } from '@subql/common';\nimport { EthereumProjectNetworkConfig } from '@subql/common-ethereum';\nimport {\n  IConfig,\n  MinConfig,\n  NodeConfig,\n  getLogger,\n  setLevel,\n} from '@subql/node-core';\nimport { camelCase, last, omitBy, isNil } from 'lodash';\nimport { yargsOptions } from '../yargs';\nimport { SubqueryProject } from './SubqueryProject';\n\nconst logger = getLogger('configure');\n\nconst YargsNameMapping: Record<string, string> = {};\n\ntype Args = typeof yargsOptions.argv['argv'];\n\nfunction yargsToIConfig(yargs: Args): Partial<IConfig> {\n  return Object.entries(yargs).reduce((acc, [key, value]) => {\n    if (['_', '$0'].includes(key)) return acc;\n\n    if (key === 'network-registry') {\n      try {\n        value = JSON.parse(value as string);\n      } catch (e) {\n        throw new Error('Argument `network-registry` is not valid JSON');\n      }\n    }\n    acc[YargsNameMapping[key] ?? camelCase(key)] = value;\n    return acc;\n  }, {} as any);\n}\n\nfunction defaultSubqueryName(config: Partial<IConfig>): MinConfig {\n  const ipfsMatch = config.subquery.match(IPFS_REGEX);\n  return {\n    ...config,\n    subqueryName:\n      config.subqueryName ??\n      (ipfsMatch\n        ? config.subquery.replace(IPFS_REGEX, '')\n        : last(\n            getProjectRootAndManifest(config.subquery).root.split(path.sep),\n          )),\n  } as MinConfig;\n}\n\n// Check if a subquery name is a valid schema name\nexport function validDbSchemaName(name: string): boolean {\n  if (name.length === 0) {\n    return false;\n  } else {\n    name = name.toLowerCase();\n    const regexp = new RegExp('^[a-zA-Z_][a-zA-Z0-9_\\\\-\\\\/]{0,62}$');\n    const flag0 = !name.startsWith('pg_'); // Reserved identifier\n    const flag1 = regexp.test(name); // <= Valid characters, less than 63 bytes\n    if (!flag0) {\n      logger.error(\n        `Invalid schema name '${name}', schema name must not be prefixed with 'pg_'`,\n      );\n    }\n    if (!flag1) {\n      logger.error(\n        `Invalid schema name '${name}', schema name must start with a letter or underscore, \n         be less than 63 bytes and must contain only valid alphanumeric characters (can include characters '_-/')`,\n      );\n    }\n    return flag0 && flag1;\n  }\n}\n\nfunction warnDeprecations() {\n  const { argv } = yargsOptions;\n  if (argv['subquery-name']) {\n    logger.warn(\n      'Note that argument --subquery-name has been deprecated in favour of --db-schema',\n    );\n  }\n  if (argv.local) {\n    logger.warn('Note that argument --local has been deprecated');\n  }\n}\n\n@Global()\n@Module({})\nexport class ConfigureModule {\n  static registerWithConfig(config: NodeConfig): DynamicModule {\n    if (!validDbSchemaName(config.dbSchema)) {\n      process.exit(1);\n    }\n\n    if (config.debug) {\n      setLevel('debug');\n    }\n\n    const project = async () => {\n      const p = await SubqueryProject.create(\n        config.subquery,\n        omitBy<EthereumProjectNetworkConfig>(\n          {\n            endpoint: config.networkEndpoints,\n            dictionary: config.networkDictionary,\n          },\n          isNil,\n        ),\n        {\n          ipfs: config.ipfs,\n        },\n      ).catch((err) => {\n        logger.error(err, 'Create Subquery project from given path failed!');\n        process.exit(1);\n      });\n      return p;\n    };\n\n    return {\n      module: ConfigureModule,\n      providers: [\n        {\n          provide: NodeConfig,\n          useValue: config,\n        },\n        {\n          provide: SubqueryProject,\n          useFactory: project,\n        },\n      ],\n      exports: [NodeConfig, SubqueryProject],\n    };\n  }\n  static register(): DynamicModule {\n    const { argv } = yargsOptions;\n    let config: NodeConfig;\n    if (argv.config) {\n      config = NodeConfig.fromFile(argv.config, yargsToIConfig(argv));\n    } else {\n      if (!argv.subquery) {\n        logger.error(\n          'Subquery path is missing neither in cli options nor in config file',\n        );\n        yargsOptions.showHelp();\n        process.exit(1);\n      }\n\n      warnDeprecations();\n      assert(argv.subquery, 'subquery path is missing');\n      config = new NodeConfig(defaultSubqueryName(yargsToIConfig(argv)));\n    }\n\n    if (!validDbSchemaName(config.dbSchema)) {\n      process.exit(1);\n    }\n\n    if (config.debug) {\n      setLevel('debug');\n    }\n\n    const project = async () => {\n      const p = await SubqueryProject.create(\n        argv.subquery,\n        omitBy<Partial<EthereumProjectNetworkConfig>>(\n          {\n            endpoint: config.networkEndpoints,\n            dictionary: config.networkDictionary,\n          },\n          isNil,\n        ),\n        {\n          ipfs: config.ipfs,\n        },\n      ).catch((err) => {\n        logger.error(err, 'Create Subquery project from given path failed!');\n        process.exit(1);\n      });\n      return p;\n    };\n\n    return {\n      module: ConfigureModule,\n      providers: [\n        {\n          provide: NodeConfig,\n          useValue: config,\n        },\n        {\n          provide: 'ISubqueryProject',\n          useFactory: project,\n        },\n      ],\n      exports: [NodeConfig, 'ISubqueryProject'],\n    };\n  }\n}\n"]}

package/dist/indexer/indexer.manager.d.ts

@@ -18,6 +18,7 @@ export declare class IndexerManager implements IIndexerManager<EthereumBlockWrap
     indexBlock(blockContent: EthereumBlockWrapper, dataSources: SubqlProjectDs[]): Promise<ProcessBlockResponse>;
     start(): Promise<void>;
     private processUnfinalizedBlocks;
+    private filterDataSources;
     private assertDataSources;
     private indexBlockData;
     private indexBlockContent;

package/dist/indexer/indexer.manager.js

@@ -41,12 +41,12 @@ let IndexerManager = class IndexerManager {
         const { block, blockHeight } = blockContent;
         let dynamicDsCreated = false;
         let reindexBlockHeight = null;
-        // Check that we have valid datasources
-        this.assertDataSources(dataSources, blockHeight);
+        const filteredDataSources = this.filterDataSources(blockHeight, dataSources);
+        this.assertDataSources(filteredDataSources, blockHeight);
         reindexBlockHeight = await this.processUnfinalizedBlocks(block);
         // Only index block if we're not going to reindex
         if (!reindexBlockHeight) {
-            await this.indexBlockData(blockContent, dataSources, 
+            await this.indexBlockData(blockContent, filteredDataSources, 
             // eslint-disable-next-line @typescript-eslint/require-await
             async (ds) => {
                 const vm = this.sandboxService.getDsProcessorWrapper(ds, this.apiService.api, blockContent);
@@ -58,7 +58,7 @@ let IndexerManager = class IndexerManager {
                         startBlock: blockHeight,
                     });
                     // Push the newly created dynamic ds to be processed this block on any future extrinsics/events
-                    dataSources.push(newDs);
+                    filteredDataSources.push(newDs);
                     dynamicDsCreated = true;
                 }, 'createDynamicDatasource');
                 return vm;
@@ -80,6 +80,30 @@ let IndexerManager = class IndexerManager {
         }
         return null;
     }
+    filterDataSources(nextProcessingHeight, dataSources) {
+        let filteredDs;
+        filteredDs = dataSources.filter((ds) => ds.startBlock <= nextProcessingHeight);
+        if (filteredDs.length === 0) {
+            logger.error(`Did not find any matching datasouces`);
+            process.exit(1);
+        }
+        // perform filter for custom ds
+        filteredDs = filteredDs.filter((ds) => {
+            if ((0, common_ethereum_1.isCustomDs)(ds)) {
+                return this.dsProcessorService
+                    .getDsProcessor(ds)
+                    .dsFilterProcessor(ds, this.apiService.api);
+            }
+            else {
+                return true;
+            }
+        });
+        if (!filteredDs.length) {
+            logger.error(`Did not find any datasources with associated processor`);
+            process.exit(1);
+        }
+        return filteredDs;
+    }
     assertDataSources(ds, blockHeight) {
         if (!ds.length) {
             logger.error(`Your start block is greater than the current indexed block height in your database. Either change your startBlock (project.yaml) to <= ${blockHeight}

package/dist/indexer/indexer.manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.manager.js","sourceRoot":"","sources":["../../src/indexer/indexer.manager.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,4DAUgC;AAChC,gDAS0B;AAU1B,+DAAkE;AAClE,oCAAwC;AACxC,iEAGgC;AAChC,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAGzB,YACU,UAAsB,EACtB,UAAsB,EACtB,cAA8B,EAC9B,kBAAsC,EACtC,gBAAkC,EAClC,wBAAkD,EACvB,cAA8B;QANzD,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAC9B,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QACvB,mBAAc,GAAd,cAAc,CAAgB;QAEjE,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACvC,CAAC;IAGK,AAAN,KAAK,CAAC,UAAU,CACd,YAAkC,EAClC,WAA6B;QAE7B,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;QAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,kBAAkB,GAAG,IAAI,CAAC;QAE9B,uCAAuC;QACvC,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACjD,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAEhE,iDAAiD;QACjD,IAAI,CAAC,kBAAkB,EAAE;YACvB,MAAM,IAAI,CAAC,cAAc,CACvB,YAAY,EACZ,WAAW;YACX,4DAA4D;YAC5D,KAAK,EAAE,EAAkB,EAAE,EAAE;gBAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAClD,EAAE,EACF,IAAI,CAAC,UAAU,CAAC,GAAG,EACnB,YAAY,CACb,CAAC;gBAEF,uCAAuC;gBACvC,EAAE,CAAC,MAAM,CACP,KAAK,EAAE,YAAoB,EAAE,IAA8B,EAAE,EAAE;oBAC7D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,uBAAuB,CAC/D;wBACE,YAAY;wBACZ,IAAI;wBACJ,UAAU,EAAE,WAAW;qBACxB,CACF,CAAC;oBACF,+FAA+F;oBAC/F,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACxB,gBAAgB,GAAG,IAAI,CAAC;gBAC1B,CAAC,EACD,yBAAyB,CAC1B,CAAC;gBAEF,OAAO,EAAE,CAAC;YACZ,CAAC,CACF,CAAC;SACH;QAED,OAAO;YACL,gBAAgB;YAChB,SAAS,EAAE,KAAK,CAAC,IAAI;YACrB,kBAAkB;SACnB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,KAAoB;QAEpB,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,OAAO,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;SACtE;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iBAAiB,CAAC,EAAoB,EAAE,WAAmB;QACjE,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YACd,MAAM,CAAC,KAAK,CACV,0IAA0I,WAAW;yFACpE,CAClF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,EAAE,KAAK,EAAE,YAAY,EAAwB,EAC7C,WAA6B,EAC7B,KAAqD;;QAErD,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAExD,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE;YAC7B,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAEpD,KAAK,MAAM,GAAG,IAAI,MAAA,EAAE,CAAC,IAAI,mCAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;aAChD;SACF;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,KAAoB,EACpB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACnE;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,EAAuB,EACvB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SAC/D;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,GAAgB,EAChB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACjE;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,IAAO,EACP,IAAuC,EACvC,EAAkB,EAClB,KAAsD;QAEtD,IAAI,EAAkB,CAAC;QACvB,IAAI,IAAA,6BAAW,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,QAAkC,CAAC,MAAM,CACpE,CAAC,CAAC,EAAE,EAAE;;gBACJ,OAAA,CAAC,CAAC,IAAI,KAAK,IAAI;oBACf,aAAa,CAAC,IAAI,CAAC,CACjB,IAAW,EACX,CAAC,CAAC,MAAa,EACf,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,CACpB,CAAA;aAAA,CACJ,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YACD,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,IAAI,CAAC,UAAU,CAAC,QAAQ;oBACtB,CAAC,CAAC,MAAM,IAAA,wBAAY,EAChB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,EACvB,oBAAoB,EACpB,OAAO,CAAC,OAAO,CAChB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;oBAClC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;aACzD;SACF;aAAM,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAC1C,EAAE,EACF,IAAI,EACJ,gBAAgB,CAAC,IAAI,CAAC,EACtB,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;gBACnB,QAAQ,IAAI,EAAE;oBACZ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,qBAAqB,CAC/C,IAAqB,EACrB,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,IAAI;wBAC3B,OAAO,qCAAoB,CAAC,2BAA2B,CACrD,IAA2B,EAC3B,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,mBAAmB,CAC7C,IAAmB,EACnB,UAAU,CACX,CAAC;oBACJ;wBACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;iBAC/C;YACH,CAAC,CACF,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YAED,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,MAAM,IAAI,CAAC,2BAA2B,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;aACrE;SACF;IACH,CAAC;IAEO,sBAAsB,CAC5B,EAA8C,EAC9C,IAAuC,EACvC,gBAAqC,EACrC,UAGY;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAE1D,OAAO,EAAE,CAAC,OAAO,CAAC,QAAQ;aACvB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,IAAI,gBAAgB,CAAC,SAAS,CAAC,EAAE;gBAC/B,SAAS,CAAC,UAAU,CAAC;gBAErB,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;aAC/C;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;YAEF,IAAI;gBACF,OAAO,SAAS,CAAC,eAAe,CAAC;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,IAAI;oBACX,EAAE;iBACH,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;gBACtD,MAAM,CAAC,CAAC;aACT;QACH,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,EAA8C,EAC9C,EAAkB,EAClB,OAA2B,EAC3B,IAAuC;QAEvC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAE3D,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;QAEF,MAAM,eAAe,GAAG,MAAM,SAAS;aACpC,WAAW,CAAC;YACX,KAAK,EAAE,IAAI;YACX,EAAE;YACF,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG;YACxB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM;SACP,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,6CAA6C,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEL,+FAA+F;QAC/F,+FAA+F;QAC/F,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE;YACnC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;SAChD;IACH,CAAC;CACF,CAAA;AAnRO;IADL,IAAA,oBAAQ,EAAC,oBAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;;;;gDAqDpC;AApEU,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAWR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCANN,sBAAU;QACV,sBAAU;QACN,gCAAc;QACV,yCAAkB;QACpB,qCAAgB;QACR,oDAAwB;QACP,gCAAc;GAVxD,cAAc,CAmS1B;AAnSY,wCAAc;AA2S3B,MAAM,gBAAgB,GAAG;IACvB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,wCAAsB;CACnD,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,qBAAqB;IACvE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,mBAAmB;IACrE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,qCAAoB,CAAC,2BAA2B;CAC7E,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAmB,EAAE,EAAE,CAAC,IAAI;IAChE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;IACzE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAC/C,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC;CACjC,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  isBlockHandlerProcessor,\n  isCallHandlerProcessor,\n  isEventHandlerProcessor,\n  isCustomDs,\n  isRuntimeDs,\n  SubqlEthereumCustomDataSource,\n  SubqlCustomHandler,\n  EthereumHandlerKind,\n  EthereumRuntimeHandlerInputMap,\n} from '@subql/common-ethereum';\nimport {\n  NodeConfig,\n  getLogger,\n  profiler,\n  profilerWrap,\n  IndexerSandbox,\n  ProcessBlockResponse,\n  ApiService,\n  IIndexerManager,\n} from '@subql/node-core';\nimport {\n  EthereumTransaction,\n  EthereumLog,\n  SubqlRuntimeHandler,\n  EthereumBlockWrapper,\n  EthereumBlock,\n} from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../configure/SubqueryProject';\nimport { EthereumApi } from '../ethereum';\nimport { EthereumBlockWrapped } from '../ethereum/block.ethereum';\nimport { yargsOptions } from '../yargs';\nimport {\n  asSecondLayerHandlerProcessor_1_0_0,\n  DsProcessorService,\n} from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('indexer');\n\n@Injectable()\nexport class IndexerManager\n  implements IIndexerManager<EthereumBlockWrapper, SubqlProjectDs>\n{\n  constructor(\n    private apiService: ApiService,\n    private nodeConfig: NodeConfig,\n    private sandboxService: SandboxService,\n    private dsProcessorService: DsProcessorService,\n    private dynamicDsService: DynamicDsService,\n    private unfinalizedBlocksService: UnfinalizedBlocksService,\n    @Inject('IProjectService') private projectService: ProjectService,\n  ) {\n    logger.info('indexer manager start');\n  }\n\n  @profiler(yargsOptions.argv.profiler)\n  async indexBlock(\n    blockContent: EthereumBlockWrapper,\n    dataSources: SubqlProjectDs[],\n  ): Promise<ProcessBlockResponse> {\n    const { block, blockHeight } = blockContent;\n    let dynamicDsCreated = false;\n    let reindexBlockHeight = null;\n\n    // Check that we have valid datasources\n    this.assertDataSources(dataSources, blockHeight);\n    reindexBlockHeight = await this.processUnfinalizedBlocks(block);\n\n    // Only index block if we're not going to reindex\n    if (!reindexBlockHeight) {\n      await this.indexBlockData(\n        blockContent,\n        dataSources,\n        // eslint-disable-next-line @typescript-eslint/require-await\n        async (ds: SubqlProjectDs) => {\n          const vm = this.sandboxService.getDsProcessorWrapper(\n            ds,\n            this.apiService.api,\n            blockContent,\n          );\n\n          // Inject function to create ds into vm\n          vm.freeze(\n            async (templateName: string, args?: Record<string, unknown>) => {\n              const newDs = await this.dynamicDsService.createDynamicDatasource(\n                {\n                  templateName,\n                  args,\n                  startBlock: blockHeight,\n                },\n              );\n              // Push the newly created dynamic ds to be processed this block on any future extrinsics/events\n              dataSources.push(newDs);\n              dynamicDsCreated = true;\n            },\n            'createDynamicDatasource',\n          );\n\n          return vm;\n        },\n      );\n    }\n\n    return {\n      dynamicDsCreated,\n      blockHash: block.hash,\n      reindexBlockHeight,\n    };\n  }\n\n  async start(): Promise<void> {\n    await this.projectService.init();\n    logger.info('indexer manager started');\n  }\n\n  private async processUnfinalizedBlocks(\n    block: EthereumBlock,\n  ): Promise<number | null> {\n    if (this.nodeConfig.unfinalizedBlocks) {\n      return this.unfinalizedBlocksService.processUnfinalizedBlocks(block);\n    }\n    return null;\n  }\n\n  private assertDataSources(ds: SubqlProjectDs[], blockHeight: number) {\n    if (!ds.length) {\n      logger.error(\n        `Your start block is greater than the current indexed block height in your database. Either change your startBlock (project.yaml) to <= ${blockHeight}\n         or delete your database and start again from the currently specified startBlock`,\n      );\n      process.exit(1);\n    }\n  }\n\n  private async indexBlockData(\n    { block, transactions }: EthereumBlockWrapper,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    await this.indexBlockContent(block, dataSources, getVM);\n\n    for (const tx of transactions) {\n      await this.indexTransaction(tx, dataSources, getVM);\n\n      for (const log of tx.logs ?? []) {\n        await this.indexEvent(log, dataSources, getVM);\n      }\n    }\n  }\n\n  private async indexBlockContent(\n    block: EthereumBlock,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Block, block, ds, getVM);\n    }\n  }\n\n  private async indexTransaction(\n    tx: EthereumTransaction,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Call, tx, ds, getVM);\n    }\n  }\n\n  private async indexEvent(\n    log: EthereumLog,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Event, log, ds, getVM);\n    }\n  }\n\n  private async indexData<K extends EthereumHandlerKind>(\n    kind: K,\n    data: EthereumRuntimeHandlerInputMap[K],\n    ds: SubqlProjectDs,\n    getVM: (ds: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    let vm: IndexerSandbox;\n    if (isRuntimeDs(ds)) {\n      const handlers = (ds.mapping.handlers as SubqlRuntimeHandler[]).filter(\n        (h) =>\n          h.kind === kind &&\n          FilterTypeMap[kind](\n            data as any,\n            h.filter as any,\n            ds.options?.address,\n          ),\n      );\n\n      if (!handlers.length) {\n        return;\n      }\n      const parsedData = await DataAbiParser[kind](this.apiService.api)(\n        data,\n        ds,\n      );\n\n      for (const handler of handlers) {\n        vm = vm ?? (await getVM(ds));\n        this.nodeConfig.profiler\n          ? await profilerWrap(\n              vm.securedExec.bind(vm),\n              'handlerPerformance',\n              handler.handler,\n            )(handler.handler, [parsedData])\n          : await vm.securedExec(handler.handler, [parsedData]);\n      }\n    } else if (isCustomDs(ds)) {\n      const handlers = this.filterCustomDsHandlers<K>(\n        ds,\n        data,\n        ProcessorTypeMap[kind],\n        (data, baseFilter) => {\n          switch (kind) {\n            case EthereumHandlerKind.Block:\n              return EthereumBlockWrapped.filterBlocksProcessor(\n                data as EthereumBlock,\n                baseFilter,\n              );\n            case EthereumHandlerKind.Call:\n              return EthereumBlockWrapped.filterTransactionsProcessor(\n                data as EthereumTransaction,\n                baseFilter,\n              );\n            case EthereumHandlerKind.Event:\n              return EthereumBlockWrapped.filterLogsProcessor(\n                data as EthereumLog,\n                baseFilter,\n              );\n            default:\n              throw new Error('Unsupported handler kind');\n          }\n        },\n      );\n\n      if (!handlers.length) {\n        return;\n      }\n\n      const parsedData = await DataAbiParser[kind](this.apiService.api)(\n        data,\n        ds,\n      );\n\n      for (const handler of handlers) {\n        vm = vm ?? (await getVM(ds));\n        await this.transformAndExecuteCustomDs(ds, vm, handler, parsedData);\n      }\n    }\n  }\n\n  private filterCustomDsHandlers<K extends EthereumHandlerKind>(\n    ds: SubqlEthereumCustomDataSource<string, any>,\n    data: EthereumRuntimeHandlerInputMap[K],\n    baseHandlerCheck: ProcessorTypeMap[K],\n    baseFilter: (\n      data: EthereumRuntimeHandlerInputMap[K],\n      baseFilter: any,\n    ) => boolean,\n  ): SubqlCustomHandler[] {\n    const plugin = this.dsProcessorService.getDsProcessor(ds);\n\n    return ds.mapping.handlers\n      .filter((handler) => {\n        const processor = plugin.handlerProcessors[handler.kind];\n        if (baseHandlerCheck(processor)) {\n          processor.baseFilter;\n\n          return baseFilter(data, processor.baseFilter);\n        }\n        return false;\n      })\n      .filter((handler) => {\n        const processor = asSecondLayerHandlerProcessor_1_0_0(\n          plugin.handlerProcessors[handler.kind],\n        );\n\n        try {\n          return processor.filterProcessor({\n            filter: handler.filter,\n            input: data,\n            ds,\n          });\n        } catch (e) {\n          logger.error(e, 'Failed to run ds processer filter.');\n          throw e;\n        }\n      });\n  }\n\n  private async transformAndExecuteCustomDs<K extends EthereumHandlerKind>(\n    ds: SubqlEthereumCustomDataSource<string, any>,\n    vm: IndexerSandbox,\n    handler: SubqlCustomHandler,\n    data: EthereumRuntimeHandlerInputMap[K],\n  ): Promise<void> {\n    const plugin = this.dsProcessorService.getDsProcessor(ds);\n    const assets = await this.dsProcessorService.getAssets(ds);\n\n    const processor = asSecondLayerHandlerProcessor_1_0_0(\n      plugin.handlerProcessors[handler.kind],\n    );\n\n    const transformedData = await processor\n      .transformer({\n        input: data,\n        ds,\n        api: this.apiService.api,\n        filter: handler.filter,\n        assets,\n      })\n      .catch((e) => {\n        logger.error(e, 'Failed to transform data with ds processor.');\n        throw e;\n      });\n\n    // We can not run this in parallel. the transformed data items may be dependent on one another.\n    // An example of this is with Acala EVM packing multiple EVM logs into a single Substrate event\n    for (const _data of transformedData) {\n      await vm.securedExec(handler.handler, [_data]);\n    }\n  }\n}\n\ntype ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: typeof isBlockHandlerProcessor;\n  [EthereumHandlerKind.Event]: typeof isEventHandlerProcessor;\n  [EthereumHandlerKind.Call]: typeof isCallHandlerProcessor;\n};\n\nconst ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: isBlockHandlerProcessor,\n  [EthereumHandlerKind.Event]: isEventHandlerProcessor,\n  [EthereumHandlerKind.Call]: isCallHandlerProcessor,\n};\n\nconst FilterTypeMap = {\n  [EthereumHandlerKind.Block]: EthereumBlockWrapped.filterBlocksProcessor,\n  [EthereumHandlerKind.Event]: EthereumBlockWrapped.filterLogsProcessor,\n  [EthereumHandlerKind.Call]: EthereumBlockWrapped.filterTransactionsProcessor,\n};\n\nconst DataAbiParser = {\n  [EthereumHandlerKind.Block]: () => (data: EthereumBlock) => data,\n  [EthereumHandlerKind.Event]: (api: EthereumApi) => api.parseLog.bind(api),\n  [EthereumHandlerKind.Call]: (api: EthereumApi) =>\n    api.parseTransaction.bind(api),\n};\n"]}
+{"version":3,"file":"indexer.manager.js","sourceRoot":"","sources":["../../src/indexer/indexer.manager.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,4DAUgC;AAChC,gDAS0B;AAU1B,+DAAkE;AAClE,oCAAwC;AACxC,iEAGgC;AAChC,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AAEvE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAGzB,YACU,UAAsB,EACtB,UAAsB,EACtB,cAA8B,EAC9B,kBAAsC,EACtC,gBAAkC,EAClC,wBAAkD,EACvB,cAA8B;QANzD,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;QAC9B,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,6BAAwB,GAAxB,wBAAwB,CAA0B;QACvB,mBAAc,GAAd,cAAc,CAAgB;QAEjE,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACvC,CAAC;IAGK,AAAN,KAAK,CAAC,UAAU,CACd,YAAkC,EAClC,WAA6B;QAE7B,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;QAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,kBAAkB,GAAkB,IAAI,CAAC;QAE7C,MAAM,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,CAChD,WAAW,EACX,WAAW,CACZ,CAAC;QAEF,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAC;QACzD,kBAAkB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAEhE,iDAAiD;QACjD,IAAI,CAAC,kBAAkB,EAAE;YACvB,MAAM,IAAI,CAAC,cAAc,CACvB,YAAY,EACZ,mBAAmB;YACnB,4DAA4D;YAC5D,KAAK,EAAE,EAAkB,EAAE,EAAE;gBAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAClD,EAAE,EACF,IAAI,CAAC,UAAU,CAAC,GAAG,EACnB,YAAY,CACb,CAAC;gBAEF,uCAAuC;gBACvC,EAAE,CAAC,MAAM,CACP,KAAK,EAAE,YAAoB,EAAE,IAA8B,EAAE,EAAE;oBAC7D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,uBAAuB,CAC/D;wBACE,YAAY;wBACZ,IAAI;wBACJ,UAAU,EAAE,WAAW;qBACxB,CACF,CAAC;oBACF,+FAA+F;oBAC/F,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAChC,gBAAgB,GAAG,IAAI,CAAC;gBAC1B,CAAC,EACD,yBAAyB,CAC1B,CAAC;gBAEF,OAAO,EAAE,CAAC;YACZ,CAAC,CACF,CAAC;SACH;QAED,OAAO;YACL,gBAAgB;YAChB,SAAS,EAAE,KAAK,CAAC,IAAI;YACrB,kBAAkB;SACnB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,KAAoB;QAEpB,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE;YACrC,OAAO,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;SACtE;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iBAAiB,CACvB,oBAA4B,EAC5B,WAA6B;QAE7B,IAAI,UAA4B,CAAC;QAEjC,UAAU,GAAG,WAAW,CAAC,MAAM,CAC7B,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,oBAAoB,CAC9C,CAAC;QAEF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,+BAA+B;QAC/B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;YACpC,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;gBAClB,OAAO,IAAI,CAAC,kBAAkB;qBAC3B,cAAc,CAAC,EAAE,CAAC;qBAClB,iBAAiB,CAAC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aAC/C;iBAAM;gBACL,OAAO,IAAI,CAAC;aACb;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;YACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,iBAAiB,CAAC,EAAoB,EAAE,WAAmB;QACjE,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;YACd,MAAM,CAAC,KAAK,CACV,0IAA0I,WAAW;yFACpE,CAClF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,EAAE,KAAK,EAAE,YAAY,EAAwB,EAC7C,WAA6B,EAC7B,KAAqD;;QAErD,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAExD,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE;YAC7B,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAEpD,KAAK,MAAM,GAAG,IAAI,MAAA,EAAE,CAAC,IAAI,mCAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;aAChD;SACF;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,KAAoB,EACpB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACnE;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,EAAuB,EACvB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SAC/D;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,GAAgB,EAChB,WAA6B,EAC7B,KAAqD;QAErD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;YAC5B,MAAM,IAAI,CAAC,SAAS,CAAC,qCAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;SACjE;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,IAAO,EACP,IAAuC,EACvC,EAAkB,EAClB,KAAsD;QAEtD,IAAI,EAAkB,CAAC;QACvB,IAAI,IAAA,6BAAW,EAAC,EAAE,CAAC,EAAE;YACnB,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,QAAkC,CAAC,MAAM,CACpE,CAAC,CAAC,EAAE,EAAE;;gBACJ,OAAA,CAAC,CAAC,IAAI,KAAK,IAAI;oBACf,aAAa,CAAC,IAAI,CAAC,CACjB,IAAW,EACX,CAAC,CAAC,MAAa,EACf,MAAA,EAAE,CAAC,OAAO,0CAAE,OAAO,CACpB,CAAA;aAAA,CACJ,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YACD,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,IAAI,CAAC,UAAU,CAAC,QAAQ;oBACtB,CAAC,CAAC,MAAM,IAAA,wBAAY,EAChB,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,EACvB,oBAAoB,EACpB,OAAO,CAAC,OAAO,CAChB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;oBAClC,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;aACzD;SACF;aAAM,IAAI,IAAA,4BAAU,EAAC,EAAE,CAAC,EAAE;YACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAC1C,EAAE,EACF,IAAI,EACJ,gBAAgB,CAAC,IAAI,CAAC,EACtB,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;gBACnB,QAAQ,IAAI,EAAE;oBACZ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,qBAAqB,CAC/C,IAAqB,EACrB,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,IAAI;wBAC3B,OAAO,qCAAoB,CAAC,2BAA2B,CACrD,IAA2B,EAC3B,UAAU,CACX,CAAC;oBACJ,KAAK,qCAAmB,CAAC,KAAK;wBAC5B,OAAO,qCAAoB,CAAC,mBAAmB,CAC7C,IAAmB,EACnB,UAAU,CACX,CAAC;oBACJ;wBACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;iBAC/C;YACH,CAAC,CACF,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBACpB,OAAO;aACR;YAED,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC/D,IAAI,EACJ,EAAE,CACH,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC9B,EAAE,GAAG,EAAE,aAAF,EAAE,cAAF,EAAE,GAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,MAAM,IAAI,CAAC,2BAA2B,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;aACrE;SACF;IACH,CAAC;IAEO,sBAAsB,CAC5B,EAA8C,EAC9C,IAAuC,EACvC,gBAAqC,EACrC,UAGY;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAE1D,OAAO,EAAE,CAAC,OAAO,CAAC,QAAQ;aACvB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,IAAI,gBAAgB,CAAC,SAAS,CAAC,EAAE;gBAC/B,SAAS,CAAC,UAAU,CAAC;gBAErB,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;aAC/C;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;YAEF,IAAI;gBACF,OAAO,SAAS,CAAC,eAAe,CAAC;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,IAAI;oBACX,EAAE;iBACH,CAAC,CAAC;aACJ;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;gBACtD,MAAM,CAAC,CAAC;aACT;QACH,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,EAA8C,EAC9C,EAAkB,EAClB,OAA2B,EAC3B,IAAuC;QAEvC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAE3D,MAAM,SAAS,GAAG,IAAA,0DAAmC,EACnD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CACvC,CAAC;QAEF,MAAM,eAAe,GAAG,MAAM,SAAS;aACpC,WAAW,CAAC;YACX,KAAK,EAAE,IAAI;YACX,EAAE;YACF,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG;YACxB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM;SACP,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,6CAA6C,CAAC,CAAC;YAC/D,MAAM,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEL,+FAA+F;QAC/F,+FAA+F;QAC/F,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE;YACnC,MAAM,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;SAChD;IACH,CAAC;CACF,CAAA;AAvTO;IADL,IAAA,oBAAQ,EAAC,oBAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;;;;gDAyDpC;AAxEU,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAWR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;qCANN,sBAAU;QACV,sBAAU;QACN,gCAAc;QACV,yCAAkB;QACpB,qCAAgB;QACR,oDAAwB;QACP,gCAAc;GAVxD,cAAc,CAuU1B;AAvUY,wCAAc;AA+U3B,MAAM,gBAAgB,GAAG;IACvB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,yCAAuB;IACpD,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,wCAAsB;CACnD,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,qBAAqB;IACvE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,qCAAoB,CAAC,mBAAmB;IACrE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,qCAAoB,CAAC,2BAA2B;CAC7E,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAmB,EAAE,EAAE,CAAC,IAAI;IAChE,CAAC,qCAAmB,CAAC,KAAK,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;IACzE,CAAC,qCAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,GAAgB,EAAE,EAAE,CAC/C,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC;CACjC,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  isBlockHandlerProcessor,\n  isCallHandlerProcessor,\n  isEventHandlerProcessor,\n  isCustomDs,\n  isRuntimeDs,\n  SubqlEthereumCustomDataSource,\n  SubqlCustomHandler,\n  EthereumHandlerKind,\n  EthereumRuntimeHandlerInputMap,\n} from '@subql/common-ethereum';\nimport {\n  NodeConfig,\n  getLogger,\n  profiler,\n  profilerWrap,\n  IndexerSandbox,\n  ProcessBlockResponse,\n  ApiService,\n  IIndexerManager,\n} from '@subql/node-core';\nimport {\n  EthereumTransaction,\n  EthereumLog,\n  SubqlRuntimeHandler,\n  EthereumBlockWrapper,\n  EthereumBlock,\n} from '@subql/types-ethereum';\nimport { SubqlProjectDs } from '../configure/SubqueryProject';\nimport { EthereumApi } from '../ethereum';\nimport { EthereumBlockWrapped } from '../ethereum/block.ethereum';\nimport { yargsOptions } from '../yargs';\nimport {\n  asSecondLayerHandlerProcessor_1_0_0,\n  DsProcessorService,\n} from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\nconst logger = getLogger('indexer');\n\n@Injectable()\nexport class IndexerManager\n  implements IIndexerManager<EthereumBlockWrapper, SubqlProjectDs>\n{\n  constructor(\n    private apiService: ApiService,\n    private nodeConfig: NodeConfig,\n    private sandboxService: SandboxService,\n    private dsProcessorService: DsProcessorService,\n    private dynamicDsService: DynamicDsService,\n    private unfinalizedBlocksService: UnfinalizedBlocksService,\n    @Inject('IProjectService') private projectService: ProjectService,\n  ) {\n    logger.info('indexer manager start');\n  }\n\n  @profiler(yargsOptions.argv.profiler)\n  async indexBlock(\n    blockContent: EthereumBlockWrapper,\n    dataSources: SubqlProjectDs[],\n  ): Promise<ProcessBlockResponse> {\n    const { block, blockHeight } = blockContent;\n    let dynamicDsCreated = false;\n    let reindexBlockHeight: number | null = null;\n\n    const filteredDataSources = this.filterDataSources(\n      blockHeight,\n      dataSources,\n    );\n\n    this.assertDataSources(filteredDataSources, blockHeight);\n    reindexBlockHeight = await this.processUnfinalizedBlocks(block);\n\n    // Only index block if we're not going to reindex\n    if (!reindexBlockHeight) {\n      await this.indexBlockData(\n        blockContent,\n        filteredDataSources,\n        // eslint-disable-next-line @typescript-eslint/require-await\n        async (ds: SubqlProjectDs) => {\n          const vm = this.sandboxService.getDsProcessorWrapper(\n            ds,\n            this.apiService.api,\n            blockContent,\n          );\n\n          // Inject function to create ds into vm\n          vm.freeze(\n            async (templateName: string, args?: Record<string, unknown>) => {\n              const newDs = await this.dynamicDsService.createDynamicDatasource(\n                {\n                  templateName,\n                  args,\n                  startBlock: blockHeight,\n                },\n              );\n              // Push the newly created dynamic ds to be processed this block on any future extrinsics/events\n              filteredDataSources.push(newDs);\n              dynamicDsCreated = true;\n            },\n            'createDynamicDatasource',\n          );\n\n          return vm;\n        },\n      );\n    }\n\n    return {\n      dynamicDsCreated,\n      blockHash: block.hash,\n      reindexBlockHeight,\n    };\n  }\n\n  async start(): Promise<void> {\n    await this.projectService.init();\n    logger.info('indexer manager started');\n  }\n\n  private async processUnfinalizedBlocks(\n    block: EthereumBlock,\n  ): Promise<number | null> {\n    if (this.nodeConfig.unfinalizedBlocks) {\n      return this.unfinalizedBlocksService.processUnfinalizedBlocks(block);\n    }\n    return null;\n  }\n\n  private filterDataSources(\n    nextProcessingHeight: number,\n    dataSources: SubqlProjectDs[],\n  ): SubqlProjectDs[] {\n    let filteredDs: SubqlProjectDs[];\n\n    filteredDs = dataSources.filter(\n      (ds) => ds.startBlock <= nextProcessingHeight,\n    );\n\n    if (filteredDs.length === 0) {\n      logger.error(`Did not find any matching datasouces`);\n      process.exit(1);\n    }\n    // perform filter for custom ds\n    filteredDs = filteredDs.filter((ds) => {\n      if (isCustomDs(ds)) {\n        return this.dsProcessorService\n          .getDsProcessor(ds)\n          .dsFilterProcessor(ds, this.apiService.api);\n      } else {\n        return true;\n      }\n    });\n\n    if (!filteredDs.length) {\n      logger.error(`Did not find any datasources with associated processor`);\n      process.exit(1);\n    }\n    return filteredDs;\n  }\n\n  private assertDataSources(ds: SubqlProjectDs[], blockHeight: number) {\n    if (!ds.length) {\n      logger.error(\n        `Your start block is greater than the current indexed block height in your database. Either change your startBlock (project.yaml) to <= ${blockHeight}\n         or delete your database and start again from the currently specified startBlock`,\n      );\n      process.exit(1);\n    }\n  }\n\n  private async indexBlockData(\n    { block, transactions }: EthereumBlockWrapper,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    await this.indexBlockContent(block, dataSources, getVM);\n\n    for (const tx of transactions) {\n      await this.indexTransaction(tx, dataSources, getVM);\n\n      for (const log of tx.logs ?? []) {\n        await this.indexEvent(log, dataSources, getVM);\n      }\n    }\n  }\n\n  private async indexBlockContent(\n    block: EthereumBlock,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Block, block, ds, getVM);\n    }\n  }\n\n  private async indexTransaction(\n    tx: EthereumTransaction,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Call, tx, ds, getVM);\n    }\n  }\n\n  private async indexEvent(\n    log: EthereumLog,\n    dataSources: SubqlProjectDs[],\n    getVM: (d: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    for (const ds of dataSources) {\n      await this.indexData(EthereumHandlerKind.Event, log, ds, getVM);\n    }\n  }\n\n  private async indexData<K extends EthereumHandlerKind>(\n    kind: K,\n    data: EthereumRuntimeHandlerInputMap[K],\n    ds: SubqlProjectDs,\n    getVM: (ds: SubqlProjectDs) => Promise<IndexerSandbox>,\n  ): Promise<void> {\n    let vm: IndexerSandbox;\n    if (isRuntimeDs(ds)) {\n      const handlers = (ds.mapping.handlers as SubqlRuntimeHandler[]).filter(\n        (h) =>\n          h.kind === kind &&\n          FilterTypeMap[kind](\n            data as any,\n            h.filter as any,\n            ds.options?.address,\n          ),\n      );\n\n      if (!handlers.length) {\n        return;\n      }\n      const parsedData = await DataAbiParser[kind](this.apiService.api)(\n        data,\n        ds,\n      );\n\n      for (const handler of handlers) {\n        vm = vm ?? (await getVM(ds));\n        this.nodeConfig.profiler\n          ? await profilerWrap(\n              vm.securedExec.bind(vm),\n              'handlerPerformance',\n              handler.handler,\n            )(handler.handler, [parsedData])\n          : await vm.securedExec(handler.handler, [parsedData]);\n      }\n    } else if (isCustomDs(ds)) {\n      const handlers = this.filterCustomDsHandlers<K>(\n        ds,\n        data,\n        ProcessorTypeMap[kind],\n        (data, baseFilter) => {\n          switch (kind) {\n            case EthereumHandlerKind.Block:\n              return EthereumBlockWrapped.filterBlocksProcessor(\n                data as EthereumBlock,\n                baseFilter,\n              );\n            case EthereumHandlerKind.Call:\n              return EthereumBlockWrapped.filterTransactionsProcessor(\n                data as EthereumTransaction,\n                baseFilter,\n              );\n            case EthereumHandlerKind.Event:\n              return EthereumBlockWrapped.filterLogsProcessor(\n                data as EthereumLog,\n                baseFilter,\n              );\n            default:\n              throw new Error('Unsupported handler kind');\n          }\n        },\n      );\n\n      if (!handlers.length) {\n        return;\n      }\n\n      const parsedData = await DataAbiParser[kind](this.apiService.api)(\n        data,\n        ds,\n      );\n\n      for (const handler of handlers) {\n        vm = vm ?? (await getVM(ds));\n        await this.transformAndExecuteCustomDs(ds, vm, handler, parsedData);\n      }\n    }\n  }\n\n  private filterCustomDsHandlers<K extends EthereumHandlerKind>(\n    ds: SubqlEthereumCustomDataSource<string, any>,\n    data: EthereumRuntimeHandlerInputMap[K],\n    baseHandlerCheck: ProcessorTypeMap[K],\n    baseFilter: (\n      data: EthereumRuntimeHandlerInputMap[K],\n      baseFilter: any,\n    ) => boolean,\n  ): SubqlCustomHandler[] {\n    const plugin = this.dsProcessorService.getDsProcessor(ds);\n\n    return ds.mapping.handlers\n      .filter((handler) => {\n        const processor = plugin.handlerProcessors[handler.kind];\n        if (baseHandlerCheck(processor)) {\n          processor.baseFilter;\n\n          return baseFilter(data, processor.baseFilter);\n        }\n        return false;\n      })\n      .filter((handler) => {\n        const processor = asSecondLayerHandlerProcessor_1_0_0(\n          plugin.handlerProcessors[handler.kind],\n        );\n\n        try {\n          return processor.filterProcessor({\n            filter: handler.filter,\n            input: data,\n            ds,\n          });\n        } catch (e) {\n          logger.error(e, 'Failed to run ds processer filter.');\n          throw e;\n        }\n      });\n  }\n\n  private async transformAndExecuteCustomDs<K extends EthereumHandlerKind>(\n    ds: SubqlEthereumCustomDataSource<string, any>,\n    vm: IndexerSandbox,\n    handler: SubqlCustomHandler,\n    data: EthereumRuntimeHandlerInputMap[K],\n  ): Promise<void> {\n    const plugin = this.dsProcessorService.getDsProcessor(ds);\n    const assets = await this.dsProcessorService.getAssets(ds);\n\n    const processor = asSecondLayerHandlerProcessor_1_0_0(\n      plugin.handlerProcessors[handler.kind],\n    );\n\n    const transformedData = await processor\n      .transformer({\n        input: data,\n        ds,\n        api: this.apiService.api,\n        filter: handler.filter,\n        assets,\n      })\n      .catch((e) => {\n        logger.error(e, 'Failed to transform data with ds processor.');\n        throw e;\n      });\n\n    // We can not run this in parallel. the transformed data items may be dependent on one another.\n    // An example of this is with Acala EVM packing multiple EVM logs into a single Substrate event\n    for (const _data of transformedData) {\n      await vm.securedExec(handler.handler, [_data]);\n    }\n  }\n}\n\ntype ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: typeof isBlockHandlerProcessor;\n  [EthereumHandlerKind.Event]: typeof isEventHandlerProcessor;\n  [EthereumHandlerKind.Call]: typeof isCallHandlerProcessor;\n};\n\nconst ProcessorTypeMap = {\n  [EthereumHandlerKind.Block]: isBlockHandlerProcessor,\n  [EthereumHandlerKind.Event]: isEventHandlerProcessor,\n  [EthereumHandlerKind.Call]: isCallHandlerProcessor,\n};\n\nconst FilterTypeMap = {\n  [EthereumHandlerKind.Block]: EthereumBlockWrapped.filterBlocksProcessor,\n  [EthereumHandlerKind.Event]: EthereumBlockWrapped.filterLogsProcessor,\n  [EthereumHandlerKind.Call]: EthereumBlockWrapped.filterTransactionsProcessor,\n};\n\nconst DataAbiParser = {\n  [EthereumHandlerKind.Block]: () => (data: EthereumBlock) => data,\n  [EthereumHandlerKind.Event]: (api: EthereumApi) => api.parseLog.bind(api),\n  [EthereumHandlerKind.Call]: (api: EthereumApi) =>\n    api.parseTransaction.bind(api),\n};\n"]}

package/dist/indexer/project.service.js

@@ -32,7 +32,6 @@ const dynamic_ds_service_1 = require("./dynamic-ds.service");
 const unfinalizedBlocks_service_1 = require("./unfinalizedBlocks.service");
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { version: packageVersion } = require('../../package.json');
-const DEFAULT_DB_SCHEMA = 'public';
 const logger = (0, node_core_1.getLogger)('Project');
 let ProjectService = class ProjectService {
     constructor(dsProcessorService, apiService, poiService, mmrService, sequelize, project, storeService, nodeConfig, dynamicDsService, eventEmitter, unfinalizedBlockService) {
@@ -115,17 +114,10 @@ let ProjectService = class ProjectService {
         return schema;
     }
     async createProjectSchema() {
-        let schema;
-        if (this.nodeConfig.localMode) {
-            // create tables in default schema if local mode is enabled
-            schema = DEFAULT_DB_SCHEMA;
-        }
-        else {
-            schema = this.nodeConfig.dbSchema;
-            const schemas = await this.sequelize.showAllSchemas(undefined);
-            if (!schemas.includes(schema)) {
-                await this.sequelize.createSchema(`"${schema}"`, undefined);
-            }
+        const schema = this.nodeConfig.dbSchema;
+        const schemas = await this.sequelize.showAllSchemas(undefined);
+        if (!schemas.includes(schema)) {
+            await this.sequelize.createSchema(`"${schema}"`, undefined);
         }
         return schema;
     }

package/dist/indexer/project.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAAoD;AACpD,yDAAsD;AACtD,gDAU0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAqE;AACrE,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AACxD,2EAAuE;AAEvE,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAKzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B,EAC3B,uBAAiD;QAVxC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACQ,YAAO,GAAP,OAAO,CAAiB;QACpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,4BAAuB,GAAvB,uBAAuB,CAA0B;IACxD,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAElE,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAEhD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;gBAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CACxB,CAAC;YAEF,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC7B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;aACjC;YAED,2CAA2C;YAC3C,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;SACrD;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAErD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5D,CAAC;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAEvD,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,aAAa;YACb,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;SACd,CAAC;QAEX,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE/C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE5C,QAAQ,CAAC,OAAO,CAAC;gBACf,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBACvC,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;gBACjD,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;gBACzC,EAAE,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;aACpD,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;SAC1C;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SAC9B;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;SACpC;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,QAAQ,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;SACxC;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,QAAQ,CAAC,GAAG,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC;SACzC;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC;SAClE;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY,KAAK,SAAS;YAC/B,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,uBAAuB,EAC5B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;QAEtE,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,CAAC,MAAM,CAC/C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,WAAW,CACrC,CAAC;IACJ,CAAC;CACF,CAAA;AArRY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAYR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALU,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;QACF,oDAAwB;GAhBhD,cAAc,CAqR1B;AArRY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  IProjectService,\n  NodeConfig,\n  IndexerEvent,\n  StoreService,\n  PoiService,\n  MmrService,\n  getLogger,\n  getExistingProjectSchema,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n  generateTimestampReferenceForBlockFilters,\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema, initHotSchemaReload } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService implements IProjectService<SubqlProjectDs> {\n  private _schema: string;\n  private _startHeight: number;\n  private _blockOffset: number;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly apiService: ApiService,\n    private readonly poiService: PoiService,\n    protected readonly mmrService: MmrService,\n    private readonly sequelize: Sequelize,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly dynamicDsService: DynamicDsService,\n    private eventEmitter: EventEmitter2,\n    private unfinalizedBlockService: UnfinalizedBlocksService,\n  ) {}\n\n  get schema(): string {\n    return this._schema;\n  }\n\n  get dataSources(): SubqlProjectDs[] {\n    return this.project.dataSources;\n  }\n\n  get blockOffset(): number {\n    return this._blockOffset;\n  }\n\n  get startHeight(): number {\n    return this._startHeight;\n  }\n\n  get isHistorical(): boolean {\n    return this.storeService.historical;\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  async init(): Promise<void> {\n    // Used to load assets into DS-processor, has to be done in any thread\n    await this.dsProcessorService.validateProjectCustomDatasources();\n    // Do extra work on main thread to setup stuff\n    this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n      this.project.dataSources,\n      this.apiService.api,\n    );\n    if (isMainThread) {\n      this._schema = await this.ensureProject();\n      await this.initDbSchema();\n      await this.ensureMetadata();\n      this.dynamicDsService.init(this.storeService.storeCache.metadata);\n\n      await this.initHotSchemaReload();\n\n      if (this.nodeConfig.proofOfIndex) {\n        const blockOffset = await this.getMetadataBlockOffset();\n        void this.setBlockOffset(Number(blockOffset));\n        await this.poiService.init();\n      }\n\n      this._startHeight = await this.getStartHeight();\n\n      if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n        logger.error(\n          'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n        );\n        process.exit(1);\n      }\n\n      const reindexedTo = await this.unfinalizedBlockService.init(\n        this.reindex.bind(this),\n      );\n\n      if (reindexedTo !== undefined) {\n        this._startHeight = reindexedTo;\n      }\n\n      // Flush any pending operations to setup DB\n      await this.storeService.storeCache.flushCache(true);\n    } else {\n      this._schema = await this.getExistingProjectSchema();\n\n      await this.sequelize.sync();\n\n      assert(this._schema, 'Schema should be created in main thread');\n      await this.initDbSchema();\n\n      if (this.nodeConfig.proofOfIndex) {\n        await this.poiService.init();\n      }\n    }\n  }\n\n  private async ensureProject(): Promise<string> {\n    let schema = await this.getExistingProjectSchema();\n    if (!schema) {\n      schema = await this.createProjectSchema();\n    }\n    this.eventEmitter.emit(IndexerEvent.Ready, {\n      value: true,\n    });\n\n    return schema;\n  }\n\n  private async createProjectSchema(): Promise<string> {\n    let schema: string;\n    if (this.nodeConfig.localMode) {\n      // create tables in default schema if local mode is enabled\n      schema = DEFAULT_DB_SCHEMA;\n    } else {\n      schema = this.nodeConfig.dbSchema;\n      const schemas = await this.sequelize.showAllSchemas(undefined);\n      if (!(schemas as unknown as string[]).includes(schema)) {\n        await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n      }\n    }\n\n    return schema;\n  }\n\n  private async initHotSchemaReload(): Promise<void> {\n    await initHotSchemaReload(this.schema, this.storeService);\n  }\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private async ensureMetadata(): Promise<void> {\n    const metadata = this.storeService.storeCache.metadata;\n\n    this.eventEmitter.emit(\n      IndexerEvent.NetworkMetadata,\n      this.apiService.networkMeta,\n    );\n\n    const keys = [\n      'lastProcessedHeight',\n      'blockOffset',\n      'indexerNodeVersion',\n      'chain',\n      'specName',\n      'genesisHash',\n      'startHeight',\n      'processedBlockCount',\n      'lastFinalizedVerifiedHeight',\n      'schemaMigrationCount',\n    ] as const;\n\n    const existing = await metadata.findMany(keys);\n\n    const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n    if (this.project.runner) {\n      const { node, query } = this.project.runner;\n\n      metadata.setBulk([\n        { key: 'runnerNode', value: node.name },\n        { key: 'runnerNodeVersion', value: node.version },\n        { key: 'runnerQuery', value: query.name },\n        { key: 'runnerQueryVersion', value: query.version },\n      ]);\n    }\n    if (!existing.genesisHash) {\n      metadata.set('genesisHash', genesisHash);\n    } else {\n      // Check if the configured genesisHash matches the currently stored genesisHash\n      assert(\n        // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n        (this.project.network.genesisHash ?? genesisHash) ===\n          existing.genesisHash,\n        'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n      );\n    }\n    if (existing.chain !== chain) {\n      metadata.set('chain', chain);\n    }\n\n    if (existing.specName !== specName) {\n      metadata.set('specName', specName);\n    }\n\n    // If project was created before this feature, don't add the key. If it is project created after, add this key.\n    if (!existing.processedBlockCount && !existing.lastProcessedHeight) {\n      metadata.set('processedBlockCount', 0);\n    }\n\n    if (existing.indexerNodeVersion !== packageVersion) {\n      metadata.set('indexerNodeVersion', packageVersion);\n    }\n    if (!existing.schemaMigrationCount) {\n      metadata.set('schemaMigrationCount', 0);\n    }\n    if (!existing.startHeight) {\n      metadata.set('startHeight', this.getStartBlockFromDataSources());\n    }\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('blockOffset');\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('lastProcessedHeight');\n  }\n\n  private async getStartHeight(): Promise<number> {\n    let startHeight: number;\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n    if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n      startHeight = Number(lastProcessedHeight) + 1;\n    } else {\n      startHeight = this.getStartBlockFromDataSources();\n    }\n    return startHeight;\n  }\n\n  async setBlockOffset(offset: number): Promise<void> {\n    if (\n      this._blockOffset !== undefined ||\n      offset === null ||\n      offset === undefined ||\n      isNaN(offset)\n    ) {\n      return;\n    }\n    logger.info(`set blockOffset to ${offset}`);\n    this._blockOffset = offset;\n    return this.mmrService.syncFileBaseFromPoi(offset).catch((err) => {\n      logger.error(err, 'failed to sync poi to mmr');\n      process.exit(1);\n    });\n  }\n\n  private getStartBlockFromDataSources() {\n    const startBlocksList = this.project.dataSources.map(\n      (item) => item.startBlock ?? 1,\n    );\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n\n    return reindex(\n      this.getStartBlockFromDataSources(),\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlockService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      /* Not providing force clean service, it should never be needed */\n    );\n  }\n\n  async getAllDataSources(blockHeight: number): Promise<SubqlProjectDs[]> {\n    const dynamicDs = await this.dynamicDsService.getDynamicDatasources();\n\n    return [...this.dataSources, ...dynamicDs].filter(\n      (ds) => ds.startBlock <= blockHeight,\n    );\n  }\n}\n"]}
+{"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAAoD;AACpD,yDAAsD;AACtD,gDAU0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAqE;AACrE,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AACxD,2EAAuE;AAEvE,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAKzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B,EAC3B,uBAAiD;QAVxC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACQ,YAAO,GAAP,OAAO,CAAiB;QACpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,4BAAuB,GAAvB,uBAAuB,CAA0B;IACxD,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAElE,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAEhD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;gBAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CACxB,CAAC;YAEF,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC7B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;aACjC;YAED,2CAA2C;YAC3C,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;SACrD;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAErD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;SAC7D;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5D,CAAC;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAEvD,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,aAAa;YACb,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;SACd,CAAC;QAEX,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE/C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE5C,QAAQ,CAAC,OAAO,CAAC;gBACf,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBACvC,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;gBACjD,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;gBACzC,EAAE,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;aACpD,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;SAC1C;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SAC9B;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;SACpC;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,QAAQ,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;SACxC;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,QAAQ,CAAC,GAAG,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC;SACzC;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC;SAClE;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY,KAAK,SAAS;YAC/B,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,uBAAuB,EAC5B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;QAEtE,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,CAAC,MAAM,CAC/C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,WAAW,CACrC,CAAC;IACJ,CAAC;CACF,CAAA;AA9QY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAYR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALU,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;QACF,oDAAwB;GAhBhD,cAAc,CA8Q1B;AA9QY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  IProjectService,\n  NodeConfig,\n  IndexerEvent,\n  StoreService,\n  PoiService,\n  MmrService,\n  getLogger,\n  getExistingProjectSchema,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n  generateTimestampReferenceForBlockFilters,\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema, initHotSchemaReload } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService implements IProjectService<SubqlProjectDs> {\n  private _schema: string;\n  private _startHeight: number;\n  private _blockOffset: number;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly apiService: ApiService,\n    private readonly poiService: PoiService,\n    protected readonly mmrService: MmrService,\n    private readonly sequelize: Sequelize,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly dynamicDsService: DynamicDsService,\n    private eventEmitter: EventEmitter2,\n    private unfinalizedBlockService: UnfinalizedBlocksService,\n  ) {}\n\n  get schema(): string {\n    return this._schema;\n  }\n\n  get dataSources(): SubqlProjectDs[] {\n    return this.project.dataSources;\n  }\n\n  get blockOffset(): number {\n    return this._blockOffset;\n  }\n\n  get startHeight(): number {\n    return this._startHeight;\n  }\n\n  get isHistorical(): boolean {\n    return this.storeService.historical;\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  async init(): Promise<void> {\n    // Used to load assets into DS-processor, has to be done in any thread\n    await this.dsProcessorService.validateProjectCustomDatasources();\n    // Do extra work on main thread to setup stuff\n    this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n      this.project.dataSources,\n      this.apiService.api,\n    );\n    if (isMainThread) {\n      this._schema = await this.ensureProject();\n      await this.initDbSchema();\n      await this.ensureMetadata();\n      this.dynamicDsService.init(this.storeService.storeCache.metadata);\n\n      await this.initHotSchemaReload();\n\n      if (this.nodeConfig.proofOfIndex) {\n        const blockOffset = await this.getMetadataBlockOffset();\n        void this.setBlockOffset(Number(blockOffset));\n        await this.poiService.init();\n      }\n\n      this._startHeight = await this.getStartHeight();\n\n      if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n        logger.error(\n          'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n        );\n        process.exit(1);\n      }\n\n      const reindexedTo = await this.unfinalizedBlockService.init(\n        this.reindex.bind(this),\n      );\n\n      if (reindexedTo !== undefined) {\n        this._startHeight = reindexedTo;\n      }\n\n      // Flush any pending operations to setup DB\n      await this.storeService.storeCache.flushCache(true);\n    } else {\n      this._schema = await this.getExistingProjectSchema();\n\n      await this.sequelize.sync();\n\n      assert(this._schema, 'Schema should be created in main thread');\n      await this.initDbSchema();\n\n      if (this.nodeConfig.proofOfIndex) {\n        await this.poiService.init();\n      }\n    }\n  }\n\n  private async ensureProject(): Promise<string> {\n    let schema = await this.getExistingProjectSchema();\n    if (!schema) {\n      schema = await this.createProjectSchema();\n    }\n    this.eventEmitter.emit(IndexerEvent.Ready, {\n      value: true,\n    });\n\n    return schema;\n  }\n\n  private async createProjectSchema(): Promise<string> {\n    const schema = this.nodeConfig.dbSchema;\n    const schemas = await this.sequelize.showAllSchemas(undefined);\n    if (!(schemas as unknown as string[]).includes(schema)) {\n      await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n    }\n    return schema;\n  }\n\n  private async initHotSchemaReload(): Promise<void> {\n    await initHotSchemaReload(this.schema, this.storeService);\n  }\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private async ensureMetadata(): Promise<void> {\n    const metadata = this.storeService.storeCache.metadata;\n\n    this.eventEmitter.emit(\n      IndexerEvent.NetworkMetadata,\n      this.apiService.networkMeta,\n    );\n\n    const keys = [\n      'lastProcessedHeight',\n      'blockOffset',\n      'indexerNodeVersion',\n      'chain',\n      'specName',\n      'genesisHash',\n      'startHeight',\n      'processedBlockCount',\n      'lastFinalizedVerifiedHeight',\n      'schemaMigrationCount',\n    ] as const;\n\n    const existing = await metadata.findMany(keys);\n\n    const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n    if (this.project.runner) {\n      const { node, query } = this.project.runner;\n\n      metadata.setBulk([\n        { key: 'runnerNode', value: node.name },\n        { key: 'runnerNodeVersion', value: node.version },\n        { key: 'runnerQuery', value: query.name },\n        { key: 'runnerQueryVersion', value: query.version },\n      ]);\n    }\n    if (!existing.genesisHash) {\n      metadata.set('genesisHash', genesisHash);\n    } else {\n      // Check if the configured genesisHash matches the currently stored genesisHash\n      assert(\n        // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n        (this.project.network.genesisHash ?? genesisHash) ===\n          existing.genesisHash,\n        'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n      );\n    }\n    if (existing.chain !== chain) {\n      metadata.set('chain', chain);\n    }\n\n    if (existing.specName !== specName) {\n      metadata.set('specName', specName);\n    }\n\n    // If project was created before this feature, don't add the key. If it is project created after, add this key.\n    if (!existing.processedBlockCount && !existing.lastProcessedHeight) {\n      metadata.set('processedBlockCount', 0);\n    }\n\n    if (existing.indexerNodeVersion !== packageVersion) {\n      metadata.set('indexerNodeVersion', packageVersion);\n    }\n    if (!existing.schemaMigrationCount) {\n      metadata.set('schemaMigrationCount', 0);\n    }\n    if (!existing.startHeight) {\n      metadata.set('startHeight', this.getStartBlockFromDataSources());\n    }\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('blockOffset');\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('lastProcessedHeight');\n  }\n\n  private async getStartHeight(): Promise<number> {\n    let startHeight: number;\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n    if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n      startHeight = Number(lastProcessedHeight) + 1;\n    } else {\n      startHeight = this.getStartBlockFromDataSources();\n    }\n    return startHeight;\n  }\n\n  async setBlockOffset(offset: number): Promise<void> {\n    if (\n      this._blockOffset !== undefined ||\n      offset === null ||\n      offset === undefined ||\n      isNaN(offset)\n    ) {\n      return;\n    }\n    logger.info(`set blockOffset to ${offset}`);\n    this._blockOffset = offset;\n    return this.mmrService.syncFileBaseFromPoi(offset).catch((err) => {\n      logger.error(err, 'failed to sync poi to mmr');\n      process.exit(1);\n    });\n  }\n\n  private getStartBlockFromDataSources() {\n    const startBlocksList = this.project.dataSources.map(\n      (item) => item.startBlock ?? 1,\n    );\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n\n    return reindex(\n      this.getStartBlockFromDataSources(),\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlockService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      /* Not providing force clean service, it should never be needed */\n    );\n  }\n\n  async getAllDataSources(blockHeight: number): Promise<SubqlProjectDs[]> {\n    const dynamicDs = await this.dynamicDsService.getDynamicDatasources();\n\n    return [...this.dataSources, ...dynamicDs].filter(\n      (ds) => ds.startBlock <= blockHeight,\n    );\n  }\n}\n"]}

package/dist/subcommands/reindex.service.js

@@ -89,7 +89,7 @@ let ReindexService = class ReindexService {
             this.getLastProcessedHeight(),
         ]);
         await (0, reindex_1.reindex)(startHeight, await this.getMetadataBlockOffset(), targetBlockHeight, lastProcessedHeight, this.storeService, this.unfinalizedBlocksService, this.dynamicDsService, this.mmrService, this.sequelize, this.forceCleanService);
-        await this.storeService.storeCache.flushCache();
+        await this.storeService.storeCache.flushCache(true, true);
     }
 };
 ReindexService = __decorate([

package/dist/subcommands/reindex.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAO0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,oFAAgF;AAChF,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAwB,EACpD,iBAAoC,EACpC,wBAAkD,EAClD,gBAAkC;QAPlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACpD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAE1D,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,oCAAoC,CACxC,WAAmB;QAEnB,wBAAwB;QACvB,IAAI,CAAC,wBAAgC,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACxE,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,wBAAwB,CAAC,4BAA4B,EAAE,CAAC;QACrE,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CACzC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAC9D,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,OAAO,WAAW,CAAC;SACpB;QACD,MAAM,CAAC,cAAc,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,4BAA4B;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,MAAM,IAAA,iBAAO,EACX,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,wBAAwB,EAC7B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;QACF,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;IAClD,CAAC;CACF,CAAA;AApGY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAJC,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACe,iCAAe;QACjC,sCAAiB;QACV,oDAAwB;QAChC,qCAAgB;GAZ1C,cAAc,CAoG1B;AApGY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  CacheMetadataModel,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: CacheMetadataModel;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly unfinalizedBlocksService: UnfinalizedBlocksService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n\n    this.metadataRepo = this.storeService.storeCache.metadata;\n\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  async getTargetHeightWithUnfinalizedBlocks(\n    inputHeight: number,\n  ): Promise<number> {\n    // Why does this happen?\n    (this.unfinalizedBlocksService as any).metadataRepo = this.metadataRepo;\n    const unfinalizedBlocks =\n      await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();\n    const bestBlocks = unfinalizedBlocks.filter(\n      ([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight,\n    );\n    if (bestBlocks.length === 0) {\n      return inputHeight;\n    }\n    const [firstBestBlock] = bestBlocks[0];\n    return Math.min(inputHeight, firstBestBlock);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.metadataRepo.find('lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.metadataRepo.find('blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    return this.metadataRepo.find('specName');\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private getStartBlockFromDataSources(): number {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    await reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlocksService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n    await this.storeService.storeCache.flushCache();\n  }\n}\n"]}
+{"version":3,"file":"reindex.service.js","sourceRoot":"","sources":["../../src/subcommands/reindex.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;AAEtC,2CAAoD;AACpD,gDAO0B;AAC1B,yCAAsC;AACtC,kEAA+D;AAC/D,sEAAiE;AACjE,oFAAgF;AAChF,8CAAgD;AAChD,8CAA2C;AAE3C,6DAAyD;AAEzD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAG7B,IAAM,cAAc,GAApB,MAAM,cAAc;IAIzB,YACmB,SAAoB,EACpB,UAAsB,EACtB,YAA0B,EAC1B,UAAsB,EACM,OAAwB,EACpD,iBAAoC,EACpC,wBAAkD,EAClD,gBAAkC;QAPlC,cAAS,GAAT,SAAS,CAAW;QACpB,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACM,YAAO,GAAP,OAAO,CAAiB;QACpD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,6BAAwB,GAAxB,wBAAwB,CAA0B;QAClD,qBAAgB,GAAhB,gBAAgB,CAAkB;IAClD,CAAC;IAEJ,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAE1D,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,oCAAoC,CACxC,WAAmB;QAEnB,wBAAwB;QACvB,IAAI,CAAC,wBAAgC,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACxE,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,wBAAwB,CAAC,4BAA4B,EAAE,CAAC;QACrE,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CACzC,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,WAAW,CAC9D,CAAC;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAC3B,OAAO,WAAW,CAAC;SACpB;QACD,MAAM,CAAC,cAAc,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,4BAA4B;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAE7C,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QACxE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,CAAC,4BAA4B,EAAE;YACnC,IAAI,CAAC,sBAAsB,EAAE;SAC9B,CAAC,CAAC;QAEH,MAAM,IAAA,iBAAO,EACX,WAAW,EACX,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,wBAAwB,EAC7B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,iBAAiB,CACvB,CAAC;QACF,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;CACF,CAAA;AApGY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAUR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAJC,qBAAS;QACR,sBAAU;QACR,wBAAY;QACd,sBAAU;QACe,iCAAe;QACjC,sCAAiB;QACV,oDAAwB;QAChC,qCAAgB;GAZ1C,cAAc,CAoG1B;AApGY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { Inject, Injectable } from '@nestjs/common';\nimport {\n  getLogger,\n  MmrService,\n  NodeConfig,\n  StoreService,\n  getExistingProjectSchema,\n  CacheMetadataModel,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { DynamicDsService } from '../indexer/dynamic-ds.service';\nimport { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';\nimport { initDbSchema } from '../utils/project';\nimport { reindex } from '../utils/reindex';\n\nimport { ForceCleanService } from './forceClean.service';\n\nconst logger = getLogger('Reindex');\n\n@Injectable()\nexport class ReindexService {\n  private schema: string;\n  private metadataRepo: CacheMetadataModel;\n\n  constructor(\n    private readonly sequelize: Sequelize,\n    private readonly nodeConfig: NodeConfig,\n    private readonly storeService: StoreService,\n    private readonly mmrService: MmrService,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly forceCleanService: ForceCleanService,\n    private readonly unfinalizedBlocksService: UnfinalizedBlocksService,\n    private readonly dynamicDsService: DynamicDsService,\n  ) {}\n\n  async init(): Promise<void> {\n    this.schema = await this.getExistingProjectSchema();\n\n    if (!this.schema) {\n      logger.error('Unable to locate schema');\n      throw new Error('Schema does not exist.');\n    }\n    await this.initDbSchema();\n\n    this.metadataRepo = this.storeService.storeCache.metadata;\n\n    this.dynamicDsService.init(this.metadataRepo);\n  }\n\n  async getTargetHeightWithUnfinalizedBlocks(\n    inputHeight: number,\n  ): Promise<number> {\n    // Why does this happen?\n    (this.unfinalizedBlocksService as any).metadataRepo = this.metadataRepo;\n    const unfinalizedBlocks =\n      await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks();\n    const bestBlocks = unfinalizedBlocks.filter(\n      ([bestBlockHeight]) => Number(bestBlockHeight) <= inputHeight,\n    );\n    if (bestBlocks.length === 0) {\n      return inputHeight;\n    }\n    const [firstBestBlock] = bestBlocks[0];\n    return Math.min(inputHeight, firstBestBlock);\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.metadataRepo.find('lastProcessedHeight');\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.metadataRepo.find('blockOffset');\n  }\n\n  private async getMetadataSpecName(): Promise<string | undefined> {\n    return this.metadataRepo.find('specName');\n  }\n\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private getStartBlockFromDataSources(): number {\n    const datasources = this.project.dataSources;\n\n    const startBlocksList = datasources.map((item) => item.startBlock ?? 1);\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const [startHeight, lastProcessedHeight] = await Promise.all([\n      this.getStartBlockFromDataSources(),\n      this.getLastProcessedHeight(),\n    ]);\n\n    await reindex(\n      startHeight,\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlocksService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      this.forceCleanService,\n    );\n    await this.storeService.storeCache.flushCache(true, true);\n  }\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subql/node-ethereum",
-  "version": "1.0.3-4",
+  "version": "1.0.3-6",
   "description": "",
   "author": "Ian He",
   "license": "Apache-2.0",
@@ -26,7 +26,7 @@
     "@nestjs/schedule": "^1.0.2",
     "@subql/common": "^2.0.0",
     "@subql/common-ethereum": "1.0.1-1",
-    "@subql/node-core": "2.0.1-3",
+    "@subql/node-core": "^2.0.2",
     "@subql/testing": "^2.0.0",
     "@subql/types-ethereum": "1.0.2-0",
     "@subql/utils": "^2.0.0",
@@ -76,5 +76,5 @@
     "/dist",
     "/bin"
   ],
-  "stableVersion": "1.0.3-3"
+  "stableVersion": "1.0.3-5"
 }