@subql/node-ethereum 1.0.0 → 1.0.1-0

package/dist/ethereum/api.ethereum.d.ts

@@ -7,6 +7,7 @@ export declare class EthereumApi implements ApiWrapper<EthereumBlockWrapper> {
     private endpoint;
     private eventEmitter;
     private client;
+    private nonBatchClient?;
     private genesisBlock;
     private contractInterfaces;
     private chainId;

package/dist/ethereum/api.ethereum.js

@@ -17,6 +17,7 @@ const utils_1 = require("ethers/lib/utils");
 const project_1 = require("../utils/project");
 const block_ethereum_1 = require("./block.ethereum");
 const json_rpc_batch_provider_1 = require("./ethers/json-rpc-batch-provider");
+const json_rpc_provider_1 = require("./ethers/json-rpc-provider");
 const safe_api_1 = __importDefault(require("./safe-api"));
 const utils_ethereum_1 = require("./utils.ethereum");
 // eslint-disable-next-line @typescript-eslint/no-var-requires
@@ -79,6 +80,7 @@ class EthereumApi {
                 connection.headers[name] = value;
             });
             this.client = new json_rpc_batch_provider_1.JsonRpcBatchProvider(connection);
+            this.nonBatchClient = new json_rpc_provider_1.JsonRpcProvider(connection);
         }
         else if (protocolStr === 'ws' || protocolStr === 'wss') {
             this.client = new providers_1.WebSocketProvider(this.endpoint);
@@ -105,7 +107,7 @@ class EthereumApi {
     async getFinalizedBlockHeight() {
         try {
             if (this.supportsFinalization) {
-                return (await this.client.getBlock('finalised')).number;
+                return (await this.client.getBlock('finalized')).number;
             }
             else {
                 // TODO make number of blocks finalised configurable
@@ -163,7 +165,7 @@ class EthereumApi {
         const ret = new block_ethereum_1.EthereumBlockWrapped(block, includeTx
             ? block.transactions.map((tx) => (Object.assign(Object.assign({}, (0, utils_ethereum_1.formatTransaction)(tx)), { 
                 // TODO memoise
-                receipt: () => this.getTransactionReceipt(tx).then((r) => (0, utils_ethereum_1.formatReceipt)(r, block)) })))
+                receipt: () => this.getTransactionReceipt(tx.hash).then((r) => (0, utils_ethereum_1.formatReceipt)(r, block)) })))
             : [], logs.map((l) => (0, utils_ethereum_1.formatLog)(l, block)));
         this.eventEmitter.emit('fetchBlock');
         return ret;
@@ -172,7 +174,11 @@ class EthereumApi {
         return Promise.all(bufferBlocks.map(async (num) => this.fetchBlock(num, true)));
     }
     freezeApi(processor, blockContent) {
-        processor.freeze(new safe_api_1.default(this.client, blockContent.blockHeight), 'api');
+        // We cannot use a batch http client because OnF don't support routing historical queries in batches to an archive nodes
+        const client = this.client instanceof providers_1.WebSocketProvider
+            ? this.client
+            : this.nonBatchClient;
+        processor.freeze(new safe_api_1.default(client, blockContent.blockHeight), 'api');
     }
     buildInterface(abiName, assets) {
         if (!assets[abiName]) {

package/dist/ethereum/api.ethereum.js.map

@@ -1 +1 @@
-{"version":3,"file":"api.ethereum.js","sourceRoot":"","sources":["../../src/ethereum/api.ethereum.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;AAEtC,4CAAoB;AACpB,gDAAwB;AACxB,kDAA0B;AAC1B,4CAA+C;AAE/C,wDAA6D;AAG7D,gDAA6C;AAS7C,wEAA+C;AAC/C,4CAA0D;AAC1D,8CAAkD;AAClD,qDAAwD;AACxD,8EAAwE;AAGxE,0DAAyC;AACzC,qDAK0B;AAE1B,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;AAEzC,KAAK,UAAU,UAAU,CACvB,EAA2B;IAE3B,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QACd,OAAO,EAAE,CAAC;KACX;IACD,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE;QACxD,IAAI;YACF,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,YAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;SAC5D;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa;IACpB,8CAA8C;IAC9C,uDAAuD;IACvD,MAAM,MAAM,GAAG,IAAI,0BAAe,EAAE,CAAC;IAErC,MAAM,OAAO,GAAsB;QACjC,SAAS,EAAE,IAAI;QACf,qBAAqB;KACtB,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,cAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,eAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAE5C,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1B,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAE3B,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,UAAU;KAClB,CAAC;AACJ,CAAC;AAED,MAAa,WAAW;IAStB,YAAoB,QAAgB,EAAU,YAA2B;QAArD,aAAQ,GAAR,QAAQ,CAAQ;QAAU,iBAAY,GAAZ,YAAY,CAAe;QANjE,uBAAkB,GAA8B,EAAE,CAAC;QAG3D,eAAe;QACP,yBAAoB,GAAG,IAAI,CAAC;QAGlC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE/D,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAE9C,MAAM,CAAC,IAAI,CAAC,aAAa,QAAQ,aAAa,WAAW,EAAE,CAAC,CAAC;QAC7D,IAAI,WAAW,KAAK,OAAO,IAAI,WAAW,KAAK,MAAM,EAAE;YACrD,MAAM,UAAU,GAAmB;gBACjC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAChC,OAAO,EAAE;oBACP,YAAY,EAAE,iBAAiB,cAAc,EAAE;iBAChD;gBACD,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,CAAC;gBAChB,oBAAoB,EAAE,CAAC;gBACvB,MAAM,EAAE,aAAa,EAAE;aACxB,CAAC;YACF,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE;gBAChD,UAAU,CAAC,OAAe,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC5C,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,GAAG,IAAI,8CAAoB,CAAC,UAAU,CAAC,CAAC;SACpD;aAAM,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,KAAK,EAAE;YACxD,IAAI,CAAC,MAAM,GAAG,IAAI,6BAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SACpD;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;SACtD;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAElD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACjC,CAAC;IAEO,YAAY;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE;YACzC,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE;gBACjB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAClC,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YACvB,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,IAAI;YACF,IAAI,IAAI,CAAC,oBAAoB,EAAE;gBAC7B,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;aACzD;iBAAM;gBACL,oDAAoD;gBACpD,OAAO,CAAC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,+BAA+B;aAC/E;SACF;QAAC,OAAO,CAAC,EAAE;YACV,sCAAsC;YACtC,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;YAClC,OAAO,IAAI,CAAC,uBAAuB,EAAE,CAAC;SACvC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC1D,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC;IAED,eAAe;QACb,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;IAChC,CAAC;IAED,WAAW;QACT,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,YAA6B;QACxD,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE;YACpC,YAAY,GAAG,IAAA,gBAAQ,EAAC,YAAY,CAAC,CAAC;SACvC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAW,EAAE,SAAS,GAAG,IAAI;QACzD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;YAC9D,IAAA,gBAAQ,EAAC,GAAG,CAAC;YACb,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,IAAA,4BAAW,EAAC,QAAQ,CAAC,CAAC;QAEpC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAE9D,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,eAAyC;QAEzC,OAAO,IAAA,wBAAc,EACnB,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CACrE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CACd,WAAmB,EACnB,SAAmB;QAEnB,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACtC,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,SAAS,CAAC;YAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAClB,SAAS,EAAE,IAAA,gBAAQ,EAAC,WAAW,CAAC;gBAChC,OAAO,EAAE,IAAA,gBAAQ,EAAC,WAAW,CAAC;aAC/B,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,IAAI,qCAAoB,CAClC,KAAK,EACL,SAAS;YACP,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,iCAC1B,IAAA,kCAAiB,EAAC,EAAE,CAAC;gBACxB,eAAe;gBACf,OAAO,EAAE,GAAG,EAAE,CACZ,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CACxC,IAAA,8BAAa,EAAC,CAAC,EAAE,KAAK,CAAC,CACxB,IACH,CAAC;YACL,CAAC,CAAC,EAAE,EACN,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,0BAAS,EAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CACrC,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACrC,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,YAAsB;QACtC,OAAO,OAAO,CAAC,GAAG,CAChB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAC5D,CAAC;IACJ,CAAC;IAED,SAAS,CAAC,SAAc,EAAE,YAA0B;QAClD,SAAS,CAAC,MAAM,CACd,IAAI,kBAAe,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,WAAW,CAAC,EAC1D,KAAK,CACN,CAAC;IACJ,CAAC;IAEO,cAAc,CACpB,OAAe,EACf,MAA8B;QAE9B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;YACpB,MAAM,IAAI,KAAK,CAAC,cAAc,OAAO,4BAA4B,CAAC,CAAC;SACpE;QAED,uFAAuF;QACvF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;YACrC,+CAA+C;YAC/C,IAAI;gBACF,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAEzC;;;mBAGG;gBACH,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE;oBACxC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC;iBACrB;gBAED,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,GAAG,IAAI,eAAS,CAAC,MAAM,CAAC,CAAC;aAC1D;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;aACnC;SACF;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,GAAgB,EAChB,EAA2B;;QAE3B,IAAI;YACF,IAAI,CAAC,CAAA,MAAA,EAAE,aAAF,EAAE,uBAAF,EAAE,CAAE,OAAO,0CAAE,GAAG,CAAA,EAAE;gBACrB,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;gBAC9C,OAAO,GAAG,CAAC;aACZ;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;YACxE,uCACK,GAAG,KACN,IAAI,EAAE,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,QAAQ,CAAC,GAAG,EAAE,IAAS,IACpC;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACtD,OAAO,GAAG,CAAC;SACZ;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,WAAgC,EAChC,EAA2B;;QAE3B,IAAI;YACF,IAAI,CAAC,CAAA,MAAA,EAAE,aAAF,EAAE,uBAAF,EAAE,CAAE,OAAO,0CAAE,GAAG,CAAA,EAAE;gBACrB,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;gBAC9C,OAAO,WAAW,CAAC;aACpB;YACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,IAAA,oBAAY,EAAC,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACtE,MAAM,IAAI,GAAG,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAM,CAAC;YACpE,uCACK,WAAW,KACd,IAAI,IACJ;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9D,OAAO,WAAW,CAAC;SACpB;IACH,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,OAAO;QACX,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,MAAM,YAAY,6BAAiB,EAAE;YAC5C,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;SAC7B;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;SACnD;IACH,CAAC;CACF;AA1PD,kCA0PC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport http from 'http';\nimport https from 'https';\nimport { Interface } from '@ethersproject/abi';\nimport { Block, TransactionReceipt } from '@ethersproject/abstract-provider';\nimport { WebSocketProvider } from '@ethersproject/providers';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { RuntimeDataSourceV0_2_0 } from '@subql/common-ethereum';\nimport { getLogger } from '@subql/node-core';\nimport {\n  ApiWrapper,\n  BlockWrapper,\n  EthereumBlockWrapper,\n  EthereumTransaction,\n  EthereumResult,\n  EthereumLog,\n} from '@subql/types-ethereum';\nimport CacheableLookup from 'cacheable-lookup';\nimport { hexDataSlice, hexValue } from 'ethers/lib/utils';\nimport { retryOnFailEth } from '../utils/project';\nimport { EthereumBlockWrapped } from './block.ethereum';\nimport { JsonRpcBatchProvider } from './ethers/json-rpc-batch-provider';\nimport { JsonRpcProvider } from './ethers/json-rpc-provider';\nimport { ConnectionInfo } from './ethers/web';\nimport SafeEthProvider from './safe-api';\nimport {\n  formatBlock,\n  formatLog,\n  formatReceipt,\n  formatTransaction,\n} from './utils.ethereum';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst logger = getLogger('api.ethereum');\n\nasync function loadAssets(\n  ds: RuntimeDataSourceV0_2_0,\n): Promise<Record<string, string>> {\n  if (!ds.assets) {\n    return {};\n  }\n  const res: Record<string, string> = {};\n\n  for (const [name, { file }] of Object.entries(ds.assets)) {\n    try {\n      res[name] = await fs.promises.readFile(file, { encoding: 'utf8' });\n    } catch (e) {\n      throw new Error(`Failed to load datasource asset ${file}`);\n    }\n  }\n\n  return res;\n}\n\nfunction getHttpAgents() {\n  // By default Nodejs doesn't cache DNS lookups\n  // https://httptoolkit.com/blog/configuring-nodejs-dns/\n  const lookup = new CacheableLookup();\n\n  const options: http.AgentOptions = {\n    keepAlive: true,\n    /*, maxSockets: 100*/\n  };\n\n  const httpAgent = new http.Agent(options);\n  const httpsAgent = new https.Agent(options);\n\n  lookup.install(httpAgent);\n  lookup.install(httpsAgent);\n\n  return {\n    http: httpAgent,\n    https: httpsAgent,\n  };\n}\n\nexport class EthereumApi implements ApiWrapper<EthereumBlockWrapper> {\n  private client: JsonRpcProvider;\n  private genesisBlock: Record<string, any>;\n  private contractInterfaces: Record<string, Interface> = {};\n  private chainId: number;\n\n  // Ethereum POS\n  private supportsFinalization = true;\n\n  constructor(private endpoint: string, private eventEmitter: EventEmitter2) {\n    const { hostname, protocol, searchParams } = new URL(endpoint);\n\n    const protocolStr = protocol.replace(':', '');\n\n    logger.info(`Api host: ${hostname}, method: ${protocolStr}`);\n    if (protocolStr === 'https' || protocolStr === 'http') {\n      const connection: ConnectionInfo = {\n        url: this.endpoint.split('?')[0],\n        headers: {\n          'User-Agent': `Subquery-Node ${packageVersion}`,\n        },\n        allowGzip: true,\n        throttleLimit: 5,\n        throttleSlotInterval: 1,\n        agents: getHttpAgents(),\n      };\n      searchParams.forEach((value, name, searchParams) => {\n        (connection.headers as any)[name] = value;\n      });\n      this.client = new JsonRpcBatchProvider(connection);\n    } else if (protocolStr === 'ws' || protocolStr === 'wss') {\n      this.client = new WebSocketProvider(this.endpoint);\n    } else {\n      throw new Error(`Unsupported protocol: ${protocol}`);\n    }\n  }\n\n  async init(): Promise<void> {\n    this.injectClient();\n    this.genesisBlock = await this.client.getBlock(0);\n\n    const network = await this.client.getNetwork();\n    this.chainId = network.chainId;\n  }\n\n  private injectClient(): void {\n    const orig = this.client.send.bind(this.client);\n    Object.defineProperty(this.client, 'send', {\n      value: (...args) => {\n        this.eventEmitter.emit('rpcCall');\n        return orig(...args);\n      },\n    });\n  }\n\n  async getFinalizedBlockHeight(): Promise<number> {\n    try {\n      if (this.supportsFinalization) {\n        return (await this.client.getBlock('finalised')).number;\n      } else {\n        // TODO make number of blocks finalised configurable\n        return (await this.getBestBlockHeight()) - 15; // Consider 15 blocks finalized\n      }\n    } catch (e) {\n      // TODO handle specific error for this\n      this.supportsFinalization = false;\n      return this.getFinalizedBlockHeight();\n    }\n  }\n\n  async getBestBlockHeight(): Promise<number> {\n    const tag = this.supportsFinalization ? 'safe' : 'latest';\n    return (await this.client.getBlock(tag)).number;\n  }\n\n  getRuntimeChain(): string {\n    return 'ethereum';\n  }\n\n  getChainId(): number {\n    return this.chainId;\n  }\n\n  getGenesisHash(): string {\n    return this.genesisBlock.hash;\n  }\n\n  getSpecName(): string {\n    return 'ethereum';\n  }\n\n  async getBlockByHeightOrHash(heightOrHash: number | string): Promise<Block> {\n    if (typeof heightOrHash === 'number') {\n      heightOrHash = hexValue(heightOrHash);\n    }\n    return this.client.getBlock(heightOrHash);\n  }\n\n  private async getBlockPromise(num: number, includeTx = true): Promise<any> {\n    const rawBlock = await this.client.send('eth_getBlockByNumber', [\n      hexValue(num),\n      includeTx,\n    ]);\n\n    const block = formatBlock(rawBlock);\n\n    block.stateRoot = this.client.formatter.hash(block.stateRoot);\n\n    return block;\n  }\n\n  async getTransactionReceipt(\n    transactionHash: string | Promise<string>,\n  ): Promise<TransactionReceipt> {\n    return retryOnFailEth<TransactionReceipt>(\n      this.client.getTransactionReceipt.bind(this.client, transactionHash),\n    );\n  }\n\n  async fetchBlock(\n    blockNumber: number,\n    includeTx?: boolean,\n  ): Promise<EthereumBlockWrapped> {\n    const [block, logs] = await Promise.all([\n      this.getBlockPromise(blockNumber, includeTx),\n      this.client.getLogs({\n        fromBlock: hexValue(blockNumber),\n        toBlock: hexValue(blockNumber),\n      }),\n    ]);\n\n    const ret = new EthereumBlockWrapped(\n      block,\n      includeTx\n        ? block.transactions.map((tx) => ({\n            ...formatTransaction(tx),\n            // TODO memoise\n            receipt: () =>\n              this.getTransactionReceipt(tx).then((r) =>\n                formatReceipt(r, block),\n              ),\n          }))\n        : [],\n      logs.map((l) => formatLog(l, block)),\n    );\n    this.eventEmitter.emit('fetchBlock');\n    return ret;\n  }\n\n  async fetchBlocks(bufferBlocks: number[]): Promise<EthereumBlockWrapper[]> {\n    return Promise.all(\n      bufferBlocks.map(async (num) => this.fetchBlock(num, true)),\n    );\n  }\n\n  freezeApi(processor: any, blockContent: BlockWrapper): void {\n    processor.freeze(\n      new SafeEthProvider(this.client, blockContent.blockHeight),\n      'api',\n    );\n  }\n\n  private buildInterface(\n    abiName: string,\n    assets: Record<string, string>,\n  ): Interface | undefined {\n    if (!assets[abiName]) {\n      throw new Error(`ABI named \"${abiName}\" not referenced in assets`);\n    }\n\n    // This assumes that all datasources have a different abi name or they are the same abi\n    if (!this.contractInterfaces[abiName]) {\n      // Constructing the interface validates the ABI\n      try {\n        let abiObj = JSON.parse(assets[abiName]);\n\n        /*\n         * Allows parsing JSON artifacts as well as ABIs\n         * https://trufflesuite.github.io/artifact-updates/background.html#what-are-artifacts\n         */\n        if (!Array.isArray(abiObj) && abiObj.abi) {\n          abiObj = abiObj.abi;\n        }\n\n        this.contractInterfaces[abiName] = new Interface(abiObj);\n      } catch (e) {\n        logger.error(`Unable to parse ABI: ${e.message}`);\n        throw new Error('ABI is invalid');\n      }\n    }\n\n    return this.contractInterfaces[abiName];\n  }\n\n  async parseLog<T extends EthereumResult = EthereumResult>(\n    log: EthereumLog,\n    ds: RuntimeDataSourceV0_2_0,\n  ): Promise<EthereumLog<T> | EthereumLog> {\n    try {\n      if (!ds?.options?.abi) {\n        logger.warn('No ABI provided for datasource');\n        return log;\n      }\n      const iface = this.buildInterface(ds.options.abi, await loadAssets(ds));\n      return {\n        ...log,\n        args: iface?.parseLog(log).args as T,\n      };\n    } catch (e) {\n      logger.warn(`Failed to parse log data: ${e.message}`);\n      return log;\n    }\n  }\n\n  async parseTransaction<T extends EthereumResult = EthereumResult>(\n    transaction: EthereumTransaction,\n    ds: RuntimeDataSourceV0_2_0,\n  ): Promise<EthereumTransaction<T> | EthereumTransaction> {\n    try {\n      if (!ds?.options?.abi) {\n        logger.warn('No ABI provided for datasource');\n        return transaction;\n      }\n      const assets = await loadAssets(ds);\n      const iface = this.buildInterface(ds.options.abi, assets);\n      const func = iface.getFunction(hexDataSlice(transaction.input, 0, 4));\n      const args = iface.decodeFunctionData(func, transaction.input) as T;\n      return {\n        ...transaction,\n        args,\n      };\n    } catch (e) {\n      logger.warn(`Failed to parse transaction data: ${e.message}`);\n      return transaction;\n    }\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  async connect(): Promise<void> {\n    logger.error('Ethereum API connect is not implemented');\n    throw new Error('Not implemented');\n  }\n\n  async disconnect(): Promise<void> {\n    if (this.client instanceof WebSocketProvider) {\n      await this.client.destroy();\n    } else {\n      logger.warn('Disconnect called on HTTP provider');\n    }\n  }\n}\n"]}
+{"version":3,"file":"api.ethereum.js","sourceRoot":"","sources":["../../src/ethereum/api.ethereum.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;AAEtC,4CAAoB;AACpB,gDAAwB;AACxB,kDAA0B;AAC1B,4CAA+C;AAE/C,wDAA4E;AAG5E,gDAA6C;AAS7C,wEAA+C;AAC/C,4CAA0D;AAC1D,8CAAkD;AAClD,qDAAwD;AACxD,8EAAwE;AACxE,kEAA6D;AAE7D,0DAAyC;AACzC,qDAK0B;AAE1B,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;AAEzC,KAAK,UAAU,UAAU,CACvB,EAA2B;IAE3B,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QACd,OAAO,EAAE,CAAC;KACX;IACD,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE;QACxD,IAAI;YACF,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,YAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;SAC5D;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa;IACpB,8CAA8C;IAC9C,uDAAuD;IACvD,MAAM,MAAM,GAAG,IAAI,0BAAe,EAAE,CAAC;IAErC,MAAM,OAAO,GAAsB;QACjC,SAAS,EAAE,IAAI;QACf,qBAAqB;KACtB,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,cAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,eAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAE5C,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1B,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAE3B,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,UAAU;KAClB,CAAC;AACJ,CAAC;AAED,MAAa,WAAW;IAYtB,YAAoB,QAAgB,EAAU,YAA2B;QAArD,aAAQ,GAAR,QAAQ,CAAQ;QAAU,iBAAY,GAAZ,YAAY,CAAe;QANjE,uBAAkB,GAA8B,EAAE,CAAC;QAG3D,eAAe;QACP,yBAAoB,GAAG,IAAI,CAAC;QAGlC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE/D,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAE9C,MAAM,CAAC,IAAI,CAAC,aAAa,QAAQ,aAAa,WAAW,EAAE,CAAC,CAAC;QAC7D,IAAI,WAAW,KAAK,OAAO,IAAI,WAAW,KAAK,MAAM,EAAE;YACrD,MAAM,UAAU,GAAmB;gBACjC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAChC,OAAO,EAAE;oBACP,YAAY,EAAE,iBAAiB,cAAc,EAAE;iBAChD;gBACD,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,CAAC;gBAChB,oBAAoB,EAAE,CAAC;gBACvB,MAAM,EAAE,aAAa,EAAE;aACxB,CAAC;YACF,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE;gBAChD,UAAU,CAAC,OAAe,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC5C,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,GAAG,IAAI,8CAAoB,CAAC,UAAU,CAAC,CAAC;YACnD,IAAI,CAAC,cAAc,GAAG,IAAI,mCAAe,CAAC,UAAU,CAAC,CAAC;SACvD;aAAM,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,KAAK,EAAE;YACxD,IAAI,CAAC,MAAM,GAAG,IAAI,6BAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SACpD;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;SACtD;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAElD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACjC,CAAC;IAEO,YAAY;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE;YACzC,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE;gBACjB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAClC,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YACvB,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,IAAI;YACF,IAAI,IAAI,CAAC,oBAAoB,EAAE;gBAC7B,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;aACzD;iBAAM;gBACL,oDAAoD;gBACpD,OAAO,CAAC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,+BAA+B;aAC/E;SACF;QAAC,OAAO,CAAC,EAAE;YACV,sCAAsC;YACtC,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;YAClC,OAAO,IAAI,CAAC,uBAAuB,EAAE,CAAC;SACvC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC1D,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC;IAED,eAAe;QACb,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;IAChC,CAAC;IAED,WAAW;QACT,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,YAA6B;QACxD,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE;YACpC,YAAY,GAAG,IAAA,gBAAQ,EAAC,YAAY,CAAC,CAAC;SACvC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAW,EAAE,SAAS,GAAG,IAAI;QACzD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;YAC9D,IAAA,gBAAQ,EAAC,GAAG,CAAC;YACb,SAAS;SACV,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,IAAA,4BAAW,EAAC,QAAQ,CAAC,CAAC;QAEpC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAE9D,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,eAAyC;QAEzC,OAAO,IAAA,wBAAc,EACnB,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CACrE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CACd,WAAmB,EACnB,SAAmB;QAEnB,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACtC,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,SAAS,CAAC;YAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAClB,SAAS,EAAE,IAAA,gBAAQ,EAAC,WAAW,CAAC;gBAChC,OAAO,EAAE,IAAA,gBAAQ,EAAC,WAAW,CAAC;aAC/B,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,IAAI,qCAAoB,CAClC,KAAK,EACL,SAAS;YACP,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,iCAC1B,IAAA,kCAAiB,EAAC,EAAE,CAAC;gBACxB,eAAe;gBACf,OAAO,EAAE,GAAG,EAAE,CACZ,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAA,8BAAa,EAAC,CAAC,EAAE,KAAK,CAAC,CACxB,IACH,CAAC;YACL,CAAC,CAAC,EAAE,EACN,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,0BAAS,EAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CACrC,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACrC,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,YAAsB;QACtC,OAAO,OAAO,CAAC,GAAG,CAChB,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAC5D,CAAC;IACJ,CAAC;IAED,SAAS,CAAC,SAAc,EAAE,YAA0B;QAClD,wHAAwH;QACxH,MAAM,MAAM,GACV,IAAI,CAAC,MAAM,YAAY,6BAAiB;YACtC,CAAC,CAAC,IAAI,CAAC,MAAM;YACb,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC;QAE1B,SAAS,CAAC,MAAM,CACd,IAAI,kBAAe,CAAC,MAAM,EAAE,YAAY,CAAC,WAAW,CAAC,EACrD,KAAK,CACN,CAAC;IACJ,CAAC;IAEO,cAAc,CACpB,OAAe,EACf,MAA8B;QAE9B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;YACpB,MAAM,IAAI,KAAK,CAAC,cAAc,OAAO,4BAA4B,CAAC,CAAC;SACpE;QAED,uFAAuF;QACvF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE;YACrC,+CAA+C;YAC/C,IAAI;gBACF,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAEzC;;;mBAGG;gBACH,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE;oBACxC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC;iBACrB;gBAED,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,GAAG,IAAI,eAAS,CAAC,MAAM,CAAC,CAAC;aAC1D;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;aACnC;SACF;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,GAAgB,EAChB,EAA2B;;QAE3B,IAAI;YACF,IAAI,CAAC,CAAA,MAAA,EAAE,aAAF,EAAE,uBAAF,EAAE,CAAE,OAAO,0CAAE,GAAG,CAAA,EAAE;gBACrB,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;gBAC9C,OAAO,GAAG,CAAC;aACZ;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;YACxE,uCACK,GAAG,KACN,IAAI,EAAE,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,QAAQ,CAAC,GAAG,EAAE,IAAS,IACpC;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACtD,OAAO,GAAG,CAAC;SACZ;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,WAAgC,EAChC,EAA2B;;QAE3B,IAAI;YACF,IAAI,CAAC,CAAA,MAAA,EAAE,aAAF,EAAE,uBAAF,EAAE,CAAE,OAAO,0CAAE,GAAG,CAAA,EAAE;gBACrB,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;gBAC9C,OAAO,WAAW,CAAC;aACpB;YACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,IAAA,oBAAY,EAAC,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACtE,MAAM,IAAI,GAAG,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAM,CAAC;YACpE,uCACK,WAAW,KACd,IAAI,IACJ;SACH;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9D,OAAO,WAAW,CAAC;SACpB;IACH,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,OAAO;QACX,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,MAAM,YAAY,6BAAiB,EAAE;YAC5C,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;SAC7B;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;SACnD;IACH,CAAC;CACF;AApQD,kCAoQC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport fs from 'fs';\nimport http from 'http';\nimport https from 'https';\nimport { Interface } from '@ethersproject/abi';\nimport { Block, TransactionReceipt } from '@ethersproject/abstract-provider';\nimport { JsonRpcSigner, WebSocketProvider } from '@ethersproject/providers';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport { RuntimeDataSourceV0_2_0 } from '@subql/common-ethereum';\nimport { getLogger } from '@subql/node-core';\nimport {\n  ApiWrapper,\n  BlockWrapper,\n  EthereumBlockWrapper,\n  EthereumTransaction,\n  EthereumResult,\n  EthereumLog,\n} from '@subql/types-ethereum';\nimport CacheableLookup from 'cacheable-lookup';\nimport { hexDataSlice, hexValue } from 'ethers/lib/utils';\nimport { retryOnFailEth } from '../utils/project';\nimport { EthereumBlockWrapped } from './block.ethereum';\nimport { JsonRpcBatchProvider } from './ethers/json-rpc-batch-provider';\nimport { JsonRpcProvider } from './ethers/json-rpc-provider';\nimport { ConnectionInfo } from './ethers/web';\nimport SafeEthProvider from './safe-api';\nimport {\n  formatBlock,\n  formatLog,\n  formatReceipt,\n  formatTransaction,\n} from './utils.ethereum';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst logger = getLogger('api.ethereum');\n\nasync function loadAssets(\n  ds: RuntimeDataSourceV0_2_0,\n): Promise<Record<string, string>> {\n  if (!ds.assets) {\n    return {};\n  }\n  const res: Record<string, string> = {};\n\n  for (const [name, { file }] of Object.entries(ds.assets)) {\n    try {\n      res[name] = await fs.promises.readFile(file, { encoding: 'utf8' });\n    } catch (e) {\n      throw new Error(`Failed to load datasource asset ${file}`);\n    }\n  }\n\n  return res;\n}\n\nfunction getHttpAgents() {\n  // By default Nodejs doesn't cache DNS lookups\n  // https://httptoolkit.com/blog/configuring-nodejs-dns/\n  const lookup = new CacheableLookup();\n\n  const options: http.AgentOptions = {\n    keepAlive: true,\n    /*, maxSockets: 100*/\n  };\n\n  const httpAgent = new http.Agent(options);\n  const httpsAgent = new https.Agent(options);\n\n  lookup.install(httpAgent);\n  lookup.install(httpsAgent);\n\n  return {\n    http: httpAgent,\n    https: httpsAgent,\n  };\n}\n\nexport class EthereumApi implements ApiWrapper<EthereumBlockWrapper> {\n  private client: JsonRpcProvider;\n\n  // This is used within the sandbox when HTTP is used\n  private nonBatchClient?: JsonRpcProvider;\n  private genesisBlock: Record<string, any>;\n  private contractInterfaces: Record<string, Interface> = {};\n  private chainId: number;\n\n  // Ethereum POS\n  private supportsFinalization = true;\n\n  constructor(private endpoint: string, private eventEmitter: EventEmitter2) {\n    const { hostname, protocol, searchParams } = new URL(endpoint);\n\n    const protocolStr = protocol.replace(':', '');\n\n    logger.info(`Api host: ${hostname}, method: ${protocolStr}`);\n    if (protocolStr === 'https' || protocolStr === 'http') {\n      const connection: ConnectionInfo = {\n        url: this.endpoint.split('?')[0],\n        headers: {\n          'User-Agent': `Subquery-Node ${packageVersion}`,\n        },\n        allowGzip: true,\n        throttleLimit: 5,\n        throttleSlotInterval: 1,\n        agents: getHttpAgents(),\n      };\n      searchParams.forEach((value, name, searchParams) => {\n        (connection.headers as any)[name] = value;\n      });\n      this.client = new JsonRpcBatchProvider(connection);\n      this.nonBatchClient = new JsonRpcProvider(connection);\n    } else if (protocolStr === 'ws' || protocolStr === 'wss') {\n      this.client = new WebSocketProvider(this.endpoint);\n    } else {\n      throw new Error(`Unsupported protocol: ${protocol}`);\n    }\n  }\n\n  async init(): Promise<void> {\n    this.injectClient();\n    this.genesisBlock = await this.client.getBlock(0);\n\n    const network = await this.client.getNetwork();\n    this.chainId = network.chainId;\n  }\n\n  private injectClient(): void {\n    const orig = this.client.send.bind(this.client);\n    Object.defineProperty(this.client, 'send', {\n      value: (...args) => {\n        this.eventEmitter.emit('rpcCall');\n        return orig(...args);\n      },\n    });\n  }\n\n  async getFinalizedBlockHeight(): Promise<number> {\n    try {\n      if (this.supportsFinalization) {\n        return (await this.client.getBlock('finalized')).number;\n      } else {\n        // TODO make number of blocks finalised configurable\n        return (await this.getBestBlockHeight()) - 15; // Consider 15 blocks finalized\n      }\n    } catch (e) {\n      // TODO handle specific error for this\n      this.supportsFinalization = false;\n      return this.getFinalizedBlockHeight();\n    }\n  }\n\n  async getBestBlockHeight(): Promise<number> {\n    const tag = this.supportsFinalization ? 'safe' : 'latest';\n    return (await this.client.getBlock(tag)).number;\n  }\n\n  getRuntimeChain(): string {\n    return 'ethereum';\n  }\n\n  getChainId(): number {\n    return this.chainId;\n  }\n\n  getGenesisHash(): string {\n    return this.genesisBlock.hash;\n  }\n\n  getSpecName(): string {\n    return 'ethereum';\n  }\n\n  async getBlockByHeightOrHash(heightOrHash: number | string): Promise<Block> {\n    if (typeof heightOrHash === 'number') {\n      heightOrHash = hexValue(heightOrHash);\n    }\n    return this.client.getBlock(heightOrHash);\n  }\n\n  private async getBlockPromise(num: number, includeTx = true): Promise<any> {\n    const rawBlock = await this.client.send('eth_getBlockByNumber', [\n      hexValue(num),\n      includeTx,\n    ]);\n\n    const block = formatBlock(rawBlock);\n\n    block.stateRoot = this.client.formatter.hash(block.stateRoot);\n\n    return block;\n  }\n\n  async getTransactionReceipt(\n    transactionHash: string | Promise<string>,\n  ): Promise<TransactionReceipt> {\n    return retryOnFailEth<TransactionReceipt>(\n      this.client.getTransactionReceipt.bind(this.client, transactionHash),\n    );\n  }\n\n  async fetchBlock(\n    blockNumber: number,\n    includeTx?: boolean,\n  ): Promise<EthereumBlockWrapped> {\n    const [block, logs] = await Promise.all([\n      this.getBlockPromise(blockNumber, includeTx),\n      this.client.getLogs({\n        fromBlock: hexValue(blockNumber),\n        toBlock: hexValue(blockNumber),\n      }),\n    ]);\n\n    const ret = new EthereumBlockWrapped(\n      block,\n      includeTx\n        ? block.transactions.map((tx) => ({\n            ...formatTransaction(tx),\n            // TODO memoise\n            receipt: () =>\n              this.getTransactionReceipt(tx.hash).then((r) =>\n                formatReceipt(r, block),\n              ),\n          }))\n        : [],\n      logs.map((l) => formatLog(l, block)),\n    );\n    this.eventEmitter.emit('fetchBlock');\n    return ret;\n  }\n\n  async fetchBlocks(bufferBlocks: number[]): Promise<EthereumBlockWrapper[]> {\n    return Promise.all(\n      bufferBlocks.map(async (num) => this.fetchBlock(num, true)),\n    );\n  }\n\n  freezeApi(processor: any, blockContent: BlockWrapper): void {\n    // We cannot use a batch http client because OnF don't support routing historical queries in batches to an archive nodes\n    const client =\n      this.client instanceof WebSocketProvider\n        ? this.client\n        : this.nonBatchClient;\n\n    processor.freeze(\n      new SafeEthProvider(client, blockContent.blockHeight),\n      'api',\n    );\n  }\n\n  private buildInterface(\n    abiName: string,\n    assets: Record<string, string>,\n  ): Interface | undefined {\n    if (!assets[abiName]) {\n      throw new Error(`ABI named \"${abiName}\" not referenced in assets`);\n    }\n\n    // This assumes that all datasources have a different abi name or they are the same abi\n    if (!this.contractInterfaces[abiName]) {\n      // Constructing the interface validates the ABI\n      try {\n        let abiObj = JSON.parse(assets[abiName]);\n\n        /*\n         * Allows parsing JSON artifacts as well as ABIs\n         * https://trufflesuite.github.io/artifact-updates/background.html#what-are-artifacts\n         */\n        if (!Array.isArray(abiObj) && abiObj.abi) {\n          abiObj = abiObj.abi;\n        }\n\n        this.contractInterfaces[abiName] = new Interface(abiObj);\n      } catch (e) {\n        logger.error(`Unable to parse ABI: ${e.message}`);\n        throw new Error('ABI is invalid');\n      }\n    }\n\n    return this.contractInterfaces[abiName];\n  }\n\n  async parseLog<T extends EthereumResult = EthereumResult>(\n    log: EthereumLog,\n    ds: RuntimeDataSourceV0_2_0,\n  ): Promise<EthereumLog<T> | EthereumLog> {\n    try {\n      if (!ds?.options?.abi) {\n        logger.warn('No ABI provided for datasource');\n        return log;\n      }\n      const iface = this.buildInterface(ds.options.abi, await loadAssets(ds));\n      return {\n        ...log,\n        args: iface?.parseLog(log).args as T,\n      };\n    } catch (e) {\n      logger.warn(`Failed to parse log data: ${e.message}`);\n      return log;\n    }\n  }\n\n  async parseTransaction<T extends EthereumResult = EthereumResult>(\n    transaction: EthereumTransaction,\n    ds: RuntimeDataSourceV0_2_0,\n  ): Promise<EthereumTransaction<T> | EthereumTransaction> {\n    try {\n      if (!ds?.options?.abi) {\n        logger.warn('No ABI provided for datasource');\n        return transaction;\n      }\n      const assets = await loadAssets(ds);\n      const iface = this.buildInterface(ds.options.abi, assets);\n      const func = iface.getFunction(hexDataSlice(transaction.input, 0, 4));\n      const args = iface.decodeFunctionData(func, transaction.input) as T;\n      return {\n        ...transaction,\n        args,\n      };\n    } catch (e) {\n      logger.warn(`Failed to parse transaction data: ${e.message}`);\n      return transaction;\n    }\n  }\n\n  // eslint-disable-next-line @typescript-eslint/require-await\n  async connect(): Promise<void> {\n    logger.error('Ethereum API connect is not implemented');\n    throw new Error('Not implemented');\n  }\n\n  async disconnect(): Promise<void> {\n    if (this.client instanceof WebSocketProvider) {\n      await this.client.destroy();\n    } else {\n      logger.warn('Disconnect called on HTTP provider');\n    }\n  }\n}\n"]}

package/dist/ethereum/api.service.ethereum.test.js

@@ -10,7 +10,7 @@ const lodash_1 = require("lodash");
 const api_service_ethereum_1 = require("./api.service.ethereum");
 // Add api key to work
 const WS_ENDPOINT = 'wss://eth.api.onfinality.io/ws?apikey=';
-const HTTP_ENDPOINT = 'https://eth.api.onfinality.io/rpc?apikey=';
+const HTTP_ENDPOINT = 'https://eth.api.onfinality.io/public';
 function testSubqueryProject(endpoint) {
     return {
         network: {
@@ -24,38 +24,50 @@ function testSubqueryProject(endpoint) {
         templates: [],
     };
 }
+const prepareApiService = async (endpoint = HTTP_ENDPOINT) => {
+    const module = await testing_1.Test.createTestingModule({
+        providers: [
+            node_core_1.ConnectionPoolService,
+            {
+                provide: node_core_1.NodeConfig,
+                useFactory: () => ({}),
+            },
+            {
+                provide: 'ISubqueryProject',
+                useFactory: () => testSubqueryProject(endpoint),
+            },
+            api_service_ethereum_1.EthereumApiService,
+        ],
+        imports: [event_emitter_1.EventEmitterModule.forRoot()],
+    }).compile();
+    const app = module.createNestApplication();
+    await app.init();
+    const apiService = app.get(api_service_ethereum_1.EthereumApiService);
+    await apiService.init();
+    return [apiService, app];
+};
 jest.setTimeout(90000);
 describe('ApiService', () => {
+    let apiService;
     let app;
+    beforeEach(async () => {
+        [apiService, app] = await prepareApiService();
+    });
     afterEach(async () => {
         return app === null || app === void 0 ? void 0 : app.close();
     });
-    const prepareApiService = async (endpoint = HTTP_ENDPOINT) => {
-        const module = await testing_1.Test.createTestingModule({
-            providers: [
-                {
-                    provide: 'ISubqueryProject',
-                    useFactory: () => testSubqueryProject(endpoint),
-                },
-                api_service_ethereum_1.EthereumApiService,
-            ],
-            imports: [event_emitter_1.EventEmitterModule.forRoot()],
-        }).compile();
-        app = module.createNestApplication();
-        await app.init();
-        const apiService = app.get(api_service_ethereum_1.EthereumApiService);
-        await apiService.init();
-        return apiService;
-    };
     it('can instantiate api', async () => {
-        const apiService = await prepareApiService();
         console.log(apiService.api.getChainId());
         await (0, node_core_1.delay)(0.5);
     });
     it('can fetch blocks', async () => {
-        const apiService = await prepareApiService();
         await apiService.api.fetchBlocks((0, lodash_1.range)(12369621, 12369651));
         await (0, node_core_1.delay)(0.5);
     });
+    it('can get the finalized height', async () => {
+        const height = await apiService.api.getFinalizedBlockHeight();
+        console.log('Finalized height', height);
+        expect(height).toBeGreaterThan(16000000);
+    });
 });
 //# sourceMappingURL=api.service.ethereum.test.js.map

package/dist/ethereum/api.service.ethereum.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"api.service.ethereum.test.js","sourceRoot":"","sources":["../../src/ethereum/api.service.ethereum.test.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;AAGtC,yDAA2D;AAC3D,6CAAuC;AACvC,gDAAyC;AACzC,qCAAwC;AACxC,mCAA+B;AAE/B,iEAA4D;AAE5D,sBAAsB;AACtB,MAAM,WAAW,GAAG,wCAAwC,CAAC;AAC7D,MAAM,aAAa,GAAG,2CAA2C,CAAC;AAElE,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,OAAO;QACL,OAAO,EAAE;YACP,QAAQ;YACR,OAAO,EAAE,GAAG;SACb;QACD,WAAW,EAAE,EAAE;QACf,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,IAAI,uBAAa,CAAC,EAAE,CAAC;QAC7B,SAAS,EAAE,EAAE;KACd,CAAC;AACJ,CAAC;AAED,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AACvB,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,IAAI,GAAqB,CAAC;IAE1B,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,EAAE,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,MAAM,iBAAiB,GAAG,KAAK,EAC7B,WAAmB,aAAa,EACH,EAAE;QAC/B,MAAM,MAAM,GAAG,MAAM,cAAI,CAAC,mBAAmB,CAAC;YAC5C,SAAS,EAAE;gBACT;oBACE,OAAO,EAAE,kBAAkB;oBAC3B,UAAU,EAAE,GAAG,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC;iBAChD;gBACD,yCAAkB;aACnB;YACD,OAAO,EAAE,CAAC,kCAAkB,CAAC,OAAO,EAAE,CAAC;SACxC,CAAC,CAAC,OAAO,EAAE,CAAC;QAEb,GAAG,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACrC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,yCAAkB,CAAC,CAAC;QAC/C,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC;IAEF,EAAE,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;QACnC,MAAM,UAAU,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACzC,MAAM,IAAA,iBAAK,EAAC,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QAChC,MAAM,UAAU,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAC7C,MAAM,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAA,cAAK,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC5D,MAAM,IAAA,iBAAK,EAAC,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { INestApplication } from '@nestjs/common';\nimport { EventEmitterModule } from '@nestjs/event-emitter';\nimport { Test } from '@nestjs/testing';\nimport { delay } from '@subql/node-core';\nimport { GraphQLSchema } from 'graphql';\nimport { range } from 'lodash';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from './api.service.ethereum';\n\n// Add api key to work\nconst WS_ENDPOINT = 'wss://eth.api.onfinality.io/ws?apikey=';\nconst HTTP_ENDPOINT = 'https://eth.api.onfinality.io/rpc?apikey=';\n\nfunction testSubqueryProject(endpoint: string): SubqueryProject {\n  return {\n    network: {\n      endpoint,\n      chainId: '1',\n    },\n    dataSources: [],\n    id: 'test',\n    root: './',\n    schema: new GraphQLSchema({}),\n    templates: [],\n  };\n}\n\njest.setTimeout(90000);\ndescribe('ApiService', () => {\n  let app: INestApplication;\n\n  afterEach(async () => {\n    return app?.close();\n  });\n\n  const prepareApiService = async (\n    endpoint: string = HTTP_ENDPOINT,\n  ): Promise<EthereumApiService> => {\n    const module = await Test.createTestingModule({\n      providers: [\n        {\n          provide: 'ISubqueryProject',\n          useFactory: () => testSubqueryProject(endpoint),\n        },\n        EthereumApiService,\n      ],\n      imports: [EventEmitterModule.forRoot()],\n    }).compile();\n\n    app = module.createNestApplication();\n    await app.init();\n    const apiService = app.get(EthereumApiService);\n    await apiService.init();\n    return apiService;\n  };\n\n  it('can instantiate api', async () => {\n    const apiService = await prepareApiService();\n    console.log(apiService.api.getChainId());\n    await delay(0.5);\n  });\n\n  it('can fetch blocks', async () => {\n    const apiService = await prepareApiService();\n    await apiService.api.fetchBlocks(range(12369621, 12369651));\n    await delay(0.5);\n  });\n});\n"]}
+{"version":3,"file":"api.service.ethereum.test.js","sourceRoot":"","sources":["../../src/ethereum/api.service.ethereum.test.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;AAGtC,yDAA2D;AAC3D,6CAAuC;AACvC,gDAA4E;AAC5E,qCAAwC;AACxC,mCAA+B;AAE/B,iEAA4D;AAE5D,sBAAsB;AACtB,MAAM,WAAW,GAAG,wCAAwC,CAAC;AAC7D,MAAM,aAAa,GAAG,sCAAsC,CAAC;AAE7D,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,OAAO;QACL,OAAO,EAAE;YACP,QAAQ;YACR,OAAO,EAAE,GAAG;SACb;QACD,WAAW,EAAE,EAAE;QACf,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,IAAI,uBAAa,CAAC,EAAE,CAAC;QAC7B,SAAS,EAAE,EAAE;KACd,CAAC;AACJ,CAAC;AAED,MAAM,iBAAiB,GAAG,KAAK,EAC7B,WAAmB,aAAa,EACiB,EAAE;IACnD,MAAM,MAAM,GAAG,MAAM,cAAI,CAAC,mBAAmB,CAAC;QAC5C,SAAS,EAAE;YACT,iCAAqB;YACrB;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;aACvB;YACD;gBACE,OAAO,EAAE,kBAAkB;gBAC3B,UAAU,EAAE,GAAG,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC;aAChD;YACD,yCAAkB;SACnB;QACD,OAAO,EAAE,CAAC,kCAAkB,CAAC,OAAO,EAAE,CAAC;KACxC,CAAC,CAAC,OAAO,EAAE,CAAC;IAEb,MAAM,GAAG,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;IAC3C,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IACjB,MAAM,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,yCAAkB,CAAC,CAAC;IAC/C,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;IACxB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;AAC3B,CAAC,CAAC;AAEF,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AACvB,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,IAAI,UAA8B,CAAC;IACnC,IAAI,GAAqB,CAAC;IAE1B,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,CAAC,UAAU,EAAE,GAAG,CAAC,GAAG,MAAM,iBAAiB,EAAE,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,OAAO,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,EAAE,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;QACnC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QACzC,MAAM,IAAA,iBAAK,EAAC,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QAChC,MAAM,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAA,cAAK,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC5D,MAAM,IAAA,iBAAK,EAAC,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC5C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC;QAE9D,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,QAAU,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { INestApplication } from '@nestjs/common';\nimport { EventEmitterModule } from '@nestjs/event-emitter';\nimport { Test } from '@nestjs/testing';\nimport { ConnectionPoolService, delay, NodeConfig } from '@subql/node-core';\nimport { GraphQLSchema } from 'graphql';\nimport { range } from 'lodash';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from './api.service.ethereum';\n\n// Add api key to work\nconst WS_ENDPOINT = 'wss://eth.api.onfinality.io/ws?apikey=';\nconst HTTP_ENDPOINT = 'https://eth.api.onfinality.io/public';\n\nfunction testSubqueryProject(endpoint: string): SubqueryProject {\n  return {\n    network: {\n      endpoint,\n      chainId: '1',\n    },\n    dataSources: [],\n    id: 'test',\n    root: './',\n    schema: new GraphQLSchema({}),\n    templates: [],\n  };\n}\n\nconst prepareApiService = async (\n  endpoint: string = HTTP_ENDPOINT,\n): Promise<[EthereumApiService, INestApplication]> => {\n  const module = await Test.createTestingModule({\n    providers: [\n      ConnectionPoolService,\n      {\n        provide: NodeConfig,\n        useFactory: () => ({}),\n      },\n      {\n        provide: 'ISubqueryProject',\n        useFactory: () => testSubqueryProject(endpoint),\n      },\n      EthereumApiService,\n    ],\n    imports: [EventEmitterModule.forRoot()],\n  }).compile();\n\n  const app = module.createNestApplication();\n  await app.init();\n  const apiService = app.get(EthereumApiService);\n  await apiService.init();\n  return [apiService, app];\n};\n\njest.setTimeout(90000);\ndescribe('ApiService', () => {\n  let apiService: EthereumApiService;\n  let app: INestApplication;\n\n  beforeEach(async () => {\n    [apiService, app] = await prepareApiService();\n  });\n\n  afterEach(async () => {\n    return app?.close();\n  });\n\n  it('can instantiate api', async () => {\n    console.log(apiService.api.getChainId());\n    await delay(0.5);\n  });\n\n  it('can fetch blocks', async () => {\n    await apiService.api.fetchBlocks(range(12369621, 12369651));\n    await delay(0.5);\n  });\n\n  it('can get the finalized height', async () => {\n    const height = await apiService.api.getFinalizedBlockHeight();\n\n    console.log('Finalized height', height);\n    expect(height).toBeGreaterThan(16_000_000);\n  });\n});\n"]}

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js

@@ -37,7 +37,6 @@ async function createIndexerWorker(store, dynamicDsService, unfinalizedBlocksSer
         'getMemoryLeft',
         'waitForWorkerBatchSize',
     ], {
-        storeCount: store.count.bind(store),
         storeGet: store.get.bind(store),
         storeGetByField: store.getByField.bind(store),
         storeGetOneByField: store.getOneByField.bind(store),

package/dist/indexer/blockDispatcher/worker-block-dispatcher.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,gDAAwB;AACxB,2CAA2E;AAC3E,yDAAsD;AACtD,gDAa0B;AAG1B,qEAGyC;AACzC,8DAAyD;AACzD,4EAGsC;AAItC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAMzD,KAAK,UAAU,mBAAmB,CAChC,KAAY,EACZ,gBAAmD,EACnD,wBAAmD;IAEnD,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CAIjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,eAAe;QACf,wBAAwB;KACzB,EACD;QACE,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;QACnC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7C,kBAAkB,EAAE,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC;QACnD,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7C,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7C,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QACrC,gCAAgC,EAC9B,gBAAgB,CAAC,uBAAuB,CAAC,IAAI,CAAC,gBAAgB,CAAC;QACjE,8BAA8B,EAC5B,gBAAgB,CAAC,qBAAqB,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAC/D,wBAAwB,EACtB,wBAAwB,CAAC,wBAAwB,CAAC,IAAI,CACpD,wBAAwB,CACzB;KACJ,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGD,IAAa,4BAA4B,GAAzC,MAAa,4BACX,SAAQ,iCAAoD;IAG5D,YACE,UAAsB,EACtB,YAA2B,EACA,cAA+B,EAC1D,iBAAoC,EACpC,YAA0B,EAC1B,iBAAoC,EACpC,UAAsB,EACM,OAAwB,EACpD,gBAAkC,EAClC,uBAAiD;QAEjD,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,GAAG,EAAE,CACH,mBAAmB,CACjB,YAAY,CAAC,QAAQ,EAAE,EACvB,gBAAgB,EAChB,uBAAuB,CACxB,CACJ,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,UAAU,CACxB,MAAqB,EACrB,MAAc;QAEd,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,oDAAoD;QACpD,yBAAyB;QACzB,iBAAiB;QACjB,yEAAyE;QACzE,OAAO;QACP,+BAA+B;QAC/B,iBAAiB;QACjB,4EAA4E;QAC5E,OAAO;QACP,IAAI;IACN,CAAC;CACF,CAAA;AAtDY,4BAA4B;IADxC,IAAA,mBAAU,GAAE;IAQR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;IAKzB,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAPf,sBAAU;QACR,6BAAa,UAER,6BAAiB;QACtB,wBAAY;QACP,6BAAiB;QACxB,sBAAU;QACe,iCAAe;QAClC,qCAAgB;QACT,oDAAwB;GAdxC,4BAA4B,CAsDxC;AAtDY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport path from 'path';\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  getLogger,\n  NodeConfig,\n  Worker,\n  SmartBatchService,\n  StoreService,\n  PoiService,\n  StoreCacheService,\n  IProjectService,\n  IDynamicDsService,\n  HostStore,\n  HostDynamicDS,\n  WorkerBlockDispatcher,\n} from '@subql/node-core';\nimport { Store } from '@subql/types';\nimport chalk from 'chalk';\nimport {\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../../configure/SubqueryProject';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport {\n  IUnfinalizedBlocksService,\n  UnfinalizedBlocksService,\n} from '../unfinalizedBlocks.service';\nimport { IIndexerWorker, IInitIndexerWorker } from '../worker/worker';\nimport { HostUnfinalizedBlocks } from '../worker/worker.unfinalizedBlocks.service';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IndexerWorker = IIndexerWorker & {\n  terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(\n  store: Store,\n  dynamicDsService: IDynamicDsService<SubqlProjectDs>,\n  unfinalizedBlocksService: IUnfinalizedBlocksService,\n): Promise<IndexerWorker> {\n  const indexerWorker = Worker.create<\n    IInitIndexerWorker,\n    HostDynamicDS<SubqlProjectDs> & HostStore & HostUnfinalizedBlocks\n  >(\n    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n    [\n      'initWorker',\n      'processBlock',\n      'fetchBlock',\n      'numFetchedBlocks',\n      'numFetchingBlocks',\n      'getStatus',\n      'getMemoryLeft',\n      'waitForWorkerBatchSize',\n    ],\n    {\n      storeCount: store.count.bind(store),\n      storeGet: store.get.bind(store),\n      storeGetByField: store.getByField.bind(store),\n      storeGetOneByField: store.getOneByField.bind(store),\n      storeSet: store.set.bind(store),\n      storeBulkCreate: store.bulkCreate.bind(store),\n      storeBulkUpdate: store.bulkUpdate.bind(store),\n      storeRemove: store.remove.bind(store),\n      dynamicDsCreateDynamicDatasource:\n        dynamicDsService.createDynamicDatasource.bind(dynamicDsService),\n      dynamicDsGetDynamicDatasources:\n        dynamicDsService.getDynamicDatasources.bind(dynamicDsService),\n      unfinalizedBlocksProcess:\n        unfinalizedBlocksService.processUnfinalizedBlocks.bind(\n          unfinalizedBlocksService,\n        ),\n    },\n  );\n\n  await indexerWorker.initWorker();\n\n  return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n  extends WorkerBlockDispatcher<SubqlProjectDs, IndexerWorker>\n  implements OnApplicationShutdown\n{\n  constructor(\n    nodeConfig: NodeConfig,\n    eventEmitter: EventEmitter2,\n    @Inject('IProjectService') projectService: IProjectService,\n    smartBatchService: SmartBatchService,\n    storeService: StoreService,\n    storeCacheService: StoreCacheService,\n    poiService: PoiService,\n    @Inject('ISubqueryProject') project: SubqueryProject,\n    dynamicDsService: DynamicDsService,\n    unfinalizedBlocksSevice: UnfinalizedBlocksService,\n  ) {\n    super(\n      nodeConfig,\n      eventEmitter,\n      projectService,\n      smartBatchService,\n      storeService,\n      storeCacheService,\n      poiService,\n      project,\n      dynamicDsService,\n      () =>\n        createIndexerWorker(\n          storeService.getStore(),\n          dynamicDsService,\n          unfinalizedBlocksSevice,\n        ),\n    );\n  }\n\n  protected async fetchBlock(\n    worker: IndexerWorker,\n    height: number,\n  ): Promise<void> {\n    const start = new Date();\n    await worker.fetchBlock(height);\n    const end = new Date();\n\n    // const waitTime = end.getTime() - start.getTime();\n    // if (waitTime > 1000) {\n    //   logger.info(\n    //     `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n    //   );\n    // } else if (waitTime > 200) {\n    //   logger.info(\n    //     `Waiting to fetch block ${height}: ${chalk.yellow(`${waitTime}ms`)}`,\n    //   );\n    // }\n  }\n}\n"]}
+{"version":3,"file":"worker-block-dispatcher.service.js","sourceRoot":"","sources":["../../../src/indexer/blockDispatcher/worker-block-dispatcher.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,gDAAwB;AACxB,2CAA2E;AAC3E,yDAAsD;AACtD,gDAa0B;AAG1B,qEAGyC;AACzC,8DAAyD;AACzD,4EAGsC;AAItC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,8BAA8B,CAAC,CAAC;AAMzD,KAAK,UAAU,mBAAmB,CAChC,KAAY,EACZ,gBAAmD,EACnD,wBAAmD;IAEnD,MAAM,aAAa,GAAG,kBAAM,CAAC,MAAM,CAIjC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,wCAAwC,CAAC,EACjE;QACE,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,mBAAmB;QACnB,WAAW;QACX,eAAe;QACf,wBAAwB;KACzB,EACD;QACE,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7C,kBAAkB,EAAE,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC;QACnD,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7C,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7C,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QACrC,gCAAgC,EAC9B,gBAAgB,CAAC,uBAAuB,CAAC,IAAI,CAAC,gBAAgB,CAAC;QACjE,8BAA8B,EAC5B,gBAAgB,CAAC,qBAAqB,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAC/D,wBAAwB,EACtB,wBAAwB,CAAC,wBAAwB,CAAC,IAAI,CACpD,wBAAwB,CACzB;KACJ,CACF,CAAC;IAEF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;IAEjC,OAAO,aAAa,CAAC;AACvB,CAAC;AAGD,IAAa,4BAA4B,GAAzC,MAAa,4BACX,SAAQ,iCAAoD;IAG5D,YACE,UAAsB,EACtB,YAA2B,EACA,cAA+B,EAC1D,iBAAoC,EACpC,YAA0B,EAC1B,iBAAoC,EACpC,UAAsB,EACM,OAAwB,EACpD,gBAAkC,EAClC,uBAAiD;QAEjD,KAAK,CACH,UAAU,EACV,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,GAAG,EAAE,CACH,mBAAmB,CACjB,YAAY,CAAC,QAAQ,EAAE,EACvB,gBAAgB,EAChB,uBAAuB,CACxB,CACJ,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,UAAU,CACxB,MAAqB,EACrB,MAAc;QAEd,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,oDAAoD;QACpD,yBAAyB;QACzB,iBAAiB;QACjB,yEAAyE;QACzE,OAAO;QACP,+BAA+B;QAC/B,iBAAiB;QACjB,4EAA4E;QAC5E,OAAO;QACP,IAAI;IACN,CAAC;CACF,CAAA;AAtDY,4BAA4B;IADxC,IAAA,mBAAU,GAAE;IAQR,WAAA,IAAA,eAAM,EAAC,iBAAiB,CAAC,CAAA;IAKzB,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCAPf,sBAAU;QACR,6BAAa,UAER,6BAAiB;QACtB,wBAAY;QACP,6BAAiB;QACxB,sBAAU;QACe,iCAAe;QAClC,qCAAgB;QACT,oDAAwB;GAdxC,4BAA4B,CAsDxC;AAtDY,oEAA4B","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport path from 'path';\nimport { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  getLogger,\n  NodeConfig,\n  Worker,\n  SmartBatchService,\n  StoreService,\n  PoiService,\n  StoreCacheService,\n  IProjectService,\n  IDynamicDsService,\n  HostStore,\n  HostDynamicDS,\n  WorkerBlockDispatcher,\n} from '@subql/node-core';\nimport { Store } from '@subql/types';\nimport chalk from 'chalk';\nimport {\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../../configure/SubqueryProject';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport {\n  IUnfinalizedBlocksService,\n  UnfinalizedBlocksService,\n} from '../unfinalizedBlocks.service';\nimport { IIndexerWorker, IInitIndexerWorker } from '../worker/worker';\nimport { HostUnfinalizedBlocks } from '../worker/worker.unfinalizedBlocks.service';\n\nconst logger = getLogger('WorkerBlockDispatcherService');\n\ntype IndexerWorker = IIndexerWorker & {\n  terminate: () => Promise<number>;\n};\n\nasync function createIndexerWorker(\n  store: Store,\n  dynamicDsService: IDynamicDsService<SubqlProjectDs>,\n  unfinalizedBlocksService: IUnfinalizedBlocksService,\n): Promise<IndexerWorker> {\n  const indexerWorker = Worker.create<\n    IInitIndexerWorker,\n    HostDynamicDS<SubqlProjectDs> & HostStore & HostUnfinalizedBlocks\n  >(\n    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),\n    [\n      'initWorker',\n      'processBlock',\n      'fetchBlock',\n      'numFetchedBlocks',\n      'numFetchingBlocks',\n      'getStatus',\n      'getMemoryLeft',\n      'waitForWorkerBatchSize',\n    ],\n    {\n      storeGet: store.get.bind(store),\n      storeGetByField: store.getByField.bind(store),\n      storeGetOneByField: store.getOneByField.bind(store),\n      storeSet: store.set.bind(store),\n      storeBulkCreate: store.bulkCreate.bind(store),\n      storeBulkUpdate: store.bulkUpdate.bind(store),\n      storeRemove: store.remove.bind(store),\n      dynamicDsCreateDynamicDatasource:\n        dynamicDsService.createDynamicDatasource.bind(dynamicDsService),\n      dynamicDsGetDynamicDatasources:\n        dynamicDsService.getDynamicDatasources.bind(dynamicDsService),\n      unfinalizedBlocksProcess:\n        unfinalizedBlocksService.processUnfinalizedBlocks.bind(\n          unfinalizedBlocksService,\n        ),\n    },\n  );\n\n  await indexerWorker.initWorker();\n\n  return indexerWorker;\n}\n\n@Injectable()\nexport class WorkerBlockDispatcherService\n  extends WorkerBlockDispatcher<SubqlProjectDs, IndexerWorker>\n  implements OnApplicationShutdown\n{\n  constructor(\n    nodeConfig: NodeConfig,\n    eventEmitter: EventEmitter2,\n    @Inject('IProjectService') projectService: IProjectService,\n    smartBatchService: SmartBatchService,\n    storeService: StoreService,\n    storeCacheService: StoreCacheService,\n    poiService: PoiService,\n    @Inject('ISubqueryProject') project: SubqueryProject,\n    dynamicDsService: DynamicDsService,\n    unfinalizedBlocksSevice: UnfinalizedBlocksService,\n  ) {\n    super(\n      nodeConfig,\n      eventEmitter,\n      projectService,\n      smartBatchService,\n      storeService,\n      storeCacheService,\n      poiService,\n      project,\n      dynamicDsService,\n      () =>\n        createIndexerWorker(\n          storeService.getStore(),\n          dynamicDsService,\n          unfinalizedBlocksSevice,\n        ),\n    );\n  }\n\n  protected async fetchBlock(\n    worker: IndexerWorker,\n    height: number,\n  ): Promise<void> {\n    const start = new Date();\n    await worker.fetchBlock(height);\n    const end = new Date();\n\n    // const waitTime = end.getTime() - start.getTime();\n    // if (waitTime > 1000) {\n    //   logger.info(\n    //     `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`,\n    //   );\n    // } else if (waitTime > 200) {\n    //   logger.info(\n    //     `Waiting to fetch block ${height}: ${chalk.yellow(`${waitTime}ms`)}`,\n    //   );\n    // }\n  }\n}\n"]}

package/dist/indexer/indexer.module.js

@@ -58,7 +58,6 @@ IndexerModule = __decorate([
                 useClass: project_service_1.ProjectService,
             },
             worker_service_1.WorkerService,
-            unfinalizedBlocks_service_1.UnfinalizedBlocksService,
             {
                 provide: unfinalizedBlocks_service_1.UnfinalizedBlocksService,
                 useFactory: () => {

package/dist/indexer/indexer.module.js.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.module.js","sourceRoot":"","sources":["../../src/indexer/indexer.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAwC;AACxC,yDAAsD;AACtD,gDAQ0B;AAE1B,0CAAiD;AAEjD,iEAA4D;AAC5D,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AACvE,4DAAwD;AACxD,gGAA2F;AAwD3F,IAAa,aAAa,GAA1B,MAAa,aAAa;CAAG,CAAA;AAAhB,aAAa;IAtDzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,gCAAc;YACd,6BAAiB;YACjB,wBAAY;YACZ,iCAAqB;YACrB;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,qBAAmE,EACnE,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,6BAAkB,CACvC,OAAO,EACP,qBAAqB,EACrB,YAAY,CACb,CAAC;oBACF,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,iCAAqB,EAAE,6BAAa,CAAC;aACnE;YACD,gCAAc;YACd,yCAAkB;YAClB;gBACE,OAAO,EAAE,qCAAgB;gBACzB,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,kCAAsB,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAC1D,CAAC;aACF;YACD,sBAAU;YACV,sBAAU;YACV;gBACE,OAAO,EAAE,iBAAiB;gBAC1B,QAAQ,EAAE,gCAAc;aACzB;YACD,8BAAa;YACb,oDAAwB;YACxB;gBACE,OAAO,EAAE,oDAAwB;gBACjC,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,iEAA8B,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAClE,CAAC;aACF;SACF;QACD,OAAO,EAAE,CAAC,wBAAY,CAAC;KACxB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  StoreService,\n  PoiService,\n  MmrService,\n  ConnectionPoolService,\n  StoreCacheService,\n  WorkerDynamicDsService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum';\nimport { EthereumApiConnection } from '../ethereum/api.connection';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\nimport { WorkerService } from './worker/worker.service';\nimport { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    IndexerManager,\n    StoreCacheService,\n    StoreService,\n    ConnectionPoolService,\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        connectionPoolService: ConnectionPoolService<EthereumApiConnection>,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new EthereumApiService(\n          project,\n          connectionPoolService,\n          eventEmitter,\n        );\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2],\n    },\n    SandboxService,\n    DsProcessorService,\n    {\n      provide: DynamicDsService,\n      useFactory: () => {\n        if (isMainThread) {\n          throw new Error('Expected to be worker thread');\n        }\n        return new WorkerDynamicDsService((global as any).host);\n      },\n    },\n    PoiService,\n    MmrService,\n    {\n      provide: 'IProjectService',\n      useClass: ProjectService,\n    },\n    WorkerService,\n    UnfinalizedBlocksService,\n    {\n      provide: UnfinalizedBlocksService,\n      useFactory: () => {\n        if (isMainThread) {\n          throw new Error('Expected to be worker thread');\n        }\n        return new WorkerUnfinalizedBlocksService((global as any).host);\n      },\n    },\n  ],\n  exports: [StoreService],\n})\nexport class IndexerModule {}\n"]}
+{"version":3,"file":"indexer.module.js","sourceRoot":"","sources":["../../src/indexer/indexer.module.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;AAEtC,mDAA8C;AAC9C,2CAAwC;AACxC,yDAAsD;AACtD,gDAQ0B;AAE1B,0CAAiD;AAEjD,iEAA4D;AAC5D,6DAAwD;AACxD,uDAAmD;AACnD,uDAAmD;AACnD,uDAAmD;AACnD,2EAAuE;AACvE,4DAAwD;AACxD,gGAA2F;AAuD3F,IAAa,aAAa,GAA1B,MAAa,aAAa;CAAG,CAAA;AAAhB,aAAa;IArDzB,IAAA,eAAM,EAAC;QACN,SAAS,EAAE;YACT,gCAAc;YACd,6BAAiB;YACjB,wBAAY;YACZ,iCAAqB;YACrB;gBACE,OAAO,EAAE,sBAAU;gBACnB,UAAU,EAAE,KAAK,EACf,OAAwB,EACxB,qBAAmE,EACnE,YAA2B,EAC3B,EAAE;oBACF,MAAM,UAAU,GAAG,IAAI,6BAAkB,CACvC,OAAO,EACP,qBAAqB,EACrB,YAAY,CACb,CAAC;oBACF,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBACxB,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,MAAM,EAAE,CAAC,kBAAkB,EAAE,iCAAqB,EAAE,6BAAa,CAAC;aACnE;YACD,gCAAc;YACd,yCAAkB;YAClB;gBACE,OAAO,EAAE,qCAAgB;gBACzB,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,kCAAsB,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAC1D,CAAC;aACF;YACD,sBAAU;YACV,sBAAU;YACV;gBACE,OAAO,EAAE,iBAAiB;gBAC1B,QAAQ,EAAE,gCAAc;aACzB;YACD,8BAAa;YACb;gBACE,OAAO,EAAE,oDAAwB;gBACjC,UAAU,EAAE,GAAG,EAAE;oBACf,IAAI,6BAAY,EAAE;wBAChB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;qBACjD;oBACD,OAAO,IAAI,iEAA8B,CAAE,MAAc,CAAC,IAAI,CAAC,CAAC;gBAClE,CAAC;aACF;SACF;QACD,OAAO,EAAE,CAAC,wBAAY,CAAC;KACxB,CAAC;GACW,aAAa,CAAG;AAAhB,sCAAa","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport { isMainThread } from 'worker_threads';\nimport { Module } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  StoreService,\n  PoiService,\n  MmrService,\n  ConnectionPoolService,\n  StoreCacheService,\n  WorkerDynamicDsService,\n} from '@subql/node-core';\nimport { SubqueryProject } from '../configure/SubqueryProject';\nimport { EthereumApiService } from '../ethereum';\nimport { EthereumApiConnection } from '../ethereum/api.connection';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { IndexerManager } from './indexer.manager';\nimport { ProjectService } from './project.service';\nimport { SandboxService } from './sandbox.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\nimport { WorkerService } from './worker/worker.service';\nimport { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlocks.service';\n\n@Module({\n  providers: [\n    IndexerManager,\n    StoreCacheService,\n    StoreService,\n    ConnectionPoolService,\n    {\n      provide: ApiService,\n      useFactory: async (\n        project: SubqueryProject,\n        connectionPoolService: ConnectionPoolService<EthereumApiConnection>,\n        eventEmitter: EventEmitter2,\n      ) => {\n        const apiService = new EthereumApiService(\n          project,\n          connectionPoolService,\n          eventEmitter,\n        );\n        await apiService.init();\n        return apiService;\n      },\n      inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2],\n    },\n    SandboxService,\n    DsProcessorService,\n    {\n      provide: DynamicDsService,\n      useFactory: () => {\n        if (isMainThread) {\n          throw new Error('Expected to be worker thread');\n        }\n        return new WorkerDynamicDsService((global as any).host);\n      },\n    },\n    PoiService,\n    MmrService,\n    {\n      provide: 'IProjectService',\n      useClass: ProjectService,\n    },\n    WorkerService,\n    {\n      provide: UnfinalizedBlocksService,\n      useFactory: () => {\n        if (isMainThread) {\n          throw new Error('Expected to be worker thread');\n        }\n        return new WorkerUnfinalizedBlocksService((global as any).host);\n      },\n    },\n  ],\n  exports: [StoreService],\n})\nexport class IndexerModule {}\n"]}

package/dist/indexer/project.service.js

@@ -92,7 +92,7 @@ let ProjectService = class ProjectService {
                 this._startHeight = reindexedTo;
             }
             // Flush any pending operations to setup DB
-            await this.storeService.storeCache.flushCache();
+            await this.storeService.storeCache.flushCache(true);
         }
         else {
             this._schema = await this.getExistingProjectSchema();

package/dist/indexer/project.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAAoD;AACpD,yDAAsD;AACtD,gDAU0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAqE;AACrE,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AACxD,2EAAuE;AAEvE,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAGpC,IAAa,cAAc,GAA3B,MAAa,cAAc;IAKzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B,EAC3B,uBAAiD;QAVxC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACQ,YAAO,GAAP,OAAO,CAAiB;QACpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,4BAAuB,GAAvB,uBAAuB,CAA0B;IACxD,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAElE,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAEhD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;gBAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CACxB,CAAC;YAEF,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC7B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;aACjC;YAED,2CAA2C;YAC3C,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;SACjD;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAErD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5D,CAAC;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAEvD,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,aAAa;YACb,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;SACd,CAAC;QAEX,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE/C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE5C,QAAQ,CAAC,OAAO,CAAC;gBACf,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBACvC,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;gBACjD,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;gBACzC,EAAE,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;aACpD,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;SAC1C;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SAC9B;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;SACpC;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,QAAQ,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;SACxC;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,QAAQ,CAAC,GAAG,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC;SACzC;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC;SAClE;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY,KAAK,SAAS;YAC/B,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,uBAAuB,EAC5B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;QAEtE,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,CAAC,MAAM,CAC/C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,WAAW,CACrC,CAAC;IACJ,CAAC;CACF,CAAA;AArRY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAYR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALU,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;QACF,oDAAwB;GAhBhD,cAAc,CAqR1B;AArRY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  IProjectService,\n  NodeConfig,\n  IndexerEvent,\n  StoreService,\n  PoiService,\n  MmrService,\n  getLogger,\n  getExistingProjectSchema,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n  generateTimestampReferenceForBlockFilters,\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema, initHotSchemaReload } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService implements IProjectService {\n  private _schema: string;\n  private _startHeight: number;\n  private _blockOffset: number;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly apiService: ApiService,\n    private readonly poiService: PoiService,\n    protected readonly mmrService: MmrService,\n    private readonly sequelize: Sequelize,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly dynamicDsService: DynamicDsService,\n    private eventEmitter: EventEmitter2,\n    private unfinalizedBlockService: UnfinalizedBlocksService,\n  ) {}\n\n  get schema(): string {\n    return this._schema;\n  }\n\n  get dataSources(): SubqlProjectDs[] {\n    return this.project.dataSources;\n  }\n\n  get blockOffset(): number {\n    return this._blockOffset;\n  }\n\n  get startHeight(): number {\n    return this._startHeight;\n  }\n\n  get isHistorical(): boolean {\n    return this.storeService.historical;\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  async init(): Promise<void> {\n    // Used to load assets into DS-processor, has to be done in any thread\n    await this.dsProcessorService.validateProjectCustomDatasources();\n    // Do extra work on main thread to setup stuff\n    this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n      this.project.dataSources,\n      this.apiService.api,\n    );\n    if (isMainThread) {\n      this._schema = await this.ensureProject();\n      await this.initDbSchema();\n      await this.ensureMetadata();\n      this.dynamicDsService.init(this.storeService.storeCache.metadata);\n\n      await this.initHotSchemaReload();\n\n      if (this.nodeConfig.proofOfIndex) {\n        const blockOffset = await this.getMetadataBlockOffset();\n        void this.setBlockOffset(Number(blockOffset));\n        await this.poiService.init();\n      }\n\n      this._startHeight = await this.getStartHeight();\n\n      if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n        logger.error(\n          'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n        );\n        process.exit(1);\n      }\n\n      const reindexedTo = await this.unfinalizedBlockService.init(\n        this.reindex.bind(this),\n      );\n\n      if (reindexedTo !== undefined) {\n        this._startHeight = reindexedTo;\n      }\n\n      // Flush any pending operations to setup DB\n      await this.storeService.storeCache.flushCache();\n    } else {\n      this._schema = await this.getExistingProjectSchema();\n\n      await this.sequelize.sync();\n\n      assert(this._schema, 'Schema should be created in main thread');\n      await this.initDbSchema();\n\n      if (this.nodeConfig.proofOfIndex) {\n        await this.poiService.init();\n      }\n    }\n  }\n\n  private async ensureProject(): Promise<string> {\n    let schema = await this.getExistingProjectSchema();\n    if (!schema) {\n      schema = await this.createProjectSchema();\n    }\n    this.eventEmitter.emit(IndexerEvent.Ready, {\n      value: true,\n    });\n\n    return schema;\n  }\n\n  private async createProjectSchema(): Promise<string> {\n    let schema: string;\n    if (this.nodeConfig.localMode) {\n      // create tables in default schema if local mode is enabled\n      schema = DEFAULT_DB_SCHEMA;\n    } else {\n      schema = this.nodeConfig.dbSchema;\n      const schemas = await this.sequelize.showAllSchemas(undefined);\n      if (!(schemas as unknown as string[]).includes(schema)) {\n        await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n      }\n    }\n\n    return schema;\n  }\n\n  private async initHotSchemaReload(): Promise<void> {\n    await initHotSchemaReload(this.schema, this.storeService);\n  }\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private async ensureMetadata(): Promise<void> {\n    const metadata = this.storeService.storeCache.metadata;\n\n    this.eventEmitter.emit(\n      IndexerEvent.NetworkMetadata,\n      this.apiService.networkMeta,\n    );\n\n    const keys = [\n      'lastProcessedHeight',\n      'blockOffset',\n      'indexerNodeVersion',\n      'chain',\n      'specName',\n      'genesisHash',\n      'startHeight',\n      'processedBlockCount',\n      'lastFinalizedVerifiedHeight',\n      'schemaMigrationCount',\n    ] as const;\n\n    const existing = await metadata.findMany(keys);\n\n    const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n    if (this.project.runner) {\n      const { node, query } = this.project.runner;\n\n      metadata.setBulk([\n        { key: 'runnerNode', value: node.name },\n        { key: 'runnerNodeVersion', value: node.version },\n        { key: 'runnerQuery', value: query.name },\n        { key: 'runnerQueryVersion', value: query.version },\n      ]);\n    }\n    if (!existing.genesisHash) {\n      metadata.set('genesisHash', genesisHash);\n    } else {\n      // Check if the configured genesisHash matches the currently stored genesisHash\n      assert(\n        // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n        (this.project.network.genesisHash ?? genesisHash) ===\n          existing.genesisHash,\n        'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n      );\n    }\n    if (existing.chain !== chain) {\n      metadata.set('chain', chain);\n    }\n\n    if (existing.specName !== specName) {\n      metadata.set('specName', specName);\n    }\n\n    // If project was created before this feature, don't add the key. If it is project created after, add this key.\n    if (!existing.processedBlockCount && !existing.lastProcessedHeight) {\n      metadata.set('processedBlockCount', 0);\n    }\n\n    if (existing.indexerNodeVersion !== packageVersion) {\n      metadata.set('indexerNodeVersion', packageVersion);\n    }\n    if (!existing.schemaMigrationCount) {\n      metadata.set('schemaMigrationCount', 0);\n    }\n    if (!existing.startHeight) {\n      metadata.set('startHeight', this.getStartBlockFromDataSources());\n    }\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('blockOffset');\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('lastProcessedHeight');\n  }\n\n  private async getStartHeight(): Promise<number> {\n    let startHeight: number;\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n    if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n      startHeight = Number(lastProcessedHeight) + 1;\n    } else {\n      startHeight = this.getStartBlockFromDataSources();\n    }\n    return startHeight;\n  }\n\n  async setBlockOffset(offset: number): Promise<void> {\n    if (\n      this._blockOffset !== undefined ||\n      offset === null ||\n      offset === undefined ||\n      isNaN(offset)\n    ) {\n      return;\n    }\n    logger.info(`set blockOffset to ${offset}`);\n    this._blockOffset = offset;\n    return this.mmrService.syncFileBaseFromPoi(offset).catch((err) => {\n      logger.error(err, 'failed to sync poi to mmr');\n      process.exit(1);\n    });\n  }\n\n  private getStartBlockFromDataSources() {\n    const startBlocksList = this.project.dataSources.map(\n      (item) => item.startBlock ?? 1,\n    );\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n\n    return reindex(\n      this.getStartBlockFromDataSources(),\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlockService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      /* Not providing force clean service, it should never be needed */\n    );\n  }\n\n  async getAllDataSources(blockHeight: number): Promise<SubqlProjectDs[]> {\n    const dynamicDs = await this.dynamicDsService.getDynamicDatasources();\n\n    return [...this.dataSources, ...dynamicDs].filter(\n      (ds) => ds.startBlock <= blockHeight,\n    );\n  }\n}\n"]}
+{"version":3,"file":"project.service.js","sourceRoot":"","sources":["../../src/indexer/project.service.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;;;;;;;;;;;;;;AAEtC,oDAA4B;AAC5B,mDAA8C;AAC9C,2CAAoD;AACpD,yDAAsD;AACtD,gDAU0B;AAC1B,yCAAsC;AACtC,kEAIsC;AACtC,8CAAqE;AACrE,8CAA2C;AAC3C,iEAA4D;AAC5D,6DAAwD;AACxD,2EAAuE;AAEvE,8DAA8D;AAC9D,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAElE,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAEnC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,SAAS,CAAC,CAAC;AAGpC,IAAa,cAAc,GAA3B,MAAa,cAAc;IAKzB,YACmB,kBAAsC,EACtC,UAAsB,EACtB,UAAsB,EACpB,UAAsB,EACxB,SAAoB,EACQ,OAAwB,EACpD,YAA0B,EAC1B,UAAsB,EACtB,gBAAkC,EAC3C,YAA2B,EAC3B,uBAAiD;QAVxC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,eAAU,GAAV,UAAU,CAAY;QACtB,eAAU,GAAV,UAAU,CAAY;QACpB,eAAU,GAAV,UAAU,CAAY;QACxB,cAAS,GAAT,SAAS,CAAW;QACQ,YAAO,GAAP,OAAO,CAAiB;QACpD,iBAAY,GAAZ,YAAY,CAAc;QAC1B,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAC3C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,4BAAuB,GAAvB,uBAAuB,CAA0B;IACxD,CAAC;IAEJ,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,OAAO,IAAA,oCAAwB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,sEAAsE;QACtE,MAAM,IAAI,CAAC,kBAAkB,CAAC,gCAAgC,EAAE,CAAC;QACjE,8CAA8C;QAC9C,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,IAAA,2DAAyC,EACxE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,UAAU,CAAC,GAAG,CACpB,CAAC;QACF,IAAI,6BAAY,EAAE;YAChB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAElE,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACxD,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;YAED,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAEhD,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;gBAC3D,MAAM,CAAC,KAAK,CACV,qHAAqH,CACtH,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,CACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CACxB,CAAC;YAEF,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC7B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;aACjC;YAED,2CAA2C;YAC3C,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;SACrD;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAErD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAA,gBAAM,EAAC,IAAI,CAAC,OAAO,EAAE,yCAAyC,CAAC,CAAC;YAChE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;gBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aAC9B;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC3C;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,wBAAY,CAAC,KAAK,EAAE;YACzC,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,MAAc,CAAC;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;YAC7B,2DAA2D;YAC3D,MAAM,GAAG,iBAAiB,CAAC;SAC5B;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAE,OAA+B,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;aAC7D;SACF;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,MAAM,IAAA,6BAAmB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5D,CAAC;IACO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAA,sBAAY,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,cAAc;;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC;QAEvD,IAAI,CAAC,YAAY,CAAC,IAAI,CACpB,wBAAY,CAAC,eAAe,EAC5B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QAEF,MAAM,IAAI,GAAG;YACX,qBAAqB;YACrB,aAAa;YACb,oBAAoB;YACpB,OAAO;YACP,UAAU;YACV,aAAa;YACb,aAAa;YACb,qBAAqB;YACrB,6BAA6B;YAC7B,sBAAsB;SACd,CAAC;QAEX,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE/C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAErE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACvB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAE5C,QAAQ,CAAC,OAAO,CAAC;gBACf,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;gBACvC,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;gBACjD,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;gBACzC,EAAE,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;aACpD,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;SAC1C;aAAM;YACL,+EAA+E;YAC/E,IAAA,gBAAM;YACJ,uHAAuH;YACvH,CAAC,MAAA,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,mCAAI,WAAW,CAAC;gBAC/C,QAAQ,CAAC,WAAW,EACtB,sJAAsJ,CACvJ,CAAC;SACH;QACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE;YAC5B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SAC9B;QAED,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;SACpC;QAED,+GAA+G;QAC/G,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE;YAClE,QAAQ,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;SACxC;QAED,IAAI,QAAQ,CAAC,kBAAkB,KAAK,cAAc,EAAE;YAClD,QAAQ,CAAC,GAAG,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;SACpD;QACD,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE;YAClC,QAAQ,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC;SACzC;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;YACzB,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,4BAA4B,EAAE,CAAC,CAAC;SAClE;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,WAAmB,CAAC;QACxB,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChE,IAAI,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,KAAK,SAAS,EAAE;YACrE,WAAW,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;SAC/C;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,4BAA4B,EAAE,CAAC;SACnD;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,IACE,IAAI,CAAC,YAAY,KAAK,SAAS;YAC/B,MAAM,KAAK,IAAI;YACf,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,CAAC,EACb;YACA,OAAO;SACR;QACD,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,2BAA2B,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,4BAA4B;QAClC,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAClD,CAAC,IAAI,EAAE,EAAE,WAAC,OAAA,MAAA,IAAI,CAAC,UAAU,mCAAI,CAAC,CAAA,EAAA,CAC/B,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CACV,2FAA2F,CAC5F,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;aAAM;YACL,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,iBAAyB;QACrC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEhE,OAAO,IAAA,iBAAO,EACZ,IAAI,CAAC,4BAA4B,EAAE,EACnC,MAAM,IAAI,CAAC,sBAAsB,EAAE,EACnC,iBAAiB,EACjB,mBAAmB,EACnB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,uBAAuB,EAC5B,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,CAEf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,CAAC;QAEtE,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,CAAC,MAAM,CAC/C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,IAAI,WAAW,CACrC,CAAC;IACJ,CAAC;CACF,CAAA;AArRY,cAAc;IAD1B,IAAA,mBAAU,GAAE;IAYR,WAAA,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAA;qCALU,yCAAkB;QAC1B,sBAAU;QACV,sBAAU;QACR,sBAAU;QACb,qBAAS;QACiB,iCAAe;QACtC,wBAAY;QACd,sBAAU;QACJ,qCAAgB;QAC7B,6BAAa;QACF,oDAAwB;GAhBhD,cAAc,CAqR1B;AArRY,wCAAc","sourcesContent":["// Copyright 2020-2022 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\nimport assert from 'assert';\nimport { isMainThread } from 'worker_threads';\nimport { Inject, Injectable } from '@nestjs/common';\nimport { EventEmitter2 } from '@nestjs/event-emitter';\nimport {\n  ApiService,\n  IProjectService,\n  NodeConfig,\n  IndexerEvent,\n  StoreService,\n  PoiService,\n  MmrService,\n  getLogger,\n  getExistingProjectSchema,\n} from '@subql/node-core';\nimport { Sequelize } from 'sequelize';\nimport {\n  generateTimestampReferenceForBlockFilters,\n  SubqlProjectDs,\n  SubqueryProject,\n} from '../configure/SubqueryProject';\nimport { initDbSchema, initHotSchemaReload } from '../utils/project';\nimport { reindex } from '../utils/reindex';\nimport { DsProcessorService } from './ds-processor.service';\nimport { DynamicDsService } from './dynamic-ds.service';\nimport { UnfinalizedBlocksService } from './unfinalizedBlocks.service';\n\n// eslint-disable-next-line @typescript-eslint/no-var-requires\nconst { version: packageVersion } = require('../../package.json');\n\nconst DEFAULT_DB_SCHEMA = 'public';\n\nconst logger = getLogger('Project');\n\n@Injectable()\nexport class ProjectService implements IProjectService {\n  private _schema: string;\n  private _startHeight: number;\n  private _blockOffset: number;\n\n  constructor(\n    private readonly dsProcessorService: DsProcessorService,\n    private readonly apiService: ApiService,\n    private readonly poiService: PoiService,\n    protected readonly mmrService: MmrService,\n    private readonly sequelize: Sequelize,\n    @Inject('ISubqueryProject') private readonly project: SubqueryProject,\n    private readonly storeService: StoreService,\n    private readonly nodeConfig: NodeConfig,\n    private readonly dynamicDsService: DynamicDsService,\n    private eventEmitter: EventEmitter2,\n    private unfinalizedBlockService: UnfinalizedBlocksService,\n  ) {}\n\n  get schema(): string {\n    return this._schema;\n  }\n\n  get dataSources(): SubqlProjectDs[] {\n    return this.project.dataSources;\n  }\n\n  get blockOffset(): number {\n    return this._blockOffset;\n  }\n\n  get startHeight(): number {\n    return this._startHeight;\n  }\n\n  get isHistorical(): boolean {\n    return this.storeService.historical;\n  }\n\n  private async getExistingProjectSchema(): Promise<string> {\n    return getExistingProjectSchema(this.nodeConfig, this.sequelize);\n  }\n\n  async init(): Promise<void> {\n    // Used to load assets into DS-processor, has to be done in any thread\n    await this.dsProcessorService.validateProjectCustomDatasources();\n    // Do extra work on main thread to setup stuff\n    this.project.dataSources = await generateTimestampReferenceForBlockFilters(\n      this.project.dataSources,\n      this.apiService.api,\n    );\n    if (isMainThread) {\n      this._schema = await this.ensureProject();\n      await this.initDbSchema();\n      await this.ensureMetadata();\n      this.dynamicDsService.init(this.storeService.storeCache.metadata);\n\n      await this.initHotSchemaReload();\n\n      if (this.nodeConfig.proofOfIndex) {\n        const blockOffset = await this.getMetadataBlockOffset();\n        void this.setBlockOffset(Number(blockOffset));\n        await this.poiService.init();\n      }\n\n      this._startHeight = await this.getStartHeight();\n\n      if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) {\n        logger.error(\n          'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical',\n        );\n        process.exit(1);\n      }\n\n      const reindexedTo = await this.unfinalizedBlockService.init(\n        this.reindex.bind(this),\n      );\n\n      if (reindexedTo !== undefined) {\n        this._startHeight = reindexedTo;\n      }\n\n      // Flush any pending operations to setup DB\n      await this.storeService.storeCache.flushCache(true);\n    } else {\n      this._schema = await this.getExistingProjectSchema();\n\n      await this.sequelize.sync();\n\n      assert(this._schema, 'Schema should be created in main thread');\n      await this.initDbSchema();\n\n      if (this.nodeConfig.proofOfIndex) {\n        await this.poiService.init();\n      }\n    }\n  }\n\n  private async ensureProject(): Promise<string> {\n    let schema = await this.getExistingProjectSchema();\n    if (!schema) {\n      schema = await this.createProjectSchema();\n    }\n    this.eventEmitter.emit(IndexerEvent.Ready, {\n      value: true,\n    });\n\n    return schema;\n  }\n\n  private async createProjectSchema(): Promise<string> {\n    let schema: string;\n    if (this.nodeConfig.localMode) {\n      // create tables in default schema if local mode is enabled\n      schema = DEFAULT_DB_SCHEMA;\n    } else {\n      schema = this.nodeConfig.dbSchema;\n      const schemas = await this.sequelize.showAllSchemas(undefined);\n      if (!(schemas as unknown as string[]).includes(schema)) {\n        await this.sequelize.createSchema(`\"${schema}\"`, undefined);\n      }\n    }\n\n    return schema;\n  }\n\n  private async initHotSchemaReload(): Promise<void> {\n    await initHotSchemaReload(this.schema, this.storeService);\n  }\n  private async initDbSchema(): Promise<void> {\n    await initDbSchema(this.project, this.schema, this.storeService);\n  }\n\n  private async ensureMetadata(): Promise<void> {\n    const metadata = this.storeService.storeCache.metadata;\n\n    this.eventEmitter.emit(\n      IndexerEvent.NetworkMetadata,\n      this.apiService.networkMeta,\n    );\n\n    const keys = [\n      'lastProcessedHeight',\n      'blockOffset',\n      'indexerNodeVersion',\n      'chain',\n      'specName',\n      'genesisHash',\n      'startHeight',\n      'processedBlockCount',\n      'lastFinalizedVerifiedHeight',\n      'schemaMigrationCount',\n    ] as const;\n\n    const existing = await metadata.findMany(keys);\n\n    const { chain, genesisHash, specName } = this.apiService.networkMeta;\n\n    if (this.project.runner) {\n      const { node, query } = this.project.runner;\n\n      metadata.setBulk([\n        { key: 'runnerNode', value: node.name },\n        { key: 'runnerNodeVersion', value: node.version },\n        { key: 'runnerQuery', value: query.name },\n        { key: 'runnerQueryVersion', value: query.version },\n      ]);\n    }\n    if (!existing.genesisHash) {\n      metadata.set('genesisHash', genesisHash);\n    } else {\n      // Check if the configured genesisHash matches the currently stored genesisHash\n      assert(\n        // Configured project yaml genesisHash only exists in specVersion v0.2.0, fallback to api fetched genesisHash on v0.0.1\n        (this.project.network.genesisHash ?? genesisHash) ===\n          existing.genesisHash,\n        'Specified project manifest chain id / genesis hash does not match database stored genesis hash, consider cleaning project schema using --force-clean',\n      );\n    }\n    if (existing.chain !== chain) {\n      metadata.set('chain', chain);\n    }\n\n    if (existing.specName !== specName) {\n      metadata.set('specName', specName);\n    }\n\n    // If project was created before this feature, don't add the key. If it is project created after, add this key.\n    if (!existing.processedBlockCount && !existing.lastProcessedHeight) {\n      metadata.set('processedBlockCount', 0);\n    }\n\n    if (existing.indexerNodeVersion !== packageVersion) {\n      metadata.set('indexerNodeVersion', packageVersion);\n    }\n    if (!existing.schemaMigrationCount) {\n      metadata.set('schemaMigrationCount', 0);\n    }\n    if (!existing.startHeight) {\n      metadata.set('startHeight', this.getStartBlockFromDataSources());\n    }\n  }\n\n  private async getMetadataBlockOffset(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('blockOffset');\n  }\n\n  private async getLastProcessedHeight(): Promise<number | undefined> {\n    return this.storeService.storeCache.metadata.find('lastProcessedHeight');\n  }\n\n  private async getStartHeight(): Promise<number> {\n    let startHeight: number;\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n    if (lastProcessedHeight !== null && lastProcessedHeight !== undefined) {\n      startHeight = Number(lastProcessedHeight) + 1;\n    } else {\n      startHeight = this.getStartBlockFromDataSources();\n    }\n    return startHeight;\n  }\n\n  async setBlockOffset(offset: number): Promise<void> {\n    if (\n      this._blockOffset !== undefined ||\n      offset === null ||\n      offset === undefined ||\n      isNaN(offset)\n    ) {\n      return;\n    }\n    logger.info(`set blockOffset to ${offset}`);\n    this._blockOffset = offset;\n    return this.mmrService.syncFileBaseFromPoi(offset).catch((err) => {\n      logger.error(err, 'failed to sync poi to mmr');\n      process.exit(1);\n    });\n  }\n\n  private getStartBlockFromDataSources() {\n    const startBlocksList = this.project.dataSources.map(\n      (item) => item.startBlock ?? 1,\n    );\n    if (startBlocksList.length === 0) {\n      logger.error(\n        `Failed to find a valid datasource, Please check your endpoint if specName filter is used.`,\n      );\n      process.exit(1);\n    } else {\n      return Math.min(...startBlocksList);\n    }\n  }\n\n  async reindex(targetBlockHeight: number): Promise<void> {\n    const lastProcessedHeight = await this.getLastProcessedHeight();\n\n    return reindex(\n      this.getStartBlockFromDataSources(),\n      await this.getMetadataBlockOffset(),\n      targetBlockHeight,\n      lastProcessedHeight,\n      this.storeService,\n      this.unfinalizedBlockService,\n      this.dynamicDsService,\n      this.mmrService,\n      this.sequelize,\n      /* Not providing force clean service, it should never be needed */\n    );\n  }\n\n  async getAllDataSources(blockHeight: number): Promise<SubqlProjectDs[]> {\n    const dynamicDs = await this.dynamicDsService.getDynamicDatasources();\n\n    return [...this.dataSources, ...dynamicDs].filter(\n      (ds) => ds.startBlock <= blockHeight,\n    );\n  }\n}\n"]}

package/dist/indexer/worker/worker.js

@@ -18,7 +18,6 @@ const node_worker_threads_1 = require("node:worker_threads");
 const v8_1 = require("v8");
 const core_1 = require("@nestjs/core");
 const node_core_1 = require("@subql/node-core");
-const dynamic_ds_service_1 = require("../dynamic-ds.service");
 const indexer_manager_1 = require("../indexer.manager");
 const worker_module_1 = require("./worker.module");
 const worker_service_1 = require("./worker.service");
@@ -53,13 +52,7 @@ async function fetchBlock(height) {
 }
 async function processBlock(height) {
     (0, assert_1.default)(workerService, 'Not initialised');
-    const res = await workerService.processBlock(height);
-    // Clean up the temp ds records for worker thread instance
-    if (res.dynamicDsCreated) {
-        const dynamicDsService = app.get(dynamic_ds_service_1.DynamicDsService);
-        dynamicDsService.deleteTempDsRecords(height);
-    }
-    return res;
+    return workerService.processBlock(height);
 }
 // eslint-disable-next-line @typescript-eslint/require-await
 async function numFetchedBlocks() {

package/dist/indexer/worker/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAC/C,2BAAuC;AAEvC,uCAA2C;AAC3C,gDAS0B;AAE1B,8DAAyD;AACzD,wDAAoD;AACpD,mDAA+C;AAC/C,qDAK0B;AAC1B,yFAG4C;AAC5C,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;AAEjC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;KACxC;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAErD,0DAA0D;IAC1D,IAAI,GAAG,CAAC,gBAAgB,EAAE;QACxB,MAAM,gBAAgB,GAAG,GAAG,CAAC,GAAG,CAAC,qCAAgB,CAAC,CAAC;QACnD,gBAAgB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;KAC9C;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,aAAa;IAC1B,MAAM,SAAS,GAAG,IAAA,sBAAiB,GAAE,CAAC,eAAe,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;IAEhD,OAAO,SAAS,GAAG,QAAQ,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,eAAuB;IAC3D,MAAM,IAAA,4BAAgB,EAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,wDAAwD;AACvD,MAAc,CAAC,IAAI,GAAG,sBAAU,CAAC,MAAM,CAItC,CAAC,GAAG,yBAAa,EAAE,GAAG,6BAAiB,EAAE,GAAG,4DAAyB,CAAC,EACtE;IACE,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,aAAa;IACb,sBAAsB;CACvB,EACD,MAAM,CACP,CAAC;AA0BF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { getHeapStatistics } from 'v8';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport {\n  waitForBatchSize,\n  WorkerHost,\n  getLogger,\n  NestLogger,\n  hostStoreKeys,\n  HostStore,\n  hostDynamicDsKeys,\n  HostDynamicDS,\n} from '@subql/node-core';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n  FetchBlockResponse,\n  ProcessBlockResponse,\n  WorkerService,\n  WorkerStatusResponse,\n} from './worker.service';\nimport {\n  HostUnfinalizedBlocks,\n  hostUnfinalizedBlocksKeys,\n} from './worker.unfinalizedBlocks.service';\nlet app: INestApplication;\nlet workerService: WorkerService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n  try {\n    if (app) {\n      logger.warn('Worker already initialised');\n      return;\n    }\n\n    app = await NestFactory.create(WorkerModule, {\n      logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n    });\n\n    await app.init();\n\n    const indexerManager = app.get(IndexerManager);\n    // Initialise async services, we do this here rather than in factories so we can capture one off events\n    await indexerManager.start();\n\n    workerService = app.get(WorkerService);\n  } catch (e) {\n    console.log('Failed to start worker', e);\n    logger.error(e, 'Failed to start worker');\n    throw e;\n  }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n  assert(workerService, 'Not initialised');\n  return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n  assert(workerService, 'Not initialised');\n\n  const res = await workerService.processBlock(height);\n\n  // Clean up the temp ds records for worker thread instance\n  if (res.dynamicDsCreated) {\n    const dynamicDsService = app.get(DynamicDsService);\n    dynamicDsService.deleteTempDsRecords(height);\n  }\n\n  return res;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n  return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n  return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n  return {\n    threadId,\n    fetchedBlocks: workerService.numFetchedBlocks,\n    toFetchBlocks: workerService.numFetchingBlocks,\n    isIndexing: workerService.isIndexing,\n  };\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getMemoryLeft(): Promise<number> {\n  const totalHeap = getHeapStatistics().heap_size_limit;\n  const heapUsed = process.memoryUsage().heapUsed;\n\n  return totalHeap - heapUsed;\n}\n\nasync function waitForWorkerBatchSize(heapSizeInBytes: number): Promise<void> {\n  await waitForBatchSize(heapSizeInBytes);\n}\n\n// Register these functions to be exposed to worker host\n(global as any).host = WorkerHost.create<\n  HostStore & HostDynamicDS<SubqlProjectDs> & HostUnfinalizedBlocks,\n  IInitIndexerWorker\n>(\n  [...hostStoreKeys, ...hostDynamicDsKeys, ...hostUnfinalizedBlocksKeys],\n  {\n    initWorker,\n    fetchBlock,\n    processBlock,\n    numFetchedBlocks,\n    numFetchingBlocks,\n    getStatus,\n    getMemoryLeft,\n    waitForWorkerBatchSize,\n  },\n  logger,\n);\n\n// Export types to be used on the parent\ntype InitWorker = typeof initWorker;\ntype FetchBlock = typeof fetchBlock;\ntype ProcessBlock = typeof processBlock;\ntype NumFetchedBlocks = typeof numFetchedBlocks;\ntype NumFetchingBlocks = typeof numFetchingBlocks;\ntype GetWorkerStatus = typeof getStatus;\ntype GetMemoryLeft = typeof getMemoryLeft;\ntype WaitForWorkerBatchSize = typeof waitForWorkerBatchSize;\n\nexport type IIndexerWorker = {\n  processBlock: ProcessBlock;\n  fetchBlock: FetchBlock;\n  numFetchedBlocks: NumFetchedBlocks;\n  numFetchingBlocks: NumFetchingBlocks;\n  getStatus: GetWorkerStatus;\n  getMemoryLeft: GetMemoryLeft;\n  waitForWorkerBatchSize: WaitForWorkerBatchSize;\n};\n\nexport type IInitIndexerWorker = IIndexerWorker & {\n  initWorker: InitWorker;\n};\n\nprocess.on('uncaughtException', (e) => {\n  logger.error(e, 'Uncaught Exception');\n  throw e;\n});\n"]}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/indexer/worker/worker.ts"],"names":[],"mappings":";AAAA,gEAAgE;AAChE,sCAAsC;;;;;AAEtC,iEAAiE;AACjE,kDAAkD;AAClD,wCAAwC;AACxC,oDAAqD;AACrD,wCAAwC;AACxC,uCAA2C;AAE3C,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAY,CAAC;AAE9B,IAAA,mBAAU,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,SAA+B,EACpC,IAAI,CAAC,QAA8B,CACpC,CAAC;AAEF,oDAA4B;AAC5B,6DAA+C;AAC/C,2BAAuC;AAEvC,uCAA2C;AAC3C,gDAS0B;AAG1B,wDAAoD;AACpD,mDAA+C;AAC/C,qDAK0B;AAC1B,yFAG4C;AAC5C,IAAI,GAAqB,CAAC;AAC1B,IAAI,aAA4B,CAAC;AAEjC,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,WAAW,8BAAQ,EAAE,CAAC,CAAC;AAEhD,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,IAAI,GAAG,EAAE;YACP,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC1C,OAAO;SACR;QAED,GAAG,GAAG,MAAM,kBAAW,CAAC,MAAM,CAAC,4BAAY,EAAE;YAC3C,MAAM,EAAE,IAAI,sBAAU,EAAE,EAAE,0EAA0E;SACrG,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAEjB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,gCAAc,CAAC,CAAC;QAC/C,uGAAuG;QACvG,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;QAE7B,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,8BAAa,CAAC,CAAC;KACxC;IAAC,OAAO,CAAC,EAAE;QACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1C,MAAM,CAAC,CAAC;KACT;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACzC,OAAO,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAA,gBAAM,EAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAEzC,OAAO,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,gBAAgB;IAC7B,OAAO,aAAa,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,iBAAiB;IAC9B,OAAO,aAAa,CAAC,iBAAiB,CAAC;AACzC,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,SAAS;IACtB,OAAO;QACL,QAAQ,EAAR,8BAAQ;QACR,aAAa,EAAE,aAAa,CAAC,gBAAgB;QAC7C,aAAa,EAAE,aAAa,CAAC,iBAAiB;QAC9C,UAAU,EAAE,aAAa,CAAC,UAAU;KACrC,CAAC;AACJ,CAAC;AAED,4DAA4D;AAC5D,KAAK,UAAU,aAAa;IAC1B,MAAM,SAAS,GAAG,IAAA,sBAAiB,GAAE,CAAC,eAAe,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;IAEhD,OAAO,SAAS,GAAG,QAAQ,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,eAAuB;IAC3D,MAAM,IAAA,4BAAgB,EAAC,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED,wDAAwD;AACvD,MAAc,CAAC,IAAI,GAAG,sBAAU,CAAC,MAAM,CAItC,CAAC,GAAG,yBAAa,EAAE,GAAG,6BAAiB,EAAE,GAAG,4DAAyB,CAAC,EACtE;IACE,UAAU;IACV,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,SAAS;IACT,aAAa;IACb,sBAAsB;CACvB,EACD,MAAM,CACP,CAAC;AA0BF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;IACpC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACtC,MAAM,CAAC,CAAC;AACV,CAAC,CAAC,CAAC","sourcesContent":["// Copyright 2020-2021 OnFinality Limited authors & contributors\n// SPDX-License-Identifier: Apache-2.0\n\n// initlogger and yargs must be imported before all other imports\n// making sure logger is defined before its called\n// eslint-disable-next-line import/order\nimport { initLogger } from '@subql/node-core/logger';\n// eslint-disable-next-line import/order\nimport { yargsOptions } from '../../yargs';\n\nconst { argv } = yargsOptions;\n\ninitLogger(\n  argv.debug,\n  argv.outputFmt as 'json' | 'colored',\n  argv.logLevel as string | undefined,\n);\n\nimport assert from 'assert';\nimport { threadId } from 'node:worker_threads';\nimport { getHeapStatistics } from 'v8';\nimport { INestApplication } from '@nestjs/common';\nimport { NestFactory } from '@nestjs/core';\nimport {\n  waitForBatchSize,\n  WorkerHost,\n  getLogger,\n  NestLogger,\n  hostStoreKeys,\n  HostStore,\n  hostDynamicDsKeys,\n  HostDynamicDS,\n} from '@subql/node-core';\nimport { SubqlProjectDs } from '../../configure/SubqueryProject';\nimport { DynamicDsService } from '../dynamic-ds.service';\nimport { IndexerManager } from '../indexer.manager';\nimport { WorkerModule } from './worker.module';\nimport {\n  FetchBlockResponse,\n  ProcessBlockResponse,\n  WorkerService,\n  WorkerStatusResponse,\n} from './worker.service';\nimport {\n  HostUnfinalizedBlocks,\n  hostUnfinalizedBlocksKeys,\n} from './worker.unfinalizedBlocks.service';\nlet app: INestApplication;\nlet workerService: WorkerService;\n\nconst logger = getLogger(`worker #${threadId}`);\n\nasync function initWorker(): Promise<void> {\n  try {\n    if (app) {\n      logger.warn('Worker already initialised');\n      return;\n    }\n\n    app = await NestFactory.create(WorkerModule, {\n      logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging\n    });\n\n    await app.init();\n\n    const indexerManager = app.get(IndexerManager);\n    // Initialise async services, we do this here rather than in factories so we can capture one off events\n    await indexerManager.start();\n\n    workerService = app.get(WorkerService);\n  } catch (e) {\n    console.log('Failed to start worker', e);\n    logger.error(e, 'Failed to start worker');\n    throw e;\n  }\n}\n\nasync function fetchBlock(height: number): Promise<FetchBlockResponse> {\n  assert(workerService, 'Not initialised');\n  return workerService.fetchBlock(height);\n}\n\nasync function processBlock(height: number): Promise<ProcessBlockResponse> {\n  assert(workerService, 'Not initialised');\n\n  return workerService.processBlock(height);\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchedBlocks(): Promise<number> {\n  return workerService.numFetchedBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function numFetchingBlocks(): Promise<number> {\n  return workerService.numFetchingBlocks;\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getStatus(): Promise<WorkerStatusResponse> {\n  return {\n    threadId,\n    fetchedBlocks: workerService.numFetchedBlocks,\n    toFetchBlocks: workerService.numFetchingBlocks,\n    isIndexing: workerService.isIndexing,\n  };\n}\n\n// eslint-disable-next-line @typescript-eslint/require-await\nasync function getMemoryLeft(): Promise<number> {\n  const totalHeap = getHeapStatistics().heap_size_limit;\n  const heapUsed = process.memoryUsage().heapUsed;\n\n  return totalHeap - heapUsed;\n}\n\nasync function waitForWorkerBatchSize(heapSizeInBytes: number): Promise<void> {\n  await waitForBatchSize(heapSizeInBytes);\n}\n\n// Register these functions to be exposed to worker host\n(global as any).host = WorkerHost.create<\n  HostStore & HostDynamicDS<SubqlProjectDs> & HostUnfinalizedBlocks,\n  IInitIndexerWorker\n>(\n  [...hostStoreKeys, ...hostDynamicDsKeys, ...hostUnfinalizedBlocksKeys],\n  {\n    initWorker,\n    fetchBlock,\n    processBlock,\n    numFetchedBlocks,\n    numFetchingBlocks,\n    getStatus,\n    getMemoryLeft,\n    waitForWorkerBatchSize,\n  },\n  logger,\n);\n\n// Export types to be used on the parent\ntype InitWorker = typeof initWorker;\ntype FetchBlock = typeof fetchBlock;\ntype ProcessBlock = typeof processBlock;\ntype NumFetchedBlocks = typeof numFetchedBlocks;\ntype NumFetchingBlocks = typeof numFetchingBlocks;\ntype GetWorkerStatus = typeof getStatus;\ntype GetMemoryLeft = typeof getMemoryLeft;\ntype WaitForWorkerBatchSize = typeof waitForWorkerBatchSize;\n\nexport type IIndexerWorker = {\n  processBlock: ProcessBlock;\n  fetchBlock: FetchBlock;\n  numFetchedBlocks: NumFetchedBlocks;\n  numFetchingBlocks: NumFetchingBlocks;\n  getStatus: GetWorkerStatus;\n  getMemoryLeft: GetMemoryLeft;\n  waitForWorkerBatchSize: WaitForWorkerBatchSize;\n};\n\nexport type IInitIndexerWorker = IIndexerWorker & {\n  initWorker: InitWorker;\n};\n\nprocess.on('uncaughtException', (e) => {\n  logger.error(e, 'Uncaught Exception');\n  throw e;\n});\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subql/node-ethereum",
-  "version": "1.0.0",
+  "version": "1.0.1-0",
   "description": "",
   "author": "Ian He",
   "license": "Apache-2.0",
@@ -26,8 +26,8 @@
     "@nestjs/schedule": "^1.0.2",
     "@subql/common": "^1.8.1",
     "@subql/common-ethereum": "1.0.0",
-    "@subql/node-core": "1.11.2-storeCache-0.0.2",
-    "@subql/types-ethereum": "1.0.0",
+    "@subql/node-core": "1.11.2-storeCache-0.0.4",
+    "@subql/types-ethereum": "1.0.1-0",
     "@subql/utils": "^1.4.1",
     "@subql/x-merkle-mountain-range": "2.0.0-0.1.2",
     "@willsoto/nestjs-prometheus": "^4.4.0",
@@ -74,5 +74,6 @@
   "files": [
     "/dist",
     "/bin"
-  ]
+  ],
+  "stableVersion": "1.0.0"
 }