envio 2.31.0 → 2.31.1-rc.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.31.0",
+  "version": "v2.31.1-rc.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.31.0",
-    "envio-linux-arm64": "v2.31.0",
-    "envio-darwin-x64": "v2.31.0",
-    "envio-darwin-arm64": "v2.31.0"
+    "envio-linux-x64": "v2.31.1-rc.0",
+    "envio-linux-arm64": "v2.31.1-rc.0",
+    "envio-darwin-x64": "v2.31.1-rc.0",
+    "envio-darwin-arm64": "v2.31.1-rc.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.6",

package/src/ReorgDetection.res

@@ -37,8 +37,6 @@ let reorgDetectedToLogParams = (reorgDetected: reorgDetected, ~shouldRollbackOnR
 }
 
 type reorgResult = NoReorg | ReorgDetected(reorgDetected)
-type validBlockError = NotFound | AlreadyReorgedHashes
-type validBlockResult = result<blockDataWithTimestamp, validBlockError>
 
 type t = {
   // Whether to rollback on reorg
@@ -186,14 +184,13 @@ let registerReorgGuard = (
 }
 
 /**
-Returns the latest block data which matches block number and hashes in the provided array
-If it doesn't exist in the reorg threshold it returns None or the latest scanned block outside of the reorg threshold
+Returns the latest block number which matches block number and hashes in the provided array
+If it doesn't exist in the reorg threshold it returns NotFound
 */
 let getLatestValidScannedBlock = (
   self: t,
   ~blockNumbersAndHashes: array<blockDataWithTimestamp>,
   ~currentBlockHeight,
-  ~skipReorgDuplicationCheck=false,
 ) => {
   let verifiedDataByBlockNumber = Js.Dict.empty()
   for idx in 0 to blockNumbersAndHashes->Array.length - 1 {
@@ -201,68 +198,38 @@ let getLatestValidScannedBlock = (
     verifiedDataByBlockNumber->Js.Dict.set(blockData.blockNumber->Int.toString, blockData)
   }
 
-  /*
-     Let's say we indexed block X with hash A.
-     The next query we got the block X with hash B.
-     We assume that the hash A is reorged since we received it earlier than B.
-     So when we try to detect the reorg depth, we consider hash A as already invalid,
-     and retry the block hashes query if we receive one. (since it could come from a different instance and cause a double reorg)
-     But the assumption that A is reorged might be wrong sometimes,
-     for example if we got B from instance which didn't handle a reorg A.
-     Theoretically, it's possible with high partition concurrency.
-     So to handle this and prevent entering an infinite loop,
-     we can skip the reorg duplication check if we're sure that the block hashes query
-     is not coming from a different instance. (let's say we tried several times)
- */
-  let isAlreadyReorgedResponse = skipReorgDuplicationCheck
-    ? false
-    : switch self.detectedReorgBlock {
-      | Some(detectedReorgBlock) =>
-        switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(
-          detectedReorgBlock.blockNumber->Int.toString,
-        ) {
-        | Some(verifiedBlockData) => verifiedBlockData.blockHash === detectedReorgBlock.blockHash
-        | None => false
-        }
-      | None => false
-      }
+  let dataByBlockNumber = self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
+  // Js engine automatically orders numeric object keys
+  let ascBlockNumberKeys = dataByBlockNumber->Js.Dict.keys
 
-  if isAlreadyReorgedResponse {
-    Error(AlreadyReorgedHashes)
-  } else {
-    let dataByBlockNumber = self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
-    // Js engine automatically orders numeric object keys
-    let ascBlockNumberKeys = dataByBlockNumber->Js.Dict.keys
-
-    let getPrevScannedBlock = idx =>
-      switch ascBlockNumberKeys
-      ->Belt.Array.get(idx - 1)
-      ->Option.flatMap(key => {
-        // We should already validate that the block number is verified at the point
-        verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(key)
-      }) {
-      | Some(data) => Ok(data)
-      | None => Error(NotFound)
+  let getPrevScannedBlockNumber = idx =>
+    ascBlockNumberKeys
+    ->Belt.Array.get(idx - 1)
+    ->Option.flatMap(key => {
+      // We should already validate that the block number is verified at the point
+      switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(key) {
+      | Some(v) => Some(v.blockNumber)
+      | None => None
       }
+    })
 
-    let rec loop = idx => {
-      switch ascBlockNumberKeys->Belt.Array.get(idx) {
-      | Some(blockNumberKey) =>
-        let scannedBlock = dataByBlockNumber->Js.Dict.unsafeGet(blockNumberKey)
-        switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(blockNumberKey) {
-        | None =>
-          Js.Exn.raiseError(
-            `Unexpected case. Couldn't find verified hash for block number ${blockNumberKey}`,
-          )
-        | Some(verifiedBlockData) if verifiedBlockData.blockHash === scannedBlock.blockHash =>
-          loop(idx + 1)
-        | Some(_) => getPrevScannedBlock(idx)
-        }
-      | None => getPrevScannedBlock(idx)
+  let rec loop = idx => {
+    switch ascBlockNumberKeys->Belt.Array.get(idx) {
+    | Some(blockNumberKey) =>
+      let scannedBlock = dataByBlockNumber->Js.Dict.unsafeGet(blockNumberKey)
+      switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(blockNumberKey) {
+      | None =>
+        Js.Exn.raiseError(
+          `Unexpected case. Couldn't find verified hash for block number ${blockNumberKey}`,
+        )
+      | Some(verifiedBlockData) if verifiedBlockData.blockHash === scannedBlock.blockHash =>
+        loop(idx + 1)
+      | Some(_) => getPrevScannedBlockNumber(idx)
       }
+    | None => getPrevScannedBlockNumber(idx)
     }
-    loop(0)
   }
+  loop(0)
 }
 
 /**
@@ -303,11 +270,22 @@ let rollbackToValidBlockNumber = (
   }
 }
 
-let getThresholdBlockNumbers = (self: t, ~currentBlockHeight) => {
-  let dataByBlockNumberCopyInThreshold =
-    self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
+let getThresholdBlockNumbersBelowBlock = (self: t, ~blockNumber: int, ~currentBlockHeight) => {
+  let arr = []
+
+  // Js engine automatically orders numeric object keys
+  let ascBlockNumberKeys = self.dataByBlockNumber->Js.Dict.keys
+  let thresholdBlockNumber = currentBlockHeight - self.maxReorgDepth
 
-  dataByBlockNumberCopyInThreshold->Js.Dict.values->Js.Array2.map(v => v.blockNumber)
+  for idx in 0 to ascBlockNumberKeys->Array.length - 1 {
+    let blockNumberKey = ascBlockNumberKeys->Js.Array2.unsafe_get(idx)
+    let scannedBlock = self.dataByBlockNumber->Js.Dict.unsafeGet(blockNumberKey)
+    let isInReorgThreshold = scannedBlock.blockNumber >= thresholdBlockNumber
+    if isInReorgThreshold && scannedBlock.blockNumber < blockNumber {
+      arr->Array.push(scannedBlock.blockNumber)
+    }
+  }
+  arr
 }
 
 let getHashByBlockNumber = (reorgDetection: t, ~blockNumber) => {

package/src/ReorgDetection.res.js

@@ -2,7 +2,6 @@
 'use strict';
 
 var Js_exn = require("rescript/lib/js/js_exn.js");
-var Js_dict = require("rescript/lib/js/js_dict.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var Belt_Option = require("rescript/lib/js/belt_Option.js");
 
@@ -109,63 +108,37 @@ function registerReorgGuard(self, reorgGuard, currentBlockHeight) {
   }
 }
 
-function getLatestValidScannedBlock(self, blockNumbersAndHashes, currentBlockHeight, skipReorgDuplicationCheckOpt) {
-  var skipReorgDuplicationCheck = skipReorgDuplicationCheckOpt !== undefined ? skipReorgDuplicationCheckOpt : false;
+function getLatestValidScannedBlock(self, blockNumbersAndHashes, currentBlockHeight) {
   var verifiedDataByBlockNumber = {};
   for(var idx = 0 ,idx_finish = blockNumbersAndHashes.length; idx < idx_finish; ++idx){
     var blockData = blockNumbersAndHashes[idx];
     verifiedDataByBlockNumber[String(blockData.blockNumber)] = blockData;
   }
-  var isAlreadyReorgedResponse;
-  if (skipReorgDuplicationCheck) {
-    isAlreadyReorgedResponse = false;
-  } else {
-    var detectedReorgBlock = self.detectedReorgBlock;
-    if (detectedReorgBlock !== undefined) {
-      var verifiedBlockData = verifiedDataByBlockNumber[String(detectedReorgBlock.blockNumber)];
-      isAlreadyReorgedResponse = verifiedBlockData !== undefined ? verifiedBlockData.blockHash === detectedReorgBlock.blockHash : false;
-    } else {
-      isAlreadyReorgedResponse = false;
-    }
-  }
-  if (isAlreadyReorgedResponse) {
-    return {
-            TAG: "Error",
-            _0: "AlreadyReorgedHashes"
-          };
-  }
   var dataByBlockNumber = getDataByBlockNumberCopyInThreshold(self, currentBlockHeight);
   var ascBlockNumberKeys = Object.keys(dataByBlockNumber);
-  var getPrevScannedBlock = function (idx) {
-    var data = Belt_Option.flatMap(Belt_Array.get(ascBlockNumberKeys, idx - 1 | 0), (function (key) {
-            return verifiedDataByBlockNumber[key];
-          }));
-    if (data !== undefined) {
-      return {
-              TAG: "Ok",
-              _0: data
-            };
-    } else {
-      return {
-              TAG: "Error",
-              _0: "NotFound"
-            };
-    }
+  var getPrevScannedBlockNumber = function (idx) {
+    return Belt_Option.flatMap(Belt_Array.get(ascBlockNumberKeys, idx - 1 | 0), (function (key) {
+                  var v = verifiedDataByBlockNumber[key];
+                  if (v !== undefined) {
+                    return v.blockNumber;
+                  }
+                  
+                }));
   };
   var _idx = 0;
   while(true) {
     var idx$1 = _idx;
     var blockNumberKey = Belt_Array.get(ascBlockNumberKeys, idx$1);
     if (blockNumberKey === undefined) {
-      return getPrevScannedBlock(idx$1);
+      return getPrevScannedBlockNumber(idx$1);
     }
     var scannedBlock = dataByBlockNumber[blockNumberKey];
-    var verifiedBlockData$1 = verifiedDataByBlockNumber[blockNumberKey];
-    if (verifiedBlockData$1 === undefined) {
+    var verifiedBlockData = verifiedDataByBlockNumber[blockNumberKey];
+    if (verifiedBlockData === undefined) {
       return Js_exn.raiseError("Unexpected case. Couldn't find verified hash for block number " + blockNumberKey);
     }
-    if (verifiedBlockData$1.blockHash !== scannedBlock.blockHash) {
-      return getPrevScannedBlock(idx$1);
+    if (verifiedBlockData.blockHash !== scannedBlock.blockHash) {
+      return getPrevScannedBlockNumber(idx$1);
     }
     _idx = idx$1 + 1 | 0;
     continue ;
@@ -202,11 +175,20 @@ function rollbackToValidBlockNumber(param, blockNumber) {
         };
 }
 
-function getThresholdBlockNumbers(self, currentBlockHeight) {
-  var dataByBlockNumberCopyInThreshold = getDataByBlockNumberCopyInThreshold(self, currentBlockHeight);
-  return Js_dict.values(dataByBlockNumberCopyInThreshold).map(function (v) {
-              return v.blockNumber;
-            });
+function getThresholdBlockNumbersBelowBlock(self, blockNumber, currentBlockHeight) {
+  var arr = [];
+  var ascBlockNumberKeys = Object.keys(self.dataByBlockNumber);
+  var thresholdBlockNumber = currentBlockHeight - self.maxReorgDepth | 0;
+  for(var idx = 0 ,idx_finish = ascBlockNumberKeys.length; idx < idx_finish; ++idx){
+    var blockNumberKey = ascBlockNumberKeys[idx];
+    var scannedBlock = self.dataByBlockNumber[blockNumberKey];
+    var isInReorgThreshold = scannedBlock.blockNumber >= thresholdBlockNumber;
+    if (isInReorgThreshold && scannedBlock.blockNumber < blockNumber) {
+      arr.push(scannedBlock.blockNumber);
+    }
+    
+  }
+  return arr;
 }
 
 function getHashByBlockNumber(reorgDetection, blockNumber) {
@@ -224,6 +206,6 @@ exports.getDataByBlockNumberCopyInThreshold = getDataByBlockNumberCopyInThreshol
 exports.registerReorgGuard = registerReorgGuard;
 exports.getLatestValidScannedBlock = getLatestValidScannedBlock;
 exports.rollbackToValidBlockNumber = rollbackToValidBlockNumber;
-exports.getThresholdBlockNumbers = getThresholdBlockNumbers;
+exports.getThresholdBlockNumbersBelowBlock = getThresholdBlockNumbersBelowBlock;
 exports.getHashByBlockNumber = getHashByBlockNumber;
 /* No side effect */

package/src/sources/RpcSource.res

@@ -522,35 +522,38 @@ let make = (
     },
   )
 
-  let blockLoader = LazyLoader.make(
-    ~loaderFn=blockNumber =>
-      getKnownBlockWithBackoff(
-        ~provider,
-        ~sourceName=name,
-        ~chain,
-        ~backoffMsOnFailure=1000,
-        ~blockNumber,
-        ~lowercaseAddresses,
-      ),
-    ~onError=(am, ~exn) => {
-      Logging.error({
-        "err": exn->Utils.prettifyExn,
-        "msg": `EE1100: Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ${(am._retryDelayMillis / 1000)
-            ->Belt.Int.toString} seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the "suggestedFix" in the metadata of this command`,
-        "source": name,
-        "chainId": chain->ChainMap.Chain.toChainId,
-        "metadata": {
-          {
-            "asyncTaskName": "blockLoader: fetching block data - `getBlock` rpc call",
-            "suggestedFix": "This likely means the RPC url you are using is not responding correctly. Please try another RPC endipoint.",
-          }
-        },
-      })
-    },
-  )
+  let makeBlockLoader = () =>
+    LazyLoader.make(
+      ~loaderFn=blockNumber =>
+        getKnownBlockWithBackoff(
+          ~provider,
+          ~sourceName=name,
+          ~chain,
+          ~backoffMsOnFailure=1000,
+          ~blockNumber,
+          ~lowercaseAddresses,
+        ),
+      ~onError=(am, ~exn) => {
+        Logging.error({
+          "err": exn->Utils.prettifyExn,
+          "msg": `EE1100: Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ${(am._retryDelayMillis / 1000)
+              ->Belt.Int.toString} seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the "suggestedFix" in the metadata of this command`,
+          "source": name,
+          "chainId": chain->ChainMap.Chain.toChainId,
+          "metadata": {
+            {
+              "asyncTaskName": "blockLoader: fetching block data - `getBlock` rpc call",
+              "suggestedFix": "This likely means the RPC url you are using is not responding correctly. Please try another RPC endipoint.",
+            }
+          },
+        })
+      },
+    )
+
+  let blockLoader = ref(makeBlockLoader())
 
   let getEventBlockOrThrow = makeThrowingGetEventBlock(~getBlock=blockNumber =>
-    blockLoader->LazyLoader.get(blockNumber)
+    blockLoader.contents->LazyLoader.get(blockNumber)
   )
   let getEventTransactionOrThrow = makeThrowingGetEventTransaction(
     ~getTransactionFields=Ethers.JsonRpcProvider.makeGetTransactionFields(
@@ -625,7 +628,7 @@ let make = (
 
     let firstBlockParentPromise =
       fromBlock > 0
-        ? blockLoader->LazyLoader.get(fromBlock - 1)->Promise.thenResolve(res => res->Some)
+        ? blockLoader.contents->LazyLoader.get(fromBlock - 1)->Promise.thenResolve(res => res->Some)
         : Promise.resolve(None)
 
     let {getLogSelectionOrThrow} = getSelectionConfig(selection)
@@ -636,7 +639,7 @@ let make = (
       ~toBlock=suggestedToBlock,
       ~addresses,
       ~topicQuery,
-      ~loadBlock=blockNumber => blockLoader->LazyLoader.get(blockNumber),
+      ~loadBlock=blockNumber => blockLoader.contents->LazyLoader.get(blockNumber),
       ~syncConfig,
       ~provider,
       ~mutSuggestedBlockIntervals,
@@ -873,8 +876,13 @@ let make = (
   }
 
   let getBlockHashes = (~blockNumbers, ~logger as _currentlyUnusedLogger) => {
+    // Clear cache by creating a fresh LazyLoader
+    // This is important, since we call this
+    // function when a reorg is detected
+    blockLoader := makeBlockLoader()
+
     blockNumbers
-    ->Array.map(blockNum => blockLoader->LazyLoader.get(blockNum))
+    ->Array.map(blockNum => blockLoader.contents->LazyLoader.get(blockNum))
     ->Promise.all
     ->Promise.thenResolve(blocks => {
       blocks

package/src/sources/RpcSource.res.js

@@ -533,22 +533,27 @@ function make(param) {
                 }
               });
         }), undefined, undefined, undefined, undefined);
-  var blockLoader = LazyLoader.make((function (blockNumber) {
-          return getKnownBlockWithBackoff(provider, name, chain, blockNumber, 1000, lowercaseAddresses);
-        }), (function (am, exn) {
-          Logging.error({
-                err: Utils.prettifyExn(exn),
-                msg: "EE1100: Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in " + String(am._retryDelayMillis / 1000 | 0) + " seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the \"suggestedFix\" in the metadata of this command",
-                source: name,
-                chainId: chain,
-                metadata: {
-                  asyncTaskName: "blockLoader: fetching block data - `getBlock` rpc call",
-                  suggestedFix: "This likely means the RPC url you are using is not responding correctly. Please try another RPC endipoint."
-                }
-              });
-        }), undefined, undefined, undefined, undefined);
+  var makeBlockLoader = function () {
+    return LazyLoader.make((function (blockNumber) {
+                  return getKnownBlockWithBackoff(provider, name, chain, blockNumber, 1000, lowercaseAddresses);
+                }), (function (am, exn) {
+                  Logging.error({
+                        err: Utils.prettifyExn(exn),
+                        msg: "EE1100: Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in " + String(am._retryDelayMillis / 1000 | 0) + " seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the \"suggestedFix\" in the metadata of this command",
+                        source: name,
+                        chainId: chain,
+                        metadata: {
+                          asyncTaskName: "blockLoader: fetching block data - `getBlock` rpc call",
+                          suggestedFix: "This likely means the RPC url you are using is not responding correctly. Please try another RPC endipoint."
+                        }
+                      });
+                }), undefined, undefined, undefined, undefined);
+  };
+  var blockLoader = {
+    contents: makeBlockLoader()
+  };
   var getEventBlockOrThrow = makeThrowingGetEventBlock(function (blockNumber) {
-        return LazyLoader.get(blockLoader, blockNumber);
+        return LazyLoader.get(blockLoader.contents, blockNumber);
       });
   var getEventTransactionOrThrow = makeThrowingGetEventTransaction(Ethers.JsonRpcProvider.makeGetTransactionFields((function (__x) {
               return LazyLoader.get(transactionLoader, __x);
@@ -601,13 +606,13 @@ function make(param) {
     var suggestedBlockInterval = maxSuggestedBlockInterval !== undefined ? maxSuggestedBlockInterval : Belt_Option.getWithDefault(mutSuggestedBlockIntervals[partitionId], syncConfig.initialBlockInterval);
     var toBlock$1 = toBlock !== undefined && toBlock < currentBlockHeight ? toBlock : currentBlockHeight;
     var suggestedToBlock = Caml.int_max(Caml.int_min((fromBlock + suggestedBlockInterval | 0) - 1 | 0, toBlock$1), fromBlock);
-    var firstBlockParentPromise = fromBlock > 0 ? LazyLoader.get(blockLoader, fromBlock - 1 | 0).then(function (res) {
+    var firstBlockParentPromise = fromBlock > 0 ? LazyLoader.get(blockLoader.contents, fromBlock - 1 | 0).then(function (res) {
             return res;
           }) : Promise.resolve(undefined);
     var match = getSelectionConfig(selection);
     var match$1 = match.getLogSelectionOrThrow(addressesByContractName);
     var match$2 = await getNextPage(fromBlock, suggestedToBlock, match$1.addresses, match$1.topicQuery, (function (blockNumber) {
-            return LazyLoader.get(blockLoader, blockNumber);
+            return LazyLoader.get(blockLoader.contents, blockNumber);
           }), syncConfig, provider, mutSuggestedBlockIntervals, partitionId);
     var latestFetchedBlock = match$2.latestFetchedBlock;
     var logs = match$2.logs;
@@ -784,8 +789,9 @@ function make(param) {
           };
   };
   var getBlockHashes = function (blockNumbers, _currentlyUnusedLogger) {
+    blockLoader.contents = makeBlockLoader();
     return $$Promise.$$catch(Promise.all(Belt_Array.map(blockNumbers, (function (blockNum) {
-                            return LazyLoader.get(blockLoader, blockNum);
+                            return LazyLoader.get(blockLoader.contents, blockNum);
                           }))).then(function (blocks) {
                     return {
                             TAG: "Ok",