envio 2.21.1 → 2.21.3-rc

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.21.1",
+  "version": "v2.21.3-rc",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.21.1",
-    "envio-linux-arm64": "v2.21.1",
-    "envio-darwin-x64": "v2.21.1",
-    "envio-darwin-arm64": "v2.21.1"
+    "envio-linux-x64": "v2.21.3-rc",
+    "envio-linux-arm64": "v2.21.3-rc",
+    "envio-darwin-x64": "v2.21.3-rc",
+    "envio-darwin-arm64": "v2.21.3-rc"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/FetchState.res

@@ -940,14 +940,14 @@ let queueItemBlockNumber = (queueItem: queueItem) => {
 let queueItemIsInReorgThreshold = (
   queueItem: queueItem,
   ~currentBlockHeight,
-  ~heighestBlockBelowThreshold,
+  ~highestBlockBelowThreshold,
 ) => {
   if currentBlockHeight === 0 {
     false
   } else {
     switch queueItem {
-    | Item(_) => queueItem->queueItemBlockNumber > heighestBlockBelowThreshold
-    | NoItem(_) => queueItem->queueItemBlockNumber > heighestBlockBelowThreshold
+    | Item(_) => queueItem->queueItemBlockNumber > highestBlockBelowThreshold
+    | NoItem(_) => queueItem->queueItemBlockNumber > highestBlockBelowThreshold
     }
   }
 }

package/src/Prometheus.res

@@ -83,6 +83,7 @@ module MakeSafePromMetric = (
   let handleInt: (t<'a>, ~labels: 'a, ~value: int) => unit
   let handleFloat: (t<'a>, ~labels: 'a, ~value: float) => unit
   let increment: (t<'a>, ~labels: 'a) => unit
+  let incrementMany: (t<'a>, ~labels: 'a, ~value: int) => unit
 } => {
   type t<'a> = {metric: M.t, labelSchema: S.t<'a>}
 
@@ -121,6 +122,13 @@ module MakeSafePromMetric = (
       ->M.labels(labels->S.reverseConvertToJsonOrThrow(labelSchema))
       ->Obj.magic
     )["inc"]()
+
+  let incrementMany = ({metric, labelSchema}: t<'a>, ~labels: 'a, ~value) =>
+    (
+      metric
+      ->M.labels(labels->S.reverseConvertToJsonOrThrow(labelSchema))
+      ->Obj.magic
+    )["inc"](value)
 }
 
 module SafeCounter = MakeSafePromMetric({
@@ -356,6 +364,30 @@ module IndexingPartitions = {
   }
 }
 
+module IndexingIdleTime = {
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_indexing_idle_time",
+    ~help="The number of milliseconds the indexer source syncing has been idle. A high value may indicate the source sync is a bottleneck.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+}
+
+module IndexingSourceWaitingTime = {
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_indexing_source_waiting_time",
+    ~help="The number of milliseconds the indexer has been waiting for new blocks.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+}
+
+module IndexingQueryTime = {
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_indexing_query_time",
+    ~help="The number of milliseconds spent performing queries to the chain data-source.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+}
+
 module IndexingBufferSize = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_indexing_buffer_size",

package/src/ReorgDetection.res

@@ -15,8 +15,8 @@ type blockData = {
 external generalizeBlockDataWithTimestamp: blockDataWithTimestamp => blockData = "%identity"
 
 type reorgGuard = {
-  lastBlockScannedData: blockData,
-  firstBlockParentNumberAndHash: option<blockData>,
+  rangeLastBlock: blockData,
+  prevRangeLastBlock: option<blockData>,
 }
 
 type reorgDetected = {
@@ -36,22 +36,31 @@ let reorgDetectedToLogParams = (reorgDetected: reorgDetected, ~shouldRollbackOnR
   }
 }
 
+type reorgResult = NoReorg | ReorgDetected(reorgDetected)
+type validBlockError = NotFound | AlreadyReorgedHashes
+type validBlockResult = result<blockDataWithTimestamp, validBlockError>
+
 module LastBlockScannedHashes: {
   type t
   /**Instantiat t with existing data*/
-  let makeWithData: (array<blockData>, ~confirmedBlockThreshold: int) => t
+  let makeWithData: (
+    array<blockData>,
+    ~confirmedBlockThreshold: int,
+    ~detectedReorgBlock: blockData=?,
+  ) => t
 
   /**Instantiat empty t with no block data*/
   let empty: (~confirmedBlockThreshold: int) => t
 
-  /** Registers a new reorg guard, prunes unnened data and returns the updated data 
-   or an error if a reorg has occured 
-  */
+  /** Registers a new reorg guard, prunes unneeded data, and returns the updated state.
+   * Resets internal state if shouldRollbackOnReorg is false (detect-only mode)
+   */
   let registerReorgGuard: (
     t,
     ~reorgGuard: reorgGuard,
     ~currentBlockHeight: int,
-  ) => result<t, reorgDetected>
+    ~shouldRollbackOnReorg: bool,
+  ) => (t, reorgResult)
 
   /**
   Returns the latest block data which matches block number and hashes in the provided array
@@ -61,7 +70,7 @@ module LastBlockScannedHashes: {
     t,
     ~blockNumbersAndHashes: array<blockDataWithTimestamp>,
     ~currentBlockHeight: int,
-  ) => option<blockDataWithTimestamp>
+  ) => validBlockResult
 
   let getThresholdBlockNumbers: (t, ~currentBlockHeight: int) => array<int>
 
@@ -76,9 +85,14 @@ module LastBlockScannedHashes: {
     // A hash map of recent blockdata by block number to make comparison checks
     // for reorgs.
     dataByBlockNumber: dict<blockData>,
+    // The latest block which detected a reorg
+    // and should never be valid.
+    // We keep track of this to avoid responses
+    // with the stale data from other data-source instances.
+    detectedReorgBlock: option<blockData>,
   }
 
-  let makeWithData = (blocks, ~confirmedBlockThreshold) => {
+  let makeWithData = (blocks, ~confirmedBlockThreshold, ~detectedReorgBlock=?) => {
     let dataByBlockNumber = Js.Dict.empty()
 
     blocks->Belt.Array.forEach(block => {
@@ -88,12 +102,14 @@ module LastBlockScannedHashes: {
     {
       confirmedBlockThreshold,
       dataByBlockNumber,
+      detectedReorgBlock,
     }
   }
   //Instantiates empty LastBlockHashes
   let empty = (~confirmedBlockThreshold) => {
     confirmedBlockThreshold,
     dataByBlockNumber: Js.Dict.empty(),
+    detectedReorgBlock: None,
   }
 
   let getDataByBlockNumberCopyInThreshold = (
@@ -122,33 +138,33 @@ module LastBlockScannedHashes: {
     {confirmedBlockThreshold} as self: t,
     ~reorgGuard: reorgGuard,
     ~currentBlockHeight,
+    ~shouldRollbackOnReorg,
   ) => {
     let dataByBlockNumberCopyInThreshold =
       self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
 
-    let {lastBlockScannedData, firstBlockParentNumberAndHash} = reorgGuard
+    let {rangeLastBlock, prevRangeLastBlock} = reorgGuard
 
     let maybeReorgDetected = switch dataByBlockNumberCopyInThreshold->Utils.Dict.dangerouslyGetNonOption(
-      lastBlockScannedData.blockNumber->Int.toString,
+      rangeLastBlock.blockNumber->Int.toString,
     ) {
-    | Some(scannedBlock) if scannedBlock.blockHash !== lastBlockScannedData.blockHash =>
+    | Some(scannedBlock) if scannedBlock.blockHash !== rangeLastBlock.blockHash =>
       Some({
-        receivedBlock: lastBlockScannedData,
+        receivedBlock: rangeLastBlock,
         scannedBlock,
       })
     | _ =>
-      switch firstBlockParentNumberAndHash {
+      switch prevRangeLastBlock {
       //If parentHash is None, then it's the genesis block (no reorg)
       //Need to check that parentHash matches because of the dynamic contracts
       | None => None
-      | Some(firstBlockParentNumberAndHash) =>
+      | Some(prevRangeLastBlock) =>
         switch dataByBlockNumberCopyInThreshold->Utils.Dict.dangerouslyGetNonOption(
-          firstBlockParentNumberAndHash.blockNumber->Int.toString,
+          prevRangeLastBlock.blockNumber->Int.toString,
         ) {
-        | Some(scannedBlock)
-          if scannedBlock.blockHash !== firstBlockParentNumberAndHash.blockHash =>
+        | Some(scannedBlock) if scannedBlock.blockHash !== prevRangeLastBlock.blockHash =>
           Some({
-            receivedBlock: firstBlockParentNumberAndHash,
+            receivedBlock: prevRangeLastBlock,
             scannedBlock,
           })
         | _ => None
@@ -157,25 +173,37 @@ module LastBlockScannedHashes: {
     }
 
     switch maybeReorgDetected {
-    | Some(reorgDetected) => Error(reorgDetected)
+    | Some(reorgDetected) => (
+        shouldRollbackOnReorg
+          ? {
+              ...self,
+              detectedReorgBlock: Some(reorgDetected.scannedBlock),
+            }
+          : empty(~confirmedBlockThreshold),
+        ReorgDetected(reorgDetected),
+      )
     | None => {
         dataByBlockNumberCopyInThreshold->Js.Dict.set(
-          lastBlockScannedData.blockNumber->Int.toString,
-          lastBlockScannedData,
+          rangeLastBlock.blockNumber->Int.toString,
+          rangeLastBlock,
         )
-        switch firstBlockParentNumberAndHash {
+        switch prevRangeLastBlock {
         | None => ()
-        | Some(firstBlockParentNumberAndHash) =>
+        | Some(prevRangeLastBlock) =>
           dataByBlockNumberCopyInThreshold->Js.Dict.set(
-            firstBlockParentNumberAndHash.blockNumber->Int.toString,
-            firstBlockParentNumberAndHash,
+            prevRangeLastBlock.blockNumber->Int.toString,
+            prevRangeLastBlock,
           )
         }
 
-        Ok({
-          confirmedBlockThreshold,
-          dataByBlockNumber: dataByBlockNumberCopyInThreshold,
-        })
+        (
+          {
+            confirmedBlockThreshold,
+            dataByBlockNumber: dataByBlockNumberCopyInThreshold,
+            detectedReorgBlock: None,
+          },
+          NoReorg,
+        )
       }
     }
   }
@@ -186,39 +214,58 @@ module LastBlockScannedHashes: {
     ~currentBlockHeight,
   ) => {
     let verifiedDataByBlockNumber = Js.Dict.empty()
-    blockNumbersAndHashes->Array.forEach(blockData => {
+    for idx in 0 to blockNumbersAndHashes->Array.length - 1 {
+      let blockData = blockNumbersAndHashes->Array.getUnsafe(idx)
       verifiedDataByBlockNumber->Js.Dict.set(blockData.blockNumber->Int.toString, blockData)
-    })
+    }
 
-    let dataByBlockNumber = self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
-    // Js engine automatically orders numeric object keys
-    let ascBlockNumberKeys = dataByBlockNumber->Js.Dict.keys
+    let isAlreadyReorgedResponse = switch self.detectedReorgBlock {
+    | Some(detectedReorgBlock) =>
+      switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(
+        detectedReorgBlock.blockNumber->Int.toString,
+      ) {
+      | Some(verifiedBlockData) => verifiedBlockData.blockHash === detectedReorgBlock.blockHash
+      | None => false
+      }
+    | None => false
+    }
 
-    let getPrevScannedBlock = idx =>
-      ascBlockNumberKeys
-      ->Belt.Array.get(idx - 1)
-      ->Option.flatMap(key => {
-        // We should already validate that the block number is verified at the point
-        verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(key)
-      })
+    if isAlreadyReorgedResponse {
+      Error(AlreadyReorgedHashes)
+    } else {
+      let dataByBlockNumber = self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
+      // Js engine automatically orders numeric object keys
+      let ascBlockNumberKeys = dataByBlockNumber->Js.Dict.keys
+
+      let getPrevScannedBlock = idx =>
+        switch ascBlockNumberKeys
+        ->Belt.Array.get(idx - 1)
+        ->Option.flatMap(key => {
+          // We should already validate that the block number is verified at the point
+          verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(key)
+        }) {
+        | Some(data) => Ok(data)
+        | None => Error(NotFound)
+        }
 
-    let rec loop = idx => {
-      switch ascBlockNumberKeys->Belt.Array.get(idx) {
-      | Some(blockNumberKey) =>
-        let scannedBlock = dataByBlockNumber->Js.Dict.unsafeGet(blockNumberKey)
-        switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(blockNumberKey) {
-        | None =>
-          Js.Exn.raiseError(
-            `Unexpected case. Couldn't find verified hash for block number ${blockNumberKey}`,
-          )
-        | Some(verifiedBlockData) if verifiedBlockData.blockHash === scannedBlock.blockHash =>
-          loop(idx + 1)
-        | Some(_) => getPrevScannedBlock(idx)
+      let rec loop = idx => {
+        switch ascBlockNumberKeys->Belt.Array.get(idx) {
+        | Some(blockNumberKey) =>
+          let scannedBlock = dataByBlockNumber->Js.Dict.unsafeGet(blockNumberKey)
+          switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(blockNumberKey) {
+          | None =>
+            Js.Exn.raiseError(
+              `Unexpected case. Couldn't find verified hash for block number ${blockNumberKey}`,
+            )
+          | Some(verifiedBlockData) if verifiedBlockData.blockHash === scannedBlock.blockHash =>
+            loop(idx + 1)
+          | Some(_) => getPrevScannedBlock(idx)
+          }
+        | None => getPrevScannedBlock(idx)
         }
-      | None => getPrevScannedBlock(idx)
       }
+      loop(0)
     }
-    loop(0)
   }
 
   /**
@@ -254,6 +301,7 @@ module LastBlockScannedHashes: {
     {
       confirmedBlockThreshold,
       dataByBlockNumber: newDataByBlockNumber,
+      detectedReorgBlock: None,
     }
   }
 

package/src/sources/SourceManager.res

@@ -1,5 +1,7 @@
 open Belt
 
+type sourceManagerStatus = Idle | WaitingForNewBlock | Querieng
+
 // Ideally the ChainFetcher name suits this better
 // But currently the ChainFetcher module is immutable
 // and handles both processing and fetching.
@@ -7,6 +9,8 @@ open Belt
 // with a mutable state for easier reasoning and testing.
 type t = {
   sources: Utils.Set.t<Source.t>,
+  mutable statusStart: Hrtime.timeRef,
+  mutable status: sourceManagerStatus,
   maxPartitionConcurrency: int,
   newBlockFallbackStallTimeout: int,
   stalledPollingInterval: int,
@@ -66,7 +70,23 @@ let make = (
     newBlockFallbackStallTimeout,
     stalledPollingInterval,
     getHeightRetryInterval,
+    statusStart: Hrtime.makeTimer(),
+    status: Idle,
+  }
+}
+
+let trackNewStatus = (sourceManager: t, ~newStatus) => {
+  let promCounter = switch newStatus {
+  | Idle => Prometheus.IndexingIdleTime.counter
+  | WaitingForNewBlock => Prometheus.IndexingSourceWaitingTime.counter
+  | Querieng => Prometheus.IndexingQueryTime.counter
   }
+  promCounter->Prometheus.SafeCounter.incrementMany(
+    ~labels=sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
+    ~value=sourceManager.statusStart->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis,
+  )
+  sourceManager.statusStart = Hrtime.makeTimer()
+  sourceManager.status = newStatus
 }
 
 let fetchNext = async (
@@ -96,10 +116,12 @@ let fetchNext = async (
     | Some(waitingStateId) if waitingStateId >= stateId => ()
     | Some(_) // Case for the prev state before a rollback
     | None =>
+      sourceManager->trackNewStatus(~newStatus=WaitingForNewBlock)
       sourceManager.waitingForNewBlockStateId = Some(stateId)
       let currentBlockHeight = await waitForNewBlock(~currentBlockHeight)
       switch sourceManager.waitingForNewBlockStateId {
       | Some(waitingStateId) if waitingStateId === stateId => {
+          sourceManager->trackNewStatus(~newStatus=Idle)
           sourceManager.waitingForNewBlockStateId = None
           onNewBlock(~currentBlockHeight)
         }
@@ -115,6 +137,7 @@ let fetchNext = async (
         ~concurrency=sourceManager.fetchingPartitionsCount,
         ~chainId=sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
       )
+      sourceManager->trackNewStatus(~newStatus=Querieng)
       let _ =
         await queries
         ->Array.map(q => {
@@ -125,6 +148,9 @@ let fetchNext = async (
               ~concurrency=sourceManager.fetchingPartitionsCount,
               ~chainId=sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
             )
+            if sourceManager.fetchingPartitionsCount === 0 {
+              sourceManager->trackNewStatus(~newStatus=Idle)
+            }
           })
           promise
         })