envio 2.21.0 → 2.21.2

package/src/Prometheus.res

@@ -83,6 +83,7 @@ module MakeSafePromMetric = (
   let handleInt: (t<'a>, ~labels: 'a, ~value: int) => unit
   let handleFloat: (t<'a>, ~labels: 'a, ~value: float) => unit
   let increment: (t<'a>, ~labels: 'a) => unit
+  let incrementMany: (t<'a>, ~labels: 'a, ~value: int) => unit
 } => {
   type t<'a> = {metric: M.t, labelSchema: S.t<'a>}
 
@@ -121,6 +122,13 @@ module MakeSafePromMetric = (
       ->M.labels(labels->S.reverseConvertToJsonOrThrow(labelSchema))
       ->Obj.magic
     )["inc"]()
+
+  let incrementMany = ({metric, labelSchema}: t<'a>, ~labels: 'a, ~value) =>
+    (
+      metric
+      ->M.labels(labels->S.reverseConvertToJsonOrThrow(labelSchema))
+      ->Obj.magic
+    )["inc"](value)
 }
 
 module SafeCounter = MakeSafePromMetric({
@@ -335,7 +343,7 @@ module IndexingMaxConcurrency = {
 module IndexingConcurrency = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_indexing_concurrency",
-    ~help="The current number of concurrent queries to the chain data-source.",
+    ~help="The number of executing concurrent queries to the chain data-source.",
     ~labelSchema=chainIdLabelsSchema,
   )
 
@@ -344,6 +352,42 @@ module IndexingConcurrency = {
   }
 }
 
+module IndexingPartitions = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_indexing_partitions",
+    ~help="The number of partitions used to split fetching logic by addresses and block ranges.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~partitionsCount, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=partitionsCount)
+  }
+}
+
+module IndexingIdleTime = {
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_indexing_idle_time",
+    ~help="The number of milliseconds the indexer source syncing has been idle. A high value may indicate the source sync is a bottleneck.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+}
+
+module IndexingSourceWaitingTime = {
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_indexing_source_waiting_time",
+    ~help="The number of milliseconds the indexer has been waiting for new blocks.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+}
+
+module IndexingQueryTime = {
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_indexing_query_time",
+    ~help="The number of milliseconds spent performing queries to the chain data-source.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+}
+
 module IndexingBufferSize = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_indexing_buffer_size",

package/src/ReorgDetection.res

@@ -15,8 +15,8 @@ type blockData = {
 external generalizeBlockDataWithTimestamp: blockDataWithTimestamp => blockData = "%identity"
 
 type reorgGuard = {
-  lastBlockScannedData: blockData,
-  firstBlockParentNumberAndHash: option<blockData>,
+  rangeLastBlock: blockData,
+  prevRangeLastBlock: option<blockData>,
 }
 
 type reorgDetected = {
@@ -36,22 +36,31 @@ let reorgDetectedToLogParams = (reorgDetected: reorgDetected, ~shouldRollbackOnR
   }
 }
 
+type reorgResult = NoReorg | ReorgDetected(reorgDetected)
+type validBlockError = NotFound | AlreadyReorgedHashes
+type validBlockResult = result<blockDataWithTimestamp, validBlockError>
+
 module LastBlockScannedHashes: {
   type t
   /**Instantiat t with existing data*/
-  let makeWithData: (array<blockData>, ~confirmedBlockThreshold: int) => t
+  let makeWithData: (
+    array<blockData>,
+    ~confirmedBlockThreshold: int,
+    ~detectedReorgBlock: blockData=?,
+  ) => t
 
   /**Instantiat empty t with no block data*/
   let empty: (~confirmedBlockThreshold: int) => t
 
-  /** Registers a new reorg guard, prunes unnened data and returns the updated data 
-   or an error if a reorg has occured 
-  */
+  /** Registers a new reorg guard, prunes unneeded data, and returns the updated state.
+   * Resets internal state if shouldRollbackOnReorg is false (detect-only mode)
+   */
   let registerReorgGuard: (
     t,
     ~reorgGuard: reorgGuard,
     ~currentBlockHeight: int,
-  ) => result<t, reorgDetected>
+    ~shouldRollbackOnReorg: bool,
+  ) => (t, reorgResult)
 
   /**
   Returns the latest block data which matches block number and hashes in the provided array
@@ -61,7 +70,7 @@ module LastBlockScannedHashes: {
     t,
     ~blockNumbersAndHashes: array<blockDataWithTimestamp>,
     ~currentBlockHeight: int,
-  ) => option<blockDataWithTimestamp>
+  ) => validBlockResult
 
   let getThresholdBlockNumbers: (t, ~currentBlockHeight: int) => array<int>
 
@@ -76,9 +85,14 @@ module LastBlockScannedHashes: {
     // A hash map of recent blockdata by block number to make comparison checks
     // for reorgs.
     dataByBlockNumber: dict<blockData>,
+    // The latest block which detected a reorg
+    // and should never be valid.
+    // We keep track of this to avoid responses
+    // with the stale data from other data-source instances.
+    detectedReorgBlock: option<blockData>,
   }
 
-  let makeWithData = (blocks, ~confirmedBlockThreshold) => {
+  let makeWithData = (blocks, ~confirmedBlockThreshold, ~detectedReorgBlock=?) => {
     let dataByBlockNumber = Js.Dict.empty()
 
     blocks->Belt.Array.forEach(block => {
@@ -88,12 +102,14 @@ module LastBlockScannedHashes: {
     {
       confirmedBlockThreshold,
       dataByBlockNumber,
+      detectedReorgBlock,
     }
   }
   //Instantiates empty LastBlockHashes
   let empty = (~confirmedBlockThreshold) => {
     confirmedBlockThreshold,
     dataByBlockNumber: Js.Dict.empty(),
+    detectedReorgBlock: None,
   }
 
   let getDataByBlockNumberCopyInThreshold = (
@@ -122,33 +138,33 @@ module LastBlockScannedHashes: {
     {confirmedBlockThreshold} as self: t,
     ~reorgGuard: reorgGuard,
     ~currentBlockHeight,
+    ~shouldRollbackOnReorg,
   ) => {
     let dataByBlockNumberCopyInThreshold =
       self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
 
-    let {lastBlockScannedData, firstBlockParentNumberAndHash} = reorgGuard
+    let {rangeLastBlock, prevRangeLastBlock} = reorgGuard
 
     let maybeReorgDetected = switch dataByBlockNumberCopyInThreshold->Utils.Dict.dangerouslyGetNonOption(
-      lastBlockScannedData.blockNumber->Int.toString,
+      rangeLastBlock.blockNumber->Int.toString,
     ) {
-    | Some(scannedBlock) if scannedBlock.blockHash !== lastBlockScannedData.blockHash =>
+    | Some(scannedBlock) if scannedBlock.blockHash !== rangeLastBlock.blockHash =>
       Some({
-        receivedBlock: lastBlockScannedData,
+        receivedBlock: rangeLastBlock,
         scannedBlock,
       })
     | _ =>
-      switch firstBlockParentNumberAndHash {
+      switch prevRangeLastBlock {
       //If parentHash is None, then it's the genesis block (no reorg)
       //Need to check that parentHash matches because of the dynamic contracts
       | None => None
-      | Some(firstBlockParentNumberAndHash) =>
+      | Some(prevRangeLastBlock) =>
         switch dataByBlockNumberCopyInThreshold->Utils.Dict.dangerouslyGetNonOption(
-          firstBlockParentNumberAndHash.blockNumber->Int.toString,
+          prevRangeLastBlock.blockNumber->Int.toString,
         ) {
-        | Some(scannedBlock)
-          if scannedBlock.blockHash !== firstBlockParentNumberAndHash.blockHash =>
+        | Some(scannedBlock) if scannedBlock.blockHash !== prevRangeLastBlock.blockHash =>
           Some({
-            receivedBlock: firstBlockParentNumberAndHash,
+            receivedBlock: prevRangeLastBlock,
             scannedBlock,
           })
         | _ => None
@@ -157,25 +173,37 @@ module LastBlockScannedHashes: {
     }
 
     switch maybeReorgDetected {
-    | Some(reorgDetected) => Error(reorgDetected)
+    | Some(reorgDetected) => (
+        shouldRollbackOnReorg
+          ? {
+              ...self,
+              detectedReorgBlock: Some(reorgDetected.scannedBlock),
+            }
+          : empty(~confirmedBlockThreshold),
+        ReorgDetected(reorgDetected),
+      )
     | None => {
         dataByBlockNumberCopyInThreshold->Js.Dict.set(
-          lastBlockScannedData.blockNumber->Int.toString,
-          lastBlockScannedData,
+          rangeLastBlock.blockNumber->Int.toString,
+          rangeLastBlock,
         )
-        switch firstBlockParentNumberAndHash {
+        switch prevRangeLastBlock {
         | None => ()
-        | Some(firstBlockParentNumberAndHash) =>
+        | Some(prevRangeLastBlock) =>
           dataByBlockNumberCopyInThreshold->Js.Dict.set(
-            firstBlockParentNumberAndHash.blockNumber->Int.toString,
-            firstBlockParentNumberAndHash,
+            prevRangeLastBlock.blockNumber->Int.toString,
+            prevRangeLastBlock,
           )
         }
 
-        Ok({
-          confirmedBlockThreshold,
-          dataByBlockNumber: dataByBlockNumberCopyInThreshold,
-        })
+        (
+          {
+            confirmedBlockThreshold,
+            dataByBlockNumber: dataByBlockNumberCopyInThreshold,
+            detectedReorgBlock: None,
+          },
+          NoReorg,
+        )
       }
     }
   }
@@ -186,39 +214,58 @@ module LastBlockScannedHashes: {
     ~currentBlockHeight,
   ) => {
     let verifiedDataByBlockNumber = Js.Dict.empty()
-    blockNumbersAndHashes->Array.forEach(blockData => {
+    for idx in 0 to blockNumbersAndHashes->Array.length - 1 {
+      let blockData = blockNumbersAndHashes->Array.getUnsafe(idx)
       verifiedDataByBlockNumber->Js.Dict.set(blockData.blockNumber->Int.toString, blockData)
-    })
+    }
 
-    let dataByBlockNumber = self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
-    // Js engine automatically orders numeric object keys
-    let ascBlockNumberKeys = dataByBlockNumber->Js.Dict.keys
+    let isAlreadyReorgedResponse = switch self.detectedReorgBlock {
+    | Some(detectedReorgBlock) =>
+      switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(
+        detectedReorgBlock.blockNumber->Int.toString,
+      ) {
+      | Some(verifiedBlockData) => verifiedBlockData.blockHash === detectedReorgBlock.blockHash
+      | None => false
+      }
+    | None => false
+    }
 
-    let getPrevScannedBlock = idx =>
-      ascBlockNumberKeys
-      ->Belt.Array.get(idx - 1)
-      ->Option.flatMap(key => {
-        // We should already validate that the block number is verified at the point
-        verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(key)
-      })
+    if isAlreadyReorgedResponse {
+      Error(AlreadyReorgedHashes)
+    } else {
+      let dataByBlockNumber = self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
+      // Js engine automatically orders numeric object keys
+      let ascBlockNumberKeys = dataByBlockNumber->Js.Dict.keys
+
+      let getPrevScannedBlock = idx =>
+        switch ascBlockNumberKeys
+        ->Belt.Array.get(idx - 1)
+        ->Option.flatMap(key => {
+          // We should already validate that the block number is verified at the point
+          verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(key)
+        }) {
+        | Some(data) => Ok(data)
+        | None => Error(NotFound)
+        }
 
-    let rec loop = idx => {
-      switch ascBlockNumberKeys->Belt.Array.get(idx) {
-      | Some(blockNumberKey) =>
-        let scannedBlock = dataByBlockNumber->Js.Dict.unsafeGet(blockNumberKey)
-        switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(blockNumberKey) {
-        | None =>
-          Js.Exn.raiseError(
-            `Unexpected case. Couldn't find verified hash for block number ${blockNumberKey}`,
-          )
-        | Some(verifiedBlockData) if verifiedBlockData.blockHash === scannedBlock.blockHash =>
-          loop(idx + 1)
-        | Some(_) => getPrevScannedBlock(idx)
+      let rec loop = idx => {
+        switch ascBlockNumberKeys->Belt.Array.get(idx) {
+        | Some(blockNumberKey) =>
+          let scannedBlock = dataByBlockNumber->Js.Dict.unsafeGet(blockNumberKey)
+          switch verifiedDataByBlockNumber->Utils.Dict.dangerouslyGetNonOption(blockNumberKey) {
+          | None =>
+            Js.Exn.raiseError(
+              `Unexpected case. Couldn't find verified hash for block number ${blockNumberKey}`,
+            )
+          | Some(verifiedBlockData) if verifiedBlockData.blockHash === scannedBlock.blockHash =>
+            loop(idx + 1)
+          | Some(_) => getPrevScannedBlock(idx)
+          }
+        | None => getPrevScannedBlock(idx)
         }
-      | None => getPrevScannedBlock(idx)
       }
+      loop(0)
     }
-    loop(0)
   }
 
   /**
@@ -254,6 +301,7 @@ module LastBlockScannedHashes: {
     {
       confirmedBlockThreshold,
       dataByBlockNumber: newDataByBlockNumber,
+      detectedReorgBlock: None,
     }
   }
 

package/src/sources/Source.res

@@ -0,0 +1,59 @@
+/**
+A set of stats for logging about the block range fetch
+*/
+type blockRangeFetchStats = {
+  @as("total time elapsed (ms)") totalTimeElapsed: int,
+  @as("parsing time (ms)") parsingTimeElapsed?: int,
+  @as("page fetch time (ms)") pageFetchTime?: int,
+}
+
+/**
+Thes response returned from a block range fetch
+*/
+type blockRangeFetchResponse = {
+  currentBlockHeight: int,
+  reorgGuard: ReorgDetection.reorgGuard,
+  parsedQueueItems: array<Internal.eventItem>,
+  fromBlockQueried: int,
+  latestFetchedBlockNumber: int,
+  latestFetchedBlockTimestamp: int,
+  stats: blockRangeFetchStats,
+}
+
+type getItemsRetry =
+  | WithSuggestedToBlock({toBlock: int})
+  | WithBackoff({message: string, backoffMillis: int})
+
+type getItemsError =
+  | UnsupportedSelection({message: string})
+  | FailedGettingFieldSelection({exn: exn, blockNumber: int, logIndex: int, message: string})
+  | FailedParsingItems({exn: exn, blockNumber: int, logIndex: int, message: string})
+  | FailedGettingItems({exn: exn, attemptedToBlock: int, retry: getItemsRetry})
+
+exception GetItemsError(getItemsError)
+
+type sourceFor = Sync | Fallback
+type t = {
+  name: string,
+  sourceFor: sourceFor,
+  chain: ChainMap.Chain.t,
+  poweredByHyperSync: bool,
+  /* Frequency (in ms) used when polling for new events on this network. */
+  pollingInterval: int,
+  getBlockHashes: (
+    ~blockNumbers: array<int>,
+    ~logger: Pino.t,
+  ) => promise<result<array<ReorgDetection.blockDataWithTimestamp>, exn>>,
+  getHeightOrThrow: unit => promise<int>,
+  getItemsOrThrow: (
+    ~fromBlock: int,
+    ~toBlock: option<int>,
+    ~addressesByContractName: dict<array<Address.t>>,
+    ~indexingContracts: dict<FetchState.indexingContract>,
+    ~currentBlockHeight: int,
+    ~partitionId: string,
+    ~selection: FetchState.selection,
+    ~retry: int,
+    ~logger: Pino.t,
+  ) => promise<blockRangeFetchResponse>,
+}