envio 2.32.3 → 2.32.7

package/src/sources/HyperSyncSource.res.js

@@ -19,6 +19,7 @@ var HyperSyncClient = require("./HyperSyncClient.res.js");
 var Caml_splice_call = require("rescript/lib/js/caml_splice_call.js");
 var HyperSyncJsonApi = require("./HyperSyncJsonApi.res.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
+var HyperSyncHeightStream = require("./HyperSyncHeightStream.res.js");
 
 function getSelectionConfig(selection, chain) {
   var nonOptionalBlockFieldNames = new Set();
@@ -52,7 +53,13 @@ function getSelectionConfig(selection, chain) {
           Caml_splice_call.spliceObjApply(noAddressesTopicSelections, "push", [tmp]);
         }));
   var fieldSelection_block = Array.from(capitalizedBlockFields);
-  var fieldSelection_transaction = Array.from(capitalizedTransactionFields);
+  var fieldSelection_transaction = Belt_Array.map(Array.from(capitalizedTransactionFields), (function (field) {
+          if (field !== "Kind") {
+            return field;
+          } else {
+            return "Type";
+          }
+        }));
   var fieldSelection_log = [
     "Address",
     "Data",
@@ -132,7 +139,7 @@ function make(param) {
   var chain = param.chain;
   var getSelectionConfig = memoGetSelectionConfig(chain);
   var apiToken = Belt_Option.getWithDefault(param.apiToken, "3dc856dd-b0ea-494f-b27e-017b8b6b7e07");
-  var client = HyperSyncClient.make(endpointUrl, apiToken, param.clientTimeoutMillis, param.clientMaxRetries, !lowercaseAddresses);
+  var client = HyperSyncClient.make(endpointUrl, apiToken, param.clientTimeoutMillis, param.clientMaxRetries, !lowercaseAddresses, param.serializationFormat, param.enableQueryCaching, undefined, undefined, undefined);
   var hscDecoder = {
     contents: undefined
   };
@@ -407,7 +414,10 @@ function make(param) {
                 return Js_exn.raiseError(height);
               }
             }),
-          getItemsOrThrow: getItemsOrThrow
+          getItemsOrThrow: getItemsOrThrow,
+          createHeightSubscription: (function (onHeight) {
+              return HyperSyncHeightStream.subscribe(endpointUrl, apiToken, onHeight);
+            })
         };
 }
 

package/src/sources/Source.res

@@ -56,4 +56,5 @@ type t = {
     ~retry: int,
     ~logger: Pino.t,
   ) => promise<blockRangeFetchResponse>,
+  createHeightSubscription?: (~onHeight: int => unit) => unit => unit,
 }

package/src/sources/SourceManager.res

@@ -2,13 +2,21 @@ open Belt
 
 type sourceManagerStatus = Idle | WaitingForNewBlock | Querieng
 
+type sourceState = {
+  source: Source.t,
+  mutable knownHeight: int,
+  mutable unsubscribe: option<unit => unit>,
+  mutable pendingHeightResolvers: array<int => unit>,
+  mutable disabled: bool,
+}
+
 // Ideally the ChainFetcher name suits this better
 // But currently the ChainFetcher module is immutable
 // and handles both processing and fetching.
 // So this module is to encapsulate the fetching logic only
 // with a mutable state for easier reasoning and testing.
 type t = {
-  sources: Utils.Set.t<Source.t>,
+  sourcesState: array<sourceState>,
   mutable statusStart: Hrtime.timeRef,
   mutable status: sourceManagerStatus,
   maxPartitionConcurrency: int,
@@ -63,7 +71,13 @@ let make = (
   )
   {
     maxPartitionConcurrency,
-    sources: Utils.Set.fromArray(sources),
+    sourcesState: sources->Array.map(source => {
+      source,
+      knownHeight: 0,
+      unsubscribe: None,
+      pendingHeightResolvers: [],
+      disabled: false,
+    }),
     activeSource: initialActiveSource,
     waitingForNewBlockStateId: None,
     fetchingPartitionsCount: 0,
@@ -100,13 +114,15 @@ let fetchNext = async (
 ) => {
   let {maxPartitionConcurrency} = sourceManager
 
-  switch fetchState->FetchState.getNextQuery(
+  let nextQuery = fetchState->FetchState.getNextQuery(
     ~concurrencyLimit={
       maxPartitionConcurrency - sourceManager.fetchingPartitionsCount
     },
     ~currentBlockHeight,
     ~stateId,
-  ) {
+  )
+
+  switch nextQuery {
   | ReachedMaxConcurrency
   | NothingToQuery => ()
   | WaitingForNewBlock =>
@@ -116,12 +132,12 @@ let fetchNext = async (
     | None =>
       sourceManager->trackNewStatus(~newStatus=WaitingForNewBlock)
       sourceManager.waitingForNewBlockStateId = Some(stateId)
-      let currentBlockHeight = await waitForNewBlock(~currentBlockHeight)
+      let knownHeight = await waitForNewBlock(~knownHeight=currentBlockHeight)
       switch sourceManager.waitingForNewBlockStateId {
       | Some(waitingStateId) if waitingStateId === stateId => {
           sourceManager->trackNewStatus(~newStatus=Idle)
           sourceManager.waitingForNewBlockStateId = None
-          onNewBlock(~currentBlockHeight)
+          onNewBlock(~knownHeight)
         }
       | Some(_) // Don't reset it if we are waiting for another state
       | None => ()
@@ -159,82 +175,135 @@ let fetchNext = async (
 
 type status = Active | Stalled | Done
 
+let disableSource = (sourceState: sourceState) => {
+  if !sourceState.disabled {
+    sourceState.disabled = true
+    switch sourceState.unsubscribe {
+    | Some(unsubscribe) => unsubscribe()
+    | None => ()
+    }
+    true
+  } else {
+    false
+  }
+}
+
 let getSourceNewHeight = async (
   sourceManager,
-  ~source: Source.t,
-  ~currentBlockHeight,
+  ~sourceState: sourceState,
+  ~knownHeight,
   ~status: ref<status>,
   ~logger,
 ) => {
-  let newHeight = ref(0)
+  let source = sourceState.source
+  let initialHeight = sourceState.knownHeight
+  let newHeight = ref(initialHeight)
   let retry = ref(0)
 
-  while newHeight.contents <= currentBlockHeight && status.contents !== Done {
-    try {
-      // Use to detect if the source is taking too long to respond
-      let endTimer = Prometheus.SourceGetHeightDuration.startTimer({
-        "source": source.name,
-        "chainId": source.chain->ChainMap.Chain.toChainId,
+  while newHeight.contents <= knownHeight && status.contents !== Done {
+    // If subscription exists, wait for next height event
+    switch sourceState.unsubscribe {
+    | Some(_) =>
+      let height = await Promise.make((resolve, _reject) => {
+        sourceState.pendingHeightResolvers->Array.push(resolve)
       })
-      let height = await source.getHeightOrThrow()
-      endTimer()
-
-      newHeight := height
-      if height <= currentBlockHeight {
-        retry := 0
-        // Slowdown polling when the chain isn't progressing
-        let pollingInterval = if status.contents === Stalled {
-          sourceManager.stalledPollingInterval
-        } else {
-          source.pollingInterval
+
+      // Only accept heights greater than initialHeight
+      if height > initialHeight {
+        newHeight := height
+      }
+    | None =>
+      // No subscription, use REST polling
+      try {
+        // Use to detect if the source is taking too long to respond
+        let endTimer = Prometheus.SourceGetHeightDuration.startTimer({
+          "source": source.name,
+          "chainId": source.chain->ChainMap.Chain.toChainId,
+        })
+        let height = await source.getHeightOrThrow()
+        endTimer()
+
+        newHeight := height
+        if height <= knownHeight {
+          retry := 0
+
+          // If createHeightSubscription is available and height hasn't changed,
+          // create subscription instead of polling
+          switch source.createHeightSubscription {
+          | Some(createSubscription) =>
+            let unsubscribe = createSubscription(~onHeight=newHeight => {
+              sourceState.knownHeight = newHeight
+              // Resolve all pending height resolvers
+              let resolvers = sourceState.pendingHeightResolvers
+              sourceState.pendingHeightResolvers = []
+              resolvers->Array.forEach(resolve => resolve(newHeight))
+            })
+            sourceState.unsubscribe = Some(unsubscribe)
+          | None =>
+            // Slowdown polling when the chain isn't progressing
+            let pollingInterval = if status.contents === Stalled {
+              sourceManager.stalledPollingInterval
+            } else {
+              source.pollingInterval
+            }
+            await Utils.delay(pollingInterval)
+          }
         }
-        await Utils.delay(pollingInterval)
+      } catch {
+      | exn =>
+        let retryInterval = sourceManager.getHeightRetryInterval(~retry=retry.contents)
+        logger->Logging.childTrace({
+          "msg": `Height retrieval from ${source.name} source failed. Retrying in ${retryInterval->Int.toString}ms.`,
+          "source": source.name,
+          "err": exn->Utils.prettifyExn,
+        })
+        retry := retry.contents + 1
+        await Utils.delay(retryInterval)
       }
-    } catch {
-    | exn =>
-      let retryInterval = sourceManager.getHeightRetryInterval(~retry=retry.contents)
-      logger->Logging.childTrace({
-        "msg": `Height retrieval from ${source.name} source failed. Retrying in ${retryInterval->Int.toString}ms.`,
-        "source": source.name,
-        "err": exn->Utils.prettifyExn,
-      })
-      retry := retry.contents + 1
-      await Utils.delay(retryInterval)
     }
   }
-  Prometheus.SourceHeight.set(
-    ~sourceName=source.name,
-    ~chainId=source.chain->ChainMap.Chain.toChainId,
-    ~blockNumber=newHeight.contents,
-  )
+
+  // Update Prometheus only if height increased
+  if newHeight.contents > initialHeight {
+    Prometheus.SourceHeight.set(
+      ~sourceName=source.name,
+      ~chainId=source.chain->ChainMap.Chain.toChainId,
+      ~blockNumber=newHeight.contents,
+    )
+  }
+
   newHeight.contents
 }
 
 // Polls for a block height greater than the given block number to ensure a new block is available for indexing.
-let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
-  let {sources} = sourceManager
+let waitForNewBlock = async (sourceManager: t, ~knownHeight) => {
+  let {sourcesState} = sourceManager
 
   let logger = Logging.createChild(
     ~params={
       "chainId": sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
-      "currentBlockHeight": currentBlockHeight,
+      "knownHeight": knownHeight,
     },
   )
   logger->Logging.childTrace("Initiating check for new blocks.")
 
   let syncSources = []
   let fallbackSources = []
-  sources->Utils.Set.forEach(source => {
-    if (
+  sourcesState->Array.forEach(sourceState => {
+    let source = sourceState.source
+    if sourceState.disabled {
+      // Skip disabled sources
+      ()
+    } else if (
       source.sourceFor === Sync ||
         // Even if the active source is a fallback, still include
         // it to the list. So we don't wait for a timeout again
         // if all main sync sources are still not valid
         source === sourceManager.activeSource
     ) {
-      syncSources->Array.push(source)
+      syncSources->Array.push(sourceState)
     } else {
-      fallbackSources->Array.push(source)
+      fallbackSources->Array.push(sourceState)
     }
   })
 
@@ -242,10 +311,10 @@ let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
 
   let (source, newBlockHeight) = await Promise.race(
     syncSources
-    ->Array.map(async source => {
+    ->Array.map(async sourceState => {
       (
-        source,
-        await sourceManager->getSourceNewHeight(~source, ~currentBlockHeight, ~status, ~logger),
+        sourceState.source,
+        await sourceManager->getSourceNewHeight(~sourceState, ~knownHeight, ~status, ~logger),
       )
     })
     ->Array.concat([
@@ -269,15 +338,10 @@ let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
         // Promise.race will be forever pending if fallbackSources is empty
         // which is good for this use case
         Promise.race(
-          fallbackSources->Array.map(async source => {
+          fallbackSources->Array.map(async sourceState => {
             (
-              source,
-              await sourceManager->getSourceNewHeight(
-                ~source,
-                ~currentBlockHeight,
-                ~status,
-                ~logger,
-              ),
+              sourceState.source,
+              await sourceManager->getSourceNewHeight(~sourceState, ~knownHeight, ~status, ~logger),
             )
           }),
         )
@@ -300,11 +364,11 @@ let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
   newBlockHeight
 }
 
-let getNextSyncSource = (
+let getNextSyncSourceState = (
   sourceManager,
   // This is needed to include the Fallback source to rotation
-  ~initialSource,
-  ~currentSource,
+  ~initialSourceState: sourceState,
+  ~currentSourceState: sourceState,
   // After multiple failures start returning fallback sources as well
   // But don't try it when main sync sources fail because of invalid configuration
   // note: The logic might be changed in the future
@@ -315,16 +379,21 @@ let getNextSyncSource = (
 
   let hasActive = ref(false)
 
-  sourceManager.sources->Utils.Set.forEach(source => {
-    if source === currentSource {
+  sourceManager.sourcesState->Array.forEach(sourceState => {
+    let source = sourceState.source
+
+    // Skip disabled sources
+    if sourceState.disabled {
+      ()
+    } else if sourceState === currentSourceState {
       hasActive := true
     } else if (
       switch source.sourceFor {
       | Sync => true
-      | Fallback => attemptFallbacks || source === initialSource
+      | Fallback => attemptFallbacks || sourceState === initialSourceState
       }
     ) {
-      (hasActive.contents ? after : before)->Array.push(source)
+      (hasActive.contents ? after : before)->Array.push(sourceState)
     }
   })
 
@@ -333,7 +402,7 @@ let getNextSyncSource = (
   | None =>
     switch before->Array.get(0) {
     | Some(s) => s
-    | None => currentSource
+    | None => currentSourceState
     }
   }
 }
@@ -349,12 +418,16 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
   )
   let responseRef = ref(None)
   let retryRef = ref(0)
-  let initialSource = sourceManager.activeSource
-  let sourceRef = ref(initialSource)
+  let initialSourceState =
+    sourceManager.sourcesState
+    ->Js.Array2.find(s => s.source === sourceManager.activeSource)
+    ->Option.getUnsafe
+  let sourceStateRef = ref(initialSourceState)
   let shouldUpdateActiveSource = ref(false)
 
   while responseRef.contents->Option.isNone {
-    let source = sourceRef.contents
+    let sourceState = sourceStateRef.contents
+    let source = sourceState.source
     let toBlock = toBlockRef.contents
     let retry = retryRef.contents
 
@@ -395,15 +468,19 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
       switch error {
       | UnsupportedSelection(_)
       | FailedGettingFieldSelection(_) => {
-          let nextSource = sourceManager->getNextSyncSource(~initialSource, ~currentSource=source)
+          let nextSourceState =
+            sourceManager->getNextSyncSourceState(
+              ~initialSourceState,
+              ~currentSourceState=sourceState,
+            )
 
-          // These errors are impossible to recover, so we delete the source
-          // from sourceManager so it's not attempted anymore
-          let notAlreadyDeleted = sourceManager.sources->Utils.Set.delete(source)
+          // These errors are impossible to recover, so we disable the source
+          // so it's not attempted anymore
+          let notAlreadyDisabled = disableSource(sourceState)
 
           // In case there are multiple partitions
           // failing at the same time. Log only once
-          if notAlreadyDeleted {
+          if notAlreadyDisabled {
             switch error {
             | UnsupportedSelection({message}) => logger->Logging.childError(message)
             | FailedGettingFieldSelection({exn, message, blockNumber, logIndex}) =>
@@ -417,7 +494,7 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
             }
           }
 
-          if nextSource === source {
+          if nextSourceState === sourceState {
             %raw(`null`)->ErrorHandling.mkLogAndRaise(
               ~logger,
               ~msg="The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.",
@@ -425,9 +502,9 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
           } else {
             logger->Logging.childInfo({
               "msg": "Switching to another data-source",
-              "source": nextSource.name,
+              "source": nextSourceState.source.name,
             })
-            sourceRef := nextSource
+            sourceStateRef := nextSourceState
             shouldUpdateActiveSource := true
             retryRef := 0
           }
@@ -441,14 +518,14 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
         toBlockRef := Some(toBlock)
         retryRef := 0
       | FailedGettingItems({exn, attemptedToBlock, retry: ImpossibleForTheQuery({message})}) =>
-        let nextSource =
-          sourceManager->getNextSyncSource(
-            ~initialSource,
-            ~currentSource=source,
+        let nextSourceState =
+          sourceManager->getNextSyncSourceState(
+            ~initialSourceState,
+            ~currentSourceState=sourceState,
             ~attemptFallbacks=true,
           )
 
-        let hasAnotherSource = nextSource !== initialSource
+        let hasAnotherSource = nextSourceState !== initialSourceState
 
         logger->Logging.childWarn({
           "msg": message ++ (hasAnotherSource ? " - Attempting to another source" : ""),
@@ -462,7 +539,7 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
             ~msg="The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.",
           )
         } else {
-          sourceRef := nextSource
+          sourceStateRef := nextSourceState
           shouldUpdateActiveSource := false
           retryRef := 0
         }
@@ -477,19 +554,19 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
         // just keep the value high
         let attemptFallbacks = retry >= 10
 
-        let nextSource = switch retry {
+        let nextSourceState = switch retry {
         // Don't attempt a switch on first two failure
-        | 0 | 1 => source
+        | 0 | 1 => sourceState
         | _ =>
           // Then try to switch every second failure
           if retry->mod(2) === 0 {
-            sourceManager->getNextSyncSource(
-              ~initialSource,
+            sourceManager->getNextSyncSourceState(
+              ~initialSourceState,
               ~attemptFallbacks,
-              ~currentSource=source,
+              ~currentSourceState=sourceState,
             )
           } else {
-            source
+            sourceState
           }
         }
 
@@ -503,13 +580,13 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
           "err": exn->Utils.prettifyExn,
         })
 
-        let shouldSwitch = nextSource !== source
+        let shouldSwitch = nextSourceState !== sourceState
         if shouldSwitch {
           logger->Logging.childInfo({
             "msg": "Switching to another data-source",
-            "source": nextSource.name,
+            "source": nextSourceState.source.name,
           })
-          sourceRef := nextSource
+          sourceStateRef := nextSourceState
           shouldUpdateActiveSource := true
         } else {
           await Utils.delay(Pervasives.min(backoffMillis, 60_000))
@@ -523,7 +600,7 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
   }
 
   if shouldUpdateActiveSource.contents {
-    sourceManager.activeSource = sourceRef.contents
+    sourceManager.activeSource = sourceStateRef.contents.source
   }
 
   responseRef.contents->Option.getUnsafe