envio 3.0.0-alpha.2 → 3.0.0-alpha.21

package/src/sources/SourceManager.res

@@ -2,27 +2,57 @@ open Belt
 
 type sourceManagerStatus = Idle | WaitingForNewBlock | Querieng
 
+type sourceState = {
+  source: Source.t,
+  mutable knownHeight: int,
+  mutable unsubscribe: option<unit => unit>,
+  mutable pendingHeightResolvers: array<int => unit>,
+  mutable disabled: bool,
+  // Timestamp (ms) when this source last failed during executeQuery.
+  // Used to decide when to attempt recovery to this source.
+  mutable lastFailedAt: option<float>,
+}
+
 // Ideally the ChainFetcher name suits this better
 // But currently the ChainFetcher module is immutable
 // and handles both processing and fetching.
 // So this module is to encapsulate the fetching logic only
 // with a mutable state for easier reasoning and testing.
 type t = {
-  sources: Utils.Set.t<Source.t>,
+  sourcesState: array<sourceState>,
   mutable statusStart: Hrtime.timeRef,
   mutable status: sourceManagerStatus,
   maxPartitionConcurrency: int,
-  newBlockFallbackStallTimeout: int,
+  newBlockStallTimeout: int,
+  newBlockStallTimeoutLive: int,
   stalledPollingInterval: int,
   getHeightRetryInterval: (~retry: int) => int,
   mutable activeSource: Source.t,
   mutable waitingForNewBlockStateId: option<int>,
   // Should take into consideration partitions fetching for previous states (before rollback)
   mutable fetchingPartitionsCount: int,
+  recoveryTimeout: float,
+  mutable hasLive: bool,
 }
 
 let getActiveSource = sourceManager => sourceManager.activeSource
 
+type sourceRole = Primary | Secondary
+
+// Determines whether a source is Primary or Secondary given the current mode.
+// isLive=false (backfill): Sync=Primary, Fallback=Secondary, Live=ignored (None).
+// isLive=true with hasLive: Live=Primary, Sync+Fallback=Secondary.
+// isLive=true without hasLive: Sync=Primary, Fallback=Secondary.
+let getSourceRole = (~sourceFor: Source.sourceFor, ~isLive, ~hasLive) =>
+  switch (isLive, sourceFor) {
+  | (false, Sync) => Some(Primary)
+  | (false, Fallback) => Some(Secondary)
+  | (false, Live) => None
+  | (true, Live) => Some(Primary)
+  | (true, Sync) => hasLive ? Some(Secondary) : Some(Primary)
+  | (true, Fallback) => Some(Secondary)
+  }
+
 let makeGetHeightRetryInterval = (
   ~initialRetryInterval,
   ~backoffMultiplicative,
@@ -41,17 +71,23 @@ let makeGetHeightRetryInterval = (
 let make = (
   ~sources: array<Source.t>,
   ~maxPartitionConcurrency,
-  ~newBlockFallbackStallTimeout=20_000,
+  ~isLive,
+  ~newBlockStallTimeout=60_000,
+  ~newBlockStallTimeoutLive=20_000,
   ~stalledPollingInterval=5_000,
+  ~recoveryTimeout=60_000.0,
   ~getHeightRetryInterval=makeGetHeightRetryInterval(
     ~initialRetryInterval=1000,
     ~backoffMultiplicative=2,
     ~maxRetryInterval=60_000,
   ),
 ) => {
-  let initialActiveSource = switch sources->Js.Array2.find(source => source.sourceFor === Sync) {
-  | None => Js.Exn.raiseError("Invalid configuration, no data-source for historical sync provided")
+  let hasLive = sources->Js.Array2.some(s => s.sourceFor === Live)
+  let initialActiveSource = switch sources->Js.Array2.find(source =>
+    getSourceRole(~sourceFor=source.sourceFor, ~isLive, ~hasLive) === Some(Primary)
+  ) {
   | Some(source) => source
+  | None => Js.Exn.raiseError("Invalid configuration, no data-source for historical sync provided")
   }
   Prometheus.IndexingMaxConcurrency.set(
     ~maxConcurrency=maxPartitionConcurrency,
@@ -63,15 +99,25 @@ let make = (
   )
   {
     maxPartitionConcurrency,
-    sources: Utils.Set.fromArray(sources),
+    sourcesState: sources->Array.map(source => {
+      source,
+      knownHeight: 0,
+      unsubscribe: None,
+      pendingHeightResolvers: [],
+      disabled: false,
+      lastFailedAt: None,
+    }),
     activeSource: initialActiveSource,
     waitingForNewBlockStateId: None,
     fetchingPartitionsCount: 0,
-    newBlockFallbackStallTimeout,
+    newBlockStallTimeout,
+    newBlockStallTimeoutLive,
     stalledPollingInterval,
     getHeightRetryInterval,
+    recoveryTimeout,
     statusStart: Hrtime.makeTimer(),
     status: Idle,
+    hasLive,
   }
 }
 
@@ -81,9 +127,9 @@ let trackNewStatus = (sourceManager: t, ~newStatus) => {
   | WaitingForNewBlock => Prometheus.IndexingSourceWaitingTime.counter
   | Querieng => Prometheus.IndexingQueryTime.counter
   }
-  promCounter->Prometheus.SafeCounter.incrementMany(
+  promCounter->Prometheus.SafeCounter.handleFloat(
     ~labels=sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
-    ~value=sourceManager.statusStart->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis,
+    ~value=sourceManager.statusStart->Hrtime.timeSince->Hrtime.toSecondsFloat,
   )
   sourceManager.statusStart = Hrtime.makeTimer()
   sourceManager.status = newStatus
@@ -92,7 +138,6 @@ let trackNewStatus = (sourceManager: t, ~newStatus) => {
 let fetchNext = async (
   sourceManager: t,
   ~fetchState: FetchState.t,
-  ~currentBlockHeight,
   ~executeQuery,
   ~waitForNewBlock,
   ~onNewBlock,
@@ -100,13 +145,13 @@ let fetchNext = async (
 ) => {
   let {maxPartitionConcurrency} = sourceManager
 
-  switch fetchState->FetchState.getNextQuery(
+  let nextQuery = fetchState->FetchState.getNextQuery(
     ~concurrencyLimit={
       maxPartitionConcurrency - sourceManager.fetchingPartitionsCount
     },
-    ~currentBlockHeight,
-    ~stateId,
-  ) {
+  )
+
+  switch nextQuery {
   | ReachedMaxConcurrency
   | NothingToQuery => ()
   | WaitingForNewBlock =>
@@ -116,19 +161,19 @@ let fetchNext = async (
     | None =>
       sourceManager->trackNewStatus(~newStatus=WaitingForNewBlock)
       sourceManager.waitingForNewBlockStateId = Some(stateId)
-      let currentBlockHeight = await waitForNewBlock(~currentBlockHeight)
+      let knownHeight = await waitForNewBlock(~knownHeight=fetchState.knownHeight)
       switch sourceManager.waitingForNewBlockStateId {
       | Some(waitingStateId) if waitingStateId === stateId => {
           sourceManager->trackNewStatus(~newStatus=Idle)
           sourceManager.waitingForNewBlockStateId = None
-          onNewBlock(~currentBlockHeight)
+          onNewBlock(~knownHeight)
         }
       | Some(_) // Don't reset it if we are waiting for another state
       | None => ()
       }
     }
   | Ready(queries) => {
-      fetchState->FetchState.startFetchingQueries(~queries, ~stateId)
+      fetchState->FetchState.startFetchingQueries(~queries)
       sourceManager.fetchingPartitionsCount =
         sourceManager.fetchingPartitionsCount + queries->Array.length
       Prometheus.IndexingConcurrency.set(
@@ -159,131 +204,279 @@ let fetchNext = async (
 
 type status = Active | Stalled | Done
 
+let disableSource = (sourceManager: t, sourceState: sourceState) => {
+  if !sourceState.disabled {
+    sourceState.disabled = true
+    switch sourceState.unsubscribe {
+    | Some(unsubscribe) => unsubscribe()
+    | None => ()
+    }
+    if sourceState.source.sourceFor === Live {
+      // Only clear hasLive if no other non-disabled Live sources remain
+      let hasOtherLive =
+        sourceManager.sourcesState->Js.Array2.some(s =>
+          s !== sourceState && !s.disabled && s.source.sourceFor === Live
+        )
+      sourceManager.hasLive = hasOtherLive
+    }
+    true
+  } else {
+    false
+  }
+}
+
 let getSourceNewHeight = async (
   sourceManager,
-  ~source: Source.t,
-  ~currentBlockHeight,
+  ~sourceState: sourceState,
+  ~knownHeight,
+  ~stallTimeout,
+  ~isLive,
   ~status: ref<status>,
   ~logger,
 ) => {
-  let newHeight = ref(0)
+  let source = sourceState.source
+  let initialHeight = sourceState.knownHeight
+  let newHeight = ref(initialHeight)
   let retry = ref(0)
 
-  while newHeight.contents <= currentBlockHeight && status.contents !== Done {
-    try {
-      // Use to detect if the source is taking too long to respond
-      let endTimer = Prometheus.SourceGetHeightDuration.startTimer({
-        "source": source.name,
-        "chainId": source.chain->ChainMap.Chain.toChainId,
+  while newHeight.contents <= knownHeight && status.contents !== Done {
+    // If subscription exists, wait for next height event
+    switch sourceState.unsubscribe {
+    | Some(_) =>
+      let subscriptionPromise = Promise.make((resolve, _reject) => {
+        sourceState.pendingHeightResolvers->Array.push(resolve)
       })
-      let height = await source.getHeightOrThrow()
-      endTimer()
-
-      newHeight := height
-      if height <= currentBlockHeight {
-        retry := 0
-        // Slowdown polling when the chain isn't progressing
-        let pollingInterval = if status.contents === Stalled {
-          sourceManager.stalledPollingInterval
-        } else {
-          source.pollingInterval
+      // If subscription goes quiet for half the stall timeout, fall back to REST polling
+      let pollingFallback = Utils.delay(stallTimeout / 2)->Promise.then(async () => {
+        logger->Logging.childTrace({
+          "msg": "onHeight subscription stale, switching to polling fallback",
+          "source": source.name,
+          "chainId": source.chain->ChainMap.Chain.toChainId,
+        })
+        let h = ref(initialHeight)
+        while h.contents <= knownHeight && !(newHeight.contents > initialHeight) {
+          try {
+            h := (await source.getHeightOrThrow())
+          } catch {
+          | _ => ()
+          }
+          if h.contents <= knownHeight && !(newHeight.contents > initialHeight) {
+            await Utils.delay(source.pollingInterval)
+          }
         }
-        await Utils.delay(pollingInterval)
-      }
-    } catch {
-    | exn =>
-      let retryInterval = sourceManager.getHeightRetryInterval(~retry=retry.contents)
-      logger->Logging.childTrace({
-        "msg": `Height retrieval from ${source.name} source failed. Retrying in ${retryInterval->Int.toString}ms.`,
-        "source": source.name,
-        "err": exn->Utils.prettifyExn,
+        h.contents
       })
-      retry := retry.contents + 1
-      await Utils.delay(retryInterval)
+      let height = await Promise.race([subscriptionPromise, pollingFallback])
+
+      // Only accept heights greater than initialHeight
+      if height > initialHeight {
+        newHeight := height
+      }
+    | None =>
+      // No subscription, use REST polling
+      try {
+        let height = await source.getHeightOrThrow()
+
+        newHeight := height
+        if height <= knownHeight {
+          retry := 0
+
+          // If createHeightSubscription is available and height hasn't changed,
+          // create subscription instead of polling
+          switch source.createHeightSubscription {
+          | Some(createSubscription) if isLive =>
+            let unsubscribe = createSubscription(~onHeight=newHeight => {
+              sourceState.knownHeight = newHeight
+              // Resolve all pending height resolvers
+              let resolvers = sourceState.pendingHeightResolvers
+              sourceState.pendingHeightResolvers = []
+              resolvers->Array.forEach(resolve => resolve(newHeight))
+            })
+            sourceState.unsubscribe = Some(unsubscribe)
+          | _ =>
+            // Slowdown polling when the chain isn't progressing
+            let pollingInterval = if status.contents === Stalled {
+              sourceManager.stalledPollingInterval
+            } else {
+              source.pollingInterval
+            }
+            await Utils.delay(pollingInterval)
+          }
+        }
+      } catch {
+      | exn =>
+        let retryInterval = sourceManager.getHeightRetryInterval(~retry=retry.contents)
+        logger->Logging.childTrace({
+          "msg": `Height retrieval from ${source.name} source failed. Retrying in ${retryInterval->Int.toString}ms.`,
+          "source": source.name,
+          "err": exn->Utils.prettifyExn,
+        })
+        retry := retry.contents + 1
+        await Utils.delay(retryInterval)
+      }
     }
   }
-  Prometheus.SourceHeight.set(
-    ~sourceName=source.name,
-    ~chainId=source.chain->ChainMap.Chain.toChainId,
-    ~blockNumber=newHeight.contents,
-  )
+
+  // Update Prometheus only if height increased
+  if newHeight.contents > initialHeight {
+    Prometheus.SourceHeight.set(
+      ~sourceName=source.name,
+      ~chainId=source.chain->ChainMap.Chain.toChainId,
+      ~blockNumber=newHeight.contents,
+    )
+  }
+
   newHeight.contents
 }
 
+let compareByOldestFailure = (a: sourceState, b: sourceState) =>
+  switch (a.lastFailedAt, b.lastFailedAt) {
+  | (None, Some(_)) => -1
+  | (Some(_), None) => 1
+  | (Some(a), Some(b)) => a < b ? -1 : a > b ? 1 : 0
+  | (None, None) => 0
+  }
+
+// Priority: working primaries > working secondaries > all primaries.
+let getNextSources = (sourceManager, ~isLive, ~excludedSources=?) => {
+  let now = Js.Date.now()
+  let workingPrimarySources = []
+  let allPrimarySources = []
+  let workingSecondarySources = []
+  for i in 0 to sourceManager.sourcesState->Array.length - 1 {
+    let sourceState = sourceManager.sourcesState->Array.getUnsafe(i)
+    if !sourceState.disabled {
+      let isExcluded = switch excludedSources {
+      | Some(set) => set->Utils.Set.has(sourceState)
+      | None => false
+      }
+      if !isExcluded {
+        let isWorking = switch sourceState.lastFailedAt {
+        | Some(failedAt) => now -. failedAt >= sourceManager.recoveryTimeout
+        | None => true
+        }
+        switch getSourceRole(
+          ~sourceFor=sourceState.source.sourceFor,
+          ~isLive,
+          ~hasLive=sourceManager.hasLive,
+        ) {
+        | Some(Primary) =>
+          allPrimarySources->Array.push(sourceState)
+          if isWorking {
+            workingPrimarySources->Array.push(sourceState)
+          }
+        | Some(Secondary) if isWorking => workingSecondarySources->Array.push(sourceState)
+        | _ => ()
+        }
+      }
+    }
+  }
+  if workingPrimarySources->Array.length > 0 {
+    workingPrimarySources
+  } else if workingSecondarySources->Array.length > 0 {
+    workingSecondarySources
+  } else {
+    // All primaries in recovery — sort by oldest lastFailedAt (closest to recovery first)
+    allPrimarySources->Js.Array2.sortInPlaceWith(compareByOldestFailure)
+  }
+}
+
+// Single source selection from getNextSources.
+// Prefers activeSource if it's in the candidates. Fast path: check first item.
+let getNextSource = (sourceManager, ~isLive, ~excludedSources=?) => {
+  let sources = sourceManager->getNextSources(~isLive, ~excludedSources?)
+  switch sources->Array.get(0) {
+  | None => None
+  | Some(first) if first.source === sourceManager.activeSource => Some(first)
+  | _ =>
+    switch sources->Js.Array2.find(s => s.source === sourceManager.activeSource) {
+    | Some(_) as result => result
+    | None => sources->Array.get(0)
+    }
+  }
+}
+
 // Polls for a block height greater than the given block number to ensure a new block is available for indexing.
-let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
-  let {sources} = sourceManager
+let waitForNewBlock = async (sourceManager: t, ~knownHeight, ~isLive) => {
+  let {sourcesState} = sourceManager
 
   let logger = Logging.createChild(
     ~params={
       "chainId": sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
-      "currentBlockHeight": currentBlockHeight,
+      "knownHeight": knownHeight,
     },
   )
   logger->Logging.childTrace("Initiating check for new blocks.")
 
-  // Only include Live sources if we've actually synced some blocks
-  // (currentBlockHeight > 0 means we've fetched at least one batch)
-  // This prevents Live RPC from winning the initial height race and
-  // becoming activeSource, which would bypass HyperSync's smart block detection
-  let isInitialHeightFetch = currentBlockHeight === 0
-
-  let syncSources = []
-  let fallbackSources = []
-  sources->Utils.Set.forEach(source => {
-    if (
-      source.sourceFor === Sync ||
-        // Include Live sources only after initial sync has started
-        // Live sources are optimized for real-time indexing with lower latency
-        (source.sourceFor === Live && !isInitialHeightFetch) ||
-        // Even if the active source is a fallback, still include
-        // it to the list. So we don't wait for a timeout again
-        // if all main sync sources are still not valid
-        source === sourceManager.activeSource
-    ) {
-      syncSources->Array.push(source)
-    } else {
-      fallbackSources->Array.push(source)
-    }
-  })
+  let mainSources = sourceManager->getNextSources(~isLive)
 
   let status = ref(Active)
 
+  let stallTimeout = if isLive {
+    sourceManager.newBlockStallTimeoutLive
+  } else {
+    sourceManager.newBlockStallTimeout
+  }
+
   let (source, newBlockHeight) = await Promise.race(
-    syncSources
-    ->Array.map(async source => {
+    mainSources
+    ->Array.map(async sourceState => {
       (
-        source,
-        await sourceManager->getSourceNewHeight(~source, ~currentBlockHeight, ~status, ~logger),
+        sourceState.source,
+        await sourceManager->getSourceNewHeight(
+          ~sourceState,
+          ~knownHeight,
+          ~stallTimeout,
+          ~isLive,
+          ~status,
+          ~logger,
+        ),
       )
     })
     ->Array.concat([
-      Utils.delay(sourceManager.newBlockFallbackStallTimeout)->Promise.then(() => {
+      Utils.delay(stallTimeout)->Promise.then(() => {
+        // Build fallback: sources not in mainSources with a valid role, even with recent lastFailedAt
+        let fallbackSources = []
+        sourcesState->Array.forEach(sourceState => {
+          if (
+            !(mainSources->Js.Array2.includes(sourceState)) &&
+            getSourceRole(
+              ~sourceFor=sourceState.source.sourceFor,
+              ~isLive,
+              ~hasLive=sourceManager.hasLive,
+            )->Option.isSome
+          ) {
+            fallbackSources->Array.push(sourceState)
+          }
+        })
+
         if status.contents !== Done {
           status := Stalled
 
           switch fallbackSources {
           | [] =>
             logger->Logging.childWarn(
-              `No new blocks detected within ${(sourceManager.newBlockFallbackStallTimeout / 1000)
+              `No new blocks detected within ${(stallTimeout / 1000)
                   ->Int.toString}s. Polling will continue at a reduced rate. For better reliability, refer to our RPC fallback guide: https://docs.envio.dev/docs/HyperIndex/rpc-sync`,
             )
           | _ =>
             logger->Logging.childWarn(
-              `No new blocks detected within ${(sourceManager.newBlockFallbackStallTimeout / 1000)
-                  ->Int.toString}s. Continuing polling with fallback RPC sources from the configuration.`,
+              `No new blocks detected within ${(stallTimeout / 1000)
+                  ->Int.toString}s. Continuing polling with secondary RPC sources from the configuration.`,
             )
           }
         }
         // Promise.race will be forever pending if fallbackSources is empty
         // which is good for this use case
         Promise.race(
-          fallbackSources->Array.map(async source => {
+          fallbackSources->Array.map(async sourceState => {
             (
-              source,
+              sourceState.source,
               await sourceManager->getSourceNewHeight(
-                ~source,
-                ~currentBlockHeight,
+                ~sourceState,
+                ~knownHeight,
+                ~stallTimeout,
+                ~isLive,
                 ~status,
                 ~logger,
               ),
@@ -296,7 +489,7 @@ let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
 
   sourceManager.activeSource = source
 
-  // Show a higher level log if we displayed a warning/error after newBlockFallbackStallTimeout
+  // Show a higher level log if we displayed a warning/error after newBlockStallTimeout
   let log = status.contents === Stalled ? Logging.childInfo : Logging.childTrace
   logger->log({
     "msg": `New blocks successfully found.`,
@@ -309,63 +502,43 @@ let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
   newBlockHeight
 }
 
-let getNextSyncSource = (
-  sourceManager,
-  // This is needed to include the Fallback source to rotation
-  ~initialSource,
-  ~currentSource,
-  // After multiple failures start returning fallback sources as well
-  // But don't try it when main sync sources fail because of invalid configuration
-  // note: The logic might be changed in the future
-  ~attemptFallbacks=false,
-) => {
-  let before = []
-  let after = []
-
-  let hasActive = ref(false)
-
-  sourceManager.sources->Utils.Set.forEach(source => {
-    if source === currentSource {
-      hasActive := true
-    } else if (
-      switch source.sourceFor {
-      | Sync => true
-      // Live sources should NOT be used for historical sync rotation
-      // They are only meant for real-time indexing once synced
-      | Live | Fallback => attemptFallbacks || source === initialSource
-      }
-    ) {
-      (hasActive.contents ? after : before)->Array.push(source)
-    }
-  })
+let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~knownHeight, ~isLive) => {
+  let noSourcesError = "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team."
 
-  switch after->Array.get(0) {
-  | Some(s) => s
-  | None =>
-    switch before->Array.get(0) {
-    | Some(s) => s
-    | None => currentSource
-    }
-  }
-}
+  // Sources where the query is impossible — lazily allocated, excluded for the duration of this query
+  let excludedSourcesRef = ref(None)
 
-let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBlockHeight) => {
-  let toBlockRef = ref(
-    switch query.target {
-    | Head => None
-    | EndBlock({toBlock})
-    | Merge({toBlock}) =>
-      Some(toBlock)
-    },
-  )
+  let toBlockRef = ref(query.toBlock)
   let responseRef = ref(None)
   let retryRef = ref(0)
-  let initialSource = sourceManager.activeSource
-  let sourceRef = ref(initialSource)
-  let shouldUpdateActiveSource = ref(false)
 
   while responseRef.contents->Option.isNone {
-    let source = sourceRef.contents
+    // Select the best source at the start of every iteration
+    let sourceState = switch sourceManager->getNextSource(
+      ~isLive,
+      ~excludedSources=?excludedSourcesRef.contents,
+    ) {
+    | Some(s) =>
+      if s.source !== sourceManager.activeSource {
+        let logger = Logging.createChild(
+          ~params={"chainId": sourceManager.activeSource.chain->ChainMap.Chain.toChainId},
+        )
+        logger->Logging.childInfo({
+          "msg": "Switching data-source",
+          "source": s.source.name,
+          "previousSource": sourceManager.activeSource.name,
+          "fromBlock": query.fromBlock,
+        })
+      }
+      s
+    | None =>
+      let logger = Logging.createChild(
+        ~params={"chainId": sourceManager.activeSource.chain->ChainMap.Chain.toChainId},
+      )
+      %raw(`null`)->ErrorHandling.mkLogAndRaise(~logger, ~msg=noSourcesError)
+    }
+    sourceManager.activeSource = sourceState.source
+    let source = sourceState.source
     let toBlock = toBlockRef.contents
     let retry = retryRef.contents
 
@@ -389,7 +562,7 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
         ~addressesByContractName=query.addressesByContractName,
         ~indexingContracts=query.indexingContracts,
         ~partitionId=query.partitionId,
-        ~currentBlockHeight,
+        ~knownHeight,
         ~selection=query.selection,
         ~retry,
         ~logger,
@@ -400,21 +573,20 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
         "numEvents": response.parsedQueueItems->Array.length,
         "stats": response.stats,
       })
+      sourceState.lastFailedAt = None
       responseRef := Some(response)
     } catch {
     | Source.GetItemsError(error) =>
       switch error {
       | UnsupportedSelection(_)
       | FailedGettingFieldSelection(_) => {
-          let nextSource = sourceManager->getNextSyncSource(~initialSource, ~currentSource=source)
-
-          // These errors are impossible to recover, so we delete the source
-          // from sourceManager so it's not attempted anymore
-          let notAlreadyDeleted = sourceManager.sources->Utils.Set.delete(source)
+          // These errors are impossible to recover, so we disable the source
+          // so it's not attempted anymore
+          let notAlreadyDisabled = sourceManager->disableSource(sourceState)
 
           // In case there are multiple partitions
           // failing at the same time. Log only once
-          if notAlreadyDeleted {
+          if notAlreadyDisabled {
             switch error {
             | UnsupportedSelection({message}) => logger->Logging.childError(message)
             | FailedGettingFieldSelection({exn, message, blockNumber, logIndex}) =>
@@ -428,20 +600,7 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
             }
           }
 
-          if nextSource === source {
-            %raw(`null`)->ErrorHandling.mkLogAndRaise(
-              ~logger,
-              ~msg="The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.",
-            )
-          } else {
-            logger->Logging.childInfo({
-              "msg": "Switching to another data-source",
-              "source": nextSource.name,
-            })
-            sourceRef := nextSource
-            shouldUpdateActiveSource := true
-            retryRef := 0
-          }
+          retryRef := 0
         }
       | FailedGettingItems({attemptedToBlock, retry: WithSuggestedToBlock({toBlock})}) =>
         logger->Logging.childTrace({
@@ -452,58 +611,24 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
         toBlockRef := Some(toBlock)
         retryRef := 0
       | FailedGettingItems({exn, attemptedToBlock, retry: ImpossibleForTheQuery({message})}) =>
-        let nextSource =
-          sourceManager->getNextSyncSource(
-            ~initialSource,
-            ~currentSource=source,
-            ~attemptFallbacks=true,
-          )
-
-        let hasAnotherSource = nextSource !== initialSource
+        // Don't set lastFailedAt — the source isn't broken, the query just can't work on it
+        let excludedSources = switch excludedSourcesRef.contents {
+        | Some(s) => s
+        | None =>
+          let s = Utils.Set.make()
+          excludedSourcesRef := Some(s)
+          s
+        }
+        excludedSources->Utils.Set.add(sourceState)->ignore
 
         logger->Logging.childWarn({
-          "msg": message ++ (hasAnotherSource ? " - Attempting to another source" : ""),
+          "msg": message ++ " - Attempting another source",
           "toBlock": attemptedToBlock,
           "err": exn->Utils.prettifyExn,
         })
-
-        if !hasAnotherSource {
-          %raw(`null`)->ErrorHandling.mkLogAndRaise(
-            ~logger,
-            ~msg="The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.",
-          )
-        } else {
-          sourceRef := nextSource
-          shouldUpdateActiveSource := false
-          retryRef := 0
-        }
+        retryRef := 0
 
       | FailedGettingItems({exn, attemptedToBlock, retry: WithBackoff({message, backoffMillis})}) =>
-        // Starting from the 11th failure (retry=10)
-        // include fallback sources for switch
-        // (previously it would consider only sync sources or the initial one)
-        // This is a little bit tricky to find the right number,
-        // because meaning between RPC and HyperSync is different for the error
-        // but since Fallback was initially designed to be used only for height check
-        // just keep the value high
-        let attemptFallbacks = retry >= 10
-
-        let nextSource = switch retry {
-        // Don't attempt a switch on first two failure
-        | 0 | 1 => source
-        | _ =>
-          // Then try to switch every second failure
-          if retry->mod(2) === 0 {
-            sourceManager->getNextSyncSource(
-              ~initialSource,
-              ~attemptFallbacks,
-              ~currentSource=source,
-            )
-          } else {
-            source
-          }
-        }
-
         // Start displaying warnings after 4 failures
         let log = retry >= 4 ? Logging.childWarn : Logging.childTrace
         logger->log({
@@ -514,14 +639,30 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
           "err": exn->Utils.prettifyExn,
         })
 
-        let shouldSwitch = nextSource !== source
+        let shouldSwitch = switch retry {
+        // Don't attempt a switch on first two failures
+        | 0 | 1 => false
+        // Then try to switch every second failure
+        | _ => retry->mod(2) === 0
+        }
+
         if shouldSwitch {
-          logger->Logging.childInfo({
-            "msg": "Switching to another data-source",
-            "source": nextSource.name,
-          })
-          sourceRef := nextSource
-          shouldUpdateActiveSource := true
+          let now = Js.Date.now()
+          sourceState.lastFailedAt = Some(now)
+          // Check if there's a working (recovered) source to switch to immediately
+          let nextSource =
+            sourceManager->getNextSource(~isLive, ~excludedSources=?excludedSourcesRef.contents)
+          let hasWorkingAlternative = switch nextSource {
+          | Some(s) =>
+            switch s.lastFailedAt {
+            | None => true
+            | Some(failedAt) => now -. failedAt >= sourceManager.recoveryTimeout
+            }
+          | None => false
+          }
+          if !hasWorkingAlternative {
+            await Utils.delay(Pervasives.min(backoffMillis, 60_000))
+          }
         } else {
           await Utils.delay(Pervasives.min(backoffMillis, 60_000))
         }
@@ -533,9 +674,5 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
     }
   }
 
-  if shouldUpdateActiveSource.contents {
-    sourceManager.activeSource = sourceRef.contents
-  }
-
   responseRef.contents->Option.getUnsafe
 }