envio 3.0.0-alpha.2 → 3.0.0-alpha.21

package/src/sources/SourceManager.res.mjs

@@ -10,6 +10,7 @@ import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
 import * as FetchState from "../FetchState.res.mjs";
 import * as Prometheus from "../Prometheus.res.mjs";
 import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
 import * as ErrorHandling from "../ErrorHandling.res.mjs";
 import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 
@@ -17,6 +18,34 @@ function getActiveSource(sourceManager) {
   return sourceManager.activeSource;
 }
 
+function getSourceRole(sourceFor, isLive, hasLive) {
+  if (isLive) {
+    switch (sourceFor) {
+      case "Sync" :
+          if (hasLive) {
+            return "Secondary";
+          } else {
+            return "Primary";
+          }
+      case "Fallback" :
+          return "Secondary";
+      case "Live" :
+          return "Primary";
+      
+    }
+  } else {
+    switch (sourceFor) {
+      case "Sync" :
+          return "Primary";
+      case "Fallback" :
+          return "Secondary";
+      case "Live" :
+          return ;
+      
+    }
+  }
+}
+
 function makeGetHeightRetryInterval(initialRetryInterval, backoffMultiplicative, maxRetryInterval) {
   return function (retry) {
     var backoff = retry === 0 ? 1 : Math.imul(retry, backoffMultiplicative);
@@ -24,27 +53,44 @@ function makeGetHeightRetryInterval(initialRetryInterval, backoffMultiplicative,
   };
 }
 
-function make(sources, maxPartitionConcurrency, newBlockFallbackStallTimeoutOpt, stalledPollingIntervalOpt, getHeightRetryIntervalOpt) {
-  var newBlockFallbackStallTimeout = newBlockFallbackStallTimeoutOpt !== undefined ? newBlockFallbackStallTimeoutOpt : 20000;
+function make(sources, maxPartitionConcurrency, isLive, newBlockStallTimeoutOpt, newBlockStallTimeoutLiveOpt, stalledPollingIntervalOpt, recoveryTimeoutOpt, getHeightRetryIntervalOpt) {
+  var newBlockStallTimeout = newBlockStallTimeoutOpt !== undefined ? newBlockStallTimeoutOpt : 60000;
+  var newBlockStallTimeoutLive = newBlockStallTimeoutLiveOpt !== undefined ? newBlockStallTimeoutLiveOpt : 20000;
   var stalledPollingInterval = stalledPollingIntervalOpt !== undefined ? stalledPollingIntervalOpt : 5000;
+  var recoveryTimeout = recoveryTimeoutOpt !== undefined ? recoveryTimeoutOpt : 60000.0;
   var getHeightRetryInterval = getHeightRetryIntervalOpt !== undefined ? getHeightRetryIntervalOpt : makeGetHeightRetryInterval(1000, 2, 60000);
+  var hasLive = sources.some(function (s) {
+        return s.sourceFor === "Live";
+      });
   var source = sources.find(function (source) {
-        return source.sourceFor === "Sync";
+        return getSourceRole(source.sourceFor, isLive, hasLive) === "Primary";
       });
   var initialActiveSource = source !== undefined ? source : Js_exn.raiseError("Invalid configuration, no data-source for historical sync provided");
   Prometheus.IndexingMaxConcurrency.set(maxPartitionConcurrency, initialActiveSource.chain);
   Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
   return {
-          sources: new Set(sources),
+          sourcesState: Belt_Array.map(sources, (function (source) {
+                  return {
+                          source: source,
+                          knownHeight: 0,
+                          unsubscribe: undefined,
+                          pendingHeightResolvers: [],
+                          disabled: false,
+                          lastFailedAt: undefined
+                        };
+                })),
           statusStart: Hrtime.makeTimer(),
           status: "Idle",
           maxPartitionConcurrency: maxPartitionConcurrency,
-          newBlockFallbackStallTimeout: newBlockFallbackStallTimeout,
+          newBlockStallTimeout: newBlockStallTimeout,
+          newBlockStallTimeoutLive: newBlockStallTimeoutLive,
           stalledPollingInterval: stalledPollingInterval,
           getHeightRetryInterval: getHeightRetryInterval,
           activeSource: initialActiveSource,
           waitingForNewBlockStateId: undefined,
-          fetchingPartitionsCount: 0
+          fetchingPartitionsCount: 0,
+          recoveryTimeout: recoveryTimeout,
+          hasLive: hasLive
         };
 }
 
@@ -63,15 +109,15 @@ function trackNewStatus(sourceManager, newStatus) {
         break;
     
   }
-  Prometheus.SafeCounter.incrementMany(promCounter, sourceManager.activeSource.chain, Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(sourceManager.statusStart))));
+  Prometheus.SafeCounter.handleFloat(promCounter, sourceManager.activeSource.chain, Hrtime.toSecondsFloat(Hrtime.timeSince(sourceManager.statusStart)));
   sourceManager.statusStart = Hrtime.makeTimer();
   sourceManager.status = newStatus;
 }
 
-async function fetchNext(sourceManager, fetchState, currentBlockHeight, executeQuery, waitForNewBlock, onNewBlock, stateId) {
-  var queries = FetchState.getNextQuery(fetchState, sourceManager.maxPartitionConcurrency - sourceManager.fetchingPartitionsCount | 0, currentBlockHeight, stateId);
-  if (typeof queries !== "object") {
-    switch (queries) {
+async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, stateId) {
+  var nextQuery = FetchState.getNextQuery(fetchState, sourceManager.maxPartitionConcurrency - sourceManager.fetchingPartitionsCount | 0);
+  if (typeof nextQuery !== "object") {
+    switch (nextQuery) {
       case "WaitingForNewBlock" :
           var waitingStateId = sourceManager.waitingForNewBlockStateId;
           if (waitingStateId !== undefined && waitingStateId >= stateId) {
@@ -79,12 +125,12 @@ async function fetchNext(sourceManager, fetchState, currentBlockHeight, executeQ
           }
           trackNewStatus(sourceManager, "WaitingForNewBlock");
           sourceManager.waitingForNewBlockStateId = stateId;
-          var currentBlockHeight$1 = await waitForNewBlock(currentBlockHeight);
+          var knownHeight = await waitForNewBlock(fetchState.knownHeight);
           var waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
           if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
             trackNewStatus(sourceManager, "Idle");
             sourceManager.waitingForNewBlockStateId = undefined;
-            return onNewBlock(currentBlockHeight$1);
+            return onNewBlock(knownHeight);
           } else {
             return ;
           }
@@ -94,12 +140,12 @@ async function fetchNext(sourceManager, fetchState, currentBlockHeight, executeQ
       
     }
   } else {
-    var queries$1 = queries._0;
-    FetchState.startFetchingQueries(fetchState, queries$1, stateId);
-    sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries$1.length | 0;
+    var queries = nextQuery._0;
+    FetchState.startFetchingQueries(fetchState, queries);
+    sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries.length | 0;
     Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
     trackNewStatus(sourceManager, "Querieng");
-    await Promise.all(Belt_Array.map(queries$1, (function (q) {
+    await Promise.all(Belt_Array.map(queries, (function (q) {
                 var promise = executeQuery(q);
                 promise.then(function (param) {
                       sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount - 1 | 0;
@@ -115,78 +161,241 @@ async function fetchNext(sourceManager, fetchState, currentBlockHeight, executeQ
   }
 }
 
-async function getSourceNewHeight(sourceManager, source, currentBlockHeight, status, logger) {
-  var newHeight = 0;
+function disableSource(sourceManager, sourceState) {
+  if (sourceState.disabled) {
+    return false;
+  }
+  sourceState.disabled = true;
+  var unsubscribe = sourceState.unsubscribe;
+  if (unsubscribe !== undefined) {
+    unsubscribe();
+  }
+  if (sourceState.source.sourceFor === "Live") {
+    var hasOtherLive = sourceManager.sourcesState.some(function (s) {
+          if (s !== sourceState && !s.disabled) {
+            return s.source.sourceFor === "Live";
+          } else {
+            return false;
+          }
+        });
+    sourceManager.hasLive = hasOtherLive;
+  }
+  return true;
+}
+
+async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stallTimeout, isLive, status, logger) {
+  var source = sourceState.source;
+  var initialHeight = sourceState.knownHeight;
+  var newHeight = {
+    contents: initialHeight
+  };
   var retry = 0;
-  while(newHeight <= currentBlockHeight && status.contents !== "Done") {
-    try {
-      var endTimer = Prometheus.SourceGetHeightDuration.startTimer({
-            source: source.name,
-            chainId: source.chain
+  while(newHeight.contents <= knownHeight && status.contents !== "Done") {
+    var match = sourceState.unsubscribe;
+    if (match !== undefined) {
+      var subscriptionPromise = new Promise((function (resolve, _reject) {
+              sourceState.pendingHeightResolvers.push(resolve);
+            }));
+      var pollingFallback = Utils.delay(stallTimeout / 2 | 0).then(async function () {
+            Logging.childTrace(logger, {
+                  msg: "onHeight subscription stale, switching to polling fallback",
+                  source: source.name,
+                  chainId: source.chain
+                });
+            var h = initialHeight;
+            while(h <= knownHeight && newHeight.contents <= initialHeight) {
+              try {
+                h = await source.getHeightOrThrow();
+              }
+              catch (exn){
+                
+              }
+              if (h <= knownHeight && newHeight.contents <= initialHeight) {
+                await Utils.delay(source.pollingInterval);
+              }
+              
+            };
+            return h;
           });
-      var height = await source.getHeightOrThrow();
-      endTimer();
-      newHeight = height;
-      if (height <= currentBlockHeight) {
-        retry = 0;
-        var pollingInterval = status.contents === "Stalled" ? sourceManager.stalledPollingInterval : source.pollingInterval;
-        await Utils.delay(pollingInterval);
+      var height = await Promise.race([
+            subscriptionPromise,
+            pollingFallback
+          ]);
+      if (height > initialHeight) {
+        newHeight.contents = height;
       }
       
-    }
-    catch (raw_exn){
-      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
-      var retryInterval = sourceManager.getHeightRetryInterval(retry);
-      Logging.childTrace(logger, {
-            msg: "Height retrieval from " + source.name + " source failed. Retrying in " + String(retryInterval) + "ms.",
-            source: source.name,
-            err: Utils.prettifyExn(exn)
-          });
-      retry = retry + 1 | 0;
-      await Utils.delay(retryInterval);
+    } else {
+      try {
+        var height$1 = await source.getHeightOrThrow();
+        newHeight.contents = height$1;
+        if (height$1 <= knownHeight) {
+          retry = 0;
+          var createSubscription = source.createHeightSubscription;
+          var exit = 0;
+          if (createSubscription !== undefined && isLive) {
+            var unsubscribe = createSubscription(function (newHeight) {
+                  sourceState.knownHeight = newHeight;
+                  var resolvers = sourceState.pendingHeightResolvers;
+                  sourceState.pendingHeightResolvers = [];
+                  Belt_Array.forEach(resolvers, (function (resolve) {
+                          resolve(newHeight);
+                        }));
+                });
+            sourceState.unsubscribe = unsubscribe;
+          } else {
+            exit = 1;
+          }
+          if (exit === 1) {
+            var pollingInterval = status.contents === "Stalled" ? sourceManager.stalledPollingInterval : source.pollingInterval;
+            await Utils.delay(pollingInterval);
+          }
+          
+        }
+        
+      }
+      catch (raw_exn){
+        var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+        var retryInterval = sourceManager.getHeightRetryInterval(retry);
+        Logging.childTrace(logger, {
+              msg: "Height retrieval from " + source.name + " source failed. Retrying in " + String(retryInterval) + "ms.",
+              source: source.name,
+              err: Utils.prettifyExn(exn)
+            });
+        retry = retry + 1 | 0;
+        await Utils.delay(retryInterval);
+      }
     }
   };
-  Prometheus.SourceHeight.set(source.name, source.chain, newHeight);
-  return newHeight;
+  if (newHeight.contents > initialHeight) {
+    Prometheus.SourceHeight.set(source.name, source.chain, newHeight.contents);
+  }
+  return newHeight.contents;
 }
 
-async function waitForNewBlock(sourceManager, currentBlockHeight) {
+function compareByOldestFailure(a, b) {
+  var match = a.lastFailedAt;
+  var match$1 = b.lastFailedAt;
+  if (match !== undefined) {
+    if (match$1 !== undefined) {
+      if (match < match$1) {
+        return -1;
+      } else if (match > match$1) {
+        return 1;
+      } else {
+        return 0;
+      }
+    } else {
+      return 1;
+    }
+  } else if (match$1 !== undefined) {
+    return -1;
+  } else {
+    return 0;
+  }
+}
+
+function getNextSources(sourceManager, isLive, excludedSources) {
+  var now = Date.now();
+  var workingPrimarySources = [];
+  var allPrimarySources = [];
+  var workingSecondarySources = [];
+  for(var i = 0 ,i_finish = sourceManager.sourcesState.length; i < i_finish; ++i){
+    var sourceState = sourceManager.sourcesState[i];
+    if (!sourceState.disabled) {
+      var isExcluded = excludedSources !== undefined ? Caml_option.valFromOption(excludedSources).has(sourceState) : false;
+      if (!isExcluded) {
+        var failedAt = sourceState.lastFailedAt;
+        var isWorking = failedAt !== undefined ? now - failedAt >= sourceManager.recoveryTimeout : true;
+        var match = getSourceRole(sourceState.source.sourceFor, isLive, sourceManager.hasLive);
+        if (match !== undefined) {
+          if (match === "Primary") {
+            allPrimarySources.push(sourceState);
+            if (isWorking) {
+              workingPrimarySources.push(sourceState);
+            }
+            
+          } else if (isWorking) {
+            workingSecondarySources.push(sourceState);
+          }
+          
+        }
+        
+      }
+      
+    }
+    
+  }
+  if (workingPrimarySources.length !== 0) {
+    return workingPrimarySources;
+  } else if (workingSecondarySources.length !== 0) {
+    return workingSecondarySources;
+  } else {
+    return allPrimarySources.sort(compareByOldestFailure);
+  }
+}
+
+function getNextSource(sourceManager, isLive, excludedSources) {
+  var sources = getNextSources(sourceManager, isLive, excludedSources);
+  var first = Belt_Array.get(sources, 0);
+  if (first === undefined) {
+    return ;
+  }
+  if (first.source === sourceManager.activeSource) {
+    return first;
+  }
+  var result = sources.find(function (s) {
+        return s.source === sourceManager.activeSource;
+      });
+  if (result !== undefined) {
+    if (result === undefined) {
+      return ;
+    } else {
+      return Caml_option.some(result);
+    }
+  } else {
+    return Belt_Array.get(sources, 0);
+  }
+}
+
+async function waitForNewBlock(sourceManager, knownHeight, isLive) {
+  var sourcesState = sourceManager.sourcesState;
   var logger = Logging.createChild({
         chainId: sourceManager.activeSource.chain,
-        currentBlockHeight: currentBlockHeight
+        knownHeight: knownHeight
       });
   Logging.childTrace(logger, "Initiating check for new blocks.");
-  var isInitialHeightFetch = currentBlockHeight === 0;
-  var syncSources = [];
-  var fallbackSources = [];
-  sourceManager.sources.forEach(function (source) {
-        if (source.sourceFor === "Sync" || source.sourceFor === "Live" && !isInitialHeightFetch || source === sourceManager.activeSource) {
-          syncSources.push(source);
-        } else {
-          fallbackSources.push(source);
-        }
-      });
+  var mainSources = getNextSources(sourceManager, isLive, undefined);
   var status = {
     contents: "Active"
   };
-  var match = await Promise.race(Belt_Array.concat(Belt_Array.map(syncSources, (async function (source) {
+  var stallTimeout = isLive ? sourceManager.newBlockStallTimeoutLive : sourceManager.newBlockStallTimeout;
+  var match = await Promise.race(Belt_Array.concat(Belt_Array.map(mainSources, (async function (sourceState) {
                   return [
-                          source,
-                          await getSourceNewHeight(sourceManager, source, currentBlockHeight, status, logger)
+                          sourceState.source,
+                          await getSourceNewHeight(sourceManager, sourceState, knownHeight, stallTimeout, isLive, status, logger)
                         ];
-                })), [Utils.delay(sourceManager.newBlockFallbackStallTimeout).then(function () {
+                })), [Utils.delay(stallTimeout).then(function () {
+                  var fallbackSources = [];
+                  Belt_Array.forEach(sourcesState, (function (sourceState) {
+                          if (!mainSources.includes(sourceState) && Belt_Option.isSome(getSourceRole(sourceState.source.sourceFor, isLive, sourceManager.hasLive))) {
+                            fallbackSources.push(sourceState);
+                            return ;
+                          }
+                          
+                        }));
                   if (status.contents !== "Done") {
                     status.contents = "Stalled";
                     if (fallbackSources.length !== 0) {
-                      Logging.childWarn(logger, "No new blocks detected within " + String(sourceManager.newBlockFallbackStallTimeout / 1000 | 0) + "s. Continuing polling with fallback RPC sources from the configuration.");
+                      Logging.childWarn(logger, "No new blocks detected within " + String(stallTimeout / 1000 | 0) + "s. Continuing polling with secondary RPC sources from the configuration.");
                     } else {
-                      Logging.childWarn(logger, "No new blocks detected within " + String(sourceManager.newBlockFallbackStallTimeout / 1000 | 0) + "s. Polling will continue at a reduced rate. For better reliability, refer to our RPC fallback guide: https://docs.envio.dev/docs/HyperIndex/rpc-sync");
+                      Logging.childWarn(logger, "No new blocks detected within " + String(stallTimeout / 1000 | 0) + "s. Polling will continue at a reduced rate. For better reliability, refer to our RPC fallback guide: https://docs.envio.dev/docs/HyperIndex/rpc-sync");
                     }
                   }
-                  return Promise.race(Belt_Array.map(fallbackSources, (async function (source) {
+                  return Promise.race(Belt_Array.map(fallbackSources, (async function (sourceState) {
                                     return [
-                                            source,
-                                            await getSourceNewHeight(sourceManager, source, currentBlockHeight, status, logger)
+                                            sourceState.source,
+                                            await getSourceNewHeight(sourceManager, sourceState, knownHeight, stallTimeout, isLive, status, logger)
                                           ];
                                   })));
                 })]));
@@ -203,66 +412,38 @@ async function waitForNewBlock(sourceManager, currentBlockHeight) {
   return newBlockHeight;
 }
 
-function getNextSyncSource(sourceManager, initialSource, currentSource, attemptFallbacksOpt) {
-  var attemptFallbacks = attemptFallbacksOpt !== undefined ? attemptFallbacksOpt : false;
-  var before = [];
-  var after = [];
-  var hasActive = {
-    contents: false
-  };
-  sourceManager.sources.forEach(function (source) {
-        if (source === currentSource) {
-          hasActive.contents = true;
-          return ;
-        }
-        var match = source.sourceFor;
-        var tmp;
-        switch (match) {
-          case "Sync" :
-              tmp = true;
-              break;
-          case "Fallback" :
-          case "Live" :
-              tmp = attemptFallbacks || source === initialSource;
-              break;
-          
-        }
-        if (tmp) {
-          (
-              hasActive.contents ? after : before
-            ).push(source);
-          return ;
-        }
-        
-      });
-  var s = Belt_Array.get(after, 0);
-  if (s !== undefined) {
-    return s;
-  }
-  var s$1 = Belt_Array.get(before, 0);
-  if (s$1 !== undefined) {
-    return s$1;
-  } else {
-    return currentSource;
-  }
-}
-
-async function executeQuery(sourceManager, query, currentBlockHeight) {
-  var match = query.target;
-  var toBlockRef;
-  toBlockRef = typeof match !== "object" ? undefined : (
-      match.TAG === "EndBlock" ? match.toBlock : match.toBlock
-    );
+async function executeQuery(sourceManager, query, knownHeight, isLive) {
+  var excludedSourcesRef;
+  var toBlockRef = query.toBlock;
   var responseRef;
   var retryRef = 0;
-  var initialSource = sourceManager.activeSource;
-  var sourceRef = initialSource;
-  var shouldUpdateActiveSource = false;
   while(Belt_Option.isNone(responseRef)) {
-    var source = sourceRef;
+    var s = getNextSource(sourceManager, isLive, excludedSourcesRef);
+    var sourceState;
+    if (s !== undefined) {
+      if (s.source !== sourceManager.activeSource) {
+        var logger = Logging.createChild({
+              chainId: sourceManager.activeSource.chain
+            });
+        Logging.childInfo(logger, {
+              msg: "Switching data-source",
+              source: s.source.name,
+              previousSource: sourceManager.activeSource.name,
+              fromBlock: query.fromBlock
+            });
+      }
+      sourceState = s;
+    } else {
+      var logger$1 = Logging.createChild({
+            chainId: sourceManager.activeSource.chain
+          });
+      sourceState = ErrorHandling.mkLogAndRaise(logger$1, "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.", null);
+    }
+    sourceManager.activeSource = sourceState.source;
+    var source = sourceState.source;
     var toBlock = toBlockRef;
     var retry = retryRef;
-    var logger = Logging.createChild({
+    var logger$2 = Logging.createChild({
           chainId: source.chain,
           logType: "Block Range Query",
           partitionId: query.partitionId,
@@ -273,13 +454,14 @@ async function executeQuery(sourceManager, query, currentBlockHeight) {
           retry: retry
         });
     try {
-      var response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, query.indexingContracts, currentBlockHeight, query.partitionId, query.selection, retry, logger);
-      Logging.childTrace(logger, {
+      var response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, query.indexingContracts, knownHeight, query.partitionId, query.selection, retry, logger$2);
+      Logging.childTrace(logger$2, {
             msg: "Fetched block range from server",
             toBlock: response.latestFetchedBlockNumber,
             numEvents: response.parsedQueueItems.length,
             stats: response.stats
           });
+      sourceState.lastFailedAt = undefined;
       responseRef = response;
     }
     catch (raw_error){
@@ -293,13 +475,13 @@ async function executeQuery(sourceManager, query, currentBlockHeight) {
               exit = 1;
               break;
           case "FailedGettingItems" :
-              var match$1 = error$1.retry;
+              var match = error$1.retry;
               var attemptedToBlock = error$1.attemptedToBlock;
               var exn = error$1.exn;
-              switch (match$1.TAG) {
+              switch (match.TAG) {
                 case "WithSuggestedToBlock" :
-                    var toBlock$1 = match$1.toBlock;
-                    Logging.childTrace(logger, {
+                    var toBlock$1 = match.toBlock;
+                    Logging.childTrace(logger$2, {
                           msg: "Failed getting data for the block range. Immediately retrying with the suggested block range from response.",
                           toBlock: attemptedToBlock,
                           suggestedToBlock: toBlock$1
@@ -308,47 +490,53 @@ async function executeQuery(sourceManager, query, currentBlockHeight) {
                     retryRef = 0;
                     break;
                 case "WithBackoff" :
-                    var backoffMillis = match$1.backoffMillis;
-                    var attemptFallbacks = retry >= 10;
-                    var nextSource = !(retry === 0 || retry === 1) && retry % 2 === 0 ? getNextSyncSource(sourceManager, initialSource, source, attemptFallbacks) : source;
+                    var backoffMillis = match.backoffMillis;
                     var log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
-                    log(logger, {
-                          msg: match$1.message,
+                    log(logger$2, {
+                          msg: match.message,
                           toBlock: attemptedToBlock,
                           backOffMilliseconds: backoffMillis,
                           retry: retry,
                           err: Utils.prettifyExn(exn)
                         });
-                    var shouldSwitch = nextSource !== source;
+                    var shouldSwitch = retry === 0 || retry === 1 ? false : retry % 2 === 0;
                     if (shouldSwitch) {
-                      Logging.childInfo(logger, {
-                            msg: "Switching to another data-source",
-                            source: nextSource.name
-                          });
-                      sourceRef = nextSource;
-                      shouldUpdateActiveSource = true;
+                      var now = Date.now();
+                      sourceState.lastFailedAt = now;
+                      var nextSource = getNextSource(sourceManager, isLive, excludedSourcesRef);
+                      var hasWorkingAlternative;
+                      if (nextSource !== undefined) {
+                        var failedAt = nextSource.lastFailedAt;
+                        hasWorkingAlternative = failedAt !== undefined ? now - failedAt >= sourceManager.recoveryTimeout : true;
+                      } else {
+                        hasWorkingAlternative = false;
+                      }
+                      if (!hasWorkingAlternative) {
+                        await Utils.delay(backoffMillis < 60000 ? backoffMillis : 60000);
+                      }
+                      
                     } else {
                       await Utils.delay(backoffMillis < 60000 ? backoffMillis : 60000);
                     }
                     retryRef = retryRef + 1 | 0;
                     break;
                 case "ImpossibleForTheQuery" :
-                    var nextSource$1 = getNextSyncSource(sourceManager, initialSource, source, true);
-                    var hasAnotherSource = nextSource$1 !== initialSource;
-                    Logging.childWarn(logger, {
-                          msg: match$1.message + (
-                            hasAnotherSource ? " - Attempting to another source" : ""
-                          ),
+                    var s$1 = excludedSourcesRef;
+                    var excludedSources;
+                    if (s$1 !== undefined) {
+                      excludedSources = Caml_option.valFromOption(s$1);
+                    } else {
+                      var s$2 = new Set();
+                      excludedSourcesRef = Caml_option.some(s$2);
+                      excludedSources = s$2;
+                    }
+                    excludedSources.add(sourceState);
+                    Logging.childWarn(logger$2, {
+                          msg: match.message + " - Attempting another source",
                           toBlock: attemptedToBlock,
                           err: Utils.prettifyExn(exn)
                         });
-                    if (hasAnotherSource) {
-                      sourceRef = nextSource$1;
-                      shouldUpdateActiveSource = false;
-                      retryRef = 0;
-                    } else {
-                      ErrorHandling.mkLogAndRaise(logger, "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.", null);
-                    }
+                    retryRef = 0;
                     break;
                 
               }
@@ -356,15 +544,14 @@ async function executeQuery(sourceManager, query, currentBlockHeight) {
           
         }
         if (exit === 1) {
-          var nextSource$2 = getNextSyncSource(sourceManager, initialSource, source, undefined);
-          var notAlreadyDeleted = sourceManager.sources.delete(source);
-          if (notAlreadyDeleted) {
+          var notAlreadyDisabled = disableSource(sourceManager, sourceState);
+          if (notAlreadyDisabled) {
             switch (error$1.TAG) {
               case "UnsupportedSelection" :
-                  Logging.childError(logger, error$1.message);
+                  Logging.childError(logger$2, error$1.message);
                   break;
               case "FailedGettingFieldSelection" :
-                  Logging.childError(logger, {
+                  Logging.childError(logger$2, {
                         msg: error$1.message,
                         err: Utils.prettifyExn(error$1.exn),
                         blockNumber: error$1.blockNumber,
@@ -376,31 +563,19 @@ async function executeQuery(sourceManager, query, currentBlockHeight) {
               
             }
           }
-          if (nextSource$2 === source) {
-            ErrorHandling.mkLogAndRaise(logger, "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.", null);
-          } else {
-            Logging.childInfo(logger, {
-                  msg: "Switching to another data-source",
-                  source: nextSource$2.name
-                });
-            sourceRef = nextSource$2;
-            shouldUpdateActiveSource = true;
-            retryRef = 0;
-          }
+          retryRef = 0;
         }
         
       } else {
-        ErrorHandling.mkLogAndRaise(logger, "Failed to fetch block Range", error);
+        ErrorHandling.mkLogAndRaise(logger$2, "Failed to fetch block Range", error);
       }
     }
   };
-  if (shouldUpdateActiveSource) {
-    sourceManager.activeSource = sourceRef;
-  }
   return responseRef;
 }
 
 export {
+  getSourceRole ,
   make ,
   getActiveSource ,
   fetchNext ,