envio 3.13.0-alpha.0 → 3.13.0

package/src/sources/SimulateSource.res

@@ -17,7 +17,7 @@ let make = (
     pollingInterval: 0,
     getBlockHashes: (~blockNumbers as _, ~logger as _) => {
       Promise.resolve({
-        Source.result: Ok(BlockStore.fromJs([], ~ecosystem, ~shouldChecksum=false)),
+        Source.result: Ok(BlockStore.fromJs([], ~ecosystem)),
         requestStats: [],
       })
     },
@@ -112,10 +112,7 @@ let make = (
         // EVM/Fuel keep transaction and block inline. SVM simulate returns a
         // store page so materialize + activity attach run the same path as HyperSync.
         transactionStore,
-        blockStore: blockStore->Option.getOr(
-          BlockStore.fromJs([], ~ecosystem, ~shouldChecksum=false),
-        ),
-        fromBlockQueried: fromBlock,
+        blockStore: blockStore->Option.getOr(BlockStore.fromJs([], ~ecosystem)),
         latestFetchedBlockNumber: toBlockQueried,
         stats: {
           totalTimeElapsed: 0.,

package/src/sources/SimulateSource.res.mjs

@@ -19,7 +19,7 @@ function make(items, endBlock, chainId, addressStore, ecosystemOpt, transactionS
     getBlockHashes: (param, param$1) => Promise.resolve({
       result: {
         TAG: "Ok",
-        _0: BlockStore.fromJs([], ecosystem, false)
+        _0: BlockStore.fromJs([], ecosystem)
       },
       requestStats: []
     }),
@@ -80,8 +80,7 @@ function make(items, endBlock, chainId, addressStore, ecosystemOpt, transactionS
         knownHeight: reportedHeight,
         parsedQueueItems: parsedQueueItems,
         transactionStore: transactionStore,
-        blockStore: Stdlib_Option.getOr(blockStore, BlockStore.fromJs([], ecosystem, false)),
-        fromBlockQueried: fromBlock,
+        blockStore: Stdlib_Option.getOr(blockStore, BlockStore.fromJs([], ecosystem)),
         latestFetchedBlockNumber: toBlockQueried,
         stats: {
           "total time elapsed (s)": 0

package/src/sources/Source.res

@@ -78,14 +78,13 @@ type blockRangeFetchResponse = {
   parsedQueueItems: array<Internal.item>,
   // Page of transactions for this response's items, keyed by (blockNumber,
   // transactionIndex); merged into the chain's store on apply. `None` for
-  // sources that keep the transaction inline on the payload (RPC/Fuel/Simulate).
+  // sources that keep the transaction inline on the payload (Fuel/Simulate).
   transactionStore: option<TransactionStore.t>,
   // Page of blocks observed while fetching this range, keyed by block number;
   // merged into the chain's store on apply, where its hashes drive reorg
-  // detection. Sources that keep the block inline on the payload (RPC/Simulate)
-  // contribute hash-only rows built from the block hashes they saw.
+  // detection. A source that keeps the block inline on the payload (Simulate)
+  // contributes hash-only rows built from the block hashes it saw.
   blockStore: BlockStore.t,
-  fromBlockQueried: int,
   latestFetchedBlockNumber: int,
   stats: blockRangeFetchStats,
   requestStats: array<requestStat>,
@@ -122,13 +121,35 @@ type getItemsRetry =
 type rateLimited = {resetMs: int, requestStats: array<requestStat>}
 exception RateLimited(rateLimited)
 
+// The two failures that can follow requests carry their timings, so a page
+// that ends in a retry still counts towards the source's metrics — the same
+// contract `RateLimited` and `SourceBehindHead` keep.
 type getItemsError =
   | UnsupportedSelection({message: string})
-  | FailedGettingFieldSelection({exn: exn, blockNumber: int, logIndex: int, message: string})
-  | FailedGettingItems({exn: exn, attemptedToBlock: int, retry: getItemsRetry})
+  | FailedGettingFieldSelection({
+      // The cause, where the source has one beyond `message`.
+      exn?: exn,
+      blockNumber: int,
+      message: string,
+      requestStats: array<requestStat>,
+    })
+  | FailedGettingItems({
+      // The cause, where the source has one beyond the retry's own message.
+      exn?: exn,
+      attemptedToBlock: int,
+      retry: getItemsRetry,
+      requestStats: array<requestStat>,
+    })
 
 exception GetItemsError(getItemsError)
 
+let getItemsErrorRequestStats = (error: getItemsError) =>
+  switch error {
+  // Raised before anything is requested.
+  | UnsupportedSelection(_) => []
+  | FailedGettingFieldSelection({requestStats}) | FailedGettingItems({requestStats}) => requestStats
+  }
+
 type sourceFor = Sync | Fallback | Realtime
 
 // Why a height subscription's connection ended. Closed, so a consumer decides
@@ -222,11 +243,12 @@ type t = {
     ~onHeight: int => unit,
     ~onStatus: heightSubscriptionStatus => unit,
   ) => unit => unit,
-  // Invoked when a reorg or internally inconsistent response means local state
-  // may point at an orphaned chain (e.g. the RPC block cache): drop all of it.
-  // Deliberately takes no rollback target — the deepest reorged block isn't
-  // known until the depth search runs, and that search reads back through this
-  // very state, so pruning relative to a target would keep exactly the entries
-  // that make it answer wrong.
+  // Invoked when a reorg or an internally inconsistent response means what the
+  // source is holding may describe an orphaned chain — for RPC, the reads still
+  // in flight, which anything asking afterwards would otherwise join. The
+  // stores are rolled back separately; this is only about what the source
+  // itself carries. Deliberately takes no rollback target: the deepest reorged
+  // block isn't known until the depth search runs, and that search reads back
+  // through this very state.
   onReorg?: unit => unit,
 }

package/src/sources/Source.res.mjs

@@ -50,6 +50,16 @@ let RateLimited = /* @__PURE__ */Primitive_exceptions.create("Source.RateLimited
 
 let GetItemsError = /* @__PURE__ */Primitive_exceptions.create("Source.GetItemsError");
 
+function getItemsErrorRequestStats(error) {
+  switch (error.TAG) {
+    case "UnsupportedSelection" :
+      return [];
+    case "FailedGettingFieldSelection" :
+    case "FailedGettingItems" :
+      return error.requestStats;
+  }
+}
+
 function downReasonLabel(reason) {
   if (typeof reason === "object") {
     return `http-` + reason._0.toString();
@@ -82,6 +92,7 @@ export {
   SourceBehindHead,
   RateLimited,
   GetItemsError,
+  getItemsErrorRequestStats,
   downReasonLabel,
 }
 /* nativeFailurePayloadSchema Not a pure module */

package/src/sources/SourceManager.res

@@ -84,6 +84,10 @@ type t = {
   // Should take into consideration partitions fetching for previous states (before rollback)
   mutable fetchingPartitionsCount: int,
   recoveryTimeout: float,
+  // Why a source was last disabled for good. A disabled source is never
+  // retried, so by the time the manager runs out of sources this is the reason
+  // it did — and the only thing that tells a caller what to change.
+  mutable disableReason: option<string>,
   mutable hasRealtime: bool,
   mutable committedRateLimitTimeMs: float,
   mutable rateLimitWaiters: int,
@@ -386,6 +390,7 @@ let make = (
     idleSeconds: 0.,
     waitingForNewBlockSeconds: 0.,
     queryingSeconds: 0.,
+    disableReason: None,
     hasRealtime,
     committedRateLimitTimeMs: 0.0,
     rateLimitWaiters: 0,
@@ -637,10 +642,20 @@ let retryBehindHead = async (
   )
 }
 
-// A source that keeps contradicting itself is not mid-reorg, it is broken, and
-// no amount of retrying moves the chain forward. Roughly five minutes of the
-// backoff schedule below.
-let inconsistentResponseStallRetries = 13
+// A query that has failed this many times running has outlived every failover
+// the schedule above can offer: it has rotated through the chain's sources
+// several times over and none of them answered. Whatever the reported cause,
+// the chain is no longer making progress and the operator has to hear about it.
+let stallRetries = 13
+
+// Which level a retry is reported at, and what it says once the retries have
+// outlasted the rotation.
+let stalledLog = (~retry, ~warnAfter, ~msg, ~stalled) =>
+  if retry >= stallRetries {
+    (Logging.childError, msg ++ stalled)
+  } else {
+    (retry >= warnAfter ? Logging.childWarn : Logging.childTrace, msg)
+  }
 
 // The response contradicted itself (the same block twice with different hashes,
 // or a requested hash missing). It may be a reorg mid-request, so refetch before
@@ -657,14 +672,12 @@ let retryInconsistentResponse = async (
 ) => {
   let backoffMillis = retry->retryBackoffMillis
   let msg = `Received a partial indicator of a possible reorg from the ${sourceState.source.name} source while fetching ${method}. Retrying the request to better identify whether a reorg happened.`
-  let (log, msg) = if retry >= inconsistentResponseStallRetries {
-    (
-      Logging.childError,
-      msg ++ " It has disagreed with itself on every attempt for several minutes now, so this chain has stopped making progress - the endpoint is likely serving blocks and logs from nodes on different chains.",
-    )
-  } else {
-    (retry >= 2 ? Logging.childWarn : Logging.childTrace, msg)
-  }
+  let (log, msg) = stalledLog(
+    ~retry,
+    ~warnAfter=2,
+    ~msg,
+    ~stalled=" It has disagreed with itself on every attempt for several minutes now, so this chain has stopped making progress - the endpoint is likely serving blocks and logs from nodes on different chains.",
+  )
   logger->log({
     "msg": msg,
     "method": method,
@@ -686,6 +699,45 @@ let retryInconsistentResponse = async (
   )
 }
 
+// The source could not serve the page and reported nothing that changes the
+// query, so the only move left is to wait and let the schedule try another
+// source. Escalates once the retries have outlasted every failover.
+let retryFailedPage = async (
+  sourceManager: t,
+  sourceState: sourceState,
+  ~retry,
+  ~isRealtime,
+  ~logger: Pino.t,
+  ~msg: string,
+  ~attemptedToBlock: int,
+  ~backoffMillis: int,
+  ~minBackoffMillis=0,
+  ~err: option<exn>,
+  ~excludedSources=?,
+) => {
+  let (log, msg) = stalledLog(
+    ~retry,
+    ~warnAfter=4,
+    ~msg,
+    ~stalled=" No source available to this chain has served this query on any attempt since, so indexing has stopped making progress.",
+  )
+  logger->log({
+    "msg": msg,
+    "toBlock": attemptedToBlock,
+    "backOffMilliseconds": backoffMillis,
+    "retry": retry,
+    "err": err->Option.map(Utils.prettifyExn),
+  })
+  await sourceManager->backoffBeforeRetry(
+    sourceState,
+    ~retry,
+    ~isRealtime,
+    ~backoffMillis,
+    ~minBackoffMillis,
+    ~excludedSources?,
+  )
+}
+
 // Polls for a block height greater than the given block number to ensure a new block is available for indexing.
 /*
 Resolves at the first height above `knownHeight` that any eligible source
@@ -895,7 +947,13 @@ let executeQuery = async (
   ~knownHeight,
   ~isRealtime,
 ) => {
-  let noSourcesError = "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team."
+  // Read when the manager actually runs out of sources, not on the way in: the
+  // reason is recorded by the failure that disabled the last one.
+  let noSourcesError = () => switch sourceManager.disableReason {
+  | Some(reason) =>
+    `The indexer doesn't have data-sources which can continue fetching. The last one was disabled because ${reason}`
+  | None => "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team."
+  }
 
   // Sources where the query is impossible - lazily allocated, excluded for the duration of this query
   let excludedSourcesRef = ref(None)
@@ -923,7 +981,7 @@ let executeQuery = async (
       s
     | None =>
       let logger = Logging.createChild(~params={"chainId": sourceManager.activeSource.chainId})
-      %raw(`null`)->ErrorHandling.mkLogAndRaise(~logger, ~msg=noSourcesError)
+      %raw(`null`)->ErrorHandling.mkLogAndRaise(~logger, ~msg=noSourcesError())
     }
     sourceManager.activeSource = sourceState.source
     let source = sourceState.source
@@ -999,6 +1057,9 @@ let executeQuery = async (
       retryRef := retryRef.contents + 1
 
     | Source.GetItemsError(error) =>
+      // A page that ends in a retry still made requests; count them, as the
+      // rate-limit and behind-head arms above do.
+      sourceState->recordRequestStats(error->Source.getItemsErrorRequestStats)
       switch error {
       | UnsupportedSelection(_)
       | FailedGettingFieldSelection(_) => {
@@ -1010,13 +1071,15 @@ let executeQuery = async (
           // failing at the same time. Log only once
           if notAlreadyDisabled {
             switch error {
-            | UnsupportedSelection({message}) => logger->Logging.childError(message)
-            | FailedGettingFieldSelection({exn, message, blockNumber, logIndex}) =>
+            | UnsupportedSelection({message}) =>
+              sourceManager.disableReason = Some(message)
+              logger->Logging.childError(message)
+            | FailedGettingFieldSelection({?exn, message, blockNumber}) =>
+              sourceManager.disableReason = Some(message)
               logger->Logging.childError({
                 "msg": message,
-                "err": exn->Utils.prettifyExn,
+                "err": exn->Option.map(Utils.prettifyExn),
                 "blockNumber": blockNumber,
-                "logIndex": logIndex,
               })
             | _ => ()
             }
@@ -1024,6 +1087,26 @@ let executeQuery = async (
 
           retryRef := 0
         }
+      // A suggestion that doesn't narrow the range asks for the query that just
+      // failed. Retrying it unchanged and without a wait makes no progress and
+      // never counts against the source, so pace it like any other failure and
+      // let the schedule move on to one that can answer.
+      | FailedGettingItems({?exn, attemptedToBlock, retry: WithSuggestedToBlock({toBlock})})
+        if toBlock >= attemptedToBlock =>
+        await sourceManager->retryFailedPage(
+          sourceState,
+          ~retry,
+          ~isRealtime,
+          ~logger,
+          ~msg=`The ${source.name} source rejected the block range and suggested #${toBlock->Int.toString}, which is no narrower than the #${attemptedToBlock->Int.toString} it just refused.`,
+          ~attemptedToBlock,
+          ~backoffMillis=retry->retryBackoffMillis,
+          ~minBackoffMillis=minRecoverableBackoffMillis,
+          ~err=exn,
+          ~excludedSources=?excludedSourcesRef.contents,
+        )
+        retryRef := retryRef.contents + 1
+
       | FailedGettingItems({attemptedToBlock, retry: WithSuggestedToBlock({toBlock})}) =>
         logger->Logging.childTrace({
           "msg": "Failed getting data for the block range. Immediately retrying with the suggested block range from response.",
@@ -1031,8 +1114,10 @@ let executeQuery = async (
           "suggestedToBlock": toBlock,
         })
         toBlockRef := Some(toBlock)
+        // The next attempt asks a strictly smaller question, so it starts a
+        // fresh schedule rather than continuing this one.
         retryRef := 0
-      | FailedGettingItems({exn, attemptedToBlock, retry: ImpossibleForTheQuery({message})}) =>
+      | FailedGettingItems({?exn, attemptedToBlock, retry: ImpossibleForTheQuery({message})}) =>
         // Don't set lastFailedAt - the source isn't broken, the query just can't work on it
         let excludedSources = switch excludedSourcesRef.contents {
         | Some(s) => s
@@ -1046,25 +1131,20 @@ let executeQuery = async (
         logger->Logging.childWarn({
           "msg": message ++ " - Attempting another source",
           "toBlock": attemptedToBlock,
-          "err": exn->Utils.prettifyExn,
+          "err": exn->Option.map(Utils.prettifyExn),
         })
         retryRef := 0
 
-      | FailedGettingItems({exn, attemptedToBlock, retry: WithBackoff({message, backoffMillis})}) =>
-        // Start displaying warnings after 4 failures
-        let log = retry >= 4 ? Logging.childWarn : Logging.childTrace
-        logger->log({
-          "msg": message,
-          "toBlock": attemptedToBlock,
-          "backOffMilliseconds": backoffMillis,
-          "retry": retry,
-          "err": exn->Utils.prettifyExn,
-        })
-        await sourceManager->backoffBeforeRetry(
+      | FailedGettingItems({?exn, attemptedToBlock, retry: WithBackoff({message, backoffMillis})}) =>
+        await sourceManager->retryFailedPage(
           sourceState,
           ~retry,
           ~isRealtime,
+          ~logger,
+          ~msg=message,
+          ~attemptedToBlock,
           ~backoffMillis,
+          ~err=exn,
           ~excludedSources=?excludedSourcesRef.contents,
         )
         retryRef := retryRef.contents + 1

package/src/sources/SourceManager.res.mjs

@@ -311,6 +311,7 @@ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeout
     waitingLogged: false,
     fetchingPartitionsCount: 0,
     recoveryTimeout: recoveryTimeout,
+    disableReason: undefined,
     hasRealtime: hasRealtime,
     committedRateLimitTimeMs: 0.0,
     rateLimitWaiters: 0,
@@ -509,16 +510,24 @@ async function retryBehindHead(sourceManager, sourceState, retry, isRealtime, lo
   return await backoffBeforeRetry(sourceManager, sourceState, retry, isRealtime, backoffMillis, 50, excludedSources);
 }
 
-async function retryInconsistentResponse(sourceManager, sourceState, retry, isRealtime, logger, method, err, excludedSources) {
-  let backoffMillis = Utils.expBackoff(100, retry, 60000);
-  let msg = `Received a partial indicator of a possible reorg from the ` + sourceState.source.name + ` source while fetching ` + method + `. Retrying the request to better identify whether a reorg happened.`;
-  let match = retry >= 13 ? [
+function stalledLog(retry, warnAfter, msg, stalled) {
+  if (retry >= 13) {
+    return [
       Logging.childError,
-      msg + " It has disagreed with itself on every attempt for several minutes now, so this chain has stopped making progress - the endpoint is likely serving blocks and logs from nodes on different chains."
-    ] : [
-      retry >= 2 ? Logging.childWarn : Logging.childTrace,
+      msg + stalled
+    ];
+  } else {
+    return [
+      retry >= warnAfter ? Logging.childWarn : Logging.childTrace,
       msg
     ];
+  }
+}
+
+async function retryInconsistentResponse(sourceManager, sourceState, retry, isRealtime, logger, method, err, excludedSources) {
+  let backoffMillis = Utils.expBackoff(100, retry, 60000);
+  let msg = `Received a partial indicator of a possible reorg from the ` + sourceState.source.name + ` source while fetching ` + method + `. Retrying the request to better identify whether a reorg happened.`;
+  let match = stalledLog(retry, 2, msg, " It has disagreed with itself on every attempt for several minutes now, so this chain has stopped making progress - the endpoint is likely serving blocks and logs from nodes on different chains.");
   match[0](logger, {
     msg: match[1],
     method: method,
@@ -530,6 +539,19 @@ async function retryInconsistentResponse(sourceManager, sourceState, retry, isRe
   return await backoffBeforeRetry(sourceManager, sourceState, retry, isRealtime, backoffMillis, 50, excludedSources);
 }
 
+async function retryFailedPage(sourceManager, sourceState, retry, isRealtime, logger, msg, attemptedToBlock, backoffMillis, minBackoffMillisOpt, err, excludedSources) {
+  let minBackoffMillis = minBackoffMillisOpt !== undefined ? minBackoffMillisOpt : 0;
+  let match = stalledLog(retry, 4, msg, " No source available to this chain has served this query on any attempt since, so indexing has stopped making progress.");
+  match[0](logger, {
+    msg: match[1],
+    toBlock: attemptedToBlock,
+    backOffMilliseconds: backoffMillis,
+    retry: retry,
+    err: Stdlib_Option.map(err, Utils.prettifyExn)
+  });
+  return await backoffBeforeRetry(sourceManager, sourceState, retry, isRealtime, backoffMillis, minBackoffMillis, excludedSources);
+}
+
 function waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling) {
   let sourcesState = sourceManager.sourcesState;
   let logger = Logging.createChild({
@@ -650,6 +672,14 @@ function waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling)
 }
 
 async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
+  let noSourcesError = () => {
+    let reason = sourceManager.disableReason;
+    if (reason !== undefined) {
+      return `The indexer doesn't have data-sources which can continue fetching. The last one was disabled because ` + reason;
+    } else {
+      return "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.";
+    }
+  };
   let excludedSourcesRef;
   let toBlockRef = query.toBlock;
   let responseRef;
@@ -674,7 +704,7 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
       let logger$1 = Logging.createChild({
         chainId: sourceManager.activeSource.chainId
       });
-      sourceState = ErrorHandling.mkLogAndRaise(logger$1, "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.", null);
+      sourceState = ErrorHandling.mkLogAndRaise(logger$1, noSourcesError(), null);
     }
     sourceManager.activeSource = sourceState.source;
     let source = sourceState.source;
@@ -713,6 +743,8 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
         retryRef = retryRef + 1 | 0;
       } else if (err.RE_EXN_ID === Source.GetItemsError) {
         let error = err._1;
+        let requestStats = Source.getItemsErrorRequestStats(error);
+        recordStatsInto(sourceState.requestStats, requestStats);
         let exit = 0;
         switch (error.TAG) {
           case "UnsupportedSelection" :
@@ -726,25 +758,21 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
             switch (match$1.TAG) {
               case "WithSuggestedToBlock" :
                 let toBlock$1 = match$1.toBlock;
-                Logging.childTrace(logger$2, {
-                  msg: "Failed getting data for the block range. Immediately retrying with the suggested block range from response.",
-                  toBlock: attemptedToBlock,
-                  suggestedToBlock: toBlock$1
-                });
-                toBlockRef = toBlock$1;
-                retryRef = 0;
+                if (toBlock$1 >= attemptedToBlock) {
+                  await retryFailedPage(sourceManager, sourceState, retry, isRealtime, logger$2, `The ` + source.name + ` source rejected the block range and suggested #` + toBlock$1.toString() + `, which is no narrower than the #` + attemptedToBlock.toString() + ` it just refused.`, attemptedToBlock, Utils.expBackoff(100, retry, 60000), 50, exn, excludedSourcesRef);
+                  retryRef = retryRef + 1 | 0;
+                } else {
+                  Logging.childTrace(logger$2, {
+                    msg: "Failed getting data for the block range. Immediately retrying with the suggested block range from response.",
+                    toBlock: attemptedToBlock,
+                    suggestedToBlock: toBlock$1
+                  });
+                  toBlockRef = toBlock$1;
+                  retryRef = 0;
+                }
                 break;
               case "WithBackoff" :
-                let backoffMillis = match$1.backoffMillis;
-                let log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
-                log(logger$2, {
-                  msg: match$1.message,
-                  toBlock: attemptedToBlock,
-                  backOffMilliseconds: backoffMillis,
-                  retry: retry,
-                  err: Utils.prettifyExn(exn)
-                });
-                await backoffBeforeRetry(sourceManager, sourceState, retry, isRealtime, backoffMillis, undefined, excludedSourcesRef);
+                await retryFailedPage(sourceManager, sourceState, retry, isRealtime, logger$2, match$1.message, attemptedToBlock, match$1.backoffMillis, undefined, exn, excludedSourcesRef);
                 retryRef = retryRef + 1 | 0;
                 break;
               case "ImpossibleForTheQuery" :
@@ -761,7 +789,7 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
                 Logging.childWarn(logger$2, {
                   msg: match$1.message + " - Attempting another source",
                   toBlock: attemptedToBlock,
-                  err: Utils.prettifyExn(exn)
+                  err: Stdlib_Option.map(exn, Utils.prettifyExn)
                 });
                 retryRef = 0;
                 break;
@@ -773,14 +801,17 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
           if (notAlreadyDisabled) {
             switch (error.TAG) {
               case "UnsupportedSelection" :
-                Logging.childError(logger$2, error.message);
+                let message = error.message;
+                sourceManager.disableReason = message;
+                Logging.childError(logger$2, message);
                 break;
               case "FailedGettingFieldSelection" :
+                let message$1 = error.message;
+                sourceManager.disableReason = message$1;
                 Logging.childError(logger$2, {
-                  msg: error.message,
-                  err: Utils.prettifyExn(error.exn),
-                  blockNumber: error.blockNumber,
-                  logIndex: error.logIndex
+                  msg: message$1,
+                  err: Stdlib_Option.map(error.exn, Utils.prettifyExn),
+                  blockNumber: error.blockNumber
                 });
                 break;
               case "FailedGettingItems" :
@@ -863,6 +894,8 @@ async function getBlockHashes(sourceManager, blockNumbers, isRealtime) {
   return responseRef;
 }
 
+let stallRetries = 13;
+
 export {
   getSourceRole,
   make,
@@ -877,6 +910,7 @@ export {
   onReorg,
   dispatch,
   waitForNewBlock,
+  stallRetries,
   executeQuery,
   makeGetHeightRetryInterval,
   getBlockHashes,

package/src/sources/SourceManager.resi

@@ -74,6 +74,10 @@ let waitForNewBlock: (
   ~reducedPolling: bool,
 ) => promise<int>
 
+// The retry at which a query that no source can answer stops being a hiccup
+// and gets reported as a stalled chain.
+let stallRetries: int
+
 let executeQuery: (
   t,
   ~query: FetchState.query,

package/src/sources/StartBlockResolver.res

@@ -14,12 +14,15 @@
 // sources the chain goes on to index with; those are built later, once
 // registrations exist.
 let makeProbeSources = (chainConfig: Config.chain, ~lowercaseAddresses): array<Source.t> => {
-  let addressStore = AddressStore.make(
-    ~ecosystem=chainConfig.ecosystem,
-    ~shouldChecksum=!lowercaseAddresses,
-    ~contracts=[],
+  let addressStore = AddressStore.make(~ecosystem=chainConfig.ecosystem, ~contracts=[])
+  ChainSources.make(
+    ~chainConfig,
+    ~onEventRegistrations=[],
+    ~addressStore,
+    ~lowercaseAddresses,
+    ~blockStore=BlockStore.make(~ecosystem=chainConfig.ecosystem),
+    ~transactionStore=TransactionStore.make(~ecosystem=chainConfig.ecosystem),
   )
-  ChainSources.make(~chainConfig, ~onEventRegistrations=[], ~addressStore, ~lowercaseAddresses)
 }
 
 // How long to keep asking a chain that won't answer.

package/src/sources/StartBlockResolver.res.mjs

@@ -3,16 +3,18 @@
 import * as Utils from "../Utils.res.mjs";
 import * as ChainId from "../ChainId.res.mjs";
 import * as Logging from "../Logging.res.mjs";
+import * as BlockStore from "./BlockStore.res.mjs";
 import * as AddressStore from "./AddressStore.res.mjs";
 import * as ChainSources from "./ChainSources.res.mjs";
 import * as SourceManager from "./SourceManager.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as TransactionStore from "./TransactionStore.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 function makeProbeSources(chainConfig, lowercaseAddresses) {
-  let addressStore = AddressStore.make(chainConfig.ecosystem, !lowercaseAddresses, []);
-  return ChainSources.make(chainConfig, [], addressStore, lowercaseAddresses);
+  let addressStore = AddressStore.make(chainConfig.ecosystem, []);
+  return ChainSources.make(chainConfig, [], addressStore, lowercaseAddresses, BlockStore.make(chainConfig.ecosystem), TransactionStore.make(chainConfig.ecosystem));
 }
 
 function candidateSources(sources) {

package/src/sources/SvmHyperSyncSource.res

@@ -158,6 +158,8 @@ let make = (
     let totalTimeRef = Performance.now()
     let pageFetchRef = Performance.now()
 
+    let fetchStats = () => RequestStat.single(~method="getInstructions", ~sentAt=pageFetchRef)
+
     let query: SvmHyperSyncClient.EventItems.query = {
       fromSlot: fromBlock,
       toSlot: toBlock,
@@ -179,6 +181,7 @@ let make = (
       throw(
         Source.GetItemsError(
           Source.FailedGettingItems({
+            requestStats: fetchStats(),
             exn,
             attemptedToBlock: toBlock->Option.getOr(knownHeight),
             retry: WithBackoff({
@@ -193,7 +196,7 @@ let make = (
       )
     }
     let pageFetchTime = pageFetchRef->Performance.secondsSince
-    let requestStats = [{Source.method: "getInstructions", seconds: pageFetchTime}]
+    let requestStats = fetchStats()
 
     let parsingRef = Performance.now()
 
@@ -244,7 +247,6 @@ let make = (
       latestFetchedBlockNumber: highestSlot,
       stats: {totalTimeElapsed, parsingTimeElapsed, pageFetchTime},
       knownHeight,
-      fromBlockQueried: fromBlock,
       requestStats,
     }
   }