envio 3.0.2 → 3.1.0-rc.1

package/src/sources/SimulateSource.res

@@ -39,13 +39,7 @@ let make = (~items: array<Internal.item>, ~endBlock: int, ~chain: ChainMap.Chain
       let reportedHeight = max(endBlock, 1)
       Promise.resolve({
         Source.knownHeight: reportedHeight,
-        reorgGuard: {
-          rangeLastBlock: {
-            blockHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
-            blockNumber: reportedHeight,
-          },
-          prevRangeLastBlock: None,
-        },
+        blockHashes: [],
         parsedQueueItems: result,
         fromBlockQueried: 0,
         latestFetchedBlockNumber: reportedHeight,

package/src/sources/SimulateSource.res.mjs

@@ -22,13 +22,7 @@ function make(items, endBlock, chain) {
       let reportedHeight = Primitive_int.max(endBlock, 1);
       return Promise.resolve({
         knownHeight: reportedHeight,
-        reorgGuard: {
-          rangeLastBlock: {
-            blockHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
-            blockNumber: reportedHeight
-          },
-          prevRangeLastBlock: undefined
-        },
+        blockHashes: [],
         parsedQueueItems: result,
         fromBlockQueried: 0,
         latestFetchedBlockNumber: reportedHeight,

package/src/sources/Source.res

@@ -12,7 +12,11 @@ Thes response returned from a block range fetch
 */
 type blockRangeFetchResponse = {
   knownHeight: int,
-  reorgGuard: ReorgDetection.reorgGuard,
+  // Best-effort (blockNumber, blockHash) pairs observed while fetching this range.
+  // Used by reorg detection; gaps are OK, no extra requests are made to fill them.
+  // Duplicates with the same block number are allowed — registerReorgGuard treats
+  // a within-array hash mismatch on the same block number as a reorg.
+  blockHashes: array<ReorgDetection.blockData>,
   parsedQueueItems: array<Internal.item>,
   fromBlockQueried: int,
   latestFetchedBlockNumber: int,
@@ -25,6 +29,8 @@ type getItemsRetry =
   | WithBackoff({message: string, backoffMillis: int})
   | ImpossibleForTheQuery({message: string})
 
+exception RateLimited({resetMs: int})
+
 type getItemsError =
   | UnsupportedSelection({message: string})
   | FailedGettingFieldSelection({exn: exn, blockNumber: int, logIndex: int, message: string})
@@ -58,4 +64,7 @@ type t = {
     ~logger: Pino.t,
   ) => promise<blockRangeFetchResponse>,
   createHeightSubscription?: (~onHeight: int => unit) => unit => unit,
+  // Invoked by SourceManager once a rollback target is known so the source can
+  // drop any state that may now point at an orphaned chain (e.g. RPC block cache).
+  onReorg?: (~rollbackTargetBlock: int) => unit,
 }

package/src/sources/Source.res.mjs

@@ -2,9 +2,12 @@
 
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
+let RateLimited = /* @__PURE__ */Primitive_exceptions.create("Source.RateLimited");
+
 let GetItemsError = /* @__PURE__ */Primitive_exceptions.create("Source.GetItemsError");
 
 export {
+  RateLimited,
   GetItemsError,
 }
 /* No side effect */

package/src/sources/SourceManager.res

@@ -32,10 +32,92 @@ type t = {
   mutable fetchingPartitionsCount: int,
   recoveryTimeout: float,
   mutable hasRealtime: bool,
+  mutable committedRateLimitTimeMs: float,
+  mutable rateLimitWaiters: int,
+  // Wall-clock timestamp (Date.now()) when the current rate-limit window
+  // started, or None if not currently waiting. Wall-clock so consumers
+  // (TUI) can compute elapsed time with their own Date.now() reads.
+  mutable activeRateLimitStartMs: option<float>,
+  // Wall-clock timestamp by which the server expects the longest current
+  // wait to clear. Tracks the latest reset across concurrent waiters so
+  // the displayed countdown reflects when the indexer will actually retry.
+  mutable activeRateLimitResetAtMs: option<float>,
 }
 
 let getActiveSource = sourceManager => sourceManager.activeSource
 
+let getRateLimitTimeMs = sourceManager =>
+  sourceManager.committedRateLimitTimeMs +.
+  switch sourceManager.activeRateLimitStartMs {
+  | Some(startMs) => Date.now() -. startMs
+  | None => 0.0
+  }
+
+let isRateLimited = sourceManager => sourceManager.activeRateLimitStartMs->Option.isSome
+
+let getRateLimitResetInMs = sourceManager =>
+  switch sourceManager.activeRateLimitResetAtMs {
+  | Some(resetAt) =>
+    let remaining = resetAt -. Date.now()
+    remaining > 0.0 ? Some(remaining) : None
+  | None => None
+  }
+
+let startRateLimitTimeout = (sourceManager, ~resetMs) => {
+  let now = Date.now()
+  if sourceManager.rateLimitWaiters === 0 {
+    sourceManager.activeRateLimitStartMs = Some(now)
+  }
+  let resetAt = now +. resetMs->Int.toFloat
+  sourceManager.activeRateLimitResetAtMs = switch sourceManager.activeRateLimitResetAtMs {
+  | Some(existing) => Some(Pervasives.max(existing, resetAt))
+  | None => Some(resetAt)
+  }
+  sourceManager.rateLimitWaiters = sourceManager.rateLimitWaiters + 1
+}
+
+let stopRateLimitTimeout = sourceManager => {
+  sourceManager.rateLimitWaiters = sourceManager.rateLimitWaiters - 1
+  if sourceManager.rateLimitWaiters === 0 {
+    switch sourceManager.activeRateLimitStartMs {
+    | Some(startMs) =>
+      sourceManager.committedRateLimitTimeMs =
+        sourceManager.committedRateLimitTimeMs +. Date.now() -. startMs
+      sourceManager.activeRateLimitStartMs = None
+    | None => ()
+    }
+    sourceManager.activeRateLimitResetAtMs = None
+  }
+}
+
+// Shared between executeQuery and getBlockHashes: wait out the server's
+// suggested reset window. Cap at 5 minutes to protect against
+// pathologically large server values. Escalates the log from trace to
+// warn after the second consecutive retry so the indexer doesn't go
+// silent under chronic throttling.
+let waitForRateLimitReset = async (sourceManager: t, ~resetMs, ~retry, ~logger) => {
+  let waitMs = Pervasives.min(resetMs, 300_000)
+  let log = retry >= 2 ? Logging.childWarn : Logging.childTrace
+  logger->log({
+    "msg": `HyperSync source is rate-limited — not critical, the indexer will retry in ${(waitMs / 1000)
+        ->Int.toString}s. For higher limits upgrade your plan at https://envio.dev/app/api-tokens.`,
+    "retry": retry,
+    "waitMs": waitMs,
+  })
+  sourceManager->startRateLimitTimeout(~resetMs=waitMs)
+  await Utils.delay(waitMs)
+  sourceManager->stopRateLimitTimeout
+}
+
+let onReorg = (sourceManager: t, ~rollbackTargetBlock) => {
+  sourceManager.sourcesState->Array.forEach(({source}) => {
+    switch source.onReorg {
+    | Some(cb) => cb(~rollbackTargetBlock)
+    | None => ()
+    }
+  })
+}
+
 type sourceRole = Primary | Secondary
 
 // Determines whether a source is Primary or Secondary given the current mode.
@@ -120,6 +202,10 @@ let make = (
     statusStart: Hrtime.makeTimer(),
     status: Idle,
     hasRealtime,
+    committedRateLimitTimeMs: 0.0,
+    rateLimitWaiters: 0,
+    activeRateLimitStartMs: None,
+    activeRateLimitResetAtMs: None,
   }
 }
 
@@ -242,14 +328,18 @@ let getSourceNewHeight = async (
   let retry = ref(0)
 
   while newHeight.contents <= knownHeight && status.contents !== Done {
-    // If subscription exists, wait for next height event
     switch sourceState.unsubscribe {
     | Some(_) =>
       let subscriptionPromise = Promise.make((resolve, _reject) => {
         sourceState.pendingHeightResolvers->Array.push(resolve)
       })
-      // If subscription goes quiet for half the stall timeout, fall back to REST polling
-      let pollingFallback = Utils.delay(stallTimeout / 2)->Promise.then(async () => {
+      // If the subscription goes quiet for half the stall timeout, fall back to REST
+      // polling. Jitter the trigger across [stallTimeout/2, stallTimeout) so indexers
+      // that go quiet together don't all start polling at the same instant.
+      let half = stallTimeout / 2
+      let pollingFallback = Utils.delay(
+        half + (Math.random() *. half->Belt.Int.toFloat)->Belt.Float.toInt,
+      )->Promise.then(async () => {
         logger->Logging.childTrace({
           "msg": "onHeight subscription stale, switching to polling fallback",
           "source": source.name,
@@ -288,11 +378,15 @@ let getSourceNewHeight = async (
           switch source.createHeightSubscription {
           | Some(createSubscription) if isRealtime =>
             let unsubscribe = createSubscription(~onHeight=newHeight => {
-              sourceState.knownHeight = newHeight
-              // Resolve all pending height resolvers
-              let resolvers = sourceState.pendingHeightResolvers
-              sourceState.pendingHeightResolvers = []
-              resolvers->Array.forEach(resolve => resolve(newHeight))
+              // Ignore non-increasing heights. The height stream re-emits the current
+              // head on every (re)connect; waking the wait loop on a height we already
+              // know spins it and leaks fallback pollers (#1270).
+              if newHeight > sourceState.knownHeight {
+                sourceState.knownHeight = newHeight
+                let resolvers = sourceState.pendingHeightResolvers
+                sourceState.pendingHeightResolvers = []
+                resolvers->Array.forEach(resolve => resolve(newHeight))
+              }
             })
             sourceState.unsubscribe = Some(unsubscribe)
           | _ =>
@@ -599,6 +693,10 @@ let executeQuery = async (
       sourceState.lastFailedAt = None
       responseRef := Some(response)
     } catch {
+    | Source.RateLimited({resetMs}) =>
+      await sourceManager->waitForRateLimitReset(~resetMs, ~retry, ~logger)
+      retryRef := retryRef.contents + 1
+
     | Source.GetItemsError(error) =>
       switch error {
       | UnsupportedSelection(_)
@@ -699,3 +797,74 @@ let executeQuery = async (
 
   responseRef.contents->Option.getUnsafe
 }
+
+let getBlockHashes = async (sourceManager: t, ~blockNumbers: array<int>, ~isRealtime: bool) => {
+  let responseRef = ref(None)
+  let retryRef = ref(0)
+
+  while responseRef.contents->Option.isNone {
+    let sourceState = switch sourceManager->getNextSource(~isRealtime) {
+    | Some(s) => s
+    | None =>
+      let logger = Logging.createChild(
+        ~params={"chainId": sourceManager.activeSource.chain->ChainMap.Chain.toChainId},
+      )
+      %raw(`null`)->ErrorHandling.mkLogAndRaise(
+        ~logger,
+        ~msg="No data-sources available for fetching block hashes.",
+      )
+    }
+    sourceManager.activeSource = sourceState.source
+    let source = sourceState.source
+    let retry = retryRef.contents
+
+    let logger = Logging.createChild(
+      ~params={
+        "chainId": source.chain->ChainMap.Chain.toChainId,
+        "logType": "Block Hash Query",
+        "source": source.name,
+        "retry": retry,
+      },
+    )
+
+    try {
+      let res = await source.getBlockHashes(~blockNumbers, ~logger)
+      switch res {
+      | Ok(data) =>
+        sourceState.lastFailedAt = None
+        responseRef := Some(data)
+      | Error(exn) => throw(exn)
+      }
+    } catch {
+    | Source.RateLimited({resetMs}) =>
+      await sourceManager->waitForRateLimitReset(~resetMs, ~retry, ~logger)
+      retryRef := retryRef.contents + 1
+
+    | exn =>
+      let backoffMillis = switch retry {
+      | 0 => 500
+      | _ => 1000 * retry
+      }
+      let log = retry >= 4 ? Logging.childWarn : Logging.childTrace
+      logger->log({
+        "msg": "Failed to fetch block hashes. Retrying.",
+        "retry": retry,
+        "backOffMilliseconds": backoffMillis,
+        "err": exn->Utils.prettifyExn,
+      })
+
+      let shouldSwitch = switch retry {
+      | 0 | 1 => false
+      | _ => retry->mod(2) === 0
+      }
+
+      if shouldSwitch {
+        sourceState.lastFailedAt = Some(Date.now())
+      }
+      await Utils.delay(Pervasives.min(backoffMillis, 60_000))
+      retryRef := retryRef.contents + 1
+    }
+  }
+
+  responseRef.contents->Option.getUnsafe
+}

package/src/sources/SourceManager.res.mjs

@@ -10,6 +10,7 @@ import * as ErrorHandling from "../ErrorHandling.res.mjs";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
@@ -17,6 +18,70 @@ function getActiveSource(sourceManager) {
   return sourceManager.activeSource;
 }
 
+function getRateLimitTimeMs(sourceManager) {
+  let startMs = sourceManager.activeRateLimitStartMs;
+  return sourceManager.committedRateLimitTimeMs + (
+    startMs !== undefined ? Date.now() - startMs : 0.0
+  );
+}
+
+function isRateLimited(sourceManager) {
+  return Stdlib_Option.isSome(sourceManager.activeRateLimitStartMs);
+}
+
+function getRateLimitResetInMs(sourceManager) {
+  let resetAt = sourceManager.activeRateLimitResetAtMs;
+  if (resetAt === undefined) {
+    return;
+  }
+  let remaining = resetAt - Date.now();
+  if (remaining > 0.0) {
+    return remaining;
+  }
+}
+
+function startRateLimitTimeout(sourceManager, resetMs) {
+  let now = Date.now();
+  if (sourceManager.rateLimitWaiters === 0) {
+    sourceManager.activeRateLimitStartMs = now;
+  }
+  let resetAt = now + resetMs;
+  let existing = sourceManager.activeRateLimitResetAtMs;
+  sourceManager.activeRateLimitResetAtMs = existing !== undefined ? Primitive_float.max(existing, resetAt) : resetAt;
+  sourceManager.rateLimitWaiters = sourceManager.rateLimitWaiters + 1 | 0;
+}
+
+async function waitForRateLimitReset(sourceManager, resetMs, retry, logger) {
+  let waitMs = Primitive_int.min(resetMs, 300000);
+  let log = retry >= 2 ? Logging.childWarn : Logging.childTrace;
+  log(logger, {
+    msg: `HyperSync source is rate-limited — not critical, the indexer will retry in ` + (waitMs / 1000 | 0).toString() + `s. For higher limits upgrade your plan at https://envio.dev/app/api-tokens.`,
+    retry: retry,
+    waitMs: waitMs
+  });
+  startRateLimitTimeout(sourceManager, waitMs);
+  await Utils.delay(waitMs);
+  sourceManager.rateLimitWaiters = sourceManager.rateLimitWaiters - 1 | 0;
+  if (sourceManager.rateLimitWaiters !== 0) {
+    return;
+  }
+  let startMs = sourceManager.activeRateLimitStartMs;
+  if (startMs !== undefined) {
+    sourceManager.committedRateLimitTimeMs = sourceManager.committedRateLimitTimeMs + Date.now() - startMs;
+    sourceManager.activeRateLimitStartMs = undefined;
+  }
+  sourceManager.activeRateLimitResetAtMs = undefined;
+}
+
+function onReorg(sourceManager, rollbackTargetBlock) {
+  sourceManager.sourcesState.forEach(param => {
+    let cb = param.source.onReorg;
+    if (cb !== undefined) {
+      return cb(rollbackTargetBlock);
+    }
+  });
+}
+
 function getSourceRole(sourceFor, isRealtime, hasRealtime) {
   if (isRealtime) {
     switch (sourceFor) {
@@ -83,7 +148,11 @@ function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeout
     waitingForNewBlockStateId: undefined,
     fetchingPartitionsCount: 0,
     recoveryTimeout: recoveryTimeout,
-    hasRealtime: hasRealtime
+    hasRealtime: hasRealtime,
+    committedRateLimitTimeMs: 0.0,
+    rateLimitWaiters: 0,
+    activeRateLimitStartMs: undefined,
+    activeRateLimitResetAtMs: undefined
   };
 }
 
@@ -186,7 +255,8 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
       let subscriptionPromise = new Promise((resolve, _reject) => {
         sourceState.pendingHeightResolvers.push(resolve);
       });
-      let pollingFallback = Utils.delay(stallTimeout / 2 | 0).then(async () => {
+      let half = stallTimeout / 2 | 0;
+      let pollingFallback = Utils.delay(half + (Math.random() * half | 0) | 0).then(async () => {
         Logging.childTrace(logger, {
           msg: "onHeight subscription stale, switching to polling fallback",
           source: source.name,
@@ -222,6 +292,9 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
           let exit = 0;
           if (createSubscription !== undefined && isRealtime) {
             let unsubscribe = createSubscription(newHeight => {
+              if (newHeight <= sourceState.knownHeight) {
+                return;
+              }
               sourceState.knownHeight = newHeight;
               let resolvers = sourceState.pendingHeightResolvers;
               sourceState.pendingHeightResolvers = [];
@@ -440,7 +513,10 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
       responseRef = response;
     } catch (raw_error) {
       let error = Primitive_exceptions.internalToException(raw_error);
-      if (error.RE_EXN_ID === Source.GetItemsError) {
+      if (error.RE_EXN_ID === Source.RateLimited) {
+        await waitForRateLimitReset(sourceManager, error.resetMs, retry, logger$2);
+        retryRef = retryRef + 1 | 0;
+      } else if (error.RE_EXN_ID === Source.GetItemsError) {
         let error$1 = error._1;
         let exit = 0;
         switch (error$1.TAG) {
@@ -543,13 +619,75 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
   return responseRef;
 }
 
+async function getBlockHashes(sourceManager, blockNumbers, isRealtime) {
+  let responseRef;
+  let retryRef = 0;
+  while (Stdlib_Option.isNone(responseRef)) {
+    let s = getNextSource(sourceManager, isRealtime, undefined);
+    let sourceState;
+    if (s !== undefined) {
+      sourceState = s;
+    } else {
+      let logger = Logging.createChild({
+        chainId: sourceManager.activeSource.chain
+      });
+      sourceState = ErrorHandling.mkLogAndRaise(logger, "No data-sources available for fetching block hashes.", null);
+    }
+    sourceManager.activeSource = sourceState.source;
+    let source = sourceState.source;
+    let retry = retryRef;
+    let logger$1 = Logging.createChild({
+      chainId: source.chain,
+      logType: "Block Hash Query",
+      source: source.name,
+      retry: retry
+    });
+    try {
+      let res = await source.getBlockHashes(blockNumbers, logger$1);
+      if (res.TAG === "Ok") {
+        sourceState.lastFailedAt = undefined;
+        responseRef = res._0;
+      } else {
+        throw res._0;
+      }
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      if (exn.RE_EXN_ID === Source.RateLimited) {
+        await waitForRateLimitReset(sourceManager, exn.resetMs, retry, logger$1);
+        retryRef = retryRef + 1 | 0;
+      } else {
+        let backoffMillis = retry !== 0 ? 1000 * retry | 0 : 500;
+        let log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
+        log(logger$1, {
+          msg: "Failed to fetch block hashes. Retrying.",
+          retry: retry,
+          backOffMilliseconds: backoffMillis,
+          err: Utils.prettifyExn(exn)
+        });
+        let shouldSwitch = retry !== 0 && retry !== 1 ? retry % 2 === 0 : false;
+        if (shouldSwitch) {
+          sourceState.lastFailedAt = Date.now();
+        }
+        await Utils.delay(Primitive_int.min(backoffMillis, 60000));
+        retryRef = retryRef + 1 | 0;
+      }
+    }
+  };
+  return responseRef;
+}
+
 export {
   getSourceRole,
   make,
   getActiveSource,
+  onReorg,
   fetchNext,
   waitForNewBlock,
   executeQuery,
   makeGetHeightRetryInterval,
+  getBlockHashes,
+  getRateLimitTimeMs,
+  isRateLimited,
+  getRateLimitResetInMs,
 }
 /* Utils Not a pure module */

package/src/sources/SourceManager.resi

@@ -22,6 +22,8 @@ let make: (
 
 let getActiveSource: t => Source.t
 
+let onReorg: (t, ~rollbackTargetBlock: int) => unit
+
 let fetchNext: (
   t,
   ~fetchState: FetchState.t,
@@ -50,3 +52,20 @@ let makeGetHeightRetryInterval: (
   ~backoffMultiplicative: int,
   ~maxRetryInterval: int,
 ) => (~retry: int) => int
+
+let getBlockHashes: (
+  t,
+  ~blockNumbers: array<int>,
+  ~isRealtime: bool,
+) => promise<array<ReorgDetection.blockDataWithTimestamp>>
+
+// Total time the indexer spent waiting on rate limits, including any
+// currently-in-progress window measured against Date.now() at call time.
+let getRateLimitTimeMs: t => float
+
+// True if at least one request is currently waiting on a rate-limit reset.
+let isRateLimited: t => bool
+
+// Milliseconds remaining until the longest active rate-limit window resets,
+// or None if not currently waiting (or the window has already elapsed).
+let getRateLimitResetInMs: t => option<float>

package/src/tui/Tui.res

@@ -92,7 +92,7 @@ module EventsPerSecond = {
     }
   }
 
-  let use = (~totalEventsProcessed: float) => {
+  let use = (~totalEventsProcessed: float, ~tick: int) => {
     let (samples, setSamples) = React.useState((): array<sample> => [])
 
     React.useEffect1(() => {
@@ -103,7 +103,7 @@ module EventsPerSecond = {
         kept->Array.concat([{time: now, events: totalEventsProcessed}])
       })
       None
-    }, [totalEventsProcessed])
+    }, [tick])
 
     computeEps(samples)
   }
@@ -132,12 +132,15 @@ module App = {
   @react.component
   let make = (~getState) => {
     let stdoutColumns = Hooks.useStdoutColumns()
-    let (state: GlobalState.t, setState) = React.useState(() => getState())
+    // GlobalState is mutated in place — passing the same ref to useState
+    // would bail out via Object.is and skip the re-render. Tick a counter
+    // instead and read state freshly from getState() on every render.
+    let (tick, setTick) = React.useState(() => 0)
+    let state: GlobalState.t = getState()
 
-    // useEffect to refresh state every 500ms
     React.useEffect(() => {
       let intervalId = setInterval(() => {
-        setState(_ => getState())
+        setTick(t => t + 1)
       }, 500)
 
       Some(
@@ -210,6 +213,9 @@ module App = {
             poweredByHyperSync: (
               cf.sourceManager->SourceManager.getActiveSource
             ).poweredByHyperSync,
+            rateLimitTimeMs: cf.sourceManager->SourceManager.getRateLimitTimeMs,
+            isRateLimited: cf.sourceManager->SourceManager.isRateLimited,
+            rateLimitResetInMs: cf.sourceManager->SourceManager.getRateLimitResetInMs,
           }: TuiData.chain
         )
       })
@@ -225,7 +231,7 @@ module App = {
         acc
       }
     })
-    let eventsPerSecond = EventsPerSecond.use(~totalEventsProcessed)
+    let eventsPerSecond = EventsPerSecond.use(~totalEventsProcessed, ~tick)
 
     <Box flexDirection={Column}>
       <BigText
@@ -257,6 +263,38 @@ module App = {
         eventsPerSecond={SyncETA.isIndexerFullySynced(chains) ? None : eventsPerSecond}
       />
       <SyncETA chains indexerStartTime=state.indexerStartTime />
+      {
+        let maxRateLimitTimeMs =
+          chains->Array.reduce(0., (acc, chain) => Pervasives.max(acc, chain.rateLimitTimeMs))
+        let maxResetInMs =
+          chains->Array.reduce(0.0, (acc, chain) =>
+            Pervasives.max(acc, chain.rateLimitResetInMs->Option.getOr(0.0))
+          )
+        maxRateLimitTimeMs > 1000.
+          ? {
+              let rateLimitSecs = Math.round(maxRateLimitTimeMs /. 1000.)
+              let activeSuffix = if maxResetInMs > 0.0 {
+                let resetSecs = Pervasives.max(1.0, Math.ceil(maxResetInMs /. 1000.))
+                ` (⏳ ${resetSecs->TuiData.formatFloatLocaleString}s until reset)`
+              } else {
+                ""
+              }
+              <Box flexDirection={Column}>
+                <Newline />
+                <Text color={Danger}>
+                  {`Backfill ${rateLimitSecs->TuiData.formatFloatLocaleString}s slower due to your plan's rate limit${activeSuffix}`->React.string}
+                </Text>
+                <Text color={Danger}>
+                  <Text color={Danger}> {"Upgrade at "->React.string} </Text>
+                  <Text color={Danger} underline=true>
+                    {"https://envio.dev/app/api-tokens"->React.string}
+                  </Text>
+                  <Text color={Danger}> {" for higher rate limits."->React.string} </Text>
+                </Text>
+              </Box>
+            }
+          : React.null
+      }
       <Newline />
       <Box flexDirection={Row}>
         <Text> {"GraphQL: "->React.string} </Text>