envio 3.3.0-alpha.0 → 3.3.0-alpha.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.3.0-alpha.0",
+  "version": "3.3.0-alpha.1",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.3.0-alpha.0",
-    "envio-linux-x64-musl": "3.3.0-alpha.0",
-    "envio-linux-arm64": "3.3.0-alpha.0",
-    "envio-darwin-x64": "3.3.0-alpha.0",
-    "envio-darwin-arm64": "3.3.0-alpha.0"
+    "envio-linux-x64": "3.3.0-alpha.1",
+    "envio-linux-x64-musl": "3.3.0-alpha.1",
+    "envio-linux-arm64": "3.3.0-alpha.1",
+    "envio-darwin-x64": "3.3.0-alpha.1",
+    "envio-darwin-arm64": "3.3.0-alpha.1"
   }
 }

package/src/ChainFetching.res

@@ -308,8 +308,7 @@ let finishWaitingForNewBlock = (
 let fetchChain = async (
   state: IndexerState.t,
   chain,
-  ~concurrencyLimit,
-  ~bufferLimit,
+  ~action,
   ~stateId,
   ~scheduleFetch,
   ~scheduleProcessing,
@@ -321,14 +320,14 @@ let fetchChain = async (
     let isRealtime = state->IndexerState.isRealtime
     let sourceManager = chainState->ChainState.sourceManager
 
-    // Only affects the WaitingForNewBlock branch of fetchNext, where
+    // Only affects the WaitingForNewBlock branch of dispatch, where
     // there's nothing to fetch. During backfill any such chain is idle.
     let reducedPolling = !isRealtime
 
     // Owns its error boundary: launch doesn't catch, so any failure here (the
-    // query, response handling, or fetchNext itself) must stop the indexer.
+    // query, response handling, or dispatch itself) must stop the indexer.
     try {
-      await sourceManager->SourceManager.fetchNext(
+      await sourceManager->SourceManager.dispatch(
         ~fetchState,
         ~waitForNewBlock=(~knownHeight) =>
           sourceManager->SourceManager.waitForNewBlock(~knownHeight, ~isRealtime, ~reducedPolling),
@@ -343,7 +342,7 @@ let fetchChain = async (
           ),
         ~executeQuery=async query => {
           // Caught here (not just by the outer try) so the query promise never
-          // rejects: fetchNext spins a side-chain off it that would otherwise
+          // rejects: dispatch spins a side-chain off it that would otherwise
           // become an unhandled rejection.
           try {
             let response = await sourceManager->SourceManager.executeQuery(
@@ -363,8 +362,7 @@ let fetchChain = async (
           | exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
           }
         },
-        ~concurrencyLimit,
-        ~bufferLimit,
+        ~action,
         ~stateId,
       )
     } catch {

package/src/ChainFetching.res.mjs

@@ -167,7 +167,7 @@ function finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFe
   scheduleProcessing();
 }
 
-async function fetchChain(state, chain, concurrencyLimit, bufferLimit, stateId, scheduleFetch, scheduleProcessing, scheduleRollback) {
+async function fetchChain(state, chain, action, stateId, scheduleFetch, scheduleProcessing, scheduleRollback) {
   let chainState = IndexerState.getChainState(state, chain);
   if (IndexerState.isResolvingReorg(state) || IndexerState.isStopped(state)) {
     return;
@@ -177,7 +177,7 @@ async function fetchChain(state, chain, concurrencyLimit, bufferLimit, stateId,
   let sourceManager = ChainState.sourceManager(chainState);
   let reducedPolling = !isRealtime;
   try {
-    return await SourceManager.fetchNext(sourceManager, fetchState, async query => {
+    return await SourceManager.dispatch(sourceManager, fetchState, async query => {
       try {
         let response = await SourceManager.executeQuery(sourceManager, query, fetchState.knownHeight, isRealtime);
         return await onQueryResponse(state, {
@@ -189,7 +189,7 @@ async function fetchChain(state, chain, concurrencyLimit, bufferLimit, stateId,
         let exn = Primitive_exceptions.internalToException(raw_exn);
         return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
       }
-    }, knownHeight => SourceManager.waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling), knownHeight => finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFetch, scheduleProcessing), concurrencyLimit, bufferLimit, stateId);
+    }, knownHeight => SourceManager.waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling), knownHeight => finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFetch, scheduleProcessing), action, stateId);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, IndexerState.unexpectedErrorMsg));

package/src/ChainState.res

@@ -11,6 +11,10 @@ type t = {
   mutable timestampCaughtUpToHeadOrEndblock: option<Date.t>,
   mutable committedProgressBlockNumber: int,
   mutable numEventsProcessed: float,
+  // Running sum of in-flight queries' estResponseSize, kept here so the
+  // scheduler doesn't re-sum pending queries on every tick. Incremented when
+  // queries are dispatched, decremented as their responses land.
+  mutable pendingBudget: float,
   mutable reorgDetection: ReorgDetection.t,
   mutable safeCheckpointTracking: option<SafeCheckpointTracking.t>,
 }
@@ -49,6 +53,7 @@ let make = (
   timestampCaughtUpToHeadOrEndblock,
   committedProgressBlockNumber,
   numEventsProcessed,
+  pendingBudget: 0.,
   reorgDetection,
   safeCheckpointTracking,
 }
@@ -380,8 +385,25 @@ let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
 let isProgressAtHead = (cs: t) => cs.isProgressAtHead
 let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
 let numEventsProcessed = (cs: t) => cs.numEventsProcessed
+let pendingBudget = (cs: t) => cs.pendingBudget
 let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock
 
+// Mark queries as in flight and reserve their estimated size against the shared
+// buffer budget in one step, so the counter stays in sync with the pending
+// queries it tracks.
+let startFetchingQueries = (cs: t, ~queries: array<FetchState.query>) => {
+  cs.fetchState->FetchState.startFetchingQueries(~queries)
+  cs.pendingBudget =
+    cs.pendingBudget +. queries->Array.reduce(0., (acc, query) => acc +. query.estResponseSize)
+}
+
+// Drop every in-flight query and release their reservations together, keeping
+// pendingBudget coupled to the pending queries it tracks.
+let resetPendingQueries = (cs: t) => {
+  cs.fetchState = cs.fetchState->FetchState.resetPendingQueries
+  cs.pendingBudget = 0.
+}
+
 // --- Derived (pure). ---
 
 let hasProcessedToEndblock = (cs: t) => {
@@ -426,6 +448,9 @@ let handleQueryResult = (
     fs
     ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
     ->FetchState.updateKnownHeight(~knownHeight)
+
+  // The query is no longer in flight, so release its reservation.
+  cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.estResponseSize)
 }
 
 // Run reorg detection against a fetch response and commit the updated guard.
@@ -447,7 +472,7 @@ let prepareReorg = (cs: t, ~eventsProcessedDiff) => {
   | Some(diff) => cs.numEventsProcessed = cs.numEventsProcessed +. diff
   | None => ()
   }
-  cs.fetchState = cs.fetchState->FetchState.resetPendingQueries
+  cs->resetPendingQueries
 }
 
 let updateKnownHeight = (cs: t, ~knownHeight) =>

package/src/ChainState.res.mjs

@@ -17,6 +17,7 @@ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as ReorgDetection from "./ReorgDetection.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as HyperFuelSource from "./sources/HyperFuelSource.res.mjs";
+import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as SvmHyperSyncSource from "./sources/SvmHyperSyncSource.res.mjs";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
@@ -49,6 +50,7 @@ function make(chainConfig, fetchState, sourceManager, reorgDetection, committedP
     timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
     committedProgressBlockNumber: committedProgressBlockNumber,
     numEventsProcessed: numEventsProcessed,
+    pendingBudget: 0,
     reorgDetection: reorgDetection,
     safeCheckpointTracking: safeCheckpointTracking
   };
@@ -225,10 +227,19 @@ function numEventsProcessed(cs) {
   return cs.numEventsProcessed;
 }
 
+function pendingBudget(cs) {
+  return cs.pendingBudget;
+}
+
 function timestampCaughtUpToHeadOrEndblock(cs) {
   return cs.timestampCaughtUpToHeadOrEndblock;
 }
 
+function startFetchingQueries(cs, queries) {
+  FetchState.startFetchingQueries(cs.fetchState, queries);
+  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.estResponseSize);
+}
+
 function hasProcessedToEndblock(cs) {
   let fetchState = cs.fetchState;
   let committedProgressBlockNumber = cs.committedProgressBlockNumber;
@@ -264,6 +275,7 @@ function isFetchingAtHead(cs) {
 function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight) {
   let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, newItemsWithDcs) : cs.fetchState;
   cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
+  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.estResponseSize);
 }
 
 function registerReorgGuard(cs, blockHashes, knownHeight) {
@@ -277,6 +289,7 @@ function prepareReorg(cs, eventsProcessedDiff) {
     cs.numEventsProcessed = cs.numEventsProcessed + eventsProcessedDiff;
   }
   cs.fetchState = FetchState.resetPendingQueries(cs.fetchState);
+  cs.pendingBudget = 0;
 }
 
 function updateKnownHeight(cs, knownHeight) {
@@ -441,6 +454,8 @@ export {
   isProgressAtHead,
   committedProgressBlockNumber,
   numEventsProcessed,
+  pendingBudget,
+  startFetchingQueries,
   timestampCaughtUpToHeadOrEndblock,
   hasProcessedToEndblock,
   getHighestBlockBelowThreshold,

package/src/ChainState.resi

@@ -46,6 +46,8 @@ let safeCheckpointTracking: t => option<SafeCheckpointTracking.t>
 let isProgressAtHead: t => bool
 let committedProgressBlockNumber: t => int
 let numEventsProcessed: t => float
+let pendingBudget: t => float
+let startFetchingQueries: (t, ~queries: array<FetchState.query>) => unit
 let timestampCaughtUpToHeadOrEndblock: t => option<Date.t>
 
 // Derived (pure).

package/src/CrossChainState.res

@@ -11,10 +11,6 @@ type t = {
   // True once every chain has caught up to head/endBlock. Monotonic during a run.
   mutable isRealtime: bool,
   mutable isInReorgThreshold: bool,
-  // Indexer-wide caps on concurrent data-source queries, shared across all
-  // chains. The realtime budget applies once every chain is at head.
-  maxBackfillConcurrency: int,
-  maxRealtimeConcurrency: int,
   // Indexer-wide fetch buffer pool (item count), shared across all chains.
   targetBufferSize: int,
 }
@@ -26,18 +22,10 @@ let calculateTargetBufferSize = () =>
   | None => 100_000
   }
 
-// The concurrency budget in force for the current phase.
-let maxConcurrency = (crossChainState: t) =>
-  crossChainState.isRealtime
-    ? crossChainState.maxRealtimeConcurrency
-    : crossChainState.maxBackfillConcurrency
-
 let make = (
   ~chainStates,
   ~isInReorgThreshold,
   ~isRealtime,
-  ~maxBackfillConcurrency=Env.maxBackfillConcurrency,
-  ~maxRealtimeConcurrency=Env.maxRealtimeConcurrency,
   ~targetBufferSize=calculateTargetBufferSize(),
 ): t => {
   let crossChainState = {
@@ -45,11 +33,8 @@ let make = (
     chainIds: chainStates->Dict.valuesToArray->Array.map(cs => (cs->ChainState.chainConfig).id),
     isRealtime,
     isInReorgThreshold,
-    maxBackfillConcurrency,
-    maxRealtimeConcurrency,
     targetBufferSize,
   }
-  Prometheus.IndexingMaxConcurrency.set(~maxConcurrency=crossChainState->maxConcurrency)
   Prometheus.IndexingTargetBufferSize.set(~targetBufferSize)
   crossChainState
 }
@@ -65,17 +50,6 @@ let chainStates = (crossChainState: t) => crossChainState.chainStates
 let isRealtime = (crossChainState: t) => crossChainState.isRealtime
 let isInReorgThreshold = (crossChainState: t) => crossChainState.isInReorgThreshold
 
-// Partition queries in flight across every chain — the live draw against
-// maxConcurrency.
-let inFlight = (crossChainState: t) => {
-  let total = ref(0)
-  for i in 0 to crossChainState.chainIds->Array.length - 1 {
-    let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
-    total := total.contents + cs->ChainState.sourceManager->SourceManager.inFlightCount
-  }
-  total.contents
-}
-
 // Ready-to-process items across every chain — the live draw against
 // targetBufferSize, which is a budget of processable events (items stuck behind
 // a gap don't count toward the goal of keeping ~targetBufferSize ready).
@@ -199,18 +173,13 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t) => {
     Prometheus.ProgressReady.setAllReady()
   }
 
-  let wasRealtime = crossChainState.isRealtime
   crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady.contents
-  if !wasRealtime && crossChainState.isRealtime {
-    // The realtime budget takes over now that every chain is at head.
-    Prometheus.IndexingMaxConcurrency.set(~maxConcurrency=crossChainState->maxConcurrency)
-  }
 }
 
 // --- Fetch control. ---
 
-// Chains ordered furthest-behind first, so the shared concurrency and buffer
-// pools go to the chains with the most backfill work before the rest.
+// Chains ordered furthest-behind first, so the shared buffer pool goes to the
+// chains with the most backfill work before the rest.
 let priorityOrder = (crossChainState: t) =>
   crossChainState.chainStates
   ->Dict.valuesToArray
@@ -221,49 +190,105 @@ let priorityOrder = (crossChainState: t) =>
     )
   )
 
-// Dispatch a fetch tick across every chain in priority order, drawing from the
-// shared concurrency and buffer pools. Chains are visited in turn; fetchChain
-// bumps the in-flight count synchronously before it suspends, so a later chain
-// sees the slots an earlier one already claimed and the budget is honored
-// indexer-wide.
-//
-// bufferLimit is each chain's slice of the shared pool: it may grow its buffer
-// into whatever the other chains leave free, so a lone backfilling chain can use
-// the whole pool while head-following chains stay shallow.
+// In-flight estimated items across every chain — the live draw against
+// targetBufferSize alongside totalReadyCount, so the pool isn't re-dispatched
+// while queries are still being fetched.
+let totalReservedSize = (crossChainState: t) => {
+  let total = ref(0.)
+  for i in 0 to crossChainState.chainIds->Array.length - 1 {
+    let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
+    total := total.contents +. cs->ChainState.pendingBudget
+  }
+  total.contents
+}
+
+// Furthest-behind first: order candidate queries by the chain progress % at
+// their fromBlock, so the most behind ranges across all chains are admitted
+// before the rest.
+let compareByProgress = (a: FetchState.query, b: FetchState.query) =>
+  Float.compare(a.progress, b.progress)
+
+// Dispatch a fetch tick across the whole indexer from one shared pool of
+// ~targetBufferSize ready events. Every chain proposes its candidate queries
+// (each carrying an estimated response size) against the full free budget; the
+// candidates are then pooled, ordered by chain progress (furthest-behind first),
+// and admitted until the budget is consumed. So the budget is split per query
+// across chains rather than per chain — a chain that can only use a little
+// leaves the rest for the others automatically.
 let checkAndFetch = async (
   crossChainState: t,
-  ~fetchChain: (
-    ~chain: ChainMap.Chain.t,
-    ~concurrencyLimit: int,
-    ~bufferLimit: int,
-  ) => promise<unit>,
+  ~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
 ) => {
-  let maxConcurrency = crossChainState->maxConcurrency
-  // Pool is a budget of ready-to-process items, but the cap fed to getNextQuery
-  // is still a buffer position (it tracks the sorted buffer, ready items first),
-  // so add this chain's own buffer size back rather than its ready count. A chain
-  // with a gap therefore gets extra headroom to fetch the not-ready overhang
-  // while still aiming for its share of ~targetBufferSize ready events.
-  let totalReady = crossChainState->totalReadyCount
-  // Track the in-flight total as a running counter (summed once up front, then
-  // adjusted by each chain's delta) instead of re-summing every chain — O(chains)
-  // per tick rather than O(chains^2). fetchChain bumps the chain's count
-  // synchronously, so the delta is observable right after the call.
-  let inFlight = ref(crossChainState->inFlight)
-  let priorityOrdered = crossChainState->priorityOrder
+  let remaining = Pervasives.max(
+    0,
+    crossChainState.targetBufferSize -
+    crossChainState->totalReadyCount -
+    crossChainState->totalReservedSize->Float.toInt,
+  )
+
+  let chainIds = crossChainState.chainIds
+  let actionByChain = Dict.make()
+  // Candidate queries from every chain. Each query carries its chain id and the
+  // chain progress % at its fromBlock (the admission sort key), set here so the
+  // pool can be ordered without a side tuple per query.
+  let candidates = []
+  for i in 0 to chainIds->Array.length - 1 {
+    let chainId = chainIds->Array.getUnsafe(i)
+    let cs = crossChainState->getChainState(chainId)
+    let fetchState = cs->ChainState.fetchState
+    switch fetchState->FetchState.getNextQuery(
+      ~budget=remaining,
+      ~chainPendingBudget=cs->ChainState.pendingBudget,
+    ) {
+    | (WaitingForNewBlock | NothingToQuery) as action =>
+      actionByChain->Utils.Dict.setByInt(chainId, action)
+    | Ready(queries) =>
+      // Default to NothingToQuery; replaced below if any candidate is admitted.
+      actionByChain->Utils.Dict.setByInt(chainId, FetchState.NothingToQuery)
+      queries->Array.forEach(query => {
+        query.chainId = chainId
+        query.progress =
+          fetchState->FetchState.getProgressPercentageAt(~blockNumber=query.fromBlock)
+        candidates->Array.push(query)
+      })
+    }
+  }
+
+  candidates->Array.sort(compareByProgress)
+
+  // Admit furthest-behind first until the budget runs out. The condition is
+  // checked before each query, so as long as there's any budget left we admit
+  // the next one even when its estimate alone exceeds the remainder — otherwise a
+  // chain whose only query is bigger than the budget would never make progress.
+  let admittedByChain = Dict.make()
+  let running = ref(0.)
+  let remainingF = remaining->Int.toFloat
+  let idx = ref(0)
+  while running.contents < remainingF && idx.contents < candidates->Array.length {
+    let query = candidates->Array.getUnsafe(idx.contents)
+    admittedByChain->Utils.Dict.push(query.chainId->Int.toString, query)
+    running := running.contents +. query.estResponseSize
+    idx := idx.contents + 1
+  }
+  admittedByChain->Dict.forEachWithKey((queries, chainId) => {
+    actionByChain->Dict.set(chainId, FetchState.Ready(queries))
+    // Mark the admitted queries in flight and reserve their size against the
+    // shared budget; released as each response lands in handleQueryResult.
+    crossChainState
+    ->getChainState(chainId->Int.fromString->Option.getUnsafe)
+    ->ChainState.startFetchingQueries(~queries)
+  })
+
   let promises = []
-  for i in 0 to priorityOrdered->Array.length - 1 {
-    let cs = priorityOrdered->Array.getUnsafe(i)
-    let chain = ChainMap.Chain.makeUnsafe(~chainId=(cs->ChainState.chainConfig).id)
-    let sourceManager = cs->ChainState.sourceManager
-    let concurrencyLimit = Pervasives.max(0, maxConcurrency - inFlight.contents)
-    let bufferLimit =
-      crossChainState.targetBufferSize -
-      (totalReady -
-      cs->ChainState.fetchState->FetchState.bufferSize)
-    let inFlightBefore = sourceManager->SourceManager.inFlightCount
-    promises->Array.push(fetchChain(~chain, ~concurrencyLimit, ~bufferLimit))
-    inFlight := inFlight.contents + (sourceManager->SourceManager.inFlightCount - inFlightBefore)
+  for i in 0 to chainIds->Array.length - 1 {
+    let chainId = chainIds->Array.getUnsafe(i)
+    switch actionByChain->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
+    | Some(NothingToQuery)
+    | None => ()
+    | Some(action) =>
+      let chain = ChainMap.Chain.makeUnsafe(~chainId)
+      promises->Array.push(dispatchChain(~chain, ~action))
+    }
   }
   let _ = await promises->Promise.all
 }

package/src/CrossChainState.res.mjs

@@ -8,8 +8,9 @@ import * as ChainMap from "./ChainMap.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
+import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
+import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
-import * as SourceManager from "./sources/SourceManager.res.mjs";
 import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
@@ -21,17 +22,7 @@ function calculateTargetBufferSize() {
   }
 }
 
-function maxConcurrency(crossChainState) {
-  if (crossChainState.isRealtime) {
-    return crossChainState.maxRealtimeConcurrency;
-  } else {
-    return crossChainState.maxBackfillConcurrency;
-  }
-}
-
-function make(chainStates, isInReorgThreshold, isRealtime, maxBackfillConcurrencyOpt, maxRealtimeConcurrencyOpt, targetBufferSizeOpt) {
-  let maxBackfillConcurrency = maxBackfillConcurrencyOpt !== undefined ? maxBackfillConcurrencyOpt : Env.maxBackfillConcurrency;
-  let maxRealtimeConcurrency = maxRealtimeConcurrencyOpt !== undefined ? maxRealtimeConcurrencyOpt : Env.maxRealtimeConcurrency;
+function make(chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt) {
   let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : (
       Env.targetBufferSize !== undefined ? Env.targetBufferSize : 100000
     );
@@ -40,11 +31,8 @@ function make(chainStates, isInReorgThreshold, isRealtime, maxBackfillConcurrenc
     chainIds: Object.values(chainStates).map(cs => ChainState.chainConfig(cs).id),
     isRealtime: isRealtime,
     isInReorgThreshold: isInReorgThreshold,
-    maxBackfillConcurrency: maxBackfillConcurrency,
-    maxRealtimeConcurrency: maxRealtimeConcurrency,
     targetBufferSize: targetBufferSize
   };
-  Prometheus.IndexingMaxConcurrency.set(maxConcurrency(crossChainState));
   Prometheus.IndexingTargetBufferSize.set(targetBufferSize);
   return crossChainState;
 }
@@ -61,15 +49,6 @@ function isInReorgThreshold(crossChainState) {
   return crossChainState.isInReorgThreshold;
 }
 
-function inFlight(crossChainState) {
-  let total = 0;
-  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
-    let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
-    total = total + SourceManager.inFlightCount(ChainState.sourceManager(cs)) | 0;
-  }
-  return total;
-}
-
 function totalReadyCount(crossChainState) {
   let total = 0;
   for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
@@ -148,32 +127,79 @@ function applyBatchProgress(crossChainState, batch) {
   if (allChainsReady) {
     Prometheus.ProgressReady.setAllReady();
   }
-  let wasRealtime = crossChainState.isRealtime;
   crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady;
-  if (!wasRealtime && crossChainState.isRealtime) {
-    return Prometheus.IndexingMaxConcurrency.set(maxConcurrency(crossChainState));
-  }
 }
 
 function priorityOrder(crossChainState) {
   return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(FetchState.getProgressPercentage(ChainState.fetchState(a)), FetchState.getProgressPercentage(ChainState.fetchState(b))));
 }
 
-async function checkAndFetch(crossChainState, fetchChain) {
-  let maxConcurrency$1 = maxConcurrency(crossChainState);
-  let totalReady = totalReadyCount(crossChainState);
-  let inFlight$1 = inFlight(crossChainState);
-  let priorityOrdered = priorityOrder(crossChainState);
+function totalReservedSize(crossChainState) {
+  let total = 0;
+  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
+    let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
+    total = total + ChainState.pendingBudget(cs);
+  }
+  return total;
+}
+
+function compareByProgress(a, b) {
+  return Primitive_float.compare(a.progress, b.progress);
+}
+
+async function checkAndFetch(crossChainState, dispatchChain) {
+  let remaining = Primitive_int.max(0, (crossChainState.targetBufferSize - totalReadyCount(crossChainState) | 0) - (totalReservedSize(crossChainState) | 0) | 0);
+  let chainIds = crossChainState.chainIds;
+  let actionByChain = {};
+  let candidates = [];
+  for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
+    let chainId = chainIds[i];
+    let cs = crossChainState.chainStates[chainId];
+    let fetchState = ChainState.fetchState(cs);
+    let action = FetchState.getNextQuery(fetchState, remaining, ChainState.pendingBudget(cs));
+    if (typeof action !== "object") {
+      actionByChain[chainId] = action;
+    } else {
+      actionByChain[chainId] = "NothingToQuery";
+      action._0.forEach(query => {
+        query.chainId = chainId;
+        query.progress = FetchState.getProgressPercentageAt(fetchState, query.fromBlock);
+        candidates.push(query);
+      });
+    }
+  }
+  candidates.sort(compareByProgress);
+  let admittedByChain = {};
+  let running = 0;
+  let idx = 0;
+  while (running < remaining && idx < candidates.length) {
+    let query = candidates[idx];
+    Utils.Dict.push(admittedByChain, query.chainId.toString(), query);
+    running = running + query.estResponseSize;
+    idx = idx + 1 | 0;
+  };
+  Stdlib_Dict.forEachWithKey(admittedByChain, (queries, chainId) => {
+    actionByChain[chainId] = {
+      TAG: "Ready",
+      _0: queries
+    };
+    let chainId$1 = Stdlib_Int.fromString(chainId, undefined);
+    ChainState.startFetchingQueries(crossChainState.chainStates[chainId$1], queries);
+  });
   let promises = [];
-  for (let i = 0, i_finish = priorityOrdered.length; i < i_finish; ++i) {
-    let cs = priorityOrdered[i];
-    let chain = ChainMap.Chain.makeUnsafe(ChainState.chainConfig(cs).id);
-    let sourceManager = ChainState.sourceManager(cs);
-    let concurrencyLimit = Primitive_int.max(0, maxConcurrency$1 - inFlight$1 | 0);
-    let bufferLimit = crossChainState.targetBufferSize - (totalReady - FetchState.bufferSize(ChainState.fetchState(cs)) | 0) | 0;
-    let inFlightBefore = SourceManager.inFlightCount(sourceManager);
-    promises.push(fetchChain(chain, concurrencyLimit, bufferLimit));
-    inFlight$1 = inFlight$1 + (SourceManager.inFlightCount(sourceManager) - inFlightBefore | 0) | 0;
+  for (let i$1 = 0, i_finish$1 = chainIds.length; i$1 < i_finish$1; ++i$1) {
+    let chainId$1 = chainIds[i$1];
+    let action$1 = actionByChain[chainId$1];
+    if (action$1 !== undefined) {
+      let exit = 0;
+      if (typeof action$1 === "object" || action$1 === "WaitingForNewBlock") {
+        exit = 1;
+      }
+      if (exit === 1) {
+        let chain = ChainMap.Chain.makeUnsafe(chainId$1);
+        promises.push(dispatchChain(chain, action$1));
+      }
+    }
   }
   await Promise.all(promises);
 }
@@ -184,8 +210,6 @@ export {
   chainStates,
   isRealtime,
   isInReorgThreshold,
-  maxConcurrency,
-  inFlight,
   nextItemIsNone,
   getSafeCheckpointId,
   createBatch,

package/src/CrossChainState.resi

@@ -9,8 +9,6 @@ let make: (
   ~chainStates: dict<ChainState.t>,
   ~isInReorgThreshold: bool,
   ~isRealtime: bool,
-  ~maxBackfillConcurrency: int=?,
-  ~maxRealtimeConcurrency: int=?,
   ~targetBufferSize: int=?,
 ) => t
 
@@ -18,8 +16,6 @@ let make: (
 let chainStates: t => dict<ChainState.t>
 let isRealtime: t => bool
 let isInReorgThreshold: t => bool
-let maxConcurrency: t => int
-let inFlight: t => int
 
 // Derived (pure).
 let nextItemIsNone: t => bool
@@ -39,9 +35,5 @@ let applyBatchProgress: (t, ~batch: Batch.t) => unit
 let priorityOrder: t => array<ChainState.t>
 let checkAndFetch: (
   t,
-  ~fetchChain: (
-    ~chain: ChainMap.Chain.t,
-    ~concurrencyLimit: int,
-    ~bufferLimit: int,
-  ) => promise<unit>,
+  ~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
 ) => promise<unit>

package/src/Env.res

@@ -8,13 +8,6 @@ let updateSyncTimeOnRestart =
   envSafe->EnvSafe.get("UPDATE_SYNC_TIME_ON_RESTART", S.bool, ~fallback=true)
 let targetBufferSize = envSafe->EnvSafe.get("ENVIO_INDEXING_MAX_BUFFER_SIZE", S.option(S.int))
 let maxAddrInPartition = envSafe->EnvSafe.get("MAX_PARTITION_SIZE", S.int, ~fallback=5_000)
-// Global caps on concurrent data-source queries across the whole indexer (all
-// chains share the budget), not per chain. Realtime allows far more since head
-// queries are small and many chains follow the head at once.
-let maxBackfillConcurrency =
-  envSafe->EnvSafe.get("ENVIO_MAX_BACKFILL_CONCURRENCY", S.int, ~fallback=30)
-let maxRealtimeConcurrency =
-  envSafe->EnvSafe.get("ENVIO_MAX_REALTIME_CONCURRENCY", S.int, ~fallback=200)
 
 // Target number of in-memory objects (uncommitted entity/effect changes plus
 // unwritten batch items) the store holds before processing waits for the write