envio 3.3.0-alpha.0 → 3.3.0-alpha.10

package/src/CrossChainState.res

@@ -11,10 +11,6 @@ type t = {
   // True once every chain has caught up to head/endBlock. Monotonic during a run.
   mutable isRealtime: bool,
   mutable isInReorgThreshold: bool,
-  // Indexer-wide caps on concurrent data-source queries, shared across all
-  // chains. The realtime budget applies once every chain is at head.
-  maxBackfillConcurrency: int,
-  maxRealtimeConcurrency: int,
   // Indexer-wide fetch buffer pool (item count), shared across all chains.
   targetBufferSize: int,
 }
@@ -26,18 +22,10 @@ let calculateTargetBufferSize = () =>
   | None => 100_000
   }
 
-// The concurrency budget in force for the current phase.
-let maxConcurrency = (crossChainState: t) =>
-  crossChainState.isRealtime
-    ? crossChainState.maxRealtimeConcurrency
-    : crossChainState.maxBackfillConcurrency
-
 let make = (
   ~chainStates,
   ~isInReorgThreshold,
   ~isRealtime,
-  ~maxBackfillConcurrency=Env.maxBackfillConcurrency,
-  ~maxRealtimeConcurrency=Env.maxRealtimeConcurrency,
   ~targetBufferSize=calculateTargetBufferSize(),
 ): t => {
   let crossChainState = {
@@ -45,11 +33,8 @@ let make = (
     chainIds: chainStates->Dict.valuesToArray->Array.map(cs => (cs->ChainState.chainConfig).id),
     isRealtime,
     isInReorgThreshold,
-    maxBackfillConcurrency,
-    maxRealtimeConcurrency,
     targetBufferSize,
   }
-  Prometheus.IndexingMaxConcurrency.set(~maxConcurrency=crossChainState->maxConcurrency)
   Prometheus.IndexingTargetBufferSize.set(~targetBufferSize)
   crossChainState
 }
@@ -65,17 +50,6 @@ let chainStates = (crossChainState: t) => crossChainState.chainStates
 let isRealtime = (crossChainState: t) => crossChainState.isRealtime
 let isInReorgThreshold = (crossChainState: t) => crossChainState.isInReorgThreshold
 
-// Partition queries in flight across every chain — the live draw against
-// maxConcurrency.
-let inFlight = (crossChainState: t) => {
-  let total = ref(0)
-  for i in 0 to crossChainState.chainIds->Array.length - 1 {
-    let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
-    total := total.contents + cs->ChainState.sourceManager->SourceManager.inFlightCount
-  }
-  total.contents
-}
-
 // Ready-to-process items across every chain — the live draw against
 // targetBufferSize, which is a budget of processable events (items stuck behind
 // a gap don't count toward the goal of keeping ~targetBufferSize ready).
@@ -83,7 +57,7 @@ let totalReadyCount = (crossChainState: t) => {
   let total = ref(0)
   for i in 0 to crossChainState.chainIds->Array.length - 1 {
     let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
-    total := total.contents + cs->ChainState.fetchState->FetchState.bufferReadyCount
+    total := total.contents + cs->ChainState.bufferReadyCount
   }
   total.contents
 }
@@ -91,9 +65,7 @@ let totalReadyCount = (crossChainState: t) => {
 // --- Derived (pure). ---
 
 let nextItemIsNone = (crossChainState: t): bool =>
-  !Batch.hasReadyItem(
-    crossChainState.chainStates->Dict.valuesToArray->Array.map(ChainState.fetchState),
-  )
+  !(crossChainState.chainStates->Dict.valuesToArray->Array.some(ChainState.hasReadyItem))
 
 let getSafeCheckpointId = (crossChainState: t) => {
   let result: ref<option<bigint>> = ref(None)
@@ -105,7 +77,7 @@ let getSafeCheckpointId = (crossChainState: t) => {
     | Some(safeCheckpointTracking) =>
       let safeCheckpointId =
         safeCheckpointTracking->SafeCheckpointTracking.getSafeCheckpointId(
-          ~sourceBlockNumber=(cs->ChainState.fetchState).knownHeight,
+          ~sourceBlockNumber=cs->ChainState.knownHeight,
         )
       switch result.contents {
       | None => result := Some(safeCheckpointId)
@@ -137,16 +109,9 @@ let createBatch = (
       // in an append-only ClickHouse insert.
       isRollback ? 1n : 0n,
     ),
-    ~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues((
-      cs
-    ): Batch.chainBeforeBatch => {
-      fetchState: cs->ChainState.fetchState,
-      progressBlockNumber: cs->ChainState.committedProgressBlockNumber,
-      totalEventsProcessed: cs->ChainState.numEventsProcessed,
-      sourceBlockNumber: (cs->ChainState.fetchState).knownHeight,
-      reorgDetection: cs->ChainState.reorgDetection,
-      chainConfig: cs->ChainState.chainConfig,
-    }),
+    ~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
+      ChainState.toChainBeforeBatch,
+    ),
     ~batchSizeTarget,
   )
 }
@@ -170,13 +135,13 @@ let enterReorgThreshold = (crossChainState: t) => {
 // whole indexer. A chain is marked caught up only once EVERY chain is caught up
 // (reached endblock or fetched/processed to head) with no processable events
 // left — so no chain flips to ready while another is still backfilling.
-let applyBatchProgress = (crossChainState: t, ~batch: Batch.t) => {
+let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampName: string) => {
   let chainIds = crossChainState.chainIds
 
   let everyChainCaughtUp = ref(true)
   for i in 0 to chainIds->Array.length - 1 {
     let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
-    cs->ChainState.applyBatchProgress(~batch)
+    cs->ChainState.applyBatchProgress(~batch, ~blockTimestampName)
     if !(cs->ChainState.hasProcessedToEndblock || cs->ChainState.isProgressAtHead) {
       everyChainCaughtUp := false
     }
@@ -199,71 +164,155 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t) => {
     Prometheus.ProgressReady.setAllReady()
   }
 
-  let wasRealtime = crossChainState.isRealtime
   crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady.contents
-  if !wasRealtime && crossChainState.isRealtime {
-    // The realtime budget takes over now that every chain is at head.
-    Prometheus.IndexingMaxConcurrency.set(~maxConcurrency=crossChainState->maxConcurrency)
-  }
 }
 
 // --- Fetch control. ---
 
-// Chains ordered furthest-behind first, so the shared concurrency and buffer
-// pools go to the chains with the most backfill work before the rest.
+// Chains ordered furthest-behind first, so the shared buffer pool goes to the
+// chains with the most backfill work before the rest.
 let priorityOrder = (crossChainState: t) =>
   crossChainState.chainStates
   ->Dict.valuesToArray
   ->Array.toSorted((a, b) =>
-    Float.compare(
-      a->ChainState.fetchState->FetchState.getProgressPercentage,
-      b->ChainState.fetchState->FetchState.getProgressPercentage,
-    )
+    Float.compare(a->ChainState.getProgressPercentage, b->ChainState.getProgressPercentage)
   )
 
-// Dispatch a fetch tick across every chain in priority order, drawing from the
-// shared concurrency and buffer pools. Chains are visited in turn; fetchChain
-// bumps the in-flight count synchronously before it suspends, so a later chain
-// sees the slots an earlier one already claimed and the budget is honored
-// indexer-wide.
-//
-// bufferLimit is each chain's slice of the shared pool: it may grow its buffer
-// into whatever the other chains leave free, so a lone backfilling chain can use
-// the whole pool while head-following chains stay shallow.
+// In-flight estimated items across every chain — the live draw against
+// targetBufferSize alongside totalReadyCount, so the pool isn't re-dispatched
+// while queries are still being fetched.
+let totalReservedSize = (crossChainState: t) => {
+  let total = ref(0.)
+  for i in 0 to crossChainState.chainIds->Array.length - 1 {
+    let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
+    total := total.contents +. cs->ChainState.pendingBudget
+  }
+  total.contents
+}
+
+// Dispatch a fetch tick across the whole indexer from one shared pool of
+// ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
+// first, hand each the budget remaining at that point (plus its own
+// already-reserved share, since a chain's pending queries aren't "spent" —
+// they're this chain's), let it turn that into queries sized against its own
+// chain-density-derived target block, then subtract what it actually used
+// before moving to the next chain. So a chain that can only use a little
+// (density too low, or already caught up) leaves the rest for the others
+// automatically. A chain visited after the budget ran out simply doesn't
+// query this round — the leader's reservations release as its responses land,
+// so the next tick redistributes. Chains that do get budget are additionally
+// capped at the most-behind chain's target progress mapped onto their own
+// range, so no chain runs further ahead than the chain the pool is
+// prioritizing.
 let checkAndFetch = async (
   crossChainState: t,
-  ~fetchChain: (
-    ~chain: ChainMap.Chain.t,
-    ~concurrencyLimit: int,
-    ~bufferLimit: int,
-  ) => promise<unit>,
+  ~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
 ) => {
-  let maxConcurrency = crossChainState->maxConcurrency
-  // Pool is a budget of ready-to-process items, but the cap fed to getNextQuery
-  // is still a buffer position (it tracks the sorted buffer, ready items first),
-  // so add this chain's own buffer size back rather than its ready count. A chain
-  // with a gap therefore gets extra headroom to fetch the not-ready overhang
-  // while still aiming for its share of ~targetBufferSize ready events.
-  let totalReady = crossChainState->totalReadyCount
-  // Track the in-flight total as a running counter (summed once up front, then
-  // adjusted by each chain's delta) instead of re-summing every chain — O(chains)
-  // per tick rather than O(chains^2). fetchChain bumps the chain's count
-  // synchronously, so the delta is observable right after the call.
-  let inFlight = ref(crossChainState->inFlight)
-  let priorityOrdered = crossChainState->priorityOrder
+  let remaining = ref(
+    Pervasives.max(
+      0.,
+      crossChainState.targetBufferSize->Int.toFloat -.
+      crossChainState->totalReadyCount->Int.toFloat -.
+      crossChainState->totalReservedSize,
+    ),
+  )
+
+  // A chain with no density signal probes blind, so it only gets a bounded
+  // slice of the pool — one unknown chain shouldn't hold the whole budget
+  // while it takes its first measurements.
+  let coldChainBudget = Pervasives.min(5_000., crossChainState.targetBufferSize->Int.toFloat)
+
+  // Chunk reservations get headroom over the density estimate so a
+  // denser-than-expected range doesn't truncate at the server cap; realtime
+  // gets more since a forced catch-up query there costs a head-poll roundtrip.
+  let chunkItemsMultiplier = crossChainState.isRealtime ? 3. : 1.5
+
+  let actionByChain = Dict.make()
+  let maxProgress = ref(None)
+  crossChainState
+  ->priorityOrder
+  ->Array.forEach(cs => {
+    let chainId = (cs->ChainState.chainConfig).id
+    if cs->ChainState.knownHeight == 0 {
+      // No height yet — nothing to size a budget or an alignment line
+      // against. Skip without consuming budget or claiming leadership, so a
+      // chain whose source hasn't reported doesn't unconstrain everyone else.
+      actionByChain->Utils.Dict.setByInt(chainId, FetchState.WaitingForNewBlock)
+    } else {
+      let isCold = cs->ChainState.effectiveDensity === None
+      let chainTargetItems =
+        (isCold ? Pervasives.min(remaining.contents, coldChainBudget) : remaining.contents) +.
+        cs->ChainState.pendingBudget
+      let maxTargetBlock = switch maxProgress.contents {
+      | None if isCold =>
+        // A cold chain's target is a guess — it doesn't set the alignment
+        // line; the first density-bearing chain after it does.
+        None
+      | None =>
+        // The most-behind chain sets the alignment line for everyone after it.
+        maxProgress :=
+          Some(
+            cs->ChainState.progressAtBlock(
+              ~blockNumber=cs->ChainState.targetBlock(~chainTargetItems),
+            ),
+          )
+        None
+      // 5% margin past the leader's line: chains whose progress tracks the
+      // leader closely would otherwise flap in and out of the clamp as
+      // leadership alternates, stalling their pipeline every other tick.
+      | Some(progress) => Some(cs->ChainState.blockAtProgress(~progress=progress +. 0.05))
+      }
+      switch cs->ChainState.getNextQuery(
+        ~chainTargetItems,
+        ~chunkItemsMultiplier,
+        ~maxTargetBlock?,
+      ) {
+      | (WaitingForNewBlock | NothingToQuery) as action =>
+        actionByChain->Utils.Dict.setByInt(chainId, action)
+      | Ready(queries) => {
+          let consumed =
+            queries->Array.reduce(0., (acc, query: FetchState.query) =>
+              acc +. query.itemsEst->Int.toFloat
+            )
+
+          let partitions = Dict.make()
+          queries->Array.forEach((query: FetchState.query) =>
+            partitions->Dict.set(
+              query.partitionId,
+              {
+                "fromBlock": query.fromBlock,
+                "targetBlock": query.toBlock,
+                "targetEvents": query.itemsTarget,
+              },
+            )
+          )
+          Logging.trace({
+            "msg": "Started querying",
+            "chainId": chainId,
+            "partitions": partitions,
+          })
+
+          actionByChain->Utils.Dict.setByInt(chainId, FetchState.Ready(queries))
+          // Mark the queries in flight and reserve their size against the
+          // shared budget; released as each response lands in
+          // handleQueryResult.
+          cs->ChainState.startFetchingQueries(~queries)
+          remaining := Pervasives.max(0., remaining.contents -. consumed)
+        }
+      }
+    }
+  })
+
   let promises = []
-  for i in 0 to priorityOrdered->Array.length - 1 {
-    let cs = priorityOrdered->Array.getUnsafe(i)
-    let chain = ChainMap.Chain.makeUnsafe(~chainId=(cs->ChainState.chainConfig).id)
-    let sourceManager = cs->ChainState.sourceManager
-    let concurrencyLimit = Pervasives.max(0, maxConcurrency - inFlight.contents)
-    let bufferLimit =
-      crossChainState.targetBufferSize -
-      (totalReady -
-      cs->ChainState.fetchState->FetchState.bufferSize)
-    let inFlightBefore = sourceManager->SourceManager.inFlightCount
-    promises->Array.push(fetchChain(~chain, ~concurrencyLimit, ~bufferLimit))
-    inFlight := inFlight.contents + (sourceManager->SourceManager.inFlightCount - inFlightBefore)
+  for i in 0 to crossChainState.chainIds->Array.length - 1 {
+    let chainId = crossChainState.chainIds->Array.getUnsafe(i)
+    switch actionByChain->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
+    | Some(NothingToQuery)
+    | None => ()
+    | Some(action) =>
+      let chain = ChainMap.Chain.makeUnsafe(~chainId)
+      promises->Array.push(dispatchChain(~chain, ~action))
+    }
   }
   let _ = await promises->Promise.all
 }

package/src/CrossChainState.res.mjs

@@ -6,10 +6,8 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
-import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
-import * as SourceManager from "./sources/SourceManager.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
@@ -21,17 +19,7 @@ function calculateTargetBufferSize() {
   }
 }
 
-function maxConcurrency(crossChainState) {
-  if (crossChainState.isRealtime) {
-    return crossChainState.maxRealtimeConcurrency;
-  } else {
-    return crossChainState.maxBackfillConcurrency;
-  }
-}
-
-function make(chainStates, isInReorgThreshold, isRealtime, maxBackfillConcurrencyOpt, maxRealtimeConcurrencyOpt, targetBufferSizeOpt) {
-  let maxBackfillConcurrency = maxBackfillConcurrencyOpt !== undefined ? maxBackfillConcurrencyOpt : Env.maxBackfillConcurrency;
-  let maxRealtimeConcurrency = maxRealtimeConcurrencyOpt !== undefined ? maxRealtimeConcurrencyOpt : Env.maxRealtimeConcurrency;
+function make(chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt) {
   let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : (
       Env.targetBufferSize !== undefined ? Env.targetBufferSize : 100000
     );
@@ -40,11 +28,8 @@ function make(chainStates, isInReorgThreshold, isRealtime, maxBackfillConcurrenc
     chainIds: Object.values(chainStates).map(cs => ChainState.chainConfig(cs).id),
     isRealtime: isRealtime,
     isInReorgThreshold: isInReorgThreshold,
-    maxBackfillConcurrency: maxBackfillConcurrency,
-    maxRealtimeConcurrency: maxRealtimeConcurrency,
     targetBufferSize: targetBufferSize
   };
-  Prometheus.IndexingMaxConcurrency.set(maxConcurrency(crossChainState));
   Prometheus.IndexingTargetBufferSize.set(targetBufferSize);
   return crossChainState;
 }
@@ -61,26 +46,17 @@ function isInReorgThreshold(crossChainState) {
   return crossChainState.isInReorgThreshold;
 }
 
-function inFlight(crossChainState) {
-  let total = 0;
-  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
-    let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
-    total = total + SourceManager.inFlightCount(ChainState.sourceManager(cs)) | 0;
-  }
-  return total;
-}
-
 function totalReadyCount(crossChainState) {
   let total = 0;
   for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
     let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
-    total = total + FetchState.bufferReadyCount(ChainState.fetchState(cs)) | 0;
+    total = total + ChainState.bufferReadyCount(cs) | 0;
   }
   return total;
 }
 
 function nextItemIsNone(crossChainState) {
-  return !Batch.hasReadyItem(Object.values(crossChainState.chainStates).map(ChainState.fetchState));
+  return !Object.values(crossChainState.chainStates).some(ChainState.hasReadyItem);
 }
 
 function getSafeCheckpointId(crossChainState) {
@@ -89,7 +65,7 @@ function getSafeCheckpointId(crossChainState) {
     let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
     let safeCheckpointTracking = ChainState.safeCheckpointTracking(cs);
     if (safeCheckpointTracking !== undefined) {
-      let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.fetchState(cs).knownHeight);
+      let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.knownHeight(cs));
       let current = result;
       if (!(current !== undefined && safeCheckpointId >= current)) {
         result = safeCheckpointId;
@@ -105,14 +81,7 @@ function getSafeCheckpointId(crossChainState) {
 function createBatch(crossChainState, processedCheckpointId, batchSizeTarget, isRollback) {
   return Batch.make(processedCheckpointId + (
     isRollback ? 1n : 0n
-  ), Utils.Dict.mapValues(crossChainState.chainStates, cs => ({
-    fetchState: ChainState.fetchState(cs),
-    reorgDetection: ChainState.reorgDetection(cs),
-    progressBlockNumber: ChainState.committedProgressBlockNumber(cs),
-    sourceBlockNumber: ChainState.fetchState(cs).knownHeight,
-    totalEventsProcessed: ChainState.numEventsProcessed(cs),
-    chainConfig: ChainState.chainConfig(cs)
-  })), batchSizeTarget, crossChainState.isInReorgThreshold);
+  ), Utils.Dict.mapValues(crossChainState.chainStates, ChainState.toChainBeforeBatch), batchSizeTarget, crossChainState.isInReorgThreshold);
 }
 
 function enterReorgThreshold(crossChainState) {
@@ -124,12 +93,12 @@ function enterReorgThreshold(crossChainState) {
   crossChainState.isInReorgThreshold = true;
 }
 
-function applyBatchProgress(crossChainState, batch) {
+function applyBatchProgress(crossChainState, batch, blockTimestampName) {
   let chainIds = crossChainState.chainIds;
   let everyChainCaughtUp = true;
   for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
     let cs = crossChainState.chainStates[chainIds[i]];
-    ChainState.applyBatchProgress(cs, batch);
+    ChainState.applyBatchProgress(cs, batch, blockTimestampName);
     if (!(ChainState.hasProcessedToEndblock(cs) || ChainState.isProgressAtHead(cs))) {
       everyChainCaughtUp = false;
     }
@@ -148,32 +117,93 @@ function applyBatchProgress(crossChainState, batch) {
   if (allChainsReady) {
     Prometheus.ProgressReady.setAllReady();
   }
-  let wasRealtime = crossChainState.isRealtime;
   crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady;
-  if (!wasRealtime && crossChainState.isRealtime) {
-    return Prometheus.IndexingMaxConcurrency.set(maxConcurrency(crossChainState));
-  }
 }
 
 function priorityOrder(crossChainState) {
-  return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(FetchState.getProgressPercentage(ChainState.fetchState(a)), FetchState.getProgressPercentage(ChainState.fetchState(b))));
+  return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(ChainState.getProgressPercentage(a), ChainState.getProgressPercentage(b)));
+}
+
+function totalReservedSize(crossChainState) {
+  let total = 0;
+  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
+    let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
+    total = total + ChainState.pendingBudget(cs);
+  }
+  return total;
 }
 
-async function checkAndFetch(crossChainState, fetchChain) {
-  let maxConcurrency$1 = maxConcurrency(crossChainState);
-  let totalReady = totalReadyCount(crossChainState);
-  let inFlight$1 = inFlight(crossChainState);
-  let priorityOrdered = priorityOrder(crossChainState);
+async function checkAndFetch(crossChainState, dispatchChain) {
+  let remaining = {
+    contents: Primitive_float.max(0, crossChainState.targetBufferSize - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
+  };
+  let coldChainBudget = Primitive_float.min(5000, crossChainState.targetBufferSize);
+  let chunkItemsMultiplier = crossChainState.isRealtime ? 3 : 1.5;
+  let actionByChain = {};
+  let maxProgress = {
+    contents: undefined
+  };
+  priorityOrder(crossChainState).forEach(cs => {
+    let chainId = ChainState.chainConfig(cs).id;
+    if (ChainState.knownHeight(cs) === 0) {
+      actionByChain[chainId] = "WaitingForNewBlock";
+      return;
+    }
+    let isCold = ChainState.effectiveDensity(cs) === undefined;
+    let chainTargetItems = (
+      isCold ? Primitive_float.min(remaining.contents, coldChainBudget) : remaining.contents
+    ) + ChainState.pendingBudget(cs);
+    let progress = maxProgress.contents;
+    let maxTargetBlock = progress !== undefined ? ChainState.blockAtProgress(cs, progress + 0.05) : (
+        isCold ? undefined : (maxProgress.contents = ChainState.progressAtBlock(cs, ChainState.targetBlock(cs, chainTargetItems)), undefined)
+      );
+    let action = ChainState.getNextQuery(cs, chainTargetItems, chunkItemsMultiplier, maxTargetBlock);
+    if (typeof action !== "object") {
+      if (action === "WaitingForNewBlock") {
+        actionByChain[chainId] = action;
+        return;
+      }
+      actionByChain[chainId] = action;
+      return;
+    } else {
+      let queries = action._0;
+      let consumed = Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsEst);
+      let partitions = {};
+      queries.forEach(query => {
+        partitions[query.partitionId] = {
+          fromBlock: query.fromBlock,
+          targetBlock: query.toBlock,
+          targetEvents: query.itemsTarget
+        };
+      });
+      Logging.trace({
+        msg: "Started querying",
+        chainId: chainId,
+        partitions: partitions
+      });
+      actionByChain[chainId] = {
+        TAG: "Ready",
+        _0: queries
+      };
+      ChainState.startFetchingQueries(cs, queries);
+      remaining.contents = Primitive_float.max(0, remaining.contents - consumed);
+      return;
+    }
+  });
   let promises = [];
-  for (let i = 0, i_finish = priorityOrdered.length; i < i_finish; ++i) {
-    let cs = priorityOrdered[i];
-    let chain = ChainMap.Chain.makeUnsafe(ChainState.chainConfig(cs).id);
-    let sourceManager = ChainState.sourceManager(cs);
-    let concurrencyLimit = Primitive_int.max(0, maxConcurrency$1 - inFlight$1 | 0);
-    let bufferLimit = crossChainState.targetBufferSize - (totalReady - FetchState.bufferSize(ChainState.fetchState(cs)) | 0) | 0;
-    let inFlightBefore = SourceManager.inFlightCount(sourceManager);
-    promises.push(fetchChain(chain, concurrencyLimit, bufferLimit));
-    inFlight$1 = inFlight$1 + (SourceManager.inFlightCount(sourceManager) - inFlightBefore | 0) | 0;
+  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
+    let chainId = crossChainState.chainIds[i];
+    let action = actionByChain[chainId];
+    if (action !== undefined) {
+      let exit = 0;
+      if (typeof action === "object" || action === "WaitingForNewBlock") {
+        exit = 1;
+      }
+      if (exit === 1) {
+        let chain = ChainMap.Chain.makeUnsafe(chainId);
+        promises.push(dispatchChain(chain, action));
+      }
+    }
   }
   await Promise.all(promises);
 }
@@ -184,8 +214,6 @@ export {
   chainStates,
   isRealtime,
   isInReorgThreshold,
-  maxConcurrency,
-  inFlight,
   nextItemIsNone,
   getSafeCheckpointId,
   createBatch,

package/src/CrossChainState.resi

@@ -9,8 +9,6 @@ let make: (
   ~chainStates: dict<ChainState.t>,
   ~isInReorgThreshold: bool,
   ~isRealtime: bool,
-  ~maxBackfillConcurrency: int=?,
-  ~maxRealtimeConcurrency: int=?,
   ~targetBufferSize: int=?,
 ) => t
 
@@ -18,8 +16,6 @@ let make: (
 let chainStates: t => dict<ChainState.t>
 let isRealtime: t => bool
 let isInReorgThreshold: t => bool
-let maxConcurrency: t => int
-let inFlight: t => int
 
 // Derived (pure).
 let nextItemIsNone: t => bool
@@ -33,15 +29,11 @@ let createBatch: (
   ~isRollback: bool,
 ) => Batch.t
 let enterReorgThreshold: t => unit
-let applyBatchProgress: (t, ~batch: Batch.t) => unit
+let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
 
 // Fetch control.
 let priorityOrder: t => array<ChainState.t>
 let checkAndFetch: (
   t,
-  ~fetchChain: (
-    ~chain: ChainMap.Chain.t,
-    ~concurrencyLimit: int,
-    ~bufferLimit: int,
-  ) => promise<unit>,
+  ~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
 ) => promise<unit>

package/src/Ecosystem.res

@@ -2,8 +2,6 @@ type name = | @as("evm") Evm | @as("fuel") Fuel | @as("svm") Svm
 
 type t = {
   name: name,
-  blockFields: array<string>,
-  transactionFields: array<string>,
   blockNumberName: string,
   blockTimestampName: string,
   blockHashName: string,
@@ -33,7 +31,8 @@ type t = {
       loggers (see `getItemLogger`). */
   logger: Pino.t,
   /** Materialise the user-facing event handed to handlers and contract
-      registration from an item's opaque payload. */
+      registration from an item's opaque payload. `event.transaction` is written
+      onto the payload at batch prep (HyperSync) or inline (RPC/simulate). */
   toEvent: Internal.eventItem => Internal.event,
   /** Build the per-item child logger for an event item, with
       ecosystem-specific log fields (EVM/Fuel: contract/event/address; SVM:
@@ -70,12 +69,12 @@ let getItemLogger = {
     | None =>
       let logger = switch item {
       | Internal.Event(_) => ecosystem.toEventLogger(item->Internal.castUnsafeEventItem)
-      | Block({blockNumber, onBlockConfig}) =>
+      | Block({blockNumber, onBlockRegistration}) =>
         Logging.createChildFrom(
           ~logger=ecosystem.logger,
           ~params={
-            "onBlock": onBlockConfig.name,
-            "chainId": onBlockConfig.chainId,
+            "onBlock": onBlockRegistration.name,
+            "chainId": onBlockRegistration.chainId,
             "block": blockNumber,
           },
         )

package/src/Ecosystem.res.mjs

@@ -26,10 +26,10 @@ function getItemLogger(item, ecosystem) {
   if (item.kind === 0) {
     logger$1 = ecosystem.toEventLogger(item);
   } else {
-    let onBlockConfig = item.onBlockConfig;
+    let onBlockRegistration = item.onBlockRegistration;
     logger$1 = Logging.createChildFrom(ecosystem.logger, {
-      onBlock: onBlockConfig.name,
-      chainId: onBlockConfig.chainId,
+      onBlock: onBlockRegistration.name,
+      chainId: onBlockRegistration.chainId,
       block: item.blockNumber
     });
   }

package/src/Env.res

@@ -8,13 +8,6 @@ let updateSyncTimeOnRestart =
   envSafe->EnvSafe.get("UPDATE_SYNC_TIME_ON_RESTART", S.bool, ~fallback=true)
 let targetBufferSize = envSafe->EnvSafe.get("ENVIO_INDEXING_MAX_BUFFER_SIZE", S.option(S.int))
 let maxAddrInPartition = envSafe->EnvSafe.get("MAX_PARTITION_SIZE", S.int, ~fallback=5_000)
-// Global caps on concurrent data-source queries across the whole indexer (all
-// chains share the budget), not per chain. Realtime allows far more since head
-// queries are small and many chains follow the head at once.
-let maxBackfillConcurrency =
-  envSafe->EnvSafe.get("ENVIO_MAX_BACKFILL_CONCURRENCY", S.int, ~fallback=30)
-let maxRealtimeConcurrency =
-  envSafe->EnvSafe.get("ENVIO_MAX_REALTIME_CONCURRENCY", S.int, ~fallback=200)
 
 // Target number of in-memory objects (uncommitted entity/effect changes plus
 // unwritten batch items) the store holds before processing waits for the write