envio 3.3.0-alpha.6 → 3.3.0-alpha.8

package/src/FetchState.res

@@ -7,15 +7,19 @@ type blockNumberAndTimestamp = {
 
 type blockNumberAndLogIndex = {blockNumber: int, logIndex: int}
 
-type selection = {eventConfigs: array<Internal.eventConfig>, dependsOnAddresses: bool}
+type selection = {
+  onEventRegistrations: array<Internal.onEventRegistration>,
+  dependsOnAddresses: bool,
+}
 
 type pendingQuery = {
   fromBlock: int,
   toBlock: option<int>,
   isChunk: bool,
-  // Estimated items this in-flight query will add, carried from the query so the
-  // shared buffer budget can account for what's already being fetched.
-  estResponseSize: float,
+  // Items this in-flight query is targeting (server maxNumLogs-style cap), carried
+  // from the query so the shared buffer budget can account for what's already
+  // being fetched.
+  itemsTarget: float,
   // Stores latestFetchedBlock when query completes. Only needed to persist
   // timestamp while earlier queries are still pending before updating
   // the partition's latestFetchedBlock.
@@ -59,15 +63,12 @@ type query = {
   fromBlock: int,
   toBlock: option<int>,
   isChunk: bool,
-  // Estimated number of items the query will return, from the partition's
-  // density and the query's block range. Used to admit queries against the
-  // shared buffer budget.
-  estResponseSize: float,
-  // Owning chain and the chain progress % at the query's fromBlock. Set by the
-  // cross-chain scheduler so candidate queries can be pooled and ordered
-  // (furthest-behind first) without allocating a side tuple per query.
-  mutable chainId: int,
-  mutable progress: float,
+  // Items this query targets: the server-side maxNumLogs-style cap, and the
+  // unit the chain's per-tick budget is reserved/consumed in. For a partition
+  // with known density this is density × the query's block range; for a
+  // partition with no signal yet it's whatever budget share the query was
+  // sized against.
+  itemsTarget: float,
   selection: selection,
   addressesByContractName: dict<array<Address.t>>,
 }
@@ -90,24 +91,28 @@ let deriveContractNameByAddress: dict<array<Address.t>> => dict<
   result
 })
 
-// Default estimate for a query whose partition hasn't responded yet, so the
-// shared budget still accounts for unknown queries instead of treating them as
-// free.
-let defaultEstResponseSize = 10_000.
-
-// Estimated items a query will return, from the partition's event density
-// (items/block derived from its last response) and the query's block range.
-// toBlock None is the open-ended tail, capped at maxQueryBlockNumber. A partition
-// that responded with no items has density 0, so its queries cost 0 — correct,
-// they don't fill the buffer. Only a partition that has never responded
-// (prevQueryRange 0) has no signal, so it falls back to defaultEstResponseSize.
-let calculateEstResponseSize = (p: partition, ~fromBlock, ~toBlock, ~maxQueryBlockNumber) =>
-  if p.prevQueryRange > 0 {
-    let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
-    (toBlock->Option.getOr(maxQueryBlockNumber) - fromBlock + 1)->Int.toFloat *. density
-  } else {
-    defaultEstResponseSize
-  }
+// Floor for a query's itemsTarget. Also used by SourceManager as the floor for
+// its own maxNumLogs-style safety net.
+let minItemsTarget = 2_000.
+
+// Ceiling for an unknown-density probe query's itemsTarget. A partition with no
+// trusted density yet gets one open-ended probe; without a cap it would claim its
+// full even share of the chain's (possibly whole-pool) budget and starve sibling
+// chains needing their own first probe in the same cross-chain tick.
+let maxItemsTarget = 10_000.
+
+// itemsTarget for a query over [fromBlock, toBlock], from the partition's event
+// density (items/block derived from its last response). toBlock None is the
+// open-ended tail, capped at chainTargetBlock — the soft per-tick horizon the
+// owning chain wants to reach (see getNextQuery). A partition that responded
+// with no items has density 0, so its queries cost 0 — correct, they don't
+// fill the buffer. Only called for partitions with a known density
+// (prevQueryRange > 0); partitions with no signal yet are sized instead
+// against their even split of the chain's range budget.
+let densityItemsTarget = (p: partition, ~fromBlock, ~toBlock, ~chainTargetBlock) => {
+  let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
+  (toBlock->Option.getOr(chainTargetBlock) - fromBlock + 1)->Int.toFloat *. density
+}
 
 // Calculate the chunk range from history using min-of-last-3-ranges heuristic
 let getMinHistoryRange = (p: partition) => {
@@ -545,10 +550,10 @@ type t = {
   // Buffer of items ordered from earliest to latest
   buffer: array<Internal.item>,
   // Caps how far ahead onBlock items are pre-generated (set to 2x the batch
-  // size). Fetch depth is bounded separately by getNextQuery's itemBudget, the
-  // chain's per-tick slice of the indexer-wide pool.
+  // size). Event fetch depth is bounded separately, by CrossChainState's
+  // cross-chain admission against the indexer-wide buffer pool.
   maxOnBlockBufferSize: int,
-  onBlockConfigs: array<Internal.onBlockConfig>,
+  onBlockRegistrations: array<Internal.onBlockRegistration>,
   knownHeight: int,
   firstEventBlock: option<int>,
 }
@@ -624,7 +629,7 @@ let blockItemLogIndex = 16777216
 // are generated at once to prevent OOM.
 let appendOnBlockItems = (
   ~mutItems: array<Internal.item>,
-  ~onBlockConfigs: array<Internal.onBlockConfig>,
+  ~onBlockRegistrations: array<Internal.onBlockRegistration>,
   ~indexerStartBlock,
   ~fromBlock,
   ~maxBlockNumber,
@@ -644,27 +649,27 @@ let appendOnBlockItems = (
     let blockNumber = latestOnBlockBlockNumber.contents + 1
     latestOnBlockBlockNumber := blockNumber
 
-    for configIdx in 0 to onBlockConfigs->Array.length - 1 {
-      let onBlockConfig = onBlockConfigs->Array.getUnsafe(configIdx)
+    for configIdx in 0 to onBlockRegistrations->Array.length - 1 {
+      let onBlockRegistration = onBlockRegistrations->Array.getUnsafe(configIdx)
 
-      let handlerStartBlock = switch onBlockConfig.startBlock {
+      let handlerStartBlock = switch onBlockRegistration.startBlock {
       | Some(startBlock) => startBlock
       | None => indexerStartBlock
       }
 
       if (
         blockNumber >= handlerStartBlock &&
-        switch onBlockConfig.endBlock {
+        switch onBlockRegistration.endBlock {
         | Some(endBlock) => blockNumber <= endBlock
         | None => true
         } &&
-        (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockConfig.interval) === 0
+        (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockRegistration.interval) === 0
       ) {
         mutItems->Array.push(
           Block({
-            onBlockConfig,
+            onBlockRegistration,
             blockNumber,
-            logIndex: blockItemLogIndex + onBlockConfig.index,
+            logIndex: blockItemLogIndex + onBlockRegistration.index,
           }),
         )
         newItemsCounter := newItemsCounter.contents + 1
@@ -688,9 +693,9 @@ let updateInternal = (
 ): t => {
   let mutItemsRef = ref(mutItems)
 
-  let latestOnBlockBlockNumber = switch fetchState.onBlockConfigs {
+  let latestOnBlockBlockNumber = switch fetchState.onBlockRegistrations {
   | [] => knownHeight
-  | onBlockConfigs => {
+  | onBlockRegistrations => {
       // Calculate the max block number we are going to create items for
       // Use maxOnBlockBufferSize to get the last target item in the buffer
       //
@@ -718,7 +723,7 @@ let updateInternal = (
 
       appendOnBlockItems(
         ~mutItems,
-        ~onBlockConfigs,
+        ~onBlockRegistrations,
         ~indexerStartBlock=fetchState.startBlock,
         ~fromBlock=fetchState.latestOnBlockBlockNumber,
         ~maxBlockNumber,
@@ -733,7 +738,7 @@ let updateInternal = (
     contractConfigs: fetchState.contractConfigs,
     normalSelection: fetchState.normalSelection,
     chainId: fetchState.chainId,
-    onBlockConfigs: fetchState.onBlockConfigs,
+    onBlockRegistrations: fetchState.onBlockRegistrations,
     maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     optimizedPartitions,
     latestOnBlockBlockNumber,
@@ -999,7 +1004,7 @@ let registerDynamicContracts = (
   // Might contain duplicates which we should filter out
   items: array<Internal.item>,
 ) => {
-  if fetchState.normalSelection.eventConfigs->Utils.Array.isEmpty {
+  if fetchState.normalSelection.onEventRegistrations->Utils.Array.isEmpty {
     // Can the normalSelection be empty?
     JsError.throwWithMessage(
       "Invalid configuration. No events to fetch for the dynamic contract registration.",
@@ -1283,8 +1288,8 @@ let handleQueryResult = (
   // param-level analogue of EventRouter's srcAddress effectiveStartBlock check.
   let newItems = newItems->Array.filter(item =>
     switch item {
-    | Internal.Event({eventConfig, payload, blockNumber}) =>
-      switch eventConfig.clientAddressFilter {
+    | Internal.Event({onEventRegistration, payload, blockNumber}) =>
+      switch onEventRegistration.clientAddressFilter {
       | Some(filter) =>
         filter(payload, blockNumber, indexingAddresses->IndexingAddresses.rawForFilter)
       | None => true
@@ -1322,7 +1327,7 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
       fromBlock: q.fromBlock,
       toBlock: q.toBlock,
       isChunk: q.isChunk,
-      estResponseSize: q.estResponseSize,
+      itemsTarget: q.itemsTarget,
       fetchedBlock: None,
     }
 
@@ -1346,45 +1351,50 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
 // Most parallel in-flight chunk queries a single partition may have at once.
 let maxPendingChunksPerPartition = 10
 
-@inline
-let pushQueriesForRange = (
+// Fills a gap range (a hole left between completed/pending chunks, e.g. from an
+// out-of-order partial response) unconditionally against the partition's real
+// density — gaps are already-committed range, not subject to this tick's
+// water-fill budget. Returns the created queries' total itemsTarget.
+let pushGapFillQueries = (
   queries: array<query>,
   ~partitionId: string,
   ~rangeFromBlock: int,
   ~rangeEndBlock: option<int>,
-  ~maxQueryBlockNumber: int,
+  ~knownHeight: int,
+  ~chainTargetBlock: int,
   ~maybeChunkRange: option<int>,
   ~maxChunks: int,
   ~partition: partition,
   ~selection: selection,
   ~addressesByContractName: dict<array<Address.t>>,
 ) => {
-  if rangeFromBlock <= maxQueryBlockNumber && maxChunks > 0 {
+  let cost = ref(0.)
+  if rangeFromBlock <= knownHeight && maxChunks > 0 {
     switch rangeEndBlock {
     | Some(endBlock) if rangeFromBlock > endBlock => ()
     | _ =>
       switch maybeChunkRange {
       | None =>
+        let itemsTarget = densityItemsTarget(
+          partition,
+          ~fromBlock=rangeFromBlock,
+          ~toBlock=rangeEndBlock,
+          ~chainTargetBlock,
+        )
         queries->Array.push({
           partitionId,
           fromBlock: rangeFromBlock,
           toBlock: rangeEndBlock,
           selection,
           isChunk: false,
-          estResponseSize: calculateEstResponseSize(
-            partition,
-            ~fromBlock=rangeFromBlock,
-            ~toBlock=rangeEndBlock,
-            ~maxQueryBlockNumber,
-          ),
-          chainId: 0,
-          progress: 0.,
+          itemsTarget,
           addressesByContractName,
         })
+        cost := cost.contents +. itemsTarget
       | Some(chunkRange) =>
         let maxBlock = switch rangeEndBlock {
         | Some(eb) => eb
-        | None => maxQueryBlockNumber
+        | None => chainTargetBlock
         }
         let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
         if rangeFromBlock + chunkSize * 2 - 1 <= maxBlock {
@@ -1394,102 +1404,119 @@ let pushQueriesForRange = (
             chunkIdx.contents < maxChunks && chunkFromBlock.contents + chunkSize - 1 <= maxBlock
           ) {
             let chunkToBlock = chunkFromBlock.contents + chunkSize - 1
+            let itemsTarget = densityItemsTarget(
+              partition,
+              ~fromBlock=chunkFromBlock.contents,
+              ~toBlock=Some(chunkToBlock),
+              ~chainTargetBlock,
+            )
             queries->Array.push({
               partitionId,
               fromBlock: chunkFromBlock.contents,
               toBlock: Some(chunkToBlock),
               isChunk: true,
               selection,
-              estResponseSize: calculateEstResponseSize(
-                partition,
-                ~fromBlock=chunkFromBlock.contents,
-                ~toBlock=Some(chunkToBlock),
-                ~maxQueryBlockNumber,
-              ),
-              chainId: 0,
-              progress: 0.,
+              itemsTarget,
               addressesByContractName,
             })
+            cost := cost.contents +. itemsTarget
             chunkFromBlock := chunkToBlock + 1
             chunkIdx := chunkIdx.contents + 1
           }
         } else {
           // Not enough room for 2 chunks, fall back to a single query
+          let itemsTarget = densityItemsTarget(
+            partition,
+            ~fromBlock=rangeFromBlock,
+            ~toBlock=rangeEndBlock,
+            ~chainTargetBlock,
+          )
           queries->Array.push({
             partitionId,
             fromBlock: rangeFromBlock,
             toBlock: rangeEndBlock,
             selection,
             isChunk: rangeEndBlock !== None,
-            estResponseSize: calculateEstResponseSize(
-              partition,
-              ~fromBlock=rangeFromBlock,
-              ~toBlock=rangeEndBlock,
-              ~maxQueryBlockNumber,
-            ),
-            chainId: 0,
-            progress: 0.,
+            itemsTarget,
             addressesByContractName,
           })
+          cost := cost.contents +. itemsTarget
         }
       }
     }
   }
+  cost.contents
 }
 
+// Per-partition mutable state threaded through the water-fill rounds below.
+type waterFillState = {
+  partitionId: string,
+  p: partition,
+  mutable cursor: int,
+  mutable chunksUsedThisCall: int,
+  // Existing mutPendingQueries count before this call — fixed for the call,
+  // used with chunksUsedThisCall against maxPendingChunksPerPartition.
+  pendingCount: int,
+  queryEndBlock: option<int>,
+  maybeChunkRange: option<int>,
+  // This partition's own slot in queriesByPartitionIndex — gap-fill and every
+  // water-fill round push here directly, so query order falls out of
+  // idsInAscOrder for free (see getNextQuery) instead of a final sort pass.
+  bucket: array<query>,
+}
+
+// Candidate queries are sized against chainTargetBlock, the soft querying
+// horizon the owning chain wants to reach this tick — derived by ChainState
+// from its share of the indexer-wide buffer budget and its chain-level event
+// density. chainTargetBlock is never used as a hard query end, only to (a)
+// select which partitions are "in range" this tick and (b) size an open-ended
+// query with no other ceiling: the true hard bounds stay
+// endBlock/mergeBlock/the lagged head.
+//
+// In-range partitions evenly split chainTargetItems (this chain's target
+// total footprint — in-flight plus new). A partition already holding more
+// than its even share (from earlier ticks' in-flight queries) is skipped this
+// round so its share flows to the others; the split is recomputed every round
+// against the shrinking set of partitions still with range left, so leftover
+// keeps redistributing until the budget or the range runs out.
+//
+// A partition with a trusted density (two or more responses — see
+// getMinHistoryRange) always emits at least one full-size chunk/query once
+// included in a round, sized by its real density — uncapped, may overshoot
+// that round's share (the server also enforces itemsTarget via a
+// maxNumLogs-style cap, so an overshoot truncates the response rather than
+// the buffer). A partition with no trusted density yet (zero or one
+// response — a single sample is too noisy to size by) emits one query sized
+// exactly to its round's share instead.
 let getNextQuery = (
-  {buffer, optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight} as fetchState: t,
-  ~budget,
-  ~chainPendingBudget,
+  {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
+  ~chainTargetBlock: int,
+  ~chainTargetItems: float,
 ) => {
   let headBlockNumber = knownHeight - blockLag
   if headBlockNumber <= 0 {
     WaitingForNewBlock
-  } else if budget <= 0 {
-    // No room left in the shared buffer pool for this chain; wait for processing
-    // to drain before fetching more.
-    NothingToQuery
   } else {
     let isOnBlockBehindTheHead = latestOnBlockBlockNumber < headBlockNumber
     let shouldWaitForNewBlock = ref(
-      switch fetchState.endBlock {
+      switch endBlock {
       | Some(endBlock) => headBlockNumber < endBlock
       | None => true
       } &&
       !isOnBlockBehindTheHead,
     )
 
-    // Fetch at most `budget` items past the ready frontier (plus what's already
-    // in flight) so processing always has buffer without ballooning memory.
-    // budget already excludes items at/below the frontier (they're in the shared
-    // totalReadyCount), so offset the index by bufferReadyCount — otherwise the
-    // ready prefix is subtracted twice and the buffer caps at a fraction of its
-    // target. A partition that fetched further is skipped until the buffer drains.
-    let maxQueryBlockNumber = {
-      switch buffer->Array.get(
-        fetchState->bufferReadyCount + budget + chainPendingBudget->Float.toInt - 1,
-      ) {
-      | Some(item) =>
-        // Just in case check that we don't query beyond the current block
-        Pervasives.min(item->Internal.getItemBlockNumber, knownHeight)
-      | None => knownHeight
-      }
-    }
-
-    let queries = []
-
     let partitionsCount = optimizedPartitions.idsInAscOrder->Array.length
-    let idxRef = ref(0)
-    while idxRef.contents < partitionsCount {
-      let idx = idxRef.contents
+
+    // Every partition is visited once here regardless of whether it gets a
+    // query pushed below — waiting-for-new-block bookkeeping shouldn't depend
+    // on this tick's budget.
+    for idx in 0 to partitionsCount - 1 {
       let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
       let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
-
-      let pendingCount = p.mutPendingQueries->Array.length
-      let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber
-      let hasPendingQueries = pendingCount > 0
-
-      if hasPendingQueries || isBehindTheHead {
+      if (
+        p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock.blockNumber < headBlockNumber
+      ) {
         // Even if there are some partitions waiting for the new block
         // We still want to wait for all partitions reaching the head
         // because they might update knownHeight in their response
@@ -1497,54 +1524,84 @@ let getNextQuery = (
         // and we don't want to poll the head for a few small partitions
         shouldWaitForNewBlock := false
       }
+    }
 
-      let partitionQueriesStart = queries->Array.length
-
-      // Compute queryEndBlock for this partition
-      let queryEndBlock = Utils.Math.minOptInt(fetchState.endBlock, p.mergeBlock)
-      let queryEndBlock = switch blockLag {
+    // One bucket per partition, in idsInAscOrder order — gap-fill and
+    // water-fill both push into a partition's own bucket, so flattening at
+    // the end (see below) reproduces idsInAscOrder without a sort.
+    let queriesByPartitionIndex: array<
+      array<query>,
+    > = Array.fromInitializer(~length=partitionsCount, _ => [])
+
+    // Compute queryEndBlock for this partition
+    let computeQueryEndBlock = (p: partition) => {
+      let queryEndBlock = Utils.Math.minOptInt(endBlock, p.mergeBlock)
+      switch blockLag {
       | 0 => queryEndBlock
       | _ =>
         // Force head block as an endBlock when blockLag is set
         // because otherwise HyperSync might return bigger range
         Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock)
       }
-      // Enforce the response range up until target block
-      // Otherwise for indexers with 100+ partitions
-      // we might blow up the buffer size to more than 600k events
-      // simply because of HyperSync returning extra blocks
-      let queryEndBlock = switch (queryEndBlock, maxQueryBlockNumber < knownHeight) {
-      | (Some(endBlock), true) => Some(Pervasives.min(maxQueryBlockNumber, endBlock))
-      | (None, true) => Some(maxQueryBlockNumber)
-      | (_, false) => queryEndBlock
+    }
+
+    // Each partition's existing in-flight itemsTarget, summed once and reused
+    // both for chainReserved (the call-wide fresh-budget ceiling below) and to
+    // seed each partition's water-fill footprint — so a partition already
+    // holding in-flight queries is counted toward its even share and doesn't
+    // get topped up past the others.
+    let existingReservedByPartition = Dict.make()
+    let chainReserved = ref(0.)
+    for idx in 0 to partitionsCount - 1 {
+      let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
+      let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
+      let cost = ref(0.)
+      for pqIdx in 0 to p.mutPendingQueries->Array.length - 1 {
+        cost := cost.contents +. (p.mutPendingQueries->Array.getUnsafe(pqIdx)).itemsTarget
       }
+      existingReservedByPartition->Dict.set(partitionId, cost.contents)
+      chainReserved := chainReserved.contents +. cost.contents
+    }
 
+    // Phase A: gap-fill. Walk each partition's pending queries once,
+    // unconditionally filling any hole (e.g. from an out-of-order partial
+    // chunk response) — uncapped, same as before this tick's water-fill was
+    // introduced. This also determines each partition's post-gap cursor and
+    // whether it's blocked on an unresolved single-shot query.
+    let fillStates = []
+    let gapFillCost = ref(0.)
+    for idx in 0 to partitionsCount - 1 {
+      let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
+      let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
+      let bucket = queriesByPartitionIndex->Array.getUnsafe(idx)
+      let pendingCount = p.mutPendingQueries->Array.length
+      let queryEndBlock = computeQueryEndBlock(p)
       let maybeChunkRange = getMinHistoryRange(p)
 
-      // Walk pending queries to find open ranges and create queries for each
       let cursor = ref(p.latestFetchedBlock.blockNumber + 1)
       let canContinue = ref(true)
+      let chunksUsedThisCall = ref(0)
       let pqIdx = ref(0)
-      while pqIdx.contents < p.mutPendingQueries->Array.length && canContinue.contents {
+      while pqIdx.contents < pendingCount && canContinue.contents {
         let pq = p.mutPendingQueries->Array.getUnsafe(pqIdx.contents)
 
-        // Gap before this pending query → create queries for the gap range
         if pq.fromBlock > cursor.contents {
-          pushQueriesForRange(
-            queries,
+          let beforeLen = bucket->Array.length
+          let cost = pushGapFillQueries(
+            bucket,
             ~partitionId,
             ~rangeFromBlock=cursor.contents,
             ~rangeEndBlock=Utils.Math.minOptInt(Some(pq.fromBlock - 1), queryEndBlock),
-            ~maxQueryBlockNumber,
+            ~knownHeight,
+            ~chainTargetBlock,
             ~maybeChunkRange,
-            ~maxChunks=maxPendingChunksPerPartition -
-            pendingCount -
-            (queries->Array.length -
-            partitionQueriesStart),
+            ~maxChunks=maxPendingChunksPerPartition - pendingCount - chunksUsedThisCall.contents,
             ~partition=p,
             ~selection=p.selection,
             ~addressesByContractName=p.addressesByContractName,
           )
+          chunksUsedThisCall := chunksUsedThisCall.contents + (bucket->Array.length - beforeLen)
+          gapFillCost := gapFillCost.contents +. cost
         }
         switch pq {
         | {isChunk: true, toBlock: Some(toBlock), fetchedBlock: Some({blockNumber})}
@@ -1556,28 +1613,180 @@ let getNextQuery = (
         pqIdx := pqIdx.contents + 1
       }
 
-      // Tail range after all pending queries
       if canContinue.contents {
-        pushQueriesForRange(
-          queries,
-          ~partitionId,
-          ~rangeFromBlock=cursor.contents,
-          ~rangeEndBlock=queryEndBlock,
-          ~maxQueryBlockNumber,
-          ~maybeChunkRange,
-          ~maxChunks=maxPendingChunksPerPartition -
-          pendingCount -
-          (queries->Array.length -
-          partitionQueriesStart),
-          ~partition=p,
-          ~selection=p.selection,
-          ~addressesByContractName=p.addressesByContractName,
-        )
+        fillStates
+        ->Array.push({
+          partitionId,
+          p,
+          cursor: cursor.contents,
+          chunksUsedThisCall: chunksUsedThisCall.contents,
+          pendingCount,
+          queryEndBlock,
+          maybeChunkRange,
+          bucket,
+        })
+        ->ignore
       }
+    }
 
-      idxRef := idxRef.contents + 1
+    // Fresh budget for this call — what's left of chainTargetItems after
+    // existing in-flight queries and this tick's gap-fill. The water-fill
+    // rounds below hand it out; a partition already holding a large share
+    // (seeded below) gets proportionally less so totals stay even.
+    let rangeItemsTarget = Pervasives.max(
+      0.,
+      chainTargetItems -. chainReserved.contents -. gapFillCost.contents,
+    )
+
+    // Each in-range partition's running footprint: existing in-flight plus any
+    // gap-fill just pushed into its bucket. The water-fill levels every
+    // partition up toward a shared line, so this seed is what a partition
+    // starts the fill already holding.
+    let reservedByPartition = Dict.make()
+    fillStates->Array.forEach(fs => {
+      let cost = ref(existingReservedByPartition->Dict.getUnsafe(fs.partitionId))
+      for i in 0 to fs.bucket->Array.length - 1 {
+        cost := cost.contents +. (fs.bucket->Array.getUnsafe(i)).itemsTarget
+      }
+      reservedByPartition->Dict.set(fs.partitionId, cost.contents)
+    })
+
+    // Emits this round's queries for one partition, given its share of the
+    // range budget. Mutates fs.cursor/chunksUsedThisCall and returns the
+    // itemsTarget consumed.
+    //
+    // Density is trusted only once the chunking heuristic is active
+    // (maybeChunkRange requires prevQueryRange AND prevPrevQueryRange, i.e.
+    // two responses — see getMinHistoryRange) — a single response is noisy
+    // (a first query that happens to land on an unusually dense/sparse block
+    // would otherwise mis-size the very next one), so it's treated the same
+    // as a partition with no signal at all: sized exactly to this round's
+    // share instead of an uncapped, one-sample density estimate.
+    let emitQueries = (fs: waterFillState, ~budget: float) => {
+      let p = fs.p
+      let maxBlock = switch fs.queryEndBlock {
+      | Some(eb) => eb
+      | None => chainTargetBlock
+      }
+      switch fs.maybeChunkRange {
+      | Some(minHistoryRange) =>
+        // Chunking active: strict chunks with a hard endBlock, uncapped real
+        // density — at least one full chunk this round even if budget falls
+        // short.
+        let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
+        let chunkSize = Js.Math.ceil_int(minHistoryRange->Int.toFloat *. 1.8)
+        let chunkCost = density *. chunkSize->Int.toFloat
+        let maxChunksRemaining =
+          maxPendingChunksPerPartition - fs.pendingCount - fs.chunksUsedThisCall
+        let affordable = if chunkCost > 0. {
+          Math.floor(budget /. chunkCost)->Float.toInt
+        } else {
+          maxChunksRemaining
+        }
+        let numChunks = Pervasives.max(1, Pervasives.min(affordable, maxChunksRemaining))
+        let consumed = ref(0.)
+        let created = ref(0)
+        let chunkFromBlock = ref(fs.cursor)
+        while created.contents < numChunks && chunkFromBlock.contents <= maxBlock {
+          let chunkToBlock = Pervasives.min(chunkFromBlock.contents + chunkSize - 1, maxBlock)
+          let itemsTarget = density *. (chunkToBlock - chunkFromBlock.contents + 1)->Int.toFloat
+          fs.bucket->Array.push({
+            partitionId: fs.partitionId,
+            fromBlock: chunkFromBlock.contents,
+            toBlock: Some(chunkToBlock),
+            isChunk: true,
+            selection: p.selection,
+            itemsTarget,
+            addressesByContractName: p.addressesByContractName,
+          })
+          consumed := consumed.contents +. itemsTarget
+          chunkFromBlock := chunkToBlock + 1
+          created := created.contents + 1
+        }
+        fs.cursor = chunkFromBlock.contents
+        fs.chunksUsedThisCall = fs.chunksUsedThisCall + created.contents
+        consumed.contents
+      | None =>
+        // No trusted density yet (zero or one response): one open query sized
+        // to this round's share, capped so a single probe can't swallow the
+        // whole (possibly cross-chain-wide) budget and starve other partitions
+        // or chains before they get their own first probe.
+        let itemsTarget = Pervasives.min(Math.round(budget), maxItemsTarget)
+        fs.bucket->Array.push({
+          partitionId: fs.partitionId,
+          fromBlock: fs.cursor,
+          toBlock: fs.queryEndBlock,
+          isChunk: false,
+          selection: p.selection,
+          itemsTarget,
+          addressesByContractName: p.addressesByContractName,
+        })
+        fs.cursor = maxBlock + 1
+        fs.chunksUsedThisCall = fs.chunksUsedThisCall + 1
+        itemsTarget
+      }
     }
 
+    let isInRange = (fs: waterFillState) =>
+      fs.cursor <= chainTargetBlock &&
+      switch fs.queryEndBlock {
+      | Some(eb) => fs.cursor <= eb
+      | None => true
+      } &&
+      fs.pendingCount + fs.chunksUsedThisCall < maxPendingChunksPerPartition
+
+    // Water-fill. Range membership is fixed by chainTargetBlock/queryEndBlock,
+    // so the in-range partitions are filtered once up front; each round levels
+    // every still-not-filled partition up toward a shared line and keeps only
+    // those still in range for the next round.
+    //
+    // The line is (remaining fresh budget + those partitions' current
+    // footprint) / count: a partition already holding more than the line gets
+    // nothing and drops out (its head start is its whole share), while the
+    // rest are topped up to the line — so leftover from an under-using or
+    // filled partition redistributes to whoever can still use it, and totals
+    // stay even regardless of processing order (the line is fixed before the
+    // round's emits).
+    //
+    // No explicit round cap: a partition survives a round only by advancing
+    // chunksUsedThisCall (capped at maxPendingChunksPerPartition) or by
+    // consuming fresh budget, so the not-filled set drains on its own and the
+    // loop also stops once the whole budget is reserved.
+    let notFilledPartitions = ref(fillStates->Array.filter(isInRange))
+    let reservedFromRange = ref(0.)
+    while (
+      notFilledPartitions.contents->Array.length > 0 &&
+        reservedFromRange.contents < rangeItemsTarget
+    ) {
+      let n = notFilledPartitions.contents->Array.length
+      let footprintSum = ref(0.)
+      notFilledPartitions.contents->Array.forEach(fs =>
+        footprintSum := footprintSum.contents +. reservedByPartition->Dict.getUnsafe(fs.partitionId)
+      )
+      let line =
+        (rangeItemsTarget -. reservedFromRange.contents +. footprintSum.contents) /. n->Int.toFloat
+      let next = []
+      notFilledPartitions.contents->Array.forEach(fs => {
+        let reserved = reservedByPartition->Dict.getUnsafe(fs.partitionId)
+        let budget = line -. reserved
+        if budget > 0. {
+          let consumed = emitQueries(fs, ~budget)
+          reservedByPartition->Dict.set(fs.partitionId, reserved +. consumed)
+          reservedFromRange := reservedFromRange.contents +. consumed
+          if fs->isInRange {
+            next->Array.push(fs)->ignore
+          }
+        }
+      })
+      notFilledPartitions := next
+    }
+
+    // Each partition pushed only into its own bucket (indexed by its
+    // idsInAscOrder position), so flattening reproduces idsInAscOrder order
+    // directly — no sort needed even though water-fill rounds interleave
+    // across partitions.
+    let queries = queriesByPartitionIndex->Array.flat
+
     if queries->Utils.Array.isEmpty {
       if shouldWaitForNewBlock.contents {
         WaitingForNewBlock
@@ -1626,7 +1835,7 @@ Instantiates a fetch state with partitions for initial addresses
 let make = (
   ~startBlock,
   ~endBlock,
-  ~eventConfigs: array<Internal.eventConfig>,
+  ~onEventRegistrations: array<Internal.onEventRegistration>,
   ~contractConfigs: dict<IndexingAddresses.contractConfig>,
   ~addresses: array<Internal.indexingAddress>,
   ~maxAddrInPartition,
@@ -1634,7 +1843,7 @@ let make = (
   ~maxOnBlockBufferSize,
   ~knownHeight,
   ~progressBlockNumber=startBlock - 1,
-  ~onBlockConfigs=[],
+  ~onBlockRegistrations=[],
   ~blockLag=0,
   ~firstEventBlock=None,
 ): t => {
@@ -1644,15 +1853,15 @@ let make = (
   }
 
   let notDependingOnAddresses = []
-  let normalEventConfigs = []
+  let normalRegistrations = []
   let contractNamesWithNormalEvents = Utils.Set.make()
 
-  eventConfigs->Array.forEach(ec => {
-    if ec.dependsOnAddresses {
-      normalEventConfigs->Array.push(ec)
-      contractNamesWithNormalEvents->Utils.Set.add(ec.contractName)->ignore
+  onEventRegistrations->Array.forEach(reg => {
+    if reg.dependsOnAddresses {
+      normalRegistrations->Array.push(reg)
+      contractNamesWithNormalEvents->Utils.Set.add(reg.eventConfig.contractName)->ignore
     } else {
-      notDependingOnAddresses->Array.push(ec)
+      notDependingOnAddresses->Array.push(reg)
     }
   })
 
@@ -1664,7 +1873,7 @@ let make = (
       latestFetchedBlock,
       selection: {
         dependsOnAddresses: false,
-        eventConfigs: notDependingOnAddresses,
+        onEventRegistrations: notDependingOnAddresses,
       },
       addressesByContractName: Dict.make(),
       mergeBlock: None,
@@ -1679,7 +1888,7 @@ let make = (
 
   let normalSelection = {
     dependsOnAddresses: true,
-    eventConfigs: normalEventConfigs,
+    onEventRegistrations: normalRegistrations,
   }
 
   let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
@@ -1715,12 +1924,15 @@ let make = (
     ~progressBlockNumber,
   )
 
-  if optimizedPartitions->OptimizedPartitions.count === 0 && onBlockConfigs->Utils.Array.isEmpty {
+  if (
+    optimizedPartitions->OptimizedPartitions.count === 0 &&
+      onBlockRegistrations->Utils.Array.isEmpty
+  ) {
     JsError.throwWithMessage(
       `Invalid configuration: Nothing to fetch on chain ${chainId->Int.toString}. ` ++
       `addresses=${addresses->Array.length->Int.toString}, ` ++
-      `eventConfigs=${eventConfigs->Array.length->Int.toString}, ` ++
-      `normalEventConfigs=${normalEventConfigs
+      `onEventRegistrations=${onEventRegistrations->Array.length->Int.toString}, ` ++
+      `normalRegistrations=${normalRegistrations
         ->Array.length
         ->Int.toString}. ` ++ `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`,
     )
@@ -1731,14 +1943,14 @@ let make = (
   // fetching, so without seeding the buffer here getNextQuery would return
   // NothingToQuery and the indexer would get stuck.
   let buffer = []
-  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockConfigs->Utils.Array.notEmpty {
+  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockRegistrations->Utils.Array.notEmpty {
     let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
     | None => knownHeight
     | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock.blockNumber
     }
     appendOnBlockItems(
       ~mutItems=buffer,
-      ~onBlockConfigs,
+      ~onBlockRegistrations,
       ~indexerStartBlock=startBlock,
       ~fromBlock=progressBlockNumber,
       ~maxBlockNumber,
@@ -1757,7 +1969,7 @@ let make = (
     latestOnBlockBlockNumber,
     normalSelection,
     blockLag,
-    onBlockConfigs,
+    onBlockRegistrations,
     maxOnBlockBufferSize,
     knownHeight,
     buffer,
@@ -2007,17 +2219,6 @@ let getProgressPercentage = (fetchState: t) => {
   }
 }
 
-// Progress a specific block sits at along the chain, used to order queries from
-// different chains: a query starting further back (lower %) is fetched first.
-let getProgressPercentageAt = (fetchState: t, ~blockNumber) => {
-  switch fetchState.firstEventBlock {
-  | None => 0.
-  | Some(firstEventBlock) =>
-    let totalRange = fetchState.knownHeight - firstEventBlock
-    totalRange <= 0 ? 0. : (blockNumber - firstEventBlock)->Int.toFloat /. totalRange->Int.toFloat
-  }
-}
-
 let sortForBatch = {
   let hasFullBatch = ({buffer} as fetchState: t, ~batchSizeTarget) => {
     switch buffer->Array.get(batchSizeTarget - 1) {