envio 3.3.0-alpha.7 → 3.3.0-alpha.9

package/src/FetchState.res

@@ -7,15 +7,19 @@ type blockNumberAndTimestamp = {
 
 type blockNumberAndLogIndex = {blockNumber: int, logIndex: int}
 
-type selection = {eventConfigs: array<Internal.eventConfig>, dependsOnAddresses: bool}
+type selection = {
+  onEventRegistrations: array<Internal.onEventRegistration>,
+  dependsOnAddresses: bool,
+}
 
 type pendingQuery = {
   fromBlock: int,
   toBlock: option<int>,
   isChunk: bool,
-  // Estimated items this in-flight query will add, carried from the query so the
-  // shared buffer budget can account for what's already being fetched.
-  estResponseSize: float,
+  // Items this in-flight query is targeting (server maxNumLogs-style cap), carried
+  // from the query so the shared buffer budget can account for what's already
+  // being fetched.
+  itemsTarget: int,
   // Stores latestFetchedBlock when query completes. Only needed to persist
   // timestamp while earlier queries are still pending before updating
   // the partition's latestFetchedBlock.
@@ -59,15 +63,12 @@ type query = {
   fromBlock: int,
   toBlock: option<int>,
   isChunk: bool,
-  // Estimated number of items the query will return, from the partition's
-  // density and the query's block range. Used to admit queries against the
-  // shared buffer budget.
-  estResponseSize: float,
-  // Owning chain and the chain progress % at the query's fromBlock. Set by the
-  // cross-chain scheduler so candidate queries can be pooled and ordered
-  // (furthest-behind first) without allocating a side tuple per query.
-  mutable chainId: int,
-  mutable progress: float,
+  // Items this query targets: the server-side maxNumLogs-style cap, and the
+  // unit the chain's per-tick budget is reserved/consumed in. For a partition
+  // with known density this is density × the query's block range; for a
+  // partition with no signal yet it's whatever budget share the query was
+  // sized against.
+  itemsTarget: int,
   selection: selection,
   addressesByContractName: dict<array<Address.t>>,
 }
@@ -90,42 +91,20 @@ let deriveContractNameByAddress: dict<array<Address.t>> => dict<
   result
 })
 
-// Bounds for the no-history default estimate below. Also used by SourceManager
-// as the floor for its own maxNumLogs-style safety net.
-let minEstResponseSize = 2_000.
-let maxEstResponseSize = 10_000.
-
-// Default estimate for a query whose partition hasn't responded yet, so the
-// shared budget still accounts for unknown queries instead of treating them as
-// free. Scaled down as partitionsCount grows: assuming every partition's first
-// query is "big" starves the rest of that tick's admission when many
-// partitions share the same buffer, so the per-partition default shrinks
-// accordingly — clamped to [minEstResponseSize, maxEstResponseSize] either way.
-let calculateDefaultEstResponseSize = (~partitionsCount) =>
+// itemsTarget for a query over [fromBlock, toBlock] at the given event density
+// (items/block). toBlock None is the open-ended tail, capped at
+// chainTargetBlock — the soft per-tick horizon the owning chain wants to reach
+// (see getNextQuery).
+let densityItemsTarget = (~density, ~fromBlock, ~toBlock, ~chainTargetBlock) => {
+  // Floor at 1: the reservation must equal the server-side cap SourceManager
+  // sends, and a 0 cap would ask the backend for nothing.
   Pervasives.max(
-    minEstResponseSize,
-    Pervasives.min(maxEstResponseSize, 20_000. /. partitionsCount->Int.toFloat),
+    1,
+    ((toBlock->Option.getOr(chainTargetBlock) - fromBlock + 1)->Int.toFloat *. density)
+    ->Math.ceil
+    ->Float.toInt,
   )
-
-// Estimated items a query will return, from the partition's event density
-// (items/block derived from its last response) and the query's block range.
-// toBlock None is the open-ended tail, capped at knownHeight. A partition
-// that responded with no items has density 0, so its queries cost 0 — correct,
-// they don't fill the buffer. Only a partition that has never responded
-// (prevQueryRange 0) has no signal, so it falls back to calculateDefaultEstResponseSize.
-let calculateEstResponseSize = (
-  p: partition,
-  ~fromBlock,
-  ~toBlock,
-  ~knownHeight,
-  ~partitionsCount,
-) =>
-  if p.prevQueryRange > 0 {
-    let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
-    (toBlock->Option.getOr(knownHeight) - fromBlock + 1)->Int.toFloat *. density
-  } else {
-    calculateDefaultEstResponseSize(~partitionsCount)
-  }
+}
 
 // Calculate the chunk range from history using min-of-last-3-ranges heuristic
 let getMinHistoryRange = (p: partition) => {
@@ -135,6 +114,15 @@ let getMinHistoryRange = (p: partition) => {
   }
 }
 
+// Density (items/block) from the last response, trusted only after two
+// responses — a single sample is too noisy to size the next query by.
+let getTrustedDensity = (p: partition) => {
+  switch (p.prevQueryRange, p.prevPrevQueryRange) {
+  | (0, _) | (_, 0) => None
+  | (prevQueryRange, _) => Some(p.prevRangeSize->Int.toFloat /. prevQueryRange->Int.toFloat)
+  }
+}
+
 let getMinQueryRange = (partitions: array<partition>) => {
   let min = ref(0)
   for i in 0 to partitions->Array.length - 1 {
@@ -214,6 +202,12 @@ module OptimizedPartitions = {
       let newId = nextPartitionIndexRef.contents->Int.toString
       nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
       let minRange = getMinQueryRange([p1, p2])
+      // The merged partition indexes both parents' addresses, so its expected
+      // event rate is the sum of their densities. Parents without a trusted
+      // density contribute 0; if none has one, prevRangeSize stays 0 and the
+      // partition probes for a fresh signal instead of chunking.
+      let inheritedDensity =
+        p1->getTrustedDensity->Option.getOr(0.) +. p2->getTrustedDensity->Option.getOr(0.)
       {
         id: newId,
         dynamicContract: Some(contractName),
@@ -224,7 +218,7 @@ module OptimizedPartitions = {
         mutPendingQueries: [],
         prevQueryRange: minRange,
         prevPrevQueryRange: minRange,
-        prevRangeSize: 0,
+        prevRangeSize: (inheritedDensity *. minRange->Int.toFloat)->Math.ceil->Float.toInt,
         latestBlockRangeUpdateBlock: 0,
       }
     }
@@ -566,7 +560,7 @@ type t = {
   // size). Event fetch depth is bounded separately, by CrossChainState's
   // cross-chain admission against the indexer-wide buffer pool.
   maxOnBlockBufferSize: int,
-  onBlockConfigs: array<Internal.onBlockConfig>,
+  onBlockRegistrations: array<Internal.onBlockRegistration>,
   knownHeight: int,
   firstEventBlock: option<int>,
 }
@@ -642,7 +636,7 @@ let blockItemLogIndex = 16777216
 // are generated at once to prevent OOM.
 let appendOnBlockItems = (
   ~mutItems: array<Internal.item>,
-  ~onBlockConfigs: array<Internal.onBlockConfig>,
+  ~onBlockRegistrations: array<Internal.onBlockRegistration>,
   ~indexerStartBlock,
   ~fromBlock,
   ~maxBlockNumber,
@@ -662,27 +656,27 @@ let appendOnBlockItems = (
     let blockNumber = latestOnBlockBlockNumber.contents + 1
     latestOnBlockBlockNumber := blockNumber
 
-    for configIdx in 0 to onBlockConfigs->Array.length - 1 {
-      let onBlockConfig = onBlockConfigs->Array.getUnsafe(configIdx)
+    for configIdx in 0 to onBlockRegistrations->Array.length - 1 {
+      let onBlockRegistration = onBlockRegistrations->Array.getUnsafe(configIdx)
 
-      let handlerStartBlock = switch onBlockConfig.startBlock {
+      let handlerStartBlock = switch onBlockRegistration.startBlock {
       | Some(startBlock) => startBlock
       | None => indexerStartBlock
       }
 
       if (
         blockNumber >= handlerStartBlock &&
-        switch onBlockConfig.endBlock {
+        switch onBlockRegistration.endBlock {
         | Some(endBlock) => blockNumber <= endBlock
         | None => true
         } &&
-        (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockConfig.interval) === 0
+        (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockRegistration.interval) === 0
       ) {
         mutItems->Array.push(
           Block({
-            onBlockConfig,
+            onBlockRegistration,
             blockNumber,
-            logIndex: blockItemLogIndex + onBlockConfig.index,
+            logIndex: blockItemLogIndex + onBlockRegistration.index,
           }),
         )
         newItemsCounter := newItemsCounter.contents + 1
@@ -706,9 +700,9 @@ let updateInternal = (
 ): t => {
   let mutItemsRef = ref(mutItems)
 
-  let latestOnBlockBlockNumber = switch fetchState.onBlockConfigs {
+  let latestOnBlockBlockNumber = switch fetchState.onBlockRegistrations {
   | [] => knownHeight
-  | onBlockConfigs => {
+  | onBlockRegistrations => {
       // Calculate the max block number we are going to create items for
       // Use maxOnBlockBufferSize to get the last target item in the buffer
       //
@@ -736,7 +730,7 @@ let updateInternal = (
 
       appendOnBlockItems(
         ~mutItems,
-        ~onBlockConfigs,
+        ~onBlockRegistrations,
         ~indexerStartBlock=fetchState.startBlock,
         ~fromBlock=fetchState.latestOnBlockBlockNumber,
         ~maxBlockNumber,
@@ -751,7 +745,7 @@ let updateInternal = (
     contractConfigs: fetchState.contractConfigs,
     normalSelection: fetchState.normalSelection,
     chainId: fetchState.chainId,
-    onBlockConfigs: fetchState.onBlockConfigs,
+    onBlockRegistrations: fetchState.onBlockRegistrations,
     maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     optimizedPartitions,
     latestOnBlockBlockNumber,
@@ -1017,7 +1011,7 @@ let registerDynamicContracts = (
   // Might contain duplicates which we should filter out
   items: array<Internal.item>,
 ) => {
-  if fetchState.normalSelection.eventConfigs->Utils.Array.isEmpty {
+  if fetchState.normalSelection.onEventRegistrations->Utils.Array.isEmpty {
     // Can the normalSelection be empty?
     JsError.throwWithMessage(
       "Invalid configuration. No events to fetch for the dynamic contract registration.",
@@ -1301,8 +1295,8 @@ let handleQueryResult = (
   // param-level analogue of EventRouter's srcAddress effectiveStartBlock check.
   let newItems = newItems->Array.filter(item =>
     switch item {
-    | Internal.Event({eventConfig, payload, blockNumber}) =>
-      switch eventConfig.clientAddressFilter {
+    | Internal.Event({onEventRegistration, payload, blockNumber}) =>
+      switch onEventRegistration.clientAddressFilter {
       | Some(filter) =>
         filter(payload, blockNumber, indexingAddresses->IndexingAddresses.rawForFilter)
       | None => true
@@ -1340,7 +1334,7 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
       fromBlock: q.fromBlock,
       toBlock: q.toBlock,
       isChunk: q.isChunk,
-      estResponseSize: q.estResponseSize,
+      itemsTarget: q.itemsTarget,
       fetchedBlock: None,
     }
 
@@ -1364,48 +1358,58 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
 // Most parallel in-flight chunk queries a single partition may have at once.
 let maxPendingChunksPerPartition = 10
 
-@inline
-let pushQueriesForRange = (
+// Fills a gap range (a hole left between completed/pending chunks, e.g. from an
+// out-of-order partial response) unconditionally — gaps are already-committed
+// range, not subject to this tick's water-fill budget. Priced by the
+// partition's trusted density when it has one; otherwise by the "available
+// density" — the partition's equal-divide budget spread over its remaining
+// range this tick — so a small gap reserves proportionally little instead of a
+// noisy one-sample estimate. Chunks only on a trusted POSITIVE density, same
+// rule as the water-fill. Returns the created queries' total itemsTarget.
+let pushGapFillQueries = (
   queries: array<query>,
   ~partitionId: string,
   ~rangeFromBlock: int,
   ~rangeEndBlock: option<int>,
   ~knownHeight: int,
+  ~chainTargetBlock: int,
   ~maybeChunkRange: option<int>,
   ~maxChunks: int,
   ~partition: partition,
+  ~partitionBudget: float,
   ~selection: selection,
   ~addressesByContractName: dict<array<Address.t>>,
-  ~partitionsCount: int,
 ) => {
+  let cost = ref(0.)
   if rangeFromBlock <= knownHeight && maxChunks > 0 {
     switch rangeEndBlock {
     | Some(endBlock) if rangeFromBlock > endBlock => ()
     | _ =>
-      switch maybeChunkRange {
-      | None =>
+      let trustedDensity = partition->getTrustedDensity
+      let maxBlock = switch rangeEndBlock {
+      | Some(eb) => eb
+      | None => chainTargetBlock
+      }
+      let pushSingleQuery = (~density, ~isChunk) => {
+        let itemsTarget = densityItemsTarget(
+          ~density,
+          ~fromBlock=rangeFromBlock,
+          ~toBlock=rangeEndBlock,
+          ~chainTargetBlock,
+        )
         queries->Array.push({
           partitionId,
           fromBlock: rangeFromBlock,
           toBlock: rangeEndBlock,
           selection,
-          isChunk: false,
-          estResponseSize: calculateEstResponseSize(
-            partition,
-            ~fromBlock=rangeFromBlock,
-            ~toBlock=rangeEndBlock,
-            ~knownHeight,
-            ~partitionsCount,
-          ),
-          chainId: 0,
-          progress: 0.,
+          isChunk,
+          itemsTarget,
           addressesByContractName,
         })
-      | Some(chunkRange) =>
-        let maxBlock = switch rangeEndBlock {
-        | Some(eb) => eb
-        | None => knownHeight
-        }
+        cost := cost.contents +. itemsTarget->Int.toFloat
+      }
+      switch (trustedDensity, maybeChunkRange) {
+      | (Some(density), Some(chunkRange)) if density > 0. =>
         let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
         if rangeFromBlock + chunkSize * 2 - 1 <= maxBlock {
           let chunkFromBlock = ref(rangeFromBlock)
@@ -1414,58 +1418,111 @@ let pushQueriesForRange = (
             chunkIdx.contents < maxChunks && chunkFromBlock.contents + chunkSize - 1 <= maxBlock
           ) {
             let chunkToBlock = chunkFromBlock.contents + chunkSize - 1
+            let itemsTarget = densityItemsTarget(
+              ~density,
+              ~fromBlock=chunkFromBlock.contents,
+              ~toBlock=Some(chunkToBlock),
+              ~chainTargetBlock,
+            )
             queries->Array.push({
               partitionId,
               fromBlock: chunkFromBlock.contents,
               toBlock: Some(chunkToBlock),
               isChunk: true,
               selection,
-              estResponseSize: calculateEstResponseSize(
-                partition,
-                ~fromBlock=chunkFromBlock.contents,
-                ~toBlock=Some(chunkToBlock),
-                ~knownHeight,
-                ~partitionsCount,
-              ),
-              chainId: 0,
-              progress: 0.,
+              itemsTarget,
               addressesByContractName,
             })
+            cost := cost.contents +. itemsTarget->Int.toFloat
             chunkFromBlock := chunkToBlock + 1
             chunkIdx := chunkIdx.contents + 1
           }
         } else {
           // Not enough room for 2 chunks, fall back to a single query
-          queries->Array.push({
-            partitionId,
-            fromBlock: rangeFromBlock,
-            toBlock: rangeEndBlock,
-            selection,
-            isChunk: rangeEndBlock !== None,
-            estResponseSize: calculateEstResponseSize(
-              partition,
-              ~fromBlock=rangeFromBlock,
-              ~toBlock=rangeEndBlock,
-              ~knownHeight,
-              ~partitionsCount,
-            ),
-            chainId: 0,
-            progress: 0.,
-            addressesByContractName,
-          })
+          pushSingleQuery(~density, ~isChunk=rangeEndBlock !== None)
         }
+      | (Some(density), _) => pushSingleQuery(~density, ~isChunk=false)
+      | (None, _) =>
+        let remainingRange = Pervasives.max(1, chainTargetBlock - rangeFromBlock + 1)
+        pushSingleQuery(~density=partitionBudget /. remainingRange->Int.toFloat, ~isChunk=false)
       }
     }
   }
+  cost.contents
+}
+
+// The level every partition ends at when `budget` is poured across partitions
+// already holding `footprints`: the unique L with Σ max(0, L - fᵢ) = budget.
+// Partitions above L get nothing (their head start is their share); the rest
+// are topped up exactly to L, so the pour equals the budget no matter how
+// uneven the footprints are.
+let waterLevel = (~budget: float, ~footprints: array<float>) => {
+  let sorted = footprints->Array.toSorted(Float.compare)
+  let n = sorted->Array.length
+  let prefix = ref(0.)
+  let level = ref(None)
+  let idx = ref(0)
+  while level.contents == None && idx.contents < n {
+    let i = idx.contents
+    prefix := prefix.contents +. sorted->Array.getUnsafe(i)
+    // The level if only the i+1 lowest footprints receive water — correct once
+    // it doesn't reach the next footprint up.
+    let candidate = (budget +. prefix.contents) /. (i + 1)->Int.toFloat
+    if i == n - 1 || candidate <= sorted->Array.getUnsafe(i + 1) {
+      level := Some(candidate)
+    }
+    idx := idx.contents + 1
+  }
+  level.contents->Option.getOr(0.)
+}
+
+// Per-partition mutable state threaded through the water-fill rounds below.
+type waterFillState = {
+  partitionId: string,
+  p: partition,
+  mutable cursor: int,
+  mutable chunksUsedThisCall: int,
+  // Existing mutPendingQueries count before this call — fixed for the call,
+  // used with chunksUsedThisCall against maxPendingChunksPerPartition.
+  pendingCount: int,
+  queryEndBlock: option<int>,
+  maybeChunkRange: option<int>,
+  // This partition's own slot in queriesByPartitionIndex — gap-fill and every
+  // water-fill round push here directly, so query order falls out of
+  // idsInAscOrder for free (see getNextQuery) instead of a final sort pass.
+  bucket: array<query>,
 }
 
-// Candidate queries are sized against the natural ceiling (head/endBlock/mergeBlock),
-// not a share of the shared buffer — CrossChainState.checkAndFetch's admission loop
-// is what actually bounds how much gets fetched per tick, by estResponseSize (which
-// the HyperSync/SVM sources also enforce server-side via a maxNumLogs-style cap, so
-// a wrong estimate truncates the response instead of overshooting the buffer).
+// Candidate queries are sized against chainTargetBlock, the soft querying
+// horizon the owning chain wants to reach this tick — derived by ChainState
+// from its share of the indexer-wide buffer budget and its chain-level event
+// density. chainTargetBlock is never used as a hard query end, only to (a)
+// select which partitions are "in range" this tick and (b) size an open-ended
+// query with no other ceiling: the true hard bounds stay
+// endBlock/mergeBlock/the lagged head.
+//
+// In-range partitions share chainTargetItems (this chain's target total
+// footprint — in-flight plus new) by water-fill: each round pours exactly the
+// remaining fresh budget at the level computed by waterLevel, so a partition
+// already holding more than the level (from earlier ticks' in-flight queries)
+// gets nothing and its implicit share flows to the others, while totals stay
+// even and the pour never exceeds the budget. Rounds repeat only because
+// emits are quantized: a partition may consume less than its allotment (range
+// or chunk-cap runs out) or slightly more (min one chunk), and the leftover
+// re-pours over whoever still has range left.
+//
+// A partition with a trusted positive density (two or more responses — see
+// getMinHistoryRange) always emits at least one full-size chunk/query once
+// given an allotment, sized by its real density — may overshoot the
+// allotment by at most one chunk (the server also enforces itemsTarget via a
+// maxNumLogs-style cap, so an overshoot truncates the response rather than
+// the buffer). Any other partition (no signal, one noisy sample, or a
+// density-0 estimate) emits one open-ended probe sized exactly at its
+// allotment instead.
 let getNextQuery = (
   {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
+  ~chainTargetBlock: int,
+  ~chainTargetItems: float,
 ) => {
   let headBlockNumber = knownHeight - blockLag
   if headBlockNumber <= 0 {
@@ -1480,20 +1537,17 @@ let getNextQuery = (
       !isOnBlockBehindTheHead,
     )
 
-    let queries = []
-
     let partitionsCount = optimizedPartitions.idsInAscOrder->Array.length
-    let idxRef = ref(0)
-    while idxRef.contents < partitionsCount {
-      let idx = idxRef.contents
+
+    // Every partition is visited once here regardless of whether it gets a
+    // query pushed below — waiting-for-new-block bookkeeping shouldn't depend
+    // on this tick's budget.
+    for idx in 0 to partitionsCount - 1 {
       let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
       let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
-
-      let pendingCount = p.mutPendingQueries->Array.length
-      let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber
-      let hasPendingQueries = pendingCount > 0
-
-      if hasPendingQueries || isBehindTheHead {
+      if (
+        p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock.blockNumber < headBlockNumber
+      ) {
         // Even if there are some partitions waiting for the new block
         // We still want to wait for all partitions reaching the head
         // because they might update knownHeight in their response
@@ -1501,46 +1555,86 @@ let getNextQuery = (
         // and we don't want to poll the head for a few small partitions
         shouldWaitForNewBlock := false
       }
+    }
 
-      let partitionQueriesStart = queries->Array.length
+    // One bucket per partition, in idsInAscOrder order — gap-fill and
+    // water-fill both push into a partition's own bucket, so flattening at
+    // the end (see below) reproduces idsInAscOrder without a sort.
+    let queriesByPartitionIndex: array<
+      array<query>,
+    > = Array.fromInitializer(~length=partitionsCount, _ => [])
 
-      // Compute queryEndBlock for this partition
+    // Compute queryEndBlock for this partition
+    let computeQueryEndBlock = (p: partition) => {
       let queryEndBlock = Utils.Math.minOptInt(endBlock, p.mergeBlock)
-      let queryEndBlock = switch blockLag {
+      switch blockLag {
       | 0 => queryEndBlock
       | _ =>
         // Force head block as an endBlock when blockLag is set
         // because otherwise HyperSync might return bigger range
         Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock)
       }
+    }
+
+    // Each partition's existing in-flight itemsTarget, summed once and reused
+    // both for chainReserved (the call-wide fresh-budget ceiling below) and to
+    // seed each partition's water-fill footprint — so a partition already
+    // holding in-flight queries is counted toward its even share and doesn't
+    // get topped up past the others.
+    let existingReservedByPartition = Dict.make()
+    let chainReserved = ref(0.)
+    for idx in 0 to partitionsCount - 1 {
+      let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
+      let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
+      let cost = ref(0.)
+      for pqIdx in 0 to p.mutPendingQueries->Array.length - 1 {
+        cost :=
+          cost.contents +. (p.mutPendingQueries->Array.getUnsafe(pqIdx)).itemsTarget->Int.toFloat
+      }
+      existingReservedByPartition->Dict.set(partitionId, cost.contents)
+      chainReserved := chainReserved.contents +. cost.contents
+    }
 
+    // Phase A: gap-fill. Walk each partition's pending queries once,
+    // unconditionally filling any hole (e.g. from an out-of-order partial
+    // chunk response) — uncapped, same as before this tick's water-fill was
+    // introduced. This also determines each partition's post-gap cursor and
+    // whether it's blocked on an unresolved single-shot query.
+    let fillStates = []
+    let gapFillCost = ref(0.)
+    for idx in 0 to partitionsCount - 1 {
+      let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
+      let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
+      let bucket = queriesByPartitionIndex->Array.getUnsafe(idx)
+      let pendingCount = p.mutPendingQueries->Array.length
+      let queryEndBlock = computeQueryEndBlock(p)
       let maybeChunkRange = getMinHistoryRange(p)
 
-      // Walk pending queries to find open ranges and create queries for each
       let cursor = ref(p.latestFetchedBlock.blockNumber + 1)
       let canContinue = ref(true)
+      let chunksUsedThisCall = ref(0)
       let pqIdx = ref(0)
-      while pqIdx.contents < p.mutPendingQueries->Array.length && canContinue.contents {
+      while pqIdx.contents < pendingCount && canContinue.contents {
         let pq = p.mutPendingQueries->Array.getUnsafe(pqIdx.contents)
 
-        // Gap before this pending query → create queries for the gap range
         if pq.fromBlock > cursor.contents {
-          pushQueriesForRange(
-            queries,
+          let beforeLen = bucket->Array.length
+          let cost = pushGapFillQueries(
+            bucket,
             ~partitionId,
             ~rangeFromBlock=cursor.contents,
             ~rangeEndBlock=Utils.Math.minOptInt(Some(pq.fromBlock - 1), queryEndBlock),
             ~knownHeight,
+            ~chainTargetBlock,
             ~maybeChunkRange,
-            ~maxChunks=maxPendingChunksPerPartition -
-            pendingCount -
-            (queries->Array.length -
-            partitionQueriesStart),
+            ~maxChunks=maxPendingChunksPerPartition - pendingCount - chunksUsedThisCall.contents,
             ~partition=p,
+            ~partitionBudget=chainTargetItems /. partitionsCount->Int.toFloat,
             ~selection=p.selection,
             ~addressesByContractName=p.addressesByContractName,
-            ~partitionsCount,
           )
+          chunksUsedThisCall := chunksUsedThisCall.contents + (bucket->Array.length - beforeLen)
+          gapFillCost := gapFillCost.contents +. cost
         }
         switch pq {
         | {isChunk: true, toBlock: Some(toBlock), fetchedBlock: Some({blockNumber})}
@@ -1552,29 +1646,168 @@ let getNextQuery = (
         pqIdx := pqIdx.contents + 1
       }
 
-      // Tail range after all pending queries
       if canContinue.contents {
-        pushQueriesForRange(
-          queries,
-          ~partitionId,
-          ~rangeFromBlock=cursor.contents,
-          ~rangeEndBlock=queryEndBlock,
-          ~knownHeight,
-          ~maybeChunkRange,
-          ~maxChunks=maxPendingChunksPerPartition -
-          pendingCount -
-          (queries->Array.length -
-          partitionQueriesStart),
-          ~partition=p,
-          ~selection=p.selection,
-          ~addressesByContractName=p.addressesByContractName,
-          ~partitionsCount,
-        )
+        fillStates
+        ->Array.push({
+          partitionId,
+          p,
+          cursor: cursor.contents,
+          chunksUsedThisCall: chunksUsedThisCall.contents,
+          pendingCount,
+          queryEndBlock,
+          maybeChunkRange,
+          bucket,
+        })
+        ->ignore
       }
+    }
 
-      idxRef := idxRef.contents + 1
+    // Fresh budget for this call — what's left of chainTargetItems after
+    // existing in-flight queries and this tick's gap-fill. The water-fill
+    // rounds below hand it out; a partition already holding a large share
+    // (seeded below) gets proportionally less so totals stay even.
+    let rangeItemsTarget = Pervasives.max(
+      0.,
+      chainTargetItems -. chainReserved.contents -. gapFillCost.contents,
+    )
+
+    // Each in-range partition's running footprint: existing in-flight plus any
+    // gap-fill just pushed into its bucket. The water-fill levels every
+    // partition up toward a shared line, so this seed is what a partition
+    // starts the fill already holding.
+    let reservedByPartition = Dict.make()
+    fillStates->Array.forEach(fs => {
+      let cost = ref(existingReservedByPartition->Dict.getUnsafe(fs.partitionId))
+      for i in 0 to fs.bucket->Array.length - 1 {
+        cost := cost.contents +. (fs.bucket->Array.getUnsafe(i)).itemsTarget->Int.toFloat
+      }
+      reservedByPartition->Dict.set(fs.partitionId, cost.contents)
+    })
+
+    let isInRange = (fs: waterFillState) =>
+      fs.cursor <= chainTargetBlock &&
+      switch fs.queryEndBlock {
+      | Some(eb) => fs.cursor <= eb
+      | None => true
+      } &&
+      fs.pendingCount + fs.chunksUsedThisCall < maxPendingChunksPerPartition
+
+    let inRangeStates = fillStates->Array.filter(isInRange)
+
+    // Emits this round's queries for one partition, given its water-fill
+    // allotment. Mutates fs.cursor/chunksUsedThisCall and returns the
+    // itemsTarget consumed.
+    //
+    // Chunks require a POSITIVE trusted density: density 0 prices every chunk
+    // at ~nothing, letting a partition flood its full chunk pipeline with
+    // hard-bounded queries that crawl 1.8× per two responses — an open-ended
+    // probe instead gets the server's full scan range in one response.
+    let emitQueries = (fs: waterFillState, ~budget: float) => {
+      let p = fs.p
+      let maxBlock = switch fs.queryEndBlock {
+      | Some(eb) => eb
+      | None => chainTargetBlock
+      }
+      switch (fs.maybeChunkRange, p->getTrustedDensity) {
+      | (Some(minHistoryRange), Some(density)) if density > 0. =>
+        // Chunking active: strict chunks with a hard endBlock, uncapped real
+        // density — at least one full chunk this round even if budget falls
+        // short.
+        let chunkSize = Js.Math.ceil_int(minHistoryRange->Int.toFloat *. 1.8)
+        let chunkCost = density *. chunkSize->Int.toFloat
+        let maxChunksRemaining =
+          maxPendingChunksPerPartition - fs.pendingCount - fs.chunksUsedThisCall
+        let affordable = Math.floor(budget /. chunkCost)->Float.toInt
+        let numChunks = Pervasives.max(1, Pervasives.min(affordable, maxChunksRemaining))
+        let consumed = ref(0.)
+        let created = ref(0)
+        let chunkFromBlock = ref(fs.cursor)
+        while created.contents < numChunks && chunkFromBlock.contents <= maxBlock {
+          let chunkToBlock = Pervasives.min(chunkFromBlock.contents + chunkSize - 1, maxBlock)
+          let itemsTarget = Pervasives.max(
+            1,
+            (density *. (chunkToBlock - chunkFromBlock.contents + 1)->Int.toFloat)
+            ->Math.ceil
+            ->Float.toInt,
+          )
+          fs.bucket->Array.push({
+            partitionId: fs.partitionId,
+            fromBlock: chunkFromBlock.contents,
+            toBlock: Some(chunkToBlock),
+            isChunk: true,
+            selection: p.selection,
+            itemsTarget,
+            addressesByContractName: p.addressesByContractName,
+          })
+          consumed := consumed.contents +. itemsTarget->Int.toFloat
+          chunkFromBlock := chunkToBlock + 1
+          created := created.contents + 1
+        }
+        fs.cursor = chunkFromBlock.contents
+        fs.chunksUsedThisCall = fs.chunksUsedThisCall + created.contents
+        consumed.contents
+      | _ =>
+        let itemsTarget = Pervasives.max(1, Math.round(budget)->Float.toInt)
+        fs.bucket->Array.push({
+          partitionId: fs.partitionId,
+          fromBlock: fs.cursor,
+          toBlock: fs.queryEndBlock,
+          isChunk: false,
+          selection: p.selection,
+          itemsTarget,
+          addressesByContractName: p.addressesByContractName,
+        })
+        fs.cursor = maxBlock + 1
+        fs.chunksUsedThisCall = fs.chunksUsedThisCall + 1
+        itemsTarget->Int.toFloat
+      }
     }
 
+    // Water-fill. Range membership is fixed by chainTargetBlock/queryEndBlock,
+    // so the in-range partitions are filtered once up front; each round pours
+    // the remaining fresh budget at the waterLevel of the still-not-filled
+    // partitions' footprints and keeps only those still in range for the next
+    // round. Allotments sum to exactly the poured budget, so the outcome is
+    // order-independent and never exceeds it — the only overshoot left is the
+    // min-one-chunk quantization in emitQueries, bounded by one chunk per
+    // partition per round.
+    //
+    // No explicit round cap: a partition survives a round only by advancing
+    // chunksUsedThisCall (capped at maxPendingChunksPerPartition) or by
+    // consuming fresh budget (every emit consumes at least 1), so the
+    // not-filled set drains on its own and the loop also stops once the whole
+    // budget is poured.
+    let notFilledPartitions = ref(inRangeStates)
+    let remainingBudget = ref(rangeItemsTarget)
+    while notFilledPartitions.contents->Array.length > 0 && remainingBudget.contents > 0. {
+      let level = waterLevel(
+        ~budget=remainingBudget.contents,
+        ~footprints=notFilledPartitions.contents->Array.map(fs =>
+          reservedByPartition->Dict.getUnsafe(fs.partitionId)
+        ),
+      )
+      let next = []
+      notFilledPartitions.contents->Array.forEach(fs => {
+        let reserved = reservedByPartition->Dict.getUnsafe(fs.partitionId)
+        let budget = level -. reserved
+        if budget > 0. {
+          let consumed = emitQueries(fs, ~budget)
+          reservedByPartition->Dict.set(fs.partitionId, reserved +. consumed)
+          remainingBudget := remainingBudget.contents -. consumed
+          if fs->isInRange {
+            next->Array.push(fs)->ignore
+          }
+        }
+      })
+      notFilledPartitions := next
+    }
+
+    // Each partition pushed only into its own bucket (indexed by its
+    // idsInAscOrder position), so flattening reproduces idsInAscOrder order
+    // directly — no sort needed even though water-fill rounds interleave
+    // across partitions.
+    let queries = queriesByPartitionIndex->Array.flat
+
     if queries->Utils.Array.isEmpty {
       if shouldWaitForNewBlock.contents {
         WaitingForNewBlock
@@ -1623,7 +1856,7 @@ Instantiates a fetch state with partitions for initial addresses
 let make = (
   ~startBlock,
   ~endBlock,
-  ~eventConfigs: array<Internal.eventConfig>,
+  ~onEventRegistrations: array<Internal.onEventRegistration>,
   ~contractConfigs: dict<IndexingAddresses.contractConfig>,
   ~addresses: array<Internal.indexingAddress>,
   ~maxAddrInPartition,
@@ -1631,7 +1864,7 @@ let make = (
   ~maxOnBlockBufferSize,
   ~knownHeight,
   ~progressBlockNumber=startBlock - 1,
-  ~onBlockConfigs=[],
+  ~onBlockRegistrations=[],
   ~blockLag=0,
   ~firstEventBlock=None,
 ): t => {
@@ -1641,15 +1874,15 @@ let make = (
   }
 
   let notDependingOnAddresses = []
-  let normalEventConfigs = []
+  let normalRegistrations = []
   let contractNamesWithNormalEvents = Utils.Set.make()
 
-  eventConfigs->Array.forEach(ec => {
-    if ec.dependsOnAddresses {
-      normalEventConfigs->Array.push(ec)
-      contractNamesWithNormalEvents->Utils.Set.add(ec.contractName)->ignore
+  onEventRegistrations->Array.forEach(reg => {
+    if reg.dependsOnAddresses {
+      normalRegistrations->Array.push(reg)
+      contractNamesWithNormalEvents->Utils.Set.add(reg.eventConfig.contractName)->ignore
     } else {
-      notDependingOnAddresses->Array.push(ec)
+      notDependingOnAddresses->Array.push(reg)
     }
   })
 
@@ -1661,7 +1894,7 @@ let make = (
       latestFetchedBlock,
       selection: {
         dependsOnAddresses: false,
-        eventConfigs: notDependingOnAddresses,
+        onEventRegistrations: notDependingOnAddresses,
       },
       addressesByContractName: Dict.make(),
       mergeBlock: None,
@@ -1676,7 +1909,7 @@ let make = (
 
   let normalSelection = {
     dependsOnAddresses: true,
-    eventConfigs: normalEventConfigs,
+    onEventRegistrations: normalRegistrations,
   }
 
   let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
@@ -1712,12 +1945,15 @@ let make = (
     ~progressBlockNumber,
   )
 
-  if optimizedPartitions->OptimizedPartitions.count === 0 && onBlockConfigs->Utils.Array.isEmpty {
+  if (
+    optimizedPartitions->OptimizedPartitions.count === 0 &&
+      onBlockRegistrations->Utils.Array.isEmpty
+  ) {
     JsError.throwWithMessage(
       `Invalid configuration: Nothing to fetch on chain ${chainId->Int.toString}. ` ++
       `addresses=${addresses->Array.length->Int.toString}, ` ++
-      `eventConfigs=${eventConfigs->Array.length->Int.toString}, ` ++
-      `normalEventConfigs=${normalEventConfigs
+      `onEventRegistrations=${onEventRegistrations->Array.length->Int.toString}, ` ++
+      `normalRegistrations=${normalRegistrations
         ->Array.length
         ->Int.toString}. ` ++ `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`,
     )
@@ -1728,14 +1964,14 @@ let make = (
   // fetching, so without seeding the buffer here getNextQuery would return
   // NothingToQuery and the indexer would get stuck.
   let buffer = []
-  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockConfigs->Utils.Array.notEmpty {
+  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockRegistrations->Utils.Array.notEmpty {
     let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
     | None => knownHeight
     | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock.blockNumber
     }
     appendOnBlockItems(
       ~mutItems=buffer,
-      ~onBlockConfigs,
+      ~onBlockRegistrations,
       ~indexerStartBlock=startBlock,
       ~fromBlock=progressBlockNumber,
       ~maxBlockNumber,
@@ -1754,7 +1990,7 @@ let make = (
     latestOnBlockBlockNumber,
     normalSelection,
     blockLag,
-    onBlockConfigs,
+    onBlockRegistrations,
     maxOnBlockBufferSize,
     knownHeight,
     buffer,
@@ -2004,17 +2240,6 @@ let getProgressPercentage = (fetchState: t) => {
   }
 }
 
-// Progress a specific block sits at along the chain, used to order queries from
-// different chains: a query starting further back (lower %) is fetched first.
-let getProgressPercentageAt = (fetchState: t, ~blockNumber) => {
-  switch fetchState.firstEventBlock {
-  | None => 0.
-  | Some(firstEventBlock) =>
-    let totalRange = fetchState.knownHeight - firstEventBlock
-    totalRange <= 0 ? 0. : (blockNumber - firstEventBlock)->Int.toFloat /. totalRange->Int.toFloat
-  }
-}
-
 let sortForBatch = {
   let hasFullBatch = ({buffer} as fetchState: t, ~batchSizeTarget) => {
     switch buffer->Array.get(batchSizeTarget - 1) {