envio 3.3.0-alpha.0 → 3.3.0-alpha.10

package/src/FetchState.res

@@ -1,11 +1,5 @@
-type contractConfig = {startBlock: option<int>}
-
 type indexingAddress = Internal.indexingContract
 
-let deriveEffectiveStartBlock = (~registrationBlock: int, ~contractStartBlock: option<int>) => {
-  Pervasives.max(Pervasives.max(registrationBlock, 0), contractStartBlock->Option.getOr(0))
-}
-
 type blockNumberAndTimestamp = {
   blockNumber: int,
   blockTimestamp: int,
@@ -13,12 +7,21 @@ type blockNumberAndTimestamp = {
 
 type blockNumberAndLogIndex = {blockNumber: int, logIndex: int}
 
-type selection = {eventConfigs: array<Internal.eventConfig>, dependsOnAddresses: bool}
+type selection = {
+  onEventRegistrations: array<Internal.onEventRegistration>,
+  dependsOnAddresses: bool,
+}
 
 type pendingQuery = {
   fromBlock: int,
   toBlock: option<int>,
   isChunk: bool,
+  // Items this in-flight query is targeting (server maxNumLogs-style cap).
+  itemsTarget: int,
+  // Estimated items this in-flight query will actually return (no headroom),
+  // carried from the query so the shared buffer budget can account for what's
+  // already being fetched without the cap's safety margin inflating it.
+  itemsEst: int,
   // Stores latestFetchedBlock when query completes. Only needed to persist
   // timestamp while earlier queries are still pending before updating
   // the partition's latestFetchedBlock.
@@ -47,6 +50,10 @@ type partition = {
   // Track last 3 successful query ranges for chunking heuristic (0 means no data)
   prevQueryRange: int,
   prevPrevQueryRange: int,
+  // Item count of the response that set prevQueryRange. Paired with it to derive
+  // the partition's event density (prevRangeSize / prevQueryRange) for sizing
+  // queries against the buffer budget.
+  prevRangeSize: int,
   // Tracks the latestFetchedBlock.blockNumber of the most recent response
   // that updated prevQueryRange. Prevents degradation of the chunking heuristic
   // when parallel query responses arrive out of order.
@@ -58,9 +65,50 @@ type query = {
   fromBlock: int,
   toBlock: option<int>,
   isChunk: bool,
+  // Items this query targets: the server-side maxNumLogs-style cap, sized
+  // with headroom (chunkItemsMultiplier) so a denser-than-expected range
+  // doesn't truncate the response.
+  itemsTarget: int,
+  // Expected items without headroom: density × the query's block range for a
+  // known-density partition, the query's budget share otherwise. This is the
+  // unit the chain's per-tick budget is reserved/consumed in — reserving the
+  // headroomed cap instead would throttle the pipeline by the safety margin.
+  itemsEst: int,
   selection: selection,
   addressesByContractName: dict<array<Address.t>>,
-  indexingAddresses: dict<indexingAddress>,
+}
+
+// Invert addressesByContractName into address→contractName for log-ownership
+// routing. 1:1 today (each address belongs to one contract), so no key
+// collisions. Memoized on the addressesByContractName object so a partition's
+// many responses share one derivation and a large factory never rebuilds the
+// whole index; sound because the dict is immutable after construction (every
+// mutation produces a new dict).
+let deriveContractNameByAddress: dict<array<Address.t>> => dict<
+  string,
+> = Utils.WeakMap.memoize(addressesByContractName => {
+  let result = Dict.make()
+  addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
+    for i in 0 to addresses->Array.length - 1 {
+      result->Dict.set(addresses->Array.getUnsafe(i)->Address.toString, contractName)
+    }
+  })
+  result
+})
+
+// itemsTarget for a query over [fromBlock, toBlock] at the given event density
+// (items/block). toBlock None is the open-ended tail, capped at
+// chainTargetBlock — the soft per-tick horizon the owning chain wants to reach
+// (see getNextQuery).
+let densityItemsTarget = (~density, ~fromBlock, ~toBlock, ~chainTargetBlock) => {
+  // Floor at 1: the reservation must equal the server-side cap SourceManager
+  // sends, and a 0 cap would ask the backend for nothing.
+  Pervasives.max(
+    1,
+    ((toBlock->Option.getOr(chainTargetBlock) - fromBlock + 1)->Int.toFloat *. density)
+    ->Math.ceil
+    ->Float.toInt,
+  )
 }
 
 // Calculate the chunk range from history using min-of-last-3-ranges heuristic
@@ -71,6 +119,15 @@ let getMinHistoryRange = (p: partition) => {
   }
 }
 
+// Density (items/block) from the last response, trusted only after two
+// responses — a single sample is too noisy to size the next query by.
+let getTrustedDensity = (p: partition) => {
+  switch (p.prevQueryRange, p.prevPrevQueryRange) {
+  | (0, _) | (_, 0) => None
+  | (prevQueryRange, _) => Some(p.prevRangeSize->Int.toFloat /. prevQueryRange->Int.toFloat)
+  }
+}
+
 let getMinQueryRange = (partitions: array<partition>) => {
   let min = ref(0)
   for i in 0 to partitions->Array.length - 1 {
@@ -150,6 +207,12 @@ module OptimizedPartitions = {
       let newId = nextPartitionIndexRef.contents->Int.toString
       nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
       let minRange = getMinQueryRange([p1, p2])
+      // The merged partition indexes both parents' addresses, so its expected
+      // event rate is the sum of their densities. Parents without a trusted
+      // density contribute 0; if none has one, prevRangeSize stays 0 and the
+      // partition probes for a fresh signal instead of chunking.
+      let inheritedDensity =
+        p1->getTrustedDensity->Option.getOr(0.) +. p2->getTrustedDensity->Option.getOr(0.)
       {
         id: newId,
         dynamicContract: Some(contractName),
@@ -160,6 +223,7 @@ module OptimizedPartitions = {
         mutPendingQueries: [],
         prevQueryRange: minRange,
         prevPrevQueryRange: minRange,
+        prevRangeSize: (inheritedDensity *. minRange->Int.toFloat)->Math.ceil->Float.toInt,
         latestBlockRangeUpdateBlock: 0,
       }
     }
@@ -397,6 +461,7 @@ module OptimizedPartitions = {
     optimizedPartitions: t,
     ~query,
     ~knownHeight,
+    ~itemsCount,
     ~latestFetchedBlock: blockNumberAndTimestamp,
   ) => {
     let p = optimizedPartitions->getOrThrow(~partitionId=query.partitionId)
@@ -416,18 +481,25 @@ module OptimizedPartitions = {
         switch query.toBlock {
         | None => latestFetchedBlock.blockNumber < knownHeight - 10 // Don't update block range when very close to the head
         | Some(queryToBlock) =>
-          // Update on partial response (direct capacity evidence),
-          // or when the query's intended range covers at least the partition's
-          // current chunk range — meaning it was a capacity-based split chunk,
-          // not a small gap-fill whose toBlock is an artificial boundary.
-          latestFetchedBlock.blockNumber < queryToBlock ||
+          if latestFetchedBlock.blockNumber < queryToBlock {
+            // Partial response is direct capacity evidence — unless it was
+            // truncated by our own itemsTarget cap: that reflects the
+            // reservation we asked for, not what the server could return.
+            itemsCount < query.itemsTarget
+          } else {
+            // A full response updates only when the query's intended range
+            // covers at least the partition's current chunk range — meaning it
+            // was a capacity-based split chunk, not a small gap-fill whose
+            // toBlock is an artificial boundary.
             switch getMinHistoryRange(p) {
             | None => false // Chunking not active yet, don't update
             | Some(minHistoryRange) => queryToBlock - query.fromBlock + 1 >= minHistoryRange
             }
+          }
         }
     let updatedPrevQueryRange = shouldUpdateBlockRange ? blockRange : p.prevQueryRange
     let updatedPrevPrevQueryRange = shouldUpdateBlockRange ? p.prevQueryRange : p.prevPrevQueryRange
+    let updatedPrevRangeSize = shouldUpdateBlockRange ? itemsCount : p.prevRangeSize
 
     // Process fetched queries from front of queue for main partition
     let updatedLatestFetchedBlock = consumeFetchedQueries(
@@ -449,6 +521,7 @@ module OptimizedPartitions = {
         latestFetchedBlock: updatedLatestFetchedBlock,
         prevQueryRange: updatedPrevQueryRange,
         prevPrevQueryRange: updatedPrevPrevQueryRange,
+        prevRangeSize: updatedPrevRangeSize,
         latestBlockRangeUpdateBlock: shouldUpdateBlockRange
           ? latestFetchedBlock.blockNumber
           : p.latestBlockRangeUpdateBlock,
@@ -480,10 +553,8 @@ type t = {
   startBlock: int,
   endBlock: option<int>,
   normalSelection: selection,
-  // By address
-  indexingAddresses: dict<indexingAddress>,
   // By contract name
-  contractConfigs: dict<contractConfig>,
+  contractConfigs: dict<IndexingAddresses.contractConfig>,
   // Not used for logic - only metadata
   chainId: int,
   // The block number of the latest block which was added to the queue
@@ -497,10 +568,10 @@ type t = {
   // Buffer of items ordered from earliest to latest
   buffer: array<Internal.item>,
   // Caps how far ahead onBlock items are pre-generated (set to 2x the batch
-  // size). Fetch depth is bounded separately by getNextQuery's bufferLimit, the
-  // chain's per-tick slice of the indexer-wide pool.
+  // size). Event fetch depth is bounded separately, by CrossChainState's
+  // cross-chain admission against the indexer-wide buffer pool.
   maxOnBlockBufferSize: int,
-  onBlockConfigs: array<Internal.onBlockConfig>,
+  onBlockRegistrations: array<Internal.onBlockRegistration>,
   knownHeight: int,
   firstEventBlock: option<int>,
 }
@@ -536,6 +607,25 @@ let bufferBlock = ({optimizedPartitions, latestOnBlockBlockNumber}: t) => {
   }
 }
 
+// Number of buffered items at or below the ready frontier (processable now,
+// i.e. not stuck behind a gap from a lagging partition or out-of-order chunk).
+// The buffer is kept sorted, so binary-search the frontier in O(log n).
+let bufferReadyCount = (fetchState: t) => {
+  let frontier = fetchState->bufferBlockNumber
+  let buffer = fetchState.buffer
+  let lo = ref(0)
+  let hi = ref(buffer->Array.length)
+  while lo.contents < hi.contents {
+    let mid = (lo.contents + hi.contents) / 2
+    if buffer->Array.getUnsafe(mid)->Internal.getItemBlockNumber <= frontier {
+      lo := mid + 1
+    } else {
+      hi := mid
+    }
+  }
+  lo.contents
+}
+
 /*
 Comparitor for two events from the same chain. No need for chain id or timestamp
 */
@@ -551,15 +641,13 @@ let compareBufferItem = (a: Internal.item, b: Internal.item) => {
 // Some big number which should be bigger than any log index
 let blockItemLogIndex = 16777216
 
-let numAddresses = fetchState => fetchState.indexingAddresses->Utils.Dict.size
-
 // Appends Block items produced by the onBlock handlers for every block in
 // (fromBlock, maxBlockNumber] into mutItems and returns the new
 // latestOnBlockBlockNumber pointer. maxOnBlockBufferSize bounds how many items
 // are generated at once to prevent OOM.
 let appendOnBlockItems = (
   ~mutItems: array<Internal.item>,
-  ~onBlockConfigs: array<Internal.onBlockConfig>,
+  ~onBlockRegistrations: array<Internal.onBlockRegistration>,
   ~indexerStartBlock,
   ~fromBlock,
   ~maxBlockNumber,
@@ -579,27 +667,27 @@ let appendOnBlockItems = (
     let blockNumber = latestOnBlockBlockNumber.contents + 1
     latestOnBlockBlockNumber := blockNumber
 
-    for configIdx in 0 to onBlockConfigs->Array.length - 1 {
-      let onBlockConfig = onBlockConfigs->Array.getUnsafe(configIdx)
+    for configIdx in 0 to onBlockRegistrations->Array.length - 1 {
+      let onBlockRegistration = onBlockRegistrations->Array.getUnsafe(configIdx)
 
-      let handlerStartBlock = switch onBlockConfig.startBlock {
+      let handlerStartBlock = switch onBlockRegistration.startBlock {
       | Some(startBlock) => startBlock
       | None => indexerStartBlock
       }
 
       if (
         blockNumber >= handlerStartBlock &&
-        switch onBlockConfig.endBlock {
+        switch onBlockRegistration.endBlock {
         | Some(endBlock) => blockNumber <= endBlock
         | None => true
         } &&
-        (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockConfig.interval) === 0
+        (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockRegistration.interval) === 0
       ) {
         mutItems->Array.push(
           Block({
-            onBlockConfig,
+            onBlockRegistration,
             blockNumber,
-            logIndex: blockItemLogIndex + onBlockConfig.index,
+            logIndex: blockItemLogIndex + onBlockRegistration.index,
           }),
         )
         newItemsCounter := newItemsCounter.contents + 1
@@ -617,16 +705,15 @@ Runs partition optimization when partitions change.
 let updateInternal = (
   fetchState: t,
   ~optimizedPartitions=fetchState.optimizedPartitions,
-  ~indexingAddresses=fetchState.indexingAddresses,
   ~mutItems=?,
   ~blockLag=fetchState.blockLag,
   ~knownHeight=fetchState.knownHeight,
 ): t => {
   let mutItemsRef = ref(mutItems)
 
-  let latestOnBlockBlockNumber = switch fetchState.onBlockConfigs {
+  let latestOnBlockBlockNumber = switch fetchState.onBlockRegistrations {
   | [] => knownHeight
-  | onBlockConfigs => {
+  | onBlockRegistrations => {
       // Calculate the max block number we are going to create items for
       // Use maxOnBlockBufferSize to get the last target item in the buffer
       //
@@ -654,7 +741,7 @@ let updateInternal = (
 
       appendOnBlockItems(
         ~mutItems,
-        ~onBlockConfigs,
+        ~onBlockRegistrations,
         ~indexerStartBlock=fetchState.startBlock,
         ~fromBlock=fetchState.latestOnBlockBlockNumber,
         ~maxBlockNumber,
@@ -669,11 +756,10 @@ let updateInternal = (
     contractConfigs: fetchState.contractConfigs,
     normalSelection: fetchState.normalSelection,
     chainId: fetchState.chainId,
-    onBlockConfigs: fetchState.onBlockConfigs,
+    onBlockRegistrations: fetchState.onBlockRegistrations,
     maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     optimizedPartitions,
     latestOnBlockBlockNumber,
-    indexingAddresses,
     blockLag,
     knownHeight,
     buffer: switch mutItemsRef.contents {
@@ -701,12 +787,6 @@ let updateInternal = (
     ~blockNumber=updatedFetchState->bufferBlockNumber,
     ~chainId=fetchState.chainId,
   )
-  if indexingAddresses !== fetchState.indexingAddresses {
-    Prometheus.IndexingAddresses.set(
-      ~addressesCount=updatedFetchState->numAddresses,
-      ~chainId=fetchState.chainId,
-    )
-  }
 
   updatedFetchState
 }
@@ -838,6 +918,7 @@ OptimizedPartitions.t => {
             mutPendingQueries: [],
             prevQueryRange: 0,
             prevPrevQueryRange: 0,
+            prevRangeSize: 0,
             latestBlockRangeUpdateBlock: 0,
           })
           nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
@@ -936,22 +1017,22 @@ OptimizedPartitions.t => {
 
 let registerDynamicContracts = (
   fetchState: t,
+  ~indexingAddresses: IndexingAddresses.t,
   // These are raw items which might have dynamic contracts received from contractRegister call.
   // Might contain duplicates which we should filter out
   items: array<Internal.item>,
 ) => {
-  if fetchState.normalSelection.eventConfigs->Utils.Array.isEmpty {
+  if fetchState.normalSelection.onEventRegistrations->Utils.Array.isEmpty {
     // Can the normalSelection be empty?
     JsError.throwWithMessage(
       "Invalid configuration. No events to fetch for the dynamic contract registration.",
     )
   }
 
-  let indexingAddresses = fetchState.indexingAddresses
   let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
   let earliestRegisteringEventBlockNumber = ref(%raw(`Infinity`))
   // Addresses registered for contracts without matching events. These are not
-  // added to partitions, but they are tracked on fetchState.indexingAddresses
+  // added to partitions, but they are tracked on indexingAddresses
   // so that later conflicting registrations are detected, and are persisted
   // to envio_addresses so they can be picked up on restart with updated config.
   let noEventsAddresses: dict<indexingAddress> = Dict.make()
@@ -977,15 +1058,13 @@ let registerDynamicContracts = (
             address: dc.address,
             contractName: dc.contractName,
             registrationBlock: dc.registrationBlock,
-            effectiveStartBlock: deriveEffectiveStartBlock(
+            effectiveStartBlock: IndexingAddresses.deriveEffectiveStartBlock(
               ~registrationBlock=dc.registrationBlock,
               ~contractStartBlock,
             ),
           }
           // Prevent registering already indexing contracts
-          switch indexingAddresses->Utils.Dict.dangerouslyGetNonOption(
-            dc.address->Address.toString,
-          ) {
+          switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
           | Some(existingContract) =>
             // FIXME: Instead of filtering out duplicates,
             // we should check the block number first.
@@ -1040,7 +1119,7 @@ let registerDynamicContracts = (
             address: dc.address,
             contractName: dc.contractName,
             registrationBlock: dc.registrationBlock,
-            effectiveStartBlock: deriveEffectiveStartBlock(
+            effectiveStartBlock: IndexingAddresses.deriveEffectiveStartBlock(
               ~registrationBlock=dc.registrationBlock,
               ~contractStartBlock=None,
             ),
@@ -1048,9 +1127,7 @@ let registerDynamicContracts = (
           // Prevent duplicate logging/persistence when the same address is
           // already tracked on fetchState, either from the db on startup or
           // from an earlier registration in this batch.
-          switch indexingAddresses->Utils.Dict.dangerouslyGetNonOption(
-            dc.address->Address.toString,
-          ) {
+          switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
           | Some(existingContract) =>
             if existingContract.contractName != dc.contractName {
               fetchState->warnDifferentContractType(~existingContract, ~dc=dcAsIndexingAddress)
@@ -1104,12 +1181,10 @@ let registerDynamicContracts = (
     // Only dcs for contracts without events. Track them on
     // indexingAddresses so subsequent registrations see them, but don't touch
     // partitions since there's nothing to fetch for them.
-    let newIndexingContracts = indexingAddresses->Utils.Dict.shallowCopy
-    let _ = Utils.Dict.mergeInPlace(newIndexingContracts, noEventsAddresses)
-    fetchState->updateInternal(~indexingAddresses=newIndexingContracts)
+    indexingAddresses->IndexingAddresses.register(noEventsAddresses)
+    fetchState
   | (_, _) => {
       let newPartitions = []
-      let newIndexingAddresses = indexingAddresses->Utils.Dict.shallowCopy
       let dynamicContractsRef = ref(fetchState.optimizedPartitions.dynamicContracts)
       let mutExistingPartitions = fetchState.optimizedPartitions.entities->Dict.valuesToArray
 
@@ -1179,6 +1254,7 @@ let registerDynamicContracts = (
                         mutPendingQueries: p.mutPendingQueries,
                         prevQueryRange: p.prevQueryRange,
                         prevPrevQueryRange: p.prevPrevQueryRange,
+                        prevRangeSize: p.prevRangeSize,
                         latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock,
                       })
                     }
@@ -1190,10 +1266,10 @@ let registerDynamicContracts = (
         }
 
         let registeringContracts = registeringContractsByContract->Dict.getUnsafe(contractName)
-        let _ = Utils.Dict.mergeInPlace(newIndexingAddresses, registeringContracts)
+        indexingAddresses->IndexingAddresses.register(registeringContracts)
       }
       // Include no-events dcs so later batches detect conflicts against them.
-      let _ = Utils.Dict.mergeInPlace(newIndexingAddresses, noEventsAddresses)
+      indexingAddresses->IndexingAddresses.register(noEventsAddresses)
 
       let optimizedPartitions = createPartitionsFromIndexingAddresses(
         ~registeringContractsByContract,
@@ -1206,7 +1282,7 @@ let registerDynamicContracts = (
         ~progressBlockNumber=0,
       )
 
-      fetchState->updateInternal(~optimizedPartitions, ~indexingAddresses=newIndexingAddresses)
+      fetchState->updateInternal(~optimizedPartitions)
     }
   }
 }
@@ -1219,6 +1295,7 @@ newItems are ordered earliest to latest (as they are returned from the worker)
 */
 let handleQueryResult = (
   fetchState: t,
+  ~indexingAddresses: IndexingAddresses.t,
   ~query: query,
   ~latestFetchedBlock: blockNumberAndTimestamp,
   ~newItems,
@@ -1229,9 +1306,10 @@ let handleQueryResult = (
   // param-level analogue of EventRouter's srcAddress effectiveStartBlock check.
   let newItems = newItems->Array.filter(item =>
     switch item {
-    | Internal.Event({eventConfig, payload, blockNumber}) =>
-      switch eventConfig.clientAddressFilter {
-      | Some(filter) => filter(payload, blockNumber, fetchState.indexingAddresses)
+    | Internal.Event({onEventRegistration, payload, blockNumber}) =>
+      switch onEventRegistration.clientAddressFilter {
+      | Some(filter) =>
+        filter(payload, blockNumber, indexingAddresses->IndexingAddresses.rawForFilter)
       | None => true
       }
     | _ => true
@@ -1241,6 +1319,7 @@ let handleQueryResult = (
     ~optimizedPartitions=fetchState.optimizedPartitions->OptimizedPartitions.handleQueryResponse(
       ~query,
       ~knownHeight=fetchState.knownHeight,
+      ~itemsCount=newItems->Array.length,
       ~latestFetchedBlock,
     ),
     ~mutItems=?{
@@ -1253,7 +1332,6 @@ let handleQueryResult = (
 }
 
 type nextQuery =
-  | ReachedMaxConcurrency
   | WaitingForNewBlock
   | NothingToQuery
   | Ready(array<query>)
@@ -1267,6 +1345,8 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
       fromBlock: q.fromBlock,
       toBlock: q.toBlock,
       isChunk: q.isChunk,
+      itemsTarget: q.itemsTarget,
+      itemsEst: q.itemsEst,
       fetchedBlock: None,
     }
 
@@ -1287,141 +1367,218 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
   }
 }
 
-@inline
-let pushQueriesForRange = (
+// Most parallel in-flight chunk queries a single partition may have at once.
+let maxPendingChunksPerPartition = 12
+
+// Fills a gap range (a hole left between completed/pending chunks, e.g. from an
+// out-of-order partial response) unconditionally — gaps are already-committed
+// range, not subject to this tick's water-fill budget. Priced by the
+// partition's trusted density when it has one; otherwise by the "available
+// density" — the partition's equal-divide budget spread over its remaining
+// range this tick — so a small gap reserves proportionally little instead of a
+// noisy one-sample estimate. Chunks only on a trusted POSITIVE density, same
+// rule as the water-fill. Returns the created queries' total itemsEst.
+let pushGapFillQueries = (
   queries: array<query>,
   ~partitionId: string,
   ~rangeFromBlock: int,
   ~rangeEndBlock: option<int>,
-  ~maxQueryBlockNumber: int,
+  ~knownHeight: int,
+  ~chainTargetBlock: int,
   ~maybeChunkRange: option<int>,
+  ~maxChunks: int,
+  ~partition: partition,
+  ~partitionBudget: float,
+  ~chunkItemsMultiplier: float,
   ~selection: selection,
   ~addressesByContractName: dict<array<Address.t>>,
-  ~indexingAddresses: dict<indexingAddress>,
 ) => {
-  if rangeFromBlock <= maxQueryBlockNumber {
+  let cost = ref(0.)
+
+  // Gaps past the chain's target block wait: they regenerate from the
+  // pending-walk each tick and fill once the target reaches them.
+  if rangeFromBlock <= Pervasives.min(knownHeight, chainTargetBlock) && maxChunks > 0 {
     switch rangeEndBlock {
     | Some(endBlock) if rangeFromBlock > endBlock => ()
     | _ =>
-      switch maybeChunkRange {
-      | None =>
+      let trustedDensity = partition->getTrustedDensity
+      let maxBlock = switch rangeEndBlock {
+      | Some(eb) => eb
+      | None => chainTargetBlock
+      }
+      let pushSingleQuery = (~density, ~isChunk) => {
+        let itemsTarget = densityItemsTarget(
+          ~density=density *. chunkItemsMultiplier,
+          ~fromBlock=rangeFromBlock,
+          ~toBlock=rangeEndBlock,
+          ~chainTargetBlock,
+        )
+        let itemsEst = densityItemsTarget(
+          ~density,
+          ~fromBlock=rangeFromBlock,
+          ~toBlock=rangeEndBlock,
+          ~chainTargetBlock,
+        )
         queries->Array.push({
           partitionId,
           fromBlock: rangeFromBlock,
           toBlock: rangeEndBlock,
           selection,
-          isChunk: false,
+          isChunk,
+          itemsTarget,
+          itemsEst,
           addressesByContractName,
-          indexingAddresses,
         })
-      | Some(chunkRange) =>
-        let maxBlock = switch rangeEndBlock {
-        | Some(eb) => eb
-        | None => maxQueryBlockNumber
-        }
+        cost := cost.contents +. itemsEst->Int.toFloat
+      }
+      switch (trustedDensity, maybeChunkRange) {
+      | (Some(density), Some(chunkRange)) if density > 0. =>
         let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
-        // Probe with two smaller chunks first so their responses come back
-        // quickly and refresh the chunking heuristic, then three full-size chunks.
-        let probeSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 0.9)
-        let getChunkSize = chunkIdx => chunkIdx < 2 ? probeSize : chunkSize
-        if rangeFromBlock + getChunkSize(0) + getChunkSize(1) - 1 <= maxBlock {
+        if rangeFromBlock + chunkSize * 2 - 1 <= maxBlock {
           let chunkFromBlock = ref(rangeFromBlock)
           let chunkIdx = ref(0)
           while (
-            chunkIdx.contents < 5 &&
-              chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1 <= maxBlock
+            chunkIdx.contents < maxChunks && chunkFromBlock.contents + chunkSize - 1 <= maxBlock
           ) {
-            let chunkToBlock = chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1
+            let chunkToBlock = chunkFromBlock.contents + chunkSize - 1
+            let itemsTarget = densityItemsTarget(
+              ~density=density *. chunkItemsMultiplier,
+              ~fromBlock=chunkFromBlock.contents,
+              ~toBlock=Some(chunkToBlock),
+              ~chainTargetBlock,
+            )
+            let itemsEst = densityItemsTarget(
+              ~density,
+              ~fromBlock=chunkFromBlock.contents,
+              ~toBlock=Some(chunkToBlock),
+              ~chainTargetBlock,
+            )
             queries->Array.push({
               partitionId,
               fromBlock: chunkFromBlock.contents,
               toBlock: Some(chunkToBlock),
               isChunk: true,
               selection,
+              itemsTarget,
+              itemsEst,
               addressesByContractName,
-              indexingAddresses,
             })
+            cost := cost.contents +. itemsEst->Int.toFloat
             chunkFromBlock := chunkToBlock + 1
             chunkIdx := chunkIdx.contents + 1
           }
         } else {
           // Not enough room for 2 chunks, fall back to a single query
-          queries->Array.push({
-            partitionId,
-            fromBlock: rangeFromBlock,
-            toBlock: rangeEndBlock,
-            selection,
-            isChunk: rangeEndBlock !== None,
-            addressesByContractName,
-            indexingAddresses,
-          })
+          pushSingleQuery(~density, ~isChunk=rangeEndBlock !== None)
         }
+      | (Some(density), _) => pushSingleQuery(~density, ~isChunk=false)
+      | (None, _) =>
+        let remainingRange = Pervasives.max(1, chainTargetBlock - rangeFromBlock + 1)
+        pushSingleQuery(~density=partitionBudget /. remainingRange->Int.toFloat, ~isChunk=false)
       }
     }
   }
+  cost.contents
 }
 
-// Most parallel in-flight chunk queries a single partition may have at once.
-let maxPendingChunksPerPartition = 10
+// The level every partition ends at when `budget` is poured across partitions
+// already holding `footprints`: the unique L with Σ max(0, L - fᵢ) = budget.
+// Partitions above L get nothing (their head start is their share); the rest
+// are topped up exactly to L, so the pour equals the budget no matter how
+// uneven the footprints are.
+let waterLevel = (~budget: float, ~footprints: array<float>) => {
+  let sorted = footprints->Array.toSorted(Float.compare)
+  let n = sorted->Array.length
+  let prefix = ref(0.)
+  let level = ref(None)
+  let idx = ref(0)
+  while level.contents == None && idx.contents < n {
+    let i = idx.contents
+    prefix := prefix.contents +. sorted->Array.getUnsafe(i)
+    // The level if only the i+1 lowest footprints receive water — correct once
+    // it doesn't reach the next footprint up.
+    let candidate = (budget +. prefix.contents) /. (i + 1)->Int.toFloat
+    if i == n - 1 || candidate <= sorted->Array.getUnsafe(i + 1) {
+      level := Some(candidate)
+    }
+    idx := idx.contents + 1
+  }
+  level.contents->Option.getOr(0.)
+}
+
+// Per-partition mutable state threaded through the water-fill rounds below.
+type waterFillState = {
+  partitionId: string,
+  p: partition,
+  mutable cursor: int,
+  mutable chunksUsedThisCall: int,
+  // Existing mutPendingQueries count before this call — fixed for the call,
+  // used with chunksUsedThisCall against maxPendingChunksPerPartition.
+  pendingCount: int,
+  queryEndBlock: option<int>,
+  maybeChunkRange: option<int>,
+  // This partition's own slot in queriesByPartitionIndex — gap-fill and every
+  // water-fill round push here directly, so query order falls out of
+  // idsInAscOrder for free (see getNextQuery) instead of a final sort pass.
+  bucket: array<query>,
+}
 
+// Candidate queries are sized against chainTargetBlock, the soft querying
+// horizon the owning chain wants to reach this tick — derived by ChainState
+// from its share of the indexer-wide buffer budget and its chain-level event
+// density. chainTargetBlock is never used as a hard query end, only to (a)
+// select which partitions are "in range" this tick and (b) size an open-ended
+// query with no other ceiling: the true hard bounds stay
+// endBlock/mergeBlock/the lagged head.
+//
+// In-range partitions share chainTargetItems (this chain's target total
+// footprint — in-flight plus new) by water-fill: each round pours exactly the
+// remaining fresh budget at the level computed by waterLevel, so a partition
+// already holding more than the level (from earlier ticks' in-flight queries)
+// gets nothing and its implicit share flows to the others, while totals stay
+// even and the pour never exceeds the budget. Rounds repeat only because
+// emits are quantized: a partition may consume less than its allotment (range
+// or chunk-cap runs out) or slightly more (min one chunk), and the leftover
+// re-pours over whoever still has range left.
+//
+// A partition with a trusted positive density (two or more responses — see
+// getMinHistoryRange) always emits at least one full-size chunk/query once
+// given an allotment, sized by its real density — may overshoot the
+// allotment by at most one chunk (the server also enforces itemsTarget via a
+// maxNumLogs-style cap, so an overshoot truncates the response rather than
+// the buffer). Any other partition (no signal, one noisy sample, or a
+// density-0 estimate) emits one open-ended probe sized exactly at its
+// allotment instead.
 let getNextQuery = (
-  {
-    buffer,
-    optimizedPartitions,
-    indexingAddresses,
-    blockLag,
-    latestOnBlockBlockNumber,
-    knownHeight,
-  } as fetchState: t,
-  ~concurrencyLimit,
-  ~bufferLimit,
+  {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
+  ~chainTargetBlock: int,
+  ~chainTargetItems: float,
+  ~chunkItemsMultiplier: float=1.,
 ) => {
   let headBlockNumber = knownHeight - blockLag
   if headBlockNumber <= 0 {
     WaitingForNewBlock
-  } else if concurrencyLimit === 0 {
-    ReachedMaxConcurrency
-  } else if bufferLimit <= 0 {
-    // No room left in the shared buffer pool for this chain; wait for processing
-    // to drain before fetching more.
-    NothingToQuery
   } else {
     let isOnBlockBehindTheHead = latestOnBlockBlockNumber < headBlockNumber
     let shouldWaitForNewBlock = ref(
-      switch fetchState.endBlock {
+      switch endBlock {
       | Some(endBlock) => headBlockNumber < endBlock
       | None => true
       } &&
       !isOnBlockBehindTheHead,
     )
 
-    // Limit how far ahead we fetch to bufferLimit items (this chain's share of
-    // the indexer-wide buffer pool) so processing always has buffer without
-    // ballooning memory. A partition that fetched further is skipped until the
-    // buffer drains.
-    let maxQueryBlockNumber = {
-      switch buffer->Array.get(bufferLimit - 1) {
-      | Some(item) =>
-        // Just in case check that we don't query beyond the current block
-        Pervasives.min(item->Internal.getItemBlockNumber, knownHeight)
-      | None => knownHeight
-      }
-    }
-
-    let queries = []
-
     let partitionsCount = optimizedPartitions.idsInAscOrder->Array.length
-    let idxRef = ref(0)
-    while idxRef.contents < partitionsCount {
-      let idx = idxRef.contents
+
+    // Every partition is visited once here regardless of whether it gets a
+    // query pushed below — waiting-for-new-block bookkeeping shouldn't depend
+    // on this tick's budget.
+    for idx in 0 to partitionsCount - 1 {
       let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
       let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
-
-      let pendingCount = p.mutPendingQueries->Array.length
-      let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber
-      let hasPendingQueries = pendingCount > 0
-
-      if hasPendingQueries || isBehindTheHead {
+      if (
+        p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock.blockNumber < headBlockNumber
+      ) {
         // Even if there are some partitions waiting for the new block
         // We still want to wait for all partitions reaching the head
         // because they might update knownHeight in their response
@@ -1429,50 +1586,86 @@ let getNextQuery = (
         // and we don't want to poll the head for a few small partitions
         shouldWaitForNewBlock := false
       }
+    }
 
-      let partitionQueriesStart = queries->Array.length
-
-      // Compute queryEndBlock for this partition
-      let queryEndBlock = Utils.Math.minOptInt(fetchState.endBlock, p.mergeBlock)
-      let queryEndBlock = switch blockLag {
+    // One bucket per partition, in idsInAscOrder order — gap-fill and
+    // water-fill both push into a partition's own bucket, so flattening at
+    // the end (see below) reproduces idsInAscOrder without a sort.
+    let queriesByPartitionIndex: array<
+      array<query>,
+    > = Array.fromInitializer(~length=partitionsCount, _ => [])
+
+    // Compute queryEndBlock for this partition
+    let computeQueryEndBlock = (p: partition) => {
+      let queryEndBlock = Utils.Math.minOptInt(endBlock, p.mergeBlock)
+      switch blockLag {
       | 0 => queryEndBlock
       | _ =>
         // Force head block as an endBlock when blockLag is set
         // because otherwise HyperSync might return bigger range
         Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock)
       }
-      // Enforce the response range up until target block
-      // Otherwise for indexers with 100+ partitions
-      // we might blow up the buffer size to more than 600k events
-      // simply because of HyperSync returning extra blocks
-      let queryEndBlock = switch (queryEndBlock, maxQueryBlockNumber < knownHeight) {
-      | (Some(endBlock), true) => Some(Pervasives.min(maxQueryBlockNumber, endBlock))
-      | (None, true) => Some(maxQueryBlockNumber)
-      | (_, false) => queryEndBlock
+    }
+
+    // Each partition's existing in-flight itemsEst, summed once and reused
+    // both for chainReserved (the call-wide fresh-budget ceiling below) and to
+    // seed each partition's water-fill footprint — so a partition already
+    // holding in-flight queries is counted toward its even share and doesn't
+    // get topped up past the others.
+    let existingReservedByPartition = Dict.make()
+    let chainReserved = ref(0.)
+    for idx in 0 to partitionsCount - 1 {
+      let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
+      let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
+      let cost = ref(0.)
+      for pqIdx in 0 to p.mutPendingQueries->Array.length - 1 {
+        cost := cost.contents +. (p.mutPendingQueries->Array.getUnsafe(pqIdx)).itemsEst->Int.toFloat
       }
+      existingReservedByPartition->Dict.set(partitionId, cost.contents)
+      chainReserved := chainReserved.contents +. cost.contents
+    }
 
+    // Phase A: gap-fill. Walk each partition's pending queries once,
+    // unconditionally filling any hole (e.g. from an out-of-order partial
+    // chunk response) — uncapped, same as before this tick's water-fill was
+    // introduced. This also determines each partition's post-gap cursor and
+    // whether it's blocked on an unresolved single-shot query.
+    let fillStates = []
+    let gapFillCost = ref(0.)
+    for idx in 0 to partitionsCount - 1 {
+      let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
+      let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
+      let bucket = queriesByPartitionIndex->Array.getUnsafe(idx)
+      let pendingCount = p.mutPendingQueries->Array.length
+      let queryEndBlock = computeQueryEndBlock(p)
       let maybeChunkRange = getMinHistoryRange(p)
 
-      // Walk pending queries to find open ranges and create queries for each
       let cursor = ref(p.latestFetchedBlock.blockNumber + 1)
       let canContinue = ref(true)
+      let chunksUsedThisCall = ref(0)
       let pqIdx = ref(0)
-      while pqIdx.contents < p.mutPendingQueries->Array.length && canContinue.contents {
+      while pqIdx.contents < pendingCount && canContinue.contents {
         let pq = p.mutPendingQueries->Array.getUnsafe(pqIdx.contents)
 
-        // Gap before this pending query → create queries for the gap range
         if pq.fromBlock > cursor.contents {
-          pushQueriesForRange(
-            queries,
+          let beforeLen = bucket->Array.length
+          let cost = pushGapFillQueries(
+            bucket,
             ~partitionId,
             ~rangeFromBlock=cursor.contents,
             ~rangeEndBlock=Utils.Math.minOptInt(Some(pq.fromBlock - 1), queryEndBlock),
-            ~maxQueryBlockNumber,
+            ~knownHeight,
+            ~chainTargetBlock,
             ~maybeChunkRange,
+            ~maxChunks=maxPendingChunksPerPartition - pendingCount - chunksUsedThisCall.contents,
+            ~partition=p,
+            ~partitionBudget=chainTargetItems /. partitionsCount->Int.toFloat,
+            ~chunkItemsMultiplier,
             ~selection=p.selection,
             ~addressesByContractName=p.addressesByContractName,
-            ~indexingAddresses,
           )
+          chunksUsedThisCall := chunksUsedThisCall.contents + (bucket->Array.length - beforeLen)
+          gapFillCost := gapFillCost.contents +. cost
         }
         switch pq {
         | {isChunk: true, toBlock: Some(toBlock), fetchedBlock: Some({blockNumber})}
@@ -1484,40 +1677,180 @@ let getNextQuery = (
         pqIdx := pqIdx.contents + 1
       }
 
-      // Tail range after all pending queries
       if canContinue.contents {
-        pushQueriesForRange(
-          queries,
-          ~partitionId,
-          ~rangeFromBlock=cursor.contents,
-          ~rangeEndBlock=queryEndBlock,
-          ~maxQueryBlockNumber,
-          ~maybeChunkRange,
-          ~selection=p.selection,
-          ~addressesByContractName=p.addressesByContractName,
-          ~indexingAddresses,
-        )
+        fillStates
+        ->Array.push({
+          partitionId,
+          p,
+          cursor: cursor.contents,
+          chunksUsedThisCall: chunksUsedThisCall.contents,
+          pendingCount,
+          queryEndBlock,
+          maybeChunkRange,
+          bucket,
+        })
+        ->ignore
       }
+    }
 
-      // Cap parallel in-flight chunks per partition so a single partition can't
-      // drain the whole concurrency budget (especially the high realtime one).
-      // Keep the earliest new chunks; the furthest-ahead ones wait for the next
-      // round once these resolve.
-      let maxNewChunks = Pervasives.max(0, maxPendingChunksPerPartition - pendingCount)
-      let generatedCount = queries->Array.length - partitionQueriesStart
-      if generatedCount > maxNewChunks {
-        queries
-        ->Array.splice(
-          ~start=partitionQueriesStart + maxNewChunks,
-          ~remove=generatedCount - maxNewChunks,
-          ~insert=[],
-        )
-        ->ignore
+    // Fresh budget for this call — what's left of chainTargetItems after
+    // existing in-flight queries and this tick's gap-fill. The water-fill
+    // rounds below hand it out; a partition already holding a large share
+    // (seeded below) gets proportionally less so totals stay even.
+    let rangeItemsTarget = Pervasives.max(
+      0.,
+      chainTargetItems -. chainReserved.contents -. gapFillCost.contents,
+    )
+
+    // Each in-range partition's running footprint: existing in-flight plus any
+    // gap-fill just pushed into its bucket. The water-fill levels every
+    // partition up toward a shared line, so this seed is what a partition
+    // starts the fill already holding.
+    let reservedByPartition = Dict.make()
+    fillStates->Array.forEach(fs => {
+      let cost = ref(existingReservedByPartition->Dict.getUnsafe(fs.partitionId))
+      for i in 0 to fs.bucket->Array.length - 1 {
+        cost := cost.contents +. (fs.bucket->Array.getUnsafe(i)).itemsEst->Int.toFloat
       }
+      reservedByPartition->Dict.set(fs.partitionId, cost.contents)
+    })
 
-      idxRef := idxRef.contents + 1
+    let isInRange = (fs: waterFillState) =>
+      fs.cursor <= chainTargetBlock &&
+      switch fs.queryEndBlock {
+      | Some(eb) => fs.cursor <= eb
+      | None => true
+      } &&
+      fs.pendingCount + fs.chunksUsedThisCall < maxPendingChunksPerPartition
+
+    let inRangeStates = fillStates->Array.filter(isInRange)
+
+    // Emits this round's queries for one partition, given its water-fill
+    // allotment. Mutates fs.cursor/chunksUsedThisCall and returns the
+    // itemsEst consumed.
+    //
+    // Chunks require a POSITIVE trusted density: density 0 prices every chunk
+    // at ~nothing, letting a partition flood its full chunk pipeline with
+    // hard-bounded queries that crawl 1.8× per two responses — an open-ended
+    // probe instead gets the server's full scan range in one response.
+    let emitQueries = (fs: waterFillState, ~budget: float) => {
+      let p = fs.p
+      let maxBlock = switch fs.queryEndBlock {
+      | Some(eb) => eb
+      | None => chainTargetBlock
+      }
+      switch (fs.maybeChunkRange, p->getTrustedDensity) {
+      | (Some(minHistoryRange), Some(density)) if density > 0. =>
+        // Chunking active: strict chunks with a hard endBlock, sized by real
+        // density with the chunk headroom multiplier — at least one full
+        // chunk this round even if the allotment falls short.
+        let chunkSize = Js.Math.ceil_int(minHistoryRange->Int.toFloat *. 1.8)
+        let maxChunksRemaining =
+          maxPendingChunksPerPartition - fs.pendingCount - fs.chunksUsedThisCall
+        let consumed = ref(0.)
+        let created = ref(0)
+        let chunkFromBlock = ref(fs.cursor)
+        let budgetLeft = ref(true)
+        // No chunk starts past chainTargetBlock; an emitted chunk still keeps
+        // its full span (chunkToBlock may exceed the target — only
+        // endBlock/mergeBlock are hard bounds).
+        while (
+          budgetLeft.contents &&
+          created.contents < maxChunksRemaining &&
+          chunkFromBlock.contents <= Pervasives.min(maxBlock, chainTargetBlock)
+        ) {
+          let chunkToBlock = Pervasives.min(chunkFromBlock.contents + chunkSize - 1, maxBlock)
+          let rawEst = density *. (chunkToBlock - chunkFromBlock.contents + 1)->Int.toFloat
+          let itemsEst = Pervasives.max(1, rawEst->Math.ceil->Float.toInt)
+          let itemsTarget = Pervasives.max(
+            1,
+            (rawEst *. chunkItemsMultiplier)->Math.ceil->Float.toInt,
+          )
+          if consumed.contents == 0. || consumed.contents +. itemsEst->Int.toFloat <= budget {
+            fs.bucket->Array.push({
+              partitionId: fs.partitionId,
+              fromBlock: chunkFromBlock.contents,
+              toBlock: Some(chunkToBlock),
+              isChunk: true,
+              selection: p.selection,
+              itemsTarget,
+              itemsEst,
+              addressesByContractName: p.addressesByContractName,
+            })
+            consumed := consumed.contents +. itemsEst->Int.toFloat
+            chunkFromBlock := chunkToBlock + 1
+            created := created.contents + 1
+          } else {
+            budgetLeft := false
+          }
+        }
+        fs.cursor = chunkFromBlock.contents
+        fs.chunksUsedThisCall = fs.chunksUsedThisCall + created.contents
+        consumed.contents
+      | _ =>
+        let itemsTarget = Pervasives.max(1, Math.round(budget)->Float.toInt)
+        let itemsEst = itemsTarget
+        fs.bucket->Array.push({
+          partitionId: fs.partitionId,
+          fromBlock: fs.cursor,
+          toBlock: fs.queryEndBlock,
+          isChunk: false,
+          selection: p.selection,
+          itemsTarget,
+          itemsEst,
+          addressesByContractName: p.addressesByContractName,
+        })
+        fs.cursor = maxBlock + 1
+        fs.chunksUsedThisCall = fs.chunksUsedThisCall + 1
+        itemsEst->Int.toFloat
+      }
+    }
+
+    // Water-fill. Range membership is fixed by chainTargetBlock/queryEndBlock,
+    // so the in-range partitions are filtered once up front; each round pours
+    // the remaining fresh budget at the waterLevel of the still-not-filled
+    // partitions' footprints and keeps only those still in range for the next
+    // round. Allotments sum to exactly the poured budget, so the outcome is
+    // order-independent and never exceeds it — the only overshoot left is the
+    // min-one-chunk quantization in emitQueries, bounded by one chunk per
+    // partition per round.
+    //
+    // No explicit round cap: every emit advances chunksUsedThisCall (capped at
+    // maxPendingChunksPerPartition) and consumes a positive amount of fresh
+    // budget, so the not-filled set drains on its own and the loop also stops
+    // once the whole budget is poured.
+    let notFilledPartitions = ref(inRangeStates)
+    let remainingBudget = ref(rangeItemsTarget)
+    while notFilledPartitions.contents->Array.length > 0 && remainingBudget.contents > 0. {
+      let level = waterLevel(
+        ~budget=remainingBudget.contents,
+        ~footprints=notFilledPartitions.contents->Array.map(fs =>
+          reservedByPartition->Dict.getUnsafe(fs.partitionId)
+        ),
+      )
+      let next = []
+      notFilledPartitions.contents->Array.forEach(fs => {
+        let reserved = reservedByPartition->Dict.getUnsafe(fs.partitionId)
+        let budget = level -. reserved
+        if budget > 0. {
+          let consumed = emitQueries(fs, ~budget)
+          reservedByPartition->Dict.set(fs.partitionId, reserved +. consumed)
+          remainingBudget := remainingBudget.contents -. consumed
+
+          if fs->isInRange {
+            next->Array.push(fs)->ignore
+          }
+        }
+      })
+      notFilledPartitions := next
     }
 
+    // Each partition pushed only into its own bucket (indexed by its
+    // idsInAscOrder position), so flattening reproduces idsInAscOrder order
+    // directly — no sort needed even though water-fill rounds interleave
+    // across partitions.
+    let queries = queriesByPartitionIndex->Array.flat
+
     if queries->Utils.Array.isEmpty {
       if shouldWaitForNewBlock.contents {
         WaitingForNewBlock
@@ -1525,13 +1858,6 @@ let getNextQuery = (
         NothingToQuery
       }
     } else {
-      // Enforce concurrency limit: sort by fromBlock and take the first concurrencyLimit
-      let queries = if queries->Array.length > concurrencyLimit {
-        queries->Array.sort((a, b) => Int.compare(a.fromBlock, b.fromBlock))
-        queries->Array.slice(~start=0, ~end=concurrencyLimit)
-      } else {
-        queries
-      }
       Ready(queries)
     }
   }
@@ -1573,14 +1899,15 @@ Instantiates a fetch state with partitions for initial addresses
 let make = (
   ~startBlock,
   ~endBlock,
-  ~eventConfigs: array<Internal.eventConfig>,
+  ~onEventRegistrations: array<Internal.onEventRegistration>,
+  ~contractConfigs: dict<IndexingAddresses.contractConfig>,
   ~addresses: array<Internal.indexingAddress>,
   ~maxAddrInPartition,
   ~chainId,
   ~maxOnBlockBufferSize,
   ~knownHeight,
   ~progressBlockNumber=startBlock - 1,
-  ~onBlockConfigs=[],
+  ~onBlockRegistrations=[],
   ~blockLag=0,
   ~firstEventBlock=None,
 ): t => {
@@ -1590,38 +1917,15 @@ let make = (
   }
 
   let notDependingOnAddresses = []
-  let normalEventConfigs = []
+  let normalRegistrations = []
   let contractNamesWithNormalEvents = Utils.Set.make()
-  let indexingAddresses = Dict.make()
-  let contractConfigs: dict<contractConfig> = Dict.make()
-
-  eventConfigs->Array.forEach(ec => {
-    switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(ec.contractName) {
-    | Some({startBlock}) =>
-      contractConfigs->Dict.set(
-        ec.contractName,
-        {
-          startBlock: switch (startBlock, ec.startBlock) {
-          | (Some(a), Some(b)) => Some(Pervasives.min(a, b))
-          | (Some(_) as s, None) | (None, Some(_) as s) => s
-          | (None, None) => None
-          },
-        },
-      )
-    | None =>
-      contractConfigs->Dict.set(
-        ec.contractName,
-        {
-          startBlock: ec.startBlock,
-        },
-      )
-    }
 
-    if ec.dependsOnAddresses {
-      normalEventConfigs->Array.push(ec)
-      contractNamesWithNormalEvents->Utils.Set.add(ec.contractName)->ignore
+  onEventRegistrations->Array.forEach(reg => {
+    if reg.dependsOnAddresses {
+      normalRegistrations->Array.push(reg)
+      contractNamesWithNormalEvents->Utils.Set.add(reg.eventConfig.contractName)->ignore
     } else {
-      notDependingOnAddresses->Array.push(ec)
+      notDependingOnAddresses->Array.push(reg)
     }
   })
 
@@ -1633,7 +1937,7 @@ let make = (
       latestFetchedBlock,
       selection: {
         dependsOnAddresses: false,
-        eventConfigs: notDependingOnAddresses,
+        onEventRegistrations: notDependingOnAddresses,
       },
       addressesByContractName: Dict.make(),
       mergeBlock: None,
@@ -1641,13 +1945,14 @@ let make = (
       mutPendingQueries: [],
       prevQueryRange: 0,
       prevPrevQueryRange: 0,
+      prevRangeSize: 0,
       latestBlockRangeUpdateBlock: 0,
     })
   }
 
   let normalSelection = {
     dependsOnAddresses: true,
-    eventConfigs: normalEventConfigs,
+    onEventRegistrations: normalRegistrations,
   }
 
   let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
@@ -1655,32 +1960,16 @@ let make = (
 
   addresses->Array.forEach(contract => {
     let contractName = contract.contractName
-    let contractStartBlock = switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(
-      contractName,
-    ) {
-    | Some({startBlock}) => startBlock
-    | None => None
-    }
-    let ia: indexingAddress = {
-      address: contract.address,
-      contractName: contract.contractName,
-      registrationBlock: contract.registrationBlock,
-      effectiveStartBlock: deriveEffectiveStartBlock(
-        ~registrationBlock=contract.registrationBlock,
-        ~contractStartBlock,
-      ),
-    }
-    // Track the address on fetchState regardless of whether it currently has
-    // matching events. This way, if the config is updated later to add events
-    // for this contract, the address is already known.
-    indexingAddresses->Dict.set(contract.address->Address.toString, ia)
 
     // Only addresses whose contract has events that depend on addresses get
     // registered for active fetching via partitions.
     if contractNamesWithNormalEvents->Utils.Set.has(contractName) {
-      let registeringContracts =
-        registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(contractName)
-      registeringContracts->Dict.set(contract.address->Address.toString, ia)
+      registeringContractsByContract
+      ->Utils.Dict.getOrInsertEmptyDict(contractName)
+      ->Dict.set(
+        contract.address->Address.toString,
+        IndexingAddresses.makeIndexingAddress(~contract, ~contractConfigs),
+      )
 
       // Detect dynamic contracts by registrationBlock
       if contract.registrationBlock !== -1 {
@@ -1699,12 +1988,15 @@ let make = (
     ~progressBlockNumber,
   )
 
-  if optimizedPartitions->OptimizedPartitions.count === 0 && onBlockConfigs->Utils.Array.isEmpty {
+  if (
+    optimizedPartitions->OptimizedPartitions.count === 0 &&
+      onBlockRegistrations->Utils.Array.isEmpty
+  ) {
     JsError.throwWithMessage(
       `Invalid configuration: Nothing to fetch on chain ${chainId->Int.toString}. ` ++
       `addresses=${addresses->Array.length->Int.toString}, ` ++
-      `eventConfigs=${eventConfigs->Array.length->Int.toString}, ` ++
-      `normalEventConfigs=${normalEventConfigs
+      `onEventRegistrations=${onEventRegistrations->Array.length->Int.toString}, ` ++
+      `normalRegistrations=${normalRegistrations
         ->Array.length
         ->Int.toString}. ` ++ `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`,
     )
@@ -1715,14 +2007,14 @@ let make = (
   // fetching, so without seeding the buffer here getNextQuery would return
   // NothingToQuery and the indexer would get stuck.
   let buffer = []
-  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockConfigs->Utils.Array.notEmpty {
+  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockRegistrations->Utils.Array.notEmpty {
     let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
     | None => knownHeight
     | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock.blockNumber
     }
     appendOnBlockItems(
       ~mutItems=buffer,
-      ~onBlockConfigs,
+      ~onBlockRegistrations,
       ~indexerStartBlock=startBlock,
       ~fromBlock=progressBlockNumber,
       ~maxBlockNumber,
@@ -1740,17 +2032,14 @@ let make = (
     endBlock,
     latestOnBlockBlockNumber,
     normalSelection,
-    indexingAddresses,
     blockLag,
-    onBlockConfigs,
+    onBlockRegistrations,
     maxOnBlockBufferSize,
     knownHeight,
     buffer,
     firstEventBlock,
   }
 
-  let numAddresses = indexingAddresses->Utils.Dict.size
-  Prometheus.IndexingAddresses.set(~addressesCount=numAddresses, ~chainId)
   Prometheus.IndexingPartitions.set(
     ~partitionsCount=optimizedPartitions->OptimizedPartitions.count,
     ~chainId,
@@ -1767,25 +2056,6 @@ let make = (
 
 let bufferSize = ({buffer}: t) => buffer->Array.length
 
-// Number of buffered items at or below the ready frontier (processable now,
-// i.e. not stuck behind a gap from a lagging partition or out-of-order chunk).
-// The buffer is kept sorted, so binary-search the frontier in O(log n).
-let bufferReadyCount = (fetchState: t) => {
-  let frontier = fetchState->bufferBlockNumber
-  let buffer = fetchState.buffer
-  let lo = ref(0)
-  let hi = ref(buffer->Array.length)
-  while lo.contents < hi.contents {
-    let mid = (lo.contents + hi.contents) / 2
-    if buffer->Array.getUnsafe(mid)->Internal.getItemBlockNumber <= frontier {
-      lo := mid + 1
-    } else {
-      hi := mid
-    }
-  }
-  lo.contents
-}
-
 let rollbackPendingQueries = (mutPendingQueries: array<pendingQuery>, ~targetBlockNumber) => {
   // - Remove queries where fromBlock > target
   // - Cap fetchedBlock at target where fetchedBlock > target
@@ -1817,18 +2087,11 @@ Always recreates optimized partitions to avoid duplicate addresses:
 - Non-wildcard with lfb <= target: keep, adjust pending queries and mergeBlock
 - Non-wildcard with lfb > target: delete, track addresses for recreation
 */
-let rollback = (fetchState: t, ~targetBlockNumber) => {
-  // Step 1: Build addressesToRemove and surviving indexingAddresses
-  let addressesToRemove = Utils.Set.make()
-  let indexingAddresses = Dict.make()
-
-  fetchState.indexingAddresses->Utils.Dict.forEachWithKey((indexingContract, address) => {
-    if indexingContract.registrationBlock > targetBlockNumber {
-      let _ = addressesToRemove->Utils.Set.add(address->Address.unsafeFromString)
-    } else {
-      indexingAddresses->Dict.set(address, indexingContract)
-    }
-  })
+let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetBlockNumber) => {
+  // Step 1: Prune addresses registered after the target block. The pruned index is
+  // then the source of truth for partition cleanup below — an address survives iff
+  // it's still in the index.
+  indexingAddresses->IndexingAddresses.rollbackInPlace(~targetBlockNumber)
 
   // Step 2: Categorize partitions
   let keptPartitions = []
@@ -1858,18 +2121,12 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
     | _ if p.latestFetchedBlock.blockNumber > targetBlockNumber =>
       p.addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
         addresses->Array.forEach(address => {
-          if (
-            !(addressesToRemove->Utils.Set.has(address)) &&
-            indexingAddresses
-            ->Utils.Dict.dangerouslyGetNonOption(address->Address.toString)
-            ->Option.isSome
-          ) {
+          switch indexingAddresses->IndexingAddresses.get(address->Address.toString) {
+          | Some(indexingContract) =>
             let registeringContracts =
               registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(contractName)
-            registeringContracts->Dict.set(
-              address->Address.toString,
-              indexingAddresses->Dict.getUnsafe(address->Address.toString),
-            )
+            registeringContracts->Dict.set(address->Address.toString, indexingContract)
+          | None => ()
           }
         })
       })
@@ -1882,11 +2139,13 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
         | other => other
         }
 
-        // Remove addresses that should be removed
+        // Drop addresses pruned from the index
         let rollbackedAddressesByContractName = Dict.make()
         addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
           let keptAddresses =
-            addresses->Array.filter(address => !(addressesToRemove->Utils.Set.has(address)))
+            addresses->Array.filter(address =>
+              indexingAddresses->IndexingAddresses.get(address->Address.toString)->Option.isSome
+            )
           if keptAddresses->Array.length > 0 {
             rollbackedAddressesByContractName->Dict.set(contractName, keptAddresses)
           }
@@ -1930,7 +2189,6 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
     ),
   }->updateInternal(
     ~optimizedPartitions,
-    ~indexingAddresses,
     ~mutItems=fetchState.buffer->Array.filter(item =>
       switch item {
       | Event({blockNumber})