envio 3.3.0-alpha.1 → 3.3.0-alpha.11

package/src/FetchState.res

@@ -1,11 +1,5 @@
-type contractConfig = {startBlock: option<int>}
-
 type indexingAddress = Internal.indexingContract
 
-let deriveEffectiveStartBlock = (~registrationBlock: int, ~contractStartBlock: option<int>) => {
-  Pervasives.max(Pervasives.max(registrationBlock, 0), contractStartBlock->Option.getOr(0))
-}
-
 type blockNumberAndTimestamp = {
   blockNumber: int,
   blockTimestamp: int,
@@ -13,15 +7,21 @@ type blockNumberAndTimestamp = {
 
 type blockNumberAndLogIndex = {blockNumber: int, logIndex: int}
 
-type selection = {eventConfigs: array<Internal.eventConfig>, dependsOnAddresses: bool}
+type selection = {
+  onEventRegistrations: array<Internal.onEventRegistration>,
+  dependsOnAddresses: bool,
+}
 
 type pendingQuery = {
   fromBlock: int,
   toBlock: option<int>,
   isChunk: bool,
-  // Estimated items this in-flight query will add, carried from the query so the
-  // shared buffer budget can account for what's already being fetched.
-  estResponseSize: float,
+  // Items this in-flight query is targeting (server maxNumLogs-style cap).
+  itemsTarget: int,
+  // Estimated items this in-flight query will actually return (no headroom),
+  // carried from the query so the shared buffer budget can account for what's
+  // already being fetched without the cap's safety margin inflating it.
+  itemsEst: int,
   // Stores latestFetchedBlock when query completes. Only needed to persist
   // timestamp while earlier queries are still pending before updating
   // the partition's latestFetchedBlock.
@@ -65,38 +65,51 @@ type query = {
   fromBlock: int,
   toBlock: option<int>,
   isChunk: bool,
-  // Estimated number of items the query will return, from the partition's
-  // density and the query's block range. Used to admit queries against the
-  // shared buffer budget.
-  estResponseSize: float,
-  // Owning chain and the chain progress % at the query's fromBlock. Set by the
-  // cross-chain scheduler so candidate queries can be pooled and ordered
-  // (furthest-behind first) without allocating a side tuple per query.
-  mutable chainId: int,
-  mutable progress: float,
+  // Items this query targets: the server-side maxNumLogs-style cap, sized
+  // with headroom (chunkItemsMultiplier) so a denser-than-expected range
+  // doesn't truncate the response.
+  itemsTarget: int,
+  // Expected items without headroom: density × the query's block range for a
+  // known-density partition, the query's budget share otherwise. This is the
+  // unit the chain's per-tick budget is reserved/consumed in — reserving the
+  // headroomed cap instead would throttle the pipeline by the safety margin.
+  itemsEst: int,
   selection: selection,
   addressesByContractName: dict<array<Address.t>>,
-  indexingAddresses: dict<indexingAddress>,
 }
 
-// Default estimate for a query whose partition hasn't responded yet, so the
-// shared budget still accounts for unknown queries instead of treating them as
-// free.
-let defaultEstResponseSize = 10_000.
-
-// Estimated items a query will return, from the partition's event density
-// (items/block derived from its last response) and the query's block range.
-// toBlock None is the open-ended tail, capped at maxQueryBlockNumber. A partition
-// that responded with no items has density 0, so its queries cost 0 — correct,
-// they don't fill the buffer. Only a partition that has never responded
-// (prevQueryRange 0) has no signal, so it falls back to defaultEstResponseSize.
-let calculateEstResponseSize = (p: partition, ~fromBlock, ~toBlock, ~maxQueryBlockNumber) =>
-  if p.prevQueryRange > 0 {
-    let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
-    (toBlock->Option.getOr(maxQueryBlockNumber) - fromBlock + 1)->Int.toFloat *. density
-  } else {
-    defaultEstResponseSize
-  }
+// Invert addressesByContractName into address→contractName for log-ownership
+// routing. 1:1 today (each address belongs to one contract), so no key
+// collisions. Memoized on the addressesByContractName object so a partition's
+// many responses share one derivation and a large factory never rebuilds the
+// whole index; sound because the dict is immutable after construction (every
+// mutation produces a new dict).
+let deriveContractNameByAddress: dict<array<Address.t>> => dict<
+  string,
+> = Utils.WeakMap.memoize(addressesByContractName => {
+  let result = Dict.make()
+  addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
+    for i in 0 to addresses->Array.length - 1 {
+      result->Dict.set(addresses->Array.getUnsafe(i)->Address.toString, contractName)
+    }
+  })
+  result
+})
+
+// itemsTarget for a query over [fromBlock, toBlock] at the given event density
+// (items/block). toBlock None is the open-ended tail, capped at
+// chainTargetBlock — the soft per-tick horizon the owning chain wants to reach
+// (see getNextQuery).
+let densityItemsTarget = (~density, ~fromBlock, ~toBlock, ~chainTargetBlock) => {
+  // Floor at 1: the reservation must equal the server-side cap SourceManager
+  // sends, and a 0 cap would ask the backend for nothing.
+  Pervasives.max(
+    1,
+    ((toBlock->Option.getOr(chainTargetBlock) - fromBlock + 1)->Int.toFloat *. density)
+    ->Math.ceil
+    ->Float.toInt,
+  )
+}
 
 // Calculate the chunk range from history using min-of-last-3-ranges heuristic
 let getMinHistoryRange = (p: partition) => {
@@ -106,6 +119,15 @@ let getMinHistoryRange = (p: partition) => {
   }
 }
 
+// Density (items/block) from the last response, trusted only after two
+// responses — a single sample is too noisy to size the next query by.
+let getTrustedDensity = (p: partition) => {
+  switch (p.prevQueryRange, p.prevPrevQueryRange) {
+  | (0, _) | (_, 0) => None
+  | (prevQueryRange, _) => Some(p.prevRangeSize->Int.toFloat /. prevQueryRange->Int.toFloat)
+  }
+}
+
 let getMinQueryRange = (partitions: array<partition>) => {
   let min = ref(0)
   for i in 0 to partitions->Array.length - 1 {
@@ -185,6 +207,12 @@ module OptimizedPartitions = {
       let newId = nextPartitionIndexRef.contents->Int.toString
       nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
       let minRange = getMinQueryRange([p1, p2])
+      // The merged partition indexes both parents' addresses, so its expected
+      // event rate is the sum of their densities. Parents without a trusted
+      // density contribute 0; if none has one, prevRangeSize stays 0 and the
+      // partition probes for a fresh signal instead of chunking.
+      let inheritedDensity =
+        p1->getTrustedDensity->Option.getOr(0.) +. p2->getTrustedDensity->Option.getOr(0.)
       {
         id: newId,
         dynamicContract: Some(contractName),
@@ -195,7 +223,7 @@ module OptimizedPartitions = {
         mutPendingQueries: [],
         prevQueryRange: minRange,
         prevPrevQueryRange: minRange,
-        prevRangeSize: 0,
+        prevRangeSize: (inheritedDensity *. minRange->Int.toFloat)->Math.ceil->Float.toInt,
         latestBlockRangeUpdateBlock: 0,
       }
     }
@@ -453,15 +481,21 @@ module OptimizedPartitions = {
         switch query.toBlock {
         | None => latestFetchedBlock.blockNumber < knownHeight - 10 // Don't update block range when very close to the head
         | Some(queryToBlock) =>
-          // Update on partial response (direct capacity evidence),
-          // or when the query's intended range covers at least the partition's
-          // current chunk range — meaning it was a capacity-based split chunk,
-          // not a small gap-fill whose toBlock is an artificial boundary.
-          latestFetchedBlock.blockNumber < queryToBlock ||
+          if latestFetchedBlock.blockNumber < queryToBlock {
+            // Partial response is direct capacity evidence — unless it was
+            // truncated by our own itemsTarget cap: that reflects the
+            // reservation we asked for, not what the server could return.
+            itemsCount < query.itemsTarget
+          } else {
+            // A full response updates only when the query's intended range
+            // covers at least the partition's current chunk range — meaning it
+            // was a capacity-based split chunk, not a small gap-fill whose
+            // toBlock is an artificial boundary.
             switch getMinHistoryRange(p) {
             | None => false // Chunking not active yet, don't update
             | Some(minHistoryRange) => queryToBlock - query.fromBlock + 1 >= minHistoryRange
             }
+          }
         }
     let updatedPrevQueryRange = shouldUpdateBlockRange ? blockRange : p.prevQueryRange
     let updatedPrevPrevQueryRange = shouldUpdateBlockRange ? p.prevQueryRange : p.prevPrevQueryRange
@@ -519,10 +553,8 @@ type t = {
   startBlock: int,
   endBlock: option<int>,
   normalSelection: selection,
-  // By address
-  indexingAddresses: dict<indexingAddress>,
   // By contract name
-  contractConfigs: dict<contractConfig>,
+  contractConfigs: dict<IndexingAddresses.contractConfig>,
   // Not used for logic - only metadata
   chainId: int,
   // The block number of the latest block which was added to the queue
@@ -536,10 +568,10 @@ type t = {
   // Buffer of items ordered from earliest to latest
   buffer: array<Internal.item>,
   // Caps how far ahead onBlock items are pre-generated (set to 2x the batch
-  // size). Fetch depth is bounded separately by getNextQuery's itemBudget, the
-  // chain's per-tick slice of the indexer-wide pool.
+  // size). Event fetch depth is bounded separately, by CrossChainState's
+  // cross-chain admission against the indexer-wide buffer pool.
   maxOnBlockBufferSize: int,
-  onBlockConfigs: array<Internal.onBlockConfig>,
+  onBlockRegistrations: array<Internal.onBlockRegistration>,
   knownHeight: int,
   firstEventBlock: option<int>,
 }
@@ -575,30 +607,124 @@ let bufferBlock = ({optimizedPartitions, latestOnBlockBlockNumber}: t) => {
   }
 }
 
+// Number of buffered items at or below the ready frontier (processable now,
+// i.e. not stuck behind a gap from a lagging partition or out-of-order chunk).
+// The buffer is kept sorted, so binary-search the frontier in O(log n).
+let bufferReadyCount = (fetchState: t) => {
+  let frontier = fetchState->bufferBlockNumber
+  let buffer = fetchState.buffer
+  let lo = ref(0)
+  let hi = ref(buffer->Array.length)
+  while lo.contents < hi.contents {
+    let mid = (lo.contents + hi.contents) / 2
+    if buffer->Array.getUnsafe(mid)->Internal.getItemBlockNumber <= frontier {
+      lo := mid + 1
+    } else {
+      hi := mid
+    }
+  }
+  lo.contents
+}
+
 /*
 Comparitor for two events from the same chain. No need for chain id or timestamp
 */
-let compareBufferItem = (a: Internal.item, b: Internal.item) => {
-  let blockOrdering = Int.compare(a->Internal.getItemBlockNumber, b->Internal.getItemBlockNumber)
-  if blockOrdering === Ordering.equal {
-    Int.compare(a->Internal.getItemLogIndex, b->Internal.getItemLogIndex)
+let getRegistrationIndex = (item: Internal.item): int =>
+  switch item {
+  | Event({onEventRegistration}) => onEventRegistration.index
+  | Block({onBlockRegistration}) => onBlockRegistration.index
+  }
+
+// Total order on buffer items: block, then logIndex, then registration index.
+// Returns a plain int (-1/0/1) with explicit field comparisons so it can be
+// called directly from the merge/insertion loops below — no Array.sort callback,
+// no allocated key. `0` means a true duplicate: same log routed to the same
+// registration (two registrations for one log differ by index and are kept).
+let compareBufferItem = (a: Internal.item, b: Internal.item): int => {
+  let ba = a->Internal.getItemBlockNumber
+  let bb = b->Internal.getItemBlockNumber
+  if ba != bb {
+    ba < bb ? -1 : 1
   } else {
-    blockOrdering
+    let la = a->Internal.getItemLogIndex
+    let lb = b->Internal.getItemLogIndex
+    if la != lb {
+      la < lb ? -1 : 1
+    } else {
+      let ia = a->getRegistrationIndex
+      let ib = b->getRegistrationIndex
+      ia < ib ? -1 : ia > ib ? 1 : 0
+    }
   }
 }
 
+// Merge a maybe-unsorted `newItems` run into the already-sorted, already-deduped
+// `buffer`, dropping items equal on (blockNumber, logIndex, registration index).
+// Single linear pass over both runs after ordering `newItems` in place; every
+// comparison is a direct `compareBufferItem` call (V8 inlines it) rather than a
+// callback through `Array.sort`.
+let mergeIntoBuffer = (buffer: array<Internal.item>, newItems: array<Internal.item>): array<
+  Internal.item,
+> => {
+  let n = newItems->Array.length
+  // Insertion sort: a source response is small and usually already ascending,
+  // so this is ~O(n) here.
+  for i in 1 to n - 1 {
+    let x = newItems->Array.getUnsafe(i)
+    let j = ref(i - 1)
+    while j.contents >= 0 && compareBufferItem(newItems->Array.getUnsafe(j.contents), x) > 0 {
+      newItems->Array.setUnsafe(j.contents + 1, newItems->Array.getUnsafe(j.contents))
+      j := j.contents - 1
+    }
+    newItems->Array.setUnsafe(j.contents + 1, x)
+  }
+
+  let m = buffer->Array.length
+  let merged = []
+  let last = ref(None)
+  let push = item =>
+    switch last.contents {
+    | Some(l) if compareBufferItem(l, item) === 0 => ()
+    | _ => {
+        merged->Array.push(item)
+        last := Some(item)
+      }
+    }
+
+  let i = ref(0)
+  let j = ref(0)
+  while i.contents < m && j.contents < n {
+    let a = buffer->Array.getUnsafe(i.contents)
+    let b = newItems->Array.getUnsafe(j.contents)
+    if compareBufferItem(a, b) <= 0 {
+      push(a)
+      i := i.contents + 1
+    } else {
+      push(b)
+      j := j.contents + 1
+    }
+  }
+  while i.contents < m {
+    push(buffer->Array.getUnsafe(i.contents))
+    i := i.contents + 1
+  }
+  while j.contents < n {
+    push(newItems->Array.getUnsafe(j.contents))
+    j := j.contents + 1
+  }
+  merged
+}
+
 // Some big number which should be bigger than any log index
 let blockItemLogIndex = 16777216
 
-let numAddresses = fetchState => fetchState.indexingAddresses->Utils.Dict.size
-
 // Appends Block items produced by the onBlock handlers for every block in
 // (fromBlock, maxBlockNumber] into mutItems and returns the new
 // latestOnBlockBlockNumber pointer. maxOnBlockBufferSize bounds how many items
 // are generated at once to prevent OOM.
 let appendOnBlockItems = (
   ~mutItems: array<Internal.item>,
-  ~onBlockConfigs: array<Internal.onBlockConfig>,
+  ~onBlockRegistrations: array<Internal.onBlockRegistration>,
   ~indexerStartBlock,
   ~fromBlock,
   ~maxBlockNumber,
@@ -618,27 +744,27 @@ let appendOnBlockItems = (
     let blockNumber = latestOnBlockBlockNumber.contents + 1
     latestOnBlockBlockNumber := blockNumber
 
-    for configIdx in 0 to onBlockConfigs->Array.length - 1 {
-      let onBlockConfig = onBlockConfigs->Array.getUnsafe(configIdx)
+    for configIdx in 0 to onBlockRegistrations->Array.length - 1 {
+      let onBlockRegistration = onBlockRegistrations->Array.getUnsafe(configIdx)
 
-      let handlerStartBlock = switch onBlockConfig.startBlock {
+      let handlerStartBlock = switch onBlockRegistration.startBlock {
       | Some(startBlock) => startBlock
       | None => indexerStartBlock
       }
 
       if (
         blockNumber >= handlerStartBlock &&
-        switch onBlockConfig.endBlock {
+        switch onBlockRegistration.endBlock {
         | Some(endBlock) => blockNumber <= endBlock
         | None => true
         } &&
-        (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockConfig.interval) === 0
+        (blockNumber - handlerStartBlock)->Pervasives.mod(onBlockRegistration.interval) === 0
       ) {
         mutItems->Array.push(
           Block({
-            onBlockConfig,
+            onBlockRegistration,
             blockNumber,
-            logIndex: blockItemLogIndex + onBlockConfig.index,
+            logIndex: blockItemLogIndex + onBlockRegistration.index,
           }),
         )
         newItemsCounter := newItemsCounter.contents + 1
@@ -656,50 +782,47 @@ Runs partition optimization when partitions change.
 let updateInternal = (
   fetchState: t,
   ~optimizedPartitions=fetchState.optimizedPartitions,
-  ~indexingAddresses=fetchState.indexingAddresses,
   ~mutItems=?,
+  // Set when the caller already passes a sorted, deduped buffer (hot paths merge
+  // via mergeIntoBuffer or filter the sorted buffer). Otherwise mutItems is
+  // normalized here, so callers can hand over items in any order.
+  ~mutItemsSorted=false,
   ~blockLag=fetchState.blockLag,
   ~knownHeight=fetchState.knownHeight,
 ): t => {
-  let mutItemsRef = ref(mutItems)
+  // The buffer to build on: the caller's items (normalized to sorted if needed),
+  // or the current buffer when only onBlock items change.
+  let base = switch mutItems {
+  | Some(items) => mutItemsSorted ? items : []->mergeIntoBuffer(items)
+  | None => fetchState.buffer
+  }
 
-  let latestOnBlockBlockNumber = switch fetchState.onBlockConfigs {
+  // onBlock items are generated as their own ascending (block, logIndex) run and
+  // folded into `base` by the single merge below.
+  let blockItems = []
+  let latestOnBlockBlockNumber = switch fetchState.onBlockRegistrations {
   | [] => knownHeight
-  | onBlockConfigs => {
-      // Calculate the max block number we are going to create items for
-      // Use maxOnBlockBufferSize to get the last target item in the buffer
-      //
-      // mutItems is not very reliable, since it might not be sorted,
-      // but the chances for it happen are very low and not critical
-      //
-      // All this needed to prevent OOM when adding too many block items to the queue
-      let maxBlockNumber = switch switch mutItemsRef.contents {
-      | Some(mutItems) => mutItems
-      | None => fetchState.buffer
-      }->Array.get(fetchState.maxOnBlockBufferSize - 1) {
-      | Some(item) => item->Internal.getItemBlockNumber
-      | None =>
-        switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
-        | None => knownHeight
-        | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock.blockNumber
-        }
-      }
-
-      let mutItems = switch mutItemsRef.contents {
-      | Some(mutItems) => mutItems
-      | None => fetchState.buffer->Array.copy
+  | onBlockRegistrations =>
+    // Calculate the max block number we are going to create items for
+    // Use maxOnBlockBufferSize to get the last target item in the buffer
+    // (sorted, so this is the highest-block item within the buffer cap).
+    // All this needed to prevent OOM when adding too many block items to the queue
+    let maxBlockNumber = switch base->Array.get(fetchState.maxOnBlockBufferSize - 1) {
+    | Some(item) => item->Internal.getItemBlockNumber
+    | None =>
+      switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
+      | None => knownHeight
+      | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock.blockNumber
       }
-      mutItemsRef := Some(mutItems)
-
-      appendOnBlockItems(
-        ~mutItems,
-        ~onBlockConfigs,
-        ~indexerStartBlock=fetchState.startBlock,
-        ~fromBlock=fetchState.latestOnBlockBlockNumber,
-        ~maxBlockNumber,
-        ~maxOnBlockBufferSize=fetchState.maxOnBlockBufferSize,
-      )
     }
+    appendOnBlockItems(
+      ~mutItems=blockItems,
+      ~onBlockRegistrations,
+      ~indexerStartBlock=fetchState.startBlock,
+      ~fromBlock=fetchState.latestOnBlockBlockNumber,
+      ~maxBlockNumber,
+      ~maxOnBlockBufferSize=fetchState.maxOnBlockBufferSize,
+    )
   }
 
   let updatedFetchState = {
@@ -708,22 +831,16 @@ let updateInternal = (
     contractConfigs: fetchState.contractConfigs,
     normalSelection: fetchState.normalSelection,
     chainId: fetchState.chainId,
-    onBlockConfigs: fetchState.onBlockConfigs,
+    onBlockRegistrations: fetchState.onBlockRegistrations,
     maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     optimizedPartitions,
     latestOnBlockBlockNumber,
-    indexingAddresses,
     blockLag,
     knownHeight,
-    buffer: switch mutItemsRef.contents {
-    // Theoretically it could be faster to asume that
-    // the items are sorted, but there are cases
-    // when the data source returns them unsorted
-    | Some(mutItems) => {
-        mutItems->Array.sort(compareBufferItem)
-        mutItems
-      }
-    | None => fetchState.buffer
+    // Single merge point: fold any onBlock items into the sorted base buffer.
+    buffer: switch blockItems {
+    | [] => base
+    | blockItems => base->mergeIntoBuffer(blockItems)
     },
     firstEventBlock: fetchState.firstEventBlock,
   }
@@ -740,12 +857,6 @@ let updateInternal = (
     ~blockNumber=updatedFetchState->bufferBlockNumber,
     ~chainId=fetchState.chainId,
   )
-  if indexingAddresses !== fetchState.indexingAddresses {
-    Prometheus.IndexingAddresses.set(
-      ~addressesCount=updatedFetchState->numAddresses,
-      ~chainId=fetchState.chainId,
-    )
-  }
 
   updatedFetchState
 }
@@ -976,22 +1087,22 @@ OptimizedPartitions.t => {
 
 let registerDynamicContracts = (
   fetchState: t,
+  ~indexingAddresses: IndexingAddresses.t,
   // These are raw items which might have dynamic contracts received from contractRegister call.
   // Might contain duplicates which we should filter out
   items: array<Internal.item>,
 ) => {
-  if fetchState.normalSelection.eventConfigs->Utils.Array.isEmpty {
+  if fetchState.normalSelection.onEventRegistrations->Utils.Array.isEmpty {
     // Can the normalSelection be empty?
     JsError.throwWithMessage(
       "Invalid configuration. No events to fetch for the dynamic contract registration.",
     )
   }
 
-  let indexingAddresses = fetchState.indexingAddresses
   let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
   let earliestRegisteringEventBlockNumber = ref(%raw(`Infinity`))
   // Addresses registered for contracts without matching events. These are not
-  // added to partitions, but they are tracked on fetchState.indexingAddresses
+  // added to partitions, but they are tracked on indexingAddresses
   // so that later conflicting registrations are detected, and are persisted
   // to envio_addresses so they can be picked up on restart with updated config.
   let noEventsAddresses: dict<indexingAddress> = Dict.make()
@@ -1017,15 +1128,13 @@ let registerDynamicContracts = (
             address: dc.address,
             contractName: dc.contractName,
             registrationBlock: dc.registrationBlock,
-            effectiveStartBlock: deriveEffectiveStartBlock(
+            effectiveStartBlock: IndexingAddresses.deriveEffectiveStartBlock(
               ~registrationBlock=dc.registrationBlock,
               ~contractStartBlock,
             ),
           }
           // Prevent registering already indexing contracts
-          switch indexingAddresses->Utils.Dict.dangerouslyGetNonOption(
-            dc.address->Address.toString,
-          ) {
+          switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
           | Some(existingContract) =>
             // FIXME: Instead of filtering out duplicates,
             // we should check the block number first.
@@ -1080,7 +1189,7 @@ let registerDynamicContracts = (
             address: dc.address,
             contractName: dc.contractName,
             registrationBlock: dc.registrationBlock,
-            effectiveStartBlock: deriveEffectiveStartBlock(
+            effectiveStartBlock: IndexingAddresses.deriveEffectiveStartBlock(
               ~registrationBlock=dc.registrationBlock,
               ~contractStartBlock=None,
             ),
@@ -1088,9 +1197,7 @@ let registerDynamicContracts = (
           // Prevent duplicate logging/persistence when the same address is
           // already tracked on fetchState, either from the db on startup or
           // from an earlier registration in this batch.
-          switch indexingAddresses->Utils.Dict.dangerouslyGetNonOption(
-            dc.address->Address.toString,
-          ) {
+          switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
           | Some(existingContract) =>
             if existingContract.contractName != dc.contractName {
               fetchState->warnDifferentContractType(~existingContract, ~dc=dcAsIndexingAddress)
@@ -1144,12 +1251,10 @@ let registerDynamicContracts = (
     // Only dcs for contracts without events. Track them on
     // indexingAddresses so subsequent registrations see them, but don't touch
     // partitions since there's nothing to fetch for them.
-    let newIndexingContracts = indexingAddresses->Utils.Dict.shallowCopy
-    let _ = Utils.Dict.mergeInPlace(newIndexingContracts, noEventsAddresses)
-    fetchState->updateInternal(~indexingAddresses=newIndexingContracts)
+    indexingAddresses->IndexingAddresses.register(noEventsAddresses)
+    fetchState
   | (_, _) => {
       let newPartitions = []
-      let newIndexingAddresses = indexingAddresses->Utils.Dict.shallowCopy
       let dynamicContractsRef = ref(fetchState.optimizedPartitions.dynamicContracts)
       let mutExistingPartitions = fetchState.optimizedPartitions.entities->Dict.valuesToArray
 
@@ -1231,10 +1336,10 @@ let registerDynamicContracts = (
         }
 
         let registeringContracts = registeringContractsByContract->Dict.getUnsafe(contractName)
-        let _ = Utils.Dict.mergeInPlace(newIndexingAddresses, registeringContracts)
+        indexingAddresses->IndexingAddresses.register(registeringContracts)
       }
       // Include no-events dcs so later batches detect conflicts against them.
-      let _ = Utils.Dict.mergeInPlace(newIndexingAddresses, noEventsAddresses)
+      indexingAddresses->IndexingAddresses.register(noEventsAddresses)
 
       let optimizedPartitions = createPartitionsFromIndexingAddresses(
         ~registeringContractsByContract,
@@ -1247,11 +1352,31 @@ let registerDynamicContracts = (
         ~progressBlockNumber=0,
       )
 
-      fetchState->updateInternal(~optimizedPartitions, ~indexingAddresses=newIndexingAddresses)
+      fetchState->updateInternal(~optimizedPartitions)
     }
   }
 }
 
+// Drop events an address-param filter rejects. A merged partition may
+// over-fetch a wildcard event whose indexed address param references an
+// address registered after the log's block; `clientAddressFilter` is the
+// param-level analogue of EventRouter's srcAddress effectiveStartBlock check.
+let filterByClientAddress = (
+  items: array<Internal.item>,
+  ~indexingAddresses: IndexingAddresses.t,
+): array<Internal.item> =>
+  items->Array.filter(item =>
+    switch item {
+    | Internal.Event({payload, blockNumber}) as item =>
+      switch (item->Internal.castUnsafeEventItem).onEventRegistration.clientAddressFilter {
+      | Some(filter) =>
+        filter(payload, blockNumber, indexingAddresses->IndexingAddresses.rawForFilter)
+      | None => true
+      }
+    | _ => true
+    }
+  )
+
 /*
 Updates fetchState with a response for a given query.
 Returns Error if the partition with given query cannot be found (unexpected)
@@ -1264,20 +1389,6 @@ let handleQueryResult = (
   ~latestFetchedBlock: blockNumberAndTimestamp,
   ~newItems,
 ): t => {
-  // Drop events an address-param filter rejects. A merged partition may
-  // over-fetch a wildcard event whose indexed address param references an
-  // address registered after the log's block; `clientAddressFilter` is the
-  // param-level analogue of EventRouter's srcAddress effectiveStartBlock check.
-  let newItems = newItems->Array.filter(item =>
-    switch item {
-    | Internal.Event({eventConfig, payload, blockNumber}) =>
-      switch eventConfig.clientAddressFilter {
-      | Some(filter) => filter(payload, blockNumber, fetchState.indexingAddresses)
-      | None => true
-      }
-    | _ => true
-    }
-  )
   fetchState->updateInternal(
     ~optimizedPartitions=fetchState.optimizedPartitions->OptimizedPartitions.handleQueryResponse(
       ~query,
@@ -1285,10 +1396,14 @@ let handleQueryResult = (
       ~itemsCount=newItems->Array.length,
       ~latestFetchedBlock,
     ),
+    // Merge the response into the sorted buffer, dropping duplicates an
+    // overlapping query may re-deliver (e.g. an over-fetched log matched by two
+    // partitions). Absorbs sorting too, so updateInternal doesn't re-sort.
+    ~mutItemsSorted=true,
     ~mutItems=?{
       switch newItems {
       | [] => None
-      | _ => Some(fetchState.buffer->Array.concat(newItems))
+      | _ => Some(fetchState.buffer->mergeIntoBuffer(newItems))
       }
     },
   )
@@ -1308,7 +1423,8 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
       fromBlock: q.fromBlock,
       toBlock: q.toBlock,
       isChunk: q.isChunk,
-      estResponseSize: q.estResponseSize,
+      itemsTarget: q.itemsTarget,
+      itemsEst: q.itemsEst,
       fetchedBlock: None,
     }
 
@@ -1329,163 +1445,191 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
   }
 }
 
-@inline
-let pushQueriesForRange = (
+// Most parallel in-flight chunk queries a single partition may have at once.
+let maxPendingChunksPerPartition = 12
+
+// Generates candidate queries for a gap range (a hole left between
+// completed/pending chunks, e.g. from an out-of-order partial response). Gaps
+// carry the range's low fromBlock, so the acceptance pass takes them before
+// forward progress. Priced by the partition's trusted density when it has one;
+// otherwise by the "available density" — the partition's equal-divide budget
+// spread over its remaining range this tick — so a small gap reserves
+// proportionally little instead of a noisy one-sample estimate. Chunks only on
+// a trusted POSITIVE density. Returns the created queries' total itemsEst.
+let pushGapFillQueries = (
   queries: array<query>,
   ~partitionId: string,
   ~rangeFromBlock: int,
   ~rangeEndBlock: option<int>,
-  ~maxQueryBlockNumber: int,
+  ~knownHeight: int,
+  ~chainTargetBlock: int,
   ~maybeChunkRange: option<int>,
+  ~maxChunks: int,
   ~partition: partition,
+  ~partitionBudget: float,
+  ~chunkItemsMultiplier: float,
   ~selection: selection,
   ~addressesByContractName: dict<array<Address.t>>,
-  ~indexingAddresses: dict<indexingAddress>,
 ) => {
-  if rangeFromBlock <= maxQueryBlockNumber {
+  let cost = ref(0.)
+
+  // Gaps past the chain's target block wait: they regenerate from the
+  // pending-walk each tick and fill once the target reaches them.
+  if rangeFromBlock <= Pervasives.min(knownHeight, chainTargetBlock) && maxChunks > 0 {
     switch rangeEndBlock {
     | Some(endBlock) if rangeFromBlock > endBlock => ()
     | _ =>
-      switch maybeChunkRange {
-      | None =>
+      let trustedDensity = partition->getTrustedDensity
+      let maxBlock = switch rangeEndBlock {
+      | Some(eb) => eb
+      | None => chainTargetBlock
+      }
+      let pushSingleQuery = (~density, ~isChunk) => {
+        let itemsTarget = densityItemsTarget(
+          ~density=density *. chunkItemsMultiplier,
+          ~fromBlock=rangeFromBlock,
+          ~toBlock=rangeEndBlock,
+          ~chainTargetBlock,
+        )
+        let itemsEst = densityItemsTarget(
+          ~density,
+          ~fromBlock=rangeFromBlock,
+          ~toBlock=rangeEndBlock,
+          ~chainTargetBlock,
+        )
         queries->Array.push({
           partitionId,
           fromBlock: rangeFromBlock,
           toBlock: rangeEndBlock,
           selection,
-          isChunk: false,
-          estResponseSize: calculateEstResponseSize(
-            partition,
-            ~fromBlock=rangeFromBlock,
-            ~toBlock=rangeEndBlock,
-            ~maxQueryBlockNumber,
-          ),
-          chainId: 0,
-          progress: 0.,
+          isChunk,
+          itemsTarget,
+          itemsEst,
           addressesByContractName,
-          indexingAddresses,
         })
-      | Some(chunkRange) =>
-        let maxBlock = switch rangeEndBlock {
-        | Some(eb) => eb
-        | None => maxQueryBlockNumber
-        }
+        cost := cost.contents +. itemsEst->Int.toFloat
+      }
+      switch (trustedDensity, maybeChunkRange) {
+      | (Some(density), Some(chunkRange)) if density > 0. =>
         let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
-        // Probe with two smaller chunks first so their responses come back
-        // quickly and refresh the chunking heuristic, then three full-size chunks.
-        let probeSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 0.9)
-        let getChunkSize = chunkIdx => chunkIdx < 2 ? probeSize : chunkSize
-        if rangeFromBlock + getChunkSize(0) + getChunkSize(1) - 1 <= maxBlock {
+        if rangeFromBlock + chunkSize * 2 - 1 <= maxBlock {
           let chunkFromBlock = ref(rangeFromBlock)
           let chunkIdx = ref(0)
           while (
-            chunkIdx.contents < 5 &&
-              chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1 <= maxBlock
+            chunkIdx.contents < maxChunks && chunkFromBlock.contents + chunkSize - 1 <= maxBlock
           ) {
-            let chunkToBlock = chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1
+            let chunkToBlock = chunkFromBlock.contents + chunkSize - 1
+            let itemsTarget = densityItemsTarget(
+              ~density=density *. chunkItemsMultiplier,
+              ~fromBlock=chunkFromBlock.contents,
+              ~toBlock=Some(chunkToBlock),
+              ~chainTargetBlock,
+            )
+            let itemsEst = densityItemsTarget(
+              ~density,
+              ~fromBlock=chunkFromBlock.contents,
+              ~toBlock=Some(chunkToBlock),
+              ~chainTargetBlock,
+            )
             queries->Array.push({
               partitionId,
               fromBlock: chunkFromBlock.contents,
               toBlock: Some(chunkToBlock),
               isChunk: true,
               selection,
-              estResponseSize: calculateEstResponseSize(
-                partition,
-                ~fromBlock=chunkFromBlock.contents,
-                ~toBlock=Some(chunkToBlock),
-                ~maxQueryBlockNumber,
-              ),
-              chainId: 0,
-              progress: 0.,
+              itemsTarget,
+              itemsEst,
               addressesByContractName,
-              indexingAddresses,
             })
+            cost := cost.contents +. itemsEst->Int.toFloat
             chunkFromBlock := chunkToBlock + 1
             chunkIdx := chunkIdx.contents + 1
           }
         } else {
           // Not enough room for 2 chunks, fall back to a single query
-          queries->Array.push({
-            partitionId,
-            fromBlock: rangeFromBlock,
-            toBlock: rangeEndBlock,
-            selection,
-            isChunk: rangeEndBlock !== None,
-            estResponseSize: calculateEstResponseSize(
-              partition,
-              ~fromBlock=rangeFromBlock,
-              ~toBlock=rangeEndBlock,
-              ~maxQueryBlockNumber,
-            ),
-            chainId: 0,
-            progress: 0.,
-            addressesByContractName,
-            indexingAddresses,
-          })
+          pushSingleQuery(~density, ~isChunk=rangeEndBlock !== None)
         }
+      | (Some(density), _) => pushSingleQuery(~density, ~isChunk=false)
+      | (None, _) =>
+        let remainingRange = Pervasives.max(1, chainTargetBlock - rangeFromBlock + 1)
+        pushSingleQuery(~density=partitionBudget /. remainingRange->Int.toFloat, ~isChunk=false)
       }
     }
   }
+  cost.contents
 }
 
-// Most parallel in-flight chunk queries a single partition may have at once.
-let maxPendingChunksPerPartition = 10
+// Per-partition state carried from the gap-fill/cursor walk to candidate
+// generation.
+type partitionFillState = {
+  partitionId: string,
+  p: partition,
+  cursor: int,
+  // Chunks already generated for this partition during gap-fill — used with
+  // pendingCount against maxPendingChunksPerPartition.
+  chunksUsedThisCall: int,
+  // Existing mutPendingQueries count before this call — fixed for the call.
+  pendingCount: int,
+  queryEndBlock: option<int>,
+  maybeChunkRange: option<int>,
+}
 
+// Candidate queries are sized against chainTargetBlock, the soft querying
+// horizon the owning chain wants to reach this tick — derived by ChainState
+// from its share of the indexer-wide buffer budget and its chain-level event
+// density. chainTargetBlock is never used as a hard query end, only to (a)
+// select which partitions are "in range" this tick and (b) size an open-ended
+// query with no other ceiling: the true hard bounds stay
+// endBlock/mergeBlock/the lagged head.
+//
+// The tick's budget is chainTargetItems minus what's already in flight. Every
+// candidate query — gap-fill holes, plus each in-range partition's chunks or
+// open-ended probe toward the target — is generated with no budget check, then
+// the candidates are sorted by fromBlock and accepted in that order while the
+// budget stays positive. The query that tips it negative is still accepted (a
+// single overshoot); everything after it waits for a tick with more budget.
+// Sorting by fromBlock spends the budget on the earliest blocks across all
+// partitions first, so no partition is starved by generation order and the
+// frontier advances evenly. In-flight reservations release as responses land,
+// so acceptance redistributes across ticks.
+//
+// A partition with a trusted positive density (two or more responses — see
+// getMinHistoryRange) generates real, density-sized chunks toward the target.
+// Any other partition (no signal, one noisy sample, or a density-0 estimate)
+// generates one open-ended probe sized to the events its range to the target is
+// expected to hold — rangeTargetDensity × (chainTargetBlock − fromBlock + 1) /
+// inRangeCount — so several unknown-density partitions probe in parallel within
+// one budget, each scaled by how much of the range it still has to cover.
 let getNextQuery = (
-  {
-    buffer,
-    optimizedPartitions,
-    indexingAddresses,
-    blockLag,
-    latestOnBlockBlockNumber,
-    knownHeight,
-  } as fetchState: t,
-  ~budget,
-  ~chainPendingBudget,
+  {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
+  ~chainTargetBlock: int,
+  ~chainTargetItems: float,
+  ~chunkItemsMultiplier: float=1.,
 ) => {
   let headBlockNumber = knownHeight - blockLag
   if headBlockNumber <= 0 {
     WaitingForNewBlock
-  } else if budget <= 0 {
-    // No room left in the shared buffer pool for this chain; wait for processing
-    // to drain before fetching more.
-    NothingToQuery
   } else {
     let isOnBlockBehindTheHead = latestOnBlockBlockNumber < headBlockNumber
     let shouldWaitForNewBlock = ref(
-      switch fetchState.endBlock {
+      switch endBlock {
       | Some(endBlock) => headBlockNumber < endBlock
       | None => true
       } &&
       !isOnBlockBehindTheHead,
     )
 
-    // Limit how far ahead we fetch to budget items (plus what's already in
-    // flight) so processing always has buffer without ballooning memory. A
-    // partition that fetched further is skipped until the buffer drains.
-    let maxQueryBlockNumber = {
-      switch buffer->Array.get(budget + chainPendingBudget->Float.toInt - 1) {
-      | Some(item) =>
-        // Just in case check that we don't query beyond the current block
-        Pervasives.min(item->Internal.getItemBlockNumber, knownHeight)
-      | None => knownHeight
-      }
-    }
-
-    let queries = []
-
     let partitionsCount = optimizedPartitions.idsInAscOrder->Array.length
-    let idxRef = ref(0)
-    while idxRef.contents < partitionsCount {
-      let idx = idxRef.contents
+
+    // Every partition is visited once here regardless of whether it gets a
+    // query pushed below — waiting-for-new-block bookkeeping shouldn't depend
+    // on this tick's budget.
+    for idx in 0 to partitionsCount - 1 {
       let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
       let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
-
-      let pendingCount = p.mutPendingQueries->Array.length
-      let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber
-      let hasPendingQueries = pendingCount > 0
-
-      if hasPendingQueries || isBehindTheHead {
+      if (
+        p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock.blockNumber < headBlockNumber
+      ) {
         // Even if there are some partitions waiting for the new block
         // We still want to wait for all partitions reaching the head
         // because they might update knownHeight in their response
@@ -1493,51 +1637,99 @@ let getNextQuery = (
         // and we don't want to poll the head for a few small partitions
         shouldWaitForNewBlock := false
       }
+    }
 
-      let partitionQueriesStart = queries->Array.length
-
-      // Compute queryEndBlock for this partition
-      let queryEndBlock = Utils.Math.minOptInt(fetchState.endBlock, p.mergeBlock)
-      let queryEndBlock = switch blockLag {
+    // One bucket per partition, in idsInAscOrder order — gap-fill and the
+    // budget pass both push into a partition's own bucket, so flattening at
+    // the end (see below) reproduces idsInAscOrder without a sort.
+    let queriesByPartitionIndex: array<
+      array<query>,
+    > = Array.fromInitializer(~length=partitionsCount, _ => [])
+
+    // Compute queryEndBlock for this partition
+    let computeQueryEndBlock = (p: partition) => {
+      let queryEndBlock = Utils.Math.minOptInt(endBlock, p.mergeBlock)
+      switch blockLag {
       | 0 => queryEndBlock
       | _ =>
         // Force head block as an endBlock when blockLag is set
         // because otherwise HyperSync might return bigger range
         Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock)
       }
-      // Enforce the response range up until target block
-      // Otherwise for indexers with 100+ partitions
-      // we might blow up the buffer size to more than 600k events
-      // simply because of HyperSync returning extra blocks
-      let queryEndBlock = switch (queryEndBlock, maxQueryBlockNumber < knownHeight) {
-      | (Some(endBlock), true) => Some(Pervasives.min(maxQueryBlockNumber, endBlock))
-      | (None, true) => Some(maxQueryBlockNumber)
-      | (_, false) => queryEndBlock
-      }
+    }
 
+    // In-flight itemsEst summed over queries still being fetched
+    // (fetchedBlock === None). A query whose response already landed has had its
+    // reservation released by ChainState even while it lingers in
+    // mutPendingQueries behind an unfilled gap, so counting it here would
+    // understate the budget. Sizes fresh forward work below.
+    let chainReserved = ref(0.)
+
+    // (fromBlock, itemsEst) of each still-in-flight query. The acceptance pass
+    // merges these into the candidate stream and draws them down in fromBlock
+    // order, so a gap-fill sitting before an in-flight query claims budget ahead
+    // of it and the buffer unblocks without waiting for that query to return.
+    let reservations = []
+
+    // Position of each partition in idsInAscOrder, so an accepted query routes
+    // back to its bucket and the output stays in idsInAscOrder. Filled in the
+    // Phase A sweep below.
+    let partitionIndexById = Dict.make()
+
+    // Every candidate query for this tick — gap-fill holes (Phase A) plus each
+    // in-range partition's chunks/probe toward the target (Phase B) — generated
+    // with no budget check, then merged with the in-flight reservations, sorted
+    // by fromBlock, and accepted while the budget lasts (acceptance pass).
+    // Selecting by fromBlock spends the budget on the earliest blocks across all
+    // partitions first, so no partition is starved by iteration order and the
+    // frontier advances evenly.
+    let candidates = []
+
+    // Phase A: gap-fill. Walk each partition's pending queries once, generating
+    // a candidate for any hole (e.g. from an out-of-order partial chunk
+    // response). This also determines each partition's post-gap cursor and
+    // whether it's blocked on an unresolved single-shot query.
+    let fillStates = []
+    for idx in 0 to partitionsCount - 1 {
+      let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
+      let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
+      partitionIndexById->Dict.set(partitionId, idx)
+      let pendingCount = p.mutPendingQueries->Array.length
+      for pqIdx in 0 to pendingCount - 1 {
+        let pq = p.mutPendingQueries->Array.getUnsafe(pqIdx)
+        if pq.fetchedBlock === None {
+          chainReserved := chainReserved.contents +. pq.itemsEst->Int.toFloat
+          reservations->Array.push((pq.fromBlock, pq.itemsEst))->ignore
+        }
+      }
+      let queryEndBlock = computeQueryEndBlock(p)
       let maybeChunkRange = getMinHistoryRange(p)
 
-      // Walk pending queries to find open ranges and create queries for each
       let cursor = ref(p.latestFetchedBlock.blockNumber + 1)
       let canContinue = ref(true)
+      let chunksUsedThisCall = ref(0)
       let pqIdx = ref(0)
-      while pqIdx.contents < p.mutPendingQueries->Array.length && canContinue.contents {
+      while pqIdx.contents < pendingCount && canContinue.contents {
         let pq = p.mutPendingQueries->Array.getUnsafe(pqIdx.contents)
 
-        // Gap before this pending query → create queries for the gap range
         if pq.fromBlock > cursor.contents {
-          pushQueriesForRange(
-            queries,
+          let beforeLen = candidates->Array.length
+          pushGapFillQueries(
+            candidates,
             ~partitionId,
             ~rangeFromBlock=cursor.contents,
             ~rangeEndBlock=Utils.Math.minOptInt(Some(pq.fromBlock - 1), queryEndBlock),
-            ~maxQueryBlockNumber,
+            ~knownHeight,
+            ~chainTargetBlock,
             ~maybeChunkRange,
+            ~maxChunks=maxPendingChunksPerPartition - pendingCount - chunksUsedThisCall.contents,
             ~partition=p,
+            ~partitionBudget=chainTargetItems /. partitionsCount->Int.toFloat,
+            ~chunkItemsMultiplier,
             ~selection=p.selection,
             ~addressesByContractName=p.addressesByContractName,
-            ~indexingAddresses,
-          )
+          )->ignore
+          chunksUsedThisCall := chunksUsedThisCall.contents + (candidates->Array.length - beforeLen)
         }
         switch pq {
         | {isChunk: true, toBlock: Some(toBlock), fetchedBlock: Some({blockNumber})}
@@ -1549,40 +1741,195 @@ let getNextQuery = (
         pqIdx := pqIdx.contents + 1
       }
 
-      // Tail range after all pending queries
       if canContinue.contents {
-        pushQueriesForRange(
-          queries,
-          ~partitionId,
-          ~rangeFromBlock=cursor.contents,
-          ~rangeEndBlock=queryEndBlock,
-          ~maxQueryBlockNumber,
-          ~maybeChunkRange,
-          ~partition=p,
-          ~selection=p.selection,
-          ~addressesByContractName=p.addressesByContractName,
-          ~indexingAddresses,
-        )
+        fillStates
+        ->Array.push({
+          partitionId,
+          p,
+          cursor: cursor.contents,
+          chunksUsedThisCall: chunksUsedThisCall.contents,
+          pendingCount,
+          queryEndBlock,
+          maybeChunkRange,
+        })
+        ->ignore
       }
+    }
 
-      // Cap parallel in-flight chunks per partition so a single partition can't
-      // monopolize fetching. Keep the earliest new chunks; the furthest-ahead
-      // ones wait for the next round once these resolve.
-      let maxNewChunks = Pervasives.max(0, maxPendingChunksPerPartition - pendingCount)
-      let generatedCount = queries->Array.length - partitionQueriesStart
-      if generatedCount > maxNewChunks {
-        queries
-        ->Array.splice(
-          ~start=partitionQueriesStart + maxNewChunks,
-          ~remove=generatedCount - maxNewChunks,
-          ~insert=[],
-        )
+    // Budget for fresh forward work: chainTargetItems minus what's still in
+    // flight. Sizes probes and bounds chunk generation below; the acceptance
+    // pass does the final budgeting against the full chainTargetItems.
+    let freshBudget = Pervasives.max(0., chainTargetItems -. chainReserved.contents)
+
+    let isInRange = (fs: partitionFillState) =>
+      fs.cursor <= chainTargetBlock &&
+      switch fs.queryEndBlock {
+      | Some(eb) => fs.cursor <= eb
+      | None => true
+      } &&
+      fs.pendingCount + fs.chunksUsedThisCall < maxPendingChunksPerPartition
+
+    let inRangeStates = fillStates->Array.filter(isInRange)
+    let inRangeCount = inRangeStates->Array.length
+    // Even share of the fresh budget across the partitions actually fetching
+    // this tick (not every partition — so budget isn't stranded on ones below
+    // the head, waiting, or already done). The fallback when there's no range to
+    // the target.
+    let probeShare = inRangeCount == 0 ? 0. : freshBudget /. inRangeCount->Int.toFloat
+    // Items/block the budget implies over the range those partitions cover this
+    // tick — from the furthest-behind in-range cursor to the target. A probe
+    // covering less of that range (its partition sits further ahead) gets
+    // proportionally fewer items; one starting at the frontier gets the full
+    // even share.
+    let frontierCursor =
+      inRangeStates->Array.reduce(chainTargetBlock, (min, fs) => fs.cursor < min ? fs.cursor : min)
+    let rangeToTarget = chainTargetBlock - frontierCursor + 1
+    let rangeTargetDensity =
+      inRangeCount > 0 && rangeToTarget > 0 ? freshBudget /. rangeToTarget->Int.toFloat : 0.
+
+    // Phase B: generate each in-range partition's candidates — strict chunks
+    // toward the target sized by real density (up to the pending-chunk cap), or,
+    // for a partition without a trusted positive density, a single open-ended
+    // probe at its even share of the budget. No budget check here; the
+    // acceptance pass below decides which candidates make the cut.
+    //
+    // Chunks require a POSITIVE trusted density: density 0 prices every chunk at
+    // ~nothing, so an open-ended probe (full server scan range in one response)
+    // beats a pipeline of hard-bounded chunks that crawl 1.8× per two responses.
+    inRangeStates->Array.forEach(fs => {
+      let p = fs.p
+      let maxBlock = switch fs.queryEndBlock {
+      | Some(eb) => eb
+      | None => chainTargetBlock
+      }
+      switch (fs.maybeChunkRange, p->getTrustedDensity) {
+      | (Some(minHistoryRange), Some(density)) if density > 0. =>
+        let chunkSize = Js.Math.ceil_int(minHistoryRange->Int.toFloat *. 1.8)
+        let maxChunksRemaining =
+          maxPendingChunksPerPartition - fs.pendingCount - fs.chunksUsedThisCall
+        // No chunk starts past chainTargetBlock; an emitted chunk still keeps
+        // its full span (chunkToBlock may exceed the target — only
+        // endBlock/mergeBlock are hard bounds).
+        let chunkStartCeiling = Pervasives.min(maxBlock, chainTargetBlock)
+        let created = ref(0)
+        let chunkFromBlock = ref(fs.cursor)
+        // Stop once this partition alone has generated more than the whole fresh
+        // budget: the acceptance pass can hand a single partition at most the
+        // budget plus one overshoot query, so further chunks could never be
+        // accepted. Bounds generation (and the candidate sort) when the budget
+        // is small relative to the pending-chunk cap.
+        let generatedItems = ref(0.)
+        while (
+          created.contents < maxChunksRemaining &&
+          chunkFromBlock.contents <= chunkStartCeiling &&
+          generatedItems.contents <= freshBudget
+        ) {
+          let chunkToBlock = Pervasives.min(chunkFromBlock.contents + chunkSize - 1, maxBlock)
+          let rawEst = density *. (chunkToBlock - chunkFromBlock.contents + 1)->Int.toFloat
+          let itemsEst = Pervasives.max(1, rawEst->Math.ceil->Float.toInt)
+          let itemsTarget = Pervasives.max(
+            1,
+            (rawEst *. chunkItemsMultiplier)->Math.ceil->Float.toInt,
+          )
+          candidates
+          ->Array.push({
+            partitionId: fs.partitionId,
+            fromBlock: chunkFromBlock.contents,
+            toBlock: Some(chunkToBlock),
+            isChunk: true,
+            selection: p.selection,
+            itemsTarget,
+            itemsEst,
+            addressesByContractName: p.addressesByContractName,
+          })
+          ->ignore
+          generatedItems := generatedItems.contents +. itemsEst->Int.toFloat
+          chunkFromBlock := chunkToBlock + 1
+          created := created.contents + 1
+        }
+      | _ =>
+        // Size the probe by the events its range to the target is expected to
+        // hold — rangeTargetDensity × (chainTargetBlock − fromBlock + 1), split
+        // across the partitions fetching this tick. With no range to the target
+        // fall back to an even share of the fresh budget, so cold chains and
+        // caught-up partitions still probe.
+        let itemsTarget = if rangeToTarget > 0 {
+          Pervasives.max(
+            1,
+            Math.round(
+              rangeTargetDensity *.
+              (chainTargetBlock - fs.cursor + 1)->Int.toFloat /.
+              inRangeCount->Int.toFloat,
+            )->Float.toInt,
+          )
+        } else {
+          Pervasives.max(1, Math.round(probeShare)->Float.toInt)
+        }
+        candidates
+        ->Array.push({
+          partitionId: fs.partitionId,
+          fromBlock: fs.cursor,
+          toBlock: fs.queryEndBlock,
+          isChunk: false,
+          selection: p.selection,
+          itemsTarget,
+          itemsEst: itemsTarget,
+          addressesByContractName: p.addressesByContractName,
+        })
         ->ignore
       }
+    })
 
-      idxRef := idxRef.contents + 1
+    // Acceptance: merge fresh candidates (Some) with the in-flight reservations
+    // (None) and walk them in fromBlock order, starting from the full
+    // chainTargetItems. A reservation just draws down the budget — its query is
+    // already sent — while a candidate draws down the budget and is emitted.
+    // Because a gap-fill's fromBlock precedes the in-flight query it unblocks,
+    // it claims budget ahead of that reservation, so the buffer never deadlocks
+    // waiting on a hole it can't fund. The candidate that tips the budget
+    // negative is still emitted (a single overshoot); everything after it waits
+    // for a tick with more budget. Accepted queries route back to their
+    // partition bucket, so the output stays in idsInAscOrder with each
+    // partition's queries in fromBlock order.
+    let acceptanceStream = []
+    candidates->Array.forEach(query =>
+      acceptanceStream->Array.push((query.fromBlock, query.itemsEst, Some(query)))->ignore
+    )
+    reservations->Array.forEach(((fromBlock, itemsEst)) =>
+      acceptanceStream->Array.push((fromBlock, itemsEst, None))->ignore
+    )
+    // Sort by fromBlock; on a tie charge the in-flight reservation (None) before
+    // a fresh candidate (Some), so a same-block candidate can't overshoot the
+    // target buffer. Only a strictly-earlier candidate — a gap-fill, whose
+    // fromBlock precedes the query it unblocks — borrows ahead of a reservation.
+    acceptanceStream->Array.sort(((aFrom, _, aQuery), (bFrom, _, bQuery)) =>
+      if aFrom !== bFrom {
+        Int.compare(aFrom, bFrom)
+      } else {
+        switch (aQuery, bQuery) {
+        | (None, Some(_)) => Ordering.less
+        | (Some(_), None) => Ordering.greater
+        | (None, None) | (Some(_), Some(_)) => Ordering.equal
+        }
+      }
+    )
+    let streamCount = acceptanceStream->Array.length
+    let remainingBudget = ref(chainTargetItems)
+    let acceptIdx = ref(0)
+    while remainingBudget.contents > 0. && acceptIdx.contents < streamCount {
+      let (_, itemsEst, maybeQuery) = acceptanceStream->Array.getUnsafe(acceptIdx.contents)
+      switch maybeQuery {
+      | Some(query) =>
+        let partitionIdx = partitionIndexById->Dict.getUnsafe(query.partitionId)
+        queriesByPartitionIndex->Array.getUnsafe(partitionIdx)->Array.push(query)->ignore
+      | None => ()
+      }
+      remainingBudget := remainingBudget.contents -. itemsEst->Int.toFloat
+      acceptIdx := acceptIdx.contents + 1
     }
 
+    let queries = queriesByPartitionIndex->Array.flat
+
     if queries->Utils.Array.isEmpty {
       if shouldWaitForNewBlock.contents {
         WaitingForNewBlock
@@ -1631,14 +1978,15 @@ Instantiates a fetch state with partitions for initial addresses
 let make = (
   ~startBlock,
   ~endBlock,
-  ~eventConfigs: array<Internal.eventConfig>,
+  ~onEventRegistrations: array<Internal.onEventRegistration>,
+  ~contractConfigs: dict<IndexingAddresses.contractConfig>,
   ~addresses: array<Internal.indexingAddress>,
   ~maxAddrInPartition,
   ~chainId,
   ~maxOnBlockBufferSize,
   ~knownHeight,
   ~progressBlockNumber=startBlock - 1,
-  ~onBlockConfigs=[],
+  ~onBlockRegistrations=[],
   ~blockLag=0,
   ~firstEventBlock=None,
 ): t => {
@@ -1648,38 +1996,15 @@ let make = (
   }
 
   let notDependingOnAddresses = []
-  let normalEventConfigs = []
+  let normalRegistrations = []
   let contractNamesWithNormalEvents = Utils.Set.make()
-  let indexingAddresses = Dict.make()
-  let contractConfigs: dict<contractConfig> = Dict.make()
-
-  eventConfigs->Array.forEach(ec => {
-    switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(ec.contractName) {
-    | Some({startBlock}) =>
-      contractConfigs->Dict.set(
-        ec.contractName,
-        {
-          startBlock: switch (startBlock, ec.startBlock) {
-          | (Some(a), Some(b)) => Some(Pervasives.min(a, b))
-          | (Some(_) as s, None) | (None, Some(_) as s) => s
-          | (None, None) => None
-          },
-        },
-      )
-    | None =>
-      contractConfigs->Dict.set(
-        ec.contractName,
-        {
-          startBlock: ec.startBlock,
-        },
-      )
-    }
 
-    if ec.dependsOnAddresses {
-      normalEventConfigs->Array.push(ec)
-      contractNamesWithNormalEvents->Utils.Set.add(ec.contractName)->ignore
+  onEventRegistrations->Array.forEach(reg => {
+    if reg.dependsOnAddresses {
+      normalRegistrations->Array.push(reg)
+      contractNamesWithNormalEvents->Utils.Set.add(reg.eventConfig.contractName)->ignore
     } else {
-      notDependingOnAddresses->Array.push(ec)
+      notDependingOnAddresses->Array.push(reg)
     }
   })
 
@@ -1691,7 +2016,7 @@ let make = (
       latestFetchedBlock,
       selection: {
         dependsOnAddresses: false,
-        eventConfigs: notDependingOnAddresses,
+        onEventRegistrations: notDependingOnAddresses,
       },
       addressesByContractName: Dict.make(),
       mergeBlock: None,
@@ -1706,7 +2031,7 @@ let make = (
 
   let normalSelection = {
     dependsOnAddresses: true,
-    eventConfigs: normalEventConfigs,
+    onEventRegistrations: normalRegistrations,
   }
 
   let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
@@ -1714,32 +2039,16 @@ let make = (
 
   addresses->Array.forEach(contract => {
     let contractName = contract.contractName
-    let contractStartBlock = switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(
-      contractName,
-    ) {
-    | Some({startBlock}) => startBlock
-    | None => None
-    }
-    let ia: indexingAddress = {
-      address: contract.address,
-      contractName: contract.contractName,
-      registrationBlock: contract.registrationBlock,
-      effectiveStartBlock: deriveEffectiveStartBlock(
-        ~registrationBlock=contract.registrationBlock,
-        ~contractStartBlock,
-      ),
-    }
-    // Track the address on fetchState regardless of whether it currently has
-    // matching events. This way, if the config is updated later to add events
-    // for this contract, the address is already known.
-    indexingAddresses->Dict.set(contract.address->Address.toString, ia)
 
     // Only addresses whose contract has events that depend on addresses get
     // registered for active fetching via partitions.
     if contractNamesWithNormalEvents->Utils.Set.has(contractName) {
-      let registeringContracts =
-        registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(contractName)
-      registeringContracts->Dict.set(contract.address->Address.toString, ia)
+      registeringContractsByContract
+      ->Utils.Dict.getOrInsertEmptyDict(contractName)
+      ->Dict.set(
+        contract.address->Address.toString,
+        IndexingAddresses.makeIndexingAddress(~contract, ~contractConfigs),
+      )
 
       // Detect dynamic contracts by registrationBlock
       if contract.registrationBlock !== -1 {
@@ -1758,12 +2067,15 @@ let make = (
     ~progressBlockNumber,
   )
 
-  if optimizedPartitions->OptimizedPartitions.count === 0 && onBlockConfigs->Utils.Array.isEmpty {
+  if (
+    optimizedPartitions->OptimizedPartitions.count === 0 &&
+      onBlockRegistrations->Utils.Array.isEmpty
+  ) {
     JsError.throwWithMessage(
       `Invalid configuration: Nothing to fetch on chain ${chainId->Int.toString}. ` ++
       `addresses=${addresses->Array.length->Int.toString}, ` ++
-      `eventConfigs=${eventConfigs->Array.length->Int.toString}, ` ++
-      `normalEventConfigs=${normalEventConfigs
+      `onEventRegistrations=${onEventRegistrations->Array.length->Int.toString}, ` ++
+      `normalRegistrations=${normalRegistrations
         ->Array.length
         ->Int.toString}. ` ++ `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`,
     )
@@ -1774,14 +2086,14 @@ let make = (
   // fetching, so without seeding the buffer here getNextQuery would return
   // NothingToQuery and the indexer would get stuck.
   let buffer = []
-  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockConfigs->Utils.Array.notEmpty {
+  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockRegistrations->Utils.Array.notEmpty {
     let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
     | None => knownHeight
     | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock.blockNumber
     }
     appendOnBlockItems(
       ~mutItems=buffer,
-      ~onBlockConfigs,
+      ~onBlockRegistrations,
       ~indexerStartBlock=startBlock,
       ~fromBlock=progressBlockNumber,
       ~maxBlockNumber,
@@ -1799,17 +2111,14 @@ let make = (
     endBlock,
     latestOnBlockBlockNumber,
     normalSelection,
-    indexingAddresses,
     blockLag,
-    onBlockConfigs,
+    onBlockRegistrations,
     maxOnBlockBufferSize,
     knownHeight,
     buffer,
     firstEventBlock,
   }
 
-  let numAddresses = indexingAddresses->Utils.Dict.size
-  Prometheus.IndexingAddresses.set(~addressesCount=numAddresses, ~chainId)
   Prometheus.IndexingPartitions.set(
     ~partitionsCount=optimizedPartitions->OptimizedPartitions.count,
     ~chainId,
@@ -1826,25 +2135,6 @@ let make = (
 
 let bufferSize = ({buffer}: t) => buffer->Array.length
 
-// Number of buffered items at or below the ready frontier (processable now,
-// i.e. not stuck behind a gap from a lagging partition or out-of-order chunk).
-// The buffer is kept sorted, so binary-search the frontier in O(log n).
-let bufferReadyCount = (fetchState: t) => {
-  let frontier = fetchState->bufferBlockNumber
-  let buffer = fetchState.buffer
-  let lo = ref(0)
-  let hi = ref(buffer->Array.length)
-  while lo.contents < hi.contents {
-    let mid = (lo.contents + hi.contents) / 2
-    if buffer->Array.getUnsafe(mid)->Internal.getItemBlockNumber <= frontier {
-      lo := mid + 1
-    } else {
-      hi := mid
-    }
-  }
-  lo.contents
-}
-
 let rollbackPendingQueries = (mutPendingQueries: array<pendingQuery>, ~targetBlockNumber) => {
   // - Remove queries where fromBlock > target
   // - Cap fetchedBlock at target where fetchedBlock > target
@@ -1876,18 +2166,11 @@ Always recreates optimized partitions to avoid duplicate addresses:
 - Non-wildcard with lfb <= target: keep, adjust pending queries and mergeBlock
 - Non-wildcard with lfb > target: delete, track addresses for recreation
 */
-let rollback = (fetchState: t, ~targetBlockNumber) => {
-  // Step 1: Build addressesToRemove and surviving indexingAddresses
-  let addressesToRemove = Utils.Set.make()
-  let indexingAddresses = Dict.make()
-
-  fetchState.indexingAddresses->Utils.Dict.forEachWithKey((indexingContract, address) => {
-    if indexingContract.registrationBlock > targetBlockNumber {
-      let _ = addressesToRemove->Utils.Set.add(address->Address.unsafeFromString)
-    } else {
-      indexingAddresses->Dict.set(address, indexingContract)
-    }
-  })
+let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetBlockNumber) => {
+  // Step 1: Prune addresses registered after the target block. The pruned index is
+  // then the source of truth for partition cleanup below — an address survives iff
+  // it's still in the index.
+  indexingAddresses->IndexingAddresses.rollbackInPlace(~targetBlockNumber)
 
   // Step 2: Categorize partitions
   let keptPartitions = []
@@ -1917,18 +2200,12 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
     | _ if p.latestFetchedBlock.blockNumber > targetBlockNumber =>
       p.addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
         addresses->Array.forEach(address => {
-          if (
-            !(addressesToRemove->Utils.Set.has(address)) &&
-            indexingAddresses
-            ->Utils.Dict.dangerouslyGetNonOption(address->Address.toString)
-            ->Option.isSome
-          ) {
+          switch indexingAddresses->IndexingAddresses.get(address->Address.toString) {
+          | Some(indexingContract) =>
             let registeringContracts =
               registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(contractName)
-            registeringContracts->Dict.set(
-              address->Address.toString,
-              indexingAddresses->Dict.getUnsafe(address->Address.toString),
-            )
+            registeringContracts->Dict.set(address->Address.toString, indexingContract)
+          | None => ()
           }
         })
       })
@@ -1941,11 +2218,13 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
         | other => other
         }
 
-        // Remove addresses that should be removed
+        // Drop addresses pruned from the index
         let rollbackedAddressesByContractName = Dict.make()
         addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
           let keptAddresses =
-            addresses->Array.filter(address => !(addressesToRemove->Utils.Set.has(address)))
+            addresses->Array.filter(address =>
+              indexingAddresses->IndexingAddresses.get(address->Address.toString)->Option.isSome
+            )
           if keptAddresses->Array.length > 0 {
             rollbackedAddressesByContractName->Dict.set(contractName, keptAddresses)
           }
@@ -1989,7 +2268,8 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
     ),
   }->updateInternal(
     ~optimizedPartitions,
-    ~indexingAddresses,
+    // Filtering the sorted buffer keeps it sorted and deduped.
+    ~mutItemsSorted=true,
     ~mutItems=fetchState.buffer->Array.filter(item =>
       switch item {
       | Event({blockNumber})
@@ -2084,17 +2364,6 @@ let getProgressPercentage = (fetchState: t) => {
   }
 }
 
-// Progress a specific block sits at along the chain, used to order queries from
-// different chains: a query starting further back (lower %) is fetched first.
-let getProgressPercentageAt = (fetchState: t, ~blockNumber) => {
-  switch fetchState.firstEventBlock {
-  | None => 0.
-  | Some(firstEventBlock) =>
-    let totalRange = fetchState.knownHeight - firstEventBlock
-    totalRange <= 0 ? 0. : (blockNumber - firstEventBlock)->Int.toFloat /. totalRange->Int.toFloat
-  }
-}
-
 let sortForBatch = {
   let hasFullBatch = ({buffer} as fetchState: t, ~batchSizeTarget) => {
     switch buffer->Array.get(batchSizeTarget - 1) {