envio 3.15.0 → 3.16.0-subgraph

package/src/FetchState.res

@@ -864,6 +864,22 @@ module OptimizedPartitions = {
   }
 }
 
+// Subgraph mode: where a dynamically registered address was registered and in
+// which round of its block (1 for a data source created by the block's own
+// triggers, 2 for one created by those, and so on).
+type creationBlock = {block: int, round: int}
+
+// When a contract's addresses leave the server-side filter. A switched contract
+// is fetched by the single address-free partition and gated against the address
+// store on our side. The switch is sticky for the run.
+type clientFilterPolicy = {
+  // Past this many registered addresses. None never switches on count alone.
+  addressThreshold: option<int>,
+  // Every dynamic contract, whatever its address count. Keeps a chain at one
+  // query per block, of one shape, however many contracts it registers.
+  allDynamicContracts: bool,
+}
+
 type t = {
   optimizedPartitions: OptimizedPartitions.t,
   startBlock: int,
@@ -879,6 +895,7 @@ type t = {
   // How much blocks behind the head we should query
   // Needed to query before entering reorg threshold
   blockLag: int,
+  blockThrottle: int,
   // Buffer of items ordered from earliest to latest
   buffer: array<Internal.item>,
   // Caps how far ahead onBlock items are pre-generated (set to 2x the batch
@@ -890,10 +907,17 @@ type t = {
   onBlockRegistrations: array<Internal.onBlockRegistration>,
   knownHeight: int,
   firstEventBlock: option<int>,
-  // Per-contract registered-address count past which a dynamic contract is
-  // switched to client-side filtering. None disables the switch, leaving every
-  // contract filtered server-side.
-  clientFilterAddressThreshold: option<int>,
+  clientFilterPolicy: clientFilterPolicy,
+  // Subgraph mode only: the block each dynamically registered address was
+  // registered in, keyed by contract and lowercase address, until fetching
+  // passes it. Its events in that block run after the rest of the block, as
+  // graph-node runs a new data source's creation-block triggers.
+  creationBlocks?: dict<creationBlock>,
+  // Set when an onBlock registration has a contractRegister: the last block
+  // whose block-handler register passes have run and whose registrations are
+  // applied. No block item past it is generated, so none is processed before
+  // the data sources its block created are fetched.
+  blockRegisterThrough?: int,
 }
 
 // The latest block whose items are all in the buffer: the lowest partition
@@ -983,6 +1007,65 @@ let compareTiebreak = (a: Internal.item, b: Internal.item): int => {
   }
 }
 
+// Within a block, graph-node runs a subgraph's triggers in this order: logs and
+// calls by transaction — a transaction's logs, then its calls in trace order —
+// then the block triggers of `filter: call` handlers, one per call, then the
+// block handlers. A call item is told apart by its rank (see
+// `Internal.getItemCallRank`), and only a run that has one pays for this
+// comparison. Creation-block items (rank 4) keep their place after everything
+// else, ordered among themselves by the same transaction rule. A `once`
+// handler's item (rank -1) comes before all of them.
+let blockOrderRank = (item: Internal.item) =>
+  switch item->Internal.getItemCreationRound {
+  | Some(round) => 3 + round
+  | None =>
+    if item->Internal.getItemKind !== 0 {
+      item->Internal.itemRunsFirst ? -1 : 3
+    } else if item->Internal.getItemCallRank === Some(2) {
+      2
+    } else {
+      0
+    }
+  }
+
+let compareWithCalls = (a: Internal.item, b: Internal.item): int => {
+  let ra = blockOrderRank(a)
+  let rb = blockOrderRank(b)
+  if ra !== rb {
+    ra < rb ? -1 : 1
+  } else if ra === 3 || ra === -1 {
+    // Block handlers: as before.
+    let la = a->Internal.getItemLogIndex
+    let lb = b->Internal.getItemLogIndex
+    if la != lb {
+      la < lb ? -1 : 1
+    } else {
+      compareTiebreak(a, b)
+    }
+  } else {
+    let ta = a->Internal.getItemTransactionIndex
+    let tb = b->Internal.getItemTransactionIndex
+    if ta != tb {
+      ta < tb ? -1 : 1
+    } else {
+      // A transaction's logs precede its calls.
+      let ca = a->Internal.getItemCallRank->Option.isSome ? 1 : 0
+      let cb = b->Internal.getItemCallRank->Option.isSome ? 1 : 0
+      if ca !== cb {
+        ca < cb ? -1 : 1
+      } else {
+        let la = a->Internal.getItemLogIndex
+        let lb = b->Internal.getItemLogIndex
+        if la != lb {
+          la < lb ? -1 : 1
+        } else {
+          compareTiebreak(a, b)
+        }
+      }
+    }
+  }
+}
+
 // Total order on buffer items: block, then item kind, then the ecosystem's
 // within-block order, then registration index. Returns a plain int (-1/0/1)
 // with explicit field comparisons so it can be called directly from the
@@ -994,15 +1077,32 @@ let compareTiebreak = (a: Internal.item, b: Internal.item): int => {
 // block's handlers by construction. A sentinel log index for block items
 // would put the same guarantee at the mercy of how large an ecosystem's key
 // grows — which is how SVM, keyed by transaction index, came to run slot
-// handlers ahead of most of a slot's instructions.
+// handlers ahead of most of a slot's instructions. A subgraph's `once` handler
+// is the exception: it runs ahead of its block's events.
 let compareBufferItem = (a: Internal.item, b: Internal.item): int => {
   let ba = a->Internal.getItemBlockNumber
   let bb = b->Internal.getItemBlockNumber
   if ba != bb {
     ba < bb ? -1 : 1
+  } else if Internal.callOrderingActive.contents {
+    compareWithCalls(a, b)
   } else {
-    let ka = a->Internal.getItemKind
-    let kb = b->Internal.getItemKind
+    let ka = switch a->Internal.getItemCreationRound {
+    | Some(round) => 1 + round
+    | None =>
+      switch a->Internal.getItemKind {
+      | 0 => 0
+      | kind => a->Internal.itemRunsFirst ? -1 : kind
+      }
+    }
+    let kb = switch b->Internal.getItemCreationRound {
+    | Some(round) => 1 + round
+    | None =>
+      switch b->Internal.getItemKind {
+      | 0 => 0
+      | kind => b->Internal.itemRunsFirst ? -1 : kind
+      }
+    }
     if ka !== kb {
       ka < kb ? -1 : 1
     } else {
@@ -1024,6 +1124,7 @@ let compareBufferItem = (a: Internal.item, b: Internal.item): int => {
 let isSameLog = (a: Internal.item, b: Internal.item): bool =>
   a->Internal.getItemKind === 0 &&
   b->Internal.getItemKind === 0 &&
+  a->Internal.getItemCallRank === b->Internal.getItemCallRank &&
   a->Internal.getItemBlockNumber === b->Internal.getItemBlockNumber &&
   a->Internal.getItemLogIndex === b->Internal.getItemLogIndex &&
   switch (a->Internal.getItemOrderPath, b->Internal.getItemOrderPath) {
@@ -1177,6 +1278,10 @@ let updateInternal = (
       },
       knownHeight,
     )
+    let maxBlockNumber = switch fetchState.blockRegisterThrough {
+    | Some(through) => Pervasives.min(maxBlockNumber, through)
+    | None => maxBlockNumber
+    }
     appendOnBlockItems(
       ~mutItems=blockItems,
       ~onBlockRegistrations,
@@ -1197,6 +1302,7 @@ let updateInternal = (
     optimizedPartitions,
     latestOnBlockBlockNumber,
     blockLag,
+    blockThrottle: fetchState.blockThrottle,
     knownHeight,
     // Single merge point: fold any onBlock items into the sorted base buffer.
     buffer: switch blockItems {
@@ -1204,7 +1310,9 @@ let updateInternal = (
     | blockItems => base->mergeIntoBuffer(blockItems)
     },
     firstEventBlock: fetchState.firstEventBlock,
-    clientFilterAddressThreshold: fetchState.clientFilterAddressThreshold,
+    clientFilterPolicy: fetchState.clientFilterPolicy,
+    creationBlocks: ?fetchState.creationBlocks,
+    blockRegisterThrough: ?fetchState.blockRegisterThrough,
   }
 
   updatedFetchState
@@ -1232,22 +1340,40 @@ type notice =
       requestedFromBlock: int,
     })
 
-// Move a contract to client-side address filtering, recording why.
-let addClientFilteredContract = (
+// The client-filtered set once `policy` has judged `contractNames`. A copy, so
+// older fetch state snapshots keep theirs: Utils.Set.add mutates in place.
+let switchToClientSideFiltering = (
   clientFilteredContracts: Utils.Set.t<string>,
-  ~contractName,
+  ~contractNames: array<string>,
+  ~policy: clientFilterPolicy,
+  ~dynamicContracts: Utils.Set.t<string>,
+  ~addressStore: AddressStore.t,
   ~onNotice,
-  ~addressCount,
-  ~threshold,
   // A resumed fetch state re-derives the switch from the persisted addresses,
-  // but it already happened - and was reported - in the run that crossed the
-  // threshold.
+  // but it already happened - and was reported - in an earlier run.
   ~isAlreadyReported=false,
 ) => {
-  clientFilteredContracts->Utils.Set.add(contractName)->ignore
-  if !isAlreadyReported {
-    onNotice(SwitchedToClientSideFiltering({contractName, addressCount, threshold}))
-  }
+  let switched = clientFilteredContracts->Utils.Set.toArray->Utils.Set.fromArray
+  contractNames->Array.forEach(contractName =>
+    if !(switched->Utils.Set.has(contractName)) {
+      if policy.allDynamicContracts && dynamicContracts->Utils.Set.has(contractName) {
+        switched->Utils.Set.add(contractName)->ignore
+      } else {
+        switch policy.addressThreshold {
+        | Some(threshold) =>
+          let addressCount = addressStore->AddressStore.contractCount(contractName)
+          if addressCount > threshold {
+            switched->Utils.Set.add(contractName)->ignore
+            if !isAlreadyReported {
+              onNotice(SwitchedToClientSideFiltering({contractName, addressCount, threshold}))
+            }
+          }
+        | None => ()
+        }
+      }
+    }
+  )
+  switched
 }
 
 // The block a partition will have fetched to once everything on its pending
@@ -1723,6 +1849,54 @@ OptimizedPartitions.t => {
   )
 }
 
+let creationBlockKey = (~contractName, ~address: Address.t) =>
+  `${contractName}:${address->Address.toString->String.toLowerCase}`
+
+// Marks the response's events that were logged in the block their address was
+// registered in, once the registrations fetching has already passed are
+// dropped: those can't receive another item.
+let markCreationBlockEvents = (creationBlocks: dict<creationBlock>, ~newItems, ~fetchedThrough) => {
+  creationBlocks
+  ->Dict.toArray
+  ->Array.forEach(((key, {block})) =>
+    if block < fetchedThrough {
+      creationBlocks->Utils.Dict.deleteInPlace(key)
+    }
+  )
+  newItems->Array.forEach(item =>
+    if item->Internal.getItemKind === 0 {
+      let eventItem = item->Internal.castUnsafeEventItem
+      let key = creationBlockKey(
+        ~contractName=eventItem.onEventRegistration.eventConfig.contractName,
+        ~address=eventItem.payload->Internal.getPayloadSrcAddress,
+      )
+      switch creationBlocks->Utils.Dict.dangerouslyGetNonOption(key) {
+      | Some({block, round}) if block === eventItem.blockNumber =>
+        item->Internal.markItemCreationRound(round)
+      | _ => ()
+      }
+    }
+  )
+}
+
+// The round a registration made by `item`'s handler belongs to: one past the
+// round `item` itself runs in, which is its address's registration round when
+// the event was logged in the block that address was registered in.
+let creationRoundOfRegistration = (fetchState: t, ~item: Internal.item) =>
+  switch fetchState.creationBlocks {
+  | None => 1
+  | Some(creationBlocks) =>
+    let eventItem = item->Internal.castUnsafeEventItem
+    let key = creationBlockKey(
+      ~contractName=eventItem.onEventRegistration.eventConfig.contractName,
+      ~address=eventItem.payload->Internal.getPayloadSrcAddress,
+    )
+    switch creationBlocks->Utils.Dict.dangerouslyGetNonOption(key) {
+    | Some({block, round}) if block === eventItem.blockNumber => round + 1
+    | _ => 1
+    }
+  }
+
 let registerDynamicContracts = (
   fetchState: t,
   ~addressStore: AddressStore.t,
@@ -1761,6 +1935,17 @@ let registerDynamicContracts = (
     let verdict = verdicts->Array.getUnsafe(idx)
     switch verdict {
     | Added({fetchable: true}) =>
+      switch fetchState.creationBlocks {
+      | Some(creationBlocks) =>
+        creationBlocks->Dict.set(
+          creationBlockKey(~contractName=registration.contractName, ~address=registration.address),
+          {
+            block: registration.registrationBlock,
+            round: registration.creationRound->Option.getOr(1),
+          },
+        )
+      | None => ()
+      }
       if !(registeringContractNames->Array.includes(registration.contractName)) {
         registeringContractNames->Array.push(registration.contractName)->ignore
       }
@@ -1862,34 +2047,16 @@ let registerDynamicContracts = (
         }
       }
 
-      // Switch any dynamic contract that has just crossed the server-side
-      // address threshold to client-side filtering. Sticky: the set only grows, and
-      // collapse in createPartitions folds the contract's
-      // partitions into the single address-free partition.
-      // Clone the sticky set before mutating so this update owns its copy and
-      // older fetchState snapshots keep theirs (Utils.Set.add mutates in place).
-      let clientFilteredContracts = switch fetchState.clientFilterAddressThreshold {
-      | Some(threshold) =>
-        let clientFilteredContracts =
-          fetchState.optimizedPartitions.clientFilteredContracts
-          ->Utils.Set.toArray
-          ->Utils.Set.fromArray
-        dynamicContractsRef.contents
-        ->Utils.Set.toArray
-        ->Array.forEach(contractName => {
-          let addressCount = addressStore->AddressStore.contractCount(contractName)
-          if !(clientFilteredContracts->Utils.Set.has(contractName)) && addressCount > threshold {
-            clientFilteredContracts->addClientFilteredContract(
-              ~contractName,
-              ~onNotice,
-              ~addressCount,
-              ~threshold,
-            )
-          }
-        })
-        clientFilteredContracts
-      | None => fetchState.optimizedPartitions.clientFilteredContracts
-      }
+      // Collapse in createPartitions folds a switched contract's partitions
+      // into the single address-free partition.
+      let clientFilteredContracts =
+        fetchState.optimizedPartitions.clientFilteredContracts->switchToClientSideFiltering(
+          ~contractNames=dynamicContractsRef.contents->Utils.Set.toArray,
+          ~policy=fetchState.clientFilterPolicy,
+          ~dynamicContracts=dynamicContractsRef.contents,
+          ~addressStore,
+          ~onNotice,
+        )
 
       // Only this batch's additions need partitions; everything already
       // registered has one.
@@ -1920,6 +2087,48 @@ let registerDynamicContracts = (
   }
 }
 
+// Every dynamic contract, the ones registered from now on included, is fetched
+// address-free and filtered on our side. Its partitions fold into the single
+// address-free partition through the same collapse the address threshold uses.
+let switchDynamicContractsToClientSideFiltering = (fetchState: t, ~addressStore: AddressStore.t) =>
+  if fetchState.clientFilterPolicy.allDynamicContracts {
+    fetchState
+  } else {
+    let fetchState = {
+      ...fetchState,
+      clientFilterPolicy: {...fetchState.clientFilterPolicy, allDynamicContracts: true},
+    }
+    let {
+      dynamicContracts,
+      clientFilteredContracts,
+      maxAddrInPartition,
+      nextPartitionIndex,
+    } = fetchState.optimizedPartitions
+    let switched =
+      clientFilteredContracts
+      ->Utils.Set.toArray
+      ->Array.concat(dynamicContracts->Utils.Set.toArray)
+      ->Utils.Set.fromArray
+    if switched->Utils.Set.size === clientFilteredContracts->Utils.Set.size {
+      fetchState
+    } else {
+      fetchState->updateInternal(
+        ~optimizedPartitions=createPartitions(
+          ~registeringSetsByContract=Dict.make(),
+          ~addressStore,
+          ~dynamicContracts,
+          ~clientFilteredContracts=switched,
+          ~normalSelection=fetchState.normalSelection,
+          ~maxAddrInPartition,
+          ~nextPartitionIndex,
+          ~existingPartitions=fetchState.optimizedPartitions.entities->Dict.valuesToArray,
+          ~progressBlockNumber=0,
+          ~knownHeight=fetchState.knownHeight,
+        ),
+      )
+    }
+  }
+
 /*
 Updates fetchState with a response for a given query.
 Throws if the partition with given query cannot be found (unexpected)
@@ -1941,12 +2150,80 @@ let handleQueryResult = (fetchState: t, ~query: query, ~latestFetchedBlock: int,
     ~mutItems=?{
       switch newItems {
       | [] => None
-      | _ => Some(fetchState.buffer->mergeIntoBuffer(newItems))
+      | _ =>
+        switch fetchState.creationBlocks {
+        | Some(creationBlocks) =>
+          creationBlocks->markCreationBlockEvents(
+            ~newItems,
+            ~fetchedThrough=fetchState->bufferBlockNumber,
+          )
+        | None => ()
+        }
+        Some(fetchState.buffer->mergeIntoBuffer(newItems))
       }
     },
   )
 }
 
+// The items of the block handlers' register passes for the blocks after
+// `after`, through `through`, at most about `limit` of them in whole blocks;
+// and the last block they cover. `nextBlockRegisterItems` starts after the
+// last block passed.
+let capAtEndBlock = (fetchState: t, block) =>
+  switch fetchState.endBlock {
+  | Some(endBlock) => Pervasives.min(block, endBlock)
+  | None => block
+  }
+
+let canAdvanceBlockRegisters = (fetchState: t, ~through) =>
+  switch fetchState.blockRegisterThrough {
+  | Some(after) => fetchState->capAtEndBlock(through) > after
+  | None => false
+  }
+
+let blockRegisterItemsAfter = (fetchState: t, ~after, ~through, ~limit) =>
+  switch fetchState.blockRegisterThrough {
+  | Some(_) if fetchState->capAtEndBlock(through) > after =>
+    let through = fetchState->capAtEndBlock(through)
+    let items = []
+    let covered = appendOnBlockItems(
+      ~mutItems=items,
+      ~onBlockRegistrations=fetchState.onBlockRegistrations->Array.filter((
+        r: Internal.onBlockRegistration,
+      ) => r.contractRegister !== None),
+      ~indexerStartBlock=fetchState.startBlock,
+      ~fromBlock=after,
+      ~maxBlockNumber=through,
+      ~maxOnBlockBufferSize=limit,
+    )
+    Some((items, covered))
+  | _ => None
+  }
+
+let nextBlockRegisterItems = (fetchState: t, ~through, ~limit) =>
+  switch fetchState.blockRegisterThrough {
+  | Some(after) => fetchState->blockRegisterItemsAfter(~after, ~through, ~limit)
+  | None => None
+  }
+
+let setBlockRegisterThrough = (fetchState: t, ~through) =>
+  switch fetchState.blockRegisterThrough {
+  | Some(current) if through > current =>
+    {...fetchState, blockRegisterThrough: through}->updateInternal
+  | _ => fetchState
+  }
+
+// How far block-handler register passes can run with no response to run them
+// with: through the block every partition has fetched, or the head when there
+// is no partition, and never past the chain's end block.
+let blockRegisterFrontier = (fetchState: t) =>
+  fetchState->capAtEndBlock(
+    switch fetchState.optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
+    | Some(block) => Pervasives.min(block, fetchState.knownHeight)
+    | None => fetchState.knownHeight
+    },
+  )
+
 type nextQuery =
   | WaitingForNewBlock
   | NothingToQuery
@@ -2387,6 +2664,20 @@ let acceptCandidates = (
   }
 }
 
+// A throttled chain lets `blockThrottle - 1` new blocks build up before it
+// fetches them, so a frontier that far behind the lagged head is waiting, not
+// behind. An end block the head has reached is fetched without waiting.
+let blocksBehindHeadAllowedByThrottle = ({blockThrottle, endBlock, knownHeight, blockLag}: t) =>
+  switch endBlock {
+  | Some(endBlock) if knownHeight - blockLag >= endBlock => 0
+  | _ => Pervasives.max(0, blockThrottle - 1)
+  }
+
+// The highest block the chain fetches up to right now: the lagged head, less the
+// blocks a throttled chain lets build up first.
+let headFetchTarget = ({knownHeight, blockLag} as fetchState: t) =>
+  knownHeight - blockLag - fetchState->blocksBehindHeadAllowedByThrottle
+
 // Candidate queries are sized against chainTargetBlock, the soft querying
 // horizon the owning chain wants to reach this tick — derived by ChainState
 // from its share of the indexer-wide buffer budget and its chain-level event
@@ -2415,7 +2706,7 @@ let acceptCandidates = (
 // rangeTargetDensity × (chainTargetBlock − fromBlock + 1) / inRangeCount — so
 // unknown-density partitions probe in parallel within one budget.
 let getNextQuery = (
-  {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
+  {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock} as fetchState: t,
   ~chainTargetBlock: int,
   ~chainTargetItems: float,
 ) => {
@@ -2423,7 +2714,8 @@ let getNextQuery = (
   if headBlockNumber <= 0 {
     WaitingForNewBlock
   } else {
-    let isOnBlockBehindTheHead = latestOnBlockBlockNumber < headBlockNumber
+    let caughtUpBlockNumber = headBlockNumber - fetchState->blocksBehindHeadAllowedByThrottle
+    let isOnBlockBehindTheHead = latestOnBlockBlockNumber < caughtUpBlockNumber
     let shouldWaitForNewBlock = ref(
       switch endBlock {
       | Some(endBlock) => headBlockNumber < endBlock
@@ -2468,7 +2760,7 @@ let getNextQuery = (
         }
       }
       inFlightCounts->Array.setUnsafe(idx, inFlightCount.contents)
-      if p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock < headBlockNumber {
+      if p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock < caughtUpBlockNumber {
         // Even if there are some partitions waiting for the new block
         // We still want to wait for all partitions reaching the head
         // because they might update knownHeight in their response
@@ -2647,9 +2939,11 @@ let make = (
   ~progressBlockNumber=startBlock - 1,
   ~onBlockRegistrations=[],
   ~blockLag=0,
+  ~blockThrottle=0,
   ~firstEventBlock=None,
   ~clientFilterAddressThreshold=None,
   ~isResumed=false,
+  ~deferCreationBlockEvents=false,
 ): t => {
   let latestFetchedBlock = progressBlockNumber
 
@@ -2729,7 +3023,6 @@ let make = (
   })
 
   let dynamicContracts = Utils.Set.make()
-  let clientFilteredContracts = Utils.Set.make()
   let registeringSetsByContract = Dict.make()
 
   // What each contract needs a partition for is read back off the store rather
@@ -2753,26 +3046,24 @@ let make = (
     }
   })
 
-  // Switch any contract already over the server-side address threshold to
-  // client-side filtering at creation — a config contract with a large static
-  // address list, or a dynamic contract restored from a large persisted set.
-  switch clientFilterAddressThreshold {
-  | Some(threshold) =>
-    registeringSetsByContract->Utils.Dict.forEachWithKey((set, contractName) => {
-      let addressCount = set->AddressSet.size
-      if addressCount > threshold {
-        clientFilteredContracts->addClientFilteredContract(
-          ~contractName,
-          ~onNotice,
-          ~addressCount,
-          ~threshold,
-          ~isAlreadyReported=isResumed,
-        )
-      }
-    })
-  | None => ()
+  let clientFilterPolicy = {
+    addressThreshold: clientFilterAddressThreshold,
+    allDynamicContracts: false,
   }
 
+  // Switch at creation whatever the threshold already covers: a config
+  // contract with a large static address list, or a dynamic contract restored
+  // from a large persisted set.
+  let clientFilteredContracts =
+    Utils.Set.make()->switchToClientSideFiltering(
+      ~contractNames=registeringSetsByContract->Dict.keysToArray,
+      ~policy=clientFilterPolicy,
+      ~dynamicContracts,
+      ~addressStore,
+      ~onNotice,
+      ~isAlreadyReported=isResumed,
+    )
+
   let optimizedPartitions = createPartitions(
     ~registeringSetsByContract,
     ~addressStore,
@@ -2805,6 +3096,14 @@ let make = (
   // fetching, so without seeding the buffer here getNextQuery would return
   // NothingToQuery and the indexer would get stuck.
   let buffer = []
+  // Every block up to the progress block was processed, so its register passes
+  // ran and what they registered is in the seeded rows.
+  let blockRegisterThrough =
+    onBlockRegistrations->Array.some((r: Internal.onBlockRegistration) =>
+      r.contractRegister !== None
+    )
+      ? Some(progressBlockNumber)
+      : None
   let latestOnBlockBlockNumber = switch onBlockRegistrations {
   // As updateInternal keeps it: with nothing to generate per block the pointer
   // is the head cap on the buffer frontier, and left at the progress block it
@@ -2817,6 +3116,10 @@ let make = (
     | None => knownHeight
     | Some(latestFullyFetchedBlock) => Pervasives.min(latestFullyFetchedBlock, knownHeight)
     }
+    let maxBlockNumber = switch blockRegisterThrough {
+    | Some(through) => Pervasives.min(maxBlockNumber, through)
+    | None => maxBlockNumber
+    }
     appendOnBlockItems(
       ~mutItems=buffer,
       ~onBlockRegistrations,
@@ -2836,12 +3139,15 @@ let make = (
     latestOnBlockBlockNumber,
     normalSelection,
     blockLag,
+    blockThrottle,
     onBlockRegistrations,
     maxOnBlockBufferSize,
     knownHeight,
     buffer,
     firstEventBlock,
-    clientFilterAddressThreshold,
+    clientFilterPolicy,
+    creationBlocks: ?(deferCreationBlockEvents ? Some(Dict.make()) : None),
+    ?blockRegisterThrough,
   }
 
   fetchState
@@ -2993,6 +3299,13 @@ let rollback = (
       fetchState.latestOnBlockBlockNumber,
       targetBlockNumber,
     ),
+    // The passes past the target run again on the refetched blocks: what they
+    // registered was pruned from the store above.
+    blockRegisterThrough: ?(
+      fetchState.blockRegisterThrough->Option.map(through =>
+        Pervasives.min(through, targetBlockNumber)
+      )
+    ),
   }->updateInternal(
     ~optimizedPartitions,
     // Filtering the sorted buffer keeps it sorted and deduped.
@@ -3054,15 +3367,37 @@ let isActivelyIndexing = ({endBlock} as fetchState: t) => {
 // True once the fetch frontier has reached the (lagged) head or endBlock,
 // regardless of whether the buffer has been consumed yet. Unlike
 // isReadyToEnterReorgThreshold, fetched-but-unprocessed items still count.
-let isFetchingAtHead = ({endBlock, blockLag, knownHeight} as fetchState: t) => {
+let isFetchingAtHead = ({endBlock, knownHeight} as fetchState: t) => {
   let bufferBlockNumber = fetchState->bufferBlockNumber
   knownHeight !== 0 &&
     switch endBlock {
     | Some(endBlock) if bufferBlockNumber >= endBlock => true
-    | _ => bufferBlockNumber >= knownHeight - blockLag
+    | _ => bufferBlockNumber >= fetchState->headFetchTarget
     }
 }
 
+// Counted in blocks whatever the chain's block time: the chain still fetches
+// block by block near the head, and an address-free query must never reach back
+// into a range a partition is still backfilling.
+let headClientFilteringDistance = 200
+
+// Every partition is within `headClientFilteringDistance` of the fetch target. A
+// chain with no head to follow never is: no known height yet, or an end block it
+// stops at before the head.
+let isWithinHeadClientFilteringDistance = (
+  {endBlock, blockLag, knownHeight, optimizedPartitions} as fetchState: t,
+) =>
+  knownHeight !== 0 &&
+  switch endBlock {
+  | Some(endBlock) => endBlock > knownHeight - blockLag
+  | None => true
+  } &&
+  switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
+  | Some(latestFetchedBlock) =>
+    fetchState->headFetchTarget - latestFetchedBlock <= headClientFilteringDistance
+  | None => true
+  }
+
 // Frontier at (or within `tolerance` of) the lagged head with no processable
 // items left — the moment the chain can cross into the reorg threshold.
 // `tolerance` absorbs the head advancing between a chain catching up and this