envio 3.15.0-subgraph → 3.16.0-subgraph

package/src/FetchState.res

@@ -869,6 +869,17 @@ module OptimizedPartitions = {
 // triggers, 2 for one created by those, and so on).
 type creationBlock = {block: int, round: int}
 
+// When a contract's addresses leave the server-side filter. A switched contract
+// is fetched by the single address-free partition and gated against the address
+// store on our side. The switch is sticky for the run.
+type clientFilterPolicy = {
+  // Past this many registered addresses. None never switches on count alone.
+  addressThreshold: option<int>,
+  // Every dynamic contract, whatever its address count. Keeps a chain at one
+  // query per block, of one shape, however many contracts it registers.
+  allDynamicContracts: bool,
+}
+
 type t = {
   optimizedPartitions: OptimizedPartitions.t,
   startBlock: int,
@@ -884,6 +895,7 @@ type t = {
   // How much blocks behind the head we should query
   // Needed to query before entering reorg threshold
   blockLag: int,
+  blockThrottle: int,
   // Buffer of items ordered from earliest to latest
   buffer: array<Internal.item>,
   // Caps how far ahead onBlock items are pre-generated (set to 2x the batch
@@ -895,10 +907,7 @@ type t = {
   onBlockRegistrations: array<Internal.onBlockRegistration>,
   knownHeight: int,
   firstEventBlock: option<int>,
-  // Per-contract registered-address count past which a dynamic contract is
-  // switched to client-side filtering. None disables the switch, leaving every
-  // contract filtered server-side.
-  clientFilterAddressThreshold: option<int>,
+  clientFilterPolicy: clientFilterPolicy,
   // Subgraph mode only: the block each dynamically registered address was
   // registered in, keyed by contract and lowercase address, until fetching
   // passes it. Its events in that block run after the rest of the block, as
@@ -1293,6 +1302,7 @@ let updateInternal = (
     optimizedPartitions,
     latestOnBlockBlockNumber,
     blockLag,
+    blockThrottle: fetchState.blockThrottle,
     knownHeight,
     // Single merge point: fold any onBlock items into the sorted base buffer.
     buffer: switch blockItems {
@@ -1300,7 +1310,7 @@ let updateInternal = (
     | blockItems => base->mergeIntoBuffer(blockItems)
     },
     firstEventBlock: fetchState.firstEventBlock,
-    clientFilterAddressThreshold: fetchState.clientFilterAddressThreshold,
+    clientFilterPolicy: fetchState.clientFilterPolicy,
     creationBlocks: ?fetchState.creationBlocks,
     blockRegisterThrough: ?fetchState.blockRegisterThrough,
   }
@@ -1330,22 +1340,40 @@ type notice =
       requestedFromBlock: int,
     })
 
-// Move a contract to client-side address filtering, recording why.
-let addClientFilteredContract = (
+// The client-filtered set once `policy` has judged `contractNames`. A copy, so
+// older fetch state snapshots keep theirs: Utils.Set.add mutates in place.
+let switchToClientSideFiltering = (
   clientFilteredContracts: Utils.Set.t<string>,
-  ~contractName,
+  ~contractNames: array<string>,
+  ~policy: clientFilterPolicy,
+  ~dynamicContracts: Utils.Set.t<string>,
+  ~addressStore: AddressStore.t,
   ~onNotice,
-  ~addressCount,
-  ~threshold,
   // A resumed fetch state re-derives the switch from the persisted addresses,
-  // but it already happened - and was reported - in the run that crossed the
-  // threshold.
+  // but it already happened - and was reported - in an earlier run.
   ~isAlreadyReported=false,
 ) => {
-  clientFilteredContracts->Utils.Set.add(contractName)->ignore
-  if !isAlreadyReported {
-    onNotice(SwitchedToClientSideFiltering({contractName, addressCount, threshold}))
-  }
+  let switched = clientFilteredContracts->Utils.Set.toArray->Utils.Set.fromArray
+  contractNames->Array.forEach(contractName =>
+    if !(switched->Utils.Set.has(contractName)) {
+      if policy.allDynamicContracts && dynamicContracts->Utils.Set.has(contractName) {
+        switched->Utils.Set.add(contractName)->ignore
+      } else {
+        switch policy.addressThreshold {
+        | Some(threshold) =>
+          let addressCount = addressStore->AddressStore.contractCount(contractName)
+          if addressCount > threshold {
+            switched->Utils.Set.add(contractName)->ignore
+            if !isAlreadyReported {
+              onNotice(SwitchedToClientSideFiltering({contractName, addressCount, threshold}))
+            }
+          }
+        | None => ()
+        }
+      }
+    }
+  )
+  switched
 }
 
 // The block a partition will have fetched to once everything on its pending
@@ -2019,34 +2047,16 @@ let registerDynamicContracts = (
         }
       }
 
-      // Switch any dynamic contract that has just crossed the server-side
-      // address threshold to client-side filtering. Sticky: the set only grows, and
-      // collapse in createPartitions folds the contract's
-      // partitions into the single address-free partition.
-      // Clone the sticky set before mutating so this update owns its copy and
-      // older fetchState snapshots keep theirs (Utils.Set.add mutates in place).
-      let clientFilteredContracts = switch fetchState.clientFilterAddressThreshold {
-      | Some(threshold) =>
-        let clientFilteredContracts =
-          fetchState.optimizedPartitions.clientFilteredContracts
-          ->Utils.Set.toArray
-          ->Utils.Set.fromArray
-        dynamicContractsRef.contents
-        ->Utils.Set.toArray
-        ->Array.forEach(contractName => {
-          let addressCount = addressStore->AddressStore.contractCount(contractName)
-          if !(clientFilteredContracts->Utils.Set.has(contractName)) && addressCount > threshold {
-            clientFilteredContracts->addClientFilteredContract(
-              ~contractName,
-              ~onNotice,
-              ~addressCount,
-              ~threshold,
-            )
-          }
-        })
-        clientFilteredContracts
-      | None => fetchState.optimizedPartitions.clientFilteredContracts
-      }
+      // Collapse in createPartitions folds a switched contract's partitions
+      // into the single address-free partition.
+      let clientFilteredContracts =
+        fetchState.optimizedPartitions.clientFilteredContracts->switchToClientSideFiltering(
+          ~contractNames=dynamicContractsRef.contents->Utils.Set.toArray,
+          ~policy=fetchState.clientFilterPolicy,
+          ~dynamicContracts=dynamicContractsRef.contents,
+          ~addressStore,
+          ~onNotice,
+        )
 
       // Only this batch's additions need partitions; everything already
       // registered has one.
@@ -2077,6 +2087,48 @@ let registerDynamicContracts = (
   }
 }
 
+// Every dynamic contract, the ones registered from now on included, is fetched
+// address-free and filtered on our side. Its partitions fold into the single
+// address-free partition through the same collapse the address threshold uses.
+let switchDynamicContractsToClientSideFiltering = (fetchState: t, ~addressStore: AddressStore.t) =>
+  if fetchState.clientFilterPolicy.allDynamicContracts {
+    fetchState
+  } else {
+    let fetchState = {
+      ...fetchState,
+      clientFilterPolicy: {...fetchState.clientFilterPolicy, allDynamicContracts: true},
+    }
+    let {
+      dynamicContracts,
+      clientFilteredContracts,
+      maxAddrInPartition,
+      nextPartitionIndex,
+    } = fetchState.optimizedPartitions
+    let switched =
+      clientFilteredContracts
+      ->Utils.Set.toArray
+      ->Array.concat(dynamicContracts->Utils.Set.toArray)
+      ->Utils.Set.fromArray
+    if switched->Utils.Set.size === clientFilteredContracts->Utils.Set.size {
+      fetchState
+    } else {
+      fetchState->updateInternal(
+        ~optimizedPartitions=createPartitions(
+          ~registeringSetsByContract=Dict.make(),
+          ~addressStore,
+          ~dynamicContracts,
+          ~clientFilteredContracts=switched,
+          ~normalSelection=fetchState.normalSelection,
+          ~maxAddrInPartition,
+          ~nextPartitionIndex,
+          ~existingPartitions=fetchState.optimizedPartitions.entities->Dict.valuesToArray,
+          ~progressBlockNumber=0,
+          ~knownHeight=fetchState.knownHeight,
+        ),
+      )
+    }
+  }
+
 /*
 Updates fetchState with a response for a given query.
 Throws if the partition with given query cannot be found (unexpected)
@@ -2612,6 +2664,20 @@ let acceptCandidates = (
   }
 }
 
+// A throttled chain lets `blockThrottle - 1` new blocks build up before it
+// fetches them, so a frontier that far behind the lagged head is waiting, not
+// behind. An end block the head has reached is fetched without waiting.
+let blocksBehindHeadAllowedByThrottle = ({blockThrottle, endBlock, knownHeight, blockLag}: t) =>
+  switch endBlock {
+  | Some(endBlock) if knownHeight - blockLag >= endBlock => 0
+  | _ => Pervasives.max(0, blockThrottle - 1)
+  }
+
+// The highest block the chain fetches up to right now: the lagged head, less the
+// blocks a throttled chain lets build up first.
+let headFetchTarget = ({knownHeight, blockLag} as fetchState: t) =>
+  knownHeight - blockLag - fetchState->blocksBehindHeadAllowedByThrottle
+
 // Candidate queries are sized against chainTargetBlock, the soft querying
 // horizon the owning chain wants to reach this tick — derived by ChainState
 // from its share of the indexer-wide buffer budget and its chain-level event
@@ -2640,7 +2706,7 @@ let acceptCandidates = (
 // rangeTargetDensity × (chainTargetBlock − fromBlock + 1) / inRangeCount — so
 // unknown-density partitions probe in parallel within one budget.
 let getNextQuery = (
-  {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
+  {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock} as fetchState: t,
   ~chainTargetBlock: int,
   ~chainTargetItems: float,
 ) => {
@@ -2648,7 +2714,8 @@ let getNextQuery = (
   if headBlockNumber <= 0 {
     WaitingForNewBlock
   } else {
-    let isOnBlockBehindTheHead = latestOnBlockBlockNumber < headBlockNumber
+    let caughtUpBlockNumber = headBlockNumber - fetchState->blocksBehindHeadAllowedByThrottle
+    let isOnBlockBehindTheHead = latestOnBlockBlockNumber < caughtUpBlockNumber
     let shouldWaitForNewBlock = ref(
       switch endBlock {
       | Some(endBlock) => headBlockNumber < endBlock
@@ -2693,7 +2760,7 @@ let getNextQuery = (
         }
       }
       inFlightCounts->Array.setUnsafe(idx, inFlightCount.contents)
-      if p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock < headBlockNumber {
+      if p.mutPendingQueries->Array.length > 0 || p.latestFetchedBlock < caughtUpBlockNumber {
         // Even if there are some partitions waiting for the new block
         // We still want to wait for all partitions reaching the head
         // because they might update knownHeight in their response
@@ -2872,6 +2939,7 @@ let make = (
   ~progressBlockNumber=startBlock - 1,
   ~onBlockRegistrations=[],
   ~blockLag=0,
+  ~blockThrottle=0,
   ~firstEventBlock=None,
   ~clientFilterAddressThreshold=None,
   ~isResumed=false,
@@ -2955,7 +3023,6 @@ let make = (
   })
 
   let dynamicContracts = Utils.Set.make()
-  let clientFilteredContracts = Utils.Set.make()
   let registeringSetsByContract = Dict.make()
 
   // What each contract needs a partition for is read back off the store rather
@@ -2979,26 +3046,24 @@ let make = (
     }
   })
 
-  // Switch any contract already over the server-side address threshold to
-  // client-side filtering at creation — a config contract with a large static
-  // address list, or a dynamic contract restored from a large persisted set.
-  switch clientFilterAddressThreshold {
-  | Some(threshold) =>
-    registeringSetsByContract->Utils.Dict.forEachWithKey((set, contractName) => {
-      let addressCount = set->AddressSet.size
-      if addressCount > threshold {
-        clientFilteredContracts->addClientFilteredContract(
-          ~contractName,
-          ~onNotice,
-          ~addressCount,
-          ~threshold,
-          ~isAlreadyReported=isResumed,
-        )
-      }
-    })
-  | None => ()
+  let clientFilterPolicy = {
+    addressThreshold: clientFilterAddressThreshold,
+    allDynamicContracts: false,
   }
 
+  // Switch at creation whatever the threshold already covers: a config
+  // contract with a large static address list, or a dynamic contract restored
+  // from a large persisted set.
+  let clientFilteredContracts =
+    Utils.Set.make()->switchToClientSideFiltering(
+      ~contractNames=registeringSetsByContract->Dict.keysToArray,
+      ~policy=clientFilterPolicy,
+      ~dynamicContracts,
+      ~addressStore,
+      ~onNotice,
+      ~isAlreadyReported=isResumed,
+    )
+
   let optimizedPartitions = createPartitions(
     ~registeringSetsByContract,
     ~addressStore,
@@ -3074,12 +3139,13 @@ let make = (
     latestOnBlockBlockNumber,
     normalSelection,
     blockLag,
+    blockThrottle,
     onBlockRegistrations,
     maxOnBlockBufferSize,
     knownHeight,
     buffer,
     firstEventBlock,
-    clientFilterAddressThreshold,
+    clientFilterPolicy,
     creationBlocks: ?(deferCreationBlockEvents ? Some(Dict.make()) : None),
     ?blockRegisterThrough,
   }
@@ -3301,15 +3367,37 @@ let isActivelyIndexing = ({endBlock} as fetchState: t) => {
 // True once the fetch frontier has reached the (lagged) head or endBlock,
 // regardless of whether the buffer has been consumed yet. Unlike
 // isReadyToEnterReorgThreshold, fetched-but-unprocessed items still count.
-let isFetchingAtHead = ({endBlock, blockLag, knownHeight} as fetchState: t) => {
+let isFetchingAtHead = ({endBlock, knownHeight} as fetchState: t) => {
   let bufferBlockNumber = fetchState->bufferBlockNumber
   knownHeight !== 0 &&
     switch endBlock {
     | Some(endBlock) if bufferBlockNumber >= endBlock => true
-    | _ => bufferBlockNumber >= knownHeight - blockLag
+    | _ => bufferBlockNumber >= fetchState->headFetchTarget
     }
 }
 
+// Counted in blocks whatever the chain's block time: the chain still fetches
+// block by block near the head, and an address-free query must never reach back
+// into a range a partition is still backfilling.
+let headClientFilteringDistance = 200
+
+// Every partition is within `headClientFilteringDistance` of the fetch target. A
+// chain with no head to follow never is: no known height yet, or an end block it
+// stops at before the head.
+let isWithinHeadClientFilteringDistance = (
+  {endBlock, blockLag, knownHeight, optimizedPartitions} as fetchState: t,
+) =>
+  knownHeight !== 0 &&
+  switch endBlock {
+  | Some(endBlock) => endBlock > knownHeight - blockLag
+  | None => true
+  } &&
+  switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
+  | Some(latestFetchedBlock) =>
+    fetchState->headFetchTarget - latestFetchedBlock <= headClientFilteringDistance
+  | None => true
+  }
+
 // Frontier at (or within `tolerance` of) the lagged head with no processable
 // items left — the moment the chain can cross into the reorg threshold.
 // `tolerance` absorbs the head advancing between a chain catching up and this

package/src/FetchState.res.mjs

@@ -1036,28 +1036,49 @@ function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, mutItemsSo
     chainId: fetchState.chainId,
     latestOnBlockBlockNumber: latestOnBlockBlockNumber,
     blockLag: blockLag,
+    blockThrottle: fetchState.blockThrottle,
     buffer: blockItems.length !== 0 ? mergeIntoBuffer(base, blockItems) : base,
     maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
     onBlockRegistrations: fetchState.onBlockRegistrations,
     knownHeight: knownHeight,
     firstEventBlock: fetchState.firstEventBlock,
-    clientFilterAddressThreshold: fetchState.clientFilterAddressThreshold,
+    clientFilterPolicy: fetchState.clientFilterPolicy,
     creationBlocks: fetchState.creationBlocks,
     blockRegisterThrough: fetchState.blockRegisterThrough
   };
 }
 
-function addClientFilteredContract(clientFilteredContracts, contractName, onNotice, addressCount, threshold, isAlreadyReportedOpt) {
+function switchToClientSideFiltering(clientFilteredContracts, contractNames, policy, dynamicContracts, addressStore, onNotice, isAlreadyReportedOpt) {
   let isAlreadyReported = isAlreadyReportedOpt !== undefined ? isAlreadyReportedOpt : false;
-  clientFilteredContracts.add(contractName);
-  if (!isAlreadyReported) {
-    return onNotice({
-      TAG: "SwitchedToClientSideFiltering",
-      contractName: contractName,
-      addressCount: addressCount,
-      threshold: threshold
-    });
-  }
+  let switched = new Set(Array.from(clientFilteredContracts));
+  contractNames.forEach(contractName => {
+    if (switched.has(contractName)) {
+      return;
+    }
+    if (policy.allDynamicContracts && dynamicContracts.has(contractName)) {
+      switched.add(contractName);
+      return;
+    }
+    let threshold = policy.addressThreshold;
+    if (threshold === undefined) {
+      return;
+    }
+    let addressCount = addressStore.contractCount(contractName);
+    if (addressCount > threshold) {
+      switched.add(contractName);
+      if (!isAlreadyReported) {
+        return onNotice({
+          TAG: "SwitchedToClientSideFiltering",
+          contractName: contractName,
+          addressCount: addressCount,
+          threshold: threshold
+        });
+      } else {
+        return;
+      }
+    }
+  });
+  return switched;
 }
 
 function claimedFetchedBlock(p, knownHeight) {
@@ -1505,20 +1526,7 @@ function registerDynamicContracts(fetchState, addressStore, onNotice, claimCeili
       }
     }
   }
-  let threshold = fetchState.clientFilterAddressThreshold;
-  let clientFilteredContracts;
-  if (threshold !== undefined) {
-    let clientFilteredContracts$1 = new Set(Array.from(fetchState.optimizedPartitions.clientFilteredContracts));
-    Array.from(dynamicContractsRef).forEach(contractName => {
-      let addressCount = addressStore.contractCount(contractName);
-      if (!clientFilteredContracts$1.has(contractName) && addressCount > threshold) {
-        return addClientFilteredContract(clientFilteredContracts$1, contractName, onNotice, addressCount, threshold, undefined);
-      }
-    });
-    clientFilteredContracts = clientFilteredContracts$1;
-  } else {
-    clientFilteredContracts = fetchState.optimizedPartitions.clientFilteredContracts;
-  }
+  let clientFilteredContracts = switchToClientSideFiltering(fetchState.optimizedPartitions.clientFilteredContracts, Array.from(dynamicContractsRef), fetchState.clientFilterPolicy, dynamicContractsRef, addressStore, onNotice, undefined);
   let registeringSetsByContract = {};
   registeringContractNames.forEach(contractName => {
     registeringSetsByContract[contractName] = AddressStore.makeSet(addressStore, contractName, {
@@ -1529,6 +1537,27 @@ function registerDynamicContracts(fetchState, addressStore, onNotice, claimCeili
   return updateInternal(fetchState, optimizedPartitions, undefined, undefined, undefined, undefined);
 }
 
+function switchDynamicContractsToClientSideFiltering(fetchState, addressStore) {
+  if (fetchState.clientFilterPolicy.allDynamicContracts) {
+    return fetchState;
+  }
+  let newrecord = {...fetchState};
+  let init = fetchState.clientFilterPolicy;
+  newrecord.clientFilterPolicy = {
+    addressThreshold: init.addressThreshold,
+    allDynamicContracts: true
+  };
+  let match = newrecord.optimizedPartitions;
+  let clientFilteredContracts = match.clientFilteredContracts;
+  let dynamicContracts = match.dynamicContracts;
+  let switched = new Set(Array.from(clientFilteredContracts).concat(Array.from(dynamicContracts)));
+  if (switched.size === clientFilteredContracts.size) {
+    return newrecord;
+  } else {
+    return updateInternal(newrecord, createPartitions({}, addressStore, dynamicContracts, switched, newrecord.normalSelection, match.maxAddrInPartition, match.nextPartitionIndex, Object.values(newrecord.optimizedPartitions.entities), 0, newrecord.knownHeight), undefined, undefined, undefined, undefined);
+  }
+}
+
 function handleQueryResult(fetchState, query, latestFetchedBlock, newItems) {
   let tmp;
   if (newItems.length !== 0) {
@@ -1838,15 +1867,30 @@ function acceptCandidates(candidates, reservations, chainTargetItems, partitionI
   };
 }
 
-function getNextQuery(param, chainTargetBlock, chainTargetItems) {
-  let blockLag = param.blockLag;
+function blocksBehindHeadAllowedByThrottle(param) {
+  let blockThrottle = param.blockThrottle;
   let endBlock = param.endBlock;
-  let optimizedPartitions = param.optimizedPartitions;
-  let headBlockNumber = param.knownHeight - blockLag | 0;
+  if (endBlock !== undefined && (param.knownHeight - param.blockLag | 0) >= endBlock) {
+    return 0;
+  } else {
+    return Primitive_int.max(0, blockThrottle - 1 | 0);
+  }
+}
+
+function headFetchTarget(fetchState) {
+  return (fetchState.knownHeight - fetchState.blockLag | 0) - blocksBehindHeadAllowedByThrottle(fetchState) | 0;
+}
+
+function getNextQuery(fetchState, chainTargetBlock, chainTargetItems) {
+  let blockLag = fetchState.blockLag;
+  let endBlock = fetchState.endBlock;
+  let optimizedPartitions = fetchState.optimizedPartitions;
+  let headBlockNumber = fetchState.knownHeight - blockLag | 0;
   if (headBlockNumber <= 0) {
     return "WaitingForNewBlock";
   }
-  let isOnBlockBehindTheHead = param.latestOnBlockBlockNumber < headBlockNumber;
+  let caughtUpBlockNumber = headBlockNumber - blocksBehindHeadAllowedByThrottle(fetchState) | 0;
+  let isOnBlockBehindTheHead = fetchState.latestOnBlockBlockNumber < caughtUpBlockNumber;
   let shouldWaitForNewBlock = (
     endBlock !== undefined ? headBlockNumber < endBlock : true
   ) && !isOnBlockBehindTheHead;
@@ -1872,7 +1916,7 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems) {
       }
     }
     inFlightCounts[idx] = inFlightCount;
-    if (p.mutPendingQueries.length !== 0 || p.latestFetchedBlock < headBlockNumber) {
+    if (p.mutPendingQueries.length !== 0 || p.latestFetchedBlock < caughtUpBlockNumber) {
       shouldWaitForNewBlock = false;
     }
   }
@@ -1963,10 +2007,11 @@ function getReadyItemsCount(fetchState, targetSize, fromItem) {
   return acc;
 }
 
-function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, maxAddrInPartition, chainId, onNotice, maxOnBlockBufferSize, knownHeight, progressBlockNumberOpt, onBlockRegistrationsOpt, blockLagOpt, firstEventBlockOpt, clientFilterAddressThresholdOpt, isResumedOpt, deferCreationBlockEventsOpt) {
+function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, maxAddrInPartition, chainId, onNotice, maxOnBlockBufferSize, knownHeight, progressBlockNumberOpt, onBlockRegistrationsOpt, blockLagOpt, blockThrottleOpt, firstEventBlockOpt, clientFilterAddressThresholdOpt, isResumedOpt, deferCreationBlockEventsOpt) {
   let progressBlockNumber = progressBlockNumberOpt !== undefined ? progressBlockNumberOpt : startBlock - 1 | 0;
   let onBlockRegistrations = onBlockRegistrationsOpt !== undefined ? onBlockRegistrationsOpt : [];
   let blockLag = blockLagOpt !== undefined ? blockLagOpt : 0;
+  let blockThrottle = blockThrottleOpt !== undefined ? blockThrottleOpt : 0;
   let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
   let clientFilterAddressThreshold = clientFilterAddressThresholdOpt !== undefined ? Primitive_option.valFromOption(clientFilterAddressThresholdOpt) : undefined;
   let isResumed = isResumedOpt !== undefined ? isResumedOpt : false;
@@ -2016,7 +2061,6 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
     requestedFromBlock: rejected.effectiveStartBlock
   }));
   let dynamicContracts = new Set();
-  let clientFilteredContracts = new Set();
   let registeringSetsByContract = {};
   Array.from(contractNamesWithNormalEvents).forEach(contractName => {
     if (addressStore.contractCount(contractName) > 0) {
@@ -2030,14 +2074,11 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
       return;
     }
   });
-  if (clientFilterAddressThreshold !== undefined) {
-    Utils.Dict.forEachWithKey(registeringSetsByContract, (set, contractName) => {
-      let addressCount = set.size();
-      if (addressCount > clientFilterAddressThreshold) {
-        return addClientFilteredContract(clientFilteredContracts, contractName, onNotice, addressCount, clientFilterAddressThreshold, isResumed);
-      }
-    });
-  }
+  let clientFilterPolicy = {
+    addressThreshold: clientFilterAddressThreshold,
+    allDynamicContracts: false
+  };
+  let clientFilteredContracts = switchToClientSideFiltering(new Set(), Object.keys(registeringSetsByContract), clientFilterPolicy, dynamicContracts, addressStore, onNotice, isResumed);
   let optimizedPartitions = createPartitions(registeringSetsByContract, addressStore, dynamicContracts, clientFilteredContracts, normalSelection, maxAddrInPartition, partitions.length, partitions, progressBlockNumber, knownHeight);
   if (optimizedPartitions.idsInAscOrder.length === 0 && Utils.$$Array.isEmpty(onBlockRegistrations$1)) {
     Stdlib_JsError.throwWithMessage(`Invalid configuration: Nothing to fetch on chain ` + ChainId.toString(chainId) + `. ` + (`addresses=` + addressRows.addresses.length.toString() + `, `) + (`onEventRegistrations=` + onEventRegistrations.length.toString() + `, `) + (`normalRegistrations=` + normalRegistrations.length.toString() + `. `) + `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`);
@@ -2070,12 +2111,13 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
     chainId: chainId,
     latestOnBlockBlockNumber: latestOnBlockBlockNumber,
     blockLag: blockLag,
+    blockThrottle: blockThrottle,
     buffer: buffer,
     maxOnBlockBufferSize: maxOnBlockBufferSize,
     onBlockRegistrations: onBlockRegistrations$1,
     knownHeight: knownHeight,
     firstEventBlock: firstEventBlock,
-    clientFilterAddressThreshold: clientFilterAddressThreshold,
+    clientFilterPolicy: clientFilterPolicy,
     creationBlocks: deferCreationBlockEvents ? ({}) : undefined,
     blockRegisterThrough: blockRegisterThrough
   };
@@ -2238,21 +2280,37 @@ function isActivelyIndexing(fetchState) {
 }
 
 function isFetchingAtHead(fetchState) {
-  let knownHeight = fetchState.knownHeight;
-  let blockLag = fetchState.blockLag;
   let endBlock = fetchState.endBlock;
   let bufferBlockNumber$1 = bufferBlockNumber(fetchState);
-  if (knownHeight !== 0) {
+  if (fetchState.knownHeight !== 0) {
     if (endBlock !== undefined && bufferBlockNumber$1 >= endBlock) {
       return true;
     } else {
-      return bufferBlockNumber$1 >= (knownHeight - blockLag | 0);
+      return bufferBlockNumber$1 >= headFetchTarget(fetchState);
     }
   } else {
     return false;
   }
 }
 
+function isWithinHeadClientFilteringDistance(fetchState) {
+  let knownHeight = fetchState.knownHeight;
+  let endBlock = fetchState.endBlock;
+  let optimizedPartitions = fetchState.optimizedPartitions;
+  if (!(knownHeight !== 0 && (
+      endBlock !== undefined ? endBlock > (knownHeight - fetchState.blockLag | 0) : true
+    ))) {
+    return false;
+  }
+  let id = optimizedPartitions.idsInAscOrder[0];
+  let latestFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
+  if (latestFetchedBlock !== undefined) {
+    return (headFetchTarget(fetchState) - latestFetchedBlock | 0) <= 200;
+  } else {
+    return true;
+  }
+}
+
 function isReadyToEnterReorgThreshold(tolerance, fetchState) {
   let knownHeight = fetchState.knownHeight;
   let endBlock = fetchState.endBlock;
@@ -2339,6 +2397,8 @@ let maxChainConcurrency = Env.maxChainConcurrency;
 
 let chunkRangeGrowthFactor = 1.8;
 
+let headClientFilteringDistance = 200;
+
 export {
   deriveSelectionStartBlock,
   makeSelection,
@@ -2364,7 +2424,7 @@ export {
   mergeIntoBuffer,
   appendOnBlockItems,
   updateInternal,
-  addClientFilteredContract,
+  switchToClientSideFiltering,
   claimedFetchedBlock,
   collapseClientFilteredContracts,
   reportRejectedRegistration,
@@ -2373,6 +2433,7 @@ export {
   markCreationBlockEvents,
   creationRoundOfRegistration,
   registerDynamicContracts,
+  switchDynamicContractsToClientSideFiltering,
   handleQueryResult,
   capAtEndBlock,
   canAdvanceBlockRegisters,
@@ -2389,6 +2450,8 @@ export {
   walkPartitionPending,
   pushForwardCandidates,
   acceptCandidates,
+  blocksBehindHeadAllowedByThrottle,
+  headFetchTarget,
   getNextQuery,
   hasReadyItem,
   getReadyItemsCount,
@@ -2400,6 +2463,8 @@ export {
   resetPendingQueries,
   isActivelyIndexing,
   isFetchingAtHead,
+  headClientFilteringDistance,
+  isWithinHeadClientFilteringDistance,
   isReadyToEnterReorgThreshold,
   getProgressPercentage,
   sortForBatch,

package/src/FinalizeBackfill.res

@@ -40,7 +40,7 @@ let runOnce = async (state: IndexerState.t) => {
     let storage = persistence->Persistence.getInitializedStorageOrThrow
     let readyAt = Date.make()
 
-    await storage.finalizeBackfill(~entities=persistence.allEntities, ~chainIds, ~readyAt)
+    await storage.finalizeBackfill(~chainIds, ~readyAt)
 
     // Only after the commit: in-memory readiness must never run ahead of the
     // `ready_at` a restart would read back.
@@ -68,10 +68,8 @@ let run = (state: IndexerState.t) =>
 // Best-effort and not awaited by the loop: indexing is already live and correct
 // without the index, just slower.
 let repairSchemaIndexes = (state: IndexerState.t) => {
-  let persistence = state->IndexerState.persistence
-  let storage = persistence->Persistence.getInitializedStorageOrThrow
+  let storage = state->IndexerState.persistence->Persistence.getInitializedStorageOrThrow
   storage.ensureSchemaIndexes(
-    ~entities=persistence.allEntities,
     ~chainIds=state->IndexerState.crossChainState->CrossChainState.chainIds,
   )
 }