envio 3.3.0-alpha.9 → 3.3.0-rc.0

package/src/ChainState.res

@@ -4,6 +4,8 @@
 
 type t = {
   logger: Pino.t,
+  // The registrations used to build this chain's sources and route native items.
+  onEventRegistrations: array<Internal.onEventRegistration>,
   mutable fetchState: FetchState.t,
   // The chain-wide address index. Not `mutable`: the dict is mutated in place by
   // register/rollback, so the reference is stable across fetchState versions.
@@ -13,8 +15,13 @@ type t = {
   mutable isProgressAtHead: bool,
   mutable timestampCaughtUpToHeadOrEndblock: option<Date.t>,
   mutable committedProgressBlockNumber: int,
+  // Progress of the batch currently being processed. The buffer is consumed at
+  // batch creation (see advanceAfterBatch), so this runs ahead of
+  // committedProgressBlockNumber until the batch commits — it's the true lower
+  // boundary of the remaining buffer's block span.
+  mutable processingBlockNumber: int,
   mutable numEventsProcessed: float,
-  // Running sum of in-flight queries' itemsTarget, kept here so the
+  // Running sum of in-flight queries' itemsEst, kept here so the
   // scheduler doesn't re-sum pending queries on every tick. Incremented when
   // queries are dispatched, decremented as their responses land.
   mutable pendingBudget: float,
@@ -66,9 +73,22 @@ let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddres
   addresses
 }
 
+let validateOnEventRegistrations = (
+  ~chainId: int,
+  registrations: array<Internal.onEventRegistration>,
+) =>
+  registrations->Array.forEachWithIndex((registration, expectedIndex) => {
+    if registration.index !== expectedIndex {
+      JsError.throwWithMessage(
+        `Invalid onEvent registration index for chain ${chainId->Int.toString}: ${registration.eventConfig.contractName}.${registration.eventConfig.name} has index ${registration.index->Int.toString}, but its ChainState position is ${expectedIndex->Int.toString}.`,
+      )
+    }
+  })
+
 let make = (
   ~chainConfig: Config.chain,
   ~fetchState: FetchState.t,
+  ~onEventRegistrations=[],
   ~indexingAddresses: IndexingAddresses.t,
   ~sourceManager: SourceManager.t,
   ~reorgDetection: ReorgDetection.t,
@@ -82,21 +102,26 @@ let make = (
   ~blockStore=BlockStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
   ~logger: Pino.t,
 ): t => {
-  logger,
-  fetchState,
-  indexingAddresses,
-  sourceManager,
-  chainConfig,
-  isProgressAtHead,
-  timestampCaughtUpToHeadOrEndblock,
-  committedProgressBlockNumber,
-  numEventsProcessed,
-  pendingBudget: 0.,
-  chainDensity,
-  reorgDetection,
-  safeCheckpointTracking,
-  transactionStore,
-  blockStore,
+  validateOnEventRegistrations(~chainId=chainConfig.id, onEventRegistrations)
+  {
+    logger,
+    onEventRegistrations,
+    fetchState,
+    indexingAddresses,
+    sourceManager,
+    chainConfig,
+    isProgressAtHead,
+    timestampCaughtUpToHeadOrEndblock,
+    committedProgressBlockNumber,
+    processingBlockNumber: committedProgressBlockNumber,
+    numEventsProcessed,
+    pendingBudget: 0.,
+    chainDensity,
+    reorgDetection,
+    safeCheckpointTracking,
+    transactionStore,
+    blockStore,
+  }
 }
 
 let makeInternal = (
@@ -232,6 +257,7 @@ let makeInternal = (
   make(
     ~chainConfig,
     ~fetchState,
+    ~onEventRegistrations,
     ~indexingAddresses=indexingAddressIndex,
     ~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
     ~reorgDetection=ReorgDetection.make(
@@ -367,7 +393,7 @@ let startFetchingQueries = (cs: t, ~queries: array<FetchState.query>) => {
   cs.fetchState->FetchState.startFetchingQueries(~queries)
   cs.pendingBudget =
     cs.pendingBudget +.
-    queries->Array.reduce(0., (acc, query) => acc +. query.itemsTarget->Int.toFloat)
+    queries->Array.reduce(0., (acc, query) => acc +. query.itemsEst->Int.toFloat)
 }
 
 // Drop every in-flight query and release their reservations together, keeping
@@ -384,44 +410,74 @@ let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
 // nudge it.
 let densityBlendWindow = 100.
 
-// The last block this chain can fetch right now: the head, or endBlock when
-// it's below the head.
+// The last block this chain can fetch right now: the lagged head
+// (knownHeight - blockLag, the frontier when a reorg-threshold blockLag holds
+// back the tip), or endBlock when it's below that head. Matching isFetchingAtHead's
+// definition of the head is what lets a chain parked at its lagged head read
+// 100% progress instead of looking behind against blocks it can't fetch.
 let fetchCeiling = (cs: t) => {
   let fetchState = cs.fetchState
+  let head = Pervasives.max(0, fetchState.knownHeight - fetchState.blockLag)
   switch fetchState.endBlock {
-  | Some(endBlock) => Pervasives.min(endBlock, fetchState.knownHeight)
-  | None => fetchState.knownHeight
+  | Some(endBlock) => Pervasives.min(endBlock, head)
+  | None => head
+  }
+}
+
+// Events/block over the ready part of the buffer — a live signal that reacts
+// to what fetching just found, unlike the processing EMA which only moves as
+// batches commit.
+let readyBufferDensity = (cs: t) => {
+  let readyCount = cs.fetchState->FetchState.bufferReadyCount
+  let span = cs.fetchState->FetchState.bufferBlockNumber - cs.processingBlockNumber
+  if readyCount > 0 && span > 0 {
+    Some(readyCount->Int.toFloat /. span->Int.toFloat)
+  } else {
+    None
   }
 }
 
+// Density used for query sizing: the higher of the processing EMA and the
+// ready-buffer density, so a stale-low EMA can't undersize queries while the
+// buffer proves the range is dense. None means the chain is cold — no density
+// signal at all.
+let effectiveDensity = (cs: t) =>
+  switch (cs.chainDensity, cs->readyBufferDensity) {
+  | (Some(ema), Some(buffer)) => Some(Pervasives.max(ema, buffer))
+  | (Some(_) as density, None) | (None, Some(_) as density) => density
+  | (None, None) => None
+  }
+
+// How far past the frontier a chain with no density signal targets while it
+// takes its first measurements.
+let coldTargetRange = 20_000
+
 // This chain's share of the indexer-wide buffer budget turned into a soft
-// target block: via chain density, or the fetch ceiling when density isn't
-// known yet.
+// target block: via density, or frontier + coldTargetRange when there's no
+// density signal yet.
 let targetBlock = (cs: t, ~chainTargetItems: float) => {
   let fetchState = cs.fetchState
   let fetchCeiling = cs->fetchCeiling
   let bufferBlockNumber = fetchState->FetchState.bufferBlockNumber
-  switch cs.chainDensity {
+  switch cs->effectiveDensity {
   | Some(density) if density > 0. =>
     Pervasives.min(
       fetchCeiling,
       bufferBlockNumber + Math.ceil(chainTargetItems /. density)->Float.toInt,
     )
-  | _ => fetchCeiling
+  | _ => Pervasives.min(bufferBlockNumber + coldTargetRange, fetchCeiling)
   }
 }
 
 // Block range that cross-chain progress alignment maps fractions over: from
-// the first block that can hold this chain's events to the last block it will
-// fetch.
+// the chain's startBlock to the last block it can fetch right now. The lower
+// bound is deliberately static — anchoring it at firstEventBlock would make
+// two chains sitting at the same block read different progress fractions
+// depending on whether each has discovered its first event yet, so the
+// anchor's line could map below a follower's own frontier and stall it.
 let progressRange = (cs: t) => {
   let fetchState = cs.fetchState
-  let lower = fetchState.firstEventBlock->Option.getOr(fetchState.startBlock)
-  let upper = switch fetchState.endBlock {
-  | Some(endBlock) => endBlock
-  | None => fetchState.knownHeight
-  }
-  (lower, upper)
+  (fetchState.startBlock, cs->fetchCeiling)
 }
 
 // A degenerate range (chain already at or past its last block) maps to 1 so it
@@ -442,13 +498,25 @@ let blockAtProgress = (cs: t, ~progress) => {
   lower + Math.ceil(progress *. (upper - lower)->Int.toFloat)->Float.toInt
 }
 
+// The fetch frontier as a fraction of the alignable range — what the
+// cross-chain waterfall aligns other chains against. Based on blocks actually
+// fetched, so it holds up even while this chain's queries are in flight.
+let frontierProgress = (cs: t) =>
+  cs->progressAtBlock(~blockNumber=cs.fetchState->FetchState.bufferBlockNumber)
+
 // Propose queries sized against this chain's target block. Called by
 // CrossChainState's waterfall, furthest-behind chain first, with
 // chainTargetItems set to whatever budget remains at that point and
 // maxTargetBlock set to the most-behind chain's progress mapped onto this
 // chain, so a chain with budget can't run further ahead than the chain the
 // whole pool is prioritizing.
-let getNextQuery = (cs: t, ~chainTargetItems: float, ~maxTargetBlock=?) => {
+let getNextQuery = (
+  cs: t,
+  ~chainTargetItems: float,
+  ~chunkItemsMultiplier=1.,
+  ~itemsTargetFloor=0,
+  ~maxTargetBlock=?,
+) => {
   let chainTargetBlock = cs->targetBlock(~chainTargetItems)
   let chainTargetBlock = switch maxTargetBlock {
   | Some(maxTargetBlock) => Pervasives.min(chainTargetBlock, maxTargetBlock)
@@ -460,25 +528,25 @@ let getNextQuery = (cs: t, ~chainTargetItems: float, ~maxTargetBlock=?) => {
   // top: they're already accounted and shouldn't crowd out new partitions), so
   // the waterfall's leftover flows to the next chain in the same tick instead
   // of being held by an oversized probe.
-  let chainTargetItems = switch cs.chainDensity {
+  let chainTargetItems = switch cs->effectiveDensity {
   | Some(density) if density > 0. =>
+    // No extra headroom here: the budget is reserved in honest itemsEst units,
+    // and truncation safety lives in the itemsTarget server cap (sized with
+    // chunkItemsMultiplier at query creation) — multiplying the budget cap too
+    // would compound the two and hold budget away from other chains.
     let rangeCost =
       density *. (chainTargetBlock - cs.fetchState->FetchState.bufferBlockNumber)->Int.toFloat
-    // 3x headroom only for a chain already caught up once and polling the
-    // head: there a single query covers the whole remaining range, and a
-    // slightly denser-than-expected range would otherwise truncate at the
-    // server cap and force an immediate catch-up query for the last blocks.
-    // During backfill the range never fits one query anyway, so headroom
-    // would just hold budget away from other chains.
-    let rangeCost =
-      chainTargetBlock >= cs->fetchCeiling && cs->isReady ? rangeCost *. 3. : rangeCost
     Pervasives.min(chainTargetItems, Math.ceil(rangeCost) +. cs.pendingBudget)
-  | _ =>
-    // No density signal yet: bound the probe so one unknown chain doesn't
-    // hold the whole pool while it takes its first measurements.
-    Pervasives.min(chainTargetItems, 5_000. +. cs.pendingBudget)
+  // No density signal: the cross-chain waterfall already clamped the handed
+  // budget to the cold-chain cap, so it's used as-is.
+  | _ => chainTargetItems
   }
-  cs.fetchState->FetchState.getNextQuery(~chainTargetBlock, ~chainTargetItems)
+  cs.fetchState->FetchState.getNextQuery(
+    ~chainTargetBlock,
+    ~chainTargetItems,
+    ~chunkItemsMultiplier,
+    ~itemsTargetFloor,
+  )
 }
 
 // Run a fetch tick for this chain against its sources, feeding the owned fetch
@@ -712,6 +780,9 @@ let materializePageItems = async (
   ))
 }
 
+let filterByClientAddress = (cs: t, items: array<Internal.item>): array<Internal.item> =>
+  items->FetchState.filterByClientAddress(~indexingAddresses=cs.indexingAddresses)
+
 let handleQueryResult = (
   cs: t,
   ~query: FetchState.query,
@@ -744,16 +815,11 @@ let handleQueryResult = (
 
   cs.fetchState =
     fs
-    ->FetchState.handleQueryResult(
-      ~indexingAddresses=cs.indexingAddresses,
-      ~query,
-      ~latestFetchedBlock,
-      ~newItems,
-    )
+    ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
     ->FetchState.updateKnownHeight(~knownHeight)
 
   // The query is no longer in flight, so release its reservation.
-  cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsTarget->Int.toFloat)
+  cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsEst->Int.toFloat)
 }
 
 // Run reorg detection against a fetch response and commit the updated guard.
@@ -854,6 +920,10 @@ let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
     cs.fetchState = enteringReorgThreshold
       ? chainAfterBatch.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
       : chainAfterBatch.fetchState
+
+    // The batch's items just left the buffer, so the remaining buffer's span
+    // starts at the batch's progress.
+    cs.processingBlockNumber = chainAfterBatch.progressBlockNumber
   | None => ()
   }
 
@@ -940,6 +1010,13 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
       }
 
       cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber
+
+      // Normally already set by advanceAfterBatch at batch creation; catch up
+      // here for paths that commit progress without it.
+      cs.processingBlockNumber = Pervasives.max(
+        cs.processingBlockNumber,
+        chainAfterBatch.progressBlockNumber,
+      )
       cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed
       // Processed blocks' transactions and blocks are no longer needed.
       cs.transactionStore->TransactionStore.prune(chainAfterBatch.progressBlockNumber)
@@ -1024,6 +1101,7 @@ let rollback = (
     cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
     cs.blockStore->BlockStore.rollback(newProgressBlockNumber)
     cs.committedProgressBlockNumber = newProgressBlockNumber
+    cs.processingBlockNumber = newProgressBlockNumber
     cs.numEventsProcessed = newTotalEventsProcessed
   | None =>
     if isReorgChain {
@@ -1042,6 +1120,7 @@ let rollback = (
         cs.committedProgressBlockNumber,
         rollbackTargetBlockNumber,
       )
+      cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, rollbackTargetBlockNumber)
     }
   }
 }

package/src/ChainState.res.mjs

@@ -40,7 +40,16 @@ function configAddresses(chainConfig) {
   return addresses;
 }
 
-function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, logger) {
+function validateOnEventRegistrations(chainId, registrations) {
+  registrations.forEach((registration, expectedIndex) => {
+    if (registration.index !== expectedIndex) {
+      return Stdlib_JsError.throwWithMessage(`Invalid onEvent registration index for chain ` + chainId.toString() + `: ` + registration.eventConfig.contractName + `.` + registration.eventConfig.name + ` has index ` + registration.index.toString() + `, but its ChainState position is ` + expectedIndex.toString() + `.`);
+    }
+  });
+}
+
+function make(chainConfig, fetchState, onEventRegistrationsOpt, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, logger) {
+  let onEventRegistrations = onEventRegistrationsOpt !== undefined ? onEventRegistrationsOpt : [];
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
   let timestampCaughtUpToHeadOrEndblock = timestampCaughtUpToHeadOrEndblockOpt !== undefined ? Primitive_option.valFromOption(timestampCaughtUpToHeadOrEndblockOpt) : undefined;
@@ -48,8 +57,10 @@ function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDe
   let transactionStore = transactionStoreOpt !== undefined ? Primitive_option.valFromOption(transactionStoreOpt) : TransactionStore.make("evm", false);
   let chainDensity = chainDensityOpt !== undefined ? Primitive_option.valFromOption(chainDensityOpt) : undefined;
   let blockStore = blockStoreOpt !== undefined ? Primitive_option.valFromOption(blockStoreOpt) : BlockStore.make("evm", false);
+  validateOnEventRegistrations(chainConfig.id, onEventRegistrations);
   return {
     logger: logger,
+    onEventRegistrations: onEventRegistrations,
     fetchState: fetchState,
     indexingAddresses: indexingAddresses,
     sourceManager: sourceManager,
@@ -57,6 +68,7 @@ function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDe
     isProgressAtHead: isProgressAtHead,
     timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
     committedProgressBlockNumber: committedProgressBlockNumber,
+    processingBlockNumber: committedProgressBlockNumber,
     numEventsProcessed: numEventsProcessed,
     pendingBudget: 0,
     chainDensity: chainDensity,
@@ -135,7 +147,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
   }
   let firstEventBlock$1 = fetchState.firstEventBlock;
   let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
-  return make(chainConfig, fetchState, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), logger);
+  return make(chainConfig, fetchState, onEventRegistrations, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), logger);
 }
 
 function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight) {
@@ -233,7 +245,7 @@ function isReadyToEnterReorgThreshold(cs) {
 
 function startFetchingQueries(cs, queries) {
   FetchState.startFetchingQueries(cs.fetchState, queries);
-  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsTarget);
+  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsEst);
 }
 
 function isReady(cs) {
@@ -242,11 +254,36 @@ function isReady(cs) {
 
 function fetchCeiling(cs) {
   let fetchState = cs.fetchState;
+  let head = Primitive_int.max(0, fetchState.knownHeight - fetchState.blockLag | 0);
   let endBlock = fetchState.endBlock;
   if (endBlock !== undefined) {
-    return Primitive_int.min(endBlock, fetchState.knownHeight);
+    return Primitive_int.min(endBlock, head);
+  } else {
+    return head;
+  }
+}
+
+function readyBufferDensity(cs) {
+  let readyCount = FetchState.bufferReadyCount(cs.fetchState);
+  let span = FetchState.bufferBlockNumber(cs.fetchState) - cs.processingBlockNumber | 0;
+  if (readyCount > 0 && span > 0) {
+    return readyCount / span;
+  }
+}
+
+function effectiveDensity(cs) {
+  let match = cs.chainDensity;
+  let match$1 = readyBufferDensity(cs);
+  if (match !== undefined) {
+    if (match$1 !== undefined) {
+      return Primitive_float.max(match, match$1);
+    } else {
+      return match;
+    }
+  } else if (match$1 !== undefined) {
+    return match$1;
   } else {
-    return fetchState.knownHeight;
+    return;
   }
 }
 
@@ -254,26 +291,30 @@ function targetBlock(cs, chainTargetItems) {
   let fetchState = cs.fetchState;
   let fetchCeiling$1 = fetchCeiling(cs);
   let bufferBlockNumber = FetchState.bufferBlockNumber(fetchState);
-  let density = cs.chainDensity;
+  let density = effectiveDensity(cs);
   if (density !== undefined && density > 0) {
     return Primitive_int.min(fetchCeiling$1, bufferBlockNumber + (Math.ceil(chainTargetItems / density) | 0) | 0);
   } else {
-    return fetchCeiling$1;
+    return Primitive_int.min(bufferBlockNumber + 20000 | 0, fetchCeiling$1);
   }
 }
 
 function progressRange(cs) {
   let fetchState = cs.fetchState;
-  let lower = Stdlib_Option.getOr(fetchState.firstEventBlock, fetchState.startBlock);
-  let endBlock = fetchState.endBlock;
-  let upper = endBlock !== undefined ? endBlock : fetchState.knownHeight;
   return [
-    lower,
-    upper
+    fetchState.startBlock,
+    fetchCeiling(cs)
   ];
 }
 
-function progressAtBlock(cs, blockNumber) {
+function blockAtProgress(cs, progress) {
+  let match = progressRange(cs);
+  let lower = match[0];
+  return lower + (Math.ceil(progress * (match[1] - lower | 0)) | 0) | 0;
+}
+
+function frontierProgress(cs) {
+  let blockNumber = FetchState.bufferBlockNumber(cs.fetchState);
   let match = progressRange(cs);
   let upper = match[1];
   let lower = match[0];
@@ -284,29 +325,20 @@ function progressAtBlock(cs, blockNumber) {
   }
 }
 
-function blockAtProgress(cs, progress) {
-  let match = progressRange(cs);
-  let lower = match[0];
-  return lower + (Math.ceil(progress * (match[1] - lower | 0)) | 0) | 0;
-}
-
-function getNextQuery(cs, chainTargetItems, maxTargetBlock) {
+function getNextQuery(cs, chainTargetItems, chunkItemsMultiplierOpt, itemsTargetFloorOpt, maxTargetBlock) {
+  let chunkItemsMultiplier = chunkItemsMultiplierOpt !== undefined ? chunkItemsMultiplierOpt : 1;
+  let itemsTargetFloor = itemsTargetFloorOpt !== undefined ? itemsTargetFloorOpt : 0;
   let chainTargetBlock = targetBlock(cs, chainTargetItems);
   let chainTargetBlock$1 = maxTargetBlock !== undefined ? Primitive_int.min(chainTargetBlock, maxTargetBlock) : chainTargetBlock;
-  let density = cs.chainDensity;
+  let density = effectiveDensity(cs);
   let chainTargetItems$1;
-  if (density !== undefined) {
-    if (density > 0) {
-      let rangeCost = density * (chainTargetBlock$1 - FetchState.bufferBlockNumber(cs.fetchState) | 0);
-      let rangeCost$1 = chainTargetBlock$1 >= fetchCeiling(cs) && cs.timestampCaughtUpToHeadOrEndblock !== undefined ? rangeCost * 3 : rangeCost;
-      chainTargetItems$1 = Primitive_float.min(chainTargetItems, Math.ceil(rangeCost$1) + cs.pendingBudget);
-    } else {
-      chainTargetItems$1 = Primitive_float.min(chainTargetItems, 5000 + cs.pendingBudget);
-    }
+  if (density !== undefined && density > 0) {
+    let rangeCost = density * (chainTargetBlock$1 - FetchState.bufferBlockNumber(cs.fetchState) | 0);
+    chainTargetItems$1 = Primitive_float.min(chainTargetItems, Math.ceil(rangeCost) + cs.pendingBudget);
   } else {
-    chainTargetItems$1 = Primitive_float.min(chainTargetItems, 5000 + cs.pendingBudget);
+    chainTargetItems$1 = chainTargetItems;
   }
-  return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1);
+  return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1, chunkItemsMultiplier, itemsTargetFloor);
 }
 
 function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
@@ -477,6 +509,10 @@ async function materializePageItems(items, transactionStore, blockStore, ecosyst
   ]);
 }
 
+function filterByClientAddress(cs, items) {
+  return FetchState.filterByClientAddress(items, cs.indexingAddresses);
+}
+
 function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, txPage, blockPage) {
   if (txPage !== undefined) {
     cs.transactionStore.merge(Primitive_option.valFromOption(txPage));
@@ -485,8 +521,8 @@ function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBl
     cs.blockStore.merge(Primitive_option.valFromOption(blockPage));
   }
   let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.indexingAddresses, newItemsWithDcs) : cs.fetchState;
-  cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, cs.indexingAddresses, query, latestFetchedBlock, newItems), knownHeight);
-  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsTarget);
+  cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
+  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsEst);
 }
 
 function registerReorgGuard(cs, blockHashes, knownHeight) {
@@ -550,7 +586,7 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
 }
 
 function enterReorgThreshold(cs) {
-  cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined, cs.chainConfig.blockLag, undefined);
+  cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined, undefined, cs.chainConfig.blockLag, undefined);
 }
 
 function toChainMetadata(cs) {
@@ -598,7 +634,8 @@ function isReadyToEnterReorgThresholdAfterBatch(cs, batch) {
 function advanceAfterBatch(cs, batch, enteringReorgThreshold) {
   let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
   if (chainAfterBatch !== undefined) {
-    cs.fetchState = enteringReorgThreshold ? FetchState.updateInternal(chainAfterBatch.fetchState, undefined, undefined, cs.chainConfig.blockLag, undefined) : chainAfterBatch.fetchState;
+    cs.fetchState = enteringReorgThreshold ? FetchState.updateInternal(chainAfterBatch.fetchState, undefined, undefined, undefined, cs.chainConfig.blockLag, undefined) : chainAfterBatch.fetchState;
+    cs.processingBlockNumber = chainAfterBatch.progressBlockNumber;
     return;
   }
 }
@@ -665,6 +702,7 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
     }
   }
   cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber;
+  cs.processingBlockNumber = Primitive_int.max(cs.processingBlockNumber, chainAfterBatch.progressBlockNumber);
   cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed;
   cs.transactionStore.prune(chainAfterBatch.progressBlockNumber);
   cs.blockStore.prune(chainAfterBatch.progressBlockNumber);
@@ -692,6 +730,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
       cs.transactionStore.rollback(rollbackTargetBlockNumber);
       cs.blockStore.rollback(rollbackTargetBlockNumber);
       cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
+      cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, rollbackTargetBlockNumber);
       return;
     } else {
       return;
@@ -715,6 +754,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
   cs.transactionStore.rollback(newProgressBlockNumber);
   cs.blockStore.rollback(newProgressBlockNumber);
   cs.committedProgressBlockNumber = newProgressBlockNumber;
+  cs.processingBlockNumber = newProgressBlockNumber;
   cs.numEventsProcessed = newTotalEventsProcessed;
 }
 
@@ -740,11 +780,12 @@ export {
   bufferReadyCount,
   getProgressPercentage,
   chainDensity,
+  effectiveDensity,
   hasReadyItem,
   isReadyToEnterReorgThreshold,
   targetBlock,
-  progressAtBlock,
   blockAtProgress,
+  frontierProgress,
   getNextQuery,
   dispatch,
   toChainData,
@@ -757,6 +798,7 @@ export {
   isReady,
   isFetchingAtHead,
   isAtHeadWithoutEndBlock,
+  filterByClientAddress,
   handleQueryResult,
   materializeBatchItems,
   materializePageItems,

package/src/ChainState.resi

@@ -8,6 +8,7 @@ let configAddresses: Config.chain => array<Internal.indexingAddress>
 let make: (
   ~chainConfig: Config.chain,
   ~fetchState: FetchState.t,
+  ~onEventRegistrations: array<Internal.onEventRegistration>=?,
   ~indexingAddresses: IndexingAddresses.t,
   ~sourceManager: SourceManager.t,
   ~reorgDetection: ReorgDetection.t,
@@ -60,14 +61,21 @@ let bufferSize: t => int
 let bufferReadyCount: t => int
 let getProgressPercentage: t => float
 let chainDensity: t => option<float>
+let effectiveDensity: t => option<float>
 let hasReadyItem: t => bool
 let isReadyToEnterReorgThreshold: t => bool
 
 // Fetch control.
 let targetBlock: (t, ~chainTargetItems: float) => int
-let progressAtBlock: (t, ~blockNumber: int) => float
 let blockAtProgress: (t, ~progress: float) => int
-let getNextQuery: (t, ~chainTargetItems: float, ~maxTargetBlock: int=?) => FetchState.nextQuery
+let frontierProgress: t => float
+let getNextQuery: (
+  t,
+  ~chainTargetItems: float,
+  ~chunkItemsMultiplier: float=?,
+  ~itemsTargetFloor: int=?,
+  ~maxTargetBlock: int=?,
+) => FetchState.nextQuery
 let dispatch: (
   t,
   ~executeQuery: FetchState.query => promise<unit>,
@@ -106,6 +114,10 @@ let isReady: t => bool
 let isFetchingAtHead: t => bool
 let isAtHeadWithoutEndBlock: t => bool
 
+// Drop over-fetched events an address-param filter rejects, before contract
+// registration runs on them. Uses the chain's current indexing addresses.
+let filterByClientAddress: (t, array<Internal.item>) => array<Internal.item>
+
 // State transitions. The chain state is mutated only through these.
 let handleQueryResult: (
   t,

package/src/Config.res

@@ -200,14 +200,18 @@ let svmEventDescriptorSchema = S.schema(s =>
     "transactionFields": s.matches(S.array(Internal.svmTransactionFieldSchema)),
     "blockFields": s.matches(S.option(S.array(Internal.svmBlockFieldSchema))),
     "includeLogs": s.matches(S.bool),
+    // An array of AND-groups OR-ed together: the CLI normalizes both the flat
+    // and `any_of` YAML shapes to `Vec<Vec<SvmAccountFilterJson>>`.
     "accountFilters": s.matches(
       S.option(
         S.array(
-          S.schema(s =>
-            {
-              "position": s.matches(S.int),
-              "values": s.matches(S.array(S.string)),
-            }
+          S.array(
+            S.schema(s =>
+              {
+                "position": s.matches(S.int),
+                "values": s.matches(S.array(S.string)),
+              }
+            ),
           ),
         ),
       ),

package/src/Config.res.mjs

@@ -115,10 +115,10 @@ let svmEventDescriptorSchema = S$RescriptSchema.schema(s => ({
   transactionFields: s.m(S$RescriptSchema.array(Internal.svmTransactionFieldSchema)),
   blockFields: s.m(S$RescriptSchema.option(S$RescriptSchema.array(Internal.svmBlockFieldSchema))),
   includeLogs: s.m(S$RescriptSchema.bool),
-  accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
+  accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
     position: s.m(S$RescriptSchema.int),
     values: s.m(S$RescriptSchema.array(S$RescriptSchema.string))
-  }))))),
+  })))))),
   isInner: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
   accounts: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.string))),
   args: s.m(S$RescriptSchema.option(S$RescriptSchema.json(false)))