envio 3.3.0-alpha.8 → 3.3.0-rc.0

package/src/ChainState.res

@@ -4,6 +4,8 @@
 
 type t = {
   logger: Pino.t,
+  // The registrations used to build this chain's sources and route native items.
+  onEventRegistrations: array<Internal.onEventRegistration>,
   mutable fetchState: FetchState.t,
   // The chain-wide address index. Not `mutable`: the dict is mutated in place by
   // register/rollback, so the reference is stable across fetchState versions.
@@ -13,8 +15,13 @@ type t = {
   mutable isProgressAtHead: bool,
   mutable timestampCaughtUpToHeadOrEndblock: option<Date.t>,
   mutable committedProgressBlockNumber: int,
+  // Progress of the batch currently being processed. The buffer is consumed at
+  // batch creation (see advanceAfterBatch), so this runs ahead of
+  // committedProgressBlockNumber until the batch commits — it's the true lower
+  // boundary of the remaining buffer's block span.
+  mutable processingBlockNumber: int,
   mutable numEventsProcessed: float,
-  // Running sum of in-flight queries' itemsTarget, kept here so the
+  // Running sum of in-flight queries' itemsEst, kept here so the
   // scheduler doesn't re-sum pending queries on every tick. Incremented when
   // queries are dispatched, decremented as their responses land.
   mutable pendingBudget: float,
@@ -66,9 +73,22 @@ let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddres
   addresses
 }
 
+let validateOnEventRegistrations = (
+  ~chainId: int,
+  registrations: array<Internal.onEventRegistration>,
+) =>
+  registrations->Array.forEachWithIndex((registration, expectedIndex) => {
+    if registration.index !== expectedIndex {
+      JsError.throwWithMessage(
+        `Invalid onEvent registration index for chain ${chainId->Int.toString}: ${registration.eventConfig.contractName}.${registration.eventConfig.name} has index ${registration.index->Int.toString}, but its ChainState position is ${expectedIndex->Int.toString}.`,
+      )
+    }
+  })
+
 let make = (
   ~chainConfig: Config.chain,
   ~fetchState: FetchState.t,
+  ~onEventRegistrations=[],
   ~indexingAddresses: IndexingAddresses.t,
   ~sourceManager: SourceManager.t,
   ~reorgDetection: ReorgDetection.t,
@@ -82,21 +102,26 @@ let make = (
   ~blockStore=BlockStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
   ~logger: Pino.t,
 ): t => {
-  logger,
-  fetchState,
-  indexingAddresses,
-  sourceManager,
-  chainConfig,
-  isProgressAtHead,
-  timestampCaughtUpToHeadOrEndblock,
-  committedProgressBlockNumber,
-  numEventsProcessed,
-  pendingBudget: 0.,
-  chainDensity,
-  reorgDetection,
-  safeCheckpointTracking,
-  transactionStore,
-  blockStore,
+  validateOnEventRegistrations(~chainId=chainConfig.id, onEventRegistrations)
+  {
+    logger,
+    onEventRegistrations,
+    fetchState,
+    indexingAddresses,
+    sourceManager,
+    chainConfig,
+    isProgressAtHead,
+    timestampCaughtUpToHeadOrEndblock,
+    committedProgressBlockNumber,
+    processingBlockNumber: committedProgressBlockNumber,
+    numEventsProcessed,
+    pendingBudget: 0.,
+    chainDensity,
+    reorgDetection,
+    safeCheckpointTracking,
+    transactionStore,
+    blockStore,
+  }
 }
 
 let makeInternal = (
@@ -136,25 +161,6 @@ let makeInternal = (
     }
   })
 
-  // TODO: Move it to the HandlerRegister module
-  // so the error is thrown with better stack trace
-  onBlockRegistrations->Array.forEach(onBlockRegistration => {
-    if onBlockRegistration.startBlock->Option.getOr(startBlock) < startBlock {
-      JsError.throwWithMessage(
-        `The start block for onBlock handler "${onBlockRegistration.name}" is less than the chain start block (${startBlock->Int.toString}). This is not supported yet.`,
-      )
-    }
-    switch endBlock {
-    | Some(chainEndBlock) =>
-      if onBlockRegistration.endBlock->Option.getOr(chainEndBlock) > chainEndBlock {
-        JsError.throwWithMessage(
-          `The end block for onBlock handler "${onBlockRegistration.name}" is greater than the chain end block (${chainEndBlock->Int.toString}). This is not supported yet.`,
-        )
-      }
-    | None => ()
-    }
-  })
-
   let contractConfigs = IndexingAddresses.makeContractConfigs(~onEventRegistrations)
   let indexingAddressIndex = IndexingAddresses.make(~contractConfigs, ~addresses=indexingAddresses)
 
@@ -251,6 +257,7 @@ let makeInternal = (
   make(
     ~chainConfig,
     ~fetchState,
+    ~onEventRegistrations,
     ~indexingAddresses=indexingAddressIndex,
     ~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
     ~reorgDetection=ReorgDetection.make(
@@ -385,7 +392,8 @@ let isReadyToEnterReorgThreshold = (cs: t) => cs.fetchState->FetchState.isReadyT
 let startFetchingQueries = (cs: t, ~queries: array<FetchState.query>) => {
   cs.fetchState->FetchState.startFetchingQueries(~queries)
   cs.pendingBudget =
-    cs.pendingBudget +. queries->Array.reduce(0., (acc, query) => acc +. query.itemsTarget)
+    cs.pendingBudget +.
+    queries->Array.reduce(0., (acc, query) => acc +. query.itemsEst->Int.toFloat)
 }
 
 // Drop every in-flight query and release their reservations together, keeping
@@ -395,24 +403,150 @@ let resetPendingQueries = (cs: t) => {
   cs.pendingBudget = 0.
 }
 
-// Turn this chain's share of the indexer-wide buffer budget into a soft
-// target block (via chain density, or knownHeight when density isn't known
-// yet), then propose queries sized against it. Called by CrossChainState's
-// waterfall, furthest-behind chain first, with chainTargetItems set to
-// whatever budget remains at that point in the waterfall.
-let getNextQuery = (cs: t, ~chainTargetItems: float) => {
+let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
+
+// Block span over which a batch fully replaces the chain density estimate;
+// smaller batches blend in proportionally, so a few sparse/dense blocks only
+// nudge it.
+let densityBlendWindow = 100.
+
+// The last block this chain can fetch right now: the lagged head
+// (knownHeight - blockLag, the frontier when a reorg-threshold blockLag holds
+// back the tip), or endBlock when it's below that head. Matching isFetchingAtHead's
+// definition of the head is what lets a chain parked at its lagged head read
+// 100% progress instead of looking behind against blocks it can't fetch.
+let fetchCeiling = (cs: t) => {
+  let fetchState = cs.fetchState
+  let head = Pervasives.max(0, fetchState.knownHeight - fetchState.blockLag)
+  switch fetchState.endBlock {
+  | Some(endBlock) => Pervasives.min(endBlock, head)
+  | None => head
+  }
+}
+
+// Events/block over the ready part of the buffer — a live signal that reacts
+// to what fetching just found, unlike the processing EMA which only moves as
+// batches commit.
+let readyBufferDensity = (cs: t) => {
+  let readyCount = cs.fetchState->FetchState.bufferReadyCount
+  let span = cs.fetchState->FetchState.bufferBlockNumber - cs.processingBlockNumber
+  if readyCount > 0 && span > 0 {
+    Some(readyCount->Int.toFloat /. span->Int.toFloat)
+  } else {
+    None
+  }
+}
+
+// Density used for query sizing: the higher of the processing EMA and the
+// ready-buffer density, so a stale-low EMA can't undersize queries while the
+// buffer proves the range is dense. None means the chain is cold — no density
+// signal at all.
+let effectiveDensity = (cs: t) =>
+  switch (cs.chainDensity, cs->readyBufferDensity) {
+  | (Some(ema), Some(buffer)) => Some(Pervasives.max(ema, buffer))
+  | (Some(_) as density, None) | (None, Some(_) as density) => density
+  | (None, None) => None
+  }
+
+// How far past the frontier a chain with no density signal targets while it
+// takes its first measurements.
+let coldTargetRange = 20_000
+
+// This chain's share of the indexer-wide buffer budget turned into a soft
+// target block: via density, or frontier + coldTargetRange when there's no
+// density signal yet.
+let targetBlock = (cs: t, ~chainTargetItems: float) => {
   let fetchState = cs.fetchState
-  let knownHeight = fetchState.knownHeight
+  let fetchCeiling = cs->fetchCeiling
   let bufferBlockNumber = fetchState->FetchState.bufferBlockNumber
-  let chainTargetBlock = switch cs.chainDensity {
+  switch cs->effectiveDensity {
   | Some(density) if density > 0. =>
     Pervasives.min(
-      knownHeight,
+      fetchCeiling,
       bufferBlockNumber + Math.ceil(chainTargetItems /. density)->Float.toInt,
     )
-  | _ => knownHeight
+  | _ => Pervasives.min(bufferBlockNumber + coldTargetRange, fetchCeiling)
   }
-  fetchState->FetchState.getNextQuery(~chainTargetBlock, ~chainTargetItems)
+}
+
+// Block range that cross-chain progress alignment maps fractions over: from
+// the chain's startBlock to the last block it can fetch right now. The lower
+// bound is deliberately static — anchoring it at firstEventBlock would make
+// two chains sitting at the same block read different progress fractions
+// depending on whether each has discovered its first event yet, so the
+// anchor's line could map below a follower's own frontier and stall it.
+let progressRange = (cs: t) => {
+  let fetchState = cs.fetchState
+  (fetchState.startBlock, cs->fetchCeiling)
+}
+
+// A degenerate range (chain already at or past its last block) maps to 1 so it
+// never constrains the other chains. Clamped at 0 for the initial -1 fetch
+// frontier — the only possible blockNumber below the range's lower bound.
+let progressAtBlock = (cs: t, ~blockNumber) => {
+  let (lower, upper) = cs->progressRange
+  upper <= lower
+    ? 1.
+    : Pervasives.max(
+        0.,
+        Pervasives.min(1., (blockNumber - lower)->Int.toFloat /. (upper - lower)->Int.toFloat),
+      )
+}
+
+let blockAtProgress = (cs: t, ~progress) => {
+  let (lower, upper) = cs->progressRange
+  lower + Math.ceil(progress *. (upper - lower)->Int.toFloat)->Float.toInt
+}
+
+// The fetch frontier as a fraction of the alignable range — what the
+// cross-chain waterfall aligns other chains against. Based on blocks actually
+// fetched, so it holds up even while this chain's queries are in flight.
+let frontierProgress = (cs: t) =>
+  cs->progressAtBlock(~blockNumber=cs.fetchState->FetchState.bufferBlockNumber)
+
+// Propose queries sized against this chain's target block. Called by
+// CrossChainState's waterfall, furthest-behind chain first, with
+// chainTargetItems set to whatever budget remains at that point and
+// maxTargetBlock set to the most-behind chain's progress mapped onto this
+// chain, so a chain with budget can't run further ahead than the chain the
+// whole pool is prioritizing.
+let getNextQuery = (
+  cs: t,
+  ~chainTargetItems: float,
+  ~chunkItemsMultiplier=1.,
+  ~itemsTargetFloor=0,
+  ~maxTargetBlock=?,
+) => {
+  let chainTargetBlock = cs->targetBlock(~chainTargetItems)
+  let chainTargetBlock = switch maxTargetBlock {
+  | Some(maxTargetBlock) => Pervasives.min(chainTargetBlock, maxTargetBlock)
+  | None => chainTargetBlock
+  }
+  // When the target block is clamped (head/endBlock/cross-chain alignment) a
+  // known-density chain can't use the whole handed budget — cap the fresh part
+  // at what the clamped range actually costs (in-flight reservations stay on
+  // top: they're already accounted and shouldn't crowd out new partitions), so
+  // the waterfall's leftover flows to the next chain in the same tick instead
+  // of being held by an oversized probe.
+  let chainTargetItems = switch cs->effectiveDensity {
+  | Some(density) if density > 0. =>
+    // No extra headroom here: the budget is reserved in honest itemsEst units,
+    // and truncation safety lives in the itemsTarget server cap (sized with
+    // chunkItemsMultiplier at query creation) — multiplying the budget cap too
+    // would compound the two and hold budget away from other chains.
+    let rangeCost =
+      density *. (chainTargetBlock - cs.fetchState->FetchState.bufferBlockNumber)->Int.toFloat
+    Pervasives.min(chainTargetItems, Math.ceil(rangeCost) +. cs.pendingBudget)
+  // No density signal: the cross-chain waterfall already clamped the handed
+  // budget to the cold-chain cap, so it's used as-is.
+  | _ => chainTargetItems
+  }
+  cs.fetchState->FetchState.getNextQuery(
+    ~chainTargetBlock,
+    ~chainTargetItems,
+    ~chunkItemsMultiplier,
+    ~itemsTargetFloor,
+  )
 }
 
 // Run a fetch tick for this chain against its sources, feeding the owned fetch
@@ -451,8 +585,6 @@ let getHighestBlockBelowThreshold = (cs: t): int => {
 
 let isActivelyIndexing = (cs: t) => cs.fetchState->FetchState.isActivelyIndexing
 
-let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
-
 // True once the fetch frontier has reached the head/endBlock for this chain.
 let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
 
@@ -648,6 +780,9 @@ let materializePageItems = async (
   ))
 }
 
+let filterByClientAddress = (cs: t, items: array<Internal.item>): array<Internal.item> =>
+  items->FetchState.filterByClientAddress(~indexingAddresses=cs.indexingAddresses)
+
 let handleQueryResult = (
   cs: t,
   ~query: FetchState.query,
@@ -680,16 +815,11 @@ let handleQueryResult = (
 
   cs.fetchState =
     fs
-    ->FetchState.handleQueryResult(
-      ~indexingAddresses=cs.indexingAddresses,
-      ~query,
-      ~latestFetchedBlock,
-      ~newItems,
-    )
+    ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
     ->FetchState.updateKnownHeight(~knownHeight)
 
   // The query is no longer in flight, so release its reservation.
-  cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsTarget)
+  cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsEst->Int.toFloat)
 }
 
 // Run reorg detection against a fetch response and commit the updated guard.
@@ -790,6 +920,10 @@ let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
     cs.fetchState = enteringReorgThreshold
       ? chainAfterBatch.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
       : chainAfterBatch.fetchState
+
+    // The batch's items just left the buffer, so the remaining buffer's span
+    // starts at the batch's progress.
+    cs.processingBlockNumber = chainAfterBatch.progressBlockNumber
   | None => ()
   }
 
@@ -852,8 +986,9 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
       }
 
       // Chain-wide density update: seed with the batch's own events/block on
-      // the first update, then smooth incrementally so a run of a few
-      // sparse/dense blocks doesn't swing the target block estimate.
+      // the first update, then blend weighted by the batch's block span — a
+      // few sparse/dense blocks barely nudge the estimate, while a
+      // window-sized batch replaces it.
       let deltaBlocks = chainAfterBatch.progressBlockNumber - cs.committedProgressBlockNumber
       if deltaBlocks > 0 {
         let deltaEvents = chainAfterBatch.totalEventsProcessed -. cs.numEventsProcessed
@@ -866,13 +1001,22 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
           cs.chainDensity = Some(
             switch cs.chainDensity {
             | None => batchDensity
-            | Some(oldDensity) => (oldDensity +. batchDensity) /. 2.
+            | Some(oldDensity) =>
+              let alpha = Pervasives.min(1., deltaBlocks->Int.toFloat /. densityBlendWindow)
+              oldDensity *. (1. -. alpha) +. batchDensity *. alpha
             },
           )
         }
       }
 
       cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber
+
+      // Normally already set by advanceAfterBatch at batch creation; catch up
+      // here for paths that commit progress without it.
+      cs.processingBlockNumber = Pervasives.max(
+        cs.processingBlockNumber,
+        chainAfterBatch.progressBlockNumber,
+      )
       cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed
       // Processed blocks' transactions and blocks are no longer needed.
       cs.transactionStore->TransactionStore.prune(chainAfterBatch.progressBlockNumber)
@@ -957,6 +1101,7 @@ let rollback = (
     cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
     cs.blockStore->BlockStore.rollback(newProgressBlockNumber)
     cs.committedProgressBlockNumber = newProgressBlockNumber
+    cs.processingBlockNumber = newProgressBlockNumber
     cs.numEventsProcessed = newTotalEventsProcessed
   | None =>
     if isReorgChain {
@@ -975,6 +1120,7 @@ let rollback = (
         cs.committedProgressBlockNumber,
         rollbackTargetBlockNumber,
       )
+      cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, rollbackTargetBlockNumber)
     }
   }
 }

package/src/ChainState.res.mjs

@@ -40,7 +40,16 @@ function configAddresses(chainConfig) {
   return addresses;
 }
 
-function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, logger) {
+function validateOnEventRegistrations(chainId, registrations) {
+  registrations.forEach((registration, expectedIndex) => {
+    if (registration.index !== expectedIndex) {
+      return Stdlib_JsError.throwWithMessage(`Invalid onEvent registration index for chain ` + chainId.toString() + `: ` + registration.eventConfig.contractName + `.` + registration.eventConfig.name + ` has index ` + registration.index.toString() + `, but its ChainState position is ` + expectedIndex.toString() + `.`);
+    }
+  });
+}
+
+function make(chainConfig, fetchState, onEventRegistrationsOpt, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, logger) {
+  let onEventRegistrations = onEventRegistrationsOpt !== undefined ? onEventRegistrationsOpt : [];
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
   let timestampCaughtUpToHeadOrEndblock = timestampCaughtUpToHeadOrEndblockOpt !== undefined ? Primitive_option.valFromOption(timestampCaughtUpToHeadOrEndblockOpt) : undefined;
@@ -48,8 +57,10 @@ function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDe
   let transactionStore = transactionStoreOpt !== undefined ? Primitive_option.valFromOption(transactionStoreOpt) : TransactionStore.make("evm", false);
   let chainDensity = chainDensityOpt !== undefined ? Primitive_option.valFromOption(chainDensityOpt) : undefined;
   let blockStore = blockStoreOpt !== undefined ? Primitive_option.valFromOption(blockStoreOpt) : BlockStore.make("evm", false);
+  validateOnEventRegistrations(chainConfig.id, onEventRegistrations);
   return {
     logger: logger,
+    onEventRegistrations: onEventRegistrations,
     fetchState: fetchState,
     indexingAddresses: indexingAddresses,
     sourceManager: sourceManager,
@@ -57,6 +68,7 @@ function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDe
     isProgressAtHead: isProgressAtHead,
     timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
     committedProgressBlockNumber: committedProgressBlockNumber,
+    processingBlockNumber: committedProgressBlockNumber,
     numEventsProcessed: numEventsProcessed,
     pendingBudget: 0,
     chainDensity: chainDensity,
@@ -74,7 +86,6 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
     onEventRegistrations: [],
     onBlockRegistrations: []
   });
-  let onBlockRegistrations = match.onBlockRegistrations;
   let onEventRegistrations = match.onEventRegistrations;
   chainConfig.contracts.forEach(contract => {
     let startBlock = contract.startBlock;
@@ -82,17 +93,9 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
       return Stdlib_JsError.throwWithMessage(`The start block for contract "` + contract.name + `" is less than the chain start block. This is not supported yet.`);
     }
   });
-  onBlockRegistrations.forEach(onBlockRegistration => {
-    if (Stdlib_Option.getOr(onBlockRegistration.startBlock, startBlock) < startBlock) {
-      Stdlib_JsError.throwWithMessage(`The start block for onBlock handler "` + onBlockRegistration.name + `" is less than the chain start block (` + startBlock.toString() + `). This is not supported yet.`);
-    }
-    if (endBlock !== undefined && Stdlib_Option.getOr(onBlockRegistration.endBlock, endBlock) > endBlock) {
-      return Stdlib_JsError.throwWithMessage(`The end block for onBlock handler "` + onBlockRegistration.name + `" is greater than the chain end block (` + endBlock.toString() + `). This is not supported yet.`);
-    }
-  });
   let contractConfigs = IndexingAddresses.makeContractConfigs(onEventRegistrations);
   let indexingAddressIndex = IndexingAddresses.make(contractConfigs, indexingAddresses);
-  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
+  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
   let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
     if (reorgCheckpoint.chain_id === chainConfig.id) {
       return reorgCheckpoint;
@@ -144,7 +147,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
   }
   let firstEventBlock$1 = fetchState.firstEventBlock;
   let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
-  return make(chainConfig, fetchState, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), logger);
+  return make(chainConfig, fetchState, onEventRegistrations, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), logger);
 }
 
 function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight) {
@@ -242,16 +245,100 @@ function isReadyToEnterReorgThreshold(cs) {
 
 function startFetchingQueries(cs, queries) {
   FetchState.startFetchingQueries(cs.fetchState, queries);
-  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsTarget);
+  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsEst);
 }
 
-function getNextQuery(cs, chainTargetItems) {
+function isReady(cs) {
+  return cs.timestampCaughtUpToHeadOrEndblock !== undefined;
+}
+
+function fetchCeiling(cs) {
   let fetchState = cs.fetchState;
-  let knownHeight = fetchState.knownHeight;
+  let head = Primitive_int.max(0, fetchState.knownHeight - fetchState.blockLag | 0);
+  let endBlock = fetchState.endBlock;
+  if (endBlock !== undefined) {
+    return Primitive_int.min(endBlock, head);
+  } else {
+    return head;
+  }
+}
+
+function readyBufferDensity(cs) {
+  let readyCount = FetchState.bufferReadyCount(cs.fetchState);
+  let span = FetchState.bufferBlockNumber(cs.fetchState) - cs.processingBlockNumber | 0;
+  if (readyCount > 0 && span > 0) {
+    return readyCount / span;
+  }
+}
+
+function effectiveDensity(cs) {
+  let match = cs.chainDensity;
+  let match$1 = readyBufferDensity(cs);
+  if (match !== undefined) {
+    if (match$1 !== undefined) {
+      return Primitive_float.max(match, match$1);
+    } else {
+      return match;
+    }
+  } else if (match$1 !== undefined) {
+    return match$1;
+  } else {
+    return;
+  }
+}
+
+function targetBlock(cs, chainTargetItems) {
+  let fetchState = cs.fetchState;
+  let fetchCeiling$1 = fetchCeiling(cs);
   let bufferBlockNumber = FetchState.bufferBlockNumber(fetchState);
-  let density = cs.chainDensity;
-  let chainTargetBlock = density !== undefined && density > 0 ? Primitive_int.min(knownHeight, bufferBlockNumber + (Math.ceil(chainTargetItems / density) | 0) | 0) : knownHeight;
-  return FetchState.getNextQuery(fetchState, chainTargetBlock, chainTargetItems);
+  let density = effectiveDensity(cs);
+  if (density !== undefined && density > 0) {
+    return Primitive_int.min(fetchCeiling$1, bufferBlockNumber + (Math.ceil(chainTargetItems / density) | 0) | 0);
+  } else {
+    return Primitive_int.min(bufferBlockNumber + 20000 | 0, fetchCeiling$1);
+  }
+}
+
+function progressRange(cs) {
+  let fetchState = cs.fetchState;
+  return [
+    fetchState.startBlock,
+    fetchCeiling(cs)
+  ];
+}
+
+function blockAtProgress(cs, progress) {
+  let match = progressRange(cs);
+  let lower = match[0];
+  return lower + (Math.ceil(progress * (match[1] - lower | 0)) | 0) | 0;
+}
+
+function frontierProgress(cs) {
+  let blockNumber = FetchState.bufferBlockNumber(cs.fetchState);
+  let match = progressRange(cs);
+  let upper = match[1];
+  let lower = match[0];
+  if (upper <= lower) {
+    return 1;
+  } else {
+    return Primitive_float.max(0, Primitive_float.min(1, (blockNumber - lower | 0) / (upper - lower | 0)));
+  }
+}
+
+function getNextQuery(cs, chainTargetItems, chunkItemsMultiplierOpt, itemsTargetFloorOpt, maxTargetBlock) {
+  let chunkItemsMultiplier = chunkItemsMultiplierOpt !== undefined ? chunkItemsMultiplierOpt : 1;
+  let itemsTargetFloor = itemsTargetFloorOpt !== undefined ? itemsTargetFloorOpt : 0;
+  let chainTargetBlock = targetBlock(cs, chainTargetItems);
+  let chainTargetBlock$1 = maxTargetBlock !== undefined ? Primitive_int.min(chainTargetBlock, maxTargetBlock) : chainTargetBlock;
+  let density = effectiveDensity(cs);
+  let chainTargetItems$1;
+  if (density !== undefined && density > 0) {
+    let rangeCost = density * (chainTargetBlock$1 - FetchState.bufferBlockNumber(cs.fetchState) | 0);
+    chainTargetItems$1 = Primitive_float.min(chainTargetItems, Math.ceil(rangeCost) + cs.pendingBudget);
+  } else {
+    chainTargetItems$1 = chainTargetItems;
+  }
+  return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1, chunkItemsMultiplier, itemsTargetFloor);
 }
 
 function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
@@ -282,10 +369,6 @@ function isActivelyIndexing(cs) {
   return FetchState.isActivelyIndexing(cs.fetchState);
 }
 
-function isReady(cs) {
-  return cs.timestampCaughtUpToHeadOrEndblock !== undefined;
-}
-
 function isFetchingAtHead(cs) {
   return FetchState.isFetchingAtHead(cs.fetchState);
 }
@@ -426,6 +509,10 @@ async function materializePageItems(items, transactionStore, blockStore, ecosyst
   ]);
 }
 
+function filterByClientAddress(cs, items) {
+  return FetchState.filterByClientAddress(items, cs.indexingAddresses);
+}
+
 function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, txPage, blockPage) {
   if (txPage !== undefined) {
     cs.transactionStore.merge(Primitive_option.valFromOption(txPage));
@@ -434,8 +521,8 @@ function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBl
     cs.blockStore.merge(Primitive_option.valFromOption(blockPage));
   }
   let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.indexingAddresses, newItemsWithDcs) : cs.fetchState;
-  cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, cs.indexingAddresses, query, latestFetchedBlock, newItems), knownHeight);
-  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsTarget);
+  cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
+  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsEst);
 }
 
 function registerReorgGuard(cs, blockHashes, knownHeight) {
@@ -499,7 +586,7 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
 }
 
 function enterReorgThreshold(cs) {
-  cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined, cs.chainConfig.blockLag, undefined);
+  cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined, undefined, cs.chainConfig.blockLag, undefined);
 }
 
 function toChainMetadata(cs) {
@@ -547,7 +634,8 @@ function isReadyToEnterReorgThresholdAfterBatch(cs, batch) {
 function advanceAfterBatch(cs, batch, enteringReorgThreshold) {
   let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
   if (chainAfterBatch !== undefined) {
-    cs.fetchState = enteringReorgThreshold ? FetchState.updateInternal(chainAfterBatch.fetchState, undefined, undefined, cs.chainConfig.blockLag, undefined) : chainAfterBatch.fetchState;
+    cs.fetchState = enteringReorgThreshold ? FetchState.updateInternal(chainAfterBatch.fetchState, undefined, undefined, undefined, cs.chainConfig.blockLag, undefined) : chainAfterBatch.fetchState;
+    cs.processingBlockNumber = chainAfterBatch.progressBlockNumber;
     return;
   }
 }
@@ -603,10 +691,18 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
     if (exit === 1) {
       let batchDensity = deltaEvents / deltaBlocks;
       let oldDensity = cs.chainDensity;
-      cs.chainDensity = oldDensity !== undefined ? (oldDensity + batchDensity) / 2 : batchDensity;
+      let tmp;
+      if (oldDensity !== undefined) {
+        let alpha = Primitive_float.min(1, deltaBlocks / 100);
+        tmp = oldDensity * (1 - alpha) + batchDensity * alpha;
+      } else {
+        tmp = batchDensity;
+      }
+      cs.chainDensity = tmp;
     }
   }
   cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber;
+  cs.processingBlockNumber = Primitive_int.max(cs.processingBlockNumber, chainAfterBatch.progressBlockNumber);
   cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed;
   cs.transactionStore.prune(chainAfterBatch.progressBlockNumber);
   cs.blockStore.prune(chainAfterBatch.progressBlockNumber);
@@ -634,6 +730,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
       cs.transactionStore.rollback(rollbackTargetBlockNumber);
       cs.blockStore.rollback(rollbackTargetBlockNumber);
       cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
+      cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, rollbackTargetBlockNumber);
       return;
     } else {
       return;
@@ -657,6 +754,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
   cs.transactionStore.rollback(newProgressBlockNumber);
   cs.blockStore.rollback(newProgressBlockNumber);
   cs.committedProgressBlockNumber = newProgressBlockNumber;
+  cs.processingBlockNumber = newProgressBlockNumber;
   cs.numEventsProcessed = newTotalEventsProcessed;
 }
 
@@ -682,8 +780,12 @@ export {
   bufferReadyCount,
   getProgressPercentage,
   chainDensity,
+  effectiveDensity,
   hasReadyItem,
   isReadyToEnterReorgThreshold,
+  targetBlock,
+  blockAtProgress,
+  frontierProgress,
   getNextQuery,
   dispatch,
   toChainData,
@@ -696,6 +798,7 @@ export {
   isReady,
   isFetchingAtHead,
   isAtHeadWithoutEndBlock,
+  filterByClientAddress,
   handleQueryResult,
   materializeBatchItems,
   materializePageItems,