envio 3.3.0-alpha.5 → 3.3.0-alpha.7

package/src/FetchState.res

@@ -90,23 +90,41 @@ let deriveContractNameByAddress: dict<array<Address.t>> => dict<
   result
 })
 
+// Bounds for the no-history default estimate below. Also used by SourceManager
+// as the floor for its own maxNumLogs-style safety net.
+let minEstResponseSize = 2_000.
+let maxEstResponseSize = 10_000.
+
 // Default estimate for a query whose partition hasn't responded yet, so the
 // shared budget still accounts for unknown queries instead of treating them as
-// free.
-let defaultEstResponseSize = 10_000.
+// free. Scaled down as partitionsCount grows: assuming every partition's first
+// query is "big" starves the rest of that tick's admission when many
+// partitions share the same buffer, so the per-partition default shrinks
+// accordingly — clamped to [minEstResponseSize, maxEstResponseSize] either way.
+let calculateDefaultEstResponseSize = (~partitionsCount) =>
+  Pervasives.max(
+    minEstResponseSize,
+    Pervasives.min(maxEstResponseSize, 20_000. /. partitionsCount->Int.toFloat),
+  )
 
 // Estimated items a query will return, from the partition's event density
 // (items/block derived from its last response) and the query's block range.
-// toBlock None is the open-ended tail, capped at maxQueryBlockNumber. A partition
+// toBlock None is the open-ended tail, capped at knownHeight. A partition
 // that responded with no items has density 0, so its queries cost 0 — correct,
 // they don't fill the buffer. Only a partition that has never responded
-// (prevQueryRange 0) has no signal, so it falls back to defaultEstResponseSize.
-let calculateEstResponseSize = (p: partition, ~fromBlock, ~toBlock, ~maxQueryBlockNumber) =>
+// (prevQueryRange 0) has no signal, so it falls back to calculateDefaultEstResponseSize.
+let calculateEstResponseSize = (
+  p: partition,
+  ~fromBlock,
+  ~toBlock,
+  ~knownHeight,
+  ~partitionsCount,
+) =>
   if p.prevQueryRange > 0 {
     let density = p.prevRangeSize->Int.toFloat /. p.prevQueryRange->Int.toFloat
-    (toBlock->Option.getOr(maxQueryBlockNumber) - fromBlock + 1)->Int.toFloat *. density
+    (toBlock->Option.getOr(knownHeight) - fromBlock + 1)->Int.toFloat *. density
   } else {
-    defaultEstResponseSize
+    calculateDefaultEstResponseSize(~partitionsCount)
   }
 
 // Calculate the chunk range from history using min-of-last-3-ranges heuristic
@@ -545,8 +563,8 @@ type t = {
   // Buffer of items ordered from earliest to latest
   buffer: array<Internal.item>,
   // Caps how far ahead onBlock items are pre-generated (set to 2x the batch
-  // size). Fetch depth is bounded separately by getNextQuery's itemBudget, the
-  // chain's per-tick slice of the indexer-wide pool.
+  // size). Event fetch depth is bounded separately, by CrossChainState's
+  // cross-chain admission against the indexer-wide buffer pool.
   maxOnBlockBufferSize: int,
   onBlockConfigs: array<Internal.onBlockConfig>,
   knownHeight: int,
@@ -1352,14 +1370,15 @@ let pushQueriesForRange = (
   ~partitionId: string,
   ~rangeFromBlock: int,
   ~rangeEndBlock: option<int>,
-  ~maxQueryBlockNumber: int,
+  ~knownHeight: int,
   ~maybeChunkRange: option<int>,
   ~maxChunks: int,
   ~partition: partition,
   ~selection: selection,
   ~addressesByContractName: dict<array<Address.t>>,
+  ~partitionsCount: int,
 ) => {
-  if rangeFromBlock <= maxQueryBlockNumber && maxChunks > 0 {
+  if rangeFromBlock <= knownHeight && maxChunks > 0 {
     switch rangeEndBlock {
     | Some(endBlock) if rangeFromBlock > endBlock => ()
     | _ =>
@@ -1375,7 +1394,8 @@ let pushQueriesForRange = (
             partition,
             ~fromBlock=rangeFromBlock,
             ~toBlock=rangeEndBlock,
-            ~maxQueryBlockNumber,
+            ~knownHeight,
+            ~partitionsCount,
           ),
           chainId: 0,
           progress: 0.,
@@ -1384,7 +1404,7 @@ let pushQueriesForRange = (
       | Some(chunkRange) =>
         let maxBlock = switch rangeEndBlock {
         | Some(eb) => eb
-        | None => maxQueryBlockNumber
+        | None => knownHeight
         }
         let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
         if rangeFromBlock + chunkSize * 2 - 1 <= maxBlock {
@@ -1404,7 +1424,8 @@ let pushQueriesForRange = (
                 partition,
                 ~fromBlock=chunkFromBlock.contents,
                 ~toBlock=Some(chunkToBlock),
-                ~maxQueryBlockNumber,
+                ~knownHeight,
+                ~partitionsCount,
               ),
               chainId: 0,
               progress: 0.,
@@ -1425,7 +1446,8 @@ let pushQueriesForRange = (
               partition,
               ~fromBlock=rangeFromBlock,
               ~toBlock=rangeEndBlock,
-              ~maxQueryBlockNumber,
+              ~knownHeight,
+              ~partitionsCount,
             ),
             chainId: 0,
             progress: 0.,
@@ -1437,45 +1459,27 @@ let pushQueriesForRange = (
   }
 }
 
+// Candidate queries are sized against the natural ceiling (head/endBlock/mergeBlock),
+// not a share of the shared buffer — CrossChainState.checkAndFetch's admission loop
+// is what actually bounds how much gets fetched per tick, by estResponseSize (which
+// the HyperSync/SVM sources also enforce server-side via a maxNumLogs-style cap, so
+// a wrong estimate truncates the response instead of overshooting the buffer).
 let getNextQuery = (
-  {buffer, optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight} as fetchState: t,
-  ~budget,
-  ~chainPendingBudget,
+  {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
 ) => {
   let headBlockNumber = knownHeight - blockLag
   if headBlockNumber <= 0 {
     WaitingForNewBlock
-  } else if budget <= 0 {
-    // No room left in the shared buffer pool for this chain; wait for processing
-    // to drain before fetching more.
-    NothingToQuery
   } else {
     let isOnBlockBehindTheHead = latestOnBlockBlockNumber < headBlockNumber
     let shouldWaitForNewBlock = ref(
-      switch fetchState.endBlock {
+      switch endBlock {
       | Some(endBlock) => headBlockNumber < endBlock
       | None => true
       } &&
       !isOnBlockBehindTheHead,
     )
 
-    // Fetch at most `budget` items past the ready frontier (plus what's already
-    // in flight) so processing always has buffer without ballooning memory.
-    // budget already excludes items at/below the frontier (they're in the shared
-    // totalReadyCount), so offset the index by bufferReadyCount — otherwise the
-    // ready prefix is subtracted twice and the buffer caps at a fraction of its
-    // target. A partition that fetched further is skipped until the buffer drains.
-    let maxQueryBlockNumber = {
-      switch buffer->Array.get(
-        fetchState->bufferReadyCount + budget + chainPendingBudget->Float.toInt - 1,
-      ) {
-      | Some(item) =>
-        // Just in case check that we don't query beyond the current block
-        Pervasives.min(item->Internal.getItemBlockNumber, knownHeight)
-      | None => knownHeight
-      }
-    }
-
     let queries = []
 
     let partitionsCount = optimizedPartitions.idsInAscOrder->Array.length
@@ -1501,7 +1505,7 @@ let getNextQuery = (
       let partitionQueriesStart = queries->Array.length
 
       // Compute queryEndBlock for this partition
-      let queryEndBlock = Utils.Math.minOptInt(fetchState.endBlock, p.mergeBlock)
+      let queryEndBlock = Utils.Math.minOptInt(endBlock, p.mergeBlock)
       let queryEndBlock = switch blockLag {
       | 0 => queryEndBlock
       | _ =>
@@ -1509,15 +1513,6 @@ let getNextQuery = (
         // because otherwise HyperSync might return bigger range
         Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock)
       }
-      // Enforce the response range up until target block
-      // Otherwise for indexers with 100+ partitions
-      // we might blow up the buffer size to more than 600k events
-      // simply because of HyperSync returning extra blocks
-      let queryEndBlock = switch (queryEndBlock, maxQueryBlockNumber < knownHeight) {
-      | (Some(endBlock), true) => Some(Pervasives.min(maxQueryBlockNumber, endBlock))
-      | (None, true) => Some(maxQueryBlockNumber)
-      | (_, false) => queryEndBlock
-      }
 
       let maybeChunkRange = getMinHistoryRange(p)
 
@@ -1535,7 +1530,7 @@ let getNextQuery = (
             ~partitionId,
             ~rangeFromBlock=cursor.contents,
             ~rangeEndBlock=Utils.Math.minOptInt(Some(pq.fromBlock - 1), queryEndBlock),
-            ~maxQueryBlockNumber,
+            ~knownHeight,
             ~maybeChunkRange,
             ~maxChunks=maxPendingChunksPerPartition -
             pendingCount -
@@ -1544,6 +1539,7 @@ let getNextQuery = (
             ~partition=p,
             ~selection=p.selection,
             ~addressesByContractName=p.addressesByContractName,
+            ~partitionsCount,
           )
         }
         switch pq {
@@ -1563,7 +1559,7 @@ let getNextQuery = (
           ~partitionId,
           ~rangeFromBlock=cursor.contents,
           ~rangeEndBlock=queryEndBlock,
-          ~maxQueryBlockNumber,
+          ~knownHeight,
           ~maybeChunkRange,
           ~maxChunks=maxPendingChunksPerPartition -
           pendingCount -
@@ -1572,6 +1568,7 @@ let getNextQuery = (
           ~partition=p,
           ~selection=p.selection,
           ~addressesByContractName=p.addressesByContractName,
+          ~partitionsCount,
         )
       }
 

package/src/FetchState.res.mjs

@@ -8,6 +8,7 @@ import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as IndexingAddresses from "./IndexingAddresses.res.mjs";
 
@@ -21,12 +22,16 @@ let deriveContractNameByAddress = Utils.$$WeakMap.memoize(addressesByContractNam
   return result;
 });
 
-function calculateEstResponseSize(p, fromBlock, toBlock, maxQueryBlockNumber) {
+function calculateDefaultEstResponseSize(partitionsCount) {
+  return Primitive_float.max(2000, Primitive_float.min(10000, 20000 / partitionsCount));
+}
+
+function calculateEstResponseSize(p, fromBlock, toBlock, knownHeight, partitionsCount) {
   if (p.prevQueryRange <= 0) {
-    return 10000;
+    return calculateDefaultEstResponseSize(partitionsCount);
   }
   let density = p.prevRangeSize / p.prevQueryRange;
-  return ((Stdlib_Option.getOr(toBlock, maxQueryBlockNumber) - fromBlock | 0) + 1 | 0) * density;
+  return ((Stdlib_Option.getOr(toBlock, knownHeight) - fromBlock | 0) + 1 | 0) * density;
 }
 
 function getMinHistoryRange(p) {
@@ -977,15 +982,15 @@ function startFetchingQueries(param, queries) {
   }
 }
 
-function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber, maybeChunkRange, maxChunks, partition, selection, addressesByContractName) {
-  if (!(rangeFromBlock <= maxQueryBlockNumber && maxChunks > 0)) {
+function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock, knownHeight, maybeChunkRange, maxChunks, partition, selection, addressesByContractName, partitionsCount) {
+  if (!(rangeFromBlock <= knownHeight && maxChunks > 0)) {
     return;
   }
   if (rangeEndBlock !== undefined && rangeFromBlock > rangeEndBlock) {
     return;
   }
   if (maybeChunkRange !== undefined) {
-    let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : maxQueryBlockNumber;
+    let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : knownHeight;
     let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
     if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) {
       let chunkFromBlock = rangeFromBlock;
@@ -997,7 +1002,7 @@ function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock
           fromBlock: chunkFromBlock,
           toBlock: chunkToBlock,
           isChunk: true,
-          estResponseSize: calculateEstResponseSize(partition, chunkFromBlock, chunkToBlock, maxQueryBlockNumber),
+          estResponseSize: calculateEstResponseSize(partition, chunkFromBlock, chunkToBlock, knownHeight, partitionsCount),
           chainId: 0,
           progress: 0,
           selection: selection,
@@ -1013,7 +1018,7 @@ function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock
       fromBlock: rangeFromBlock,
       toBlock: rangeEndBlock,
       isChunk: rangeEndBlock !== undefined,
-      estResponseSize: calculateEstResponseSize(partition, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber),
+      estResponseSize: calculateEstResponseSize(partition, rangeFromBlock, rangeEndBlock, knownHeight, partitionsCount),
       chainId: 0,
       progress: 0,
       selection: selection,
@@ -1026,7 +1031,7 @@ function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock
     fromBlock: rangeFromBlock,
     toBlock: rangeEndBlock,
     isChunk: false,
-    estResponseSize: calculateEstResponseSize(partition, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber),
+    estResponseSize: calculateEstResponseSize(partition, rangeFromBlock, rangeEndBlock, knownHeight, partitionsCount),
     chainId: 0,
     progress: 0,
     selection: selection,
@@ -1034,24 +1039,19 @@ function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock
   });
 }
 
-function getNextQuery(fetchState, budget, chainPendingBudget) {
-  let knownHeight = fetchState.knownHeight;
-  let blockLag = fetchState.blockLag;
-  let optimizedPartitions = fetchState.optimizedPartitions;
+function getNextQuery(param) {
+  let knownHeight = param.knownHeight;
+  let blockLag = param.blockLag;
+  let endBlock = param.endBlock;
+  let optimizedPartitions = param.optimizedPartitions;
   let headBlockNumber = knownHeight - blockLag | 0;
   if (headBlockNumber <= 0) {
     return "WaitingForNewBlock";
   }
-  if (budget <= 0) {
-    return "NothingToQuery";
-  }
-  let isOnBlockBehindTheHead = fetchState.latestOnBlockBlockNumber < headBlockNumber;
-  let endBlock = fetchState.endBlock;
+  let isOnBlockBehindTheHead = param.latestOnBlockBlockNumber < headBlockNumber;
   let shouldWaitForNewBlock = (
     endBlock !== undefined ? headBlockNumber < endBlock : true
   ) && !isOnBlockBehindTheHead;
-  let item = fetchState.buffer[((bufferReadyCount(fetchState) + budget | 0) + (chainPendingBudget | 0) | 0) - 1 | 0];
-  let maxQueryBlockNumber = item !== undefined ? Primitive_int.min(item.blockNumber, knownHeight) : knownHeight;
   let queries = [];
   let partitionsCount = optimizedPartitions.idsInAscOrder.length;
   let idxRef = 0;
@@ -1066,14 +1066,8 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
       shouldWaitForNewBlock = false;
     }
     let partitionQueriesStart = queries.length;
-    let queryEndBlock = Utils.$$Math.minOptInt(fetchState.endBlock, p.mergeBlock);
+    let queryEndBlock = Utils.$$Math.minOptInt(endBlock, p.mergeBlock);
     let queryEndBlock$1 = blockLag !== 0 ? Utils.$$Math.minOptInt(headBlockNumber, queryEndBlock) : queryEndBlock;
-    let match = maxQueryBlockNumber < knownHeight;
-    let queryEndBlock$2 = queryEndBlock$1 !== undefined ? (
-        match ? Primitive_int.min(maxQueryBlockNumber, queryEndBlock$1) : queryEndBlock$1
-      ) : (
-        match ? maxQueryBlockNumber : queryEndBlock$1
-      );
     let maybeChunkRange = getMinHistoryRange(p);
     let cursor = p.latestFetchedBlock.blockNumber + 1 | 0;
     let canContinue = true;
@@ -1084,16 +1078,16 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
         let addressesByContractName = p.addressesByContractName;
         let selection = p.selection;
         let maxChunks = (10 - pendingCount | 0) - (queries.length - partitionQueriesStart | 0) | 0;
-        let rangeEndBlock = Utils.$$Math.minOptInt(pq.fromBlock - 1 | 0, queryEndBlock$2);
+        let rangeEndBlock = Utils.$$Math.minOptInt(pq.fromBlock - 1 | 0, queryEndBlock$1);
         let rangeFromBlock = cursor;
-        if (rangeFromBlock <= maxQueryBlockNumber && maxChunks > 0) {
+        if (rangeFromBlock <= knownHeight && maxChunks > 0) {
           let exit = 0;
           if (!(rangeEndBlock !== undefined && rangeFromBlock > rangeEndBlock)) {
             exit = 1;
           }
           if (exit === 1) {
             if (maybeChunkRange !== undefined) {
-              let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : maxQueryBlockNumber;
+              let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : knownHeight;
               let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8);
               if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) {
                 let chunkFromBlock = rangeFromBlock;
@@ -1105,7 +1099,7 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
                     fromBlock: chunkFromBlock,
                     toBlock: chunkToBlock,
                     isChunk: true,
-                    estResponseSize: calculateEstResponseSize(p, chunkFromBlock, chunkToBlock, maxQueryBlockNumber),
+                    estResponseSize: calculateEstResponseSize(p, chunkFromBlock, chunkToBlock, knownHeight, partitionsCount),
                     chainId: 0,
                     progress: 0,
                     selection: selection,
@@ -1120,7 +1114,7 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
                   fromBlock: rangeFromBlock,
                   toBlock: rangeEndBlock,
                   isChunk: rangeEndBlock !== undefined,
-                  estResponseSize: calculateEstResponseSize(p, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber),
+                  estResponseSize: calculateEstResponseSize(p, rangeFromBlock, rangeEndBlock, knownHeight, partitionsCount),
                   chainId: 0,
                   progress: 0,
                   selection: selection,
@@ -1133,7 +1127,7 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
                 fromBlock: rangeFromBlock,
                 toBlock: rangeEndBlock,
                 isChunk: false,
-                estResponseSize: calculateEstResponseSize(p, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber),
+                estResponseSize: calculateEstResponseSize(p, rangeFromBlock, rangeEndBlock, knownHeight, partitionsCount),
                 chainId: 0,
                 progress: 0,
                 selection: selection,
@@ -1145,9 +1139,9 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
       }
       let toBlock = pq.toBlock;
       if (toBlock !== undefined && pq.isChunk) {
-        let match$1 = pq.fetchedBlock;
-        if (match$1 !== undefined) {
-          let blockNumber = match$1.blockNumber;
+        let match = pq.fetchedBlock;
+        if (match !== undefined) {
+          let blockNumber = match.blockNumber;
           cursor = blockNumber < toBlock ? blockNumber + 1 | 0 : toBlock + 1 | 0;
         } else {
           cursor = toBlock + 1 | 0;
@@ -1162,14 +1156,14 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
       let selection$1 = p.selection;
       let maxChunks$1 = (10 - pendingCount | 0) - (queries.length - partitionQueriesStart | 0) | 0;
       let rangeFromBlock$1 = cursor;
-      if (rangeFromBlock$1 <= maxQueryBlockNumber && maxChunks$1 > 0) {
+      if (rangeFromBlock$1 <= knownHeight && maxChunks$1 > 0) {
         let exit$1 = 0;
-        if (!(queryEndBlock$2 !== undefined && rangeFromBlock$1 > queryEndBlock$2)) {
+        if (!(queryEndBlock$1 !== undefined && rangeFromBlock$1 > queryEndBlock$1)) {
           exit$1 = 1;
         }
         if (exit$1 === 1) {
           if (maybeChunkRange !== undefined) {
-            let maxBlock$1 = queryEndBlock$2 !== undefined ? queryEndBlock$2 : maxQueryBlockNumber;
+            let maxBlock$1 = queryEndBlock$1 !== undefined ? queryEndBlock$1 : knownHeight;
             let chunkSize$1 = Js_math.ceil_int(maybeChunkRange * 1.8);
             if (((rangeFromBlock$1 + (chunkSize$1 << 1) | 0) - 1 | 0) <= maxBlock$1) {
               let chunkFromBlock$1 = rangeFromBlock$1;
@@ -1181,7 +1175,7 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
                   fromBlock: chunkFromBlock$1,
                   toBlock: chunkToBlock$1,
                   isChunk: true,
-                  estResponseSize: calculateEstResponseSize(p, chunkFromBlock$1, chunkToBlock$1, maxQueryBlockNumber),
+                  estResponseSize: calculateEstResponseSize(p, chunkFromBlock$1, chunkToBlock$1, knownHeight, partitionsCount),
                   chainId: 0,
                   progress: 0,
                   selection: selection$1,
@@ -1194,9 +1188,9 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
               queries.push({
                 partitionId: partitionId,
                 fromBlock: rangeFromBlock$1,
-                toBlock: queryEndBlock$2,
-                isChunk: queryEndBlock$2 !== undefined,
-                estResponseSize: calculateEstResponseSize(p, rangeFromBlock$1, queryEndBlock$2, maxQueryBlockNumber),
+                toBlock: queryEndBlock$1,
+                isChunk: queryEndBlock$1 !== undefined,
+                estResponseSize: calculateEstResponseSize(p, rangeFromBlock$1, queryEndBlock$1, knownHeight, partitionsCount),
                 chainId: 0,
                 progress: 0,
                 selection: selection$1,
@@ -1207,9 +1201,9 @@ function getNextQuery(fetchState, budget, chainPendingBudget) {
             queries.push({
               partitionId: partitionId,
               fromBlock: rangeFromBlock$1,
-              toBlock: queryEndBlock$2,
+              toBlock: queryEndBlock$1,
               isChunk: false,
-              estResponseSize: calculateEstResponseSize(p, rangeFromBlock$1, queryEndBlock$2, maxQueryBlockNumber),
+              estResponseSize: calculateEstResponseSize(p, rangeFromBlock$1, queryEndBlock$1, knownHeight, partitionsCount),
               chainId: 0,
               progress: 0,
               selection: selection$1,
@@ -1656,7 +1650,9 @@ function updateKnownHeight(fetchState, knownHeight) {
   }
 }
 
-let defaultEstResponseSize = 10000;
+let minEstResponseSize = 2000;
+
+let maxEstResponseSize = 10000;
 
 let blockItemLogIndex = 16777216;
 
@@ -1664,7 +1660,9 @@ let maxPendingChunksPerPartition = 10;
 
 export {
   deriveContractNameByAddress,
-  defaultEstResponseSize,
+  minEstResponseSize,
+  maxEstResponseSize,
+  calculateDefaultEstResponseSize,
   calculateEstResponseSize,
   getMinHistoryRange,
   getMinQueryRange,

package/src/IndexerState.res

@@ -108,6 +108,8 @@ type t = {
   // waitForNewBlock waiter is bound to the old, pre-realtime source). A fetch
   // response or waiter carrying an older epoch than this is discarded.
   mutable epoch: int,
+  // None off the simulate path.
+  simulateDeadInputTracker: option<SimulateDeadInputTracker.t>,
 }
 
 let make = (
@@ -169,6 +171,7 @@ let make = (
     onError,
     isStopped: false,
     epoch: 0,
+    simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config),
   }
 }
 
@@ -400,6 +403,7 @@ let exitAfterFirstEventBlock = (state: t) => state.exitAfterFirstEventBlock
 let isStopped = (state: t) => state.isStopped
 let epoch = (state: t) => state.epoch
 let pruneStaleEntityHistoryThrottler = (state: t) => state.writeThrottlers.pruneStaleEntityHistory
+let simulateDeadInputTracker = (state: t) => state.simulateDeadInputTracker
 
 // --- Store domain operations. ---
 

package/src/IndexerState.res.mjs

@@ -18,6 +18,7 @@ import * as CrossChainState from "./CrossChainState.res.mjs";
 import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
 
 function make() {
   let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis;
@@ -91,7 +92,8 @@ function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime
     exitAfterFirstEventBlock: exitAfterFirstEventBlock,
     onError: onError,
     isStopped: false,
-    epoch: 0
+    epoch: 0,
+    simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config)
   };
 }
 
@@ -348,6 +350,10 @@ function pruneStaleEntityHistoryThrottler(state) {
   return state.writeThrottlers.pruneStaleEntityHistory;
 }
 
+function simulateDeadInputTracker(state) {
+  return state.simulateDeadInputTracker;
+}
+
 function queueProcessedBatch(state, batch) {
   state.processedBatches.push(batch);
   let checkpointId = Utils.$$Array.last(batch.checkpointIds);
@@ -527,6 +533,7 @@ export {
   isStopped,
   epoch,
   pruneStaleEntityHistoryThrottler,
+  simulateDeadInputTracker,
   queueProcessedBatch,
   drainBatchRun,
   takeRollback,

package/src/IndexerState.resi

@@ -107,6 +107,7 @@ let exitAfterFirstEventBlock: t => bool
 let isStopped: t => bool
 let epoch: t => int
 let pruneStaleEntityHistoryThrottler: t => Throttler.t
+let simulateDeadInputTracker: t => option<SimulateDeadInputTracker.t>
 
 // Store domain operations.
 let queueProcessedBatch: (t, ~batch: Batch.t) => unit

package/src/SimulateDeadInputTracker.res

@@ -0,0 +1,85 @@
+// Lives on IndexerState and is fed each processed batch, so ChainState and the
+// fetch loop carry no simulate-specific state.
+
+// Match a provided item to a processed one by its (chain, block, logIndex)
+// coordinate rather than object identity, so matching survives any copy or
+// transform of the item between the source and the batch.
+let itemKey = (item: Internal.item): string =>
+  switch item {
+  | Internal.Event({chain, blockNumber, logIndex}) =>
+    `${chain
+      ->ChainMap.Chain.toChainId
+      ->Int.toString}:${blockNumber->Int.toString}:${logIndex->Int.toString}`
+  | _ => "" // non-event items are never a provided simulate input
+  }
+
+// `index` is the item's position in its chain's `simulate` array, reported back
+// so a user finds it without echoing its fields.
+type entry = {chainId: int, index: int, key: string}
+
+type t = {mutable unprocessed: array<entry>}
+
+let makeFromConfig = (config: Config.t): option<t> => {
+  let entries =
+    config.chainMap
+    ->ChainMap.values
+    ->Array.flatMap(chainConfig =>
+      switch chainConfig.sourceConfig {
+      | Config.CustomSources(sources) =>
+        sources->Array.flatMap(source =>
+          source.simulateItems
+          ->Option.getOr([])
+          ->Array.mapWithIndex(
+            (item, index) => {
+              chainId: chainConfig.id,
+              index,
+              key: itemKey(item),
+            },
+          )
+        )
+      | _ => []
+      }
+    )
+  switch entries {
+  | [] => None
+  | _ => Some({unprocessed: entries})
+  }
+}
+
+let recordProcessed = (t: t, ~batch: Batch.t) => {
+  let processedKeys = batch.items->Array.map(itemKey)->Utils.Set.fromArray
+  t.unprocessed = t.unprocessed->Array.filter(entry => !(processedKeys->Utils.Set.has(entry.key)))
+}
+
+// Unrouted item indices grouped by chain, in the order chains were first seen.
+let unroutedByChain = (t: t): array<(int, array<int>)> => {
+  let indicesByChain = Dict.make()
+  let chainOrder = []
+  t.unprocessed->Array.forEach(entry => {
+    let key = entry.chainId->Int.toString
+    if indicesByChain->Dict.get(key)->Option.isNone {
+      chainOrder->Array.push(entry.chainId)->ignore
+    }
+    indicesByChain->Utils.Dict.push(key, entry.index)
+  })
+  chainOrder->Array.map(chainId => (chainId, indicesByChain->Dict.getUnsafe(chainId->Int.toString)))
+}
+
+let failureMessage = (t: t): option<string> =>
+  switch t->unroutedByChain {
+  | [] => None
+  | byChain =>
+    let count = byChain->Array.reduce(0, (acc, (_chainId, indices)) => acc + indices->Array.length)
+    let itemWord = count === 1 ? "item" : "items"
+    let lines =
+      byChain
+      ->Array.map(((chainId, indices)) =>
+        `  - chain ${chainId->Int.toString}: ${indices
+          ->Array.map(index => index->Int.toString)
+          ->Array.join(", ")}`
+      )
+      ->Array.join("\n")
+    Some(
+      `simulate: ${count->Int.toString} ${itemWord} you passed to simulate never reached a handler, so nothing ran for them. Each was filtered out before the handler — usually a non-wildcard srcAddress that isn't indexed for the contract, or a where/block filter that excluded the event. Unrouted items, by index in each chain's simulate array:\n${lines}`,
+    )
+  }