envio 3.12.0 → 3.12.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.12.0",
+  "version": "3.12.1",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -68,10 +68,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.12.0",
-    "envio-linux-x64-musl": "3.12.0",
-    "envio-linux-arm64": "3.12.0",
-    "envio-darwin-x64": "3.12.0",
-    "envio-darwin-arm64": "3.12.0"
+    "envio-linux-x64": "3.12.1",
+    "envio-linux-x64-musl": "3.12.1",
+    "envio-linux-arm64": "3.12.1",
+    "envio-darwin-x64": "3.12.1",
+    "envio-darwin-arm64": "3.12.1"
   }
 }

package/src/ChainFetching.res

@@ -326,9 +326,13 @@ let fetchChain = async (
     let isRealtime = state->IndexerState.isRealtime
     let sourceManager = chainState->ChainState.sourceManager
 
-    // Only affects the WaitingForNewBlock branch of dispatch, where
-    // there's nothing to fetch. During backfill any such chain is idle.
-    let reducedPolling = !isRealtime
+    // Only affects the WaitingForNewBlock branch of dispatch, where there's
+    // nothing to fetch. The line is whether any height is known at all, not
+    // whether the head is fetchable yet: a chain held back by its lag is still
+    // tracking a head it has, and asking faster would buy nothing. A chain with
+    // no height has nothing to show for itself until the first one lands, so it
+    // polls at the source's own cadence and earns the normal stall window.
+    let reducedPolling = !isRealtime && chainState->ChainState.knownHeight > 0
 
     // Owns its error boundary: launch doesn't catch, so any failure here (the
     // query, response handling, or dispatch itself) must stop the indexer.

package/src/ChainFetching.res.mjs

@@ -180,7 +180,7 @@ async function fetchChain(state, chainId, action, stateId, scheduleFetch, schedu
   }
   let isRealtime = IndexerState.isRealtime(state);
   let sourceManager = ChainState.sourceManager(chainState);
-  let reducedPolling = !isRealtime;
+  let reducedPolling = !isRealtime && ChainState.knownHeight(chainState) > 0;
   try {
     return await ChainState.dispatch(chainState, async query => {
       try {

package/src/ChainState.res

@@ -1021,7 +1021,7 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
 // in-threshold block hashes are copied up front because the build reuses the
 // whole set; the block time comes as a lookup instead, since the key it needs -
 // the batch's own progress block - isn't known until the build computes it.
-let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
+let toChainBeforeBatch = (cs: t, ~isRealtime): Batch.chainBeforeBatch => {
   let {blockNumbers, hashes} =
     cs.blockStore->BlockStore.getHashes(
       ~fromBlock=Pervasives.max(cs.fetchState.knownHeight - cs.maxReorgDepth, 0),
@@ -1037,8 +1037,13 @@ let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
     totalEventsProcessed: cs.numEventsProcessed,
     sourceBlockNumber: cs.fetchState.knownHeight,
     scannedHashes: {blockNumbers, hashByBlockNumber},
+    // A slot that produced no block only counts as skipped where the query
+    // covered every slot in its range, which is what a chain at the head asks
+    // for.
     blockTimeAt: blockNumber =>
-      cs.blockStore->BlockStore.getTimestamp(blockNumber)->Null.toOption,
+      cs.blockStore
+      ->BlockStore.getTimestamp(blockNumber, ~allowSkippedSlot=isRealtime)
+      ->Null.toOption,
     shouldRollbackOnReorg: cs.shouldRollbackOnReorg,
     chainConfig: cs.chainConfig,
   }
@@ -1194,8 +1199,12 @@ let markReady = (cs: t, ~readyAt) =>
 let rollbackCommittedProgress = (cs: t, blockNumber) =>
   if blockNumber !== cs.committedProgressBlockNumber {
     cs.committedProgressBlockNumber = blockNumber
+    // Exact block only: the rolled-back region is about to be refetched, and
+    // the next batch re-establishes the time either way.
     cs.committedProgressBlockTime =
-      cs.blockStore->BlockStore.getTimestamp(blockNumber)->Null.toOption
+      cs.blockStore
+      ->BlockStore.getTimestamp(blockNumber, ~allowSkippedSlot=false)
+      ->Null.toOption
   }
 
 type progressDiff = {blockNumber: int, eventsProcessed: float}

package/src/ChainState.res.mjs

@@ -718,7 +718,7 @@ function toMetrics(cs) {
   };
 }
 
-function toChainBeforeBatch(cs) {
+function toChainBeforeBatch(cs, isRealtime) {
   let match = cs.blockStore.getHashes(Primitive_int.max(cs.fetchState.knownHeight - cs.maxReorgDepth | 0, 0), cs.fetchState.knownHeight + 1 | 0);
   let hashes = match.hashes;
   let blockNumbers = match.blockNumbers;
@@ -732,7 +732,7 @@ function toChainBeforeBatch(cs) {
       blockNumbers: blockNumbers,
       hashByBlockNumber: hashByBlockNumber
     },
-    blockTimeAt: blockNumber => Primitive_option.fromNull(cs.blockStore.getTimestamp(blockNumber)),
+    blockTimeAt: blockNumber => Primitive_option.fromNull(cs.blockStore.getTimestamp(blockNumber, isRealtime)),
     shouldRollbackOnReorg: cs.shouldRollbackOnReorg,
     progressBlockNumber: cs.committedProgressBlockNumber,
     sourceBlockNumber: cs.fetchState.knownHeight,
@@ -835,7 +835,7 @@ function markReady(cs, readyAt) {
 function rollbackCommittedProgress(cs, blockNumber) {
   if (blockNumber !== cs.committedProgressBlockNumber) {
     cs.committedProgressBlockNumber = blockNumber;
-    cs.committedProgressBlockTime = Primitive_option.fromNull(cs.blockStore.getTimestamp(blockNumber));
+    cs.committedProgressBlockTime = Primitive_option.fromNull(cs.blockStore.getTimestamp(blockNumber, false));
     return;
   }
 }

package/src/ChainState.resi

@@ -119,7 +119,7 @@ let dispatch: (
 // Views.
 let toMetrics: t => Metrics.chainMetrics
 let toChainMetadata: t => InternalTable.Chains.metaFields
-let toChainBeforeBatch: t => Batch.chainBeforeBatch
+let toChainBeforeBatch: (t, ~isRealtime: bool) => Batch.chainBeforeBatch
 let isReadyToEnterReorgThresholdAfterBatch: (t, ~batch: Batch.t) => bool
 
 // Derived (pure).

package/src/CrossChainState.res

@@ -114,7 +114,7 @@ let createBatch = (
     ~history=config->HistoryPolicy.decide(~shouldSaveHistory=crossChainState->shouldSaveHistory),
     ~frontier,
     ~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
-      ChainState.toChainBeforeBatch,
+      ChainState.toChainBeforeBatch(~isRealtime=crossChainState.isRealtime, ...),
     ),
     ~batchSizeTarget,
   )

package/src/CrossChainState.res.mjs

@@ -91,7 +91,7 @@ function getSafeCheckpointIdByChain(crossChainState, sequence, committedFrontier
 }
 
 function createBatch(crossChainState, config, frontier, batchSizeTarget) {
-  return Batch.make(config.checkpointSequence, frontier, Utils.Dict.mapValues(crossChainState.chainStates, ChainState.toChainBeforeBatch), batchSizeTarget, HistoryPolicy.decide(config, shouldSaveHistory(crossChainState)));
+  return Batch.make(config.checkpointSequence, frontier, Utils.Dict.mapValues(crossChainState.chainStates, none => ChainState.toChainBeforeBatch(none, crossChainState.isRealtime)), batchSizeTarget, HistoryPolicy.decide(config, shouldSaveHistory(crossChainState)));
 }
 
 function enterReorgThreshold(crossChainState) {

package/src/FetchState.res

@@ -50,6 +50,22 @@ let makeSelection = (~onEventRegistrations, ~dependsOnAddresses, ~clientFiltered
   startBlock: ?deriveSelectionStartBlock(onEventRegistrations),
 }
 
+// A partition's frontier never sits below the block before its selection can
+// first match: the blocks below hold nothing for it, so they count as fetched
+// the same way the blocks before an address's start block do. The chain's
+// buffer frontier is its lowest partition frontier, so a partition left at the
+// chain start would hold every other partition's events back as unprocessable
+// and keep the chain's query target — sized from that frontier — short of
+// anything worth asking for. A start block the chain has not reached yet leaves
+// the partition waiting there, like an address partition does, rather than
+// scanning empty ranges up to it; bufferBlockNumber caps the chain's frontier at
+// the head meanwhile.
+let floorAtSelectionStart = (latestFetchedBlock, ~selection) =>
+  switch selection.startBlock {
+  | Some(startBlock) => Pervasives.max(latestFetchedBlock, startBlock - 1)
+  | None => latestFetchedBlock
+  }
+
 type pendingQuery = {
   fromBlock: int,
   toBlock: option<int>,
@@ -398,6 +414,10 @@ module OptimizedPartitions = {
     ~dynamicContracts: Utils.Set.t<string>,
     ~clientFilteredContracts: Utils.Set.t<string>,
   ) => {
+    let partitions = partitions->Array.map(p => {
+      let floored = p.latestFetchedBlock->floorAtSelectionStart(~selection=p.selection)
+      floored === p.latestFetchedBlock ? p : {...p, latestFetchedBlock: floored}
+    })
     let newPartitions = []
     let mergingPartitions = Dict.make()
     let nextPartitionIndexRef = ref(nextPartitionIndex)
@@ -847,28 +867,17 @@ type t = {
   clientFilterAddressThreshold: option<int>,
 }
 
+// The latest block whose items are all in the buffer: the lowest partition
+// frontier, held back by the onBlock pointer. Without onBlock registrations the
+// pointer tracks the known height, so this also caps the frontier at the head: a
+// partition can sit past it, on a response that reported a block the chain
+// hadn't heard of yet or on a start block the chain hasn't reached.
 @inline
 let bufferBlockNumber = ({latestOnBlockBlockNumber, optimizedPartitions}: t) => {
   switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
   | None => latestOnBlockBlockNumber
   | Some(latestFullyFetchedBlock) =>
-    latestOnBlockBlockNumber < latestFullyFetchedBlock
-      ? latestOnBlockBlockNumber
-      : latestFullyFetchedBlock
-  }
-}
-
-/**
-* Returns the latest block which is ready to be consumed
-*/
-@inline
-let bufferBlock = ({optimizedPartitions, latestOnBlockBlockNumber}: t) => {
-  switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
-  | None => latestOnBlockBlockNumber
-  | Some(latestFullyFetchedBlock) =>
-    latestOnBlockBlockNumber < latestFullyFetchedBlock
-      ? latestOnBlockBlockNumber
-      : latestFullyFetchedBlock
+    Pervasives.min(latestOnBlockBlockNumber, latestFullyFetchedBlock)
   }
 }
 
@@ -1138,14 +1147,20 @@ let updateInternal = (
     // Use maxOnBlockBufferSize to get the last target item in the buffer
     // (sorted, so this is the highest-block item within the buffer cap).
     // All this needed to prevent OOM when adding too many block items to the queue
-    let maxBlockNumber = switch base->Array.get(fetchState.maxOnBlockBufferSize - 1) {
-    | Some(item) => item->Internal.getItemBlockNumber
-    | None =>
-      switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
-      | None => knownHeight
-      | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
-      }
-    }
+    // Never past the head: a partition can sit there, on a start block the
+    // chain hasn't reached, and a response is applied before the height it
+    // reported, so its events can run past the height still known here.
+    let maxBlockNumber = Pervasives.min(
+      switch base->Array.get(fetchState.maxOnBlockBufferSize - 1) {
+      | Some(item) => item->Internal.getItemBlockNumber
+      | None =>
+        switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
+        | None => knownHeight
+        | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
+        }
+      },
+      knownHeight,
+    )
     appendOnBlockItems(
       ~mutItems=blockItems,
       ~onBlockRegistrations,
@@ -2165,27 +2180,11 @@ let walkPartitionPending = (
     pqIdx := pqIdx.contents + 1
   }
 
-  // Nothing in this partition's selection can match below its earliest start
-  // block, so forward work skips straight to it instead of scanning up to it and
-  // discarding whole pages. Only the cursor moves — `latestFetchedBlock` still
-  // advances solely on a response, so no block is ever reported fetched that
-  // wasn't. Bounded by the head and the query end block: past either, the
-  // partition would offer no candidate at all, so it queries as before rather
-  // than going quiet until the chain reaches its start block.
-  let cursor = switch p.selection.startBlock {
-  | Some(startBlock) if startBlock > cursor.contents =>
-    switch Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock) {
-    | Some(reachable) if startBlock <= reachable => startBlock
-    | _ => cursor.contents
-    }
-  | _ => cursor.contents
-  }
-
   canContinue.contents
     ? Some({
         partitionId,
         p,
-        cursor,
+        cursor: cursor.contents,
         chunksUsedThisCall: chunksUsedThisCall.contents,
         inFlightCount,
         queryEndBlock,
@@ -2780,10 +2779,17 @@ let make = (
   // fetching, so without seeding the buffer here getNextQuery would return
   // NothingToQuery and the indexer would get stuck.
   let buffer = []
-  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockRegistrations->Utils.Array.notEmpty {
+  let latestOnBlockBlockNumber = switch onBlockRegistrations {
+  // As updateInternal keeps it: with nothing to generate per block the pointer
+  // is the head cap on the buffer frontier, and left at the progress block it
+  // would hold the frontier there until the first height update — below a
+  // partition that starts later, whose queries the chain sizes off that
+  // frontier and so never reaches.
+  | [] if knownHeight > 0 => knownHeight
+  | onBlockRegistrations if knownHeight > 0 =>
     let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
     | None => knownHeight
-    | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
+    | Some(latestFullyFetchedBlock) => Pervasives.min(latestFullyFetchedBlock, knownHeight)
     }
     appendOnBlockItems(
       ~mutItems=buffer,
@@ -2793,8 +2799,7 @@ let make = (
       ~maxBlockNumber,
       ~maxOnBlockBufferSize,
     )
-  } else {
-    progressBlockNumber
+  | _ => progressBlockNumber
   }
 
   let fetchState = {

package/src/FetchState.res.mjs

@@ -47,6 +47,15 @@ function makeSelection(onEventRegistrations, dependsOnAddresses, clientFilteredC
   };
 }
 
+function floorAtSelectionStart(latestFetchedBlock, selection) {
+  let startBlock = selection.startBlock;
+  if (startBlock !== undefined) {
+    return Primitive_int.max(latestFetchedBlock, startBlock - 1 | 0);
+  } else {
+    return latestFetchedBlock;
+  }
+}
+
 function withAddresses(p, addresses) {
   return {
     id: p.id,
@@ -305,6 +314,26 @@ function getAnchorSafeBlock(p) {
 }
 
 function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContracts, clientFilteredContracts) {
+  let partitions$1 = partitions.map(p => {
+    let floored = floorAtSelectionStart(p.latestFetchedBlock, p.selection);
+    if (floored === p.latestFetchedBlock) {
+      return p;
+    } else {
+      return {
+        id: p.id,
+        latestFetchedBlock: floored,
+        selection: p.selection,
+        addresses: p.addresses,
+        mergeBlock: p.mergeBlock,
+        dynamicContract: p.dynamicContract,
+        mutPendingQueries: p.mutPendingQueries,
+        sourceRangeCapacity: p.sourceRangeCapacity,
+        prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+        eventDensity: p.eventDensity,
+        latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock
+      };
+    }
+  });
   let newPartitions = [];
   let mergingPartitions = {};
   let nextPartitionIndexRef = {
@@ -313,7 +342,7 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
   let anchorSafeBlocks = {};
   let anchoredContractsSet = new Set();
   if (clientFilteredContracts.size > 0) {
-    partitions.forEach(p => {
+    partitions$1.forEach(p => {
       let match = p.mergeBlock;
       let match$1 = p.selection.clientFilteredContracts;
       if (match !== undefined) {
@@ -332,8 +361,8 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
       });
     });
   }
-  for (let idx = 0, idx_finish = partitions.length; idx < idx_finish; ++idx) {
-    let p = partitions[idx];
+  for (let idx = 0, idx_finish = partitions$1.length; idx < idx_finish; ++idx) {
+    let p = partitions$1[idx];
     let mergeBlock = p.mergeBlock;
     if (mergeBlock !== undefined) {
       if (p.latestFetchedBlock < mergeBlock || isFetching(p)) {
@@ -676,20 +705,8 @@ function bufferBlockNumber(param) {
   let optimizedPartitions = param.optimizedPartitions;
   let id = optimizedPartitions.idsInAscOrder[0];
   let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
-  if (latestFullyFetchedBlock !== undefined && latestOnBlockBlockNumber >= latestFullyFetchedBlock) {
-    return latestFullyFetchedBlock;
-  } else {
-    return latestOnBlockBlockNumber;
-  }
-}
-
-function bufferBlock(param) {
-  let latestOnBlockBlockNumber = param.latestOnBlockBlockNumber;
-  let optimizedPartitions = param.optimizedPartitions;
-  let id = optimizedPartitions.idsInAscOrder[0];
-  let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
-  if (latestFullyFetchedBlock !== undefined && latestOnBlockBlockNumber >= latestFullyFetchedBlock) {
-    return latestFullyFetchedBlock;
+  if (latestFullyFetchedBlock !== undefined) {
+    return Primitive_int.min(latestOnBlockBlockNumber, latestFullyFetchedBlock);
   } else {
     return latestOnBlockBlockNumber;
   }
@@ -897,14 +914,15 @@ function updateInternal(fetchState, optimizedPartitionsOpt, mutItems, mutItemsSo
   let latestOnBlockBlockNumber;
   if (onBlockRegistrations.length !== 0) {
     let item = base[fetchState.maxOnBlockBufferSize - 1 | 0];
-    let maxBlockNumber;
+    let tmp;
     if (item !== undefined) {
-      maxBlockNumber = item.blockNumber;
+      tmp = item.blockNumber;
     } else {
       let id = optimizedPartitions.idsInAscOrder[0];
       let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
-      maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock : knownHeight;
+      tmp = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock : knownHeight;
     }
+    let maxBlockNumber = Primitive_int.min(tmp, knownHeight);
     latestOnBlockBlockNumber = appendOnBlockItems(blockItems, onBlockRegistrations, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.maxOnBlockBufferSize);
   } else {
     latestOnBlockBlockNumber = knownHeight;
@@ -1486,19 +1504,11 @@ function walkPartitionPending(p, partitionId, inFlightCount, candidates, headBlo
     }
     pqIdx = pqIdx + 1 | 0;
   };
-  let startBlock = p.selection.startBlock;
-  let cursor$1;
-  if (startBlock !== undefined && startBlock > cursor) {
-    let reachable = Utils.$$Math.minOptInt(headBlockNumber, queryEndBlock);
-    cursor$1 = reachable !== undefined && startBlock <= reachable ? startBlock : cursor;
-  } else {
-    cursor$1 = cursor;
-  }
   if (canContinue) {
     return {
       partitionId: partitionId,
       p: p,
-      cursor: cursor$1,
+      cursor: cursor,
       chunksUsedThisCall: chunksUsedThisCall,
       inFlightCount: inFlightCount,
       queryEndBlock: queryEndBlock,
@@ -1800,13 +1810,21 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
   }
   let buffer = [];
   let latestOnBlockBlockNumber;
-  if (knownHeight > 0 && Utils.$$Array.notEmpty(onBlockRegistrations)) {
-    let id = optimizedPartitions.idsInAscOrder[0];
-    let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
-    let maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock : knownHeight;
-    latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockRegistrations, startBlock, progressBlockNumber, maxBlockNumber, maxOnBlockBufferSize);
+  let exit = 0;
+  if (onBlockRegistrations.length !== 0 || knownHeight <= 0) {
+    exit = 1;
   } else {
-    latestOnBlockBlockNumber = progressBlockNumber;
+    latestOnBlockBlockNumber = knownHeight;
+  }
+  if (exit === 1) {
+    if (knownHeight > 0) {
+      let id = optimizedPartitions.idsInAscOrder[0];
+      let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
+      let maxBlockNumber = latestFullyFetchedBlock !== undefined ? Primitive_int.min(latestFullyFetchedBlock, knownHeight) : knownHeight;
+      latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockRegistrations, startBlock, progressBlockNumber, maxBlockNumber, maxOnBlockBufferSize);
+    } else {
+      latestOnBlockBlockNumber = progressBlockNumber;
+    }
   }
   return {
     optimizedPartitions: optimizedPartitions,
@@ -2111,6 +2129,7 @@ let chunkRangeGrowthFactor = 1.8;
 export {
   deriveSelectionStartBlock,
   makeSelection,
+  floorAtSelectionStart,
   withAddresses,
   narrowSelectionToRange,
   densityItemsTarget,
@@ -2120,7 +2139,6 @@ export {
   getMinQueryRange,
   OptimizedPartitions,
   bufferBlockNumber,
-  bufferBlock,
   bufferReadyCount,
   getRegistrationIndex,
   comparePath,