envio 3.11.0 → 3.12.1

package/src/CrossChainState.res.mjs

@@ -6,8 +6,10 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as HistoryPolicy from "./HistoryPolicy.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
+import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
 function calculateTargetBufferSize() {
@@ -18,7 +20,7 @@ function calculateTargetBufferSize() {
   }
 }
 
-function make(chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt) {
+function make(chainStates, isRealtime, targetBufferSizeOpt) {
   let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : (
       Env.targetBufferSize !== undefined ? Env.targetBufferSize : 100000
     );
@@ -27,7 +29,6 @@ function make(chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt)
     chainIds: Object.values(chainStates).map(cs => ChainState.chainConfig(cs).id),
     isRealtime: isRealtime,
     isCaughtUp: isRealtime,
-    isInReorgThreshold: isInReorgThreshold,
     targetBufferSize: targetBufferSize
   };
 }
@@ -40,18 +41,31 @@ function isRealtime(crossChainState) {
   return crossChainState.isRealtime;
 }
 
+function chainIds(crossChainState) {
+  return crossChainState.chainIds;
+}
+
 function isCaughtUp(crossChainState) {
   return crossChainState.isCaughtUp;
 }
 
 function isInReorgThreshold(crossChainState) {
-  return crossChainState.isInReorgThreshold;
+  let chainStates = Object.values(crossChainState.chainStates);
+  if (Utils.$$Array.notEmpty(chainStates)) {
+    return chainStates.every(ChainState.isInReorgThreshold);
+  } else {
+    return false;
+  }
 }
 
 function targetBufferSize(crossChainState) {
   return crossChainState.targetBufferSize;
 }
 
+function shouldSaveHistory(crossChainState) {
+  return Utils.Dict.mapValues(crossChainState.chainStates, ChainState.shouldSaveHistory);
+}
+
 function totalReadyCount(crossChainState) {
   let total = 0;
   for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
@@ -65,21 +79,19 @@ function nextItemIsNone(crossChainState) {
   return !Object.values(crossChainState.chainStates).some(ChainState.hasReadyItem);
 }
 
-function getSafeCheckpointIdByChain(crossChainState, committedCheckpointId) {
+function getSafeCheckpointIdByChain(crossChainState, sequence, committedFrontier) {
   return crossChainState.chainIds.map(chainId => {
     let cs = crossChainState.chainStates[chainId];
     let tracking = ChainState.safeCheckpointTracking(cs);
     return [
       chainId,
-      tracking !== undefined ? SafeCheckpointTracking.getSafeCheckpointId(tracking, ChainState.knownHeight(cs)) : committedCheckpointId
+      tracking !== undefined ? SafeCheckpointTracking.getSafeCheckpointId(tracking, ChainState.knownHeight(cs)) : CheckpointSequence.position(sequence, committedFrontier, chainId)
     ];
   });
 }
 
-function createBatch(crossChainState, processedCheckpointId, batchSizeTarget, isRollback) {
-  return Batch.make(processedCheckpointId + (
-    isRollback ? 1n : 0n
-  ), Utils.Dict.mapValues(crossChainState.chainStates, ChainState.toChainBeforeBatch), batchSizeTarget, crossChainState.isInReorgThreshold);
+function createBatch(crossChainState, config, frontier, batchSizeTarget) {
+  return Batch.make(config.checkpointSequence, frontier, Utils.Dict.mapValues(crossChainState.chainStates, none => ChainState.toChainBeforeBatch(none, crossChainState.isRealtime)), batchSizeTarget, HistoryPolicy.decide(config, shouldSaveHistory(crossChainState)));
 }
 
 function enterReorgThreshold(crossChainState) {
@@ -87,7 +99,6 @@ function enterReorgThreshold(crossChainState) {
   for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
     ChainState.enterReorgThreshold(crossChainState.chainStates[crossChainState.chainIds[i]]);
   }
-  crossChainState.isInReorgThreshold = true;
 }
 
 function applyBatchProgress(crossChainState, batch, blockTimestampName) {
@@ -101,13 +112,6 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
     }
   }
   crossChainState.isCaughtUp = crossChainState.isCaughtUp || nextItemIsNone(crossChainState) && everyChainCaughtUp;
-  let allChainsReady = true;
-  for (let i$1 = 0, i_finish$1 = chainIds.length; i$1 < i_finish$1; ++i$1) {
-    if (!ChainState.isReady(crossChainState.chainStates[chainIds[i$1]])) {
-      allChainsReady = false;
-    }
-  }
-  crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady;
 }
 
 function isSettledAtHead(crossChainState) {
@@ -129,17 +133,13 @@ function markCaughtUpIfSettled(crossChainState) {
 
 function markCaughtUpOnResume(crossChainState) {
   let everyChainCaughtUp = crossChainState.chainIds.length !== 0;
-  let everyChainReady = true;
   for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
     let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
     if (!ChainState.isDurablyCaughtUp(cs)) {
       everyChainCaughtUp = false;
     }
-    if (!ChainState.isReady(cs)) {
-      everyChainReady = false;
-    }
   }
-  if (everyChainCaughtUp && !everyChainReady && nextItemIsNone(crossChainState)) {
+  if (everyChainCaughtUp && !crossChainState.isRealtime && nextItemIsNone(crossChainState)) {
     crossChainState.isCaughtUp = true;
     return;
   }
@@ -199,7 +199,7 @@ async function checkAndFetch(crossChainState, dispatchChain) {
     let chainTargetItems = (
       isCold ? Primitive_float.min(remaining.contents, minimumAdmissionBudget) : remaining.contents
     ) + ChainState.pendingBudget(cs);
-    let maxTargetBlock = alignment !== undefined && alignment[0] !== chainId ? ChainState.blockAtProgress(cs, alignment[1] + 0.1) : undefined;
+    let maxTargetBlock = alignment !== undefined && alignment[0] !== chainId ? ChainState.blockAtProgress(cs, alignment[1] + 0.2) : undefined;
     let action = ChainState.getNextQuery(cs, chainTargetItems, maxTargetBlock);
     if (typeof action !== "object") {
       if (action === "WaitingForNewBlock") {
@@ -249,9 +249,11 @@ export {
   make,
   chainStates,
   isRealtime,
+  chainIds,
   isCaughtUp,
   isInReorgThreshold,
   targetBufferSize,
+  shouldSaveHistory,
   nextItemIsNone,
   getSafeCheckpointIdByChain,
   createBatch,

package/src/CrossChainState.resi

@@ -5,34 +5,27 @@ type t
 
 let calculateTargetBufferSize: unit => int
 
-let make: (
-  ~chainStates: dict<ChainState.t>,
-  ~isInReorgThreshold: bool,
-  ~isRealtime: bool,
-  ~targetBufferSize: int=?,
-) => t
+let make: (~chainStates: dict<ChainState.t>, ~isRealtime: bool, ~targetBufferSize: int=?) => t
 
 // Accessors.
 let chainStates: t => dict<ChainState.t>
 let isRealtime: t => bool
+let chainIds: t => array<ChainId.t>
 let isCaughtUp: t => bool
 let isInReorgThreshold: t => bool
 let targetBufferSize: t => int
+let shouldSaveHistory: t => dict<bool>
 
 // Derived (pure).
 let nextItemIsNone: t => bool
 let getSafeCheckpointIdByChain: (
   t,
-  ~committedCheckpointId: Internal.checkpointId,
+  ~sequence: CheckpointSequence.t,
+  ~committedFrontier: Frontier.t,
 ) => array<(ChainId.t, option<Internal.checkpointId>)>
 
 // Cross-chain transitions.
-let createBatch: (
-  t,
-  ~processedCheckpointId: Internal.checkpointId,
-  ~batchSizeTarget: int,
-  ~isRollback: bool,
-) => Batch.t
+let createBatch: (t, ~config: Config.t, ~frontier: Frontier.t, ~batchSizeTarget: int) => Batch.t
 let enterReorgThreshold: t => unit
 let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
 let markCaughtUpIfSettled: t => unit

package/src/EffectState.res

@@ -36,15 +36,24 @@ type effectStats = {
   mutable hasCache: bool,
 }
 
+// A handler output, stamped with the checkpoint that produced it on the chain
+// whose handler ran it. A cross-chain effect's entries come from several
+// chains, and under per-chain sequences an id only compares against its own
+// chain's committed one.
+type cacheEntry = {
+  output: Internal.effectOutput,
+  chainId: ChainId.t,
+  checkpointId: Internal.checkpointId,
+}
+
 type effectCacheInMemTable = {
   // Cache keys whose handler output is persisted on the next write. Drained
   // each write; eviction is driven by the per-entry checkpointId instead.
   mutable idsToStore: array<string>,
   mutable invalidationsCount: int,
-  // Each entry is stamped with the checkpoint that referenced it (or
-  // loadedFromDbCheckpointId for db reads), so committed entries can be
-  // dropped once persisted/re-derivable, mirroring entity changes.
-  mutable dict: dict<Change.t<Internal.effectOutput>>,
+  // Committed entries are dropped once persisted or re-derivable, mirroring
+  // entity changes; a db read is stamped loadedFromDbCheckpointId.
+  mutable dict: dict<cacheEntry>,
   mutable changesCount: float,
   effect: Internal.effect,
   // The scope this cache is for and its resolved db table (the address). The

package/src/EffectState.resi

@@ -11,10 +11,16 @@ type effectRateLimitState = {
 // toMetrics.
 type effectStats
 
+type cacheEntry = {
+  output: Internal.effectOutput,
+  chainId: ChainId.t,
+  checkpointId: Internal.checkpointId,
+}
+
 type effectCacheInMemTable = {
   mutable idsToStore: array<string>,
   mutable invalidationsCount: int,
-  mutable dict: dict<Change.t<Internal.effectOutput>>,
+  mutable dict: dict<cacheEntry>,
   mutable changesCount: float,
   effect: Internal.effect,
   scope: Internal.chainScope,

package/src/EventProcessing.res

@@ -157,9 +157,9 @@ let preloadBatchOrThrow = async (
 
   for checkpointIdx in 0 to batch.checkpointIds->Array.length - 1 {
     let checkpointId = batch.checkpointIds->Array.getUnsafe(checkpointIdx)
-    let checkpointEventsProcessed = batch.checkpointEventsProcessed->Array.getUnsafe(checkpointIdx)
+    let checkpointItemsCount = batch.checkpointItemsCount->Array.getUnsafe(checkpointIdx)
 
-    for idx in 0 to checkpointEventsProcessed - 1 {
+    for idx in 0 to checkpointItemsCount - 1 {
       let item = batch.items->Array.getUnsafe(itemIdx.contents + idx)
       switch item {
       | Event(_) =>
@@ -233,7 +233,7 @@ let preloadBatchOrThrow = async (
       }
     }
 
-    itemIdx := itemIdx.contents + checkpointEventsProcessed
+    itemIdx := itemIdx.contents + checkpointItemsCount
   }
 
   let _ = await Promise.all(promises)
@@ -251,9 +251,9 @@ let runBatchHandlersOrThrow = async (
 
   for checkpointIdx in 0 to batch.checkpointIds->Array.length - 1 {
     let checkpointId = batch.checkpointIds->Array.getUnsafe(checkpointIdx)
-    let checkpointEventsProcessed = batch.checkpointEventsProcessed->Array.getUnsafe(checkpointIdx)
+    let checkpointItemsCount = batch.checkpointItemsCount->Array.getUnsafe(checkpointIdx)
 
-    for idx in 0 to checkpointEventsProcessed - 1 {
+    for idx in 0 to checkpointItemsCount - 1 {
       let item = batch.items->Array.getUnsafe(itemIdx.contents + idx)
 
       await runHandlerOrThrow(
@@ -266,7 +266,7 @@ let runBatchHandlersOrThrow = async (
         ~chains,
       )
     }
-    itemIdx := itemIdx.contents + checkpointEventsProcessed
+    itemIdx := itemIdx.contents + checkpointItemsCount
   }
 }
 

package/src/EventProcessing.res.mjs

@@ -110,8 +110,8 @@ async function preloadBatchOrThrow(batch, loadManager, persistence, config, inde
   let itemIdx = 0;
   for (let checkpointIdx = 0, checkpointIdx_finish = batch.checkpointIds.length; checkpointIdx < checkpointIdx_finish; ++checkpointIdx) {
     let checkpointId = batch.checkpointIds[checkpointIdx];
-    let checkpointEventsProcessed = batch.checkpointEventsProcessed[checkpointIdx];
-    for (let idx = 0; idx < checkpointEventsProcessed; ++idx) {
+    let checkpointItemsCount = batch.checkpointItemsCount[checkpointIdx];
+    for (let idx = 0; idx < checkpointItemsCount; ++idx) {
       let item = batch.items[itemIdx + idx | 0];
       if (item.kind === 0) {
         let match = item.onEventRegistration;
@@ -158,7 +158,7 @@ async function preloadBatchOrThrow(batch, loadManager, persistence, config, inde
         }
       }
     }
-    itemIdx = itemIdx + checkpointEventsProcessed | 0;
+    itemIdx = itemIdx + checkpointItemsCount | 0;
   }
   await Promise.all(promises);
 }
@@ -167,12 +167,12 @@ async function runBatchHandlersOrThrow(batch, indexerState, loadManager, persist
   let itemIdx = 0;
   for (let checkpointIdx = 0, checkpointIdx_finish = batch.checkpointIds.length; checkpointIdx < checkpointIdx_finish; ++checkpointIdx) {
     let checkpointId = batch.checkpointIds[checkpointIdx];
-    let checkpointEventsProcessed = batch.checkpointEventsProcessed[checkpointIdx];
-    for (let idx = 0; idx < checkpointEventsProcessed; ++idx) {
+    let checkpointItemsCount = batch.checkpointItemsCount[checkpointIdx];
+    for (let idx = 0; idx < checkpointItemsCount; ++idx) {
       let item = batch.items[itemIdx + idx | 0];
       await runHandlerOrThrow(item, checkpointId, indexerState, loadManager, persistence, config, chains);
     }
-    itemIdx = itemIdx + checkpointEventsProcessed | 0;
+    itemIdx = itemIdx + checkpointItemsCount | 0;
   }
 }
 

package/src/FetchState.res

@@ -50,6 +50,22 @@ let makeSelection = (~onEventRegistrations, ~dependsOnAddresses, ~clientFiltered
   startBlock: ?deriveSelectionStartBlock(onEventRegistrations),
 }
 
+// A partition's frontier never sits below the block before its selection can
+// first match: the blocks below hold nothing for it, so they count as fetched
+// the same way the blocks before an address's start block do. The chain's
+// buffer frontier is its lowest partition frontier, so a partition left at the
+// chain start would hold every other partition's events back as unprocessable
+// and keep the chain's query target — sized from that frontier — short of
+// anything worth asking for. A start block the chain has not reached yet leaves
+// the partition waiting there, like an address partition does, rather than
+// scanning empty ranges up to it; bufferBlockNumber caps the chain's frontier at
+// the head meanwhile.
+let floorAtSelectionStart = (latestFetchedBlock, ~selection) =>
+  switch selection.startBlock {
+  | Some(startBlock) => Pervasives.max(latestFetchedBlock, startBlock - 1)
+  | None => latestFetchedBlock
+  }
+
 type pendingQuery = {
   fromBlock: int,
   toBlock: option<int>,
@@ -398,6 +414,10 @@ module OptimizedPartitions = {
     ~dynamicContracts: Utils.Set.t<string>,
     ~clientFilteredContracts: Utils.Set.t<string>,
   ) => {
+    let partitions = partitions->Array.map(p => {
+      let floored = p.latestFetchedBlock->floorAtSelectionStart(~selection=p.selection)
+      floored === p.latestFetchedBlock ? p : {...p, latestFetchedBlock: floored}
+    })
     let newPartitions = []
     let mergingPartitions = Dict.make()
     let nextPartitionIndexRef = ref(nextPartitionIndex)
@@ -847,28 +867,17 @@ type t = {
   clientFilterAddressThreshold: option<int>,
 }
 
+// The latest block whose items are all in the buffer: the lowest partition
+// frontier, held back by the onBlock pointer. Without onBlock registrations the
+// pointer tracks the known height, so this also caps the frontier at the head: a
+// partition can sit past it, on a response that reported a block the chain
+// hadn't heard of yet or on a start block the chain hasn't reached.
 @inline
 let bufferBlockNumber = ({latestOnBlockBlockNumber, optimizedPartitions}: t) => {
   switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
   | None => latestOnBlockBlockNumber
   | Some(latestFullyFetchedBlock) =>
-    latestOnBlockBlockNumber < latestFullyFetchedBlock
-      ? latestOnBlockBlockNumber
-      : latestFullyFetchedBlock
-  }
-}
-
-/**
-* Returns the latest block which is ready to be consumed
-*/
-@inline
-let bufferBlock = ({optimizedPartitions, latestOnBlockBlockNumber}: t) => {
-  switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
-  | None => latestOnBlockBlockNumber
-  | Some(latestFullyFetchedBlock) =>
-    latestOnBlockBlockNumber < latestFullyFetchedBlock
-      ? latestOnBlockBlockNumber
-      : latestFullyFetchedBlock
+    Pervasives.min(latestOnBlockBlockNumber, latestFullyFetchedBlock)
   }
 }
 
@@ -979,6 +988,20 @@ let compareBufferItem = (a: Internal.item, b: Internal.item): int => {
   }
 }
 
+// Whether two adjacent buffer items came from one log routed to two
+// registrations: everything `compareBufferItem` orders on except the
+// registration index is equal. Only meaningful on neighbours of a sorted
+// buffer, where such items sit next to each other.
+let isSameLog = (a: Internal.item, b: Internal.item): bool =>
+  a->Internal.getItemKind === 0 &&
+  b->Internal.getItemKind === 0 &&
+  a->Internal.getItemBlockNumber === b->Internal.getItemBlockNumber &&
+  a->Internal.getItemLogIndex === b->Internal.getItemLogIndex &&
+  switch (a->Internal.getItemOrderPath, b->Internal.getItemOrderPath) {
+  | (Value(pa), Value(pb)) => comparePath(pa, pb) === 0
+  | _ => true
+  }
+
 // Merge a maybe-unsorted `newItems` run into the already-sorted, already-deduped
 // `buffer`, dropping items equal on every component of `compareBufferItem`.
 // Single linear pass over both runs after ordering `newItems` in place; every
@@ -1124,14 +1147,20 @@ let updateInternal = (
     // Use maxOnBlockBufferSize to get the last target item in the buffer
     // (sorted, so this is the highest-block item within the buffer cap).
     // All this needed to prevent OOM when adding too many block items to the queue
-    let maxBlockNumber = switch base->Array.get(fetchState.maxOnBlockBufferSize - 1) {
-    | Some(item) => item->Internal.getItemBlockNumber
-    | None =>
-      switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
-      | None => knownHeight
-      | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
-      }
-    }
+    // Never past the head: a partition can sit there, on a start block the
+    // chain hasn't reached, and a response is applied before the height it
+    // reported, so its events can run past the height still known here.
+    let maxBlockNumber = Pervasives.min(
+      switch base->Array.get(fetchState.maxOnBlockBufferSize - 1) {
+      | Some(item) => item->Internal.getItemBlockNumber
+      | None =>
+        switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
+        | None => knownHeight
+        | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
+        }
+      },
+      knownHeight,
+    )
     appendOnBlockItems(
       ~mutItems=blockItems,
       ~onBlockRegistrations,
@@ -2151,27 +2180,11 @@ let walkPartitionPending = (
     pqIdx := pqIdx.contents + 1
   }
 
-  // Nothing in this partition's selection can match below its earliest start
-  // block, so forward work skips straight to it instead of scanning up to it and
-  // discarding whole pages. Only the cursor moves — `latestFetchedBlock` still
-  // advances solely on a response, so no block is ever reported fetched that
-  // wasn't. Bounded by the head and the query end block: past either, the
-  // partition would offer no candidate at all, so it queries as before rather
-  // than going quiet until the chain reaches its start block.
-  let cursor = switch p.selection.startBlock {
-  | Some(startBlock) if startBlock > cursor.contents =>
-    switch Utils.Math.minOptInt(Some(headBlockNumber), queryEndBlock) {
-    | Some(reachable) if startBlock <= reachable => startBlock
-    | _ => cursor.contents
-    }
-  | _ => cursor.contents
-  }
-
   canContinue.contents
     ? Some({
         partitionId,
         p,
-        cursor,
+        cursor: cursor.contents,
         chunksUsedThisCall: chunksUsedThisCall.contents,
         inFlightCount,
         queryEndBlock,
@@ -2766,10 +2779,17 @@ let make = (
   // fetching, so without seeding the buffer here getNextQuery would return
   // NothingToQuery and the indexer would get stuck.
   let buffer = []
-  let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockRegistrations->Utils.Array.notEmpty {
+  let latestOnBlockBlockNumber = switch onBlockRegistrations {
+  // As updateInternal keeps it: with nothing to generate per block the pointer
+  // is the head cap on the buffer frontier, and left at the progress block it
+  // would hold the frontier there until the first height update — below a
+  // partition that starts later, whose queries the chain sizes off that
+  // frontier and so never reaches.
+  | [] if knownHeight > 0 => knownHeight
+  | onBlockRegistrations if knownHeight > 0 =>
     let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
     | None => knownHeight
-    | Some(latestFullyFetchedBlock) => latestFullyFetchedBlock
+    | Some(latestFullyFetchedBlock) => Pervasives.min(latestFullyFetchedBlock, knownHeight)
     }
     appendOnBlockItems(
       ~mutItems=buffer,
@@ -2779,8 +2799,7 @@ let make = (
       ~maxBlockNumber,
       ~maxOnBlockBufferSize,
     )
-  } else {
-    progressBlockNumber
+  | _ => progressBlockNumber
   }
 
   let fetchState = {