envio 3.13.0-alpha.0 → 3.13.0

package/src/ChainState.res

@@ -43,6 +43,10 @@ type t = {
   maxReorgDepth: int,
   // One-way: past it, everything the chain writes can still be rolled back.
   mutable isInReorgThreshold: bool,
+  // How this chain spells the addresses materialised out of its stores, from
+  // `lowercaseAddresses`. The stores hold raw bytes; the spelling is decided
+  // where they are read.
+  shouldChecksum: bool,
   // Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
   // transactionIndex). Fetch responses merge their page in; entries are pruned
   // as the chain progresses and dropped above the target on rollback.
@@ -58,9 +62,10 @@ type t = {
   mutable blockRangeFetchCount: float,
   mutable blockRangeFetchedEvents: float,
   mutable blockRangeFetchedBlocks: float,
-  // What this chain last reported reaching. The head moves, so a chain reaches
-  // it again on every catch-up; only a new milestone is worth a line.
-  mutable reportedFetchedTo: option<string>,
+  // Whether this chain has said where it finished indexing. Being finished
+  // stays true for the rest of the run, and every pass over the chains would
+  // say so again.
+  mutable reportedFinished: bool,
   mutable reorgCount: int,
   mutable reorgDetectedBlock: option<int>,
   mutable rollbackTargetBlock: option<int>,
@@ -139,9 +144,10 @@ let make = (
   ~numEventsProcessed=0.,
   ~timestampCaughtUpToHeadOrEndblock=None,
   ~isProgressAtHead=false,
-  ~transactionStore=TransactionStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
+  ~transactionStore=TransactionStore.make(~ecosystem=Ecosystem.Evm),
   ~chainDensity=None,
-  ~blockStore=BlockStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
+  ~blockStore=BlockStore.make(~ecosystem=Ecosystem.Evm),
+  ~shouldChecksum=false,
   ~reorgThresholdReadyTolerance=100,
   ~perChainEntities: array<Internal.entityConfig>=[],
   ~logger: Pino.t,
@@ -168,13 +174,14 @@ let make = (
     isInReorgThreshold,
     transactionStore,
     blockStore,
+    shouldChecksum,
     reorgThresholdReadyTolerance,
     blockRangeFetchSeconds: 0.,
     blockRangeParseSeconds: 0.,
     blockRangeFetchCount: 0.,
     blockRangeFetchedEvents: 0.,
     blockRangeFetchedBlocks: 0.,
-    reportedFetchedTo: None,
+    reportedFinished: false,
     reorgCount: 0,
     reorgDetectedBlock: None,
     rollbackTargetBlock: None,
@@ -232,7 +239,6 @@ let makeInternal = (
   // chain's addresses into it, and every source client holds the same handle.
   let addressStore = AddressStore.make(
     ~ecosystem=config.ecosystem.name,
-    ~shouldChecksum=!lowercaseAddresses,
     ~contracts=AddressStore.contractsOf(~onEventRegistrations, ~contractMapping),
   )
 
@@ -277,17 +283,18 @@ let makeInternal = (
     }
   })
 
+  // Before the sources, which are constructed holding them.
+  let blockStore = BlockStore.make(~ecosystem=config.ecosystem.name)
+  let transactionStore = TransactionStore.make(~ecosystem=config.ecosystem.name)
+
   // Create sources lazily here - this is where API token validation happens
   let sources = ChainSources.make(
     ~chainConfig,
     ~onEventRegistrations,
     ~addressStore,
     ~lowercaseAddresses,
-  )
-
-  let blockStore = BlockStore.make(
-    ~ecosystem=config.ecosystem.name,
-    ~shouldChecksum=!lowercaseAddresses,
+    ~blockStore,
+    ~transactionStore,
   )
 
   // Seed the stored reorg checkpoints (hash-only rows) so detection resumes
@@ -299,7 +306,6 @@ let makeInternal = (
         blockHash: cp.blockHash,
       }),
       ~ecosystem=config.ecosystem.name,
-      ~shouldChecksum=!lowercaseAddresses,
     )
     blockStore
     ->BlockStore.merge(seedPage, ~fromBlock=0, ~reportOnly=false)
@@ -333,12 +339,10 @@ let makeInternal = (
     ~perChainEntities=config.userEntities->EntityTables.perChain,
     ~timestampCaughtUpToHeadOrEndblock,
     ~numEventsProcessed,
-    ~transactionStore=TransactionStore.make(
-      ~ecosystem=config.ecosystem.name,
-      ~shouldChecksum=!lowercaseAddresses,
-    ),
+    ~transactionStore,
     ~chainDensity,
     ~blockStore,
+    ~shouldChecksum=!lowercaseAddresses,
     ~reorgThresholdReadyTolerance=config.reorgThresholdReadyTolerance,
     ~logger,
   )
@@ -398,6 +402,8 @@ let makeFromDbState = (
 
 let logger = (cs: t) => cs.logger
 let blockStore = (cs: t) => cs.blockStore
+let transactionStore = (cs: t) => cs.transactionStore
+let shouldChecksum = (cs: t) => cs.shouldChecksum
 let entities = (cs: t) => cs.entities
 
 // Rollback discards every uncommitted change, so this chain's partition is
@@ -420,7 +426,6 @@ let getLatestValidScannedBlock = (cs: t, ~blockStore: BlockStore.t, ~blockNumber
   ->BlockStore.latestValidBlockFromStore(blockStore, blockNumbers)
   ->Null.toOption
 let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
-let isProgressAtHead = (cs: t) => cs.isProgressAtHead
 let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
 let committedProgressBlockTime = (cs: t) => cs.committedProgressBlockTime
 let numEventsProcessed = (cs: t) => cs.numEventsProcessed
@@ -454,7 +459,7 @@ let setRollbackTargetBlock = (cs: t, ~blockNumber) => cs.rollbackTargetBlock = S
 // rather than reaching into it.
 let knownHeight = (cs: t) => cs.fetchState.knownHeight
 let contractAddresses = (cs: t, ~contractName) =>
-  cs.addressStore->AddressStore.contractAddresses(contractName)
+  cs.addressStore->AddressStore.contractAddresses(contractName, ~shouldChecksum=cs.shouldChecksum)
 
 // Hands over the dynamically registered addresses the database hasn't seen yet,
 // up to the block the caller is about to commit, each paired with the checkpoint
@@ -646,37 +651,6 @@ let dispatch = (
 
 // --- Derived (pure). ---
 
-%%private(
-  // Where the fetch frontier has just landed. Below the reorg threshold a chain
-  // can only fetch the finalized range, so reaching the end of it is not
-  // reaching the head — the rest opens up once the indexer enters the threshold.
-  let fetchedTo = (cs: t) =>
-  switch cs.fetchState.endBlock {
-  | Some(endBlock) if endBlock <= cs.fetchState.knownHeight - cs.fetchState.blockLag => (
-      "the end block",
-      endBlock,
-    )
-  | _ =>
-    cs.isInReorgThreshold
-      ? ("the chain head", cs.fetchState.knownHeight)
-      : ("the safe block", cs.fetchState.knownHeight - cs.fetchState.blockLag)
-  }
-)
-
-// What this chain has just reached, the first time it reaches it. `None` once
-// it has been reported: a chain catches up to a moving head over and over, and
-// the milestone is the same one every time. A new one — the head past the
-// threshold, an end block — is a line of its own.
-let takeFetchedTo = (cs: t) => {
-  let (target, block) = cs->fetchedTo
-  if cs.reportedFetchedTo == Some(target) {
-    None
-  } else {
-    cs.reportedFetchedTo = Some(target)
-    Some((target, block))
-  }
-}
-
 let hasProcessedToEndblock = (cs: t) => {
   let {committedProgressBlockNumber, fetchState} = cs
   switch fetchState.endBlock {
@@ -709,6 +683,45 @@ let isDurablyCaughtUp = (cs: t) => {
   atEndBlock || atHead
 }
 
+// This chain has indexed everything there was to index: it reached its end
+// block, or its progress reached the head. The single reading of that, for
+// everything that turns on it — what the chain reports, and whether the whole
+// process may finalize.
+//
+// Both spellings, because neither covers the other: `isProgressAtHead` latches
+// the moment a batch's progress met the head known when it was created, which a
+// head that has since moved on would read as behind; `isDurablyCaughtUp` judges
+// the head as it stands, which is all a run resumed at the head has — it has no
+// batch to latch anything.
+let hasCaughtUp = (cs: t) => cs.isProgressAtHead || cs->isDurablyCaughtUp
+
+// Where this chain has finished indexing, the first time it gets there.
+// `EndBlock` is terminal: the chain indexed everything it was configured to.
+// `Backfill` is the rest of the history, up to the point where blocks can
+// still be reorged, which is as far as a chain indexes before the indexer
+// crosses into them.
+type finished = EndBlock(int) | Backfill(int)
+
+let takeFinished = (cs: t) =>
+  if cs.reportedFinished {
+    None
+  } else {
+    switch (cs.fetchState.endBlock, cs->hasProcessedToEndblock, cs->hasCaughtUp) {
+    | (Some(endBlock), true, _) => {
+        cs.reportedFinished = true
+        Some(EndBlock(endBlock))
+      }
+    | (_, _, true) => {
+        cs.reportedFinished = true
+        // Finishing a backfill happens in a run, and a chain resumed with its
+        // `ready_at` already committed did none in this one. Being at the end
+        // block, above, stays true across runs and is said on every one.
+        cs->isReady ? None : Some(Backfill(cs.committedProgressBlockNumber))
+      }
+    | _ => None
+    }
+  }
+
 let getHighestBlockBelowThreshold = (cs: t): int => {
   let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.maxReorgDepth
   highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
@@ -765,7 +778,7 @@ let groupBatchItems = (items: array<Internal.item>): (transactionGroups, blockGr
       let {blockNumber} = eventItem
 
       switch eventItem.payload->Internal.getPayloadTransaction->Nullable.toOption {
-      | Some(_) => () // RPC/simulate/Fuel carry the transaction inline.
+      | Some(_) => () // Fuel and Simulate carry the transaction inline.
       | None =>
         let {transactionIndex} = eventItem
         let mask = eventItem.onEventRegistration.fieldSelection.transactionMask
@@ -792,7 +805,7 @@ let groupBatchItems = (items: array<Internal.item>): (transactionGroups, blockGr
       }
 
       switch eventItem.payload->Internal.getPayloadBlock->Nullable.toOption {
-      | Some(_) => () // RPC/simulate carry the block inline.
+      | Some(_) => () // Simulate carries the block inline.
       | None =>
         let mask = eventItem.onEventRegistration.fieldSelection.blockMask
         let last = blockItemGroups->Array.length - 1
@@ -831,13 +844,18 @@ let groupBatchItems = (items: array<Internal.item>): (transactionGroups, blockGr
 // `groupBatchItems`). Store-backed items always get a transaction object - the
 // selected fields, or `{}` when nothing was selected - so `event.transaction`
 // is never `undefined` (matching the inline sources).
-let applyTransactionGroups = async (store: TransactionStore.t, g: transactionGroups) => {
+let applyTransactionGroups = async (
+  store: TransactionStore.t,
+  g: transactionGroups,
+  ~shouldChecksum,
+) => {
   if g.payloadGroups->Utils.Array.notEmpty {
     if g.anyTransactionFieldSelected {
       let txs = await store->TransactionStore.materialize(
         ~blockNumbers=g.txBlockNumbers,
         ~transactionIndices=g.transactionIndices,
         ~masks=g.transactionMasks,
+        ~shouldChecksum,
       )
       g.payloadGroups->Array.forEachWithIndex((payloads, i) => {
         let tx = txs->Array.getUnsafe(i)
@@ -858,11 +876,12 @@ let applyTransactionGroups = async (store: TransactionStore.t, g: transactionGro
 }
 
 // Materialise a `BlockStore` against precomputed groups (see `groupBatchItems`).
-let applyBlockGroups = async (store: BlockStore.t, g: blockGroups) => {
+let applyBlockGroups = async (store: BlockStore.t, g: blockGroups, ~shouldChecksum) => {
   if g.blockItemGroups->Utils.Array.notEmpty {
     let blocks = await store->BlockStore.materialize(
       ~blockNumbers=g.blockBlockNumbers,
       ~masks=g.blockMasks,
+      ~shouldChecksum,
     )
     g.blockItemGroups->Array.forEachWithIndex((group, i) => {
       let block = blocks->Array.getUnsafe(i)
@@ -878,27 +897,26 @@ let applyBlockGroups = async (store: BlockStore.t, g: blockGroups) => {
 let materializeBatchItems = async (cs: t, ~items: array<Internal.item>) => {
   let (txGroups, blockGroups) = items->groupBatchItems
   let _ = await Promise.all2((
-    cs.transactionStore->applyTransactionGroups(txGroups),
-    cs.blockStore->applyBlockGroups(blockGroups),
+    cs.transactionStore->applyTransactionGroups(txGroups, ~shouldChecksum=cs.shouldChecksum),
+    cs.blockStore->applyBlockGroups(blockGroups, ~shouldChecksum=cs.shouldChecksum),
   ))
 }
 
 // Materialise a fetch-response's transactions and blocks onto its items before
-// contract-register handlers read them. Transactions come from the response's
-// page (`None` when kept inline); blocks come from the chain store, which the
-// response's page was already merged into by `registerReorgGuard`.
+// contract-register handlers read them. Both come from the chain stores, which
+// the response's pages were merged into once the reorg guard passed: a source
+// that consults those stores to decide what to fetch leaves out what they
+// already hold, so its page alone does not cover its own items.
 let materializePageItems = async (
   ~items: array<Internal.item>,
-  ~transactionStore: option<TransactionStore.t>,
+  ~transactionStore: TransactionStore.t,
   ~blockStore: BlockStore.t,
+  ~shouldChecksum: bool,
 ) => {
   let (txGroups, blockGroups) = items->groupBatchItems
   let _ = await Promise.all2((
-    switch transactionStore {
-    | Some(store) => store->applyTransactionGroups(txGroups)
-    | None => Promise.resolve()
-    },
-    blockStore->applyBlockGroups(blockGroups),
+    transactionStore->applyTransactionGroups(txGroups, ~shouldChecksum),
+    blockStore->applyBlockGroups(blockGroups, ~shouldChecksum),
   ))
 }
 
@@ -909,16 +927,7 @@ let handleQueryResult = (
   ~newRegistrations,
   ~latestFetchedBlock: int,
   ~knownHeight,
-  ~transactionStore as txPage: option<TransactionStore.t>,
 ) => {
-  // Merge this response's transaction page into the chain store in lockstep
-  // with appending its items to the buffer. Inline sources contribute no page;
-  // the block page was already merged by `registerReorgGuard`.
-  switch txPage {
-  | Some(page) => cs.transactionStore->TransactionStore.merge(page)
-  | None => ()
-  }
-
   let fs = switch newRegistrations {
   | [] => cs.fetchState
   | _ =>
@@ -999,6 +1008,34 @@ let isInReorgThreshold = (cs: t) => cs.isInReorgThreshold
 // progress has run.
 let shouldSaveHistory = (cs: t) =>
   cs.shouldRollbackOnReorg && cs.maxReorgDepth > 0 && cs.isInReorgThreshold
+// What crossing into the recent blocks changed for this chain. Until now it
+// stopped short of the head by its reorg depth, because it kept nothing it
+// could roll back with; crossing lifts both at once. The second half is the
+// answer to why the indexer starts writing more than it was.
+// Whether crossing gives this chain anything more to index: the last block it
+// may fetch afterwards against the last it may fetch now. Read before the
+// crossing, while the lag being lifted is still the one in place.
+//
+// False wherever the lag was never holding this chain back, which is every
+// configuration that also keeps no history: a chain that isn't rolled back on a
+// reorg, or has no reorg depth, already fetches as far as it ever will. It is
+// false too for a chain whose end block sits below the blocks being opened up,
+// which will never reach one of them.
+let reorgThresholdLiftsCeiling = (cs: t) => {
+  let ceiling = (~blockLag) => {
+    let head = Pervasives.max(0, cs.fetchState.knownHeight - blockLag)
+    switch cs.fetchState.endBlock {
+    | Some(endBlock) => Pervasives.min(endBlock, head)
+    | None => head
+    }
+  }
+  ceiling(~blockLag=cs.chainConfig.blockLag) > ceiling(~blockLag=cs.fetchState.blockLag)
+}
+
+// Only ever said by a chain the crossing lifts, and lifting it takes a reorg
+// depth to have been held back by, which is the same thing that makes the
+// history below worth keeping. So there is no second form without it.
+let reorgThresholdEntryMessage = "Indexing the latest blocks now. These can be reorged, so the indexer starts storing a history of every change to roll back with."
 
 // Snapshot the chain's metadata fields for staging into the chains table.
 let toChainMetadata = (cs: t): InternalTable.Chains.metaFields => {
@@ -1090,9 +1127,7 @@ let toChainBeforeBatch = (cs: t, ~isRealtime): Batch.chainBeforeBatch => {
 // batch would leave it. Entering the threshold is what lifts the pre-threshold
 // lag, so a chain waiting to enter it has fetched as far as it can.
 let isReadyToEnterReorgThreshold = (cs: t) =>
-  cs.fetchState->FetchState.isReadyToEnterReorgThreshold(
-    ~tolerance=cs.reorgThresholdReadyTolerance,
-  )
+  cs.fetchState->FetchState.isReadyToEnterReorgThreshold(~tolerance=cs.reorgThresholdReadyTolerance)
 
 let isReadyToEnterReorgThresholdAfterBatch = (cs: t, ~batch: Batch.t) => {
   let fetchState = switch batch.progressedChainsById->ChainId.Dict.dangerouslyGetNonOption(
@@ -1242,6 +1277,7 @@ let markReady = (cs: t, ~readyAt) =>
 let rollbackCommittedProgress = (cs: t, blockNumber) =>
   if blockNumber !== cs.committedProgressBlockNumber {
     cs.committedProgressBlockNumber = blockNumber
+
     // Exact block only: the rolled-back region is about to be refetched, and
     // the next batch re-establishes the time either way.
     cs.committedProgressBlockTime =