envio 3.12.1 → 3.13.0-rc.0

package/src/ChainState.res

@@ -43,6 +43,10 @@ type t = {
   maxReorgDepth: int,
   // One-way: past it, everything the chain writes can still be rolled back.
   mutable isInReorgThreshold: bool,
+  // How this chain spells the addresses materialised out of its stores, from
+  // `lowercaseAddresses`. The stores hold raw bytes; the spelling is decided
+  // where they are read.
+  shouldChecksum: bool,
   // Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
   // transactionIndex). Fetch responses merge their page in; entries are pruned
   // as the chain progresses and dropped above the target on rollback.
@@ -58,6 +62,10 @@ type t = {
   mutable blockRangeFetchCount: float,
   mutable blockRangeFetchedEvents: float,
   mutable blockRangeFetchedBlocks: float,
+  // Whether this chain has said where it finished indexing. Being finished
+  // stays true for the rest of the run, and every pass over the chains would
+  // say so again.
+  mutable reportedFinished: bool,
   mutable reorgCount: int,
   mutable reorgDetectedBlock: option<int>,
   mutable rollbackTargetBlock: option<int>,
@@ -136,9 +144,10 @@ let make = (
   ~numEventsProcessed=0.,
   ~timestampCaughtUpToHeadOrEndblock=None,
   ~isProgressAtHead=false,
-  ~transactionStore=TransactionStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
+  ~transactionStore=TransactionStore.make(~ecosystem=Ecosystem.Evm),
   ~chainDensity=None,
-  ~blockStore=BlockStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
+  ~blockStore=BlockStore.make(~ecosystem=Ecosystem.Evm),
+  ~shouldChecksum=false,
   ~reorgThresholdReadyTolerance=100,
   ~perChainEntities: array<Internal.entityConfig>=[],
   ~logger: Pino.t,
@@ -165,12 +174,14 @@ let make = (
     isInReorgThreshold,
     transactionStore,
     blockStore,
+    shouldChecksum,
     reorgThresholdReadyTolerance,
     blockRangeFetchSeconds: 0.,
     blockRangeParseSeconds: 0.,
     blockRangeFetchCount: 0.,
     blockRangeFetchedEvents: 0.,
     blockRangeFetchedBlocks: 0.,
+    reportedFinished: false,
     reorgCount: 0,
     reorgDetectedBlock: None,
     rollbackTargetBlock: None,
@@ -228,7 +239,6 @@ let makeInternal = (
   // chain's addresses into it, and every source client holds the same handle.
   let addressStore = AddressStore.make(
     ~ecosystem=config.ecosystem.name,
-    ~shouldChecksum=!lowercaseAddresses,
     ~contracts=AddressStore.contractsOf(~onEventRegistrations, ~contractMapping),
   )
 
@@ -273,17 +283,18 @@ let makeInternal = (
     }
   })
 
+  // Before the sources, which are constructed holding them.
+  let blockStore = BlockStore.make(~ecosystem=config.ecosystem.name)
+  let transactionStore = TransactionStore.make(~ecosystem=config.ecosystem.name)
+
   // Create sources lazily here - this is where API token validation happens
   let sources = ChainSources.make(
     ~chainConfig,
     ~onEventRegistrations,
     ~addressStore,
     ~lowercaseAddresses,
-  )
-
-  let blockStore = BlockStore.make(
-    ~ecosystem=config.ecosystem.name,
-    ~shouldChecksum=!lowercaseAddresses,
+    ~blockStore,
+    ~transactionStore,
   )
 
   // Seed the stored reorg checkpoints (hash-only rows) so detection resumes
@@ -295,7 +306,6 @@ let makeInternal = (
         blockHash: cp.blockHash,
       }),
       ~ecosystem=config.ecosystem.name,
-      ~shouldChecksum=!lowercaseAddresses,
     )
     blockStore
     ->BlockStore.merge(seedPage, ~fromBlock=0, ~reportOnly=false)
@@ -329,12 +339,10 @@ let makeInternal = (
     ~perChainEntities=config.userEntities->EntityTables.perChain,
     ~timestampCaughtUpToHeadOrEndblock,
     ~numEventsProcessed,
-    ~transactionStore=TransactionStore.make(
-      ~ecosystem=config.ecosystem.name,
-      ~shouldChecksum=!lowercaseAddresses,
-    ),
+    ~transactionStore,
     ~chainDensity,
     ~blockStore,
+    ~shouldChecksum=!lowercaseAddresses,
     ~reorgThresholdReadyTolerance=config.reorgThresholdReadyTolerance,
     ~logger,
   )
@@ -394,6 +402,8 @@ let makeFromDbState = (
 
 let logger = (cs: t) => cs.logger
 let blockStore = (cs: t) => cs.blockStore
+let transactionStore = (cs: t) => cs.transactionStore
+let shouldChecksum = (cs: t) => cs.shouldChecksum
 let entities = (cs: t) => cs.entities
 
 // Rollback discards every uncommitted change, so this chain's partition is
@@ -450,7 +460,7 @@ let setRollbackTargetBlock = (cs: t, ~blockNumber) => cs.rollbackTargetBlock = S
 // rather than reaching into it.
 let knownHeight = (cs: t) => cs.fetchState.knownHeight
 let contractAddresses = (cs: t, ~contractName) =>
-  cs.addressStore->AddressStore.contractAddresses(contractName)
+  cs.addressStore->AddressStore.contractAddresses(contractName, ~shouldChecksum=cs.shouldChecksum)
 
 // Hands over the dynamically registered addresses the database hasn't seen yet,
 // up to the block the caller is about to commit, each paired with the checkpoint
@@ -650,6 +660,30 @@ let hasProcessedToEndblock = (cs: t) => {
   }
 }
 
+// Where this chain has finished indexing, the first time it gets there.
+// `EndBlock` is terminal: the chain indexed everything it was configured to.
+// `Backfill` is the rest of the history, up to the point where blocks can
+// still be reorged, which is as far as a chain indexes before the indexer
+// crosses into them.
+type finished = EndBlock(int) | Backfill(int)
+
+let takeFinished = (cs: t) =>
+  if cs.reportedFinished {
+    None
+  } else {
+    switch (cs.fetchState.endBlock, cs->hasProcessedToEndblock, cs.isProgressAtHead) {
+    | (Some(endBlock), true, _) => {
+        cs.reportedFinished = true
+        Some(EndBlock(endBlock))
+      }
+    | (_, _, true) => {
+        cs.reportedFinished = true
+        Some(Backfill(cs.committedProgressBlockNumber))
+      }
+    | _ => None
+    }
+  }
+
 // Caught up as judged by persisted values alone: progress reached the endBlock,
 // or the head the previous run had already observed (less the lag that holds the
 // tip back). Unlike `isFetchingAtHead` this doesn't move when a fresh height
@@ -730,7 +764,7 @@ let groupBatchItems = (items: array<Internal.item>): (transactionGroups, blockGr
       let {blockNumber} = eventItem
 
       switch eventItem.payload->Internal.getPayloadTransaction->Nullable.toOption {
-      | Some(_) => () // RPC/simulate/Fuel carry the transaction inline.
+      | Some(_) => () // Fuel and Simulate carry the transaction inline.
       | None =>
         let {transactionIndex} = eventItem
         let mask = eventItem.onEventRegistration.fieldSelection.transactionMask
@@ -757,7 +791,7 @@ let groupBatchItems = (items: array<Internal.item>): (transactionGroups, blockGr
       }
 
       switch eventItem.payload->Internal.getPayloadBlock->Nullable.toOption {
-      | Some(_) => () // RPC/simulate carry the block inline.
+      | Some(_) => () // Simulate carries the block inline.
       | None =>
         let mask = eventItem.onEventRegistration.fieldSelection.blockMask
         let last = blockItemGroups->Array.length - 1
@@ -796,13 +830,18 @@ let groupBatchItems = (items: array<Internal.item>): (transactionGroups, blockGr
 // `groupBatchItems`). Store-backed items always get a transaction object - the
 // selected fields, or `{}` when nothing was selected - so `event.transaction`
 // is never `undefined` (matching the inline sources).
-let applyTransactionGroups = async (store: TransactionStore.t, g: transactionGroups) => {
+let applyTransactionGroups = async (
+  store: TransactionStore.t,
+  g: transactionGroups,
+  ~shouldChecksum,
+) => {
   if g.payloadGroups->Utils.Array.notEmpty {
     if g.anyTransactionFieldSelected {
       let txs = await store->TransactionStore.materialize(
         ~blockNumbers=g.txBlockNumbers,
         ~transactionIndices=g.transactionIndices,
         ~masks=g.transactionMasks,
+        ~shouldChecksum,
       )
       g.payloadGroups->Array.forEachWithIndex((payloads, i) => {
         let tx = txs->Array.getUnsafe(i)
@@ -823,11 +862,12 @@ let applyTransactionGroups = async (store: TransactionStore.t, g: transactionGro
 }
 
 // Materialise a `BlockStore` against precomputed groups (see `groupBatchItems`).
-let applyBlockGroups = async (store: BlockStore.t, g: blockGroups) => {
+let applyBlockGroups = async (store: BlockStore.t, g: blockGroups, ~shouldChecksum) => {
   if g.blockItemGroups->Utils.Array.notEmpty {
     let blocks = await store->BlockStore.materialize(
       ~blockNumbers=g.blockBlockNumbers,
       ~masks=g.blockMasks,
+      ~shouldChecksum,
     )
     g.blockItemGroups->Array.forEachWithIndex((group, i) => {
       let block = blocks->Array.getUnsafe(i)
@@ -843,27 +883,26 @@ let applyBlockGroups = async (store: BlockStore.t, g: blockGroups) => {
 let materializeBatchItems = async (cs: t, ~items: array<Internal.item>) => {
   let (txGroups, blockGroups) = items->groupBatchItems
   let _ = await Promise.all2((
-    cs.transactionStore->applyTransactionGroups(txGroups),
-    cs.blockStore->applyBlockGroups(blockGroups),
+    cs.transactionStore->applyTransactionGroups(txGroups, ~shouldChecksum=cs.shouldChecksum),
+    cs.blockStore->applyBlockGroups(blockGroups, ~shouldChecksum=cs.shouldChecksum),
   ))
 }
 
 // Materialise a fetch-response's transactions and blocks onto its items before
-// contract-register handlers read them. Transactions come from the response's
-// page (`None` when kept inline); blocks come from the chain store, which the
-// response's page was already merged into by `registerReorgGuard`.
+// contract-register handlers read them. Both come from the chain stores, which
+// the response's pages were merged into once the reorg guard passed: a source
+// that consults those stores to decide what to fetch leaves out what they
+// already hold, so its page alone does not cover its own items.
 let materializePageItems = async (
   ~items: array<Internal.item>,
-  ~transactionStore: option<TransactionStore.t>,
+  ~transactionStore: TransactionStore.t,
   ~blockStore: BlockStore.t,
+  ~shouldChecksum: bool,
 ) => {
   let (txGroups, blockGroups) = items->groupBatchItems
   let _ = await Promise.all2((
-    switch transactionStore {
-    | Some(store) => store->applyTransactionGroups(txGroups)
-    | None => Promise.resolve()
-    },
-    blockStore->applyBlockGroups(blockGroups),
+    transactionStore->applyTransactionGroups(txGroups, ~shouldChecksum),
+    blockStore->applyBlockGroups(blockGroups, ~shouldChecksum),
   ))
 }
 
@@ -874,16 +913,7 @@ let handleQueryResult = (
   ~newRegistrations,
   ~latestFetchedBlock: int,
   ~knownHeight,
-  ~transactionStore as txPage: option<TransactionStore.t>,
 ) => {
-  // Merge this response's transaction page into the chain store in lockstep
-  // with appending its items to the buffer. Inline sources contribute no page;
-  // the block page was already merged by `registerReorgGuard`.
-  switch txPage {
-  | Some(page) => cs.transactionStore->TransactionStore.merge(page)
-  | None => ()
-  }
-
   let fs = switch newRegistrations {
   | [] => cs.fetchState
   | _ =>
@@ -964,6 +994,34 @@ let isInReorgThreshold = (cs: t) => cs.isInReorgThreshold
 // progress has run.
 let shouldSaveHistory = (cs: t) =>
   cs.shouldRollbackOnReorg && cs.maxReorgDepth > 0 && cs.isInReorgThreshold
+// What crossing into the recent blocks changed for this chain. Until now it
+// stopped short of the head by its reorg depth, because it kept nothing it
+// could roll back with; crossing lifts both at once. The second half is the
+// answer to why the indexer starts writing more than it was.
+// Whether crossing gives this chain anything more to index: the last block it
+// may fetch afterwards against the last it may fetch now. Read before the
+// crossing, while the lag being lifted is still the one in place.
+//
+// False wherever the lag was never holding this chain back, which is every
+// configuration that also keeps no history: a chain that isn't rolled back on a
+// reorg, or has no reorg depth, already fetches as far as it ever will. It is
+// false too for a chain whose end block sits below the blocks being opened up,
+// which will never reach one of them.
+let reorgThresholdLiftsCeiling = (cs: t) => {
+  let ceiling = (~blockLag) => {
+    let head = Pervasives.max(0, cs.fetchState.knownHeight - blockLag)
+    switch cs.fetchState.endBlock {
+    | Some(endBlock) => Pervasives.min(endBlock, head)
+    | None => head
+    }
+  }
+  ceiling(~blockLag=cs.chainConfig.blockLag) > ceiling(~blockLag=cs.fetchState.blockLag)
+}
+
+// Only ever said by a chain the crossing lifts, and lifting it takes a reorg
+// depth to have been held back by, which is the same thing that makes the
+// history below worth keeping. So there is no second form without it.
+let reorgThresholdEntryMessage = "Indexing the latest blocks now. These can be reorged, so the indexer starts storing a history of every change to roll back with."
 
 // Snapshot the chain's metadata fields for staging into the chains table.
 let toChainMetadata = (cs: t): InternalTable.Chains.metaFields => {
@@ -1051,6 +1109,12 @@ let toChainBeforeBatch = (cs: t, ~isRealtime): Batch.chainBeforeBatch => {
 
 // Whether the chain's post-batch fetch frontier is ready to cross into the reorg
 // threshold, using the batch's progressed frontier when this chain advanced.
+// The same question asked of where the chain stands now rather than of where a
+// batch would leave it. Entering the threshold is what lifts the pre-threshold
+// lag, so a chain waiting to enter it has fetched as far as it can.
+let isReadyToEnterReorgThreshold = (cs: t) =>
+  cs.fetchState->FetchState.isReadyToEnterReorgThreshold(~tolerance=cs.reorgThresholdReadyTolerance)
+
 let isReadyToEnterReorgThresholdAfterBatch = (cs: t, ~batch: Batch.t) => {
   let fetchState = switch batch.progressedChainsById->ChainId.Dict.dangerouslyGetNonOption(
     cs.fetchState.chainId,
@@ -1199,6 +1263,7 @@ let markReady = (cs: t, ~readyAt) =>
 let rollbackCommittedProgress = (cs: t, blockNumber) =>
   if blockNumber !== cs.committedProgressBlockNumber {
     cs.committedProgressBlockNumber = blockNumber
+
     // Exact block only: the rolled-back region is about to be refetched, and
     // the next batch re-establishes the time either way.
     cs.committedProgressBlockTime =

package/src/ChainState.res.mjs

@@ -55,7 +55,7 @@ function validateOnEventRegistrations(chainId, registrations) {
   });
 }
 
-function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, sourceManager, committedProgressBlockNumber, committedProgressBlockTimeOpt, safeCheckpointTrackingOpt, shouldRollbackOnReorg, maxReorgDepth, isInReorgThresholdOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, perChainEntitiesOpt, logger) {
+function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, sourceManager, committedProgressBlockNumber, committedProgressBlockTimeOpt, safeCheckpointTrackingOpt, shouldRollbackOnReorg, maxReorgDepth, isInReorgThresholdOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, shouldChecksumOpt, reorgThresholdReadyToleranceOpt, perChainEntitiesOpt, logger) {
   let onEventRegistrations = onEventRegistrationsOpt !== undefined ? onEventRegistrationsOpt : [];
   let committedProgressBlockTime = committedProgressBlockTimeOpt !== undefined ? Primitive_option.valFromOption(committedProgressBlockTimeOpt) : undefined;
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
@@ -63,9 +63,10 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
   let timestampCaughtUpToHeadOrEndblock = timestampCaughtUpToHeadOrEndblockOpt !== undefined ? Primitive_option.valFromOption(timestampCaughtUpToHeadOrEndblockOpt) : undefined;
   let isProgressAtHead = isProgressAtHeadOpt !== undefined ? isProgressAtHeadOpt : false;
-  let transactionStore = transactionStoreOpt !== undefined ? Primitive_option.valFromOption(transactionStoreOpt) : TransactionStore.make("evm", false);
+  let transactionStore = transactionStoreOpt !== undefined ? Primitive_option.valFromOption(transactionStoreOpt) : TransactionStore.make("evm");
   let chainDensity = chainDensityOpt !== undefined ? Primitive_option.valFromOption(chainDensityOpt) : undefined;
-  let blockStore = blockStoreOpt !== undefined ? Primitive_option.valFromOption(blockStoreOpt) : BlockStore.make("evm", false);
+  let blockStore = blockStoreOpt !== undefined ? Primitive_option.valFromOption(blockStoreOpt) : BlockStore.make("evm");
+  let shouldChecksum = shouldChecksumOpt !== undefined ? shouldChecksumOpt : false;
   let reorgThresholdReadyTolerance = reorgThresholdReadyToleranceOpt !== undefined ? reorgThresholdReadyToleranceOpt : 100;
   let perChainEntities = perChainEntitiesOpt !== undefined ? perChainEntitiesOpt : [];
   validateOnEventRegistrations(chainConfig.id, onEventRegistrations);
@@ -88,6 +89,7 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
     shouldRollbackOnReorg: shouldRollbackOnReorg,
     maxReorgDepth: maxReorgDepth,
     isInReorgThreshold: isInReorgThreshold,
+    shouldChecksum: shouldChecksum,
     transactionStore: transactionStore,
     blockStore: blockStore,
     reorgThresholdReadyTolerance: reorgThresholdReadyTolerance,
@@ -96,6 +98,7 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
     blockRangeFetchCount: 0,
     blockRangeFetchedEvents: 0,
     blockRangeFetchedBlocks: 0,
+    reportedFinished: false,
     reorgCount: 0,
     reorgDetectedBlock: undefined,
     rollbackTargetBlock: undefined,
@@ -136,7 +139,7 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
     }
   });
   let lowercaseAddresses = config.lowercaseAddresses;
-  let addressStore = AddressStore.make(config.ecosystem.name, !lowercaseAddresses, AddressStore.contractsOf(onEventRegistrations, contractMapping));
+  let addressStore = AddressStore.make(config.ecosystem.name, AddressStore.contractsOf(onEventRegistrations, contractMapping));
   let match$1 = config.ecosystem.name;
   let tmp;
   switch (match$1) {
@@ -154,18 +157,19 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
       return reorgCheckpoint;
     }
   });
-  let sources = ChainSources.make(chainConfig, onEventRegistrations, addressStore, lowercaseAddresses);
-  let blockStore = BlockStore.make(config.ecosystem.name, !lowercaseAddresses);
+  let blockStore = BlockStore.make(config.ecosystem.name);
+  let transactionStore = TransactionStore.make(config.ecosystem.name);
+  let sources = ChainSources.make(chainConfig, onEventRegistrations, addressStore, lowercaseAddresses, blockStore, transactionStore);
   if (Utils.$$Array.notEmpty(chainReorgCheckpoints)) {
     let seedPage = BlockStore.fromJs(chainReorgCheckpoints.map(cp => ({
       blockNumber: cp.block_number,
       blockHash: cp.block_hash
-    })), config.ecosystem.name, !lowercaseAddresses);
+    })), config.ecosystem.name);
     blockStore.merge(seedPage, 0, false);
   }
   let firstEventBlock$1 = fetchState.firstEventBlock;
   let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
-  return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(committedProgressBlockTime), Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, isInReorgThreshold, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.userEntities), logger);
+  return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(committedProgressBlockTime), Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, isInReorgThreshold, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(transactionStore), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), !lowercaseAddresses, config.reorgThresholdReadyTolerance, EntityTables.perChain(config.userEntities), logger);
 }
 
 function logger(cs) {
@@ -176,6 +180,14 @@ function blockStore(cs) {
   return cs.blockStore;
 }
 
+function transactionStore(cs) {
+  return cs.transactionStore;
+}
+
+function shouldChecksum(cs) {
+  return cs.shouldChecksum;
+}
+
 function entities(cs) {
   return cs.entities;
 }
@@ -254,7 +266,7 @@ function knownHeight(cs) {
 }
 
 function contractAddresses(cs, contractName) {
-  return cs.addressStore.contractAddresses(contractName);
+  return cs.addressStore.contractAddresses(contractName, cs.shouldChecksum);
 }
 
 function drainAddressesForWrite(cs, toBlockInclusive, checkpointBlockNumbers) {
@@ -409,6 +421,29 @@ function hasProcessedToEndblock(cs) {
   }
 }
 
+function takeFinished(cs) {
+  if (cs.reportedFinished) {
+    return;
+  }
+  let match = cs.fetchState.endBlock;
+  let match$1 = hasProcessedToEndblock(cs);
+  let match$2 = cs.isProgressAtHead;
+  if (match !== undefined && match$1) {
+    cs.reportedFinished = true;
+    return {
+      TAG: "EndBlock",
+      _0: match
+    };
+  }
+  if (match$2) {
+    cs.reportedFinished = true;
+    return {
+      TAG: "Backfill",
+      _0: cs.committedProgressBlockNumber
+    };
+  }
+}
+
 function isDurablyCaughtUp(cs) {
   let fetchState = cs.fetchState;
   let committedProgressBlockNumber = cs.committedProgressBlockNumber;
@@ -511,12 +546,12 @@ function groupBatchItems(items) {
   ];
 }
 
-async function applyTransactionGroups(store, g) {
+async function applyTransactionGroups(store, g, shouldChecksum) {
   if (!Utils.$$Array.notEmpty(g.payloadGroups)) {
     return;
   }
   if (g.anyTransactionFieldSelected) {
-    let txs = await store.materialize(g.txBlockNumbers, g.transactionIndices, g.transactionMasks);
+    let txs = await store.materialize(g.txBlockNumbers, g.transactionIndices, g.transactionMasks, shouldChecksum);
     g.payloadGroups.forEach((payloads, i) => {
       let tx = txs[i];
       payloads.forEach(payload => {
@@ -537,11 +572,11 @@ async function applyTransactionGroups(store, g) {
   });
 }
 
-async function applyBlockGroups(store, g) {
+async function applyBlockGroups(store, g, shouldChecksum) {
   if (!Utils.$$Array.notEmpty(g.blockItemGroups)) {
     return;
   }
-  let blocks = await store.materialize(g.blockBlockNumbers, g.blockMasks);
+  let blocks = await store.materialize(g.blockBlockNumbers, g.blockMasks, shouldChecksum);
   g.blockItemGroups.forEach((group, i) => {
     let block = blocks[i];
     group.forEach(ei => {
@@ -553,23 +588,20 @@ async function applyBlockGroups(store, g) {
 async function materializeBatchItems(cs, items) {
   let match = groupBatchItems(items);
   await Promise.all([
-    applyTransactionGroups(cs.transactionStore, match[0]),
-    applyBlockGroups(cs.blockStore, match[1])
+    applyTransactionGroups(cs.transactionStore, match[0], cs.shouldChecksum),
+    applyBlockGroups(cs.blockStore, match[1], cs.shouldChecksum)
   ]);
 }
 
-async function materializePageItems(items, transactionStore, blockStore) {
+async function materializePageItems(items, transactionStore, blockStore, shouldChecksum) {
   let match = groupBatchItems(items);
   await Promise.all([
-    transactionStore !== undefined ? applyTransactionGroups(Primitive_option.valFromOption(transactionStore), match[0]) : Promise.resolve(),
-    applyBlockGroups(blockStore, match[1])
+    applyTransactionGroups(transactionStore, match[0], shouldChecksum),
+    applyBlockGroups(blockStore, match[1], shouldChecksum)
   ]);
 }
 
-function handleQueryResult(cs, query, newItems, newRegistrations, latestFetchedBlock, knownHeight, txPage) {
-  if (txPage !== undefined) {
-    cs.transactionStore.merge(Primitive_option.valFromOption(txPage));
-  }
+function handleQueryResult(cs, query, newItems, newRegistrations, latestFetchedBlock, knownHeight) {
   let fs = newRegistrations.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.addressStore, Primitive_int.max(knownHeight, latestFetchedBlock), newRegistrations) : cs.fetchState;
   cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
   cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsEst);
@@ -666,6 +698,19 @@ function shouldSaveHistory(cs) {
   }
 }
 
+function reorgThresholdLiftsCeiling(cs) {
+  let ceiling = blockLag => {
+    let head = Primitive_int.max(0, cs.fetchState.knownHeight - blockLag | 0);
+    let endBlock = cs.fetchState.endBlock;
+    if (endBlock !== undefined) {
+      return Primitive_int.min(endBlock, head);
+    } else {
+      return head;
+    }
+  };
+  return ceiling(cs.chainConfig.blockLag) > ceiling(cs.fetchState.blockLag);
+}
+
 function toChainMetadata(cs) {
   return {
     first_event_block: Stdlib_Null.fromOption(cs.fetchState.firstEventBlock),
@@ -741,6 +786,10 @@ function toChainBeforeBatch(cs, isRealtime) {
   };
 }
 
+function isReadyToEnterReorgThreshold(cs) {
+  return FetchState.isReadyToEnterReorgThreshold(cs.reorgThresholdReadyTolerance, cs.fetchState);
+}
+
 function isReadyToEnterReorgThresholdAfterBatch(cs, batch) {
   let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
   let fetchState = chainAfterBatch !== undefined ? chainAfterBatch.fetchState : cs.fetchState;
@@ -878,6 +927,8 @@ function rollback(cs, rolledBackTo) {
   return rolledBackAddresses$1;
 }
 
+let reorgThresholdEntryMessage = "Indexing the latest blocks now. These can be reorged, so the indexer starts storing a history of every change to roll back with.";
+
 export {
   configStorageRows,
   seedConfigAddresses,
@@ -885,6 +936,8 @@ export {
   makeFromDbState,
   logger,
   blockStore,
+  transactionStore,
+  shouldChecksum,
   entities,
   resetEntities,
   sourceManager,
@@ -921,7 +974,11 @@ export {
   toMetrics,
   toChainMetadata,
   toChainBeforeBatch,
+  isReadyToEnterReorgThreshold,
   isReadyToEnterReorgThresholdAfterBatch,
+  takeFinished,
+  reorgThresholdLiftsCeiling,
+  reorgThresholdEntryMessage,
   hasProcessedToEndblock,
   isDurablyCaughtUp,
   getHighestBlockBelowThreshold,

package/src/ChainState.resi

@@ -34,6 +34,7 @@ let make: (
   ~transactionStore: TransactionStore.t=?,
   ~chainDensity: option<float>=?,
   ~blockStore: BlockStore.t=?,
+  ~shouldChecksum: bool=?,
   ~reorgThresholdReadyTolerance: int=?,
   ~perChainEntities: array<Internal.entityConfig>=?,
   ~logger: Pino.t,
@@ -54,6 +55,8 @@ let makeFromDbState: (
 // Accessors.
 let logger: t => Pino.t
 let blockStore: t => BlockStore.t
+let transactionStore: t => TransactionStore.t
+let shouldChecksum: t => bool
 let entities: t => EntityTables.t
 let resetEntities: (t, ~perChainEntities: array<Internal.entityConfig>) => unit
 let sourceManager: t => SourceManager.t
@@ -120,9 +123,14 @@ let dispatch: (
 let toMetrics: t => Metrics.chainMetrics
 let toChainMetadata: t => InternalTable.Chains.metaFields
 let toChainBeforeBatch: (t, ~isRealtime: bool) => Batch.chainBeforeBatch
+let isReadyToEnterReorgThreshold: t => bool
 let isReadyToEnterReorgThresholdAfterBatch: (t, ~batch: Batch.t) => bool
 
 // Derived (pure).
+type finished = EndBlock(int) | Backfill(int)
+let takeFinished: t => option<finished>
+let reorgThresholdLiftsCeiling: t => bool
+let reorgThresholdEntryMessage: string
 let hasProcessedToEndblock: t => bool
 let isDurablyCaughtUp: t => bool
 let getHighestBlockBelowThreshold: t => int
@@ -139,14 +147,18 @@ let handleQueryResult: (
   ~newRegistrations: array<AddressStore.registration>,
   ~latestFetchedBlock: int,
   ~knownHeight: int,
-  ~transactionStore: option<TransactionStore.t>,
 ) => unit
 let materializeBatchItems: (t, ~items: array<Internal.item>) => promise<unit>
 let materializePageItems: (
   ~items: array<Internal.item>,
-  ~transactionStore: option<TransactionStore.t>,
+  ~transactionStore: TransactionStore.t,
   ~blockStore: BlockStore.t,
+  ~shouldChecksum: bool,
 ) => promise<unit>
+// Merges the page's blocks into the chain's store as it checks them: the check
+// is the merge's own conflict report, so a page that passes is already applied.
+// Its transactions are a separate page, merged by the caller once this returns
+// no reorg.
 let registerReorgGuard: (
   t,
   ~blockStore: BlockStore.t,

package/src/Config.res

@@ -625,6 +625,16 @@ let getChain = (config, ~chainId) =>
         "No chain with id " ++ chainId->ChainId.toString ++ " found in config.yaml",
       )
 
+// Whether every entity belongs to exactly one chain. Read off the checkpoint
+// sequence rather than the entities again: a chain gets a counter of its own
+// only when no other chain can reach its rows, which is the same fact and the
+// one that makes splitting a run across processes safe.
+let isPerChain = (config: t) =>
+  switch config.checkpointSequence {
+  | PerChain => true
+  | SharedAcrossChains => false
+  }
+
 // Narrows a config to the chains one `envio start --chain` process drives.
 // `contractMapping` is deliberately left whole: its ids are what the migration
 // that created the schema stored, and one rebuilt from a subset would hand the
@@ -1221,6 +1231,24 @@ let prime = (json: JSON.t): unit => {
   cached := None
 }
 
+// What the command decided rather than the project's files: which chains this
+// process drives, and whether the run is a dev run. A worker parses the same
+// files its supervisor did, so these are the only two it cannot arrive at on
+// its own.
+let withCommandFields = (json: JSON.t, ~chainIds, ~isDev) =>
+  switch json->JSON.Decode.object {
+  | Some(fields) => {
+      let narrowed = fields->Dict.copy
+      narrowed->Dict.set(
+        "isolatedChains",
+        chainIds->S.reverseConvertToJsonOrThrow(S.array(ChainId.schema)),
+      )
+      narrowed->Dict.set("isDev", JSON.Encode.bool(isDev))
+      JSON.Object(narrowed)
+    }
+  | None => JsError.throwWithMessage("Invalid indexer config: not an object")
+  }
+
 let getPublicConfigJson = () =>
   switch primedJson.contents {
   | Some(json) => json
@@ -1266,6 +1294,10 @@ let stripSensitiveData = (json: JSON.t): JSON.t => {
   cloned
 }
 
+// What the storage layer records as the config this schema was built from,
+// and checks a resuming run against.
+let envioInfo = () => getPublicConfigJson()->stripSensitiveData
+
 // Postgres jsonb doesn't preserve key order, so canonicalize with sorted
 // keys before string-comparing.
 let rec canonicalJson = (json: JSON.t): JSON.t =>