envio 3.3.0-alpha.1 → 3.3.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.3.0-alpha.1",
+  "version": "3.3.0-alpha.2",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.3.0-alpha.1",
-    "envio-linux-x64-musl": "3.3.0-alpha.1",
-    "envio-linux-arm64": "3.3.0-alpha.1",
-    "envio-darwin-x64": "3.3.0-alpha.1",
-    "envio-darwin-arm64": "3.3.0-alpha.1"
+    "envio-linux-x64": "3.3.0-alpha.2",
+    "envio-linux-x64-musl": "3.3.0-alpha.2",
+    "envio-linux-arm64": "3.3.0-alpha.2",
+    "envio-darwin-x64": "3.3.0-alpha.2",
+    "envio-darwin-arm64": "3.3.0-alpha.2"
   }
 }

package/src/Batch.res

@@ -34,12 +34,6 @@ type t = {
   checkpointEventsProcessed: array<int>,
 }
 
-let hasReadyItem = (fetchStates: array<FetchState.t>) => {
-  fetchStates->Array.some(fetchState => {
-    fetchState->FetchState.isActivelyIndexing && fetchState->FetchState.hasReadyItem
-  })
-}
-
 let getProgressedChainsById = {
   let getChainAfterBatchIfProgressed = (
     ~chainBeforeBatch: chainBeforeBatch,

package/src/Batch.res.mjs

@@ -4,16 +4,6 @@ import * as Utils from "./Utils.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
 import * as ReorgDetection from "./ReorgDetection.res.mjs";
 
-function hasReadyItem(fetchStates) {
-  return fetchStates.some(fetchState => {
-    if (FetchState.isActivelyIndexing(fetchState)) {
-      return FetchState.hasReadyItem(fetchState);
-    } else {
-      return false;
-    }
-  });
-}
-
 function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
   if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
     return {
@@ -208,7 +198,6 @@ function findLastEventItem(batch, chainId) {
 }
 
 export {
-  hasReadyItem,
   getProgressedChainsById,
   addReorgCheckpoints,
   prepareBatch,

package/src/BatchProcessing.res

@@ -77,15 +77,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
     state
     ->IndexerState.chainStates
     ->Dict.valuesToArray
-    ->Array.every(cs => {
-      let fetchState = switch progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
-        (cs->ChainState.fetchState).chainId,
-      ) {
-      | Some(chainAfterBatch) => chainAfterBatch.fetchState
-      | None => cs->ChainState.fetchState
-      }
-      fetchState->FetchState.isReadyToEnterReorgThreshold
-    })
+    ->Array.every(cs => cs->ChainState.isReadyToEnterReorgThresholdAfterBatch(~batch))
 
   if shouldEnterReorgThreshold {
     IndexerState.enterReorgThreshold(state)
@@ -171,9 +163,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
           state
           ->IndexerState.chainStates
           ->Dict.valuesToArray
-          ->Array.every(cs =>
-            cs->ChainState.isProgressAtHead && (cs->ChainState.fetchState).endBlock->Option.isNone
-          )
+          ->Array.every(ChainState.isAtHeadWithoutEndBlock)
         ) {
           IndexerState.errorExit(
             state,

package/src/BatchProcessing.res.mjs

@@ -4,12 +4,10 @@ import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
-import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as ExitOnCaughtUp from "./ExitOnCaughtUp.res.mjs";
 import * as EventProcessing from "./EventProcessing.res.mjs";
 import * as PruneStaleHistory from "./PruneStaleHistory.res.mjs";
@@ -30,10 +28,7 @@ async function processNextBatch(state, scheduleFetch) {
   let batch = IndexerState.createBatch(state, IndexerState.processedCheckpointId(state), IndexerState.config(state).batchSize, isRollbackBatch);
   let progressedChainsById = batch.progressedChainsById;
   let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
-  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => {
-    let chainAfterBatch = progressedChainsById[ChainState.fetchState(cs).chainId];
-    return FetchState.isReadyToEnterReorgThreshold(chainAfterBatch !== undefined ? chainAfterBatch.fetchState : ChainState.fetchState(cs));
-  });
+  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));
   if (shouldEnterReorgThreshold) {
     IndexerState.enterReorgThreshold(state);
   }
@@ -75,7 +70,7 @@ async function processNextBatch(state, scheduleFetch) {
   }
   if (allCaughtUp && !IndexerState.keepProcessAlive(state)) {
     return await ExitOnCaughtUp.run(state);
-  } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isProgressAtHead(cs) ? Stdlib_Option.isNone(ChainState.fetchState(cs).endBlock) : false)) {
+  } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(ChainState.isAtHeadWithoutEndBlock)) {
     return IndexerState.errorExit(state, ErrorHandling.make(new Error("No events found between startBlock and chain head. Cannot auto-detect endBlock."), undefined, undefined));
   } else {
     ChainMetadata.stage(state);

package/src/ChainFetching.res

@@ -11,7 +11,14 @@ type partitionQueryResponse = {
 let runContractRegistersOrThrow = async (
   ~itemsWithContractRegister: array<Internal.item>,
   ~config: Config.t,
+  ~chainState: ChainState.t,
+  ~page: option<TransactionStore.t>,
 ) => {
+  // contractRegister handlers can read event.transaction, so materialise the
+  // selected fields onto the payloads before running them. All items belong to
+  // the chain being fetched, hence its single page store and mask.
+  await chainState->ChainState.materializePageItems(~items=itemsWithContractRegister, ~page)
+
   let itemsWithDcs = []
 
   let onRegister = (~item: Internal.item, ~contractAddress, ~contractName) => {
@@ -106,6 +113,7 @@ let rec onQueryResponse = async (
     let chainState = state->IndexerState.getChainState(~chain)
     let {
       parsedQueueItems,
+      transactionStore,
       latestFetchedBlockNumber,
       latestFetchedBlockTimestamp,
       stats,
@@ -114,7 +122,7 @@ let rec onQueryResponse = async (
       fromBlockQueried,
     } = response
 
-    if knownHeight > (chainState->ChainState.fetchState).knownHeight {
+    if knownHeight > chainState->ChainState.knownHeight {
       Prometheus.SourceHeight.set(
         ~blockNumber=knownHeight,
         ~chainId=(chainState->ChainState.chainConfig).id,
@@ -212,6 +220,7 @@ let rec onQueryResponse = async (
               blockTimestamp: latestFetchedBlockTimestamp,
             },
             ~query,
+            ~transactionStore,
           )
           ChainMetadata.stage(state)
           scheduleFetch()
@@ -224,6 +233,8 @@ let rec onQueryResponse = async (
         switch await runContractRegistersOrThrow(
           ~itemsWithContractRegister,
           ~config=state->IndexerState.config,
+          ~chainState,
+          ~page=transactionStore,
         ) {
         | exception exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
         | newItemsWithDcs => proceed(~newItemsWithDcs)
@@ -240,6 +251,7 @@ and applyQueryResponse = (
   ~knownHeight,
   ~latestFetchedBlock,
   ~query,
+  ~transactionStore,
 ) => {
   let chainState = state->IndexerState.getChainState(~chain)
   let wasFetchingAtHead = chainState->ChainState.isFetchingAtHead
@@ -250,6 +262,7 @@ and applyQueryResponse = (
     ~newItems,
     ~newItemsWithDcs,
     ~knownHeight,
+    ~transactionStore,
   )
 
   // In auto-exit mode, set endBlock to the first event's block when events arrive.
@@ -294,7 +307,7 @@ let finishWaitingForNewBlock = (
       ->IndexerState.chainStates
       ->Dict.valuesToArray
       ->Array.every(cs => {
-        cs->ChainState.fetchState->FetchState.isReadyToEnterReorgThreshold
+        cs->ChainState.isReadyToEnterReorgThreshold
       })
 
     // Kick processing in case there are block handlers to run.
@@ -316,7 +329,6 @@ let fetchChain = async (
 ) => {
   let chainState = state->IndexerState.getChainState(~chain)
   if !(state->IndexerState.isResolvingReorg) && !(state->IndexerState.isStopped) {
-    let fetchState = chainState->ChainState.fetchState
     let isRealtime = state->IndexerState.isRealtime
     let sourceManager = chainState->ChainState.sourceManager
 
@@ -327,8 +339,7 @@ let fetchChain = async (
     // Owns its error boundary: launch doesn't catch, so any failure here (the
     // query, response handling, or dispatch itself) must stop the indexer.
     try {
-      await sourceManager->SourceManager.dispatch(
-        ~fetchState,
+      await chainState->ChainState.dispatch(
         ~waitForNewBlock=(~knownHeight) =>
           sourceManager->SourceManager.waitForNewBlock(~knownHeight, ~isRealtime, ~reducedPolling),
         ~onNewBlock=(~knownHeight) =>
@@ -347,7 +358,7 @@ let fetchChain = async (
           try {
             let response = await sourceManager->SourceManager.executeQuery(
               ~query,
-              ~knownHeight=fetchState.knownHeight,
+              ~knownHeight=chainState->ChainState.knownHeight,
               ~isRealtime,
             )
             await onQueryResponse(

package/src/ChainFetching.res.mjs

@@ -4,7 +4,6 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
@@ -17,7 +16,8 @@ import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 import * as ContractRegisterContext from "./ContractRegisterContext.res.mjs";
 
-async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
+async function runContractRegistersOrThrow(itemsWithContractRegister, config, chainState, page) {
+  await ChainState.materializePageItems(chainState, itemsWithContractRegister, page);
   let itemsWithDcs = [];
   let onRegister = (item, contractAddress, contractName) => {
     let dc_registrationBlock = item.blockNumber;
@@ -71,10 +71,10 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
   return itemsWithDcs;
 }
 
-function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query) {
+function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query, transactionStore) {
   let chainState = IndexerState.getChainState(state, chain);
   let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
-  ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight);
+  ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, transactionStore);
   if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
     ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
   }
@@ -94,9 +94,10 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   let stats = response.stats;
   let latestFetchedBlockTimestamp = response.latestFetchedBlockTimestamp;
   let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
+  let transactionStore = response.transactionStore;
   let parsedQueueItems = response.parsedQueueItems;
   let knownHeight = response.knownHeight;
-  if (knownHeight > ChainState.fetchState(chainState).knownHeight) {
+  if (knownHeight > ChainState.knownHeight(chainState)) {
     Prometheus.SourceHeight.set(SourceManager.getActiveSource(ChainState.sourceManager(chainState)).name, ChainState.chainConfig(chainState).id, knownHeight);
   }
   Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - response.fromBlockQueried | 0) + 1 | 0);
@@ -133,7 +134,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
       applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, {
         blockNumber: latestFetchedBlockNumber,
         blockTimestamp: latestFetchedBlockTimestamp
-      }, query);
+      }, query, transactionStore);
       ChainMetadata.stage(state);
       scheduleFetch();
       return scheduleProcessing();
@@ -144,7 +145,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   }
   let newItemsWithDcs;
   try {
-    newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state));
+    newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), chainState, transactionStore);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
@@ -159,7 +160,7 @@ function finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFe
   let chainState = IndexerState.getChainState(state, chain);
   ChainState.updateKnownHeight(chainState, knownHeight);
   let isBelowReorgThreshold = !IndexerState.isInReorgThreshold(state) && IndexerState.config(state).shouldRollbackOnReorg;
-  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => FetchState.isReadyToEnterReorgThreshold(ChainState.fetchState(cs)));
+  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(ChainState.isReadyToEnterReorgThreshold);
   if (shouldEnterReorgThreshold) {
     IndexerState.enterReorgThreshold(state);
   }
@@ -172,14 +173,13 @@ async function fetchChain(state, chain, action, stateId, scheduleFetch, schedule
   if (IndexerState.isResolvingReorg(state) || IndexerState.isStopped(state)) {
     return;
   }
-  let fetchState = ChainState.fetchState(chainState);
   let isRealtime = IndexerState.isRealtime(state);
   let sourceManager = ChainState.sourceManager(chainState);
   let reducedPolling = !isRealtime;
   try {
-    return await SourceManager.dispatch(sourceManager, fetchState, async query => {
+    return await ChainState.dispatch(chainState, async query => {
       try {
-        let response = await SourceManager.executeQuery(sourceManager, query, fetchState.knownHeight, isRealtime);
+        let response = await SourceManager.executeQuery(sourceManager, query, ChainState.knownHeight(chainState), isRealtime);
         return await onQueryResponse(state, {
           chain: chain,
           response: response,

package/src/ChainMetadata.res

@@ -7,19 +7,9 @@ let stage = (state: IndexerState.t) => {
   ->IndexerState.chainStates
   ->Dict.valuesToArray
   ->Array.forEach(cs => {
-    let fetchState = cs->ChainState.fetchState
     chainsData->Dict.set(
       (cs->ChainState.chainConfig).id->Int.toString,
-      {
-        firstEventBlockNumber: fetchState.firstEventBlock->Null.fromOption,
-        isHyperSync: (
-          cs->ChainState.sourceManager->SourceManager.getActiveSource
-        ).poweredByHyperSync,
-        latestFetchedBlockNumber: fetchState->FetchState.bufferBlockNumber,
-        timestampCaughtUpToHeadOrEndblock: cs
-        ->ChainState.timestampCaughtUpToHeadOrEndblock
-        ->Null.fromOption,
-      },
+      cs->ChainState.toChainMetadata,
     )
   })
 

package/src/ChainMetadata.res.mjs

@@ -2,21 +2,12 @@
 
 import * as Writing from "./Writing.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
-import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as IndexerState from "./IndexerState.res.mjs";
-import * as SourceManager from "./sources/SourceManager.res.mjs";
 
 function stage(state) {
   let chainsData = {};
   Object.values(IndexerState.chainStates(state)).forEach(cs => {
-    let fetchState = ChainState.fetchState(cs);
-    chainsData[ChainState.chainConfig(cs).id.toString()] = {
-      first_event_block: Stdlib_Null.fromOption(fetchState.firstEventBlock),
-      buffer_block: FetchState.bufferBlockNumber(fetchState),
-      ready_at: Stdlib_Null.fromOption(ChainState.timestampCaughtUpToHeadOrEndblock(cs)),
-      _is_hyper_sync: SourceManager.getActiveSource(ChainState.sourceManager(cs)).poweredByHyperSync
-    };
+    chainsData[ChainState.chainConfig(cs).id.toString()] = ChainState.toChainMetadata(cs);
   });
   Writing.setChainMeta(state, chainsData);
 }

package/src/ChainState.res

@@ -17,6 +17,30 @@ type t = {
   mutable pendingBudget: float,
   mutable reorgDetection: ReorgDetection.t,
   mutable safeCheckpointTracking: option<SafeCheckpointTracking.t>,
+  // Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
+  // transactionIndex). Fetch responses merge their page in; entries are pruned
+  // as the chain progresses and dropped above the target on rollback.
+  transactionStore: TransactionStore.t,
+  // Bitmask of the transaction fields the store materialises at batch prep.
+  transactionFieldMask: float,
+}
+
+// Per-chain shape returned by the status API.
+type chainData = {
+  chainId: float,
+  poweredByHyperSync: bool,
+  firstEventBlockNumber: option<int>,
+  latestProcessedBlock: option<int>,
+  timestampCaughtUpToHeadOrEndblock: option<Date.t>,
+  numEventsProcessed: float,
+  latestFetchedBlockNumber: int,
+  // Need this for API backwards compatibility
+  @as("currentBlockHeight")
+  knownHeight: int,
+  numBatchesFetched: int,
+  startBlock: int,
+  endBlock: option<int>,
+  numAddresses: int,
 }
 
 let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddress> => {
@@ -43,6 +67,7 @@ let make = (
   ~numEventsProcessed=0.,
   ~timestampCaughtUpToHeadOrEndblock=None,
   ~isProgressAtHead=false,
+  ~transactionFieldMask=0.,
   ~logger: Pino.t,
 ): t => {
   logger,
@@ -56,6 +81,8 @@ let make = (
   pendingBudget: 0.,
   reorgDetection,
   safeCheckpointTracking,
+  transactionStore: TransactionStore.make(),
+  transactionFieldMask,
 }
 
 let makeInternal = (
@@ -301,6 +328,7 @@ let makeInternal = (
     ~committedProgressBlockNumber=progressBlockNumber,
     ~timestampCaughtUpToHeadOrEndblock,
     ~numEventsProcessed,
+    ~transactionFieldMask=config.ecosystem.transactionFieldMask(eventConfigs),
     ~logger,
   )
 }
@@ -377,7 +405,6 @@ let makeFromDbState = (
 // --- Read accessors. ---
 
 let logger = (cs: t) => cs.logger
-let fetchState = (cs: t) => cs.fetchState
 let sourceManager = (cs: t) => cs.sourceManager
 let chainConfig = (cs: t) => cs.chainConfig
 let reorgDetection = (cs: t) => cs.reorgDetection
@@ -388,6 +415,19 @@ let numEventsProcessed = (cs: t) => cs.numEventsProcessed
 let pendingBudget = (cs: t) => cs.pendingBudget
 let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock
 
+// Fetch-frontier reads. The FetchState is owned here; callers go through these
+// rather than reaching into it.
+let knownHeight = (cs: t) => cs.fetchState.knownHeight
+let indexingAddresses = (cs: t) => cs.fetchState.indexingAddresses
+let bufferSize = (cs: t) => cs.fetchState->FetchState.bufferSize
+let bufferReadyCount = (cs: t) => cs.fetchState->FetchState.bufferReadyCount
+let getProgressPercentage = (cs: t) => cs.fetchState->FetchState.getProgressPercentage
+let getProgressPercentageAt = (cs: t, ~blockNumber) =>
+  cs.fetchState->FetchState.getProgressPercentageAt(~blockNumber)
+let hasReadyItem = (cs: t) =>
+  cs.fetchState->FetchState.isActivelyIndexing && cs.fetchState->FetchState.hasReadyItem
+let isReadyToEnterReorgThreshold = (cs: t) => cs.fetchState->FetchState.isReadyToEnterReorgThreshold
+
 // Mark queries as in flight and reserve their estimated size against the shared
 // buffer budget in one step, so the counter stays in sync with the pending
 // queries it tracks.
@@ -404,6 +444,30 @@ let resetPendingQueries = (cs: t) => {
   cs.pendingBudget = 0.
 }
 
+// Propose the chain's candidate queries against the shared buffer budget,
+// accounting for this chain's own in-flight reservations.
+let getNextQuery = (cs: t, ~budget) =>
+  cs.fetchState->FetchState.getNextQuery(~budget, ~chainPendingBudget=cs.pendingBudget)
+
+// Run a fetch tick for this chain against its sources, feeding the owned fetch
+// frontier to the source manager.
+let dispatch = (
+  cs: t,
+  ~executeQuery,
+  ~waitForNewBlock,
+  ~onNewBlock,
+  ~action: FetchState.nextQuery,
+  ~stateId,
+) =>
+  cs.sourceManager->SourceManager.dispatch(
+    ~fetchState=cs.fetchState,
+    ~executeQuery,
+    ~waitForNewBlock,
+    ~onNewBlock,
+    ~action,
+    ~stateId,
+  )
+
 // --- Derived (pure). ---
 
 let hasProcessedToEndblock = (cs: t) => {
@@ -426,11 +490,34 @@ let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
 // True once the fetch frontier has reached the head/endBlock for this chain.
 let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
 
+// Reached head on a chain with no configured endBlock — used by auto-exit to
+// detect that no events were found in the start..head range.
+let isAtHeadWithoutEndBlock = (cs: t) =>
+  cs.isProgressAtHead && cs.fetchState.endBlock->Option.isNone
+
 // --- State transitions. The chain state is mutated only through these; each
 // owns a cohesive update so callers don't juggle individual fields. ---
 
 // Apply a fetch response: register any new dynamic contracts, append the items
 // to the buffer and advance the known head.
+// Materialise the chain store's selected transaction fields onto a batch's
+// items at batch prep (the persistent-store path).
+let materializeBatchItems = (cs: t, ~items: array<Internal.item>) =>
+  cs.transactionStore->TransactionStore.materializeItems(~items, ~mask=cs.transactionFieldMask)
+
+// Materialise a fetch-response page's transactions onto its items before
+// contract-register handlers read them. `None` pages (RPC/Fuel/Simulate keep the
+// transaction inline) are a no-op.
+let materializePageItems = (
+  cs: t,
+  ~items: array<Internal.item>,
+  ~page: option<TransactionStore.t>,
+) =>
+  switch page {
+  | Some(store) => store->TransactionStore.materializeItems(~items, ~mask=cs.transactionFieldMask)
+  | None => Promise.resolve()
+  }
+
 let handleQueryResult = (
   cs: t,
   ~query: FetchState.query,
@@ -438,7 +525,15 @@ let handleQueryResult = (
   ~newItemsWithDcs,
   ~latestFetchedBlock,
   ~knownHeight,
+  ~transactionStore as page: option<TransactionStore.t>,
 ) => {
+  // Merge this response's page into the chain store in lockstep with appending
+  // its items to the buffer. Inline-transaction sources contribute no page.
+  switch page {
+  | Some(page) => cs.transactionStore->TransactionStore.merge(page)
+  | None => ()
+  }
+
   let fs = switch newItemsWithDcs {
   | [] => cs.fetchState
   | _ => cs.fetchState->FetchState.registerDynamicContracts(newItemsWithDcs)
@@ -491,6 +586,56 @@ let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>
 let enterReorgThreshold = (cs: t) =>
   cs.fetchState = cs.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
 
+// Snapshot the chain's metadata fields for staging into the chains table.
+let toChainMetadata = (cs: t): InternalTable.Chains.metaFields => {
+  firstEventBlockNumber: cs.fetchState.firstEventBlock->Null.fromOption,
+  isHyperSync: (cs.sourceManager->SourceManager.getActiveSource).poweredByHyperSync,
+  latestFetchedBlockNumber: cs.fetchState->FetchState.bufferBlockNumber,
+  timestampCaughtUpToHeadOrEndblock: cs.timestampCaughtUpToHeadOrEndblock->Null.fromOption,
+}
+
+// Snapshot the chain's view for the status API.
+let toChainData = (cs: t): chainData => {
+  chainId: cs.chainConfig.id->Int.toFloat,
+  poweredByHyperSync: (cs.sourceManager->SourceManager.getActiveSource).poweredByHyperSync,
+  firstEventBlockNumber: cs.fetchState.firstEventBlock,
+  latestProcessedBlock: cs.committedProgressBlockNumber === -1
+    ? None
+    : Some(cs.committedProgressBlockNumber),
+  timestampCaughtUpToHeadOrEndblock: cs.timestampCaughtUpToHeadOrEndblock,
+  numEventsProcessed: cs.numEventsProcessed,
+  latestFetchedBlockNumber: Pervasives.max(cs.fetchState->FetchState.bufferBlockNumber, 0),
+  knownHeight: cs->hasProcessedToEndblock
+    ? cs.fetchState.endBlock->Option.getOr(cs.fetchState.knownHeight)
+    : cs.fetchState.knownHeight,
+  numBatchesFetched: 0,
+  startBlock: cs.fetchState.startBlock,
+  endBlock: cs.fetchState.endBlock,
+  numAddresses: cs.fetchState->FetchState.numAddresses,
+}
+
+// Snapshot the inputs a batch build needs from this chain.
+let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
+  fetchState: cs.fetchState,
+  progressBlockNumber: cs.committedProgressBlockNumber,
+  totalEventsProcessed: cs.numEventsProcessed,
+  sourceBlockNumber: cs.fetchState.knownHeight,
+  reorgDetection: cs.reorgDetection,
+  chainConfig: cs.chainConfig,
+}
+
+// Whether the chain's post-batch fetch frontier is ready to cross into the reorg
+// threshold, using the batch's progressed frontier when this chain advanced.
+let isReadyToEnterReorgThresholdAfterBatch = (cs: t, ~batch: Batch.t) => {
+  let fetchState = switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
+    cs.fetchState.chainId,
+  ) {
+  | Some(chainAfterBatch) => chainAfterBatch.fetchState
+  | None => cs.fetchState
+  }
+  fetchState->FetchState.isReadyToEnterReorgThreshold
+}
+
 // Commit the post-batch fetch frontier for a chain that progressed in the batch,
 // applying blockLag when this batch also crosses into the reorg threshold.
 let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
@@ -551,6 +696,8 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t) => {
 
       cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber
       cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed
+      // Processed blocks' transactions are no longer needed.
+      cs.transactionStore->TransactionStore.prune(chainAfterBatch.progressBlockNumber)
       cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated
       switch cs.safeCheckpointTracking {
       | Some(safeCheckpointTracking) =>
@@ -624,6 +771,7 @@ let rollback = (
     | None => ()
     }
     cs.fetchState = cs.fetchState->FetchState.rollback(~targetBlockNumber=newProgressBlockNumber)
+    cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
     cs.committedProgressBlockNumber = newProgressBlockNumber
     cs.numEventsProcessed = newTotalEventsProcessed
   | None =>
@@ -634,6 +782,7 @@ let rollback = (
         )
       cs.fetchState =
         cs.fetchState->FetchState.rollback(~targetBlockNumber=rollbackTargetBlockNumber)
+      cs.transactionStore->TransactionStore.rollback(rollbackTargetBlockNumber)
       cs.committedProgressBlockNumber = Pervasives.min(
         cs.committedProgressBlockNumber,
         rollbackTargetBlockNumber,