envio 3.3.0-alpha.0 → 3.3.0-alpha.2

package/src/ChainState.res

@@ -11,8 +11,36 @@ type t = {
   mutable timestampCaughtUpToHeadOrEndblock: option<Date.t>,
   mutable committedProgressBlockNumber: int,
   mutable numEventsProcessed: float,
+  // Running sum of in-flight queries' estResponseSize, kept here so the
+  // scheduler doesn't re-sum pending queries on every tick. Incremented when
+  // queries are dispatched, decremented as their responses land.
+  mutable pendingBudget: float,
   mutable reorgDetection: ReorgDetection.t,
   mutable safeCheckpointTracking: option<SafeCheckpointTracking.t>,
+  // Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
+  // transactionIndex). Fetch responses merge their page in; entries are pruned
+  // as the chain progresses and dropped above the target on rollback.
+  transactionStore: TransactionStore.t,
+  // Bitmask of the transaction fields the store materialises at batch prep.
+  transactionFieldMask: float,
+}
+
+// Per-chain shape returned by the status API.
+type chainData = {
+  chainId: float,
+  poweredByHyperSync: bool,
+  firstEventBlockNumber: option<int>,
+  latestProcessedBlock: option<int>,
+  timestampCaughtUpToHeadOrEndblock: option<Date.t>,
+  numEventsProcessed: float,
+  latestFetchedBlockNumber: int,
+  // Need this for API backwards compatibility
+  @as("currentBlockHeight")
+  knownHeight: int,
+  numBatchesFetched: int,
+  startBlock: int,
+  endBlock: option<int>,
+  numAddresses: int,
 }
 
 let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddress> => {
@@ -39,6 +67,7 @@ let make = (
   ~numEventsProcessed=0.,
   ~timestampCaughtUpToHeadOrEndblock=None,
   ~isProgressAtHead=false,
+  ~transactionFieldMask=0.,
   ~logger: Pino.t,
 ): t => {
   logger,
@@ -49,8 +78,11 @@ let make = (
   timestampCaughtUpToHeadOrEndblock,
   committedProgressBlockNumber,
   numEventsProcessed,
+  pendingBudget: 0.,
   reorgDetection,
   safeCheckpointTracking,
+  transactionStore: TransactionStore.make(),
+  transactionFieldMask,
 }
 
 let makeInternal = (
@@ -296,6 +328,7 @@ let makeInternal = (
     ~committedProgressBlockNumber=progressBlockNumber,
     ~timestampCaughtUpToHeadOrEndblock,
     ~numEventsProcessed,
+    ~transactionFieldMask=config.ecosystem.transactionFieldMask(eventConfigs),
     ~logger,
   )
 }
@@ -372,7 +405,6 @@ let makeFromDbState = (
 // --- Read accessors. ---
 
 let logger = (cs: t) => cs.logger
-let fetchState = (cs: t) => cs.fetchState
 let sourceManager = (cs: t) => cs.sourceManager
 let chainConfig = (cs: t) => cs.chainConfig
 let reorgDetection = (cs: t) => cs.reorgDetection
@@ -380,8 +412,62 @@ let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
 let isProgressAtHead = (cs: t) => cs.isProgressAtHead
 let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
 let numEventsProcessed = (cs: t) => cs.numEventsProcessed
+let pendingBudget = (cs: t) => cs.pendingBudget
 let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock
 
+// Fetch-frontier reads. The FetchState is owned here; callers go through these
+// rather than reaching into it.
+let knownHeight = (cs: t) => cs.fetchState.knownHeight
+let indexingAddresses = (cs: t) => cs.fetchState.indexingAddresses
+let bufferSize = (cs: t) => cs.fetchState->FetchState.bufferSize
+let bufferReadyCount = (cs: t) => cs.fetchState->FetchState.bufferReadyCount
+let getProgressPercentage = (cs: t) => cs.fetchState->FetchState.getProgressPercentage
+let getProgressPercentageAt = (cs: t, ~blockNumber) =>
+  cs.fetchState->FetchState.getProgressPercentageAt(~blockNumber)
+let hasReadyItem = (cs: t) =>
+  cs.fetchState->FetchState.isActivelyIndexing && cs.fetchState->FetchState.hasReadyItem
+let isReadyToEnterReorgThreshold = (cs: t) => cs.fetchState->FetchState.isReadyToEnterReorgThreshold
+
+// Mark queries as in flight and reserve their estimated size against the shared
+// buffer budget in one step, so the counter stays in sync with the pending
+// queries it tracks.
+let startFetchingQueries = (cs: t, ~queries: array<FetchState.query>) => {
+  cs.fetchState->FetchState.startFetchingQueries(~queries)
+  cs.pendingBudget =
+    cs.pendingBudget +. queries->Array.reduce(0., (acc, query) => acc +. query.estResponseSize)
+}
+
+// Drop every in-flight query and release their reservations together, keeping
+// pendingBudget coupled to the pending queries it tracks.
+let resetPendingQueries = (cs: t) => {
+  cs.fetchState = cs.fetchState->FetchState.resetPendingQueries
+  cs.pendingBudget = 0.
+}
+
+// Propose the chain's candidate queries against the shared buffer budget,
+// accounting for this chain's own in-flight reservations.
+let getNextQuery = (cs: t, ~budget) =>
+  cs.fetchState->FetchState.getNextQuery(~budget, ~chainPendingBudget=cs.pendingBudget)
+
+// Run a fetch tick for this chain against its sources, feeding the owned fetch
+// frontier to the source manager.
+let dispatch = (
+  cs: t,
+  ~executeQuery,
+  ~waitForNewBlock,
+  ~onNewBlock,
+  ~action: FetchState.nextQuery,
+  ~stateId,
+) =>
+  cs.sourceManager->SourceManager.dispatch(
+    ~fetchState=cs.fetchState,
+    ~executeQuery,
+    ~waitForNewBlock,
+    ~onNewBlock,
+    ~action,
+    ~stateId,
+  )
+
 // --- Derived (pure). ---
 
 let hasProcessedToEndblock = (cs: t) => {
@@ -404,11 +490,34 @@ let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
 // True once the fetch frontier has reached the head/endBlock for this chain.
 let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
 
+// Reached head on a chain with no configured endBlock — used by auto-exit to
+// detect that no events were found in the start..head range.
+let isAtHeadWithoutEndBlock = (cs: t) =>
+  cs.isProgressAtHead && cs.fetchState.endBlock->Option.isNone
+
 // --- State transitions. The chain state is mutated only through these; each
 // owns a cohesive update so callers don't juggle individual fields. ---
 
 // Apply a fetch response: register any new dynamic contracts, append the items
 // to the buffer and advance the known head.
+// Materialise the chain store's selected transaction fields onto a batch's
+// items at batch prep (the persistent-store path).
+let materializeBatchItems = (cs: t, ~items: array<Internal.item>) =>
+  cs.transactionStore->TransactionStore.materializeItems(~items, ~mask=cs.transactionFieldMask)
+
+// Materialise a fetch-response page's transactions onto its items before
+// contract-register handlers read them. `None` pages (RPC/Fuel/Simulate keep the
+// transaction inline) are a no-op.
+let materializePageItems = (
+  cs: t,
+  ~items: array<Internal.item>,
+  ~page: option<TransactionStore.t>,
+) =>
+  switch page {
+  | Some(store) => store->TransactionStore.materializeItems(~items, ~mask=cs.transactionFieldMask)
+  | None => Promise.resolve()
+  }
+
 let handleQueryResult = (
   cs: t,
   ~query: FetchState.query,
@@ -416,7 +525,15 @@ let handleQueryResult = (
   ~newItemsWithDcs,
   ~latestFetchedBlock,
   ~knownHeight,
+  ~transactionStore as page: option<TransactionStore.t>,
 ) => {
+  // Merge this response's page into the chain store in lockstep with appending
+  // its items to the buffer. Inline-transaction sources contribute no page.
+  switch page {
+  | Some(page) => cs.transactionStore->TransactionStore.merge(page)
+  | None => ()
+  }
+
   let fs = switch newItemsWithDcs {
   | [] => cs.fetchState
   | _ => cs.fetchState->FetchState.registerDynamicContracts(newItemsWithDcs)
@@ -426,6 +543,9 @@ let handleQueryResult = (
     fs
     ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
     ->FetchState.updateKnownHeight(~knownHeight)
+
+  // The query is no longer in flight, so release its reservation.
+  cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.estResponseSize)
 }
 
 // Run reorg detection against a fetch response and commit the updated guard.
@@ -447,7 +567,7 @@ let prepareReorg = (cs: t, ~eventsProcessedDiff) => {
   | Some(diff) => cs.numEventsProcessed = cs.numEventsProcessed +. diff
   | None => ()
   }
-  cs.fetchState = cs.fetchState->FetchState.resetPendingQueries
+  cs->resetPendingQueries
 }
 
 let updateKnownHeight = (cs: t, ~knownHeight) =>
@@ -466,6 +586,56 @@ let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>
 let enterReorgThreshold = (cs: t) =>
   cs.fetchState = cs.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
 
+// Snapshot the chain's metadata fields for staging into the chains table.
+let toChainMetadata = (cs: t): InternalTable.Chains.metaFields => {
+  firstEventBlockNumber: cs.fetchState.firstEventBlock->Null.fromOption,
+  isHyperSync: (cs.sourceManager->SourceManager.getActiveSource).poweredByHyperSync,
+  latestFetchedBlockNumber: cs.fetchState->FetchState.bufferBlockNumber,
+  timestampCaughtUpToHeadOrEndblock: cs.timestampCaughtUpToHeadOrEndblock->Null.fromOption,
+}
+
+// Snapshot the chain's view for the status API.
+let toChainData = (cs: t): chainData => {
+  chainId: cs.chainConfig.id->Int.toFloat,
+  poweredByHyperSync: (cs.sourceManager->SourceManager.getActiveSource).poweredByHyperSync,
+  firstEventBlockNumber: cs.fetchState.firstEventBlock,
+  latestProcessedBlock: cs.committedProgressBlockNumber === -1
+    ? None
+    : Some(cs.committedProgressBlockNumber),
+  timestampCaughtUpToHeadOrEndblock: cs.timestampCaughtUpToHeadOrEndblock,
+  numEventsProcessed: cs.numEventsProcessed,
+  latestFetchedBlockNumber: Pervasives.max(cs.fetchState->FetchState.bufferBlockNumber, 0),
+  knownHeight: cs->hasProcessedToEndblock
+    ? cs.fetchState.endBlock->Option.getOr(cs.fetchState.knownHeight)
+    : cs.fetchState.knownHeight,
+  numBatchesFetched: 0,
+  startBlock: cs.fetchState.startBlock,
+  endBlock: cs.fetchState.endBlock,
+  numAddresses: cs.fetchState->FetchState.numAddresses,
+}
+
+// Snapshot the inputs a batch build needs from this chain.
+let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
+  fetchState: cs.fetchState,
+  progressBlockNumber: cs.committedProgressBlockNumber,
+  totalEventsProcessed: cs.numEventsProcessed,
+  sourceBlockNumber: cs.fetchState.knownHeight,
+  reorgDetection: cs.reorgDetection,
+  chainConfig: cs.chainConfig,
+}
+
+// Whether the chain's post-batch fetch frontier is ready to cross into the reorg
+// threshold, using the batch's progressed frontier when this chain advanced.
+let isReadyToEnterReorgThresholdAfterBatch = (cs: t, ~batch: Batch.t) => {
+  let fetchState = switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
+    cs.fetchState.chainId,
+  ) {
+  | Some(chainAfterBatch) => chainAfterBatch.fetchState
+  | None => cs.fetchState
+  }
+  fetchState->FetchState.isReadyToEnterReorgThreshold
+}
+
 // Commit the post-batch fetch frontier for a chain that progressed in the batch,
 // applying blockLag when this batch also crosses into the reorg threshold.
 let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
@@ -526,6 +696,8 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t) => {
 
       cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber
       cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed
+      // Processed blocks' transactions are no longer needed.
+      cs.transactionStore->TransactionStore.prune(chainAfterBatch.progressBlockNumber)
       cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated
       switch cs.safeCheckpointTracking {
       | Some(safeCheckpointTracking) =>
@@ -599,6 +771,7 @@ let rollback = (
     | None => ()
     }
     cs.fetchState = cs.fetchState->FetchState.rollback(~targetBlockNumber=newProgressBlockNumber)
+    cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
     cs.committedProgressBlockNumber = newProgressBlockNumber
     cs.numEventsProcessed = newTotalEventsProcessed
   | None =>
@@ -609,6 +782,7 @@ let rollback = (
         )
       cs.fetchState =
         cs.fetchState->FetchState.rollback(~targetBlockNumber=rollbackTargetBlockNumber)
+      cs.transactionStore->TransactionStore.rollback(rollbackTargetBlockNumber)
       cs.committedProgressBlockNumber = Pervasives.min(
         cs.committedProgressBlockNumber,
         rollbackTargetBlockNumber,

package/src/ChainState.res.mjs

@@ -10,6 +10,7 @@ import * as EvmChain from "./sources/EvmChain.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as EventRouter from "./sources/EventRouter.res.mjs";
+import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as SourceManager from "./sources/SourceManager.res.mjs";
@@ -17,7 +18,9 @@ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as ReorgDetection from "./ReorgDetection.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as HyperFuelSource from "./sources/HyperFuelSource.res.mjs";
+import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+import * as TransactionStore from "./sources/TransactionStore.res.mjs";
 import * as SvmHyperSyncSource from "./sources/SvmHyperSyncSource.res.mjs";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
@@ -35,11 +38,12 @@ function configAddresses(chainConfig) {
   return addresses;
 }
 
-function make(chainConfig, fetchState, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, logger) {
+function make(chainConfig, fetchState, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionFieldMaskOpt, logger) {
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
   let timestampCaughtUpToHeadOrEndblock = timestampCaughtUpToHeadOrEndblockOpt !== undefined ? Primitive_option.valFromOption(timestampCaughtUpToHeadOrEndblockOpt) : undefined;
   let isProgressAtHead = isProgressAtHeadOpt !== undefined ? isProgressAtHeadOpt : false;
+  let transactionFieldMask = transactionFieldMaskOpt !== undefined ? transactionFieldMaskOpt : 0;
   return {
     logger: logger,
     fetchState: fetchState,
@@ -49,8 +53,11 @@ function make(chainConfig, fetchState, sourceManager, reorgDetection, committedP
     timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
     committedProgressBlockNumber: committedProgressBlockNumber,
     numEventsProcessed: numEventsProcessed,
+    pendingBudget: 0,
     reorgDetection: reorgDetection,
-    safeCheckpointTracking: safeCheckpointTracking
+    safeCheckpointTracking: safeCheckpointTracking,
+    transactionStore: TransactionStore.make(),
+    transactionFieldMask: transactionFieldMask
   };
 }
 
@@ -169,7 +176,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
       sources$1 = sources._0;
       break;
   }
-  return make(chainConfig, fetchState, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, logger);
+  return make(chainConfig, fetchState, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, config.ecosystem.transactionFieldMask(eventConfigs), logger);
 }
 
 function makeFromConfig(chainConfig, config, registrations, knownHeight) {
@@ -193,10 +200,6 @@ function logger(cs) {
   return cs.logger;
 }
 
-function fetchState(cs) {
-  return cs.fetchState;
-}
-
 function sourceManager(cs) {
   return cs.sourceManager;
 }
@@ -225,10 +228,63 @@ function numEventsProcessed(cs) {
   return cs.numEventsProcessed;
 }
 
+function pendingBudget(cs) {
+  return cs.pendingBudget;
+}
+
 function timestampCaughtUpToHeadOrEndblock(cs) {
   return cs.timestampCaughtUpToHeadOrEndblock;
 }
 
+function knownHeight(cs) {
+  return cs.fetchState.knownHeight;
+}
+
+function indexingAddresses(cs) {
+  return cs.fetchState.indexingAddresses;
+}
+
+function bufferSize(cs) {
+  return FetchState.bufferSize(cs.fetchState);
+}
+
+function bufferReadyCount(cs) {
+  return FetchState.bufferReadyCount(cs.fetchState);
+}
+
+function getProgressPercentage(cs) {
+  return FetchState.getProgressPercentage(cs.fetchState);
+}
+
+function getProgressPercentageAt(cs, blockNumber) {
+  return FetchState.getProgressPercentageAt(cs.fetchState, blockNumber);
+}
+
+function hasReadyItem(cs) {
+  if (FetchState.isActivelyIndexing(cs.fetchState)) {
+    return FetchState.hasReadyItem(cs.fetchState);
+  } else {
+    return false;
+  }
+}
+
+function isReadyToEnterReorgThreshold(cs) {
+  return FetchState.isReadyToEnterReorgThreshold(cs.fetchState);
+}
+
+function startFetchingQueries(cs, queries) {
+  FetchState.startFetchingQueries(cs.fetchState, queries);
+  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.estResponseSize);
+}
+
+function getNextQuery(cs, budget) {
+  return FetchState.getNextQuery(cs.fetchState, budget, cs.pendingBudget);
+}
+
+function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
+  return SourceManager.dispatch(cs.sourceManager, cs.fetchState, executeQuery, waitForNewBlock, onNewBlock, action, stateId);
+}
+
 function hasProcessedToEndblock(cs) {
   let fetchState = cs.fetchState;
   let committedProgressBlockNumber = cs.committedProgressBlockNumber;
@@ -261,9 +317,33 @@ function isFetchingAtHead(cs) {
   return FetchState.isFetchingAtHead(cs.fetchState);
 }
 
-function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight) {
+function isAtHeadWithoutEndBlock(cs) {
+  if (cs.isProgressAtHead) {
+    return Stdlib_Option.isNone(cs.fetchState.endBlock);
+  } else {
+    return false;
+  }
+}
+
+function materializeBatchItems(cs, items) {
+  return TransactionStore.materializeItems(cs.transactionStore, items, cs.transactionFieldMask);
+}
+
+function materializePageItems(cs, items, page) {
+  if (page !== undefined) {
+    return TransactionStore.materializeItems(Primitive_option.valFromOption(page), items, cs.transactionFieldMask);
+  } else {
+    return Promise.resolve();
+  }
+}
+
+function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, page) {
+  if (page !== undefined) {
+    cs.transactionStore.merge(Primitive_option.valFromOption(page));
+  }
   let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, newItemsWithDcs) : cs.fetchState;
   cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
+  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.estResponseSize);
 }
 
 function registerReorgGuard(cs, blockHashes, knownHeight) {
@@ -277,6 +357,7 @@ function prepareReorg(cs, eventsProcessedDiff) {
     cs.numEventsProcessed = cs.numEventsProcessed + eventsProcessedDiff;
   }
   cs.fetchState = FetchState.resetPendingQueries(cs.fetchState);
+  cs.pendingBudget = 0;
 }
 
 function updateKnownHeight(cs, knownHeight) {
@@ -331,6 +412,48 @@ function enterReorgThreshold(cs) {
   cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined, undefined, cs.chainConfig.blockLag, undefined);
 }
 
+function toChainMetadata(cs) {
+  return {
+    first_event_block: Stdlib_Null.fromOption(cs.fetchState.firstEventBlock),
+    buffer_block: FetchState.bufferBlockNumber(cs.fetchState),
+    ready_at: Stdlib_Null.fromOption(cs.timestampCaughtUpToHeadOrEndblock),
+    _is_hyper_sync: SourceManager.getActiveSource(cs.sourceManager).poweredByHyperSync
+  };
+}
+
+function toChainData(cs) {
+  return {
+    chainId: cs.chainConfig.id,
+    poweredByHyperSync: SourceManager.getActiveSource(cs.sourceManager).poweredByHyperSync,
+    firstEventBlockNumber: cs.fetchState.firstEventBlock,
+    latestProcessedBlock: cs.committedProgressBlockNumber === -1 ? undefined : cs.committedProgressBlockNumber,
+    timestampCaughtUpToHeadOrEndblock: cs.timestampCaughtUpToHeadOrEndblock,
+    numEventsProcessed: cs.numEventsProcessed,
+    latestFetchedBlockNumber: Primitive_int.max(FetchState.bufferBlockNumber(cs.fetchState), 0),
+    currentBlockHeight: hasProcessedToEndblock(cs) ? Stdlib_Option.getOr(cs.fetchState.endBlock, cs.fetchState.knownHeight) : cs.fetchState.knownHeight,
+    numBatchesFetched: 0,
+    startBlock: cs.fetchState.startBlock,
+    endBlock: cs.fetchState.endBlock,
+    numAddresses: FetchState.numAddresses(cs.fetchState)
+  };
+}
+
+function toChainBeforeBatch(cs) {
+  return {
+    fetchState: cs.fetchState,
+    reorgDetection: cs.reorgDetection,
+    progressBlockNumber: cs.committedProgressBlockNumber,
+    sourceBlockNumber: cs.fetchState.knownHeight,
+    totalEventsProcessed: cs.numEventsProcessed,
+    chainConfig: cs.chainConfig
+  };
+}
+
+function isReadyToEnterReorgThresholdAfterBatch(cs, batch) {
+  let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
+  return FetchState.isReadyToEnterReorgThreshold(chainAfterBatch !== undefined ? chainAfterBatch.fetchState : cs.fetchState);
+}
+
 function advanceAfterBatch(cs, batch, enteringReorgThreshold) {
   let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
   if (chainAfterBatch !== undefined) {
@@ -381,6 +504,7 @@ function applyBatchProgress(cs, batch) {
   }
   cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber;
   cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed;
+  cs.transactionStore.prune(chainAfterBatch.progressBlockNumber);
   cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated;
   let safeCheckpointTracking = cs.safeCheckpointTracking;
   if (safeCheckpointTracking !== undefined) {
@@ -402,6 +526,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
     if (isReorgChain) {
       cs.reorgDetection = ReorgDetection.rollbackToValidBlockNumber(cs.reorgDetection, rollbackTargetBlockNumber);
       cs.fetchState = FetchState.rollback(cs.fetchState, rollbackTargetBlockNumber);
+      cs.transactionStore.rollback(rollbackTargetBlockNumber);
       cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
       return;
     } else {
@@ -423,6 +548,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
     cs.safeCheckpointTracking = SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber);
   }
   cs.fetchState = FetchState.rollback(cs.fetchState, newProgressBlockNumber);
+  cs.transactionStore.rollback(newProgressBlockNumber);
   cs.committedProgressBlockNumber = newProgressBlockNumber;
   cs.numEventsProcessed = newTotalEventsProcessed;
 }
@@ -433,7 +559,6 @@ export {
   makeFromConfig,
   makeFromDbState,
   logger,
-  fetchState,
   sourceManager,
   chainConfig,
   reorgDetection,
@@ -441,13 +566,32 @@ export {
   isProgressAtHead,
   committedProgressBlockNumber,
   numEventsProcessed,
+  pendingBudget,
+  startFetchingQueries,
   timestampCaughtUpToHeadOrEndblock,
+  knownHeight,
+  indexingAddresses,
+  bufferSize,
+  bufferReadyCount,
+  getProgressPercentage,
+  getProgressPercentageAt,
+  hasReadyItem,
+  isReadyToEnterReorgThreshold,
+  getNextQuery,
+  dispatch,
+  toChainData,
+  toChainMetadata,
+  toChainBeforeBatch,
+  isReadyToEnterReorgThresholdAfterBatch,
   hasProcessedToEndblock,
   getHighestBlockBelowThreshold,
   isActivelyIndexing,
   isReady,
   isFetchingAtHead,
+  isAtHeadWithoutEndBlock,
   handleQueryResult,
+  materializeBatchItems,
+  materializePageItems,
   registerReorgGuard,
   prepareReorg,
   updateKnownHeight,

package/src/ChainState.resi

@@ -15,6 +15,7 @@ let make: (
   ~numEventsProcessed: float=?,
   ~timestampCaughtUpToHeadOrEndblock: option<Date.t>=?,
   ~isProgressAtHead: bool=?,
+  ~transactionFieldMask: float=?,
   ~logger: Pino.t,
 ) => t
 
@@ -38,7 +39,6 @@ let makeFromDbState: (
 
 // Accessors.
 let logger: t => Pino.t
-let fetchState: t => FetchState.t
 let sourceManager: t => SourceManager.t
 let chainConfig: t => Config.chain
 let reorgDetection: t => ReorgDetection.t
@@ -46,14 +46,59 @@ let safeCheckpointTracking: t => option<SafeCheckpointTracking.t>
 let isProgressAtHead: t => bool
 let committedProgressBlockNumber: t => int
 let numEventsProcessed: t => float
+let pendingBudget: t => float
+let startFetchingQueries: (t, ~queries: array<FetchState.query>) => unit
 let timestampCaughtUpToHeadOrEndblock: t => option<Date.t>
 
+// Fetch-frontier reads.
+let knownHeight: t => int
+let indexingAddresses: t => dict<FetchState.indexingAddress>
+let bufferSize: t => int
+let bufferReadyCount: t => int
+let getProgressPercentage: t => float
+let getProgressPercentageAt: (t, ~blockNumber: int) => float
+let hasReadyItem: t => bool
+let isReadyToEnterReorgThreshold: t => bool
+
+// Fetch control.
+let getNextQuery: (t, ~budget: int) => FetchState.nextQuery
+let dispatch: (
+  t,
+  ~executeQuery: FetchState.query => promise<unit>,
+  ~waitForNewBlock: (~knownHeight: int) => promise<int>,
+  ~onNewBlock: (~knownHeight: int) => unit,
+  ~action: FetchState.nextQuery,
+  ~stateId: int,
+) => promise<unit>
+
+// Views.
+type chainData = {
+  chainId: float,
+  poweredByHyperSync: bool,
+  firstEventBlockNumber: option<int>,
+  latestProcessedBlock: option<int>,
+  timestampCaughtUpToHeadOrEndblock: option<Date.t>,
+  numEventsProcessed: float,
+  latestFetchedBlockNumber: int,
+  @as("currentBlockHeight")
+  knownHeight: int,
+  numBatchesFetched: int,
+  startBlock: int,
+  endBlock: option<int>,
+  numAddresses: int,
+}
+let toChainData: t => chainData
+let toChainMetadata: t => InternalTable.Chains.metaFields
+let toChainBeforeBatch: t => Batch.chainBeforeBatch
+let isReadyToEnterReorgThresholdAfterBatch: (t, ~batch: Batch.t) => bool
+
 // Derived (pure).
 let hasProcessedToEndblock: t => bool
 let getHighestBlockBelowThreshold: t => int
 let isActivelyIndexing: t => bool
 let isReady: t => bool
 let isFetchingAtHead: t => bool
+let isAtHeadWithoutEndBlock: t => bool
 
 // State transitions. The chain state is mutated only through these.
 let handleQueryResult: (
@@ -63,7 +108,14 @@ let handleQueryResult: (
   ~newItemsWithDcs: array<Internal.item>,
   ~latestFetchedBlock: FetchState.blockNumberAndTimestamp,
   ~knownHeight: int,
+  ~transactionStore: option<TransactionStore.t>,
 ) => unit
+let materializeBatchItems: (t, ~items: array<Internal.item>) => promise<unit>
+let materializePageItems: (
+  t,
+  ~items: array<Internal.item>,
+  ~page: option<TransactionStore.t>,
+) => promise<unit>
 let registerReorgGuard: (
   t,
   ~blockHashes: array<ReorgDetection.blockData>,

package/src/Config.res

@@ -195,9 +195,8 @@ let svmEventDescriptorSchema = S.schema(s =>
   {
     "discriminator": s.matches(S.option(S.string)),
     "discriminatorByteLen": s.matches(S.int),
-    "includeTransaction": s.matches(S.bool),
+    "transactionFields": s.matches(S.array(Internal.svmTransactionFieldSchema)),
     "includeLogs": s.matches(S.bool),
-    "includeTokenBalances": s.matches(S.bool),
     "accountFilters": s.matches(
       S.option(
         S.array(
@@ -704,9 +703,8 @@ let fromPublic = (publicConfigJson: JSON.t) => {
                 "svm": option<{
                   "discriminator": option<string>,
                   "discriminatorByteLen": int,
-                  "includeTransaction": bool,
+                  "transactionFields": array<Internal.svmTransactionField>,
                   "includeLogs": bool,
-                  "includeTokenBalances": bool,
                   "accountFilters": option<
                     array<array<{"position": int, "values": array<string>}>>,
                   >,
@@ -740,9 +738,8 @@ let fromPublic = (publicConfigJson: JSON.t) => {
             ~programId,
             ~discriminator=svm["discriminator"],
             ~discriminatorByteLen=svm["discriminatorByteLen"],
-            ~includeTransaction=svm["includeTransaction"],
+            ~transactionFields=svm["transactionFields"],
             ~includeLogs=svm["includeLogs"],
-            ~includeTokenBalances=svm["includeTokenBalances"],
             ~accountFilters,
             ~isInner=svm["isInner"],
             ~isWildcard=false,