envio 3.3.0-alpha.0 → 3.3.0-alpha.10

package/src/ChainState.res.mjs

@@ -7,9 +7,11 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as EvmChain from "./sources/EvmChain.res.mjs";
+import * as FieldMask from "./sources/FieldMask.res.mjs";
+import * as BlockStore from "./sources/BlockStore.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
-import * as EventRouter from "./sources/EventRouter.res.mjs";
+import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as SourceManager from "./sources/SourceManager.res.mjs";
@@ -17,7 +19,10 @@ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as ReorgDetection from "./ReorgDetection.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as HyperFuelSource from "./sources/HyperFuelSource.res.mjs";
+import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+import * as TransactionStore from "./sources/TransactionStore.res.mjs";
+import * as IndexingAddresses from "./IndexingAddresses.res.mjs";
 import * as SvmHyperSyncSource from "./sources/SvmHyperSyncSource.res.mjs";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
@@ -35,85 +40,51 @@ function configAddresses(chainConfig) {
   return addresses;
 }
 
-function make(chainConfig, fetchState, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, logger) {
+function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, logger) {
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
   let timestampCaughtUpToHeadOrEndblock = timestampCaughtUpToHeadOrEndblockOpt !== undefined ? Primitive_option.valFromOption(timestampCaughtUpToHeadOrEndblockOpt) : undefined;
   let isProgressAtHead = isProgressAtHeadOpt !== undefined ? isProgressAtHeadOpt : false;
+  let transactionStore = transactionStoreOpt !== undefined ? Primitive_option.valFromOption(transactionStoreOpt) : TransactionStore.make("evm", false);
+  let chainDensity = chainDensityOpt !== undefined ? Primitive_option.valFromOption(chainDensityOpt) : undefined;
+  let blockStore = blockStoreOpt !== undefined ? Primitive_option.valFromOption(blockStoreOpt) : BlockStore.make("evm", false);
   return {
     logger: logger,
     fetchState: fetchState,
+    indexingAddresses: indexingAddresses,
     sourceManager: sourceManager,
     chainConfig: chainConfig,
     isProgressAtHead: isProgressAtHead,
     timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
     committedProgressBlockNumber: committedProgressBlockNumber,
+    processingBlockNumber: committedProgressBlockNumber,
     numEventsProcessed: numEventsProcessed,
+    pendingBudget: 0,
+    chainDensity: chainDensity,
     reorgDetection: reorgDetection,
-    safeCheckpointTracking: safeCheckpointTracking
+    safeCheckpointTracking: safeCheckpointTracking,
+    transactionStore: transactionStore,
+    blockStore: blockStore
   };
 }
 
-function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrations, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, reducedPollingInterval) {
+function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrationsByChainId, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, reducedPollingInterval) {
   let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
   let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
-  let eventRouter = EventRouter.empty();
-  let eventConfigs = [];
-  let notRegisteredEvents = [];
+  let match = Stdlib_Option.getOr(registrationsByChainId[chainConfig.id.toString()], {
+    onEventRegistrations: [],
+    onBlockRegistrations: []
+  });
+  let onEventRegistrations = match.onEventRegistrations;
   chainConfig.contracts.forEach(contract => {
-    let contractName = contract.name;
-    contract.events.forEach(eventConfig => {
-      let hasContractRegister = Stdlib_Option.isSome(eventConfig.contractRegister);
-      EventRouter.addOrThrow(eventRouter, eventConfig.id, undefined, contractName, eventConfig.isWildcard, eventConfig.name, ChainMap.Chain.makeUnsafe(chainConfig.id));
-      let shouldBeIncluded;
-      if (config.enableRawEvents) {
-        shouldBeIncluded = true;
-      } else {
-        let isRegistered = hasContractRegister || Stdlib_Option.isSome(eventConfig.handler);
-        if (!isRegistered) {
-          notRegisteredEvents.push(eventConfig);
-        }
-        shouldBeIncluded = isRegistered;
-      }
-      let shouldSkip;
-      try {
-        let getEventFiltersOrThrow = eventConfig.getEventFiltersOrThrow;
-        if (getEventFiltersOrThrow) {
-          let match = getEventFiltersOrThrow(ChainMap.Chain.makeUnsafe(chainConfig.id));
-          shouldSkip = match.TAG === "Static" ? match._0.length === 0 : false;
-        } else {
-          shouldSkip = false;
-        }
-      } catch (exn) {
-        shouldSkip = false;
-      }
-      if (shouldBeIncluded && !shouldSkip) {
-        eventConfigs.push(eventConfig);
-        return;
-      }
-    });
     let startBlock = contract.startBlock;
     if (startBlock !== undefined && startBlock < chainConfig.startBlock) {
-      return Stdlib_JsError.throwWithMessage(`The start block for contract "` + contractName + `" is less than the chain start block. This is not supported yet.`);
+      return Stdlib_JsError.throwWithMessage(`The start block for contract "` + contract.name + `" is less than the chain start block. This is not supported yet.`);
     }
   });
-  if (Utils.$$Array.notEmpty(notRegisteredEvents)) {
-    Logging.childInfo(logger, `The event` + (
-      notRegisteredEvents.length > 1 ? "s" : ""
-    ) + ` ` + notRegisteredEvents.map(eventConfig => eventConfig.contractName + `.` + eventConfig.name).join(", ") + ` don't have an event handler and skipped for indexing.`);
-  }
-  let onBlockConfigs = registrations.onBlockByChainId[chainConfig.id.toString()];
-  if (onBlockConfigs !== undefined) {
-    onBlockConfigs.forEach(onBlockConfig => {
-      if (Stdlib_Option.getOr(onBlockConfig.startBlock, startBlock) < startBlock) {
-        Stdlib_JsError.throwWithMessage(`The start block for onBlock handler "` + onBlockConfig.name + `" is less than the chain start block (` + startBlock.toString() + `). This is not supported yet.`);
-      }
-      if (endBlock !== undefined && Stdlib_Option.getOr(onBlockConfig.endBlock, endBlock) > endBlock) {
-        return Stdlib_JsError.throwWithMessage(`The end block for onBlock handler "` + onBlockConfig.name + `" is greater than the chain end block (` + endBlock.toString() + `). This is not supported yet.`);
-      }
-    });
-  }
-  let fetchState = FetchState.make(startBlock, endBlock, eventConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, onBlockConfigs, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
+  let contractConfigs = IndexingAddresses.makeContractConfigs(onEventRegistrations);
+  let indexingAddressIndex = IndexingAddresses.make(contractConfigs, indexingAddresses);
+  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
   let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
     if (reorgCheckpoint.chain_id === chainConfig.id) {
       return reorgCheckpoint;
@@ -125,22 +96,19 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
   let sources$1;
   switch (sources.TAG) {
     case "EvmSourceConfig" :
-      let evmContracts = chainConfig.contracts.map(contract => ({
-        name: contract.name,
-        abi: contract.abi,
-        events: contract.events
-      }));
       let evmRpcs = sources.rpcs.map(rpc => {
         let syncConfig = rpc.syncConfig;
         let ws = rpc.ws;
+        let headers = rpc.headers;
         return {
           url: rpc.url,
           sourceFor: rpc.sourceFor,
           syncConfig: syncConfig,
-          ws: ws
+          ws: ws,
+          headers: headers
         };
       });
-      sources$1 = EvmChain.makeSources(chain, evmContracts, sources.hypersync, evmRpcs, lowercaseAddresses);
+      sources$1 = EvmChain.makeSources(chain, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses);
       break;
     case "FuelSourceConfig" :
       sources$1 = [HyperFuelSource.make({
@@ -152,51 +120,46 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
     case "SvmSourceConfig" :
       let rpc = sources.rpc;
       let hypersync = sources.hypersync;
-      if (hypersync !== undefined) {
-        let svmEventConfigs = chainConfig.contracts.flatMap(contract => contract.events);
-        sources$1 = [SvmHyperSyncSource.make({
+      sources$1 = hypersync !== undefined ? [SvmHyperSyncSource.make({
             chain: chain,
             endpointUrl: hypersync,
             apiToken: Env.envioApiToken,
-            eventConfigs: svmEventConfigs,
+            onEventRegistrations: onEventRegistrations,
             clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis
-          })];
-      } else {
-        sources$1 = rpc !== undefined ? [Svm.makeRPCSource(chain, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + chain.toString() + ` has no SVM data source`);
-      }
+          })] : (
+          rpc !== undefined ? [Svm.makeRPCSource(chain, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + chain.toString() + ` has no SVM data source`)
+        );
       break;
     case "CustomSources" :
       sources$1 = sources._0;
       break;
   }
-  return make(chainConfig, fetchState, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, logger);
+  let firstEventBlock$1 = fetchState.firstEventBlock;
+  let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
+  return make(chainConfig, fetchState, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), logger);
 }
 
-function makeFromConfig(chainConfig, config, registrations, knownHeight) {
+function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight) {
   let logger = Logging.createChild({
     chainId: chainConfig.id
   });
-  return makeInternal(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined);
+  return makeInternal(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrationsByChainId, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined);
 }
 
-function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrations, reducedPollingInterval) {
+function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval) {
   let chainId = chainConfig.id;
   let logger = Logging.createChild({
     chainId: chainId
   });
   Prometheus.ProgressEventsCount.set(resumedChainState.numEventsProcessed, chainId);
   let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
-  return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrations, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, reducedPollingInterval);
+  return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrationsByChainId, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, reducedPollingInterval);
 }
 
 function logger(cs) {
   return cs.logger;
 }
 
-function fetchState(cs) {
-  return cs.fetchState;
-}
-
 function sourceManager(cs) {
   return cs.sourceManager;
 }
@@ -225,10 +188,152 @@ function numEventsProcessed(cs) {
   return cs.numEventsProcessed;
 }
 
+function pendingBudget(cs) {
+  return cs.pendingBudget;
+}
+
 function timestampCaughtUpToHeadOrEndblock(cs) {
   return cs.timestampCaughtUpToHeadOrEndblock;
 }
 
+function knownHeight(cs) {
+  return cs.fetchState.knownHeight;
+}
+
+function contractAddresses(cs, contractName) {
+  return IndexingAddresses.getContractAddresses(cs.indexingAddresses, contractName);
+}
+
+function bufferSize(cs) {
+  return FetchState.bufferSize(cs.fetchState);
+}
+
+function bufferReadyCount(cs) {
+  return FetchState.bufferReadyCount(cs.fetchState);
+}
+
+function getProgressPercentage(cs) {
+  return FetchState.getProgressPercentage(cs.fetchState);
+}
+
+function chainDensity(cs) {
+  return cs.chainDensity;
+}
+
+function hasReadyItem(cs) {
+  if (FetchState.isActivelyIndexing(cs.fetchState)) {
+    return FetchState.hasReadyItem(cs.fetchState);
+  } else {
+    return false;
+  }
+}
+
+function isReadyToEnterReorgThreshold(cs) {
+  return FetchState.isReadyToEnterReorgThreshold(cs.fetchState);
+}
+
+function startFetchingQueries(cs, queries) {
+  FetchState.startFetchingQueries(cs.fetchState, queries);
+  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsEst);
+}
+
+function isReady(cs) {
+  return cs.timestampCaughtUpToHeadOrEndblock !== undefined;
+}
+
+function fetchCeiling(cs) {
+  let fetchState = cs.fetchState;
+  let endBlock = fetchState.endBlock;
+  if (endBlock !== undefined) {
+    return Primitive_int.min(endBlock, fetchState.knownHeight);
+  } else {
+    return fetchState.knownHeight;
+  }
+}
+
+function readyBufferDensity(cs) {
+  let readyCount = FetchState.bufferReadyCount(cs.fetchState);
+  let span = FetchState.bufferBlockNumber(cs.fetchState) - cs.processingBlockNumber | 0;
+  if (readyCount > 0 && span > 0) {
+    return readyCount / span;
+  }
+}
+
+function effectiveDensity(cs) {
+  let match = cs.chainDensity;
+  let match$1 = readyBufferDensity(cs);
+  if (match !== undefined) {
+    if (match$1 !== undefined) {
+      return Primitive_float.max(match, match$1);
+    } else {
+      return match;
+    }
+  } else if (match$1 !== undefined) {
+    return match$1;
+  } else {
+    return;
+  }
+}
+
+function targetBlock(cs, chainTargetItems) {
+  let fetchState = cs.fetchState;
+  let fetchCeiling$1 = fetchCeiling(cs);
+  let bufferBlockNumber = FetchState.bufferBlockNumber(fetchState);
+  let density = effectiveDensity(cs);
+  if (density !== undefined && density > 0) {
+    return Primitive_int.min(fetchCeiling$1, bufferBlockNumber + (Math.ceil(chainTargetItems / density) | 0) | 0);
+  } else {
+    return Primitive_int.min(bufferBlockNumber + 20000 | 0, fetchCeiling$1);
+  }
+}
+
+function progressRange(cs) {
+  let fetchState = cs.fetchState;
+  let lower = Stdlib_Option.getOr(fetchState.firstEventBlock, fetchState.startBlock);
+  let endBlock = fetchState.endBlock;
+  let upper = endBlock !== undefined ? endBlock : fetchState.knownHeight;
+  return [
+    lower,
+    upper
+  ];
+}
+
+function progressAtBlock(cs, blockNumber) {
+  let match = progressRange(cs);
+  let upper = match[1];
+  let lower = match[0];
+  if (upper <= lower) {
+    return 1;
+  } else {
+    return Primitive_float.max(0, Primitive_float.min(1, (blockNumber - lower | 0) / (upper - lower | 0)));
+  }
+}
+
+function blockAtProgress(cs, progress) {
+  let match = progressRange(cs);
+  let lower = match[0];
+  return lower + (Math.ceil(progress * (match[1] - lower | 0)) | 0) | 0;
+}
+
+function getNextQuery(cs, chainTargetItems, chunkItemsMultiplierOpt, maxTargetBlock) {
+  let chunkItemsMultiplier = chunkItemsMultiplierOpt !== undefined ? chunkItemsMultiplierOpt : 1;
+  let chainTargetBlock = targetBlock(cs, chainTargetItems);
+  let chainTargetBlock$1 = maxTargetBlock !== undefined ? Primitive_int.min(chainTargetBlock, maxTargetBlock) : chainTargetBlock;
+  let density = effectiveDensity(cs);
+  let chainTargetItems$1;
+  if (density !== undefined && density > 0) {
+    let rangeCost = density * (chainTargetBlock$1 - FetchState.bufferBlockNumber(cs.fetchState) | 0);
+    chainTargetItems$1 = Primitive_float.min(chainTargetItems, Math.ceil(rangeCost) + cs.pendingBudget);
+  } else {
+    chainTargetItems$1 = chainTargetItems;
+  }
+  return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1, chunkItemsMultiplier);
+}
+
+function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
+  return SourceManager.dispatch(cs.sourceManager, cs.fetchState, executeQuery, waitForNewBlock, onNewBlock, action, stateId);
+}
+
 function hasProcessedToEndblock(cs) {
   let fetchState = cs.fetchState;
   let committedProgressBlockNumber = cs.committedProgressBlockNumber;
@@ -253,17 +358,156 @@ function isActivelyIndexing(cs) {
   return FetchState.isActivelyIndexing(cs.fetchState);
 }
 
-function isReady(cs) {
-  return cs.timestampCaughtUpToHeadOrEndblock !== undefined;
-}
-
 function isFetchingAtHead(cs) {
   return FetchState.isFetchingAtHead(cs.fetchState);
 }
 
-function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight) {
-  let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, newItemsWithDcs) : cs.fetchState;
-  cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
+function isAtHeadWithoutEndBlock(cs) {
+  if (cs.isProgressAtHead) {
+    return Stdlib_Option.isNone(cs.fetchState.endBlock);
+  } else {
+    return false;
+  }
+}
+
+function groupBatchItems(items, includeBlocks) {
+  let txBlockNumbers = [];
+  let transactionIndices = [];
+  let transactionMasks = [];
+  let payloadGroups = [];
+  let anyTransactionFieldSelected = {
+    contents: false
+  };
+  let blockBlockNumbers = [];
+  let blockMasks = [];
+  let blockItemGroups = [];
+  items.forEach(item => {
+    if (item.kind !== 0) {
+      return;
+    }
+    let blockNumber = item.blockNumber;
+    let match = item.payload.transaction;
+    if (match == null) {
+      let transactionIndex = item.transactionIndex;
+      let mask = item.onEventRegistration.eventConfig.transactionFieldMask;
+      if (mask !== 0) {
+        anyTransactionFieldSelected.contents = true;
+      }
+      let last = payloadGroups.length - 1 | 0;
+      if (last >= 0 && txBlockNumbers[last] === blockNumber && transactionIndices[last] === transactionIndex) {
+        payloadGroups[last].push(item.payload);
+        transactionMasks[last] = FieldMask.orMask(transactionMasks[last], mask);
+      } else {
+        txBlockNumbers.push(blockNumber);
+        transactionIndices.push(transactionIndex);
+        transactionMasks.push(mask);
+        payloadGroups.push([item.payload]);
+      }
+    }
+    if (!includeBlocks) {
+      return;
+    }
+    let match$1 = item.payload.block;
+    if (!(match$1 == null)) {
+      return;
+    }
+    let mask$1 = item.onEventRegistration.eventConfig.blockFieldMask;
+    let last$1 = blockItemGroups.length - 1 | 0;
+    if (last$1 >= 0 && blockBlockNumbers[last$1] === blockNumber) {
+      blockItemGroups[last$1].push(item);
+      blockMasks[last$1] = FieldMask.orMask(blockMasks[last$1], mask$1);
+    } else {
+      blockBlockNumbers.push(blockNumber);
+      blockMasks.push(mask$1);
+      blockItemGroups.push([item]);
+    }
+  });
+  return [
+    {
+      txBlockNumbers: txBlockNumbers,
+      transactionIndices: transactionIndices,
+      transactionMasks: transactionMasks,
+      payloadGroups: payloadGroups,
+      anyTransactionFieldSelected: anyTransactionFieldSelected.contents
+    },
+    {
+      blockBlockNumbers: blockBlockNumbers,
+      blockMasks: blockMasks,
+      blockItemGroups: blockItemGroups
+    }
+  ];
+}
+
+async function applyTransactionGroups(store, g) {
+  if (!Utils.$$Array.notEmpty(g.payloadGroups)) {
+    return;
+  }
+  if (g.anyTransactionFieldSelected) {
+    let txs = await store.materialize(g.txBlockNumbers, g.transactionIndices, g.transactionMasks);
+    g.payloadGroups.forEach((payloads, i) => {
+      let tx = txs[i];
+      payloads.forEach(payload => {
+        payload.transaction = tx;
+      });
+    });
+    return;
+  }
+  g.payloadGroups.forEach(payloads => {
+    payloads.forEach(payload => {
+      payload.transaction = {};
+    });
+  });
+}
+
+async function applyBlockGroups(store, g) {
+  if (!Utils.$$Array.notEmpty(g.blockItemGroups)) {
+    return;
+  }
+  let blocks = await store.materialize(g.blockBlockNumbers, g.blockMasks);
+  g.blockItemGroups.forEach((group, i) => {
+    let block = blocks[i];
+    group.forEach(ei => {
+      ei.payload.block = block;
+    });
+  });
+}
+
+function includeBlocksForEcosystem(ecosystem) {
+  switch (ecosystem) {
+    case "fuel" :
+      return false;
+    case "evm" :
+    case "svm" :
+      return true;
+  }
+}
+
+async function materializeBatchItems(cs, items, ecosystem) {
+  let match = groupBatchItems(items, includeBlocksForEcosystem(ecosystem));
+  await Promise.all([
+    applyTransactionGroups(cs.transactionStore, match[0]),
+    applyBlockGroups(cs.blockStore, match[1])
+  ]);
+}
+
+async function materializePageItems(items, transactionStore, blockStore, ecosystem) {
+  let match = groupBatchItems(items, includeBlocksForEcosystem(ecosystem));
+  await Promise.all([
+    transactionStore !== undefined ? applyTransactionGroups(Primitive_option.valFromOption(transactionStore), match[0]) : Promise.resolve(),
+    blockStore !== undefined ? applyBlockGroups(Primitive_option.valFromOption(blockStore), match[1]) : Promise.resolve()
+  ]);
+}
+
+function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, txPage, blockPage) {
+  if (txPage !== undefined) {
+    cs.transactionStore.merge(Primitive_option.valFromOption(txPage));
+  }
+  if (blockPage !== undefined) {
+    cs.blockStore.merge(Primitive_option.valFromOption(blockPage));
+  }
+  let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.indexingAddresses, newItemsWithDcs) : cs.fetchState;
+  cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, cs.indexingAddresses, query, latestFetchedBlock, newItems), knownHeight);
+  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsEst);
 }
 
 function registerReorgGuard(cs, blockHashes, knownHeight) {
@@ -277,6 +521,7 @@ function prepareReorg(cs, eventsProcessedDiff) {
     cs.numEventsProcessed = cs.numEventsProcessed + eventsProcessedDiff;
   }
   cs.fetchState = FetchState.resetPendingQueries(cs.fetchState);
+  cs.pendingBudget = 0;
 }
 
 function updateKnownHeight(cs, knownHeight) {
@@ -295,14 +540,13 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
       startBlock: init.startBlock,
       endBlock: blockNumber,
       normalSelection: init.normalSelection,
-      indexingAddresses: init.indexingAddresses,
       contractConfigs: init.contractConfigs,
       chainId: init.chainId,
       latestOnBlockBlockNumber: init.latestOnBlockBlockNumber,
       blockLag: init.blockLag,
       buffer: init.buffer,
       maxOnBlockBufferSize: init.maxOnBlockBufferSize,
-      onBlockConfigs: init.onBlockConfigs,
+      onBlockRegistrations: init.onBlockRegistrations,
       knownHeight: init.knownHeight,
       firstEventBlock: init.firstEventBlock
     };
@@ -314,32 +558,74 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
     startBlock: init$1.startBlock,
     endBlock: blockNumber,
     normalSelection: init$1.normalSelection,
-    indexingAddresses: init$1.indexingAddresses,
     contractConfigs: init$1.contractConfigs,
     chainId: init$1.chainId,
     latestOnBlockBlockNumber: init$1.latestOnBlockBlockNumber,
     blockLag: init$1.blockLag,
     buffer: init$1.buffer,
     maxOnBlockBufferSize: init$1.maxOnBlockBufferSize,
-    onBlockConfigs: init$1.onBlockConfigs,
+    onBlockRegistrations: init$1.onBlockRegistrations,
     knownHeight: init$1.knownHeight,
     firstEventBlock: init$1.firstEventBlock
   };
 }
 
 function enterReorgThreshold(cs) {
-  cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined, undefined, cs.chainConfig.blockLag, undefined);
+  cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined, cs.chainConfig.blockLag, undefined);
+}
+
+function toChainMetadata(cs) {
+  return {
+    first_event_block: Stdlib_Null.fromOption(cs.fetchState.firstEventBlock),
+    buffer_block: FetchState.bufferBlockNumber(cs.fetchState),
+    ready_at: Stdlib_Null.fromOption(cs.timestampCaughtUpToHeadOrEndblock),
+    _is_hyper_sync: SourceManager.getActiveSource(cs.sourceManager).poweredByHyperSync
+  };
+}
+
+function toChainData(cs) {
+  return {
+    chainId: cs.chainConfig.id,
+    poweredByHyperSync: SourceManager.getActiveSource(cs.sourceManager).poweredByHyperSync,
+    firstEventBlockNumber: cs.fetchState.firstEventBlock,
+    latestProcessedBlock: cs.committedProgressBlockNumber === -1 ? undefined : cs.committedProgressBlockNumber,
+    timestampCaughtUpToHeadOrEndblock: cs.timestampCaughtUpToHeadOrEndblock,
+    numEventsProcessed: cs.numEventsProcessed,
+    latestFetchedBlockNumber: Primitive_int.max(FetchState.bufferBlockNumber(cs.fetchState), 0),
+    currentBlockHeight: hasProcessedToEndblock(cs) ? Stdlib_Option.getOr(cs.fetchState.endBlock, cs.fetchState.knownHeight) : cs.fetchState.knownHeight,
+    numBatchesFetched: 0,
+    startBlock: cs.fetchState.startBlock,
+    endBlock: cs.fetchState.endBlock,
+    numAddresses: IndexingAddresses.size(cs.indexingAddresses)
+  };
+}
+
+function toChainBeforeBatch(cs) {
+  return {
+    fetchState: cs.fetchState,
+    reorgDetection: cs.reorgDetection,
+    progressBlockNumber: cs.committedProgressBlockNumber,
+    sourceBlockNumber: cs.fetchState.knownHeight,
+    totalEventsProcessed: cs.numEventsProcessed,
+    chainConfig: cs.chainConfig
+  };
+}
+
+function isReadyToEnterReorgThresholdAfterBatch(cs, batch) {
+  let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
+  return FetchState.isReadyToEnterReorgThreshold(chainAfterBatch !== undefined ? chainAfterBatch.fetchState : cs.fetchState);
 }
 
 function advanceAfterBatch(cs, batch, enteringReorgThreshold) {
   let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
   if (chainAfterBatch !== undefined) {
-    cs.fetchState = enteringReorgThreshold ? FetchState.updateInternal(chainAfterBatch.fetchState, undefined, undefined, undefined, cs.chainConfig.blockLag, undefined) : chainAfterBatch.fetchState;
+    cs.fetchState = enteringReorgThreshold ? FetchState.updateInternal(chainAfterBatch.fetchState, undefined, undefined, cs.chainConfig.blockLag, undefined) : chainAfterBatch.fetchState;
+    cs.processingBlockNumber = chainAfterBatch.progressBlockNumber;
     return;
   }
 }
 
-function applyBatchProgress(cs, batch) {
+function applyBatchProgress(cs, batch, blockTimestampName) {
   let chainId = cs.chainConfig.id;
   let chainAfterBatch = batch.progressedChainsById[chainId];
   if (chainAfterBatch === undefined) {
@@ -351,9 +637,9 @@ function applyBatchProgress(cs, batch) {
   if (cs.numEventsProcessed !== chainAfterBatch.totalEventsProcessed) {
     Prometheus.ProgressEventsCount.set(chainAfterBatch.totalEventsProcessed, chainId);
   }
-  let eventItem = Batch.findLastEventItem(batch, chainId);
-  if (eventItem !== undefined) {
-    let blockTimestampMs = eventItem.timestamp * 1000 | 0;
+  let blockTimestamp = Stdlib_Option.flatMap(Stdlib_Option.flatMap(Batch.findLastEventItem(batch, chainId), eventItem => Primitive_option.fromNullable(eventItem.payload.block)), block => block[blockTimestampName]);
+  if (blockTimestamp !== undefined) {
+    let blockTimestampMs = Primitive_option.valFromOption(blockTimestamp) * 1000 | 0;
     Prometheus.ProgressLatency.set((Date.now() | 0) - blockTimestampMs | 0, chainId);
   }
   let match = cs.fetchState.firstEventBlock;
@@ -366,21 +652,45 @@ function applyBatchProgress(cs, batch) {
         startBlock: init.startBlock,
         endBlock: init.endBlock,
         normalSelection: init.normalSelection,
-        indexingAddresses: init.indexingAddresses,
         contractConfigs: init.contractConfigs,
         chainId: init.chainId,
         latestOnBlockBlockNumber: init.latestOnBlockBlockNumber,
         blockLag: init.blockLag,
         buffer: init.buffer,
         maxOnBlockBufferSize: init.maxOnBlockBufferSize,
-        onBlockConfigs: init.onBlockConfigs,
+        onBlockRegistrations: init.onBlockRegistrations,
         knownHeight: init.knownHeight,
         firstEventBlock: firstEventBlock
       };
     }
   }
+  let deltaBlocks = chainAfterBatch.progressBlockNumber - cs.committedProgressBlockNumber | 0;
+  if (deltaBlocks > 0) {
+    let deltaEvents = chainAfterBatch.totalEventsProcessed - cs.numEventsProcessed;
+    let match$1 = cs.chainDensity;
+    let match$2 = deltaEvents > 0;
+    let exit = 0;
+    if (match$1 !== undefined || match$2) {
+      exit = 1;
+    }
+    if (exit === 1) {
+      let batchDensity = deltaEvents / deltaBlocks;
+      let oldDensity = cs.chainDensity;
+      let tmp;
+      if (oldDensity !== undefined) {
+        let alpha = Primitive_float.min(1, deltaBlocks / 100);
+        tmp = oldDensity * (1 - alpha) + batchDensity * alpha;
+      } else {
+        tmp = batchDensity;
+      }
+      cs.chainDensity = tmp;
+    }
+  }
   cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber;
+  cs.processingBlockNumber = Primitive_int.max(cs.processingBlockNumber, chainAfterBatch.progressBlockNumber);
   cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed;
+  cs.transactionStore.prune(chainAfterBatch.progressBlockNumber);
+  cs.blockStore.prune(chainAfterBatch.progressBlockNumber);
   cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated;
   let safeCheckpointTracking = cs.safeCheckpointTracking;
   if (safeCheckpointTracking !== undefined) {
@@ -401,8 +711,11 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
   if (newProgressBlockNumber === undefined) {
     if (isReorgChain) {
       cs.reorgDetection = ReorgDetection.rollbackToValidBlockNumber(cs.reorgDetection, rollbackTargetBlockNumber);
-      cs.fetchState = FetchState.rollback(cs.fetchState, rollbackTargetBlockNumber);
+      cs.fetchState = FetchState.rollback(cs.fetchState, cs.indexingAddresses, rollbackTargetBlockNumber);
+      cs.transactionStore.rollback(rollbackTargetBlockNumber);
+      cs.blockStore.rollback(rollbackTargetBlockNumber);
       cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
+      cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, rollbackTargetBlockNumber);
       return;
     } else {
       return;
@@ -422,8 +735,11 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
   if (safeCheckpointTracking !== undefined) {
     cs.safeCheckpointTracking = SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber);
   }
-  cs.fetchState = FetchState.rollback(cs.fetchState, newProgressBlockNumber);
+  cs.fetchState = FetchState.rollback(cs.fetchState, cs.indexingAddresses, newProgressBlockNumber);
+  cs.transactionStore.rollback(newProgressBlockNumber);
+  cs.blockStore.rollback(newProgressBlockNumber);
   cs.committedProgressBlockNumber = newProgressBlockNumber;
+  cs.processingBlockNumber = newProgressBlockNumber;
   cs.numEventsProcessed = newTotalEventsProcessed;
 }
 
@@ -433,7 +749,6 @@ export {
   makeFromConfig,
   makeFromDbState,
   logger,
-  fetchState,
   sourceManager,
   chainConfig,
   reorgDetection,
@@ -441,13 +756,36 @@ export {
   isProgressAtHead,
   committedProgressBlockNumber,
   numEventsProcessed,
+  pendingBudget,
+  startFetchingQueries,
   timestampCaughtUpToHeadOrEndblock,
+  knownHeight,
+  contractAddresses,
+  bufferSize,
+  bufferReadyCount,
+  getProgressPercentage,
+  chainDensity,
+  effectiveDensity,
+  hasReadyItem,
+  isReadyToEnterReorgThreshold,
+  targetBlock,
+  progressAtBlock,
+  blockAtProgress,
+  getNextQuery,
+  dispatch,
+  toChainData,
+  toChainMetadata,
+  toChainBeforeBatch,
+  isReadyToEnterReorgThresholdAfterBatch,
   hasProcessedToEndblock,
   getHighestBlockBelowThreshold,
   isActivelyIndexing,
   isReady,
   isFetchingAtHead,
+  isAtHeadWithoutEndBlock,
   handleQueryResult,
+  materializeBatchItems,
+  materializePageItems,
   registerReorgGuard,
   prepareReorg,
   updateKnownHeight,