envio 3.3.0-alpha.7 → 3.3.0-alpha.9

package/src/ChainState.res.mjs

@@ -7,9 +7,10 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as EvmChain from "./sources/EvmChain.res.mjs";
+import * as FieldMask from "./sources/FieldMask.res.mjs";
+import * as BlockStore from "./sources/BlockStore.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
-import * as EventRouter from "./sources/EventRouter.res.mjs";
 import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
@@ -39,12 +40,14 @@ function configAddresses(chainConfig) {
   return addresses;
 }
 
-function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, logger) {
+function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, logger) {
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
   let timestampCaughtUpToHeadOrEndblock = timestampCaughtUpToHeadOrEndblockOpt !== undefined ? Primitive_option.valFromOption(timestampCaughtUpToHeadOrEndblockOpt) : undefined;
   let isProgressAtHead = isProgressAtHeadOpt !== undefined ? isProgressAtHeadOpt : false;
   let transactionStore = transactionStoreOpt !== undefined ? Primitive_option.valFromOption(transactionStoreOpt) : TransactionStore.make("evm", false);
+  let chainDensity = chainDensityOpt !== undefined ? Primitive_option.valFromOption(chainDensityOpt) : undefined;
+  let blockStore = blockStoreOpt !== undefined ? Primitive_option.valFromOption(blockStoreOpt) : BlockStore.make("evm", false);
   return {
     logger: logger,
     fetchState: fetchState,
@@ -56,74 +59,31 @@ function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDe
     committedProgressBlockNumber: committedProgressBlockNumber,
     numEventsProcessed: numEventsProcessed,
     pendingBudget: 0,
+    chainDensity: chainDensity,
     reorgDetection: reorgDetection,
     safeCheckpointTracking: safeCheckpointTracking,
-    transactionStore: transactionStore
+    transactionStore: transactionStore,
+    blockStore: blockStore
   };
 }
 
-function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrations, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, reducedPollingInterval) {
+function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrationsByChainId, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, reducedPollingInterval) {
   let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
   let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
-  let eventRouter = EventRouter.empty();
-  let eventConfigs = [];
-  let notRegisteredEvents = [];
+  let match = Stdlib_Option.getOr(registrationsByChainId[chainConfig.id.toString()], {
+    onEventRegistrations: [],
+    onBlockRegistrations: []
+  });
+  let onEventRegistrations = match.onEventRegistrations;
   chainConfig.contracts.forEach(contract => {
-    let contractName = contract.name;
-    contract.events.forEach(eventConfig => {
-      let hasContractRegister = Stdlib_Option.isSome(eventConfig.contractRegister);
-      EventRouter.addOrThrow(eventRouter, eventConfig.id, undefined, contractName, eventConfig.isWildcard, eventConfig.name, ChainMap.Chain.makeUnsafe(chainConfig.id));
-      let shouldBeIncluded;
-      if (config.enableRawEvents) {
-        shouldBeIncluded = true;
-      } else {
-        let isRegistered = hasContractRegister || Stdlib_Option.isSome(eventConfig.handler);
-        if (!isRegistered) {
-          notRegisteredEvents.push(eventConfig);
-        }
-        shouldBeIncluded = isRegistered;
-      }
-      let shouldSkip;
-      try {
-        let getEventFiltersOrThrow = eventConfig.getEventFiltersOrThrow;
-        if (getEventFiltersOrThrow) {
-          let match = getEventFiltersOrThrow(ChainMap.Chain.makeUnsafe(chainConfig.id));
-          shouldSkip = match.TAG === "Static" ? match._0.length === 0 : false;
-        } else {
-          shouldSkip = false;
-        }
-      } catch (exn) {
-        shouldSkip = false;
-      }
-      if (shouldBeIncluded && !shouldSkip) {
-        eventConfigs.push(eventConfig);
-        return;
-      }
-    });
     let startBlock = contract.startBlock;
     if (startBlock !== undefined && startBlock < chainConfig.startBlock) {
-      return Stdlib_JsError.throwWithMessage(`The start block for contract "` + contractName + `" is less than the chain start block. This is not supported yet.`);
+      return Stdlib_JsError.throwWithMessage(`The start block for contract "` + contract.name + `" is less than the chain start block. This is not supported yet.`);
     }
   });
-  if (Utils.$$Array.notEmpty(notRegisteredEvents)) {
-    Logging.childInfo(logger, `The event` + (
-      notRegisteredEvents.length > 1 ? "s" : ""
-    ) + ` ` + notRegisteredEvents.map(eventConfig => eventConfig.contractName + `.` + eventConfig.name).join(", ") + ` don't have an event handler and skipped for indexing.`);
-  }
-  let onBlockConfigs = registrations.onBlockByChainId[chainConfig.id.toString()];
-  if (onBlockConfigs !== undefined) {
-    onBlockConfigs.forEach(onBlockConfig => {
-      if (Stdlib_Option.getOr(onBlockConfig.startBlock, startBlock) < startBlock) {
-        Stdlib_JsError.throwWithMessage(`The start block for onBlock handler "` + onBlockConfig.name + `" is less than the chain start block (` + startBlock.toString() + `). This is not supported yet.`);
-      }
-      if (endBlock !== undefined && Stdlib_Option.getOr(onBlockConfig.endBlock, endBlock) > endBlock) {
-        return Stdlib_JsError.throwWithMessage(`The end block for onBlock handler "` + onBlockConfig.name + `" is greater than the chain end block (` + endBlock.toString() + `). This is not supported yet.`);
-      }
-    });
-  }
-  let contractConfigs = IndexingAddresses.makeContractConfigs(eventConfigs);
+  let contractConfigs = IndexingAddresses.makeContractConfigs(onEventRegistrations);
   let indexingAddressIndex = IndexingAddresses.make(contractConfigs, indexingAddresses);
-  let fetchState = FetchState.make(startBlock, endBlock, eventConfigs, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, onBlockConfigs, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
+  let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
   let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
     if (reorgCheckpoint.chain_id === chainConfig.id) {
       return reorgCheckpoint;
@@ -135,11 +95,6 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
   let sources$1;
   switch (sources.TAG) {
     case "EvmSourceConfig" :
-      let evmContracts = chainConfig.contracts.map(contract => ({
-        name: contract.name,
-        abi: contract.abi,
-        events: contract.events
-      }));
       let evmRpcs = sources.rpcs.map(rpc => {
         let syncConfig = rpc.syncConfig;
         let ws = rpc.ws;
@@ -152,7 +107,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
           headers: headers
         };
       });
-      sources$1 = EvmChain.makeSources(chain, evmContracts, sources.hypersync, evmRpcs, lowercaseAddresses);
+      sources$1 = EvmChain.makeSources(chain, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses);
       break;
     case "FuelSourceConfig" :
       sources$1 = [HyperFuelSource.make({
@@ -164,41 +119,40 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
     case "SvmSourceConfig" :
       let rpc = sources.rpc;
       let hypersync = sources.hypersync;
-      if (hypersync !== undefined) {
-        let svmEventConfigs = chainConfig.contracts.flatMap(contract => contract.events);
-        sources$1 = [SvmHyperSyncSource.make({
+      sources$1 = hypersync !== undefined ? [SvmHyperSyncSource.make({
             chain: chain,
             endpointUrl: hypersync,
             apiToken: Env.envioApiToken,
-            eventConfigs: svmEventConfigs,
+            onEventRegistrations: onEventRegistrations,
             clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis
-          })];
-      } else {
-        sources$1 = rpc !== undefined ? [Svm.makeRPCSource(chain, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + chain.toString() + ` has no SVM data source`);
-      }
+          })] : (
+          rpc !== undefined ? [Svm.makeRPCSource(chain, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + chain.toString() + ` has no SVM data source`)
+        );
       break;
     case "CustomSources" :
       sources$1 = sources._0;
       break;
   }
-  return make(chainConfig, fetchState, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), logger);
+  let firstEventBlock$1 = fetchState.firstEventBlock;
+  let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
+  return make(chainConfig, fetchState, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), logger);
 }
 
-function makeFromConfig(chainConfig, config, registrations, knownHeight) {
+function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight) {
   let logger = Logging.createChild({
     chainId: chainConfig.id
   });
-  return makeInternal(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined);
+  return makeInternal(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrationsByChainId, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined);
 }
 
-function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrations, reducedPollingInterval) {
+function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval) {
   let chainId = chainConfig.id;
   let logger = Logging.createChild({
     chainId: chainId
   });
   Prometheus.ProgressEventsCount.set(resumedChainState.numEventsProcessed, chainId);
   let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
-  return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrations, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, reducedPollingInterval);
+  return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrationsByChainId, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, reducedPollingInterval);
 }
 
 function logger(cs) {
@@ -261,8 +215,8 @@ function getProgressPercentage(cs) {
   return FetchState.getProgressPercentage(cs.fetchState);
 }
 
-function getProgressPercentageAt(cs, blockNumber) {
-  return FetchState.getProgressPercentageAt(cs.fetchState, blockNumber);
+function chainDensity(cs) {
+  return cs.chainDensity;
 }
 
 function hasReadyItem(cs) {
@@ -279,11 +233,80 @@ function isReadyToEnterReorgThreshold(cs) {
 
 function startFetchingQueries(cs, queries) {
   FetchState.startFetchingQueries(cs.fetchState, queries);
-  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.estResponseSize);
+  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsTarget);
+}
+
+function isReady(cs) {
+  return cs.timestampCaughtUpToHeadOrEndblock !== undefined;
+}
+
+function fetchCeiling(cs) {
+  let fetchState = cs.fetchState;
+  let endBlock = fetchState.endBlock;
+  if (endBlock !== undefined) {
+    return Primitive_int.min(endBlock, fetchState.knownHeight);
+  } else {
+    return fetchState.knownHeight;
+  }
+}
+
+function targetBlock(cs, chainTargetItems) {
+  let fetchState = cs.fetchState;
+  let fetchCeiling$1 = fetchCeiling(cs);
+  let bufferBlockNumber = FetchState.bufferBlockNumber(fetchState);
+  let density = cs.chainDensity;
+  if (density !== undefined && density > 0) {
+    return Primitive_int.min(fetchCeiling$1, bufferBlockNumber + (Math.ceil(chainTargetItems / density) | 0) | 0);
+  } else {
+    return fetchCeiling$1;
+  }
+}
+
+function progressRange(cs) {
+  let fetchState = cs.fetchState;
+  let lower = Stdlib_Option.getOr(fetchState.firstEventBlock, fetchState.startBlock);
+  let endBlock = fetchState.endBlock;
+  let upper = endBlock !== undefined ? endBlock : fetchState.knownHeight;
+  return [
+    lower,
+    upper
+  ];
+}
+
+function progressAtBlock(cs, blockNumber) {
+  let match = progressRange(cs);
+  let upper = match[1];
+  let lower = match[0];
+  if (upper <= lower) {
+    return 1;
+  } else {
+    return Primitive_float.max(0, Primitive_float.min(1, (blockNumber - lower | 0) / (upper - lower | 0)));
+  }
 }
 
-function getNextQuery(cs) {
-  return FetchState.getNextQuery(cs.fetchState);
+function blockAtProgress(cs, progress) {
+  let match = progressRange(cs);
+  let lower = match[0];
+  return lower + (Math.ceil(progress * (match[1] - lower | 0)) | 0) | 0;
+}
+
+function getNextQuery(cs, chainTargetItems, maxTargetBlock) {
+  let chainTargetBlock = targetBlock(cs, chainTargetItems);
+  let chainTargetBlock$1 = maxTargetBlock !== undefined ? Primitive_int.min(chainTargetBlock, maxTargetBlock) : chainTargetBlock;
+  let density = cs.chainDensity;
+  let chainTargetItems$1;
+  if (density !== undefined) {
+    if (density > 0) {
+      let rangeCost = density * (chainTargetBlock$1 - FetchState.bufferBlockNumber(cs.fetchState) | 0);
+      let rangeCost$1 = chainTargetBlock$1 >= fetchCeiling(cs) && cs.timestampCaughtUpToHeadOrEndblock !== undefined ? rangeCost * 3 : rangeCost;
+      chainTargetItems$1 = Primitive_float.min(chainTargetItems, Math.ceil(rangeCost$1) + cs.pendingBudget);
+    } else {
+      chainTargetItems$1 = Primitive_float.min(chainTargetItems, 5000 + cs.pendingBudget);
+    }
+  } else {
+    chainTargetItems$1 = Primitive_float.min(chainTargetItems, 5000 + cs.pendingBudget);
+  }
+  return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1);
 }
 
 function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
@@ -314,10 +337,6 @@ function isActivelyIndexing(cs) {
   return FetchState.isActivelyIndexing(cs.fetchState);
 }
 
-function isReady(cs) {
-  return cs.timestampCaughtUpToHeadOrEndblock !== undefined;
-}
-
 function isFetchingAtHead(cs) {
   return FetchState.isFetchingAtHead(cs.fetchState);
 }
@@ -330,25 +349,144 @@ function isAtHeadWithoutEndBlock(cs) {
   }
 }
 
-function materializeBatchItems(cs, items) {
-  return TransactionStore.materializeItems(cs.transactionStore, items);
+function groupBatchItems(items, includeBlocks) {
+  let txBlockNumbers = [];
+  let transactionIndices = [];
+  let transactionMasks = [];
+  let payloadGroups = [];
+  let anyTransactionFieldSelected = {
+    contents: false
+  };
+  let blockBlockNumbers = [];
+  let blockMasks = [];
+  let blockItemGroups = [];
+  items.forEach(item => {
+    if (item.kind !== 0) {
+      return;
+    }
+    let blockNumber = item.blockNumber;
+    let match = item.payload.transaction;
+    if (match == null) {
+      let transactionIndex = item.transactionIndex;
+      let mask = item.onEventRegistration.eventConfig.transactionFieldMask;
+      if (mask !== 0) {
+        anyTransactionFieldSelected.contents = true;
+      }
+      let last = payloadGroups.length - 1 | 0;
+      if (last >= 0 && txBlockNumbers[last] === blockNumber && transactionIndices[last] === transactionIndex) {
+        payloadGroups[last].push(item.payload);
+        transactionMasks[last] = FieldMask.orMask(transactionMasks[last], mask);
+      } else {
+        txBlockNumbers.push(blockNumber);
+        transactionIndices.push(transactionIndex);
+        transactionMasks.push(mask);
+        payloadGroups.push([item.payload]);
+      }
+    }
+    if (!includeBlocks) {
+      return;
+    }
+    let match$1 = item.payload.block;
+    if (!(match$1 == null)) {
+      return;
+    }
+    let mask$1 = item.onEventRegistration.eventConfig.blockFieldMask;
+    let last$1 = blockItemGroups.length - 1 | 0;
+    if (last$1 >= 0 && blockBlockNumbers[last$1] === blockNumber) {
+      blockItemGroups[last$1].push(item);
+      blockMasks[last$1] = FieldMask.orMask(blockMasks[last$1], mask$1);
+    } else {
+      blockBlockNumbers.push(blockNumber);
+      blockMasks.push(mask$1);
+      blockItemGroups.push([item]);
+    }
+  });
+  return [
+    {
+      txBlockNumbers: txBlockNumbers,
+      transactionIndices: transactionIndices,
+      transactionMasks: transactionMasks,
+      payloadGroups: payloadGroups,
+      anyTransactionFieldSelected: anyTransactionFieldSelected.contents
+    },
+    {
+      blockBlockNumbers: blockBlockNumbers,
+      blockMasks: blockMasks,
+      blockItemGroups: blockItemGroups
+    }
+  ];
 }
 
-function materializePageItems(items, page) {
-  if (page !== undefined) {
-    return TransactionStore.materializeItems(Primitive_option.valFromOption(page), items);
-  } else {
-    return Promise.resolve();
+async function applyTransactionGroups(store, g) {
+  if (!Utils.$$Array.notEmpty(g.payloadGroups)) {
+    return;
+  }
+  if (g.anyTransactionFieldSelected) {
+    let txs = await store.materialize(g.txBlockNumbers, g.transactionIndices, g.transactionMasks);
+    g.payloadGroups.forEach((payloads, i) => {
+      let tx = txs[i];
+      payloads.forEach(payload => {
+        payload.transaction = tx;
+      });
+    });
+    return;
   }
+  g.payloadGroups.forEach(payloads => {
+    payloads.forEach(payload => {
+      payload.transaction = {};
+    });
+  });
 }
 
-function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, page) {
-  if (page !== undefined) {
-    cs.transactionStore.merge(Primitive_option.valFromOption(page));
+async function applyBlockGroups(store, g) {
+  if (!Utils.$$Array.notEmpty(g.blockItemGroups)) {
+    return;
+  }
+  let blocks = await store.materialize(g.blockBlockNumbers, g.blockMasks);
+  g.blockItemGroups.forEach((group, i) => {
+    let block = blocks[i];
+    group.forEach(ei => {
+      ei.payload.block = block;
+    });
+  });
+}
+
+function includeBlocksForEcosystem(ecosystem) {
+  switch (ecosystem) {
+    case "fuel" :
+      return false;
+    case "evm" :
+    case "svm" :
+      return true;
+  }
+}
+
+async function materializeBatchItems(cs, items, ecosystem) {
+  let match = groupBatchItems(items, includeBlocksForEcosystem(ecosystem));
+  await Promise.all([
+    applyTransactionGroups(cs.transactionStore, match[0]),
+    applyBlockGroups(cs.blockStore, match[1])
+  ]);
+}
+
+async function materializePageItems(items, transactionStore, blockStore, ecosystem) {
+  let match = groupBatchItems(items, includeBlocksForEcosystem(ecosystem));
+  await Promise.all([
+    transactionStore !== undefined ? applyTransactionGroups(Primitive_option.valFromOption(transactionStore), match[0]) : Promise.resolve(),
+    blockStore !== undefined ? applyBlockGroups(Primitive_option.valFromOption(blockStore), match[1]) : Promise.resolve()
+  ]);
+}
+
+function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, txPage, blockPage) {
+  if (txPage !== undefined) {
+    cs.transactionStore.merge(Primitive_option.valFromOption(txPage));
+  }
+  if (blockPage !== undefined) {
+    cs.blockStore.merge(Primitive_option.valFromOption(blockPage));
   }
   let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.indexingAddresses, newItemsWithDcs) : cs.fetchState;
   cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, cs.indexingAddresses, query, latestFetchedBlock, newItems), knownHeight);
-  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.estResponseSize);
+  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsTarget);
 }
 
 function registerReorgGuard(cs, blockHashes, knownHeight) {
@@ -387,7 +525,7 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
       blockLag: init.blockLag,
       buffer: init.buffer,
       maxOnBlockBufferSize: init.maxOnBlockBufferSize,
-      onBlockConfigs: init.onBlockConfigs,
+      onBlockRegistrations: init.onBlockRegistrations,
       knownHeight: init.knownHeight,
       firstEventBlock: init.firstEventBlock
     };
@@ -405,7 +543,7 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
     blockLag: init$1.blockLag,
     buffer: init$1.buffer,
     maxOnBlockBufferSize: init$1.maxOnBlockBufferSize,
-    onBlockConfigs: init$1.onBlockConfigs,
+    onBlockRegistrations: init$1.onBlockRegistrations,
     knownHeight: init$1.knownHeight,
     firstEventBlock: init$1.firstEventBlock
   };
@@ -465,7 +603,7 @@ function advanceAfterBatch(cs, batch, enteringReorgThreshold) {
   }
 }
 
-function applyBatchProgress(cs, batch) {
+function applyBatchProgress(cs, batch, blockTimestampName) {
   let chainId = cs.chainConfig.id;
   let chainAfterBatch = batch.progressedChainsById[chainId];
   if (chainAfterBatch === undefined) {
@@ -477,9 +615,9 @@ function applyBatchProgress(cs, batch) {
   if (cs.numEventsProcessed !== chainAfterBatch.totalEventsProcessed) {
     Prometheus.ProgressEventsCount.set(chainAfterBatch.totalEventsProcessed, chainId);
   }
-  let eventItem = Batch.findLastEventItem(batch, chainId);
-  if (eventItem !== undefined) {
-    let blockTimestampMs = eventItem.timestamp * 1000 | 0;
+  let blockTimestamp = Stdlib_Option.flatMap(Stdlib_Option.flatMap(Batch.findLastEventItem(batch, chainId), eventItem => Primitive_option.fromNullable(eventItem.payload.block)), block => block[blockTimestampName]);
+  if (blockTimestamp !== undefined) {
+    let blockTimestampMs = Primitive_option.valFromOption(blockTimestamp) * 1000 | 0;
     Prometheus.ProgressLatency.set((Date.now() | 0) - blockTimestampMs | 0, chainId);
   }
   let match = cs.fetchState.firstEventBlock;
@@ -498,15 +636,38 @@ function applyBatchProgress(cs, batch) {
         blockLag: init.blockLag,
         buffer: init.buffer,
         maxOnBlockBufferSize: init.maxOnBlockBufferSize,
-        onBlockConfigs: init.onBlockConfigs,
+        onBlockRegistrations: init.onBlockRegistrations,
         knownHeight: init.knownHeight,
         firstEventBlock: firstEventBlock
       };
     }
   }
+  let deltaBlocks = chainAfterBatch.progressBlockNumber - cs.committedProgressBlockNumber | 0;
+  if (deltaBlocks > 0) {
+    let deltaEvents = chainAfterBatch.totalEventsProcessed - cs.numEventsProcessed;
+    let match$1 = cs.chainDensity;
+    let match$2 = deltaEvents > 0;
+    let exit = 0;
+    if (match$1 !== undefined || match$2) {
+      exit = 1;
+    }
+    if (exit === 1) {
+      let batchDensity = deltaEvents / deltaBlocks;
+      let oldDensity = cs.chainDensity;
+      let tmp;
+      if (oldDensity !== undefined) {
+        let alpha = Primitive_float.min(1, deltaBlocks / 100);
+        tmp = oldDensity * (1 - alpha) + batchDensity * alpha;
+      } else {
+        tmp = batchDensity;
+      }
+      cs.chainDensity = tmp;
+    }
+  }
   cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber;
   cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed;
   cs.transactionStore.prune(chainAfterBatch.progressBlockNumber);
+  cs.blockStore.prune(chainAfterBatch.progressBlockNumber);
   cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated;
   let safeCheckpointTracking = cs.safeCheckpointTracking;
   if (safeCheckpointTracking !== undefined) {
@@ -529,6 +690,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
       cs.reorgDetection = ReorgDetection.rollbackToValidBlockNumber(cs.reorgDetection, rollbackTargetBlockNumber);
       cs.fetchState = FetchState.rollback(cs.fetchState, cs.indexingAddresses, rollbackTargetBlockNumber);
       cs.transactionStore.rollback(rollbackTargetBlockNumber);
+      cs.blockStore.rollback(rollbackTargetBlockNumber);
       cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
       return;
     } else {
@@ -551,6 +713,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
   }
   cs.fetchState = FetchState.rollback(cs.fetchState, cs.indexingAddresses, newProgressBlockNumber);
   cs.transactionStore.rollback(newProgressBlockNumber);
+  cs.blockStore.rollback(newProgressBlockNumber);
   cs.committedProgressBlockNumber = newProgressBlockNumber;
   cs.numEventsProcessed = newTotalEventsProcessed;
 }
@@ -576,9 +739,12 @@ export {
   bufferSize,
   bufferReadyCount,
   getProgressPercentage,
-  getProgressPercentageAt,
+  chainDensity,
   hasReadyItem,
   isReadyToEnterReorgThreshold,
+  targetBlock,
+  progressAtBlock,
+  blockAtProgress,
   getNextQuery,
   dispatch,
   toChainData,

package/src/ChainState.resi

@@ -17,13 +17,15 @@ let make: (
   ~timestampCaughtUpToHeadOrEndblock: option<Date.t>=?,
   ~isProgressAtHead: bool=?,
   ~transactionStore: TransactionStore.t=?,
+  ~chainDensity: option<float>=?,
+  ~blockStore: BlockStore.t=?,
   ~logger: Pino.t,
 ) => t
 
 let makeFromConfig: (
   Config.chain,
   ~config: Config.t,
-  ~registrations: HandlerRegister.registrations,
+  ~registrationsByChainId: HandlerRegister.registrationsByChainId,
   ~knownHeight: int,
 ) => t
 
@@ -34,7 +36,7 @@ let makeFromDbState: (
   ~isInReorgThreshold: bool,
   ~isRealtime: bool,
   ~config: Config.t,
-  ~registrations: HandlerRegister.registrations,
+  ~registrationsByChainId: HandlerRegister.registrationsByChainId,
   ~reducedPollingInterval: int=?,
 ) => t
 
@@ -57,12 +59,15 @@ let contractAddresses: (t, ~contractName: string) => array<Address.t>
 let bufferSize: t => int
 let bufferReadyCount: t => int
 let getProgressPercentage: t => float
-let getProgressPercentageAt: (t, ~blockNumber: int) => float
+let chainDensity: t => option<float>
 let hasReadyItem: t => bool
 let isReadyToEnterReorgThreshold: t => bool
 
 // Fetch control.
-let getNextQuery: t => FetchState.nextQuery
+let targetBlock: (t, ~chainTargetItems: float) => int
+let progressAtBlock: (t, ~blockNumber: int) => float
+let blockAtProgress: (t, ~progress: float) => int
+let getNextQuery: (t, ~chainTargetItems: float, ~maxTargetBlock: int=?) => FetchState.nextQuery
 let dispatch: (
   t,
   ~executeQuery: FetchState.query => promise<unit>,
@@ -110,11 +115,18 @@ let handleQueryResult: (
   ~latestFetchedBlock: FetchState.blockNumberAndTimestamp,
   ~knownHeight: int,
   ~transactionStore: option<TransactionStore.t>,
+  ~blockStore: option<BlockStore.t>,
 ) => unit
-let materializeBatchItems: (t, ~items: array<Internal.item>) => promise<unit>
+let materializeBatchItems: (
+  t,
+  ~items: array<Internal.item>,
+  ~ecosystem: Ecosystem.name,
+) => promise<unit>
 let materializePageItems: (
   ~items: array<Internal.item>,
-  ~page: option<TransactionStore.t>,
+  ~transactionStore: option<TransactionStore.t>,
+  ~blockStore: option<BlockStore.t>,
+  ~ecosystem: Ecosystem.name,
 ) => promise<unit>
 let registerReorgGuard: (
   t,
@@ -126,7 +138,7 @@ let updateKnownHeight: (t, ~knownHeight: int) => unit
 let setEndBlockToFirstEvent: (t, ~blockNumber: int) => unit
 let enterReorgThreshold: t => unit
 let advanceAfterBatch: (t, ~batch: Batch.t, ~enteringReorgThreshold: bool) => unit
-let applyBatchProgress: (t, ~batch: Batch.t) => unit
+let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
 let markReady: t => unit
 let rollback: (
   t,