envio 3.0.0-alpha.2 → 3.0.0-alpha.3

package/src/ChainFetcher.res.mjs

@@ -0,0 +1,281 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Env from "./Env.res.mjs";
+import * as Caml from "rescript/lib/es6/caml.js";
+import * as Utils from "./Utils.res.mjs";
+import * as Js_exn from "rescript/lib/es6/js_exn.js";
+import * as Logging from "./Logging.res.mjs";
+import * as $$Promise from "./bindings/Promise.res.mjs";
+import * as ChainMap from "./ChainMap.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as FetchState from "./FetchState.res.mjs";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as Belt_Result from "rescript/lib/es6/belt_Result.js";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
+import * as EventRouter from "./sources/EventRouter.res.mjs";
+import * as UserContext from "./UserContext.res.mjs";
+import * as ErrorHandling from "./ErrorHandling.res.mjs";
+import * as SourceManager from "./sources/SourceManager.res.mjs";
+import * as ReorgDetection from "./ReorgDetection.res.mjs";
+import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
+import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
+
+function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlockNumber, progressBlockNumber, config, registrations, targetBufferSize, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, numBatchesFetched, isInReorgThreshold, reorgCheckpoints, maxReorgDepth) {
+  var eventRouter = EventRouter.empty();
+  var contracts = [];
+  var eventConfigs = [];
+  var notRegisteredEvents = [];
+  Belt_Array.forEach(chainConfig.contracts, (function (contract) {
+          var contractName = contract.name;
+          Belt_Array.forEach(contract.events, (function (eventConfig) {
+                  var hasContractRegister = Belt_Option.isSome(eventConfig.contractRegister);
+                  EventRouter.addOrThrow(eventRouter, eventConfig.id, undefined, contractName, eventConfig.isWildcard, eventConfig.name, ChainMap.Chain.makeUnsafe(chainConfig.id));
+                  var shouldBeIncluded;
+                  if (config.enableRawEvents) {
+                    shouldBeIncluded = true;
+                  } else {
+                    var isRegistered = hasContractRegister || Belt_Option.isSome(eventConfig.handler);
+                    if (!isRegistered) {
+                      notRegisteredEvents.push(eventConfig);
+                    }
+                    shouldBeIncluded = isRegistered;
+                  }
+                  if (shouldBeIncluded) {
+                    eventConfigs.push(eventConfig);
+                    return ;
+                  }
+                  
+                }));
+          var startBlock = contract.startBlock;
+          if (startBlock !== undefined && startBlock < chainConfig.startBlock) {
+            Js_exn.raiseError("The start block for contract \"" + contractName + "\" is less than the chain start block. This is not supported yet.");
+          }
+          Belt_Array.forEach(contract.addresses, (function (address) {
+                  var startBlock = contract.startBlock;
+                  contracts.push({
+                        address: address,
+                        contractName: contract.name,
+                        startBlock: startBlock !== undefined ? startBlock : chainConfig.startBlock,
+                        registrationBlock: undefined
+                      });
+                }));
+        }));
+  Belt_Array.forEach(dynamicContracts, (function (dc) {
+          contracts.push(dc);
+        }));
+  if (Utils.$$Array.notEmpty(notRegisteredEvents)) {
+    Logging.childInfo(logger, "The event" + (
+          notRegisteredEvents.length > 1 ? "s" : ""
+        ) + " " + Belt_Array.map(notRegisteredEvents, (function (eventConfig) {
+                  return eventConfig.contractName + "." + eventConfig.name;
+                })).join(", ") + " don't have an event handler and skipped for indexing.");
+  }
+  var onBlockConfigs = registrations.onBlockByChainId[String(chainConfig.id)];
+  if (onBlockConfigs !== undefined) {
+    Belt_Array.forEach(onBlockConfigs, (function (onBlockConfig) {
+            if (Belt_Option.getWithDefault(onBlockConfig.startBlock, startBlock) < startBlock) {
+              Js_exn.raiseError("The start block for onBlock handler \"" + onBlockConfig.name + "\" is less than the chain start block (" + String(startBlock) + "). This is not supported yet.");
+            }
+            if (endBlock !== undefined && Belt_Option.getWithDefault(onBlockConfig.endBlock, endBlock) > endBlock) {
+              return Js_exn.raiseError("The end block for onBlock handler \"" + onBlockConfig.name + "\" is greater than the chain end block (" + String(endBlock) + "). This is not supported yet.");
+            }
+            
+          }));
+  }
+  var fetchState = FetchState.make(startBlock, endBlock, eventConfigs, contracts, config.maxAddrInPartition, chainConfig.id, targetBufferSize, 0, progressBlockNumber, onBlockConfigs, Caml.int_max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, Belt_Option.getWithDefault(Env.indexingBlockLag, 0)));
+  var chainReorgCheckpoints = Belt_Array.keepMapU(reorgCheckpoints, (function (reorgCheckpoint) {
+          if (reorgCheckpoint.chain_id === chainConfig.id) {
+            return reorgCheckpoint;
+          }
+          
+        }));
+  return {
+          logger: logger,
+          fetchState: fetchState,
+          sourceManager: SourceManager.make(chainConfig.sources, Env.maxPartitionConcurrency, undefined, undefined, undefined),
+          chainConfig: chainConfig,
+          isProgressAtHead: false,
+          timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
+          committedProgressBlockNumber: progressBlockNumber,
+          firstEventBlockNumber: firstEventBlockNumber,
+          numEventsProcessed: numEventsProcessed,
+          numBatchesFetched: numBatchesFetched,
+          reorgDetection: ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg),
+          safeCheckpointTracking: SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)
+        };
+}
+
+function makeFromConfig(chainConfig, config, registrations, targetBufferSize) {
+  var logger = Logging.createChild({
+        chainId: chainConfig.id
+      });
+  return make(chainConfig, [], chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, targetBufferSize, logger, undefined, 0, 0, false, [], chainConfig.maxReorgDepth);
+}
+
+async function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, config, registrations, targetBufferSize) {
+  var chainId = chainConfig.id;
+  var logger = Logging.createChild({
+        chainId: chainId
+      });
+  Prometheus.ProgressEventsCount.set(resumedChainState.numEventsProcessed, chainId);
+  var progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
+  return make(chainConfig, resumedChainState.dynamicContracts, resumedChainState.startBlock, resumedChainState.endBlock, resumedChainState.firstEventBlockNumber, progressBlockNumber, config, registrations, targetBufferSize, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, 0, isInReorgThreshold, reorgCheckpoints, resumedChainState.maxReorgDepth);
+}
+
+function getContractStartBlock(config, chain, contractName) {
+  var chainConfig = ChainMap.get(config.chainMap, chain);
+  return Belt_Option.flatMap(Caml_option.undefined_to_opt(chainConfig.contracts.find(function (contract) {
+                      return contract.name === contractName;
+                    })), (function (contract) {
+                return contract.startBlock;
+              }));
+}
+
+async function runContractRegistersOrThrow(itemsWithContractRegister, chain, config) {
+  var itemsWithDcs = [];
+  var onRegister = function (item, contractAddress, contractName) {
+    var blockNumber = item.blockNumber;
+    var configuredStartBlock = getContractStartBlock(config, chain, contractName);
+    var contractStartBlock = configuredStartBlock !== undefined ? configuredStartBlock : blockNumber;
+    var dc_registrationBlock = blockNumber;
+    var dc = {
+      address: contractAddress,
+      contractName: contractName,
+      startBlock: contractStartBlock,
+      registrationBlock: dc_registrationBlock
+    };
+    var dcs = item.dcs;
+    if (dcs !== undefined) {
+      dcs.push(dc);
+    } else {
+      item.dcs = [dc];
+      itemsWithDcs.push(item);
+    }
+  };
+  var promises = [];
+  for(var idx = 0 ,idx_finish = itemsWithContractRegister.length; idx < idx_finish; ++idx){
+    var item = itemsWithContractRegister[idx];
+    var match = item.eventConfig;
+    var contractRegister = match.contractRegister;
+    var contractRegister$1 = contractRegister !== undefined ? contractRegister : Js_exn.raiseError("Contract register is not set for event " + match.name);
+    var errorMessage = "Event contractRegister failed, please fix the error to keep the indexer running smoothly";
+    try {
+      var params = {
+        item: item,
+        onRegister: onRegister,
+        config: config,
+        isResolved: false
+      };
+      var result = contractRegister$1(UserContext.getContractRegisterArgs(params));
+      if ($$Promise.isCatchable(result)) {
+        promises.push($$Promise.$$catch(result.then((function(params){
+                    return function (r) {
+                      params.isResolved = true;
+                      
+                    }
+                    }(params))), (function(item,params){
+                return function (exn) {
+                  params.isResolved = true;
+                  return ErrorHandling.mkLogAndRaise(Logging.getItemLogger(item), errorMessage, exn);
+                }
+                }(item,params))));
+      } else {
+        params.isResolved = true;
+      }
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      ErrorHandling.mkLogAndRaise(Logging.getItemLogger(item), errorMessage, exn);
+    }
+  }
+  if (Utils.$$Array.notEmpty(promises)) {
+    await Promise.all(promises);
+  }
+  return itemsWithDcs;
+}
+
+function handleQueryResult(chainFetcher, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight) {
+  var fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(chainFetcher.fetchState, newItemsWithDcs) : chainFetcher.fetchState;
+  return Belt_Result.map(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), (function (fs) {
+                return {
+                        logger: chainFetcher.logger,
+                        fetchState: FetchState.updateKnownHeight(fs, knownHeight),
+                        sourceManager: chainFetcher.sourceManager,
+                        chainConfig: chainFetcher.chainConfig,
+                        isProgressAtHead: chainFetcher.isProgressAtHead,
+                        timestampCaughtUpToHeadOrEndblock: chainFetcher.timestampCaughtUpToHeadOrEndblock,
+                        committedProgressBlockNumber: chainFetcher.committedProgressBlockNumber,
+                        firstEventBlockNumber: chainFetcher.firstEventBlockNumber,
+                        numEventsProcessed: chainFetcher.numEventsProcessed,
+                        numBatchesFetched: chainFetcher.numBatchesFetched,
+                        reorgDetection: chainFetcher.reorgDetection,
+                        safeCheckpointTracking: chainFetcher.safeCheckpointTracking
+                      };
+              }));
+}
+
+function hasProcessedToEndblock(self) {
+  var endBlock = self.fetchState.endBlock;
+  if (endBlock !== undefined) {
+    return self.committedProgressBlockNumber >= endBlock;
+  } else {
+    return false;
+  }
+}
+
+function hasNoMoreEventsToProcess(self) {
+  return FetchState.bufferSize(self.fetchState) === 0;
+}
+
+function getHighestBlockBelowThreshold(cf) {
+  var highestBlockBelowThreshold = cf.fetchState.knownHeight - cf.chainConfig.maxReorgDepth | 0;
+  if (highestBlockBelowThreshold < 0) {
+    return 0;
+  } else {
+    return highestBlockBelowThreshold;
+  }
+}
+
+async function getLastKnownValidBlock(chainFetcher, reorgBlockNumber, getBlockHashesOpt) {
+  var getBlockHashes = getBlockHashesOpt !== undefined ? getBlockHashesOpt : SourceManager.getActiveSource(chainFetcher.sourceManager).getBlockHashes;
+  var scannedBlockNumbers = ReorgDetection.getThresholdBlockNumbersBelowBlock(chainFetcher.reorgDetection, reorgBlockNumber, chainFetcher.fetchState.knownHeight);
+  var getBlockHashes$1 = function (blockNumbers) {
+    return getBlockHashes(blockNumbers, chainFetcher.logger).then(function (res) {
+                if (res.TAG === "Ok") {
+                  return res._0;
+                } else {
+                  return ErrorHandling.mkLogAndRaise(undefined, "Failed to fetch blockHashes for given blockNumbers during rollback", res._0);
+                }
+              });
+  };
+  if (scannedBlockNumbers.length === 0) {
+    return getHighestBlockBelowThreshold(chainFetcher);
+  }
+  var blockNumbersAndHashes = await getBlockHashes$1(scannedBlockNumbers);
+  var blockNumber = ReorgDetection.getLatestValidScannedBlock(chainFetcher.reorgDetection, blockNumbersAndHashes);
+  if (blockNumber !== undefined) {
+    return blockNumber;
+  } else {
+    return getHighestBlockBelowThreshold(chainFetcher);
+  }
+}
+
+function isActivelyIndexing(chainFetcher) {
+  return FetchState.isActivelyIndexing(chainFetcher.fetchState);
+}
+
+export {
+  make ,
+  makeFromConfig ,
+  makeFromDbState ,
+  getContractStartBlock ,
+  runContractRegistersOrThrow ,
+  handleQueryResult ,
+  hasProcessedToEndblock ,
+  hasNoMoreEventsToProcess ,
+  getHighestBlockBelowThreshold ,
+  getLastKnownValidBlock ,
+  isActivelyIndexing ,
+}
+/* Env Not a pure module */

package/src/ChainManager.res

@@ -0,0 +1,179 @@
+open Belt
+
+type t = {
+  committedCheckpointId: float,
+  chainFetchers: ChainMap.t<ChainFetcher.t>,
+  multichain: Config.multichain,
+  isInReorgThreshold: bool,
+}
+
+let calculateTargetBufferSize = (~activeChainsCount, ~config: Config.t) => {
+  let targetBatchesInBuffer = 3
+  switch Env.targetBufferSize {
+  | Some(size) => size
+  | None =>
+    config.batchSize * (activeChainsCount > targetBatchesInBuffer ? 1 : targetBatchesInBuffer)
+  }
+}
+
+let makeFromConfig = (~config: Config.t, ~registrations): t => {
+  let targetBufferSize = calculateTargetBufferSize(
+    ~activeChainsCount=config.chainMap->ChainMap.size,
+    ~config,
+  )
+  let chainFetchers =
+    config.chainMap->ChainMap.map(
+      ChainFetcher.makeFromConfig(_, ~config, ~registrations, ~targetBufferSize),
+    )
+  {
+    committedCheckpointId: 0.,
+    chainFetchers,
+    multichain: config.multichain,
+    isInReorgThreshold: false,
+  }
+}
+
+let makeFromDbState = async (
+  ~initialState: Persistence.initialState,
+  ~config: Config.t,
+  ~registrations,
+  ~persistence: Persistence.t,
+): t => {
+  let isInReorgThreshold = if initialState.cleanRun {
+    false
+  } else {
+    // TODO: Move to Persistence.initialState
+    // Since now it's possible not to have rows in the history table
+    // even after the indexer started saving history (entered reorg threshold),
+    // This rows check might incorrectly return false for recovering the isInReorgThreshold option.
+    // But this is not a problem. There's no history anyways, and the indexer will be able to
+    // correctly calculate isInReorgThreshold as it starts.
+    let hasStartedSavingHistory = await persistence.storage.hasEntityHistoryRows()
+
+    //If we have started saving history, continue to save history
+    //as regardless of whether we are still in a reorg threshold
+    hasStartedSavingHistory
+  }
+
+  let targetBufferSize = calculateTargetBufferSize(
+    ~activeChainsCount=initialState.chains->Array.length,
+    ~config,
+  )
+  Prometheus.ProcessingMaxBatchSize.set(~maxBatchSize=config.batchSize)
+  Prometheus.IndexingTargetBufferSize.set(~targetBufferSize)
+  Prometheus.ReorgThreshold.set(~isInReorgThreshold)
+  initialState.cache->Utils.Dict.forEach(({effectName, count}) => {
+    Prometheus.EffectCacheCount.set(~count, ~effectName)
+  })
+
+  let chainFetchersArr =
+    await initialState.chains
+    ->Array.map(async (resumedChainState: Persistence.initialChainState) => {
+      let chain = Config.getChain(config, ~chainId=resumedChainState.id)
+      let chainConfig = config.chainMap->ChainMap.get(chain)
+
+      (
+        chain,
+        await chainConfig->ChainFetcher.makeFromDbState(
+          ~resumedChainState,
+          ~reorgCheckpoints=initialState.reorgCheckpoints,
+          ~isInReorgThreshold,
+          ~targetBufferSize,
+          ~config,
+          ~registrations,
+        ),
+      )
+    })
+    ->Promise.all
+
+  let chainFetchers = ChainMap.fromArrayUnsafe(chainFetchersArr)
+
+  {
+    committedCheckpointId: initialState.checkpointId,
+    multichain: config.multichain,
+    chainFetchers,
+    isInReorgThreshold,
+  }
+}
+
+let getChainFetcher = (chainManager: t, ~chain: ChainMap.Chain.t): ChainFetcher.t => {
+  chainManager.chainFetchers->ChainMap.get(chain)
+}
+
+let setChainFetcher = (chainManager: t, chainFetcher: ChainFetcher.t) => {
+  {
+    ...chainManager,
+    chainFetchers: chainManager.chainFetchers->ChainMap.set(
+      ChainMap.Chain.makeUnsafe(~chainId=chainFetcher.chainConfig.id),
+      chainFetcher,
+    ),
+  }
+}
+
+let nextItemIsNone = (chainManager: t): bool => {
+  !Batch.hasMultichainReadyItem(
+    chainManager.chainFetchers->ChainMap.map(cf => {
+      cf.fetchState
+    }),
+    ~multichain=chainManager.multichain,
+  )
+}
+
+let createBatch = (chainManager: t, ~batchSizeTarget: int, ~isRollback: bool): Batch.t => {
+  Batch.make(
+    ~checkpointIdBeforeBatch=chainManager.committedCheckpointId +. (
+      // Since for rollback we have a diff checkpoint id.
+      // This is needed to currectly overwrite old state
+      // in an append-only ClickHouse insert.
+      isRollback ? 1. : 0.
+    ),
+    ~chainsBeforeBatch=chainManager.chainFetchers->ChainMap.map((cf): Batch.chainBeforeBatch => {
+      fetchState: cf.fetchState,
+      progressBlockNumber: cf.committedProgressBlockNumber,
+      totalEventsProcessed: cf.numEventsProcessed,
+      sourceBlockNumber: cf.fetchState.knownHeight,
+      reorgDetection: cf.reorgDetection,
+    }),
+    ~multichain=chainManager.multichain,
+    ~batchSizeTarget,
+  )
+}
+
+let isProgressAtHead = chainManager =>
+  chainManager.chainFetchers
+  ->ChainMap.values
+  ->Js.Array2.every(cf => cf.isProgressAtHead)
+
+let isActivelyIndexing = chainManager =>
+  chainManager.chainFetchers
+  ->ChainMap.values
+  ->Js.Array2.every(ChainFetcher.isActivelyIndexing)
+
+let getSafeCheckpointId = (chainManager: t) => {
+  let chainFetchers = chainManager.chainFetchers->ChainMap.values
+
+  let infinity = (%raw(`Infinity`): float)
+  let result = ref(infinity)
+
+  for idx in 0 to chainFetchers->Array.length - 1 {
+    let chainFetcher = chainFetchers->Array.getUnsafe(idx)
+    switch chainFetcher.safeCheckpointTracking {
+    | None => () // Skip chains with maxReorgDepth = 0
+    | Some(safeCheckpointTracking) => {
+        let safeCheckpointId =
+          safeCheckpointTracking->SafeCheckpointTracking.getSafeCheckpointId(
+            ~sourceBlockNumber=chainFetcher.fetchState.knownHeight,
+          )
+        if safeCheckpointId < result.contents {
+          result := safeCheckpointId
+        }
+      }
+    }
+  }
+
+  if result.contents === infinity || result.contents === 0. {
+    None // No safe checkpoint found
+  } else {
+    Some(result.contents)
+  }
+}

package/src/ChainManager.res.mjs

@@ -0,0 +1,139 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Env from "./Env.res.mjs";
+import * as Batch from "./Batch.res.mjs";
+import * as Utils from "./Utils.res.mjs";
+import * as Config from "./Config.res.mjs";
+import * as ChainMap from "./ChainMap.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as ChainFetcher from "./ChainFetcher.res.mjs";
+import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
+
+function calculateTargetBufferSize(activeChainsCount, config) {
+  if (Env.targetBufferSize !== undefined) {
+    return Env.targetBufferSize;
+  } else {
+    return Math.imul(config.batchSize, activeChainsCount > 3 ? 1 : 3);
+  }
+}
+
+function makeFromConfig(config, registrations) {
+  var targetBufferSize = calculateTargetBufferSize(ChainMap.size(config.chainMap), config);
+  var chainFetchers = ChainMap.map(config.chainMap, (function (__x) {
+          return ChainFetcher.makeFromConfig(__x, config, registrations, targetBufferSize);
+        }));
+  return {
+          committedCheckpointId: 0,
+          chainFetchers: chainFetchers,
+          multichain: config.multichain,
+          isInReorgThreshold: false
+        };
+}
+
+async function makeFromDbState(initialState, config, registrations, persistence) {
+  var isInReorgThreshold = initialState.cleanRun ? false : await persistence.storage.hasEntityHistoryRows();
+  var targetBufferSize = calculateTargetBufferSize(initialState.chains.length, config);
+  Prometheus.ProcessingMaxBatchSize.set(config.batchSize);
+  Prometheus.IndexingTargetBufferSize.set(targetBufferSize);
+  Prometheus.ReorgThreshold.set(isInReorgThreshold);
+  Utils.Dict.forEach(initialState.cache, (function (param) {
+          var count = param.count;
+          Prometheus.EffectCacheCount.set(count, param.effectName);
+        }));
+  var chainFetchersArr = await Promise.all(Belt_Array.map(initialState.chains, (async function (resumedChainState) {
+              var chain = Config.getChain(config, resumedChainState.id);
+              var chainConfig = ChainMap.get(config.chainMap, chain);
+              return [
+                      chain,
+                      await ChainFetcher.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, config, registrations, targetBufferSize)
+                    ];
+            })));
+  var chainFetchers = ChainMap.fromArrayUnsafe(chainFetchersArr);
+  return {
+          committedCheckpointId: initialState.checkpointId,
+          chainFetchers: chainFetchers,
+          multichain: config.multichain,
+          isInReorgThreshold: isInReorgThreshold
+        };
+}
+
+function getChainFetcher(chainManager, chain) {
+  return ChainMap.get(chainManager.chainFetchers, chain);
+}
+
+function setChainFetcher(chainManager, chainFetcher) {
+  return {
+          committedCheckpointId: chainManager.committedCheckpointId,
+          chainFetchers: ChainMap.set(chainManager.chainFetchers, ChainMap.Chain.makeUnsafe(chainFetcher.chainConfig.id), chainFetcher),
+          multichain: chainManager.multichain,
+          isInReorgThreshold: chainManager.isInReorgThreshold
+        };
+}
+
+function nextItemIsNone(chainManager) {
+  return !Batch.hasMultichainReadyItem(ChainMap.map(chainManager.chainFetchers, (function (cf) {
+                    return cf.fetchState;
+                  })), chainManager.multichain);
+}
+
+function createBatch(chainManager, batchSizeTarget, isRollback) {
+  return Batch.make(chainManager.committedCheckpointId + (
+              isRollback ? 1 : 0
+            ), ChainMap.map(chainManager.chainFetchers, (function (cf) {
+                    return {
+                            fetchState: cf.fetchState,
+                            reorgDetection: cf.reorgDetection,
+                            progressBlockNumber: cf.committedProgressBlockNumber,
+                            sourceBlockNumber: cf.fetchState.knownHeight,
+                            totalEventsProcessed: cf.numEventsProcessed
+                          };
+                  })), chainManager.multichain, batchSizeTarget);
+}
+
+function isProgressAtHead(chainManager) {
+  return ChainMap.values(chainManager.chainFetchers).every(function (cf) {
+              return cf.isProgressAtHead;
+            });
+}
+
+function isActivelyIndexing(chainManager) {
+  return ChainMap.values(chainManager.chainFetchers).every(ChainFetcher.isActivelyIndexing);
+}
+
+function getSafeCheckpointId(chainManager) {
+  var chainFetchers = ChainMap.values(chainManager.chainFetchers);
+  var infinity = Infinity;
+  var result = infinity;
+  for(var idx = 0 ,idx_finish = chainFetchers.length; idx < idx_finish; ++idx){
+    var chainFetcher = chainFetchers[idx];
+    var safeCheckpointTracking = chainFetcher.safeCheckpointTracking;
+    if (safeCheckpointTracking !== undefined) {
+      var safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, chainFetcher.fetchState.knownHeight);
+      if (safeCheckpointId < result) {
+        result = safeCheckpointId;
+      }
+      
+    }
+    
+  }
+  if (result === infinity || result === 0) {
+    return ;
+  } else {
+    return result;
+  }
+}
+
+export {
+  calculateTargetBufferSize ,
+  makeFromConfig ,
+  makeFromDbState ,
+  getChainFetcher ,
+  setChainFetcher ,
+  nextItemIsNone ,
+  createBatch ,
+  isProgressAtHead ,
+  isActivelyIndexing ,
+  getSafeCheckpointId ,
+}
+/* Env Not a pure module */

package/src/Config.res

@@ -51,6 +51,7 @@ type t = {
   batchSize: int,
   lowercaseAddresses: bool,
   addContractNameToContractNameMapping: dict<string>,
+  userEntitiesByName: dict<Internal.entityConfig>,
 }
 
 let make = (
@@ -64,6 +65,7 @@ let make = (
   ~multichain=Unordered,
   ~shouldUseHypersyncClientDecoder=true,
   ~maxAddrInPartition=5000,
+  ~userEntities: array<Internal.entityConfig>=[],
 ) => {
   // Validate that lowercase addresses is not used with viem decoder
   if lowercaseAddresses && !shouldUseHypersyncClientDecoder {
@@ -88,7 +90,18 @@ let make = (
     })
   })
 
-  let ecosystem = Ecosystem.fromName(ecosystem)
+  let ecosystem = switch ecosystem {
+  | Ecosystem.Evm => Evm.ecosystem
+  | Ecosystem.Fuel => Fuel.ecosystem
+  | Ecosystem.Solana => Solana.ecosystem
+  }
+
+  let userEntitiesByName =
+    userEntities
+    ->Js.Array2.map(entityConfig => {
+      (entityConfig.name, entityConfig)
+    })
+    ->Js.Dict.fromArray
 
   {
     shouldRollbackOnReorg,
@@ -102,6 +115,7 @@ let make = (
     batchSize,
     lowercaseAddresses,
     addContractNameToContractNameMapping,
+    userEntitiesByName,
   }
 }