envio 3.2.1 → 3.3.0-alpha.0

package/src/ChainFetcher.res

@@ -1,519 +0,0 @@
-//A filter should return true if the event should be kept and isValid should return
-//false when the filter should be removed/cleaned up
-type processingFilter = {
-  filter: Internal.item => bool,
-  isValid: (~fetchState: FetchState.t) => bool,
-}
-
-type t = {
-  logger: Pino.t,
-  fetchState: FetchState.t,
-  sourceManager: SourceManager.t,
-  chainConfig: Config.chain,
-  isProgressAtHead: bool,
-  timestampCaughtUpToHeadOrEndblock: option<Date.t>,
-  committedProgressBlockNumber: int,
-  numEventsProcessed: float,
-  reorgDetection: ReorgDetection.t,
-  safeCheckpointTracking: option<SafeCheckpointTracking.t>,
-}
-
-//CONSTRUCTION
-let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddress> => {
-  let addresses = []
-  chainConfig.contracts->Array.forEach(contract => {
-    contract.addresses->Array.forEach(address => {
-      addresses->Array.push({
-        Internal.address,
-        contractName: contract.name,
-        registrationBlock: -1,
-      })
-    })
-  })
-  addresses
-}
-
-let make = (
-  ~chainConfig: Config.chain,
-  ~indexingAddresses: array<Internal.indexingAddress>,
-  ~startBlock,
-  ~endBlock,
-  ~firstEventBlock=None,
-  ~progressBlockNumber,
-  ~config: Config.t,
-  ~registrations: HandlerRegister.registrations,
-  ~targetBufferSize,
-  ~logger,
-  ~timestampCaughtUpToHeadOrEndblock,
-  ~numEventsProcessed,
-  ~isInReorgThreshold,
-  ~isRealtime,
-  ~reorgCheckpoints: array<Internal.reorgCheckpoint>,
-  ~maxReorgDepth,
-  ~knownHeight=0,
-  ~reducedPollingInterval=?,
-): t => {
-  // We don't need the router itself, but only validation logic,
-  // since now event router is created for selection of events
-  // and validation doesn't work correctly in routers.
-  // Ideally to split it into two different parts.
-  let eventRouter = EventRouter.empty()
-
-  // Aggregate events we want to fetch
-  let eventConfigs: array<Internal.eventConfig> = []
-
-  let notRegisteredEvents = []
-
-  chainConfig.contracts->Array.forEach(contract => {
-    let contractName = contract.name
-
-    contract.events->Array.forEach(eventConfig => {
-      let {isWildcard} = eventConfig
-      let hasContractRegister = eventConfig.contractRegister->Option.isSome
-
-      // Should validate the events
-      eventRouter->EventRouter.addOrThrow(
-        eventConfig.id,
-        (),
-        ~contractName,
-        ~chain=ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id),
-        ~eventName=eventConfig.name,
-        ~isWildcard,
-      )
-
-      // Filter out non-preRegistration events on preRegistration phase
-      // so we don't care about it in fetch state and workers anymore
-      let shouldBeIncluded = if config.enableRawEvents {
-        true
-      } else {
-        let isRegistered = hasContractRegister || eventConfig.handler->Option.isSome
-        if !isRegistered {
-          notRegisteredEvents->Array.push(eventConfig)
-        }
-        isRegistered
-      }
-
-      // Check if event has Static([]) filters (from a dynamic where
-      // callback returning `false` / SkipAll for this chain).
-      // If so, skip it entirely - it should never be fetched
-      let shouldSkip = try {
-        let getEventFiltersOrThrow = (
-          eventConfig->(Utils.magic: Internal.eventConfig => Internal.evmEventConfig)
-        ).getEventFiltersOrThrow
-
-        // Check for non-evm chains
-        if (
-          getEventFiltersOrThrow->(Utils.magic: (ChainMap.Chain.t => Internal.eventFilters) => bool)
-        ) {
-          switch getEventFiltersOrThrow(ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)) {
-          | Static([]) => true
-          | _ => false
-          }
-        } else {
-          false
-        }
-      } catch {
-      // Can throw when filter is invalid
-      // Don't skip an event in this case. Let it throw in a better place - source code
-      | _ => false
-      }
-
-      if shouldBeIncluded && !shouldSkip {
-        eventConfigs->Array.push(eventConfig)
-      }
-    })
-
-    switch contract.startBlock {
-    | Some(startBlock) if startBlock < chainConfig.startBlock =>
-      JsError.throwWithMessage(
-        `The start block for contract "${contractName}" is less than the chain start block. This is not supported yet.`,
-      )
-    | _ => ()
-    }
-  })
-
-  if notRegisteredEvents->Utils.Array.notEmpty {
-    logger->Logging.childInfo(
-      `The event${if notRegisteredEvents->Array.length > 1 {
-          "s"
-        } else {
-          ""
-        }} ${notRegisteredEvents
-        ->Array.map(eventConfig => `${eventConfig.contractName}.${eventConfig.name}`)
-        ->Array.joinUnsafe(", ")} don't have an event handler and skipped for indexing.`,
-    )
-  }
-
-  let onBlockConfigs =
-    registrations.onBlockByChainId->Utils.Dict.dangerouslyGetNonOption(chainConfig.id->Int.toString)
-  switch onBlockConfigs {
-  | Some(onBlockConfigs) =>
-    // TODO: Move it to the HandlerRegister module
-    // so the error is thrown with better stack trace
-    onBlockConfigs->Array.forEach(onBlockConfig => {
-      if onBlockConfig.startBlock->Option.getOr(startBlock) < startBlock {
-        JsError.throwWithMessage(
-          `The start block for onBlock handler "${onBlockConfig.name}" is less than the chain start block (${startBlock->Int.toString}). This is not supported yet.`,
-        )
-      }
-      switch endBlock {
-      | Some(chainEndBlock) =>
-        if onBlockConfig.endBlock->Option.getOr(chainEndBlock) > chainEndBlock {
-          JsError.throwWithMessage(
-            `The end block for onBlock handler "${onBlockConfig.name}" is greater than the chain end block (${chainEndBlock->Int.toString}). This is not supported yet.`,
-          )
-        }
-      | None => ()
-      }
-    })
-  | None => ()
-  }
-
-  let fetchState = FetchState.make(
-    ~maxAddrInPartition=config.maxAddrInPartition,
-    ~addresses=indexingAddresses,
-    ~progressBlockNumber,
-    ~startBlock,
-    ~endBlock,
-    ~eventConfigs,
-    ~targetBufferSize,
-    ~knownHeight,
-    ~chainId=chainConfig.id,
-    // FIXME: Shouldn't set with full history
-    ~blockLag=Pervasives.max(
-      !config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth,
-      chainConfig.blockLag,
-    ),
-    ~onBlockConfigs?,
-    ~firstEventBlock,
-  )
-
-  let chainReorgCheckpoints = reorgCheckpoints->Array.filterMap(reorgCheckpoint => {
-    if reorgCheckpoint.chainId === chainConfig.id {
-      Some(reorgCheckpoint)
-    } else {
-      None
-    }
-  })
-
-  // Create sources lazily here - this is where API token validation happens
-  let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
-  let lowercaseAddresses = config.lowercaseAddresses
-  let sources = switch chainConfig.sourceConfig {
-  | Config.EvmSourceConfig({hypersync, rpcs}) =>
-    // Build Internal.evmContractConfig from contracts for EvmChain.makeSources
-    let evmContracts: array<Internal.evmContractConfig> = chainConfig.contracts->Array.map((
-      contract
-    ): Internal.evmContractConfig => {
-      name: contract.name,
-      abi: contract.abi,
-      events: contract.events->(
-        Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
-      ),
-    })
-    let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
-      let syncConfig = rpc.syncConfig
-      let ws = rpc.ws
-      {
-        url: rpc.url,
-        sourceFor: rpc.sourceFor,
-        ?syncConfig,
-        ?ws,
-      }
-    })
-    EvmChain.makeSources(
-      ~chain,
-      ~contracts=evmContracts,
-      ~hyperSync=hypersync,
-      ~rpcs=evmRpcs,
-      ~lowercaseAddresses,
-    )
-  | Config.FuelSourceConfig({hypersync}) => [HyperFuelSource.make({chain, endpointUrl: hypersync})]
-  | Config.SvmSourceConfig({hypersync, rpc}) =>
-    switch (hypersync, rpc) {
-    | (None, None) =>
-      JsError.throwWithMessage(
-        `Chain ${chain->ChainMap.Chain.toChainId->Int.toString} has no SVM data source`,
-      )
-    | (None, Some(rpc)) => [Svm.makeRPCSource(~chain, ~rpc)]
-    | (Some(hypersyncUrl), _) =>
-      // HyperSync drives instruction sync. A configured RPC is ignored for now
-      // (RPC fallback isn't wired up yet).
-      let svmEventConfigs =
-        chainConfig.contracts
-        ->Array.flatMap(contract => contract.events)
-        ->(Utils.magic: array<Internal.eventConfig> => array<Internal.svmInstructionEventConfig>)
-      let apiToken = Env.envioApiToken
-      [
-        SvmHyperSyncSource.make({
-          chain,
-          endpointUrl: hypersyncUrl,
-          apiToken,
-          eventConfigs: svmEventConfigs,
-          clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
-        }),
-      ]
-    }
-  // For tests: use ready-to-use sources directly
-  | Config.CustomSources(sources) => sources
-  }
-
-  {
-    logger,
-    chainConfig,
-    sourceManager: SourceManager.make(
-      ~sources,
-      ~maxPartitionConcurrency=Env.maxPartitionConcurrency,
-      ~isRealtime,
-      ~reducedPollingInterval?,
-    ),
-    reorgDetection: ReorgDetection.make(
-      ~chainReorgCheckpoints,
-      ~maxReorgDepth,
-      ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
-    ),
-    safeCheckpointTracking: SafeCheckpointTracking.make(
-      ~maxReorgDepth,
-      ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
-      ~chainReorgCheckpoints,
-    ),
-    isProgressAtHead: false,
-    fetchState,
-    committedProgressBlockNumber: progressBlockNumber,
-    timestampCaughtUpToHeadOrEndblock,
-    numEventsProcessed,
-  }
-}
-
-let makeFromConfig = (
-  chainConfig: Config.chain,
-  ~config,
-  ~registrations,
-  ~targetBufferSize,
-  ~knownHeight,
-) => {
-  let logger = Logging.createChild(~params={"chainId": chainConfig.id})
-
-  make(
-    ~chainConfig,
-    ~config,
-    ~registrations,
-    ~startBlock=chainConfig.startBlock,
-    ~endBlock=chainConfig.endBlock,
-    ~reorgCheckpoints=[],
-    ~maxReorgDepth=chainConfig.maxReorgDepth,
-    ~progressBlockNumber=-1,
-    ~timestampCaughtUpToHeadOrEndblock=None,
-    ~numEventsProcessed=0.,
-    ~targetBufferSize,
-    ~logger,
-    ~indexingAddresses=configAddresses(chainConfig),
-    ~isInReorgThreshold=false,
-    ~isRealtime=false,
-    ~knownHeight,
-  )
-}
-
-/**
- * This function allows a chain fetcher to be created from metadata, in particular this is useful for restarting an indexer and making sure it fetches blocks from the same place.
- */
-let makeFromDbState = (
-  chainConfig: Config.chain,
-  ~resumedChainState: Persistence.initialChainState,
-  ~reorgCheckpoints,
-  ~isInReorgThreshold,
-  ~isRealtime,
-  ~config,
-  ~registrations,
-  ~targetBufferSize,
-  ~reducedPollingInterval=?,
-) => {
-  let chainId = chainConfig.id
-  let logger = Logging.createChild(~params={"chainId": chainId})
-
-  Prometheus.ProgressEventsCount.set(~processedCount=resumedChainState.numEventsProcessed, ~chainId)
-
-  let progressBlockNumber =
-    // Can be -1 when not set
-    resumedChainState.progressBlockNumber >= 0
-      ? resumedChainState.progressBlockNumber
-      : resumedChainState.startBlock - 1
-
-  make(
-    ~indexingAddresses=resumedChainState.indexingAddresses,
-    ~chainConfig,
-    ~startBlock=resumedChainState.startBlock,
-    ~endBlock=resumedChainState.endBlock,
-    ~config,
-    ~registrations,
-    ~reorgCheckpoints,
-    ~maxReorgDepth=resumedChainState.maxReorgDepth,
-    ~firstEventBlock=resumedChainState.firstEventBlockNumber,
-    ~progressBlockNumber,
-    ~timestampCaughtUpToHeadOrEndblock=Env.updateSyncTimeOnRestart
-      ? None
-      : resumedChainState.timestampCaughtUpToHeadOrEndblock,
-    ~numEventsProcessed=resumedChainState.numEventsProcessed,
-    ~logger,
-    ~targetBufferSize,
-    ~isInReorgThreshold,
-    ~isRealtime,
-    ~knownHeight=resumedChainState.sourceBlockNumber,
-    ~reducedPollingInterval?,
-  )
-}
-
-let runContractRegistersOrThrow = async (
-  ~itemsWithContractRegister: array<Internal.item>,
-  ~config: Config.t,
-) => {
-  let itemsWithDcs = []
-
-  let onRegister = (~item: Internal.item, ~contractAddress, ~contractName) => {
-    let eventItem = item->Internal.castUnsafeEventItem
-    let {blockNumber} = eventItem
-
-    let dc: Internal.indexingAddress = {
-      address: contractAddress,
-      contractName,
-      registrationBlock: blockNumber,
-    }
-
-    switch item->Internal.getItemDcs {
-    | None => {
-        item->Internal.setItemDcs([dc])
-        itemsWithDcs->Array.push(item)
-      }
-    | Some(dcs) => dcs->Array.push(dc)
-    }
-  }
-
-  let promises = []
-  for idx in 0 to itemsWithContractRegister->Array.length - 1 {
-    let item = itemsWithContractRegister->Array.getUnsafe(idx)
-    let eventItem = item->Internal.castUnsafeEventItem
-    let contractRegister = switch eventItem {
-    | {eventConfig: {contractRegister: Some(contractRegister)}} => contractRegister
-    | {eventConfig: {contractRegister: None, name: eventName}} =>
-      // Unexpected case, since we should pass only events with contract register to this function
-      JsError.throwWithMessage("Contract register is not set for event " ++ eventName)
-    }
-
-    let errorMessage = "Event contractRegister failed, please fix the error to keep the indexer running smoothly"
-
-    // Catch sync and async errors
-    try {
-      let params: UserContext.contractRegisterParams = {
-        item,
-        onRegister,
-        config,
-        isResolved: false,
-      }
-      let result = contractRegister(UserContext.getContractRegisterArgs(params))
-
-      // Even though `contractRegister` always returns a promise,
-      // in the ReScript type, but it might return a non-promise value for TS API.
-      if result->Utils.Promise.isCatchable {
-        promises->Array.push(
-          result
-          ->Promise.thenResolve(r => {
-            params.isResolved = true
-            r
-          })
-          ->Promise.catch(exn => {
-            params.isResolved = true
-            exn->ErrorHandling.mkLogAndRaise(~msg=errorMessage, ~logger=item->Logging.getItemLogger)
-          }),
-        )
-      } else {
-        params.isResolved = true
-      }
-    } catch {
-    | exn =>
-      exn->ErrorHandling.mkLogAndRaise(~msg=errorMessage, ~logger=item->Logging.getItemLogger)
-    }
-  }
-
-  if promises->Utils.Array.notEmpty {
-    let _ = await Promise.all(promises)
-  }
-
-  itemsWithDcs
-}
-
-let handleQueryResult = (
-  chainFetcher: t,
-  ~query: FetchState.query,
-  ~newItems,
-  ~newItemsWithDcs,
-  ~latestFetchedBlock,
-  ~knownHeight,
-) => {
-  let fs = switch newItemsWithDcs {
-  | [] => chainFetcher.fetchState
-  | _ => chainFetcher.fetchState->FetchState.registerDynamicContracts(newItemsWithDcs)
-  }
-
-  {
-    ...chainFetcher,
-    fetchState: fs
-    ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
-    ->FetchState.updateKnownHeight(~knownHeight),
-  }
-}
-
-/**
-Gets the latest item on the front of the queue and returns updated fetcher
-*/
-let hasProcessedToEndblock = (self: t) => {
-  let {committedProgressBlockNumber, fetchState} = self
-  switch fetchState.endBlock {
-  | Some(endBlock) => committedProgressBlockNumber >= endBlock
-  | None => false
-  }
-}
-
-let hasNoMoreEventsToProcess = (self: t) => {
-  self.fetchState->FetchState.bufferSize === 0
-}
-
-let getHighestBlockBelowThreshold = (cf: t): int => {
-  let highestBlockBelowThreshold = cf.fetchState.knownHeight - cf.chainConfig.maxReorgDepth
-  highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
-}
-
-/**
-Finds the last known valid block number below the reorg block
-If not found, returns the highest block below threshold
-*/
-let getLastKnownValidBlock = async (chainFetcher: t, ~reorgBlockNumber: int, ~isRealtime: bool) => {
-  // Don't include the reorg block itself — different source instances
-  // may have mismatching hashes at the head, so we always rollback
-  // the block where we detected the reorg.
-  let scannedBlockNumbers =
-    chainFetcher.reorgDetection->ReorgDetection.getThresholdBlockNumbersBelowBlock(
-      ~blockNumber=reorgBlockNumber,
-      ~knownHeight=chainFetcher.fetchState.knownHeight,
-    )
-
-  switch scannedBlockNumbers {
-  | [] => chainFetcher->getHighestBlockBelowThreshold
-  | _ => {
-      let blockNumbersAndHashes = await chainFetcher.sourceManager->SourceManager.getBlockHashes(
-        ~blockNumbers=scannedBlockNumbers,
-        ~isRealtime,
-      )
-
-      switch chainFetcher.reorgDetection->ReorgDetection.getLatestValidScannedBlock(
-        ~blockNumbersAndHashes,
-      ) {
-      | Some(blockNumber) => blockNumber
-      | None => chainFetcher->getHighestBlockBelowThreshold
-      }
-    }
-  }
-}
-
-let isActivelyIndexing = (chainFetcher: t) => chainFetcher.fetchState->FetchState.isActivelyIndexing
-
-let isReady = (chainFetcher: t) => chainFetcher.timestampCaughtUpToHeadOrEndblock !== None