envio 3.3.0-alpha.1 → 3.3.0-alpha.11

package/src/sources/HyperSyncHeightStream.res.mjs

@@ -2,7 +2,6 @@
 
 import * as Utils from "../Utils.res.mjs";
 import * as Logging from "../Logging.res.mjs";
-import * as Prometheus from "../Prometheus.res.mjs";
 import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
 import * as Eventsource from "eventsource";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
@@ -83,7 +82,6 @@ function subscribe(hyperSyncUrl, apiToken, chainId, onHeight) {
     es$1.addEventListener("height", event => {
       let height = Stdlib_Int.fromString(event.data, undefined);
       if (height !== undefined) {
-        Prometheus.SourceRequestCount.increment("HyperSync", chainId, "heightStream");
         updateTimeoutId();
         return onHeight(height);
       } else {

package/src/sources/HyperSyncSource.res

@@ -1,132 +1,5 @@
 open Source
 
-type selectionConfig = {
-  getLogSelectionOrThrow: (
-    ~addressesByContractName: dict<array<Address.t>>,
-  ) => array<LogSelection.t>,
-  fieldSelection: HyperSyncClient.QueryTypes.fieldSelection,
-}
-
-let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
-  let capitalizedBlockFields = Utils.Set.make()
-  let capitalizedTransactionFields = Utils.Set.make()
-
-  let staticTopicSelectionsByContract = Dict.make()
-  let dynamicEventFiltersByContract = Dict.make()
-  let dynamicWildcardEventFiltersByContract = Dict.make()
-  let noAddressesTopicSelections = []
-  let contractNames = Utils.Set.make()
-
-  selection.eventConfigs
-  ->(Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>)
-  ->Array.forEach(({
-    dependsOnAddresses,
-    contractName,
-    getEventFiltersOrThrow,
-    selectedBlockFields,
-    selectedTransactionFields,
-    isWildcard,
-  }) => {
-    selectedBlockFields
-    ->Utils.Set.toArray
-    ->Array.forEach(name =>
-      capitalizedBlockFields
-      ->Utils.Set.add((name :> string)->Utils.String.capitalize)
-      ->ignore
-    )
-    selectedTransactionFields
-    ->Utils.Set.toArray
-    ->Array.forEach(name =>
-      capitalizedTransactionFields
-      ->Utils.Set.add((name :> string)->Utils.String.capitalize)
-      ->ignore
-    )
-
-    let eventFilters = getEventFiltersOrThrow(chain)
-    if dependsOnAddresses {
-      let _ = contractNames->Utils.Set.add(contractName)
-      switch eventFilters {
-      | Static(topicSelections) =>
-        staticTopicSelectionsByContract->Utils.Dict.pushMany(contractName, topicSelections)
-      | Dynamic(fn) =>
-        (
-          isWildcard ? dynamicWildcardEventFiltersByContract : dynamicEventFiltersByContract
-        )->Utils.Dict.push(contractName, fn)
-      }
-    } else {
-      noAddressesTopicSelections
-      ->Array.pushMany(
-        switch eventFilters {
-        | Static(s) => s
-        | Dynamic(fn) => fn([])
-        },
-      )
-      ->ignore
-    }
-  })
-
-  let fieldSelection: HyperSyncClient.QueryTypes.fieldSelection = {
-    log: [Address, Data, LogIndex, Topic0, Topic1, Topic2, Topic3],
-    block: capitalizedBlockFields
-    ->Utils.Set.toArray
-    ->(Utils.magic: array<string> => array<HyperSyncClient.QueryTypes.blockField>),
-    transaction: capitalizedTransactionFields
-    ->Utils.Set.toArray
-    ->(Utils.magic: array<string> => array<HyperSyncClient.QueryTypes.transactionField>),
-  }
-
-  let noAddressesLogSelection = LogSelection.make(
-    ~addresses=[],
-    ~topicSelections=noAddressesTopicSelections,
-  )
-
-  let getLogSelectionOrThrow = (~addressesByContractName): array<LogSelection.t> => {
-    let logSelections = []
-    if noAddressesLogSelection.topicSelections->Utils.Array.isEmpty->not {
-      logSelections->Array.push(noAddressesLogSelection)
-    }
-    contractNames->Utils.Set.forEach(contractName => {
-      switch addressesByContractName->Utils.Dict.dangerouslyGetNonOption(contractName) {
-      | None
-      | Some([]) => ()
-      | Some(addresses) =>
-        switch staticTopicSelectionsByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
-        | None => ()
-        | Some(topicSelections) =>
-          logSelections->Array.push(LogSelection.make(~addresses, ~topicSelections))
-        }
-        switch dynamicEventFiltersByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
-        | None => ()
-        | Some(fns) =>
-          logSelections->Array.push(
-            LogSelection.make(~addresses, ~topicSelections=fns->Array.flatMap(fn => fn(addresses))),
-          )
-        }
-        switch dynamicWildcardEventFiltersByContract->Utils.Dict.dangerouslyGetNonOption(
-          contractName,
-        ) {
-        | None => ()
-        | Some(fns) =>
-          logSelections->Array.push(
-            LogSelection.make(
-              ~addresses=[],
-              ~topicSelections=fns->Array.flatMap(fn => fn(addresses)),
-            ),
-          )
-        }
-      }
-    })
-    logSelections
-  }
-
-  {
-    getLogSelectionOrThrow,
-    fieldSelection,
-  }
-}
-
-let memoGetSelectionConfig = (~chain) =>
-  Utils.WeakMap.memoize(selection => selection->getSelectionConfig(~chain))
 
 // Surfaced by HyperSyncClient.getHeight (Rust) when HyperSync rejects the API
 // token. The corrupted-token test feeds the real server error through this
@@ -136,8 +9,8 @@ let isUnauthorizedError = (message: string) => message->String.includes("401 Una
 type options = {
   chain: ChainMap.Chain.t,
   endpointUrl: string,
-  allEventParams: array<HyperSyncClient.Decoder.eventParamsInput>,
-  eventRouter: EventRouter.t<Internal.evmEventConfig>,
+  // The chain's registrations, indexed by their sequential `index`.
+  onEventRegistrations: array<Internal.evmOnEventRegistration>,
   apiToken: option<string>,
   clientTimeoutMillis: int,
   lowercaseAddresses: bool,
@@ -150,8 +23,7 @@ let make = (
   {
     chain,
     endpointUrl,
-    allEventParams,
-    eventRouter,
+    onEventRegistrations,
     apiToken,
     clientTimeoutMillis,
     lowercaseAddresses,
@@ -162,8 +34,6 @@ let make = (
 ): t => {
   let name = "HyperSync"
 
-  let getSelectionConfig = memoGetSelectionConfig(~chain)
-
   let apiToken = switch apiToken {
   | Some(token) => token
   | None =>
@@ -176,7 +46,9 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     ~url=endpointUrl,
     ~apiToken,
     ~httpReqTimeoutMillis=clientTimeoutMillis,
-    ~eventParams=allEventParams,
+    ~eventRegistrations=HyperSyncClient.Registration.fromOnEventRegistrations(
+      onEventRegistrations,
+    ),
     ~enableChecksumAddresses=!lowercaseAddresses,
     ~serializationFormat,
     ~enableQueryCaching,
@@ -189,32 +61,25 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     )
   }
 
-  exception UndefinedValue
-
   let makeEventBatchQueueItem = (
     item: HyperSyncClient.EventItems.item,
-    ~params: Internal.eventParams,
-    ~eventConfig: Internal.evmEventConfig,
+    ~onEventRegistration: Internal.evmOnEventRegistration,
   ): Internal.item => {
-    let {block, transaction, logIndex, srcAddress} = item
-    let chainId = chain->ChainMap.Chain.toChainId
+    let {transactionIndex, logIndex, srcAddress} = item
 
     Internal.Event({
-      eventConfig: (eventConfig :> Internal.eventConfig),
-      timestamp: block.timestamp->Option.getUnsafe,
+      onEventRegistration: (onEventRegistration :> Internal.onEventRegistration),
       chain,
-      blockNumber: block.number->Option.getUnsafe,
-      blockHash: block.hash->Option.getUnsafe,
+      blockNumber: item.blockNumber,
       logIndex,
+      transactionIndex,
+      // `block` and `transaction` are omitted; they're materialised from the
+      // per-chain stores onto the payload at batch prep.
       payload: {
-        contractName: eventConfig.contractName,
-        eventName: eventConfig.name,
-        chainId,
-        params,
-        transaction: transaction->(
-          Utils.magic: HyperSyncClient.ResponseTypes.transaction => Internal.eventTransaction
-        ),
-        block: block->(Utils.magic: HyperSyncClient.ResponseTypes.block => Internal.eventBlock),
+        contractName: onEventRegistration.eventConfig.contractName,
+        eventName: onEventRegistration.eventConfig.name,
+        chainId: chain->ChainMap.Chain.toChainId,
+        params: item.params,
         srcAddress,
         logIndex,
       }->Evm.fromPayload,
@@ -225,36 +90,26 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     ~fromBlock,
     ~toBlock,
     ~addressesByContractName,
-    ~indexingAddresses,
+    ~contractNameByAddress as _,
     ~knownHeight,
     ~partitionId as _,
-    ~selection,
+    ~selection: FetchState.selection,
+    ~itemsTarget,
     ~retry,
-    ~logger,
+    ~logger as _,
   ) => {
-    let totalTimeRef = Hrtime.makeTimer()
+    let totalTimeRef = Performance.now()
 
-    let selectionConfig = selection->getSelectionConfig
-
-    let logSelections = try selectionConfig.getLogSelectionOrThrow(~addressesByContractName) catch {
-    | exn =>
-      exn->ErrorHandling.mkLogAndRaise(~logger, ~msg="Failed getting log selection for the query")
-    }
-
-    let startFetchingBatchTimeRef = Hrtime.makeTimer()
+    let startFetchingBatchTimeRef = Performance.now()
 
     //fetch batch
-    Prometheus.SourceRequestCount.increment(
-      ~sourceName=name,
-      ~chainId=chain->ChainMap.Chain.toChainId,
-      ~method="getLogs",
-    )
     let pageUnsafe = try await HyperSync.GetLogs.query(
       ~client,
       ~fromBlock,
       ~toBlock,
-      ~logSelections,
-      ~fieldSelection=selectionConfig.fieldSelection,
+      ~maxNumLogs=itemsTarget,
+      ~registrationIndexes=selection.onEventRegistrations->Array.map(reg => reg.index),
+      ~addressesByContractName,
     ) catch {
     | HyperSync.GetLogs.Error(error) =>
       throw(
@@ -301,7 +156,8 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       )
     }
 
-    let pageFetchTime = startFetchingBatchTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let pageFetchTime = startFetchingBatchTimeRef->Performance.secondsSince
+    let requestStats = [{Source.method: "getLogs", seconds: pageFetchTime}]
 
     //set height and next from block
     let knownHeight = pageUnsafe.archiveHeight
@@ -311,82 +167,37 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     //In the query
     let heighestBlockQueried = pageUnsafe.nextBlock - 1
 
-    let parsingTimeRef = Hrtime.makeTimer()
+    let parsingTimeRef = Performance.now()
 
     //Parse page items into queue items
     let parsedQueueItems = []
 
-    let handleDecodeFailure = (
-      ~eventConfig: Internal.evmEventConfig,
-      ~logIndex,
-      ~blockNumber,
-      ~chainId,
-      ~exn,
-    ) => {
-      if !eventConfig.isWildcard {
-        //Wildcard events can be parsed as undefined if the number of topics
-        //don't match the event with the given topic0
-        //Non wildcard events should be expected to be parsed
-        let msg = `Event ${eventConfig.name} was unexpectedly parsed as undefined`
-        let logger = Logging.createChildFrom(
-          ~logger,
-          ~params={
-            "chainId": chainId,
-            "blockNumber": blockNumber,
-            "logIndex": logIndex,
-            "decoder": "hypersync-client",
-          },
-        )
-        exn->ErrorHandling.mkLogAndRaise(~msg, ~logger)
-      }
-    }
+    // Block headers are returned once per number; items reference them by blockNumber.
+    let blocksByNumber = Utils.Map.make()
+    pageUnsafe.blocks->Array.forEach(block => {
+      blocksByNumber->Utils.Map.set(block.number, block)->ignore
+    })
+    let getBlock = blockNumber => blocksByNumber->Utils.Map.unsafeGet(blockNumber)
 
     pageUnsafe.items->Array.forEach(item => {
-      let chainId = chain->ChainMap.Chain.toChainId
-      let maybeEventConfig =
-        eventRouter->EventRouter.get(
-          ~tag=EventRouter.getEvmEventId(
-            ~sighash=item.topic0->EvmTypes.Hex.toString,
-            ~topicCount=item.topicCount,
-          ),
-          ~indexingAddresses,
-          ~contractAddress=item.srcAddress,
-          ~blockNumber=item.block.number->Option.getUnsafe,
-        )
-
-      switch maybeEventConfig {
-      | None => () //ignore events that aren't registered
-      | Some(eventConfig) =>
-        switch item.params
-        ->Nullable.toOption
-        ->Option.flatMap(Dict.get(_, eventConfig.contractName)) {
-        | Some(params) =>
-          parsedQueueItems->Array.push(makeEventBatchQueueItem(item, ~params, ~eventConfig))->ignore
-        | None =>
-          handleDecodeFailure(
-            ~eventConfig,
-            ~logIndex=item.logIndex,
-            ~blockNumber=item.block.number->Option.getUnsafe,
-            ~chainId,
-            ~exn=UndefinedValue,
-          )
-        }
-      }
+      let onEventRegistration =
+        onEventRegistrations->Array.getUnsafe(item.onEventRegistrationIndex)
+      parsedQueueItems
+      ->Array.push(makeEventBatchQueueItem(item, ~onEventRegistration))
+      ->ignore
     })
 
-    let parsingTimeElapsed = parsingTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let parsingTimeElapsed = parsingTimeRef->Performance.secondsSince
 
     // Collect (blockNumber, blockHash) pairs we already have from the response —
-    // one per item's block plus, when present, the rollbackGuard's head block
+    // one per returned block plus, when present, the rollbackGuard's head block
     // and the parent of the range's first block. Duplicates are allowed; reorg
     // detection notices same-block-number-different-hash collisions itself.
     let blockHashes = []
-    pageUnsafe.items->Array.forEach(({block}) => {
-      switch (block.number, block.hash) {
-      | (Some(blockNumber), Some(blockHash)) =>
-        blockHashes->Array.push({ReorgDetection.blockNumber, blockHash})->ignore
-      | _ => ()
-      }
+    pageUnsafe.blocks->Array.forEach(block => {
+      blockHashes
+      ->Array.push({ReorgDetection.blockNumber: block.number, blockHash: block.hash})
+      ->ignore
     })
     switch pageUnsafe.rollbackGuard {
     | None => ()
@@ -409,13 +220,13 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     | Some({timestamp}) => timestamp
     | None =>
       switch pageUnsafe.items->Array.get(pageUnsafe.items->Array.length - 1) {
-      | Some({block}) if block.number->Option.getUnsafe == heighestBlockQueried =>
-        block.timestamp->Option.getUnsafe
+      | Some(item) if item.blockNumber == heighestBlockQueried =>
+        getBlock(item.blockNumber).timestamp
       | _ => 0
       }
     }
 
-    let totalTimeElapsed = totalTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let totalTimeElapsed = totalTimeRef->Performance.secondsSince
 
     let stats = {
       totalTimeElapsed,
@@ -426,11 +237,14 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     {
       latestFetchedBlockTimestamp,
       parsedQueueItems,
+      transactionStore: Some(pageUnsafe.transactionStore),
+      blockStore: Some(pageUnsafe.blockStore),
       latestFetchedBlockNumber: heighestBlockQueried,
       stats,
       knownHeight,
       blockHashes,
       fromBlockQueried: fromBlock,
+      requestStats,
     }
   }
 
@@ -441,7 +255,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       ~sourceName=name,
       ~chainId=chain->ChainMap.Chain.toChainId,
       ~logger,
-    )->Promise.thenResolve(HyperSync.mapExn)
+    )->Promise.thenResolve(((queryRes, requestStats)) => {
+      Source.result: queryRes->HyperSync.mapExn,
+      requestStats,
+    })
 
   {
     name,
@@ -451,8 +268,8 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     poweredByHyperSync: true,
     getBlockHashes,
     getHeightOrThrow: async () => {
-      let timerRef = Hrtime.makeTimer()
-      let result = try {
+      let timerRef = Performance.now()
+      let height = try {
         await client.getHeight()
       } catch {
       | JsExn(e) =>
@@ -465,19 +282,8 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
         | _ => throw(JsExn(e))
         }
       }
-      let seconds = timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat
-      Prometheus.SourceRequestCount.increment(
-        ~sourceName=name,
-        ~chainId=chain->ChainMap.Chain.toChainId,
-        ~method="getHeight",
-      )
-      Prometheus.SourceRequestCount.addSeconds(
-        ~sourceName=name,
-        ~chainId=chain->ChainMap.Chain.toChainId,
-        ~method="getHeight",
-        ~seconds,
-      )
-      result
+      let seconds = timerRef->Performance.secondsSince
+      {height, requestStats: [{method: "getHeight", seconds}]}
     },
     getItemsOrThrow,
     createHeightSubscription: (~onHeight) =>