envio 3.3.0-alpha.1 → 3.3.0-alpha.10

package/src/sources/HyperSyncSource.res

@@ -7,26 +7,23 @@ type selectionConfig = {
   fieldSelection: HyperSyncClient.QueryTypes.fieldSelection,
 }
 
-let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
+let getSelectionConfig = (selection: FetchState.selection) => {
   let capitalizedBlockFields = Utils.Set.make()
   let capitalizedTransactionFields = Utils.Set.make()
 
-  let staticTopicSelectionsByContract = Dict.make()
-  let dynamicEventFiltersByContract = Dict.make()
-  let dynamicWildcardEventFiltersByContract = Dict.make()
+  let topicSelectionsByContract = Dict.make()
+  let wildcardTopicSelectionsByContract = Dict.make()
   let noAddressesTopicSelections = []
   let contractNames = Utils.Set.make()
 
-  selection.eventConfigs
-  ->(Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>)
-  ->Array.forEach(({
-    dependsOnAddresses,
-    contractName,
-    getEventFiltersOrThrow,
-    selectedBlockFields,
-    selectedTransactionFields,
-    isWildcard,
-  }) => {
+  selection.onEventRegistrations
+  ->(Utils.magic: array<Internal.onEventRegistration> => array<Internal.evmOnEventRegistration>)
+  ->Array.forEach(reg => {
+    let eventConfig =
+      reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.evmEventConfig)
+    let contractName = eventConfig.contractName
+    let {selectedBlockFields, selectedTransactionFields} = eventConfig
+    let {dependsOnAddresses, resolvedWhere, isWildcard} = reg
     selectedBlockFields
     ->Utils.Set.toArray
     ->Array.forEach(name =>
@@ -36,30 +33,26 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
     )
     selectedTransactionFields
     ->Utils.Set.toArray
-    ->Array.forEach(name =>
-      capitalizedTransactionFields
-      ->Utils.Set.add((name :> string)->Utils.String.capitalize)
-      ->ignore
-    )
+    ->Array.forEach(name => {
+      // transactionIndex is read off the log (the store key), so it never needs
+      // to be requested as a transaction column — and requesting it alone would
+      // pull the whole transaction table for nothing.
+      let fieldName = (name :> string)
+      if fieldName != "transactionIndex" {
+        capitalizedTransactionFields->Utils.Set.add(fieldName->Utils.String.capitalize)->ignore
+      }
+    })
 
-    let eventFilters = getEventFiltersOrThrow(chain)
     if dependsOnAddresses {
       let _ = contractNames->Utils.Set.add(contractName)
-      switch eventFilters {
-      | Static(topicSelections) =>
-        staticTopicSelectionsByContract->Utils.Dict.pushMany(contractName, topicSelections)
-      | Dynamic(fn) =>
-        (
-          isWildcard ? dynamicWildcardEventFiltersByContract : dynamicEventFiltersByContract
-        )->Utils.Dict.push(contractName, fn)
-      }
+
+      (
+        isWildcard ? wildcardTopicSelectionsByContract : topicSelectionsByContract
+      )->Utils.Dict.pushMany(contractName, resolvedWhere.topicSelections)
     } else {
       noAddressesTopicSelections
       ->Array.pushMany(
-        switch eventFilters {
-        | Static(s) => s
-        | Dynamic(fn) => fn([])
-        },
+        resolvedWhere.topicSelections->LogSelection.materializeTopicSelections(~addresses=[]),
       )
       ->ignore
     }
@@ -90,27 +83,26 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
       | None
       | Some([]) => ()
       | Some(addresses) =>
-        switch staticTopicSelectionsByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
+        switch topicSelectionsByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
         | None => ()
         | Some(topicSelections) =>
-          logSelections->Array.push(LogSelection.make(~addresses, ~topicSelections))
-        }
-        switch dynamicEventFiltersByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
-        | None => ()
-        | Some(fns) =>
           logSelections->Array.push(
-            LogSelection.make(~addresses, ~topicSelections=fns->Array.flatMap(fn => fn(addresses))),
+            LogSelection.make(
+              ~addresses,
+              ~topicSelections=topicSelections->LogSelection.materializeTopicSelections(~addresses),
+            ),
           )
         }
-        switch dynamicWildcardEventFiltersByContract->Utils.Dict.dangerouslyGetNonOption(
-          contractName,
-        ) {
+        // Wildcard events that filter an indexed param by registered addresses:
+        // the addresses fold into the topics, so the query itself stays
+        // address-unbound.
+        switch wildcardTopicSelectionsByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
         | None => ()
-        | Some(fns) =>
+        | Some(topicSelections) =>
           logSelections->Array.push(
             LogSelection.make(
               ~addresses=[],
-              ~topicSelections=fns->Array.flatMap(fn => fn(addresses)),
+              ~topicSelections=topicSelections->LogSelection.materializeTopicSelections(~addresses),
             ),
           )
         }
@@ -125,8 +117,7 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
   }
 }
 
-let memoGetSelectionConfig = (~chain) =>
-  Utils.WeakMap.memoize(selection => selection->getSelectionConfig(~chain))
+let memoGetSelectionConfig = () => Utils.WeakMap.memoize(getSelectionConfig)
 
 // Surfaced by HyperSyncClient.getHeight (Rust) when HyperSync rejects the API
 // token. The corrupted-token test feeds the real server error through this
@@ -137,7 +128,7 @@ type options = {
   chain: ChainMap.Chain.t,
   endpointUrl: string,
   allEventParams: array<HyperSyncClient.Decoder.eventParamsInput>,
-  eventRouter: EventRouter.t<Internal.evmEventConfig>,
+  eventRouter: EventRouter.t<Internal.evmOnEventRegistration>,
   apiToken: option<string>,
   clientTimeoutMillis: int,
   lowercaseAddresses: bool,
@@ -162,7 +153,7 @@ let make = (
 ): t => {
   let name = "HyperSync"
 
-  let getSelectionConfig = memoGetSelectionConfig(~chain)
+  let getSelectionConfig = memoGetSelectionConfig()
 
   let apiToken = switch apiToken {
   | Some(token) => token
@@ -194,27 +185,24 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
   let makeEventBatchQueueItem = (
     item: HyperSyncClient.EventItems.item,
     ~params: Internal.eventParams,
-    ~eventConfig: Internal.evmEventConfig,
+    ~onEventRegistration: Internal.evmOnEventRegistration,
   ): Internal.item => {
-    let {block, transaction, logIndex, srcAddress} = item
+    let {transactionIndex, logIndex, srcAddress} = item
     let chainId = chain->ChainMap.Chain.toChainId
 
     Internal.Event({
-      eventConfig: (eventConfig :> Internal.eventConfig),
-      timestamp: block.timestamp->Option.getUnsafe,
+      onEventRegistration: (onEventRegistration :> Internal.onEventRegistration),
       chain,
-      blockNumber: block.number->Option.getUnsafe,
-      blockHash: block.hash->Option.getUnsafe,
+      blockNumber: item.blockNumber,
       logIndex,
+      transactionIndex,
+      // `block` and `transaction` are omitted; they're materialised from the
+      // per-chain stores onto the payload at batch prep.
       payload: {
-        contractName: eventConfig.contractName,
-        eventName: eventConfig.name,
+        contractName: onEventRegistration.eventConfig.contractName,
+        eventName: onEventRegistration.eventConfig.name,
         chainId,
         params,
-        transaction: transaction->(
-          Utils.magic: HyperSyncClient.ResponseTypes.transaction => Internal.eventTransaction
-        ),
-        block: block->(Utils.magic: HyperSyncClient.ResponseTypes.block => Internal.eventBlock),
         srcAddress,
         logIndex,
       }->Evm.fromPayload,
@@ -225,14 +213,15 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     ~fromBlock,
     ~toBlock,
     ~addressesByContractName,
-    ~indexingAddresses,
+    ~contractNameByAddress,
     ~knownHeight,
     ~partitionId as _,
     ~selection,
+    ~itemsTarget,
     ~retry,
     ~logger,
   ) => {
-    let totalTimeRef = Hrtime.makeTimer()
+    let totalTimeRef = Performance.now()
 
     let selectionConfig = selection->getSelectionConfig
 
@@ -241,20 +230,16 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       exn->ErrorHandling.mkLogAndRaise(~logger, ~msg="Failed getting log selection for the query")
     }
 
-    let startFetchingBatchTimeRef = Hrtime.makeTimer()
+    let startFetchingBatchTimeRef = Performance.now()
 
     //fetch batch
-    Prometheus.SourceRequestCount.increment(
-      ~sourceName=name,
-      ~chainId=chain->ChainMap.Chain.toChainId,
-      ~method="getLogs",
-    )
     let pageUnsafe = try await HyperSync.GetLogs.query(
       ~client,
       ~fromBlock,
       ~toBlock,
       ~logSelections,
       ~fieldSelection=selectionConfig.fieldSelection,
+      ~maxNumLogs=itemsTarget,
     ) catch {
     | HyperSync.GetLogs.Error(error) =>
       throw(
@@ -301,7 +286,8 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       )
     }
 
-    let pageFetchTime = startFetchingBatchTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let pageFetchTime = startFetchingBatchTimeRef->Performance.secondsSince
+    let requestStats = [{Source.method: "getLogs", seconds: pageFetchTime}]
 
     //set height and next from block
     let knownHeight = pageUnsafe.archiveHeight
@@ -311,23 +297,30 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     //In the query
     let heighestBlockQueried = pageUnsafe.nextBlock - 1
 
-    let parsingTimeRef = Hrtime.makeTimer()
+    let parsingTimeRef = Performance.now()
 
     //Parse page items into queue items
     let parsedQueueItems = []
 
+    // Block headers are returned once per number; items reference them by blockNumber.
+    let blocksByNumber = Utils.Map.make()
+    pageUnsafe.blocks->Array.forEach(block => {
+      blocksByNumber->Utils.Map.set(block.number, block)->ignore
+    })
+    let getBlock = blockNumber => blocksByNumber->Utils.Map.unsafeGet(blockNumber)
+
     let handleDecodeFailure = (
-      ~eventConfig: Internal.evmEventConfig,
+      ~onEventRegistration: Internal.evmOnEventRegistration,
       ~logIndex,
       ~blockNumber,
       ~chainId,
       ~exn,
     ) => {
-      if !eventConfig.isWildcard {
+      if !onEventRegistration.isWildcard {
         //Wildcard events can be parsed as undefined if the number of topics
         //don't match the event with the given topic0
         //Non wildcard events should be expected to be parsed
-        let msg = `Event ${eventConfig.name} was unexpectedly parsed as undefined`
+        let msg = `Event ${onEventRegistration.eventConfig.name} was unexpectedly parsed as undefined`
         let logger = Logging.createChildFrom(
           ~logger,
           ~params={
@@ -349,24 +342,25 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
             ~sighash=item.topic0->EvmTypes.Hex.toString,
             ~topicCount=item.topicCount,
           ),
-          ~indexingAddresses,
+          ~contractNameByAddress,
           ~contractAddress=item.srcAddress,
-          ~blockNumber=item.block.number->Option.getUnsafe,
         )
 
       switch maybeEventConfig {
       | None => () //ignore events that aren't registered
-      | Some(eventConfig) =>
+      | Some(onEventRegistration) =>
         switch item.params
         ->Nullable.toOption
-        ->Option.flatMap(Dict.get(_, eventConfig.contractName)) {
+        ->Option.flatMap(Dict.get(_, onEventRegistration.eventConfig.contractName)) {
         | Some(params) =>
-          parsedQueueItems->Array.push(makeEventBatchQueueItem(item, ~params, ~eventConfig))->ignore
+          parsedQueueItems
+          ->Array.push(makeEventBatchQueueItem(item, ~params, ~onEventRegistration))
+          ->ignore
         | None =>
           handleDecodeFailure(
-            ~eventConfig,
+            ~onEventRegistration,
             ~logIndex=item.logIndex,
-            ~blockNumber=item.block.number->Option.getUnsafe,
+            ~blockNumber=item.blockNumber,
             ~chainId,
             ~exn=UndefinedValue,
           )
@@ -374,19 +368,17 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       }
     })
 
-    let parsingTimeElapsed = parsingTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let parsingTimeElapsed = parsingTimeRef->Performance.secondsSince
 
     // Collect (blockNumber, blockHash) pairs we already have from the response —
-    // one per item's block plus, when present, the rollbackGuard's head block
+    // one per returned block plus, when present, the rollbackGuard's head block
     // and the parent of the range's first block. Duplicates are allowed; reorg
     // detection notices same-block-number-different-hash collisions itself.
     let blockHashes = []
-    pageUnsafe.items->Array.forEach(({block}) => {
-      switch (block.number, block.hash) {
-      | (Some(blockNumber), Some(blockHash)) =>
-        blockHashes->Array.push({ReorgDetection.blockNumber, blockHash})->ignore
-      | _ => ()
-      }
+    pageUnsafe.blocks->Array.forEach(block => {
+      blockHashes
+      ->Array.push({ReorgDetection.blockNumber: block.number, blockHash: block.hash})
+      ->ignore
     })
     switch pageUnsafe.rollbackGuard {
     | None => ()
@@ -409,13 +401,13 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     | Some({timestamp}) => timestamp
     | None =>
       switch pageUnsafe.items->Array.get(pageUnsafe.items->Array.length - 1) {
-      | Some({block}) if block.number->Option.getUnsafe == heighestBlockQueried =>
-        block.timestamp->Option.getUnsafe
+      | Some(item) if item.blockNumber == heighestBlockQueried =>
+        getBlock(item.blockNumber).timestamp
       | _ => 0
       }
     }
 
-    let totalTimeElapsed = totalTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let totalTimeElapsed = totalTimeRef->Performance.secondsSince
 
     let stats = {
       totalTimeElapsed,
@@ -426,11 +418,14 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     {
       latestFetchedBlockTimestamp,
       parsedQueueItems,
+      transactionStore: Some(pageUnsafe.transactionStore),
+      blockStore: Some(pageUnsafe.blockStore),
       latestFetchedBlockNumber: heighestBlockQueried,
       stats,
       knownHeight,
       blockHashes,
       fromBlockQueried: fromBlock,
+      requestStats,
     }
   }
 
@@ -441,7 +436,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       ~sourceName=name,
       ~chainId=chain->ChainMap.Chain.toChainId,
       ~logger,
-    )->Promise.thenResolve(HyperSync.mapExn)
+    )->Promise.thenResolve(((queryRes, requestStats)) => {
+      Source.result: queryRes->HyperSync.mapExn,
+      requestStats,
+    })
 
   {
     name,
@@ -451,8 +449,8 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     poweredByHyperSync: true,
     getBlockHashes,
     getHeightOrThrow: async () => {
-      let timerRef = Hrtime.makeTimer()
-      let result = try {
+      let timerRef = Performance.now()
+      let height = try {
         await client.getHeight()
       } catch {
       | JsExn(e) =>
@@ -465,19 +463,8 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
         | _ => throw(JsExn(e))
         }
       }
-      let seconds = timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat
-      Prometheus.SourceRequestCount.increment(
-        ~sourceName=name,
-        ~chainId=chain->ChainMap.Chain.toChainId,
-        ~method="getHeight",
-      )
-      Prometheus.SourceRequestCount.addSeconds(
-        ~sourceName=name,
-        ~chainId=chain->ChainMap.Chain.toChainId,
-        ~method="getHeight",
-        ~seconds,
-      )
-      result
+      let seconds = timerRef->Performance.secondsSince
+      {height, requestStats: [{method: "getHeight", seconds}]}
     },
     getItemsOrThrow,
     createHeightSubscription: (~onHeight) =>