envio 3.3.0-alpha.7 → 3.3.0-alpha.9

package/src/sources/RpcSource.res

@@ -48,18 +48,16 @@ let getKnownRawBlockWithBackoff = async (
   ~chain,
   ~blockNumber,
   ~backoffMsOnFailure,
+  ~recordRequest: (~method: string, ~seconds: float) => unit,
 ) => {
   let currentBackoff = ref(backoffMsOnFailure)
   let result = ref(None)
 
   while result.contents->Option.isNone {
-    Prometheus.SourceRequestCount.increment(
-      ~sourceName,
-      ~chainId=chain->ChainMap.Chain.toChainId,
-      ~method="eth_getBlockByNumber",
-    )
+    let timerRef = Performance.now()
     switch await getKnownRawBlock(~client, ~blockNumber) {
     | exception err =>
+      recordRequest(~method="eth_getBlockByNumber", ~seconds=timerRef->Performance.secondsSince)
       Logging.warn({
         "err": err->Utils.prettifyExn,
         "msg": `Issue while running fetching batch of events from the RPC. Will wait ${currentBackoff.contents->Int.toString}ms and try again.`,
@@ -69,7 +67,9 @@ let getKnownRawBlockWithBackoff = async (
       })
       await Time.resolvePromiseAfterDelay(~delayMilliseconds=currentBackoff.contents)
       currentBackoff := currentBackoff.contents * 2
-    | json => result := Some(json)
+    | json =>
+      recordRequest(~method="eth_getBlockByNumber", ~seconds=timerRef->Performance.secondsSince)
+      result := Some(json)
     }
   }
   result.contents->Option.getOrThrow
@@ -300,8 +300,7 @@ let getNextPage = (
   ~rpcClient: EvmRpcClient.t,
   ~mutSuggestedBlockIntervals,
   ~partitionId,
-  ~sourceName,
-  ~chainId,
+  ~recordRequest: (~method: string, ~seconds: float) => unit,
 ): promise<eventBatchQuery> => {
   //If the query hangs for longer than this, reject this promise to reduce the block interval
   let queryTimoutPromise =
@@ -314,7 +313,7 @@ let getNextPage = (
   let latestFetchedBlockPromise = loadBlock(toBlock)
 
   let queryLogs = ({addresses, topicQuery}: logSelection) => {
-    Prometheus.SourceRequestCount.increment(~sourceName, ~chainId, ~method="eth_getLogs")
+    let timerRef = Performance.now()
     rpcClient.getLogs({
       fromBlock,
       toBlock,
@@ -326,6 +325,9 @@ let getNextPage = (
         | Multiple(topics) => Nullable.make(topics)
         }
       ),
+    })->Promise.thenResolve(items => {
+      recordRequest(~method="eth_getLogs", ~seconds=timerRef->Performance.secondsSince)
+      items
     })
   }
 
@@ -410,13 +412,13 @@ type selectionConfig = {
   ) => array<logSelection>,
 }
 
-let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
-  let evmEventConfigs =
-    selection.eventConfigs->(
-      Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
+let getSelectionConfig = (selection: FetchState.selection) => {
+  let evmOnEventRegistrations =
+    selection.onEventRegistrations->(
+      Utils.magic: array<Internal.onEventRegistration> => array<Internal.evmOnEventRegistration>
     )
 
-  if evmEventConfigs->Utils.Array.isEmpty {
+  if evmOnEventRegistrations->Utils.Array.isEmpty {
     throw(
       Source.GetItemsError(
         UnsupportedSelection({
@@ -434,36 +436,23 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
   // re-applies it). Pure-wildcard events carry no address constraint, so they're
   // pooled and resolved once.
   let noAddressTopicSelections = []
-  let staticByContract = Dict.make()
-  let dynamicByContract = Dict.make()
-  let dynamicWildcardByContract = Dict.make()
+  let byContract = Dict.make()
+  let wildcardByContract = Dict.make()
   let contractNames = Utils.Set.make()
 
-  evmEventConfigs->Array.forEach(({
-    contractName,
-    isWildcard,
-    dependsOnAddresses,
-    getEventFiltersOrThrow,
-  }) => {
-    let eventFilters = getEventFiltersOrThrow(chain)
+  evmOnEventRegistrations->Array.forEach(reg => {
+    let contractName = reg.eventConfig.contractName
+    let {isWildcard, dependsOnAddresses, resolvedWhere} = reg
     if dependsOnAddresses {
       contractNames->Utils.Set.add(contractName)->ignore
-      switch eventFilters {
-      | Internal.Static(topicSelections) =>
-        staticByContract->Utils.Dict.pushMany(contractName, topicSelections)
-      | Dynamic(fn) =>
-        (isWildcard ? dynamicWildcardByContract : dynamicByContract)->Utils.Dict.push(
-          contractName,
-          fn,
-        )
-      }
+      (isWildcard ? wildcardByContract : byContract)->Utils.Dict.pushMany(
+        contractName,
+        resolvedWhere.topicSelections,
+      )
     } else {
       noAddressTopicSelections
       ->Array.pushMany(
-        switch eventFilters {
-        | Static(s) => s
-        | Dynamic(fn) => fn([])
-        },
+        resolvedWhere.topicSelections->LogSelection.materializeTopicSelections(~addresses=[]),
       )
       ->ignore
     }
@@ -492,30 +481,30 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
         | None
         | Some([]) => ()
         | Some(addresses) =>
-          // Static + dynamic non-wildcard filters, scoped to this contract's addresses.
-          let addressedTopicSelections = []
-          switch staticByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
-          | Some(s) => addressedTopicSelections->Array.pushMany(s)->ignore
-          | None => ()
-          }
-          switch dynamicByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
-          | Some(fns) =>
-            fns->Array.forEach(fn =>
-              addressedTopicSelections->Array.pushMany(fn(addresses))->ignore
+          // Non-wildcard filters, scoped to this contract's addresses.
+          switch byContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
+          | Some(topicSelections) =>
+            logSelections
+            ->Array.pushMany(
+              toLogSelections(
+                ~addresses=Some(addresses),
+                topicSelections->LogSelection.materializeTopicSelections(~addresses),
+              ),
             )
+            ->ignore
           | None => ()
           }
-          logSelections
-          ->Array.pushMany(toLogSelections(~addresses=Some(addresses), addressedTopicSelections))
-          ->ignore
 
-          // Dynamic wildcard-by-address filters fold the address into the topics,
+          // Wildcard-by-address filters fold the address into the topics,
           // so they still match any address.
-          switch dynamicWildcardByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
-          | Some(fns) =>
+          switch wildcardByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
+          | Some(topicSelections) =>
             logSelections
             ->Array.pushMany(
-              toLogSelections(~addresses=None, fns->Array.flatMap(fn => fn(addresses))),
+              toLogSelections(
+                ~addresses=None,
+                topicSelections->LogSelection.materializeTopicSelections(~addresses),
+              ),
             )
             ->ignore
           | None => ()
@@ -531,8 +520,7 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
   }
 }
 
-let memoGetSelectionConfig = (~chain) =>
-  Utils.WeakMap.memoize(selection => selection->getSelectionConfig(~chain))
+let memoGetSelectionConfig = () => Utils.WeakMap.memoize(getSelectionConfig)
 
 // Type-erase a schema for storage in the field registry
 external toFieldSchema: S.t<'a> => S.t<JSON.t> = "%identity"
@@ -674,11 +662,9 @@ let makeThrowingGetEventBlock = (
   }
 }
 
-// `number`, `timestamp` and `hash` are always part of the selected block
-// fields, so they can be read from the assembled block at item construction.
+// `number` is always part of the selected block fields, so it can be read
+// from the assembled block for the item's own `blockNumber`.
 @get external getBlockNumber: Internal.eventBlock => int = "number"
-@get external getBlockTimestamp: Internal.eventBlock => int = "timestamp"
-@get external getBlockHash: Internal.eventBlock => string = "hash"
 
 // Field source classification for RPC calls
 type fieldSource = TransactionOnly | ReceiptOnly | Both
@@ -994,7 +980,7 @@ type options = {
   syncConfig: Config.sourceSync,
   url: string,
   chain: ChainMap.Chain.t,
-  eventRouter: EventRouter.t<Internal.evmEventConfig>,
+  eventRouter: EventRouter.t<Internal.evmOnEventRegistration>,
   allEventParams: array<HyperSyncClient.Decoder.eventParamsInput>,
   lowercaseAddresses: bool,
   ws?: string,
@@ -1024,7 +1010,7 @@ let make = (
   }
   let name = `RPC (${urlHost})`
 
-  let getSelectionConfig = memoGetSelectionConfig(~chain)
+  let getSelectionConfig = memoGetSelectionConfig()
 
   // Per-partition adaptive block interval (AIMD), keyed by partitionId. The
   // `max` key holds a source-wide ceiling that only ever tightens, set by
@@ -1040,15 +1026,37 @@ let make = (
     ~headers?,
   )
 
+  // Requests are made from shared, memoized loaders, so they can't be
+  // attributed to a single getItemsOrThrow/getHeightOrThrow/getBlockHashes
+  // call at its call site. Every actual request (cache/dedup hits never reach
+  // recordRequest) pushes here; each method drains whatever is pending when it
+  // returns. Since a push always lands in exactly one drain, per-source totals
+  // stay exact even with concurrent in-flight calls — which call happens to
+  // drain a given entry doesn't matter, since SourceManager aggregates by
+  // (source, method) regardless of which call returned it.
+  let pendingRequestStats: array<Source.requestStat> = []
+  let recordRequest = (~method, ~seconds) => {
+    pendingRequestStats->Array.push({Source.method, seconds})->ignore
+  }
+  let drainRequestStats = () => {
+    let stats = pendingRequestStats->Utils.Array.copy
+    pendingRequestStats->Utils.Array.clearInPlace
+    stats
+  }
+
   let makeTransactionLoader = () =>
     LazyLoader.make(
       ~loaderFn=transactionHash => {
-        Prometheus.SourceRequestCount.increment(
-          ~sourceName=name,
-          ~chainId=chain->ChainMap.Chain.toChainId,
-          ~method="eth_getTransactionByHash",
-        )
-        Rpc.GetTransactionByHash.rawRoute->Rest.fetch(transactionHash, ~client)
+        let timerRef = Performance.now()
+        Rpc.GetTransactionByHash.rawRoute
+        ->Rest.fetch(transactionHash, ~client)
+        ->Promise.thenResolve(res => {
+          recordRequest(
+            ~method="eth_getTransactionByHash",
+            ~seconds=timerRef->Performance.secondsSince,
+          )
+          res
+        })
       },
       ~onError=(am, ~exn) => {
         Logging.error({
@@ -1076,6 +1084,7 @@ let make = (
           ~chain,
           ~backoffMsOnFailure=1000,
           ~blockNumber,
+          ~recordRequest,
         )
       },
       ~onError=(am, ~exn) => {
@@ -1098,12 +1107,16 @@ let make = (
   let makeReceiptLoader = () =>
     LazyLoader.make(
       ~loaderFn=transactionHash => {
-        Prometheus.SourceRequestCount.increment(
-          ~sourceName=name,
-          ~chainId=chain->ChainMap.Chain.toChainId,
-          ~method="eth_getTransactionReceipt",
-        )
-        Rpc.GetTransactionReceipt.rawRoute->Rest.fetch(transactionHash, ~client)
+        let timerRef = Performance.now()
+        Rpc.GetTransactionReceipt.rawRoute
+        ->Rest.fetch(transactionHash, ~client)
+        ->Promise.thenResolve(res => {
+          recordRequest(
+            ~method="eth_getTransactionReceipt",
+            ~seconds=timerRef->Performance.secondsSince,
+          )
+          res
+        })
       },
       ~onError=(am, ~exn) => {
         Logging.error({
@@ -1211,8 +1224,7 @@ let make = (
       ~rpcClient,
       ~mutSuggestedBlockIntervals,
       ~partitionId,
-      ~sourceName=name,
-      ~chainId=chain->ChainMap.Chain.toChainId,
+      ~recordRequest,
     )
 
     let executedBlockInterval = suggestedToBlock - fromBlock + 1
@@ -1246,7 +1258,11 @@ let make = (
         ~contractAddress=routedAddress,
       ) {
       | None => None
-      | Some(eventConfig) =>
+      | Some(onEventRegistration) =>
+        let eventConfig =
+          onEventRegistration.eventConfig->(
+            Utils.magic: Internal.eventConfig => Internal.evmEventConfig
+          )
         switch maybeDecodedEvent
         ->Nullable.toOption
         ->Option.flatMap(Dict.get(_, eventConfig.contractName)) {
@@ -1293,10 +1309,8 @@ let make = (
                 }
 
                 Internal.Event({
-                  eventConfig: (eventConfig :> Internal.eventConfig),
-                  timestamp: block->getBlockTimestamp,
+                  onEventRegistration: (onEventRegistration :> Internal.onEventRegistration),
                   blockNumber: block->getBlockNumber,
-                  blockHash: block->getBlockHash,
                   chain,
                   logIndex: log.logIndex,
                   transactionIndex: log.transactionIndex,
@@ -1350,14 +1364,16 @@ let make = (
       latestFetchedBlockTimestamp: latestFetchedBlockInfo.timestamp,
       latestFetchedBlockNumber: latestFetchedBlockInfo.number,
       parsedQueueItems,
-      // RPC keeps the transaction inline on the payload; no store page.
+      // RPC keeps the transaction and block inline on the payload; no store pages.
       transactionStore: None,
+      blockStore: None,
       stats: {
         totalTimeElapsed: totalTimeElapsed,
       },
       knownHeight,
       blockHashes,
       fromBlockQueried: fromBlock,
+      requestStats: drainRequestStats(),
     }
   }
 
@@ -1374,21 +1390,25 @@ let make = (
     ->Array.map(blockNum => blockLoader.contents->LazyLoader.get(blockNum))
     ->Promise.all
     ->Promise.thenResolve(rawBlocks => {
-      rawBlocks
-      ->Array.map(json => {
-        let b = parseBlockInfo(json)
-
-        (
-          {
-            blockNumber: b.number,
-            blockHash: b.hash,
-            blockTimestamp: b.timestamp,
-          }: ReorgDetection.blockDataWithTimestamp
-        )
-      })
-      ->Ok
+      let result =
+        rawBlocks
+        ->Array.map(json => {
+          let b = parseBlockInfo(json)
+
+          (
+            {
+              blockNumber: b.number,
+              blockHash: b.hash,
+              blockTimestamp: b.timestamp,
+            }: ReorgDetection.blockDataWithTimestamp
+          )
+        })
+        ->Ok
+      {Source.result, requestStats: drainRequestStats()}
     })
-    ->Promise.catch(exn => exn->Error->Promise.resolve)
+    ->Promise.catch(exn =>
+      {Source.result: Error(exn), requestStats: drainRequestStats()}->Promise.resolve
+    )
   }
 
   let createHeightSubscription =
@@ -1410,33 +1430,11 @@ let make = (
         await rpcClient.getHeight()
       } catch {
       | exn =>
-        let seconds = timerRef->Performance.secondsSince
-        Prometheus.SourceRequestCount.increment(
-          ~sourceName=name,
-          ~chainId=chain->ChainMap.Chain.toChainId,
-          ~method="eth_blockNumber",
-        )
-        Prometheus.SourceRequestCount.addSeconds(
-          ~sourceName=name,
-          ~chainId=chain->ChainMap.Chain.toChainId,
-          ~method="eth_blockNumber",
-          ~seconds,
-        )
+        recordRequest(~method="eth_blockNumber", ~seconds=timerRef->Performance.secondsSince)
         exn->throw
       }
-      let seconds = timerRef->Performance.secondsSince
-      Prometheus.SourceRequestCount.increment(
-        ~sourceName=name,
-        ~chainId=chain->ChainMap.Chain.toChainId,
-        ~method="eth_blockNumber",
-      )
-      Prometheus.SourceRequestCount.addSeconds(
-        ~sourceName=name,
-        ~chainId=chain->ChainMap.Chain.toChainId,
-        ~method="eth_blockNumber",
-        ~seconds,
-      )
-      height
+      recordRequest(~method="eth_blockNumber", ~seconds=timerRef->Performance.secondsSince)
+      {height, requestStats: drainRequestStats()}
     },
     getItemsOrThrow,
     ?createHeightSubscription,