envio 3.0.0-alpha.2 → 3.0.0-alpha.20

package/src/sources/HyperSyncHeightStream.res.mjs

@@ -1,16 +1,24 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "../Utils.res.mjs";
-import * as Js_exn from "rescript/lib/es6/js_exn.js";
 import * as Js_dict from "rescript/lib/es6/js_dict.js";
 import * as Logging from "../Logging.res.mjs";
 import * as Belt_Int from "rescript/lib/es6/belt_Int.js";
-import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as Caml_obj from "rescript/lib/es6/caml_obj.js";
+import * as Prometheus from "../Prometheus.res.mjs";
 import * as Caml_option from "rescript/lib/es6/caml_option.js";
 import * as Eventsource from "eventsource";
 
-function make(hyperSyncUrl, apiToken) {
-  var updateTimeoutId = function (eventsourceRef, timeoutIdRef, hyperSyncUrl, apiToken, heightRef, errorRef) {
+function subscribe(hyperSyncUrl, apiToken, chainId, onHeight) {
+  var eventsourceRef = {
+    contents: undefined
+  };
+  var timeoutIdRef = {
+    contents: setTimeout((function () {
+            
+          }), 0)
+  };
+  var updateTimeoutId = function () {
     clearTimeout(timeoutIdRef.contents);
     var newTimeoutId = setTimeout((function () {
             Logging.trace({
@@ -18,30 +26,33 @@ function make(hyperSyncUrl, apiToken) {
                   url: hyperSyncUrl,
                   staleTimeMillis: 15000
                 });
-            refreshEventSource(eventsourceRef, hyperSyncUrl, apiToken, heightRef, errorRef, timeoutIdRef);
+            refreshEventSource();
           }), 15000);
     timeoutIdRef.contents = newTimeoutId;
   };
-  var refreshEventSource = function (eventsourceRef, hyperSyncUrl, apiToken, heightRef, errorRef, timeoutIdRef) {
+  var refreshEventSource = function () {
     var es = eventsourceRef.contents;
     if (es !== undefined) {
       Caml_option.valFromOption(es).close();
     }
     var userAgent = "hyperindex/" + Utils.EnvioPackage.value.version;
     var es$1 = new Eventsource.EventSource(hyperSyncUrl + "/height/sse", {
-          headers: Js_dict.fromArray([
-                [
-                  "Authorization",
-                  "Bearer " + apiToken
-                ],
-                [
-                  "User-Agent",
-                  userAgent
-                ]
-              ])
+          fetch: (function (url, args) {
+              var newrecord = Caml_obj.obj_dup(args);
+              return fetch(url, (newrecord.headers = Js_dict.fromArray([
+                                [
+                                  "Authorization",
+                                  "Bearer " + apiToken
+                                ],
+                                [
+                                  "User-Agent",
+                                  userAgent
+                                ]
+                              ]), newrecord));
+            })
         });
     eventsourceRef.contents = Caml_option.some(es$1);
-    updateTimeoutId(eventsourceRef, timeoutIdRef, hyperSyncUrl, apiToken, heightRef, errorRef);
+    updateTimeoutId();
     es$1.onopen = (function () {
         Logging.trace({
               msg: "SSE connection opened for height stream",
@@ -53,75 +64,37 @@ function make(hyperSyncUrl, apiToken) {
               msg: "EventSource error",
               error: error.message
             });
-        errorRef.contents = Belt_Option.getWithDefault(error.message, "Unexpected no error.message");
       });
     es$1.addEventListener("ping", (function (_event) {
-            updateTimeoutId(eventsourceRef, timeoutIdRef, hyperSyncUrl, apiToken, heightRef, errorRef);
-            errorRef.contents = undefined;
+            updateTimeoutId();
           }));
     es$1.addEventListener("height", (function ($$event) {
             var height = Belt_Int.fromString($$event.data);
             if (height !== undefined) {
-              updateTimeoutId(eventsourceRef, timeoutIdRef, hyperSyncUrl, apiToken, heightRef, errorRef);
-              errorRef.contents = undefined;
-              heightRef.contents = height;
+              Prometheus.SourceRequestCount.increment("HyperSync", chainId, "heightStream");
+              updateTimeoutId();
+              return onHeight(height);
             } else {
-              Logging.trace({
-                    msg: "Height was not a number in event.data",
-                    data: $$event.data
-                  });
-              errorRef.contents = "Height was not a number in event.data";
+              return Logging.trace({
+                          msg: "Height was not a number in event.data",
+                          data: $$event.data
+                        });
             }
           }));
   };
-  var heightRef = {
-    contents: 0
-  };
-  var errorRef = {
-    contents: undefined
-  };
-  var eventsourceRef = {
-    contents: undefined
-  };
-  var timeoutIdRef = {
-    contents: setTimeout((function () {
-            
-          }), 0)
-  };
-  refreshEventSource(eventsourceRef, hyperSyncUrl, apiToken, heightRef, errorRef, timeoutIdRef);
-  return {
-          heightRef: heightRef,
-          errorRef: errorRef,
-          timeoutIdRef: timeoutIdRef,
-          eventsourceRef: eventsourceRef
-        };
-}
-
-async function getHeight(t) {
-  while(t.heightRef.contents === 0 && t.errorRef.contents === undefined) {
-    await Utils.delay(200);
+  refreshEventSource();
+  return function () {
+    clearTimeout(timeoutIdRef.contents);
+    var es = eventsourceRef.contents;
+    if (es !== undefined) {
+      Caml_option.valFromOption(es).close();
+      return ;
+    }
+    
   };
-  var error = t.errorRef.contents;
-  if (error !== undefined) {
-    return Js_exn.raiseError(error);
-  } else {
-    return t.heightRef.contents;
-  }
-}
-
-function close(t) {
-  clearTimeout(t.timeoutIdRef.contents);
-  var es = t.eventsourceRef.contents;
-  if (es !== undefined) {
-    Caml_option.valFromOption(es).close();
-    return ;
-  }
-  
 }
 
 export {
-  make ,
-  getHeight ,
-  close ,
+  subscribe ,
 }
 /* Utils Not a pure module */

package/src/sources/HyperSyncSource.res

@@ -28,20 +28,28 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
     dependsOnAddresses,
     contractName,
     getEventFiltersOrThrow,
-    blockSchema,
-    transactionSchema,
+    selectedBlockFields,
+    selectedTransactionFields,
     isWildcard,
   }) => {
-    nonOptionalBlockFieldNames->Utils.Set.addMany(
-      blockSchema->Utils.Schema.getNonOptionalFieldNames,
-    )
-    nonOptionalTransactionFieldNames->Utils.Set.addMany(
-      transactionSchema->Utils.Schema.getNonOptionalFieldNames,
-    )
-    capitalizedBlockFields->Utils.Set.addMany(blockSchema->Utils.Schema.getCapitalizedFieldNames)
-    capitalizedTransactionFields->Utils.Set.addMany(
-      transactionSchema->Utils.Schema.getCapitalizedFieldNames,
-    )
+    selectedBlockFields
+    ->Utils.Set.toArray
+    ->Array.forEach(name => {
+      let nameStr = (name :> string)
+      if !(Internal.evmNullableBlockFields->Utils.Set.has(name)) {
+        nonOptionalBlockFieldNames->Utils.Set.add(nameStr)->ignore
+      }
+      capitalizedBlockFields->Utils.Set.add(nameStr->Utils.String.capitalize)->ignore
+    })
+    selectedTransactionFields
+    ->Utils.Set.toArray
+    ->Array.forEach(name => {
+      let nameStr = (name :> string)
+      if !(Internal.evmNullableTransactionFields->Utils.Set.has(name)) {
+        nonOptionalTransactionFieldNames->Utils.Set.add(nameStr)->ignore
+      }
+      capitalizedTransactionFields->Utils.Set.add(nameStr->Utils.String.capitalize)->ignore
+    })
 
     let eventFilters = getEventFiltersOrThrow(chain)
     if dependsOnAddresses {
@@ -131,25 +139,13 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
   }
 }
 
-let memoGetSelectionConfig = (~chain) => {
-  let cache = Utils.WeakMap.make()
-  selection =>
-    switch cache->Utils.WeakMap.get(selection) {
-    | Some(c) => c
-    | None => {
-        let c = selection->getSelectionConfig(~chain)
-        let _ = cache->Utils.WeakMap.set(selection, c)
-        c
-      }
-    }
-}
+let memoGetSelectionConfig = (~chain) =>
+  Utils.WeakMap.memoize(selection => selection->getSelectionConfig(~chain))
 
 type options = {
-  contracts: array<Internal.evmContractConfig>,
   chain: ChainMap.Chain.t,
   endpointUrl: string,
   allEventSignatures: array<string>,
-  shouldUseHypersyncClientDecoder: bool,
   eventRouter: EventRouter.t<Internal.evmEventConfig>,
   apiToken: option<string>,
   clientMaxRetries: int,
@@ -161,11 +157,9 @@ type options = {
 
 let make = (
   {
-    contracts,
     chain,
     endpointUrl,
     allEventSignatures,
-    shouldUseHypersyncClientDecoder,
     eventRouter,
     apiToken,
     clientMaxRetries,
@@ -179,7 +173,13 @@ let make = (
 
   let getSelectionConfig = memoGetSelectionConfig(~chain)
 
-  let apiToken = apiToken->Belt.Option.getWithDefault("3dc856dd-b0ea-494f-b27e-017b8b6b7e07")
+  let apiToken = switch apiToken {
+  | Some(token) => token
+  | None =>
+    Js.Exn.raiseError(`An API token is required for using HyperSync as a data-source.
+Set the ENVIO_API_TOKEN environment variable in your .env file.
+Learn more or get a free API token at: https://envio.dev/app/api-tokens`)
+  }
 
   let client = HyperSyncClient.make(
     ~url=endpointUrl,
@@ -199,7 +199,7 @@ let make = (
       switch HyperSyncClient.Decoder.fromSignatures(allEventSignatures) {
       | exception exn =>
         exn->ErrorHandling.mkLogAndRaise(
-          ~msg="Failed to instantiate a decoder from hypersync client, please double check your ABI or try using 'event_decoder: viem' config option",
+          ~msg="Failed to instantiate a decoder from hypersync client, please double check your ABI",
         )
       | decoder =>
         if lowercaseAddresses {
@@ -239,17 +239,12 @@ let make = (
     })
   }
 
-  let contractNameAbiMapping = Js.Dict.empty()
-  contracts->Belt.Array.forEach(contract => {
-    contractNameAbiMapping->Js.Dict.set(contract.name, contract.abi)
-  })
-
   let getItemsOrThrow = async (
     ~fromBlock,
     ~toBlock,
     ~addressesByContractName,
     ~indexingContracts,
-    ~currentBlockHeight,
+    ~knownHeight,
     ~partitionId as _,
     ~selection,
     ~retry,
@@ -268,6 +263,11 @@ let make = (
     let startFetchingBatchTimeRef = Hrtime.makeTimer()
 
     //fetch batch
+    Prometheus.SourceRequestCount.increment(
+      ~sourceName=name,
+      ~chainId=chain->ChainMap.Chain.toChainId,
+      ~method="getLogs",
+    )
     let pageUnsafe = try await HyperSync.GetLogs.query(
       ~client,
       ~fromBlock,
@@ -282,7 +282,7 @@ let make = (
         Source.GetItemsError(
           Source.FailedGettingItems({
             exn: %raw(`null`),
-            attemptedToBlock: toBlock->Option.getWithDefault(currentBlockHeight),
+            attemptedToBlock: toBlock->Option.getWithDefault(knownHeight),
             retry: switch error {
             | WrongInstance =>
               let backoffMillis = switch retry {
@@ -308,7 +308,7 @@ let make = (
         Source.GetItemsError(
           Source.FailedGettingItems({
             exn,
-            attemptedToBlock: toBlock->Option.getWithDefault(currentBlockHeight),
+            attemptedToBlock: toBlock->Option.getWithDefault(knownHeight),
             retry: WithBackoff({
               message: `Unexpected issue while fetching events from HyperSync client. Attempt a retry.`,
               backoffMillis: switch retry {
@@ -321,11 +321,10 @@ let make = (
       )
     }
 
-    let pageFetchTime =
-      startFetchingBatchTimeRef->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis
+    let pageFetchTime = startFetchingBatchTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
 
     //set height and next from block
-    let currentBlockHeight = pageUnsafe.archiveHeight
+    let knownHeight = pageUnsafe.archiveHeight
 
     //The heighest (biggest) blocknumber that was accounted for in
     //Our query. Not necessarily the blocknumber of the last log returned
@@ -369,6 +368,8 @@ let make = (
           ~serverUrl=endpointUrl,
           ~apiToken,
           ~blockNumber=heighestBlockQueried,
+          ~sourceName=name,
+          ~chainId=chain->ChainMap.Chain.toChainId,
           ~logger,
         )->Promise.thenResolve(res =>
           switch res {
@@ -396,7 +397,6 @@ let make = (
 
     let handleDecodeFailure = (
       ~eventConfig: Internal.evmEventConfig,
-      ~decoder,
       ~logIndex,
       ~blockNumber,
       ~chainId,
@@ -413,72 +413,28 @@ let make = (
             "chainId": chainId,
             "blockNumber": blockNumber,
             "logIndex": logIndex,
-            "decoder": decoder,
+            "decoder": "hypersync-client",
           },
         )
         exn->ErrorHandling.mkLogAndRaise(~msg, ~logger)
       }
     }
-    if shouldUseHypersyncClientDecoder {
-      //Currently there are still issues with decoder for some cases so
-      //this can only be activated with a flag
-
-      //Parse page items into queue items
-      let parsedEvents = switch await getHscDecoder().decodeEvents(pageUnsafe.events) {
-      | exception exn =>
-        exn->mkLogAndRaise(
-          ~msg="Failed to parse events using hypersync client, please double check your ABI.",
-        )
-      | parsedEvents => parsedEvents
-      }
 
-      pageUnsafe.items->Belt.Array.forEachWithIndex((index, item) => {
-        let {block, log} = item
-        let chainId = chain->ChainMap.Chain.toChainId
-        let topic0 = log.topics->Js.Array2.unsafe_get(0)
-        let maybeEventConfig =
-          eventRouter->EventRouter.get(
-            ~tag=EventRouter.getEvmEventId(
-              ~sighash=topic0->EvmTypes.Hex.toString,
-              ~topicCount=log.topics->Array.length,
-            ),
-            ~indexingContracts,
-            ~contractAddress=log.address,
-            ~blockNumber=block.number->Belt.Option.getUnsafe,
-          )
-        let maybeDecodedEvent = parsedEvents->Js.Array2.unsafe_get(index)
-
-        switch (maybeEventConfig, maybeDecodedEvent) {
-        | (Some(eventConfig), Value(decoded)) =>
-          parsedQueueItems
-          ->Js.Array2.push(
-            makeEventBatchQueueItem(
-              item,
-              ~params=decoded->eventConfig.convertHyperSyncEventArgs,
-              ~eventConfig,
-            ),
-          )
-          ->ignore
-        | (Some(eventConfig), Null | Undefined) =>
-          handleDecodeFailure(
-            ~eventConfig,
-            ~decoder="hypersync-client",
-            ~logIndex=log.logIndex,
-            ~blockNumber=block.number->Belt.Option.getUnsafe,
-            ~chainId,
-            ~exn=UndefinedValue,
-          )
-        | (None, _) => () //ignore events that aren't registered
-        }
-      })
-    } else {
-      //Parse with viem -> slower than the HyperSyncClient
-      pageUnsafe.items->Array.forEach(item => {
-        let {block, log} = item
-        let chainId = chain->ChainMap.Chain.toChainId
-        let topic0 = log.topics->Js.Array2.unsafe_get(0)
+    //Parse page items into queue items
+    let parsedEvents = switch await getHscDecoder().decodeEvents(pageUnsafe.events) {
+    | exception exn =>
+      exn->mkLogAndRaise(
+        ~msg="Failed to parse events using hypersync client, please double check your ABI.",
+      )
+    | parsedEvents => parsedEvents
+    }
 
-        switch eventRouter->EventRouter.get(
+    pageUnsafe.items->Belt.Array.forEachWithIndex((index, item) => {
+      let {block, log} = item
+      let chainId = chain->ChainMap.Chain.toChainId
+      let topic0 = log.topics->Utils.Array.firstUnsafe
+      let maybeEventConfig =
+        eventRouter->EventRouter.get(
           ~tag=EventRouter.getEvmEventId(
             ~sighash=topic0->EvmTypes.Hex.toString,
             ~topicCount=log.topics->Array.length,
@@ -486,33 +442,33 @@ let make = (
           ~indexingContracts,
           ~contractAddress=log.address,
           ~blockNumber=block.number->Belt.Option.getUnsafe,
-        ) {
-        | Some(eventConfig) =>
-          switch contractNameAbiMapping->Viem.parseLogOrThrow(
-            ~contractName=eventConfig.contractName,
-            ~topics=log.topics,
-            ~data=log.data,
-          ) {
-          | exception exn =>
-            handleDecodeFailure(
-              ~eventConfig,
-              ~decoder="viem",
-              ~logIndex=log.logIndex,
-              ~blockNumber=block.number->Belt.Option.getUnsafe,
-              ~chainId,
-              ~exn,
-            )
-          | decodedEvent =>
-            parsedQueueItems
-            ->Js.Array2.push(makeEventBatchQueueItem(item, ~params=decodedEvent.args, ~eventConfig))
-            ->ignore
-          }
-        | None => () //Ignore events that aren't registered
-        }
-      })
-    }
+        )
+      let maybeDecodedEvent = parsedEvents->Js.Array2.unsafe_get(index)
+
+      switch (maybeEventConfig, maybeDecodedEvent) {
+      | (Some(eventConfig), Value(decoded)) =>
+        parsedQueueItems
+        ->Js.Array2.push(
+          makeEventBatchQueueItem(
+            item,
+            ~params=decoded->eventConfig.convertHyperSyncEventArgs,
+            ~eventConfig,
+          ),
+        )
+        ->ignore
+      | (Some(eventConfig), Null | Undefined) =>
+        handleDecodeFailure(
+          ~eventConfig,
+          ~logIndex=log.logIndex,
+          ~blockNumber=block.number->Belt.Option.getUnsafe,
+          ~chainId,
+          ~exn=UndefinedValue,
+        )
+      | (None, _) => () //ignore events that aren't registered
+      }
+    })
 
-    let parsingTimeElapsed = parsingTimeRef->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis
+    let parsingTimeElapsed = parsingTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
 
     let rangeLastBlock = await lastBlockQueriedPromise
 
@@ -524,7 +480,7 @@ let make = (
       }),
     }
 
-    let totalTimeElapsed = totalTimeRef->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis
+    let totalTimeElapsed = totalTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
 
     let stats = {
       totalTimeElapsed,
@@ -537,7 +493,7 @@ let make = (
       parsedQueueItems,
       latestFetchedBlockNumber: rangeLastBlock.blockNumber,
       stats,
-      currentBlockHeight,
+      knownHeight,
       reorgGuard,
       fromBlockQueried: fromBlock,
     }
@@ -548,35 +504,58 @@ let make = (
       ~serverUrl=endpointUrl,
       ~apiToken,
       ~blockNumbers,
+      ~sourceName=name,
+      ~chainId=chain->ChainMap.Chain.toChainId,
       ~logger,
     )->Promise.thenResolve(HyperSync.mapExn)
 
-  let malformedTokenMessage = `Your token is malformed. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens.`
+  let jsonApiClient = Rest.client(endpointUrl)
 
-  let heightStream = HyperSyncHeightStream.make(~hyperSyncUrl=endpointUrl, ~apiToken)
+  let malformedTokenMessage = `Your token is malformed. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens.`
 
   {
     name,
     sourceFor: Sync,
     chain,
-    pollingInterval: 32, // purely internal state polling since we now stream the height to the state
+    pollingInterval: 100,
     poweredByHyperSync: true,
     getBlockHashes,
     getHeightOrThrow: async () => {
-      try await heightStream->HyperSyncHeightStream.getHeight catch {
-      | Js.Exn.Error(exn)
-        if exn
-        ->Js.Exn.message
-        ->Option.getWithDefault("")
-        ->Js.String2.includes(malformedTokenMessage) =>
+      let timerRef = Hrtime.makeTimer()
+      let result = switch await HyperSyncJsonApi.heightRoute->Rest.fetch(
+        apiToken,
+        ~client=jsonApiClient,
+      ) {
+      | Value(height) => height
+      | ErrorMessage(m) if m === malformedTokenMessage =>
         Logging.error(`Your ENVIO_API_TOKEN is malformed. The indexer will not be able to fetch events. Update the token and restart the indexer using 'pnpm envio start'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`)
         // Don't want to retry if the token is malformed
         // So just block forever
         let _ = await Promise.make((_, _) => ())
         0
-      | exn => raise(exn)
+      | ErrorMessage(m) => Js.Exn.raiseError(m)
       }
+      let seconds = timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+      Prometheus.SourceRequestCount.increment(
+        ~sourceName=name,
+        ~chainId=chain->ChainMap.Chain.toChainId,
+        ~method="getHeight",
+      )
+      Prometheus.SourceRequestCount.addSeconds(
+        ~sourceName=name,
+        ~chainId=chain->ChainMap.Chain.toChainId,
+        ~method="getHeight",
+        ~seconds,
+      )
+      result
     },
     getItemsOrThrow,
+    createHeightSubscription: (~onHeight) =>
+      HyperSyncHeightStream.subscribe(
+        ~hyperSyncUrl=endpointUrl,
+        ~apiToken,
+        ~chainId=chain->ChainMap.Chain.toChainId,
+        ~onHeight,
+      ),
   }
 }