envio 3.0.0-alpha.2 → 3.0.0-alpha.20

package/src/sources/HyperFuelSource.res.mjs

@@ -9,6 +9,7 @@ import * as $$Promise from "../bindings/Promise.res.mjs";
 import * as HyperFuel from "./HyperFuel.res.mjs";
 import * as HyperSync from "./HyperSync.res.mjs";
 import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as Prometheus from "../Prometheus.res.mjs";
 import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
 import * as EventRouter from "./EventRouter.res.mjs";
 import * as ErrorHandling from "../ErrorHandling.res.mjs";
@@ -185,8 +186,9 @@ function memoGetSelectionConfig(chain) {
 function make(param) {
   var endpointUrl = param.endpointUrl;
   var chain = param.chain;
+  var name = "HyperFuel";
   var getSelectionConfig = memoGetSelectionConfig(chain);
-  var getItemsOrThrow = async function (fromBlock, toBlock, addressesByContractName, indexingContracts, currentBlockHeight, param, selection, retry, logger) {
+  var getItemsOrThrow = async function (fromBlock, toBlock, addressesByContractName, indexingContracts, knownHeight, param, selection, retry, logger) {
     var mkLogAndRaise = function (extra, extra$1) {
       return ErrorHandling.mkLogAndRaise(logger, extra, extra$1);
     };
@@ -194,6 +196,7 @@ function make(param) {
     var selectionConfig = getSelectionConfig(selection);
     var recieptsSelection = selectionConfig.getRecieptsSelection(addressesByContractName);
     var startFetchingBatchTimeRef = Hrtime.makeTimer();
+    Prometheus.SourceRequestCount.increment(name, chain, "getLogs");
     var pageUnsafe;
     try {
       pageUnsafe = await HyperFuel.GetLogs.query(endpointUrl, fromBlock, toBlock, recieptsSelection);
@@ -221,7 +224,7 @@ function make(param) {
               _1: {
                 TAG: "FailedGettingItems",
                 exn: null,
-                attemptedToBlock: Belt_Option.getWithDefault(toBlock, currentBlockHeight),
+                attemptedToBlock: Belt_Option.getWithDefault(toBlock, knownHeight),
                 retry: tmp
               },
               Error: new Error()
@@ -232,7 +235,7 @@ function make(param) {
             _1: {
               TAG: "FailedGettingItems",
               exn: error,
-              attemptedToBlock: Belt_Option.getWithDefault(toBlock, currentBlockHeight),
+              attemptedToBlock: Belt_Option.getWithDefault(toBlock, knownHeight),
               retry: {
                 TAG: "WithBackoff",
                 message: "Unexpected issue while fetching events from HyperFuel client. Attempt a retry.",
@@ -242,8 +245,8 @@ function make(param) {
             Error: new Error()
           };
     }
-    var pageFetchTime = Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(startFetchingBatchTimeRef)));
-    var currentBlockHeight$1 = pageUnsafe.archiveHeight;
+    var pageFetchTime = Hrtime.toSecondsFloat(Hrtime.timeSince(startFetchingBatchTimeRef));
+    var knownHeight$1 = pageUnsafe.archiveHeight;
     var heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;
     var match = Belt_Array.get(pageUnsafe.items, pageUnsafe.items.length - 1 | 0);
     var lastBlockQueriedPromise;
@@ -392,22 +395,22 @@ function make(param) {
                     }
                   };
           }));
-    var parsingTimeElapsed = Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(parsingTimeRef)));
+    var parsingTimeElapsed = Hrtime.toSecondsFloat(Hrtime.timeSince(parsingTimeRef));
     var rangeLastBlock = await lastBlockQueriedPromise;
     var reorgGuard = {
       rangeLastBlock: rangeLastBlock,
       prevRangeLastBlock: undefined
     };
-    var totalTimeElapsed = Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(totalTimeRef)));
-    var stats_parsing$unknowntime$unknown$lparms$rpar = parsingTimeElapsed;
-    var stats_page$unknownfetch$unknowntime$unknown$lparms$rpar = pageFetchTime;
+    var totalTimeElapsed = Hrtime.toSecondsFloat(Hrtime.timeSince(totalTimeRef));
+    var stats_parsing$unknowntime$unknown$lpars$rpar = parsingTimeElapsed;
+    var stats_page$unknownfetch$unknowntime$unknown$lpars$rpar = pageFetchTime;
     var stats = {
-      "total time elapsed (ms)": totalTimeElapsed,
-      "parsing time (ms)": stats_parsing$unknowntime$unknown$lparms$rpar,
-      "page fetch time (ms)": stats_page$unknownfetch$unknowntime$unknown$lparms$rpar
+      "total time elapsed (s)": totalTimeElapsed,
+      "parsing time (s)": stats_parsing$unknowntime$unknown$lpars$rpar,
+      "page fetch time (s)": stats_page$unknownfetch$unknowntime$unknown$lpars$rpar
     };
     return {
-            currentBlockHeight: currentBlockHeight$1,
+            knownHeight: knownHeight$1,
             reorgGuard: reorgGuard,
             parsedQueueItems: parsedQueueItems,
             fromBlockQueried: fromBlock,
@@ -421,14 +424,19 @@ function make(param) {
   };
   var jsonApiClient = Rest.client(endpointUrl, undefined);
   return {
-          name: "HyperFuel",
+          name: name,
           sourceFor: "Sync",
           chain: chain,
           poweredByHyperSync: true,
           pollingInterval: 100,
           getBlockHashes: getBlockHashes,
-          getHeightOrThrow: (function () {
-              return Rest.$$fetch(HyperFuel.heightRoute, undefined, jsonApiClient);
+          getHeightOrThrow: (async function () {
+              var timerRef = Hrtime.makeTimer();
+              var height = await Rest.$$fetch(HyperFuel.heightRoute, undefined, jsonApiClient);
+              var seconds = Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef));
+              Prometheus.SourceRequestCount.increment(name, chain, "getHeight");
+              Prometheus.SourceRequestCount.addSeconds(name, chain, "getHeight", seconds);
+              return height;
             }),
           getItemsOrThrow: getItemsOrThrow
         };

package/src/sources/HyperSync.res

@@ -114,7 +114,7 @@ module GetLogs = {
     }
 
     //Topics can be nullable and still need to be filtered
-    let logUnsanitized: Log.t = event.log->Utils.magic
+    let logUnsanitized: Log.t = event.log->(Utils.magic: HyperSyncClient.ResponseTypes.log => Log.t)
     let topics = event.log.topics->Option.getUnsafe->Array.keepMap(Js.Nullable.toOption)
     let address = event.log.address->Option.getUnsafe
     let log = {
@@ -125,8 +125,14 @@ module GetLogs = {
 
     {
       log,
-      block: event.block->Utils.magic,
-      transaction: event.transaction->Utils.magic,
+      block: event.block->(
+        Utils.magic: option<
+          HyperSyncClient.ResponseTypes.block,
+        > => HyperSyncClient.ResponseTypes.block
+      ),
+      transaction: event.transaction->(
+        Utils.magic: option<HyperSyncClient.ResponseTypes.transaction> => Internal.eventTransaction
+      ),
     }
   }
 
@@ -245,6 +251,8 @@ module BlockData = {
     ~apiToken,
     ~fromBlock,
     ~toBlock,
+    ~sourceName,
+    ~chainId,
     ~logger,
   ): queryResponse<array<ReorgDetection.blockDataWithTimestamp>> => {
     let body = makeRequestBody(~fromBlock, ~toBlock)
@@ -258,6 +266,7 @@ module BlockData = {
       },
     )
 
+    Prometheus.SourceRequestCount.increment(~sourceName, ~chainId, ~method="getBlockHashes")
     let maybeSuccessfulRes = switch await Time.retryAsyncWithExponentialBackOff(() =>
       HyperSyncJsonApi.queryRoute->Rest.fetch(
         {
@@ -281,7 +290,15 @@ module BlockData = {
           `Block #${fromBlock->Int.toString} not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${delayMilliseconds->Int.toString}ms.`,
         )
         await Time.resolvePromiseAfterDelay(~delayMilliseconds)
-        await queryBlockData(~serverUrl, ~apiToken, ~fromBlock, ~toBlock, ~logger)
+        await queryBlockData(
+          ~serverUrl,
+          ~apiToken,
+          ~fromBlock,
+          ~toBlock,
+          ~sourceName,
+          ~chainId,
+          ~logger,
+        )
       }
     | Some(res) =>
       switch res->convertResponse {
@@ -292,6 +309,8 @@ module BlockData = {
             ~apiToken,
             ~fromBlock=res.nextBlock,
             ~toBlock,
+            ~sourceName,
+            ~chainId,
             ~logger,
           )
           restRes->Result.map(rest => datas->Array.concat(rest))
@@ -301,7 +320,14 @@ module BlockData = {
     }
   }
 
-  let queryBlockDataMulti = async (~serverUrl, ~apiToken, ~blockNumbers, ~logger) => {
+  let queryBlockDataMulti = async (
+    ~serverUrl,
+    ~apiToken,
+    ~blockNumbers,
+    ~sourceName,
+    ~chainId,
+    ~logger,
+  ) => {
     switch blockNumbers->Array.get(0) {
     | None => Ok([])
     | Some(firstBlock) => {
@@ -328,6 +354,8 @@ module BlockData = {
           ~toBlock=toBlock.contents,
           ~serverUrl,
           ~apiToken,
+          ~sourceName,
+          ~chainId,
           ~logger,
         )
         let filtered = res->Result.map(datas => {
@@ -346,12 +374,14 @@ module BlockData = {
   }
 }
 
-let queryBlockData = (~serverUrl, ~apiToken, ~blockNumber, ~logger) =>
+let queryBlockData = (~serverUrl, ~apiToken, ~blockNumber, ~sourceName, ~chainId, ~logger) =>
   BlockData.queryBlockData(
     ~serverUrl,
     ~apiToken,
     ~fromBlock=blockNumber,
     ~toBlock=blockNumber,
+    ~sourceName,
+    ~chainId,
     ~logger,
   )->Promise.thenResolve(res => res->Result.map(res => res->Array.get(0)))
 let queryBlockDataMulti = BlockData.queryBlockDataMulti

package/src/sources/HyperSync.res.mjs

@@ -7,6 +7,7 @@ import * as $$BigInt from "../bindings/BigInt.res.mjs";
 import * as Js_exn from "rescript/lib/es6/js_exn.js";
 import * as Logging from "../Logging.res.mjs";
 import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as Prometheus from "../Prometheus.res.mjs";
 import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
 import * as Belt_Result from "rescript/lib/es6/belt_Result.js";
 import * as Caml_option from "rescript/lib/es6/caml_option.js";
@@ -226,13 +227,14 @@ function convertResponse(res) {
                   })));
 }
 
-async function queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger) {
+async function queryBlockData(serverUrl, apiToken, fromBlock, toBlock, sourceName, chainId, logger) {
   var body = makeRequestBody$1(fromBlock, toBlock);
   var logger$1 = Logging.createChildFrom(logger, {
         logType: "HyperSync get block hash query",
         fromBlock: fromBlock,
         toBlock: toBlock
       });
+  Prometheus.SourceRequestCount.increment(sourceName, chainId, "getBlockHashes");
   var maybeSuccessfulRes;
   var exit = 0;
   var res;
@@ -260,7 +262,7 @@ async function queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger) {
       return err;
     }
     var datas = err._0;
-    var restRes = await queryBlockData(serverUrl, apiToken, maybeSuccessfulRes.nextBlock, toBlock, logger$1);
+    var restRes = await queryBlockData(serverUrl, apiToken, maybeSuccessfulRes.nextBlock, toBlock, sourceName, chainId, logger$1);
     return Belt_Result.map(restRes, (function (rest) {
                   return Belt_Array.concat(datas, rest);
                 }));
@@ -270,10 +272,10 @@ async function queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger) {
       });
   Logging.childInfo(logger$2, "Block #" + String(fromBlock) + " not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in " + String(100) + "ms.");
   await Time.resolvePromiseAfterDelay(100);
-  return await queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger$2);
+  return await queryBlockData(serverUrl, apiToken, fromBlock, toBlock, sourceName, chainId, logger$2);
 }
 
-async function queryBlockDataMulti(serverUrl, apiToken, blockNumbers, logger) {
+async function queryBlockDataMulti(serverUrl, apiToken, blockNumbers, sourceName, chainId, logger) {
   var firstBlock = Belt_Array.get(blockNumbers, 0);
   if (firstBlock === undefined) {
     return {
@@ -297,7 +299,7 @@ async function queryBlockDataMulti(serverUrl, apiToken, blockNumbers, logger) {
   if ((toBlock - fromBlock | 0) > 1000) {
     Js_exn.raiseError("Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: " + String(fromBlock) + "-" + String(toBlock));
   }
-  var res = await queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger);
+  var res = await queryBlockData(serverUrl, apiToken, fromBlock, toBlock, sourceName, chainId, logger);
   var filtered = Belt_Result.map(res, (function (datas) {
           return Belt_Array.keep(datas, (function (data) {
                         return set.delete(data.blockNumber);
@@ -309,8 +311,8 @@ async function queryBlockDataMulti(serverUrl, apiToken, blockNumbers, logger) {
   return filtered;
 }
 
-function queryBlockData$1(serverUrl, apiToken, blockNumber, logger) {
-  return queryBlockData(serverUrl, apiToken, blockNumber, blockNumber, logger).then(function (res) {
+function queryBlockData$1(serverUrl, apiToken, blockNumber, sourceName, chainId, logger) {
+  return queryBlockData(serverUrl, apiToken, blockNumber, blockNumber, sourceName, chainId, logger).then(function (res) {
               return Belt_Result.map(res, (function (res) {
                             return Belt_Array.get(res, 0);
                           }));

package/src/sources/HyperSync.resi

@@ -56,6 +56,8 @@ let queryBlockData: (
   ~serverUrl: string,
   ~apiToken: string,
   ~blockNumber: int,
+  ~sourceName: string,
+  ~chainId: int,
   ~logger: Pino.t,
 ) => promise<queryResponse<option<ReorgDetection.blockDataWithTimestamp>>>
 
@@ -63,6 +65,8 @@ let queryBlockDataMulti: (
   ~serverUrl: string,
   ~apiToken: string,
   ~blockNumbers: array<int>,
+  ~sourceName: string,
+  ~chainId: int,
   ~logger: Pino.t,
 ) => promise<queryResponse<array<ReorgDetection.blockDataWithTimestamp>>>
 

package/src/sources/HyperSyncClient.res

@@ -390,7 +390,7 @@ module ResponseTypes = {
     l1Fee?: bigint,
     l1GasPrice?: bigint,
     l1GasUsed?: bigint,
-    l1FeeScalar?: int,
+    l1FeeScalar?: float,
     gasUsedForL1?: bigint,
     authorizationList?: array<authorizationList>,
   }

package/src/sources/HyperSyncHeightStream.res

@@ -1,28 +1,15 @@
 /*
-Pure js implementation of the HyperSync height stream.
+Pure subscription-based implementation of the HyperSync height stream.
 */
 
-type t = {
-  heightRef: ref<int>,
-  errorRef: ref<option<string>>,
-  timeoutIdRef: ref<Js.Global.timeoutId>,
-  eventsourceRef: ref<option<EventSource.t>>,
-}
-
-let make = (~hyperSyncUrl, ~apiToken) => {
-  /**
-  On every successful ping or height event, clear the timeout and set a new one.
+let subscribe = (~hyperSyncUrl, ~apiToken, ~chainId, ~onHeight: int => unit): (unit => unit) => {
+  let eventsourceRef = ref(None)
+  // Timeout doesn't do anything for initialization
+  let timeoutIdRef = ref(Js.Global.setTimeout(() => (), 0))
 
-  if the timeout lapses, close and reconnect the EventSource.
-  */
-  let rec updateTimeoutId = (
-    ~eventsourceRef: ref<option<EventSource.t>>,
-    ~timeoutIdRef: ref<Js.Global.timeoutId>,
-    ~hyperSyncUrl,
-    ~apiToken,
-    ~heightRef: ref<int>,
-    ~errorRef: ref<option<string>>,
-  ) => {
+  // On every successful ping or height event, clear the timeout and set a new one.
+  // If the timeout lapses, close and reconnect the EventSource.
+  let rec updateTimeoutId = () => {
     timeoutIdRef.contents->Js.Global.clearTimeout
 
     // Should receive a ping at least every 5s, so 15s is a safe margin
@@ -34,31 +21,15 @@ let make = (~hyperSyncUrl, ~apiToken) => {
         "url": hyperSyncUrl,
         "staleTimeMillis": staleTimeMillis,
       })
-      refreshEventSource(
-        ~eventsourceRef,
-        ~hyperSyncUrl,
-        ~apiToken,
-        ~heightRef,
-        ~errorRef,
-        ~timeoutIdRef,
-      )
+      refreshEventSource()
     }, staleTimeMillis)
 
     timeoutIdRef := newTimeoutId
   }
-  and /**
-    Instantiate a new EventSource and set it to the shared refs.
-    Add the necessary event listeners, handle errors
-    and update the timeout.
-    */
-  refreshEventSource = (
-    ~eventsourceRef: ref<option<EventSource.t>>,
-    ~hyperSyncUrl,
-    ~apiToken,
-    ~heightRef: ref<int>,
-    ~errorRef: ref<option<string>>,
-    ~timeoutIdRef: ref<Js.Global.timeoutId>,
-  ) => {
+  // Instantiate a new EventSource and set it to the shared refs.
+  // Add the necessary event listeners, handle errors
+  // and update the timeout.
+  and refreshEventSource = () => {
     // Close the old EventSource if it exists (on a new connection after timeout)
     switch eventsourceRef.contents {
     | Some(es) => es->EventSource.close
@@ -69,17 +40,25 @@ let make = (~hyperSyncUrl, ~apiToken) => {
     let es = EventSource.create(
       ~url=`${hyperSyncUrl}/height/sse`,
       ~options={
-        headers: Js.Dict.fromArray([
-          ("Authorization", `Bearer ${apiToken}`),
-          ("User-Agent", userAgent),
-        ]),
+        fetch: (url, ~args) => {
+          EventSource.Fetch.fetch(
+            url,
+            ~args={
+              ...args,
+              headers: Js.Dict.fromArray([
+                ("Authorization", `Bearer ${apiToken}`),
+                ("User-Agent", userAgent),
+              ]),
+            },
+          )
+        },
       },
     )
 
     // Set the new EventSource to the shared ref
     eventsourceRef := Some(es)
     // Update the timeout in case connection goes stale
-    updateTimeoutId(~eventsourceRef, ~timeoutIdRef, ~hyperSyncUrl, ~apiToken, ~heightRef, ~errorRef)
+    updateTimeoutId()
 
     es->EventSource.onopen(_ => {
       Logging.trace({"msg": "SSE connection opened for height stream", "url": hyperSyncUrl})
@@ -90,90 +69,42 @@ let make = (~hyperSyncUrl, ~apiToken) => {
         "msg": "EventSource error",
         "error": error->Js.Exn.message,
       })
-      // On errors, set the error ref
-      // so that getHeight can raise an error
-      errorRef :=
-        Some(error->Js.Exn.message->Belt.Option.getWithDefault("Unexpected no error.message"))
     })
 
     es->EventSource.addEventListener("ping", _event => {
       // ping lets us know from the server that the connection is still alive
-      // and that the height hasn't updated for 5seconds
+      // and that the height hasn't updated for 5 seconds
       // update the timeout on each successful ping received
-      updateTimeoutId(
-        ~eventsourceRef,
-        ~timeoutIdRef,
-        ~hyperSyncUrl,
-        ~apiToken,
-        ~heightRef,
-        ~errorRef,
-      )
-      // reset the error ref, since we had a successful ping
-      errorRef := None
+      updateTimeoutId()
     })
 
     es->EventSource.addEventListener("height", event => {
       switch event.data->Belt.Int.fromString {
       | Some(height) =>
-        // On a successful height event, update the timeout
-        // and reset the error ref
-        updateTimeoutId(
-          ~eventsourceRef,
-          ~timeoutIdRef,
-          ~hyperSyncUrl,
-          ~apiToken,
-          ~heightRef,
-          ~errorRef,
+        // Track the SSE height event
+        Prometheus.SourceRequestCount.increment(
+          ~sourceName="HyperSync",
+          ~chainId,
+          ~method="heightStream",
         )
-        errorRef := None
-        // Set the actual height ref
-        heightRef := height
-      | None =>
-        Logging.trace({"msg": "Height was not a number in event.data", "data": event.data})
-        errorRef := Some("Height was not a number in event.data")
+        // On a successful height event, update the timeout
+        updateTimeoutId()
+        // Call the callback with the new height
+        onHeight(height)
+      | None => Logging.trace({"msg": "Height was not a number in event.data", "data": event.data})
       }
     })
   }
 
-  // Refs used between the functions
+  // Start the EventSource connection
+  refreshEventSource()
 
-  let heightRef = ref(0)
-  let errorRef = ref(None)
-  let eventsourceRef = ref(None)
-  // Timeout doesn't do anything for initalization
-  let timeoutIdRef = ref(Js.Global.setTimeout(() => (), 0))
-  refreshEventSource(
-    ~eventsourceRef,
-    ~hyperSyncUrl,
-    ~apiToken,
-    ~heightRef,
-    ~errorRef,
-    ~timeoutIdRef,
-  )
-
-  {
-    heightRef,
-    errorRef,
-    timeoutIdRef,
-    eventsourceRef,
-  }
-}
-
-let getHeight = async (t: t) => {
-  while t.heightRef.contents == 0 && t.errorRef.contents == None {
-    // Poll internally until height is over 0
-    await Utils.delay(200)
-  }
-  switch t.errorRef.contents {
-  | None => t.heightRef.contents
-  | Some(error) => Js.Exn.raiseError(error)
-  }
-}
-
-let close = t => {
-  t.timeoutIdRef.contents->Js.Global.clearTimeout
-  switch t.eventsourceRef.contents {
-  | Some(es) => es->EventSource.close
-  | None => ()
+  // Return unsubscribe function
+  () => {
+    timeoutIdRef.contents->Js.Global.clearTimeout
+    switch eventsourceRef.contents {
+    | Some(es) => es->EventSource.close
+    | None => ()
+    }
   }
 }