envio 3.0.0-alpha.2 → 3.0.0-alpha.21

package/src/sources/RpcWebSocketHeightStream.res

@@ -0,0 +1,181 @@
+/*
+WebSocket-based implementation for real-time block height tracking.
+Uses eth_subscribe("newHeads") for low-latency block detection.
+Falls back behavior is handled by SourceManager when subscription fails.
+*/
+
+let retryCount = 9
+let baseDuration = 125
+// Close and reconnect if no new block head arrives within this period.
+// Detects silently dropped server-side subscriptions.
+let staleTimeMillis = 60_000
+
+type wsMessage =
+  | NewHead(int)
+  | SubscriptionConfirmed(string)
+  | ErrorResponse
+
+let subscribeRequestJson =
+  {"jsonrpc": "2.0", "id": 1, "method": "eth_subscribe", "params": ["newHeads"]}
+  ->(Utils.magic: {
+    "jsonrpc": string,
+    "id": int,
+    "method": string,
+    "params": array<string>,
+  } => Js.Json.t)
+  ->Js.Json.serializeExn
+
+let wsMessageSchema = S.union([
+  S.object(s => {
+    let _ = s.field("method", S.literal("eth_subscription"))
+    NewHead(
+      s.field(
+        "params",
+        S.object(s => {
+          s.field(
+            "result",
+            S.object(
+              s => {
+                s.field("number", Rpc.hexIntSchema)
+              },
+            ),
+          )
+        }),
+      ),
+    )
+  }),
+  S.object(s => {
+    SubscriptionConfirmed(s.field("result", S.string))
+  }),
+  S.object(s => {
+    let _ = s.field("error", S.unknown)
+    ErrorResponse
+  }),
+])
+
+let subscribe = (~wsUrl, ~chainId, ~onHeight: int => unit): (unit => unit) => {
+  let wsRef: ref<option<WebSocket.t>> = ref(None)
+  let isUnsubscribed = ref(false)
+  let errorCount = ref(0)
+  let staleTimeoutId: ref<option<Js.Global.timeoutId>> = ref(None)
+
+  let clearStaleTimeout = () => {
+    switch staleTimeoutId.contents {
+    | Some(id) =>
+      Js.Global.clearTimeout(id)
+      staleTimeoutId := None
+    | None => ()
+    }
+  }
+
+  let resetStaleTimeout = () => {
+    clearStaleTimeout()
+    staleTimeoutId := Some(Js.Global.setTimeout(() => {
+          // Connection went stale - close to trigger reconnect
+          switch wsRef.contents {
+          | Some(ws) => ws->WebSocket.close
+          | None => ()
+          }
+        }, staleTimeMillis))
+  }
+
+  let rec scheduleReconnect = () => {
+    if !isUnsubscribed.contents && errorCount.contents < retryCount {
+      let duration =
+        baseDuration *
+        Js.Math.pow_float(~base=2.0, ~exp=errorCount.contents->Belt.Int.toFloat)->Belt.Float.toInt
+      let _ = Js.Global.setTimeout(() => {
+        if !isUnsubscribed.contents {
+          startConnection()
+        }
+      }, duration)
+    }
+  }
+  and startConnection = () => {
+    if isUnsubscribed.contents || errorCount.contents >= retryCount {
+      ()
+    } else {
+      let ws = WebSocket.create(wsUrl)
+      wsRef := Some(ws)
+
+      ws->WebSocket.onopen(() => {
+        ws->WebSocket.send(subscribeRequestJson)
+        resetStaleTimeout()
+      })
+
+      ws->WebSocket.onmessage(event => {
+        try {
+          switch event.data->Js.Json.parseExn->S.parseOrThrow(wsMessageSchema) {
+          | NewHead(blockNumber) =>
+            errorCount := 0
+            resetStaleTimeout()
+            Prometheus.SourceRequestCount.increment(
+              ~sourceName="WebSocket",
+              ~chainId,
+              ~method="eth_subscribe",
+            )
+            onHeight(blockNumber)
+          | SubscriptionConfirmed(_) => resetStaleTimeout()
+          | ErrorResponse =>
+            if errorCount.contents < retryCount {
+              errorCount := errorCount.contents + 1
+            }
+            switch wsRef.contents {
+            | Some(ws) => ws->WebSocket.close
+            | None => ()
+            }
+          }
+        } catch {
+        | S.Raised(_) =>
+          Logging.warn({
+            "msg": "WebSocket height stream received unrecognized message",
+            "chainId": chainId,
+            "data": event.data,
+          })
+        | Js.Exn.Error(_) as e =>
+          Logging.warn({
+            "msg": "WebSocket height stream failed to parse message",
+            "chainId": chainId,
+            "err": e->Utils.prettifyExn,
+            "data": event.data,
+          })
+        | e =>
+          Logging.error({
+            "msg": "Unexpected error in WebSocket height stream message handler",
+            "chainId": chainId,
+            "err": e->Utils.prettifyExn,
+            "data": event.data,
+          })
+          raise(e)
+        }
+      })
+
+      ws->WebSocket.onerror(_error => {
+        if errorCount.contents < retryCount {
+          errorCount := errorCount.contents + 1
+        }
+        switch wsRef.contents {
+        | Some(ws) if ws->WebSocket.readyState === Open => ws->WebSocket.close
+        | _ => ()
+        }
+      })
+
+      ws->WebSocket.onclose(() => {
+        wsRef := None
+        clearStaleTimeout()
+        scheduleReconnect()
+      })
+    }
+  }
+
+  startConnection()
+
+  () => {
+    isUnsubscribed := true
+    clearStaleTimeout()
+    switch wsRef.contents {
+    | Some(ws) => ws->WebSocket.close
+    | None => ()
+    }
+  }
+}

package/src/sources/RpcWebSocketHeightStream.res.mjs

@@ -0,0 +1,196 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Rpc from "./Rpc.res.mjs";
+import * as Utils from "../Utils.res.mjs";
+import * as Js_exn from "rescript/lib/es6/js_exn.js";
+import * as Js_json from "rescript/lib/es6/js_json.js";
+import * as Logging from "../Logging.res.mjs";
+import * as Prometheus from "../Prometheus.res.mjs";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
+
+var subscribeRequestJson = Js_json.serializeExn({
+      jsonrpc: "2.0",
+      id: 1,
+      method: "eth_subscribe",
+      params: ["newHeads"]
+    });
+
+var wsMessageSchema = S$RescriptSchema.union([
+      S$RescriptSchema.object(function (s) {
+            s.f("method", S$RescriptSchema.literal("eth_subscription"));
+            return {
+                    TAG: "NewHead",
+                    _0: s.f("params", S$RescriptSchema.object(function (s) {
+                              return s.f("result", S$RescriptSchema.object(function (s) {
+                                              return s.f("number", Rpc.hexIntSchema);
+                                            }));
+                            }))
+                  };
+          }),
+      S$RescriptSchema.object(function (s) {
+            return {
+                    TAG: "SubscriptionConfirmed",
+                    _0: s.f("result", S$RescriptSchema.string)
+                  };
+          }),
+      S$RescriptSchema.object(function (s) {
+            s.f("error", S$RescriptSchema.unknown);
+            return "ErrorResponse";
+          })
+    ]);
+
+function subscribe(wsUrl, chainId, onHeight) {
+  var wsRef = {
+    contents: undefined
+  };
+  var isUnsubscribed = {
+    contents: false
+  };
+  var errorCount = {
+    contents: 0
+  };
+  var staleTimeoutId = {
+    contents: undefined
+  };
+  var clearStaleTimeout = function () {
+    var id = staleTimeoutId.contents;
+    if (id !== undefined) {
+      clearTimeout(Caml_option.valFromOption(id));
+      staleTimeoutId.contents = undefined;
+      return ;
+    }
+    
+  };
+  var resetStaleTimeout = function () {
+    clearStaleTimeout();
+    staleTimeoutId.contents = Caml_option.some(setTimeout((function () {
+                var ws = wsRef.contents;
+                if (ws !== undefined) {
+                  Caml_option.valFromOption(ws).close();
+                  return ;
+                }
+                
+              }), 60000));
+  };
+  var scheduleReconnect = function () {
+    if (!(!isUnsubscribed.contents && errorCount.contents < 9)) {
+      return ;
+    }
+    var duration = Math.imul(125, Math.pow(2.0, errorCount.contents) | 0);
+    setTimeout((function () {
+            if (!isUnsubscribed.contents) {
+              return startConnection();
+            }
+            
+          }), duration);
+  };
+  var startConnection = function () {
+    if (isUnsubscribed.contents || errorCount.contents >= 9) {
+      return ;
+    }
+    var ws = new WebSocket(wsUrl);
+    wsRef.contents = Caml_option.some(ws);
+    ws.onopen = (function () {
+        ws.send(subscribeRequestJson);
+        resetStaleTimeout();
+      });
+    ws.onmessage = (function ($$event) {
+        try {
+          var blockNumber = S$RescriptSchema.parseOrThrow(JSON.parse($$event.data), wsMessageSchema);
+          if (typeof blockNumber !== "object") {
+            if (errorCount.contents < 9) {
+              errorCount.contents = errorCount.contents + 1 | 0;
+            }
+            var ws = wsRef.contents;
+            if (ws !== undefined) {
+              Caml_option.valFromOption(ws).close();
+              return ;
+            } else {
+              return ;
+            }
+          }
+          if (blockNumber.TAG !== "NewHead") {
+            return resetStaleTimeout();
+          }
+          errorCount.contents = 0;
+          resetStaleTimeout();
+          Prometheus.SourceRequestCount.increment("WebSocket", chainId, "eth_subscribe");
+          return onHeight(blockNumber._0);
+        }
+        catch (raw_e){
+          var e = Caml_js_exceptions.internalToOCamlException(raw_e);
+          if (e.RE_EXN_ID === S$RescriptSchema.Raised) {
+            return Logging.warn({
+                        msg: "WebSocket height stream received unrecognized message",
+                        chainId: chainId,
+                        data: $$event.data
+                      });
+          }
+          if (e.RE_EXN_ID === Js_exn.$$Error) {
+            return Logging.warn({
+                        msg: "WebSocket height stream failed to parse message",
+                        chainId: chainId,
+                        err: Utils.prettifyExn(e),
+                        data: $$event.data
+                      });
+          }
+          Logging.error({
+                msg: "Unexpected error in WebSocket height stream message handler",
+                chainId: chainId,
+                err: Utils.prettifyExn(e),
+                data: $$event.data
+              });
+          throw e;
+        }
+      });
+    ws.onerror = (function (_error) {
+        if (errorCount.contents < 9) {
+          errorCount.contents = errorCount.contents + 1 | 0;
+        }
+        var ws = wsRef.contents;
+        if (ws === undefined) {
+          return ;
+        }
+        var ws$1 = Caml_option.valFromOption(ws);
+        if (ws$1.readyState === 1) {
+          ws$1.close();
+          return ;
+        }
+        
+      });
+    ws.onclose = (function () {
+        wsRef.contents = undefined;
+        clearStaleTimeout();
+        scheduleReconnect();
+      });
+  };
+  startConnection();
+  return function () {
+    isUnsubscribed.contents = true;
+    clearStaleTimeout();
+    var ws = wsRef.contents;
+    if (ws !== undefined) {
+      Caml_option.valFromOption(ws).close();
+      return ;
+    }
+    
+  };
+}
+
+var retryCount = 9;
+
+var baseDuration = 125;
+
+var staleTimeMillis = 60000;
+
+export {
+  retryCount ,
+  baseDuration ,
+  staleTimeMillis ,
+  subscribeRequestJson ,
+  wsMessageSchema ,
+  subscribe ,
+}
+/* subscribeRequestJson Not a pure module */

package/src/sources/SimulateSource.res

@@ -0,0 +1,59 @@
+let make = (~items: array<Internal.item>, ~endBlock: int, ~chain: ChainMap.Chain.t): Source.t => {
+  // getItemsOrThrow might be called multiple times with different partition ids.
+  // Return all items on the first call and empty on subsequent calls to prevent
+  // duplicate event processing.
+  let delivered = ref(false)
+
+  {
+    name: "SimulateSource",
+    sourceFor: Sync,
+    chain,
+    poweredByHyperSync: false,
+    pollingInterval: 0,
+    getBlockHashes: (~blockNumbers as _, ~logger as _) => {
+      Promise.resolve(Ok([]))
+    },
+    getHeightOrThrow: () => {
+      // Report at least height 1 so the engine doesn't treat 0 as "no blocks available"
+      Promise.resolve(max(endBlock, 1))
+    },
+    getItemsOrThrow: (
+      ~fromBlock as _,
+      ~toBlock as _,
+      ~addressesByContractName as _,
+      ~indexingContracts as _,
+      ~knownHeight as _,
+      ~partitionId as _,
+      ~selection as _,
+      ~retry as _,
+      ~logger as _,
+    ) => {
+      // Return all items on first call, empty on subsequent calls
+      let result = if delivered.contents {
+        []
+      } else {
+        delivered := true
+        items
+      }
+
+      let reportedHeight = max(endBlock, 1)
+      Promise.resolve({
+        Source.knownHeight: reportedHeight,
+        reorgGuard: {
+          rangeLastBlock: {
+            blockHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
+            blockNumber: reportedHeight,
+          },
+          prevRangeLastBlock: None,
+        },
+        parsedQueueItems: result,
+        fromBlockQueried: 0,
+        latestFetchedBlockNumber: reportedHeight,
+        latestFetchedBlockTimestamp: 0,
+        stats: {
+          totalTimeElapsed: 0.,
+        },
+      })
+    },
+  }
+}

package/src/sources/SimulateSource.res.mjs

@@ -0,0 +1,50 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+
+function make(items, endBlock, chain) {
+  var delivered = {
+    contents: false
+  };
+  return {
+          name: "SimulateSource",
+          sourceFor: "Sync",
+          chain: chain,
+          poweredByHyperSync: false,
+          pollingInterval: 0,
+          getBlockHashes: (function (param, param$1) {
+              return Promise.resolve({
+                          TAG: "Ok",
+                          _0: []
+                        });
+            }),
+          getHeightOrThrow: (function () {
+              return Promise.resolve(endBlock > 1 ? endBlock : 1);
+            }),
+          getItemsOrThrow: (function (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8) {
+              var result = delivered.contents ? [] : (delivered.contents = true, items);
+              var reportedHeight = endBlock > 1 ? endBlock : 1;
+              return Promise.resolve({
+                          knownHeight: reportedHeight,
+                          reorgGuard: {
+                            rangeLastBlock: {
+                              blockHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
+                              blockNumber: reportedHeight
+                            },
+                            prevRangeLastBlock: undefined
+                          },
+                          parsedQueueItems: result,
+                          fromBlockQueried: 0,
+                          latestFetchedBlockNumber: reportedHeight,
+                          latestFetchedBlockTimestamp: 0,
+                          stats: {
+                            "total time elapsed (s)": 0
+                          }
+                        });
+            })
+        };
+}
+
+export {
+  make ,
+}
+/* No side effect */

package/src/sources/Source.res

@@ -2,16 +2,16 @@
 A set of stats for logging about the block range fetch
 */
 type blockRangeFetchStats = {
-  @as("total time elapsed (ms)") totalTimeElapsed: int,
-  @as("parsing time (ms)") parsingTimeElapsed?: int,
-  @as("page fetch time (ms)") pageFetchTime?: int,
+  @as("total time elapsed (s)") totalTimeElapsed: float,
+  @as("parsing time (s)") parsingTimeElapsed?: float,
+  @as("page fetch time (s)") pageFetchTime?: float,
 }
 
 /**
 Thes response returned from a block range fetch
 */
 type blockRangeFetchResponse = {
-  currentBlockHeight: int,
+  knownHeight: int,
   reorgGuard: ReorgDetection.reorgGuard,
   parsedQueueItems: array<Internal.item>,
   fromBlockQueried: int,
@@ -33,6 +33,7 @@ type getItemsError =
 exception GetItemsError(getItemsError)
 
 type sourceFor = Sync | Fallback | Live
+
 type t = {
   name: string,
   sourceFor: sourceFor,
@@ -50,10 +51,11 @@ type t = {
     ~toBlock: option<int>,
     ~addressesByContractName: dict<array<Address.t>>,
     ~indexingContracts: dict<Internal.indexingContract>,
-    ~currentBlockHeight: int,
+    ~knownHeight: int,
     ~partitionId: string,
     ~selection: FetchState.selection,
     ~retry: int,
     ~logger: Pino.t,
   ) => promise<blockRangeFetchResponse>,
+  createHeightSubscription?: (~onHeight: int => unit) => unit => unit,
 }