envio 3.15.0 → 3.16.0

package/src/sources/EvmRpcClient.res

@@ -8,6 +8,7 @@ type cfg = {
   intervalCeiling: int,
   backoffMillis: int,
   queryTimeoutMillis: int,
+  maxAddressesPerRequest: int,
 }
 
 type nextPageParams = {
@@ -18,10 +19,12 @@ type nextPageParams = {
   onBlockRegistrationIndexes: array<int>,
   clientFilteredContracts: option<array<string>>,
   retry: int,
-  // Blocks at or above this one can still be reorged, so a blocks-only page
-  // reaching them is read for the hashes of its boundary blocks even with no
-  // handler block in range. None when the range is final.
+  // Blocks at or above this one can still be reorged, so each of them in the
+  // page has its header read and its logs checked against it, even a
+  // blocks-only page with no handler block in range. None when the range is
+  // final.
   reorgableFrom?: int,
+  knownHeight: int,
 }
 
 // Rust reports what to do about a page it could not read rather than throwing
@@ -77,6 +80,8 @@ external classNew: (
   array<BlockItem.registrationInput>,
   ~checksumAddresses: bool,
   ~addressStore: AddressStore.t,
+  Logger.native,
+  array<string>,
 ) => t = "new"
 
 let make = (
@@ -88,6 +93,8 @@ let make = (
   ~eventRegistrations=[],
   ~onBlockRegistrations=[],
   ~addressStore,
+  ~maxAddressesPerRequest,
+  ~logger: Logger.t,
 ) =>
   Core.getAddon().evmRpcClient->classNew(
     {
@@ -100,9 +107,12 @@ let make = (
       intervalCeiling: syncConfig.intervalCeiling,
       backoffMillis: syncConfig.backoffMillis,
       queryTimeoutMillis: syncConfig.queryTimeoutMillis,
+      maxAddressesPerRequest,
     },
     eventRegistrations,
     onBlockRegistrations->BlockItem.toRegistrationInputs,
     ~checksumAddresses,
     ~addressStore,
+    logger.output.native,
+    logger.boundFieldsJson,
   )

package/src/sources/EvmRpcClient.res.mjs

@@ -3,7 +3,7 @@
 import * as Core from "../Core.res.mjs";
 import * as BlockItem from "./BlockItem.res.mjs";
 
-function make(url, checksumAddresses, syncConfig, maxConcurrentRequests, headers, eventRegistrationsOpt, onBlockRegistrationsOpt, addressStore) {
+function make(url, checksumAddresses, syncConfig, maxConcurrentRequests, headers, eventRegistrationsOpt, onBlockRegistrationsOpt, addressStore, maxAddressesPerRequest, logger) {
   let eventRegistrations = eventRegistrationsOpt !== undefined ? eventRegistrationsOpt : [];
   let onBlockRegistrations = onBlockRegistrationsOpt !== undefined ? onBlockRegistrationsOpt : [];
   return Core.getAddon().EvmRpcClient.new({
@@ -15,8 +15,9 @@ function make(url, checksumAddresses, syncConfig, maxConcurrentRequests, headers
     accelerationAdditive: syncConfig.accelerationAdditive,
     intervalCeiling: syncConfig.intervalCeiling,
     backoffMillis: syncConfig.backoffMillis,
-    queryTimeoutMillis: syncConfig.queryTimeoutMillis
-  }, eventRegistrations, BlockItem.toRegistrationInputs(onBlockRegistrations), checksumAddresses, addressStore);
+    queryTimeoutMillis: syncConfig.queryTimeoutMillis,
+    maxAddressesPerRequest: maxAddressesPerRequest
+  }, eventRegistrations, BlockItem.toRegistrationInputs(onBlockRegistrations), checksumAddresses, addressStore, logger.output.native, logger.boundFieldsJson);
 }
 
 export {

package/src/sources/EvmRpcWs.res

@@ -81,8 +81,8 @@ let wsMessageSchema = S.union([
   }),
 ])
 
-let subscribe = (~wsUrl, ~onHeight, ~onStatus) =>
-  HeightStream.subscribe(~staleTimeout, ~onHeight, ~onStatus, ~connect=driver => {
+let subscribe = (~clock, ~wsUrl, ~onHeight, ~onStatus) =>
+  HeightStream.subscribe(~clock, ~staleTimeout, ~onHeight, ~onStatus, ~connect=driver => {
     let ws = WebSocket.create(wsUrl)
     let subscribed = ref(false)
 
@@ -103,7 +103,8 @@ let subscribe = (~wsUrl, ~onHeight, ~onStatus) =>
         | Some(Response({id: Some(id), isError: false})) if id === subscribeRequestIdText =>
           subscribed := true
           driver.onConnected()
-        | Some(Response({id, isError: true})) if id->refusesSubscribe(~subscribed=subscribed.contents) =>
+        | Some(Response({id, isError: true}))
+          if id->refusesSubscribe(~subscribed=subscribed.contents) =>
           driver.onFailure(~reason=SubscribeRejected, ~detail=text)
         // A provider talking about something other than the subscription this
         // socket asked for — a rate limit, a request nobody here made. Read

package/src/sources/EvmRpcWs.res.mjs

@@ -61,8 +61,8 @@ let wsMessageSchema = S$RescriptSchema.union([
   })
 ]);
 
-function subscribe(wsUrl, onHeight, onStatus) {
-  return HeightStream.subscribe(60000, onHeight, onStatus, driver => {
+function subscribe(clock, wsUrl, onHeight, onStatus) {
+  return HeightStream.subscribe(clock, 60000, onHeight, onStatus, driver => {
     let ws = new WebSocket(wsUrl);
     let subscribed = {
       contents: false

package/src/sources/HeightFeed.res

@@ -79,6 +79,7 @@ type t = {
   // than any healthy getHeight, so a merely slow endpoint answers first and
   // nothing here fires in normal running.
   pollTimeoutMillis: int,
+  clock: Clock.t,
   mutable knownHeight: int,
   mutable waiters: array<waiter>,
   mutable stream: streamState,
@@ -110,12 +111,14 @@ let make = (
   ~recordRequestStats,
   ~getHeightRetryInterval,
   ~pollTimeoutMillis,
+  ~clock,
 ): t => {
   source,
   logger: logger->Logger.withFields({"source": source.name}),
   recordRequestStats,
   getHeightRetryInterval,
   pollTimeoutMillis,
+  clock,
   knownHeight: 0,
   waiters: [],
   stream: NeverEnabled,
@@ -275,26 +278,43 @@ let heightWithin = (feed: t): promise<pollOutcome> => {
     request
     ->Promise.thenResolve(res => Answered(res))
     ->Promise.catch(exn => Promise.resolve(Failed(exn))),
-    Promise.make((resolve, _reject) => timeoutId := Some(setTimeout(() => {
-            timedOut := true
-            resolve(TimedOut)
-          }, feed.pollTimeoutMillis))),
+    Promise.make((resolve, _reject) =>
+      timeoutId :=
+        Some(
+          feed.clock.setTimeout(
+            ~label="height poll timeout",
+            () => {
+              timedOut := true
+              resolve(TimedOut)
+            },
+            feed.pollTimeoutMillis,
+          ),
+        )
+    ),
   ])->Promise.thenResolve(outcome => {
-    timeoutId->Utils.clearTimeoutRef
+    feed.clock->Clock.clearRef(timeoutId)
     outcome
   })
 }
 
 let sleep = (feed: t, ~kind, millis) =>
   Promise.make((resolve, _reject) => {
-    let timeoutId = setTimeout(() => {
-      feed.sleeping = None
-      resolve()
-    }, millis)
+    let label = switch kind {
+    | Cadence => "height poll cadence"
+    | Backoff => "height poll backoff"
+    }
+    let timer = feed.clock.setTimeout(
+      ~label,
+      () => {
+        feed.sleeping = None
+        resolve()
+      },
+      millis,
+    )
     feed.sleeping = Some({
       kind,
       wake: () => {
-        clearTimeout(timeoutId)
+        feed.clock.clearTimeout(timer)
         feed.sleeping = None
         resolve()
       },
@@ -521,6 +541,7 @@ let enableStream = (feed: t) =>
     }
     feed.closeStream = Some(
       createSubscription(
+        ~clock=feed.clock,
         ~onHeight=height => feed->handlePushedHeight(height),
         ~onStatus=status => feed->handleStatus(status),
       ),

package/src/sources/HeightFeed.res.mjs

@@ -1,5 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Clock from "../Clock.res.mjs";
 import * as Utils from "../Utils.res.mjs";
 import * as Logger from "../Logger.res.mjs";
 import * as Source from "./Source.res.mjs";
@@ -10,7 +11,7 @@ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js
 import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
-function make(source, logger, recordRequestStats, getHeightRetryInterval, pollTimeoutMillis) {
+function make(source, logger, recordRequestStats, getHeightRetryInterval, pollTimeoutMillis, clock) {
   return {
     source: source,
     logger: Logger.withFields(logger, {
@@ -19,6 +20,7 @@ function make(source, logger, recordRequestStats, getHeightRetryInterval, pollTi
     recordRequestStats: recordRequestStats,
     getHeightRetryInterval: getHeightRetryInterval,
     pollTimeoutMillis: pollTimeoutMillis,
+    clock: clock,
     knownHeight: 0,
     waiters: [],
     stream: "NeverEnabled",
@@ -157,27 +159,29 @@ function heightWithin(feed) {
       _0: exn
     })),
     new Promise((resolve, _reject) => {
-      timeoutId.contents = Primitive_option.some(setTimeout(() => {
+      timeoutId.contents = Primitive_option.some(feed.clock.setTimeout("height poll timeout", () => {
         timedOut.contents = true;
         resolve("TimedOut");
       }, feed.pollTimeoutMillis));
     })
   ]).then(outcome => {
-    Utils.clearTimeoutRef(timeoutId);
+    Clock.clearRef(feed.clock, timeoutId);
     return outcome;
   });
 }
 
 function sleep(feed, kind, millis) {
   return new Promise((resolve, _reject) => {
-    let timeoutId = setTimeout(() => {
+    let label;
+    label = kind === "Cadence" ? "height poll cadence" : "height poll backoff";
+    let timer = feed.clock.setTimeout(label, () => {
       feed.sleeping = undefined;
       resolve();
     }, millis);
     feed.sleeping = {
       kind: kind,
       wake: () => {
-        clearTimeout(timeoutId);
+        feed.clock.clearTimeout(timer);
         feed.sleeping = undefined;
         resolve();
       }
@@ -370,7 +374,7 @@ function enableStream(feed) {
   if (typeof match$2 !== "object" && match$2 === "NeverEnabled") {
     feed.stream = "Disconnected";
   }
-  feed.closeStream = match(height => handlePushedHeight(feed, height), status => handleStatus(feed, status));
+  feed.closeStream = match(feed.clock, height => handlePushedHeight(feed, height), status => handleStatus(feed, status));
 }
 
 function stop(feed) {

package/src/sources/HeightStream.res

@@ -58,6 +58,7 @@ let truncateDetail = detail =>
     : detail
 
 let subscribe = (
+  ~clock: Clock.t,
   ~staleTimeout: int,
   ~onHeight: int => unit,
   ~onStatus: Source.heightSubscriptionStatus => unit,
@@ -85,7 +86,7 @@ let subscribe = (
   let liveAt = ref(None)
   let minProvenMillis = staleTimeout / provenStaleFraction
 
-  let clearPendingTimeout = () => timeoutId->Utils.clearTimeoutRef
+  let clearPendingTimeout = () => clock->Clock.clearRef(timeoutId)
 
   // A transport that throws while closing must not be able to stop the retry
   // that follows, nor escape a timer callback and take the process down. Every
@@ -106,15 +107,22 @@ let subscribe = (
 
   let rec armStaleTimeout = () => {
     clearPendingTimeout()
-    timeoutId := Some(setTimeout(() => {
-          timeoutId := None
-          // Distinguishes a provider whose message shape we don't understand
-          // from a chain that simply has nothing to report.
-          switch unreadableDetail.contents {
-          | Some(detail) => fail(~reason=Source.Unreadable, ~detail)
-          | None => fail(~reason=Source.Stale)
-          }
-        }, staleTimeout))
+    timeoutId :=
+      Some(
+        clock.setTimeout(
+          ~label="height stream stale",
+          () => {
+            timeoutId := None
+            // Distinguishes a provider whose message shape we don't understand
+            // from a chain that simply has nothing to report.
+            switch unreadableDetail.contents {
+            | Some(detail) => fail(~reason=Source.Unreadable, ~detail)
+            | None => fail(~reason=Source.Stale)
+            }
+          },
+          staleTimeout,
+        ),
+      )
   }
   and fail = (~reason: Source.heightSubscriptionDownReason, ~detail=?) => {
     generation := generation.contents + 1
@@ -128,7 +136,7 @@ let subscribe = (
     // bar rises with the backoff, so an endpoint that always dies at the same
     // age keeps escalating rather than settling into a loop at a fixed delay.
     let servedMillis = switch liveAt.contents {
-    | Some(since) => (Performance.now() -. since)->Float.toInt
+    | Some(since) => (clock.now() -. since)->Float.toInt
     | None => 0
     }
     // Against the schedule, not against the jittered wait actually served. The
@@ -143,7 +151,7 @@ let subscribe = (
         failureCount.contents + 1
       }
 
-    let retryMillis = retryDelayFor(failureCount.contents)->Utils.jitter
+    let retryMillis = clock->Clock.jitter(retryDelayFor(failureCount.contents))
 
     // A clean end to a connection that had proven itself is a rotation: routine,
     // and worth telling apart from the same clean end arriving early, which is a
@@ -157,10 +165,17 @@ let subscribe = (
 
     // Scheduled before reporting, so a consumer that throws can't be what makes
     // the stream give up.
-    timeoutId := Some(setTimeout(() => {
-          timeoutId := None
-          start()
-        }, retryMillis))
+    timeoutId :=
+      Some(
+        clock.setTimeout(
+          ~label="height stream retry",
+          () => {
+            timeoutId := None
+            start()
+          },
+          retryMillis,
+        ),
+      )
 
     onStatus(Down({reason, detail: ?(detail->Option.map(truncateDetail))}))
   }
@@ -183,7 +198,7 @@ let subscribe = (
     let goLive = () => {
       armStaleTimeout()
       if liveAt.contents->Option.isNone {
-        liveAt := Some(Performance.now())
+        liveAt := Some(clock.now())
         onStatus(Live)
       }
     }

package/src/sources/HeightStream.res.mjs

@@ -1,7 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Clock from "../Clock.res.mjs";
 import * as Utils from "../Utils.res.mjs";
-import * as Performance from "../bindings/Performance.res.mjs";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
@@ -23,7 +23,7 @@ function truncateDetail(detail) {
   }
 }
 
-function subscribe(staleTimeout, onHeight, onStatus, connect) {
+function subscribe(clock, staleTimeout, onHeight, onStatus, connect) {
   let closeConnectionRef = {
     contents: undefined
   };
@@ -58,8 +58,8 @@ function subscribe(staleTimeout, onHeight, onStatus, connect) {
     closeConnectionRef.contents = undefined;
   };
   let armStaleTimeout = () => {
-    Utils.clearTimeoutRef(timeoutId);
-    timeoutId.contents = Primitive_option.some(setTimeout(() => {
+    Clock.clearRef(clock, timeoutId);
+    timeoutId.contents = Primitive_option.some(clock.setTimeout("height stream stale", () => {
       timeoutId.contents = undefined;
       let detail = unreadableDetail.contents;
       if (detail !== undefined) {
@@ -71,16 +71,16 @@ function subscribe(staleTimeout, onHeight, onStatus, connect) {
   };
   let fail = (reason, detail) => {
     generation.contents = generation.contents + 1 | 0;
-    Utils.clearTimeoutRef(timeoutId);
+    Clock.clearRef(clock, timeoutId);
     closeConnection();
     let since = liveAt.contents;
-    let servedMillis = since !== undefined ? Performance.now() - since | 0 : 0;
+    let servedMillis = since !== undefined ? clock.now() - since | 0 : 0;
     let provenMillis = Primitive_int.max(retryDelayFor(failureCount.contents), minProvenMillis);
     failureCount.contents = servedMillis >= provenMillis ? 1 : failureCount.contents + 1 | 0;
-    let retryMillis = Utils.jitter(retryDelayFor(failureCount.contents));
+    let retryMillis = Clock.jitter(clock, retryDelayFor(failureCount.contents));
     let reason$1;
     reason$1 = typeof reason !== "object" && reason === "Closed" && servedMillis >= provenMillis ? "Rotated" : reason;
-    timeoutId.contents = Primitive_option.some(setTimeout(() => {
+    timeoutId.contents = Primitive_option.some(clock.setTimeout("height stream retry", () => {
       timeoutId.contents = undefined;
       start();
     }, retryMillis));
@@ -99,7 +99,7 @@ function subscribe(staleTimeout, onHeight, onStatus, connect) {
     let goLive = () => {
       armStaleTimeout();
       if (Stdlib_Option.isNone(liveAt.contents)) {
-        liveAt.contents = Performance.now();
+        liveAt.contents = clock.now();
         return onStatus("Live");
       }
     };
@@ -169,7 +169,7 @@ function subscribe(staleTimeout, onHeight, onStatus, connect) {
   start();
   return () => {
     generation.contents = generation.contents + 1 | 0;
-    Utils.clearTimeoutRef(timeoutId);
+    Clock.clearRef(clock, timeoutId);
     closeConnection();
   };
 }

package/src/sources/HyperSyncSSE.res

@@ -14,8 +14,8 @@ let failure = (error: EventSource.errorEvent): (
   | (None, None) => (Closed, None)
   }
 
-let subscribe = (~hyperSyncUrl, ~apiToken, ~onHeight, ~onStatus) =>
-  HeightStream.subscribe(~staleTimeout, ~onHeight, ~onStatus, ~connect=driver => {
+let subscribe = (~clock, ~hyperSyncUrl, ~apiToken, ~onHeight, ~onStatus) =>
+  HeightStream.subscribe(~clock, ~staleTimeout, ~onHeight, ~onStatus, ~connect=driver => {
     let userAgent = `hyperindex/${Utils.EnvioPackage.value.version}`
     let es = EventSource.create(
       ~url=`${hyperSyncUrl}/height/sse`,
@@ -24,10 +24,8 @@ let subscribe = (~hyperSyncUrl, ~apiToken, ~onHeight, ~onStatus) =>
           let headers =
             args.headers
             ->Option.getOr(Dict.make())
-            ->Utils.Dict.merge(
-              dict{"Authorization": `Bearer ${apiToken}`, "User-Agent": userAgent},
-            )
-          EventSource.Fetch.fetch(url, ~args={...args, headers: headers})
+            ->Utils.Dict.merge(dict{"Authorization": `Bearer ${apiToken}`, "User-Agent": userAgent})
+          EventSource.Fetch.fetch(url, ~args={...args, headers})
         },
       },
     )

package/src/sources/HyperSyncSSE.res.mjs

@@ -30,8 +30,8 @@ function failure(error) {
   }
 }
 
-function subscribe(hyperSyncUrl, apiToken, onHeight, onStatus) {
-  return HeightStream.subscribe(15000, onHeight, onStatus, driver => {
+function subscribe(clock, hyperSyncUrl, apiToken, onHeight, onStatus) {
+  return HeightStream.subscribe(clock, 15000, onHeight, onStatus, driver => {
     let userAgent = `hyperindex/` + Utils.EnvioPackage.value.version;
     let es = new Eventsource.EventSource(hyperSyncUrl + `/height/sse`, {
       fetch: (url, args) => {

package/src/sources/RpcSource.res

@@ -17,6 +17,10 @@ type options = {
   transactionStore: TransactionStore.t,
   ws?: string,
   headers?: dict<string>,
+  // How many addresses one `eth_getLogs` may carry when an address-free query
+  // the provider couldn't answer is read again by address.
+  maxAddressesPerRequest: int,
+  logger: Logger.t,
 }
 
 let make = (
@@ -33,6 +37,8 @@ let make = (
     transactionStore,
     ?ws,
     ?headers,
+    maxAddressesPerRequest,
+    logger,
   }: options,
 ): t => {
   let urlHost = switch Utils.Url.getHostFromUrl(url) {
@@ -53,6 +59,8 @@ let make = (
     ~syncConfig,
     ~headers?,
     ~addressStore,
+    ~maxAddressesPerRequest,
+    ~logger,
   )
 
   let getItemsOrThrow = async (
@@ -99,6 +107,7 @@ let make = (
         clientFilteredContracts: selection.clientFilteredContracts,
         retry,
         ?reorgableFrom,
+        knownHeight,
       },
       addressSet,
       blockStore,
@@ -219,7 +228,7 @@ let make = (
 
   let createHeightSubscription =
     ws->Option.map(wsUrl =>
-      (~onHeight, ~onStatus) => EvmRpcWs.subscribe(~wsUrl, ~onHeight, ~onStatus)
+      (~clock, ~onHeight, ~onStatus) => EvmRpcWs.subscribe(~clock, ~wsUrl, ~onHeight, ~onStatus)
     )
 
   {

package/src/sources/RpcSource.res.mjs

@@ -26,7 +26,7 @@ function make(param) {
   let urlHost = host !== undefined ? host : Stdlib_JsError.throwWithMessage(`The RPC url for chain ` + ChainId.toString(chainId) + ` is in incorrect format. The RPC url needs to start with either http:// or https://`);
   let name = `RPC (` + urlHost + `)`;
   let onBlockRegistrations = Stdlib_Option.getOr(param.onBlockRegistrations, []);
-  let rpcClient = EvmRpcClient.make(url, !param.lowercaseAddresses, syncConfig, undefined, param.headers, HyperSyncClient.Registration.fromOnEventRegistrations(onEventRegistrations), onBlockRegistrations, param.addressStore);
+  let rpcClient = EvmRpcClient.make(url, !param.lowercaseAddresses, syncConfig, undefined, param.headers, HyperSyncClient.Registration.fromOnEventRegistrations(onEventRegistrations), onBlockRegistrations, param.addressStore, param.maxAddressesPerRequest, param.logger);
   let getItemsOrThrow = async (fromBlock, toBlock, addressSet, reorgableFrom, knownHeight, partitionId, selection, param, retry, param$1) => {
     let totalTimeRef = Performance.now();
     let toBlock$1 = toBlock !== undefined ? Primitive_int.min(toBlock, knownHeight) : knownHeight;
@@ -49,7 +49,8 @@ function make(param) {
       onBlockRegistrationIndexes: selection.onBlockRegistrations.map(reg => reg.index),
       clientFilteredContracts: selection.clientFilteredContracts,
       retry: retry,
-      reorgableFrom: reorgableFrom
+      reorgableFrom: reorgableFrom,
+      knownHeight: knownHeight
     }, addressSet, blockStore, transactionStore);
     let result = match[0];
     let pageFetchTime = Performance.secondsSince(pageFetchTimeRef);
@@ -148,7 +149,7 @@ function make(param) {
       requestStats: result.requestStats
     };
   };
-  let createHeightSubscription = Stdlib_Option.map(param.ws, wsUrl => ((onHeight, onStatus) => EvmRpcWs.subscribe(wsUrl, onHeight, onStatus)));
+  let createHeightSubscription = Stdlib_Option.map(param.ws, wsUrl => ((clock, onHeight, onStatus) => EvmRpcWs.subscribe(clock, wsUrl, onHeight, onStatus)));
   return {
     name: name,
     sourceFor: param.sourceFor,

package/src/sources/Source.res

@@ -240,6 +240,9 @@ type t = {
     ~logger: Logger.t,
   ) => promise<blockRangeFetchResponse>,
   createHeightSubscription?: (
+    // The stream's staleness and reconnect timers run on the clock of the
+    // manager that subscribes, not one the source was built with.
+    ~clock: Clock.t,
     ~onHeight: int => unit,
     ~onStatus: heightSubscriptionStatus => unit,
   ) => unit => unit,