envio 3.11.0 → 3.12.0

package/src/db/RollbackFloors.res

@@ -1,55 +1,48 @@
-// How far back a reorg rollback takes each chain: the last checkpoint each keeps,
-// and for a chain whose own reorg set its floor, the last block that reorg left
-// valid. Superseding an unwritten rollback with a new one is a pointwise
-// minimum, so a merge can neither lose a chain's floor nor raise it.
+// How far back a reorg rollback takes each chain: the last checkpoint each
+// keeps, and for a chain whose own reorg set its floor, the last block that
+// reorg left valid. Superseding an unwritten rollback with a new one is a
+// pointwise minimum, so a merge can neither lose a chain's floor nor raise it.
 type t = {
-  floors: CheckpointBounds.t,
+  checkpointBounds: CheckpointSequence.checkpointBoundsByChain,
   // Progress recomputed from the surviving checkpoints alone can land above the
   // fork: the blocks between it and the chain's next checkpoint carried no
   // events on the orphaned chain, but the chain replacing it can have its own.
   // Absent for a chain dragged along by another chain's reorg.
-  forkBlockNumberByChain: array<(ChainId.t, int)>,
+  forkBlockNumberByChain: dict<int>,
 }
 
-// The rollback of a schema whose chains can't have written each other's rows.
-let isolated = (~chainId, ~floorCheckpointId, ~forkBlockNumber) => {
-  floors: PerChain([(chainId, floorCheckpointId)]),
-  forkBlockNumberByChain: [(chainId, forkBlockNumber)],
-}
-
-// The rollback of a schema with a cross-chain entity: every chain goes back to
-// the same checkpoint.
-let global = (~floorCheckpointId, ~reorgChainId, ~forkBlockNumber) => {
-  floors: EveryChain(floorCheckpointId),
-  forkBlockNumberByChain: [(reorgChainId, forkBlockNumber)],
-}
-
-let mergeByChain = (pending: array<(ChainId.t, 'a)>, next: array<(ChainId.t, 'a)>) => {
-  let merged = pending->Array.copy
-  next->Array.forEach(((chainId, value)) =>
-    switch merged->Array.findIndexOpt(((mergedChainId, _)) => mergedChainId === chainId) {
-    | Some(idx) =>
-      let (_, pendingValue) = merged->Array.getUnsafe(idx)
-      merged->Array.setUnsafe(idx, (chainId, Pervasives.min(pendingValue, value)))
-    | None => merged->Array.push((chainId, value))
-    }
-  )
-  merged
+// Under one shared sequence a reorg on any chain can have changed any chain's
+// rows, so every chain goes back to the same checkpoint; with a counter per
+// chain the floor names the reorg chain alone and every sibling is left alone.
+let make = (
+  ~sequence: CheckpointSequence.t,
+  ~chainIds: array<ChainId.t>,
+  ~reorgChainId,
+  ~floorCheckpointId,
+  ~forkBlockNumber,
+) => {
+  checkpointBounds: {
+    CheckpointSequence.sequence,
+    byChain: switch sequence {
+    | SharedAcrossChains =>
+      chainIds->Array.map(chainId => (chainId, floorCheckpointId))->Frontier.fromEntries
+    | PerChain => Frontier.fromEntries([(reorgChainId, floorCheckpointId)])
+    },
+  },
+  forkBlockNumberByChain: Dict.fromArray([(reorgChainId->ChainId.toString, forkBlockNumber)]),
 }
 
 let merge = (pending: t, next: t) => {
-  floors: switch (pending.floors, next.floors) {
-  | (EveryChain(pending), EveryChain(next)) => EveryChain(Pervasives.min(pending, next))
-  | (PerChain(pending), PerChain(next)) => PerChain(mergeByChain(pending, next))
-  | (EveryChain(_), PerChain(_)) | (PerChain(_), EveryChain(_)) =>
-    JsError.throwWithMessage(
-      "Internal error: a rollback reaching every chain can't be merged with one isolated to a chain. The schema decides which a run builds, and it doesn't change.",
-    )
+  checkpointBounds: {
+    CheckpointSequence.sequence: next.checkpointBounds.sequence,
+    byChain: Frontier.mergeMin(pending.checkpointBounds.byChain, next.checkpointBounds.byChain),
   },
-  forkBlockNumberByChain: mergeByChain(pending.forkBlockNumberByChain, next.forkBlockNumberByChain),
+  forkBlockNumberByChain: Utils.Dict.mergeWith(
+    pending.forkBlockNumberByChain,
+    next.forkBlockNumberByChain,
+    Pervasives.min,
+  ),
 }
 
 let forkBlockNumber = (floors: t, chainId) =>
-  floors.forkBlockNumberByChain
-  ->Array.find(((forkChainId, _)) => forkChainId === chainId)
-  ->Option.map(((_, forkBlockNumber)) => forkBlockNumber)
+  floors.forkBlockNumberByChain->ChainId.Dict.dangerouslyGetNonOption(chainId)

package/src/db/RollbackFloors.res.mjs

@@ -1,92 +1,48 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
-import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
-import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
-import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
-import * as Primitive_bigint from "@rescript/runtime/lib/es6/Primitive_bigint.js";
-import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
+import * as Utils from "../Utils.res.mjs";
+import * as ChainId from "../ChainId.res.mjs";
+import * as Frontier from "./Frontier.res.mjs";
+import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 
-function isolated(chainId, floorCheckpointId, forkBlockNumber) {
-  return {
-    floors: {
-      TAG: "PerChain",
-      _0: [[
-          chainId,
-          floorCheckpointId
-        ]]
-    },
-    forkBlockNumberByChain: [[
-        chainId,
-        forkBlockNumber
-      ]]
-  };
-}
-
-function global(floorCheckpointId, reorgChainId, forkBlockNumber) {
+function make(sequence, chainIds, reorgChainId, floorCheckpointId, forkBlockNumber) {
+  let tmp;
+  tmp = sequence === "SharedAcrossChains" ? Frontier.fromEntries(chainIds.map(chainId => [
+      chainId,
+      floorCheckpointId
+    ])) : Frontier.fromEntries([[
+        reorgChainId,
+        floorCheckpointId
+      ]]);
   return {
-    floors: {
-      TAG: "EveryChain",
-      _0: floorCheckpointId
+    checkpointBounds: {
+      sequence: sequence,
+      byChain: tmp
     },
-    forkBlockNumberByChain: [[
-        reorgChainId,
+    forkBlockNumberByChain: Object.fromEntries([[
+        ChainId.toString(reorgChainId),
         forkBlockNumber
-      ]]
+      ]])
   };
 }
 
-function mergeByChain(pending, next) {
-  let merged = pending.slice();
-  next.forEach(param => {
-    let value = param[1];
-    let chainId = param[0];
-    let idx = Stdlib_Array.findIndexOpt(merged, param => param[0] === chainId);
-    if (idx !== undefined) {
-      let match = merged[idx];
-      merged[idx] = [
-        chainId,
-        Primitive_object.min(match[1], value)
-      ];
-      return;
-    }
-    merged.push([
-      chainId,
-      value
-    ]);
-  });
-  return merged;
-}
-
 function merge(pending, next) {
-  let match = pending.floors;
-  let match$1 = next.floors;
-  let tmp;
-  tmp = match.TAG === "EveryChain" ? (
-      match$1.TAG === "EveryChain" ? ({
-          TAG: "EveryChain",
-          _0: Primitive_bigint.min(match._0, match$1._0)
-        }) : Stdlib_JsError.throwWithMessage("Internal error: a rollback reaching every chain can't be merged with one isolated to a chain. The schema decides which a run builds, and it doesn't change.")
-    ) : (
-      match$1.TAG === "EveryChain" ? Stdlib_JsError.throwWithMessage("Internal error: a rollback reaching every chain can't be merged with one isolated to a chain. The schema decides which a run builds, and it doesn't change.") : ({
-          TAG: "PerChain",
-          _0: mergeByChain(match._0, match$1._0)
-        })
-    );
   return {
-    floors: tmp,
-    forkBlockNumberByChain: mergeByChain(pending.forkBlockNumberByChain, next.forkBlockNumberByChain)
+    checkpointBounds: {
+      sequence: next.checkpointBounds.sequence,
+      byChain: Frontier.mergeMin(pending.checkpointBounds.byChain, next.checkpointBounds.byChain)
+    },
+    forkBlockNumberByChain: Utils.Dict.mergeWith(pending.forkBlockNumberByChain, next.forkBlockNumberByChain, Primitive_int.min)
   };
 }
 
 function forkBlockNumber(floors, chainId) {
-  return Stdlib_Option.map(floors.forkBlockNumberByChain.find(param => param[0] === chainId), param => param[1]);
+  return floors.forkBlockNumberByChain[chainId];
 }
 
 export {
-  isolated,
-  global,
-  mergeByChain,
+  make,
   merge,
   forkBlockNumber,
 }
-/* No side effect */
+/* Utils Not a pure module */

package/src/sources/BlockStore.res

@@ -142,6 +142,9 @@ external materialize: (
 // Hash of a stored block, if the store still holds it.
 @send external getHash: (t, int) => Null.t<string> = "getHash"
 
+// Unix timestamp of a stored block - exactly that block, on every ecosystem.
+@send external getTimestamp: (t, int) => Null.t<int> = "getTimestamp"
+
 // Block numbers in `[fromBlock, belowBlock)` with a stored hash, ascending.
 @send
 external getHashedBlockNumbers: (t, ~fromBlock: int, ~belowBlock: int) => array<int> =

package/src/sources/EvmHyperSyncSource.res

@@ -141,7 +141,9 @@ let make = (
     }
 
     let pageFetchTime = startFetchingBatchTimeRef->Performance.secondsSince
-    let requestStats = [{Source.method: "getLogs", seconds: pageFetchTime}]
+    let requestStats = [
+      {Source.method: "getLogs", seconds: pageFetchTime, responseBlocks: pageUnsafe.responseBlocks},
+    ]
 
     //set height and next from block
     let knownHeight = pageUnsafe.archiveHeight

package/src/sources/EvmHyperSyncSource.res.mjs

@@ -100,7 +100,8 @@ function make(param) {
     let pageFetchTime = Performance.secondsSince(startFetchingBatchTimeRef);
     let requestStats = [{
         method: "getLogs",
-        seconds: pageFetchTime
+        seconds: pageFetchTime,
+        responseBlocks: pageUnsafe.responseBlocks
       }];
     let knownHeight$1 = pageUnsafe.archiveHeight;
     let heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;

package/src/sources/HyperSync.res

@@ -90,6 +90,7 @@ type logsQueryPage = {
   items: array<HyperSyncClient.EventItems.item>,
   nextBlock: int,
   archiveHeight: int,
+  responseBlocks: int,
   // Page store owning this page's raw transactions.
   transactionStore: TransactionStore.t,
   // Page store owning this page's raw blocks.
@@ -165,6 +166,7 @@ module GetLogs = {
     {
       items: res.items,
       nextBlock: res.nextBlock,
+      responseBlocks: res.responseBlocks,
       archiveHeight: res.archiveHeight->Option.getOr(0), //Archive Height is only None if height is 0
       transactionStore,
       blockStore,

package/src/sources/HyperSync.res.mjs

@@ -176,6 +176,7 @@ async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes
     items: res.items,
     nextBlock: res.nextBlock,
     archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0),
+    responseBlocks: res.responseBlocks,
     transactionStore: match[1],
     blockStore: match[2]
   };

package/src/sources/HyperSync.resi

@@ -2,6 +2,7 @@ type logsQueryPage = {
   items: array<HyperSyncClient.EventItems.item>,
   nextBlock: int,
   archiveHeight: int,
+  responseBlocks: int,
   transactionStore: TransactionStore.t,
   blockStore: BlockStore.t,
 }

package/src/sources/HyperSyncClient.res

@@ -283,6 +283,9 @@ module EventItems = {
     archiveHeight: option<int>,
     nextBlock: int,
     items: array<item>,
+    // Blocks the server returned for the query, counted before routing drops
+    // the ones no item joins to.
+    responseBlocks: int,
   }
 }
 

package/src/sources/RequestStat.res

@@ -2,4 +2,11 @@
 // in its own leaf module (rather than on `Source`) so the client FFI bindings
 // can name it without importing `Source` — which would form a dependency cycle
 // through `Env → HyperSyncClient → Source → FetchState → Env`.
-type t = {method: string, seconds: float}
+type t = {
+  method: string,
+  seconds: float,
+  // Blocks the backend returned for this request, before any client-side
+  // routing drops them. Absent for methods whose responses aren't measured in
+  // blocks, so a zero always means the request came back empty.
+  responseBlocks?: int,
+}

package/src/sources/Source.res

@@ -8,9 +8,10 @@ type blockRangeFetchStats = {
 }
 
 // A single backend request a source method actually made (cache/dedup hits
-// aren't requests), with the time it took. SourceManager aggregates these
-// per (source, method) into the envio_source_request_* metrics.
-type requestStat = RequestStat.t = {method: string, seconds: float}
+// aren't requests), with the time it took and how much it brought back.
+// SourceManager aggregates these per (source, method) into the
+// envio_source_request_* and envio_source_response_* metrics.
+type requestStat = RequestStat.t = {method: string, seconds: float, responseBlocks?: int}
 
 // Native clients wrap a failure of a multi-request operation in a structured
 // payload, so the source can still return timings when SourceManager retries

package/src/sources/SourceManager.res

@@ -1,9 +1,14 @@
 type sourceManagerStatus = Idle | WaitingForNewBlock | Querying
 
-// Cumulative per-method request count/time for a source, aggregated from the
-// requestStat arrays returned by its methods. Rendered into
-// envio_source_request_* by Metrics.renderSourceRequests.
-type requestStatAgg = {mutable count: int, mutable seconds: float}
+// Cumulative per-method request stats for a source, aggregated from the
+// requestStat arrays returned by its methods. Rendered into the
+// envio_source_request_* and envio_source_response_* metrics.
+type requestStatAgg = {
+  mutable count: int,
+  mutable seconds: float,
+  mutable responseBlocks: option<int>,
+  mutable emptyResponseCount: int,
+}
 
 type sourceState = {
   source: Source.t,
@@ -19,12 +24,23 @@ let recordStatsInto = (
   aggregates: dict<requestStatAgg>,
   requestStats: array<Source.requestStat>,
 ) => {
-  requestStats->Array.forEach(({method, seconds}) => {
-    switch aggregates->Utils.Dict.dangerouslyGetNonOption(method) {
-    | Some(agg) =>
-      agg.count = agg.count + 1
-      agg.seconds = agg.seconds +. seconds
-    | None => aggregates->Dict.set(method, {count: 1, seconds})
+  requestStats->Array.forEach(({method, seconds, responseBlocks: ?responseBlocks}) => {
+    let agg = switch aggregates->Utils.Dict.dangerouslyGetNonOption(method) {
+    | Some(agg) => agg
+    | None =>
+      let agg = {count: 0, seconds: 0., responseBlocks: None, emptyResponseCount: 0}
+      aggregates->Dict.set(method, agg)
+      agg
+    }
+    agg.count = agg.count + 1
+    agg.seconds = agg.seconds +. seconds
+    switch responseBlocks {
+    | Some(responseBlocks) =>
+      agg.responseBlocks = Some(agg.responseBlocks->Option.getOr(0) + responseBlocks)
+      if responseBlocks === 0 {
+        agg.emptyResponseCount = agg.emptyResponseCount + 1
+      }
+    | None => ()
     }
   })
 }
@@ -40,6 +56,8 @@ type requestStatSample = {
   method: string,
   count: int,
   seconds: float,
+  responseBlocks: option<int>,
+  emptyResponseCount: int,
 }
 
 // Encapsulates the fetching logic for a chain's sources.
@@ -93,6 +111,8 @@ let getRequestStatSamples = (sourceManager: t): array<requestStatSample> => {
         method,
         count: agg.count,
         seconds: agg.seconds,
+        responseBlocks: agg.responseBlocks,
+        emptyResponseCount: agg.emptyResponseCount,
       })
       ->ignore
     })

package/src/sources/SourceManager.res.mjs

@@ -16,17 +16,32 @@ import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_excep
 
 function recordStatsInto(aggregates, requestStats) {
   requestStats.forEach(param => {
-    let seconds = param.seconds;
+    let responseBlocks = param.responseBlocks;
     let method = param.method;
     let agg = aggregates[method];
+    let agg$1;
     if (agg !== undefined) {
-      agg.count = agg.count + 1 | 0;
-      agg.seconds = agg.seconds + seconds;
+      agg$1 = agg;
     } else {
-      aggregates[method] = {
-        count: 1,
-        seconds: seconds
+      let agg$2 = {
+        count: 0,
+        seconds: 0,
+        responseBlocks: undefined,
+        emptyResponseCount: 0
       };
+      aggregates[method] = agg$2;
+      agg$1 = agg$2;
+    }
+    agg$1.count = agg$1.count + 1 | 0;
+    agg$1.seconds = agg$1.seconds + param.seconds;
+    if (responseBlocks !== undefined) {
+      agg$1.responseBlocks = Stdlib_Option.getOr(agg$1.responseBlocks, 0) + responseBlocks | 0;
+      if (responseBlocks === 0) {
+        agg$1.emptyResponseCount = agg$1.emptyResponseCount + 1 | 0;
+        return;
+      } else {
+        return;
+      }
     }
   });
 }
@@ -45,7 +60,9 @@ function getRequestStatSamples(sourceManager) {
         chainId: chainId,
         method: method,
         count: agg.count,
-        seconds: agg.seconds
+        seconds: agg.seconds,
+        responseBlocks: agg.responseBlocks,
+        emptyResponseCount: agg.emptyResponseCount
       });
     });
   });

package/src/sources/SourceManager.resi

@@ -27,6 +27,8 @@ type requestStatSample = {
   method: string,
   count: int,
   seconds: float,
+  responseBlocks: option<int>,
+  emptyResponseCount: int,
 }
 
 let getRequestStatSamples: t => array<requestStatSample>