envio 3.3.0-alpha.6 → 3.3.0-alpha.8

package/src/sources/RpcWebSocketHeightStream.res.mjs

@@ -3,7 +3,6 @@
 import * as Rpc from "./Rpc.res.mjs";
 import * as Utils from "../Utils.res.mjs";
 import * as Logging from "../Logging.res.mjs";
-import * as Prometheus from "../Prometheus.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
@@ -105,7 +104,6 @@ function subscribe(wsUrl, chainId, onHeight) {
         }
         errorCount.contents = 0;
         resetStaleTimeout();
-        Prometheus.SourceRequestCount.increment("WebSocket", chainId, "eth_subscribe");
         return onHeight(blockNumber._0);
       } catch (raw_e) {
         let e = Primitive_exceptions.internalToException(raw_e);

package/src/sources/SimulateSource.res

@@ -12,11 +12,11 @@ let make = (~items: array<Internal.item>, ~endBlock: int, ~chain: ChainMap.Chain
     poweredByHyperSync: false,
     pollingInterval: 0,
     getBlockHashes: (~blockNumbers as _, ~logger as _) => {
-      Promise.resolve(Ok([]))
+      Promise.resolve({Source.result: Ok([]), requestStats: []})
     },
     getHeightOrThrow: () => {
       // Report at least height 1 so the engine doesn't treat 0 as "no blocks available"
-      Promise.resolve(max(endBlock, 1))
+      Promise.resolve({Source.height: max(endBlock, 1), requestStats: []})
     },
     getItemsOrThrow: (
       ~fromBlock as _,
@@ -26,6 +26,7 @@ let make = (~items: array<Internal.item>, ~endBlock: int, ~chain: ChainMap.Chain
       ~knownHeight as _,
       ~partitionId as _,
       ~selection as _,
+      ~itemsTarget as _,
       ~retry as _,
       ~logger as _,
     ) => {
@@ -42,14 +43,16 @@ let make = (~items: array<Internal.item>, ~endBlock: int, ~chain: ChainMap.Chain
         Source.knownHeight: reportedHeight,
         blockHashes: [],
         parsedQueueItems: result,
-        // Simulate keeps the transaction inline on the payload; no store page.
+        // Simulate keeps the transaction and block inline on the payload; no store pages.
         transactionStore: None,
+        blockStore: None,
         fromBlockQueried: 0,
         latestFetchedBlockNumber: reportedHeight,
         latestFetchedBlockTimestamp: 0,
         stats: {
           totalTimeElapsed: 0.,
         },
+        requestStats: [],
       })
     },
   }

package/src/sources/SimulateSource.res.mjs

@@ -13,11 +13,17 @@ function make(items, endBlock, chain) {
     poweredByHyperSync: false,
     pollingInterval: 0,
     getBlockHashes: (param, param$1) => Promise.resolve({
-      TAG: "Ok",
-      _0: []
+      result: {
+        TAG: "Ok",
+        _0: []
+      },
+      requestStats: []
     }),
-    getHeightOrThrow: () => Promise.resolve(Primitive_int.max(endBlock, 1)),
-    getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8) => {
+    getHeightOrThrow: () => Promise.resolve({
+      height: Primitive_int.max(endBlock, 1),
+      requestStats: []
+    }),
+    getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8, param$9) => {
       let result = delivered.contents ? [] : (delivered.contents = true, items);
       let reportedHeight = Primitive_int.max(endBlock, 1);
       return Promise.resolve({
@@ -25,12 +31,14 @@ function make(items, endBlock, chain) {
         blockHashes: [],
         parsedQueueItems: result,
         transactionStore: undefined,
+        blockStore: undefined,
         fromBlockQueried: 0,
         latestFetchedBlockNumber: reportedHeight,
         latestFetchedBlockTimestamp: 0,
         stats: {
           "total time elapsed (s)": 0
-        }
+        },
+        requestStats: []
       });
     },
     simulateItems: items

package/src/sources/Source.res

@@ -7,6 +7,11 @@ type blockRangeFetchStats = {
   @as("page fetch time (s)") pageFetchTime?: float,
 }
 
+// A single backend request a source method actually made (cache/dedup hits
+// aren't requests), with the time it took. SourceManager aggregates these
+// per (source, method) into the envio_source_request_* metrics.
+type requestStat = {method: string, seconds: float}
+
 /**
 Thes response returned from a block range fetch
 */
@@ -22,10 +27,22 @@ type blockRangeFetchResponse = {
   // transactionIndex); merged into the chain's store on apply. `None` for
   // sources that keep the transaction inline on the payload (RPC/Fuel/Simulate).
   transactionStore: option<TransactionStore.t>,
+  // Page of blocks for this response's items, keyed by block number; merged into
+  // the chain's store on apply. `None` for sources that keep the block fully
+  // inline on the payload (RPC/Fuel/Simulate).
+  blockStore: option<BlockStore.t>,
   fromBlockQueried: int,
   latestFetchedBlockNumber: int,
   latestFetchedBlockTimestamp: int,
   stats: blockRangeFetchStats,
+  requestStats: array<requestStat>,
+}
+
+type getHeightResponse = {height: int, requestStats: array<requestStat>}
+
+type getBlockHashesResponse = {
+  result: result<array<ReorgDetection.blockDataWithTimestamp>, exn>,
+  requestStats: array<requestStat>,
 }
 
 type getItemsRetry =
@@ -51,11 +68,8 @@ type t = {
   poweredByHyperSync: bool,
   /* Frequency (in ms) used when polling for new events on this network. */
   pollingInterval: int,
-  getBlockHashes: (
-    ~blockNumbers: array<int>,
-    ~logger: Pino.t,
-  ) => promise<result<array<ReorgDetection.blockDataWithTimestamp>, exn>>,
-  getHeightOrThrow: unit => promise<int>,
+  getBlockHashes: (~blockNumbers: array<int>, ~logger: Pino.t) => promise<getBlockHashesResponse>,
+  getHeightOrThrow: unit => promise<getHeightResponse>,
   getItemsOrThrow: (
     ~fromBlock: int,
     ~toBlock: option<int>,
@@ -64,6 +78,12 @@ type t = {
     ~knownHeight: int,
     ~partitionId: string,
     ~selection: FetchState.selection,
+    // Soft cap on the number of primary items (logs/instructions/receipts) the
+    // source should ask its backend for, from the query's own estResponseSize.
+    // A HyperSync-backed source enforces it server-side, so a wrong estimate
+    // truncates the response instead of overshooting the shared buffer. Sources
+    // without an equivalent lever (RPC, Fuel, Simulate) ignore it.
+    ~itemsTarget: int,
     ~retry: int,
     ~logger: Pino.t,
   ) => promise<blockRangeFetchResponse>,

package/src/sources/SourceManager.res

@@ -1,5 +1,10 @@
 type sourceManagerStatus = Idle | WaitingForNewBlock | Querieng
 
+// Cumulative per-method request count/time for a source, aggregated from the
+// requestStat arrays returned by its methods. Rendered into
+// envio_source_request_* by Metrics.renderSourceRequests.
+type requestStatAgg = {mutable count: int, mutable seconds: float}
+
 type sourceState = {
   source: Source.t,
   mutable knownHeight: int,
@@ -9,6 +14,28 @@ type sourceState = {
   // Timestamp (ms) when this source last failed during executeQuery.
   // Used to decide when to attempt recovery to this source.
   mutable lastFailedAt: option<float>,
+  requestStats: dict<requestStatAgg>,
+}
+
+let recordRequestStats = (sourceState: sourceState, requestStats: array<Source.requestStat>) => {
+  requestStats->Array.forEach(({method, seconds}) => {
+    switch sourceState.requestStats->Utils.Dict.dangerouslyGetNonOption(method) {
+    | Some(agg) =>
+      agg.count = agg.count + 1
+      agg.seconds = agg.seconds +. seconds
+    | None => sourceState.requestStats->Dict.set(method, {count: 1, seconds})
+    }
+  })
+}
+
+// Flattened (source, method) aggregates for Metrics.renderSourceRequests to
+// inline into the /metrics response.
+type requestStatSample = {
+  sourceName: string,
+  chainId: int,
+  method: string,
+  count: int,
+  seconds: float,
 }
 
 // Encapsulates the fetching logic for a chain's sources.
@@ -46,6 +73,25 @@ type t = {
 
 let getActiveSource = sourceManager => sourceManager.activeSource
 
+let getRequestStatSamples = (sourceManager: t): array<requestStatSample> => {
+  let samples = []
+  sourceManager.sourcesState->Array.forEach(sourceState => {
+    let chainId = sourceState.source.chain->ChainMap.Chain.toChainId
+    sourceState.requestStats->Utils.Dict.forEachWithKey((agg, method) => {
+      samples
+      ->Array.push({
+        sourceName: sourceState.source.name,
+        chainId,
+        method,
+        count: agg.count,
+        seconds: agg.seconds,
+      })
+      ->ignore
+    })
+  })
+  samples
+}
+
 // Partition queries currently in flight on this chain's sources. Summed across
 // chains by CrossChainState to enforce the indexer-wide concurrency budget.
 let inFlightCount = sourceManager => sourceManager.fetchingPartitionsCount
@@ -187,6 +233,7 @@ let make = (
       pendingHeightResolvers: [],
       disabled: false,
       lastFailedAt: None,
+      requestStats: Dict.make(),
     }),
     activeSource: initialActiveSource,
     waitingForNewBlockStateId: None,
@@ -344,7 +391,9 @@ let getSourceNewHeight = async (
         let h = ref(initialHeight)
         while h.contents <= knownHeight && !(newHeight.contents > initialHeight) {
           try {
-            h := (await source.getHeightOrThrow())
+            let res = await source.getHeightOrThrow()
+            sourceState->recordRequestStats(res.requestStats)
+            h := res.height
           } catch {
           | _ => ()
           }
@@ -363,7 +412,9 @@ let getSourceNewHeight = async (
     | None =>
       // No subscription, use REST polling
       try {
-        let height = await source.getHeightOrThrow()
+        let res = await source.getHeightOrThrow()
+        sourceState->recordRequestStats(res.requestStats)
+        let height = res.height
 
         newHeight := height
         if height <= knownHeight {
@@ -385,6 +436,9 @@ let getSourceNewHeight = async (
               }
             })
             sourceState.unsubscribe = Some(unsubscribe)
+            // Count a subscription (re)start rather than every pushed height —
+            // there's no request/response to time here.
+            sourceState->recordRequestStats([{Source.method: "heightSubscription", seconds: 0.}])
           | _ =>
             // Slowdown polling when the chain isn't progressing
             let pollingInterval = if reducedPolling {
@@ -680,9 +734,17 @@ let executeQuery = async (
         ~partitionId=query.partitionId,
         ~knownHeight,
         ~selection=query.selection,
+        // Ceil (not truncate) so a sub-1 target from a sparse partition over a
+        // small range doesn't round down to 0 and ask the backend to cap the
+        // response at nothing — 0 items is indistinguishable from "no signal".
+        ~itemsTarget={
+          let target = query.itemsTarget->Math.ceil->Float.toInt
+          target > 0 ? target : FetchState.minItemsTarget->Float.toInt
+        },
         ~retry,
         ~logger,
       )
+      sourceState->recordRequestStats(response.requestStats)
       sourceState.lastFailedAt = None
       responseRef := Some(response)
     } catch {
@@ -822,7 +884,8 @@ let getBlockHashes = async (sourceManager: t, ~blockNumbers: array<int>, ~isReal
 
     try {
       let res = await source.getBlockHashes(~blockNumbers, ~logger)
-      switch res {
+      sourceState->recordRequestStats(res.requestStats)
+      switch res.result {
       | Ok(data) =>
         sourceState.lastFailedAt = None
         responseRef := Some(data)

package/src/sources/SourceManager.res.mjs

@@ -14,10 +14,44 @@ import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
+function recordRequestStats(sourceState, requestStats) {
+  requestStats.forEach(param => {
+    let seconds = param.seconds;
+    let method = param.method;
+    let agg = sourceState.requestStats[method];
+    if (agg !== undefined) {
+      agg.count = agg.count + 1 | 0;
+      agg.seconds = agg.seconds + seconds;
+    } else {
+      sourceState.requestStats[method] = {
+        count: 1,
+        seconds: seconds
+      };
+    }
+  });
+}
+
 function getActiveSource(sourceManager) {
   return sourceManager.activeSource;
 }
 
+function getRequestStatSamples(sourceManager) {
+  let samples = [];
+  sourceManager.sourcesState.forEach(sourceState => {
+    let chainId = sourceState.source.chain;
+    Utils.Dict.forEachWithKey(sourceState.requestStats, (agg, method) => {
+      samples.push({
+        sourceName: sourceState.source.name,
+        chainId: chainId,
+        method: method,
+        count: agg.count,
+        seconds: agg.seconds
+      });
+    });
+  });
+  return samples;
+}
+
 function inFlightCount(sourceManager) {
   return sourceManager.fetchingPartitionsCount;
 }
@@ -137,7 +171,8 @@ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeout
       unsubscribe: undefined,
       pendingHeightResolvers: [],
       disabled: false,
-      lastFailedAt: undefined
+      lastFailedAt: undefined,
+      requestStats: {}
     })),
     statusStart: Performance.now(),
     status: "Idle",
@@ -263,7 +298,9 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
         let h = initialHeight;
         while (h <= knownHeight && newHeight.contents <= initialHeight) {
           try {
-            h = await source.getHeightOrThrow();
+            let res = await source.getHeightOrThrow();
+            recordRequestStats(sourceState, res.requestStats);
+            h = res.height;
           } catch (exn) {
             
           }
@@ -282,7 +319,9 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
       }
     } else {
       try {
-        let height$1 = await source.getHeightOrThrow();
+        let res = await source.getHeightOrThrow();
+        recordRequestStats(sourceState, res.requestStats);
+        let height$1 = res.height;
         newHeight.contents = height$1;
         if (height$1 <= knownHeight) {
           retry = 0;
@@ -299,6 +338,10 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
               resolvers.forEach(resolve => resolve(newHeight));
             });
             sourceState.unsubscribe = unsubscribe;
+            recordRequestStats(sourceState, [{
+                method: "heightSubscription",
+                seconds: 0
+              }]);
           } else {
             exit = 1;
           }
@@ -500,7 +543,9 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
       retry: retry
     });
     try {
-      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, FetchState.deriveContractNameByAddress(query.addressesByContractName), knownHeight, query.partitionId, query.selection, retry, logger$2);
+      let target = Math.ceil(query.itemsTarget) | 0;
+      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, FetchState.deriveContractNameByAddress(query.addressesByContractName), knownHeight, query.partitionId, query.selection, target > 0 ? target : FetchState.minItemsTarget | 0, retry, logger$2);
+      recordRequestStats(sourceState, response.requestStats);
       sourceState.lastFailedAt = undefined;
       responseRef = response;
     } catch (raw_error) {
@@ -636,11 +681,13 @@ async function getBlockHashes(sourceManager, blockNumbers, isRealtime) {
     });
     try {
       let res = await source.getBlockHashes(blockNumbers, logger$1);
-      if (res.TAG === "Ok") {
+      recordRequestStats(sourceState, res.requestStats);
+      let data = res.result;
+      if (data.TAG === "Ok") {
         sourceState.lastFailedAt = undefined;
-        responseRef = res._0;
+        responseRef = data._0;
       } else {
-        throw res._0;
+        throw data._0;
       }
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
@@ -672,6 +719,7 @@ export {
   getSourceRole,
   make,
   getActiveSource,
+  getRequestStatSamples,
   inFlightCount,
   onReorg,
   dispatch,

package/src/sources/SourceManager.resi

@@ -21,6 +21,16 @@ let make: (
 
 let getActiveSource: t => Source.t
 
+type requestStatSample = {
+  sourceName: string,
+  chainId: int,
+  method: string,
+  count: int,
+  seconds: float,
+}
+
+let getRequestStatSamples: t => array<requestStatSample>
+
 let inFlightCount: t => int
 
 let onReorg: (t, ~rollbackTargetBlock: int) => unit

package/src/sources/Svm.res

@@ -16,6 +16,17 @@ let transactionFields =
 // Computed per event at config build and cached on the event config.
 let eventTransactionFieldMask = TransactionStore.makeMaskFn(transactionFields)
 
+// Ordered block field names. The index of each is the field code shared with the
+// Rust store (`SvmBlockField`) — keep this order in sync.
+let blockFields = ["slot", "time", "hash", "height", "parentSlot", "parentHash"]
+
+// `slot`/`time`/`hash` are always included; every other block field is opt-in
+// via `field_selection.block_fields`. All are materialised from the store.
+//
+// One instruction's selected block fields → store selection bitmask. Computed per
+// event at config build and cached on the event config.
+let eventBlockFieldMask = BlockStore.makeMaskFn(blockFields)
+
 let make = (~logger: Pino.t): Ecosystem.t => {
   name: Svm,
   blockNumberName: "height",
@@ -37,8 +48,8 @@ let make = (~logger: Pino.t): Ecosystem.t => {
     Logging.createChildFrom(
       ~logger,
       ~params={
-        "program": eventItem.eventConfig.contractName,
-        "instruction": eventItem.eventConfig.name,
+        "program": eventItem.onEventRegistration.eventConfig.contractName,
+        "instruction": eventItem.onEventRegistration.eventConfig.name,
         "chainId": eventItem.chain->ChainMap.Chain.toChainId,
         "slot": eventItem.blockNumber,
         "programId": instruction.programId,
@@ -84,14 +95,7 @@ let makeRPCSource = (~chain, ~rpc: string, ~sourceFor: Source.sourceFor=Sync): S
       let timerRef = Performance.now()
       let height = await GetFinalizedSlot.route->Rest.fetch((), ~client)
       let seconds = timerRef->Performance.secondsSince
-      Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getSlot")
-      Prometheus.SourceRequestCount.addSeconds(
-        ~sourceName=name,
-        ~chainId,
-        ~method="getSlot",
-        ~seconds,
-      )
-      height
+      {Source.height, requestStats: [{Source.method: "getSlot", seconds}]}
     },
     getItemsOrThrow: (
       ~fromBlock as _,
@@ -101,6 +105,7 @@ let makeRPCSource = (~chain, ~rpc: string, ~sourceFor: Source.sourceFor=Sync): S
       ~knownHeight as _,
       ~partitionId as _,
       ~selection as _,
+      ~itemsTarget as _,
       ~retry as _,
       ~logger as _,
     ) => JsError.throwWithMessage("Svm does not support getting items"),

package/src/sources/Svm.res.mjs

@@ -5,7 +5,7 @@ import * as Rest from "../vendored/Rest.res.mjs";
 import * as Utils from "../Utils.res.mjs";
 import * as Logging from "../Logging.res.mjs";
 import * as Internal from "../Internal.res.mjs";
-import * as Prometheus from "../Prometheus.res.mjs";
+import * as BlockStore from "./BlockStore.res.mjs";
 import * as Performance from "../bindings/Performance.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
@@ -19,6 +19,17 @@ let cleanUpRawEventFieldsInPlace = (fields => {
 
 let eventTransactionFieldMask = TransactionStore.makeMaskFn(Internal.allSvmTransactionFields);
 
+let blockFields = [
+  "slot",
+  "time",
+  "hash",
+  "height",
+  "parentSlot",
+  "parentHash"
+];
+
+let eventBlockFieldMask = BlockStore.makeMaskFn(blockFields);
+
 function make(logger) {
   return {
     name: "svm",
@@ -34,8 +45,8 @@ function make(logger) {
     toEventLogger: eventItem => {
       let instruction = eventItem.payload;
       return Logging.createChildFrom(logger, {
-        program: eventItem.eventConfig.contractName,
-        instruction: eventItem.eventConfig.name,
+        program: eventItem.onEventRegistration.eventConfig.contractName,
+        instruction: eventItem.onEventRegistration.eventConfig.name,
         chainId: eventItem.chain,
         slot: eventItem.blockNumber,
         programId: instruction.programId
@@ -72,11 +83,15 @@ function makeRPCSource(chain, rpc, sourceForOpt) {
       let timerRef = Performance.now();
       let height = await Rest.fetch(route, undefined, client);
       let seconds = Performance.secondsSince(timerRef);
-      Prometheus.SourceRequestCount.increment(name, chain, "getSlot");
-      Prometheus.SourceRequestCount.addSeconds(name, chain, "getSlot", seconds);
-      return height;
+      return {
+        height: height,
+        requestStats: [{
+            method: "getSlot",
+            seconds: seconds
+          }]
+      };
     },
-    getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8) => Stdlib_JsError.throwWithMessage("Svm does not support getting items")
+    getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8, param$9) => Stdlib_JsError.throwWithMessage("Svm does not support getting items")
   };
 }
 
@@ -86,6 +101,8 @@ export {
   cleanUpRawEventFieldsInPlace,
   transactionFields,
   eventTransactionFieldMask,
+  blockFields,
+  eventBlockFieldMask,
   make,
   GetFinalizedSlot,
   makeRPCSource,

package/src/sources/SvmHyperSyncClient.res

@@ -138,13 +138,12 @@ module QueryTypes = {
 }
 
 module ResponseTypes = {
+  // Lean per-slot header for reorg detection and each item's slot/time; the
+  // selectable fields live in the block store and are materialised on demand.
   type block = {
     slot: int,
     blockhash: string,
-    parentSlot?: int,
-    parentBlockhash?: string,
     blockTime?: int,
-    blockHeight?: int,
   }
 
   /// Borsh-decoded view attached by the Rust client. `argsJson`/`accountsJson`
@@ -206,9 +205,9 @@ type queryResponse = ResponseTypes.queryResponse
 
 type t = {
   getHeight: unit => promise<int>,
-  // Returns the response plus a page of raw transactions (kept in Rust),
-  // keyed by (slot, transactionIndex), materialised at batch prep.
-  get: (~query: query) => promise<(queryResponse, TransactionStore.t)>,
+  // Returns the response plus pages of raw transactions and blocks (kept in
+  // Rust), keyed by (slot, transactionIndex) / slot, materialised at batch prep.
+  get: (~query: query) => promise<(queryResponse, TransactionStore.t, BlockStore.t)>,
 }
 
 @send