envio 3.3.0-alpha.7 → 3.3.0-alpha.9

package/src/sources/SourceManager.res.mjs

@@ -14,10 +14,44 @@ import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
+function recordRequestStats(sourceState, requestStats) {
+  requestStats.forEach(param => {
+    let seconds = param.seconds;
+    let method = param.method;
+    let agg = sourceState.requestStats[method];
+    if (agg !== undefined) {
+      agg.count = agg.count + 1 | 0;
+      agg.seconds = agg.seconds + seconds;
+    } else {
+      sourceState.requestStats[method] = {
+        count: 1,
+        seconds: seconds
+      };
+    }
+  });
+}
+
 function getActiveSource(sourceManager) {
   return sourceManager.activeSource;
 }
 
+function getRequestStatSamples(sourceManager) {
+  let samples = [];
+  sourceManager.sourcesState.forEach(sourceState => {
+    let chainId = sourceState.source.chain;
+    Utils.Dict.forEachWithKey(sourceState.requestStats, (agg, method) => {
+      samples.push({
+        sourceName: sourceState.source.name,
+        chainId: chainId,
+        method: method,
+        count: agg.count,
+        seconds: agg.seconds
+      });
+    });
+  });
+  return samples;
+}
+
 function inFlightCount(sourceManager) {
   return sourceManager.fetchingPartitionsCount;
 }
@@ -137,7 +171,8 @@ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeout
       unsubscribe: undefined,
       pendingHeightResolvers: [],
       disabled: false,
-      lastFailedAt: undefined
+      lastFailedAt: undefined,
+      requestStats: {}
     })),
     statusStart: Performance.now(),
     status: "Idle",
@@ -263,7 +298,9 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
         let h = initialHeight;
         while (h <= knownHeight && newHeight.contents <= initialHeight) {
           try {
-            h = await source.getHeightOrThrow();
+            let res = await source.getHeightOrThrow();
+            recordRequestStats(sourceState, res.requestStats);
+            h = res.height;
           } catch (exn) {
             
           }
@@ -282,7 +319,9 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
       }
     } else {
       try {
-        let height$1 = await source.getHeightOrThrow();
+        let res = await source.getHeightOrThrow();
+        recordRequestStats(sourceState, res.requestStats);
+        let height$1 = res.height;
         newHeight.contents = height$1;
         if (height$1 <= knownHeight) {
           retry = 0;
@@ -299,6 +338,10 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
               resolvers.forEach(resolve => resolve(newHeight));
             });
             sourceState.unsubscribe = unsubscribe;
+            recordRequestStats(sourceState, [{
+                method: "heightSubscription",
+                seconds: 0
+              }]);
           } else {
             exit = 1;
           }
@@ -500,8 +543,8 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
       retry: retry
     });
     try {
-      let est = Math.ceil(query.estResponseSize) | 0;
-      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, FetchState.deriveContractNameByAddress(query.addressesByContractName), knownHeight, query.partitionId, query.selection, est > 0 ? est : FetchState.minEstResponseSize | 0, retry, logger$2);
+      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, FetchState.deriveContractNameByAddress(query.addressesByContractName), knownHeight, query.partitionId, query.selection, query.itemsTarget, retry, logger$2);
+      recordRequestStats(sourceState, response.requestStats);
       sourceState.lastFailedAt = undefined;
       responseRef = response;
     } catch (raw_error) {
@@ -637,11 +680,13 @@ async function getBlockHashes(sourceManager, blockNumbers, isRealtime) {
     });
     try {
       let res = await source.getBlockHashes(blockNumbers, logger$1);
-      if (res.TAG === "Ok") {
+      recordRequestStats(sourceState, res.requestStats);
+      let data = res.result;
+      if (data.TAG === "Ok") {
         sourceState.lastFailedAt = undefined;
-        responseRef = res._0;
+        responseRef = data._0;
       } else {
-        throw res._0;
+        throw data._0;
       }
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
@@ -673,6 +718,7 @@ export {
   getSourceRole,
   make,
   getActiveSource,
+  getRequestStatSamples,
   inFlightCount,
   onReorg,
   dispatch,

package/src/sources/SourceManager.resi

@@ -21,6 +21,16 @@ let make: (
 
 let getActiveSource: t => Source.t
 
+type requestStatSample = {
+  sourceName: string,
+  chainId: int,
+  method: string,
+  count: int,
+  seconds: float,
+}
+
+let getRequestStatSamples: t => array<requestStatSample>
+
 let inFlightCount: t => int
 
 let onReorg: (t, ~rollbackTargetBlock: int) => unit

package/src/sources/Svm.res

@@ -16,6 +16,17 @@ let transactionFields =
 // Computed per event at config build and cached on the event config.
 let eventTransactionFieldMask = TransactionStore.makeMaskFn(transactionFields)
 
+// Ordered block field names. The index of each is the field code shared with the
+// Rust store (`SvmBlockField`) — keep this order in sync.
+let blockFields = ["slot", "time", "hash", "height", "parentSlot", "parentHash"]
+
+// `slot`/`time`/`hash` are always included; every other block field is opt-in
+// via `field_selection.block_fields`. All are materialised from the store.
+//
+// One instruction's selected block fields → store selection bitmask. Computed per
+// event at config build and cached on the event config.
+let eventBlockFieldMask = BlockStore.makeMaskFn(blockFields)
+
 let make = (~logger: Pino.t): Ecosystem.t => {
   name: Svm,
   blockNumberName: "height",
@@ -37,8 +48,8 @@ let make = (~logger: Pino.t): Ecosystem.t => {
     Logging.createChildFrom(
       ~logger,
       ~params={
-        "program": eventItem.eventConfig.contractName,
-        "instruction": eventItem.eventConfig.name,
+        "program": eventItem.onEventRegistration.eventConfig.contractName,
+        "instruction": eventItem.onEventRegistration.eventConfig.name,
         "chainId": eventItem.chain->ChainMap.Chain.toChainId,
         "slot": eventItem.blockNumber,
         "programId": instruction.programId,
@@ -84,14 +95,7 @@ let makeRPCSource = (~chain, ~rpc: string, ~sourceFor: Source.sourceFor=Sync): S
       let timerRef = Performance.now()
       let height = await GetFinalizedSlot.route->Rest.fetch((), ~client)
       let seconds = timerRef->Performance.secondsSince
-      Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getSlot")
-      Prometheus.SourceRequestCount.addSeconds(
-        ~sourceName=name,
-        ~chainId,
-        ~method="getSlot",
-        ~seconds,
-      )
-      height
+      {Source.height, requestStats: [{Source.method: "getSlot", seconds}]}
     },
     getItemsOrThrow: (
       ~fromBlock as _,

package/src/sources/Svm.res.mjs

@@ -5,7 +5,7 @@ import * as Rest from "../vendored/Rest.res.mjs";
 import * as Utils from "../Utils.res.mjs";
 import * as Logging from "../Logging.res.mjs";
 import * as Internal from "../Internal.res.mjs";
-import * as Prometheus from "../Prometheus.res.mjs";
+import * as BlockStore from "./BlockStore.res.mjs";
 import * as Performance from "../bindings/Performance.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
@@ -19,6 +19,17 @@ let cleanUpRawEventFieldsInPlace = (fields => {
 
 let eventTransactionFieldMask = TransactionStore.makeMaskFn(Internal.allSvmTransactionFields);
 
+let blockFields = [
+  "slot",
+  "time",
+  "hash",
+  "height",
+  "parentSlot",
+  "parentHash"
+];
+
+let eventBlockFieldMask = BlockStore.makeMaskFn(blockFields);
+
 function make(logger) {
   return {
     name: "svm",
@@ -34,8 +45,8 @@ function make(logger) {
     toEventLogger: eventItem => {
       let instruction = eventItem.payload;
       return Logging.createChildFrom(logger, {
-        program: eventItem.eventConfig.contractName,
-        instruction: eventItem.eventConfig.name,
+        program: eventItem.onEventRegistration.eventConfig.contractName,
+        instruction: eventItem.onEventRegistration.eventConfig.name,
         chainId: eventItem.chain,
         slot: eventItem.blockNumber,
         programId: instruction.programId
@@ -72,9 +83,13 @@ function makeRPCSource(chain, rpc, sourceForOpt) {
       let timerRef = Performance.now();
       let height = await Rest.fetch(route, undefined, client);
       let seconds = Performance.secondsSince(timerRef);
-      Prometheus.SourceRequestCount.increment(name, chain, "getSlot");
-      Prometheus.SourceRequestCount.addSeconds(name, chain, "getSlot", seconds);
-      return height;
+      return {
+        height: height,
+        requestStats: [{
+            method: "getSlot",
+            seconds: seconds
+          }]
+      };
     },
     getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8, param$9) => Stdlib_JsError.throwWithMessage("Svm does not support getting items")
   };
@@ -86,6 +101,8 @@ export {
   cleanUpRawEventFieldsInPlace,
   transactionFields,
   eventTransactionFieldMask,
+  blockFields,
+  eventBlockFieldMask,
   make,
   GetFinalizedSlot,
   makeRPCSource,

package/src/sources/SvmHyperSyncClient.res

@@ -138,13 +138,12 @@ module QueryTypes = {
 }
 
 module ResponseTypes = {
+  // Lean per-slot header for reorg detection and each item's slot/time; the
+  // selectable fields live in the block store and are materialised on demand.
   type block = {
     slot: int,
     blockhash: string,
-    parentSlot?: int,
-    parentBlockhash?: string,
     blockTime?: int,
-    blockHeight?: int,
   }
 
   /// Borsh-decoded view attached by the Rust client. `argsJson`/`accountsJson`
@@ -206,9 +205,9 @@ type queryResponse = ResponseTypes.queryResponse
 
 type t = {
   getHeight: unit => promise<int>,
-  // Returns the response plus a page of raw transactions (kept in Rust),
-  // keyed by (slot, transactionIndex), materialised at batch prep.
-  get: (~query: query) => promise<(queryResponse, TransactionStore.t)>,
+  // Returns the response plus pages of raw transactions and blocks (kept in
+  // Rust), keyed by (slot, transactionIndex) / slot, materialised at batch prep.
+  get: (~query: query) => promise<(queryResponse, TransactionStore.t, BlockStore.t)>,
 }
 
 @send

package/src/sources/SvmHyperSyncSource.res

@@ -4,7 +4,7 @@ type options = {
   chain: ChainMap.Chain.t,
   endpointUrl: string,
   apiToken: option<string>,
-  eventConfigs: array<Internal.svmInstructionEventConfig>,
+  onEventRegistrations: array<Internal.svmOnEventRegistration>,
   clientTimeoutMillis: int,
 }
 
@@ -184,12 +184,12 @@ let parseDecoded = (
   }
 }
 
+// `block` is omitted; it's materialised from the block store at batch prep.
 let toSvmInstruction = (
   instr: SvmHyperSyncClient.ResponseTypes.instruction,
   ~programName,
   ~instructionName,
   ~logs,
-  ~block,
 ): Envio.svmInstruction => {
   programName,
   instructionName,
@@ -204,14 +204,13 @@ let toSvmInstruction = (
   d8: ?instr.d8,
   params: ?(instr.decoded->Option.map(parseDecoded)),
   ?logs,
-  block,
 }
 
 // Probe the discriminator byte-length ordering longest-first. Stops at the
 // first router hit. Falls back to the `_none` key (program-wide handler) when
 // no discriminator-keyed handler matches.
 let probeRouter = (
-  router: EventRouter.t<Internal.svmInstructionEventConfig>,
+  router: EventRouter.t<Internal.svmOnEventRegistration>,
   programId: SvmTypes.Pubkey.t,
   instr: SvmHyperSyncClient.ResponseTypes.instruction,
   byteLengthsDesc: array<int>,
@@ -268,9 +267,30 @@ let toQueryTxField = (field: Internal.svmTransactionField): option<
   | TokenBalances => None
   }
 
-let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: options): t => {
+// Map a selected block field to its HyperSync query column. Slot/Blockhash/
+// BlockTime are requested unconditionally (needed for reorg detection and the
+// item's slot/timestamp), so selecting `slot`/`time`/`hash` adds no extra column.
+let toQueryBlockField = (field: Internal.svmBlockField): option<
+  SvmHyperSyncClient.QueryTypes.blockField,
+> =>
+  switch field {
+  | Slot | Time | Hash => None
+  | Height => Some(BlockHeight)
+  | ParentSlot => Some(ParentSlot)
+  | ParentHash => Some(ParentBlockhash)
+  }
+
+let make = (
+  {chain, endpointUrl, apiToken, onEventRegistrations, clientTimeoutMillis}: options,
+): t => {
   let name = "SvmHyperSync"
-  let chainId = chain->ChainMap.Chain.toChainId
+
+  // Definitions drive query/decode building; the registrations drive routing
+  // (they carry `isWildcard` and become each decoded item's `onEventRegistration`).
+  let eventConfigs =
+    onEventRegistrations->Array.map(reg =>
+      reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig)
+    )
 
   // Built once at startup and handed to the client so `get` decodes matching
   // instructions in Rust rather than per-instruction over the napi boundary.
@@ -285,7 +305,10 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     },
   )
 
-  let (eventRouter, programOrderings) = EventRouter.fromSvmEventConfigsOrThrow(eventConfigs, ~chain)
+  let (eventRouter, programOrderings) = EventRouter.fromSvmEventConfigsOrThrow(
+    onEventRegistrations,
+    ~chain,
+  )
 
   // programId.toString -> sorted-desc byte lengths
   let orderingByProgram = Dict.make()
@@ -332,6 +355,24 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       }
     )
 
+  // Union of selected block fields across the chain's events. `slot`/`time`/
+  // `hash` are always fetched (as Slot/BlockTime/Blockhash); the rest are added
+  // only when an instruction selected them.
+  let blockQueryFields = {
+    let fields: array<SvmHyperSyncClient.QueryTypes.blockField> = [Slot, Blockhash, BlockTime]
+    let selected = Utils.Set.make()
+    eventConfigs->Array.forEach(cfg =>
+      cfg.selectedBlockFields->Utils.Set.forEach(field => selected->Utils.Set.add(field)->ignore)
+    )
+    selected->Utils.Set.forEach(field =>
+      switch field->toQueryBlockField {
+      | Some(queryField) => fields->Array.push(queryField)->ignore
+      | None => ()
+      }
+    )
+    fields
+  }
+
   let getItemsOrThrow = async (
     ~fromBlock,
     ~toBlock,
@@ -353,7 +394,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     // field list returns no rows (instructions and blocks are exempt), so each
     // opted-into table needs its columns spelled out here.
     let fields: SvmHyperSyncClient.QueryTypes.fieldSelection = {
-      block: [Slot, Blockhash, BlockTime],
+      block: blockQueryFields,
       transaction: ?(needsTransactions ? Some(txQueryFields) : None),
       log: ?(needsLogs ? Some([Slot, TransactionIndex, InstructionAddress, Kind, Message]) : None),
       tokenBalance: ?(
@@ -372,9 +413,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       maxNumInstructions: itemsTarget,
     }
 
-    Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getInstructions")
-
-    let (resp, transactionStore) = try await client.get(~query) catch {
+    let (resp, transactionStore, blockStore) = try await client.get(~query) catch {
     | exn =>
       throw(
         Source.GetItemsError(
@@ -393,11 +432,13 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       )
     }
     let pageFetchTime = pageFetchRef->Performance.secondsSince
+    let requestStats = [{Source.method: "getInstructions", seconds: pageFetchTime}]
 
     let parsingRef = Performance.now()
 
-    // Per-slot unix timestamp lookup from the response's `blocks` table. Slots
-    // without a block row (rare; usually skipped slots) fall back to `None`.
+    // Per-slot blockTime lookup from the response's `blocks` table, for the
+    // batch's `latestFetchedBlockTimestamp`. Slots without a block row (rare;
+    // usually skipped slots) fall back to `None`.
     let blockTimeBySlot = Dict.make()
     resp.data.blocks->Array.forEach(b => {
       switch b.blockTime {
@@ -447,7 +488,11 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
 
       switch maybeConfig {
       | None => ()
-      | Some(eventConfig) =>
+      | Some(onEventRegistration) =>
+        let eventConfig =
+          onEventRegistration.eventConfig->(
+            Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig
+          )
         let logKey =
           instr.slot->Int.toString ++
           ":" ++
@@ -466,28 +511,19 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
             )
           )
 
-        let slotKey = instr.slot->Int.toString
-        let blockTime = blockTimeBySlot->Utils.Dict.dangerouslyGetNonOption(slotKey)
         let payload = toSvmInstruction(
           instr,
           ~programName=eventConfig.contractName,
           ~instructionName=eventConfig.name,
           ~logs=eventConfig.includeLogs ? maybeLogs : None,
-          ~block={
-            slot: instr.slot,
-            time: blockTime->Option.getOr(0),
-            hash: "",
-          },
         )
 
         parsedQueueItems
         ->Array.push(
           Internal.Event({
-            eventConfig: (eventConfig :> Internal.eventConfig),
-            timestamp: blockTime->Option.getOr(0),
+            onEventRegistration,
             chain,
             blockNumber: instr.slot,
-            blockHash: "",
             logIndex: synthLogIndex(instr),
             // The parent transaction is materialised from the store at batch prep.
             transactionIndex: instr.transactionIndex,
@@ -522,11 +558,14 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       parsedQueueItems,
       // Raw transactions kept in Rust; materialised (selected fields) at batch prep.
       transactionStore: Some(transactionStore),
+      // Raw blocks kept in Rust; materialised onto the payload at batch prep.
+      blockStore: Some(blockStore),
       latestFetchedBlockNumber: highestSlot,
       stats: {totalTimeElapsed, parsingTimeElapsed, pageFetchTime},
       knownHeight,
       blockHashes,
       fromBlockQueried: fromBlock,
+      requestStats,
     }
   }
 
@@ -535,6 +574,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
   // wire, so we request `maxSlot + 1`; the caller filters to the exact slots.
   let queryBlockDataRange = async (~fromSlot, ~toSlot) => {
     let blockDatas = []
+    let requestStats = []
     let fromRef = ref(fromSlot)
     let keepGoing = ref(true)
     while keepGoing.contents {
@@ -545,9 +585,12 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
         fields: {block: [Slot, Blockhash, BlockTime]},
         maxNumBlocks: 1000,
       }
-      Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getBlockHashes")
-      // Block-only query; the store page is empty.
-      let (resp, _) = await client.get(~query)
+      let timerRef = Performance.now()
+      // Block-only query; the store pages are empty.
+      let (resp, _, _) = await client.get(~query)
+      requestStats
+      ->Array.push({Source.method: "getBlockHashes", seconds: timerRef->Performance.secondsSince})
+      ->ignore
       resp.data.blocks->Array.forEach(b =>
         blockDatas
         ->Array.push({
@@ -566,12 +609,12 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
         fromRef := resp.nextSlot
       }
     }
-    blockDatas
+    (blockDatas, requestStats)
   }
 
   let getBlockHashes = async (~blockNumbers, ~logger as _) =>
     switch blockNumbers->Array.get(0) {
-    | None => Ok([])
+    | None => {Source.result: Ok([]), requestStats: []}
     | Some(firstSlot) =>
       try {
         let minSlot = ref(firstSlot)
@@ -586,14 +629,19 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
           }
           requested->Utils.Set.add(slot)->ignore
         })
-        let blockDatas = await queryBlockDataRange(
+        let (blockDatas, requestStats) = await queryBlockDataRange(
           ~fromSlot=minSlot.contents,
           ~toSlot=maxSlot.contents,
         )
         // Keep one entry per requested slot; drop duplicates and unrelated slots.
-        Ok(blockDatas->Array.filter(data => requested->Utils.Set.delete(data.blockNumber)))
+        {
+          Source.result: Ok(
+            blockDatas->Array.filter(data => requested->Utils.Set.delete(data.blockNumber)),
+          ),
+          requestStats,
+        }
       } catch {
-      | exn => Error(exn)
+      | exn => {Source.result: Error(exn), requestStats: []}
       }
     }
 
@@ -606,16 +654,9 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     getBlockHashes,
     getHeightOrThrow: async () => {
       let timer = Performance.now()
-      let h = await client.getHeight()
+      let height = await client.getHeight()
       let seconds = timer->Performance.secondsSince
-      Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getHeight")
-      Prometheus.SourceRequestCount.addSeconds(
-        ~sourceName=name,
-        ~chainId,
-        ~method="getHeight",
-        ~seconds,
-      )
-      h
+      {height, requestStats: [{method: "getHeight", seconds}]}
     },
     getItemsOrThrow,
   }