envio 3.3.0-alpha.6 → 3.3.0-alpha.8

package/src/sources/SvmHyperSyncSource.res

@@ -4,7 +4,7 @@ type options = {
   chain: ChainMap.Chain.t,
   endpointUrl: string,
   apiToken: option<string>,
-  eventConfigs: array<Internal.svmInstructionEventConfig>,
+  onEventRegistrations: array<Internal.svmOnEventRegistration>,
   clientTimeoutMillis: int,
 }
 
@@ -184,12 +184,12 @@ let parseDecoded = (
   }
 }
 
+// `block` is omitted; it's materialised from the block store at batch prep.
 let toSvmInstruction = (
   instr: SvmHyperSyncClient.ResponseTypes.instruction,
   ~programName,
   ~instructionName,
   ~logs,
-  ~block,
 ): Envio.svmInstruction => {
   programName,
   instructionName,
@@ -204,14 +204,13 @@ let toSvmInstruction = (
   d8: ?instr.d8,
   params: ?(instr.decoded->Option.map(parseDecoded)),
   ?logs,
-  block,
 }
 
 // Probe the discriminator byte-length ordering longest-first. Stops at the
 // first router hit. Falls back to the `_none` key (program-wide handler) when
 // no discriminator-keyed handler matches.
 let probeRouter = (
-  router: EventRouter.t<Internal.svmInstructionEventConfig>,
+  router: EventRouter.t<Internal.svmOnEventRegistration>,
   programId: SvmTypes.Pubkey.t,
   instr: SvmHyperSyncClient.ResponseTypes.instruction,
   byteLengthsDesc: array<int>,
@@ -268,9 +267,30 @@ let toQueryTxField = (field: Internal.svmTransactionField): option<
   | TokenBalances => None
   }
 
-let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: options): t => {
+// Map a selected block field to its HyperSync query column. Slot/Blockhash/
+// BlockTime are requested unconditionally (needed for reorg detection and the
+// item's slot/timestamp), so selecting `slot`/`time`/`hash` adds no extra column.
+let toQueryBlockField = (field: Internal.svmBlockField): option<
+  SvmHyperSyncClient.QueryTypes.blockField,
+> =>
+  switch field {
+  | Slot | Time | Hash => None
+  | Height => Some(BlockHeight)
+  | ParentSlot => Some(ParentSlot)
+  | ParentHash => Some(ParentBlockhash)
+  }
+
+let make = (
+  {chain, endpointUrl, apiToken, onEventRegistrations, clientTimeoutMillis}: options,
+): t => {
   let name = "SvmHyperSync"
-  let chainId = chain->ChainMap.Chain.toChainId
+
+  // Definitions drive query/decode building; the registrations drive routing
+  // (they carry `isWildcard` and become each decoded item's `onEventRegistration`).
+  let eventConfigs =
+    onEventRegistrations->Array.map(reg =>
+      reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig)
+    )
 
   // Built once at startup and handed to the client so `get` decodes matching
   // instructions in Rust rather than per-instruction over the napi boundary.
@@ -285,7 +305,10 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     },
   )
 
-  let (eventRouter, programOrderings) = EventRouter.fromSvmEventConfigsOrThrow(eventConfigs, ~chain)
+  let (eventRouter, programOrderings) = EventRouter.fromSvmEventConfigsOrThrow(
+    onEventRegistrations,
+    ~chain,
+  )
 
   // programId.toString -> sorted-desc byte lengths
   let orderingByProgram = Dict.make()
@@ -332,6 +355,24 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       }
     )
 
+  // Union of selected block fields across the chain's events. `slot`/`time`/
+  // `hash` are always fetched (as Slot/BlockTime/Blockhash); the rest are added
+  // only when an instruction selected them.
+  let blockQueryFields = {
+    let fields: array<SvmHyperSyncClient.QueryTypes.blockField> = [Slot, Blockhash, BlockTime]
+    let selected = Utils.Set.make()
+    eventConfigs->Array.forEach(cfg =>
+      cfg.selectedBlockFields->Utils.Set.forEach(field => selected->Utils.Set.add(field)->ignore)
+    )
+    selected->Utils.Set.forEach(field =>
+      switch field->toQueryBlockField {
+      | Some(queryField) => fields->Array.push(queryField)->ignore
+      | None => ()
+      }
+    )
+    fields
+  }
+
   let getItemsOrThrow = async (
     ~fromBlock,
     ~toBlock,
@@ -340,6 +381,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     ~knownHeight,
     ~partitionId as _,
     ~selection as _,
+    ~itemsTarget,
     ~retry,
     ~logger,
   ) => {
@@ -352,7 +394,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     // field list returns no rows (instructions and blocks are exempt), so each
     // opted-into table needs its columns spelled out here.
     let fields: SvmHyperSyncClient.QueryTypes.fieldSelection = {
-      block: [Slot, Blockhash, BlockTime],
+      block: blockQueryFields,
       transaction: ?(needsTransactions ? Some(txQueryFields) : None),
       log: ?(needsLogs ? Some([Slot, TransactionIndex, InstructionAddress, Kind, Message]) : None),
       tokenBalance: ?(
@@ -368,11 +410,10 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       toSlot: ?(toBlock->Option.map(toBlock => toBlock + 1)),
       instructions: instructionSelections,
       fields,
+      maxNumInstructions: itemsTarget,
     }
 
-    Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getInstructions")
-
-    let (resp, transactionStore) = try await client.get(~query) catch {
+    let (resp, transactionStore, blockStore) = try await client.get(~query) catch {
     | exn =>
       throw(
         Source.GetItemsError(
@@ -391,11 +432,13 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       )
     }
     let pageFetchTime = pageFetchRef->Performance.secondsSince
+    let requestStats = [{Source.method: "getInstructions", seconds: pageFetchTime}]
 
     let parsingRef = Performance.now()
 
-    // Per-slot unix timestamp lookup from the response's `blocks` table. Slots
-    // without a block row (rare; usually skipped slots) fall back to `None`.
+    // Per-slot blockTime lookup from the response's `blocks` table, for the
+    // batch's `latestFetchedBlockTimestamp`. Slots without a block row (rare;
+    // usually skipped slots) fall back to `None`.
     let blockTimeBySlot = Dict.make()
     resp.data.blocks->Array.forEach(b => {
       switch b.blockTime {
@@ -445,7 +488,11 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
 
       switch maybeConfig {
       | None => ()
-      | Some(eventConfig) =>
+      | Some(onEventRegistration) =>
+        let eventConfig =
+          onEventRegistration.eventConfig->(
+            Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig
+          )
         let logKey =
           instr.slot->Int.toString ++
           ":" ++
@@ -464,28 +511,19 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
             )
           )
 
-        let slotKey = instr.slot->Int.toString
-        let blockTime = blockTimeBySlot->Utils.Dict.dangerouslyGetNonOption(slotKey)
         let payload = toSvmInstruction(
           instr,
           ~programName=eventConfig.contractName,
           ~instructionName=eventConfig.name,
           ~logs=eventConfig.includeLogs ? maybeLogs : None,
-          ~block={
-            slot: instr.slot,
-            time: blockTime->Option.getOr(0),
-            hash: "",
-          },
         )
 
         parsedQueueItems
         ->Array.push(
           Internal.Event({
-            eventConfig: (eventConfig :> Internal.eventConfig),
-            timestamp: blockTime->Option.getOr(0),
+            onEventRegistration,
             chain,
             blockNumber: instr.slot,
-            blockHash: "",
             logIndex: synthLogIndex(instr),
             // The parent transaction is materialised from the store at batch prep.
             transactionIndex: instr.transactionIndex,
@@ -520,11 +558,14 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       parsedQueueItems,
       // Raw transactions kept in Rust; materialised (selected fields) at batch prep.
       transactionStore: Some(transactionStore),
+      // Raw blocks kept in Rust; materialised onto the payload at batch prep.
+      blockStore: Some(blockStore),
       latestFetchedBlockNumber: highestSlot,
       stats: {totalTimeElapsed, parsingTimeElapsed, pageFetchTime},
       knownHeight,
       blockHashes,
       fromBlockQueried: fromBlock,
+      requestStats,
     }
   }
 
@@ -533,6 +574,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
   // wire, so we request `maxSlot + 1`; the caller filters to the exact slots.
   let queryBlockDataRange = async (~fromSlot, ~toSlot) => {
     let blockDatas = []
+    let requestStats = []
     let fromRef = ref(fromSlot)
     let keepGoing = ref(true)
     while keepGoing.contents {
@@ -543,9 +585,12 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
         fields: {block: [Slot, Blockhash, BlockTime]},
         maxNumBlocks: 1000,
       }
-      Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getBlockHashes")
-      // Block-only query; the store page is empty.
-      let (resp, _) = await client.get(~query)
+      let timerRef = Performance.now()
+      // Block-only query; the store pages are empty.
+      let (resp, _, _) = await client.get(~query)
+      requestStats
+      ->Array.push({Source.method: "getBlockHashes", seconds: timerRef->Performance.secondsSince})
+      ->ignore
       resp.data.blocks->Array.forEach(b =>
         blockDatas
         ->Array.push({
@@ -564,12 +609,12 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
         fromRef := resp.nextSlot
       }
     }
-    blockDatas
+    (blockDatas, requestStats)
   }
 
   let getBlockHashes = async (~blockNumbers, ~logger as _) =>
     switch blockNumbers->Array.get(0) {
-    | None => Ok([])
+    | None => {Source.result: Ok([]), requestStats: []}
     | Some(firstSlot) =>
       try {
         let minSlot = ref(firstSlot)
@@ -584,14 +629,19 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
           }
           requested->Utils.Set.add(slot)->ignore
         })
-        let blockDatas = await queryBlockDataRange(
+        let (blockDatas, requestStats) = await queryBlockDataRange(
           ~fromSlot=minSlot.contents,
           ~toSlot=maxSlot.contents,
         )
         // Keep one entry per requested slot; drop duplicates and unrelated slots.
-        Ok(blockDatas->Array.filter(data => requested->Utils.Set.delete(data.blockNumber)))
+        {
+          Source.result: Ok(
+            blockDatas->Array.filter(data => requested->Utils.Set.delete(data.blockNumber)),
+          ),
+          requestStats,
+        }
       } catch {
-      | exn => Error(exn)
+      | exn => {Source.result: Error(exn), requestStats: []}
       }
     }
 
@@ -604,16 +654,9 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     getBlockHashes,
     getHeightOrThrow: async () => {
       let timer = Performance.now()
-      let h = await client.getHeight()
+      let height = await client.getHeight()
       let seconds = timer->Performance.secondsSince
-      Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getHeight")
-      Prometheus.SourceRequestCount.addSeconds(
-        ~sourceName=name,
-        ~chainId,
-        ~method="getHeight",
-        ~seconds,
-      )
-      h
+      {height, requestStats: [{method: "getHeight", seconds}]}
     },
     getItemsOrThrow,
   }

package/src/sources/SvmHyperSyncSource.res.mjs

@@ -1,7 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Source from "./Source.res.mjs";
-import * as Prometheus from "../Prometheus.res.mjs";
 import * as EventRouter from "./EventRouter.res.mjs";
 import * as Performance from "../bindings/Performance.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
@@ -166,7 +165,7 @@ function parseDecoded(d) {
   };
 }
 
-function toSvmInstruction(instr, programName, instructionName, logs, block) {
+function toSvmInstruction(instr, programName, instructionName, logs) {
   return {
     programName: programName,
     instructionName: instructionName,
@@ -180,8 +179,7 @@ function toSvmInstruction(instr, programName, instructionName, logs, block) {
     d4: instr.d4,
     d8: instr.d8,
     params: Stdlib_Option.map(instr.decoded, parseDecoded),
-    logs: logs,
-    block: block
+    logs: logs
   };
 }
 
@@ -248,13 +246,26 @@ function toQueryTxField(field) {
   }
 }
 
+function toQueryBlockField(field) {
+  switch (field) {
+    case "height" :
+      return "block_height";
+    case "parentSlot" :
+      return "parent_slot";
+    case "parentHash" :
+      return "parent_blockhash";
+    default:
+      return;
+  }
+}
+
 function make(param) {
-  let eventConfigs = param.eventConfigs;
+  let onEventRegistrations = param.onEventRegistrations;
   let chain = param.chain;
-  let name = "SvmHyperSync";
+  let eventConfigs = onEventRegistrations.map(reg => reg.eventConfig);
   let programSchemas = buildProgramSchemas(eventConfigs);
   let client = SvmHyperSyncClient.make(param.endpointUrl, param.apiToken, param.clientTimeoutMillis, undefined, undefined, undefined, programSchemas.length !== 0 ? programSchemas : undefined);
-  let match = EventRouter.fromSvmEventConfigsOrThrow(eventConfigs, chain);
+  let match = EventRouter.fromSvmEventConfigsOrThrow(onEventRegistrations, chain);
   let eventRouter = match[0];
   let orderingByProgram = {};
   match[1].forEach(o => {
@@ -288,15 +299,29 @@ function make(param) {
         return true;
     }
   });
-  let getItemsOrThrow = async (fromBlock, toBlock, param, contractNameByAddress, knownHeight, param$1, param$2, retry, logger) => {
+  let fields$1 = [
+    "slot",
+    "blockhash",
+    "block_time"
+  ];
+  let selected = new Set();
+  eventConfigs.forEach(cfg => {
+    cfg.selectedBlockFields.forEach(field => {
+      selected.add(field);
+    });
+  });
+  selected.forEach(field => {
+    let queryField = toQueryBlockField(field);
+    if (queryField !== undefined) {
+      fields$1.push(queryField);
+      return;
+    }
+  });
+  let getItemsOrThrow = async (fromBlock, toBlock, param, contractNameByAddress, knownHeight, param$1, param$2, itemsTarget, retry, logger) => {
     let totalTimeRef = Performance.now();
     let pageFetchRef = Performance.now();
     let instructionSelections = buildInstructionSelections(eventConfigs);
-    let fields_block = [
-      "slot",
-      "blockhash",
-      "block_time"
-    ];
+    let fields_block = fields$1;
     let fields_transaction = needsTransactions ? fields : undefined;
     let fields_log = needsLogs ? [
         "slot",
@@ -314,7 +339,7 @@ function make(param) {
         "pre_amount",
         "post_amount"
       ] : undefined;
-    let fields$1 = {
+    let fields$2 = {
       block: fields_block,
       transaction: fields_transaction,
       log: fields_log,
@@ -322,14 +347,15 @@ function make(param) {
     };
     let query_toSlot = Stdlib_Option.map(toBlock, toBlock => toBlock + 1 | 0);
     let query_instructions = instructionSelections;
-    let query_fields = fields$1;
+    let query_fields = fields$2;
+    let query_maxNumInstructions = itemsTarget;
     let query = {
       fromSlot: fromBlock,
       toSlot: query_toSlot,
       instructions: query_instructions,
-      fields: query_fields
+      fields: query_fields,
+      maxNumInstructions: query_maxNumInstructions
     };
-    Prometheus.SourceRequestCount.increment(name, chain, "getInstructions");
     let match;
     try {
       match = await client.get(query);
@@ -352,6 +378,10 @@ function make(param) {
     }
     let resp = match[0];
     let pageFetchTime = Performance.secondsSince(pageFetchRef);
+    let requestStats = [{
+        method: "getInstructions",
+        seconds: pageFetchTime
+      }];
     let parsingRef = Performance.now();
     let blockTimeBySlot = {};
     resp.data.blocks.forEach(b => {
@@ -386,25 +416,18 @@ function make(param) {
       let contractAddress = instr.programId;
       let maybeConfig = probeRouter(eventRouter, programId, instr, byteLengths, contractAddress, contractNameByAddress);
       if (maybeConfig !== undefined) {
+        let eventConfig = maybeConfig.eventConfig;
         let logKey = instr.slot.toString() + ":" + instr.transactionIndex.toString() + ":" + serializeInstructionAddress(instr.instructionAddress);
         let maybeLogs = Stdlib_Option.map(logsByKey[logKey], logs => logs.map(log => ({
           kind: Stdlib_Option.getOr(log.kind, ""),
           message: Stdlib_Option.getOr(log.message, "")
         })));
-        let slotKey = instr.slot.toString();
-        let blockTime = blockTimeBySlot[slotKey];
-        let payload = toSvmInstruction(instr, maybeConfig.contractName, maybeConfig.name, maybeConfig.includeLogs ? maybeLogs : undefined, {
-          slot: instr.slot,
-          time: Stdlib_Option.getOr(blockTime, 0),
-          hash: ""
-        });
+        let payload = toSvmInstruction(instr, eventConfig.contractName, eventConfig.name, eventConfig.includeLogs ? maybeLogs : undefined);
         parsedQueueItems.push({
           kind: 0,
-          eventConfig: maybeConfig,
-          timestamp: Stdlib_Option.getOr(blockTime, 0),
+          onEventRegistration: maybeConfig,
           chain: chain,
           blockNumber: instr.slot,
-          blockHash: "",
           logIndex: synthLogIndex(instr),
           transactionIndex: instr.transactionIndex,
           payload: payload
@@ -424,6 +447,7 @@ function make(param) {
       blockHashes: blockHashes,
       parsedQueueItems: parsedQueueItems,
       transactionStore: Primitive_option.some(match[1]),
+      blockStore: Primitive_option.some(match[2]),
       fromBlockQueried: fromBlock,
       latestFetchedBlockNumber: highestSlot,
       latestFetchedBlockTimestamp: latestBlockTime,
@@ -431,11 +455,13 @@ function make(param) {
         "total time elapsed (s)": totalTimeElapsed,
         "parsing time (s)": parsingTimeElapsed,
         "page fetch time (s)": pageFetchTime
-      }
+      },
+      requestStats: requestStats
     };
   };
   let queryBlockDataRange = async (fromSlot, toSlot) => {
     let blockDatas = [];
+    let requestStats = [];
     let fromRef = fromSlot;
     let keepGoing = true;
     while (keepGoing) {
@@ -456,9 +482,13 @@ function make(param) {
         fields: query_fields,
         maxNumBlocks: query_maxNumBlocks
       };
-      Prometheus.SourceRequestCount.increment(name, chain, "getBlockHashes");
+      let timerRef = Performance.now();
       let match = await client.get(query);
       let resp = match[0];
+      requestStats.push({
+        method: "getBlockHashes",
+        seconds: Performance.secondsSince(timerRef)
+      });
       resp.data.blocks.forEach(b => {
         blockDatas.push({
           blockHash: b.blockhash,
@@ -472,14 +502,20 @@ function make(param) {
         fromRef = resp.nextSlot;
       }
     };
-    return blockDatas;
+    return [
+      blockDatas,
+      requestStats
+    ];
   };
   let getBlockHashes = async (blockNumbers, param) => {
     let firstSlot = blockNumbers[0];
     if (firstSlot === undefined) {
       return {
-        TAG: "Ok",
-        _0: []
+        result: {
+          TAG: "Ok",
+          _0: []
+        },
+        requestStats: []
       };
     }
     try {
@@ -499,21 +535,27 @@ function make(param) {
         }
         requested.add(slot);
       });
-      let blockDatas = await queryBlockDataRange(minSlot.contents, maxSlot.contents);
+      let match = await queryBlockDataRange(minSlot.contents, maxSlot.contents);
       return {
-        TAG: "Ok",
-        _0: blockDatas.filter(data => requested.delete(data.blockNumber))
+        result: {
+          TAG: "Ok",
+          _0: match[0].filter(data => requested.delete(data.blockNumber))
+        },
+        requestStats: match[1]
       };
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
       return {
-        TAG: "Error",
-        _0: exn
+        result: {
+          TAG: "Error",
+          _0: exn
+        },
+        requestStats: []
       };
     }
   };
   return {
-    name: name,
+    name: "SvmHyperSync",
     sourceFor: "Sync",
     chain: chain,
     poweredByHyperSync: true,
@@ -521,11 +563,15 @@ function make(param) {
     getBlockHashes: getBlockHashes,
     getHeightOrThrow: async () => {
       let timer = Performance.now();
-      let h = await client.getHeight();
+      let height = await client.getHeight();
       let seconds = Performance.secondsSince(timer);
-      Prometheus.SourceRequestCount.increment(name, chain, "getHeight");
-      Prometheus.SourceRequestCount.addSeconds(name, chain, "getHeight", seconds);
-      return h;
+      return {
+        height: height,
+        requestStats: [{
+            method: "getHeight",
+            seconds: seconds
+          }]
+      };
     },
     getItemsOrThrow: getItemsOrThrow
   };
@@ -540,6 +586,7 @@ export {
   toSvmInstruction,
   probeRouter,
   toQueryTxField,
+  toQueryBlockField,
   make,
 }
-/* Prometheus Not a pure module */
+/* EventRouter Not a pure module */