envio 3.3.0-alpha.0 → 3.3.0-alpha.10

package/src/sources/SvmHyperSyncSource.res

@@ -4,7 +4,7 @@ type options = {
   chain: ChainMap.Chain.t,
   endpointUrl: string,
   apiToken: option<string>,
-  eventConfigs: array<Internal.svmInstructionEventConfig>,
+  onEventRegistrations: array<Internal.svmOnEventRegistration>,
   clientTimeoutMillis: int,
 }
 
@@ -184,13 +184,12 @@ let parseDecoded = (
   }
 }
 
+// `block` is omitted; it's materialised from the block store at batch prep.
 let toSvmInstruction = (
   instr: SvmHyperSyncClient.ResponseTypes.instruction,
   ~programName,
   ~instructionName,
-  ~transaction,
   ~logs,
-  ~block,
 ): Envio.svmInstruction => {
   programName,
   instructionName,
@@ -204,49 +203,23 @@ let toSvmInstruction = (
   d4: ?instr.d4,
   d8: ?instr.d8,
   params: ?(instr.decoded->Option.map(parseDecoded)),
-  ?transaction,
   ?logs,
-  block,
-}
-
-let toSvmTransaction = (tx: SvmHyperSyncClient.ResponseTypes.transaction): Envio.svmTransaction => {
-  signatures: tx.signatures,
-  accountKeys: tx.accountKeys->SvmTypes.Pubkey.fromStringsUnsafe,
-  feePayer: ?(tx.feePayer->Option.map(SvmTypes.Pubkey.fromStringUnsafe)),
-  success: ?tx.success,
-  err: ?tx.err,
-  // u64 lamports / compute units arrive as `int` over napi. Convert to
-  // `bigint` so the public type stays defensible even for pathological values.
-  fee: ?(tx.fee->Option.map(BigInt.fromInt)),
-  computeUnitsConsumed: ?(tx.computeUnitsConsumed->Option.map(BigInt.fromInt)),
-  recentBlockhash: ?tx.recentBlockhash,
-  version: ?tx.version,
-}
-
-let toSvmTokenBalance = (
-  tb: SvmHyperSyncClient.ResponseTypes.tokenBalance,
-): Envio.svmTokenBalance => {
-  account: ?(tb.account->Option.map(SvmTypes.Pubkey.fromStringUnsafe)),
-  mint: ?(tb.mint->Option.map(SvmTypes.Pubkey.fromStringUnsafe)),
-  owner: ?(tb.owner->Option.map(SvmTypes.Pubkey.fromStringUnsafe)),
-  preAmount: ?tb.preAmount,
-  postAmount: ?tb.postAmount,
 }
 
 // Probe the discriminator byte-length ordering longest-first. Stops at the
 // first router hit. Falls back to the `_none` key (program-wide handler) when
 // no discriminator-keyed handler matches.
 let probeRouter = (
-  router: EventRouter.t<Internal.svmInstructionEventConfig>,
+  router: EventRouter.t<Internal.svmOnEventRegistration>,
   programId: SvmTypes.Pubkey.t,
   instr: SvmHyperSyncClient.ResponseTypes.instruction,
   byteLengthsDesc: array<int>,
   ~contractAddress,
-  ~indexingAddresses,
+  ~contractNameByAddress,
 ) => {
   let probe = (dN: option<string>) => {
     let tag = EventRouter.getSvmEventId(~programId, ~discriminator=dN)
-    router->EventRouter.get(~tag, ~contractAddress, ~blockNumber=instr.slot, ~indexingAddresses)
+    router->EventRouter.get(~tag, ~contractAddress, ~contractNameByAddress)
   }
 
   let result = byteLengthsDesc->Array.reduce(None, (acc, len) =>
@@ -273,9 +246,51 @@ let probeRouter = (
   }
 }
 
-let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: options): t => {
+// Map a selected transaction field to the extra query-side column it needs.
+// `transactionIndex` is always fetched as the store key, and `tokenBalances`
+// lives in a separate table (requested via `needsTokenBalances`), so neither
+// adds a transaction column here.
+let toQueryTxField = (field: Internal.svmTransactionField): option<
+  SvmHyperSyncClient.QueryTypes.transactionField,
+> =>
+  switch field {
+  | TransactionIndex => None
+  | Signatures => Some(Signatures)
+  | FeePayer => Some(FeePayer)
+  | Success => Some(Success)
+  | Err => Some(Err)
+  | Fee => Some(Fee)
+  | ComputeUnitsConsumed => Some(ComputeUnitsConsumed)
+  | AccountKeys => Some(AccountKeys)
+  | RecentBlockhash => Some(RecentBlockhash)
+  | Version => Some(Version)
+  | TokenBalances => None
+  }
+
+// Map a selected block field to its HyperSync query column. Slot/Blockhash/
+// BlockTime are requested unconditionally (needed for reorg detection and the
+// item's slot/timestamp), so selecting `slot`/`time`/`hash` adds no extra column.
+let toQueryBlockField = (field: Internal.svmBlockField): option<
+  SvmHyperSyncClient.QueryTypes.blockField,
+> =>
+  switch field {
+  | Slot | Time | Hash => None
+  | Height => Some(BlockHeight)
+  | ParentSlot => Some(ParentSlot)
+  | ParentHash => Some(ParentBlockhash)
+  }
+
+let make = (
+  {chain, endpointUrl, apiToken, onEventRegistrations, clientTimeoutMillis}: options,
+): t => {
   let name = "SvmHyperSync"
-  let chainId = chain->ChainMap.Chain.toChainId
+
+  // Definitions drive query/decode building; the registrations drive routing
+  // (they carry `isWildcard` and become each decoded item's `onEventRegistration`).
+  let eventConfigs =
+    onEventRegistrations->Array.map(reg =>
+      reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig)
+    )
 
   // Built once at startup and handed to the client so `get` decodes matching
   // instructions in Rust rather than per-instruction over the napi boundary.
@@ -290,7 +305,10 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     },
   )
 
-  let (eventRouter, programOrderings) = EventRouter.fromSvmEventConfigsOrThrow(eventConfigs, ~chain)
+  let (eventRouter, programOrderings) = EventRouter.fromSvmEventConfigsOrThrow(
+    onEventRegistrations,
+    ~chain,
+  )
 
   // programId.toString -> sorted-desc byte lengths
   let orderingByProgram = Dict.make()
@@ -298,23 +316,77 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     orderingByProgram->Dict.set(o.programId->SvmTypes.Pubkey.toString, o.byteLengthsDesc)
   )
 
-  let needsTransactions = eventConfigs->Array.some(cfg => cfg.includeTransaction)
   let needsLogs = eventConfigs->Array.some(cfg => cfg.includeLogs)
-  let needsTokenBalances = eventConfigs->Array.some(cfg => cfg.includeTokenBalances)
+
+  // Union of selected transaction fields across the chain's events. Drives both
+  // the query column selection (fetch only what's used) and the materialisation
+  // mask. `slot`/`transactionIndex` are always fetched as the store key.
+  let selectedTxFields = Utils.Set.make()
+  eventConfigs->Array.forEach(cfg =>
+    cfg.selectedTransactionFields
+    ->(Utils.magic: Utils.Set.t<string> => Utils.Set.t<Internal.svmTransactionField>)
+    ->Utils.Set.forEach(field => selectedTxFields->Utils.Set.add(field)->ignore)
+  )
+  let needsTokenBalances = selectedTxFields->Utils.Set.has(Internal.TokenBalances)
+  let txQueryFields = {
+    // Slot + TransactionIndex are always fetched so the store can be keyed by
+    // (slot, transactionIndex).
+    let fields: array<SvmHyperSyncClient.QueryTypes.transactionField> = [Slot, TransactionIndex]
+    selectedTxFields
+    ->Utils.Set.toArray
+    ->Array.forEach(field =>
+      switch toQueryTxField(field) {
+      | Some(queryField) => fields->Array.push(queryField)
+      | None => ()
+      }
+    )
+    fields
+  }
+  // The transaction table is fetched only when a selected field is actually read
+  // off a stored transaction record. `transactionIndex` materialises from the
+  // store key and `tokenBalances` lives in its own table, so neither requires it.
+  let needsTransactions =
+    selectedTxFields
+    ->Utils.Set.toArray
+    ->Array.some(field =>
+      switch field {
+      | Internal.TransactionIndex | Internal.TokenBalances => false
+      | _ => true
+      }
+    )
+
+  // Union of selected block fields across the chain's events. `slot`/`time`/
+  // `hash` are always fetched (as Slot/BlockTime/Blockhash); the rest are added
+  // only when an instruction selected them.
+  let blockQueryFields = {
+    let fields: array<SvmHyperSyncClient.QueryTypes.blockField> = [Slot, Blockhash, BlockTime]
+    let selected = Utils.Set.make()
+    eventConfigs->Array.forEach(cfg =>
+      cfg.selectedBlockFields->Utils.Set.forEach(field => selected->Utils.Set.add(field)->ignore)
+    )
+    selected->Utils.Set.forEach(field =>
+      switch field->toQueryBlockField {
+      | Some(queryField) => fields->Array.push(queryField)->ignore
+      | None => ()
+      }
+    )
+    fields
+  }
 
   let getItemsOrThrow = async (
     ~fromBlock,
     ~toBlock,
     ~addressesByContractName as _,
-    ~indexingAddresses,
+    ~contractNameByAddress,
     ~knownHeight,
     ~partitionId as _,
     ~selection as _,
+    ~itemsTarget,
     ~retry,
     ~logger,
   ) => {
-    let totalTimeRef = Hrtime.makeTimer()
-    let pageFetchRef = Hrtime.makeTimer()
+    let totalTimeRef = Performance.now()
+    let pageFetchRef = Performance.now()
 
     let instructionSelections = buildInstructionSelections(eventConfigs)
     // Under the server's default merge mode, requesting a table's columns is
@@ -322,24 +394,8 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     // field list returns no rows (instructions and blocks are exempt), so each
     // opted-into table needs its columns spelled out here.
     let fields: SvmHyperSyncClient.QueryTypes.fieldSelection = {
-      block: [Slot, Blockhash, BlockTime],
-      transaction: ?(
-        needsTransactions
-          ? Some([
-              Slot,
-              TransactionIndex,
-              Signatures,
-              FeePayer,
-              Success,
-              Err,
-              Fee,
-              ComputeUnitsConsumed,
-              AccountKeys,
-              RecentBlockhash,
-              Version,
-            ])
-          : None
-      ),
+      block: blockQueryFields,
+      transaction: ?(needsTransactions ? Some(txQueryFields) : None),
       log: ?(needsLogs ? Some([Slot, TransactionIndex, InstructionAddress, Kind, Message]) : None),
       tokenBalance: ?(
         needsTokenBalances
@@ -354,11 +410,10 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       toSlot: ?(toBlock->Option.map(toBlock => toBlock + 1)),
       instructions: instructionSelections,
       fields,
+      maxNumInstructions: itemsTarget,
     }
 
-    Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getInstructions")
-
-    let resp = try await client.get(~query) catch {
+    let (resp, transactionStore, blockStore) = try await client.get(~query) catch {
     | exn =>
       throw(
         Source.GetItemsError(
@@ -376,12 +431,14 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
         ),
       )
     }
-    let pageFetchTime = pageFetchRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let pageFetchTime = pageFetchRef->Performance.secondsSince
+    let requestStats = [{Source.method: "getInstructions", seconds: pageFetchTime}]
 
-    let parsingRef = Hrtime.makeTimer()
+    let parsingRef = Performance.now()
 
-    // Per-slot unix timestamp lookup from the response's `blocks` table. Slots
-    // without a block row (rare; usually skipped slots) fall back to `None`.
+    // Per-slot blockTime lookup from the response's `blocks` table, for the
+    // batch's `latestFetchedBlockTimestamp`. Slots without a block row (rare;
+    // usually skipped slots) fall back to `None`.
     let blockTimeBySlot = Dict.make()
     resp.data.blocks->Array.forEach(b => {
       switch b.blockTime {
@@ -390,13 +447,6 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       }
     })
 
-    // Per (slot, transaction_index) lookup for parent transactions.
-    let txByKey = Dict.make()
-    resp.data.transactions->Array.forEach(tx => {
-      let key = tx.slot->Int.toString ++ ":" ++ tx.transactionIndex->Int.toString
-      txByKey->Dict.set(key, tx)
-    })
-
     // Per (slot, transaction_index, instruction_address) lookup for logs
     // scoped to a single instruction. `instructionAddress: None` logs are
     // attached to no instruction (rare; usually only system messages).
@@ -418,21 +468,6 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       }
     })
 
-    let tokenBalancesByTx = Dict.make()
-    if needsTokenBalances {
-      resp.data.tokenBalances->Array.forEach(tb => {
-        switch tb.transactionIndex {
-        | Some(txIdx) =>
-          let key = tb.slot->Int.toString ++ ":" ++ txIdx->Int.toString
-          switch tokenBalancesByTx->Dict.get(key) {
-          | Some(existing) => existing->Array.push(tb)
-          | None => tokenBalancesByTx->Dict.set(key, [tb])
-          }
-        | None => ()
-        }
-      })
-    }
-
     let parsedQueueItems = []
     resp.data.instructions->Array.forEach(instr => {
       let programId = instr.programId->SvmTypes.Pubkey.fromStringUnsafe
@@ -448,26 +483,16 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
         instr,
         byteLengths,
         ~contractAddress,
-        ~indexingAddresses,
+        ~contractNameByAddress,
       )
 
       switch maybeConfig {
       | None => ()
-      | Some(eventConfig) =>
-        let txKey = instr.slot->Int.toString ++ ":" ++ instr.transactionIndex->Int.toString
-        let maybeTx =
-          txByKey->Utils.Dict.dangerouslyGetNonOption(txKey)->Option.map(toSvmTransaction)
-        let maybeTx = if eventConfig.includeTokenBalances {
-          maybeTx->Option.map(tx => {
-            let maybeBalances =
-              tokenBalancesByTx
-              ->Utils.Dict.dangerouslyGetNonOption(txKey)
-              ->Option.map(bals => bals->Array.map(toSvmTokenBalance))
-            {...tx, tokenBalances: ?maybeBalances}
-          })
-        } else {
-          maybeTx
-        }
+      | Some(onEventRegistration) =>
+        let eventConfig =
+          onEventRegistration.eventConfig->(
+            Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig
+          )
         let logKey =
           instr.slot->Int.toString ++
           ":" ++
@@ -486,30 +511,22 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
             )
           )
 
-        let slotKey = instr.slot->Int.toString
-        let blockTime = blockTimeBySlot->Utils.Dict.dangerouslyGetNonOption(slotKey)
         let payload = toSvmInstruction(
           instr,
           ~programName=eventConfig.contractName,
           ~instructionName=eventConfig.name,
-          ~transaction=eventConfig.includeTransaction ? maybeTx : None,
           ~logs=eventConfig.includeLogs ? maybeLogs : None,
-          ~block={
-            slot: instr.slot,
-            time: blockTime->Option.getOr(0),
-            hash: "",
-          },
         )
 
         parsedQueueItems
         ->Array.push(
           Internal.Event({
-            eventConfig: (eventConfig :> Internal.eventConfig),
-            timestamp: blockTime->Option.getOr(0),
+            onEventRegistration,
             chain,
             blockNumber: instr.slot,
-            blockHash: "",
             logIndex: synthLogIndex(instr),
+            // The parent transaction is materialised from the store at batch prep.
+            transactionIndex: instr.transactionIndex,
             payload: payload->(Utils.magic: Envio.svmInstruction => Internal.eventPayload),
           }),
         )
@@ -519,7 +536,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       let _ = logger
     })
 
-    let parsingTimeElapsed = parsingRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let parsingTimeElapsed = parsingRef->Performance.secondsSince
     let highestSlot = resp.nextSlot - 1
     let latestBlockTime =
       blockTimeBySlot
@@ -534,16 +551,21 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       blockHash: b.blockhash,
     })
 
-    let totalTimeElapsed = totalTimeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let totalTimeElapsed = totalTimeRef->Performance.secondsSince
 
     {
       latestFetchedBlockTimestamp: latestBlockTime,
       parsedQueueItems,
+      // Raw transactions kept in Rust; materialised (selected fields) at batch prep.
+      transactionStore: Some(transactionStore),
+      // Raw blocks kept in Rust; materialised onto the payload at batch prep.
+      blockStore: Some(blockStore),
       latestFetchedBlockNumber: highestSlot,
       stats: {totalTimeElapsed, parsingTimeElapsed, pageFetchTime},
       knownHeight,
       blockHashes,
       fromBlockQueried: fromBlock,
+      requestStats,
     }
   }
 
@@ -552,6 +574,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
   // wire, so we request `maxSlot + 1`; the caller filters to the exact slots.
   let queryBlockDataRange = async (~fromSlot, ~toSlot) => {
     let blockDatas = []
+    let requestStats = []
     let fromRef = ref(fromSlot)
     let keepGoing = ref(true)
     while keepGoing.contents {
@@ -562,8 +585,12 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
         fields: {block: [Slot, Blockhash, BlockTime]},
         maxNumBlocks: 1000,
       }
-      Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getBlockHashes")
-      let resp = await client.get(~query)
+      let timerRef = Performance.now()
+      // Block-only query; the store pages are empty.
+      let (resp, _, _) = await client.get(~query)
+      requestStats
+      ->Array.push({Source.method: "getBlockHashes", seconds: timerRef->Performance.secondsSince})
+      ->ignore
       resp.data.blocks->Array.forEach(b =>
         blockDatas
         ->Array.push({
@@ -582,12 +609,12 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
         fromRef := resp.nextSlot
       }
     }
-    blockDatas
+    (blockDatas, requestStats)
   }
 
   let getBlockHashes = async (~blockNumbers, ~logger as _) =>
     switch blockNumbers->Array.get(0) {
-    | None => Ok([])
+    | None => {Source.result: Ok([]), requestStats: []}
     | Some(firstSlot) =>
       try {
         let minSlot = ref(firstSlot)
@@ -602,14 +629,19 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
           }
           requested->Utils.Set.add(slot)->ignore
         })
-        let blockDatas = await queryBlockDataRange(
+        let (blockDatas, requestStats) = await queryBlockDataRange(
           ~fromSlot=minSlot.contents,
           ~toSlot=maxSlot.contents,
         )
         // Keep one entry per requested slot; drop duplicates and unrelated slots.
-        Ok(blockDatas->Array.filter(data => requested->Utils.Set.delete(data.blockNumber)))
+        {
+          Source.result: Ok(
+            blockDatas->Array.filter(data => requested->Utils.Set.delete(data.blockNumber)),
+          ),
+          requestStats,
+        }
       } catch {
-      | exn => Error(exn)
+      | exn => {Source.result: Error(exn), requestStats: []}
       }
     }
 
@@ -621,17 +653,10 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     poweredByHyperSync: true,
     getBlockHashes,
     getHeightOrThrow: async () => {
-      let timer = Hrtime.makeTimer()
-      let h = await client.getHeight()
-      let seconds = timer->Hrtime.timeSince->Hrtime.toSecondsFloat
-      Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getHeight")
-      Prometheus.SourceRequestCount.addSeconds(
-        ~sourceName=name,
-        ~chainId,
-        ~method="getHeight",
-        ~seconds,
-      )
-      h
+      let timer = Performance.now()
+      let height = await client.getHeight()
+      let seconds = timer->Performance.secondsSince
+      {height, requestStats: [{method: "getHeight", seconds}]}
     },
     getItemsOrThrow,
   }