envio 3.3.0-alpha.6 → 3.3.0-alpha.8

package/src/sources/HyperSyncSource.res

@@ -17,16 +17,14 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
   let noAddressesTopicSelections = []
   let contractNames = Utils.Set.make()
 
-  selection.eventConfigs
-  ->(Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>)
-  ->Array.forEach(({
-    dependsOnAddresses,
-    contractName,
-    getEventFiltersOrThrow,
-    selectedBlockFields,
-    selectedTransactionFields,
-    isWildcard,
-  }) => {
+  selection.onEventRegistrations
+  ->(Utils.magic: array<Internal.onEventRegistration> => array<Internal.evmOnEventRegistration>)
+  ->Array.forEach(reg => {
+    let eventConfig =
+      reg.eventConfig->(Utils.magic: Internal.eventConfig => Internal.evmEventConfig)
+    let contractName = eventConfig.contractName
+    let {selectedBlockFields, selectedTransactionFields} = eventConfig
+    let {dependsOnAddresses, getEventFiltersOrThrow, isWildcard} = reg
     selectedBlockFields
     ->Utils.Set.toArray
     ->Array.forEach(name =>
@@ -141,7 +139,7 @@ type options = {
   chain: ChainMap.Chain.t,
   endpointUrl: string,
   allEventParams: array<HyperSyncClient.Decoder.eventParamsInput>,
-  eventRouter: EventRouter.t<Internal.evmEventConfig>,
+  eventRouter: EventRouter.t<Internal.evmOnEventRegistration>,
   apiToken: option<string>,
   clientTimeoutMillis: int,
   lowercaseAddresses: bool,
@@ -197,27 +195,25 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
 
   let makeEventBatchQueueItem = (
     item: HyperSyncClient.EventItems.item,
-    ~block: HyperSyncClient.ResponseTypes.block,
     ~params: Internal.eventParams,
-    ~eventConfig: Internal.evmEventConfig,
+    ~onEventRegistration: Internal.evmOnEventRegistration,
   ): Internal.item => {
     let {transactionIndex, logIndex, srcAddress} = item
     let chainId = chain->ChainMap.Chain.toChainId
 
     Internal.Event({
-      eventConfig: (eventConfig :> Internal.eventConfig),
-      timestamp: block.timestamp->Option.getUnsafe,
+      onEventRegistration: (onEventRegistration :> Internal.onEventRegistration),
       chain,
       blockNumber: item.blockNumber,
-      blockHash: block.hash->Option.getUnsafe,
       logIndex,
       transactionIndex,
+      // `block` and `transaction` are omitted; they're materialised from the
+      // per-chain stores onto the payload at batch prep.
       payload: {
-        contractName: eventConfig.contractName,
-        eventName: eventConfig.name,
+        contractName: onEventRegistration.eventConfig.contractName,
+        eventName: onEventRegistration.eventConfig.name,
         chainId,
         params,
-        block: block->(Utils.magic: HyperSyncClient.ResponseTypes.block => Internal.eventBlock),
         srcAddress,
         logIndex,
       }->Evm.fromPayload,
@@ -232,6 +228,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     ~knownHeight,
     ~partitionId as _,
     ~selection,
+    ~itemsTarget,
     ~retry,
     ~logger,
   ) => {
@@ -247,17 +244,13 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     let startFetchingBatchTimeRef = Performance.now()
 
     //fetch batch
-    Prometheus.SourceRequestCount.increment(
-      ~sourceName=name,
-      ~chainId=chain->ChainMap.Chain.toChainId,
-      ~method="getLogs",
-    )
     let pageUnsafe = try await HyperSync.GetLogs.query(
       ~client,
       ~fromBlock,
       ~toBlock,
       ~logSelections,
       ~fieldSelection=selectionConfig.fieldSelection,
+      ~maxNumLogs=itemsTarget,
     ) catch {
     | HyperSync.GetLogs.Error(error) =>
       throw(
@@ -305,6 +298,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     }
 
     let pageFetchTime = startFetchingBatchTimeRef->Performance.secondsSince
+    let requestStats = [{Source.method: "getLogs", seconds: pageFetchTime}]
 
     //set height and next from block
     let knownHeight = pageUnsafe.archiveHeight
@@ -319,28 +313,25 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     //Parse page items into queue items
     let parsedQueueItems = []
 
-    // Blocks are returned once per number; items reference them by blockNumber.
+    // Block headers are returned once per number; items reference them by blockNumber.
     let blocksByNumber = Utils.Map.make()
     pageUnsafe.blocks->Array.forEach(block => {
-      switch block.number {
-      | Some(number) => blocksByNumber->Utils.Map.set(number, block)->ignore
-      | None => ()
-      }
+      blocksByNumber->Utils.Map.set(block.number, block)->ignore
     })
     let getBlock = blockNumber => blocksByNumber->Utils.Map.unsafeGet(blockNumber)
 
     let handleDecodeFailure = (
-      ~eventConfig: Internal.evmEventConfig,
+      ~onEventRegistration: Internal.evmOnEventRegistration,
       ~logIndex,
       ~blockNumber,
       ~chainId,
       ~exn,
     ) => {
-      if !eventConfig.isWildcard {
+      if !onEventRegistration.isWildcard {
         //Wildcard events can be parsed as undefined if the number of topics
         //don't match the event with the given topic0
         //Non wildcard events should be expected to be parsed
-        let msg = `Event ${eventConfig.name} was unexpectedly parsed as undefined`
+        let msg = `Event ${onEventRegistration.eventConfig.name} was unexpectedly parsed as undefined`
         let logger = Logging.createChildFrom(
           ~logger,
           ~params={
@@ -368,19 +359,17 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
 
       switch maybeEventConfig {
       | None => () //ignore events that aren't registered
-      | Some(eventConfig) =>
+      | Some(onEventRegistration) =>
         switch item.params
         ->Nullable.toOption
-        ->Option.flatMap(Dict.get(_, eventConfig.contractName)) {
+        ->Option.flatMap(Dict.get(_, onEventRegistration.eventConfig.contractName)) {
         | Some(params) =>
           parsedQueueItems
-          ->Array.push(
-            makeEventBatchQueueItem(item, ~block=getBlock(item.blockNumber), ~params, ~eventConfig),
-          )
+          ->Array.push(makeEventBatchQueueItem(item, ~params, ~onEventRegistration))
           ->ignore
         | None =>
           handleDecodeFailure(
-            ~eventConfig,
+            ~onEventRegistration,
             ~logIndex=item.logIndex,
             ~blockNumber=item.blockNumber,
             ~chainId,
@@ -398,11 +387,9 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     // detection notices same-block-number-different-hash collisions itself.
     let blockHashes = []
     pageUnsafe.blocks->Array.forEach(block => {
-      switch (block.number, block.hash) {
-      | (Some(blockNumber), Some(blockHash)) =>
-        blockHashes->Array.push({ReorgDetection.blockNumber, blockHash})->ignore
-      | _ => ()
-      }
+      blockHashes
+      ->Array.push({ReorgDetection.blockNumber: block.number, blockHash: block.hash})
+      ->ignore
     })
     switch pageUnsafe.rollbackGuard {
     | None => ()
@@ -426,7 +413,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     | None =>
       switch pageUnsafe.items->Array.get(pageUnsafe.items->Array.length - 1) {
       | Some(item) if item.blockNumber == heighestBlockQueried =>
-        getBlock(item.blockNumber).timestamp->Option.getUnsafe
+        getBlock(item.blockNumber).timestamp
       | _ => 0
       }
     }
@@ -443,11 +430,13 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       latestFetchedBlockTimestamp,
       parsedQueueItems,
       transactionStore: Some(pageUnsafe.transactionStore),
+      blockStore: Some(pageUnsafe.blockStore),
       latestFetchedBlockNumber: heighestBlockQueried,
       stats,
       knownHeight,
       blockHashes,
       fromBlockQueried: fromBlock,
+      requestStats,
     }
   }
 
@@ -458,7 +447,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       ~sourceName=name,
       ~chainId=chain->ChainMap.Chain.toChainId,
       ~logger,
-    )->Promise.thenResolve(HyperSync.mapExn)
+    )->Promise.thenResolve(((queryRes, requestStats)) => {
+      Source.result: queryRes->HyperSync.mapExn,
+      requestStats,
+    })
 
   {
     name,
@@ -469,7 +461,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     getBlockHashes,
     getHeightOrThrow: async () => {
       let timerRef = Performance.now()
-      let result = try {
+      let height = try {
         await client.getHeight()
       } catch {
       | JsExn(e) =>
@@ -483,18 +475,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
         }
       }
       let seconds = timerRef->Performance.secondsSince
-      Prometheus.SourceRequestCount.increment(
-        ~sourceName=name,
-        ~chainId=chain->ChainMap.Chain.toChainId,
-        ~method="getHeight",
-      )
-      Prometheus.SourceRequestCount.addSeconds(
-        ~sourceName=name,
-        ~chainId=chain->ChainMap.Chain.toChainId,
-        ~method="getHeight",
-        ~seconds,
-      )
-      result
+      {height, requestStats: [{method: "getHeight", seconds}]}
     },
     getItemsOrThrow,
     createHeightSubscription: (~onHeight) =>

package/src/sources/HyperSyncSource.res.mjs

@@ -4,7 +4,6 @@ import * as Utils from "../Utils.res.mjs";
 import * as Source from "./Source.res.mjs";
 import * as Logging from "../Logging.res.mjs";
 import * as HyperSync from "./HyperSync.res.mjs";
-import * as Prometheus from "../Prometheus.res.mjs";
 import * as EventRouter from "./EventRouter.res.mjs";
 import * as Performance from "../bindings/Performance.res.mjs";
 import * as LogSelection from "../LogSelection.res.mjs";
@@ -25,24 +24,25 @@ function getSelectionConfig(selection, chain) {
   let dynamicWildcardEventFiltersByContract = {};
   let noAddressesTopicSelections = [];
   let contractNames = new Set();
-  selection.eventConfigs.forEach(param => {
-    let contractName = param.contractName;
-    Array.from(param.selectedBlockFields).forEach(name => {
+  selection.onEventRegistrations.forEach(reg => {
+    let eventConfig = reg.eventConfig;
+    let contractName = eventConfig.contractName;
+    Array.from(eventConfig.selectedBlockFields).forEach(name => {
       capitalizedBlockFields.add(Utils.$$String.capitalize(name));
     });
-    Array.from(param.selectedTransactionFields).forEach(name => {
+    Array.from(eventConfig.selectedTransactionFields).forEach(name => {
       if (name !== "transactionIndex") {
         capitalizedTransactionFields.add(Utils.$$String.capitalize(name));
         return;
       }
     });
-    let eventFilters = param.getEventFiltersOrThrow(chain);
-    if (param.dependsOnAddresses) {
+    let eventFilters = reg.getEventFiltersOrThrow(chain);
+    if (reg.dependsOnAddresses) {
       contractNames.add(contractName);
       if (eventFilters.TAG === "Static") {
         return Utils.Dict.pushMany(staticTopicSelectionsByContract, contractName, eventFilters._0);
       } else {
-        return Utils.Dict.push(param.isWildcard ? dynamicWildcardEventFiltersByContract : dynamicEventFiltersByContract, contractName, eventFilters._0);
+        return Utils.Dict.push(reg.isWildcard ? dynamicWildcardEventFiltersByContract : dynamicEventFiltersByContract, contractName, eventFilters._0);
       }
     }
     let tmp;
@@ -127,29 +127,26 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the hypersync client, please double check your ABI", exn);
   }
   let UndefinedValue = /* @__PURE__ */Primitive_exceptions.create("UndefinedValue");
-  let makeEventBatchQueueItem = (item, block, params, eventConfig) => {
+  let makeEventBatchQueueItem = (item, params, onEventRegistration) => {
     let logIndex = item.logIndex;
     return {
       kind: 0,
-      eventConfig: eventConfig,
-      timestamp: block.timestamp,
+      onEventRegistration: onEventRegistration,
       chain: chain,
       blockNumber: item.blockNumber,
-      blockHash: block.hash,
       logIndex: logIndex,
       transactionIndex: item.transactionIndex,
       payload: {
-        contractName: eventConfig.contractName,
-        eventName: eventConfig.name,
+        contractName: onEventRegistration.eventConfig.contractName,
+        eventName: onEventRegistration.eventConfig.name,
         params: params,
         chainId: chain,
         srcAddress: item.srcAddress,
-        logIndex: logIndex,
-        block: block
+        logIndex: logIndex
       }
     };
   };
-  let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, param, selection, retry, logger) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, param, selection, itemsTarget, retry, logger) => {
     let totalTimeRef = Performance.now();
     let selectionConfig = getSelectionConfig(selection);
     let logSelections;
@@ -160,10 +157,9 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
       logSelections = ErrorHandling.mkLogAndRaise(logger, "Failed getting log selection for the query", exn);
     }
     let startFetchingBatchTimeRef = Performance.now();
-    Prometheus.SourceRequestCount.increment(name, chain, "getLogs");
     let pageUnsafe;
     try {
-      pageUnsafe = await HyperSync.GetLogs.query(client, fromBlock, toBlock, logSelections, selectionConfig.fieldSelection);
+      pageUnsafe = await HyperSync.GetLogs.query(client, fromBlock, toBlock, logSelections, selectionConfig.fieldSelection, itemsTarget);
     } catch (raw_error) {
       let error = Primitive_exceptions.internalToException(raw_error);
       if (error.RE_EXN_ID === HyperSync.GetLogs.$$Error) {
@@ -212,26 +208,26 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
       };
     }
     let pageFetchTime = Performance.secondsSince(startFetchingBatchTimeRef);
+    let requestStats = [{
+        method: "getLogs",
+        seconds: pageFetchTime
+      }];
     let knownHeight$1 = pageUnsafe.archiveHeight;
     let heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;
     let parsingTimeRef = Performance.now();
     let parsedQueueItems = [];
     let blocksByNumber = new Map();
     pageUnsafe.blocks.forEach(block => {
-      let number = block.number;
-      if (number !== undefined) {
-        blocksByNumber.set(number, block);
-        return;
-      }
+      blocksByNumber.set(block.number, block);
     });
     pageUnsafe.items.forEach(item => {
       let maybeEventConfig = EventRouter.get(eventRouter, EventRouter.getEvmEventId(item.topic0, item.topicCount), item.srcAddress, contractNameByAddress);
       if (maybeEventConfig === undefined) {
         return;
       }
-      let params = Stdlib_Option.flatMap(Primitive_option.fromNullable(item.params), __x => __x[maybeEventConfig.contractName]);
+      let params = Stdlib_Option.flatMap(Primitive_option.fromNullable(item.params), __x => __x[maybeEventConfig.eventConfig.contractName]);
       if (params !== undefined) {
-        parsedQueueItems.push(makeEventBatchQueueItem(item, blocksByNumber.get(item.blockNumber), Primitive_option.valFromOption(params), maybeEventConfig));
+        parsedQueueItems.push(makeEventBatchQueueItem(item, Primitive_option.valFromOption(params), maybeEventConfig));
         return;
       } else {
         let logIndex = item.logIndex;
@@ -242,7 +238,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
         if (maybeEventConfig.isWildcard) {
           return;
         }
-        let msg = `Event ` + maybeEventConfig.name + ` was unexpectedly parsed as undefined`;
+        let msg = `Event ` + maybeEventConfig.eventConfig.name + ` was unexpectedly parsed as undefined`;
         let logger$1 = Logging.createChildFrom(logger, {
           chainId: chain,
           blockNumber: blockNumber,
@@ -255,15 +251,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     let parsingTimeElapsed = Performance.secondsSince(parsingTimeRef);
     let blockHashes = [];
     pageUnsafe.blocks.forEach(block => {
-      let match = block.number;
-      let match$1 = block.hash;
-      if (match !== undefined && match$1 !== undefined) {
-        blockHashes.push({
-          blockHash: match$1,
-          blockNumber: match
-        });
-        return;
-      }
+      blockHashes.push({
+        blockHash: block.hash,
+        blockNumber: block.number
+      });
     });
     let match = pageUnsafe.rollbackGuard;
     if (match !== undefined) {
@@ -297,13 +288,18 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
       blockHashes: blockHashes,
       parsedQueueItems: parsedQueueItems,
       transactionStore: Primitive_option.some(pageUnsafe.transactionStore),
+      blockStore: Primitive_option.some(pageUnsafe.blockStore),
       fromBlockQueried: fromBlock,
       latestFetchedBlockNumber: heighestBlockQueried,
       latestFetchedBlockTimestamp: latestFetchedBlockTimestamp,
-      stats: stats
+      stats: stats,
+      requestStats: requestStats
     };
   };
-  let getBlockHashes = (blockNumbers, logger) => HyperSync.queryBlockDataMulti(client, blockNumbers, name, chain, logger).then(HyperSync.mapExn);
+  let getBlockHashes = (blockNumbers, logger) => HyperSync.queryBlockDataMulti(client, blockNumbers, name, chain, logger).then(param => ({
+    result: HyperSync.mapExn(param[0]),
+    requestStats: param[1]
+  }));
   return {
     name: name,
     sourceFor: "Sync",
@@ -313,9 +309,9 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     getBlockHashes: getBlockHashes,
     getHeightOrThrow: async () => {
       let timerRef = Performance.now();
-      let result;
+      let height;
       try {
-        result = await client.getHeight();
+        height = await client.getHeight();
       } catch (raw_e) {
         let e = Primitive_exceptions.internalToException(raw_e);
         if (e.RE_EXN_ID === "JsExn") {
@@ -325,7 +321,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
             if (message.includes("401 Unauthorized")) {
               Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperSync (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`);
               await new Promise((param, param$1) => {});
-              result = 0;
+              height = 0;
             } else {
               throw {
                 RE_EXN_ID: "JsExn",
@@ -345,9 +341,13 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
         }
       }
       let seconds = Performance.secondsSince(timerRef);
-      Prometheus.SourceRequestCount.increment(name, chain, "getHeight");
-      Prometheus.SourceRequestCount.addSeconds(name, chain, "getHeight", seconds);
-      return result;
+      return {
+        height: height,
+        requestStats: [{
+            method: "getHeight",
+            seconds: seconds
+          }]
+      };
     },
     getItemsOrThrow: getItemsOrThrow,
     createHeightSubscription: onHeight => HyperSyncHeightStream.subscribe(endpointUrl, apiToken$1, chain, onHeight)