envio 3.3.2 → 3.4.0

package/src/sources/FuelHyperSyncSource.res

@@ -0,0 +1,257 @@
+open Source
+
+let isUnauthorizedError = (message: string) => message->String.includes("401 Unauthorized")
+
+type options = {
+  chain: ChainMap.Chain.t,
+  endpointUrl: string,
+  apiToken: option<string>,
+  // The chain's registrations, indexed by their sequential `index`.
+  onEventRegistrations: array<Internal.fuelOnEventRegistration>,
+}
+
+let make = ({chain, endpointUrl, apiToken, onEventRegistrations}: options): t => {
+  let name = "HyperFuel"
+
+  let apiToken = switch apiToken {
+  | Some(token) => token
+  | None =>
+    JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
+Set the ENVIO_API_TOKEN environment variable in your .env file.
+Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
+  }
+
+  let client = switch FuelHyperSyncClient.make(
+    {url: endpointUrl, apiToken},
+    ~eventRegistrations=FuelHyperSyncClient.Registration.fromOnEventRegistrations(onEventRegistrations),
+  ) {
+  | client => client
+  | exception exn =>
+    exn->ErrorHandling.mkLogAndRaise(~msg="Failed to instantiate the HyperFuel client")
+  }
+
+  let getItemsOrThrow = async (
+    ~fromBlock,
+    ~toBlock,
+    ~addressesByContractName,
+    ~contractNameByAddress as _,
+    ~knownHeight,
+    ~partitionId as _,
+    ~selection: FetchState.selection,
+    ~itemsTarget as _,
+    ~retry,
+    ~logger,
+  ) => {
+    let totalTimeRef = Performance.now()
+
+    let startFetchingBatchTimeRef = Performance.now()
+
+    //fetch batch
+    let pageUnsafe = try await FuelHyperSync.GetLogs.query(
+      ~client,
+      ~fromBlock,
+      ~toBlock,
+      ~registrationIndexes=selection.onEventRegistrations->Array.map(reg => reg.index),
+      ~addressesByContractName,
+    ) catch {
+    | FuelHyperSync.GetLogs.Error(error) =>
+      throw(
+        Source.GetItemsError(
+          Source.FailedGettingItems({
+            exn: %raw(`null`),
+            attemptedToBlock: toBlock->Option.getOr(knownHeight),
+            retry: switch error {
+            | WrongInstance =>
+              let backoffMillis = switch retry {
+              | 0 => 100
+              | _ => 500 * retry
+              }
+              WithBackoff({
+                message: `Block #${fromBlock->Int.toString} not found in FuelHyperSync. HyperFuel has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${backoffMillis->Int.toString}ms.`,
+                backoffMillis,
+              })
+            | UnexpectedMissingParams({missingParams}) =>
+              ImpossibleForTheQuery({
+                message: `Source returned invalid data with missing required fields: ${missingParams->Array.joinUnsafe(
+                    ", ",
+                  )}`,
+              })
+            },
+          }),
+        ),
+      )
+    | exn =>
+      throw(
+        Source.GetItemsError(
+          Source.FailedGettingItems({
+            exn,
+            attemptedToBlock: toBlock->Option.getOr(knownHeight),
+            retry: WithBackoff({
+              message: `Unexpected issue while fetching events from HyperFuel client. Attempt a retry.`,
+              backoffMillis: switch retry {
+              | 0 => 500
+              | _ => 1000 * retry
+              },
+            }),
+          }),
+        ),
+      )
+    }
+
+    let pageFetchTime = startFetchingBatchTimeRef->Performance.secondsSince
+    let requestStats = [{Source.method: "getLogs", seconds: pageFetchTime}]
+
+    //set height and next from block
+    let knownHeight = pageUnsafe.archiveHeight
+
+    //The heighest (biggest) blocknumber that was accounted for in
+    //Our query. Not necessarily the blocknumber of the last log returned
+    //In the query
+    let heighestBlockQueried = pageUnsafe.nextBlock - 1
+
+    let parsingTimeRef = Performance.now()
+
+    // Blocks are returned once per height; items reference them by blockHeight.
+    let blocksByHeight = Utils.Map.make()
+    pageUnsafe.blocks->Array.forEach(block => {
+      blocksByHeight->Utils.Map.set(block.height, block)->ignore
+    })
+
+    let chainId = chain->ChainMap.Chain.toChainId
+
+    let parsedQueueItems = pageUnsafe.items->Array.map(item => {
+      // Routing happened in Rust; the item references its registration by
+      // chain-scoped index.
+      let onEventRegistration = onEventRegistrations->Array.getUnsafe(item.onEventRegistrationIndex)
+      let eventConfig =
+        onEventRegistration.eventConfig->(
+          Utils.magic: Internal.eventConfig => Internal.fuelEventConfig
+        )
+      // Presence of every routed item's block is validated in Rust.
+      let block = blocksByHeight->Utils.Map.unsafeGet(item.blockHeight)
+
+      let params = switch eventConfig.kind {
+      | LogData({decode}) =>
+        // Kind-required columns are validated present in Rust before the item
+        // crosses the boundary.
+        let data = item.data->Option.getOr("")
+        try decode(data) catch {
+        | exn => {
+            let params = {
+              "chainId": chainId,
+              "blockNumber": item.blockHeight,
+              "logIndex": item.receiptIndex,
+            }
+            let logger = Logging.createChildFrom(~logger, ~params)
+            exn->ErrorHandling.mkLogAndRaise(
+              ~msg="Failed to decode Fuel LogData receipt, please double check your ABI.",
+              ~logger,
+            )
+          }
+        }
+      | Mint | Burn =>
+        (
+          {
+            subId: item.subId->Option.getOr(""),
+            amount: item.val->Option.getOr(0n),
+          }: Internal.fuelSupplyParams
+        )->Obj.magic
+      | Transfer | Call =>
+        (
+          {
+            to: item.to->Option.getOr("")->Address.unsafeFromString,
+            assetId: item.assetId->Option.getOr(""),
+            amount: item.amount->Option.getOr(0n),
+          }: Internal.fuelTransferParams
+        )->Obj.magic
+      }
+
+      Internal.Event({
+        onEventRegistration,
+        chain,
+        blockNumber: item.blockHeight,
+        logIndex: item.receiptIndex,
+        // Fuel carries the transaction inline on the payload; the store key is
+        // unused (Fuel identifies transactions by hash, kept on the payload).
+        transactionIndex: 0,
+        payload: {
+          contractName: eventConfig.contractName,
+          eventName: eventConfig.name,
+          chainId,
+          params,
+          transaction: {
+            "id": item.txId,
+          }->Obj.magic, // TODO: Obj.magic needed until the field selection types are not configurable for Fuel and Evm separately
+          block: block->Obj.magic,
+          srcAddress: item.srcAddress,
+          logIndex: item.receiptIndex,
+        }->Fuel.fromPayload,
+      })
+    })
+
+    let parsingTimeElapsed = parsingTimeRef->Performance.secondsSince
+
+    // Fuel never rolls back on reorg, so block hashes here are purely informational
+    // for detect-only logging via ReorgDetection.
+    let blockHashes = pageUnsafe.blocks->Array.map(block => {
+      ReorgDetection.blockNumber: block.height,
+      blockHash: block.id,
+    })
+
+    let latestFetchedBlockTimestamp = switch blocksByHeight->Utils.Map.get(heighestBlockQueried) {
+    | Some(block) => block.time
+    | None => 0
+    }
+
+    let totalTimeElapsed = totalTimeRef->Performance.secondsSince
+
+    let stats = {
+      totalTimeElapsed,
+      parsingTimeElapsed,
+      pageFetchTime,
+    }
+
+    {
+      latestFetchedBlockTimestamp,
+      parsedQueueItems,
+      // Fuel keeps transaction and block on the payload; no store pages.
+      transactionStore: None,
+      blockStore: None,
+      latestFetchedBlockNumber: heighestBlockQueried,
+      stats,
+      knownHeight,
+      blockHashes,
+      fromBlockQueried: fromBlock,
+      requestStats,
+    }
+  }
+
+  let getBlockHashes = (~blockNumbers as _, ~logger as _) =>
+    JsError.throwWithMessage("HyperFuel does not support getting block hashes")
+
+  {
+    name,
+    sourceFor: Sync,
+    chain,
+    getBlockHashes,
+    pollingInterval: 100,
+    poweredByHyperSync: true,
+    getHeightOrThrow: async () => {
+      let timerRef = Performance.now()
+      let height = try await client->FuelHyperSyncClient.getHeight catch {
+      | JsExn(e) =>
+        switch e->JsExn.message {
+        | Some(message) if message->isUnauthorizedError =>
+          Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`)
+          // Retrying an unauthorized request can never succeed, so block forever
+          let _ = await Promise.make((_, _) => ())
+          0
+        | _ => throw(JsExn(e))
+        }
+      }
+      let seconds = timerRef->Performance.secondsSince
+      {height, requestStats: [{method: "getHeight", seconds}]}
+    },
+    getItemsOrThrow,
+  }
+}

package/src/sources/FuelHyperSyncSource.res.mjs

@@ -0,0 +1,252 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Source from "./Source.res.mjs";
+import * as Logging from "../Logging.res.mjs";
+import * as Performance from "../bindings/Performance.res.mjs";
+import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
+import * as ErrorHandling from "../ErrorHandling.res.mjs";
+import * as FuelHyperSync from "./FuelHyperSync.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as FuelHyperSyncClient from "./FuelHyperSyncClient.res.mjs";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+function isUnauthorizedError(message) {
+  return message.includes("401 Unauthorized");
+}
+
+function make(param) {
+  let onEventRegistrations = param.onEventRegistrations;
+  let apiToken = param.apiToken;
+  let chain = param.chain;
+  let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
+Set the ENVIO_API_TOKEN environment variable in your .env file.
+Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
+  let client;
+  try {
+    client = FuelHyperSyncClient.make({
+      url: param.endpointUrl,
+      apiToken: apiToken$1
+    }, FuelHyperSyncClient.Registration.fromOnEventRegistrations(onEventRegistrations));
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the HyperFuel client", exn);
+  }
+  let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, param, knownHeight, param$1, selection, param$2, retry, logger) => {
+    let totalTimeRef = Performance.now();
+    let startFetchingBatchTimeRef = Performance.now();
+    let pageUnsafe;
+    try {
+      pageUnsafe = await FuelHyperSync.GetLogs.query(client, fromBlock, toBlock, selection.onEventRegistrations.map(reg => reg.index), addressesByContractName);
+    } catch (raw_error) {
+      let error = Primitive_exceptions.internalToException(raw_error);
+      if (error.RE_EXN_ID === FuelHyperSync.GetLogs.$$Error) {
+        let error$1 = error._1;
+        let tmp;
+        if (typeof error$1 !== "object") {
+          let backoffMillis = retry !== 0 ? 500 * retry | 0 : 100;
+          tmp = {
+            TAG: "WithBackoff",
+            message: `Block #` + fromBlock.toString() + ` not found in FuelHyperSync. HyperFuel has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ` + backoffMillis.toString() + `ms.`,
+            backoffMillis: backoffMillis
+          };
+        } else {
+          tmp = {
+            TAG: "ImpossibleForTheQuery",
+            message: `Source returned invalid data with missing required fields: ` + error$1.missingParams.join(", ")
+          };
+        }
+        throw {
+          RE_EXN_ID: Source.GetItemsError,
+          _1: {
+            TAG: "FailedGettingItems",
+            exn: null,
+            attemptedToBlock: Stdlib_Option.getOr(toBlock, knownHeight),
+            retry: tmp
+          },
+          Error: new Error()
+        };
+      }
+      throw {
+        RE_EXN_ID: Source.GetItemsError,
+        _1: {
+          TAG: "FailedGettingItems",
+          exn: error,
+          attemptedToBlock: Stdlib_Option.getOr(toBlock, knownHeight),
+          retry: {
+            TAG: "WithBackoff",
+            message: `Unexpected issue while fetching events from HyperFuel client. Attempt a retry.`,
+            backoffMillis: retry !== 0 ? 1000 * retry | 0 : 500
+          }
+        },
+        Error: new Error()
+      };
+    }
+    let pageFetchTime = Performance.secondsSince(startFetchingBatchTimeRef);
+    let requestStats = [{
+        method: "getLogs",
+        seconds: pageFetchTime
+      }];
+    let knownHeight$1 = pageUnsafe.archiveHeight;
+    let heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;
+    let parsingTimeRef = Performance.now();
+    let blocksByHeight = new Map();
+    pageUnsafe.blocks.forEach(block => {
+      blocksByHeight.set(block.height, block);
+    });
+    let parsedQueueItems = pageUnsafe.items.map(item => {
+      let onEventRegistration = onEventRegistrations[item.onEventRegistrationIndex];
+      let eventConfig = onEventRegistration.eventConfig;
+      let block = blocksByHeight.get(item.blockHeight);
+      let match = eventConfig.kind;
+      let params;
+      let exit = 0;
+      if (typeof match !== "object") {
+        switch (match) {
+          case "Mint" :
+          case "Burn" :
+            exit = 1;
+            break;
+          case "Transfer" :
+          case "Call" :
+            exit = 2;
+            break;
+        }
+      } else {
+        let data = Stdlib_Option.getOr(item.data, "");
+        try {
+          params = match.decode(data);
+        } catch (raw_exn) {
+          let exn = Primitive_exceptions.internalToException(raw_exn);
+          let params$1 = {
+            chainId: chain,
+            blockNumber: item.blockHeight,
+            logIndex: item.receiptIndex
+          };
+          let logger$1 = Logging.createChildFrom(logger, params$1);
+          params = ErrorHandling.mkLogAndRaise(logger$1, "Failed to decode Fuel LogData receipt, please double check your ABI.", exn);
+        }
+      }
+      switch (exit) {
+        case 1 :
+          params = {
+            subId: Stdlib_Option.getOr(item.subId, ""),
+            amount: Stdlib_Option.getOr(item.val, 0n)
+          };
+          break;
+        case 2 :
+          params = {
+            to: Stdlib_Option.getOr(item.to, ""),
+            assetId: Stdlib_Option.getOr(item.assetId, ""),
+            amount: Stdlib_Option.getOr(item.amount, 0n)
+          };
+          break;
+      }
+      return {
+        kind: 0,
+        onEventRegistration: onEventRegistration,
+        chain: chain,
+        blockNumber: item.blockHeight,
+        logIndex: item.receiptIndex,
+        transactionIndex: 0,
+        payload: {
+          contractName: eventConfig.contractName,
+          eventName: eventConfig.name,
+          params: params,
+          chainId: chain,
+          srcAddress: item.srcAddress,
+          logIndex: item.receiptIndex,
+          transaction: {
+            id: item.txId
+          },
+          block: block
+        }
+      };
+    });
+    let parsingTimeElapsed = Performance.secondsSince(parsingTimeRef);
+    let blockHashes = pageUnsafe.blocks.map(block => ({
+      blockHash: block.id,
+      blockNumber: block.height
+    }));
+    let block = blocksByHeight.get(heighestBlockQueried);
+    let latestFetchedBlockTimestamp = block !== undefined ? block.time : 0;
+    let totalTimeElapsed = Performance.secondsSince(totalTimeRef);
+    let stats_parsing$unknowntime$unknown$lpars$rpar = parsingTimeElapsed;
+    let stats_page$unknownfetch$unknowntime$unknown$lpars$rpar = pageFetchTime;
+    let stats = {
+      "total time elapsed (s)": totalTimeElapsed,
+      "parsing time (s)": stats_parsing$unknowntime$unknown$lpars$rpar,
+      "page fetch time (s)": stats_page$unknownfetch$unknowntime$unknown$lpars$rpar
+    };
+    return {
+      knownHeight: knownHeight$1,
+      blockHashes: blockHashes,
+      parsedQueueItems: parsedQueueItems,
+      transactionStore: undefined,
+      blockStore: undefined,
+      fromBlockQueried: fromBlock,
+      latestFetchedBlockNumber: heighestBlockQueried,
+      latestFetchedBlockTimestamp: latestFetchedBlockTimestamp,
+      stats: stats,
+      requestStats: requestStats
+    };
+  };
+  let getBlockHashes = (param, param$1) => Stdlib_JsError.throwWithMessage("HyperFuel does not support getting block hashes");
+  return {
+    name: "HyperFuel",
+    sourceFor: "Sync",
+    chain: chain,
+    poweredByHyperSync: true,
+    pollingInterval: 100,
+    getBlockHashes: getBlockHashes,
+    getHeightOrThrow: async () => {
+      let timerRef = Performance.now();
+      let height;
+      try {
+        height = await client.getHeight();
+      } catch (raw_e) {
+        let e = Primitive_exceptions.internalToException(raw_e);
+        if (e.RE_EXN_ID === "JsExn") {
+          let e$1 = e._1;
+          let message = Stdlib_JsExn.message(e$1);
+          if (message !== undefined) {
+            if (message.includes("401 Unauthorized")) {
+              Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`);
+              await new Promise((param, param$1) => {});
+              height = 0;
+            } else {
+              throw {
+                RE_EXN_ID: "JsExn",
+                _1: e$1,
+                Error: new Error()
+              };
+            }
+          } else {
+            throw {
+              RE_EXN_ID: "JsExn",
+              _1: e$1,
+              Error: new Error()
+            };
+          }
+        } else {
+          throw e;
+        }
+      }
+      let seconds = Performance.secondsSince(timerRef);
+      return {
+        height: height,
+        requestStats: [{
+            method: "getHeight",
+            seconds: seconds
+          }]
+      };
+    },
+    getItemsOrThrow: getItemsOrThrow
+  };
+}
+
+export {
+  isUnauthorizedError,
+  make,
+}
+/* Logging Not a pure module */

package/src/sources/HyperSyncClient.res

@@ -358,11 +358,11 @@ type t = {
 }
 
 @send
-external classNew: (Core.evmHypersyncClientCtor, cfg, string, array<Registration.input>) => t =
+external classNew: (Core.evmHyperSyncClientCtor, cfg, string, array<Registration.input>) => t =
   "new"
 
 let makeWithAgent = (cfg, ~userAgent, ~eventRegistrations) =>
-  Core.getAddon().evmHypersyncClient->classNew(cfg, userAgent, eventRegistrations)
+  Core.getAddon().evmHyperSyncClient->classNew(cfg, userAgent, eventRegistrations)
 
 type logLevel = [#trace | #debug | #info | #warn | #error]
 let logLevelSchema: S.t<logLevel> = S.enum([#trace, #debug, #info, #warn, #error])

package/src/sources/HyperSyncClient.res.mjs

@@ -74,7 +74,7 @@ let Registration = {
 let EventItems = {};
 
 function makeWithAgent(cfg, userAgent, eventRegistrations) {
-  return Core.getAddon().EvmHypersyncClient.new(cfg, userAgent, eventRegistrations);
+  return Core.getAddon().EvmHyperSyncClient.new(cfg, userAgent, eventRegistrations);
 }
 
 let logLevelSchema = S$RescriptSchema.$$enum([