envio 2.32.8 → 2.32.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.32.8",
+  "version": "v2.32.11",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.32.8",
-    "envio-linux-arm64": "v2.32.8",
-    "envio-darwin-x64": "v2.32.8",
-    "envio-darwin-arm64": "v2.32.8"
+    "envio-linux-x64": "v2.32.11",
+    "envio-linux-arm64": "v2.32.11",
+    "envio-darwin-x64": "v2.32.11",
+    "envio-darwin-arm64": "v2.32.11"
   },
   "dependencies": {
     "@elastic/ecs-pino-format": "1.4.0",

package/src/sources/HyperSync.res

@@ -104,9 +104,11 @@ module GetLogs = {
     // Remap "type" -> "kind" on the transaction object at runtime before validation.
     // The latest hypersync client renamed "kind" to "type" but v2 consumers expect "kind".
     let transaction: Js.Dict.t<unknown> = event.transaction->Utils.magic
-    switch transaction->Js.Dict.get("type") {
-    | Some(v) => transaction->Js.Dict.set("kind", v)
-    | None => ()
+    if transaction->Utils.magic {
+      switch transaction->Js.Dict.get("type") {
+      | Some(v) => transaction->Js.Dict.set("kind", v)
+      | None => ()
+      }
     }
 
     let missingParams = []
@@ -197,7 +199,7 @@ module GetLogs = {
 }
 
 module BlockData = {
-  let makeRequestBody = (~fromBlock, ~toBlock): HyperSyncJsonApi.QueryTypes.postQueryBody => {
+  let makeRequestBody = (~fromBlock, ~toBlock): HyperSyncClient.QueryTypes.query => {
     fromBlock,
     toBlockExclusive: toBlock + 1,
     fieldSelection: {
@@ -206,51 +208,44 @@ module BlockData = {
     includeAllBlocks: true,
   }
 
-  let convertResponse = (res: HyperSyncJsonApi.ResponseTypes.queryResponse): queryResponse<
+  let convertResponse = (res: HyperSyncClient.queryResponse): queryResponse<
     array<ReorgDetection.blockDataWithTimestamp>,
   > => {
-    res.data
-    ->Array.flatMap(item => {
-      item.blocks->Option.mapWithDefault([], blocks => {
-        blocks->Array.map(
-          block => {
-            switch block {
-            | {number: blockNumber, timestamp, hash: blockHash} =>
-              let blockTimestamp = timestamp->BigInt.toInt->Option.getExn
-              Ok(
-                (
-                  {
-                    blockTimestamp,
-                    blockNumber,
-                    blockHash,
-                  }: ReorgDetection.blockDataWithTimestamp
-                ),
-              )
-            | _ =>
-              let missingParams =
-                [
-                  block.number->Utils.Option.mapNone("block.number"),
-                  block.timestamp->Utils.Option.mapNone("block.timestamp"),
-                  block.hash->Utils.Option.mapNone("block.hash"),
-                ]->Array.keepMap(p => p)
-
-              Error(
-                UnexpectedMissingParams({
-                  queryName: "query block data HyperSync",
-                  missingParams,
-                }),
-              )
-            }
-          },
+    res.data.blocks
+    ->Option.getWithDefault([])
+    ->Array.map(block => {
+      switch block {
+      | {number: blockNumber, timestamp: blockTimestamp, hash: blockHash} =>
+        Ok(
+          (
+            {
+              blockTimestamp,
+              blockNumber,
+              blockHash,
+            }: ReorgDetection.blockDataWithTimestamp
+          ),
         )
-      })
+      | _ =>
+        let missingParams =
+          [
+            block.number->Utils.Option.mapNone("block.number"),
+            block.timestamp->Utils.Option.mapNone("block.timestamp"),
+            block.hash->Utils.Option.mapNone("block.hash"),
+          ]->Array.keepMap(p => p)
+
+        Error(
+          UnexpectedMissingParams({
+            queryName: "query block data HyperSync",
+            missingParams,
+          }),
+        )
+      }
     })
     ->Utils.Array.transposeResults
   }
 
   let rec queryBlockData = async (
-    ~serverUrl,
-    ~apiToken,
+    ~client: HyperSyncClient.t,
     ~fromBlock,
     ~toBlock,
     ~logger,
@@ -266,38 +261,28 @@ module BlockData = {
       },
     )
 
-    let maybeSuccessfulRes = switch await Time.retryAsyncWithExponentialBackOff(() =>
-      HyperSyncJsonApi.queryRoute->Rest.fetch(
-        {
-          "query": body,
-          "token": apiToken,
-        },
-        ~client=Rest.client(serverUrl),
-      )
-    , ~logger) {
+    let maybeSuccessfulRes = switch await client.get(~query=body) {
     | exception _ => None
     | res if res.nextBlock <= fromBlock => None
     | res => Some(res)
     }
 
-    // If the block is not found, retry the query. This can occur since replicas of hypersync might not hack caught up yet
+    // If the block is not found, retry the query. This can occur since replicas of hypersync might not have caught up yet
     switch maybeSuccessfulRes {
     | None => {
-        let logger = Logging.createChild(~params={"url": serverUrl})
         let delayMilliseconds = 100
         logger->Logging.childInfo(
           `Block #${fromBlock->Int.toString} not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${delayMilliseconds->Int.toString}ms.`,
         )
         await Time.resolvePromiseAfterDelay(~delayMilliseconds)
-        await queryBlockData(~serverUrl, ~apiToken, ~fromBlock, ~toBlock, ~logger)
+        await queryBlockData(~client, ~fromBlock, ~toBlock, ~logger)
       }
     | Some(res) =>
       switch res->convertResponse {
       | Error(_) as err => err
       | Ok(datas) if res.nextBlock <= toBlock => {
           let restRes = await queryBlockData(
-            ~serverUrl,
-            ~apiToken,
+            ~client,
             ~fromBlock=res.nextBlock,
             ~toBlock,
             ~logger,
@@ -309,7 +294,7 @@ module BlockData = {
     }
   }
 
-  let queryBlockDataMulti = async (~serverUrl, ~apiToken, ~blockNumbers, ~logger) => {
+  let queryBlockDataMulti = async (~client, ~blockNumbers, ~logger) => {
     switch blockNumbers->Array.get(0) {
     | None => Ok([])
     | Some(firstBlock) => {
@@ -334,8 +319,7 @@ module BlockData = {
         let res = await queryBlockData(
           ~fromBlock=fromBlock.contents,
           ~toBlock=toBlock.contents,
-          ~serverUrl,
-          ~apiToken,
+          ~client,
           ~logger,
         )
         let filtered = res->Result.map(datas => {
@@ -354,10 +338,9 @@ module BlockData = {
   }
 }
 
-let queryBlockData = (~serverUrl, ~apiToken, ~blockNumber, ~logger) =>
+let queryBlockData = (~client, ~blockNumber, ~logger) =>
   BlockData.queryBlockData(
-    ~serverUrl,
-    ~apiToken,
+    ~client,
     ~fromBlock=blockNumber,
     ~toBlock=blockNumber,
     ~logger,

package/src/sources/HyperSync.res.js

@@ -1,10 +1,8 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
-var Rest = require("../vendored/Rest.res.js");
 var Time = require("../Time.res.js");
 var Utils = require("../Utils.res.js");
-var $$BigInt = require("../bindings/BigInt.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Js_dict = require("rescript/lib/js/js_dict.js");
 var Logging = require("../Logging.res.js");
@@ -14,7 +12,6 @@ var Belt_Result = require("rescript/lib/js/belt_Result.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
 var HyperSyncClient = require("./HyperSyncClient.res.js");
-var HyperSyncJsonApi = require("./HyperSyncJsonApi.res.js");
 
 var fieldNames = [
   "address",
@@ -77,9 +74,12 @@ async function query(client, fromBlock, toBlock, logSelections, fieldSelection,
   }
   var items = Belt_Array.map(res.data, (function (item) {
           var transaction = item.transaction;
-          var v = Js_dict.get(transaction, "type");
-          if (v !== undefined) {
-            transaction["kind"] = Caml_option.valFromOption(v);
+          if (transaction) {
+            var v = Js_dict.get(transaction, "type");
+            if (v !== undefined) {
+              transaction["kind"] = Caml_option.valFromOption(v);
+            }
+            
           }
           var missingParams = [];
           var returnedObj = item.log;
@@ -174,12 +174,12 @@ async function query(client, fromBlock, toBlock, logSelections, fieldSelection,
 function makeRequestBody$1(fromBlock, toBlock) {
   return {
           fromBlock: fromBlock,
-          toBlockExclusive: toBlock + 1 | 0,
+          toBlock: toBlock + 1 | 0,
           fieldSelection: {
             block: [
-              "number",
-              "hash",
-              "timestamp"
+              "Number",
+              "Hash",
+              "Timestamp"
             ]
           },
           includeAllBlocks: true
@@ -187,52 +187,47 @@ function makeRequestBody$1(fromBlock, toBlock) {
 }
 
 function convertResponse(res) {
-  return Utils.$$Array.transposeResults(Belt_Array.flatMap(res.data, (function (item) {
-                    return Belt_Option.mapWithDefault(item.blocks, [], (function (blocks) {
-                                  return Belt_Array.map(blocks, (function (block) {
-                                                var blockNumber = block.number;
-                                                if (blockNumber !== undefined) {
-                                                  var blockHash = block.hash;
-                                                  if (blockHash !== undefined) {
-                                                    var timestamp = block.timestamp;
-                                                    if (timestamp !== undefined) {
-                                                      var blockTimestamp = Belt_Option.getExn($$BigInt.toInt(timestamp));
-                                                      return {
-                                                              TAG: "Ok",
-                                                              _0: {
-                                                                blockHash: blockHash,
-                                                                blockNumber: blockNumber,
-                                                                blockTimestamp: blockTimestamp
-                                                              }
-                                                            };
-                                                    }
-                                                    
-                                                  }
-                                                  
-                                                }
-                                                var missingParams = Belt_Array.keepMap([
-                                                      Utils.$$Option.mapNone(block.number, "block.number"),
-                                                      Utils.$$Option.mapNone(block.timestamp, "block.timestamp"),
-                                                      Utils.$$Option.mapNone(block.hash, "block.hash")
-                                                    ], (function (p) {
-                                                        return p;
-                                                      }));
-                                                return {
-                                                        TAG: "Error",
-                                                        _0: {
-                                                          TAG: "UnexpectedMissingParams",
-                                                          _0: {
-                                                            queryName: "query block data HyperSync",
-                                                            missingParams: missingParams
-                                                          }
-                                                        }
-                                                      };
-                                              }));
-                                }));
+  return Utils.$$Array.transposeResults(Belt_Array.map(Belt_Option.getWithDefault(res.data.blocks, []), (function (block) {
+                    var blockNumber = block.number;
+                    if (blockNumber !== undefined) {
+                      var blockHash = block.hash;
+                      if (blockHash !== undefined) {
+                        var blockTimestamp = block.timestamp;
+                        if (blockTimestamp !== undefined) {
+                          return {
+                                  TAG: "Ok",
+                                  _0: {
+                                    blockHash: blockHash,
+                                    blockNumber: blockNumber,
+                                    blockTimestamp: blockTimestamp
+                                  }
+                                };
+                        }
+                        
+                      }
+                      
+                    }
+                    var missingParams = Belt_Array.keepMap([
+                          Utils.$$Option.mapNone(block.number, "block.number"),
+                          Utils.$$Option.mapNone(block.timestamp, "block.timestamp"),
+                          Utils.$$Option.mapNone(block.hash, "block.hash")
+                        ], (function (p) {
+                            return p;
+                          }));
+                    return {
+                            TAG: "Error",
+                            _0: {
+                              TAG: "UnexpectedMissingParams",
+                              _0: {
+                                queryName: "query block data HyperSync",
+                                missingParams: missingParams
+                              }
+                            }
+                          };
                   })));
 }
 
-async function queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger) {
+async function queryBlockData(client, fromBlock, toBlock, logger) {
   var body = makeRequestBody$1(fromBlock, toBlock);
   var logger$1 = Logging.createChildFrom(logger, {
         logType: "HyperSync get block hash query",
@@ -243,12 +238,7 @@ async function queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger) {
   var exit = 0;
   var res;
   try {
-    res = await Time.retryAsyncWithExponentialBackOff(undefined, undefined, undefined, undefined, logger$1, (function () {
-            return Rest.$$fetch(HyperSyncJsonApi.queryRoute, {
-                        query: body,
-                        token: apiToken
-                      }, Rest.client(serverUrl, undefined));
-          }));
+    res = await client.get(body);
     exit = 1;
   }
   catch (exn){
@@ -266,20 +256,17 @@ async function queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger) {
       return err;
     }
     var datas = err._0;
-    var restRes = await queryBlockData(serverUrl, apiToken, maybeSuccessfulRes.nextBlock, toBlock, logger$1);
+    var restRes = await queryBlockData(client, maybeSuccessfulRes.nextBlock, toBlock, logger$1);
     return Belt_Result.map(restRes, (function (rest) {
                   return Belt_Array.concat(datas, rest);
                 }));
   }
-  var logger$2 = Logging.createChild({
-        url: serverUrl
-      });
-  Logging.childInfo(logger$2, "Block #" + String(fromBlock) + " not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in " + String(100) + "ms.");
+  Logging.childInfo(logger$1, "Block #" + String(fromBlock) + " not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in " + String(100) + "ms.");
   await Time.resolvePromiseAfterDelay(100);
-  return await queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger$2);
+  return await queryBlockData(client, fromBlock, toBlock, logger$1);
 }
 
-async function queryBlockDataMulti(serverUrl, apiToken, blockNumbers, logger) {
+async function queryBlockDataMulti(client, blockNumbers, logger) {
   var firstBlock = Belt_Array.get(blockNumbers, 0);
   if (firstBlock === undefined) {
     return {
@@ -303,7 +290,7 @@ async function queryBlockDataMulti(serverUrl, apiToken, blockNumbers, logger) {
   if ((toBlock - fromBlock | 0) > 1000) {
     Js_exn.raiseError("Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: " + String(fromBlock) + "-" + String(toBlock));
   }
-  var res = await queryBlockData(serverUrl, apiToken, fromBlock, toBlock, logger);
+  var res = await queryBlockData(client, fromBlock, toBlock, logger);
   var filtered = Belt_Result.map(res, (function (datas) {
           return Belt_Array.keep(datas, (function (data) {
                         return set.delete(data.blockNumber);
@@ -315,8 +302,8 @@ async function queryBlockDataMulti(serverUrl, apiToken, blockNumbers, logger) {
   return filtered;
 }
 
-function queryBlockData$1(serverUrl, apiToken, blockNumber, logger) {
-  return queryBlockData(serverUrl, apiToken, blockNumber, blockNumber, logger).then(function (res) {
+function queryBlockData$1(client, blockNumber, logger) {
+  return queryBlockData(client, blockNumber, blockNumber, logger).then(function (res) {
               return Belt_Result.map(res, (function (res) {
                             return Belt_Array.get(res, 0);
                           }));
@@ -336,4 +323,4 @@ exports.GetLogs = GetLogs;
 exports.queryBlockData = queryBlockData$1;
 exports.queryBlockDataMulti = queryBlockDataMulti;
 exports.mapExn = mapExn;
-/* Rest Not a pure module */
+/* Time Not a pure module */

package/src/sources/HyperSync.resi

@@ -53,15 +53,13 @@ module GetLogs: {
 }
 
 let queryBlockData: (
-  ~serverUrl: string,
-  ~apiToken: string,
+  ~client: HyperSyncClient.t,
   ~blockNumber: int,
   ~logger: Pino.t,
 ) => promise<queryResponse<option<ReorgDetection.blockDataWithTimestamp>>>
 
 let queryBlockDataMulti: (
-  ~serverUrl: string,
-  ~apiToken: string,
+  ~client: HyperSyncClient.t,
   ~blockNumbers: array<int>,
   ~logger: Pino.t,
 ) => promise<queryResponse<array<ReorgDetection.blockDataWithTimestamp>>>

package/src/sources/HyperSyncClient.res

@@ -284,6 +284,12 @@ module QueryTypes = {
      * JoinNothing: join nothing.
      */
     joinMode?: joinMode,
+    /**
+     * Whether to include all blocks regardless of if they are related to a returned transaction or log. Normally
+     *  the server will return only the blocks that are related to the transaction or logs in the response. But if this
+     *  is set to true, the server will return data for all blocks in the requested range [from_block, to_block).
+     */
+    includeAllBlocks?: bool,
   }
 }
 
@@ -456,10 +462,23 @@ type eventResponse = ResponseTypes.eventResponse
 
 //Todo, add bindings for these types
 type streamConfig
-type queryResponse
 type queryResponseStream
 type eventStream
 
+type queryResponseData = {
+  blocks: option<array<ResponseTypes.block>>,
+  transactions: option<array<ResponseTypes.transaction>>,
+  logs: option<array<ResponseTypes.log>>,
+}
+
+type queryResponse = {
+  archiveHeight: option<int>,
+  nextBlock: int,
+  totalExecutionTime: int,
+  data: queryResponseData,
+  rollbackGuard: option<ResponseTypes.rollbackGuard>,
+}
+
 @tag("type")
 type heightStreamEvent =
   | Height({height: int})
@@ -520,6 +539,16 @@ let make = (
   )
 }
 
+type logLevel = [#trace | #debug | #info | #warn | #error]
+let logLevelSchema: S.t<logLevel> = S.enum([#trace, #debug, #info, #warn, #error])
+
+/**
+ * Set the log level for the underlying Rust logger in hypersync-client.
+ * Must be called before creating any HypersyncClient.
+ */
+@module("@envio-dev/hypersync-client")
+external setLogLevel: logLevel => unit = "setLogLevel"
+
 module Decoder = {
   type rec decodedSolType<'a> = {val: 'a}
 

package/src/sources/HyperSyncClient.res.js

@@ -82,6 +82,14 @@ function make(url, apiToken, httpReqTimeoutMillis, maxNumRetries, enableChecksum
             }, "hyperindex/" + envioVersion);
 }
 
+var logLevelSchema = S$RescriptSchema.$$enum([
+      "trace",
+      "debug",
+      "info",
+      "warn",
+      "error"
+    ]);
+
 function toUnderlying(_d) {
   while(true) {
     var d = _d;
@@ -117,5 +125,6 @@ exports.QueryTypes = QueryTypes;
 exports.ResponseTypes = ResponseTypes;
 exports.HeightStream = HeightStream;
 exports.make = make;
+exports.logLevelSchema = logLevelSchema;
 exports.Decoder = Decoder;
 /* serializationFormatSchema Not a pure module */

package/src/sources/HyperSyncSource.res

@@ -369,8 +369,7 @@ let make = (
         //If there were no logs at all in the current page query then fetch the
         //timestamp of the heighest block accounted for
         HyperSync.queryBlockData(
-          ~serverUrl=endpointUrl,
-          ~apiToken,
+          ~client,
           ~blockNumber=heighestBlockQueried,
           ~logger,
         )->Promise.thenResolve(res =>
@@ -548,8 +547,7 @@ let make = (
 
   let getBlockHashes = (~blockNumbers, ~logger) =>
     HyperSync.queryBlockDataMulti(
-      ~serverUrl=endpointUrl,
-      ~apiToken,
+      ~client,
       ~blockNumbers,
       ~logger,
     )->Promise.thenResolve(HyperSync.mapExn)

package/src/sources/HyperSyncSource.res.js

@@ -280,7 +280,7 @@ function make(param) {
         exit = 1;
       }
       if (exit === 1) {
-        lastBlockQueriedPromise = HyperSync.queryBlockData(endpointUrl, apiToken, heighestBlockQueried, logger).then(function (res) {
+        lastBlockQueriedPromise = HyperSync.queryBlockData(client, heighestBlockQueried, logger).then(function (res) {
               if (res.TAG !== "Ok") {
                 return mkLogAndRaise("Failed to query blockData for block " + String(heighestBlockQueried), HyperSync.queryErrorToMsq(res._0));
               }
@@ -390,7 +390,7 @@ function make(param) {
           };
   };
   var getBlockHashes = function (blockNumbers, logger) {
-    return HyperSync.queryBlockDataMulti(endpointUrl, apiToken, blockNumbers, logger).then(HyperSync.mapExn);
+    return HyperSync.queryBlockDataMulti(client, blockNumbers, logger).then(HyperSync.mapExn);
   };
   var jsonApiClient = Rest.client(endpointUrl, undefined);
   return {