envio 3.0.0-alpha-main-clickhouse-sink → 3.0.0-alpha.1

package/src/sources/{HyperFuelSource.res.js → HyperFuelSource.res.mjs}

@@ -1,20 +1,19 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-'use strict';
 
-var Rest = require("../vendored/Rest.res.js");
-var Hrtime = require("../bindings/Hrtime.res.js");
-var Js_exn = require("rescript/lib/js/js_exn.js");
-var Source = require("./Source.res.js");
-var Logging = require("../Logging.res.js");
-var $$Promise = require("../bindings/Promise.res.js");
-var HyperFuel = require("./HyperFuel.res.js");
-var HyperSync = require("./HyperSync.res.js");
-var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Belt_Option = require("rescript/lib/js/belt_Option.js");
-var EventRouter = require("./EventRouter.res.js");
-var ErrorHandling = require("../ErrorHandling.res.js");
-var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
-var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
+import * as Rest from "../vendored/Rest.res.mjs";
+import * as Hrtime from "../bindings/Hrtime.res.mjs";
+import * as Js_exn from "rescript/lib/es6/js_exn.js";
+import * as Source from "./Source.res.mjs";
+import * as Logging from "../Logging.res.mjs";
+import * as $$Promise from "../bindings/Promise.res.mjs";
+import * as HyperFuel from "./HyperFuel.res.mjs";
+import * as HyperSync from "./HyperSync.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as EventRouter from "./EventRouter.res.mjs";
+import * as ErrorHandling from "../ErrorHandling.res.mjs";
+import * as Caml_exceptions from "rescript/lib/es6/caml_exceptions.js";
+import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 
 var EventRoutingFailed = /* @__PURE__ */Caml_exceptions.create("HyperFuelSource.EventRoutingFailed");
 
@@ -435,16 +434,18 @@ function make(param) {
         };
 }
 
-exports.EventRoutingFailed = EventRoutingFailed;
-exports.mintEventTag = mintEventTag;
-exports.burnEventTag = burnEventTag;
-exports.transferEventTag = transferEventTag;
-exports.callEventTag = callEventTag;
-exports.logDataReceiptTypeSelection = logDataReceiptTypeSelection;
-exports.txStatusSelection = txStatusSelection;
-exports.makeGetNormalRecieptsSelection = makeGetNormalRecieptsSelection;
-exports.makeWildcardRecieptsSelection = makeWildcardRecieptsSelection;
-exports.getSelectionConfig = getSelectionConfig;
-exports.memoGetSelectionConfig = memoGetSelectionConfig;
-exports.make = make;
+export {
+  EventRoutingFailed ,
+  mintEventTag ,
+  burnEventTag ,
+  transferEventTag ,
+  callEventTag ,
+  logDataReceiptTypeSelection ,
+  txStatusSelection ,
+  makeGetNormalRecieptsSelection ,
+  makeWildcardRecieptsSelection ,
+  getSelectionConfig ,
+  memoGetSelectionConfig ,
+  make ,
+}
 /* Rest Not a pure module */

package/src/sources/{HyperSync.res.js → HyperSync.res.mjs}

@@ -1,19 +1,18 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-'use strict';
 
-var Rest = require("../vendored/Rest.res.js");
-var Time = require("../Time.res.js");
-var Utils = require("../Utils.res.js");
-var $$BigInt = require("../bindings/BigInt.res.js");
-var Js_exn = require("rescript/lib/js/js_exn.js");
-var Logging = require("../Logging.res.js");
-var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Belt_Option = require("rescript/lib/js/belt_Option.js");
-var Belt_Result = require("rescript/lib/js/belt_Result.js");
-var Caml_option = require("rescript/lib/js/caml_option.js");
-var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
-var HyperSyncClient = require("./HyperSyncClient.res.js");
-var HyperSyncJsonApi = require("./HyperSyncJsonApi.res.js");
+import * as Rest from "../vendored/Rest.res.mjs";
+import * as Time from "../Time.res.mjs";
+import * as Utils from "../Utils.res.mjs";
+import * as $$BigInt from "../bindings/BigInt.res.mjs";
+import * as Js_exn from "rescript/lib/es6/js_exn.js";
+import * as Logging from "../Logging.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as Belt_Result from "rescript/lib/es6/belt_Result.js";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
+import * as Caml_exceptions from "rescript/lib/es6/caml_exceptions.js";
+import * as HyperSyncClient from "./HyperSyncClient.res.mjs";
+import * as HyperSyncJsonApi from "./HyperSyncJsonApi.res.mjs";
 
 var fieldNames = [
   "address",
@@ -325,10 +324,12 @@ var GetLogs = {
   query: query
 };
 
-exports.Log = Log;
-exports.queryErrorToMsq = queryErrorToMsq;
-exports.GetLogs = GetLogs;
-exports.queryBlockData = queryBlockData$1;
-exports.queryBlockDataMulti = queryBlockDataMulti;
-exports.mapExn = mapExn;
+export {
+  Log ,
+  queryErrorToMsq ,
+  GetLogs ,
+  queryBlockData$1 as queryBlockData,
+  queryBlockDataMulti ,
+  mapExn ,
+}
 /* Rest Not a pure module */

package/src/sources/HyperSyncClient.gen.ts

@@ -3,7 +3,7 @@
 /* eslint-disable */
 /* tslint:disable */
 
-import type {t as Address_t} from '../../src/Address.gen';
+import type {t as Address_t} from '../../src/Address.gen.js';
 
 export type ResponseTypes_accessList = { readonly address?: Address_t; readonly storageKeys?: string[] };
 

package/src/sources/HyperSyncClient.res

@@ -1,12 +1,33 @@
+/** Determines query serialization format for HTTP requests. */
+type serializationFormat =
+  // Use JSON serialization (default)
+  | Json
+  // Use Cap'n Proto binary serialization
+  | CapnProto
+
+let serializationFormatSchema = S.enum([Json, CapnProto])
+
 type cfg = {
-  url?: string,
-  bearerToken?: string,
+  /** HyperSync server URL. */
+  url: string,
+  /** HyperSync server api token. */
+  apiToken: string,
+  /** Milliseconds to wait for a response before timing out. Default: 30000. */
   httpReqTimeoutMillis?: int,
+  /** Number of retries to attempt before returning error. Default: 12. */
   maxNumRetries?: int,
+  /** Milliseconds that would be used for retry backoff increasing. Default: 500. */
   retryBackoffMs?: int,
+  /** Initial wait time for request backoff. Default: 200. */
   retryBaseMs?: int,
+  /** Ceiling time for request backoff. Default: 5000. */
   retryCeilingMs?: int,
+  /** Enable checksum addresses in responses. */
   enableChecksumAddresses?: bool,
+  /** Query serialization format to use for HTTP requests. Default: Json. */
+  serializationFormat?: serializationFormat,
+  /** Whether to use query caching when using CapnProto serialization format. */
+  enableQueryCaching?: bool,
 }
 
 module QueryTypes = {
@@ -67,7 +88,7 @@ module QueryTypes = {
     | GasUsed
     | ContractAddress
     | LogsBloom
-    | Kind
+    | Type
     | Root
     | Status
     | L1Fee
@@ -111,7 +132,7 @@ module QueryTypes = {
     | TraceAddress
     | TransactionHash
     | TransactionPosition
-    | Kind
+    | Type
     | Error
 
   type fieldSelection = {
@@ -133,7 +154,7 @@ module QueryTypes = {
     topic3,
   )
 
-  type logSelection = {
+  type logFilter = {
     /**
      * Address of the contract, any logs that has any of these addresses will be returned.
      * Empty means match all.
@@ -148,7 +169,7 @@ module QueryTypes = {
 
   let makeLogSelection = (~address, ~topics) => {address, topics}
 
-  type transactionSelection = {
+  type transactionFilter = {
     /**
      * Address the transaction should originate from. If transaction.from matches any of these, the transaction
      *  will be returned. Keep in mind that this has an and relationship with to filter, so each transaction should
@@ -167,7 +188,8 @@ module QueryTypes = {
     /** If tx.status matches this it will be returned. */
     status?: int,
     /** If transaction.type matches any of these values, the transaction will be returned */
-    kind?: array<int>,
+    @as("type")
+    type_?: array<int>,
     contractAddress?: array<Address.t>,
   }
 
@@ -177,7 +199,8 @@ module QueryTypes = {
     address?: array<Address.t>,
     callType?: array<string>,
     rewardType?: array<string>,
-    kind?: array<string>,
+    @as("type")
+    type_?: array<string>,
     sighash?: array<string>,
   }
 
@@ -214,12 +237,12 @@ module QueryTypes = {
      * List of log selections, these have an or relationship between them, so the query will return logs
      * that match any of these selections.
      */
-    logs?: array<logSelection>,
+    logs?: array<logFilter>,
     /**
      * List of transaction selections, the query will return transactions that match any of these selections and
      *  it will return transactions that are related to the returned logs.
      */
-    transactions?: array<transactionSelection>,
+    transactions?: array<transactionFilter>,
     /**
      * List of trace selections, the query will return traces that match any of these selections and
      *  it will re turn traces that are related to the returned logs.
@@ -360,7 +383,8 @@ module ResponseTypes = {
     gasUsed?: bigint,
     contractAddress?: string,
     logsBloom?: string,
-    kind?: int,
+    @as("type")
+    type_?: int,
     root?: string,
     status?: int,
     l1Fee?: bigint,
@@ -436,6 +460,22 @@ type streamConfig
 type queryResponse
 type queryResponseStream
 type eventStream
+
+@tag("type")
+type heightStreamEvent =
+  | Height({height: int})
+  | Connected
+  | Reconnecting({delayMillis: int, errorMsg: string})
+
+module HeightStream = {
+  type t = {
+    /** Close the height stream */
+    close: unit => promise<unit>,
+    /** Receive the next height stream event from the stream */
+    recv: unit => promise<heightStreamEvent>,
+  }
+}
+
 type t = {
   getHeight: unit => promise<int>,
   collect: (~query: query, ~config: streamConfig) => promise<queryResponse>,
@@ -445,23 +485,41 @@ type t = {
   getEvents: (~query: query) => promise<eventResponse>,
   stream: (~query: query, ~config: streamConfig) => promise<queryResponseStream>,
   streamEvents: (~query: query, ~config: streamConfig) => promise<eventStream>,
+  streamHeight: unit => promise<HeightStream.t>,
 }
 
-@module("@envio-dev/hypersync-client") @scope("HypersyncClient") external make: cfg => t = "new"
+@module("@envio-dev/hypersync-client") @scope("HypersyncClient")
+external makeWithAgent: (cfg, ~userAgent: string) => t = "newWithAgent"
+
 let make = (
   ~url,
   ~apiToken,
   ~httpReqTimeoutMillis,
   ~maxNumRetries,
   ~enableChecksumAddresses=true,
-) =>
-  make({
-    url,
-    enableChecksumAddresses,
-    bearerToken: apiToken,
-    httpReqTimeoutMillis,
-    maxNumRetries,
-  })
+  ~serializationFormat=?,
+  ~enableQueryCaching=?,
+  ~retryBaseMs=?,
+  ~retryBackoffMs=?,
+  ~retryCeilingMs=?,
+) => {
+  let envioVersion = Utils.EnvioPackage.value.version
+  makeWithAgent(
+    {
+      url,
+      enableChecksumAddresses,
+      apiToken,
+      httpReqTimeoutMillis,
+      maxNumRetries,
+      ?serializationFormat,
+      ?enableQueryCaching,
+      ?retryBaseMs,
+      ?retryBackoffMs,
+      ?retryCeilingMs,
+    },
+    ~userAgent=`hyperindex/${envioVersion}`,
+  )
+}
 
 module Decoder = {
   type rec decodedSolType<'a> = {val: 'a}

package/src/sources/{HyperSyncClient.res.js → HyperSyncClient.res.mjs}

@@ -1,10 +1,15 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-'use strict';
 
-var Address = require("../Address.res.js");
-var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var S$RescriptSchema = require("rescript-schema/src/S.res.js");
-var HypersyncClient = require("@envio-dev/hypersync-client");
+import * as Utils from "../Utils.res.mjs";
+import * as Address from "../Address.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as HypersyncClient from "@envio-dev/hypersync-client";
+
+var serializationFormatSchema = S$RescriptSchema.$$enum([
+      "Json",
+      "CapnProto"
+    ]);
 
 function makeTopicSelection(topic0Opt, topic1Opt, topic2Opt, topic3Opt) {
   var topic0 = topic0Opt !== undefined ? topic0Opt : [];
@@ -57,15 +62,23 @@ var ResponseTypes = {
   authorizationListSchema: authorizationListSchema
 };
 
-function make(url, apiToken, httpReqTimeoutMillis, maxNumRetries, enableChecksumAddressesOpt) {
+var HeightStream = {};
+
+function make(url, apiToken, httpReqTimeoutMillis, maxNumRetries, enableChecksumAddressesOpt, serializationFormat, enableQueryCaching, retryBaseMs, retryBackoffMs, retryCeilingMs) {
   var enableChecksumAddresses = enableChecksumAddressesOpt !== undefined ? enableChecksumAddressesOpt : true;
-  return HypersyncClient.HypersyncClient.new({
+  var envioVersion = Utils.EnvioPackage.value.version;
+  return HypersyncClient.HypersyncClient.newWithAgent({
               url: url,
-              bearerToken: apiToken,
+              apiToken: apiToken,
               httpReqTimeoutMillis: httpReqTimeoutMillis,
               maxNumRetries: maxNumRetries,
-              enableChecksumAddresses: enableChecksumAddresses
-            });
+              retryBackoffMs: retryBackoffMs,
+              retryBaseMs: retryBaseMs,
+              retryCeilingMs: retryCeilingMs,
+              enableChecksumAddresses: enableChecksumAddresses,
+              serializationFormat: serializationFormat,
+              enableQueryCaching: enableQueryCaching
+            }, "hyperindex/" + envioVersion);
 }
 
 function toUnderlying(_d) {
@@ -98,8 +111,12 @@ var Decoder = {
   fromSignatures: fromSignatures
 };
 
-exports.QueryTypes = QueryTypes;
-exports.ResponseTypes = ResponseTypes;
-exports.make = make;
-exports.Decoder = Decoder;
-/* accessListSchema Not a pure module */
+export {
+  serializationFormatSchema ,
+  QueryTypes ,
+  ResponseTypes ,
+  HeightStream ,
+  make ,
+  Decoder ,
+}
+/* serializationFormatSchema Not a pure module */

package/src/sources/{HyperSyncJsonApi.res.js → HyperSyncJsonApi.res.mjs}

@@ -1,10 +1,9 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-'use strict';
 
-var $$BigInt = require("../bindings/BigInt.res.js");
-var Address = require("../Address.res.js");
-var EvmTypes = require("../EvmTypes.res.js");
-var S$RescriptSchema = require("rescript-schema/src/S.res.js");
+import * as $$BigInt from "../bindings/BigInt.res.mjs";
+import * as Address from "../Address.res.mjs";
+import * as EvmTypes from "../EvmTypes.res.mjs";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 
 var blockFieldOptionsSchema = S$RescriptSchema.$$enum([
       "number",
@@ -259,8 +258,10 @@ function heightRoute() {
         };
 }
 
-exports.QueryTypes = QueryTypes;
-exports.ResponseTypes = ResponseTypes;
-exports.queryRoute = queryRoute;
-exports.heightRoute = heightRoute;
+export {
+  QueryTypes ,
+  ResponseTypes ,
+  queryRoute ,
+  heightRoute ,
+}
 /* blockFieldOptionsSchema Not a pure module */

package/src/sources/HyperSyncSource.res

@@ -155,6 +155,65 @@ type options = {
   clientMaxRetries: int,
   clientTimeoutMillis: int,
   lowercaseAddresses: bool,
+  serializationFormat: HyperSyncClient.serializationFormat,
+  enableQueryCaching: bool,
+}
+
+module HeightState: {
+  type t
+  let make: (HyperSyncClient.t, ~chainId: int) => Promise.t<t>
+  let getHeight: t => Promise.t<int>
+} = {
+  open HyperSyncClient
+  type t = {
+    heightStream: HeightStream.t,
+    mutable currentHeight: int,
+    mutable errMessage: option<string>,
+  }
+
+  let makeInternal = async (client: HyperSyncClient.t) => {
+    currentHeight: 0,
+    heightStream: await client.streamHeight(),
+    errMessage: None,
+  }
+
+  let start = async (state: t, ~chainId) => {
+    while true {
+      let height = await state.heightStream.recv()
+      switch height {
+      | Height({height}) => state.currentHeight = height
+      | Connected =>
+        Logging.trace({"msg": "HyperSync height stream is connected", "chainId": chainId})
+        state.errMessage = None
+      | Reconnecting({delayMillis, errorMsg}) =>
+        Logging.trace({
+          "msg": "HyperSync height stream is reconnecting",
+          "err": errorMsg,
+          "delayMillis": delayMillis,
+          "chainId": chainId,
+        })
+        state.errMessage = Some(errorMsg)
+      }
+    }
+  }
+
+  let make = async (client: HyperSyncClient.t, ~chainId) => {
+    let state = await makeInternal(client)
+    let _async = state->start(~chainId)
+    state
+  }
+
+  let getHeight = async (state: t) => {
+    while state.currentHeight == 0 && state.errMessage->Belt.Option.isNone {
+      // Just poll internally until its no longer 0
+      await Utils.delay(200)
+    }
+
+    switch state.errMessage {
+    | Some(errMessage) => Js.Exn.raiseError(errMessage)
+    | None => state.currentHeight
+    }
+  }
 }
 
 let make = (
@@ -169,6 +228,8 @@ let make = (
     clientMaxRetries,
     clientTimeoutMillis,
     lowercaseAddresses,
+    serializationFormat,
+    enableQueryCaching,
   }: options,
 ): t => {
   let name = "HyperSync"
@@ -183,6 +244,8 @@ let make = (
     ~maxNumRetries=clientMaxRetries,
     ~httpReqTimeoutMillis=clientTimeoutMillis,
     ~enableChecksumAddresses=!lowercaseAddresses,
+    ~serializationFormat,
+    ~enableQueryCaching,
   )
 
   let hscDecoder: ref<option<HyperSyncClient.Decoder.t>> = ref(None)
@@ -545,28 +608,33 @@ let make = (
       ~logger,
     )->Promise.thenResolve(HyperSync.mapExn)
 
-  let jsonApiClient = Rest.client(endpointUrl)
-
   let malformedTokenMessage = `Your token is malformed. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens.`
 
+  let heightStatePromise = HeightState.make(client, ~chainId=chain->ChainMap.Chain.toChainId)
+
   {
     name,
     sourceFor: Sync,
     chain,
-    pollingInterval: 100,
+    pollingInterval: 32, // purely internal state polling since we now stream the height to the state
     poweredByHyperSync: true,
     getBlockHashes,
-    getHeightOrThrow: async () =>
-      switch await HyperSyncJsonApi.heightRoute->Rest.fetch(apiToken, ~client=jsonApiClient) {
-      | Value(height) => height
-      | ErrorMessage(m) if m === malformedTokenMessage =>
+    getHeightOrThrow: async () => {
+      let heightState = await heightStatePromise
+      try await heightState->HeightState.getHeight catch {
+      | Js.Exn.Error(exn)
+        if exn
+        ->Js.Exn.message
+        ->Option.getWithDefault("")
+        ->Js.String2.includes(malformedTokenMessage) =>
         Logging.error(`Your ENVIO_API_TOKEN is malformed. The indexer will not be able to fetch events. Update the token and restart the indexer using 'pnpm envio start'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`)
         // Don't want to retry if the token is malformed
         // So just block forever
         let _ = await Promise.make((_, _) => ())
         0
-      | ErrorMessage(m) => Js.Exn.raiseError(m)
-      },
+      | exn => raise(exn)
+      }
+    },
     getItemsOrThrow,
   }
 }