envio 3.0.0-alpha-main-clickhouse-sink → 3.0.0-alpha.1

package/src/sources/{HyperSyncSource.res.js → HyperSyncSource.res.mjs}

@@ -1,24 +1,21 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-'use strict';
 
-var Rest = require("../vendored/Rest.res.js");
-var Viem = require("../bindings/Viem.res.js");
-var Utils = require("../Utils.res.js");
-var Hrtime = require("../bindings/Hrtime.res.js");
-var Js_exn = require("rescript/lib/js/js_exn.js");
-var Source = require("./Source.res.js");
-var Logging = require("../Logging.res.js");
-var HyperSync = require("./HyperSync.res.js");
-var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Belt_Option = require("rescript/lib/js/belt_Option.js");
-var EventRouter = require("./EventRouter.res.js");
-var LogSelection = require("../LogSelection.res.js");
-var ErrorHandling = require("../ErrorHandling.res.js");
-var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
-var HyperSyncClient = require("./HyperSyncClient.res.js");
-var Caml_splice_call = require("rescript/lib/js/caml_splice_call.js");
-var HyperSyncJsonApi = require("./HyperSyncJsonApi.res.js");
-var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
+import * as Viem from "../bindings/Viem.res.mjs";
+import * as Utils from "../Utils.res.mjs";
+import * as Hrtime from "../bindings/Hrtime.res.mjs";
+import * as Js_exn from "rescript/lib/es6/js_exn.js";
+import * as Source from "./Source.res.mjs";
+import * as Logging from "../Logging.res.mjs";
+import * as HyperSync from "./HyperSync.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as EventRouter from "./EventRouter.res.mjs";
+import * as LogSelection from "../LogSelection.res.mjs";
+import * as ErrorHandling from "../ErrorHandling.res.mjs";
+import * as Caml_exceptions from "rescript/lib/es6/caml_exceptions.js";
+import * as HyperSyncClient from "./HyperSyncClient.res.mjs";
+import * as Caml_splice_call from "rescript/lib/es6/caml_splice_call.js";
+import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 
 function getSelectionConfig(selection, chain) {
   var nonOptionalBlockFieldNames = new Set();
@@ -123,7 +120,62 @@ function memoGetSelectionConfig(chain) {
   };
 }
 
-function make(param) {
+async function makeInternal(client) {
+  return {
+          heightStream: await client.streamHeight(),
+          currentHeight: 0,
+          errMessage: undefined
+        };
+}
+
+async function start(state, chainId) {
+  while(true) {
+    var height = await state.heightStream.recv();
+    if (typeof height !== "object") {
+      Logging.trace({
+            msg: "HyperSync height stream is connected",
+            chainId: chainId
+          });
+      state.errMessage = undefined;
+    } else if (height.type === "Height") {
+      state.currentHeight = height.height;
+    } else {
+      var errorMsg = height.errorMsg;
+      Logging.trace({
+            msg: "HyperSync height stream is reconnecting",
+            err: errorMsg,
+            delayMillis: height.delayMillis,
+            chainId: chainId
+          });
+      state.errMessage = errorMsg;
+    }
+  };
+}
+
+async function make(client, chainId) {
+  var state = await makeInternal(client);
+  start(state, chainId);
+  return state;
+}
+
+async function getHeight(state) {
+  while(state.currentHeight === 0 && Belt_Option.isNone(state.errMessage)) {
+    await Utils.delay(200);
+  };
+  var errMessage = state.errMessage;
+  if (errMessage !== undefined) {
+    return Js_exn.raiseError(errMessage);
+  } else {
+    return state.currentHeight;
+  }
+}
+
+var HeightState = {
+  make: make,
+  getHeight: getHeight
+};
+
+function make$1(param) {
   var lowercaseAddresses = param.lowercaseAddresses;
   var eventRouter = param.eventRouter;
   var shouldUseHypersyncClientDecoder = param.shouldUseHypersyncClientDecoder;
@@ -132,7 +184,7 @@ function make(param) {
   var chain = param.chain;
   var getSelectionConfig = memoGetSelectionConfig(chain);
   var apiToken = Belt_Option.getWithDefault(param.apiToken, "3dc856dd-b0ea-494f-b27e-017b8b6b7e07");
-  var client = HyperSyncClient.make(endpointUrl, apiToken, param.clientTimeoutMillis, param.clientMaxRetries, !lowercaseAddresses);
+  var client = HyperSyncClient.make(endpointUrl, apiToken, param.clientTimeoutMillis, param.clientMaxRetries, !lowercaseAddresses, param.serializationFormat, param.enableQueryCaching, undefined, undefined, undefined);
   var hscDecoder = {
     contents: undefined
   };
@@ -385,33 +437,42 @@ function make(param) {
   var getBlockHashes = function (blockNumbers, logger) {
     return HyperSync.queryBlockDataMulti(endpointUrl, apiToken, blockNumbers, logger).then(HyperSync.mapExn);
   };
-  var jsonApiClient = Rest.client(endpointUrl, undefined);
+  var heightStatePromise = make(client, chain);
   return {
           name: "HyperSync",
           sourceFor: "Sync",
           chain: chain,
           poweredByHyperSync: true,
-          pollingInterval: 100,
+          pollingInterval: 32,
           getBlockHashes: getBlockHashes,
           getHeightOrThrow: (async function () {
-              var height = await Rest.$$fetch(HyperSyncJsonApi.heightRoute, apiToken, jsonApiClient);
-              if (typeof height === "number") {
-                return height;
-              } else if (height === "Your token is malformed. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens.") {
-                Logging.error("Your ENVIO_API_TOKEN is malformed. The indexer will not be able to fetch events. Update the token and restart the indexer using 'pnpm envio start'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens");
-                await new Promise((function (param, param$1) {
-                        
-                      }));
-                return 0;
-              } else {
-                return Js_exn.raiseError(height);
+              var heightState = await heightStatePromise;
+              try {
+                return await getHeight(heightState);
+              }
+              catch (raw_exn){
+                var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+                if (exn.RE_EXN_ID === Js_exn.$$Error) {
+                  if (Belt_Option.getWithDefault(exn._1.message, "").includes("Your token is malformed. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens.")) {
+                    Logging.error("Your ENVIO_API_TOKEN is malformed. The indexer will not be able to fetch events. Update the token and restart the indexer using 'pnpm envio start'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens");
+                    await new Promise((function (param, param$1) {
+                            
+                          }));
+                    return 0;
+                  }
+                  throw exn;
+                }
+                throw exn;
               }
             }),
           getItemsOrThrow: getItemsOrThrow
         };
 }
 
-exports.getSelectionConfig = getSelectionConfig;
-exports.memoGetSelectionConfig = memoGetSelectionConfig;
-exports.make = make;
-/* Rest Not a pure module */
+export {
+  getSelectionConfig ,
+  memoGetSelectionConfig ,
+  HeightState ,
+  make$1 as make,
+}
+/* Viem Not a pure module */

package/src/sources/{Rpc.res.js → Rpc.res.mjs}

@@ -1,13 +1,12 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-'use strict';
 
-var Viem = require("viem");
-var Utils = require("../Utils.res.js");
-var $$BigInt = require("../bindings/BigInt.res.js");
-var Address = require("../Address.res.js");
-var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Caml_option = require("rescript/lib/js/caml_option.js");
-var S$RescriptSchema = require("rescript-schema/src/S.res.js");
+import * as Viem from "viem";
+import * as Utils from "../Utils.res.mjs";
+import * as $$BigInt from "../bindings/BigInt.res.mjs";
+import * as Address from "../Address.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 
 function makeRpcRoute(method, paramsSchema, resultSchema) {
   var idSchema = S$RescriptSchema.literal(1);
@@ -188,11 +187,13 @@ var GetBlockHeight = {
   route: route$2
 };
 
-exports.makeRpcRoute = makeRpcRoute;
-exports.makeHexSchema = makeHexSchema;
-exports.hexBigintSchema = hexBigintSchema;
-exports.hexIntSchema = hexIntSchema;
-exports.GetLogs = GetLogs;
-exports.GetBlockByNumber = GetBlockByNumber;
-exports.GetBlockHeight = GetBlockHeight;
+export {
+  makeRpcRoute ,
+  makeHexSchema ,
+  hexBigintSchema ,
+  hexIntSchema ,
+  GetLogs ,
+  GetBlockByNumber ,
+  GetBlockHeight ,
+}
 /* hexBigintSchema Not a pure module */

package/src/sources/{RpcSource.res.js → RpcSource.res.mjs}

@@ -1,32 +1,31 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-'use strict';
 
-var Rpc = require("./Rpc.res.js");
-var Caml = require("rescript/lib/js/caml.js");
-var Rest = require("../vendored/Rest.res.js");
-var Time = require("../Time.res.js");
-var Viem = require("../bindings/Viem.res.js");
-var Utils = require("../Utils.res.js");
-var Ethers = require("../bindings/Ethers.res.js");
-var Hrtime = require("../bindings/Hrtime.res.js");
-var Js_exn = require("rescript/lib/js/js_exn.js");
-var Source = require("./Source.res.js");
-var Address = require("../Address.res.js");
-var Logging = require("../Logging.res.js");
-var $$Promise = require("../bindings/Promise.res.js");
-var Belt_Int = require("rescript/lib/js/belt_Int.js");
-var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var FetchState = require("../FetchState.res.js");
-var LazyLoader = require("../LazyLoader.res.js");
-var Belt_Option = require("rescript/lib/js/belt_Option.js");
-var Caml_option = require("rescript/lib/js/caml_option.js");
-var EventRouter = require("./EventRouter.res.js");
-var LogSelection = require("../LogSelection.res.js");
-var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
-var HyperSyncClient = require("./HyperSyncClient.res.js");
-var Caml_splice_call = require("rescript/lib/js/caml_splice_call.js");
-var S$RescriptSchema = require("rescript-schema/src/S.res.js");
-var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
+import * as Rpc from "./Rpc.res.mjs";
+import * as Caml from "rescript/lib/es6/caml.js";
+import * as Rest from "../vendored/Rest.res.mjs";
+import * as Time from "../Time.res.mjs";
+import * as Viem from "../bindings/Viem.res.mjs";
+import * as Utils from "../Utils.res.mjs";
+import * as Ethers from "../bindings/Ethers.res.mjs";
+import * as Hrtime from "../bindings/Hrtime.res.mjs";
+import * as Js_exn from "rescript/lib/es6/js_exn.js";
+import * as Source from "./Source.res.mjs";
+import * as Address from "../Address.res.mjs";
+import * as Logging from "../Logging.res.mjs";
+import * as $$Promise from "../bindings/Promise.res.mjs";
+import * as Belt_Int from "rescript/lib/es6/belt_Int.js";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as FetchState from "../FetchState.res.mjs";
+import * as LazyLoader from "../LazyLoader.res.mjs";
+import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
+import * as EventRouter from "./EventRouter.res.mjs";
+import * as LogSelection from "../LogSelection.res.mjs";
+import * as Caml_exceptions from "rescript/lib/es6/caml_exceptions.js";
+import * as HyperSyncClient from "./HyperSyncClient.res.mjs";
+import * as Caml_splice_call from "rescript/lib/es6/caml_splice_call.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 
 var QueryTimout = /* @__PURE__ */Caml_exceptions.create("RpcSource.QueryTimout");
 
@@ -835,16 +834,18 @@ function make(param) {
         };
 }
 
-exports.QueryTimout = QueryTimout;
-exports.getKnownBlock = getKnownBlock;
-exports.getKnownBlockWithBackoff = getKnownBlockWithBackoff;
-exports.getSuggestedBlockIntervalFromExn = getSuggestedBlockIntervalFromExn;
-exports.maxSuggestedBlockIntervalKey = maxSuggestedBlockIntervalKey;
-exports.getNextPage = getNextPage;
-exports.getSelectionConfig = getSelectionConfig;
-exports.memoGetSelectionConfig = memoGetSelectionConfig;
-exports.makeThrowingGetEventBlock = makeThrowingGetEventBlock;
-exports.makeThrowingGetEventTransaction = makeThrowingGetEventTransaction;
-exports.sanitizeUrl = sanitizeUrl;
-exports.make = make;
+export {
+  QueryTimout ,
+  getKnownBlock ,
+  getKnownBlockWithBackoff ,
+  getSuggestedBlockIntervalFromExn ,
+  maxSuggestedBlockIntervalKey ,
+  getNextPage ,
+  getSelectionConfig ,
+  memoGetSelectionConfig ,
+  makeThrowingGetEventBlock ,
+  makeThrowingGetEventTransaction ,
+  sanitizeUrl ,
+  make ,
+}
 /* Rpc Not a pure module */

package/src/sources/Source.res

@@ -32,7 +32,7 @@ type getItemsError =
 
 exception GetItemsError(getItemsError)
 
-type sourceFor = Sync | Fallback
+type sourceFor = Sync | Fallback | Live
 type t = {
   name: string,
   sourceFor: sourceFor,

package/src/sources/{Source.res.js → Source.res.mjs}

@@ -1,9 +1,10 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-'use strict';
 
-var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
+import * as Caml_exceptions from "rescript/lib/es6/caml_exceptions.js";
 
 var GetItemsError = /* @__PURE__ */Caml_exceptions.create("Source.GetItemsError");
 
-exports.GetItemsError = GetItemsError;
+export {
+  GetItemsError ,
+}
 /* No side effect */

package/src/sources/SourceManager.res

@@ -222,11 +222,20 @@ let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
   )
   logger->Logging.childTrace("Initiating check for new blocks.")
 
+  // Only include Live sources if we've actually synced some blocks
+  // (currentBlockHeight > 0 means we've fetched at least one batch)
+  // This prevents Live RPC from winning the initial height race and
+  // becoming activeSource, which would bypass HyperSync's smart block detection
+  let isInitialHeightFetch = currentBlockHeight === 0
+
   let syncSources = []
   let fallbackSources = []
   sources->Utils.Set.forEach(source => {
     if (
       source.sourceFor === Sync ||
+        // Include Live sources only after initial sync has started
+        // Live sources are optimized for real-time indexing with lower latency
+        (source.sourceFor === Live && !isInitialHeightFetch) ||
         // Even if the active source is a fallback, still include
         // it to the list. So we don't wait for a timeout again
         // if all main sync sources are still not valid
@@ -321,7 +330,9 @@ let getNextSyncSource = (
     } else if (
       switch source.sourceFor {
       | Sync => true
-      | Fallback => attemptFallbacks || source === initialSource
+      // Live sources should NOT be used for historical sync rotation
+      // They are only meant for real-time indexing once synced
+      | Live | Fallback => attemptFallbacks || source === initialSource
       }
     ) {
       (hasActive.contents ? after : before)->Array.push(source)

package/src/sources/{SourceManager.res.js → SourceManager.res.mjs}

@@ -1,18 +1,17 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
-'use strict';
 
-var Caml = require("rescript/lib/js/caml.js");
-var Utils = require("../Utils.res.js");
-var Hrtime = require("../bindings/Hrtime.res.js");
-var Js_exn = require("rescript/lib/js/js_exn.js");
-var Source = require("./Source.res.js");
-var Logging = require("../Logging.res.js");
-var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var FetchState = require("../FetchState.res.js");
-var Prometheus = require("../Prometheus.res.js");
-var Belt_Option = require("rescript/lib/js/belt_Option.js");
-var ErrorHandling = require("../ErrorHandling.res.js");
-var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
+import * as Caml from "rescript/lib/es6/caml.js";
+import * as Utils from "../Utils.res.mjs";
+import * as Hrtime from "../bindings/Hrtime.res.mjs";
+import * as Js_exn from "rescript/lib/es6/js_exn.js";
+import * as Source from "./Source.res.mjs";
+import * as Logging from "../Logging.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as FetchState from "../FetchState.res.mjs";
+import * as Prometheus from "../Prometheus.res.mjs";
+import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as ErrorHandling from "../ErrorHandling.res.mjs";
+import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 
 function getActiveSource(sourceManager) {
   return sourceManager.activeSource;
@@ -157,10 +156,11 @@ async function waitForNewBlock(sourceManager, currentBlockHeight) {
         currentBlockHeight: currentBlockHeight
       });
   Logging.childTrace(logger, "Initiating check for new blocks.");
+  var isInitialHeightFetch = currentBlockHeight === 0;
   var syncSources = [];
   var fallbackSources = [];
   sourceManager.sources.forEach(function (source) {
-        if (source.sourceFor === "Sync" || source === sourceManager.activeSource) {
+        if (source.sourceFor === "Sync" || source.sourceFor === "Live" && !isInitialHeightFetch || source === sourceManager.activeSource) {
           syncSources.push(source);
         } else {
           fallbackSources.push(source);
@@ -217,7 +217,16 @@ function getNextSyncSource(sourceManager, initialSource, currentSource, attemptF
         }
         var match = source.sourceFor;
         var tmp;
-        tmp = match === "Sync" ? true : attemptFallbacks || source === initialSource;
+        switch (match) {
+          case "Sync" :
+              tmp = true;
+              break;
+          case "Fallback" :
+          case "Live" :
+              tmp = attemptFallbacks || source === initialSource;
+              break;
+          
+        }
         if (tmp) {
           (
               hasActive.contents ? after : before
@@ -391,10 +400,12 @@ async function executeQuery(sourceManager, query, currentBlockHeight) {
   return responseRef;
 }
 
-exports.make = make;
-exports.getActiveSource = getActiveSource;
-exports.fetchNext = fetchNext;
-exports.waitForNewBlock = waitForNewBlock;
-exports.executeQuery = executeQuery;
-exports.makeGetHeightRetryInterval = makeGetHeightRetryInterval;
+export {
+  make ,
+  getActiveSource ,
+  fetchNext ,
+  waitForNewBlock ,
+  executeQuery ,
+  makeGetHeightRetryInterval ,
+}
 /* Utils Not a pure module */