envio 2.22.0 → 2.22.2

package/src/sources/Rpc.res.js

@@ -0,0 +1,198 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+'use strict';
+
+var Viem = require("viem");
+var Utils = require("../Utils.res.js");
+var $$BigInt = require("../bindings/BigInt.res.js");
+var Address = require("../Address.res.js");
+var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var Caml_option = require("rescript/lib/js/caml_option.js");
+var S$RescriptSchema = require("rescript-schema/src/S.res.js");
+
+function makeRpcRoute(method, paramsSchema, resultSchema) {
+  var idSchema = S$RescriptSchema.literal(1);
+  var versionSchema = S$RescriptSchema.literal("2.0");
+  return function () {
+    return {
+            method: "POST",
+            path: "",
+            input: (function (s) {
+                s.field("method", S$RescriptSchema.literal(method));
+                s.field("id", idSchema);
+                s.field("jsonrpc", versionSchema);
+                return s.field("params", paramsSchema);
+              }),
+            responses: [(function (s) {
+                  s.field("jsonrpc", versionSchema);
+                  s.field("id", idSchema);
+                  return s.field("result", resultSchema);
+                })]
+          };
+  };
+}
+
+function makeHexSchema(fromStr) {
+  return S$RescriptSchema.transform(S$RescriptSchema.string, (function (s) {
+                return {
+                        p: (function (str) {
+                            var v = fromStr(str);
+                            if (v !== undefined) {
+                              return Caml_option.valFromOption(v);
+                            } else {
+                              return s.fail("The string is not valid hex", undefined);
+                            }
+                          }),
+                        s: (function (value) {
+                            return Viem.toHex(value);
+                          })
+                      };
+              }));
+}
+
+var hexBigintSchema = makeHexSchema($$BigInt.fromString);
+
+var hexIntSchema = makeHexSchema(function (v) {
+      return Number(v);
+    });
+
+var topicFilterSchema = S$RescriptSchema.union([
+      S$RescriptSchema.literal(null),
+      S$RescriptSchema.schema(function (s) {
+            return s.m(S$RescriptSchema.array(S$RescriptSchema.string));
+          }),
+      S$RescriptSchema.schema(function (s) {
+            return s.m(S$RescriptSchema.string);
+          })
+    ]);
+
+var topicQuerySchema = S$RescriptSchema.array(topicFilterSchema);
+
+function makeTopicQuery(topic0Opt, topic1Opt, topic2Opt, topic3Opt) {
+  var topic0 = topic0Opt !== undefined ? topic0Opt : [];
+  var topic1 = topic1Opt !== undefined ? topic1Opt : [];
+  var topic2 = topic2Opt !== undefined ? topic2Opt : [];
+  var topic3 = topic3Opt !== undefined ? topic3Opt : [];
+  var topics = [
+    topic0,
+    topic1,
+    topic2,
+    topic3
+  ];
+  var isLastTopicEmpty = function () {
+    var match = Utils.$$Array.last(topics);
+    if (match !== undefined) {
+      return match.length === 0;
+    } else {
+      return false;
+    }
+  };
+  while(isLastTopicEmpty()) {
+    topics.pop();
+  };
+  var toTopicFilter = function (topic) {
+    var len = topic.length;
+    if (len !== 1) {
+      if (len !== 0) {
+        return topic;
+      } else {
+        return null;
+      }
+    }
+    var single = topic[0];
+    return single;
+  };
+  return Belt_Array.map(topics, toTopicFilter);
+}
+
+function mapTopicQuery(param) {
+  return makeTopicQuery(param.topic0, param.topic1, param.topic2, param.topic3);
+}
+
+var paramsSchema = S$RescriptSchema.object(function (s) {
+      return {
+              fromBlock: s.f("fromBlock", hexIntSchema),
+              toBlock: s.f("toBlock", hexIntSchema),
+              address: s.f("address", S$RescriptSchema.array(Address.schema)),
+              topics: s.f("topics", topicQuerySchema)
+            };
+    });
+
+var logSchema = S$RescriptSchema.object(function (s) {
+      return {
+              address: s.f("address", Address.schema),
+              topics: s.f("topics", S$RescriptSchema.array(S$RescriptSchema.string)),
+              data: s.f("data", S$RescriptSchema.string),
+              blockNumber: s.f("blockNumber", hexIntSchema),
+              transactionHash: s.f("transactionHash", S$RescriptSchema.string),
+              transactionIndex: s.f("transactionIndex", hexIntSchema),
+              blockHash: s.f("blockHash", S$RescriptSchema.string),
+              logIndex: s.f("logIndex", hexIntSchema),
+              removed: s.f("removed", S$RescriptSchema.bool)
+            };
+    });
+
+var route = makeRpcRoute("eth_getLogs", S$RescriptSchema.tuple1(paramsSchema), S$RescriptSchema.array(logSchema));
+
+var GetLogs = {
+  topicFilterSchema: topicFilterSchema,
+  topicQuerySchema: topicQuerySchema,
+  makeTopicQuery: makeTopicQuery,
+  mapTopicQuery: mapTopicQuery,
+  paramsSchema: paramsSchema,
+  logSchema: logSchema,
+  route: route
+};
+
+var blockSchema = S$RescriptSchema.object(function (s) {
+      return {
+              difficulty: s.f("difficulty", S$RescriptSchema.$$null(hexBigintSchema)),
+              extraData: s.f("extraData", S$RescriptSchema.string),
+              gasLimit: s.f("gasLimit", hexBigintSchema),
+              gasUsed: s.f("gasUsed", hexBigintSchema),
+              hash: s.f("hash", S$RescriptSchema.string),
+              logsBloom: s.f("logsBloom", S$RescriptSchema.string),
+              miner: s.f("miner", Address.schema),
+              mixHash: s.f("mixHash", S$RescriptSchema.$$null(S$RescriptSchema.string)),
+              nonce: s.f("nonce", S$RescriptSchema.$$null(hexBigintSchema)),
+              number: s.f("number", hexIntSchema),
+              parentHash: s.f("parentHash", S$RescriptSchema.string),
+              receiptsRoot: s.f("receiptsRoot", S$RescriptSchema.string),
+              sha3Uncles: s.f("sha3Uncles", S$RescriptSchema.string),
+              size: s.f("size", hexBigintSchema),
+              stateRoot: s.f("stateRoot", S$RescriptSchema.string),
+              timestamp: s.f("timestamp", hexIntSchema),
+              totalDifficulty: s.f("totalDifficulty", S$RescriptSchema.$$null(hexBigintSchema)),
+              transactions: s.f("transactions", S$RescriptSchema.array(S$RescriptSchema.json(false))),
+              transactionsRoot: s.f("transactionsRoot", S$RescriptSchema.string),
+              uncles: s.f("uncles", S$RescriptSchema.$$null(S$RescriptSchema.array(S$RescriptSchema.string)))
+            };
+    });
+
+var route$1 = makeRpcRoute("eth_getBlockByNumber", S$RescriptSchema.tuple(function (s) {
+          return {
+                  blockNumber: s.item(0, hexIntSchema),
+                  includeTransactions: s.item(1, S$RescriptSchema.bool)
+                };
+        }), S$RescriptSchema.$$null(blockSchema));
+
+var GetBlockByNumber = {
+  blockSchema: blockSchema,
+  route: route$1
+};
+
+var route$2 = makeRpcRoute("eth_blockNumber", S$RescriptSchema.tuple(function (param) {
+          
+        }), hexIntSchema);
+
+var GetBlockHeight = {
+  route: route$2
+};
+
+exports.makeRpcRoute = makeRpcRoute;
+exports.makeHexSchema = makeHexSchema;
+exports.hexBigintSchema = hexBigintSchema;
+exports.hexIntSchema = hexIntSchema;
+exports.GetLogs = GetLogs;
+exports.GetBlockByNumber = GetBlockByNumber;
+exports.GetBlockHeight = GetBlockHeight;
+/* hexBigintSchema Not a pure module */

package/src/sources/Source.res.js

@@ -0,0 +1,9 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+'use strict';
+
+var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
+
+var GetItemsError = /* @__PURE__ */Caml_exceptions.create("Source.GetItemsError");
+
+exports.GetItemsError = GetItemsError;
+/* No side effect */

package/src/sources/SourceManager.res.js

@@ -0,0 +1,366 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+'use strict';
+
+var Caml = require("rescript/lib/js/caml.js");
+var Utils = require("../Utils.res.js");
+var Hrtime = require("../bindings/Hrtime.res.js");
+var Js_exn = require("rescript/lib/js/js_exn.js");
+var Source = require("./Source.res.js");
+var Logging = require("../Logging.res.js");
+var Internal = require("../Internal.res.js");
+var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var FetchState = require("../FetchState.res.js");
+var Prometheus = require("../Prometheus.res.js");
+var Belt_Option = require("rescript/lib/js/belt_Option.js");
+var ErrorHandling = require("../ErrorHandling.res.js");
+var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
+
+function getActiveSource(sourceManager) {
+  return sourceManager.activeSource;
+}
+
+function makeGetHeightRetryInterval(initialRetryInterval, backoffMultiplicative, maxRetryInterval) {
+  return function (retry) {
+    var backoff = retry === 0 ? 1 : Math.imul(retry, backoffMultiplicative);
+    return Caml.int_min(Math.imul(initialRetryInterval, backoff), maxRetryInterval);
+  };
+}
+
+function make(sources, maxPartitionConcurrency, newBlockFallbackStallTimeoutOpt, stalledPollingIntervalOpt, getHeightRetryIntervalOpt) {
+  var newBlockFallbackStallTimeout = newBlockFallbackStallTimeoutOpt !== undefined ? newBlockFallbackStallTimeoutOpt : 20000;
+  var stalledPollingInterval = stalledPollingIntervalOpt !== undefined ? stalledPollingIntervalOpt : 5000;
+  var getHeightRetryInterval = getHeightRetryIntervalOpt !== undefined ? getHeightRetryIntervalOpt : makeGetHeightRetryInterval(1000, 2, 60000);
+  var source = sources.find(function (source) {
+        return source.sourceFor === "Sync";
+      });
+  var initialActiveSource = source !== undefined ? source : Js_exn.raiseError("Invalid configuration, no data-source for historical sync provided");
+  Prometheus.IndexingMaxConcurrency.set(maxPartitionConcurrency, initialActiveSource.chain);
+  Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
+  return {
+          sources: new Set(sources),
+          statusStart: Hrtime.makeTimer(),
+          status: "Idle",
+          maxPartitionConcurrency: maxPartitionConcurrency,
+          newBlockFallbackStallTimeout: newBlockFallbackStallTimeout,
+          stalledPollingInterval: stalledPollingInterval,
+          getHeightRetryInterval: getHeightRetryInterval,
+          activeSource: initialActiveSource,
+          waitingForNewBlockStateId: undefined,
+          fetchingPartitionsCount: 0
+        };
+}
+
+function trackNewStatus(sourceManager, newStatus) {
+  var promCounter;
+  switch (newStatus) {
+    case "Idle" :
+        promCounter = Prometheus.IndexingIdleTime.counter;
+        break;
+    case "WaitingForNewBlock" :
+        promCounter = Prometheus.IndexingSourceWaitingTime.counter;
+        break;
+    case "Querieng" :
+        promCounter = Prometheus.IndexingQueryTime.counter;
+        break;
+    
+  }
+  Prometheus.SafeCounter.incrementMany(promCounter, sourceManager.activeSource.chain, Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(sourceManager.statusStart))));
+  sourceManager.statusStart = Hrtime.makeTimer();
+  sourceManager.status = newStatus;
+}
+
+async function fetchNext(sourceManager, fetchState, currentBlockHeight, executeQuery, waitForNewBlock, onNewBlock, targetBufferSize, stateId) {
+  var queries = FetchState.getNextQuery(fetchState, sourceManager.maxPartitionConcurrency - sourceManager.fetchingPartitionsCount | 0, targetBufferSize, currentBlockHeight, stateId);
+  if (typeof queries !== "object") {
+    switch (queries) {
+      case "WaitingForNewBlock" :
+          var waitingStateId = sourceManager.waitingForNewBlockStateId;
+          if (waitingStateId !== undefined && waitingStateId >= stateId) {
+            return ;
+          }
+          trackNewStatus(sourceManager, "WaitingForNewBlock");
+          sourceManager.waitingForNewBlockStateId = stateId;
+          var currentBlockHeight$1 = await waitForNewBlock(currentBlockHeight);
+          var waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
+          if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
+            trackNewStatus(sourceManager, "Idle");
+            sourceManager.waitingForNewBlockStateId = undefined;
+            return onNewBlock(currentBlockHeight$1);
+          } else {
+            return ;
+          }
+      case "ReachedMaxConcurrency" :
+      case "NothingToQuery" :
+          return ;
+      
+    }
+  } else {
+    var queries$1 = queries._0;
+    FetchState.startFetchingQueries(fetchState, queries$1, stateId);
+    sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries$1.length | 0;
+    Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
+    trackNewStatus(sourceManager, "Querieng");
+    await Promise.all(Belt_Array.map(queries$1, (function (q) {
+                var promise = executeQuery(q);
+                promise.then(function (param) {
+                      sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount - 1 | 0;
+                      Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
+                      if (sourceManager.fetchingPartitionsCount === 0) {
+                        return trackNewStatus(sourceManager, "Idle");
+                      }
+                      
+                    });
+                return promise;
+              })));
+    return ;
+  }
+}
+
+async function getSourceNewHeight(sourceManager, source, currentBlockHeight, status, logger) {
+  var newHeight = 0;
+  var retry = 0;
+  while(newHeight <= currentBlockHeight && status.contents !== "Done") {
+    try {
+      var endTimer = Prometheus.SourceGetHeightDuration.startTimer({
+            source: source.name,
+            chainId: source.chain
+          });
+      var height = await source.getHeightOrThrow();
+      endTimer();
+      newHeight = height;
+      if (height <= currentBlockHeight) {
+        retry = 0;
+        var pollingInterval = status.contents === "Stalled" ? sourceManager.stalledPollingInterval : source.pollingInterval;
+        await Utils.delay(pollingInterval);
+      }
+      
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      var retryInterval = sourceManager.getHeightRetryInterval(retry);
+      Logging.childTrace(logger, {
+            msg: "Height retrieval from " + source.name + " source failed. Retrying in " + String(retryInterval) + "ms.",
+            source: source.name,
+            err: Internal.prettifyExn(exn)
+          });
+      retry = retry + 1 | 0;
+      await Utils.delay(retryInterval);
+    }
+  };
+  Prometheus.SourceHeight.set(source.name, source.chain, newHeight);
+  return newHeight;
+}
+
+async function waitForNewBlock(sourceManager, currentBlockHeight) {
+  var logger = Logging.createChild({
+        chainId: sourceManager.activeSource.chain,
+        currentBlockHeight: currentBlockHeight
+      });
+  Logging.childTrace(logger, "Initiating check for new blocks.");
+  var syncSources = [];
+  var fallbackSources = [];
+  sourceManager.sources.forEach(function (source) {
+        if (source.sourceFor === "Sync" || source === sourceManager.activeSource) {
+          syncSources.push(source);
+        } else {
+          fallbackSources.push(source);
+        }
+      });
+  var status = {
+    contents: "Active"
+  };
+  var match = await Promise.race(Belt_Array.concat(Belt_Array.map(syncSources, (async function (source) {
+                  return [
+                          source,
+                          await getSourceNewHeight(sourceManager, source, currentBlockHeight, status, logger)
+                        ];
+                })), [Utils.delay(sourceManager.newBlockFallbackStallTimeout).then(function () {
+                  if (status.contents !== "Done") {
+                    status.contents = "Stalled";
+                    if (fallbackSources.length !== 0) {
+                      Logging.childWarn(logger, "No new blocks detected within " + String(sourceManager.newBlockFallbackStallTimeout / 1000 | 0) + "s. Continuing polling with fallback RPC sources from the configuration.");
+                    } else {
+                      Logging.childWarn(logger, "No new blocks detected within " + String(sourceManager.newBlockFallbackStallTimeout / 1000 | 0) + "s. Polling will continue at a reduced rate. For better reliability, refer to our RPC fallback guide: https://docs.envio.dev/docs/HyperIndex/rpc-sync");
+                    }
+                  }
+                  return Promise.race(Belt_Array.map(fallbackSources, (async function (source) {
+                                    return [
+                                            source,
+                                            await getSourceNewHeight(sourceManager, source, currentBlockHeight, status, logger)
+                                          ];
+                                  })));
+                })]));
+  var newBlockHeight = match[1];
+  var source = match[0];
+  sourceManager.activeSource = source;
+  var log = status.contents === "Stalled" ? Logging.childInfo : Logging.childTrace;
+  log(logger, {
+        msg: "New blocks successfully found.",
+        source: source.name,
+        newBlockHeight: newBlockHeight
+      });
+  status.contents = "Done";
+  return newBlockHeight;
+}
+
+function getNextSyncSource(sourceManager, initialSource, attemptFallbacksOpt) {
+  var attemptFallbacks = attemptFallbacksOpt !== undefined ? attemptFallbacksOpt : false;
+  var before = [];
+  var after = [];
+  var hasActive = {
+    contents: false
+  };
+  sourceManager.sources.forEach(function (source) {
+        if (source === sourceManager.activeSource) {
+          hasActive.contents = true;
+          return ;
+        }
+        var match = source.sourceFor;
+        var tmp;
+        tmp = match === "Sync" ? true : attemptFallbacks || source === initialSource;
+        if (tmp) {
+          (
+              hasActive.contents ? after : before
+            ).push(source);
+          return ;
+        }
+        
+      });
+  var s = Belt_Array.get(after, 0);
+  if (s !== undefined) {
+    return s;
+  }
+  var s$1 = Belt_Array.get(before, 0);
+  if (s$1 !== undefined) {
+    return s$1;
+  } else {
+    return sourceManager.activeSource;
+  }
+}
+
+async function executeQuery(sourceManager, query, currentBlockHeight) {
+  var match = query.target;
+  var toBlockRef;
+  toBlockRef = typeof match !== "object" ? undefined : (
+      match.TAG === "EndBlock" ? match.toBlock : match.toBlock
+    );
+  var responseRef;
+  var retryRef = 0;
+  var initialSource = sourceManager.activeSource;
+  while(Belt_Option.isNone(responseRef)) {
+    var source = sourceManager.activeSource;
+    var toBlock = toBlockRef;
+    var retry = retryRef;
+    var logger = Logging.createChild({
+          chainId: source.chain,
+          logType: "Block Range Query",
+          partitionId: query.partitionId,
+          source: source.name,
+          fromBlock: query.fromBlock,
+          toBlock: toBlock,
+          addresses: FetchState.addressesByContractNameCount(query.addressesByContractName),
+          retry: retry
+        });
+    try {
+      var response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, query.indexingContracts, currentBlockHeight, query.partitionId, query.selection, retry, logger);
+      Logging.childTrace(logger, {
+            msg: "Fetched block range from server",
+            toBlock: response.latestFetchedBlockNumber,
+            numEvents: response.parsedQueueItems.length,
+            stats: response.stats
+          });
+      responseRef = response;
+    }
+    catch (raw_error){
+      var error = Caml_js_exceptions.internalToOCamlException(raw_error);
+      if (error.RE_EXN_ID === Source.GetItemsError) {
+        var error$1 = error._1;
+        if (error$1.TAG === "FailedGettingItems") {
+          var match$1 = error$1.retry;
+          var attemptedToBlock = error$1.attemptedToBlock;
+          if (match$1.TAG === "WithSuggestedToBlock") {
+            var toBlock$1 = match$1.toBlock;
+            Logging.childTrace(logger, {
+                  msg: "Failed getting data for the block range. Immediately retrying with the suggested block range from response.",
+                  toBlock: attemptedToBlock,
+                  suggestedToBlock: toBlock$1
+                });
+            toBlockRef = toBlock$1;
+            retryRef = 0;
+          } else {
+            var backoffMillis = match$1.backoffMillis;
+            var attemptFallbacks = retry >= 10;
+            var nextSource = !(retry === 0 || retry === 1) && retry % 2 === 0 ? getNextSyncSource(sourceManager, initialSource, attemptFallbacks) : source;
+            var log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
+            log(logger, {
+                  msg: match$1.message,
+                  toBlock: attemptedToBlock,
+                  backOffMilliseconds: backoffMillis,
+                  retry: retry,
+                  err: Internal.prettifyExn(error$1.exn)
+                });
+            var shouldSwitch = nextSource !== source;
+            if (shouldSwitch) {
+              Logging.childInfo(logger, {
+                    msg: "Switching to another data-source",
+                    source: nextSource.name
+                  });
+              sourceManager.activeSource = nextSource;
+            } else {
+              await Utils.delay(backoffMillis < 60000 ? backoffMillis : 60000);
+            }
+            retryRef = retryRef + 1 | 0;
+          }
+        } else {
+          var nextSource$1 = getNextSyncSource(sourceManager, initialSource, undefined);
+          var notAlreadyDeleted = sourceManager.sources.delete(source);
+          if (notAlreadyDeleted) {
+            var exit = 0;
+            switch (error$1.TAG) {
+              case "UnsupportedSelection" :
+                  Logging.childError(logger, error$1.message);
+                  break;
+              case "FailedGettingFieldSelection" :
+              case "FailedParsingItems" :
+                  exit = 1;
+                  break;
+              case "FailedGettingItems" :
+                  break;
+              
+            }
+            if (exit === 1) {
+              Logging.childError(logger, {
+                    msg: error$1.message,
+                    err: Internal.prettifyExn(error$1.exn),
+                    blockNumber: error$1.blockNumber,
+                    logIndex: error$1.logIndex
+                  });
+            }
+            
+          }
+          if (nextSource$1 === source) {
+            ErrorHandling.mkLogAndRaise(logger, "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.", null);
+          } else {
+            Logging.childInfo(logger, {
+                  msg: "Switching to another data-source",
+                  source: nextSource$1.name
+                });
+            sourceManager.activeSource = nextSource$1;
+            retryRef = 0;
+          }
+        }
+      } else {
+        ErrorHandling.mkLogAndRaise(logger, "Failed to fetch block Range", error);
+      }
+    }
+  };
+  return responseRef;
+}
+
+exports.make = make;
+exports.getActiveSource = getActiveSource;
+exports.fetchNext = fetchNext;
+exports.waitForNewBlock = waitForNewBlock;
+exports.executeQuery = executeQuery;
+exports.makeGetHeightRetryInterval = makeGetHeightRetryInterval;
+/* Utils Not a pure module */