envio 3.0.0-alpha.3 → 3.0.0-alpha.4

package/src/sources/SourceManager.res.mjs

@@ -10,6 +10,7 @@ import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
 import * as FetchState from "../FetchState.res.mjs";
 import * as Prometheus from "../Prometheus.res.mjs";
 import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
 import * as ErrorHandling from "../ErrorHandling.res.mjs";
 import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 
@@ -35,7 +36,15 @@ function make(sources, maxPartitionConcurrency, newBlockFallbackStallTimeoutOpt,
   Prometheus.IndexingMaxConcurrency.set(maxPartitionConcurrency, initialActiveSource.chain);
   Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
   return {
-          sources: new Set(sources),
+          sourcesState: Belt_Array.map(sources, (function (source) {
+                  return {
+                          source: source,
+                          knownHeight: 0,
+                          unsubscribe: undefined,
+                          pendingHeightResolvers: [],
+                          disabled: false
+                        };
+                })),
           statusStart: Hrtime.makeTimer(),
           status: "Idle",
           maxPartitionConcurrency: maxPartitionConcurrency,
@@ -115,38 +124,78 @@ async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBloc
   }
 }
 
-async function getSourceNewHeight(sourceManager, source, knownHeight, status, logger) {
-  var newHeight = 0;
+function disableSource(sourceState) {
+  if (sourceState.disabled) {
+    return false;
+  }
+  sourceState.disabled = true;
+  var unsubscribe = sourceState.unsubscribe;
+  if (unsubscribe !== undefined) {
+    unsubscribe();
+  }
+  return true;
+}
+
+async function getSourceNewHeight(sourceManager, sourceState, knownHeight, status, logger) {
+  var source = sourceState.source;
+  var initialHeight = sourceState.knownHeight;
+  var newHeight = initialHeight;
   var retry = 0;
   while(newHeight <= knownHeight && status.contents !== "Done") {
-    try {
-      var endTimer = Prometheus.SourceGetHeightDuration.startTimer({
-            source: source.name,
-            chainId: source.chain
-          });
-      var height = await source.getHeightOrThrow();
-      endTimer();
-      newHeight = height;
-      if (height <= knownHeight) {
-        retry = 0;
-        var pollingInterval = status.contents === "Stalled" ? sourceManager.stalledPollingInterval : source.pollingInterval;
-        await Utils.delay(pollingInterval);
+    var match = sourceState.unsubscribe;
+    if (match !== undefined) {
+      var height = await new Promise((function (resolve, _reject) {
+              sourceState.pendingHeightResolvers.push(resolve);
+            }));
+      if (height > initialHeight) {
+        newHeight = height;
       }
       
-    }
-    catch (raw_exn){
-      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
-      var retryInterval = sourceManager.getHeightRetryInterval(retry);
-      Logging.childTrace(logger, {
-            msg: "Height retrieval from " + source.name + " source failed. Retrying in " + String(retryInterval) + "ms.",
-            source: source.name,
-            err: Utils.prettifyExn(exn)
-          });
-      retry = retry + 1 | 0;
-      await Utils.delay(retryInterval);
+    } else {
+      try {
+        var endTimer = Prometheus.SourceGetHeightDuration.startTimer({
+              source: source.name,
+              chainId: source.chain
+            });
+        var height$1 = await source.getHeightOrThrow();
+        endTimer();
+        newHeight = height$1;
+        if (height$1 <= knownHeight) {
+          retry = 0;
+          var createSubscription = source.createHeightSubscription;
+          if (createSubscription !== undefined) {
+            var unsubscribe = createSubscription(function (newHeight) {
+                  sourceState.knownHeight = newHeight;
+                  var resolvers = sourceState.pendingHeightResolvers;
+                  sourceState.pendingHeightResolvers = [];
+                  Belt_Array.forEach(resolvers, (function (resolve) {
+                          resolve(newHeight);
+                        }));
+                });
+            sourceState.unsubscribe = unsubscribe;
+          } else {
+            var pollingInterval = status.contents === "Stalled" ? sourceManager.stalledPollingInterval : source.pollingInterval;
+            await Utils.delay(pollingInterval);
+          }
+        }
+        
+      }
+      catch (raw_exn){
+        var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+        var retryInterval = sourceManager.getHeightRetryInterval(retry);
+        Logging.childTrace(logger, {
+              msg: "Height retrieval from " + source.name + " source failed. Retrying in " + String(retryInterval) + "ms.",
+              source: source.name,
+              err: Utils.prettifyExn(exn)
+            });
+        retry = retry + 1 | 0;
+        await Utils.delay(retryInterval);
+      }
     }
   };
-  Prometheus.SourceHeight.set(source.name, source.chain, newHeight);
+  if (newHeight > initialHeight) {
+    Prometheus.SourceHeight.set(source.name, source.chain, newHeight);
+  }
   return newHeight;
 }
 
@@ -159,20 +208,26 @@ async function waitForNewBlock(sourceManager, knownHeight) {
   var isInitialHeightFetch = knownHeight === 0;
   var syncSources = [];
   var fallbackSources = [];
-  sourceManager.sources.forEach(function (source) {
-        if (source.sourceFor === "Sync" || source.sourceFor === "Live" && !isInitialHeightFetch || source === sourceManager.activeSource) {
-          syncSources.push(source);
-        } else {
-          fallbackSources.push(source);
-        }
-      });
+  Belt_Array.forEach(sourceManager.sourcesState, (function (sourceState) {
+          var source = sourceState.source;
+          if (sourceState.disabled) {
+            return ;
+          } else {
+            if (source.sourceFor === "Sync" || source.sourceFor === "Live" && !isInitialHeightFetch || source === sourceManager.activeSource) {
+              syncSources.push(sourceState);
+            } else {
+              fallbackSources.push(sourceState);
+            }
+            return ;
+          }
+        }));
   var status = {
     contents: "Active"
   };
-  var match = await Promise.race(Belt_Array.concat(Belt_Array.map(syncSources, (async function (source) {
+  var match = await Promise.race(Belt_Array.concat(Belt_Array.map(syncSources, (async function (sourceState) {
                   return [
-                          source,
-                          await getSourceNewHeight(sourceManager, source, knownHeight, status, logger)
+                          sourceState.source,
+                          await getSourceNewHeight(sourceManager, sourceState, knownHeight, status, logger)
                         ];
                 })), [Utils.delay(sourceManager.newBlockFallbackStallTimeout).then(function () {
                   if (status.contents !== "Done") {
@@ -183,10 +238,10 @@ async function waitForNewBlock(sourceManager, knownHeight) {
                       Logging.childWarn(logger, "No new blocks detected within " + String(sourceManager.newBlockFallbackStallTimeout / 1000 | 0) + "s. Polling will continue at a reduced rate. For better reliability, refer to our RPC fallback guide: https://docs.envio.dev/docs/HyperIndex/rpc-sync");
                     }
                   }
-                  return Promise.race(Belt_Array.map(fallbackSources, (async function (source) {
+                  return Promise.race(Belt_Array.map(fallbackSources, (async function (sourceState) {
                                     return [
-                                            source,
-                                            await getSourceNewHeight(sourceManager, source, knownHeight, status, logger)
+                                            sourceState.source,
+                                            await getSourceNewHeight(sourceManager, sourceState, knownHeight, status, logger)
                                           ];
                                   })));
                 })]));
@@ -203,38 +258,42 @@ async function waitForNewBlock(sourceManager, knownHeight) {
   return newBlockHeight;
 }
 
-function getNextSyncSource(sourceManager, initialSource, currentSource, attemptFallbacksOpt) {
+function getNextSyncSourceState(sourceManager, initialSourceState, currentSourceState, attemptFallbacksOpt) {
   var attemptFallbacks = attemptFallbacksOpt !== undefined ? attemptFallbacksOpt : false;
   var before = [];
   var after = [];
   var hasActive = {
     contents: false
   };
-  sourceManager.sources.forEach(function (source) {
-        if (source === currentSource) {
-          hasActive.contents = true;
-          return ;
-        }
-        var match = source.sourceFor;
-        var tmp;
-        switch (match) {
-          case "Sync" :
-              tmp = true;
-              break;
-          case "Fallback" :
-          case "Live" :
-              tmp = attemptFallbacks || source === initialSource;
-              break;
+  Belt_Array.forEach(sourceManager.sourcesState, (function (sourceState) {
+          var source = sourceState.source;
+          if (sourceState.disabled) {
+            return ;
+          }
+          if (sourceState === currentSourceState) {
+            hasActive.contents = true;
+            return ;
+          }
+          var match = source.sourceFor;
+          var tmp;
+          switch (match) {
+            case "Sync" :
+                tmp = true;
+                break;
+            case "Fallback" :
+            case "Live" :
+                tmp = attemptFallbacks || sourceState === initialSourceState;
+                break;
+            
+          }
+          if (tmp) {
+            (
+                hasActive.contents ? after : before
+              ).push(sourceState);
+            return ;
+          }
           
-        }
-        if (tmp) {
-          (
-              hasActive.contents ? after : before
-            ).push(source);
-          return ;
-        }
-        
-      });
+        }));
   var s = Belt_Array.get(after, 0);
   if (s !== undefined) {
     return s;
@@ -243,7 +302,7 @@ function getNextSyncSource(sourceManager, initialSource, currentSource, attemptF
   if (s$1 !== undefined) {
     return s$1;
   } else {
-    return currentSource;
+    return currentSourceState;
   }
 }
 
@@ -255,11 +314,15 @@ async function executeQuery(sourceManager, query, knownHeight) {
     );
   var responseRef;
   var retryRef = 0;
-  var initialSource = sourceManager.activeSource;
-  var sourceRef = initialSource;
+  var initialSourceState = sourceManager.sourcesState.find(function (s) {
+        return s.source === sourceManager.activeSource;
+      });
+  var initialSourceState$1 = initialSourceState === undefined ? undefined : Caml_option.some(initialSourceState);
+  var sourceStateRef = initialSourceState$1;
   var shouldUpdateActiveSource = false;
   while(Belt_Option.isNone(responseRef)) {
-    var source = sourceRef;
+    var sourceState = sourceStateRef;
+    var source = sourceState.source;
     var toBlock = toBlockRef;
     var retry = retryRef;
     var logger = Logging.createChild({
@@ -310,7 +373,7 @@ async function executeQuery(sourceManager, query, knownHeight) {
                 case "WithBackoff" :
                     var backoffMillis = match$1.backoffMillis;
                     var attemptFallbacks = retry >= 10;
-                    var nextSource = !(retry === 0 || retry === 1) && retry % 2 === 0 ? getNextSyncSource(sourceManager, initialSource, source, attemptFallbacks) : source;
+                    var nextSourceState = !(retry === 0 || retry === 1) && retry % 2 === 0 ? getNextSyncSourceState(sourceManager, initialSourceState$1, sourceState, attemptFallbacks) : sourceState;
                     var log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
                     log(logger, {
                           msg: match$1.message,
@@ -319,13 +382,13 @@ async function executeQuery(sourceManager, query, knownHeight) {
                           retry: retry,
                           err: Utils.prettifyExn(exn)
                         });
-                    var shouldSwitch = nextSource !== source;
+                    var shouldSwitch = nextSourceState !== sourceState;
                     if (shouldSwitch) {
                       Logging.childInfo(logger, {
                             msg: "Switching to another data-source",
-                            source: nextSource.name
+                            source: nextSourceState.source.name
                           });
-                      sourceRef = nextSource;
+                      sourceStateRef = nextSourceState;
                       shouldUpdateActiveSource = true;
                     } else {
                       await Utils.delay(backoffMillis < 60000 ? backoffMillis : 60000);
@@ -333,8 +396,8 @@ async function executeQuery(sourceManager, query, knownHeight) {
                     retryRef = retryRef + 1 | 0;
                     break;
                 case "ImpossibleForTheQuery" :
-                    var nextSource$1 = getNextSyncSource(sourceManager, initialSource, source, true);
-                    var hasAnotherSource = nextSource$1 !== initialSource;
+                    var nextSourceState$1 = getNextSyncSourceState(sourceManager, initialSourceState$1, sourceState, true);
+                    var hasAnotherSource = nextSourceState$1 !== initialSourceState$1;
                     Logging.childWarn(logger, {
                           msg: match$1.message + (
                             hasAnotherSource ? " - Attempting to another source" : ""
@@ -343,7 +406,7 @@ async function executeQuery(sourceManager, query, knownHeight) {
                           err: Utils.prettifyExn(exn)
                         });
                     if (hasAnotherSource) {
-                      sourceRef = nextSource$1;
+                      sourceStateRef = nextSourceState$1;
                       shouldUpdateActiveSource = false;
                       retryRef = 0;
                     } else {
@@ -356,9 +419,9 @@ async function executeQuery(sourceManager, query, knownHeight) {
           
         }
         if (exit === 1) {
-          var nextSource$2 = getNextSyncSource(sourceManager, initialSource, source, undefined);
-          var notAlreadyDeleted = sourceManager.sources.delete(source);
-          if (notAlreadyDeleted) {
+          var nextSourceState$2 = getNextSyncSourceState(sourceManager, initialSourceState$1, sourceState, undefined);
+          var notAlreadyDisabled = disableSource(sourceState);
+          if (notAlreadyDisabled) {
             switch (error$1.TAG) {
               case "UnsupportedSelection" :
                   Logging.childError(logger, error$1.message);
@@ -376,14 +439,14 @@ async function executeQuery(sourceManager, query, knownHeight) {
               
             }
           }
-          if (nextSource$2 === source) {
+          if (nextSourceState$2 === sourceState) {
             ErrorHandling.mkLogAndRaise(logger, "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.", null);
           } else {
             Logging.childInfo(logger, {
                   msg: "Switching to another data-source",
-                  source: nextSource$2.name
+                  source: nextSourceState$2.source.name
                 });
-            sourceRef = nextSource$2;
+            sourceStateRef = nextSourceState$2;
             shouldUpdateActiveSource = true;
             retryRef = 0;
           }
@@ -395,7 +458,7 @@ async function executeQuery(sourceManager, query, knownHeight) {
     }
   };
   if (shouldUpdateActiveSource) {
-    sourceManager.activeSource = sourceRef;
+    sourceManager.activeSource = sourceStateRef.source;
   }
   return responseRef;
 }

package/src/sources/{Solana.res → Svm.res}

@@ -9,7 +9,7 @@ let cleanUpRawEventFieldsInPlace: Js.Json.t => unit = %raw(`fields => {
   }`)
 
 let ecosystem: Ecosystem.t = {
-  name: Solana,
+  name: Svm,
   blockFields: ["slot"],
   transactionFields: [],
   blockNumberName: "height",
@@ -36,13 +36,13 @@ let makeRPCSource = (~chain, ~rpc: string): Source.t => {
   let client = Rest.client(rpc)
 
   {
-    name: "Solana",
+    name: "Svm",
     sourceFor: Sync,
     chain,
     poweredByHyperSync: false,
     pollingInterval: 10_000,
     getBlockHashes: (~blockNumbers as _, ~logger as _) =>
-      Js.Exn.raiseError("Solana does not support getting block hashes"),
+      Js.Exn.raiseError("Svm does not support getting block hashes"),
     getHeightOrThrow: () => GetFinalizedSlot.route->Rest.fetch((), ~client),
     getItemsOrThrow: (
       ~fromBlock as _,
@@ -54,6 +54,6 @@ let makeRPCSource = (~chain, ~rpc: string): Source.t => {
       ~selection as _,
       ~retry as _,
       ~logger as _,
-    ) => Js.Exn.raiseError("Solana does not support getting items"),
+    ) => Js.Exn.raiseError("Svm does not support getting items"),
   }
 }

package/src/sources/{Solana.res.mjs → Svm.res.mjs}

@@ -28,7 +28,7 @@ function ecosystem_getId(prim) {
 }
 
 var ecosystem = {
-  name: "solana",
+  name: "svm",
   blockFields: ecosystem_blockFields,
   transactionFields: ecosystem_transactionFields,
   blockNumberName: "height",
@@ -53,19 +53,19 @@ var GetFinalizedSlot = {
 function makeRPCSource(chain, rpc) {
   var client = Rest.client(rpc, undefined);
   return {
-          name: "Solana",
+          name: "Svm",
           sourceFor: "Sync",
           chain: chain,
           poweredByHyperSync: false,
           pollingInterval: 10000,
           getBlockHashes: (function (param, param$1) {
-              return Js_exn.raiseError("Solana does not support getting block hashes");
+              return Js_exn.raiseError("Svm does not support getting block hashes");
             }),
           getHeightOrThrow: (function () {
               return Rest.$$fetch(route, undefined, client);
             }),
           getItemsOrThrow: (function (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8) {
-              return Js_exn.raiseError("Solana does not support getting items");
+              return Js_exn.raiseError("Svm does not support getting items");
             })
         };
 }

package/src/bindings/Ethers.gen.ts

@@ -1,14 +0,0 @@
-/* TypeScript file generated from Ethers.res by genType. */
-
-/* eslint-disable */
-/* tslint:disable */
-
-import * as EthersJS from './Ethers.res.mjs';
-
-import type {t as Address_t} from '../../src/Address.gen.js';
-
-export const Addresses_mockAddresses: Address_t[] = EthersJS.Addresses.mockAddresses as any;
-
-export const Addresses_defaultAddress: Address_t = EthersJS.Addresses.defaultAddress as any;
-
-export const Addresses: { mockAddresses: Address_t[]; defaultAddress: Address_t } = EthersJS.Addresses as any;