envio 3.0.0-alpha.2 → 3.0.0-alpha.3

package/src/LoadLayer.res.mjs

@@ -0,0 +1,296 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as Hrtime from "./bindings/Hrtime.res.mjs";
+import * as Logging from "./Logging.res.mjs";
+import * as Internal from "./Internal.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
+import * as LoadManager from "./LoadManager.res.mjs";
+import * as Persistence from "./Persistence.res.mjs";
+import * as TableIndices from "./TableIndices.res.mjs";
+import * as ErrorHandling from "./ErrorHandling.res.mjs";
+import * as InMemoryStore from "./InMemoryStore.res.mjs";
+import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
+
+function loadById(loadManager, persistence, entityConfig, inMemoryStore, shouldGroup, item, entityId) {
+  var key = entityConfig.name + ".get";
+  var inMemTable = InMemoryStore.getInMemTable(inMemoryStore, entityConfig);
+  var load = async function (idsToLoad, param) {
+    var timerRef = Prometheus.StorageLoad.startOperation(key);
+    var dbEntities;
+    try {
+      dbEntities = await Persistence.getInitializedStorageOrThrow(persistence).loadByIdsOrThrow(idsToLoad, entityConfig.table, entityConfig.rowsSchema);
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      if (exn.RE_EXN_ID === Persistence.StorageError) {
+        dbEntities = ErrorHandling.mkLogAndRaise(Logging.getItemLogger(item), exn.message, exn.reason);
+      } else {
+        throw exn;
+      }
+    }
+    var entitiesMap = {};
+    for(var idx = 0 ,idx_finish = dbEntities.length; idx < idx_finish; ++idx){
+      var entity = dbEntities[idx];
+      entitiesMap[entity.id] = entity;
+    }
+    idsToLoad.forEach(function (entityId) {
+          InMemoryTable.Entity.initValue(inMemTable, entityId, entitiesMap[entityId], false);
+        });
+    return Prometheus.StorageLoad.endOperation(timerRef, key, idsToLoad.length, dbEntities.length);
+  };
+  return LoadManager.call(loadManager, entityId, key, load, LoadManager.noopHasher, shouldGroup, (function (hash) {
+                return InMemoryTable.hasByHash(inMemTable.table, hash);
+              }), InMemoryTable.Entity.getUnsafe(inMemTable));
+}
+
+function callEffect(effect, arg, inMemTable, timerRef, onError) {
+  var effectName = effect.name;
+  var hadActiveCalls = effect.activeCallsCount > 0;
+  effect.activeCallsCount = effect.activeCallsCount + 1 | 0;
+  Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, effectName, effect.activeCallsCount);
+  if (hadActiveCalls) {
+    var elapsed = Hrtime.millisBetween(effect.prevCallStartTimerRef, timerRef);
+    if (elapsed > 0) {
+      Prometheus.SafeCounter.incrementMany(Prometheus.EffectCalls.timeCounter, effectName, Hrtime.millisBetween(effect.prevCallStartTimerRef, timerRef));
+    }
+    
+  }
+  effect.prevCallStartTimerRef = timerRef;
+  return effect.handler(arg).then(function (output) {
+                  inMemTable.dict[arg.cacheKey] = output;
+                  if (arg.context.cache) {
+                    inMemTable.idsToStore.push(arg.cacheKey);
+                    return ;
+                  }
+                  
+                }).catch(function (exn) {
+                onError(arg.cacheKey, exn);
+              }).finally(function () {
+              effect.activeCallsCount = effect.activeCallsCount - 1 | 0;
+              Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, effectName, effect.activeCallsCount);
+              var newTimer = Hrtime.makeTimer();
+              Prometheus.SafeCounter.incrementMany(Prometheus.EffectCalls.timeCounter, effectName, Hrtime.millisBetween(effect.prevCallStartTimerRef, newTimer));
+              effect.prevCallStartTimerRef = newTimer;
+              Prometheus.SafeCounter.increment(Prometheus.EffectCalls.totalCallsCount, effectName);
+              Prometheus.SafeCounter.incrementMany(Prometheus.EffectCalls.sumTimeCounter, effectName, Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(timerRef))));
+            });
+}
+
+function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQueue) {
+  var effectName = effect.name;
+  var timerRef = Hrtime.makeTimer();
+  var promises = [];
+  var state = effect.rateLimit;
+  if (state !== undefined) {
+    var now = Date.now();
+    if (now >= state.windowStartTime + state.durationMs) {
+      state.availableCalls = state.callsPerDuration;
+      state.windowStartTime = now;
+      state.nextWindowPromise = undefined;
+    }
+    var immediateCount = Math.min(state.availableCalls, effectArgs.length);
+    var immediateArgs = Belt_Array.slice(effectArgs, 0, immediateCount);
+    var queuedArgs = Belt_Array.sliceToEnd(effectArgs, immediateCount);
+    state.availableCalls = state.availableCalls - immediateCount | 0;
+    for(var idx = 0 ,idx_finish = immediateArgs.length; idx < idx_finish; ++idx){
+      promises.push(callEffect(effect, immediateArgs[idx], inMemTable, timerRef, onError));
+    }
+    if (immediateCount > 0 && isFromQueue) {
+      state.queueCount = state.queueCount - immediateCount | 0;
+      Prometheus.EffectQueueCount.set(state.queueCount, effectName);
+    }
+    if (Utils.$$Array.notEmpty(queuedArgs)) {
+      if (!isFromQueue) {
+        state.queueCount = state.queueCount + queuedArgs.length | 0;
+        Prometheus.EffectQueueCount.set(state.queueCount, effectName);
+      }
+      var millisUntilReset = {
+        contents: 0
+      };
+      var p = state.nextWindowPromise;
+      var nextWindowPromise;
+      if (p !== undefined) {
+        nextWindowPromise = Caml_option.valFromOption(p);
+      } else {
+        millisUntilReset.contents = state.windowStartTime + state.durationMs - now | 0;
+        var p$1 = Utils.delay(millisUntilReset.contents);
+        state.nextWindowPromise = Caml_option.some(p$1);
+        nextWindowPromise = p$1;
+      }
+      promises.push(nextWindowPromise.then(function () {
+                if (millisUntilReset.contents > 0) {
+                  Prometheus.SafeCounter.incrementMany(Prometheus.EffectQueueCount.timeCounter, effectName, millisUntilReset.contents);
+                }
+                return executeWithRateLimit(effect, queuedArgs, inMemTable, onError, true);
+              }));
+    }
+    
+  } else {
+    for(var idx$1 = 0 ,idx_finish$1 = effectArgs.length; idx$1 < idx_finish$1; ++idx$1){
+      promises.push(callEffect(effect, effectArgs[idx$1], inMemTable, timerRef, onError));
+    }
+  }
+  return Promise.all(promises);
+}
+
+function loadEffect(loadManager, persistence, effect, effectArgs, inMemoryStore, shouldGroup, item) {
+  var effectName = effect.name;
+  var key = effectName + ".effect";
+  var inMemTable = InMemoryStore.getEffectInMemTable(inMemoryStore, effect);
+  var load = async function (args, onError) {
+    var idsToLoad = args.map(function (arg) {
+          return arg.cacheKey;
+        });
+    var idsFromCache = new Set();
+    var match = persistence.storageStatus;
+    var tmp;
+    tmp = typeof match !== "object" || match.TAG === "Initializing" ? false : Utils.Dict.has(match._0.cache, effectName);
+    if (tmp) {
+      var timerRef = Prometheus.StorageLoad.startOperation(key);
+      var match$1 = effect.storageMeta;
+      var outputSchema = match$1.outputSchema;
+      var dbEntities;
+      try {
+        dbEntities = await Persistence.getInitializedStorageOrThrow(persistence).loadByIdsOrThrow(idsToLoad, match$1.table, Internal.effectCacheItemRowsSchema);
+      }
+      catch (raw_exn){
+        var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+        Logging.childWarn(Logging.getItemLogger(item), {
+              msg: "Failed to load cache effect cache. The indexer will continue working, but the effect will not be able to use the cache.",
+              err: Utils.prettifyExn(exn),
+              effect: effectName
+            });
+        dbEntities = [];
+      }
+      dbEntities.forEach(function (dbEntity) {
+            try {
+              var output = S$RescriptSchema.parseOrThrow(dbEntity.output, outputSchema);
+              idsFromCache.add(dbEntity.id);
+              inMemTable.dict[dbEntity.id] = output;
+              return ;
+            }
+            catch (raw_error){
+              var error = Caml_js_exceptions.internalToOCamlException(raw_error);
+              if (error.RE_EXN_ID === S$RescriptSchema.Raised) {
+                inMemTable.invalidationsCount = inMemTable.invalidationsCount + 1 | 0;
+                Prometheus.EffectCacheInvalidationsCount.increment(effectName);
+                return Logging.childTrace(Logging.getItemLogger(item), {
+                            msg: "Invalidated effect cache",
+                            input: dbEntity.id,
+                            effect: effectName,
+                            err: S$RescriptSchema.$$Error.message(error._1)
+                          });
+              }
+              throw error;
+            }
+          });
+      Prometheus.StorageLoad.endOperation(timerRef, key, idsToLoad.length, dbEntities.length);
+    }
+    var remainingCallsCount = idsToLoad.length - idsFromCache.size | 0;
+    if (remainingCallsCount <= 0) {
+      return ;
+    }
+    var argsToCall = [];
+    for(var idx = 0 ,idx_finish = args.length; idx < idx_finish; ++idx){
+      var arg = args[idx];
+      if (!idsFromCache.has(arg.cacheKey)) {
+        argsToCall.push(arg);
+      }
+      
+    }
+    if (Utils.$$Array.notEmpty(argsToCall)) {
+      return await executeWithRateLimit(effect, argsToCall, inMemTable, onError, false);
+    }
+    
+  };
+  return LoadManager.call(loadManager, effectArgs, key, load, (function (args) {
+                return args.cacheKey;
+              }), shouldGroup, (function (hash) {
+                return Utils.Dict.has(inMemTable.dict, hash);
+              }), (function (hash) {
+                return inMemTable.dict[hash];
+              }));
+}
+
+function loadByField(loadManager, persistence, operator, entityConfig, inMemoryStore, fieldName, fieldValueSchema, shouldGroup, item, fieldValue) {
+  var operatorCallName;
+  switch (operator) {
+    case "Eq" :
+        operatorCallName = "eq";
+        break;
+    case "Gt" :
+        operatorCallName = "gt";
+        break;
+    case "Lt" :
+        operatorCallName = "lt";
+        break;
+    
+  }
+  var key = entityConfig.name + ".getWhere." + fieldName + "." + operatorCallName;
+  var inMemTable = InMemoryStore.getInMemTable(inMemoryStore, entityConfig);
+  var load = async function (fieldValues, param) {
+    var timerRef = Prometheus.StorageLoad.startOperation(key);
+    var size = {
+      contents: 0
+    };
+    var indiciesToLoad = fieldValues.map(function (fieldValue) {
+          return {
+                  fieldName: fieldName,
+                  fieldValue: fieldValue,
+                  operator: operator
+                };
+        });
+    await Promise.all(indiciesToLoad.map(async function (index) {
+              InMemoryTable.Entity.addEmptyIndex(inMemTable, index);
+              try {
+                var tmp;
+                switch (index.operator) {
+                  case "Eq" :
+                      tmp = "=";
+                      break;
+                  case "Gt" :
+                      tmp = ">";
+                      break;
+                  case "Lt" :
+                      tmp = "<";
+                      break;
+                  
+                }
+                var entities = await Persistence.getInitializedStorageOrThrow(persistence).loadByFieldOrThrow(TableIndices.Index.getFieldName(index), fieldValueSchema, index.fieldValue, tmp, entityConfig.table, entityConfig.rowsSchema);
+                Belt_Array.forEach(entities, (function (entity) {
+                        InMemoryTable.Entity.initValue(inMemTable, entity.id, entity, false);
+                      }));
+                size.contents = size.contents + entities.length | 0;
+                return ;
+              }
+              catch (raw_exn){
+                var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+                if (exn.RE_EXN_ID === Persistence.StorageError) {
+                  return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Logging.getItemLogger(item), {
+                                  operator: operatorCallName,
+                                  tableName: entityConfig.table.tableName,
+                                  fieldName: fieldName,
+                                  fieldValue: fieldValue
+                                }), exn.message, exn.reason);
+                }
+                throw exn;
+              }
+            }));
+    return Prometheus.StorageLoad.endOperation(timerRef, key, fieldValues.length, size.contents);
+  };
+  return LoadManager.call(loadManager, fieldValue, key, load, (function (fieldValue) {
+                return TableIndices.FieldValue.toString(fieldValue);
+              }), shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable, fieldName, operator), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable, fieldName, operator));
+}
+
+export {
+  loadById ,
+  loadByField ,
+  loadEffect ,
+}
+/* Utils Not a pure module */

package/src/LoadLayer.resi

@@ -0,0 +1,32 @@
+let loadById: (
+  ~loadManager: LoadManager.t,
+  ~persistence: Persistence.t,
+  ~entityConfig: Internal.entityConfig,
+  ~inMemoryStore: InMemoryStore.t,
+  ~shouldGroup: bool,
+  ~item: Internal.item,
+  ~entityId: string,
+) => promise<option<Internal.entity>>
+
+let loadByField: (
+  ~loadManager: LoadManager.t,
+  ~persistence: Persistence.t,
+  ~operator: TableIndices.Operator.t,
+  ~entityConfig: Internal.entityConfig,
+  ~inMemoryStore: InMemoryStore.t,
+  ~fieldName: string,
+  ~fieldValueSchema: RescriptSchema.S.t<'fieldValue>,
+  ~shouldGroup: bool,
+  ~item: Internal.item,
+  ~fieldValue: 'fieldValue,
+) => promise<array<Internal.entity>>
+
+let loadEffect: (
+  ~loadManager: LoadManager.t,
+  ~persistence: Persistence.t,
+  ~effect: Internal.effect,
+  ~effectArgs: Internal.effectArgs,
+  ~inMemoryStore: InMemoryStore.t,
+  ~shouldGroup: bool,
+  ~item: Internal.item,
+) => promise<Internal.effectOutput>

package/src/Prometheus.res

@@ -34,12 +34,6 @@ let allChainsSyncedToHead = PromClient.Gauge.makeGauge({
   "labelNames": [],
 })
 
-let sourceChainHeight = PromClient.Gauge.makeGauge({
-  "name": "chain_block_height",
-  "help": "Chain Height of Source Chain",
-  "labelNames": ["chainId"],
-})
-
 module Labels = {
   let rec schemaIsString = (schema: S.t<'a>) =>
     switch schema->S.classify {
@@ -233,8 +227,14 @@ let incrementStorageWriteCounter = () => {
   storageWriteCounter->PromClient.Counter.inc
 }
 
-let setSourceChainHeight = (~blockNumber, ~chainId) => {
-  sourceChainHeight
+let knownHeightGauge = PromClient.Gauge.makeGauge({
+  "name": "envio_indexing_known_height",
+  "help": "The latest known block number reported by the active indexing source. This value may lag behind the actual chain height, as it is updated only when needed.",
+  "labelNames": ["chainId"],
+})
+
+let setKnownHeight = (~blockNumber, ~chainId) => {
+  knownHeightGauge
   ->PromClient.Gauge.labels({"chainId": chainId})
   ->PromClient.Gauge.set(blockNumber)
 }

package/src/Prometheus.res.mjs

@@ -44,12 +44,6 @@ var allChainsSyncedToHead = new PromClient.Gauge({
       labelNames: []
     });
 
-var sourceChainHeight = new PromClient.Gauge({
-      name: "chain_block_height",
-      help: "Chain Height of Source Chain",
-      labelNames: ["chainId"]
-    });
-
 function schemaIsString(_schema) {
   while(true) {
     var schema = _schema;
@@ -333,8 +327,14 @@ function incrementStorageWriteCounter() {
   storageWriteCounter.inc();
 }
 
-function setSourceChainHeight(blockNumber, chainId) {
-  sourceChainHeight.labels({
+var knownHeightGauge = new PromClient.Gauge({
+      name: "envio_indexing_known_height",
+      help: "The latest known block number reported by the active indexing source. This value may lag behind the actual chain height, as it is updated only when needed.",
+      labelNames: ["chainId"]
+    });
+
+function setKnownHeight(blockNumber, chainId) {
+  knownHeightGauge.labels({
           chainId: chainId
         }).set(blockNumber);
 }
@@ -854,7 +854,6 @@ export {
   storageWriteTimeCounter ,
   storageWriteCounter ,
   allChainsSyncedToHead ,
-  sourceChainHeight ,
   Labels ,
   metricNames ,
   MakeSafePromMetric ,
@@ -867,7 +866,8 @@ export {
   incrementExecuteBatchDurationCounter ,
   incrementStorageWriteTimeCounter ,
   incrementStorageWriteCounter ,
-  setSourceChainHeight ,
+  knownHeightGauge ,
+  setKnownHeight ,
   setAllChainsSyncedToHead ,
   BenchmarkCounters ,
   chainIdLabelsSchema ,

package/src/ReorgDetection.res

@@ -77,13 +77,10 @@ let make = (
   }
 }
 
-let getDataByBlockNumberCopyInThreshold = (
-  {dataByBlockNumber, maxReorgDepth}: t,
-  ~currentBlockHeight,
-) => {
+let getDataByBlockNumberCopyInThreshold = ({dataByBlockNumber, maxReorgDepth}: t, ~knownHeight) => {
   // Js engine automatically orders numeric object keys
   let ascBlockNumberKeys = dataByBlockNumber->Js.Dict.keys
-  let thresholdBlockNumber = currentBlockHeight - maxReorgDepth
+  let thresholdBlockNumber = knownHeight - maxReorgDepth
 
   let copy = Js.Dict.empty()
 
@@ -105,10 +102,9 @@ let getDataByBlockNumberCopyInThreshold = (
 let registerReorgGuard = (
   {maxReorgDepth, shouldRollbackOnReorg} as self: t,
   ~reorgGuard: reorgGuard,
-  ~currentBlockHeight,
+  ~knownHeight,
 ) => {
-  let dataByBlockNumberCopyInThreshold =
-    self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
+  let dataByBlockNumberCopyInThreshold = self->getDataByBlockNumberCopyInThreshold(~knownHeight)
 
   let {rangeLastBlock, prevRangeLastBlock} = reorgGuard
 
@@ -248,12 +244,12 @@ let rollbackToValidBlockNumber = (
   }
 }
 
-let getThresholdBlockNumbersBelowBlock = (self: t, ~blockNumber: int, ~currentBlockHeight) => {
+let getThresholdBlockNumbersBelowBlock = (self: t, ~blockNumber: int, ~knownHeight) => {
   let arr = []
 
   // Js engine automatically orders numeric object keys
   let ascBlockNumberKeys = self.dataByBlockNumber->Js.Dict.keys
-  let thresholdBlockNumber = currentBlockHeight - self.maxReorgDepth
+  let thresholdBlockNumber = knownHeight - self.maxReorgDepth
 
   for idx in 0 to ascBlockNumberKeys->Array.length - 1 {
     let blockNumberKey = ascBlockNumberKeys->Js.Array2.unsafe_get(idx)

package/src/ReorgDetection.res.mjs

@@ -30,10 +30,10 @@ function make(chainReorgCheckpoints, maxReorgDepth, shouldRollbackOnReorg) {
         };
 }
 
-function getDataByBlockNumberCopyInThreshold(param, currentBlockHeight) {
+function getDataByBlockNumberCopyInThreshold(param, knownHeight) {
   var dataByBlockNumber = param.dataByBlockNumber;
   var ascBlockNumberKeys = Object.keys(dataByBlockNumber);
-  var thresholdBlockNumber = currentBlockHeight - param.maxReorgDepth | 0;
+  var thresholdBlockNumber = knownHeight - param.maxReorgDepth | 0;
   var copy = {};
   for(var idx = 0 ,idx_finish = ascBlockNumberKeys.length; idx < idx_finish; ++idx){
     var blockNumberKey = ascBlockNumberKeys[idx];
@@ -47,10 +47,10 @@ function getDataByBlockNumberCopyInThreshold(param, currentBlockHeight) {
   return copy;
 }
 
-function registerReorgGuard(self, reorgGuard, currentBlockHeight) {
+function registerReorgGuard(self, reorgGuard, knownHeight) {
   var maxReorgDepth = self.maxReorgDepth;
   var shouldRollbackOnReorg = self.shouldRollbackOnReorg;
-  var dataByBlockNumberCopyInThreshold = getDataByBlockNumberCopyInThreshold(self, currentBlockHeight);
+  var dataByBlockNumberCopyInThreshold = getDataByBlockNumberCopyInThreshold(self, knownHeight);
   var prevRangeLastBlock = reorgGuard.prevRangeLastBlock;
   var rangeLastBlock = reorgGuard.rangeLastBlock;
   var scannedBlock = dataByBlockNumberCopyInThreshold[String(rangeLastBlock.blockNumber)];
@@ -159,10 +159,10 @@ function rollbackToValidBlockNumber(param, blockNumber) {
         };
 }
 
-function getThresholdBlockNumbersBelowBlock(self, blockNumber, currentBlockHeight) {
+function getThresholdBlockNumbersBelowBlock(self, blockNumber, knownHeight) {
   var arr = [];
   var ascBlockNumberKeys = Object.keys(self.dataByBlockNumber);
-  var thresholdBlockNumber = currentBlockHeight - self.maxReorgDepth | 0;
+  var thresholdBlockNumber = knownHeight - self.maxReorgDepth | 0;
   for(var idx = 0 ,idx_finish = ascBlockNumberKeys.length; idx < idx_finish; ++idx){
     var blockNumberKey = ascBlockNumberKeys[idx];
     var scannedBlock = self.dataByBlockNumber[blockNumberKey];