envio 3.0.0-alpha.2 → 3.0.0-alpha.20

package/src/LoadLayer.res.mjs

@@ -0,0 +1,296 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as Hrtime from "./bindings/Hrtime.res.mjs";
+import * as Logging from "./Logging.res.mjs";
+import * as Internal from "./Internal.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
+import * as LoadManager from "./LoadManager.res.mjs";
+import * as Persistence from "./Persistence.res.mjs";
+import * as TableIndices from "./TableIndices.res.mjs";
+import * as ErrorHandling from "./ErrorHandling.res.mjs";
+import * as InMemoryStore from "./InMemoryStore.res.mjs";
+import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
+
+function loadById(loadManager, persistence, entityConfig, inMemoryStore, shouldGroup, item, entityId) {
+  var key = entityConfig.name + ".get";
+  var inMemTable = InMemoryStore.getInMemTable(inMemoryStore, entityConfig);
+  var load = async function (idsToLoad, param) {
+    var timerRef = Prometheus.StorageLoad.startOperation(key);
+    var dbEntities;
+    try {
+      dbEntities = await Persistence.getInitializedStorageOrThrow(persistence).loadByIdsOrThrow(idsToLoad, entityConfig.table, entityConfig.rowsSchema);
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      if (exn.RE_EXN_ID === Persistence.StorageError) {
+        dbEntities = ErrorHandling.mkLogAndRaise(Logging.getItemLogger(item), exn.message, exn.reason);
+      } else {
+        throw exn;
+      }
+    }
+    var entitiesMap = {};
+    for(var idx = 0 ,idx_finish = dbEntities.length; idx < idx_finish; ++idx){
+      var entity = dbEntities[idx];
+      entitiesMap[entity.id] = entity;
+    }
+    idsToLoad.forEach(function (entityId) {
+          InMemoryTable.Entity.initValue(inMemTable, entityId, entitiesMap[entityId], false);
+        });
+    return Prometheus.StorageLoad.endOperation(timerRef, key, idsToLoad.length, dbEntities.length);
+  };
+  return LoadManager.call(loadManager, entityId, key, load, LoadManager.noopHasher, shouldGroup, (function (hash) {
+                return InMemoryTable.hasByHash(inMemTable.table, hash);
+              }), InMemoryTable.Entity.getUnsafe(inMemTable));
+}
+
+function callEffect(effect, arg, inMemTable, timerRef, onError) {
+  var effectName = effect.name;
+  var hadActiveCalls = effect.activeCallsCount > 0;
+  effect.activeCallsCount = effect.activeCallsCount + 1 | 0;
+  Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, effectName, effect.activeCallsCount);
+  if (hadActiveCalls) {
+    var elapsed = Hrtime.secondsBetween(effect.prevCallStartTimerRef, timerRef);
+    if (elapsed > 0) {
+      Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, effectName, elapsed);
+    }
+    
+  }
+  effect.prevCallStartTimerRef = timerRef;
+  return effect.handler(arg).then(function (output) {
+                  inMemTable.dict[arg.cacheKey] = output;
+                  if (arg.context.cache) {
+                    inMemTable.idsToStore.push(arg.cacheKey);
+                    return ;
+                  }
+                  
+                }).catch(function (exn) {
+                onError(arg.cacheKey, exn);
+              }).finally(function () {
+              effect.activeCallsCount = effect.activeCallsCount - 1 | 0;
+              Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, effectName, effect.activeCallsCount);
+              var newTimer = Hrtime.makeTimer();
+              Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, effectName, Hrtime.secondsBetween(effect.prevCallStartTimerRef, newTimer));
+              effect.prevCallStartTimerRef = newTimer;
+              Prometheus.SafeCounter.increment(Prometheus.EffectCalls.totalCallsCount, effectName);
+              Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.sumTimeCounter, effectName, Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef)));
+            });
+}
+
+function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQueue) {
+  var effectName = effect.name;
+  var timerRef = Hrtime.makeTimer();
+  var promises = [];
+  var state = effect.rateLimit;
+  if (state !== undefined) {
+    var now = Date.now();
+    if (now >= state.windowStartTime + state.durationMs) {
+      state.availableCalls = state.callsPerDuration;
+      state.windowStartTime = now;
+      state.nextWindowPromise = undefined;
+    }
+    var immediateCount = Math.min(state.availableCalls, effectArgs.length);
+    var immediateArgs = Belt_Array.slice(effectArgs, 0, immediateCount);
+    var queuedArgs = Belt_Array.sliceToEnd(effectArgs, immediateCount);
+    state.availableCalls = state.availableCalls - immediateCount | 0;
+    for(var idx = 0 ,idx_finish = immediateArgs.length; idx < idx_finish; ++idx){
+      promises.push(callEffect(effect, immediateArgs[idx], inMemTable, timerRef, onError));
+    }
+    if (immediateCount > 0 && isFromQueue) {
+      state.queueCount = state.queueCount - immediateCount | 0;
+      Prometheus.EffectQueueCount.set(state.queueCount, effectName);
+    }
+    if (Utils.$$Array.notEmpty(queuedArgs)) {
+      if (!isFromQueue) {
+        state.queueCount = state.queueCount + queuedArgs.length | 0;
+        Prometheus.EffectQueueCount.set(state.queueCount, effectName);
+      }
+      var millisUntilReset = {
+        contents: 0
+      };
+      var p = state.nextWindowPromise;
+      var nextWindowPromise;
+      if (p !== undefined) {
+        nextWindowPromise = Caml_option.valFromOption(p);
+      } else {
+        millisUntilReset.contents = state.windowStartTime + state.durationMs - now | 0;
+        var p$1 = Utils.delay(millisUntilReset.contents);
+        state.nextWindowPromise = Caml_option.some(p$1);
+        nextWindowPromise = p$1;
+      }
+      promises.push(nextWindowPromise.then(function () {
+                if (millisUntilReset.contents > 0) {
+                  Prometheus.SafeCounter.handleFloat(Prometheus.EffectQueueCount.timeCounter, effectName, millisUntilReset.contents / 1000);
+                }
+                return executeWithRateLimit(effect, queuedArgs, inMemTable, onError, true);
+              }));
+    }
+    
+  } else {
+    for(var idx$1 = 0 ,idx_finish$1 = effectArgs.length; idx$1 < idx_finish$1; ++idx$1){
+      promises.push(callEffect(effect, effectArgs[idx$1], inMemTable, timerRef, onError));
+    }
+  }
+  return Promise.all(promises);
+}
+
+function loadEffect(loadManager, persistence, effect, effectArgs, inMemoryStore, shouldGroup, item) {
+  var effectName = effect.name;
+  var key = effectName + ".effect";
+  var inMemTable = InMemoryStore.getEffectInMemTable(inMemoryStore, effect);
+  var load = async function (args, onError) {
+    var idsToLoad = args.map(function (arg) {
+          return arg.cacheKey;
+        });
+    var idsFromCache = new Set();
+    var match = persistence.storageStatus;
+    var tmp;
+    tmp = typeof match !== "object" || match.TAG === "Initializing" ? false : Utils.Dict.has(match._0.cache, effectName);
+    if (tmp) {
+      var timerRef = Prometheus.StorageLoad.startOperation(key);
+      var match$1 = effect.storageMeta;
+      var outputSchema = match$1.outputSchema;
+      var dbEntities;
+      try {
+        dbEntities = await Persistence.getInitializedStorageOrThrow(persistence).loadByIdsOrThrow(idsToLoad, match$1.table, Internal.effectCacheItemRowsSchema);
+      }
+      catch (raw_exn){
+        var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+        Logging.childWarn(Logging.getItemLogger(item), {
+              msg: "Failed to load cache effect cache. The indexer will continue working, but the effect will not be able to use the cache.",
+              err: Utils.prettifyExn(exn),
+              effect: effectName
+            });
+        dbEntities = [];
+      }
+      dbEntities.forEach(function (dbEntity) {
+            try {
+              var output = S$RescriptSchema.parseOrThrow(dbEntity.output, outputSchema);
+              idsFromCache.add(dbEntity.id);
+              inMemTable.dict[dbEntity.id] = output;
+              return ;
+            }
+            catch (raw_error){
+              var error = Caml_js_exceptions.internalToOCamlException(raw_error);
+              if (error.RE_EXN_ID === S$RescriptSchema.Raised) {
+                inMemTable.invalidationsCount = inMemTable.invalidationsCount + 1 | 0;
+                Prometheus.EffectCacheInvalidationsCount.increment(effectName);
+                return Logging.childTrace(Logging.getItemLogger(item), {
+                            msg: "Invalidated effect cache",
+                            input: dbEntity.id,
+                            effect: effectName,
+                            err: S$RescriptSchema.$$Error.message(error._1)
+                          });
+              }
+              throw error;
+            }
+          });
+      Prometheus.StorageLoad.endOperation(timerRef, key, idsToLoad.length, dbEntities.length);
+    }
+    var remainingCallsCount = idsToLoad.length - idsFromCache.size | 0;
+    if (remainingCallsCount <= 0) {
+      return ;
+    }
+    var argsToCall = [];
+    for(var idx = 0 ,idx_finish = args.length; idx < idx_finish; ++idx){
+      var arg = args[idx];
+      if (!idsFromCache.has(arg.cacheKey)) {
+        argsToCall.push(arg);
+      }
+      
+    }
+    if (Utils.$$Array.notEmpty(argsToCall)) {
+      return await executeWithRateLimit(effect, argsToCall, inMemTable, onError, false);
+    }
+    
+  };
+  return LoadManager.call(loadManager, effectArgs, key, load, (function (args) {
+                return args.cacheKey;
+              }), shouldGroup, (function (hash) {
+                return Utils.Dict.has(inMemTable.dict, hash);
+              }), (function (hash) {
+                return inMemTable.dict[hash];
+              }));
+}
+
+function loadByField(loadManager, persistence, operator, entityConfig, inMemoryStore, fieldName, fieldValueSchema, shouldGroup, item, fieldValue) {
+  var operatorCallName;
+  switch (operator) {
+    case "Eq" :
+        operatorCallName = "eq";
+        break;
+    case "Gt" :
+        operatorCallName = "gt";
+        break;
+    case "Lt" :
+        operatorCallName = "lt";
+        break;
+    
+  }
+  var key = entityConfig.name + ".getWhere." + fieldName + "." + operatorCallName;
+  var inMemTable = InMemoryStore.getInMemTable(inMemoryStore, entityConfig);
+  var load = async function (fieldValues, param) {
+    var timerRef = Prometheus.StorageLoad.startOperation(key);
+    var size = {
+      contents: 0
+    };
+    var indiciesToLoad = fieldValues.map(function (fieldValue) {
+          return {
+                  fieldName: fieldName,
+                  fieldValue: fieldValue,
+                  operator: operator
+                };
+        });
+    await Promise.all(indiciesToLoad.map(async function (index) {
+              InMemoryTable.Entity.addEmptyIndex(inMemTable, index);
+              try {
+                var tmp;
+                switch (index.operator) {
+                  case "Eq" :
+                      tmp = "=";
+                      break;
+                  case "Gt" :
+                      tmp = ">";
+                      break;
+                  case "Lt" :
+                      tmp = "<";
+                      break;
+                  
+                }
+                var entities = await Persistence.getInitializedStorageOrThrow(persistence).loadByFieldOrThrow(TableIndices.Index.getFieldName(index), fieldValueSchema, index.fieldValue, tmp, entityConfig.table, entityConfig.rowsSchema);
+                Belt_Array.forEach(entities, (function (entity) {
+                        InMemoryTable.Entity.initValue(inMemTable, entity.id, entity, false);
+                      }));
+                size.contents = size.contents + entities.length | 0;
+                return ;
+              }
+              catch (raw_exn){
+                var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+                if (exn.RE_EXN_ID === Persistence.StorageError) {
+                  return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Logging.getItemLogger(item), {
+                                  operator: operatorCallName,
+                                  tableName: entityConfig.table.tableName,
+                                  fieldName: fieldName,
+                                  fieldValue: fieldValue
+                                }), exn.message, exn.reason);
+                }
+                throw exn;
+              }
+            }));
+    return Prometheus.StorageLoad.endOperation(timerRef, key, fieldValues.length, size.contents);
+  };
+  return LoadManager.call(loadManager, fieldValue, key, load, (function (fieldValue) {
+                return TableIndices.FieldValue.toString(fieldValue);
+              }), shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable, fieldName, operator), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable, fieldName, operator));
+}
+
+export {
+  loadById ,
+  loadByField ,
+  loadEffect ,
+}
+/* Utils Not a pure module */

package/src/LoadLayer.resi

@@ -0,0 +1,32 @@
+let loadById: (
+  ~loadManager: LoadManager.t,
+  ~persistence: Persistence.t,
+  ~entityConfig: Internal.entityConfig,
+  ~inMemoryStore: InMemoryStore.t,
+  ~shouldGroup: bool,
+  ~item: Internal.item,
+  ~entityId: string,
+) => promise<option<Internal.entity>>
+
+let loadByField: (
+  ~loadManager: LoadManager.t,
+  ~persistence: Persistence.t,
+  ~operator: TableIndices.Operator.t,
+  ~entityConfig: Internal.entityConfig,
+  ~inMemoryStore: InMemoryStore.t,
+  ~fieldName: string,
+  ~fieldValueSchema: RescriptSchema.S.t<'fieldValue>,
+  ~shouldGroup: bool,
+  ~item: Internal.item,
+  ~fieldValue: 'fieldValue,
+) => promise<array<Internal.entity>>
+
+let loadEffect: (
+  ~loadManager: LoadManager.t,
+  ~persistence: Persistence.t,
+  ~effect: Internal.effect,
+  ~effectArgs: Internal.effectArgs,
+  ~inMemoryStore: InMemoryStore.t,
+  ~shouldGroup: bool,
+  ~item: Internal.item,
+) => promise<Internal.effectOutput>

package/src/LogSelection.res

@@ -84,38 +84,44 @@ let parseEventFiltersOrThrow = {
     }
 
     let parse = (eventFilters: Js.Json.t): array<Internal.topicSelection> => {
-      switch eventFilters {
-      | Array([]) => [%raw(`{}`)]
-      | Array(a) => a
-      | _ => [eventFilters]
-      }->Js.Array2.map(eventFilter => {
-        switch eventFilter {
-        | Object(eventFilter) => {
-            let filterKeys = eventFilter->Js.Dict.keys
-            switch filterKeys {
-            | [] => default
-            | _ => {
-                filterKeys->Js.Array2.forEach(key => {
-                  if params->Js.Array2.includes(key)->not {
-                    // In TS type validation doesn't catch this
-                    // when we have eventFilters as a callback
-                    Js.Exn.raiseError(
-                      `Invalid event filters configuration. The event doesn't have an indexed parameter "${key}" and can't use it for filtering`,
-                    )
+      if eventFilters === Obj.magic(true) {
+        [default]
+      } else if eventFilters === Obj.magic(false) {
+        []
+      } else {
+        switch eventFilters {
+        | Array([]) => [%raw(`{}`)]
+        | Array(a) => a
+        | _ => [eventFilters]
+        }->Js.Array2.map(eventFilter => {
+          switch eventFilter {
+          | Object(eventFilter) => {
+              let filterKeys = eventFilter->Js.Dict.keys
+              switch filterKeys {
+              | [] => default
+              | _ => {
+                  filterKeys->Js.Array2.forEach(key => {
+                    if params->Js.Array2.includes(key)->not {
+                      // In TS type validation doesn't catch this
+                      // when we have eventFilters as a callback
+                      Js.Exn.raiseError(
+                        `Invalid event filters configuration. The event doesn't have an indexed parameter "${key}" and can't use it for filtering`,
+                      )
+                    }
+                  })
+                  {
+                    Internal.topic0,
+                    topic1: topic1(eventFilter),
+                    topic2: topic2(eventFilter),
+                    topic3: topic3(eventFilter),
                   }
-                })
-                {
-                  Internal.topic0,
-                  topic1: topic1(eventFilter),
-                  topic2: topic2(eventFilter),
-                  topic3: topic3(eventFilter),
                 }
               }
             }
+          | _ => Js.Exn.raiseError("Invalid event filters configuration. Expected an object")
           }
-        | _ => Js.Exn.raiseError("Invalid event filters configuration. Expected an object")
-        }
-      })
+        })
+      }
     }
 
     let getEventFiltersOrThrow = switch eventFilters {

package/src/LogSelection.res.mjs

@@ -100,6 +100,12 @@ function parseEventFiltersOrThrow(eventFilters, sighash, params, topic1Opt, topi
     topic3: emptyTopics
   };
   var parse = function (eventFilters) {
+    if (eventFilters === true) {
+      return [$$default];
+    }
+    if (eventFilters === false) {
+      return [];
+    }
     var tmp;
     tmp = !Array.isArray(eventFilters) && (eventFilters === null || typeof eventFilters !== "object") && typeof eventFilters !== "number" && typeof eventFilters !== "string" && typeof eventFilters !== "boolean" ? [eventFilters] : (
         Array.isArray(eventFilters) ? (

package/src/Logging.res

@@ -42,33 +42,40 @@ let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLeve
   let makeMultiStreamLogger =
     MultiStreamLogger.make(~userLogLevel, ~defaultFileLogLevel, ~customLevels=logLevels, ...)
 
+  // Empty base disables pid and hostname in logs
+  let base: Js.Json.t = %raw("{}")
+
   switch logStrategy {
   | EcsFile =>
     makeWithOptionsAndTransport(
       {
         ...Pino.ECS.make(),
         customLevels: logLevels,
+        base,
       },
       Transport.make(pinoFile),
     )
-  | EcsConsoleMultistream => makeMultiStreamLogger(~logFile=None, ~options=Some(Pino.ECS.make()))
+  | EcsConsoleMultistream =>
+    makeMultiStreamLogger(~logFile=None, ~options=Some({...Pino.ECS.make(), base}))
   | EcsConsole =>
     make({
       ...Pino.ECS.make(),
       level: userLogLevel,
       customLevels: logLevels,
+      base,
     })
   | FileOnly =>
     makeWithOptionsAndTransport(
       {
         customLevels: logLevels,
         level: defaultFileLogLevel,
+        base,
       },
       Transport.make(pinoFile),
     )
-  | ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=None)
-  | ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=None)
-  | Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=None)
+  | ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=Some({base: base}))
+  | ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=Some({base: base}))
+  | Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=Some({base: base}))
   }
 }
 
@@ -147,7 +154,9 @@ let createChildFrom = (~logger: t, ~params: 'a) => {
 let getItemLogger = {
   let cacheKey = "_logger"
   (item: Internal.item) => {
-    switch item->Utils.magic->Utils.Dict.dangerouslyGetNonOption(cacheKey) {
+    switch item
+    ->(Utils.magic: Internal.item => Js.Dict.t<Pino.t>)
+    ->Utils.Dict.dangerouslyGetNonOption(cacheKey) {
     | Some(l) => l
     | None => {
         let l = getLogger()->child(
@@ -169,7 +178,7 @@ let getItemLogger = {
             }->createChildParams
           },
         )
-        item->Utils.magic->Js.Dict.set(cacheKey, l)
+        item->(Utils.magic: Internal.item => Js.Dict.t<Pino.t>)->Js.Dict.set(cacheKey, l)
         l
       }
     }
@@ -178,7 +187,12 @@ let getItemLogger = {
 
 @inline
 let logForItem = (item, level: Pino.logLevel, message: string, ~params=?) => {
-  (item->getItemLogger->Utils.magic->Js.Dict.unsafeGet((level :> string)))(params, message)
+  (
+    item
+    ->getItemLogger
+    ->(Utils.magic: Pino.t => Js.Dict.t<(option<'a>, string) => unit>)
+    ->Js.Dict.unsafeGet((level :> string))
+  )(params, message)
 }
 
 let noopLogger: Envio.logger = {

package/src/Logging.res.mjs

@@ -71,27 +71,35 @@ function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel)
   var makeMultiStreamLogger = function (none, none$1) {
     return Pino.MultiStreamLogger.make(userLogLevel, logLevels, none, none$1, defaultFileLogLevel);
   };
+  var base = {};
   switch (logStrategy) {
     case "ecs-file" :
         var newrecord = Caml_obj.obj_dup(Pino.ECS.make());
-        return Pino$1.pino((newrecord.customLevels = logLevels, newrecord), Pino$1.transport(pinoFile));
+        return Pino$1.pino((newrecord.base = base, newrecord.customLevels = logLevels, newrecord), Pino$1.transport(pinoFile));
     case "ecs-console" :
         var newrecord$1 = Caml_obj.obj_dup(Pino.ECS.make());
-        return Pino$1.pino((newrecord$1.customLevels = logLevels, newrecord$1.level = userLogLevel, newrecord$1));
+        return Pino$1.pino((newrecord$1.base = base, newrecord$1.customLevels = logLevels, newrecord$1.level = userLogLevel, newrecord$1));
     case "ecs-console-multistream" :
-        return makeMultiStreamLogger(undefined, Pino.ECS.make());
+        var newrecord$2 = Caml_obj.obj_dup(Pino.ECS.make());
+        return makeMultiStreamLogger(undefined, (newrecord$2.base = base, newrecord$2));
     case "file-only" :
         return Pino$1.pino({
                     level: defaultFileLogLevel,
-                    customLevels: logLevels
+                    customLevels: logLevels,
+                    base: base
                   }, Pino$1.transport(pinoFile));
     case "console-raw" :
     case "console-pretty" :
-        return makeMultiStreamLogger(undefined, undefined);
+        break;
     case "both-prettyconsole" :
-        return makeMultiStreamLogger(logFilePath, undefined);
+        return makeMultiStreamLogger(logFilePath, {
+                    base: base
+                  });
     
   }
+  return makeMultiStreamLogger(undefined, {
+              base: base
+            });
 }
 
 function setLogger(l) {
@@ -180,7 +188,7 @@ var cacheKey = "_logger";
 function getItemLogger(item) {
   var l = item[cacheKey];
   if (l !== undefined) {
-    return Caml_option.valFromOption(l);
+    return l;
   }
   var tmp;
   if (item.kind === 0) {
@@ -207,7 +215,7 @@ function getItemLogger(item) {
 }
 
 function logForItem(item, level, message, params) {
-  return getItemLogger(item)[level](params, message);
+  getItemLogger(item)[level](params, message);
 }
 
 function noopLogger_debug(_message, param) {