envio 2.31.1 → 2.32.0

package/src/InMemoryTable.res.js

@@ -0,0 +1,401 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+'use strict';
+
+var Utils = require("./Utils.res.js");
+var Js_exn = require("rescript/lib/js/js_exn.js");
+var Lodash = require("./bindings/Lodash.res.js");
+var Js_dict = require("rescript/lib/js/js_dict.js");
+var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var Belt_Option = require("rescript/lib/js/belt_Option.js");
+var Caml_option = require("rescript/lib/js/caml_option.js");
+var TableIndices = require("./TableIndices.res.js");
+var ErrorHandling = require("./ErrorHandling.res.js");
+var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
+
+function make(hash) {
+  return {
+          dict: {},
+          hash: hash
+        };
+}
+
+function set(self, key, value) {
+  self.dict[self.hash(key)] = value;
+}
+
+function setByHash(self, hash, value) {
+  self.dict[hash] = value;
+}
+
+function hasByHash(self, hash) {
+  return Utils.Dict.has(self.dict, hash);
+}
+
+function getUnsafeByHash(self, hash) {
+  return self.dict[hash];
+}
+
+function get(self, key) {
+  return self.dict[self.hash(key)];
+}
+
+function values(self) {
+  return Js_dict.values(self.dict);
+}
+
+function clone(self) {
+  return {
+          dict: Lodash.cloneDeep(self.dict),
+          hash: self.hash
+        };
+}
+
+var UnexpectedIdNotDefinedOnEntity = /* @__PURE__ */Caml_exceptions.create("InMemoryTable.Entity.UnexpectedIdNotDefinedOnEntity");
+
+function getEntityIdUnsafe(entity) {
+  var id = entity.id;
+  if (id !== undefined) {
+    return Caml_option.valFromOption(id);
+  } else {
+    return ErrorHandling.mkLogAndRaise(undefined, "Property 'id' does not exist on expected entity object", {
+                RE_EXN_ID: UnexpectedIdNotDefinedOnEntity
+              });
+  }
+}
+
+function makeIndicesSerializedToValue(index, relatedEntityIdsOpt) {
+  var relatedEntityIds = relatedEntityIdsOpt !== undefined ? Caml_option.valFromOption(relatedEntityIdsOpt) : new Set();
+  var empty_dict = {};
+  var empty_hash = TableIndices.Index.toString;
+  var empty = {
+    dict: empty_dict,
+    hash: empty_hash
+  };
+  set(empty, index, [
+        index,
+        relatedEntityIds
+      ]);
+  return empty;
+}
+
+function make$1() {
+  return {
+          table: {
+            dict: {},
+            hash: (function (str) {
+                return str;
+              })
+          },
+          fieldNameIndices: {
+            dict: {},
+            hash: TableIndices.Index.getFieldName
+          }
+        };
+}
+
+var UndefinedKey = /* @__PURE__ */Caml_exceptions.create("InMemoryTable.Entity.UndefinedKey");
+
+function updateIndices(self, entity, entityIndices) {
+  entityIndices.forEach(function (index) {
+        var fieldName = TableIndices.Index.getFieldName(index);
+        var fieldValue = Js_dict.get(entity, fieldName);
+        if (!TableIndices.Index.evaluate(index, fieldName, fieldValue)) {
+          entityIndices.delete(index);
+          return ;
+        }
+        
+      });
+  Belt_Array.forEach(Object.keys(self.fieldNameIndices.dict), (function (fieldName) {
+          var match = Js_dict.get(entity, fieldName);
+          var match$1 = Js_dict.get(self.fieldNameIndices.dict, fieldName);
+          if (match !== undefined && match$1 !== undefined) {
+            var fieldValue = Caml_option.valFromOption(match);
+            return Belt_Array.forEach(Js_dict.values(match$1.dict), (function (param) {
+                          var relatedEntityIds = param[1];
+                          var index = param[0];
+                          if (TableIndices.Index.evaluate(index, fieldName, fieldValue)) {
+                            relatedEntityIds.add(getEntityIdUnsafe(entity));
+                            entityIndices.add(index);
+                          } else {
+                            relatedEntityIds.delete(getEntityIdUnsafe(entity));
+                          }
+                        }));
+          }
+          ErrorHandling.mkLogAndRaise(undefined, "Expected field name to exist on the referenced index and the provided entity", {
+                RE_EXN_ID: UndefinedKey,
+                _1: fieldName
+              });
+        }));
+}
+
+function deleteEntityFromIndices(self, entityId, entityIndices) {
+  entityIndices.forEach(function (index) {
+        var match = Belt_Option.flatMap(get(self.fieldNameIndices, index), (function (__x) {
+                return get(__x, index);
+              }));
+        if (match !== undefined) {
+          match[1].delete(entityId);
+        }
+        entityIndices.delete(index);
+      });
+}
+
+function initValue(inMemTable, key, entity, allowOverWriteEntityOpt) {
+  var allowOverWriteEntity = allowOverWriteEntityOpt !== undefined ? allowOverWriteEntityOpt : false;
+  var shouldWriteEntity = allowOverWriteEntity || Belt_Option.isNone(Js_dict.get(inMemTable.table.dict, inMemTable.table.hash(key)));
+  if (!shouldWriteEntity) {
+    return ;
+  }
+  var entityIndices = new Set();
+  var initialStoreRow;
+  if (entity !== undefined) {
+    var entity$1 = Caml_option.valFromOption(entity);
+    updateIndices(inMemTable, entity$1, entityIndices);
+    initialStoreRow = {
+      TAG: "InitialReadFromDb",
+      _0: {
+        TAG: "AlreadySet",
+        _0: entity$1
+      }
+    };
+  } else {
+    initialStoreRow = {
+      TAG: "InitialReadFromDb",
+      _0: "NotSet"
+    };
+  }
+  inMemTable.table.dict[inMemTable.table.hash(key)] = {
+    entityRow: initialStoreRow,
+    entityIndices: entityIndices
+  };
+}
+
+function set$1(inMemTable, entityUpdate, shouldSaveHistory, containsRollbackDiffChangeOpt) {
+  var containsRollbackDiffChange = containsRollbackDiffChangeOpt !== undefined ? containsRollbackDiffChangeOpt : false;
+  var match = get(inMemTable.table, entityUpdate.entityId);
+  var match$1;
+  if (match !== undefined) {
+    var previous_values = match.entityRow;
+    if (previous_values.TAG === "Updated") {
+      var previous_values$1 = previous_values._0;
+      var entityRow = {
+        TAG: "Updated",
+        _0: {
+          latest: entityUpdate,
+          history: shouldSaveHistory ? (
+              previous_values$1.latest.checkpointId === entityUpdate.checkpointId ? Utils.$$Array.setIndexImmutable(previous_values$1.history, previous_values$1.history.length - 1 | 0, entityUpdate) : Belt_Array.concatMany([
+                      previous_values$1.history,
+                      [entityUpdate]
+                    ])
+            ) : previous_values$1.history,
+          containsRollbackDiffChange: previous_values$1.containsRollbackDiffChange
+        }
+      };
+      match$1 = {
+        entityRow: entityRow,
+        entityIndices: match.entityIndices
+      };
+    } else {
+      match$1 = {
+        entityRow: {
+          TAG: "Updated",
+          _0: {
+            latest: entityUpdate,
+            history: shouldSaveHistory ? [entityUpdate] : [],
+            containsRollbackDiffChange: containsRollbackDiffChange
+          }
+        },
+        entityIndices: match.entityIndices
+      };
+    }
+  } else {
+    match$1 = {
+      entityRow: {
+        TAG: "Updated",
+        _0: {
+          latest: entityUpdate,
+          history: shouldSaveHistory ? [entityUpdate] : [],
+          containsRollbackDiffChange: containsRollbackDiffChange
+        }
+      },
+      entityIndices: new Set()
+    };
+  }
+  var entityIndices = match$1.entityIndices;
+  var entity = entityUpdate.entityUpdateAction;
+  if (typeof entity !== "object") {
+    deleteEntityFromIndices(inMemTable, entityUpdate.entityId, entityIndices);
+  } else {
+    updateIndices(inMemTable, entity._0, entityIndices);
+  }
+  set(inMemTable.table, entityUpdate.entityId, {
+        entityRow: match$1.entityRow,
+        entityIndices: entityIndices
+      });
+}
+
+function rowToEntity(row) {
+  var match = row.entityRow;
+  if (match.TAG === "Updated") {
+    var entity = match._0.latest.entityUpdateAction;
+    if (typeof entity !== "object") {
+      return ;
+    } else {
+      return Caml_option.some(entity._0);
+    }
+  }
+  var entity$1 = match._0;
+  if (typeof entity$1 !== "object") {
+    return ;
+  } else {
+    return Caml_option.some(entity$1._0);
+  }
+}
+
+function getUnsafe(inMemTable) {
+  return function (key) {
+    return rowToEntity(inMemTable.table.dict[key]);
+  };
+}
+
+function hasIndex(inMemTable, fieldName, operator) {
+  return function (fieldValueHash) {
+    var indicesSerializedToValue = inMemTable.fieldNameIndices.dict[fieldName];
+    if (indicesSerializedToValue === undefined) {
+      return false;
+    }
+    var key = fieldName + ":" + operator + ":" + fieldValueHash;
+    return indicesSerializedToValue.dict[key] !== undefined;
+  };
+}
+
+function getUnsafeOnIndex(inMemTable, fieldName, operator) {
+  var getEntity = getUnsafe(inMemTable);
+  return function (fieldValueHash) {
+    var indicesSerializedToValue = inMemTable.fieldNameIndices.dict[fieldName];
+    if (indicesSerializedToValue === undefined) {
+      return Js_exn.raiseError("Unexpected error. Must have an index on field " + fieldName);
+    }
+    var key = fieldName + ":" + operator + ":" + fieldValueHash;
+    var match = indicesSerializedToValue.dict[key];
+    if (match !== undefined) {
+      return Belt_Array.keepMap(Array.from(match[1]), (function (entityId) {
+                    if (hasByHash(inMemTable.table, entityId)) {
+                      return getEntity(entityId);
+                    }
+                    
+                  }));
+    } else {
+      return Js_exn.raiseError("Unexpected error. Must have an index for the value " + fieldValueHash + " on field " + fieldName);
+    }
+  };
+}
+
+function addEmptyIndex(inMemTable, index) {
+  var fieldName = TableIndices.Index.getFieldName(index);
+  var relatedEntityIds = new Set();
+  Belt_Array.forEach(Js_dict.values(inMemTable.table.dict), (function (row) {
+          var entity = rowToEntity(row);
+          if (entity === undefined) {
+            return ;
+          }
+          var entity$1 = Caml_option.valFromOption(entity);
+          var fieldValue = entity$1[fieldName];
+          if (TableIndices.Index.evaluate(index, fieldName, fieldValue)) {
+            row.entityIndices.add(index);
+            relatedEntityIds.add(getEntityIdUnsafe(entity$1));
+            return ;
+          }
+          
+        }));
+  var indicesSerializedToValue = get(inMemTable.fieldNameIndices, index);
+  if (indicesSerializedToValue === undefined) {
+    return set(inMemTable.fieldNameIndices, index, makeIndicesSerializedToValue(index, Caml_option.some(relatedEntityIds)));
+  }
+  var match = get(indicesSerializedToValue, index);
+  if (match !== undefined) {
+    return ;
+  } else {
+    return set(indicesSerializedToValue, index, [
+                index,
+                relatedEntityIds
+              ]);
+  }
+}
+
+function addIdToIndex(inMemTable, index, entityId) {
+  var indicesSerializedToValue = get(inMemTable.fieldNameIndices, index);
+  if (indicesSerializedToValue === undefined) {
+    return set(inMemTable.fieldNameIndices, index, makeIndicesSerializedToValue(index, Caml_option.some(new Set().add(entityId))));
+  }
+  var match = get(indicesSerializedToValue, index);
+  if (match !== undefined) {
+    match[1].add(entityId);
+    return ;
+  } else {
+    return set(indicesSerializedToValue, index, [
+                index,
+                new Set().add(entityId)
+              ]);
+  }
+}
+
+function rows(inMemTable) {
+  return Belt_Array.map(Js_dict.values(inMemTable.table.dict), (function (v) {
+                return v.entityRow;
+              }));
+}
+
+function values$1(inMemTable) {
+  return Belt_Array.keepMap(Js_dict.values(inMemTable.table.dict), rowToEntity);
+}
+
+function clone$1(param) {
+  var fieldNameIndices = param.fieldNameIndices;
+  return {
+          table: clone(param.table),
+          fieldNameIndices: {
+            dict: Js_dict.fromArray(Belt_Array.map(Js_dict.entries(fieldNameIndices.dict), (function (param) {
+                        return [
+                                param[0],
+                                clone(param[1])
+                              ];
+                      }))),
+            hash: fieldNameIndices.hash
+          }
+        };
+}
+
+var Entity = {
+  UnexpectedIdNotDefinedOnEntity: UnexpectedIdNotDefinedOnEntity,
+  getEntityIdUnsafe: getEntityIdUnsafe,
+  makeIndicesSerializedToValue: makeIndicesSerializedToValue,
+  make: make$1,
+  UndefinedKey: UndefinedKey,
+  updateIndices: updateIndices,
+  deleteEntityFromIndices: deleteEntityFromIndices,
+  initValue: initValue,
+  setRow: set,
+  set: set$1,
+  rowToEntity: rowToEntity,
+  getRow: get,
+  getUnsafe: getUnsafe,
+  hasIndex: hasIndex,
+  getUnsafeOnIndex: getUnsafeOnIndex,
+  addEmptyIndex: addEmptyIndex,
+  addIdToIndex: addIdToIndex,
+  rows: rows,
+  values: values$1,
+  clone: clone$1
+};
+
+exports.make = make;
+exports.set = set;
+exports.setByHash = setByHash;
+exports.hasByHash = hasByHash;
+exports.getUnsafeByHash = getUnsafeByHash;
+exports.get = get;
+exports.values = values;
+exports.clone = clone;
+exports.Entity = Entity;
+/* Utils Not a pure module */

package/src/Indexer.res

@@ -0,0 +1,5 @@
+type t = {
+  registrations: EventRegister.registrations,
+  config: Config.t,
+  persistence: Persistence.t,
+}

package/src/Internal.res

@@ -281,25 +281,37 @@ let makeEnumConfig = (~name, ~variants) => {
 
 type effectInput
 type effectOutput
-type effectContext
+type effectContext = private {mutable cache: bool}
 type effectArgs = {
   input: effectInput,
   context: effectContext,
   cacheKey: string,
 }
 type effectCacheItem = {id: string, output: effectOutput}
-type effectCacheMeta = {
+type effectCacheStorageMeta = {
   itemSchema: S.t<effectCacheItem>,
   outputSchema: S.t<effectOutput>,
   table: Table.table,
 }
+type rateLimitState = {
+  callsPerDuration: int,
+  durationMs: int,
+  mutable availableCalls: int,
+  mutable windowStartTime: float,
+  mutable queueCount: int,
+  mutable nextWindowPromise: option<promise<unit>>,
+}
 type effect = {
   name: string,
   handler: effectArgs => promise<effectOutput>,
-  cache: option<effectCacheMeta>,
+  storageMeta: effectCacheStorageMeta,
+  defaultShouldCache: bool,
   output: S.t<effectOutput>,
   input: S.t<effectInput>,
-  mutable callsCount: int,
+  // The number of functions that are currently running.
+  mutable activeCallsCount: int,
+  mutable prevCallStartTimerRef: Hrtime.timeRef,
+  rateLimit: option<rateLimitState>,
 }
 let cacheTablePrefix = "envio_effect_"
 let cacheOutputSchema = S.json(~validate=false)->(Utils.magic: S.t<Js.Json.t> => S.t<effectOutput>)

package/src/Persistence.res

@@ -34,7 +34,7 @@ type initialState = {
   reorgCheckpoints: array<Internal.reorgCheckpoint>,
 }
 
-type operator = [#">" | #"="]
+type operator = [#">" | #"=" | #"<"]
 
 type storage = {
   // Should return true if we already have persisted data
@@ -43,7 +43,7 @@ type storage = {
   // Should initialize the storage so we can start interacting with it
   // Eg create connection, schema, tables, etc.
   initialize: (
-    ~chainConfigs: array<InternalConfig.chain>=?,
+    ~chainConfigs: array<Config.chain>=?,
     ~entities: array<Internal.entityConfig>=?,
     ~enums: array<Internal.enumConfig<Internal.enum>>=?,
   ) => promise<initialState>,
@@ -91,7 +91,10 @@ type t = {
   allEntities: array<Internal.entityConfig>,
   allEnums: array<Internal.enumConfig<Internal.enum>>,
   mutable storageStatus: storageStatus,
-  storage: storage,
+  mutable storage: storage,
+  // FIXME: This is temporary to move it library
+  // Should be a part of the storage interface and db agnostic
+  mutable sql: Postgres.sql,
 }
 
 let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
@@ -106,6 +109,7 @@ let make = (
   // TODO: Should only pass userEnums and create internal config in runtime
   ~allEnums,
   ~storage,
+  ~sql,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([InternalTable.DynamicContractRegistry.config])
   let allEnums =
@@ -116,6 +120,7 @@ let make = (
     allEnums,
     storageStatus: Unknown,
     storage,
+    sql,
   }
 }
 

package/src/Persistence.res.js

@@ -25,7 +25,7 @@ var entityHistoryActionEnumConfig = {
   default: "SET"
 };
 
-function make(userEntities, allEnums, storage) {
+function make(userEntities, allEnums, storage, sql) {
   var allEntities = userEntities.concat([InternalTable.DynamicContractRegistry.config]);
   var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
   return {
@@ -33,7 +33,8 @@ function make(userEntities, allEnums, storage) {
           allEntities: allEntities,
           allEnums: allEnums$1,
           storageStatus: "Unknown",
-          storage: storage
+          storage: storage,
+          sql: sql
         };
 }
 

package/src/PgStorage.res

@@ -789,13 +789,7 @@ let make = (
     ~items: array<Internal.effectCacheItem>,
     ~initialize: bool,
   ) => {
-    let {table, itemSchema} = switch effect.cache {
-    | Some(cacheMeta) => cacheMeta
-    | None =>
-      Js.Exn.raiseError(
-        `Failed to set effect cache for "${effect.name}". Effect has no cache enabled.`,
-      )
-    }
+    let {table, itemSchema} = effect.storageMeta
 
     if initialize {
       let _ =

package/src/PgStorage.res.js

@@ -570,8 +570,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     return setOrThrow(sql, items, table, itemSchema, pgSchema);
   };
   var setEffectCacheOrThrow = async function (effect, items, initialize) {
-    var cacheMeta = effect.cache;
-    var match = cacheMeta !== undefined ? cacheMeta : Js_exn.raiseError("Failed to set effect cache for \"" + effect.name + "\". Effect has no cache enabled.");
+    var match = effect.storageMeta;
     var table = match.table;
     if (initialize) {
       await sql.unsafe(makeCreateTableQuery(table, pgSchema, false));

package/src/Prometheus.res

@@ -580,20 +580,46 @@ module ProgressBatchCount = {
   }
 }
 
+module ProgressLatency = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_progress_latency",
+    ~help="The latency in milliseconds between the latest processed event creation and the time it was written to storage.",
+    ~labelSchema=chainIdLabelsSchema,
+  )
+
+  let set = (~latencyMs, ~chainId) => {
+    gauge->SafeGauge.handleInt(~labels=chainId, ~value=latencyMs)
+  }
+}
+
 let effectLabelsSchema = S.object(s => {
   s.field("effect", S.string)
 })
 
-module EffectCallsCount = {
-  let gauge = SafeGauge.makeOrThrow(
+module EffectCalls = {
+  let timeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_effect_calls_time",
+    ~help="Processing time taken to call the Effect function. (milliseconds)",
+    ~labelSchema=effectLabelsSchema,
+  )
+
+  let sumTimeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_effect_calls_sum_time",
+    ~help="Cumulative time spent calling the Effect function during the indexing process. (milliseconds)",
+    ~labelSchema=effectLabelsSchema,
+  )
+
+  let totalCallsCount = SafeCounter.makeOrThrow(
     ~name="envio_effect_calls_count",
-    ~help="The number of calls to the effect. Including both handler execution and cache hits.",
+    ~help="Cumulative number of resolved Effect function calls during the indexing process.",
     ~labelSchema=effectLabelsSchema,
   )
 
-  let set = (~callsCount, ~effectName) => {
-    gauge->SafeGauge.handleInt(~labels=effectName, ~value=callsCount)
-  }
+  let activeCallsCount = SafeGauge.makeOrThrow(
+    ~name="envio_effect_active_calls_count",
+    ~help="The number of Effect function calls that are currently running.",
+    ~labelSchema=effectLabelsSchema,
+  )
 }
 
 module EffectCacheCount = {
@@ -620,6 +646,18 @@ module EffectCacheInvalidationsCount = {
   }
 }
 
+module EffectQueueCount = {
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="envio_effect_queue_count",
+    ~help="The number of effect calls waiting in the rate limit queue.",
+    ~labelSchema=effectLabelsSchema,
+  )
+
+  let set = (~count, ~effectName) => {
+    gauge->SafeGauge.handleInt(~labels=effectName, ~value=count)
+  }
+}
+
 module StorageLoad = {
   let operationLabelsSchema = S.object(s => s.field("operation", S.string))
 
@@ -629,8 +667,8 @@ module StorageLoad = {
     ~labelSchema=operationLabelsSchema,
   )
 
-  let totalTimeCounter = SafeCounter.makeOrThrow(
-    ~name="envio_storage_load_total_time",
+  let sumTimeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_storage_load_sum_time",
     ~help="Cumulative time spent loading data from storage during the indexing process. (milliseconds)",
     ~labelSchema=operationLabelsSchema,
   )
@@ -686,7 +724,7 @@ module StorageLoad = {
       )
       operations->Utils.Dict.deleteInPlace(operation)
     }
-    totalTimeCounter->SafeCounter.handleInt(
+    sumTimeCounter->SafeCounter.handleInt(
       ~labels={operation},
       ~value=timerRef->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis,
     )