envio 3.0.2 → 3.1.0-rc.1

package/src/InMemoryStore.res.mjs

@@ -2,22 +2,20 @@
 
 import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
-import * as Belt_Array from "@rescript/runtime/lib/es6/Belt_Array.js";
-import * as EventUtils from "./EventUtils.res.mjs";
-import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
+import * as Internal from "./Internal.res.mjs";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
-function hashRawEventsKey(key) {
-  return EventUtils.getEventIdKeyString(key.chainId, key.eventId);
-}
-
 let UndefinedEntity = /* @__PURE__ */Primitive_exceptions.create("InMemoryStore.EntityTables.UndefinedEntity");
 
 function make(entities) {
   let init = {};
-  Belt_Array.forEach(entities, entityConfig => {
+  entities.forEach(entityConfig => {
     init[entityConfig.name] = InMemoryTable.Entity.make();
   });
   return init;
@@ -35,40 +33,21 @@ function get(self, entityName) {
   }
 }
 
-function clone(self) {
-  return Object.fromEntries(Belt_Array.map(Object.entries(self), param => [
-    param[0],
-    InMemoryTable.Entity.clone(param[1])
-  ]));
-}
-
 let EntityTables = {
   UndefinedEntity: UndefinedEntity,
   make: make,
-  get: get,
-  clone: clone
+  get: get
 };
 
-function make$1(entities, rollbackTargetCheckpointId) {
+function make$1(entities, committedCheckpointIdOpt) {
+  let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
   return {
-    rawEvents: InMemoryTable.make(hashRawEventsKey),
+    allEntities: entities,
+    rawEvents: [],
     entities: make(entities),
     effects: {},
-    rollbackTargetCheckpointId: rollbackTargetCheckpointId
-  };
-}
-
-function clone$1(self) {
-  return {
-    rawEvents: InMemoryTable.clone(self.rawEvents),
-    entities: clone(self.entities),
-    effects: Stdlib_Dict.mapValues(self.effects, table => ({
-      idsToStore: table.idsToStore.slice(),
-      invalidationsCount: table.invalidationsCount,
-      dict: Utils.Dict.shallowCopy(table.dict),
-      effect: table.effect
-    })),
-    rollbackTargetCheckpointId: self.rollbackTargetCheckpointId
+    rollback: undefined,
+    committedCheckpointId: committedCheckpointId
   };
 }
 
@@ -93,10 +72,145 @@ function getInMemTable(inMemoryStore, entityConfig) {
 }
 
 function isRollingBack(inMemoryStore) {
-  return inMemoryStore.rollbackTargetCheckpointId !== undefined;
+  return inMemoryStore.rollback !== undefined;
+}
+
+async function writeBatch(inMemoryStore, persistence, batch, config, isInReorgThreshold) {
+  let match = persistence.storageStatus;
+  if (typeof match !== "object") {
+    return Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`);
+  }
+  if (match.TAG === "Initializing") {
+    return Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`);
+  }
+  let cache = match._0.cache;
+  let committedCheckpointId = inMemoryStore.committedCheckpointId;
+  let totalChanges = {
+    contents: 0
+  };
+  persistence.allEntities.forEach(entityConfig => {
+    totalChanges.contents = totalChanges.contents + getInMemTable(inMemoryStore, entityConfig).changesCount;
+  });
+  let keepLatestChanges = totalChanges.contents < 50000;
+  let updatedEntities = Stdlib_Array.filterMap(persistence.allEntities, entityConfig => {
+    let table = getInMemTable(inMemoryStore, entityConfig);
+    table.changesCount = table.changesCount - table.prevEntityChanges.length;
+    let changes = table.prevEntityChanges;
+    Utils.Dict.forEach(table.latestEntityChangeById, change => {
+      if (change.checkpointId > committedCheckpointId) {
+        changes.push(change);
+        return;
+      }
+    });
+    if (Utils.$$Array.isEmpty(changes)) {
+      return;
+    } else {
+      return {
+        entityConfig: entityConfig,
+        changes: changes
+      };
+    }
+  });
+  let acc = [];
+  await persistence.storage.writeBatch(batch, inMemoryStore.rawEvents, inMemoryStore.rollback, isInReorgThreshold, config, persistence.allEntities, (Utils.Dict.forEach(inMemoryStore.effects, inMemTable => {
+    let idsToStore = inMemTable.idsToStore;
+    let invalidationsCount = inMemTable.invalidationsCount;
+    let effect = inMemTable.effect;
+    let dict = inMemTable.dict;
+    if (idsToStore.length === 0) {
+      return;
+    }
+    let items = idsToStore.map(id => ({
+      id: id,
+      output: dict[id]
+    }));
+    let effectName = effect.name;
+    let c = cache[effectName];
+    let effectCacheRecord;
+    if (c !== undefined) {
+      effectCacheRecord = c;
+    } else {
+      let c$1 = {
+        effectName: effectName,
+        count: 0
+      };
+      cache[effectName] = c$1;
+      effectCacheRecord = c$1;
+    }
+    let shouldInitialize = effectCacheRecord.count === 0;
+    effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
+    Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
+    acc.push({
+      effect: effect,
+      items: items,
+      shouldInitialize: shouldInitialize
+    });
+  }), acc), updatedEntities);
+  inMemoryStore.rawEvents = [];
+  inMemoryStore.effects = {};
+  inMemoryStore.rollback = undefined;
+  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
+  inMemoryStore.committedCheckpointId = checkpointId !== undefined ? checkpointId : committedCheckpointId;
+  if (keepLatestChanges) {
+    persistence.allEntities.forEach(entityConfig => {
+      let table = getInMemTable(inMemoryStore, entityConfig);
+      inMemoryStore.entities[entityConfig.name] = InMemoryTable.Entity.resetButKeepLatestChanges(table);
+    });
+    return;
+  }
+  let loadedFromDbCount = {
+    contents: 0
+  };
+  let resetTables = persistence.allEntities.map(entityConfig => {
+    let resetTable = InMemoryTable.Entity.resetButKeepLoadedFromDbChanges(getInMemTable(inMemoryStore, entityConfig));
+    loadedFromDbCount.contents = loadedFromDbCount.contents + resetTable.changesCount;
+    return resetTable;
+  });
+  let dropEverything = loadedFromDbCount.contents >= 50000;
+  persistence.allEntities.forEach((entityConfig, idx) => {
+    inMemoryStore.entities[entityConfig.name] = dropEverything ? InMemoryTable.Entity.make() : resetTables[idx];
+  });
 }
 
-function setBatchDcs(inMemoryStore, batch, shouldSaveHistory) {
+async function prepareRollbackDiff(inMemoryStore, persistence, rollbackTargetCheckpointId, rollbackDiffCheckpointId) {
+  inMemoryStore.rawEvents = [];
+  inMemoryStore.entities = make(inMemoryStore.allEntities);
+  inMemoryStore.effects = {};
+  inMemoryStore.rollback = {
+    targetCheckpointId: rollbackTargetCheckpointId,
+    diffCheckpointId: rollbackDiffCheckpointId
+  };
+  let deletedEntities = {};
+  let setEntities = {};
+  await Promise.all(persistence.allEntities.map(async entityConfig => {
+    let entityTable = getInMemTable(inMemoryStore, entityConfig);
+    let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
+    match[0].forEach(data => {
+      Utils.Dict.push(deletedEntities, entityConfig.name, data.id);
+      InMemoryTable.Entity.set(entityTable, inMemoryStore.committedCheckpointId, {
+        type: "DELETE",
+        entityId: data.id,
+        checkpointId: rollbackDiffCheckpointId
+      });
+    });
+    let restoredEntities = S$RescriptSchema.parseOrThrow(match[1], entityConfig.rowsSchema);
+    restoredEntities.forEach(entity => {
+      Utils.Dict.push(setEntities, entityConfig.name, entity.id);
+      InMemoryTable.Entity.set(entityTable, inMemoryStore.committedCheckpointId, {
+        type: "SET",
+        entityId: entity.id,
+        entity: entity,
+        checkpointId: rollbackDiffCheckpointId
+      });
+    });
+  }));
+  return {
+    deletedEntities: deletedEntities,
+    setEntities: setEntities
+  };
+}
+
+function setBatchDcs(inMemoryStore, batch) {
   let inMemTable = getInMemTable(inMemoryStore, Config.EnvioAddresses.entityConfig);
   let itemIdx = 0;
   for (let checkpoint = 0, checkpoint_finish = batch.checkpointIds.length; checkpoint < checkpoint_finish; ++checkpoint) {
@@ -120,12 +234,12 @@ function setBatchDcs(inMemoryStore, batch, shouldSaveHistory) {
             registration_log_index: entity_registration_log_index,
             contract_name: entity_contract_name
           };
-          InMemoryTable.Entity.set(inMemTable, {
+          InMemoryTable.Entity.set(inMemTable, inMemoryStore.committedCheckpointId, {
             type: "SET",
             entityId: entity_id,
             entity: entity,
             checkpointId: checkpointId
-          }, shouldSaveHistory, undefined);
+          });
         }
       }
     }
@@ -133,14 +247,17 @@ function setBatchDcs(inMemoryStore, batch, shouldSaveHistory) {
   }
 }
 
+let keepLatestChangesLimit = 50000;
+
 export {
-  hashRawEventsKey,
   EntityTables,
   make$1 as make,
-  clone$1 as clone,
+  keepLatestChangesLimit,
   getEffectInMemTable,
   getInMemTable,
   isRollingBack,
+  writeBatch,
+  prepareRollbackDiff,
   setBatchDcs,
 }
 /* Utils Not a pure module */