envio 2.32.1 → 3.0.0-alpha-main-clickhouse-sink

package/src/Persistence.res

@@ -29,13 +29,24 @@ type initialState = {
   cleanRun: bool,
   cache: dict<effectCacheRecord>,
   chains: array<initialChainState>,
-  checkpointId: int,
+  checkpointId: Internal.checkpointId,
   // Needed to keep reorg detection logic between restarts
   reorgCheckpoints: array<Internal.reorgCheckpoint>,
 }
 
 type operator = [#">" | #"=" | #"<"]
 
+type updatedEffectCache = {
+  effect: Internal.effect,
+  items: array<Internal.effectCacheItem>,
+  shouldInitialize: bool,
+}
+
+type updatedEntity = {
+  entityConfig: Internal.entityConfig,
+  updates: array<Internal.inMemoryStoreEntityUpdate<Internal.entity>>,
+}
+
 type storage = {
   // Should return true if we already have persisted data
   // and we can skip initialization
@@ -45,7 +56,7 @@ type storage = {
   initialize: (
     ~chainConfigs: array<Config.chain>=?,
     ~entities: array<Internal.entityConfig>=?,
-    ~enums: array<Internal.enumConfig<Internal.enum>>=?,
+    ~enums: array<Table.enumConfig<Table.enum>>=?,
   ) => promise<initialState>,
   resumeInitialState: unit => promise<initialState>,
   @raises("StorageError")
@@ -77,10 +88,53 @@ type storage = {
   ) => promise<unit>,
   // This is to download cache from the database to .envio/cache
   dumpEffectCache: unit => promise<unit>,
+  // Execute raw SQL query
+  executeUnsafe: string => promise<unknown>,
+  // Check if entity history has rows
+  hasEntityHistoryRows: unit => promise<bool>,
+  // Update chain metadata
+  setChainMeta: dict<InternalTable.Chains.metaFields> => promise<unknown>,
+  // Prune old checkpoints
+  pruneStaleCheckpoints: (~safeCheckpointId: Internal.checkpointId) => promise<unit>,
+  // Prune stale entity history
+  pruneStaleEntityHistory: (
+    ~entityName: string,
+    ~entityIndex: int,
+    ~safeCheckpointId: Internal.checkpointId,
+  ) => promise<unit>,
+  // Get rollback target checkpoint
+  getRollbackTargetCheckpoint: (
+    ~reorgChainId: int,
+    ~lastKnownValidBlockNumber: int,
+  ) => promise<array<{"id": Internal.checkpointId}>>,
+  // Get rollback progress diff
+  getRollbackProgressDiff: (
+    ~rollbackTargetCheckpointId: Internal.checkpointId,
+  ) => promise<
+    array<{
+      "chain_id": int,
+      "events_processed_diff": string,
+      "new_progress_block_number": int,
+    }>,
+  >,
+  // Get rollback data for entity
+  getRollbackData: (
+    ~entityConfig: Internal.entityConfig,
+    ~rollbackTargetCheckpointId: Internal.checkpointId,
+  ) => promise<(array<{"id": string}>, array<unknown>)>,
+  // Write batch to storage
+  writeBatch: (
+    ~batch: Batch.t,
+    ~rawEvents: array<InternalTable.RawEvents.t>,
+    ~rollbackTargetCheckpointId: option<Internal.checkpointId>,
+    ~isInReorgThreshold: bool,
+    ~config: Config.t,
+    ~allEntities: array<Internal.entityConfig>,
+    ~updatedEffectsCache: array<updatedEffectCache>,
+    ~updatedEntities: array<updatedEntity>,
+  ) => promise<unit>,
 }
 
-exception StorageError({message: string, reason: exn})
-
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
@@ -89,38 +143,28 @@ type storageStatus =
 type t = {
   userEntities: array<Internal.entityConfig>,
   allEntities: array<Internal.entityConfig>,
-  allEnums: array<Internal.enumConfig<Internal.enum>>,
+  allEnums: array<Table.enumConfig<Table.enum>>,
   mutable storageStatus: storageStatus,
   mutable storage: storage,
-  // FIXME: This is temporary to move it library
-  // Should be a part of the storage interface and db agnostic
-  mutable sql: Postgres.sql,
 }
 
-let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
-  name: EntityHistory.RowAction.name,
-  variants: EntityHistory.RowAction.variants,
-  schema: EntityHistory.RowAction.schema,
-  default: SET,
-}
+exception StorageError({message: string, reason: exn})
 
 let make = (
   ~userEntities,
   // TODO: Should only pass userEnums and create internal config in runtime
   ~allEnums,
   ~storage,
-  ~sql,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([InternalTable.DynamicContractRegistry.config])
   let allEnums =
-    allEnums->Js.Array2.concat([entityHistoryActionEnumConfig->Internal.fromGenericEnumConfig])
+    allEnums->Js.Array2.concat([EntityHistory.RowAction.config->Table.fromGenericEnumConfig])
   {
     userEntities,
     allEntities,
     allEnums,
     storageStatus: Unknown,
     storage,
-    sql,
   }
 }
 
@@ -197,32 +241,140 @@ let getInitializedState = persistence => {
   }
 }
 
-let setEffectCacheOrThrow = async (
+let writeBatch = (
   persistence,
-  ~effect: Internal.effect,
-  ~items,
-  ~invalidationsCount,
-) => {
+  ~batch,
+  ~config,
+  ~inMemoryStore: InMemoryStore.t,
+  ~isInReorgThreshold,
+) =>
   switch persistence.storageStatus {
   | Unknown
   | Initializing(_) =>
     Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
-  | Ready({cache}) => {
-      let storage = persistence.storage
-      let effectName = effect.name
-      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
-      | Some(c) => c
-      | None => {
-          let c = {effectName, count: 0}
-          cache->Js.Dict.set(effectName, c)
-          c
-        }
+  | Ready({cache}) =>
+    let updatedEntities = persistence.allEntities->Belt.Array.keepMapU(entityConfig => {
+      let updates =
+        inMemoryStore
+        ->InMemoryStore.getInMemTable(~entityConfig)
+        ->InMemoryTable.Entity.updates
+      if updates->Utils.Array.isEmpty {
+        None
+      } else {
+        Some({entityConfig, updates})
       }
-      let initialize = effectCacheRecord.count === 0
-      await storage.setEffectCacheOrThrow(~effect, ~items, ~initialize)
-      effectCacheRecord.count =
-        effectCacheRecord.count + items->Js.Array2.length - invalidationsCount
-      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
-    }
+    })
+    persistence.storage.writeBatch(
+      ~batch,
+      ~rawEvents=inMemoryStore.rawEvents->InMemoryTable.values,
+      ~rollbackTargetCheckpointId=inMemoryStore.rollbackTargetCheckpointId,
+      ~isInReorgThreshold,
+      ~config,
+      ~allEntities=persistence.allEntities,
+      ~updatedEntities,
+      ~updatedEffectsCache={
+        inMemoryStore.effects
+        ->Js.Dict.keys
+        ->Belt.Array.keepMapU(effectName => {
+          let inMemTable = inMemoryStore.effects->Js.Dict.unsafeGet(effectName)
+          let {idsToStore, dict, effect, invalidationsCount} = inMemTable
+          switch idsToStore {
+          | [] => None
+          | ids => {
+              let items = Belt.Array.makeUninitializedUnsafe(ids->Belt.Array.length)
+              ids->Belt.Array.forEachWithIndex((index, id) => {
+                items->Js.Array2.unsafe_set(
+                  index,
+                  (
+                    {
+                      id,
+                      output: dict->Js.Dict.unsafeGet(id),
+                    }: Internal.effectCacheItem
+                  ),
+                )
+              })
+              Some({
+                let effectName = effect.name
+                let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(
+                  effectName,
+                ) {
+                | Some(c) => c
+                | None => {
+                    let c = {effectName, count: 0}
+                    cache->Js.Dict.set(effectName, c)
+                    c
+                  }
+                }
+                let shouldInitialize = effectCacheRecord.count === 0
+                effectCacheRecord.count =
+                  effectCacheRecord.count + items->Js.Array2.length - invalidationsCount
+                Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
+                {effect, items, shouldInitialize}
+              })
+            }
+          }
+        })
+      },
+    )
+  }
+
+let prepareRollbackDiff = async (
+  persistence: t,
+  ~rollbackTargetCheckpointId,
+  ~rollbackDiffCheckpointId,
+) => {
+  let inMemStore = InMemoryStore.make(
+    ~entities=persistence.allEntities,
+    ~rollbackTargetCheckpointId,
+  )
+
+  let deletedEntities = Js.Dict.empty()
+  let setEntities = Js.Dict.empty()
+
+  let _ =
+    await persistence.allEntities
+    ->Belt.Array.map(async entityConfig => {
+      let entityTable = inMemStore->InMemoryStore.getInMemTable(~entityConfig)
+
+      let (removedIdsResult, restoredEntitiesResult) = await persistence.storage.getRollbackData(
+        ~entityConfig,
+        ~rollbackTargetCheckpointId,
+      )
+
+      // Process removed IDs
+      removedIdsResult->Js.Array2.forEach(data => {
+        deletedEntities->Utils.Dict.push(entityConfig.name, data["id"])
+        entityTable->InMemoryTable.Entity.set(
+          Delete({
+            entityId: data["id"],
+            checkpointId: rollbackDiffCheckpointId,
+          }),
+          ~shouldSaveHistory=false,
+          ~containsRollbackDiffChange=true,
+        )
+      })
+
+      let restoredEntities = restoredEntitiesResult->S.parseOrThrow(entityConfig.rowsSchema)
+
+      // Process restored entities
+      restoredEntities->Belt.Array.forEach((entity: Internal.entity) => {
+        setEntities->Utils.Dict.push(entityConfig.name, entity.id)
+        entityTable->InMemoryTable.Entity.set(
+          Set({
+            entityId: entity.id,
+            checkpointId: rollbackDiffCheckpointId,
+            entity,
+          }),
+          ~shouldSaveHistory=false,
+          ~containsRollbackDiffChange=true,
+        )
+      })
+    })
+    ->Promise.all
+
+  {
+    "inMemStore": inMemStore,
+    "deletedEntities": deletedEntities,
+    "setEntities": setEntities,
   }
 }

package/src/Persistence.res.js

@@ -1,40 +1,31 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Utils = require("./Utils.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Logging = require("./Logging.res.js");
+var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var Prometheus = require("./Prometheus.res.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
+var InMemoryStore = require("./InMemoryStore.res.js");
+var InMemoryTable = require("./InMemoryTable.res.js");
 var InternalTable = require("./db/InternalTable.res.js");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
+var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
 var StorageError = /* @__PURE__ */Caml_exceptions.create("Persistence.StorageError");
 
-var entityHistoryActionEnumConfig_name = EntityHistory.RowAction.name;
-
-var entityHistoryActionEnumConfig_variants = EntityHistory.RowAction.variants;
-
-var entityHistoryActionEnumConfig_schema = EntityHistory.RowAction.schema;
-
-var entityHistoryActionEnumConfig = {
-  name: entityHistoryActionEnumConfig_name,
-  variants: entityHistoryActionEnumConfig_variants,
-  schema: entityHistoryActionEnumConfig_schema,
-  default: "SET"
-};
-
-function make(userEntities, allEnums, storage, sql) {
+function make(userEntities, allEnums, storage) {
   var allEntities = userEntities.concat([InternalTable.DynamicContractRegistry.config]);
-  var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
+  var allEnums$1 = allEnums.concat([EntityHistory.RowAction.config]);
   return {
           userEntities: userEntities,
           allEntities: allEntities,
           allEnums: allEnums$1,
           storageStatus: "Unknown",
-          storage: storage,
-          sql: sql
+          storage: storage
         };
 }
 
@@ -120,7 +111,7 @@ function getInitializedState(persistence) {
   }
 }
 
-async function setEffectCacheOrThrow(persistence, effect, items, invalidationsCount) {
+function writeBatch(persistence, batch, config, inMemoryStore, isInReorgThreshold) {
   var match = persistence.storageStatus;
   if (typeof match !== "object") {
     return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
@@ -129,31 +120,93 @@ async function setEffectCacheOrThrow(persistence, effect, items, invalidationsCo
     return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
   }
   var cache = match._0.cache;
-  var storage = persistence.storage;
-  var effectName = effect.name;
-  var c = cache[effectName];
-  var effectCacheRecord;
-  if (c !== undefined) {
-    effectCacheRecord = c;
-  } else {
-    var c$1 = {
-      effectName: effectName,
-      count: 0
-    };
-    cache[effectName] = c$1;
-    effectCacheRecord = c$1;
-  }
-  var initialize = effectCacheRecord.count === 0;
-  await storage.setEffectCacheOrThrow(effect, items, initialize);
-  effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
-  return Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
+  var updatedEntities = Belt_Array.keepMapU(persistence.allEntities, (function (entityConfig) {
+          var updates = InMemoryTable.Entity.updates(InMemoryStore.getInMemTable(inMemoryStore, entityConfig));
+          if (Utils.$$Array.isEmpty(updates)) {
+            return ;
+          } else {
+            return {
+                    entityConfig: entityConfig,
+                    updates: updates
+                  };
+          }
+        }));
+  return persistence.storage.writeBatch(batch, InMemoryTable.values(inMemoryStore.rawEvents), inMemoryStore.rollbackTargetCheckpointId, isInReorgThreshold, config, persistence.allEntities, Belt_Array.keepMapU(Object.keys(inMemoryStore.effects), (function (effectName) {
+                    var inMemTable = inMemoryStore.effects[effectName];
+                    var idsToStore = inMemTable.idsToStore;
+                    var invalidationsCount = inMemTable.invalidationsCount;
+                    var effect = inMemTable.effect;
+                    var dict = inMemTable.dict;
+                    if (idsToStore.length === 0) {
+                      return ;
+                    }
+                    var items = new Array(idsToStore.length);
+                    Belt_Array.forEachWithIndex(idsToStore, (function (index, id) {
+                            items[index] = {
+                              id: id,
+                              output: dict[id]
+                            };
+                          }));
+                    var effectName$1 = effect.name;
+                    var c = cache[effectName$1];
+                    var effectCacheRecord;
+                    if (c !== undefined) {
+                      effectCacheRecord = c;
+                    } else {
+                      var c$1 = {
+                        effectName: effectName$1,
+                        count: 0
+                      };
+                      cache[effectName$1] = c$1;
+                      effectCacheRecord = c$1;
+                    }
+                    var shouldInitialize = effectCacheRecord.count === 0;
+                    return effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0, Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName$1), {
+                            effect: effect,
+                            items: items,
+                            shouldInitialize: shouldInitialize
+                          };
+                  })), updatedEntities);
+}
+
+async function prepareRollbackDiff(persistence, rollbackTargetCheckpointId, rollbackDiffCheckpointId) {
+  var inMemStore = InMemoryStore.make(persistence.allEntities, rollbackTargetCheckpointId);
+  var deletedEntities = {};
+  var setEntities = {};
+  await Promise.all(Belt_Array.map(persistence.allEntities, (async function (entityConfig) {
+              var entityTable = InMemoryStore.getInMemTable(inMemStore, entityConfig);
+              var match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
+              match[0].forEach(function (data) {
+                    Utils.Dict.push(deletedEntities, entityConfig.name, data.id);
+                    InMemoryTable.Entity.set(entityTable, {
+                          type: "DELETE",
+                          entityId: data.id,
+                          checkpointId: rollbackDiffCheckpointId
+                        }, false, true);
+                  });
+              var restoredEntities = S$RescriptSchema.parseOrThrow(match[1], entityConfig.rowsSchema);
+              return Belt_Array.forEach(restoredEntities, (function (entity) {
+                            Utils.Dict.push(setEntities, entityConfig.name, entity.id);
+                            InMemoryTable.Entity.set(entityTable, {
+                                  type: "SET",
+                                  entityId: entity.id,
+                                  entity: entity,
+                                  checkpointId: rollbackDiffCheckpointId
+                                }, false, true);
+                          }));
+            })));
+  return {
+          inMemStore: inMemStore,
+          deletedEntities: deletedEntities,
+          setEntities: setEntities
+        };
 }
 
 exports.StorageError = StorageError;
-exports.entityHistoryActionEnumConfig = entityHistoryActionEnumConfig;
 exports.make = make;
 exports.init = init;
 exports.getInitializedStorageOrThrow = getInitializedStorageOrThrow;
 exports.getInitializedState = getInitializedState;
-exports.setEffectCacheOrThrow = setEffectCacheOrThrow;
-/* Logging Not a pure module */
+exports.writeBatch = writeBatch;
+exports.prepareRollbackDiff = prepareRollbackDiff;
+/* Utils Not a pure module */