envio 2.32.2 → 3.0.0-alpha-main-clickhouse-sink

package/src/InMemoryTable.res.js

@@ -147,114 +147,77 @@ function initValue(inMemTable, key, entity, allowOverWriteEntityOpt) {
     return ;
   }
   var entityIndices = new Set();
-  var initialStoreRow;
   if (entity !== undefined) {
-    var entity$1 = Caml_option.valFromOption(entity);
-    updateIndices(inMemTable, entity$1, entityIndices);
-    initialStoreRow = {
-      TAG: "InitialReadFromDb",
-      _0: {
-        TAG: "AlreadySet",
-        _0: entity$1
-      }
-    };
-  } else {
-    initialStoreRow = {
-      TAG: "InitialReadFromDb",
-      _0: "NotSet"
-    };
+    updateIndices(inMemTable, Caml_option.valFromOption(entity), entityIndices);
   }
   inMemTable.table.dict[inMemTable.table.hash(key)] = {
-    entityRow: initialStoreRow,
+    latest: entity,
+    status: "Loaded",
     entityIndices: entityIndices
   };
 }
 
-function set$1(inMemTable, entityUpdate, shouldSaveHistory, containsRollbackDiffChangeOpt) {
+function set$1(inMemTable, change, shouldSaveHistory, containsRollbackDiffChangeOpt) {
   var containsRollbackDiffChange = containsRollbackDiffChangeOpt !== undefined ? containsRollbackDiffChangeOpt : false;
-  var match = get(inMemTable.table, entityUpdate.entityId);
-  var match$1;
+  var latest;
+  latest = change.type === "SET" ? Caml_option.some(change.entity) : undefined;
+  var match = get(inMemTable.table, change.entityId);
+  var updatedEntityRecord;
   if (match !== undefined) {
-    var previous_values = match.entityRow;
-    if (previous_values.TAG === "Updated") {
-      var previous_values$1 = previous_values._0;
-      var entityRow = {
-        TAG: "Updated",
-        _0: {
-          latest: entityUpdate,
-          history: shouldSaveHistory ? (
-              previous_values$1.latest.checkpointId === entityUpdate.checkpointId ? Utils.$$Array.setIndexImmutable(previous_values$1.history, previous_values$1.history.length - 1 | 0, entityUpdate) : Belt_Array.concatMany([
-                      previous_values$1.history,
-                      [entityUpdate]
-                    ])
-            ) : previous_values$1.history,
-          containsRollbackDiffChange: previous_values$1.containsRollbackDiffChange
-        }
-      };
-      match$1 = {
-        entityRow: entityRow,
+    var previous_values = match.status;
+    if (typeof previous_values !== "object") {
+      updatedEntityRecord = {
+        latest: latest,
+        status: {
+          latestChange: change,
+          history: shouldSaveHistory ? [change] : Utils.$$Array.immutableEmpty,
+          containsRollbackDiffChange: containsRollbackDiffChange
+        },
         entityIndices: match.entityIndices
       };
     } else {
-      match$1 = {
-        entityRow: {
-          TAG: "Updated",
-          _0: {
-            latest: entityUpdate,
-            history: shouldSaveHistory ? [entityUpdate] : [],
-            containsRollbackDiffChange: containsRollbackDiffChange
-          }
-        },
+      var newStatus = {
+        latestChange: change,
+        history: shouldSaveHistory ? (
+            previous_values.latestChange.checkpointId === change.checkpointId ? Utils.$$Array.setIndexImmutable(previous_values.history, previous_values.history.length - 1 | 0, change) : Belt_Array.concatMany([
+                    previous_values.history,
+                    [change]
+                  ])
+          ) : previous_values.history,
+        containsRollbackDiffChange: previous_values.containsRollbackDiffChange
+      };
+      updatedEntityRecord = {
+        latest: latest,
+        status: newStatus,
         entityIndices: match.entityIndices
       };
     }
   } else {
-    match$1 = {
-      entityRow: {
-        TAG: "Updated",
-        _0: {
-          latest: entityUpdate,
-          history: shouldSaveHistory ? [entityUpdate] : [],
-          containsRollbackDiffChange: containsRollbackDiffChange
-        }
+    updatedEntityRecord = {
+      latest: latest,
+      status: {
+        latestChange: change,
+        history: shouldSaveHistory ? [change] : Utils.$$Array.immutableEmpty,
+        containsRollbackDiffChange: containsRollbackDiffChange
       },
       entityIndices: new Set()
     };
   }
-  var entityIndices = match$1.entityIndices;
-  var entity = entityUpdate.entityUpdateAction;
-  if (typeof entity !== "object") {
-    deleteEntityFromIndices(inMemTable, entityUpdate.entityId, entityIndices);
+  if (change.type === "SET") {
+    updateIndices(inMemTable, change.entity, updatedEntityRecord.entityIndices);
   } else {
-    updateIndices(inMemTable, entity._0, entityIndices);
+    deleteEntityFromIndices(inMemTable, change.entityId, updatedEntityRecord.entityIndices);
   }
-  set(inMemTable.table, entityUpdate.entityId, {
-        entityRow: match$1.entityRow,
-        entityIndices: entityIndices
-      });
+  set(inMemTable.table, change.entityId, updatedEntityRecord);
 }
 
 function rowToEntity(row) {
-  var match = row.entityRow;
-  if (match.TAG === "Updated") {
-    var entity = match._0.latest.entityUpdateAction;
-    if (typeof entity !== "object") {
-      return ;
-    } else {
-      return Caml_option.some(entity._0);
-    }
-  }
-  var entity$1 = match._0;
-  if (typeof entity$1 !== "object") {
-    return ;
-  } else {
-    return Caml_option.some(entity$1._0);
-  }
+  return row.latest;
 }
 
 function getUnsafe(inMemTable) {
   return function (key) {
-    return rowToEntity(inMemTable.table.dict[key]);
+    return inMemTable.table.dict[key].latest;
   };
 }
 
@@ -295,7 +258,7 @@ function addEmptyIndex(inMemTable, index) {
   var fieldName = TableIndices.Index.getFieldName(index);
   var relatedEntityIds = new Set();
   Belt_Array.forEach(Js_dict.values(inMemTable.table.dict), (function (row) {
-          var entity = rowToEntity(row);
+          var entity = row.latest;
           if (entity === undefined) {
             return ;
           }
@@ -340,9 +303,14 @@ function addIdToIndex(inMemTable, index, entityId) {
   }
 }
 
-function rows(inMemTable) {
-  return Belt_Array.map(Js_dict.values(inMemTable.table.dict), (function (v) {
-                return v.entityRow;
+function updates(inMemTable) {
+  return Belt_Array.keepMapU(Js_dict.values(inMemTable.table.dict), (function (v) {
+                var update = v.status;
+                if (typeof update !== "object") {
+                  return ;
+                } else {
+                  return update;
+                }
               }));
 }
 
@@ -384,7 +352,7 @@ var Entity = {
   getUnsafeOnIndex: getUnsafeOnIndex,
   addEmptyIndex: addEmptyIndex,
   addIdToIndex: addIdToIndex,
-  rows: rows,
+  updates: updates,
   values: values$1,
   clone: clone$1
 };

package/src/Internal.gen.ts

@@ -53,8 +53,6 @@ export type genericHandlerWithLoader<loader,handler,eventFilters> = {
   readonly preRegisterDynamicContracts?: boolean
 };
 
-export type blockEvent = { readonly number: number; readonly chainId: number };
-
 export type eventOptions<eventFilters> = {
   readonly wildcard?: boolean; 
   readonly eventFilters?: eventFilters; 

package/src/Internal.res

@@ -169,11 +169,8 @@ type eventItem = private {
   event: event,
 }
 
-@genType
-type blockEvent = {
-  number: int,
-  chainId: int,
-}
+// Opaque type to support both EVM and Fuel platforms
+type blockEvent
 
 type onBlockArgs = {
   block: blockEvent,
@@ -249,36 +246,22 @@ let fuelTransferParamsSchema = S.schema(s => {
 })
 
 type entity = private {id: string}
+type clickHouseSetUpdatesCache = {
+  tableName: string,
+  convertOrThrow: Change.t<entity> => Js.Json.t,
+}
 type genericEntityConfig<'entity> = {
   name: string,
   index: int,
   schema: S.t<'entity>,
   rowsSchema: S.t<array<'entity>>,
   table: Table.table,
-  entityHistory: EntityHistory.t<'entity>,
+  mutable clickHouseSetUpdatesCache?: clickHouseSetUpdatesCache,
+  mutable pgEntityHistoryCache?: EntityHistory.pgEntityHistory<'entity>,
 }
 type entityConfig = genericEntityConfig<entity>
 external fromGenericEntityConfig: genericEntityConfig<'entity> => entityConfig = "%identity"
 
-type enum
-type enumConfig<'enum> = {
-  name: string,
-  variants: array<'enum>,
-  schema: S.t<'enum>,
-  default: 'enum,
-}
-external fromGenericEnumConfig: enumConfig<'enum> => enumConfig<enum> = "%identity"
-
-let makeEnumConfig = (~name, ~variants) => {
-  name,
-  variants,
-  schema: S.enum(variants),
-  default: switch variants->Belt.Array.get(0) {
-  | Some(v) => v
-  | None => Js.Exn.raiseError("No variants defined for enum " ++ name)
-  },
-}
-
 type effectInput
 type effectOutput
 type effectContext = private {mutable cache: bool}
@@ -322,8 +305,8 @@ let makeCacheTable = (~effectName) => {
   Table.mkTable(
     cacheTablePrefix ++ effectName,
     ~fields=[
-      Table.mkField("id", Text, ~fieldSchema=S.string, ~isPrimaryKey=true),
-      Table.mkField("output", JsonB, ~fieldSchema=cacheOutputSchema, ~isNullable=true),
+      Table.mkField("id", String, ~fieldSchema=S.string, ~isPrimaryKey=true),
+      Table.mkField("output", Json, ~fieldSchema=cacheOutputSchema, ~isNullable=true),
     ],
   )
 }
@@ -331,9 +314,11 @@ let makeCacheTable = (~effectName) => {
 @genType.import(("./Types.ts", "Invalid"))
 type noEventFilters
 
+type checkpointId = float
+
 type reorgCheckpoint = {
   @as("id")
-  checkpointId: int,
+  checkpointId: float,
   @as("chain_id")
   chainId: int,
   @as("block_number")
@@ -342,13 +327,9 @@ type reorgCheckpoint = {
   blockHash: string,
 }
 
-type entityValueAtStartOfBatch<'entityType> =
-  | NotSet // The entity isn't in the DB yet
-  | AlreadySet('entityType)
-
-type updatedValue<'entityType> = {
-  latest: EntityHistory.entityUpdate<'entityType>,
-  history: array<EntityHistory.entityUpdate<'entityType>>,
+type inMemoryStoreEntityUpdate<'entity> = {
+  latestChange: Change.t<'entity>,
+  history: array<Change.t<'entity>>,
   // In the event of a rollback, some entity updates may have been
   // been affected by a rollback diff. If there was no rollback diff
   // this will always be false.
@@ -359,6 +340,7 @@ type updatedValue<'entityType> = {
   containsRollbackDiffChange: bool,
 }
 
-type inMemoryStoreRowEntity<'entityType> =
-  | Updated(updatedValue<'entityType>)
-  | InitialReadFromDb(entityValueAtStartOfBatch<'entityType>) // This means there is no change from the db.
+@unboxed
+type inMemoryStoreEntityStatus<'entity> =
+  | Updated(inMemoryStoreEntityUpdate<'entity>)
+  | Loaded // This means there is no change from the db.

package/src/Internal.res.js

@@ -3,10 +3,7 @@
 
 var Table = require("./db/Table.res.js");
 var $$BigInt = require("./bindings/BigInt.res.js");
-var Js_exn = require("rescript/lib/js/js_exn.js");
 var Address = require("./Address.res.js");
-var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Caml_option = require("rescript/lib/js/caml_option.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
 var fuelSupplyParamsSchema = S$RescriptSchema.schema(function (s) {
@@ -24,16 +21,6 @@ var fuelTransferParamsSchema = S$RescriptSchema.schema(function (s) {
             };
     });
 
-function makeEnumConfig(name, variants) {
-  var v = Belt_Array.get(variants, 0);
-  return {
-          name: name,
-          variants: variants,
-          schema: S$RescriptSchema.$$enum(variants),
-          default: v !== undefined ? Caml_option.valFromOption(v) : Js_exn.raiseError("No variants defined for enum " + name)
-        };
-}
-
 var cacheTablePrefix = "envio_effect_";
 
 var cacheOutputSchema = S$RescriptSchema.json(false);
@@ -47,14 +34,13 @@ var effectCacheItemRowsSchema = S$RescriptSchema.array(S$RescriptSchema.schema(f
 
 function makeCacheTable(effectName) {
   return Table.mkTable(cacheTablePrefix + effectName, undefined, [
-              Table.mkField("id", "TEXT", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
-              Table.mkField("output", "JSONB", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined)
+              Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined),
+              Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined)
             ]);
 }
 
 exports.fuelSupplyParamsSchema = fuelSupplyParamsSchema;
 exports.fuelTransferParamsSchema = fuelTransferParamsSchema;
-exports.makeEnumConfig = makeEnumConfig;
 exports.cacheTablePrefix = cacheTablePrefix;
 exports.cacheOutputSchema = cacheOutputSchema;
 exports.effectCacheItemRowsSchema = effectCacheItemRowsSchema;

package/src/Persistence.res

@@ -29,13 +29,24 @@ type initialState = {
   cleanRun: bool,
   cache: dict<effectCacheRecord>,
   chains: array<initialChainState>,
-  checkpointId: int,
+  checkpointId: Internal.checkpointId,
   // Needed to keep reorg detection logic between restarts
   reorgCheckpoints: array<Internal.reorgCheckpoint>,
 }
 
 type operator = [#">" | #"=" | #"<"]
 
+type updatedEffectCache = {
+  effect: Internal.effect,
+  items: array<Internal.effectCacheItem>,
+  shouldInitialize: bool,
+}
+
+type updatedEntity = {
+  entityConfig: Internal.entityConfig,
+  updates: array<Internal.inMemoryStoreEntityUpdate<Internal.entity>>,
+}
+
 type storage = {
   // Should return true if we already have persisted data
   // and we can skip initialization
@@ -45,7 +56,7 @@ type storage = {
   initialize: (
     ~chainConfigs: array<Config.chain>=?,
     ~entities: array<Internal.entityConfig>=?,
-    ~enums: array<Internal.enumConfig<Internal.enum>>=?,
+    ~enums: array<Table.enumConfig<Table.enum>>=?,
   ) => promise<initialState>,
   resumeInitialState: unit => promise<initialState>,
   @raises("StorageError")
@@ -77,10 +88,53 @@ type storage = {
   ) => promise<unit>,
   // This is to download cache from the database to .envio/cache
   dumpEffectCache: unit => promise<unit>,
+  // Execute raw SQL query
+  executeUnsafe: string => promise<unknown>,
+  // Check if entity history has rows
+  hasEntityHistoryRows: unit => promise<bool>,
+  // Update chain metadata
+  setChainMeta: dict<InternalTable.Chains.metaFields> => promise<unknown>,
+  // Prune old checkpoints
+  pruneStaleCheckpoints: (~safeCheckpointId: Internal.checkpointId) => promise<unit>,
+  // Prune stale entity history
+  pruneStaleEntityHistory: (
+    ~entityName: string,
+    ~entityIndex: int,
+    ~safeCheckpointId: Internal.checkpointId,
+  ) => promise<unit>,
+  // Get rollback target checkpoint
+  getRollbackTargetCheckpoint: (
+    ~reorgChainId: int,
+    ~lastKnownValidBlockNumber: int,
+  ) => promise<array<{"id": Internal.checkpointId}>>,
+  // Get rollback progress diff
+  getRollbackProgressDiff: (
+    ~rollbackTargetCheckpointId: Internal.checkpointId,
+  ) => promise<
+    array<{
+      "chain_id": int,
+      "events_processed_diff": string,
+      "new_progress_block_number": int,
+    }>,
+  >,
+  // Get rollback data for entity
+  getRollbackData: (
+    ~entityConfig: Internal.entityConfig,
+    ~rollbackTargetCheckpointId: Internal.checkpointId,
+  ) => promise<(array<{"id": string}>, array<unknown>)>,
+  // Write batch to storage
+  writeBatch: (
+    ~batch: Batch.t,
+    ~rawEvents: array<InternalTable.RawEvents.t>,
+    ~rollbackTargetCheckpointId: option<Internal.checkpointId>,
+    ~isInReorgThreshold: bool,
+    ~config: Config.t,
+    ~allEntities: array<Internal.entityConfig>,
+    ~updatedEffectsCache: array<updatedEffectCache>,
+    ~updatedEntities: array<updatedEntity>,
+  ) => promise<unit>,
 }
 
-exception StorageError({message: string, reason: exn})
-
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
@@ -89,38 +143,28 @@ type storageStatus =
 type t = {
   userEntities: array<Internal.entityConfig>,
   allEntities: array<Internal.entityConfig>,
-  allEnums: array<Internal.enumConfig<Internal.enum>>,
+  allEnums: array<Table.enumConfig<Table.enum>>,
   mutable storageStatus: storageStatus,
   mutable storage: storage,
-  // FIXME: This is temporary to move it library
-  // Should be a part of the storage interface and db agnostic
-  mutable sql: Postgres.sql,
 }
 
-let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t> = {
-  name: EntityHistory.RowAction.name,
-  variants: EntityHistory.RowAction.variants,
-  schema: EntityHistory.RowAction.schema,
-  default: SET,
-}
+exception StorageError({message: string, reason: exn})
 
 let make = (
   ~userEntities,
   // TODO: Should only pass userEnums and create internal config in runtime
   ~allEnums,
   ~storage,
-  ~sql,
 ) => {
   let allEntities = userEntities->Js.Array2.concat([InternalTable.DynamicContractRegistry.config])
   let allEnums =
-    allEnums->Js.Array2.concat([entityHistoryActionEnumConfig->Internal.fromGenericEnumConfig])
+    allEnums->Js.Array2.concat([EntityHistory.RowAction.config->Table.fromGenericEnumConfig])
   {
     userEntities,
     allEntities,
     allEnums,
     storageStatus: Unknown,
     storage,
-    sql,
   }
 }
 
@@ -197,32 +241,140 @@ let getInitializedState = persistence => {
   }
 }
 
-let setEffectCacheOrThrow = async (
+let writeBatch = (
   persistence,
-  ~effect: Internal.effect,
-  ~items,
-  ~invalidationsCount,
-) => {
+  ~batch,
+  ~config,
+  ~inMemoryStore: InMemoryStore.t,
+  ~isInReorgThreshold,
+) =>
   switch persistence.storageStatus {
   | Unknown
   | Initializing(_) =>
     Js.Exn.raiseError(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
-  | Ready({cache}) => {
-      let storage = persistence.storage
-      let effectName = effect.name
-      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
-      | Some(c) => c
-      | None => {
-          let c = {effectName, count: 0}
-          cache->Js.Dict.set(effectName, c)
-          c
-        }
+  | Ready({cache}) =>
+    let updatedEntities = persistence.allEntities->Belt.Array.keepMapU(entityConfig => {
+      let updates =
+        inMemoryStore
+        ->InMemoryStore.getInMemTable(~entityConfig)
+        ->InMemoryTable.Entity.updates
+      if updates->Utils.Array.isEmpty {
+        None
+      } else {
+        Some({entityConfig, updates})
       }
-      let initialize = effectCacheRecord.count === 0
-      await storage.setEffectCacheOrThrow(~effect, ~items, ~initialize)
-      effectCacheRecord.count =
-        effectCacheRecord.count + items->Js.Array2.length - invalidationsCount
-      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
-    }
+    })
+    persistence.storage.writeBatch(
+      ~batch,
+      ~rawEvents=inMemoryStore.rawEvents->InMemoryTable.values,
+      ~rollbackTargetCheckpointId=inMemoryStore.rollbackTargetCheckpointId,
+      ~isInReorgThreshold,
+      ~config,
+      ~allEntities=persistence.allEntities,
+      ~updatedEntities,
+      ~updatedEffectsCache={
+        inMemoryStore.effects
+        ->Js.Dict.keys
+        ->Belt.Array.keepMapU(effectName => {
+          let inMemTable = inMemoryStore.effects->Js.Dict.unsafeGet(effectName)
+          let {idsToStore, dict, effect, invalidationsCount} = inMemTable
+          switch idsToStore {
+          | [] => None
+          | ids => {
+              let items = Belt.Array.makeUninitializedUnsafe(ids->Belt.Array.length)
+              ids->Belt.Array.forEachWithIndex((index, id) => {
+                items->Js.Array2.unsafe_set(
+                  index,
+                  (
+                    {
+                      id,
+                      output: dict->Js.Dict.unsafeGet(id),
+                    }: Internal.effectCacheItem
+                  ),
+                )
+              })
+              Some({
+                let effectName = effect.name
+                let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(
+                  effectName,
+                ) {
+                | Some(c) => c
+                | None => {
+                    let c = {effectName, count: 0}
+                    cache->Js.Dict.set(effectName, c)
+                    c
+                  }
+                }
+                let shouldInitialize = effectCacheRecord.count === 0
+                effectCacheRecord.count =
+                  effectCacheRecord.count + items->Js.Array2.length - invalidationsCount
+                Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
+                {effect, items, shouldInitialize}
+              })
+            }
+          }
+        })
+      },
+    )
+  }
+
+let prepareRollbackDiff = async (
+  persistence: t,
+  ~rollbackTargetCheckpointId,
+  ~rollbackDiffCheckpointId,
+) => {
+  let inMemStore = InMemoryStore.make(
+    ~entities=persistence.allEntities,
+    ~rollbackTargetCheckpointId,
+  )
+
+  let deletedEntities = Js.Dict.empty()
+  let setEntities = Js.Dict.empty()
+
+  let _ =
+    await persistence.allEntities
+    ->Belt.Array.map(async entityConfig => {
+      let entityTable = inMemStore->InMemoryStore.getInMemTable(~entityConfig)
+
+      let (removedIdsResult, restoredEntitiesResult) = await persistence.storage.getRollbackData(
+        ~entityConfig,
+        ~rollbackTargetCheckpointId,
+      )
+
+      // Process removed IDs
+      removedIdsResult->Js.Array2.forEach(data => {
+        deletedEntities->Utils.Dict.push(entityConfig.name, data["id"])
+        entityTable->InMemoryTable.Entity.set(
+          Delete({
+            entityId: data["id"],
+            checkpointId: rollbackDiffCheckpointId,
+          }),
+          ~shouldSaveHistory=false,
+          ~containsRollbackDiffChange=true,
+        )
+      })
+
+      let restoredEntities = restoredEntitiesResult->S.parseOrThrow(entityConfig.rowsSchema)
+
+      // Process restored entities
+      restoredEntities->Belt.Array.forEach((entity: Internal.entity) => {
+        setEntities->Utils.Dict.push(entityConfig.name, entity.id)
+        entityTable->InMemoryTable.Entity.set(
+          Set({
+            entityId: entity.id,
+            checkpointId: rollbackDiffCheckpointId,
+            entity,
+          }),
+          ~shouldSaveHistory=false,
+          ~containsRollbackDiffChange=true,
+        )
+      })
+    })
+    ->Promise.all
+
+  {
+    "inMemStore": inMemStore,
+    "deletedEntities": deletedEntities,
+    "setEntities": setEntities,
   }
 }