envio 3.0.2 → 3.1.0-rc.0

package/src/Main.res

@@ -24,7 +24,6 @@ type state =
       chains: array<chainData>,
       indexerStartTime: Date.t,
       isPreRegisteringDynamicContracts: bool,
-      isUnorderedMultichainMode: bool,
       rollbackOnReorg: bool,
     })
 
@@ -51,7 +50,6 @@ let stateSchema = S.union([
     indexerStartTime: s.matches(S.datetime(S.string)),
     // Keep the field, since Dev Console expects it to be present
     isPreRegisteringDynamicContracts: false,
-    isUnorderedMultichainMode: s.matches(S.bool),
     rollbackOnReorg: s.matches(S.bool),
   })),
 ])
@@ -692,6 +690,10 @@ let start = async (
     Ctx.registrations,
     config,
     persistence,
+    inMemoryStore: InMemoryStore.make(
+      ~entities=persistence.allEntities,
+      ~committedCheckpointId=(persistence->Persistence.getInitializedState).checkpointId,
+    ),
   }
 
   let envioVersion = Utils.EnvioPackage.value.version
@@ -744,10 +746,6 @@ let start = async (
             indexerStartTime: state.indexerStartTime,
             isPreRegisteringDynamicContracts: false,
             rollbackOnReorg: ctx.config.shouldRollbackOnReorg,
-            isUnorderedMultichainMode: switch ctx.config.multichain {
-            | Unordered => true
-            | Ordered => false
-            },
           })
         }
       }

package/src/Main.res.mjs

@@ -19,6 +19,7 @@ import Yargs from "yargs/yargs";
 import * as ChainFetcher from "./ChainFetcher.res.mjs";
 import * as ChainManager from "./ChainManager.res.mjs";
 import * as HandlerLoader from "./HandlerLoader.res.mjs";
+import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as SourceManager from "./sources/SourceManager.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
@@ -58,7 +59,6 @@ let stateSchema = S$RescriptSchema.union([
     chains: s.m(S$RescriptSchema.array(chainDataSchema)),
     indexerStartTime: s.m(S$RescriptSchema.datetime(S$RescriptSchema.string, undefined)),
     isPreRegisteringDynamicContracts: false,
-    isUnorderedMultichainMode: s.m(S$RescriptSchema.bool),
     rollbackOnReorg: s.m(S$RescriptSchema.bool)
   }))
 ]);
@@ -501,19 +501,34 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
   let match = await HandlerLoader.registerAllHandlers(configWithoutRegistrations);
   let registrations = match[1];
   let config = match[0];
-  let config$1;
-  if (isTest) {
-    let newrecord = {...config};
-    newrecord.shouldRollbackOnReorg = false;
-    config$1 = newrecord;
-  } else {
-    config$1 = config;
-  }
+  let config$1 = isTest ? ({
+      name: config.name,
+      description: config.description,
+      handlers: config.handlers,
+      contractHandlers: config.contractHandlers,
+      shouldRollbackOnReorg: false,
+      shouldSaveFullHistory: config.shouldSaveFullHistory,
+      storage: config.storage,
+      chainMap: config.chainMap,
+      defaultChain: config.defaultChain,
+      ecosystem: config.ecosystem,
+      enableRawEvents: config.enableRawEvents,
+      maxAddrInPartition: config.maxAddrInPartition,
+      batchSize: config.batchSize,
+      lowercaseAddresses: config.lowercaseAddresses,
+      isDev: config.isDev,
+      userEntitiesByName: config.userEntitiesByName,
+      userEntities: config.userEntities,
+      allEntities: config.allEntities,
+      allEnums: config.allEnums
+    }) : config;
   let config$2 = patchConfig !== undefined ? patchConfig(config$1, registrations) : config$1;
+  let ctx_inMemoryStore = InMemoryStore.make(persistence$1.allEntities, Persistence.getInitializedState(persistence$1).checkpointId);
   let ctx = {
     registrations: registrations,
     config: config$2,
-    persistence: persistence$1
+    persistence: persistence$1,
+    inMemoryStore: ctx_inMemoryStore
   };
   let envioVersion = Utils.EnvioPackage.value.version;
   Prometheus.Info.set(envioVersion);
@@ -546,21 +561,17 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
           numAddresses: FetchState.numAddresses(cf.fetchState)
         };
       });
-      let match = config$2.multichain;
-      let tmp;
-      tmp = match !== "ordered";
       return {
         status: "active",
         envioVersion: envioVersion,
         chains: chains,
         indexerStartTime: state.indexerStartTime,
         isPreRegisteringDynamicContracts: false,
-        isUnorderedMultichainMode: tmp,
         rollbackOnReorg: config$2.shouldRollbackOnReorg
       };
     }, ctx, isDevelopmentMode);
   }
-  let chainManager = ChainManager.makeFromDbState(Persistence.getInitializedState(persistence$1), config$2, registrations);
+  let chainManager = ChainManager.makeFromDbState(Persistence.getInitializedState(persistence$1), config$2, registrations, undefined);
   let globalState = GlobalState.make(ctx, chainManager, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock);
   let gsManager = GlobalStateManager.make(globalState, undefined);
   if (shouldUseTui) {

package/src/Persistence.res

@@ -48,9 +48,14 @@ type updatedEffectCache = {
   shouldInitialize: bool,
 }
 
+type rollback = {
+  targetCheckpointId: Internal.checkpointId,
+  diffCheckpointId: Internal.checkpointId,
+}
+
 type updatedEntity = {
   entityConfig: Internal.entityConfig,
-  updates: array<Internal.inMemoryStoreEntityUpdate<Internal.entity>>,
+  changes: array<Change.t<Internal.entity>>,
 }
 
 type storage = {
@@ -122,7 +127,7 @@ type storage = {
   writeBatch: (
     ~batch: Batch.t,
     ~rawEvents: array<InternalTable.RawEvents.t>,
-    ~rollbackTargetCheckpointId: option<Internal.checkpointId>,
+    ~rollback: option<rollback>,
     ~isInReorgThreshold: bool,
     ~config: Config.t,
     ~allEntities: array<Internal.entityConfig>,
@@ -263,133 +268,3 @@ let getInitializedState = persistence => {
   | Ready(initialState) => initialState
   }
 }
-
-let writeBatch = (
-  persistence,
-  ~batch,
-  ~config,
-  ~inMemoryStore: InMemoryStore.t,
-  ~isInReorgThreshold,
-) =>
-  switch persistence.storageStatus {
-  | Unknown
-  | Initializing(_) =>
-    JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
-  | Ready({cache}) =>
-    let updatedEntities = persistence.allEntities->Belt.Array.keepMap(entityConfig => {
-      let updates =
-        inMemoryStore
-        ->InMemoryStore.getInMemTable(~entityConfig)
-        ->InMemoryTable.Entity.updates
-      if updates->Utils.Array.isEmpty {
-        None
-      } else {
-        Some({entityConfig, updates})
-      }
-    })
-    persistence.storage.writeBatch(
-      ~batch,
-      ~rawEvents=inMemoryStore.rawEvents->InMemoryTable.values,
-      ~rollbackTargetCheckpointId=inMemoryStore.rollbackTargetCheckpointId,
-      ~isInReorgThreshold,
-      ~config,
-      ~allEntities=persistence.allEntities,
-      ~updatedEntities,
-      ~updatedEffectsCache={
-        let acc = []
-        inMemoryStore.effects->Utils.Dict.forEach(inMemTable => {
-          let {idsToStore, dict, effect, invalidationsCount} = inMemTable
-          switch idsToStore {
-          | [] => ()
-          | ids =>
-            let items = Belt.Array.makeUninitializedUnsafe(ids->Belt.Array.length)
-            ids->Belt.Array.forEachWithIndex((index, id) => {
-              items->Array.setUnsafe(
-                index,
-                (
-                  {
-                    id,
-                    output: dict->Dict.getUnsafe(id),
-                  }: Internal.effectCacheItem
-                ),
-              )
-            })
-            let effectName = effect.name
-            let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
-            | Some(c) => c
-            | None =>
-              let c = {effectName, count: 0}
-              cache->Dict.set(effectName, c)
-              c
-            }
-            let shouldInitialize = effectCacheRecord.count === 0
-            effectCacheRecord.count =
-              effectCacheRecord.count + items->Array.length - invalidationsCount
-            Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
-            acc->Array.push({effect, items, shouldInitialize})->ignore
-          }
-        })
-        acc
-      },
-    )
-  }
-
-let prepareRollbackDiff = async (
-  persistence: t,
-  ~rollbackTargetCheckpointId,
-  ~rollbackDiffCheckpointId,
-) => {
-  let inMemStore = InMemoryStore.make(
-    ~entities=persistence.allEntities,
-    ~rollbackTargetCheckpointId,
-  )
-
-  let deletedEntities = Dict.make()
-  let setEntities = Dict.make()
-
-  let _ = await persistence.allEntities
-  ->Belt.Array.map(async entityConfig => {
-    let entityTable = inMemStore->InMemoryStore.getInMemTable(~entityConfig)
-
-    let (removedIdsResult, restoredEntitiesResult) = await persistence.storage.getRollbackData(
-      ~entityConfig,
-      ~rollbackTargetCheckpointId,
-    )
-
-    // Process removed IDs
-    removedIdsResult->Array.forEach(data => {
-      deletedEntities->Utils.Dict.push(entityConfig.name, data["id"])
-      entityTable->InMemoryTable.Entity.set(
-        Delete({
-          entityId: data["id"],
-          checkpointId: rollbackDiffCheckpointId,
-        }),
-        ~shouldSaveHistory=false,
-        ~containsRollbackDiffChange=true,
-      )
-    })
-
-    let restoredEntities = restoredEntitiesResult->S.parseOrThrow(entityConfig.rowsSchema)
-
-    // Process restored entities
-    restoredEntities->Belt.Array.forEach((entity: Internal.entity) => {
-      setEntities->Utils.Dict.push(entityConfig.name, entity.id)
-      entityTable->InMemoryTable.Entity.set(
-        Set({
-          entityId: entity.id,
-          checkpointId: rollbackDiffCheckpointId,
-          entity,
-        }),
-        ~shouldSaveHistory=false,
-        ~containsRollbackDiffChange=true,
-      )
-    })
-  })
-  ->Promise.all
-
-  {
-    "inMemStore": inMemStore,
-    "deletedEntities": deletedEntities,
-    "setEntities": setEntities,
-  }
-}

package/src/Persistence.res.mjs

@@ -1,16 +1,10 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
-import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
 import * as Logging from "./Logging.res.mjs";
-import * as Belt_Array from "@rescript/runtime/lib/es6/Belt_Array.js";
-import * as Prometheus from "./Prometheus.res.mjs";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
-import * as InMemoryStore from "./InMemoryStore.res.mjs";
-import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
-import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 let StorageError = /* @__PURE__ */Primitive_exceptions.create("Persistence.StorageError");
@@ -122,106 +116,11 @@ function getInitializedState(persistence) {
   }
 }
 
-function writeBatch(persistence, batch, config, inMemoryStore, isInReorgThreshold) {
-  let match = persistence.storageStatus;
-  if (typeof match !== "object") {
-    return Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`);
-  }
-  if (match.TAG === "Initializing") {
-    return Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`);
-  }
-  let cache = match._0.cache;
-  let updatedEntities = Belt_Array.keepMap(persistence.allEntities, entityConfig => {
-    let updates = InMemoryTable.Entity.updates(InMemoryStore.getInMemTable(inMemoryStore, entityConfig));
-    if (Utils.$$Array.isEmpty(updates)) {
-      return;
-    } else {
-      return {
-        entityConfig: entityConfig,
-        updates: updates
-      };
-    }
-  });
-  let acc = [];
-  return persistence.storage.writeBatch(batch, InMemoryTable.values(inMemoryStore.rawEvents), inMemoryStore.rollbackTargetCheckpointId, isInReorgThreshold, config, persistence.allEntities, (Utils.Dict.forEach(inMemoryStore.effects, inMemTable => {
-    let idsToStore = inMemTable.idsToStore;
-    let invalidationsCount = inMemTable.invalidationsCount;
-    let effect = inMemTable.effect;
-    let dict = inMemTable.dict;
-    if (idsToStore.length === 0) {
-      return;
-    }
-    let items = new Array(idsToStore.length);
-    Belt_Array.forEachWithIndex(idsToStore, (index, id) => {
-      items[index] = {
-        id: id,
-        output: dict[id]
-      };
-    });
-    let effectName = effect.name;
-    let c = cache[effectName];
-    let effectCacheRecord;
-    if (c !== undefined) {
-      effectCacheRecord = c;
-    } else {
-      let c$1 = {
-        effectName: effectName,
-        count: 0
-      };
-      cache[effectName] = c$1;
-      effectCacheRecord = c$1;
-    }
-    let shouldInitialize = effectCacheRecord.count === 0;
-    effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
-    Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
-    acc.push({
-      effect: effect,
-      items: items,
-      shouldInitialize: shouldInitialize
-    });
-  }), acc), updatedEntities);
-}
-
-async function prepareRollbackDiff(persistence, rollbackTargetCheckpointId, rollbackDiffCheckpointId) {
-  let inMemStore = InMemoryStore.make(persistence.allEntities, rollbackTargetCheckpointId);
-  let deletedEntities = {};
-  let setEntities = {};
-  await Promise.all(Belt_Array.map(persistence.allEntities, async entityConfig => {
-    let entityTable = InMemoryStore.getInMemTable(inMemStore, entityConfig);
-    let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
-    match[0].forEach(data => {
-      Utils.Dict.push(deletedEntities, entityConfig.name, data.id);
-      InMemoryTable.Entity.set(entityTable, {
-        type: "DELETE",
-        entityId: data.id,
-        checkpointId: rollbackDiffCheckpointId
-      }, false, true);
-    });
-    let restoredEntities = S$RescriptSchema.parseOrThrow(match[1], entityConfig.rowsSchema);
-    return Belt_Array.forEach(restoredEntities, entity => {
-      Utils.Dict.push(setEntities, entityConfig.name, entity.id);
-      InMemoryTable.Entity.set(entityTable, {
-        type: "SET",
-        entityId: entity.id,
-        entity: entity,
-        checkpointId: rollbackDiffCheckpointId
-      }, false, true);
-    });
-  }));
-  return {
-    inMemStore: inMemStore,
-    deletedEntities: deletedEntities,
-    setEntities: setEntities
-  };
-}
-
 export {
   StorageError,
   make,
   init,
   getInitializedStorageOrThrow,
   getInitializedState,
-  writeBatch,
-  prepareRollbackDiff,
 }
-/* Utils Not a pure module */
+/* Config Not a pure module */

package/src/PgStorage.res

@@ -717,7 +717,7 @@ let rec writeBatch = async (
   ~batch: Batch.t,
   ~rawEvents,
   ~pgSchema,
-  ~rollbackTargetCheckpointId,
+  ~rollback: option<Persistence.rollback>,
   ~isInReorgThreshold,
   ~config: Config.t,
   ~allEntities: array<Internal.entityConfig>,
@@ -745,14 +745,55 @@ let rec writeBatch = async (
       ~items=rawEvents,
     )
 
-    let setEntities = updatedEntities->Belt.Array.map(({entityConfig, updates}) => {
+    let setEntities = updatedEntities->Belt.Array.map(({entityConfig, changes}) => {
       let entitiesToSet = []
       let idsToDelete = []
 
-      updates->Array.forEach(row => {
-        switch row {
-        | {latestChange: Set({entity})} => entitiesToSet->Belt.Array.push(entity)
-        | {latestChange: Delete({entityId})} => idsToDelete->Belt.Array.push(entityId)
+      // The rollback-diff change is written to the entity table only, never the
+      // history table; when present it is an id's oldest change.
+      let diffCheckpointId = rollback->Option.map(r => r.diffCheckpointId)
+
+      // History batches, populated only when saving history.
+      let batchSetUpdates = []
+      let batchDeleteEntityIds = []
+      let batchDeleteCheckpointIds = []
+      let idsWithDiff = Utils.Set.make()
+
+      // Single pass over the change log: track each id's latest change (the last
+      // one seen) and, when saving history, fan every non-diff change out to the
+      // history-table batches.
+      let latestChangeById = Dict.make()
+      let orderedIds = []
+      changes->Belt.Array.forEach(change => {
+        let entityId = change->Change.getEntityId
+        if latestChangeById->Utils.Dict.dangerouslyGetNonOption(entityId)->Option.isNone {
+          orderedIds->Belt.Array.push(entityId)
+        }
+        latestChangeById->Dict.set(entityId, change)
+        if shouldSaveHistory {
+          if Some(change->Change.getCheckpointId) === diffCheckpointId {
+            idsWithDiff->Utils.Set.add(entityId)->ignore
+          } else {
+            switch change {
+            | Delete({entityId, checkpointId}) =>
+              batchDeleteEntityIds->Belt.Array.push(entityId)->ignore
+              batchDeleteCheckpointIds->Belt.Array.push(checkpointId)->ignore
+            | Set(_) => batchSetUpdates->Belt.Array.push(change)->ignore
+            }
+          }
+        }
+      })
+
+      let backfillHistoryIds = Utils.Set.make()
+      orderedIds->Belt.Array.forEach(entityId => {
+        switch latestChangeById->Dict.getUnsafe(entityId) {
+        | Set({entity}) => entitiesToSet->Belt.Array.push(entity)
+        | Delete({entityId}) => idsToDelete->Belt.Array.push(entityId)
+        }
+
+        // An id needs a history backfill iff none of its changes is the diff.
+        if shouldSaveHistory && !(idsWithDiff->Utils.Set.has(entityId)) {
+          backfillHistoryIds->Utils.Set.add(entityId)->ignore
         }
       })
 
@@ -766,34 +807,6 @@ let rec writeBatch = async (
           let promises = []
 
           if shouldSaveHistory {
-            let backfillHistoryIds = Utils.Set.make()
-            let batchSetUpdates = []
-            // Use unnest approach
-            let batchDeleteCheckpointIds = []
-            let batchDeleteEntityIds = []
-
-            updates->Array.forEach(update => {
-              switch update {
-              | {history, containsRollbackDiffChange} =>
-                history->Array.forEach(
-                  (change: Change.t<'a>) => {
-                    if !containsRollbackDiffChange {
-                      backfillHistoryIds->Utils.Set.add(change->Change.getEntityId)->ignore
-                    }
-                    switch change {
-                    | Delete({entityId}) => {
-                        batchDeleteEntityIds->Belt.Array.push(entityId)->ignore
-                        batchDeleteCheckpointIds
-                        ->Belt.Array.push(change->Change.getCheckpointId)
-                        ->ignore
-                      }
-                    | Set(_) => batchSetUpdates->Array.push(change)->ignore
-                    }
-                  },
-                )
-              }
-            })
-
             if backfillHistoryIds->Utils.Set.size !== 0 {
               // This must run before updating entity or entity history tables
               await EntityHistory.backfillHistory(
@@ -913,8 +926,8 @@ let rec writeBatch = async (
     //In the event of a rollback, rollback all meta tables based on the given
     //valid event identifier, where all rows created after this eventIdentifier should
     //be deleted
-    let rollbackTables = switch rollbackTargetCheckpointId {
-    | Some(rollbackTargetCheckpointId) =>
+    let rollbackTables = switch rollback {
+    | Some({targetCheckpointId: rollbackTargetCheckpointId}) =>
       Some(
         sql => {
           let promises = allEntities->Array.map(entityConfig => {
@@ -1026,7 +1039,7 @@ let rec writeBatch = async (
       ~rawEvents,
       ~batch,
       ~pgSchema,
-      ~rollbackTargetCheckpointId,
+      ~rollback,
       ~isInReorgThreshold,
       ~config,
       ~setEffectCacheOrThrow,
@@ -1619,7 +1632,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
   let writeBatchMethod = async (
     ~batch,
     ~rawEvents,
-    ~rollbackTargetCheckpointId,
+    ~rollback,
     ~isInReorgThreshold,
     ~config,
     ~allEntities,
@@ -1665,7 +1678,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       ~batch,
       ~rawEvents,
       ~pgSchema,
-      ~rollbackTargetCheckpointId,
+      ~rollback,
       ~isInReorgThreshold,
       ~config,
       ~allEntities,
@@ -1789,7 +1802,11 @@ let makeStorageFromEnv = (
                 secret: Env.Hasura.secret,
               },
               ~pgSchema,
-              ~tableNames,
+              ~tableConfigs=tableNames->Array.map(tableName => {
+                Hasura.tableName,
+                description: None,
+                columnDescriptions: dict{},
+              }),
             )->Promise.catch(err => {
               Logging.errorWithExn(
                 err->Utils.prettifyExn,