envio 3.1.0-rc.1 → 3.1.0

package/src/InMemoryTable.res.mjs

@@ -53,49 +53,41 @@ function make() {
   };
 }
 
-function resetButKeepLatestChanges(self) {
-  let init = {
-    latestEntityChangeById: {},
-    changesCount: 0,
-    prevEntityChanges: [],
-    indicesByEntityId: {},
-    fieldNameIndices: {}
-  };
-  return {
-    latestEntityChangeById: self.latestEntityChangeById,
-    changesCount: self.changesCount,
-    prevEntityChanges: init.prevEntityChanges,
-    indicesByEntityId: init.indicesByEntityId,
-    fieldNameIndices: init.fieldNameIndices
-  };
+function snapshotChanges(self, committedCheckpointId, upToCheckpointId) {
+  let changes = [];
+  let keptPrev = [];
+  self.prevEntityChanges.forEach(change => {
+    if (change.checkpointId > upToCheckpointId) {
+      keptPrev.push(change);
+    } else {
+      changes.push(change);
+    }
+  });
+  self.prevEntityChanges = keptPrev;
+  self.changesCount = self.changesCount - changes.length;
+  Utils.Dict.forEach(self.latestEntityChangeById, change => {
+    let checkpointId = change.checkpointId;
+    if (checkpointId > committedCheckpointId && checkpointId <= upToCheckpointId) {
+      changes.push(change);
+      return;
+    }
+  });
+  return changes;
 }
 
-function resetButKeepLoadedFromDbChanges(self) {
-  let latestEntityChangeById = {};
-  let keptCount = {
-    contents: 0
-  };
+function dropCommittedChanges(self, committedCheckpointId, keepLoadedFromDb) {
+  let keysToDelete = [];
   Utils.Dict.forEachWithKey(self.latestEntityChangeById, (change, key) => {
-    if (change.checkpointId === Internal.loadedFromDbCheckpointId) {
-      latestEntityChangeById[key] = change;
-      keptCount.contents = keptCount.contents + 1;
+    let checkpointId = change.checkpointId;
+    if (checkpointId <= committedCheckpointId && !(keepLoadedFromDb && checkpointId === Internal.loadedFromDbCheckpointId)) {
+      keysToDelete.push(key);
       return;
     }
   });
-  let init = {
-    latestEntityChangeById: {},
-    changesCount: 0,
-    prevEntityChanges: [],
-    indicesByEntityId: {},
-    fieldNameIndices: {}
-  };
-  return {
-    latestEntityChangeById: latestEntityChangeById,
-    changesCount: keptCount.contents,
-    prevEntityChanges: init.prevEntityChanges,
-    indicesByEntityId: init.indicesByEntityId,
-    fieldNameIndices: init.fieldNameIndices
-  };
+  keysToDelete.forEach(key => Utils.Dict.deleteInPlace(self.latestEntityChangeById, key));
+  self.changesCount = self.changesCount - keysToDelete.length;
+  self.indicesByEntityId = {};
+  self.fieldNameIndices = {};
 }
 
 function updateIndices(self, entity) {
@@ -259,8 +251,8 @@ let Entity = {
   getOrCreateEntityIndices: getOrCreateEntityIndices,
   makeIndicesSerializedToValue: makeIndicesSerializedToValue,
   make: make,
-  resetButKeepLatestChanges: resetButKeepLatestChanges,
-  resetButKeepLoadedFromDbChanges: resetButKeepLoadedFromDbChanges,
+  snapshotChanges: snapshotChanges,
+  dropCommittedChanges: dropCommittedChanges,
   updateIndices: updateIndices,
   deleteEntityFromIndices: deleteEntityFromIndices,
   set: set,

package/src/Internal.res

@@ -537,6 +537,9 @@ type effectArgs = {
   input: effectInput,
   context: effectContext,
   cacheKey: string,
+  // The processing checkpoint that referenced this effect; stamped on the
+  // in-memory cache entry so it's evicted once the checkpoint commits.
+  checkpointId: bigint,
 }
 type effectCacheItem = {id: string, output: effectOutput}
 type effectCacheStorageMeta = {

package/src/LoadLayer.res

@@ -89,10 +89,12 @@ let callEffect = (
 
   effect.handler(arg)
   ->Promise.thenResolve(output => {
-    inMemTable.dict->Dict.set(arg.cacheKey, output)
-    if arg.context.cache {
-      inMemTable.idsToStore->Array.push(arg.cacheKey)->ignore
-    }
+    inMemTable->InMemoryStore.setEffectOutput(
+      ~checkpointId=arg.checkpointId,
+      ~cacheKey=arg.cacheKey,
+      ~output,
+      ~shouldCache=arg.context.cache,
+    )
   })
   ->Utils.Promise.catchResolve(exn => {
     onError(~inputKey=arg.cacheKey, ~exn)
@@ -283,7 +285,7 @@ let loadEffect = (
         try {
           let output = dbEntity.output->S.parseOrThrow(outputSchema)
           idsFromCache->Utils.Set.add(dbEntity.id)->ignore
-          inMemTable.dict->Dict.set(dbEntity.id, output)
+          inMemTable->InMemoryStore.initEffectOutputFromDb(~cacheKey=dbEntity.id, ~output)
         } catch {
         | S.Raised(error) =>
           inMemTable.invalidationsCount = inMemTable.invalidationsCount + 1
@@ -334,8 +336,8 @@ let loadEffect = (
     ~load,
     ~shouldGroup,
     ~hasher=args => args.cacheKey,
-    ~getUnsafeInMemory=hash => inMemTable.dict->Dict.getUnsafe(hash),
-    ~hasInMemory=hash => inMemTable.dict->Dict.has(hash),
+    ~getUnsafeInMemory=hash => inMemTable->InMemoryStore.getEffectOutput(hash)->Option.getUnsafe,
+    ~hasInMemory=hash => inMemTable->InMemoryStore.getEffectOutput(hash)->Option.isSome,
     ~input=effectArgs,
   )
 }

package/src/LoadLayer.res.mjs

@@ -12,6 +12,7 @@ import * as TableIndices from "./TableIndices.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
@@ -55,13 +56,7 @@ function callEffect(effect, arg, inMemTable, timerRef, onError) {
     }
   }
   effect.prevCallStartTimerRef = timerRef;
-  return effect.handler(arg).then(output => {
-    inMemTable.dict[arg.cacheKey] = output;
-    if (arg.context.cache) {
-      inMemTable.idsToStore.push(arg.cacheKey);
-      return;
-    }
-  }).catch(exn => onError(arg.cacheKey, exn)).finally(() => {
+  return effect.handler(arg).then(output => InMemoryStore.setEffectOutput(inMemTable, arg.checkpointId, arg.cacheKey, output, arg.context.cache)).catch(exn => onError(arg.cacheKey, exn)).finally(() => {
     effect.activeCallsCount = effect.activeCallsCount - 1 | 0;
     Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, effectName, effect.activeCallsCount);
     let newTimer = Hrtime.makeTimer();
@@ -159,8 +154,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, inMemoryStore,
         try {
           let output = S$RescriptSchema.parseOrThrow(dbEntity.output, outputSchema);
           idsFromCache.add(dbEntity.id);
-          inMemTable.dict[dbEntity.id] = output;
-          return;
+          return InMemoryStore.initEffectOutputFromDb(inMemTable, dbEntity.id, output);
         } catch (raw_error) {
           let error = Primitive_exceptions.internalToException(raw_error);
           if (error.RE_EXN_ID === S$RescriptSchema.Raised) {
@@ -193,7 +187,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, inMemoryStore,
       return await executeWithRateLimit(effect, argsToCall, inMemTable, onError, false);
     }
   };
-  return LoadManager.call(loadManager, effectArgs, key, load, args => args.cacheKey, shouldGroup, hash => hash in inMemTable.dict, hash => inMemTable.dict[hash]);
+  return LoadManager.call(loadManager, effectArgs, key, load, args => args.cacheKey, shouldGroup, hash => Stdlib_Option.isSome(InMemoryStore.getEffectOutput(inMemTable, hash)), hash => InMemoryStore.getEffectOutput(inMemTable, hash));
 }
 
 function loadByField(loadManager, persistence, operator, entityConfig, inMemoryStore, fieldName, fieldValueSchema, shouldGroup, item, fieldValue) {

package/src/Main.res

@@ -312,6 +312,13 @@ let getGlobalIndexer = (): 'indexer => {
     )
   }
 
+  let onRollbackCommitFn = (callback: 'a) => {
+    HandlerRegister.throwIfFinishedRegistration(
+      ~methodName="~internalAndWillBeRemovedSoon_onRollbackCommit",
+    )
+    let _ = RollbackCommit.register(callback->(Utils.magic: 'a => RollbackCommit.callback))
+  }
+
   // Two-stage parse: first the ecosystem-specific outer schema unwraps the
   // wrapper (`block.number` / `block.height` / `slot`) and surfaces the
   // inner chunk as raw `unknown`; then the shared `blockRangeSchema`
@@ -453,8 +460,16 @@ let getGlobalIndexer = (): 'indexer => {
             "onEvent",
             "contractRegister",
             "onBlock",
+            "~internalAndWillBeRemovedSoon_onRollbackCommit",
+          ]
+        | Svm => [
+            "name",
+            "description",
+            "chainIds",
+            "chains",
+            "onSlot",
+            "~internalAndWillBeRemovedSoon_onRollbackCommit",
           ]
-        | Svm => ["name", "description", "chainIds", "chains", "onSlot"]
         }
         keysMemo := Some(keys)
         keys
@@ -477,6 +492,7 @@ let getGlobalIndexer = (): 'indexer => {
     | "onEvent" => onEventFn->Utils.magic
     | "contractRegister" => contractRegisterFn->Utils.magic
     | "onBlock" | "onSlot" => onBlockFn->Utils.magic
+    | "~internalAndWillBeRemovedSoon_onRollbackCommit" => onRollbackCommitFn->Utils.magic
     | _ =>
       JsError.throwWithMessage(
         `Field \`${prop}\` does not exist on \`indexer\`. Available fields: ${getKeys()->Array.join(
@@ -686,6 +702,12 @@ let start = async (
   | Some(patchConfig) => patchConfig(config, registrations)
   | None => config
   }
+  // The single fatal-error handler, shared by the store, ErrorExit, and the
+  // manager's catch.
+  let onError = (errHandler: ErrorHandling.t) => {
+    errHandler->ErrorHandling.log
+    NodeJs.process->NodeJs.exitWithCode(Failure)
+  }
   let ctx = {
     Ctx.registrations,
     config,
@@ -693,6 +715,9 @@ let start = async (
     inMemoryStore: InMemoryStore.make(
       ~entities=persistence.allEntities,
       ~committedCheckpointId=(persistence->Persistence.getInitializedState).checkpointId,
+      ~persistence,
+      ~config,
+      ~onError=exn => onError(exn->ErrorHandling.make(~msg="Failed writing batch to the database")),
     ),
   }
 
@@ -763,8 +788,12 @@ let start = async (
     ~isDevelopmentMode,
     ~shouldUseTui,
     ~exitAfterFirstEventBlock,
+    ~onError,
   )
-  let gsManager = globalState->GlobalStateManager.make
+  let gsManager =
+    globalState->GlobalStateManager.make(~onError=exn =>
+      onError(exn->ErrorHandling.make(~msg="Indexer has failed with an unexpected error"))
+    )
   if shouldUseTui {
     let _rerender = Tui.start(~getState=() => gsManager->GlobalStateManager.getState)
   }

package/src/Main.res.mjs

@@ -18,12 +18,14 @@ import * as PromClient from "prom-client";
 import Yargs from "yargs/yargs";
 import * as ChainFetcher from "./ChainFetcher.res.mjs";
 import * as ChainManager from "./ChainManager.res.mjs";
+import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as HandlerLoader from "./HandlerLoader.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as SourceManager from "./sources/SourceManager.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Helpers from "yargs/helpers";
+import * as RollbackCommit from "./RollbackCommit.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as HandlerRegister from "./HandlerRegister.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
@@ -254,6 +256,10 @@ function getGlobalIndexer() {
     let match = parseIdentityConfig(identityConfig);
     HandlerRegister.setContractRegister(match[0], match[1], handler, match[2], undefined);
   };
+  let onRollbackCommitFn = callback => {
+    HandlerRegister.throwIfFinishedRegistration("~internalAndWillBeRemovedSoon_onRollbackCommit");
+    RollbackCommit.register(callback);
+  };
   let extractRange = (filter, name, ecosystem) => {
     try {
       let inner = S$RescriptSchema.parseOrThrow(filter, ecosystem.onBlockFilterSchema);
@@ -340,7 +346,8 @@ function getGlobalIndexer() {
           "description",
           "chainIds",
           "chains",
-          "onSlot"
+          "onSlot",
+          "~internalAndWillBeRemovedSoon_onRollbackCommit"
         ];
         break;
     }
@@ -352,7 +359,8 @@ function getGlobalIndexer() {
         "chains",
         "onEvent",
         "contractRegister",
-        "onBlock"
+        "onBlock",
+        "~internalAndWillBeRemovedSoon_onRollbackCommit"
       ];
     }
     keysMemo.contents = keys;
@@ -375,6 +383,8 @@ function getGlobalIndexer() {
       case "onBlock" :
       case "onSlot" :
         return onBlockFn;
+      case "~internalAndWillBeRemovedSoon_onRollbackCommit" :
+        return onRollbackCommitFn;
       default:
         return Stdlib_JsError.throwWithMessage(`Field \`` + prop + `\` does not exist on \`indexer\`. Available fields: ` + getKeys().join(", ") + `.`);
     }
@@ -523,7 +533,11 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
       allEnums: config.allEnums
     }) : config;
   let config$2 = patchConfig !== undefined ? patchConfig(config$1, registrations) : config$1;
-  let ctx_inMemoryStore = InMemoryStore.make(persistence$1.allEntities, Persistence.getInitializedState(persistence$1).checkpointId);
+  let onError = errHandler => {
+    ErrorHandling.log(errHandler);
+    Process.exit(1);
+  };
+  let ctx_inMemoryStore = InMemoryStore.make(persistence$1.allEntities, Persistence.getInitializedState(persistence$1).checkpointId, persistence$1, config$2, exn => onError(ErrorHandling.make(exn, undefined, "Failed writing batch to the database")));
   let ctx = {
     registrations: registrations,
     config: config$2,
@@ -572,8 +586,8 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
     }, ctx, isDevelopmentMode);
   }
   let chainManager = ChainManager.makeFromDbState(Persistence.getInitializedState(persistence$1), config$2, registrations, undefined);
-  let globalState = GlobalState.make(ctx, chainManager, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock);
-  let gsManager = GlobalStateManager.make(globalState, undefined);
+  let globalState = GlobalState.make(ctx, chainManager, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
+  let gsManager = GlobalStateManager.make(globalState, exn => onError(ErrorHandling.make(exn, undefined, "Indexer has failed with an unexpected error")));
   if (shouldUseTui) {
     Tui.start(() => GlobalStateManager.getState(gsManager));
   }

package/src/Persistence.res

@@ -51,6 +51,9 @@ type updatedEffectCache = {
 type rollback = {
   targetCheckpointId: Internal.checkpointId,
   diffCheckpointId: Internal.checkpointId,
+  // Last valid block per chain affected by the rollback. Read by
+  // `RollbackCommit.fire` once the diff is durably written.
+  progressBlockNumberByChainId: dict<int>,
 }
 
 type updatedEntity = {
@@ -126,13 +129,15 @@ type storage = {
   // Write batch to storage
   writeBatch: (
     ~batch: Batch.t,
-    ~rawEvents: array<InternalTable.RawEvents.t>,
     ~rollback: option<rollback>,
     ~isInReorgThreshold: bool,
     ~config: Config.t,
     ~allEntities: array<Internal.entityConfig>,
     ~updatedEffectsCache: array<updatedEffectCache>,
     ~updatedEntities: array<updatedEntity>,
+    // Chain metadata stale since the last write, persisted in the same
+    // transaction so it never races the batch write.
+    ~chainMetaData: option<dict<InternalTable.Chains.metaFields>>,
   ) => promise<unit>,
   // Release any long-lived resources (e.g. the postgres connection pool) so
   // short-lived CLI commands like `db-migrate setup` can exit cleanly.