envio 3.1.0-rc.1 → 3.1.0

package/src/GlobalState.res

@@ -7,22 +7,8 @@ type rollbackState =
   | RollbackReady({eventsProcessedDiffByChain: dict<float>})
 
 module WriteThrottlers = {
-  type t = {
-    chainMetaData: Throttler.t,
-    pruneStaleEntityHistory: Throttler.t,
-  }
+  type t = {pruneStaleEntityHistory: Throttler.t}
   let make = (): t => {
-    let chainMetaData = {
-      let intervalMillis = Env.ThrottleWrites.chainMetadataIntervalMillis
-      let logger = Logging.createChild(
-        ~params={
-          "context": "Throttler for chain metadata writes",
-          "intervalMillis": intervalMillis,
-        },
-      )
-      Throttler.make(~intervalMillis, ~logger)
-    }
-
     let pruneStaleEntityHistory = {
       let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis
       let logger = Logging.createChild(
@@ -33,7 +19,7 @@ module WriteThrottlers = {
       )
       Throttler.make(~intervalMillis, ~logger)
     }
-    {chainMetaData, pruneStaleEntityHistory}
+    {pruneStaleEntityHistory: pruneStaleEntityHistory}
   }
 }
 
@@ -48,6 +34,8 @@ type t = {
   loadManager: LoadManager.t,
   keepProcessAlive: bool,
   exitAfterFirstEventBlock: bool,
+  // The single fatal-error handler ErrorExit delegates to.
+  onError: ErrorHandling.t => unit,
   //Initialized as 0, increments, when rollbacks occur to invalidate
   //responses based on the wrong stateId
   id: int,
@@ -59,6 +47,7 @@ let make = (
   ~isDevelopmentMode=false,
   ~shouldUseTui=false,
   ~exitAfterFirstEventBlock=false,
+  ~onError: ErrorHandling.t => unit,
 ) => {
   {
     ctx,
@@ -71,6 +60,7 @@ let make = (
     loadManager: LoadManager.make(),
     keepProcessAlive: isDevelopmentMode || shouldUseTui,
     exitAfterFirstEventBlock,
+    onError,
     id: 0,
   }
 }
@@ -145,11 +135,7 @@ type task =
   | Rollback
   | PruneStaleEntityHistory
 
-let updateChainMetadataTable = (
-  cm: ChainManager.t,
-  ~persistence: Persistence.t,
-  ~throttler: Throttler.t,
-) => {
+let updateChainMetadataTable = (cm: ChainManager.t, ~inMemoryStore: InMemoryStore.t) => {
   let chainsData: dict<InternalTable.Chains.metaFields> = Dict.make()
 
   cm.chainFetchers
@@ -166,10 +152,8 @@ let updateChainMetadataTable = (
     )
   })
 
-  //Don't await this set, it can happen in its own time
-  throttler->Throttler.schedule(() =>
-    persistence.storage.setChainMeta(chainsData)->Utils.Promise.ignoreValue
-  )
+  // Staged; the cycle folds the stale diff into the next batch write.
+  inMemoryStore->InMemoryStore.setChainMeta(chainsData)
 }
 
 /**
@@ -711,12 +695,17 @@ let actionReducer = (state: t, action: action) => {
           NoExit
         }
 
-    (
-      state,
+    // On exit, stop dispatching ProcessEventBatch: the flush is async and would
+    // otherwise keep processing further batches while it runs.
+    let tasks = switch shouldExit {
+    | ExitWithSuccess
+    | ExitWithError(_) => [UpdateChainMetaDataAndCheckForExit(shouldExit)]
+    | NoExit =>
       [UpdateChainMetaDataAndCheckForExit(shouldExit), ProcessEventBatch]->Array.concat(
         maybePruneEntityHistory,
-      ),
-    )
+      )
+    }
+    (state, tasks)
 
   | StartProcessingBatch => ({...state, currentlyProcessingBatch: true}, [])
   | StartFindingReorgDepth => ({...state, rollbackState: FindingReorgDepth}, [])
@@ -775,8 +764,7 @@ let actionReducer = (state: t, action: action) => {
       (state, [])
     }
   | ErrorExit(errHandler) =>
-    errHandler->ErrorHandling.log
-    NodeJs.process->NodeJs.exitWithCode(Failure)
+    state.onError(errHandler)
     (state, [])
   }
 }
@@ -902,23 +890,15 @@ let injectedTaskReducer = (
       state.writeThrottlers.pruneStaleEntityHistory->Throttler.schedule(runPrune)
 
     | UpdateChainMetaDataAndCheckForExit(shouldExit) =>
-      let {chainManager, writeThrottlers} = state
+      let {chainManager} = state
       switch shouldExit {
       | ExitWithSuccess =>
-        updateChainMetadataTable(
-          chainManager,
-          ~throttler=writeThrottlers.chainMetaData,
-          ~persistence=state.ctx.persistence,
-        )
+        updateChainMetadataTable(chainManager, ~inMemoryStore=state.ctx.inMemoryStore)
+        await state.ctx.inMemoryStore->InMemoryStore.flush
         dispatchAction(SuccessExit)
       | ExitWithError(msg) =>
         dispatchAction(ErrorExit(ErrorHandling.make(JsError.throwWithMessage(msg))))
-      | NoExit =>
-        updateChainMetadataTable(
-          chainManager,
-          ~throttler=writeThrottlers.chainMetaData,
-          ~persistence=state.ctx.persistence,
-        )->ignore
+      | NoExit => updateChainMetadataTable(chainManager, ~inMemoryStore=state.ctx.inMemoryStore)
       }
     | NextQuery(chainCheck) =>
       let fetchForChain = checkAndFetchForChain(
@@ -947,15 +927,13 @@ let injectedTaskReducer = (
 
         let batch =
           state.chainManager->ChainManager.createBatch(
-            ~committedCheckpointId=state.ctx.inMemoryStore.committedCheckpointId,
+            ~processedCheckpointId=state.ctx.inMemoryStore.processedCheckpointId,
             ~batchSizeTarget=state.ctx.config.batchSize,
             ~isRollback=isRollbackBatch,
           )
 
         let progressedChainsById = batch.progressedChainsById
 
-        let isInReorgThreshold = state.chainManager.isInReorgThreshold
-
         let isBelowReorgThreshold =
           !state.chainManager.isInReorgThreshold && state.ctx.config.shouldRollbackOnReorg
         let shouldEnterReorgThreshold =
@@ -982,11 +960,8 @@ let injectedTaskReducer = (
           if EventProcessing.allChainsEventsProcessedToEndblock(state.chainManager.chainFetchers) {
             Logging.info("All chains are caught up to end blocks.")
             if !state.keepProcessAlive {
-              updateChainMetadataTable(
-                state.chainManager,
-                ~persistence=state.ctx.persistence,
-                ~throttler=state.writeThrottlers.chainMetaData,
-              )
+              updateChainMetadataTable(state.chainManager, ~inMemoryStore=state.ctx.inMemoryStore)
+              await state.ctx.inMemoryStore->InMemoryStore.flush
               dispatchAction(SuccessExit)
             }
           }
@@ -1000,7 +975,6 @@ let injectedTaskReducer = (
           switch await EventProcessing.processEventBatch(
             ~batch,
             ~inMemoryStore,
-            ~isInReorgThreshold,
             ~loadManager=state.loadManager,
             ~ctx=state.ctx,
             ~chainFetchers=state.chainManager.chainFetchers,
@@ -1177,10 +1151,15 @@ let injectedTaskReducer = (
           }
         })
 
+        // Finish pending writes so committedCheckpointId reflects the db before
+        // computing the rollback diff against it.
+        await state.ctx.inMemoryStore->InMemoryStore.flush
+
         let diff = await state.ctx.inMemoryStore->InMemoryStore.prepareRollbackDiff(
           ~persistence=state.ctx.persistence,
           ~rollbackTargetCheckpointId,
           ~rollbackDiffCheckpointId=state.ctx.inMemoryStore.committedCheckpointId->BigInt.add(1n),
+          ~progressBlockNumberByChainId=newProgressBlockNumberPerChain,
         )
 
         let chainManager = {

package/src/GlobalState.res.mjs

@@ -32,20 +32,13 @@ import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_excep
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
 function make() {
-  let intervalMillis = Env.ThrottleWrites.chainMetadataIntervalMillis;
+  let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis;
   let logger = Logging.createChild({
-    context: "Throttler for chain metadata writes",
-    intervalMillis: intervalMillis
-  });
-  let chainMetaData = Throttler.make(intervalMillis, logger);
-  let intervalMillis$1 = Env.ThrottleWrites.pruneStaleDataIntervalMillis;
-  let logger$1 = Logging.createChild({
     context: "Throttler for pruning stale entity history data",
-    intervalMillis: intervalMillis$1
+    intervalMillis: intervalMillis
   });
-  let pruneStaleEntityHistory = Throttler.make(intervalMillis$1, logger$1);
+  let pruneStaleEntityHistory = Throttler.make(intervalMillis, logger);
   return {
-    chainMetaData: chainMetaData,
     pruneStaleEntityHistory: pruneStaleEntityHistory
   };
 }
@@ -54,7 +47,7 @@ let WriteThrottlers = {
   make: make
 };
 
-function make$1(ctx, chainManager, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt) {
+function make$1(ctx, chainManager, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError) {
   let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
   let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
   let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
@@ -69,6 +62,7 @@ function make$1(ctx, chainManager, isDevelopmentModeOpt, shouldUseTuiOpt, exitAf
     loadManager: LoadManager.make(),
     keepProcessAlive: isDevelopmentMode || shouldUseTui,
     exitAfterFirstEventBlock: exitAfterFirstEventBlock,
+    onError: onError,
     id: 0
   };
 }
@@ -89,6 +83,7 @@ function setChainManager(self, chainManager) {
     loadManager: self.loadManager,
     keepProcessAlive: self.keepProcessAlive,
     exitAfterFirstEventBlock: self.exitAfterFirstEventBlock,
+    onError: self.onError,
     id: self.id
   };
 }
@@ -102,7 +97,7 @@ function isPreparingRollback(state) {
   }
 }
 
-function updateChainMetadataTable(cm, persistence, throttler) {
+function updateChainMetadataTable(cm, inMemoryStore) {
   let chainsData = {};
   Belt_Array.forEach(ChainMap.values(cm.chainFetchers), cf => {
     chainsData[String(cf.chainConfig.id)] = {
@@ -112,7 +107,7 @@ function updateChainMetadataTable(cm, persistence, throttler) {
       _is_hyper_sync: SourceManager.getActiveSource(cf.sourceManager).poweredByHyperSync
     };
   });
-  Throttler.schedule(throttler, () => persistence.storage.setChainMeta(chainsData));
+  InMemoryStore.setChainMeta(inMemoryStore, chainsData);
 }
 
 function updateProgressedChains(chainManager, batch, ctx) {
@@ -299,6 +294,7 @@ function validatePartitionQueryResponse(state, partitionQueryResponse) {
   let nextState_loadManager = state.loadManager;
   let nextState_keepProcessAlive = state.keepProcessAlive;
   let nextState_exitAfterFirstEventBlock = state.exitAfterFirstEventBlock;
+  let nextState_onError = state.onError;
   let nextState_id = state.id;
   let nextState = {
     ctx: nextState_ctx,
@@ -311,6 +307,7 @@ function validatePartitionQueryResponse(state, partitionQueryResponse) {
     loadManager: nextState_loadManager,
     keepProcessAlive: nextState_keepProcessAlive,
     exitAfterFirstEventBlock: nextState_exitAfterFirstEventBlock,
+    onError: nextState_onError,
     id: nextState_id
   };
   let rollbackWithReorgDetectedBlockNumber;
@@ -394,6 +391,7 @@ function validatePartitionQueryResponse(state, partitionQueryResponse) {
       loadManager: nextState_loadManager,
       keepProcessAlive: nextState_keepProcessAlive,
       exitAfterFirstEventBlock: nextState_exitAfterFirstEventBlock,
+      onError: nextState_onError,
       id: nextState_id + 1 | 0
     },
     ["Rollback"]
@@ -491,6 +489,7 @@ function submitPartitionQueryResponse(state, newItems, newItemsWithDcs, knownHei
   let nextState_loadManager = state.loadManager;
   let nextState_keepProcessAlive = state.keepProcessAlive;
   let nextState_exitAfterFirstEventBlock = state.exitAfterFirstEventBlock;
+  let nextState_onError = state.onError;
   let nextState_id = state.id;
   let nextState = {
     ctx: nextState_ctx,
@@ -503,6 +502,7 @@ function submitPartitionQueryResponse(state, newItems, newItemsWithDcs, knownHei
     loadManager: nextState_loadManager,
     keepProcessAlive: nextState_keepProcessAlive,
     exitAfterFirstEventBlock: nextState_exitAfterFirstEventBlock,
+    onError: nextState_onError,
     id: nextState_id
   };
   return [
@@ -569,6 +569,7 @@ function updateChainFetcher(chainFetcherUpdate, state, chain) {
       loadManager: state.loadManager,
       keepProcessAlive: state.keepProcessAlive,
       exitAfterFirstEventBlock: state.exitAfterFirstEventBlock,
+      onError: state.onError,
       id: state.id
     },
     []
@@ -606,6 +607,7 @@ function onEnterReorgThreshold(state) {
     loadManager: state.loadManager,
     keepProcessAlive: state.keepProcessAlive,
     exitAfterFirstEventBlock: state.exitAfterFirstEventBlock,
+    onError: state.onError,
     id: state.id
   };
 }
@@ -626,6 +628,7 @@ function actionReducer(state, action) {
             loadManager: state.loadManager,
             keepProcessAlive: state.keepProcessAlive,
             exitAfterFirstEventBlock: state.exitAfterFirstEventBlock,
+            onError: state.onError,
             id: state.id
           },
           []
@@ -643,6 +646,7 @@ function actionReducer(state, action) {
             loadManager: state.loadManager,
             keepProcessAlive: state.keepProcessAlive,
             exitAfterFirstEventBlock: state.exitAfterFirstEventBlock,
+            onError: state.onError,
             id: state.id
           },
           []
@@ -708,6 +712,7 @@ function actionReducer(state, action) {
         let state_loadManager = state.loadManager;
         let state_keepProcessAlive = state.keepProcessAlive;
         let state_exitAfterFirstEventBlock = state.exitAfterFirstEventBlock;
+        let state_onError = state.onError;
         let state_id = state.id;
         let state$1 = {
           ctx: state_ctx,
@@ -720,6 +725,7 @@ function actionReducer(state, action) {
           loadManager: state_loadManager,
           keepProcessAlive: state_keepProcessAlive,
           exitAfterFirstEventBlock: state_exitAfterFirstEventBlock,
+          onError: state_onError,
           id: state_id
         };
         if (shouldEnterReorgThreshold) {
@@ -758,6 +764,7 @@ function actionReducer(state, action) {
         let state_loadManager$1 = state.loadManager;
         let state_keepProcessAlive$1 = state.keepProcessAlive;
         let state_exitAfterFirstEventBlock$1 = state.exitAfterFirstEventBlock;
+        let state_onError$1 = state.onError;
         let state_id$1 = state.id;
         let state$2 = {
           ctx: state_ctx$1,
@@ -770,6 +777,7 @@ function actionReducer(state, action) {
           loadManager: state_loadManager$1,
           keepProcessAlive: state_keepProcessAlive$1,
           exitAfterFirstEventBlock: state_exitAfterFirstEventBlock$1,
+          onError: state_onError$1,
           id: state_id$1
         };
         let shouldExit = EventProcessing.allChainsEventsProcessedToEndblock(state_chainManager$1.chainFetchers) ? (Logging.info("All chains are caught up to end blocks."), state_keepProcessAlive$1 ? "NoExit" : "ExitWithSuccess") : (
@@ -784,15 +792,25 @@ function actionReducer(state, action) {
                 _0: "No events found between startBlock and chain head. Cannot auto-detect endBlock."
               }) : "NoExit"
           );
-        return [
-          state$2,
-          [
-            {
+        let tasks;
+        tasks = typeof shouldExit !== "object" ? (
+            shouldExit === "ExitWithSuccess" ? [{
+                  TAG: "UpdateChainMetaDataAndCheckForExit",
+                  _0: shouldExit
+                }] : [
+                {
+                  TAG: "UpdateChainMetaDataAndCheckForExit",
+                  _0: shouldExit
+                },
+                "ProcessEventBatch"
+              ].concat(maybePruneEntityHistory)
+          ) : [{
               TAG: "UpdateChainMetaDataAndCheckForExit",
               _0: shouldExit
-            },
-            "ProcessEventBatch"
-          ].concat(maybePruneEntityHistory)
+            }];
+        return [
+          state$2,
+          tasks
         ];
       case "FindReorgDepth" :
         return [
@@ -811,6 +829,7 @@ function actionReducer(state, action) {
             loadManager: state.loadManager,
             keepProcessAlive: state.keepProcessAlive,
             exitAfterFirstEventBlock: state.exitAfterFirstEventBlock,
+            onError: state.onError,
             id: state.id
           },
           ["Rollback"]
@@ -854,6 +873,7 @@ function actionReducer(state, action) {
             loadManager: state.loadManager,
             keepProcessAlive: state.keepProcessAlive,
             exitAfterFirstEventBlock: state.exitAfterFirstEventBlock,
+            onError: state.onError,
             id: state.id
           },
           [{
@@ -862,8 +882,7 @@ function actionReducer(state, action) {
             }]
         ];
       case "ErrorExit" :
-        ErrorHandling.log(action._0);
-        Process.exit(1);
+        state.onError(action._0);
         return [
           state,
           []
@@ -884,6 +903,7 @@ function actionReducer(state, action) {
             loadManager: state.loadManager,
             keepProcessAlive: state.keepProcessAlive,
             exitAfterFirstEventBlock: state.exitAfterFirstEventBlock,
+            onError: state.onError,
             id: state.id
           },
           [
@@ -916,6 +936,7 @@ function invalidatedActionReducer(state, action) {
               loadManager: state.loadManager,
               keepProcessAlive: state.keepProcessAlive,
               exitAfterFirstEventBlock: state.exitAfterFirstEventBlock,
+              onError: state.onError,
               id: state.id
             },
             ["Rollback"]
@@ -982,9 +1003,8 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
           let match = state.rollbackState;
           let isRollbackBatch;
           isRollbackBatch = typeof match !== "object" ? false : match.TAG === "RollbackReady";
-          let batch = ChainManager.createBatch(state.chainManager, state.ctx.inMemoryStore.committedCheckpointId, state.ctx.config.batchSize, isRollbackBatch);
+          let batch = ChainManager.createBatch(state.chainManager, state.ctx.inMemoryStore.processedCheckpointId, state.ctx.config.batchSize, isRollbackBatch);
           let progressedChainsById = batch.progressedChainsById;
-          let isInReorgThreshold = state.chainManager.isInReorgThreshold;
           let isBelowReorgThreshold = !state.chainManager.isInReorgThreshold && state.ctx.config.shouldRollbackOnReorg;
           let shouldEnterReorgThreshold = isBelowReorgThreshold && ChainMap.values(state.chainManager.chainFetchers).every(chainFetcher => {
             let chainAfterBatch = progressedChainsById[chainFetcher.fetchState.chainId];
@@ -997,7 +1017,8 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
             if (EventProcessing.allChainsEventsProcessedToEndblock(state.chainManager.chainFetchers)) {
               Logging.info("All chains are caught up to end blocks.");
               if (!state.keepProcessAlive) {
-                updateChainMetadataTable(state.chainManager, state.ctx.persistence, state.writeThrottlers.chainMetaData);
+                updateChainMetadataTable(state.chainManager, state.ctx.inMemoryStore);
+                await InMemoryStore.flush(state.ctx.inMemoryStore);
                 return dispatchAction("SuccessExit");
               } else {
                 return;
@@ -1016,7 +1037,7 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
           InMemoryStore.setBatchDcs(inMemoryStore, batch);
           let res;
           try {
-            res = await EventProcessing.processEventBatch(batch, inMemoryStore, isInReorgThreshold, state.loadManager, state.ctx, state.chainManager.chainFetchers);
+            res = await EventProcessing.processEventBatch(batch, inMemoryStore, state.loadManager, state.ctx, state.chainManager.chainFetchers);
           } catch (raw_exn) {
             let exn = Primitive_exceptions.internalToException(raw_exn);
             let errHandler = ErrorHandling.make(exn, undefined, "A top level unexpected error occurred during processing");
@@ -1129,7 +1150,8 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
                   safeCheckpointTracking: safeCheckpointTracking !== undefined ? SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber) : undefined
                 };
               });
-              let diff$1 = await InMemoryStore.prepareRollbackDiff(state.ctx.inMemoryStore, state.ctx.persistence, rollbackTargetCheckpointId, state.ctx.inMemoryStore.committedCheckpointId + 1n);
+              await InMemoryStore.flush(state.ctx.inMemoryStore);
+              let diff$1 = await InMemoryStore.prepareRollbackDiff(state.ctx.inMemoryStore, state.ctx.persistence, rollbackTargetCheckpointId, state.ctx.inMemoryStore.committedCheckpointId + 1n, newProgressBlockNumberPerChain);
               let init = state.chainManager;
               let chainManager_isInReorgThreshold = init.isInReorgThreshold;
               let chainManager_isRealtime = init.isRealtime;
@@ -1198,7 +1220,6 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
           processPartitionQueryResponse(state, task._0, dispatchAction);
           return;
         case "UpdateChainMetaDataAndCheckForExit" :
-          let writeThrottlers = state.writeThrottlers;
           let chainManager$1 = state.chainManager;
           let shouldExit = task._0;
           if (typeof shouldExit === "object") {
@@ -1207,12 +1228,12 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
               _0: ErrorHandling.make(Stdlib_JsError.throwWithMessage(shouldExit._0), undefined, undefined)
             });
           }
-          if (shouldExit === "ExitWithSuccess") {
-            updateChainMetadataTable(chainManager$1, state.ctx.persistence, writeThrottlers.chainMetaData);
-            return dispatchAction("SuccessExit");
+          if (shouldExit !== "ExitWithSuccess") {
+            return updateChainMetadataTable(chainManager$1, state.ctx.inMemoryStore);
           }
-          updateChainMetadataTable(chainManager$1, state.ctx.persistence, writeThrottlers.chainMetaData);
-          return;
+          updateChainMetadataTable(chainManager$1, state.ctx.inMemoryStore);
+          await InMemoryStore.flush(state.ctx.inMemoryStore);
+          return dispatchAction("SuccessExit");
       }
     }
   };

package/src/GlobalStateManager.res

@@ -1,5 +1,3 @@
-@val external setImmediate: (unit => unit) => unit = "setImmediate"
-
 module type State = {
   type t
   type action
@@ -41,7 +39,7 @@ module MakeManager = (S: State) => {
   }
   and dispatchTask = (self, task: S.task) => {
     let stateId = self.state->S.getId
-    setImmediate(() => {
+    NodeJs.setImmediate(() => {
       if stateId !== self.state->S.getId {
         Logging.info("Invalidated task discarded")
       } else {