envio 3.12.1 → 3.13.0-rc.0

package/src/IndexerLoop.res

@@ -45,6 +45,8 @@ let start = (state: IndexerState.t) => {
     launch(state, () => FinalizeBackfill.repairSchemaIndexes(state))
   }
 
+  state->IndexerState.bindScheduleProcessing(scheduleProcessing)
+
   scheduleFetch()
   scheduleProcessing()
 }

package/src/IndexerLoop.res.mjs

@@ -42,6 +42,7 @@ function start(state) {
       work();
     }
   }
+  IndexerState.bindScheduleProcessing(state, scheduleProcessing);
   scheduleFetch();
   scheduleProcessing();
 }

package/src/IndexerState.res

@@ -115,6 +115,10 @@ type t = {
   // waitForNewBlock waiter is bound to the old, pre-realtime source). A fetch
   // response or waiter carrying an older epoch than this is discarded.
   mutable epoch: int,
+  // The loop's one door in from outside it: IndexerLoop owns scheduling and
+  // wires this when it starts, so an event the loop can't see for itself can
+  // still make it re-evaluate. A no-op before then.
+  mutable scheduleProcessing: unit => unit,
   // None off the simulate path.
   simulateDeadInputTracker: option<SimulateDeadInputTracker.t>,
   // --- Metric counters, rendered by Metrics at scrape time. ---
@@ -141,6 +145,7 @@ let make = (
   ~chainStates: dict<ChainState.t>,
   ~isRealtime: bool,
   ~targetBufferSize=CrossChainState.calculateTargetBufferSize(),
+  ~holdRealtime=false,
   ~committedFrontier=Frontier.empty(),
   ~isDevelopmentMode=false,
   ~shouldUseTui=false,
@@ -180,11 +185,17 @@ let make = (
     chainMetaDirty: false,
     chainMetaThrottler,
     isProcessing: false,
-    crossChainState: CrossChainState.make(~chainStates, ~isRealtime, ~targetBufferSize),
+    crossChainState: CrossChainState.make(
+      ~chainStates,
+      ~isRealtime,
+      ~targetBufferSize,
+      ~holdRealtime,
+    ),
     indexerStartTime: Date.make(),
     indexerStartTimeRef: Performance.now(),
     rollbackState: NoRollback,
     lastPrunedAtMillis: Dict.make(),
+    scheduleProcessing: () => (),
     loadManager: LoadManager.make(),
     keepProcessAlive: isDevelopmentMode || shouldUseTui,
     exitAfterFirstEventBlock,
@@ -227,6 +238,9 @@ let makeFromDbState = (
   ~exitAfterFirstEventBlock=false,
   ~reducedPollingInterval=?,
   ~targetBufferSize=CrossChainState.calculateTargetBufferSize(),
+  // A process driving part of a split run waits for its supervisor before
+  // entering the reorg threshold or switching to realtime.
+  ~holdRealtime=false,
   ~onError,
   ~onExit=?,
 ) => {
@@ -274,6 +288,7 @@ let makeFromDbState = (
     ~chainStates,
     ~isRealtime,
     ~targetBufferSize,
+    ~holdRealtime,
     ~committedFrontier=initialState.checkpointFrontier,
     ~isDevelopmentMode,
     ~shouldUseTui,
@@ -500,11 +515,37 @@ let isFinalizingIndexes = (state: t) =>
   state.crossChainState->CrossChainState.isCaughtUp &&
     !(state.crossChainState->CrossChainState.isRealtime)
 
+// The FinalizingIndexes phase is the transition a held process waits on: it
+// ends with `ready_at` committed and the indexer realtime.
+let shouldFinalizeIndexes = (state: t) =>
+  state->isFinalizingIndexes && !(state.crossChainState->CrossChainState.isHoldingRealtime)
+
 let markCaughtUpIfSettled = (state: t) =>
   state.crossChainState->CrossChainState.markCaughtUpIfSettled
 
+let isReadyToEnterReorgThreshold = (state: t, ~batch) =>
+  state.crossChainState->CrossChainState.isReadyToEnterReorgThreshold(~batch)
+
+let bindScheduleProcessing = (state: t, scheduleProcessing) =>
+  state.scheduleProcessing = scheduleProcessing
+
+// A process still waiting on its supervisor owes the schema the indexes its
+// chains deferred, so reaching every end block doesn't make it done.
+let isHoldingRealtime = (state: t) => state.crossChainState->CrossChainState.isHoldingRealtime
+
+let hasArrivedAtHead = (state: t) => state.crossChainState->CrossChainState.hasArrivedAtHead
+
+let releaseRealtime = (state: t) => {
+  state.crossChainState->CrossChainState.releaseRealtime
+  // Every chain is parked at the head with no batch coming, so nothing would
+  // notice the hold is gone without a pass through processing.
+  state.scheduleProcessing()
+}
+
 let markReady = (state: t, ~readyAt) => state.crossChainState->CrossChainState.markReady(~readyAt)
 
+let reportFinished = (state: t) => state.crossChainState->CrossChainState.reportFinished
+
 let rollbackState = (state: t) => state.rollbackState
 let indexerStartTime = (state: t) => state.indexerStartTime
 let loadManager = (state: t) => state.loadManager
@@ -572,6 +613,7 @@ let toMetrics = (state: t): Metrics.t => {
     elapsedSeconds: state.indexerStartTimeRef->Performance.secondsSince,
     targetBufferSize: state.crossChainState->CrossChainState.targetBufferSize,
     isInReorgThreshold: state.crossChainState->CrossChainState.isInReorgThreshold,
+    hasArrivedAtHead: state.crossChainState->CrossChainState.hasArrivedAtHead,
     rollbackEnabled: state.config.shouldRollbackOnReorg,
     maxBatchSize: state.config.batchSize,
     preloadSeconds: state.preloadSeconds,

package/src/IndexerState.res.mjs

@@ -24,8 +24,9 @@ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js
 import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
 import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
 
-function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt, committedFrontierOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError, onExit) {
+function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt, holdRealtimeOpt, committedFrontierOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError, onExit) {
   let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
+  let holdRealtime = holdRealtimeOpt !== undefined ? holdRealtimeOpt : false;
   let committedFrontier = committedFrontierOpt !== undefined ? committedFrontierOpt : Frontier.empty();
   let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
   let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
@@ -54,7 +55,7 @@ function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt,
     chainMetaDirty: false,
     chainMetaThrottler: chainMetaThrottler,
     isProcessing: false,
-    crossChainState: CrossChainState.make(chainStates, isRealtime, targetBufferSize),
+    crossChainState: CrossChainState.make(chainStates, isRealtime, targetBufferSize, holdRealtime),
     rollbackState: "NoRollback",
     indexerStartTime: new Date(),
     indexerStartTimeRef: Performance.now(),
@@ -66,6 +67,7 @@ function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt,
     onExit: onExit,
     isStopped: false,
     epoch: 0,
+    scheduleProcessing: () => {},
     simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config),
     preloadSeconds: 0,
     processingSeconds: 0,
@@ -82,11 +84,12 @@ function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt,
   };
 }
 
-function makeFromDbState(config, persistence, initialState, registrationsByChainId, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, reducedPollingInterval, targetBufferSizeOpt, onError, onExit) {
+function makeFromDbState(config, persistence, initialState, registrationsByChainId, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, reducedPollingInterval, targetBufferSizeOpt, holdRealtimeOpt, onError, onExit) {
   let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
   let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
   let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
   let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
+  let holdRealtime = holdRealtimeOpt !== undefined ? holdRealtimeOpt : false;
   let isInReorgThreshold = initialState.cleanRun ? false : initialState.chains.some(resumedChainState => {
       let progressBlockNumber = resumedChainState.progressBlockNumber;
       let sourceBlockNumber = resumedChainState.sourceBlockNumber;
@@ -104,7 +107,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
     let chainConfig = ChainMap.get(config.chainMap, chainId);
     chainStates[resumedChainState.id] = ChainState.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, isRealtime, config, initialState.contractMapping, registrationsByChainId, reducedPollingInterval);
   });
-  let state = make(config, persistence, chainStates, isRealtime, targetBufferSize, initialState.checkpointFrontier, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError, onExit);
+  let state = make(config, persistence, chainStates, isRealtime, targetBufferSize, holdRealtime, initialState.checkpointFrontier, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError, onExit);
   CrossChainState.markCaughtUpOnResume(state.crossChainState);
   Utils.Dict.forEach(initialState.cache, param => {
     let count = param.count;
@@ -350,14 +353,47 @@ function isFinalizingIndexes(state) {
   }
 }
 
+function shouldFinalizeIndexes(state) {
+  if (isFinalizingIndexes(state)) {
+    return !CrossChainState.isHoldingRealtime(state.crossChainState);
+  } else {
+    return false;
+  }
+}
+
 function markCaughtUpIfSettled(state) {
   CrossChainState.markCaughtUpIfSettled(state.crossChainState);
 }
 
+function isReadyToEnterReorgThreshold(state, batch) {
+  return CrossChainState.isReadyToEnterReorgThreshold(state.crossChainState, batch);
+}
+
+function bindScheduleProcessing(state, scheduleProcessing) {
+  state.scheduleProcessing = scheduleProcessing;
+}
+
+function isHoldingRealtime(state) {
+  return CrossChainState.isHoldingRealtime(state.crossChainState);
+}
+
+function hasArrivedAtHead(state) {
+  return CrossChainState.hasArrivedAtHead(state.crossChainState);
+}
+
+function releaseRealtime(state) {
+  CrossChainState.releaseRealtime(state.crossChainState);
+  state.scheduleProcessing();
+}
+
 function markReady(state, readyAt) {
   CrossChainState.markReady(state.crossChainState, readyAt);
 }
 
+function reportFinished(state) {
+  CrossChainState.reportFinished(state.crossChainState);
+}
+
 function rollbackState(state) {
   return state.rollbackState;
 }
@@ -447,6 +483,7 @@ function toMetrics(state) {
     elapsedSeconds: Performance.secondsSince(state.indexerStartTimeRef),
     targetBufferSize: CrossChainState.targetBufferSize(state.crossChainState),
     isInReorgThreshold: CrossChainState.isInReorgThreshold(state.crossChainState),
+    hasArrivedAtHead: CrossChainState.hasArrivedAtHead(state.crossChainState),
     rollbackEnabled: state.config.shouldRollbackOnReorg,
     maxBatchSize: state.config.batchSize,
     preloadSeconds: state.preloadSeconds,
@@ -844,7 +881,14 @@ export {
   isRealtime,
   isFinalizingIndexes,
   markCaughtUpIfSettled,
+  isReadyToEnterReorgThreshold,
+  shouldFinalizeIndexes,
+  bindScheduleProcessing,
+  isHoldingRealtime,
+  hasArrivedAtHead,
+  releaseRealtime,
   markReady,
+  reportFinished,
   rollbackState,
   indexerStartTime,
   loadManager,

package/src/IndexerState.resi

@@ -16,6 +16,7 @@ let make: (
   ~chainStates: dict<ChainState.t>,
   ~isRealtime: bool,
   ~targetBufferSize: int=?,
+  ~holdRealtime: bool=?,
   ~committedFrontier: Frontier.t=?,
   ~isDevelopmentMode: bool=?,
   ~shouldUseTui: bool=?,
@@ -34,6 +35,7 @@ let makeFromDbState: (
   ~exitAfterFirstEventBlock: bool=?,
   ~reducedPollingInterval: int=?,
   ~targetBufferSize: int=?,
+  ~holdRealtime: bool=?,
   ~onError: ErrorHandling.t => unit,
   ~onExit: unit => unit=?,
 ) => t
@@ -93,7 +95,16 @@ let shouldSaveHistory: t => dict<bool>
 let isRealtime: t => bool
 let isFinalizingIndexes: t => bool
 let markCaughtUpIfSettled: t => unit
+let isReadyToEnterReorgThreshold: (t, ~batch: Batch.t) => bool
+let shouldFinalizeIndexes: t => bool
+// Wires the loop's way back in. IndexerLoop calls this as it starts.
+let bindScheduleProcessing: (t, unit => unit) => unit
+let isHoldingRealtime: t => bool
+let hasArrivedAtHead: t => bool
+// The supervisor's go-ahead for a process driving part of a split run.
+let releaseRealtime: t => unit
 let markReady: (t, ~readyAt: Date.t) => unit
+let reportFinished: t => unit
 let rollbackState: t => rollbackState
 let indexerStartTime: t => Date.t
 let loadManager: t => LoadManager.t