envio 3.2.0 → 3.3.0-alpha.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.2.0",
+  "version": "3.3.0-alpha.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -42,7 +42,6 @@
   "dependencies": {
     "@clickhouse/client": "1.17.0",
     "@elastic/ecs-pino-format": "1.4.0",
-    "@envio-dev/hyperfuel-client": "1.2.2",
     "@fuel-ts/crypto": "0.96.1",
     "@fuel-ts/errors": "0.96.1",
     "@fuel-ts/hasher": "0.96.1",
@@ -70,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.2.0",
-    "envio-linux-x64-musl": "3.2.0",
-    "envio-linux-arm64": "3.2.0",
-    "envio-darwin-x64": "3.2.0",
-    "envio-darwin-arm64": "3.2.0"
+    "envio-linux-x64": "3.3.0-alpha.0",
+    "envio-linux-x64-musl": "3.3.0-alpha.0",
+    "envio-linux-arm64": "3.3.0-alpha.0",
+    "envio-darwin-x64": "3.3.0-alpha.0",
+    "envio-darwin-arm64": "3.3.0-alpha.0"
   }
 }

package/src/Batch.res

@@ -34,10 +34,8 @@ type t = {
   checkpointEventsProcessed: array<int>,
 }
 
-let hasReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
-  fetchStates
-  ->ChainMap.values
-  ->Array.some(fetchState => {
+let hasReadyItem = (fetchStates: array<FetchState.t>) => {
+  fetchStates->Array.some(fetchState => {
     fetchState->FetchState.isActivelyIndexing && fetchState->FetchState.hasReadyItem
   })
 }
@@ -72,7 +70,7 @@ let getProgressedChainsById = {
   }
 
   (
-    ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
+    ~chainsBeforeBatch: dict<chainBeforeBatch>,
     ~batchSizePerChain: dict<int>,
     ~progressBlockNumberPerChain: dict<int>,
   ) => {
@@ -83,7 +81,7 @@ let getProgressedChainsById = {
     // - Accumulate registered dynamic contracts to store in the db
     // - Trigger onBlock pointer update
     chainsBeforeBatch
-    ->ChainMap.values
+    ->Dict.valuesToArray
     ->Array.forEach(chainBeforeBatch => {
       let fetchState = chainBeforeBatch.fetchState
 
@@ -166,15 +164,15 @@ let addReorgCheckpoints = (
 
 let prepareBatch = (
   ~checkpointIdBeforeBatch,
-  ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
+  ~chainsBeforeBatch: dict<chainBeforeBatch>,
   ~batchSizeTarget,
   ~isInReorgThreshold,
 ) => {
   let preparedFetchStates =
     chainsBeforeBatch
-    ->ChainMap.values
+    ->Dict.valuesToArray
     ->Array.map(chainBeforeBatch => chainBeforeBatch.fetchState)
-    ->FetchState.sortForUnorderedBatch(~batchSizeTarget)
+    ->FetchState.sortForBatch(~batchSizeTarget)
 
   let chainIdx = ref(0)
   let preparedNumber = preparedFetchStates->Array.length
@@ -202,7 +200,9 @@ let prepareBatch = (
         ~fromItem=0,
       )
     let chainBeforeBatch =
-      chainsBeforeBatch->ChainMap.get(ChainMap.Chain.makeUnsafe(~chainId=fetchState.chainId))
+      chainsBeforeBatch
+      ->Utils.Dict.dangerouslyGetByIntNonOption(fetchState.chainId)
+      ->Option.getUnsafe
 
     let prevBlockNumber = ref(chainBeforeBatch.progressBlockNumber)
     if chainBatchSize > 0 {
@@ -243,10 +243,7 @@ let prepareBatch = (
         } else {
           let lastIndex = checkpointEventsProcessed->Array.length - 1
           checkpointEventsProcessed
-          ->Array.setUnsafe(
-            lastIndex,
-            checkpointEventsProcessed->Array.getUnsafe(lastIndex) + 1,
-          )
+          ->Array.setUnsafe(lastIndex, checkpointEventsProcessed->Array.getUnsafe(lastIndex) + 1)
           ->ignore
         }
 
@@ -301,7 +298,7 @@ let prepareBatch = (
 
 let make = (
   ~checkpointIdBeforeBatch,
-  ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
+  ~chainsBeforeBatch: dict<chainBeforeBatch>,
   ~batchSizeTarget,
   ~isInReorgThreshold,
 ) => {

package/src/Batch.res.mjs

@@ -1,12 +1,11 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
-import * as ChainMap from "./ChainMap.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
 import * as ReorgDetection from "./ReorgDetection.res.mjs";
 
 function hasReadyItem(fetchStates) {
-  return ChainMap.values(fetchStates).some(fetchState => {
+  return fetchStates.some(fetchState => {
     if (FetchState.isActivelyIndexing(fetchState)) {
       return FetchState.hasReadyItem(fetchState);
     } else {
@@ -30,7 +29,7 @@ function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAft
 
 function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressBlockNumberPerChain) {
   let progressedChainsById = {};
-  ChainMap.values(chainsBeforeBatch).forEach(chainBeforeBatch => {
+  Object.values(chainsBeforeBatch).forEach(chainBeforeBatch => {
     let fetchState = chainBeforeBatch.fetchState;
     let progressBlockNumber = progressBlockNumberPerChain[fetchState.chainId.toString()];
     let progressBlockNumberAfterBatch = progressBlockNumber !== undefined ? progressBlockNumber : chainBeforeBatch.progressBlockNumber;
@@ -71,7 +70,7 @@ function addReorgCheckpoints(prevCheckpointId, reorgDetection, fromBlockExclusiv
 }
 
 function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget, isInReorgThreshold) {
-  let preparedFetchStates = FetchState.sortForUnorderedBatch(ChainMap.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
+  let preparedFetchStates = FetchState.sortForBatch(Object.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
   let chainIdx = 0;
   let preparedNumber = preparedFetchStates.length;
   let totalBatchSize = 0;
@@ -87,7 +86,7 @@ function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarge
   while (totalBatchSize < batchSizeTarget && chainIdx < preparedNumber) {
     let fetchState = preparedFetchStates[chainIdx];
     let chainBatchSize = FetchState.getReadyItemsCount(fetchState, batchSizeTarget - totalBatchSize, 0);
-    let chainBeforeBatch = ChainMap.get(chainsBeforeBatch, ChainMap.Chain.makeUnsafe(fetchState.chainId));
+    let chainBeforeBatch = chainsBeforeBatch[fetchState.chainId];
     let prevBlockNumber = chainBeforeBatch.progressBlockNumber;
     if (chainBatchSize > 0) {
       for (let idx = 0, idx_finish = chainBatchSize - 1; idx <= idx_finish; ++idx) {

package/src/BatchProcessing.res

@@ -0,0 +1,199 @@
+// The single batch-processing loop. Re-enters fetch/rollback through the
+// injected schedule* effects.
+
+// Yield to the end of the current tick so fetch responses that resolved this
+// tick land before the processing loop builds its first batch (coalescing).
+@inline
+let yieldTick = () => Promise.make((resolve, _) => NodeJs.setImmediate(() => resolve()))
+
+// The single processing loop. Runs batches back-to-back while there's work and
+// no reorg is being resolved; exits when idle so producers (fetch, rollback) can
+// re-kick it. `state.isProcessing` guarantees one instance. The loop
+// decides whether to keep going by inspecting state after each batch, rather than
+// from a return value of processNextBatch.
+let rec startProcessing = async (state: IndexerState.t, ~scheduleFetch, ~scheduleRollback) => {
+  if !(state->IndexerState.isProcessing) && !(state->IndexerState.isStopped) {
+    state->IndexerState.beginProcessing
+    // FIXME: Needed only for test determinism. The mocks resolve several fetch
+    // responses synchronously in one tick; this yield lets them all land before
+    // the first createBatch so they coalesce into one batch (matching the old
+    // setImmediate model). In production responses arrive on separate network
+    // ticks and never co-arrive, so this never coalesces anything real — remove
+    // it later to avoid an unnecessary setImmediate per processing burst.
+    await yieldTick()
+    // Seeded true so the first batch always runs (it handles the caught-up exit
+    // even when there's nothing to process, eg resuming fully-synced state).
+    // Keep looping only while the last batch actually processed something: an
+    // empty/idle batch records nothing, so the counter stays put and the loop
+    // exits, leaving producers (fetch, rollback) to re-kick it.
+    let hasMoreWork = ref(true)
+    while (
+      hasMoreWork.contents &&
+      !(state->IndexerState.isStopped) &&
+      !(state->IndexerState.isResolvingReorg)
+    ) {
+      let processedBatchesBefore = state->IndexerState.processedBatchesCount
+      switch await processNextBatch(state, ~scheduleFetch) {
+      | exception exn =>
+        IndexerState.errorExit(state, exn->ErrorHandling.make(~msg=IndexerState.unexpectedErrorMsg))
+      | () => hasMoreWork := state->IndexerState.processedBatchesCount > processedBatchesBefore
+      }
+    }
+    state->IndexerState.endProcessing
+
+    // A reorg detected mid-batch parks the rollback until the loop is idle.
+    // Hand off now that no batch is in flight.
+    if state->IndexerState.isResolvingReorg {
+      scheduleRollback()
+    }
+  }
+}
+
+and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
+  // The reorg-threshold and queue updates below advance the chain states for the
+  // next round, but the current batch is created and processed against the chain
+  // states as they are now.
+  let isInReorgThresholdBeforeUpdate = state->IndexerState.isInReorgThreshold
+  let isRealtimeBeforeUpdate = state->IndexerState.isRealtime
+
+  let isRollbackBatch = switch state->IndexerState.rollbackState {
+  | RollbackReady(_) => true
+  | _ => false
+  }
+
+  let batch =
+    state->IndexerState.createBatch(
+      ~processedCheckpointId=state->IndexerState.processedCheckpointId,
+      ~batchSizeTarget=(state->IndexerState.config).batchSize,
+      ~isRollback=isRollbackBatch,
+    )
+
+  let progressedChainsById = batch.progressedChainsById
+
+  let isBelowReorgThreshold =
+    !isInReorgThresholdBeforeUpdate && (state->IndexerState.config).shouldRollbackOnReorg
+  let shouldEnterReorgThreshold =
+    isBelowReorgThreshold &&
+    state
+    ->IndexerState.chainStates
+    ->Dict.valuesToArray
+    ->Array.every(cs => {
+      let fetchState = switch progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
+        (cs->ChainState.fetchState).chainId,
+      ) {
+      | Some(chainAfterBatch) => chainAfterBatch.fetchState
+      | None => cs->ChainState.fetchState
+      }
+      fetchState->FetchState.isReadyToEnterReorgThreshold
+    })
+
+  if shouldEnterReorgThreshold {
+    IndexerState.enterReorgThreshold(state)
+  }
+
+  if progressedChainsById->Utils.Dict.isEmpty {
+    if shouldEnterReorgThreshold {
+      scheduleFetch()
+    }
+
+    // When resuming from persisted state, all events may already be processed.
+    if EventProcessing.allChainsEventsProcessedToEndblock(state->IndexerState.chainStates) {
+      Logging.info("All chains are caught up to end blocks.")
+      if !(state->IndexerState.keepProcessAlive) {
+        await ExitOnCaughtUp.run(state)
+      }
+    }
+  } else {
+    // The batch was created from pre-threshold fetch states, so advanceAfterBatch
+    // applies blockLag when crossing the threshold; enterReorgThreshold already
+    // covered the non-progressed chain states.
+    state
+    ->IndexerState.chainStates
+    ->Utils.Dict.forEach(cs =>
+      cs->ChainState.advanceAfterBatch(~batch, ~enteringReorgThreshold=shouldEnterReorgThreshold)
+    )
+    // Kick the next fetch round before awaiting the batch. A response that
+    // lands mid-batch commits only fetch-frontier fields (buffer, knownHeight),
+    // while applyBatchProgress below commits only progress fields, so the two
+    // concurrent writes are disjoint and neither clobbers the other.
+    scheduleFetch()
+
+    state->InMemoryStore.setBatchDcs(~batch)
+
+    // An exception here propagates to startProcessing's catch, the single error
+    // boundary for the loop. The Error case is not an exception, so it's handled
+    // here to preserve the handler's user-facing message.
+    switch await EventProcessing.processEventBatch(
+      ~batch,
+      ~indexerState=state,
+      ~loadManager=state->IndexerState.loadManager,
+      ~persistence=state->IndexerState.persistence,
+      ~config=state->IndexerState.config,
+      ~chainStates=state->IndexerState.chainStates,
+    ) {
+    | Error(errHandler) => IndexerState.errorExit(state, errHandler)
+    | Ok() =>
+      state->IndexerState.recordProcessedBatch
+
+      if state->IndexerState.isResolvingReorg {
+        // A reorg landed while this batch was processing. Apply its progress so
+        // the rollback diff is computed against up-to-date chain progress, but
+        // don't reset rollback state or evaluate exit. The loop exit hands off
+        // to rollback.
+        state->IndexerState.applyBatchProgress(~batch)
+      } else {
+        // Can safely reset rollback state, since overwrite is not possible.
+        state->IndexerState.clearRollback
+        state->IndexerState.applyBatchProgress(~batch)
+
+        if !isRealtimeBeforeUpdate && state->IndexerState.isRealtime {
+          // Catching up just flipped the chain to realtime, which changes the
+          // active source for waitForNewBlock (eg sync -> live). The waiter that
+          // parked during backfill is bound to the old source; bump the epoch to
+          // invalidate it and kick a fresh fetch that parks on the realtime source.
+          state->IndexerState.invalidateInflight
+          scheduleFetch()
+        }
+
+        let allCaughtUp = EventProcessing.allChainsEventsProcessedToEndblock(
+          state->IndexerState.chainStates,
+        )
+        if allCaughtUp {
+          Logging.info("All chains are caught up to end blocks.")
+        }
+
+        if allCaughtUp && !(state->IndexerState.keepProcessAlive) {
+          await ExitOnCaughtUp.run(state)
+        } else if (
+          // In auto-exit mode, error if all chains reached head with no events found
+          !allCaughtUp &&
+          state->IndexerState.exitAfterFirstEventBlock &&
+          state
+          ->IndexerState.chainStates
+          ->Dict.valuesToArray
+          ->Array.every(cs =>
+            cs->ChainState.isProgressAtHead && (cs->ChainState.fetchState).endBlock->Option.isNone
+          )
+        ) {
+          IndexerState.errorExit(
+            state,
+            ErrorHandling.make(
+              Utils.Error.make(
+                "No events found between startBlock and chain head. Cannot auto-detect endBlock.",
+              ),
+            ),
+          )
+        } else {
+          ChainMetadata.stage(state)
+          if (
+            state
+            ->IndexerState.config
+            ->Config.shouldPruneHistory(~isInReorgThreshold=state->IndexerState.isInReorgThreshold)
+          ) {
+            state->PruneStaleHistory.schedule
+          }
+        }
+      }
+    }
+  }
+}

package/src/BatchProcessing.res.mjs

@@ -0,0 +1,125 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as Config from "./Config.res.mjs";
+import * as Logging from "./Logging.res.mjs";
+import * as ChainState from "./ChainState.res.mjs";
+import * as FetchState from "./FetchState.res.mjs";
+import * as IndexerState from "./IndexerState.res.mjs";
+import * as ChainMetadata from "./ChainMetadata.res.mjs";
+import * as ErrorHandling from "./ErrorHandling.res.mjs";
+import * as InMemoryStore from "./InMemoryStore.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as ExitOnCaughtUp from "./ExitOnCaughtUp.res.mjs";
+import * as EventProcessing from "./EventProcessing.res.mjs";
+import * as PruneStaleHistory from "./PruneStaleHistory.res.mjs";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+function yieldTick() {
+  return new Promise((resolve, param) => {
+    setImmediate(() => resolve());
+  });
+}
+
+async function processNextBatch(state, scheduleFetch) {
+  let isInReorgThresholdBeforeUpdate = IndexerState.isInReorgThreshold(state);
+  let isRealtimeBeforeUpdate = IndexerState.isRealtime(state);
+  let match = IndexerState.rollbackState(state);
+  let isRollbackBatch;
+  isRollbackBatch = typeof match !== "object" ? false : match.TAG === "RollbackReady";
+  let batch = IndexerState.createBatch(state, IndexerState.processedCheckpointId(state), IndexerState.config(state).batchSize, isRollbackBatch);
+  let progressedChainsById = batch.progressedChainsById;
+  let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
+  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => {
+    let chainAfterBatch = progressedChainsById[ChainState.fetchState(cs).chainId];
+    return FetchState.isReadyToEnterReorgThreshold(chainAfterBatch !== undefined ? chainAfterBatch.fetchState : ChainState.fetchState(cs));
+  });
+  if (shouldEnterReorgThreshold) {
+    IndexerState.enterReorgThreshold(state);
+  }
+  if (Utils.Dict.isEmpty(progressedChainsById)) {
+    if (shouldEnterReorgThreshold) {
+      scheduleFetch();
+    }
+    if (EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state))) {
+      Logging.info("All chains are caught up to end blocks.");
+      if (!IndexerState.keepProcessAlive(state)) {
+        return await ExitOnCaughtUp.run(state);
+      } else {
+        return;
+      }
+    } else {
+      return;
+    }
+  }
+  Utils.Dict.forEach(IndexerState.chainStates(state), cs => ChainState.advanceAfterBatch(cs, batch, shouldEnterReorgThreshold));
+  scheduleFetch();
+  InMemoryStore.setBatchDcs(state, batch);
+  let errHandler = await EventProcessing.processEventBatch(batch, state, IndexerState.loadManager(state), IndexerState.persistence(state), IndexerState.config(state), IndexerState.chainStates(state));
+  if (errHandler.TAG !== "Ok") {
+    return IndexerState.errorExit(state, errHandler._0);
+  }
+  IndexerState.recordProcessedBatch(state);
+  if (IndexerState.isResolvingReorg(state)) {
+    return IndexerState.applyBatchProgress(state, batch);
+  }
+  IndexerState.clearRollback(state);
+  IndexerState.applyBatchProgress(state, batch);
+  if (!isRealtimeBeforeUpdate && IndexerState.isRealtime(state)) {
+    IndexerState.invalidateInflight(state);
+    scheduleFetch();
+  }
+  let allCaughtUp = EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state));
+  if (allCaughtUp) {
+    Logging.info("All chains are caught up to end blocks.");
+  }
+  if (allCaughtUp && !IndexerState.keepProcessAlive(state)) {
+    return await ExitOnCaughtUp.run(state);
+  } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isProgressAtHead(cs) ? Stdlib_Option.isNone(ChainState.fetchState(cs).endBlock) : false)) {
+    return IndexerState.errorExit(state, ErrorHandling.make(new Error("No events found between startBlock and chain head. Cannot auto-detect endBlock."), undefined, undefined));
+  } else {
+    ChainMetadata.stage(state);
+    if (Config.shouldPruneHistory(IndexerState.config(state), IndexerState.isInReorgThreshold(state))) {
+      return PruneStaleHistory.schedule(state);
+    } else {
+      return;
+    }
+  }
+}
+
+async function startProcessing(state, scheduleFetch, scheduleRollback) {
+  if (IndexerState.isProcessing(state) || IndexerState.isStopped(state)) {
+    return;
+  }
+  IndexerState.beginProcessing(state);
+  await new Promise((resolve, param) => {
+    setImmediate(() => resolve());
+  });
+  let hasMoreWork = true;
+  while (hasMoreWork && !IndexerState.isStopped(state) && !IndexerState.isResolvingReorg(state)) {
+    let processedBatchesBefore = IndexerState.processedBatchesCount(state);
+    let exit = 0;
+    let val;
+    try {
+      val = await processNextBatch(state, scheduleFetch);
+      exit = 1;
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, IndexerState.unexpectedErrorMsg));
+    }
+    if (exit === 1) {
+      hasMoreWork = IndexerState.processedBatchesCount(state) > processedBatchesBefore;
+    }
+  };
+  IndexerState.endProcessing(state);
+  if (IndexerState.isResolvingReorg(state)) {
+    return scheduleRollback();
+  }
+}
+
+export {
+  yieldTick,
+  startProcessing,
+  processNextBatch,
+}
+/* Utils Not a pure module */