envio 3.12.1 → 3.13.0-rc.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.12.1",
+  "version": "3.13.0-rc.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -49,11 +49,9 @@
     "@fuel-ts/utils": "0.96.1",
     "bignumber.js": "9.3.1",
     "eventsource": "4.1.0",
-    "js-sdsl": "4.4.2",
     "express": "4.19.2",
     "pino": "10.3.1",
     "pino-pretty": "13.1.3",
-    "yargs": "17.7.2",
     "@rescript/runtime": "12.2.0",
     "rescript-schema": "9.5.1",
     "viem": "2.54.0",
@@ -68,10 +66,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.12.1",
-    "envio-linux-x64-musl": "3.12.1",
-    "envio-linux-arm64": "3.12.1",
-    "envio-darwin-x64": "3.12.1",
-    "envio-darwin-arm64": "3.12.1"
+    "envio-linux-x64": "3.13.0-rc.0",
+    "envio-linux-x64-musl": "3.13.0-rc.0",
+    "envio-linux-arm64": "3.13.0-rc.0",
+    "envio-darwin-x64": "3.13.0-rc.0",
+    "envio-darwin-arm64": "3.13.0-rc.0"
   }
 }

package/src/BatchProcessing.res

@@ -69,18 +69,15 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
   let isInReorgThresholdBeforeUpdate = state->IndexerState.isInReorgThreshold
   let isRealtimeBeforeUpdate = state->IndexerState.isRealtime
 
-  let batch = state->IndexerState.createBatch(~batchSizeTarget=(state->IndexerState.config).batchSize)
+  let batch =
+    state->IndexerState.createBatch(~batchSizeTarget=(state->IndexerState.config).batchSize)
 
   let progressedChainsById = batch.progressedChainsById
 
   let isBelowReorgThreshold =
     !isInReorgThresholdBeforeUpdate && (state->IndexerState.config).shouldRollbackOnReorg
   let shouldEnterReorgThreshold =
-    isBelowReorgThreshold &&
-    state
-    ->IndexerState.chainStates
-    ->Dict.valuesToArray
-    ->Array.every(cs => cs->ChainState.isReadyToEnterReorgThresholdAfterBatch(~batch))
+    isBelowReorgThreshold && state->IndexerState.isReadyToEnterReorgThreshold(~batch)
 
   if shouldEnterReorgThreshold {
     IndexerState.enterReorgThreshold(state)
@@ -95,7 +92,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
     // finalizing resumes exactly here: it still owes the schema its deferred
     // indexes, and no batch will ever come along to notice.
     state->IndexerState.markCaughtUpIfSettled
-    if state->IndexerState.isFinalizingIndexes {
+    if state->IndexerState.shouldFinalizeIndexes {
       await FinalizeBackfill.run(state)
     }
 
@@ -109,9 +106,9 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
     }
 
     // When resuming from persisted state, all events may already be processed.
+    state->IndexerState.reportFinished
     if EventProcessing.allChainsEventsProcessedToEndblock(state->IndexerState.chainStates) {
-      Logging.info("All chains are caught up to end blocks.")
-      if !(state->IndexerState.keepProcessAlive) {
+      if !(state->IndexerState.keepProcessAlive) && !(state->IndexerState.isHoldingRealtime) {
         await ExitOnCaughtUp.run(state)
       }
     }
@@ -162,11 +159,14 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
         // Can safely reset rollback state, since overwrite is not possible.
         state->IndexerState.clearRollback
         state->IndexerState.applyBatchProgress(~batch)
+        // Before the finalize below, so a chain says where it finished ahead of
+        // the process saying what it does about that.
+        state->IndexerState.reportFinished
 
         // Backfilling → FinalizingIndexes → Ready. Awaiting here holds the
         // processing loop for the whole finalize, which is what pauses
         // processing while the indexes are built.
-        if state->IndexerState.isFinalizingIndexes {
+        if state->IndexerState.shouldFinalizeIndexes {
           await FinalizeBackfill.run(state)
         }
 
@@ -182,11 +182,12 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
         let allCaughtUp = EventProcessing.allChainsEventsProcessedToEndblock(
           state->IndexerState.chainStates,
         )
-        if allCaughtUp {
-          Logging.info("All chains are caught up to end blocks.")
-        }
 
-        if allCaughtUp && !(state->IndexerState.keepProcessAlive) {
+        if (
+          allCaughtUp &&
+          !(state->IndexerState.keepProcessAlive) &&
+          !(state->IndexerState.isHoldingRealtime)
+        ) {
           await ExitOnCaughtUp.run(state)
         } else if (
           // In auto-exit mode, error if all chains reached head with no events found

package/src/BatchProcessing.res.mjs

@@ -1,7 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
-import * as Logging from "./Logging.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
@@ -26,7 +25,7 @@ async function processNextBatch(state, scheduleFetch) {
   let batch = IndexerState.createBatch(state, IndexerState.config(state).batchSize);
   let progressedChainsById = batch.progressedChainsById;
   let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
-  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));
+  let shouldEnterReorgThreshold = isBelowReorgThreshold && IndexerState.isReadyToEnterReorgThreshold(state, batch);
   if (shouldEnterReorgThreshold) {
     IndexerState.enterReorgThreshold(state);
   }
@@ -35,20 +34,16 @@ async function processNextBatch(state, scheduleFetch) {
       scheduleFetch();
     }
     IndexerState.markCaughtUpIfSettled(state);
-    if (IndexerState.isFinalizingIndexes(state)) {
+    if (IndexerState.shouldFinalizeIndexes(state)) {
       await FinalizeBackfill.run(state);
     }
     if (!isRealtimeBeforeUpdate && IndexerState.isRealtime(state)) {
       IndexerState.invalidateInflight(state);
       scheduleFetch();
     }
-    if (EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state))) {
-      Logging.info("All chains are caught up to end blocks.");
-      if (!IndexerState.keepProcessAlive(state)) {
-        return await ExitOnCaughtUp.run(state);
-      } else {
-        return;
-      }
+    IndexerState.reportFinished(state);
+    if (EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state)) && !IndexerState.keepProcessAlive(state) && !IndexerState.isHoldingRealtime(state)) {
+      return await ExitOnCaughtUp.run(state);
     } else {
       return;
     }
@@ -70,7 +65,8 @@ async function processNextBatch(state, scheduleFetch) {
   }
   IndexerState.clearRollback(state);
   IndexerState.applyBatchProgress(state, batch);
-  if (IndexerState.isFinalizingIndexes(state)) {
+  IndexerState.reportFinished(state);
+  if (IndexerState.shouldFinalizeIndexes(state)) {
     await FinalizeBackfill.run(state);
   }
   if (!isRealtimeBeforeUpdate && IndexerState.isRealtime(state)) {
@@ -78,10 +74,7 @@ async function processNextBatch(state, scheduleFetch) {
     scheduleFetch();
   }
   let allCaughtUp = EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state));
-  if (allCaughtUp) {
-    Logging.info("All chains are caught up to end blocks.");
-  }
-  if (allCaughtUp && !IndexerState.keepProcessAlive(state)) {
+  if (allCaughtUp && !IndexerState.keepProcessAlive(state) && !IndexerState.isHoldingRealtime(state)) {
     return await ExitOnCaughtUp.run(state);
   } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(ChainState.isAtHeadWithoutEndBlock)) {
     return IndexerState.errorExit(state, ErrorHandling.make(new Error("No events found between startBlock and chain head. Cannot auto-detect endBlock."), undefined, undefined));

package/src/Bin.res

@@ -51,23 +51,30 @@ let applyEnv = (env: dict<JSON.t>) =>
 
 let run = async args => {
   try {
-    switch (await Core.runCli(args))->Null.toOption {
-    // Rust-only command (codegen / init / stop / docker / metrics / help /
-    // version / scripts) — nothing for JS to do, exit cleanly.
-    | None => ()
-    | Some(json) =>
-      switch decodeCommand(json->JSON.parseOrThrow) {
-      | Start({reset, cwd, env, config}) =>
-        Config.prime(config)
-        processChdir(cwd)
-        applyEnv(env)
-        await Main.start(~reset)
-      | Migrate({reset, config}) =>
-        Config.prime(config)
-        await Main.migrate(~reset)
-      | DropSchema({config}) =>
-        Config.prime(config)
-        await Main.dropSchema()
+    if Worker.isEnabled {
+      Worker.bindToSupervisor()
+      // Its working directory, its environment and the chains it drives all came
+      // with the fork, so a worker starts the same way every other process does.
+      await Main.start()
+    } else {
+      switch (await Core.runCli(args))->Null.toOption {
+      // Rust-only command (codegen / init / stop / docker / metrics / help /
+      // version / scripts) — nothing for JS to do, exit cleanly.
+      | None => ()
+      | Some(json) =>
+        switch decodeCommand(json->JSON.parseOrThrow) {
+        | Start({reset, cwd, env, config}) =>
+          Config.prime(config)
+          processChdir(cwd)
+          applyEnv(env)
+          await Main.start(~reset)
+        | Migrate({reset, config}) =>
+          Config.prime(config)
+          await Main.migrate(~reset)
+        | DropSchema({config}) =>
+          Config.prime(config)
+          await Main.dropSchema()
+        }
       }
     }
   } catch {

package/src/Bin.res.mjs

@@ -4,6 +4,7 @@ import * as Core from "./Core.res.mjs";
 import * as Main from "./Main.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
+import * as Worker from "./Worker.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Process from "process";
 import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
@@ -57,6 +58,10 @@ function applyEnv(env) {
 
 async function run(args) {
   try {
+    if (Worker.isEnabled) {
+      Worker.bindToSupervisor();
+      return await Main.start(undefined, undefined, undefined, undefined, undefined);
+    }
     let json = await Core.runCli(args);
     if (json === null) {
       return;

package/src/ChainFetching.res

@@ -12,16 +12,16 @@ let runContractRegistersOrThrow = async (
   ~itemsWithContractRegister: array<Internal.item>,
   ~config: Config.t,
   ~chainState: ChainState.t,
-  ~transactionStore: option<TransactionStore.t>,
 ) => {
   // contractRegister handlers can read event.transaction and event.block, so
   // materialise the selected fields onto the payloads before running them. All
-  // items belong to the chain being fetched: transactions come from its
-  // response page, blocks from the chain store the page was merged into.
+  // items belong to the chain being fetched, and both stores are the chain's
+  // own, which this response's pages have already been merged into.
   await ChainState.materializePageItems(
     ~items=itemsWithContractRegister,
-    ~transactionStore,
+    ~transactionStore=chainState->ChainState.transactionStore,
     ~blockStore=chainState->ChainState.blockStore,
+    ~shouldChecksum=chainState->ChainState.shouldChecksum,
   )
 
   let registrations: array<AddressStore.registration> = []
@@ -113,14 +113,13 @@ let rec onQueryResponse = async (
       latestFetchedBlockNumber,
       stats,
       knownHeight,
-      fromBlockQueried,
     } = response
 
     chainState->ChainState.recordBlockRangeFetch(
       ~totalTimeElapsed=stats.totalTimeElapsed,
       ~parsingTimeElapsed=stats.parsingTimeElapsed->Option.getOr(0.),
       ~numEvents=parsedQueueItems->Array.length,
-      ~blockRangeSize=latestFetchedBlockNumber - fromBlockQueried + 1,
+      ~blockRangeSize=latestFetchedBlockNumber - query.fromBlock + 1,
     )
 
     let numContractRegisterEvents = parsedQueueItems->Array.reduce(0, (count, item) => {
@@ -132,7 +131,7 @@ let rec onQueryResponse = async (
         "msg": "Finished querying",
         "chainId": chainId,
         "partitionId": query.partitionId,
-        "fromBlock": fromBlockQueried,
+        "fromBlock": query.fromBlock,
         "toBlock": latestFetchedBlockNumber,
         "numEvents": parsedQueueItems->Array.length,
       })
@@ -141,7 +140,7 @@ let rec onQueryResponse = async (
         "msg": "Finished querying",
         "chainId": chainId,
         "partitionId": query.partitionId,
-        "fromBlock": fromBlockQueried,
+        "fromBlock": query.fromBlock,
         "toBlock": latestFetchedBlockNumber,
         "numEvents": parsedQueueItems->Array.length,
         "numContractRegisterEvents": numContractRegisterEvents,
@@ -200,7 +199,18 @@ let rec onQueryResponse = async (
       // Advances synchronously to FindingReorgDepth, so a concurrent rollback
       // kick (eg from the processing loop quiescing) collapses into this one.
       scheduleRollback()
+    // No rollback: either the guard found no reorg, or it found one and this
+    // chain only reports them. Either way the guard has merged the blocks, so
+    // the transactions follow. A source that reads the chain store to decide
+    // what to fetch leaves out what the store already holds, so its page alone
+    // does not cover its own items — only the merged store does, and contract
+    // registers read it below.
     | None =>
+      switch transactionStore {
+      | Some(page) => chainState->ChainState.transactionStore->TransactionStore.merge(page)
+      | None => ()
+      }
+
       // Over-fetched events (a merged partition returning an address before its
       // effectiveStartBlock, a wildcard param referencing an address registered
       // after the log's block, or a registration whose own start block is later
@@ -228,7 +238,6 @@ let rec onQueryResponse = async (
             ~knownHeight,
             ~latestFetchedBlock=latestFetchedBlockNumber,
             ~query,
-            ~transactionStore,
           )
           ChainMetadata.stage(state)
           scheduleFetch()
@@ -242,7 +251,6 @@ let rec onQueryResponse = async (
           ~itemsWithContractRegister,
           ~config=state->IndexerState.config,
           ~chainState,
-          ~transactionStore,
         ) {
         | exception exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
         | newRegistrations => proceed(~newRegistrations)
@@ -259,10 +267,8 @@ and applyQueryResponse = (
   ~knownHeight,
   ~latestFetchedBlock,
   ~query,
-  ~transactionStore,
 ) => {
   let chainState = state->IndexerState.getChainState(~chainId)
-  let wasFetchingAtHead = chainState->ChainState.isFetchingAtHead
 
   chainState->ChainState.handleQueryResult(
     ~query,
@@ -270,7 +276,6 @@ and applyQueryResponse = (
     ~newItems,
     ~newRegistrations,
     ~knownHeight,
-    ~transactionStore,
   )
 
   // In auto-exit mode, set endBlock to the first event's block when events arrive.
@@ -279,17 +284,6 @@ and applyQueryResponse = (
       ~blockNumber=newItems->Array.getUnsafe(0)->Internal.getItemBlockNumber,
     )
   }
-
-  // Log the backfill→head transition once: this response brought the fetch
-  // frontier to the head. Gated on !isReady so realtime re-catch-ups (a new
-  // block arrives, gets fetched) don't spam the log after the chain is synced.
-  if (
-    !wasFetchingAtHead &&
-    !(chainState->ChainState.isReady) &&
-    chainState->ChainState.isFetchingAtHead
-  ) {
-    chainState->ChainState.logger->Logging.childInfo("All events have been fetched")
-  }
 }
 
 let finishWaitingForNewBlock = (

package/src/ChainFetching.res.mjs

@@ -13,11 +13,12 @@ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as ReorgDetection from "./ReorgDetection.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 import * as ContractRegisterContext from "./ContractRegisterContext.res.mjs";
 
-async function runContractRegistersOrThrow(itemsWithContractRegister, config, chainState, transactionStore) {
-  await ChainState.materializePageItems(itemsWithContractRegister, transactionStore, ChainState.blockStore(chainState));
+async function runContractRegistersOrThrow(itemsWithContractRegister, config, chainState) {
+  await ChainState.materializePageItems(itemsWithContractRegister, ChainState.transactionStore(chainState), ChainState.blockStore(chainState), ChainState.shouldChecksum(chainState));
   let registrations = [];
   let onRegister = (item, contractAddress, contractName) => {
     registrations.push({
@@ -63,15 +64,11 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config, ch
   return registrations;
 }
 
-function applyQueryResponse(state, chainId, newItems, newRegistrations, knownHeight, latestFetchedBlock, query, transactionStore) {
+function applyQueryResponse(state, chainId, newItems, newRegistrations, knownHeight, latestFetchedBlock, query) {
   let chainState = IndexerState.getChainState(state, chainId);
-  let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
-  ChainState.handleQueryResult(chainState, query, newItems, newRegistrations, latestFetchedBlock, knownHeight, transactionStore);
+  ChainState.handleQueryResult(chainState, query, newItems, newRegistrations, latestFetchedBlock, knownHeight);
   if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
-    ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
-  }
-  if (!wasFetchingAtHead && !ChainState.isReady(chainState) && ChainState.isFetchingAtHead(chainState)) {
-    return Logging.childInfo(ChainState.logger(chainState), "All events have been fetched");
+    return ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
   }
 }
 
@@ -85,11 +82,10 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   let chainState = IndexerState.getChainState(state, chainId);
   let stats = response.stats;
   let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
-  let fromBlockQueried = response.fromBlockQueried;
   let transactionStore = response.transactionStore;
   let parsedQueueItems = response.parsedQueueItems;
   let knownHeight = response.knownHeight;
-  ChainState.recordBlockRangeFetch(chainState, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - fromBlockQueried | 0) + 1 | 0);
+  ChainState.recordBlockRangeFetch(chainState, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - query.fromBlock | 0) + 1 | 0);
   let numContractRegisterEvents = Stdlib_Array.reduce(parsedQueueItems, 0, (count, item) => {
     if (item.onEventRegistration.contractRegister !== undefined) {
       return count + 1 | 0;
@@ -102,7 +98,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
       msg: "Finished querying",
       chainId: chainId,
       partitionId: query.partitionId,
-      fromBlock: fromBlockQueried,
+      fromBlock: query.fromBlock,
       toBlock: latestFetchedBlockNumber,
       numEvents: parsedQueueItems.length
     });
@@ -111,7 +107,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
       msg: "Finished querying",
       chainId: chainId,
       partitionId: query.partitionId,
-      fromBlock: fromBlockQueried,
+      fromBlock: query.fromBlock,
       toBlock: latestFetchedBlockNumber,
       numEvents: parsedQueueItems.length,
       numContractRegisterEvents: numContractRegisterEvents
@@ -135,6 +131,9 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
     IndexerState.beginReorg(state, chainId, rollbackWithReorgDetectedBlockNumber);
     return scheduleRollback();
   }
+  if (transactionStore !== undefined) {
+    ChainState.transactionStore(chainState).merge(Primitive_option.valFromOption(transactionStore));
+  }
   let itemsWithContractRegister = [];
   for (let idx = 0, idx_finish = parsedQueueItems.length; idx < idx_finish; ++idx) {
     let item = parsedQueueItems[idx];
@@ -144,7 +143,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   }
   let proceed = newRegistrations => {
     if (!IndexerState.isStale(state, stateId)) {
-      applyQueryResponse(state, chainId, parsedQueueItems, newRegistrations, knownHeight, latestFetchedBlockNumber, query, transactionStore);
+      applyQueryResponse(state, chainId, parsedQueueItems, newRegistrations, knownHeight, latestFetchedBlockNumber, query);
       ChainMetadata.stage(state);
       scheduleFetch();
       return scheduleProcessing();
@@ -155,7 +154,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   }
   let newRegistrations;
   try {
-    newRegistrations = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), chainState, transactionStore);
+    newRegistrations = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), chainState);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));