envio 3.13.0-alpha.0 → 3.13.0-rc.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.13.0-alpha.0",
+  "version": "3.13.0-rc.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -49,11 +49,9 @@
     "@fuel-ts/utils": "0.96.1",
     "bignumber.js": "9.3.1",
     "eventsource": "4.1.0",
-    "js-sdsl": "4.4.2",
     "express": "4.19.2",
     "pino": "10.3.1",
     "pino-pretty": "13.1.3",
-    "yargs": "17.7.2",
     "@rescript/runtime": "12.2.0",
     "rescript-schema": "9.5.1",
     "viem": "2.54.0",
@@ -68,10 +66,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.13.0-alpha.0",
-    "envio-linux-x64-musl": "3.13.0-alpha.0",
-    "envio-linux-arm64": "3.13.0-alpha.0",
-    "envio-darwin-x64": "3.13.0-alpha.0",
-    "envio-darwin-arm64": "3.13.0-alpha.0"
+    "envio-linux-x64": "3.13.0-rc.0",
+    "envio-linux-x64-musl": "3.13.0-rc.0",
+    "envio-linux-arm64": "3.13.0-rc.0",
+    "envio-darwin-x64": "3.13.0-rc.0",
+    "envio-darwin-arm64": "3.13.0-rc.0"
   }
 }

package/src/BatchProcessing.res

@@ -69,7 +69,8 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
   let isInReorgThresholdBeforeUpdate = state->IndexerState.isInReorgThreshold
   let isRealtimeBeforeUpdate = state->IndexerState.isRealtime
 
-  let batch = state->IndexerState.createBatch(~batchSizeTarget=(state->IndexerState.config).batchSize)
+  let batch =
+    state->IndexerState.createBatch(~batchSizeTarget=(state->IndexerState.config).batchSize)
 
   let progressedChainsById = batch.progressedChainsById
 
@@ -105,8 +106,8 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
     }
 
     // When resuming from persisted state, all events may already be processed.
+    state->IndexerState.reportFinished
     if EventProcessing.allChainsEventsProcessedToEndblock(state->IndexerState.chainStates) {
-      Logging.info("All chains are caught up to end blocks.")
       if !(state->IndexerState.keepProcessAlive) && !(state->IndexerState.isHoldingRealtime) {
         await ExitOnCaughtUp.run(state)
       }
@@ -158,6 +159,9 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
         // Can safely reset rollback state, since overwrite is not possible.
         state->IndexerState.clearRollback
         state->IndexerState.applyBatchProgress(~batch)
+        // Before the finalize below, so a chain says where it finished ahead of
+        // the process saying what it does about that.
+        state->IndexerState.reportFinished
 
         // Backfilling → FinalizingIndexes → Ready. Awaiting here holds the
         // processing loop for the whole finalize, which is what pauses
@@ -178,9 +182,6 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
         let allCaughtUp = EventProcessing.allChainsEventsProcessedToEndblock(
           state->IndexerState.chainStates,
         )
-        if allCaughtUp {
-          Logging.info("All chains are caught up to end blocks.")
-        }
 
         if (
           allCaughtUp &&

package/src/BatchProcessing.res.mjs

@@ -1,7 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
-import * as Logging from "./Logging.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
@@ -42,13 +41,9 @@ async function processNextBatch(state, scheduleFetch) {
       IndexerState.invalidateInflight(state);
       scheduleFetch();
     }
-    if (EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state))) {
-      Logging.info("All chains are caught up to end blocks.");
-      if (!IndexerState.keepProcessAlive(state) && !IndexerState.isHoldingRealtime(state)) {
-        return await ExitOnCaughtUp.run(state);
-      } else {
-        return;
-      }
+    IndexerState.reportFinished(state);
+    if (EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state)) && !IndexerState.keepProcessAlive(state) && !IndexerState.isHoldingRealtime(state)) {
+      return await ExitOnCaughtUp.run(state);
     } else {
       return;
     }
@@ -70,6 +65,7 @@ async function processNextBatch(state, scheduleFetch) {
   }
   IndexerState.clearRollback(state);
   IndexerState.applyBatchProgress(state, batch);
+  IndexerState.reportFinished(state);
   if (IndexerState.shouldFinalizeIndexes(state)) {
     await FinalizeBackfill.run(state);
   }
@@ -78,9 +74,6 @@ async function processNextBatch(state, scheduleFetch) {
     scheduleFetch();
   }
   let allCaughtUp = EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state));
-  if (allCaughtUp) {
-    Logging.info("All chains are caught up to end blocks.");
-  }
   if (allCaughtUp && !IndexerState.keepProcessAlive(state) && !IndexerState.isHoldingRealtime(state)) {
     return await ExitOnCaughtUp.run(state);
   } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(ChainState.isAtHeadWithoutEndBlock)) {

package/src/ChainFetching.res

@@ -12,16 +12,16 @@ let runContractRegistersOrThrow = async (
   ~itemsWithContractRegister: array<Internal.item>,
   ~config: Config.t,
   ~chainState: ChainState.t,
-  ~transactionStore: option<TransactionStore.t>,
 ) => {
   // contractRegister handlers can read event.transaction and event.block, so
   // materialise the selected fields onto the payloads before running them. All
-  // items belong to the chain being fetched: transactions come from its
-  // response page, blocks from the chain store the page was merged into.
+  // items belong to the chain being fetched, and both stores are the chain's
+  // own, which this response's pages have already been merged into.
   await ChainState.materializePageItems(
     ~items=itemsWithContractRegister,
-    ~transactionStore,
+    ~transactionStore=chainState->ChainState.transactionStore,
     ~blockStore=chainState->ChainState.blockStore,
+    ~shouldChecksum=chainState->ChainState.shouldChecksum,
   )
 
   let registrations: array<AddressStore.registration> = []
@@ -113,14 +113,13 @@ let rec onQueryResponse = async (
       latestFetchedBlockNumber,
       stats,
       knownHeight,
-      fromBlockQueried,
     } = response
 
     chainState->ChainState.recordBlockRangeFetch(
       ~totalTimeElapsed=stats.totalTimeElapsed,
       ~parsingTimeElapsed=stats.parsingTimeElapsed->Option.getOr(0.),
       ~numEvents=parsedQueueItems->Array.length,
-      ~blockRangeSize=latestFetchedBlockNumber - fromBlockQueried + 1,
+      ~blockRangeSize=latestFetchedBlockNumber - query.fromBlock + 1,
     )
 
     let numContractRegisterEvents = parsedQueueItems->Array.reduce(0, (count, item) => {
@@ -132,7 +131,7 @@ let rec onQueryResponse = async (
         "msg": "Finished querying",
         "chainId": chainId,
         "partitionId": query.partitionId,
-        "fromBlock": fromBlockQueried,
+        "fromBlock": query.fromBlock,
         "toBlock": latestFetchedBlockNumber,
         "numEvents": parsedQueueItems->Array.length,
       })
@@ -141,7 +140,7 @@ let rec onQueryResponse = async (
         "msg": "Finished querying",
         "chainId": chainId,
         "partitionId": query.partitionId,
-        "fromBlock": fromBlockQueried,
+        "fromBlock": query.fromBlock,
         "toBlock": latestFetchedBlockNumber,
         "numEvents": parsedQueueItems->Array.length,
         "numContractRegisterEvents": numContractRegisterEvents,
@@ -200,7 +199,18 @@ let rec onQueryResponse = async (
       // Advances synchronously to FindingReorgDepth, so a concurrent rollback
       // kick (eg from the processing loop quiescing) collapses into this one.
       scheduleRollback()
+    // No rollback: either the guard found no reorg, or it found one and this
+    // chain only reports them. Either way the guard has merged the blocks, so
+    // the transactions follow. A source that reads the chain store to decide
+    // what to fetch leaves out what the store already holds, so its page alone
+    // does not cover its own items — only the merged store does, and contract
+    // registers read it below.
     | None =>
+      switch transactionStore {
+      | Some(page) => chainState->ChainState.transactionStore->TransactionStore.merge(page)
+      | None => ()
+      }
+
       // Over-fetched events (a merged partition returning an address before its
       // effectiveStartBlock, a wildcard param referencing an address registered
       // after the log's block, or a registration whose own start block is later
@@ -228,7 +238,6 @@ let rec onQueryResponse = async (
             ~knownHeight,
             ~latestFetchedBlock=latestFetchedBlockNumber,
             ~query,
-            ~transactionStore,
           )
           ChainMetadata.stage(state)
           scheduleFetch()
@@ -242,7 +251,6 @@ let rec onQueryResponse = async (
           ~itemsWithContractRegister,
           ~config=state->IndexerState.config,
           ~chainState,
-          ~transactionStore,
         ) {
         | exception exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
         | newRegistrations => proceed(~newRegistrations)
@@ -259,7 +267,6 @@ and applyQueryResponse = (
   ~knownHeight,
   ~latestFetchedBlock,
   ~query,
-  ~transactionStore,
 ) => {
   let chainState = state->IndexerState.getChainState(~chainId)
 
@@ -269,7 +276,6 @@ and applyQueryResponse = (
     ~newItems,
     ~newRegistrations,
     ~knownHeight,
-    ~transactionStore,
   )
 
   // In auto-exit mode, set endBlock to the first event's block when events arrive.
@@ -278,32 +284,6 @@ and applyQueryResponse = (
       ~blockNumber=newItems->Array.getUnsafe(0)->Internal.getItemBlockNumber,
     )
   }
-
-  // Report the milestone this response brought the fetch frontier to, once.
-  // The chain reaches it again every time the head moves and it catches up, so
-  // what keeps the line off the log is the chain having already reported it,
-  // not the transition — which re-arms on every advance.
-  switch chainState->ChainState.isFetchingAtHead
-    ? chainState->ChainState.takeFetchedTo
-    : None {
-  | None => ()
-  | Some((target, block)) =>
-    // What the chain is waiting on, which is not itself. A held process is
-    // waiting on chains it doesn't drive, so it says so even when it drives
-    // only one — how the run is split is not the reader's problem.
-    let waitingOn = if (
-      state->IndexerState.isHoldingRealtime ||
-        state->IndexerState.chainStates->Dict.keysToArray->Array.length > 1
-    ) {
-      " Waiting for other chains."
-    } else {
-      ""
-    }
-    chainState->ChainState.logger->Logging.childInfo({
-      "msg": `Fetched to ${target}.${waitingOn}`,
-      "block": block,
-    })
-  }
 }
 
 let finishWaitingForNewBlock = (

package/src/ChainFetching.res.mjs

@@ -13,11 +13,12 @@ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as ReorgDetection from "./ReorgDetection.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 import * as ContractRegisterContext from "./ContractRegisterContext.res.mjs";
 
-async function runContractRegistersOrThrow(itemsWithContractRegister, config, chainState, transactionStore) {
-  await ChainState.materializePageItems(itemsWithContractRegister, transactionStore, ChainState.blockStore(chainState));
+async function runContractRegistersOrThrow(itemsWithContractRegister, config, chainState) {
+  await ChainState.materializePageItems(itemsWithContractRegister, ChainState.transactionStore(chainState), ChainState.blockStore(chainState), ChainState.shouldChecksum(chainState));
   let registrations = [];
   let onRegister = (item, contractAddress, contractName) => {
     registrations.push({
@@ -63,21 +64,12 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config, ch
   return registrations;
 }
 
-function applyQueryResponse(state, chainId, newItems, newRegistrations, knownHeight, latestFetchedBlock, query, transactionStore) {
+function applyQueryResponse(state, chainId, newItems, newRegistrations, knownHeight, latestFetchedBlock, query) {
   let chainState = IndexerState.getChainState(state, chainId);
-  ChainState.handleQueryResult(chainState, query, newItems, newRegistrations, latestFetchedBlock, knownHeight, transactionStore);
+  ChainState.handleQueryResult(chainState, query, newItems, newRegistrations, latestFetchedBlock, knownHeight);
   if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
-    ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
+    return ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
   }
-  let match = ChainState.isFetchingAtHead(chainState) ? ChainState.takeFetchedTo(chainState) : undefined;
-  if (match === undefined) {
-    return;
-  }
-  let waitingOn = IndexerState.isHoldingRealtime(state) || Object.keys(IndexerState.chainStates(state)).length > 1 ? " Waiting for other chains." : "";
-  Logging.childInfo(ChainState.logger(chainState), {
-    msg: `Fetched to ` + match[0] + `.` + waitingOn,
-    block: match[1]
-  });
 }
 
 async function onQueryResponse(state, param, stateId, scheduleFetch, scheduleProcessing, scheduleRollback) {
@@ -90,11 +82,10 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   let chainState = IndexerState.getChainState(state, chainId);
   let stats = response.stats;
   let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
-  let fromBlockQueried = response.fromBlockQueried;
   let transactionStore = response.transactionStore;
   let parsedQueueItems = response.parsedQueueItems;
   let knownHeight = response.knownHeight;
-  ChainState.recordBlockRangeFetch(chainState, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - fromBlockQueried | 0) + 1 | 0);
+  ChainState.recordBlockRangeFetch(chainState, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - query.fromBlock | 0) + 1 | 0);
   let numContractRegisterEvents = Stdlib_Array.reduce(parsedQueueItems, 0, (count, item) => {
     if (item.onEventRegistration.contractRegister !== undefined) {
       return count + 1 | 0;
@@ -107,7 +98,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
       msg: "Finished querying",
       chainId: chainId,
       partitionId: query.partitionId,
-      fromBlock: fromBlockQueried,
+      fromBlock: query.fromBlock,
       toBlock: latestFetchedBlockNumber,
       numEvents: parsedQueueItems.length
     });
@@ -116,7 +107,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
       msg: "Finished querying",
       chainId: chainId,
       partitionId: query.partitionId,
-      fromBlock: fromBlockQueried,
+      fromBlock: query.fromBlock,
       toBlock: latestFetchedBlockNumber,
       numEvents: parsedQueueItems.length,
       numContractRegisterEvents: numContractRegisterEvents
@@ -140,6 +131,9 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
     IndexerState.beginReorg(state, chainId, rollbackWithReorgDetectedBlockNumber);
     return scheduleRollback();
   }
+  if (transactionStore !== undefined) {
+    ChainState.transactionStore(chainState).merge(Primitive_option.valFromOption(transactionStore));
+  }
   let itemsWithContractRegister = [];
   for (let idx = 0, idx_finish = parsedQueueItems.length; idx < idx_finish; ++idx) {
     let item = parsedQueueItems[idx];
@@ -149,7 +143,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   }
   let proceed = newRegistrations => {
     if (!IndexerState.isStale(state, stateId)) {
-      applyQueryResponse(state, chainId, parsedQueueItems, newRegistrations, knownHeight, latestFetchedBlockNumber, query, transactionStore);
+      applyQueryResponse(state, chainId, parsedQueueItems, newRegistrations, knownHeight, latestFetchedBlockNumber, query);
       ChainMetadata.stage(state);
       scheduleFetch();
       return scheduleProcessing();
@@ -160,7 +154,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   }
   let newRegistrations;
   try {
-    newRegistrations = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), chainState, transactionStore);
+    newRegistrations = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), chainState);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));