envio 3.3.0-alpha.0 → 3.3.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.3.0-alpha.0",
+  "version": "3.3.0-alpha.2",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.3.0-alpha.0",
-    "envio-linux-x64-musl": "3.3.0-alpha.0",
-    "envio-linux-arm64": "3.3.0-alpha.0",
-    "envio-darwin-x64": "3.3.0-alpha.0",
-    "envio-darwin-arm64": "3.3.0-alpha.0"
+    "envio-linux-x64": "3.3.0-alpha.2",
+    "envio-linux-x64-musl": "3.3.0-alpha.2",
+    "envio-linux-arm64": "3.3.0-alpha.2",
+    "envio-darwin-x64": "3.3.0-alpha.2",
+    "envio-darwin-arm64": "3.3.0-alpha.2"
   }
 }

package/src/Batch.res

@@ -34,12 +34,6 @@ type t = {
   checkpointEventsProcessed: array<int>,
 }
 
-let hasReadyItem = (fetchStates: array<FetchState.t>) => {
-  fetchStates->Array.some(fetchState => {
-    fetchState->FetchState.isActivelyIndexing && fetchState->FetchState.hasReadyItem
-  })
-}
-
 let getProgressedChainsById = {
   let getChainAfterBatchIfProgressed = (
     ~chainBeforeBatch: chainBeforeBatch,

package/src/Batch.res.mjs

@@ -4,16 +4,6 @@ import * as Utils from "./Utils.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
 import * as ReorgDetection from "./ReorgDetection.res.mjs";
 
-function hasReadyItem(fetchStates) {
-  return fetchStates.some(fetchState => {
-    if (FetchState.isActivelyIndexing(fetchState)) {
-      return FetchState.hasReadyItem(fetchState);
-    } else {
-      return false;
-    }
-  });
-}
-
 function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
   if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
     return {
@@ -208,7 +198,6 @@ function findLastEventItem(batch, chainId) {
 }
 
 export {
-  hasReadyItem,
   getProgressedChainsById,
   addReorgCheckpoints,
   prepareBatch,

package/src/BatchProcessing.res

@@ -77,15 +77,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
     state
     ->IndexerState.chainStates
     ->Dict.valuesToArray
-    ->Array.every(cs => {
-      let fetchState = switch progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
-        (cs->ChainState.fetchState).chainId,
-      ) {
-      | Some(chainAfterBatch) => chainAfterBatch.fetchState
-      | None => cs->ChainState.fetchState
-      }
-      fetchState->FetchState.isReadyToEnterReorgThreshold
-    })
+    ->Array.every(cs => cs->ChainState.isReadyToEnterReorgThresholdAfterBatch(~batch))
 
   if shouldEnterReorgThreshold {
     IndexerState.enterReorgThreshold(state)
@@ -171,9 +163,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
           state
           ->IndexerState.chainStates
           ->Dict.valuesToArray
-          ->Array.every(cs =>
-            cs->ChainState.isProgressAtHead && (cs->ChainState.fetchState).endBlock->Option.isNone
-          )
+          ->Array.every(ChainState.isAtHeadWithoutEndBlock)
         ) {
           IndexerState.errorExit(
             state,

package/src/BatchProcessing.res.mjs

@@ -4,12 +4,10 @@ import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
-import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as ExitOnCaughtUp from "./ExitOnCaughtUp.res.mjs";
 import * as EventProcessing from "./EventProcessing.res.mjs";
 import * as PruneStaleHistory from "./PruneStaleHistory.res.mjs";
@@ -30,10 +28,7 @@ async function processNextBatch(state, scheduleFetch) {
   let batch = IndexerState.createBatch(state, IndexerState.processedCheckpointId(state), IndexerState.config(state).batchSize, isRollbackBatch);
   let progressedChainsById = batch.progressedChainsById;
   let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
-  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => {
-    let chainAfterBatch = progressedChainsById[ChainState.fetchState(cs).chainId];
-    return FetchState.isReadyToEnterReorgThreshold(chainAfterBatch !== undefined ? chainAfterBatch.fetchState : ChainState.fetchState(cs));
-  });
+  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));
   if (shouldEnterReorgThreshold) {
     IndexerState.enterReorgThreshold(state);
   }
@@ -75,7 +70,7 @@ async function processNextBatch(state, scheduleFetch) {
   }
   if (allCaughtUp && !IndexerState.keepProcessAlive(state)) {
     return await ExitOnCaughtUp.run(state);
-  } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isProgressAtHead(cs) ? Stdlib_Option.isNone(ChainState.fetchState(cs).endBlock) : false)) {
+  } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(ChainState.isAtHeadWithoutEndBlock)) {
     return IndexerState.errorExit(state, ErrorHandling.make(new Error("No events found between startBlock and chain head. Cannot auto-detect endBlock."), undefined, undefined));
   } else {
     ChainMetadata.stage(state);

package/src/ChainFetching.res

@@ -11,7 +11,14 @@ type partitionQueryResponse = {
 let runContractRegistersOrThrow = async (
   ~itemsWithContractRegister: array<Internal.item>,
   ~config: Config.t,
+  ~chainState: ChainState.t,
+  ~page: option<TransactionStore.t>,
 ) => {
+  // contractRegister handlers can read event.transaction, so materialise the
+  // selected fields onto the payloads before running them. All items belong to
+  // the chain being fetched, hence its single page store and mask.
+  await chainState->ChainState.materializePageItems(~items=itemsWithContractRegister, ~page)
+
   let itemsWithDcs = []
 
   let onRegister = (~item: Internal.item, ~contractAddress, ~contractName) => {
@@ -106,6 +113,7 @@ let rec onQueryResponse = async (
     let chainState = state->IndexerState.getChainState(~chain)
     let {
       parsedQueueItems,
+      transactionStore,
       latestFetchedBlockNumber,
       latestFetchedBlockTimestamp,
       stats,
@@ -114,7 +122,7 @@ let rec onQueryResponse = async (
       fromBlockQueried,
     } = response
 
-    if knownHeight > (chainState->ChainState.fetchState).knownHeight {
+    if knownHeight > chainState->ChainState.knownHeight {
       Prometheus.SourceHeight.set(
         ~blockNumber=knownHeight,
         ~chainId=(chainState->ChainState.chainConfig).id,
@@ -212,6 +220,7 @@ let rec onQueryResponse = async (
               blockTimestamp: latestFetchedBlockTimestamp,
             },
             ~query,
+            ~transactionStore,
           )
           ChainMetadata.stage(state)
           scheduleFetch()
@@ -224,6 +233,8 @@ let rec onQueryResponse = async (
         switch await runContractRegistersOrThrow(
           ~itemsWithContractRegister,
           ~config=state->IndexerState.config,
+          ~chainState,
+          ~page=transactionStore,
         ) {
         | exception exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
         | newItemsWithDcs => proceed(~newItemsWithDcs)
@@ -240,6 +251,7 @@ and applyQueryResponse = (
   ~knownHeight,
   ~latestFetchedBlock,
   ~query,
+  ~transactionStore,
 ) => {
   let chainState = state->IndexerState.getChainState(~chain)
   let wasFetchingAtHead = chainState->ChainState.isFetchingAtHead
@@ -250,6 +262,7 @@ and applyQueryResponse = (
     ~newItems,
     ~newItemsWithDcs,
     ~knownHeight,
+    ~transactionStore,
   )
 
   // In auto-exit mode, set endBlock to the first event's block when events arrive.
@@ -294,7 +307,7 @@ let finishWaitingForNewBlock = (
       ->IndexerState.chainStates
       ->Dict.valuesToArray
       ->Array.every(cs => {
-        cs->ChainState.fetchState->FetchState.isReadyToEnterReorgThreshold
+        cs->ChainState.isReadyToEnterReorgThreshold
       })
 
     // Kick processing in case there are block handlers to run.
@@ -308,8 +321,7 @@ let finishWaitingForNewBlock = (
 let fetchChain = async (
   state: IndexerState.t,
   chain,
-  ~concurrencyLimit,
-  ~bufferLimit,
+  ~action,
   ~stateId,
   ~scheduleFetch,
   ~scheduleProcessing,
@@ -317,19 +329,17 @@ let fetchChain = async (
 ) => {
   let chainState = state->IndexerState.getChainState(~chain)
   if !(state->IndexerState.isResolvingReorg) && !(state->IndexerState.isStopped) {
-    let fetchState = chainState->ChainState.fetchState
     let isRealtime = state->IndexerState.isRealtime
     let sourceManager = chainState->ChainState.sourceManager
 
-    // Only affects the WaitingForNewBlock branch of fetchNext, where
+    // Only affects the WaitingForNewBlock branch of dispatch, where
     // there's nothing to fetch. During backfill any such chain is idle.
     let reducedPolling = !isRealtime
 
     // Owns its error boundary: launch doesn't catch, so any failure here (the
-    // query, response handling, or fetchNext itself) must stop the indexer.
+    // query, response handling, or dispatch itself) must stop the indexer.
     try {
-      await sourceManager->SourceManager.fetchNext(
-        ~fetchState,
+      await chainState->ChainState.dispatch(
         ~waitForNewBlock=(~knownHeight) =>
           sourceManager->SourceManager.waitForNewBlock(~knownHeight, ~isRealtime, ~reducedPolling),
         ~onNewBlock=(~knownHeight) =>
@@ -343,12 +353,12 @@ let fetchChain = async (
           ),
         ~executeQuery=async query => {
           // Caught here (not just by the outer try) so the query promise never
-          // rejects: fetchNext spins a side-chain off it that would otherwise
+          // rejects: dispatch spins a side-chain off it that would otherwise
           // become an unhandled rejection.
           try {
             let response = await sourceManager->SourceManager.executeQuery(
               ~query,
-              ~knownHeight=fetchState.knownHeight,
+              ~knownHeight=chainState->ChainState.knownHeight,
               ~isRealtime,
             )
             await onQueryResponse(
@@ -363,8 +373,7 @@ let fetchChain = async (
           | exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
           }
         },
-        ~concurrencyLimit,
-        ~bufferLimit,
+        ~action,
         ~stateId,
       )
     } catch {

package/src/ChainFetching.res.mjs

@@ -4,7 +4,6 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
@@ -17,7 +16,8 @@ import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 import * as ContractRegisterContext from "./ContractRegisterContext.res.mjs";
 
-async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
+async function runContractRegistersOrThrow(itemsWithContractRegister, config, chainState, page) {
+  await ChainState.materializePageItems(chainState, itemsWithContractRegister, page);
   let itemsWithDcs = [];
   let onRegister = (item, contractAddress, contractName) => {
     let dc_registrationBlock = item.blockNumber;
@@ -71,10 +71,10 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
   return itemsWithDcs;
 }
 
-function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query) {
+function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query, transactionStore) {
   let chainState = IndexerState.getChainState(state, chain);
   let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
-  ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight);
+  ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, transactionStore);
   if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
     ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
   }
@@ -94,9 +94,10 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   let stats = response.stats;
   let latestFetchedBlockTimestamp = response.latestFetchedBlockTimestamp;
   let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
+  let transactionStore = response.transactionStore;
   let parsedQueueItems = response.parsedQueueItems;
   let knownHeight = response.knownHeight;
-  if (knownHeight > ChainState.fetchState(chainState).knownHeight) {
+  if (knownHeight > ChainState.knownHeight(chainState)) {
     Prometheus.SourceHeight.set(SourceManager.getActiveSource(ChainState.sourceManager(chainState)).name, ChainState.chainConfig(chainState).id, knownHeight);
   }
   Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - response.fromBlockQueried | 0) + 1 | 0);
@@ -133,7 +134,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
       applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, {
         blockNumber: latestFetchedBlockNumber,
         blockTimestamp: latestFetchedBlockTimestamp
-      }, query);
+      }, query, transactionStore);
       ChainMetadata.stage(state);
       scheduleFetch();
       return scheduleProcessing();
@@ -144,7 +145,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   }
   let newItemsWithDcs;
   try {
-    newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state));
+    newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), chainState, transactionStore);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
@@ -159,7 +160,7 @@ function finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFe
   let chainState = IndexerState.getChainState(state, chain);
   ChainState.updateKnownHeight(chainState, knownHeight);
   let isBelowReorgThreshold = !IndexerState.isInReorgThreshold(state) && IndexerState.config(state).shouldRollbackOnReorg;
-  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => FetchState.isReadyToEnterReorgThreshold(ChainState.fetchState(cs)));
+  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(ChainState.isReadyToEnterReorgThreshold);
   if (shouldEnterReorgThreshold) {
     IndexerState.enterReorgThreshold(state);
   }
@@ -167,19 +168,18 @@ function finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFe
   scheduleProcessing();
 }
 
-async function fetchChain(state, chain, concurrencyLimit, bufferLimit, stateId, scheduleFetch, scheduleProcessing, scheduleRollback) {
+async function fetchChain(state, chain, action, stateId, scheduleFetch, scheduleProcessing, scheduleRollback) {
   let chainState = IndexerState.getChainState(state, chain);
   if (IndexerState.isResolvingReorg(state) || IndexerState.isStopped(state)) {
     return;
   }
-  let fetchState = ChainState.fetchState(chainState);
   let isRealtime = IndexerState.isRealtime(state);
   let sourceManager = ChainState.sourceManager(chainState);
   let reducedPolling = !isRealtime;
   try {
-    return await SourceManager.fetchNext(sourceManager, fetchState, async query => {
+    return await ChainState.dispatch(chainState, async query => {
       try {
-        let response = await SourceManager.executeQuery(sourceManager, query, fetchState.knownHeight, isRealtime);
+        let response = await SourceManager.executeQuery(sourceManager, query, ChainState.knownHeight(chainState), isRealtime);
         return await onQueryResponse(state, {
           chain: chain,
           response: response,
@@ -189,7 +189,7 @@ async function fetchChain(state, chain, concurrencyLimit, bufferLimit, stateId,
         let exn = Primitive_exceptions.internalToException(raw_exn);
         return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
       }
-    }, knownHeight => SourceManager.waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling), knownHeight => finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFetch, scheduleProcessing), concurrencyLimit, bufferLimit, stateId);
+    }, knownHeight => SourceManager.waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling), knownHeight => finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFetch, scheduleProcessing), action, stateId);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, IndexerState.unexpectedErrorMsg));

package/src/ChainMetadata.res

@@ -7,19 +7,9 @@ let stage = (state: IndexerState.t) => {
   ->IndexerState.chainStates
   ->Dict.valuesToArray
   ->Array.forEach(cs => {
-    let fetchState = cs->ChainState.fetchState
     chainsData->Dict.set(
       (cs->ChainState.chainConfig).id->Int.toString,
-      {
-        firstEventBlockNumber: fetchState.firstEventBlock->Null.fromOption,
-        isHyperSync: (
-          cs->ChainState.sourceManager->SourceManager.getActiveSource
-        ).poweredByHyperSync,
-        latestFetchedBlockNumber: fetchState->FetchState.bufferBlockNumber,
-        timestampCaughtUpToHeadOrEndblock: cs
-        ->ChainState.timestampCaughtUpToHeadOrEndblock
-        ->Null.fromOption,
-      },
+      cs->ChainState.toChainMetadata,
     )
   })
 

package/src/ChainMetadata.res.mjs

@@ -2,21 +2,12 @@
 
 import * as Writing from "./Writing.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
-import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as IndexerState from "./IndexerState.res.mjs";
-import * as SourceManager from "./sources/SourceManager.res.mjs";
 
 function stage(state) {
   let chainsData = {};
   Object.values(IndexerState.chainStates(state)).forEach(cs => {
-    let fetchState = ChainState.fetchState(cs);
-    chainsData[ChainState.chainConfig(cs).id.toString()] = {
-      first_event_block: Stdlib_Null.fromOption(fetchState.firstEventBlock),
-      buffer_block: FetchState.bufferBlockNumber(fetchState),
-      ready_at: Stdlib_Null.fromOption(ChainState.timestampCaughtUpToHeadOrEndblock(cs)),
-      _is_hyper_sync: SourceManager.getActiveSource(ChainState.sourceManager(cs)).poweredByHyperSync
-    };
+    chainsData[ChainState.chainConfig(cs).id.toString()] = ChainState.toChainMetadata(cs);
   });
   Writing.setChainMeta(state, chainsData);
 }