envio 3.3.0-alpha.1 → 3.3.0-alpha.11

package/src/ChainFetching.res

@@ -11,7 +11,19 @@ type partitionQueryResponse = {
 let runContractRegistersOrThrow = async (
   ~itemsWithContractRegister: array<Internal.item>,
   ~config: Config.t,
+  ~transactionStore: option<TransactionStore.t>,
+  ~blockStore: option<BlockStore.t>,
 ) => {
+  // contractRegister handlers can read event.transaction and event.block, so
+  // materialise the selected fields onto the payloads before running them. All
+  // items belong to the chain being fetched, hence its single page stores.
+  await ChainState.materializePageItems(
+    ~items=itemsWithContractRegister,
+    ~transactionStore,
+    ~blockStore,
+    ~ecosystem=config.ecosystem.name,
+  )
+
   let itemsWithDcs = []
 
   let onRegister = (~item: Internal.item, ~contractAddress, ~contractName) => {
@@ -37,9 +49,9 @@ let runContractRegistersOrThrow = async (
   for idx in 0 to itemsWithContractRegister->Array.length - 1 {
     let item = itemsWithContractRegister->Array.getUnsafe(idx)
     let eventItem = item->Internal.castUnsafeEventItem
-    let contractRegister = switch eventItem {
-    | {eventConfig: {contractRegister: Some(contractRegister)}} => contractRegister
-    | {eventConfig: {contractRegister: None, name: eventName}} =>
+    let contractRegister = switch eventItem.onEventRegistration {
+    | {contractRegister: Some(contractRegister)} => contractRegister
+    | {contractRegister: None, eventConfig: {name: eventName}} =>
       // Unexpected case, since we should pass only events with contract register to this function
       JsError.throwWithMessage("Contract register is not set for event " ++ eventName)
     }
@@ -106,6 +118,8 @@ let rec onQueryResponse = async (
     let chainState = state->IndexerState.getChainState(~chain)
     let {
       parsedQueueItems,
+      transactionStore,
+      blockStore,
       latestFetchedBlockNumber,
       latestFetchedBlockTimestamp,
       stats,
@@ -114,7 +128,7 @@ let rec onQueryResponse = async (
       fromBlockQueried,
     } = response
 
-    if knownHeight > (chainState->ChainState.fetchState).knownHeight {
+    if knownHeight > chainState->ChainState.knownHeight {
       Prometheus.SourceHeight.set(
         ~blockNumber=knownHeight,
         ~chainId=(chainState->ChainState.chainConfig).id,
@@ -133,6 +147,22 @@ let rec onQueryResponse = async (
       ~blockRangeSize=latestFetchedBlockNumber - fromBlockQueried + 1,
     )
 
+    let numContractRegisterEvents = parsedQueueItems->Array.reduce(0, (count, item) => {
+      let eventItem = item->Internal.castUnsafeEventItem
+      eventItem.onEventRegistration.contractRegister !== None
+        ? count + 1
+        : count
+    })
+    Logging.trace({
+      "msg": "Finished querying",
+      "chainId": chain->ChainMap.Chain.toChainId,
+      "partitionId": query.partitionId,
+      "fromBlock": fromBlockQueried,
+      "toBlock": latestFetchedBlockNumber,
+      "numEvents": parsedQueueItems->Array.length,
+      "numContractRegisterEvents": numContractRegisterEvents,
+    })
+
     let reorgResult = chainState->ChainState.registerReorgGuard(~blockHashes, ~knownHeight)
 
     let rollbackWithReorgDetectedBlockNumber = switch reorgResult {
@@ -184,17 +214,18 @@ let rec onQueryResponse = async (
       // kick (eg from the processing loop quiescing) collapses into this one.
       scheduleRollback()
     | None =>
+      // Drop over-fetched events (a merged partition returning an address before
+      // its effectiveStartBlock, or a wildcard param referencing an address
+      // registered after the log's block) before contract registration, so they
+      // neither spawn dynamic contracts nor enter the buffer.
+      let newItems = chainState->ChainState.filterByClientAddress(parsedQueueItems)
       let itemsWithContractRegister = []
-      let newItems = []
-      for idx in 0 to parsedQueueItems->Array.length - 1 {
-        let item = parsedQueueItems->Array.getUnsafe(idx)
+      for idx in 0 to newItems->Array.length - 1 {
+        let item = newItems->Array.getUnsafe(idx)
         let eventItem = item->Internal.castUnsafeEventItem
-        if eventItem.eventConfig.contractRegister !== None {
+        if eventItem.onEventRegistration.contractRegister !== None {
           itemsWithContractRegister->Array.push(item)
         }
-        // TODO: Don't really need to keep it in the queue
-        // when there's no handler (besides raw_events, processed counter, and dcsToStore consuming)
-        newItems->Array.push(item)
       }
 
       // Re-check staleness: contract registration is async, so the chain state
@@ -212,6 +243,8 @@ let rec onQueryResponse = async (
               blockTimestamp: latestFetchedBlockTimestamp,
             },
             ~query,
+            ~transactionStore,
+            ~blockStore,
           )
           ChainMetadata.stage(state)
           scheduleFetch()
@@ -224,6 +257,8 @@ let rec onQueryResponse = async (
         switch await runContractRegistersOrThrow(
           ~itemsWithContractRegister,
           ~config=state->IndexerState.config,
+          ~transactionStore,
+          ~blockStore,
         ) {
         | exception exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
         | newItemsWithDcs => proceed(~newItemsWithDcs)
@@ -240,6 +275,8 @@ and applyQueryResponse = (
   ~knownHeight,
   ~latestFetchedBlock,
   ~query,
+  ~transactionStore,
+  ~blockStore,
 ) => {
   let chainState = state->IndexerState.getChainState(~chain)
   let wasFetchingAtHead = chainState->ChainState.isFetchingAtHead
@@ -250,6 +287,8 @@ and applyQueryResponse = (
     ~newItems,
     ~newItemsWithDcs,
     ~knownHeight,
+    ~transactionStore,
+    ~blockStore,
   )
 
   // In auto-exit mode, set endBlock to the first event's block when events arrive.
@@ -294,7 +333,7 @@ let finishWaitingForNewBlock = (
       ->IndexerState.chainStates
       ->Dict.valuesToArray
       ->Array.every(cs => {
-        cs->ChainState.fetchState->FetchState.isReadyToEnterReorgThreshold
+        cs->ChainState.isReadyToEnterReorgThreshold
       })
 
     // Kick processing in case there are block handlers to run.
@@ -316,7 +355,6 @@ let fetchChain = async (
 ) => {
   let chainState = state->IndexerState.getChainState(~chain)
   if !(state->IndexerState.isResolvingReorg) && !(state->IndexerState.isStopped) {
-    let fetchState = chainState->ChainState.fetchState
     let isRealtime = state->IndexerState.isRealtime
     let sourceManager = chainState->ChainState.sourceManager
 
@@ -327,8 +365,7 @@ let fetchChain = async (
     // Owns its error boundary: launch doesn't catch, so any failure here (the
     // query, response handling, or dispatch itself) must stop the indexer.
     try {
-      await sourceManager->SourceManager.dispatch(
-        ~fetchState,
+      await chainState->ChainState.dispatch(
         ~waitForNewBlock=(~knownHeight) =>
           sourceManager->SourceManager.waitForNewBlock(~knownHeight, ~isRealtime, ~reducedPolling),
         ~onNewBlock=(~knownHeight) =>
@@ -347,7 +384,7 @@ let fetchChain = async (
           try {
             let response = await sourceManager->SourceManager.executeQuery(
               ~query,
-              ~knownHeight=fetchState.knownHeight,
+              ~knownHeight=chainState->ChainState.knownHeight,
               ~isRealtime,
             )
             await onQueryResponse(

package/src/ChainFetching.res.mjs

@@ -4,9 +4,9 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as SourceManager from "./sources/SourceManager.res.mjs";
@@ -17,7 +17,8 @@ import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 import * as ContractRegisterContext from "./ContractRegisterContext.res.mjs";
 
-async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
+async function runContractRegistersOrThrow(itemsWithContractRegister, config, transactionStore, blockStore) {
+  await ChainState.materializePageItems(itemsWithContractRegister, transactionStore, blockStore, config.ecosystem.name);
   let itemsWithDcs = [];
   let onRegister = (item, contractAddress, contractName) => {
     let dc_registrationBlock = item.blockNumber;
@@ -37,9 +38,9 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
   let promises = [];
   for (let idx = 0, idx_finish = itemsWithContractRegister.length; idx < idx_finish; ++idx) {
     let item = itemsWithContractRegister[idx];
-    let match = item.eventConfig;
+    let match = item.onEventRegistration;
     let contractRegister = match.contractRegister;
-    let contractRegister$1 = contractRegister !== undefined ? contractRegister : Stdlib_JsError.throwWithMessage("Contract register is not set for event " + match.name);
+    let contractRegister$1 = contractRegister !== undefined ? contractRegister : Stdlib_JsError.throwWithMessage("Contract register is not set for event " + match.eventConfig.name);
     let errorMessage = "Event contractRegister failed, please fix the error to keep the indexer running smoothly";
     try {
       let params = {
@@ -71,10 +72,10 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
   return itemsWithDcs;
 }
 
-function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query) {
+function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query, transactionStore, blockStore) {
   let chainState = IndexerState.getChainState(state, chain);
   let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
-  ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight);
+  ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, transactionStore, blockStore);
   if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
     ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
   }
@@ -94,12 +95,31 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   let stats = response.stats;
   let latestFetchedBlockTimestamp = response.latestFetchedBlockTimestamp;
   let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
+  let fromBlockQueried = response.fromBlockQueried;
+  let blockStore = response.blockStore;
+  let transactionStore = response.transactionStore;
   let parsedQueueItems = response.parsedQueueItems;
   let knownHeight = response.knownHeight;
-  if (knownHeight > ChainState.fetchState(chainState).knownHeight) {
+  if (knownHeight > ChainState.knownHeight(chainState)) {
     Prometheus.SourceHeight.set(SourceManager.getActiveSource(ChainState.sourceManager(chainState)).name, ChainState.chainConfig(chainState).id, knownHeight);
   }
-  Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - response.fromBlockQueried | 0) + 1 | 0);
+  Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - fromBlockQueried | 0) + 1 | 0);
+  let numContractRegisterEvents = Stdlib_Array.reduce(parsedQueueItems, 0, (count, item) => {
+    if (item.onEventRegistration.contractRegister !== undefined) {
+      return count + 1 | 0;
+    } else {
+      return count;
+    }
+  });
+  Logging.trace({
+    msg: "Finished querying",
+    chainId: chain,
+    partitionId: query.partitionId,
+    fromBlock: fromBlockQueried,
+    toBlock: latestFetchedBlockNumber,
+    numEvents: parsedQueueItems.length,
+    numContractRegisterEvents: numContractRegisterEvents
+  });
   let reorgResult = ChainState.registerReorgGuard(chainState, response.blockHashes, knownHeight);
   let rollbackWithReorgDetectedBlockNumber;
   if (typeof reorgResult !== "object") {
@@ -119,21 +139,20 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
     IndexerState.beginReorg(state, chain, rollbackWithReorgDetectedBlockNumber);
     return scheduleRollback();
   }
+  let newItems = ChainState.filterByClientAddress(chainState, parsedQueueItems);
   let itemsWithContractRegister = [];
-  let newItems = [];
-  for (let idx = 0, idx_finish = parsedQueueItems.length; idx < idx_finish; ++idx) {
-    let item = parsedQueueItems[idx];
-    if (item.eventConfig.contractRegister !== undefined) {
+  for (let idx = 0, idx_finish = newItems.length; idx < idx_finish; ++idx) {
+    let item = newItems[idx];
+    if (item.onEventRegistration.contractRegister !== undefined) {
       itemsWithContractRegister.push(item);
     }
-    newItems.push(item);
   }
   let proceed = newItemsWithDcs => {
     if (!IndexerState.isStale(state, stateId)) {
       applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, {
         blockNumber: latestFetchedBlockNumber,
         blockTimestamp: latestFetchedBlockTimestamp
-      }, query);
+      }, query, transactionStore, blockStore);
       ChainMetadata.stage(state);
       scheduleFetch();
       return scheduleProcessing();
@@ -144,7 +163,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   }
   let newItemsWithDcs;
   try {
-    newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state));
+    newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), transactionStore, blockStore);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
@@ -159,7 +178,7 @@ function finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFe
   let chainState = IndexerState.getChainState(state, chain);
   ChainState.updateKnownHeight(chainState, knownHeight);
   let isBelowReorgThreshold = !IndexerState.isInReorgThreshold(state) && IndexerState.config(state).shouldRollbackOnReorg;
-  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => FetchState.isReadyToEnterReorgThreshold(ChainState.fetchState(cs)));
+  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(ChainState.isReadyToEnterReorgThreshold);
   if (shouldEnterReorgThreshold) {
     IndexerState.enterReorgThreshold(state);
   }
@@ -172,14 +191,13 @@ async function fetchChain(state, chain, action, stateId, scheduleFetch, schedule
   if (IndexerState.isResolvingReorg(state) || IndexerState.isStopped(state)) {
     return;
   }
-  let fetchState = ChainState.fetchState(chainState);
   let isRealtime = IndexerState.isRealtime(state);
   let sourceManager = ChainState.sourceManager(chainState);
   let reducedPolling = !isRealtime;
   try {
-    return await SourceManager.dispatch(sourceManager, fetchState, async query => {
+    return await ChainState.dispatch(chainState, async query => {
       try {
-        let response = await SourceManager.executeQuery(sourceManager, query, fetchState.knownHeight, isRealtime);
+        let response = await SourceManager.executeQuery(sourceManager, query, ChainState.knownHeight(chainState), isRealtime);
         return await onQueryResponse(state, {
           chain: chain,
           response: response,

package/src/ChainMetadata.res

@@ -7,19 +7,9 @@ let stage = (state: IndexerState.t) => {
   ->IndexerState.chainStates
   ->Dict.valuesToArray
   ->Array.forEach(cs => {
-    let fetchState = cs->ChainState.fetchState
     chainsData->Dict.set(
       (cs->ChainState.chainConfig).id->Int.toString,
-      {
-        firstEventBlockNumber: fetchState.firstEventBlock->Null.fromOption,
-        isHyperSync: (
-          cs->ChainState.sourceManager->SourceManager.getActiveSource
-        ).poweredByHyperSync,
-        latestFetchedBlockNumber: fetchState->FetchState.bufferBlockNumber,
-        timestampCaughtUpToHeadOrEndblock: cs
-        ->ChainState.timestampCaughtUpToHeadOrEndblock
-        ->Null.fromOption,
-      },
+      cs->ChainState.toChainMetadata,
     )
   })
 

package/src/ChainMetadata.res.mjs

@@ -2,21 +2,12 @@
 
 import * as Writing from "./Writing.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
-import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as IndexerState from "./IndexerState.res.mjs";
-import * as SourceManager from "./sources/SourceManager.res.mjs";
 
 function stage(state) {
   let chainsData = {};
   Object.values(IndexerState.chainStates(state)).forEach(cs => {
-    let fetchState = ChainState.fetchState(cs);
-    chainsData[ChainState.chainConfig(cs).id.toString()] = {
-      first_event_block: Stdlib_Null.fromOption(fetchState.firstEventBlock),
-      buffer_block: FetchState.bufferBlockNumber(fetchState),
-      ready_at: Stdlib_Null.fromOption(ChainState.timestampCaughtUpToHeadOrEndblock(cs)),
-      _is_hyper_sync: SourceManager.getActiveSource(ChainState.sourceManager(cs)).poweredByHyperSync
-    };
+    chainsData[ChainState.chainConfig(cs).id.toString()] = ChainState.toChainMetadata(cs);
   });
   Writing.setChainMeta(state, chainsData);
 }