envio 3.3.0-alpha.1 → 3.3.0-alpha.10

package/src/EventProcessing.res

@@ -39,7 +39,7 @@ let runEventHandlerOrThrow = async (
   let eventItem = item->Internal.castUnsafeEventItem
 
   //Include the load in time before handler
-  let timeBeforeHandler = Hrtime.makeTimer()
+  let timeBeforeHandler = Performance.now()
 
   try {
     let contextParams: UserContext.contextParams = {
@@ -72,10 +72,10 @@ let runEventHandlerOrThrow = async (
       }),
     )
   }
-  let handlerDuration = timeBeforeHandler->Hrtime.timeSince->Hrtime.toSecondsFloat
+  let handlerDuration = timeBeforeHandler->Performance.secondsSince
   Prometheus.ProcessingHandler.increment(
-    ~contract=eventItem.eventConfig.contractName,
-    ~event=eventItem.eventConfig.name,
+    ~contract=eventItem.onEventRegistration.eventConfig.contractName,
+    ~event=eventItem.onEventRegistration.eventConfig.name,
     ~duration=handlerDuration,
   )
 }
@@ -90,7 +90,7 @@ let runHandlerOrThrow = async (
   ~chains: Internal.chains,
 ) => {
   switch item {
-  | Block({onBlockConfig: {handler}, blockNumber}) =>
+  | Block({onBlockRegistration: {handler}, blockNumber}) =>
     try {
       let contextParams: UserContext.contextParams = {
         item,
@@ -121,8 +121,8 @@ let runHandlerOrThrow = async (
         }),
       )
     }
-  | Event({eventConfig}) =>
-    switch eventConfig.handler {
+  | Event({onEventRegistration}) =>
+    switch onEventRegistration.handler {
     | Some(handler) =>
       await item->runEventHandlerOrThrow(
         ~handler,
@@ -161,7 +161,7 @@ let preloadBatchOrThrow = async (
     for idx in 0 to checkpointEventsProcessed - 1 {
       let item = batch.items->Array.getUnsafe(itemIdx.contents + idx)
       switch item {
-      | Event({eventConfig: {handler, contractName, name: eventName}}) =>
+      | Event({onEventRegistration: {handler, eventConfig: {contractName, name: eventName}}}) =>
         switch handler {
         | None => ()
         | Some(handler) =>
@@ -200,7 +200,7 @@ let preloadBatchOrThrow = async (
           | _ => ()
           }
         }
-      | Block({onBlockConfig: {handler}, blockNumber}) =>
+      | Block({onBlockRegistration: {handler}, blockNumber}) =>
         try {
           promises->Array.push(
             handler({
@@ -264,11 +264,19 @@ let runBatchHandlersOrThrow = async (
   }
 }
 
-let registerProcessEventBatchMetrics = (~logger, ~loadDuration, ~handlerDuration) => {
-  logger->Logging.childTrace({
-    "msg": "Finished processing batch",
-    "loader_time_elapsed": loadDuration,
-    "handlers_time_elapsed": handlerDuration,
+let registerProcessEventBatchMetrics = (
+  ~logger,
+  ~batch: Batch.t,
+  ~loadDuration,
+  ~handlerDuration,
+) => {
+  batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
+    logger->Logging.childTrace({
+      "msg": "Finished processing",
+      "chainId": chainId->Int.fromString->Option.getUnsafe,
+      "batchSize": chainAfterBatch.batchSize,
+      "progress": chainAfterBatch.progressBlockNumber,
+    })
   })
 
   Prometheus.ProcessingBatch.registerMetrics(~loadDuration, ~handlerDuration)
@@ -281,6 +289,39 @@ type logPartitionInfo = {
   lastItemBlockNumber?: int,
 }
 
+// Off the hot path: bulk-materialise the selected transaction and block fields
+// for the batch's store-backed (HyperSync) items and write them onto the
+// payloads, so handlers read plain objects. A batch can span chains, each with
+// its own stores and field masks, so group items by chain before materialising.
+let materializeBatchEvents = async (
+  batch: Batch.t,
+  ~chainStates: dict<ChainState.t>,
+  ~ecosystem,
+) => {
+  switch chainStates->Dict.valuesToArray {
+  // Single-chain indexers (the common case): every item belongs to the one
+  // chain, so skip the per-chain grouping and its allocations.
+  | [cs] => await cs->ChainState.materializeBatchItems(~items=batch.items, ~ecosystem)
+  | _ =>
+    let itemsByChain: dict<array<Internal.item>> = Dict.make()
+    batch.items->Array.forEach(item => {
+      let chainId = item->Internal.getItemChainId->Int.toString
+      switch itemsByChain->Utils.Dict.dangerouslyGetNonOption(chainId) {
+      | Some(items) => items->Array.push(item)
+      | None => itemsByChain->Dict.set(chainId, [item])
+      }
+    })
+
+    let _ = await itemsByChain
+    ->Dict.toArray
+    ->Array.map(async ((chainId, items)) => {
+      let cs = chainStates->Dict.getUnsafe(chainId)
+      await cs->ChainState.materializeBatchItems(~items, ~ecosystem)
+    })
+    ->Promise.all
+  }
+}
+
 let processEventBatch = async (
   ~batch: Batch.t,
   ~indexerState: IndexerState.t,
@@ -289,33 +330,33 @@ let processEventBatch = async (
   ~config: Config.t,
   ~chainStates: dict<ChainState.t>,
 ) => {
-  let totalBatchSize = batch.totalBatchSize
   // Compute chains state for this batch
   let chains: Internal.chains = chainStates->computeChainsState
 
   let logger = Logging.getLogger()
-  logger->Logging.childTrace({
-    "msg": "Started processing batch",
-    "totalBatchSize": totalBatchSize,
-    "chains": batch.progressedChainsById->Utils.Dict.mapValues(chainAfterBatch => {
-      {
-        "batchSize": chainAfterBatch.batchSize,
-        "progress": chainAfterBatch.progressBlockNumber,
-      }
-    }),
+
+  batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
+    logger->Logging.childTrace({
+      "msg": "Started processing",
+      "chainId": chainId->Int.fromString->Option.getUnsafe,
+      "batchSize": chainAfterBatch.batchSize,
+    })
   })
 
   try {
     // Backpressure: keep processing within keepLatestChangesLimit of the cycle.
     await indexerState->Writing.awaitCapacity
 
-    let timeRef = Hrtime.makeTimer()
+    let timeRef = Performance.now()
 
     if batch.items->Utils.Array.notEmpty {
+      // Materialise store-backed transactions onto payloads before any handler
+      // (preload or execute) reads them.
+      await materializeBatchEvents(batch, ~chainStates, ~ecosystem=config.ecosystem.name)
       await batch->preloadBatchOrThrow(~loadManager, ~persistence, ~indexerState, ~chains, ~config)
     }
 
-    let elapsedTimeAfterLoaders = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let elapsedTimeAfterLoaders = timeRef->Performance.secondsSince
 
     if batch.items->Utils.Array.notEmpty {
       await batch->runBatchHandlersOrThrow(
@@ -327,13 +368,18 @@ let processEventBatch = async (
       )
     }
 
-    let elapsedTimeAfterProcessing = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let elapsedTimeAfterProcessing = timeRef->Performance.secondsSince
 
     indexerState->Writing.commitBatch(~batch)
 
     let loaderDuration = elapsedTimeAfterLoaders
     let handlerDuration = elapsedTimeAfterProcessing -. loaderDuration
-    registerProcessEventBatchMetrics(~logger, ~loadDuration=loaderDuration, ~handlerDuration)
+    registerProcessEventBatchMetrics(
+      ~logger,
+      ~batch,
+      ~loadDuration=loaderDuration,
+      ~handlerDuration,
+    )
     Ok()
   } catch {
   | Persistence.StorageError({message, reason}) =>

package/src/EventProcessing.res.mjs

@@ -1,13 +1,16 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
-import * as Hrtime from "./bindings/Hrtime.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Writing from "./Writing.res.mjs";
+import * as Internal from "./Internal.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
+import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
+import * as Performance from "./bindings/Performance.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
+import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
 import * as UserContext from "./UserContext.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
@@ -33,7 +36,7 @@ function computeChainsState(chainStates) {
 let ProcessingError = /* @__PURE__ */Primitive_exceptions.create("EventProcessing.ProcessingError");
 
 async function runEventHandlerOrThrow(item, checkpointId, handler, indexerState, loadManager, persistence, chains, config) {
-  let timeBeforeHandler = Hrtime.makeTimer();
+  let timeBeforeHandler = Performance.now();
   try {
     let contextParams = {
       item: item,
@@ -61,13 +64,13 @@ async function runEventHandlerOrThrow(item, checkpointId, handler, indexerState,
       Error: new Error()
     };
   }
-  let handlerDuration = Hrtime.toSecondsFloat(Hrtime.timeSince(timeBeforeHandler));
-  return Prometheus.ProcessingHandler.increment(item.eventConfig.contractName, item.eventConfig.name, handlerDuration);
+  let handlerDuration = Performance.secondsSince(timeBeforeHandler);
+  return Prometheus.ProcessingHandler.increment(item.onEventRegistration.eventConfig.contractName, item.onEventRegistration.eventConfig.name, handlerDuration);
 }
 
 async function runHandlerOrThrow(item, checkpointId, indexerState, loadManager, persistence, config, chains) {
   if (item.kind === 0) {
-    let handler = item.eventConfig.handler;
+    let handler = item.onEventRegistration.handler;
     if (handler !== undefined) {
       return await runEventHandlerOrThrow(item, checkpointId, handler, indexerState, loadManager, persistence, chains, config);
     } else {
@@ -86,7 +89,7 @@ async function runHandlerOrThrow(item, checkpointId, indexerState, loadManager,
       config: config,
       isResolved: false
     };
-    await item.onBlockConfig.handler(Ecosystem.makeOnBlockArgs(item.blockNumber, config.ecosystem, UserContext.getHandlerContext(contextParams)));
+    await item.onBlockRegistration.handler(Ecosystem.makeOnBlockArgs(item.blockNumber, config.ecosystem, UserContext.getHandlerContext(contextParams)));
     contextParams.isResolved = true;
     return;
   } catch (raw_exn) {
@@ -110,11 +113,12 @@ async function preloadBatchOrThrow(batch, loadManager, persistence, config, inde
     for (let idx = 0; idx < checkpointEventsProcessed; ++idx) {
       let item = batch.items[itemIdx + idx | 0];
       if (item.kind === 0) {
-        let match = item.eventConfig;
+        let match = item.onEventRegistration;
         let handler = match.handler;
         if (handler !== undefined) {
-          let contractName = match.contractName;
-          let eventName = match.name;
+          let match$1 = match.eventConfig;
+          let contractName = match$1.contractName;
+          let eventName = match$1.name;
           try {
             let timerRef = Prometheus.PreloadHandler.startOperation(contractName, eventName);
             promises.push(Utils.$$Promise.silentCatch(handler({
@@ -137,7 +141,7 @@ async function preloadBatchOrThrow(batch, loadManager, persistence, config, inde
         }
       } else {
         try {
-          promises.push(Utils.$$Promise.silentCatch(item.onBlockConfig.handler(Ecosystem.makeOnBlockArgs(item.blockNumber, config.ecosystem, UserContext.getHandlerContext({
+          promises.push(Utils.$$Promise.silentCatch(item.onBlockRegistration.handler(Ecosystem.makeOnBlockArgs(item.blockNumber, config.ecosystem, UserContext.getHandlerContext({
             item: item,
             checkpointId: checkpointId,
             indexerState: indexerState,
@@ -171,41 +175,62 @@ async function runBatchHandlersOrThrow(batch, indexerState, loadManager, persist
   }
 }
 
-function registerProcessEventBatchMetrics(logger, loadDuration, handlerDuration) {
-  Logging.childTrace(logger, {
-    msg: "Finished processing batch",
-    loader_time_elapsed: loadDuration,
-    handlers_time_elapsed: handlerDuration
-  });
+function registerProcessEventBatchMetrics(logger, batch, loadDuration, handlerDuration) {
+  Stdlib_Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, chainId) => Logging.childTrace(logger, {
+    msg: "Finished processing",
+    chainId: Stdlib_Int.fromString(chainId, undefined),
+    batchSize: chainAfterBatch.batchSize,
+    progress: chainAfterBatch.progressBlockNumber
+  }));
   Prometheus.ProcessingBatch.registerMetrics(loadDuration, handlerDuration);
 }
 
+async function materializeBatchEvents(batch, chainStates, ecosystem) {
+  let match = Object.values(chainStates);
+  if (match.length !== 1) {
+    let itemsByChain = {};
+    batch.items.forEach(item => {
+      let chainId = Internal.getItemChainId(item).toString();
+      let items = itemsByChain[chainId];
+      if (items !== undefined) {
+        items.push(item);
+      } else {
+        itemsByChain[chainId] = [item];
+      }
+    });
+    await Promise.all(Object.entries(itemsByChain).map(async param => {
+      let cs = chainStates[param[0]];
+      return await ChainState.materializeBatchItems(cs, param[1], ecosystem);
+    }));
+    return;
+  }
+  let cs = match[0];
+  return await ChainState.materializeBatchItems(cs, batch.items, ecosystem);
+}
+
 async function processEventBatch(batch, indexerState, loadManager, persistence, config, chainStates) {
-  let totalBatchSize = batch.totalBatchSize;
   let chains = computeChainsState(chainStates);
   let logger = Logging.getLogger();
-  Logging.childTrace(logger, {
-    msg: "Started processing batch",
-    totalBatchSize: totalBatchSize,
-    chains: Utils.Dict.mapValues(batch.progressedChainsById, chainAfterBatch => ({
-      batchSize: chainAfterBatch.batchSize,
-      progress: chainAfterBatch.progressBlockNumber
-    }))
-  });
+  Stdlib_Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, chainId) => Logging.childTrace(logger, {
+    msg: "Started processing",
+    chainId: Stdlib_Int.fromString(chainId, undefined),
+    batchSize: chainAfterBatch.batchSize
+  }));
   try {
     await Writing.awaitCapacity(indexerState);
-    let timeRef = Hrtime.makeTimer();
+    let timeRef = Performance.now();
     if (Utils.$$Array.notEmpty(batch.items)) {
+      await materializeBatchEvents(batch, chainStates, config.ecosystem.name);
       await preloadBatchOrThrow(batch, loadManager, persistence, config, indexerState, chains);
     }
-    let elapsedTimeAfterLoaders = Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef));
+    let elapsedTimeAfterLoaders = Performance.secondsSince(timeRef);
     if (Utils.$$Array.notEmpty(batch.items)) {
       await runBatchHandlersOrThrow(batch, indexerState, loadManager, persistence, config, chains);
     }
-    let elapsedTimeAfterProcessing = Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef));
+    let elapsedTimeAfterProcessing = Performance.secondsSince(timeRef);
     Writing.commitBatch(indexerState, batch);
     let handlerDuration = elapsedTimeAfterProcessing - elapsedTimeAfterLoaders;
-    registerProcessEventBatchMetrics(logger, elapsedTimeAfterLoaders, handlerDuration);
+    registerProcessEventBatchMetrics(logger, batch, elapsedTimeAfterLoaders, handlerDuration);
     return {
       TAG: "Ok",
       _0: undefined
@@ -240,6 +265,7 @@ export {
   preloadBatchOrThrow,
   runBatchHandlersOrThrow,
   registerProcessEventBatchMetrics,
+  materializeBatchEvents,
   processEventBatch,
 }
 /* Utils Not a pure module */

package/src/ExitOnCaughtUp.res

@@ -4,7 +4,15 @@ let run = async (state: IndexerState.t) => {
   ChainMetadata.stage(state)
   await state->Writing.flush
   if !(state->IndexerState.isStopped) && !(state->IndexerState.isResolvingReorg) {
-    Logging.info("Exiting with success")
-    NodeJs.process->NodeJs.exitWithCode(Success)
+    // A simulate run fails here when a provided item never reached a handler —
+    // dead test code. The tracker is None (a no-op) off the simulate path.
+    switch state
+    ->IndexerState.simulateDeadInputTracker
+    ->Option.flatMap(SimulateDeadInputTracker.failureMessage) {
+    | None =>
+      Logging.info("Exiting with success")
+      NodeJs.process->NodeJs.exitWithCode(Success)
+    | Some(message) => state->IndexerState.errorExit(ErrorHandling.make(Utils.Error.make(message)))
+    }
   }
 }

package/src/ExitOnCaughtUp.res.mjs

@@ -5,11 +5,20 @@ import * as Writing from "./Writing.res.mjs";
 import * as Process from "process";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
+import * as ErrorHandling from "./ErrorHandling.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
 
 async function run(state) {
   ChainMetadata.stage(state);
   await Writing.flush(state);
-  if (!IndexerState.isStopped(state) && !IndexerState.isResolvingReorg(state)) {
+  if (IndexerState.isStopped(state) || IndexerState.isResolvingReorg(state)) {
+    return;
+  }
+  let message = Stdlib_Option.flatMap(IndexerState.simulateDeadInputTracker(state), SimulateDeadInputTracker.failureMessage);
+  if (message !== undefined) {
+    return IndexerState.errorExit(state, ErrorHandling.make(new Error(message), undefined, undefined));
+  } else {
     Logging.info("Exiting with success");
     Process.exit(0);
     return;