envio 3.3.0-alpha.1 → 3.3.0-alpha.3

package/src/EventProcessing.res.mjs

@@ -1,13 +1,16 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
-import * as Hrtime from "./bindings/Hrtime.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Writing from "./Writing.res.mjs";
+import * as Internal from "./Internal.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
+import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
+import * as Performance from "./bindings/Performance.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
+import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
 import * as UserContext from "./UserContext.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
@@ -33,7 +36,7 @@ function computeChainsState(chainStates) {
 let ProcessingError = /* @__PURE__ */Primitive_exceptions.create("EventProcessing.ProcessingError");
 
 async function runEventHandlerOrThrow(item, checkpointId, handler, indexerState, loadManager, persistence, chains, config) {
-  let timeBeforeHandler = Hrtime.makeTimer();
+  let timeBeforeHandler = Performance.now();
   try {
     let contextParams = {
       item: item,
@@ -61,7 +64,7 @@ async function runEventHandlerOrThrow(item, checkpointId, handler, indexerState,
       Error: new Error()
     };
   }
-  let handlerDuration = Hrtime.toSecondsFloat(Hrtime.timeSince(timeBeforeHandler));
+  let handlerDuration = Performance.secondsSince(timeBeforeHandler);
   return Prometheus.ProcessingHandler.increment(item.eventConfig.contractName, item.eventConfig.name, handlerDuration);
 }
 
@@ -171,41 +174,62 @@ async function runBatchHandlersOrThrow(batch, indexerState, loadManager, persist
   }
 }
 
-function registerProcessEventBatchMetrics(logger, loadDuration, handlerDuration) {
-  Logging.childTrace(logger, {
-    msg: "Finished processing batch",
-    loader_time_elapsed: loadDuration,
-    handlers_time_elapsed: handlerDuration
-  });
+function registerProcessEventBatchMetrics(logger, batch, loadDuration, handlerDuration) {
+  Stdlib_Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, chainId) => Logging.childTrace(logger, {
+    msg: "Finished processing",
+    chainId: Stdlib_Int.fromString(chainId, undefined),
+    batchSize: chainAfterBatch.batchSize,
+    progress: chainAfterBatch.progressBlockNumber
+  }));
   Prometheus.ProcessingBatch.registerMetrics(loadDuration, handlerDuration);
 }
 
+async function materializeBatchTransactions(batch, chainStates) {
+  let match = Object.values(chainStates);
+  if (match.length !== 1) {
+    let itemsByChain = {};
+    batch.items.forEach(item => {
+      let chainId = Internal.getItemChainId(item).toString();
+      let items = itemsByChain[chainId];
+      if (items !== undefined) {
+        items.push(item);
+      } else {
+        itemsByChain[chainId] = [item];
+      }
+    });
+    await Promise.all(Object.entries(itemsByChain).map(async param => {
+      let cs = chainStates[param[0]];
+      return await ChainState.materializeBatchItems(cs, param[1]);
+    }));
+    return;
+  }
+  let cs = match[0];
+  return await ChainState.materializeBatchItems(cs, batch.items);
+}
+
 async function processEventBatch(batch, indexerState, loadManager, persistence, config, chainStates) {
-  let totalBatchSize = batch.totalBatchSize;
   let chains = computeChainsState(chainStates);
   let logger = Logging.getLogger();
-  Logging.childTrace(logger, {
-    msg: "Started processing batch",
-    totalBatchSize: totalBatchSize,
-    chains: Utils.Dict.mapValues(batch.progressedChainsById, chainAfterBatch => ({
-      batchSize: chainAfterBatch.batchSize,
-      progress: chainAfterBatch.progressBlockNumber
-    }))
-  });
+  Stdlib_Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, chainId) => Logging.childTrace(logger, {
+    msg: "Started processing",
+    chainId: Stdlib_Int.fromString(chainId, undefined),
+    batchSize: chainAfterBatch.batchSize
+  }));
   try {
     await Writing.awaitCapacity(indexerState);
-    let timeRef = Hrtime.makeTimer();
+    let timeRef = Performance.now();
     if (Utils.$$Array.notEmpty(batch.items)) {
+      await materializeBatchTransactions(batch, chainStates);
       await preloadBatchOrThrow(batch, loadManager, persistence, config, indexerState, chains);
     }
-    let elapsedTimeAfterLoaders = Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef));
+    let elapsedTimeAfterLoaders = Performance.secondsSince(timeRef);
     if (Utils.$$Array.notEmpty(batch.items)) {
       await runBatchHandlersOrThrow(batch, indexerState, loadManager, persistence, config, chains);
     }
-    let elapsedTimeAfterProcessing = Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef));
+    let elapsedTimeAfterProcessing = Performance.secondsSince(timeRef);
     Writing.commitBatch(indexerState, batch);
     let handlerDuration = elapsedTimeAfterProcessing - elapsedTimeAfterLoaders;
-    registerProcessEventBatchMetrics(logger, elapsedTimeAfterLoaders, handlerDuration);
+    registerProcessEventBatchMetrics(logger, batch, elapsedTimeAfterLoaders, handlerDuration);
     return {
       TAG: "Ok",
       _0: undefined
@@ -240,6 +264,7 @@ export {
   preloadBatchOrThrow,
   runBatchHandlersOrThrow,
   registerProcessEventBatchMetrics,
+  materializeBatchTransactions,
   processEventBatch,
 }
 /* Utils Not a pure module */

package/src/FetchState.res

@@ -41,6 +41,12 @@ type partition = {
   latestFetchedBlock: blockNumberAndTimestamp,
   selection: selection,
   addressesByContractName: dict<array<Address.t>>,
+  // Reverse index address→contractName, derived from addressesByContractName.
+  // Used by EventRouter to resolve a log's owning contract from the partition
+  // that fetched it (instead of a chain-wide snapshot). Always recomputed in
+  // OptimizedPartitions.make, so partition literals can seed it with an empty
+  // dict.
+  contractNameByAddress: dict<string>,
   mergeBlock: option<int>,
   // When set, partition indexes a single dynamic contract type.
   // The addressesByContractName must contain only addresses for this contract.
@@ -76,7 +82,23 @@ type query = {
   mutable progress: float,
   selection: selection,
   addressesByContractName: dict<array<Address.t>>,
-  indexingAddresses: dict<indexingAddress>,
+  // The owning partition's reverse index, referenced (not copied) so routing
+  // resolves ownership without a chain-wide snapshot.
+  contractNameByAddress: dict<string>,
+}
+
+// Invert addressesByContractName into address→contractName. 1:1 today (each
+// address belongs to one contract), so no key collisions.
+let deriveContractNameByAddress = (addressesByContractName: dict<array<Address.t>>): dict<
+  string,
+> => {
+  let result = Dict.make()
+  addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
+    for i in 0 to addresses->Array.length - 1 {
+      result->Dict.set(addresses->Array.getUnsafe(i)->Address.toString, contractName)
+    }
+  })
+  result
 }
 
 // Default estimate for a query whose partition hasn't responded yet, so the
@@ -192,6 +214,7 @@ module OptimizedPartitions = {
         latestFetchedBlock: {blockNumber: potentialMergeBlock, blockTimestamp: 0},
         mergeBlock: None,
         addressesByContractName: Dict.make(), // set below
+        contractNameByAddress: Dict.make(), // derived in make
         mutPendingQueries: [],
         prevQueryRange: minRange,
         prevPrevQueryRange: minRange,
@@ -374,7 +397,12 @@ module OptimizedPartitions = {
     for idx in 0 to partitionsCount - 1 {
       let p = newPartitions->Array.getUnsafe(idx)
       idsInAscOrder->Array.setUnsafe(idx, p.id)
-      entities->Dict.set(p.id, p)
+      // Single point where every partition's reverse index is (re)derived, so all
+      // construction paths (literals, spreads, merges, splits) stay consistent.
+      entities->Dict.set(
+        p.id,
+        {...p, contractNameByAddress: deriveContractNameByAddress(p.addressesByContractName)},
+      )
     }
 
     {
@@ -873,6 +901,7 @@ OptimizedPartitions.t => {
             selection: normalSelection,
             dynamicContract: isDynamic ? Some(contractName) : None,
             addressesByContractName,
+            contractNameByAddress: Dict.make(), // derived in make
             mergeBlock: None,
             mutPendingQueries: [],
             prevQueryRange: 0,
@@ -1215,6 +1244,7 @@ let registerDynamicContracts = (
                         selection: fetchState.normalSelection,
                         dynamicContract: Some(contractName),
                         addressesByContractName,
+                        contractNameByAddress: Dict.make(), // derived in make
                         mergeBlock: None,
                         mutPendingQueries: p.mutPendingQueries,
                         prevQueryRange: p.prevQueryRange,
@@ -1329,6 +1359,9 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
   }
 }
 
+// Most parallel in-flight chunk queries a single partition may have at once.
+let maxPendingChunksPerPartition = 10
+
 @inline
 let pushQueriesForRange = (
   queries: array<query>,
@@ -1337,12 +1370,12 @@ let pushQueriesForRange = (
   ~rangeEndBlock: option<int>,
   ~maxQueryBlockNumber: int,
   ~maybeChunkRange: option<int>,
+  ~maxChunks: int,
   ~partition: partition,
   ~selection: selection,
   ~addressesByContractName: dict<array<Address.t>>,
-  ~indexingAddresses: dict<indexingAddress>,
 ) => {
-  if rangeFromBlock <= maxQueryBlockNumber {
+  if rangeFromBlock <= maxQueryBlockNumber && maxChunks > 0 {
     switch rangeEndBlock {
     | Some(endBlock) if rangeFromBlock > endBlock => ()
     | _ =>
@@ -1363,7 +1396,7 @@ let pushQueriesForRange = (
           chainId: 0,
           progress: 0.,
           addressesByContractName,
-          indexingAddresses,
+          contractNameByAddress: partition.contractNameByAddress,
         })
       | Some(chunkRange) =>
         let maxBlock = switch rangeEndBlock {
@@ -1371,18 +1404,13 @@ let pushQueriesForRange = (
         | None => maxQueryBlockNumber
         }
         let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
-        // Probe with two smaller chunks first so their responses come back
-        // quickly and refresh the chunking heuristic, then three full-size chunks.
-        let probeSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 0.9)
-        let getChunkSize = chunkIdx => chunkIdx < 2 ? probeSize : chunkSize
-        if rangeFromBlock + getChunkSize(0) + getChunkSize(1) - 1 <= maxBlock {
+        if rangeFromBlock + chunkSize * 2 - 1 <= maxBlock {
           let chunkFromBlock = ref(rangeFromBlock)
           let chunkIdx = ref(0)
           while (
-            chunkIdx.contents < 5 &&
-              chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1 <= maxBlock
+            chunkIdx.contents < maxChunks && chunkFromBlock.contents + chunkSize - 1 <= maxBlock
           ) {
-            let chunkToBlock = chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1
+            let chunkToBlock = chunkFromBlock.contents + chunkSize - 1
             queries->Array.push({
               partitionId,
               fromBlock: chunkFromBlock.contents,
@@ -1398,7 +1426,7 @@ let pushQueriesForRange = (
               chainId: 0,
               progress: 0.,
               addressesByContractName,
-              indexingAddresses,
+              contractNameByAddress: partition.contractNameByAddress,
             })
             chunkFromBlock := chunkToBlock + 1
             chunkIdx := chunkIdx.contents + 1
@@ -1420,7 +1448,7 @@ let pushQueriesForRange = (
             chainId: 0,
             progress: 0.,
             addressesByContractName,
-            indexingAddresses,
+            contractNameByAddress: partition.contractNameByAddress,
           })
         }
       }
@@ -1428,18 +1456,8 @@ let pushQueriesForRange = (
   }
 }
 
-// Most parallel in-flight chunk queries a single partition may have at once.
-let maxPendingChunksPerPartition = 10
-
 let getNextQuery = (
-  {
-    buffer,
-    optimizedPartitions,
-    indexingAddresses,
-    blockLag,
-    latestOnBlockBlockNumber,
-    knownHeight,
-  } as fetchState: t,
+  {buffer, optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight} as fetchState: t,
   ~budget,
   ~chainPendingBudget,
 ) => {
@@ -1533,10 +1551,13 @@ let getNextQuery = (
             ~rangeEndBlock=Utils.Math.minOptInt(Some(pq.fromBlock - 1), queryEndBlock),
             ~maxQueryBlockNumber,
             ~maybeChunkRange,
+            ~maxChunks=maxPendingChunksPerPartition -
+            pendingCount -
+            (queries->Array.length -
+            partitionQueriesStart),
             ~partition=p,
             ~selection=p.selection,
             ~addressesByContractName=p.addressesByContractName,
-            ~indexingAddresses,
           )
         }
         switch pq {
@@ -1558,28 +1579,16 @@ let getNextQuery = (
           ~rangeEndBlock=queryEndBlock,
           ~maxQueryBlockNumber,
           ~maybeChunkRange,
+          ~maxChunks=maxPendingChunksPerPartition -
+          pendingCount -
+          (queries->Array.length -
+          partitionQueriesStart),
           ~partition=p,
           ~selection=p.selection,
           ~addressesByContractName=p.addressesByContractName,
-          ~indexingAddresses,
         )
       }
 
-      // Cap parallel in-flight chunks per partition so a single partition can't
-      // monopolize fetching. Keep the earliest new chunks; the furthest-ahead
-      // ones wait for the next round once these resolve.
-      let maxNewChunks = Pervasives.max(0, maxPendingChunksPerPartition - pendingCount)
-      let generatedCount = queries->Array.length - partitionQueriesStart
-      if generatedCount > maxNewChunks {
-        queries
-        ->Array.splice(
-          ~start=partitionQueriesStart + maxNewChunks,
-          ~remove=generatedCount - maxNewChunks,
-          ~insert=[],
-        )
-        ->ignore
-      }
-
       idxRef := idxRef.contents + 1
     }
 
@@ -1694,6 +1703,7 @@ let make = (
         eventConfigs: notDependingOnAddresses,
       },
       addressesByContractName: Dict.make(),
+      contractNameByAddress: Dict.make(), // derived in make
       mergeBlock: None,
       dynamicContract: None,
       mutPendingQueries: [],