envio 3.3.0-alpha.2 → 3.3.0-alpha.4

package/src/EventProcessing.res

@@ -39,7 +39,7 @@ let runEventHandlerOrThrow = async (
   let eventItem = item->Internal.castUnsafeEventItem
 
   //Include the load in time before handler
-  let timeBeforeHandler = Hrtime.makeTimer()
+  let timeBeforeHandler = Performance.now()
 
   try {
     let contextParams: UserContext.contextParams = {
@@ -72,7 +72,7 @@ let runEventHandlerOrThrow = async (
       }),
     )
   }
-  let handlerDuration = timeBeforeHandler->Hrtime.timeSince->Hrtime.toSecondsFloat
+  let handlerDuration = timeBeforeHandler->Performance.secondsSince
   Prometheus.ProcessingHandler.increment(
     ~contract=eventItem.eventConfig.contractName,
     ~event=eventItem.eventConfig.name,
@@ -264,11 +264,19 @@ let runBatchHandlersOrThrow = async (
   }
 }
 
-let registerProcessEventBatchMetrics = (~logger, ~loadDuration, ~handlerDuration) => {
-  logger->Logging.childTrace({
-    "msg": "Finished processing batch",
-    "loader_time_elapsed": loadDuration,
-    "handlers_time_elapsed": handlerDuration,
+let registerProcessEventBatchMetrics = (
+  ~logger,
+  ~batch: Batch.t,
+  ~loadDuration,
+  ~handlerDuration,
+) => {
+  batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
+    logger->Logging.childTrace({
+      "msg": "Finished processing",
+      "chainId": chainId->Int.fromString->Option.getUnsafe,
+      "batchSize": chainAfterBatch.batchSize,
+      "progress": chainAfterBatch.progressBlockNumber,
+    })
   })
 
   Prometheus.ProcessingBatch.registerMetrics(~loadDuration, ~handlerDuration)
@@ -286,22 +294,28 @@ type logPartitionInfo = {
 // handlers read plain objects. A batch can span chains, each with its own store
 // and field mask, so group items by chain before materialising.
 let materializeBatchTransactions = async (batch: Batch.t, ~chainStates: dict<ChainState.t>) => {
-  let itemsByChain: dict<array<Internal.item>> = Dict.make()
-  batch.items->Array.forEach(item => {
-    let chainId = item->Internal.getItemChainId->Int.toString
-    switch itemsByChain->Utils.Dict.dangerouslyGetNonOption(chainId) {
-    | Some(items) => items->Array.push(item)
-    | None => itemsByChain->Dict.set(chainId, [item])
-    }
-  })
-
-  let _ = await itemsByChain
-  ->Dict.toArray
-  ->Array.map(async ((chainId, items)) => {
-    let cs = chainStates->Dict.getUnsafe(chainId)
-    await cs->ChainState.materializeBatchItems(~items)
-  })
-  ->Promise.all
+  switch chainStates->Dict.valuesToArray {
+  // Single-chain indexers (the common case): every item belongs to the one
+  // chain, so skip the per-chain grouping and its allocations.
+  | [cs] => await cs->ChainState.materializeBatchItems(~items=batch.items)
+  | _ =>
+    let itemsByChain: dict<array<Internal.item>> = Dict.make()
+    batch.items->Array.forEach(item => {
+      let chainId = item->Internal.getItemChainId->Int.toString
+      switch itemsByChain->Utils.Dict.dangerouslyGetNonOption(chainId) {
+      | Some(items) => items->Array.push(item)
+      | None => itemsByChain->Dict.set(chainId, [item])
+      }
+    })
+
+    let _ = await itemsByChain
+    ->Dict.toArray
+    ->Array.map(async ((chainId, items)) => {
+      let cs = chainStates->Dict.getUnsafe(chainId)
+      await cs->ChainState.materializeBatchItems(~items)
+    })
+    ->Promise.all
+  }
 }
 
 let processEventBatch = async (
@@ -312,27 +326,24 @@ let processEventBatch = async (
   ~config: Config.t,
   ~chainStates: dict<ChainState.t>,
 ) => {
-  let totalBatchSize = batch.totalBatchSize
   // Compute chains state for this batch
   let chains: Internal.chains = chainStates->computeChainsState
 
   let logger = Logging.getLogger()
-  logger->Logging.childTrace({
-    "msg": "Started processing batch",
-    "totalBatchSize": totalBatchSize,
-    "chains": batch.progressedChainsById->Utils.Dict.mapValues(chainAfterBatch => {
-      {
-        "batchSize": chainAfterBatch.batchSize,
-        "progress": chainAfterBatch.progressBlockNumber,
-      }
-    }),
+
+  batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
+    logger->Logging.childTrace({
+      "msg": "Started processing",
+      "chainId": chainId->Int.fromString->Option.getUnsafe,
+      "batchSize": chainAfterBatch.batchSize,
+    })
   })
 
   try {
     // Backpressure: keep processing within keepLatestChangesLimit of the cycle.
     await indexerState->Writing.awaitCapacity
 
-    let timeRef = Hrtime.makeTimer()
+    let timeRef = Performance.now()
 
     if batch.items->Utils.Array.notEmpty {
       // Materialise store-backed transactions onto payloads before any handler
@@ -341,7 +352,7 @@ let processEventBatch = async (
       await batch->preloadBatchOrThrow(~loadManager, ~persistence, ~indexerState, ~chains, ~config)
     }
 
-    let elapsedTimeAfterLoaders = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let elapsedTimeAfterLoaders = timeRef->Performance.secondsSince
 
     if batch.items->Utils.Array.notEmpty {
       await batch->runBatchHandlersOrThrow(
@@ -353,13 +364,18 @@ let processEventBatch = async (
       )
     }
 
-    let elapsedTimeAfterProcessing = timeRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+    let elapsedTimeAfterProcessing = timeRef->Performance.secondsSince
 
     indexerState->Writing.commitBatch(~batch)
 
     let loaderDuration = elapsedTimeAfterLoaders
     let handlerDuration = elapsedTimeAfterProcessing -. loaderDuration
-    registerProcessEventBatchMetrics(~logger, ~loadDuration=loaderDuration, ~handlerDuration)
+    registerProcessEventBatchMetrics(
+      ~logger,
+      ~batch,
+      ~loadDuration=loaderDuration,
+      ~handlerDuration,
+    )
     Ok()
   } catch {
   | Persistence.StorageError({message, reason}) =>

package/src/EventProcessing.res.mjs

@@ -1,14 +1,16 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
-import * as Hrtime from "./bindings/Hrtime.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Writing from "./Writing.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
+import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
+import * as Performance from "./bindings/Performance.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
+import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
 import * as UserContext from "./UserContext.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
@@ -34,7 +36,7 @@ function computeChainsState(chainStates) {
 let ProcessingError = /* @__PURE__ */Primitive_exceptions.create("EventProcessing.ProcessingError");
 
 async function runEventHandlerOrThrow(item, checkpointId, handler, indexerState, loadManager, persistence, chains, config) {
-  let timeBeforeHandler = Hrtime.makeTimer();
+  let timeBeforeHandler = Performance.now();
   try {
     let contextParams = {
       item: item,
@@ -62,7 +64,7 @@ async function runEventHandlerOrThrow(item, checkpointId, handler, indexerState,
       Error: new Error()
     };
   }
-  let handlerDuration = Hrtime.toSecondsFloat(Hrtime.timeSince(timeBeforeHandler));
+  let handlerDuration = Performance.secondsSince(timeBeforeHandler);
   return Prometheus.ProcessingHandler.increment(item.eventConfig.contractName, item.eventConfig.name, handlerDuration);
 }
 
@@ -172,59 +174,62 @@ async function runBatchHandlersOrThrow(batch, indexerState, loadManager, persist
   }
 }
 
-function registerProcessEventBatchMetrics(logger, loadDuration, handlerDuration) {
-  Logging.childTrace(logger, {
-    msg: "Finished processing batch",
-    loader_time_elapsed: loadDuration,
-    handlers_time_elapsed: handlerDuration
-  });
+function registerProcessEventBatchMetrics(logger, batch, loadDuration, handlerDuration) {
+  Stdlib_Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, chainId) => Logging.childTrace(logger, {
+    msg: "Finished processing",
+    chainId: Stdlib_Int.fromString(chainId, undefined),
+    batchSize: chainAfterBatch.batchSize,
+    progress: chainAfterBatch.progressBlockNumber
+  }));
   Prometheus.ProcessingBatch.registerMetrics(loadDuration, handlerDuration);
 }
 
 async function materializeBatchTransactions(batch, chainStates) {
-  let itemsByChain = {};
-  batch.items.forEach(item => {
-    let chainId = Internal.getItemChainId(item).toString();
-    let items = itemsByChain[chainId];
-    if (items !== undefined) {
-      items.push(item);
-    } else {
-      itemsByChain[chainId] = [item];
-    }
-  });
-  await Promise.all(Object.entries(itemsByChain).map(async param => {
-    let cs = chainStates[param[0]];
-    return await ChainState.materializeBatchItems(cs, param[1]);
-  }));
+  let match = Object.values(chainStates);
+  if (match.length !== 1) {
+    let itemsByChain = {};
+    batch.items.forEach(item => {
+      let chainId = Internal.getItemChainId(item).toString();
+      let items = itemsByChain[chainId];
+      if (items !== undefined) {
+        items.push(item);
+      } else {
+        itemsByChain[chainId] = [item];
+      }
+    });
+    await Promise.all(Object.entries(itemsByChain).map(async param => {
+      let cs = chainStates[param[0]];
+      return await ChainState.materializeBatchItems(cs, param[1]);
+    }));
+    return;
+  }
+  let cs = match[0];
+  return await ChainState.materializeBatchItems(cs, batch.items);
 }
 
 async function processEventBatch(batch, indexerState, loadManager, persistence, config, chainStates) {
-  let totalBatchSize = batch.totalBatchSize;
   let chains = computeChainsState(chainStates);
   let logger = Logging.getLogger();
-  Logging.childTrace(logger, {
-    msg: "Started processing batch",
-    totalBatchSize: totalBatchSize,
-    chains: Utils.Dict.mapValues(batch.progressedChainsById, chainAfterBatch => ({
-      batchSize: chainAfterBatch.batchSize,
-      progress: chainAfterBatch.progressBlockNumber
-    }))
-  });
+  Stdlib_Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, chainId) => Logging.childTrace(logger, {
+    msg: "Started processing",
+    chainId: Stdlib_Int.fromString(chainId, undefined),
+    batchSize: chainAfterBatch.batchSize
+  }));
   try {
     await Writing.awaitCapacity(indexerState);
-    let timeRef = Hrtime.makeTimer();
+    let timeRef = Performance.now();
     if (Utils.$$Array.notEmpty(batch.items)) {
       await materializeBatchTransactions(batch, chainStates);
       await preloadBatchOrThrow(batch, loadManager, persistence, config, indexerState, chains);
     }
-    let elapsedTimeAfterLoaders = Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef));
+    let elapsedTimeAfterLoaders = Performance.secondsSince(timeRef);
     if (Utils.$$Array.notEmpty(batch.items)) {
       await runBatchHandlersOrThrow(batch, indexerState, loadManager, persistence, config, chains);
     }
-    let elapsedTimeAfterProcessing = Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef));
+    let elapsedTimeAfterProcessing = Performance.secondsSince(timeRef);
     Writing.commitBatch(indexerState, batch);
     let handlerDuration = elapsedTimeAfterProcessing - elapsedTimeAfterLoaders;
-    registerProcessEventBatchMetrics(logger, elapsedTimeAfterLoaders, handlerDuration);
+    registerProcessEventBatchMetrics(logger, batch, elapsedTimeAfterLoaders, handlerDuration);
     return {
       TAG: "Ok",
       _0: undefined

package/src/FetchState.res

@@ -76,9 +76,26 @@ type query = {
   mutable progress: float,
   selection: selection,
   addressesByContractName: dict<array<Address.t>>,
-  indexingAddresses: dict<indexingAddress>,
 }
 
+// Invert addressesByContractName into address→contractName for log-ownership
+// routing. 1:1 today (each address belongs to one contract), so no key
+// collisions. Memoized on the addressesByContractName object so a partition's
+// many responses share one derivation and a large factory never rebuilds the
+// whole index; sound because the dict is immutable after construction (every
+// mutation produces a new dict).
+let deriveContractNameByAddress: dict<array<Address.t>> => dict<
+  string,
+> = Utils.WeakMap.memoize(addressesByContractName => {
+  let result = Dict.make()
+  addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
+    for i in 0 to addresses->Array.length - 1 {
+      result->Dict.set(addresses->Array.getUnsafe(i)->Address.toString, contractName)
+    }
+  })
+  result
+})
+
 // Default estimate for a query whose partition hasn't responded yet, so the
 // shared budget still accounts for unknown queries instead of treating them as
 // free.
@@ -1329,6 +1346,9 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
   }
 }
 
+// Most parallel in-flight chunk queries a single partition may have at once.
+let maxPendingChunksPerPartition = 10
+
 @inline
 let pushQueriesForRange = (
   queries: array<query>,
@@ -1337,12 +1357,12 @@ let pushQueriesForRange = (
   ~rangeEndBlock: option<int>,
   ~maxQueryBlockNumber: int,
   ~maybeChunkRange: option<int>,
+  ~maxChunks: int,
   ~partition: partition,
   ~selection: selection,
   ~addressesByContractName: dict<array<Address.t>>,
-  ~indexingAddresses: dict<indexingAddress>,
 ) => {
-  if rangeFromBlock <= maxQueryBlockNumber {
+  if rangeFromBlock <= maxQueryBlockNumber && maxChunks > 0 {
     switch rangeEndBlock {
     | Some(endBlock) if rangeFromBlock > endBlock => ()
     | _ =>
@@ -1363,7 +1383,6 @@ let pushQueriesForRange = (
           chainId: 0,
           progress: 0.,
           addressesByContractName,
-          indexingAddresses,
         })
       | Some(chunkRange) =>
         let maxBlock = switch rangeEndBlock {
@@ -1371,18 +1390,13 @@ let pushQueriesForRange = (
         | None => maxQueryBlockNumber
         }
         let chunkSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 1.8)
-        // Probe with two smaller chunks first so their responses come back
-        // quickly and refresh the chunking heuristic, then three full-size chunks.
-        let probeSize = Js.Math.ceil_int(chunkRange->Int.toFloat *. 0.9)
-        let getChunkSize = chunkIdx => chunkIdx < 2 ? probeSize : chunkSize
-        if rangeFromBlock + getChunkSize(0) + getChunkSize(1) - 1 <= maxBlock {
+        if rangeFromBlock + chunkSize * 2 - 1 <= maxBlock {
           let chunkFromBlock = ref(rangeFromBlock)
           let chunkIdx = ref(0)
           while (
-            chunkIdx.contents < 5 &&
-              chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1 <= maxBlock
+            chunkIdx.contents < maxChunks && chunkFromBlock.contents + chunkSize - 1 <= maxBlock
           ) {
-            let chunkToBlock = chunkFromBlock.contents + getChunkSize(chunkIdx.contents) - 1
+            let chunkToBlock = chunkFromBlock.contents + chunkSize - 1
             queries->Array.push({
               partitionId,
               fromBlock: chunkFromBlock.contents,
@@ -1398,7 +1412,6 @@ let pushQueriesForRange = (
               chainId: 0,
               progress: 0.,
               addressesByContractName,
-              indexingAddresses,
             })
             chunkFromBlock := chunkToBlock + 1
             chunkIdx := chunkIdx.contents + 1
@@ -1420,7 +1433,6 @@ let pushQueriesForRange = (
             chainId: 0,
             progress: 0.,
             addressesByContractName,
-            indexingAddresses,
           })
         }
       }
@@ -1428,18 +1440,8 @@ let pushQueriesForRange = (
   }
 }
 
-// Most parallel in-flight chunk queries a single partition may have at once.
-let maxPendingChunksPerPartition = 10
-
 let getNextQuery = (
-  {
-    buffer,
-    optimizedPartitions,
-    indexingAddresses,
-    blockLag,
-    latestOnBlockBlockNumber,
-    knownHeight,
-  } as fetchState: t,
+  {buffer, optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight} as fetchState: t,
   ~budget,
   ~chainPendingBudget,
 ) => {
@@ -1533,10 +1535,13 @@ let getNextQuery = (
             ~rangeEndBlock=Utils.Math.minOptInt(Some(pq.fromBlock - 1), queryEndBlock),
             ~maxQueryBlockNumber,
             ~maybeChunkRange,
+            ~maxChunks=maxPendingChunksPerPartition -
+            pendingCount -
+            (queries->Array.length -
+            partitionQueriesStart),
             ~partition=p,
             ~selection=p.selection,
             ~addressesByContractName=p.addressesByContractName,
-            ~indexingAddresses,
           )
         }
         switch pq {
@@ -1558,28 +1563,16 @@ let getNextQuery = (
           ~rangeEndBlock=queryEndBlock,
           ~maxQueryBlockNumber,
           ~maybeChunkRange,
+          ~maxChunks=maxPendingChunksPerPartition -
+          pendingCount -
+          (queries->Array.length -
+          partitionQueriesStart),
           ~partition=p,
           ~selection=p.selection,
           ~addressesByContractName=p.addressesByContractName,
-          ~indexingAddresses,
         )
       }
 
-      // Cap parallel in-flight chunks per partition so a single partition can't
-      // monopolize fetching. Keep the earliest new chunks; the furthest-ahead
-      // ones wait for the next round once these resolve.
-      let maxNewChunks = Pervasives.max(0, maxPendingChunksPerPartition - pendingCount)
-      let generatedCount = queries->Array.length - partitionQueriesStart
-      if generatedCount > maxNewChunks {
-        queries
-        ->Array.splice(
-          ~start=partitionQueriesStart + maxNewChunks,
-          ~remove=generatedCount - maxNewChunks,
-          ~insert=[],
-        )
-        ->ignore
-      }
-
       idxRef := idxRef.contents + 1
     }