envio 3.3.0-alpha.6 → 3.3.0-alpha.7

package/src/sources/RpcSource.res

@@ -97,16 +97,24 @@ let getErrorMessage = (exn: exn): option<string> =>
 // never helps — the same range always re-trips the same cap. The reaction is to
 // shrink the range and retry immediately, ratcheting the max range down.
 let isResponseTooLargeError = {
+  // Provider-specific "too many logs" messages, matched loosely since each names
+  // it differently: HyperRPC ("More than 50000 logs returned"), ZkEVM ("query
+  // returned more than N results"), LlamaRPC ("query exceeds max results"),
+  // 1RPC ("response size should not..."), Optimism ("(backend) response too
+  // large"), Arbitrum ("logs matched by query exceeds limit"), Ankr ("block
+  // range is too wide"). Kept as one block rather than per-line comments — the
+  // ReScript formatter doesn't preserve comments attached to regex literals in
+  // an array.
   let patterns = [
-    /more than \d+ logs/i, // HyperRPC: "More than 50000 logs returned"
+    /more than \d+ logs/i,
     /\d+ logs returned/i,
     /too many logs/i,
-    /query returned more than \d+ results/i, // ZkEVM
-    /query exceeds max results/i, // LlamaRPC
-    /response size should not/i, // 1RPC
-    /(backend )?response too large/i, // Optimism
-    /logs matched by query exceeds limit/i, // Arbitrum
-    /block range is too wide/i, // Ankr
+    /query returned more than \d+ results/i,
+    /query exceeds max results/i,
+    /response size should not/i,
+    /(backend )?response too large/i,
+    /logs matched by query exceeds limit/i,
+    /block range is too wide/i,
   ]
   (exn: exn) =>
     switch exn->getErrorMessage {
@@ -353,8 +361,10 @@ let getNextPage = (
   ->Promise.race
   ->Promise.catch(err => {
     let executedBlockInterval = toBlock - fromBlock + 1
-    let shrunkBlockInterval =
-      Pervasives.max(1, (executedBlockInterval->Int.toFloat *. sc.backoffMultiplicative)->Float.toInt)
+    let shrunkBlockInterval = Pervasives.max(
+      1,
+      (executedBlockInterval->Int.toFloat *. sc.backoffMultiplicative)->Float.toInt,
+    )
 
     let throwFailedGettingItems = retry =>
       throw(Source.GetItemsError(FailedGettingItems({exn: err, attemptedToBlock: toBlock, retry})))
@@ -1152,13 +1162,16 @@ let make = (
     ~knownHeight,
     ~partitionId,
     ~selection: FetchState.selection,
+    ~itemsTarget as _,
     ~retry,
     ~logger as _,
   ) => {
     let startFetchingBatchTimeRef = Performance.now()
 
     let sourceMaxBlockInterval =
-      mutSuggestedBlockIntervals->getSourceMaxBlockInterval(~intervalCeiling=syncConfig.intervalCeiling)
+      mutSuggestedBlockIntervals->getSourceMaxBlockInterval(
+        ~intervalCeiling=syncConfig.intervalCeiling,
+      )
     let suggestedBlockInterval = Pervasives.min(
       mutSuggestedBlockIntervals
       ->Utils.Dict.dangerouslyGetNonOption(partitionId)

package/src/sources/RpcSource.res.mjs

@@ -1040,7 +1040,7 @@ function make(param) {
       Error: new Error()
     };
   }, lowercaseAddresses);
-  let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, partitionId, selection, retry, param) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, partitionId, selection, param, retry, param$1) => {
     let startFetchingBatchTimeRef = Performance.now();
     let sourceMaxBlockInterval = getSourceMaxBlockInterval(mutSuggestedBlockIntervals, syncConfig.intervalCeiling);
     let suggestedBlockInterval = Primitive_int.min(Stdlib_Option.getOr(mutSuggestedBlockIntervals[partitionId], syncConfig.initialBlockInterval), sourceMaxBlockInterval);

package/src/sources/SimulateSource.res

@@ -26,6 +26,7 @@ let make = (~items: array<Internal.item>, ~endBlock: int, ~chain: ChainMap.Chain
       ~knownHeight as _,
       ~partitionId as _,
       ~selection as _,
+      ~itemsTarget as _,
       ~retry as _,
       ~logger as _,
     ) => {

package/src/sources/SimulateSource.res.mjs

@@ -17,7 +17,7 @@ function make(items, endBlock, chain) {
       _0: []
     }),
     getHeightOrThrow: () => Promise.resolve(Primitive_int.max(endBlock, 1)),
-    getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8) => {
+    getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8, param$9) => {
       let result = delivered.contents ? [] : (delivered.contents = true, items);
       let reportedHeight = Primitive_int.max(endBlock, 1);
       return Promise.resolve({

package/src/sources/Source.res

@@ -64,6 +64,12 @@ type t = {
     ~knownHeight: int,
     ~partitionId: string,
     ~selection: FetchState.selection,
+    // Soft cap on the number of primary items (logs/instructions/receipts) the
+    // source should ask its backend for, from the query's own estResponseSize.
+    // A HyperSync-backed source enforces it server-side, so a wrong estimate
+    // truncates the response instead of overshooting the shared buffer. Sources
+    // without an equivalent lever (RPC, Fuel, Simulate) ignore it.
+    ~itemsTarget: int,
     ~retry: int,
     ~logger: Pino.t,
   ) => promise<blockRangeFetchResponse>,

package/src/sources/SourceManager.res

@@ -680,6 +680,13 @@ let executeQuery = async (
         ~partitionId=query.partitionId,
         ~knownHeight,
         ~selection=query.selection,
+        // Ceil (not truncate) so a sub-1 estimate from a sparse partition over a
+        // small range doesn't round down to 0 and ask the backend to cap the
+        // response at nothing — 0 items is indistinguishable from "no signal".
+        ~itemsTarget={
+          let est = query.estResponseSize->Math.ceil->Float.toInt
+          est > 0 ? est : FetchState.minEstResponseSize->Float.toInt
+        },
         ~retry,
         ~logger,
       )

package/src/sources/SourceManager.res.mjs

@@ -500,7 +500,8 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
       retry: retry
     });
     try {
-      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, FetchState.deriveContractNameByAddress(query.addressesByContractName), knownHeight, query.partitionId, query.selection, retry, logger$2);
+      let est = Math.ceil(query.estResponseSize) | 0;
+      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, FetchState.deriveContractNameByAddress(query.addressesByContractName), knownHeight, query.partitionId, query.selection, est > 0 ? est : FetchState.minEstResponseSize | 0, retry, logger$2);
       sourceState.lastFailedAt = undefined;
       responseRef = response;
     } catch (raw_error) {

package/src/sources/Svm.res

@@ -101,6 +101,7 @@ let makeRPCSource = (~chain, ~rpc: string, ~sourceFor: Source.sourceFor=Sync): S
       ~knownHeight as _,
       ~partitionId as _,
       ~selection as _,
+      ~itemsTarget as _,
       ~retry as _,
       ~logger as _,
     ) => JsError.throwWithMessage("Svm does not support getting items"),

package/src/sources/Svm.res.mjs

@@ -76,7 +76,7 @@ function makeRPCSource(chain, rpc, sourceForOpt) {
       Prometheus.SourceRequestCount.addSeconds(name, chain, "getSlot", seconds);
       return height;
     },
-    getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8) => Stdlib_JsError.throwWithMessage("Svm does not support getting items")
+    getItemsOrThrow: (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8, param$9) => Stdlib_JsError.throwWithMessage("Svm does not support getting items")
   };
 }
 

package/src/sources/SvmHyperSyncSource.res

@@ -340,6 +340,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
     ~knownHeight,
     ~partitionId as _,
     ~selection as _,
+    ~itemsTarget,
     ~retry,
     ~logger,
   ) => {
@@ -368,6 +369,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
       toSlot: ?(toBlock->Option.map(toBlock => toBlock + 1)),
       instructions: instructionSelections,
       fields,
+      maxNumInstructions: itemsTarget,
     }
 
     Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getInstructions")

package/src/sources/SvmHyperSyncSource.res.mjs

@@ -288,7 +288,7 @@ function make(param) {
         return true;
     }
   });
-  let getItemsOrThrow = async (fromBlock, toBlock, param, contractNameByAddress, knownHeight, param$1, param$2, retry, logger) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, param, contractNameByAddress, knownHeight, param$1, param$2, itemsTarget, retry, logger) => {
     let totalTimeRef = Performance.now();
     let pageFetchRef = Performance.now();
     let instructionSelections = buildInstructionSelections(eventConfigs);
@@ -323,11 +323,13 @@ function make(param) {
     let query_toSlot = Stdlib_Option.map(toBlock, toBlock => toBlock + 1 | 0);
     let query_instructions = instructionSelections;
     let query_fields = fields$1;
+    let query_maxNumInstructions = itemsTarget;
     let query = {
       fromSlot: fromBlock,
       toSlot: query_toSlot,
       instructions: query_instructions,
-      fields: query_fields
+      fields: query_fields,
+      maxNumInstructions: query_maxNumInstructions
     };
     Prometheus.SourceRequestCount.increment(name, chain, "getInstructions");
     let match;