envio 3.11.0 → 3.12.1

package/src/sources/EvmHyperSyncSource.res.mjs

@@ -44,12 +44,12 @@ function make(param) {
       }
     };
   };
-  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, knownHeight, param, selection, itemsTarget, retry, param$1) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, includeAllBlocks, knownHeight, param, selection, itemsTarget, retry, param$1) => {
     let totalTimeRef = Performance.now();
     let startFetchingBatchTimeRef = Performance.now();
     let pageUnsafe;
     try {
-      pageUnsafe = await HyperSync.GetLogs.query(client, fromBlock, toBlock, itemsTarget, selection.onEventRegistrations.map(reg => reg.index), addressSet, selection.clientFilteredContracts);
+      pageUnsafe = await HyperSync.GetLogs.query(client, fromBlock, toBlock, itemsTarget, selection.onEventRegistrations.map(reg => reg.index), addressSet, selection.clientFilteredContracts, includeAllBlocks);
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
       if (exn.RE_EXN_ID === HyperSync.GetLogs.$$Error) {
@@ -100,7 +100,8 @@ function make(param) {
     let pageFetchTime = Performance.secondsSince(startFetchingBatchTimeRef);
     let requestStats = [{
         method: "getLogs",
-        seconds: pageFetchTime
+        seconds: pageFetchTime,
+        responseBlocks: pageUnsafe.responseBlocks
       }];
     let knownHeight$1 = pageUnsafe.archiveHeight;
     let heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;

package/src/sources/FuelHyperSyncSource.res

@@ -35,6 +35,7 @@ let make = ({chainId, endpointUrl, apiToken, onEventRegistrations, addressStore}
     ~fromBlock,
     ~toBlock,
     ~addressSet,
+    ~includeAllBlocks as _,
     ~knownHeight,
     ~partitionId as _,
     ~selection: FetchState.selection,

package/src/sources/FuelHyperSyncSource.res.mjs

@@ -28,7 +28,7 @@ function make(param) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the HyperFuel client", exn);
   }
-  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, knownHeight, param, selection, param$1, retry, logger) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, param, knownHeight, param$1, selection, param$2, retry, logger) => {
     let totalTimeRef = Performance.now();
     let startFetchingBatchTimeRef = Performance.now();
     let pageUnsafe;

package/src/sources/HyperSync.res

@@ -90,6 +90,7 @@ type logsQueryPage = {
   items: array<HyperSyncClient.EventItems.item>,
   nextBlock: int,
   archiveHeight: int,
+  responseBlocks: int,
   // Page store owning this page's raw transactions.
   transactionStore: TransactionStore.t,
   // Page store owning this page's raw blocks.
@@ -136,6 +137,7 @@ module GetLogs = {
     ~registrationIndexes,
     ~addressSet,
     ~clientFilteredContracts,
+    ~includeAllBlocks,
   ): logsQueryPage => {
     let query: HyperSyncClient.EventItems.query = {
       fromBlock,
@@ -143,6 +145,9 @@ module GetLogs = {
       ?maxNumLogs,
       registrationIndexes,
       clientFilteredContracts,
+      // Absent rather than false, so a range that wants only the blocks its
+      // logs came from sends the query it always sent.
+      includeAllBlocks: ?(includeAllBlocks ? Some(true) : None),
     }
 
     let (res, transactionStore, blockStore) = switch await client.getEventItems(
@@ -165,6 +170,7 @@ module GetLogs = {
     {
       items: res.items,
       nextBlock: res.nextBlock,
+      responseBlocks: res.responseBlocks,
       archiveHeight: res.archiveHeight->Option.getOr(0), //Archive Height is only None if height is 0
       transactionStore,
       blockStore,

package/src/sources/HyperSync.res.mjs

@@ -137,13 +137,15 @@ function extractMissingParams(exn) {
   }
 }
 
-async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes, addressSet, clientFilteredContracts) {
+async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes, addressSet, clientFilteredContracts, includeAllBlocks) {
+  let query_includeAllBlocks = includeAllBlocks ? true : undefined;
   let query$1 = {
     fromBlock: fromBlock,
     toBlock: toBlock,
     maxNumLogs: maxNumLogs,
     registrationIndexes: registrationIndexes,
-    clientFilteredContracts: clientFilteredContracts
+    clientFilteredContracts: clientFilteredContracts,
+    includeAllBlocks: query_includeAllBlocks
   };
   let match;
   try {
@@ -176,6 +178,7 @@ async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes
     items: res.items,
     nextBlock: res.nextBlock,
     archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0),
+    responseBlocks: res.responseBlocks,
     transactionStore: match[1],
     blockStore: match[2]
   };

package/src/sources/HyperSync.resi

@@ -2,6 +2,7 @@ type logsQueryPage = {
   items: array<HyperSyncClient.EventItems.item>,
   nextBlock: int,
   archiveHeight: int,
+  responseBlocks: int,
   transactionStore: TransactionStore.t,
   blockStore: BlockStore.t,
 }
@@ -56,5 +57,6 @@ module GetLogs: {
     ~registrationIndexes: array<int>,
     ~addressSet: AddressSet.t,
     ~clientFilteredContracts: option<array<string>>,
+    ~includeAllBlocks: bool,
   ) => promise<logsQueryPage>
 }

package/src/sources/HyperSyncClient.res

@@ -262,6 +262,9 @@ module EventItems = {
     // depend on addresses (client-side filtering). None/empty means
     // every address-dependent contract is filtered server-side.
     clientFilteredContracts: option<array<string>>,
+    // Return a header for every block in the range, not only the ones a log
+    // landed on. Absent means only the blocks logs came from.
+    includeAllBlocks?: bool,
   }
 
   type item = {
@@ -283,6 +286,9 @@ module EventItems = {
     archiveHeight: option<int>,
     nextBlock: int,
     items: array<item>,
+    // Blocks the server returned for the query, counted before routing drops
+    // the ones no item joins to.
+    responseBlocks: int,
   }
 }
 

package/src/sources/HyperSyncSSE.res

@@ -25,7 +25,7 @@ let subscribe = (~hyperSyncUrl, ~apiToken, ~onHeight, ~onStatus) =>
             args.headers
             ->Option.getOr(Dict.make())
             ->Utils.Dict.merge(
-              Dict.fromArray([("Authorization", `Bearer ${apiToken}`), ("User-Agent", userAgent)]),
+              dict{"Authorization": `Bearer ${apiToken}`, "User-Agent": userAgent},
             )
           EventSource.Fetch.fetch(url, ~args={...args, headers: headers})
         },

package/src/sources/HyperSyncSSE.res.mjs

@@ -35,16 +35,10 @@ function subscribe(hyperSyncUrl, apiToken, onHeight, onStatus) {
     let userAgent = `hyperindex/` + Utils.EnvioPackage.value.version;
     let es = new Eventsource.EventSource(hyperSyncUrl + `/height/sse`, {
       fetch: (url, args) => {
-        let headers = Utils.Dict.merge(Stdlib_Option.getOr(args.headers, {}), Object.fromEntries([
-          [
-            "Authorization",
-            `Bearer ` + apiToken
-          ],
-          [
-            "User-Agent",
-            userAgent
-          ]
-        ]));
+        let headers = Utils.Dict.merge(Stdlib_Option.getOr(args.headers, {}), {
+          Authorization: `Bearer ` + apiToken,
+          "User-Agent": userAgent
+        });
         let newrecord = {...args};
         return fetch(url, (newrecord.headers = headers, newrecord));
       }

package/src/sources/RequestStat.res

@@ -2,4 +2,11 @@
 // in its own leaf module (rather than on `Source`) so the client FFI bindings
 // can name it without importing `Source` — which would form a dependency cycle
 // through `Env → HyperSyncClient → Source → FetchState → Env`.
-type t = {method: string, seconds: float}
+type t = {
+  method: string,
+  seconds: float,
+  // Blocks the backend returned for this request, before any client-side
+  // routing drops them. Absent for methods whose responses aren't measured in
+  // blocks, so a zero always means the request came back empty.
+  responseBlocks?: int,
+}

package/src/sources/RpcSource.res

@@ -877,6 +877,7 @@ let make = (
     ~fromBlock,
     ~toBlock,
     ~addressSet,
+    ~includeAllBlocks as _,
     ~knownHeight,
     ~partitionId,
     ~selection: FetchState.selection,

package/src/sources/RpcSource.res.mjs

@@ -874,7 +874,7 @@ function make(param) {
       Error: new Error()
     };
   }, lowercaseAddresses);
-  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, knownHeight, partitionId, selection, param, retry, param$1) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, param, knownHeight, partitionId, selection, param$1, retry, param$2) => {
     let startFetchingBatchTimeRef = Performance.now();
     let toBlock$1 = toBlock !== undefined ? Primitive_int.min(toBlock, knownHeight) : knownHeight;
     let firstBlockPromise = fromBlock > 0 && fromBlock <= toBlock$1 ? LazyLoader.get(blockLoader.contents, fromBlock).then(json => parseBlockInfo(json)) : Promise.resolve(undefined);

package/src/sources/SimulateSource.res

@@ -29,6 +29,7 @@ let make = (
       ~fromBlock,
       ~toBlock,
       ~addressSet,
+      ~includeAllBlocks as _,
       ~knownHeight as _,
       ~partitionId as _,
       ~selection: FetchState.selection,

package/src/sources/SimulateSource.res.mjs

@@ -27,7 +27,7 @@ function make(items, endBlock, chainId, addressStore, ecosystemOpt, transactionS
       height: reportedHeight,
       requestStats: []
     }),
-    getItemsOrThrow: (fromBlock, toBlock, addressSet, param, param$1, selection, param$2, param$3, param$4) => {
+    getItemsOrThrow: (fromBlock, toBlock, addressSet, param, param$1, param$2, selection, param$3, param$4, param$5) => {
       let toBlockQueried = toBlock !== undefined ? toBlock : reportedHeight;
       let selectionRegistrationIndexes = new Set();
       selection.onEventRegistrations.forEach(reg => {

package/src/sources/Source.res

@@ -8,9 +8,10 @@ type blockRangeFetchStats = {
 }
 
 // A single backend request a source method actually made (cache/dedup hits
-// aren't requests), with the time it took. SourceManager aggregates these
-// per (source, method) into the envio_source_request_* metrics.
-type requestStat = RequestStat.t = {method: string, seconds: float}
+// aren't requests), with the time it took and how much it brought back.
+// SourceManager aggregates these per (source, method) into the
+// envio_source_request_* and envio_source_response_* metrics.
+type requestStat = RequestStat.t = {method: string, seconds: float, responseBlocks?: int}
 
 // Native clients wrap a failure of a multi-request operation in a structured
 // payload, so the source can still return timings when SourceManager retries
@@ -196,6 +197,13 @@ type t = {
     // straight to its Rust client, which builds the query's address filter from
     // it and gates every returned item against the chain-wide store.
     ~addressSet: AddressSet.t,
+    // Return a header for every block in the range, not only the ones an item
+    // landed on. The progress block's timestamp is what measures how far behind
+    // chain time the indexer is, and at the head that block often carries no
+    // item of its own. Only set once the chain is at the head, where the range
+    // is a handful of blocks; over a backfill range it would be a header per
+    // block for no gain.
+    ~includeAllBlocks: bool,
     ~knownHeight: int,
     ~partitionId: string,
     ~selection: FetchState.selection,

package/src/sources/SourceManager.res

@@ -1,9 +1,14 @@
 type sourceManagerStatus = Idle | WaitingForNewBlock | Querying
 
-// Cumulative per-method request count/time for a source, aggregated from the
-// requestStat arrays returned by its methods. Rendered into
-// envio_source_request_* by Metrics.renderSourceRequests.
-type requestStatAgg = {mutable count: int, mutable seconds: float}
+// Cumulative per-method request stats for a source, aggregated from the
+// requestStat arrays returned by its methods. Rendered into the
+// envio_source_request_* and envio_source_response_* metrics.
+type requestStatAgg = {
+  mutable count: int,
+  mutable seconds: float,
+  mutable responseBlocks: option<int>,
+  mutable emptyResponseCount: int,
+}
 
 type sourceState = {
   source: Source.t,
@@ -19,12 +24,23 @@ let recordStatsInto = (
   aggregates: dict<requestStatAgg>,
   requestStats: array<Source.requestStat>,
 ) => {
-  requestStats->Array.forEach(({method, seconds}) => {
-    switch aggregates->Utils.Dict.dangerouslyGetNonOption(method) {
-    | Some(agg) =>
-      agg.count = agg.count + 1
-      agg.seconds = agg.seconds +. seconds
-    | None => aggregates->Dict.set(method, {count: 1, seconds})
+  requestStats->Array.forEach(({method, seconds, responseBlocks: ?responseBlocks}) => {
+    let agg = switch aggregates->Utils.Dict.dangerouslyGetNonOption(method) {
+    | Some(agg) => agg
+    | None =>
+      let agg = {count: 0, seconds: 0., responseBlocks: None, emptyResponseCount: 0}
+      aggregates->Dict.set(method, agg)
+      agg
+    }
+    agg.count = agg.count + 1
+    agg.seconds = agg.seconds +. seconds
+    switch responseBlocks {
+    | Some(responseBlocks) =>
+      agg.responseBlocks = Some(agg.responseBlocks->Option.getOr(0) + responseBlocks)
+      if responseBlocks === 0 {
+        agg.emptyResponseCount = agg.emptyResponseCount + 1
+      }
+    | None => ()
     }
   })
 }
@@ -40,6 +56,8 @@ type requestStatSample = {
   method: string,
   count: int,
   seconds: float,
+  responseBlocks: option<int>,
+  emptyResponseCount: int,
 }
 
 // Encapsulates the fetching logic for a chain's sources.
@@ -93,6 +111,8 @@ let getRequestStatSamples = (sourceManager: t): array<requestStatSample> => {
         method,
         count: agg.count,
         seconds: agg.seconds,
+        responseBlocks: agg.responseBlocks,
+        emptyResponseCount: agg.emptyResponseCount,
       })
       ->ignore
     })
@@ -685,7 +705,7 @@ let waitForNewBlock = (sourceManager: t, ~knownHeight, ~isRealtime, ~reducedPoll
     logger->Logging.childTrace(
       reducedPolling
         ? `Waiting for new blocks with reduced polling (${(sourceManager.reducedPollingInterval / 1000)
-              ->Int.toString}s). Chain is caught up, waiting for other chains to backfill.`
+              ->Int.toString}s) until the indexer enters realtime mode.`
         : "Initiating check for new blocks.",
     )
     sourceManager.waitingLogged = true
@@ -697,8 +717,8 @@ let waitForNewBlock = (sourceManager: t, ~knownHeight, ~isRealtime, ~reducedPoll
   // cadence the sources poll at and the level the closing line is logged at.
   let stalled = ref(false)
 
-  // Use a much longer stall timeout when reduced polling is active
-  // to avoid spurious stall warnings while waiting for other chains to backfill
+  // Use a much longer stall timeout when reduced polling is active, so a chain
+  // deliberately asking rarely doesn't report itself stalled between polls.
   let stallTimeout = if reducedPolling {
     sourceManager.reducedPollingInterval * 2
   } else if isRealtime {
@@ -928,6 +948,7 @@ let executeQuery = async (
         ~fromBlock=query.fromBlock,
         ~toBlock,
         ~addressSet=query.addresses,
+        ~includeAllBlocks=isRealtime,
         ~partitionId=query.partitionId,
         ~knownHeight,
         ~selection=query.selection->FetchState.narrowSelectionToRange(~toBlock),

package/src/sources/SourceManager.res.mjs

@@ -16,17 +16,32 @@ import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_excep
 
 function recordStatsInto(aggregates, requestStats) {
   requestStats.forEach(param => {
-    let seconds = param.seconds;
+    let responseBlocks = param.responseBlocks;
     let method = param.method;
     let agg = aggregates[method];
+    let agg$1;
     if (agg !== undefined) {
-      agg.count = agg.count + 1 | 0;
-      agg.seconds = agg.seconds + seconds;
+      agg$1 = agg;
     } else {
-      aggregates[method] = {
-        count: 1,
-        seconds: seconds
+      let agg$2 = {
+        count: 0,
+        seconds: 0,
+        responseBlocks: undefined,
+        emptyResponseCount: 0
       };
+      aggregates[method] = agg$2;
+      agg$1 = agg$2;
+    }
+    agg$1.count = agg$1.count + 1 | 0;
+    agg$1.seconds = agg$1.seconds + param.seconds;
+    if (responseBlocks !== undefined) {
+      agg$1.responseBlocks = Stdlib_Option.getOr(agg$1.responseBlocks, 0) + responseBlocks | 0;
+      if (responseBlocks === 0) {
+        agg$1.emptyResponseCount = agg$1.emptyResponseCount + 1 | 0;
+        return;
+      } else {
+        return;
+      }
     }
   });
 }
@@ -45,7 +60,9 @@ function getRequestStatSamples(sourceManager) {
         chainId: chainId,
         method: method,
         count: agg.count,
-        seconds: agg.seconds
+        seconds: agg.seconds,
+        responseBlocks: agg.responseBlocks,
+        emptyResponseCount: agg.emptyResponseCount
       });
     });
   });
@@ -520,7 +537,7 @@ function waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling)
     knownHeight: knownHeight
   });
   if (!sourceManager.waitingLogged) {
-    Logging.childTrace(logger, reducedPolling ? `Waiting for new blocks with reduced polling (` + (sourceManager.reducedPollingInterval / 1000 | 0).toString() + `s). Chain is caught up, waiting for other chains to backfill.` : "Initiating check for new blocks.");
+    Logging.childTrace(logger, reducedPolling ? `Waiting for new blocks with reduced polling (` + (sourceManager.reducedPollingInterval / 1000 | 0).toString() + `s) until the indexer enters realtime mode.` : "Initiating check for new blocks.");
     sourceManager.waitingLogged = true;
   }
   let mainSources = getNextSources(sourceManager, isRealtime, undefined);
@@ -674,7 +691,7 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
       retry: retry
     });
     try {
-      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addresses, knownHeight, query.partitionId, FetchState.narrowSelectionToRange(query.selection, toBlock), query.itemsTarget, retry, logger$2);
+      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addresses, isRealtime, knownHeight, query.partitionId, FetchState.narrowSelectionToRange(query.selection, toBlock), query.itemsTarget, retry, logger$2);
       recordStatsInto(sourceState.requestStats, response.requestStats);
       validateResponseBlockStore("getItems", response.blockStore, undefined);
       sourceState.lastFailedAt = undefined;

package/src/sources/SourceManager.resi

@@ -27,6 +27,8 @@ type requestStatSample = {
   method: string,
   count: int,
   seconds: float,
+  responseBlocks: option<int>,
+  emptyResponseCount: int,
 }
 
 let getRequestStatSamples: t => array<requestStatSample>

package/src/sources/SvmHyperSyncClient.res

@@ -143,6 +143,11 @@ module EventItems = {
     // depend on addresses (client-side filtering). None/empty means every
     // address-dependent program is filtered server-side.
     clientFilteredContracts: option<array<string>>,
+    // Return a block for every slot in the range, not only the ones an
+    // instruction landed on. Absent means only the slots instructions came
+    // from, which leaves a skipped slot indistinguishable from an unfetched
+    // one.
+    includeAllBlocks?: bool,
   }
 
   // NAPI encodes Rust `None` as `null`, never `undefined`, so an unselected

package/src/sources/SvmHyperSyncSource.res

@@ -147,6 +147,7 @@ let make = (
     ~fromBlock,
     ~toBlock,
     ~addressSet,
+    ~includeAllBlocks,
     ~knownHeight,
     ~partitionId as _,
     ~selection: FetchState.selection,
@@ -163,6 +164,8 @@ let make = (
       maxNumInstructions: ?itemsTarget,
       registrationIndexes: selection.onEventRegistrations->Array.map(reg => reg.index),
       clientFilteredContracts: selection.clientFilteredContracts,
+      // Absent rather than false, so a backfill query is the one it always was.
+      includeAllBlocks: ?(includeAllBlocks ? Some(true) : None),
     }
 
     let (resp, transactionStore, blockStore) = try await client.getEventItems(

package/src/sources/SvmHyperSyncSource.res.mjs

@@ -106,17 +106,19 @@ function make(param) {
   let chainId = param.chainId;
   let apiToken = HyperSync.requireApiToken(param.apiToken);
   let client = SvmHyperSyncClient.make(endpointUrl, apiToken, param.clientTimeoutMillis, undefined, undefined, SvmHyperSyncClient.Registration.fromOnEventRegistrations(onEventRegistrations), param.addressStore);
-  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, knownHeight, param, selection, itemsTarget, retry, param$1) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, includeAllBlocks, knownHeight, param, selection, itemsTarget, retry, param$1) => {
     let totalTimeRef = Performance.now();
     let pageFetchRef = Performance.now();
     let query_registrationIndexes = selection.onEventRegistrations.map(reg => reg.index);
     let query_clientFilteredContracts = selection.clientFilteredContracts;
+    let query_includeAllBlocks = includeAllBlocks ? true : undefined;
     let query = {
       fromSlot: fromBlock,
       toSlot: toBlock,
       maxNumInstructions: itemsTarget,
       registrationIndexes: query_registrationIndexes,
-      clientFilteredContracts: query_clientFilteredContracts
+      clientFilteredContracts: query_clientFilteredContracts,
+      includeAllBlocks: query_includeAllBlocks
     };
     let match;
     try {