envio 3.3.0 → 3.3.1-rc.0

package/src/sources/SourceManager.res

@@ -44,6 +44,11 @@ type t = {
   sourcesState: array<sourceState>,
   mutable statusStart: Performance.timeRef,
   mutable status: sourceManagerStatus,
+  // Cumulative time spent in each status, rendered into the
+  // envio_indexing_idle/source_waiting/source_querying_seconds counters.
+  mutable idleSeconds: float,
+  mutable waitingForNewBlockSeconds: float,
+  mutable queryingSeconds: float,
   newBlockStallTimeout: int,
   newBlockStallTimeoutRealtime: int,
   stalledPollingInterval: int,
@@ -92,6 +97,55 @@ let getRequestStatSamples = (sourceManager: t): array<requestStatSample> => {
   samples
 }
 
+// Per-source known heights for envio_source_known_height. Sources with no
+// observed height yet are skipped.
+type sourceHeightSample = {
+  sourceName: string,
+  chainId: int,
+  height: int,
+}
+
+let getSourceHeightSamples = (sourceManager: t): array<sourceHeightSample> => {
+  let samples = []
+  sourceManager.sourcesState->Array.forEach(sourceState => {
+    if sourceState.knownHeight > 0 {
+      samples->Array.push({
+        sourceName: sourceState.source.name,
+        chainId: sourceState.source.chain->ChainMap.Chain.toChainId,
+        height: sourceState.knownHeight,
+      })
+    }
+  })
+  samples
+}
+
+// Each accessor adds the in-progress interval of the current status, so a
+// scrape during a long idle/wait/query isn't stale until the next transition.
+let idleSeconds = (sourceManager: t) =>
+  sourceManager.idleSeconds +.
+  switch sourceManager.status {
+  | Idle => sourceManager.statusStart->Performance.secondsSince
+  | _ => 0.
+  }
+
+let waitingForNewBlockSeconds = (sourceManager: t) =>
+  sourceManager.waitingForNewBlockSeconds +.
+  switch sourceManager.status {
+  | WaitingForNewBlock => sourceManager.statusStart->Performance.secondsSince
+  | _ => 0.
+  }
+
+let queryingSeconds = (sourceManager: t) =>
+  sourceManager.queryingSeconds +.
+  switch sourceManager.status {
+  | Querying => sourceManager.statusStart->Performance.secondsSince
+  | _ => 0.
+  }
+
+// Partition queries currently in flight on this chain's sources. Summed across
+// chains by CrossChainState to enforce the indexer-wide concurrency budget.
+let inFlightCount = sourceManager => sourceManager.fetchingPartitionsCount
+
 let getRateLimitTimeMs = sourceManager =>
   sourceManager.committedRateLimitTimeMs +.
   switch sourceManager.activeRateLimitStartMs {
@@ -217,10 +271,6 @@ let make = (
   | None =>
     JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided")
   }
-  Prometheus.IndexingConcurrency.set(
-    ~concurrency=0,
-    ~chainId=initialActiveSource.chain->ChainMap.Chain.toChainId,
-  )
   {
     sourcesState: sources->Array.map(source => {
       source,
@@ -243,6 +293,9 @@ let make = (
     recoveryTimeout,
     statusStart: Performance.now(),
     status: Idle,
+    idleSeconds: 0.,
+    waitingForNewBlockSeconds: 0.,
+    queryingSeconds: 0.,
     hasRealtime,
     committedRateLimitTimeMs: 0.0,
     rateLimitWaiters: 0,
@@ -252,15 +305,13 @@ let make = (
 }
 
 let trackNewStatus = (sourceManager: t, ~newStatus) => {
-  let promCounter = switch sourceManager.status {
-  | Idle => Prometheus.IndexingIdleTime.counter
-  | WaitingForNewBlock => Prometheus.IndexingSourceWaitingTime.counter
-  | Querying => Prometheus.IndexingQueryTime.counter
+  let elapsed = sourceManager.statusStart->Performance.secondsSince
+  switch sourceManager.status {
+  | Idle => sourceManager.idleSeconds = sourceManager.idleSeconds +. elapsed
+  | WaitingForNewBlock =>
+    sourceManager.waitingForNewBlockSeconds = sourceManager.waitingForNewBlockSeconds +. elapsed
+  | Querying => sourceManager.queryingSeconds = sourceManager.queryingSeconds +. elapsed
   }
-  promCounter->Prometheus.SafeCounter.handleFloat(
-    ~labels=sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
-    ~value=sourceManager.statusStart->Performance.secondsSince,
-  )
   sourceManager.statusStart = Performance.now()
   sourceManager.status = newStatus
 }
@@ -303,20 +354,12 @@ let dispatch = async (
       // when they were admitted; here we just execute them.
       sourceManager.fetchingPartitionsCount =
         sourceManager.fetchingPartitionsCount + queries->Array.length
-      Prometheus.IndexingConcurrency.set(
-        ~concurrency=sourceManager.fetchingPartitionsCount,
-        ~chainId=sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
-      )
       sourceManager->trackNewStatus(~newStatus=Querying)
       let _ = await queries
       ->Array.map(q => {
         let promise = q->executeQuery
         let _ = promise->Promise.thenResolve(_ => {
           sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount - 1
-          Prometheus.IndexingConcurrency.set(
-            ~concurrency=sourceManager.fetchingPartitionsCount,
-            ~chainId=sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
-          )
           if sourceManager.fetchingPartitionsCount === 0 {
             sourceManager->trackNewStatus(~newStatus=Idle)
           }
@@ -461,13 +504,8 @@ let getSourceNewHeight = async (
     }
   }
 
-  // Update Prometheus only if height increased
-  if newHeight.contents > initialHeight {
-    Prometheus.SourceHeight.set(
-      ~sourceName=source.name,
-      ~chainId=source.chain->ChainMap.Chain.toChainId,
-      ~blockNumber=newHeight.contents,
-    )
+  if newHeight.contents > sourceState.knownHeight {
+    sourceState.knownHeight = newHeight.contents
   }
 
   newHeight.contents
@@ -736,6 +774,13 @@ let executeQuery = async (
       )
       sourceState->recordRequestStats(response.requestStats)
       sourceState.lastFailedAt = None
+
+      // The response carries a fresh height for exactly this source, so during a
+      // long backfill (when the wait loop doesn't run) it keeps the per-source
+      // envio_source_known_height current.
+      if response.knownHeight > sourceState.knownHeight {
+        sourceState.knownHeight = response.knownHeight
+      }
       responseRef := Some(response)
     } catch {
     | Source.RateLimited({resetMs}) =>

package/src/sources/SourceManager.res.mjs

@@ -4,7 +4,6 @@ import * as Utils from "../Utils.res.mjs";
 import * as Source from "./Source.res.mjs";
 import * as Logging from "../Logging.res.mjs";
 import * as FetchState from "../FetchState.res.mjs";
-import * as Prometheus from "../Prometheus.res.mjs";
 import * as Performance from "../bindings/Performance.res.mjs";
 import * as ErrorHandling from "../ErrorHandling.res.mjs";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
@@ -52,6 +51,70 @@ function getRequestStatSamples(sourceManager) {
   return samples;
 }
 
+function getSourceHeightSamples(sourceManager) {
+  let samples = [];
+  sourceManager.sourcesState.forEach(sourceState => {
+    if (sourceState.knownHeight > 0) {
+      samples.push({
+        sourceName: sourceState.source.name,
+        chainId: sourceState.source.chain,
+        height: sourceState.knownHeight
+      });
+      return;
+    }
+  });
+  return samples;
+}
+
+function idleSeconds(sourceManager) {
+  let match = sourceManager.status;
+  let tmp;
+  switch (match) {
+    case "Idle" :
+      tmp = Performance.secondsSince(sourceManager.statusStart);
+      break;
+    case "WaitingForNewBlock" :
+    case "Querying" :
+      tmp = 0;
+      break;
+  }
+  return sourceManager.idleSeconds + tmp;
+}
+
+function waitingForNewBlockSeconds(sourceManager) {
+  let match = sourceManager.status;
+  let tmp;
+  switch (match) {
+    case "WaitingForNewBlock" :
+      tmp = Performance.secondsSince(sourceManager.statusStart);
+      break;
+    case "Idle" :
+    case "Querying" :
+      tmp = 0;
+      break;
+  }
+  return sourceManager.waitingForNewBlockSeconds + tmp;
+}
+
+function queryingSeconds(sourceManager) {
+  let match = sourceManager.status;
+  let tmp;
+  switch (match) {
+    case "Idle" :
+    case "WaitingForNewBlock" :
+      tmp = 0;
+      break;
+    case "Querying" :
+      tmp = Performance.secondsSince(sourceManager.statusStart);
+      break;
+  }
+  return sourceManager.queryingSeconds + tmp;
+}
+
+function inFlightCount(sourceManager) {
+  return sourceManager.fetchingPartitionsCount;
+}
+
 function getRateLimitTimeMs(sourceManager) {
   let startMs = sourceManager.activeRateLimitStartMs;
   return sourceManager.committedRateLimitTimeMs + (
@@ -159,7 +222,6 @@ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeout
   let hasRealtime = sources.some(s => s.sourceFor === "Realtime");
   let source = sources.find(source => getSourceRole(source.sourceFor, isRealtime, hasRealtime) === "Primary");
   let initialActiveSource = source !== undefined ? source : Stdlib_JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided");
-  Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
   return {
     sourcesState: sources.map(source => ({
       source: source,
@@ -172,6 +234,9 @@ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeout
     })),
     statusStart: Performance.now(),
     status: "Idle",
+    idleSeconds: 0,
+    waitingForNewBlockSeconds: 0,
+    queryingSeconds: 0,
     newBlockStallTimeout: newBlockStallTimeout,
     newBlockStallTimeoutRealtime: newBlockStallTimeoutRealtime,
     stalledPollingInterval: stalledPollingInterval,
@@ -191,20 +256,19 @@ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeout
 }
 
 function trackNewStatus(sourceManager, newStatus) {
+  let elapsed = Performance.secondsSince(sourceManager.statusStart);
   let match = sourceManager.status;
-  let promCounter;
   switch (match) {
     case "Idle" :
-      promCounter = Prometheus.IndexingIdleTime.counter;
+      sourceManager.idleSeconds = sourceManager.idleSeconds + elapsed;
       break;
     case "WaitingForNewBlock" :
-      promCounter = Prometheus.IndexingSourceWaitingTime.counter;
+      sourceManager.waitingForNewBlockSeconds = sourceManager.waitingForNewBlockSeconds + elapsed;
       break;
     case "Querying" :
-      promCounter = Prometheus.IndexingQueryTime.counter;
+      sourceManager.queryingSeconds = sourceManager.queryingSeconds + elapsed;
       break;
   }
-  Prometheus.SafeCounter.handleFloat(promCounter, sourceManager.activeSource.chain, Performance.secondsSince(sourceManager.statusStart));
   sourceManager.statusStart = Performance.now();
   sourceManager.status = newStatus;
 }
@@ -232,13 +296,11 @@ async function dispatch(sourceManager, fetchState, executeQuery, waitForNewBlock
   } else {
     let queries = action._0;
     sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries.length | 0;
-    Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
     trackNewStatus(sourceManager, "Querying");
     await Promise.all(queries.map(q => {
       let promise = executeQuery(q);
       promise.then(param => {
         sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount - 1 | 0;
-        Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
         if (sourceManager.fetchingPartitionsCount === 0) {
           return trackNewStatus(sourceManager, "Idle");
         }
@@ -361,8 +423,8 @@ async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stall
       }
     }
   };
-  if (newHeight.contents > initialHeight) {
-    Prometheus.SourceHeight.set(source.name, source.chain, newHeight.contents);
+  if (newHeight.contents > sourceState.knownHeight) {
+    sourceState.knownHeight = newHeight.contents;
   }
   return newHeight.contents;
 }
@@ -542,6 +604,9 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
       let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, FetchState.deriveContractNameByAddress(query.addressesByContractName), knownHeight, query.partitionId, query.selection, query.itemsTarget, retry, logger$2);
       recordRequestStats(sourceState, response.requestStats);
       sourceState.lastFailedAt = undefined;
+      if (response.knownHeight > sourceState.knownHeight) {
+        sourceState.knownHeight = response.knownHeight;
+      }
       responseRef = response;
     } catch (raw_error) {
       let error = Primitive_exceptions.internalToException(raw_error);
@@ -715,6 +780,11 @@ export {
   make,
   getActiveSource,
   getRequestStatSamples,
+  getSourceHeightSamples,
+  idleSeconds,
+  waitingForNewBlockSeconds,
+  queryingSeconds,
+  inFlightCount,
   onReorg,
   dispatch,
   waitForNewBlock,

package/src/sources/SourceManager.resi

@@ -31,6 +31,20 @@ type requestStatSample = {
 
 let getRequestStatSamples: t => array<requestStatSample>
 
+type sourceHeightSample = {
+  sourceName: string,
+  chainId: int,
+  height: int,
+}
+
+let getSourceHeightSamples: t => array<sourceHeightSample>
+
+let idleSeconds: t => float
+let waitingForNewBlockSeconds: t => float
+let queryingSeconds: t => float
+
+let inFlightCount: t => int
+
 let onReorg: (t, ~rollbackTargetBlock: int) => unit
 
 let dispatch: (

package/src/tui/Tui.res

@@ -157,7 +157,7 @@ module App = {
       ->IndexerState.chainStates
       ->Dict.valuesToArray
       ->Array.map(cs => {
-        let data = cs->ChainState.toChainData
+        let data = cs->ChainState.toMetrics
         let numEventsProcessed = data.numEventsProcessed
         let committedProgressBlockNumber = cs->ChainState.committedProgressBlockNumber
         let timestampCaughtUpToHeadOrEndblock = data.timestampCaughtUpToHeadOrEndblock

package/src/tui/Tui.res.mjs

@@ -207,7 +207,7 @@ function Tui$App(props) {
     };
   }, [getState]);
   let chains = Object.values(IndexerState.chainStates(state)).map(cs => {
-    let data = ChainState.toChainData(cs);
+    let data = ChainState.toMetrics(cs);
     let numEventsProcessed = data.numEventsProcessed;
     let committedProgressBlockNumber = ChainState.committedProgressBlockNumber(cs);
     let timestampCaughtUpToHeadOrEndblock = data.timestampCaughtUpToHeadOrEndblock;
@@ -266,7 +266,7 @@ function Tui$App(props) {
       poweredByHyperSync: data.poweredByHyperSync,
       progress: progress,
       latestFetchedBlockNumber: latestFetchedBlockNumber,
-      knownHeight: data.currentBlockHeight,
+      knownHeight: data.knownHeight,
       blockUnit: tmp,
       rateLimitTimeMs: SourceManager.getRateLimitTimeMs(sourceManager),
       isRateLimited: SourceManager.isRateLimited(sourceManager),