envio 3.3.0-rc.0 → 3.3.1-rc.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.3.0-rc.0",
+  "version": "3.3.1-rc.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -54,7 +54,6 @@
     "express": "4.19.2",
     "pino": "10.3.1",
     "pino-pretty": "13.1.3",
-    "prom-client": "15.1.3",
     "yargs": "17.7.2",
     "@rescript/runtime": "12.2.0",
     "rescript-schema": "9.5.1",
@@ -70,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.3.0-rc.0",
-    "envio-linux-x64-musl": "3.3.0-rc.0",
-    "envio-linux-arm64": "3.3.0-rc.0",
-    "envio-darwin-x64": "3.3.0-rc.0",
-    "envio-darwin-arm64": "3.3.0-rc.0"
+    "envio-linux-x64": "3.3.1-rc.0",
+    "envio-linux-x64-musl": "3.3.1-rc.0",
+    "envio-linux-arm64": "3.3.1-rc.0",
+    "envio-darwin-x64": "3.3.1-rc.0",
+    "envio-darwin-arm64": "3.3.1-rc.0"
   }
 }

package/src/ChainFetching.res

@@ -128,19 +128,7 @@ let rec onQueryResponse = async (
       fromBlockQueried,
     } = response
 
-    if knownHeight > chainState->ChainState.knownHeight {
-      Prometheus.SourceHeight.set(
-        ~blockNumber=knownHeight,
-        ~chainId=(chainState->ChainState.chainConfig).id,
-        // The knownHeight from response won't necessarily
-        // belong to the currently active source.
-        // But for simplicity, assume it does.
-        ~sourceName=(chainState->ChainState.sourceManager->SourceManager.getActiveSource).name,
-      )
-    }
-
-    Prometheus.FetchingBlockRange.increment(
-      ~chainId=chain->ChainMap.Chain.toChainId,
+    chainState->ChainState.recordBlockRangeFetch(
       ~totalTimeElapsed=stats.totalTimeElapsed,
       ~parsingTimeElapsed=stats.parsingTimeElapsed->Option.getOr(0.),
       ~numEvents=parsedQueueItems->Array.length,
@@ -183,10 +171,8 @@ let rec onQueryResponse = async (
             ~shouldRollbackOnReorg=(state->IndexerState.config).shouldRollbackOnReorg,
           ),
         )
-        Prometheus.ReorgCount.increment(~chain)
-        Prometheus.ReorgDetectionBlockNumber.set(
+        chainState->ChainState.recordReorgDetected(
           ~blockNumber=reorgDetected.scannedBlock.blockNumber,
-          ~chain,
         )
         if (state->IndexerState.config).shouldRollbackOnReorg {
           Some(reorgDetected.scannedBlock.blockNumber)
@@ -333,22 +319,8 @@ let finishWaitingForNewBlock = (
     let chainState = state->IndexerState.getChainState(~chain)
     chainState->ChainState.updateKnownHeight(~knownHeight)
 
-    let isBelowReorgThreshold =
-      !(state->IndexerState.isInReorgThreshold) &&
-      (state->IndexerState.config).shouldRollbackOnReorg
-    let shouldEnterReorgThreshold =
-      isBelowReorgThreshold &&
-      state
-      ->IndexerState.chainStates
-      ->Dict.valuesToArray
-      ->Array.every(cs => {
-        cs->ChainState.isReadyToEnterReorgThreshold
-      })
-
-    // Kick processing in case there are block handlers to run.
-    if shouldEnterReorgThreshold {
-      IndexerState.enterReorgThreshold(state)
-    }
+    // No reorg-threshold check here: scheduleProcessing always runs at least one
+    // processNextBatch (even with no items), which owns the entry decision.
     scheduleFetch()
     scheduleProcessing()
   }

package/src/ChainFetching.res.mjs

@@ -4,7 +4,6 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as Prometheus from "./Prometheus.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
@@ -100,10 +99,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   let transactionStore = response.transactionStore;
   let parsedQueueItems = response.parsedQueueItems;
   let knownHeight = response.knownHeight;
-  if (knownHeight > ChainState.knownHeight(chainState)) {
-    Prometheus.SourceHeight.set(SourceManager.getActiveSource(ChainState.sourceManager(chainState)).name, ChainState.chainConfig(chainState).id, knownHeight);
-  }
-  Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - fromBlockQueried | 0) + 1 | 0);
+  ChainState.recordBlockRangeFetch(chainState, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - fromBlockQueried | 0) + 1 | 0);
   let numContractRegisterEvents = Stdlib_Array.reduce(parsedQueueItems, 0, (count, item) => {
     if (item.onEventRegistration.contractRegister !== undefined) {
       return count + 1 | 0;
@@ -138,8 +134,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   } else {
     let reorgDetected = reorgResult._0;
     Logging.childInfo(ChainState.logger(chainState), ReorgDetection.reorgDetectedToLogParams(reorgDetected, IndexerState.config(state).shouldRollbackOnReorg));
-    Prometheus.ReorgCount.increment(chain);
-    Prometheus.ReorgDetectionBlockNumber.set(reorgDetected.scannedBlock.blockNumber, chain);
+    ChainState.recordReorgDetected(chainState, reorgDetected.scannedBlock.blockNumber);
     rollbackWithReorgDetectedBlockNumber = IndexerState.config(state).shouldRollbackOnReorg ? reorgDetected.scannedBlock.blockNumber : undefined;
   }
   if (rollbackWithReorgDetectedBlockNumber !== undefined) {
@@ -188,11 +183,6 @@ function finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFe
   }
   let chainState = IndexerState.getChainState(state, chain);
   ChainState.updateKnownHeight(chainState, knownHeight);
-  let isBelowReorgThreshold = !IndexerState.isInReorgThreshold(state) && IndexerState.config(state).shouldRollbackOnReorg;
-  let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(ChainState.isReadyToEnterReorgThreshold);
-  if (shouldEnterReorgThreshold) {
-    IndexerState.enterReorgThreshold(state);
-  }
   scheduleFetch();
   scheduleProcessing();
 }

package/src/ChainState.res

@@ -39,24 +39,18 @@ type t = {
   // Holds this chain's blocks (kept in Rust) keyed by block number. Same merge /
   // prune / rollback lifecycle as the transaction store.
   blockStore: BlockStore.t,
-}
-
-// Per-chain shape returned by the status API.
-type chainData = {
-  chainId: float,
-  poweredByHyperSync: bool,
-  firstEventBlockNumber: option<int>,
-  latestProcessedBlock: option<int>,
-  timestampCaughtUpToHeadOrEndblock: option<Date.t>,
-  numEventsProcessed: float,
-  latestFetchedBlockNumber: int,
-  // Need this for API backwards compatibility
-  @as("currentBlockHeight")
-  knownHeight: int,
-  numBatchesFetched: int,
-  startBlock: int,
-  endBlock: option<int>,
-  numAddresses: int,
+  reorgThresholdReadyTolerance: int,
+  // --- Per-chain metric counters, rendered by Metrics at scrape time.
+  // Floats: cumulative counters outgrow int32. ---
+  mutable blockRangeFetchSeconds: float,
+  mutable blockRangeParseSeconds: float,
+  mutable blockRangeFetchCount: float,
+  mutable blockRangeFetchedEvents: float,
+  mutable blockRangeFetchedBlocks: float,
+  mutable reorgCount: int,
+  mutable reorgDetectedBlock: option<int>,
+  mutable rollbackTargetBlock: option<int>,
+  mutable progressLatencyMs: option<int>,
 }
 
 let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddress> => {
@@ -100,6 +94,7 @@ let make = (
   ~transactionStore=TransactionStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
   ~chainDensity=None,
   ~blockStore=BlockStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
+  ~reorgThresholdReadyTolerance=100,
   ~logger: Pino.t,
 ): t => {
   validateOnEventRegistrations(~chainId=chainConfig.id, onEventRegistrations)
@@ -121,6 +116,16 @@ let make = (
     safeCheckpointTracking,
     transactionStore,
     blockStore,
+    reorgThresholdReadyTolerance,
+    blockRangeFetchSeconds: 0.,
+    blockRangeParseSeconds: 0.,
+    blockRangeFetchCount: 0.,
+    blockRangeFetchedEvents: 0.,
+    blockRangeFetchedBlocks: 0.,
+    reorgCount: 0,
+    reorgDetectedBlock: None,
+    rollbackTargetBlock: None,
+    progressLatencyMs: None,
   }
 }
 
@@ -282,6 +287,7 @@ let makeInternal = (
       ~ecosystem=config.ecosystem.name,
       ~shouldChecksum=!lowercaseAddresses,
     ),
+    ~reorgThresholdReadyTolerance=config.reorgThresholdReadyTolerance,
     ~logger,
   )
 }
@@ -329,8 +335,6 @@ let makeFromDbState = (
   let chainId = chainConfig.id
   let logger = Logging.createChild(~params={"chainId": chainId})
 
-  Prometheus.ProgressEventsCount.set(~processedCount=resumedChainState.numEventsProcessed, ~chainId)
-
   let progressBlockNumber =
     // Can be -1 when not set
     resumedChainState.progressBlockNumber >= 0
@@ -373,6 +377,29 @@ let numEventsProcessed = (cs: t) => cs.numEventsProcessed
 let pendingBudget = (cs: t) => cs.pendingBudget
 let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock
 
+// --- Metric counters. ---
+
+let recordBlockRangeFetch = (
+  cs: t,
+  ~totalTimeElapsed,
+  ~parsingTimeElapsed,
+  ~numEvents,
+  ~blockRangeSize,
+) => {
+  cs.blockRangeFetchSeconds = cs.blockRangeFetchSeconds +. totalTimeElapsed
+  cs.blockRangeParseSeconds = cs.blockRangeParseSeconds +. parsingTimeElapsed
+  cs.blockRangeFetchCount = cs.blockRangeFetchCount +. 1.
+  cs.blockRangeFetchedEvents = cs.blockRangeFetchedEvents +. numEvents->Int.toFloat
+  cs.blockRangeFetchedBlocks = cs.blockRangeFetchedBlocks +. blockRangeSize->Int.toFloat
+}
+
+let recordReorgDetected = (cs: t, ~blockNumber) => {
+  cs.reorgCount = cs.reorgCount + 1
+  cs.reorgDetectedBlock = Some(blockNumber)
+}
+
+let setRollbackTargetBlock = (cs: t, ~blockNumber) => cs.rollbackTargetBlock = Some(blockNumber)
+
 // Fetch-frontier reads. The FetchState is owned here; callers go through these
 // rather than reaching into it.
 let knownHeight = (cs: t) => cs.fetchState.knownHeight
@@ -384,7 +411,6 @@ let getProgressPercentage = (cs: t) => cs.fetchState->FetchState.getProgressPerc
 let chainDensity = (cs: t) => cs.chainDensity
 let hasReadyItem = (cs: t) =>
   cs.fetchState->FetchState.isActivelyIndexing && cs.fetchState->FetchState.hasReadyItem
-let isReadyToEnterReorgThreshold = (cs: t) => cs.fetchState->FetchState.isReadyToEnterReorgThreshold
 
 // Mark queries as in flight and reserve their estimated size against the shared
 // buffer budget in one step, so the counter stays in sync with the pending
@@ -868,8 +894,7 @@ let toChainMetadata = (cs: t): InternalTable.Chains.metaFields => {
   timestampCaughtUpToHeadOrEndblock: cs.timestampCaughtUpToHeadOrEndblock->Null.fromOption,
 }
 
-// Snapshot the chain's view for the status API.
-let toChainData = (cs: t): chainData => {
+let toMetrics = (cs: t): Metrics.chainMetrics => {
   chainId: cs.chainConfig.id->Int.toFloat,
   poweredByHyperSync: (cs.sourceManager->SourceManager.getActiveSource).poweredByHyperSync,
   firstEventBlockNumber: cs.fetchState.firstEventBlock,
@@ -886,6 +911,25 @@ let toChainData = (cs: t): chainData => {
   startBlock: cs.fetchState.startBlock,
   endBlock: cs.fetchState.endBlock,
   numAddresses: cs.indexingAddresses->IndexingAddresses.size,
+  isReady: cs->isReady,
+  sourceBlockNumber: cs.fetchState.knownHeight,
+  progressBlockNumber: cs.committedProgressBlockNumber,
+  progressLatencyMs: cs.progressLatencyMs,
+  concurrency: cs.sourceManager->SourceManager.inFlightCount,
+  partitionsCount: cs.fetchState->FetchState.partitionsCount,
+  bufferSize: cs.fetchState->FetchState.bufferSize,
+  bufferBlockNumber: cs.fetchState->FetchState.bufferBlockNumber,
+  idleSeconds: cs.sourceManager->SourceManager.idleSeconds,
+  waitingForNewBlockSeconds: cs.sourceManager->SourceManager.waitingForNewBlockSeconds,
+  queryingSeconds: cs.sourceManager->SourceManager.queryingSeconds,
+  blockRangeFetchSeconds: cs.blockRangeFetchSeconds,
+  blockRangeParseSeconds: cs.blockRangeParseSeconds,
+  blockRangeFetchCount: cs.blockRangeFetchCount,
+  blockRangeFetchedEvents: cs.blockRangeFetchedEvents,
+  blockRangeFetchedBlocks: cs.blockRangeFetchedBlocks,
+  reorgCount: cs.reorgCount,
+  reorgDetectedBlock: cs.reorgDetectedBlock,
+  rollbackTargetBlock: cs.rollbackTargetBlock,
 }
 
 // Snapshot the inputs a batch build needs from this chain.
@@ -907,7 +951,7 @@ let isReadyToEnterReorgThresholdAfterBatch = (cs: t, ~batch: Batch.t) => {
   | Some(chainAfterBatch) => chainAfterBatch.fetchState
   | None => cs.fetchState
   }
-  fetchState->FetchState.isReadyToEnterReorgThreshold
+  fetchState->FetchState.isReadyToEnterReorgThreshold(~tolerance=cs.reorgThresholdReadyTolerance)
 }
 
 // Commit the post-batch fetch frontier for a chain that progressed in the batch,
@@ -928,28 +972,15 @@ let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
   }
 
 // Commit a processed batch's progress for this chain (progress block, events
-// processed, head/safe-checkpoint tracking, first event block). Emits the
-// per-chain progress metrics. Readiness is decided by CrossChainState once every
-// chain is caught up (see markReady). `blockTimestampName` is the ecosystem's
-// block-timestamp field, read off the payload block for the latency metric.
+// processed, head/safe-checkpoint tracking, first event block). Readiness is
+// decided by CrossChainState once every chain is caught up (see markReady).
+// `blockTimestampName` is the ecosystem's block-timestamp field, read off the
+// payload block for the latency metric.
 let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) => {
   let chainId = cs.chainConfig.id
 
   switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
   | Some(chainAfterBatch) => {
-      if cs.committedProgressBlockNumber !== chainAfterBatch.progressBlockNumber {
-        Prometheus.ProgressBlockNumber.set(
-          ~blockNumber=chainAfterBatch.progressBlockNumber,
-          ~chainId,
-        )
-      }
-      if cs.numEventsProcessed !== chainAfterBatch.totalEventsProcessed {
-        Prometheus.ProgressEventsCount.set(
-          ~processedCount=chainAfterBatch.totalEventsProcessed,
-          ~chainId,
-        )
-      }
-
       // Calculate and set latency metrics. The payload block is materialised or
       // inline by processing time; its timestamp may still be absent (e.g. an
       // SVM slot with no block row) — the metric is skipped then.
@@ -967,10 +998,7 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
       ) {
       | Some(blockTimestamp) =>
         let blockTimestampMs = blockTimestamp->(Utils.magic: unknown => int) * 1000
-        Prometheus.ProgressLatency.set(
-          ~latencyMs=Date.now()->Float.toInt - blockTimestampMs,
-          ~chainId,
-        )
+        cs.progressLatencyMs = Some(Date.now()->Float.toInt - blockTimestampMs)
       | None => ()
       }
 
@@ -1046,7 +1074,6 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
 let markReady = (cs: t) =>
   if !(cs->isReady) {
     cs.timestampCaughtUpToHeadOrEndblock = Date.make()->Some
-    Prometheus.ProgressReady.set(~chainId=cs.chainConfig.id)
   }
 
 // Roll a chain back to a reorg target. With a progress diff, restore fetch/
@@ -1061,7 +1088,6 @@ let rollback = (
   ~rollbackTargetBlockNumber,
   ~isReorgChain,
 ) => {
-  let chainId = cs.chainConfig.id
   switch newProgressBlockNumber {
   | Some(newProgressBlockNumber) =>
     let newTotalEventsProcessed =
@@ -1072,12 +1098,6 @@ let rollback = (
         ~message="Missing events-processed diff for rolled-back chain",
       )
 
-    if cs.committedProgressBlockNumber !== newProgressBlockNumber {
-      Prometheus.ProgressBlockNumber.set(~blockNumber=newProgressBlockNumber, ~chainId)
-    }
-    if cs.numEventsProcessed !== newTotalEventsProcessed {
-      Prometheus.ProgressEventsCount.set(~processedCount=newTotalEventsProcessed, ~chainId)
-    }
     if isReorgChain {
       cs.reorgDetection =
         cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(

package/src/ChainState.res.mjs

@@ -10,7 +10,6 @@ import * as EvmChain from "./sources/EvmChain.res.mjs";
 import * as FieldMask from "./sources/FieldMask.res.mjs";
 import * as BlockStore from "./sources/BlockStore.res.mjs";
 import * as FetchState from "./FetchState.res.mjs";
-import * as Prometheus from "./Prometheus.res.mjs";
 import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
@@ -48,7 +47,7 @@ function validateOnEventRegistrations(chainId, registrations) {
   });
 }
 
-function make(chainConfig, fetchState, onEventRegistrationsOpt, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, logger) {
+function make(chainConfig, fetchState, onEventRegistrationsOpt, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, logger) {
   let onEventRegistrations = onEventRegistrationsOpt !== undefined ? onEventRegistrationsOpt : [];
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
@@ -57,6 +56,7 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, indexingAddresse
   let transactionStore = transactionStoreOpt !== undefined ? Primitive_option.valFromOption(transactionStoreOpt) : TransactionStore.make("evm", false);
   let chainDensity = chainDensityOpt !== undefined ? Primitive_option.valFromOption(chainDensityOpt) : undefined;
   let blockStore = blockStoreOpt !== undefined ? Primitive_option.valFromOption(blockStoreOpt) : BlockStore.make("evm", false);
+  let reorgThresholdReadyTolerance = reorgThresholdReadyToleranceOpt !== undefined ? reorgThresholdReadyToleranceOpt : 100;
   validateOnEventRegistrations(chainConfig.id, onEventRegistrations);
   return {
     logger: logger,
@@ -75,7 +75,17 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, indexingAddresse
     reorgDetection: reorgDetection,
     safeCheckpointTracking: safeCheckpointTracking,
     transactionStore: transactionStore,
-    blockStore: blockStore
+    blockStore: blockStore,
+    reorgThresholdReadyTolerance: reorgThresholdReadyTolerance,
+    blockRangeFetchSeconds: 0,
+    blockRangeParseSeconds: 0,
+    blockRangeFetchCount: 0,
+    blockRangeFetchedEvents: 0,
+    blockRangeFetchedBlocks: 0,
+    reorgCount: 0,
+    reorgDetectedBlock: undefined,
+    rollbackTargetBlock: undefined,
+    progressLatencyMs: undefined
   };
 }
 
@@ -147,7 +157,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
   }
   let firstEventBlock$1 = fetchState.firstEventBlock;
   let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
-  return make(chainConfig, fetchState, onEventRegistrations, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), logger);
+  return make(chainConfig, fetchState, onEventRegistrations, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), config.reorgThresholdReadyTolerance, logger);
 }
 
 function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight) {
@@ -162,7 +172,6 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
   let logger = Logging.createChild({
     chainId: chainId
   });
-  Prometheus.ProgressEventsCount.set(resumedChainState.numEventsProcessed, chainId);
   let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
   return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrationsByChainId, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, reducedPollingInterval);
 }
@@ -207,6 +216,23 @@ function timestampCaughtUpToHeadOrEndblock(cs) {
   return cs.timestampCaughtUpToHeadOrEndblock;
 }
 
+function recordBlockRangeFetch(cs, totalTimeElapsed, parsingTimeElapsed, numEvents, blockRangeSize) {
+  cs.blockRangeFetchSeconds = cs.blockRangeFetchSeconds + totalTimeElapsed;
+  cs.blockRangeParseSeconds = cs.blockRangeParseSeconds + parsingTimeElapsed;
+  cs.blockRangeFetchCount = cs.blockRangeFetchCount + 1;
+  cs.blockRangeFetchedEvents = cs.blockRangeFetchedEvents + numEvents;
+  cs.blockRangeFetchedBlocks = cs.blockRangeFetchedBlocks + blockRangeSize;
+}
+
+function recordReorgDetected(cs, blockNumber) {
+  cs.reorgCount = cs.reorgCount + 1 | 0;
+  cs.reorgDetectedBlock = blockNumber;
+}
+
+function setRollbackTargetBlock(cs, blockNumber) {
+  cs.rollbackTargetBlock = blockNumber;
+}
+
 function knownHeight(cs) {
   return cs.fetchState.knownHeight;
 }
@@ -239,10 +265,6 @@ function hasReadyItem(cs) {
   }
 }
 
-function isReadyToEnterReorgThreshold(cs) {
-  return FetchState.isReadyToEnterReorgThreshold(cs.fetchState);
-}
-
 function startFetchingQueries(cs, queries) {
   FetchState.startFetchingQueries(cs.fetchState, queries);
   cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsEst);
@@ -598,7 +620,7 @@ function toChainMetadata(cs) {
   };
 }
 
-function toChainData(cs) {
+function toMetrics(cs) {
   return {
     chainId: cs.chainConfig.id,
     poweredByHyperSync: SourceManager.getActiveSource(cs.sourceManager).poweredByHyperSync,
@@ -607,11 +629,30 @@ function toChainData(cs) {
     timestampCaughtUpToHeadOrEndblock: cs.timestampCaughtUpToHeadOrEndblock,
     numEventsProcessed: cs.numEventsProcessed,
     latestFetchedBlockNumber: Primitive_int.max(FetchState.bufferBlockNumber(cs.fetchState), 0),
-    currentBlockHeight: hasProcessedToEndblock(cs) ? Stdlib_Option.getOr(cs.fetchState.endBlock, cs.fetchState.knownHeight) : cs.fetchState.knownHeight,
+    knownHeight: hasProcessedToEndblock(cs) ? Stdlib_Option.getOr(cs.fetchState.endBlock, cs.fetchState.knownHeight) : cs.fetchState.knownHeight,
     numBatchesFetched: 0,
     startBlock: cs.fetchState.startBlock,
     endBlock: cs.fetchState.endBlock,
-    numAddresses: IndexingAddresses.size(cs.indexingAddresses)
+    numAddresses: IndexingAddresses.size(cs.indexingAddresses),
+    isReady: cs.timestampCaughtUpToHeadOrEndblock !== undefined,
+    sourceBlockNumber: cs.fetchState.knownHeight,
+    progressBlockNumber: cs.committedProgressBlockNumber,
+    progressLatencyMs: cs.progressLatencyMs,
+    concurrency: SourceManager.inFlightCount(cs.sourceManager),
+    partitionsCount: FetchState.partitionsCount(cs.fetchState),
+    bufferSize: FetchState.bufferSize(cs.fetchState),
+    bufferBlockNumber: FetchState.bufferBlockNumber(cs.fetchState),
+    idleSeconds: SourceManager.idleSeconds(cs.sourceManager),
+    waitingForNewBlockSeconds: SourceManager.waitingForNewBlockSeconds(cs.sourceManager),
+    queryingSeconds: SourceManager.queryingSeconds(cs.sourceManager),
+    blockRangeFetchSeconds: cs.blockRangeFetchSeconds,
+    blockRangeParseSeconds: cs.blockRangeParseSeconds,
+    blockRangeFetchCount: cs.blockRangeFetchCount,
+    blockRangeFetchedEvents: cs.blockRangeFetchedEvents,
+    blockRangeFetchedBlocks: cs.blockRangeFetchedBlocks,
+    reorgCount: cs.reorgCount,
+    reorgDetectedBlock: cs.reorgDetectedBlock,
+    rollbackTargetBlock: cs.rollbackTargetBlock
   };
 }
 
@@ -628,7 +669,8 @@ function toChainBeforeBatch(cs) {
 
 function isReadyToEnterReorgThresholdAfterBatch(cs, batch) {
   let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
-  return FetchState.isReadyToEnterReorgThreshold(chainAfterBatch !== undefined ? chainAfterBatch.fetchState : cs.fetchState);
+  let fetchState = chainAfterBatch !== undefined ? chainAfterBatch.fetchState : cs.fetchState;
+  return FetchState.isReadyToEnterReorgThreshold(cs.reorgThresholdReadyTolerance, fetchState);
 }
 
 function advanceAfterBatch(cs, batch, enteringReorgThreshold) {
@@ -646,16 +688,10 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
   if (chainAfterBatch === undefined) {
     return;
   }
-  if (cs.committedProgressBlockNumber !== chainAfterBatch.progressBlockNumber) {
-    Prometheus.ProgressBlockNumber.set(chainAfterBatch.progressBlockNumber, chainId);
-  }
-  if (cs.numEventsProcessed !== chainAfterBatch.totalEventsProcessed) {
-    Prometheus.ProgressEventsCount.set(chainAfterBatch.totalEventsProcessed, chainId);
-  }
   let blockTimestamp = Stdlib_Option.flatMap(Stdlib_Option.flatMap(Batch.findLastEventItem(batch, chainId), eventItem => Primitive_option.fromNullable(eventItem.payload.block)), block => block[blockTimestampName]);
   if (blockTimestamp !== undefined) {
     let blockTimestampMs = Primitive_option.valFromOption(blockTimestamp) * 1000 | 0;
-    Prometheus.ProgressLatency.set((Date.now() | 0) - blockTimestampMs | 0, chainId);
+    cs.progressLatencyMs = (Date.now() | 0) - blockTimestampMs | 0;
   }
   let match = cs.fetchState.firstEventBlock;
   if (match === undefined) {
@@ -717,12 +753,11 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
 function markReady(cs) {
   if (cs.timestampCaughtUpToHeadOrEndblock === undefined) {
     cs.timestampCaughtUpToHeadOrEndblock = new Date();
-    return Prometheus.ProgressReady.set(cs.chainConfig.id);
+    return;
   }
 }
 
 function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTargetBlockNumber, isReorgChain) {
-  let chainId = cs.chainConfig.id;
   if (newProgressBlockNumber === undefined) {
     if (isReorgChain) {
       cs.reorgDetection = ReorgDetection.rollbackToValidBlockNumber(cs.reorgDetection, rollbackTargetBlockNumber);
@@ -737,12 +772,6 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
     }
   }
   let newTotalEventsProcessed = cs.numEventsProcessed - Stdlib_Option.getOrThrow(eventsProcessedDiff, "Missing events-processed diff for rolled-back chain");
-  if (cs.committedProgressBlockNumber !== newProgressBlockNumber) {
-    Prometheus.ProgressBlockNumber.set(newProgressBlockNumber, chainId);
-  }
-  if (cs.numEventsProcessed !== newTotalEventsProcessed) {
-    Prometheus.ProgressEventsCount.set(newTotalEventsProcessed, chainId);
-  }
   if (isReorgChain) {
     cs.reorgDetection = ReorgDetection.rollbackToValidBlockNumber(cs.reorgDetection, rollbackTargetBlockNumber);
   }
@@ -774,6 +803,9 @@ export {
   pendingBudget,
   startFetchingQueries,
   timestampCaughtUpToHeadOrEndblock,
+  recordBlockRangeFetch,
+  recordReorgDetected,
+  setRollbackTargetBlock,
   knownHeight,
   contractAddresses,
   bufferSize,
@@ -782,13 +814,12 @@ export {
   chainDensity,
   effectiveDensity,
   hasReadyItem,
-  isReadyToEnterReorgThreshold,
   targetBlock,
   blockAtProgress,
   frontierProgress,
   getNextQuery,
   dispatch,
-  toChainData,
+  toMetrics,
   toChainMetadata,
   toChainBeforeBatch,
   isReadyToEnterReorgThresholdAfterBatch,

package/src/ChainState.resi

@@ -20,6 +20,7 @@ let make: (
   ~transactionStore: TransactionStore.t=?,
   ~chainDensity: option<float>=?,
   ~blockStore: BlockStore.t=?,
+  ~reorgThresholdReadyTolerance: int=?,
   ~logger: Pino.t,
 ) => t
 
@@ -54,6 +55,17 @@ let pendingBudget: t => float
 let startFetchingQueries: (t, ~queries: array<FetchState.query>) => unit
 let timestampCaughtUpToHeadOrEndblock: t => option<Date.t>
 
+// Metric counters, rendered by Metrics at scrape time.
+let recordBlockRangeFetch: (
+  t,
+  ~totalTimeElapsed: float,
+  ~parsingTimeElapsed: float,
+  ~numEvents: int,
+  ~blockRangeSize: int,
+) => unit
+let recordReorgDetected: (t, ~blockNumber: int) => unit
+let setRollbackTargetBlock: (t, ~blockNumber: int) => unit
+
 // Fetch-frontier reads.
 let knownHeight: t => int
 let contractAddresses: (t, ~contractName: string) => array<Address.t>
@@ -63,7 +75,6 @@ let getProgressPercentage: t => float
 let chainDensity: t => option<float>
 let effectiveDensity: t => option<float>
 let hasReadyItem: t => bool
-let isReadyToEnterReorgThreshold: t => bool
 
 // Fetch control.
 let targetBlock: (t, ~chainTargetItems: float) => int
@@ -86,22 +97,7 @@ let dispatch: (
 ) => promise<unit>
 
 // Views.
-type chainData = {
-  chainId: float,
-  poweredByHyperSync: bool,
-  firstEventBlockNumber: option<int>,
-  latestProcessedBlock: option<int>,
-  timestampCaughtUpToHeadOrEndblock: option<Date.t>,
-  numEventsProcessed: float,
-  latestFetchedBlockNumber: int,
-  @as("currentBlockHeight")
-  knownHeight: int,
-  numBatchesFetched: int,
-  startBlock: int,
-  endBlock: option<int>,
-  numAddresses: int,
-}
-let toChainData: t => chainData
+let toMetrics: t => Metrics.chainMetrics
 let toChainMetadata: t => InternalTable.Chains.metaFields
 let toChainBeforeBatch: t => Batch.chainBeforeBatch
 let isReadyToEnterReorgThresholdAfterBatch: (t, ~batch: Batch.t) => bool

package/src/Config.res

@@ -79,6 +79,10 @@ type t = {
   enableRawEvents: bool,
   maxAddrInPartition: int,
   batchSize: int,
+  // Slack (in blocks) below the lagged head within which a chain still counts as
+  // ready to enter the reorg threshold, absorbing head advances between catch-up
+  // and the entry check. Overridable in tests.
+  reorgThresholdReadyTolerance: int,
   lowercaseAddresses: bool,
   isDev: bool,
   userEntitiesByName: dict<Internal.entityConfig>,
@@ -997,6 +1001,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
     ecosystem,
     maxAddrInPartition,
     batchSize: publicConfig["fullBatchSize"]->Option.getOr(5000),
+    reorgThresholdReadyTolerance: 100,
     lowercaseAddresses,
     isDev: publicConfig["isDev"]->Option.getOr(false),
     userEntitiesByName,

package/src/Config.res.mjs

@@ -698,6 +698,7 @@ function fromPublic(publicConfigJson) {
     enableRawEvents: Stdlib_Option.getOr(publicConfig.rawEvents, false),
     maxAddrInPartition: Env.maxAddrInPartition,
     batchSize: Stdlib_Option.getOr(publicConfig.fullBatchSize, 5000),
+    reorgThresholdReadyTolerance: 100,
     lowercaseAddresses: lowercaseAddresses,
     isDev: Stdlib_Option.getOr(publicConfig.isDev, false),
     userEntitiesByName: userEntitiesByName,