envio 3.11.0-svm-alpha.1 → 3.12.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.11.0-svm-alpha.1",
+  "version": "3.12.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -68,10 +68,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.11.0-svm-alpha.1",
-    "envio-linux-x64-musl": "3.11.0-svm-alpha.1",
-    "envio-linux-arm64": "3.11.0-svm-alpha.1",
-    "envio-darwin-x64": "3.11.0-svm-alpha.1",
-    "envio-darwin-arm64": "3.11.0-svm-alpha.1"
+    "envio-linux-x64": "3.12.0",
+    "envio-linux-x64-musl": "3.12.0",
+    "envio-linux-arm64": "3.12.0",
+    "envio-darwin-x64": "3.12.0",
+    "envio-darwin-arm64": "3.12.0"
   }
 }

package/src/Batch.res

@@ -4,6 +4,12 @@ open Utils.UnsafeIntOperators
 type chainAfterBatch = {
   batchSize: int,
   progressBlockNumber: int,
+  // Timestamp of `progressBlockNumber` itself, never of a lower block, so it
+  // measures how far behind chain time the committed progress is. `None` when
+  // the source didn't return that block's header - during backfill it only
+  // returns blocks carrying items, and an SVM slot may have produced no block
+  // at all.
+  progressBlockTime: option<int>,
   sourceBlockNumber: int,
   totalEventsProcessed: float,
   fetchState: FetchState.t,
@@ -23,6 +29,10 @@ type reorgHashSnapshot = {
 type chainBeforeBatch = {
   fetchState: FetchState.t,
   scannedHashes: reorgHashSnapshot,
+  // The progress block isn't known until the build computes it, so its
+  // timestamp can't be snapshotted with the rest. Reads the chain's store,
+  // which the synchronous build can't see mutate.
+  blockTimeAt: int => option<int>,
   shouldRollbackOnReorg: bool,
   progressBlockNumber: int,
   sourceBlockNumber: int,
@@ -70,6 +80,7 @@ let getProgressedChainsById = {
           {
             batchSize: counts.size,
             progressBlockNumber: progressBlockNumberAfterBatch,
+            progressBlockTime: chainBeforeBatch.blockTimeAt(progressBlockNumberAfterBatch),
             sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
             totalEventsProcessed: chainBeforeBatch.totalEventsProcessed +.
             counts.eventsProcessed->Int.toFloat,

package/src/Batch.res.mjs

@@ -10,6 +10,7 @@ function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAft
     return {
       batchSize: counts.size,
       progressBlockNumber: progressBlockNumberAfterBatch,
+      progressBlockTime: chainBeforeBatch.blockTimeAt(progressBlockNumberAfterBatch),
       sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
       totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + counts.eventsProcessed,
       fetchState: fetchStateAfterBatch,

package/src/ChainState.res

@@ -62,6 +62,9 @@ type t = {
   mutable reorgDetectedBlock: option<int>,
   mutable rollbackTargetBlock: option<int>,
   mutable progressLatencyMs: option<int>,
+  // Timestamp of the committed progress block itself. See
+  // `Batch.chainAfterBatch.progressBlockTime`.
+  mutable committedProgressBlockTime: option<int>,
 }
 
 // The chain's config-declared addresses as storage rows: what a clean run
@@ -125,6 +128,7 @@ let make = (
   ~addressStore: AddressStore.t,
   ~sourceManager: SourceManager.t,
   ~committedProgressBlockNumber: int,
+  ~committedProgressBlockTime=None,
   ~safeCheckpointTracking=None,
   ~shouldRollbackOnReorg,
   ~maxReorgDepth,
@@ -171,6 +175,7 @@ let make = (
     reorgDetectedBlock: None,
     rollbackTargetBlock: None,
     progressLatencyMs: None,
+    committedProgressBlockTime,
   }
 }
 
@@ -184,6 +189,7 @@ let makeInternal = (
   ~endBlock,
   ~firstEventBlock=None,
   ~progressBlockNumber,
+  ~committedProgressBlockTime=None,
   ~config: Config.t,
   ~registrationsByChainId: HandlerRegister.registrationsByChainId,
   ~logger,
@@ -319,6 +325,7 @@ let makeInternal = (
       ~chainReorgCheckpoints,
     ),
     ~committedProgressBlockNumber=progressBlockNumber,
+    ~committedProgressBlockTime,
     ~perChainEntities=config.userEntities->EntityTables.perChain,
     ~timestampCaughtUpToHeadOrEndblock,
     ~numEventsProcessed,
@@ -368,6 +375,10 @@ let makeFromDbState = (
     ~maxReorgDepth=resumedChainState.maxReorgDepth,
     ~firstEventBlock=resumedChainState.firstEventBlockNumber,
     ~progressBlockNumber,
+    // A block's time never changes, so the stored one stays true for the block
+    // progress is at - and keeps the metric reporting from the first scrape
+    // after a restart, rather than only once a batch commits.
+    ~committedProgressBlockTime=resumedChainState.progressBlockTime,
     ~timestampCaughtUpToHeadOrEndblock=resumedChainState.timestampCaughtUpToHeadOrEndblock,
     ~numEventsProcessed=resumedChainState.numEventsProcessed,
     ~logger,
@@ -407,6 +418,7 @@ let getLatestValidScannedBlock = (cs: t, ~blockStore: BlockStore.t, ~blockNumber
 let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
 let isProgressAtHead = (cs: t) => cs.isProgressAtHead
 let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
+let committedProgressBlockTime = (cs: t) => cs.committedProgressBlockTime
 let numEventsProcessed = (cs: t) => cs.numEventsProcessed
 let pendingBudget = (cs: t) => cs.pendingBudget
 let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock
@@ -985,6 +997,7 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
   sourceBlockNumber: cs.fetchState.knownHeight,
   progressBlockNumber: cs.committedProgressBlockNumber,
   progressLatencyMs: cs.progressLatencyMs,
+  progressBlockTime: cs.committedProgressBlockTime,
   concurrency: cs.sourceManager->SourceManager.inFlightCount,
   partitionsCount: cs.fetchState->FetchState.partitionsCount,
   bufferSize: cs.fetchState->FetchState.bufferSize,
@@ -1004,9 +1017,10 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
   rateLimitResetInMs: cs.sourceManager->SourceManager.getRateLimitResetInMs,
 }
 
-// Snapshot the inputs a batch build needs from this chain, including an
-// immutable copy of the scanned in-threshold block hashes so the batch never
-// reads the live store mid-build.
+// Snapshot the inputs a batch build needs from this chain. The scanned
+// in-threshold block hashes are copied up front because the build reuses the
+// whole set; the block time comes as a lookup instead, since the key it needs -
+// the batch's own progress block - isn't known until the build computes it.
 let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
   let {blockNumbers, hashes} =
     cs.blockStore->BlockStore.getHashes(
@@ -1023,6 +1037,8 @@ let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
     totalEventsProcessed: cs.numEventsProcessed,
     sourceBlockNumber: cs.fetchState.knownHeight,
     scannedHashes: {blockNumbers, hashByBlockNumber},
+    blockTimeAt: blockNumber =>
+      cs.blockStore->BlockStore.getTimestamp(blockNumber)->Null.toOption,
     shouldRollbackOnReorg: cs.shouldRollbackOnReorg,
     chainConfig: cs.chainConfig,
   }
@@ -1122,6 +1138,7 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
       }
 
       cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber
+      cs.committedProgressBlockTime = chainAfterBatch.progressBlockTime
 
       // Normally already set by advanceAfterBatch at batch creation; catch up
       // here for paths that commit progress without it.
@@ -1168,6 +1185,19 @@ let markReady = (cs: t, ~readyAt) =>
     cs.timestampCaughtUpToHeadOrEndblock = Some(readyAt)
   }
 
+// Rewind progress to a reorg target. Moving it means the timestamp now belongs
+// to an orphaned block, so it's re-read for the block progress actually lands
+// on - normally `None`, since the store keeps only hashes below the last
+// committed progress, until the re-fetch commits a new progress block. A target
+// at or above the progress block leaves both alone: that block was never
+// orphaned, and re-reading its long-pruned header would drop a valid timestamp.
+let rollbackCommittedProgress = (cs: t, blockNumber) =>
+  if blockNumber !== cs.committedProgressBlockNumber {
+    cs.committedProgressBlockNumber = blockNumber
+    cs.committedProgressBlockTime =
+      cs.blockStore->BlockStore.getTimestamp(blockNumber)->Null.toOption
+  }
+
 type progressDiff = {blockNumber: int, eventsProcessed: float}
 
 type rolledBackTo =
@@ -1215,7 +1245,7 @@ let rollback = (cs: t, ~rolledBackTo: rolledBackTo): array<AddressRows.key> => {
     let rolledBackAddresses = rollbackTo(newProgressBlockNumber)
     cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
     cs.blockStore->BlockStore.rollback(newProgressBlockNumber)
-    cs.committedProgressBlockNumber = newProgressBlockNumber
+    cs->rollbackCommittedProgress(newProgressBlockNumber)
     cs.processingBlockNumber = newProgressBlockNumber
     cs.numEventsProcessed = newTotalEventsProcessed
     rolledBackAddresses
@@ -1225,7 +1255,7 @@ let rollback = (cs: t, ~rolledBackTo: rolledBackTo): array<AddressRows.key> => {
     let rolledBackAddresses = rollbackTo(forkBlock)
     cs.transactionStore->TransactionStore.rollback(forkBlock)
     cs.blockStore->BlockStore.rollback(forkBlock)
-    cs.committedProgressBlockNumber = Pervasives.min(cs.committedProgressBlockNumber, forkBlock)
+    cs->rollbackCommittedProgress(Pervasives.min(cs.committedProgressBlockNumber, forkBlock))
     cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, forkBlock)
     rolledBackAddresses
   | Untouched => []

package/src/ChainState.res.mjs

@@ -55,8 +55,9 @@ function validateOnEventRegistrations(chainId, registrations) {
   });
 }
 
-function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, sourceManager, committedProgressBlockNumber, safeCheckpointTrackingOpt, shouldRollbackOnReorg, maxReorgDepth, isInReorgThresholdOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, perChainEntitiesOpt, logger) {
+function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, sourceManager, committedProgressBlockNumber, committedProgressBlockTimeOpt, safeCheckpointTrackingOpt, shouldRollbackOnReorg, maxReorgDepth, isInReorgThresholdOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, perChainEntitiesOpt, logger) {
   let onEventRegistrations = onEventRegistrationsOpt !== undefined ? onEventRegistrationsOpt : [];
+  let committedProgressBlockTime = committedProgressBlockTimeOpt !== undefined ? Primitive_option.valFromOption(committedProgressBlockTimeOpt) : undefined;
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
   let isInReorgThreshold = isInReorgThresholdOpt !== undefined ? isInReorgThresholdOpt : false;
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
@@ -98,7 +99,8 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
     reorgCount: 0,
     reorgDetectedBlock: undefined,
     rollbackTargetBlock: undefined,
-    progressLatencyMs: undefined
+    progressLatencyMs: undefined,
+    committedProgressBlockTime: committedProgressBlockTime
   };
 }
 
@@ -112,12 +114,14 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
   let startBlock = resumedChainState.startBlock;
   let endBlock = resumedChainState.endBlock;
   let firstEventBlockOpt = Primitive_option.some(resumedChainState.firstEventBlockNumber);
+  let committedProgressBlockTimeOpt = Primitive_option.some(resumedChainState.progressBlockTime);
   let timestampCaughtUpToHeadOrEndblock = resumedChainState.timestampCaughtUpToHeadOrEndblock;
   let numEventsProcessed = resumedChainState.numEventsProcessed;
   let maxReorgDepth = resumedChainState.maxReorgDepth;
   let knownHeightOpt = resumedChainState.sourceBlockNumber;
   let isResumedOpt = true;
   let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
+  let committedProgressBlockTime = committedProgressBlockTimeOpt !== undefined ? Primitive_option.valFromOption(committedProgressBlockTimeOpt) : undefined;
   let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
   let isResumed = isResumedOpt !== undefined ? isResumedOpt : false;
   let match = Stdlib_Option.getOr(registrationsByChainId[ChainId.toString(chainConfig.id)], {
@@ -161,7 +165,7 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
   }
   let firstEventBlock$1 = fetchState.firstEventBlock;
   let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
-  return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, isInReorgThreshold, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.userEntities), logger);
+  return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(committedProgressBlockTime), Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, isInReorgThreshold, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.userEntities), logger);
 }
 
 function logger(cs) {
@@ -212,6 +216,10 @@ function committedProgressBlockNumber(cs) {
   return cs.committedProgressBlockNumber;
 }
 
+function committedProgressBlockTime(cs) {
+  return cs.committedProgressBlockTime;
+}
+
 function numEventsProcessed(cs) {
   return cs.numEventsProcessed;
 }
@@ -689,6 +697,7 @@ function toMetrics(cs) {
     sourceBlockNumber: cs.fetchState.knownHeight,
     progressBlockNumber: cs.committedProgressBlockNumber,
     progressLatencyMs: cs.progressLatencyMs,
+    progressBlockTime: cs.committedProgressBlockTime,
     concurrency: SourceManager.inFlightCount(cs.sourceManager),
     partitionsCount: FetchState.partitionsCount(cs.fetchState),
     bufferSize: FetchState.bufferSize(cs.fetchState),
@@ -723,6 +732,7 @@ function toChainBeforeBatch(cs) {
       blockNumbers: blockNumbers,
       hashByBlockNumber: hashByBlockNumber
     },
+    blockTimeAt: blockNumber => Primitive_option.fromNull(cs.blockStore.getTimestamp(blockNumber)),
     shouldRollbackOnReorg: cs.shouldRollbackOnReorg,
     progressBlockNumber: cs.committedProgressBlockNumber,
     sourceBlockNumber: cs.fetchState.knownHeight,
@@ -802,6 +812,7 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
     }
   }
   cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber;
+  cs.committedProgressBlockTime = chainAfterBatch.progressBlockTime;
   cs.processingBlockNumber = Primitive_int.max(cs.processingBlockNumber, chainAfterBatch.progressBlockNumber);
   cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed;
   cs.transactionStore.prune(chainAfterBatch.progressBlockNumber);
@@ -821,6 +832,14 @@ function markReady(cs, readyAt) {
   }
 }
 
+function rollbackCommittedProgress(cs, blockNumber) {
+  if (blockNumber !== cs.committedProgressBlockNumber) {
+    cs.committedProgressBlockNumber = blockNumber;
+    cs.committedProgressBlockTime = Primitive_option.fromNull(cs.blockStore.getTimestamp(blockNumber));
+    return;
+  }
+}
+
 function rollback(cs, rolledBackTo) {
   let rollbackTo = targetBlockNumber => {
     let match = FetchState.rollback(cs.fetchState, cs.addressStore, targetBlockNumber);
@@ -845,7 +864,7 @@ function rollback(cs, rolledBackTo) {
     let rolledBackAddresses = rollbackTo(newProgressBlockNumber);
     cs.transactionStore.rollback(newProgressBlockNumber);
     cs.blockStore.rollback(newProgressBlockNumber);
-    cs.committedProgressBlockNumber = newProgressBlockNumber;
+    rollbackCommittedProgress(cs, newProgressBlockNumber);
     cs.processingBlockNumber = newProgressBlockNumber;
     cs.numEventsProcessed = newTotalEventsProcessed;
     return rolledBackAddresses;
@@ -854,7 +873,7 @@ function rollback(cs, rolledBackTo) {
   let rolledBackAddresses$1 = rollbackTo(forkBlock);
   cs.transactionStore.rollback(forkBlock);
   cs.blockStore.rollback(forkBlock);
-  cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, forkBlock);
+  rollbackCommittedProgress(cs, Primitive_int.min(cs.committedProgressBlockNumber, forkBlock));
   cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, forkBlock);
   return rolledBackAddresses$1;
 }
@@ -876,6 +895,7 @@ export {
   safeCheckpointTracking,
   isProgressAtHead,
   committedProgressBlockNumber,
+  committedProgressBlockTime,
   numEventsProcessed,
   pendingBudget,
   startFetchingQueries,

package/src/ChainState.resi

@@ -23,6 +23,7 @@ let make: (
   ~addressStore: AddressStore.t,
   ~sourceManager: SourceManager.t,
   ~committedProgressBlockNumber: int,
+  ~committedProgressBlockTime: option<int>=?,
   ~safeCheckpointTracking: option<SafeCheckpointTracking.t>=?,
   ~shouldRollbackOnReorg: bool,
   ~maxReorgDepth: int,
@@ -68,6 +69,7 @@ let getLatestValidScannedBlock: (
 let safeCheckpointTracking: t => option<SafeCheckpointTracking.t>
 let isProgressAtHead: t => bool
 let committedProgressBlockNumber: t => int
+let committedProgressBlockTime: t => option<int>
 let numEventsProcessed: t => float
 let pendingBudget: t => float
 let startFetchingQueries: (t, ~queries: array<FetchState.query>) => unit

package/src/CrossChainState.res

@@ -257,6 +257,10 @@ let idleOrWaitAction = (cs: ChainState.t) =>
     ? FetchState.NothingToQuery
     : FetchState.WaitingForNewBlock
 
+// How far past the alignment anchor's frontier progress a follower may fetch,
+// as a fraction of its own alignable range.
+let alignmentMargin = 0.2
+
 // Dispatch a fetch tick across the whole indexer from one shared pool of
 // ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
 // first, hand each the budget remaining at that point (plus its own
@@ -340,11 +344,14 @@ let checkAndFetch = async (
         (isCold ? Pervasives.min(remaining.contents, coldChainBudget) : remaining.contents) +.
         cs->ChainState.pendingBudget
       let maxTargetBlock = switch alignment {
-      // 10% margin past the anchor's line: chains whose progress tracks the
+      // 20% margin past the anchor's line: chains whose progress tracks the
       // anchor closely would otherwise flap in and out of the clamp on every
-      // small frontier move, stalling their pipeline every other tick.
+      // small frontier move, stalling their pipeline every other tick. The
+      // margin is also the headroom a follower keeps buffered while the anchor
+      // is mid-fetch, so it has to outlast a slow anchor response, not just
+      // absorb frontier jitter.
       | Some((anchorChainId, progress)) if anchorChainId !== chainId =>
-        Some(cs->ChainState.blockAtProgress(~progress=progress +. 0.1))
+        Some(cs->ChainState.blockAtProgress(~progress=progress +. alignmentMargin))
       | _ => None
       }
       switch cs->ChainState.getNextQuery(~chainTargetItems, ~maxTargetBlock?) {

package/src/CrossChainState.res.mjs

@@ -199,7 +199,7 @@ async function checkAndFetch(crossChainState, dispatchChain) {
     let chainTargetItems = (
       isCold ? Primitive_float.min(remaining.contents, minimumAdmissionBudget) : remaining.contents
     ) + ChainState.pendingBudget(cs);
-    let maxTargetBlock = alignment !== undefined && alignment[0] !== chainId ? ChainState.blockAtProgress(cs, alignment[1] + 0.1) : undefined;
+    let maxTargetBlock = alignment !== undefined && alignment[0] !== chainId ? ChainState.blockAtProgress(cs, alignment[1] + 0.2) : undefined;
     let action = ChainState.getNextQuery(cs, chainTargetItems, maxTargetBlock);
     if (typeof action !== "object") {
       if (action === "WaitingForNewBlock") {

package/src/Metrics.res

@@ -20,6 +20,7 @@ type chainMetrics = {
   // Raw committed progress (may be -1), unlike the optional latestProcessedBlock.
   progressBlockNumber: int,
   progressLatencyMs: option<int>,
+  progressBlockTime: option<int>,
   concurrency: int,
   partitionsCount: int,
   bufferSize: int,
@@ -659,6 +660,13 @@ let renderMetrics = (b: builder, metrics: t) => {
     ~entries=chains,
     ~value=m => m.numEventsProcessed,
   )
+  b->seriesOpt(
+    ~name="envio_progress_block_time_seconds",
+    ~help="Unix timestamp of the block the chain has processed up to. Subtract it from the scrape time for how far behind chain time the indexer is, which stays honest when the data source itself is behind the chain. Best effort in realtime mode, absent during backfill.",
+    ~kind="gauge",
+    ~entries=chains,
+    ~value=m => m.progressBlockTime->Option.map(Int.toFloat),
+  )
   b->seriesOpt(
     ~name="envio_progress_latency",
     ~help="The latency in milliseconds between the latest processed event creation and the time it was written to storage.",

package/src/Metrics.res.mjs

@@ -221,6 +221,7 @@ function renderMetrics(b, metrics) {
   single(b, "envio_processing_max_batch_size", "The maximum number of items to process in a single batch.", "gauge", metrics.maxBatchSize);
   series(b, "envio_progress_block", "The block number of the latest block processed and stored in the database.", "gauge", chains, m => m.progressBlockNumber);
   series(b, "envio_progress_events", "The number of events processed and reflected in the database.", "gauge", chains, m => m.numEventsProcessed);
+  seriesOpt(b, "envio_progress_block_time_seconds", "Unix timestamp of the block the chain has processed up to. Subtract it from the scrape time for how far behind chain time the indexer is, which stays honest when the data source itself is behind the chain. Best effort in realtime mode, absent during backfill.", "gauge", chains, m => Stdlib_Option.map(m.progressBlockTime, prim => prim));
   seriesOpt(b, "envio_progress_latency", "The latency in milliseconds between the latest processed event creation and the time it was written to storage.", "gauge", chains, m => Stdlib_Option.map(m.progressLatencyMs, prim => prim));
   let ifCalled = (s, value) => {
     if (s.callCount > 0 || s.activeCallsCount > 0) {

package/src/Persistence.res

@@ -24,6 +24,7 @@ type initialChainState = {
   endBlock: option<int>,
   maxReorgDepth: int,
   progressBlockNumber: int,
+  progressBlockTime: option<int>,
   numEventsProcessed: float,
   firstEventBlockNumber: option<int>,
   timestampCaughtUpToHeadOrEndblock: option<Date.t>,

package/src/PgStorage.res

@@ -1449,6 +1449,7 @@ let rec writeBatch = async (
                 ): InternalTable.Chains.progressedChain => {
                   chainId: chainAfterBatch.fetchState.chainId,
                   progressBlockNumber: chainAfterBatch.progressBlockNumber,
+                  progressBlockTime: chainAfterBatch.progressBlockTime,
                   sourceBlockNumber: chainAfterBatch.sourceBlockNumber,
                   totalEventsProcessed: chainAfterBatch.totalEventsProcessed,
                 }),
@@ -1992,6 +1993,7 @@ let make = (
         firstEventBlockNumber: None,
         timestampCaughtUpToHeadOrEndblock: None,
         addressRows: rowsByChain->Array.getUnsafe(idx)->AddressRows.seedRowsOf,
+        progressBlockTime: None,
         sourceBlockNumber: 0,
       }),
       checkpointFrontier: Frontier.empty(),
@@ -2427,6 +2429,7 @@ let make = (
             timestampCaughtUpToHeadOrEndblock: rawInitialState.timestampCaughtUpToHeadOrEndblock->Null.toOption,
             numEventsProcessed: rawInitialState.numEventsProcessed,
             progressBlockNumber: rawInitialState.progressBlockNumber,
+            progressBlockTime: rawInitialState.progressBlockTime->InternalTable.Chains.blockTimeFromDb,
             addressRows: rawInitialState.addressRows,
             sourceBlockNumber: rawInitialState.sourceBlockNumber,
           }),

package/src/PgStorage.res.mjs

@@ -914,6 +914,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, config, allEntities, s
             sql => InternalTable.Chains.setProgressedChains(sql, pgSchema, Utils.Dict.mapValuesToArray(batch.progressedChainsById, chainAfterBatch => ({
               chainId: chainAfterBatch.fetchState.chainId,
               progressBlockNumber: chainAfterBatch.progressBlockNumber,
+              progressBlockTime: chainAfterBatch.progressBlockTime,
               sourceBlockNumber: chainAfterBatch.sourceBlockNumber,
               totalEventsProcessed: chainAfterBatch.totalEventsProcessed
             }))),
@@ -1208,6 +1209,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
         endBlock: chainConfig.endBlock,
         maxReorgDepth: chainConfig.maxReorgDepth,
         progressBlockNumber: -1,
+        progressBlockTime: undefined,
         numEventsProcessed: 0,
         firstEventBlockNumber: undefined,
         timestampCaughtUpToHeadOrEndblock: undefined,
@@ -1480,6 +1482,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
             endBlock: Primitive_option.fromNull(rawInitialState.endBlock),
             maxReorgDepth: rawInitialState.maxReorgDepth,
             progressBlockNumber: rawInitialState.progressBlockNumber,
+            progressBlockTime: InternalTable.Chains.blockTimeFromDb(rawInitialState.progressBlockTime),
             numEventsProcessed: rawInitialState.numEventsProcessed,
             firstEventBlockNumber: Primitive_option.fromNull(rawInitialState.firstEventBlockNumber),
             timestampCaughtUpToHeadOrEndblock: Primitive_option.fromNull(rawInitialState.timestampCaughtUpToHeadOrEndblock),

package/src/Rollback.res

@@ -200,6 +200,7 @@ and executeRollback = async (
       ->Array.push({
         chainId,
         progressBlockNumber: toBlock,
+        progressBlockTime: cs->ChainState.committedProgressBlockTime,
         sourceBlockNumber: cs->ChainState.knownHeight,
         totalEventsProcessed: cs->ChainState.numEventsProcessed,
       })

package/src/Rollback.res.mjs

@@ -86,6 +86,7 @@ async function executeRollback(state, reorgChain, rollbackTargetBlockNumber, sch
       rolledBackChains.push({
         chainId: chainId,
         progressBlockNumber: toBlock,
+        progressBlockTime: ChainState.committedProgressBlockTime(cs),
         sourceBlockNumber: ChainState.knownHeight(cs),
         totalEventsProcessed: ChainState.numEventsProcessed(cs)
       });