envio 3.11.0 → 3.12.1

package/src/BatchProcessing.res

@@ -69,17 +69,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
   let isInReorgThresholdBeforeUpdate = state->IndexerState.isInReorgThreshold
   let isRealtimeBeforeUpdate = state->IndexerState.isRealtime
 
-  let isRollbackBatch = switch state->IndexerState.rollbackState {
-  | RollbackReady(_) => true
-  | _ => false
-  }
-
-  let batch =
-    state->IndexerState.createBatch(
-      ~processedCheckpointId=state->IndexerState.processedCheckpointId,
-      ~batchSizeTarget=(state->IndexerState.config).batchSize,
-      ~isRollback=isRollbackBatch,
-    )
+  let batch = state->IndexerState.createBatch(~batchSizeTarget=(state->IndexerState.config).batchSize)
 
   let progressedChainsById = batch.progressedChainsById
 

package/src/BatchProcessing.res.mjs

@@ -23,10 +23,7 @@ function yieldTick() {
 async function processNextBatch(state, scheduleFetch) {
   let isInReorgThresholdBeforeUpdate = IndexerState.isInReorgThreshold(state);
   let isRealtimeBeforeUpdate = IndexerState.isRealtime(state);
-  let match = IndexerState.rollbackState(state);
-  let isRollbackBatch;
-  isRollbackBatch = typeof match !== "object" ? false : match.TAG === "RollbackReady";
-  let batch = IndexerState.createBatch(state, IndexerState.processedCheckpointId(state), IndexerState.config(state).batchSize, isRollbackBatch);
+  let batch = IndexerState.createBatch(state, IndexerState.config(state).batchSize);
   let progressedChainsById = batch.progressedChainsById;
   let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
   let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));

package/src/ChainFetching.res

@@ -326,9 +326,13 @@ let fetchChain = async (
     let isRealtime = state->IndexerState.isRealtime
     let sourceManager = chainState->ChainState.sourceManager
 
-    // Only affects the WaitingForNewBlock branch of dispatch, where
-    // there's nothing to fetch. During backfill any such chain is idle.
-    let reducedPolling = !isRealtime
+    // Only affects the WaitingForNewBlock branch of dispatch, where there's
+    // nothing to fetch. The line is whether any height is known at all, not
+    // whether the head is fetchable yet: a chain held back by its lag is still
+    // tracking a head it has, and asking faster would buy nothing. A chain with
+    // no height has nothing to show for itself until the first one lands, so it
+    // polls at the source's own cadence and earns the normal stall window.
+    let reducedPolling = !isRealtime && chainState->ChainState.knownHeight > 0
 
     // Owns its error boundary: launch doesn't catch, so any failure here (the
     // query, response handling, or dispatch itself) must stop the indexer.

package/src/ChainFetching.res.mjs

@@ -180,7 +180,7 @@ async function fetchChain(state, chainId, action, stateId, scheduleFetch, schedu
   }
   let isRealtime = IndexerState.isRealtime(state);
   let sourceManager = ChainState.sourceManager(chainState);
-  let reducedPolling = !isRealtime;
+  let reducedPolling = !isRealtime && ChainState.knownHeight(chainState) > 0;
   try {
     return await ChainState.dispatch(chainState, async query => {
       try {

package/src/ChainState.res

@@ -41,6 +41,8 @@ type t = {
   // detected reorg either rolls back or is only logged.
   shouldRollbackOnReorg: bool,
   maxReorgDepth: int,
+  // One-way: past it, everything the chain writes can still be rolled back.
+  mutable isInReorgThreshold: bool,
   // Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
   // transactionIndex). Fetch responses merge their page in; entries are pruned
   // as the chain progresses and dropped above the target on rollback.
@@ -60,6 +62,9 @@ type t = {
   mutable reorgDetectedBlock: option<int>,
   mutable rollbackTargetBlock: option<int>,
   mutable progressLatencyMs: option<int>,
+  // Timestamp of the committed progress block itself. See
+  // `Batch.chainAfterBatch.progressBlockTime`.
+  mutable committedProgressBlockTime: option<int>,
 }
 
 // The chain's config-declared addresses as storage rows: what a clean run
@@ -123,9 +128,11 @@ let make = (
   ~addressStore: AddressStore.t,
   ~sourceManager: SourceManager.t,
   ~committedProgressBlockNumber: int,
+  ~committedProgressBlockTime=None,
   ~safeCheckpointTracking=None,
   ~shouldRollbackOnReorg,
   ~maxReorgDepth,
+  ~isInReorgThreshold=false,
   ~numEventsProcessed=0.,
   ~timestampCaughtUpToHeadOrEndblock=None,
   ~isProgressAtHead=false,
@@ -155,6 +162,7 @@ let make = (
     safeCheckpointTracking,
     shouldRollbackOnReorg,
     maxReorgDepth,
+    isInReorgThreshold,
     transactionStore,
     blockStore,
     reorgThresholdReadyTolerance,
@@ -167,6 +175,7 @@ let make = (
     reorgDetectedBlock: None,
     rollbackTargetBlock: None,
     progressLatencyMs: None,
+    committedProgressBlockTime,
   }
 }
 
@@ -180,6 +189,7 @@ let makeInternal = (
   ~endBlock,
   ~firstEventBlock=None,
   ~progressBlockNumber,
+  ~committedProgressBlockTime=None,
   ~config: Config.t,
   ~registrationsByChainId: HandlerRegister.registrationsByChainId,
   ~logger,
@@ -308,12 +318,14 @@ let makeInternal = (
     ~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
     ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
     ~maxReorgDepth,
+    ~isInReorgThreshold,
     ~safeCheckpointTracking=SafeCheckpointTracking.make(
       ~maxReorgDepth,
       ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
       ~chainReorgCheckpoints,
     ),
     ~committedProgressBlockNumber=progressBlockNumber,
+    ~committedProgressBlockTime,
     ~perChainEntities=config.userEntities->EntityTables.perChain,
     ~timestampCaughtUpToHeadOrEndblock,
     ~numEventsProcessed,
@@ -363,6 +375,10 @@ let makeFromDbState = (
     ~maxReorgDepth=resumedChainState.maxReorgDepth,
     ~firstEventBlock=resumedChainState.firstEventBlockNumber,
     ~progressBlockNumber,
+    // A block's time never changes, so the stored one stays true for the block
+    // progress is at - and keeps the metric reporting from the first scrape
+    // after a restart, rather than only once a batch commits.
+    ~committedProgressBlockTime=resumedChainState.progressBlockTime,
     ~timestampCaughtUpToHeadOrEndblock=resumedChainState.timestampCaughtUpToHeadOrEndblock,
     ~numEventsProcessed=resumedChainState.numEventsProcessed,
     ~logger,
@@ -402,6 +418,7 @@ let getLatestValidScannedBlock = (cs: t, ~blockStore: BlockStore.t, ~blockNumber
 let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
 let isProgressAtHead = (cs: t) => cs.isProgressAtHead
 let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
+let committedProgressBlockTime = (cs: t) => cs.committedProgressBlockTime
 let numEventsProcessed = (cs: t) => cs.numEventsProcessed
 let pendingBudget = (cs: t) => cs.pendingBudget
 let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock
@@ -640,15 +657,21 @@ let hasProcessedToEndblock = (cs: t) => {
 // head has run away from it while the indexer was down.
 let isDurablyCaughtUp = (cs: t) => {
   let {committedProgressBlockNumber, fetchState} = cs
-  switch fetchState.endBlock {
-  | Some(endBlock) => committedProgressBlockNumber >= endBlock
-  | None =>
-    // The configured lag, not the fetch state's: pre-threshold that one also
-    // carries maxReorgDepth, which would read a chain a whole reorg depth behind
-    // the head as caught up.
+  let atEndBlock =
+    fetchState.endBlock->Option.mapOr(false, endBlock => committedProgressBlockNumber >= endBlock)
+  // Either one, like `isFetchingAtHead`: an `end_block` above the head is never
+  // reached, and testing only for it would leave such a chain reading as behind
+  // however long it sits at the head.
+  //
+  // The configured lag, not the fetch state's: pre-threshold that one also
+  // carries maxReorgDepth, which would read a chain a whole reorg depth behind
+  // the head as caught up. Clamped at zero: the -1 a run that has processed
+  // nothing carries would otherwise clear a chain younger than its own lag.
+  let atHead =
     fetchState.knownHeight > 0 &&
-      committedProgressBlockNumber >= fetchState.knownHeight - cs.chainConfig.blockLag
-  }
+      committedProgressBlockNumber >=
+      Pervasives.max(0, fetchState.knownHeight - cs.chainConfig.blockLag)
+  atEndBlock || atHead
 }
 
 let getHighestBlockBelowThreshold = (cs: t): int => {
@@ -929,8 +952,18 @@ let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>
   }
 
 // Shrink the fetch buffer by the configured blockLag on entering the reorg threshold.
-let enterReorgThreshold = (cs: t) =>
+let enterReorgThreshold = (cs: t) => {
+  cs.isInReorgThreshold = true
   cs.fetchState = cs.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
+}
+
+let isInReorgThreshold = (cs: t) => cs.isInReorgThreshold
+
+// Whether the chain's writes need history: only what a rollback could still
+// reach. A chain with no reorg depth is never rolled back, however far its
+// progress has run.
+let shouldSaveHistory = (cs: t) =>
+  cs.shouldRollbackOnReorg && cs.maxReorgDepth > 0 && cs.isInReorgThreshold
 
 // Snapshot the chain's metadata fields for staging into the chains table.
 let toChainMetadata = (cs: t): InternalTable.Chains.metaFields => {
@@ -964,6 +997,7 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
   sourceBlockNumber: cs.fetchState.knownHeight,
   progressBlockNumber: cs.committedProgressBlockNumber,
   progressLatencyMs: cs.progressLatencyMs,
+  progressBlockTime: cs.committedProgressBlockTime,
   concurrency: cs.sourceManager->SourceManager.inFlightCount,
   partitionsCount: cs.fetchState->FetchState.partitionsCount,
   bufferSize: cs.fetchState->FetchState.bufferSize,
@@ -979,12 +1013,15 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
   reorgCount: cs.reorgCount,
   reorgDetectedBlock: cs.reorgDetectedBlock,
   rollbackTargetBlock: cs.rollbackTargetBlock,
+  rateLimitTimeMs: cs.sourceManager->SourceManager.getRateLimitTimeMs,
+  rateLimitResetInMs: cs.sourceManager->SourceManager.getRateLimitResetInMs,
 }
 
-// Snapshot the inputs a batch build needs from this chain, including an
-// immutable copy of the scanned in-threshold block hashes so the batch never
-// reads the live store mid-build.
-let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
+// Snapshot the inputs a batch build needs from this chain. The scanned
+// in-threshold block hashes are copied up front because the build reuses the
+// whole set; the block time comes as a lookup instead, since the key it needs -
+// the batch's own progress block - isn't known until the build computes it.
+let toChainBeforeBatch = (cs: t, ~isRealtime): Batch.chainBeforeBatch => {
   let {blockNumbers, hashes} =
     cs.blockStore->BlockStore.getHashes(
       ~fromBlock=Pervasives.max(cs.fetchState.knownHeight - cs.maxReorgDepth, 0),
@@ -1000,6 +1037,13 @@ let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
     totalEventsProcessed: cs.numEventsProcessed,
     sourceBlockNumber: cs.fetchState.knownHeight,
     scannedHashes: {blockNumbers, hashByBlockNumber},
+    // A slot that produced no block only counts as skipped where the query
+    // covered every slot in its range, which is what a chain at the head asks
+    // for.
+    blockTimeAt: blockNumber =>
+      cs.blockStore
+      ->BlockStore.getTimestamp(blockNumber, ~allowSkippedSlot=isRealtime)
+      ->Null.toOption,
     shouldRollbackOnReorg: cs.shouldRollbackOnReorg,
     chainConfig: cs.chainConfig,
   }
@@ -1099,6 +1143,7 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
       }
 
       cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber
+      cs.committedProgressBlockTime = chainAfterBatch.progressBlockTime
 
       // Normally already set by advanceAfterBatch at batch creation; catch up
       // here for paths that commit progress without it.
@@ -1135,16 +1180,33 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
 }
 
 // Mark the chain caught up to head/endblock. Called by CrossChainState only once
-// every chain in the indexer is caught up and the deferred schema indexes are
-// committed, so no chain flips to ready while another is still backfilling or
-// while an index the schema promises is still missing. `readyAt` is the
-// timestamp already committed to `envio_chains.ready_at` in that same
-// transaction. Sticky: a chain stays ready once set.
+// every chain this process drives is caught up and the indexes the schema
+// promises them are committed, so no chain flips to ready while an index it
+// promised is still missing. `readyAt` is the timestamp already committed to
+// `envio_chains.ready_at` in that same transaction. Sticky: a chain stays ready
+// once set.
 let markReady = (cs: t, ~readyAt) =>
   if !(cs->isReady) {
     cs.timestampCaughtUpToHeadOrEndblock = Some(readyAt)
   }
 
+// Rewind progress to a reorg target. Moving it means the timestamp now belongs
+// to an orphaned block, so it's re-read for the block progress actually lands
+// on - normally `None`, since the store keeps only hashes below the last
+// committed progress, until the re-fetch commits a new progress block. A target
+// at or above the progress block leaves both alone: that block was never
+// orphaned, and re-reading its long-pruned header would drop a valid timestamp.
+let rollbackCommittedProgress = (cs: t, blockNumber) =>
+  if blockNumber !== cs.committedProgressBlockNumber {
+    cs.committedProgressBlockNumber = blockNumber
+    // Exact block only: the rolled-back region is about to be refetched, and
+    // the next batch re-establishes the time either way.
+    cs.committedProgressBlockTime =
+      cs.blockStore
+      ->BlockStore.getTimestamp(blockNumber, ~allowSkippedSlot=false)
+      ->Null.toOption
+  }
+
 type progressDiff = {blockNumber: int, eventsProcessed: float}
 
 type rolledBackTo =
@@ -1192,7 +1254,7 @@ let rollback = (cs: t, ~rolledBackTo: rolledBackTo): array<AddressRows.key> => {
     let rolledBackAddresses = rollbackTo(newProgressBlockNumber)
     cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
     cs.blockStore->BlockStore.rollback(newProgressBlockNumber)
-    cs.committedProgressBlockNumber = newProgressBlockNumber
+    cs->rollbackCommittedProgress(newProgressBlockNumber)
     cs.processingBlockNumber = newProgressBlockNumber
     cs.numEventsProcessed = newTotalEventsProcessed
     rolledBackAddresses
@@ -1202,7 +1264,7 @@ let rollback = (cs: t, ~rolledBackTo: rolledBackTo): array<AddressRows.key> => {
     let rolledBackAddresses = rollbackTo(forkBlock)
     cs.transactionStore->TransactionStore.rollback(forkBlock)
     cs.blockStore->BlockStore.rollback(forkBlock)
-    cs.committedProgressBlockNumber = Pervasives.min(cs.committedProgressBlockNumber, forkBlock)
+    cs->rollbackCommittedProgress(Pervasives.min(cs.committedProgressBlockNumber, forkBlock))
     cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, forkBlock)
     rolledBackAddresses
   | Untouched => []

package/src/ChainState.res.mjs

@@ -55,9 +55,11 @@ function validateOnEventRegistrations(chainId, registrations) {
   });
 }
 
-function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, sourceManager, committedProgressBlockNumber, safeCheckpointTrackingOpt, shouldRollbackOnReorg, maxReorgDepth, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, perChainEntitiesOpt, logger) {
+function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, sourceManager, committedProgressBlockNumber, committedProgressBlockTimeOpt, safeCheckpointTrackingOpt, shouldRollbackOnReorg, maxReorgDepth, isInReorgThresholdOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, perChainEntitiesOpt, logger) {
   let onEventRegistrations = onEventRegistrationsOpt !== undefined ? onEventRegistrationsOpt : [];
+  let committedProgressBlockTime = committedProgressBlockTimeOpt !== undefined ? Primitive_option.valFromOption(committedProgressBlockTimeOpt) : undefined;
   let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
+  let isInReorgThreshold = isInReorgThresholdOpt !== undefined ? isInReorgThresholdOpt : false;
   let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
   let timestampCaughtUpToHeadOrEndblock = timestampCaughtUpToHeadOrEndblockOpt !== undefined ? Primitive_option.valFromOption(timestampCaughtUpToHeadOrEndblockOpt) : undefined;
   let isProgressAtHead = isProgressAtHeadOpt !== undefined ? isProgressAtHeadOpt : false;
@@ -85,6 +87,7 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
     safeCheckpointTracking: safeCheckpointTracking,
     shouldRollbackOnReorg: shouldRollbackOnReorg,
     maxReorgDepth: maxReorgDepth,
+    isInReorgThreshold: isInReorgThreshold,
     transactionStore: transactionStore,
     blockStore: blockStore,
     reorgThresholdReadyTolerance: reorgThresholdReadyTolerance,
@@ -96,7 +99,8 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
     reorgCount: 0,
     reorgDetectedBlock: undefined,
     rollbackTargetBlock: undefined,
-    progressLatencyMs: undefined
+    progressLatencyMs: undefined,
+    committedProgressBlockTime: committedProgressBlockTime
   };
 }
 
@@ -110,12 +114,14 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
   let startBlock = resumedChainState.startBlock;
   let endBlock = resumedChainState.endBlock;
   let firstEventBlockOpt = Primitive_option.some(resumedChainState.firstEventBlockNumber);
+  let committedProgressBlockTimeOpt = Primitive_option.some(resumedChainState.progressBlockTime);
   let timestampCaughtUpToHeadOrEndblock = resumedChainState.timestampCaughtUpToHeadOrEndblock;
   let numEventsProcessed = resumedChainState.numEventsProcessed;
   let maxReorgDepth = resumedChainState.maxReorgDepth;
   let knownHeightOpt = resumedChainState.sourceBlockNumber;
   let isResumedOpt = true;
   let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
+  let committedProgressBlockTime = committedProgressBlockTimeOpt !== undefined ? Primitive_option.valFromOption(committedProgressBlockTimeOpt) : undefined;
   let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
   let isResumed = isResumedOpt !== undefined ? isResumedOpt : false;
   let match = Stdlib_Option.getOr(registrationsByChainId[ChainId.toString(chainConfig.id)], {
@@ -159,7 +165,7 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
   }
   let firstEventBlock$1 = fetchState.firstEventBlock;
   let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
-  return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.userEntities), logger);
+  return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(committedProgressBlockTime), Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, isInReorgThreshold, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.userEntities), logger);
 }
 
 function logger(cs) {
@@ -210,6 +216,10 @@ function committedProgressBlockNumber(cs) {
   return cs.committedProgressBlockNumber;
 }
 
+function committedProgressBlockTime(cs) {
+  return cs.committedProgressBlockTime;
+}
+
 function numEventsProcessed(cs) {
   return cs.numEventsProcessed;
 }
@@ -402,13 +412,12 @@ function hasProcessedToEndblock(cs) {
 function isDurablyCaughtUp(cs) {
   let fetchState = cs.fetchState;
   let committedProgressBlockNumber = cs.committedProgressBlockNumber;
-  let endBlock = fetchState.endBlock;
-  if (endBlock !== undefined) {
-    return committedProgressBlockNumber >= endBlock;
-  } else if (fetchState.knownHeight > 0) {
-    return committedProgressBlockNumber >= (fetchState.knownHeight - cs.chainConfig.blockLag | 0);
+  let atEndBlock = Stdlib_Option.mapOr(fetchState.endBlock, false, endBlock => committedProgressBlockNumber >= endBlock);
+  let atHead = fetchState.knownHeight > 0 && committedProgressBlockNumber >= Primitive_int.max(0, fetchState.knownHeight - cs.chainConfig.blockLag | 0);
+  if (atEndBlock) {
+    return true;
   } else {
-    return false;
+    return atHead;
   }
 }
 
@@ -641,9 +650,22 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
 }
 
 function enterReorgThreshold(cs) {
+  cs.isInReorgThreshold = true;
   cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined, undefined, cs.chainConfig.blockLag, undefined);
 }
 
+function isInReorgThreshold(cs) {
+  return cs.isInReorgThreshold;
+}
+
+function shouldSaveHistory(cs) {
+  if (cs.shouldRollbackOnReorg && cs.maxReorgDepth > 0) {
+    return cs.isInReorgThreshold;
+  } else {
+    return false;
+  }
+}
+
 function toChainMetadata(cs) {
   return {
     first_event_block: Stdlib_Null.fromOption(cs.fetchState.firstEventBlock),
@@ -675,6 +697,7 @@ function toMetrics(cs) {
     sourceBlockNumber: cs.fetchState.knownHeight,
     progressBlockNumber: cs.committedProgressBlockNumber,
     progressLatencyMs: cs.progressLatencyMs,
+    progressBlockTime: cs.committedProgressBlockTime,
     concurrency: SourceManager.inFlightCount(cs.sourceManager),
     partitionsCount: FetchState.partitionsCount(cs.fetchState),
     bufferSize: FetchState.bufferSize(cs.fetchState),
@@ -689,11 +712,13 @@ function toMetrics(cs) {
     blockRangeFetchedBlocks: cs.blockRangeFetchedBlocks,
     reorgCount: cs.reorgCount,
     reorgDetectedBlock: cs.reorgDetectedBlock,
-    rollbackTargetBlock: cs.rollbackTargetBlock
+    rollbackTargetBlock: cs.rollbackTargetBlock,
+    rateLimitTimeMs: SourceManager.getRateLimitTimeMs(cs.sourceManager),
+    rateLimitResetInMs: SourceManager.getRateLimitResetInMs(cs.sourceManager)
   };
 }
 
-function toChainBeforeBatch(cs) {
+function toChainBeforeBatch(cs, isRealtime) {
   let match = cs.blockStore.getHashes(Primitive_int.max(cs.fetchState.knownHeight - cs.maxReorgDepth | 0, 0), cs.fetchState.knownHeight + 1 | 0);
   let hashes = match.hashes;
   let blockNumbers = match.blockNumbers;
@@ -707,6 +732,7 @@ function toChainBeforeBatch(cs) {
       blockNumbers: blockNumbers,
       hashByBlockNumber: hashByBlockNumber
     },
+    blockTimeAt: blockNumber => Primitive_option.fromNull(cs.blockStore.getTimestamp(blockNumber, isRealtime)),
     shouldRollbackOnReorg: cs.shouldRollbackOnReorg,
     progressBlockNumber: cs.committedProgressBlockNumber,
     sourceBlockNumber: cs.fetchState.knownHeight,
@@ -786,6 +812,7 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
     }
   }
   cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber;
+  cs.committedProgressBlockTime = chainAfterBatch.progressBlockTime;
   cs.processingBlockNumber = Primitive_int.max(cs.processingBlockNumber, chainAfterBatch.progressBlockNumber);
   cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed;
   cs.transactionStore.prune(chainAfterBatch.progressBlockNumber);
@@ -805,6 +832,14 @@ function markReady(cs, readyAt) {
   }
 }
 
+function rollbackCommittedProgress(cs, blockNumber) {
+  if (blockNumber !== cs.committedProgressBlockNumber) {
+    cs.committedProgressBlockNumber = blockNumber;
+    cs.committedProgressBlockTime = Primitive_option.fromNull(cs.blockStore.getTimestamp(blockNumber, false));
+    return;
+  }
+}
+
 function rollback(cs, rolledBackTo) {
   let rollbackTo = targetBlockNumber => {
     let match = FetchState.rollback(cs.fetchState, cs.addressStore, targetBlockNumber);
@@ -829,7 +864,7 @@ function rollback(cs, rolledBackTo) {
     let rolledBackAddresses = rollbackTo(newProgressBlockNumber);
     cs.transactionStore.rollback(newProgressBlockNumber);
     cs.blockStore.rollback(newProgressBlockNumber);
-    cs.committedProgressBlockNumber = newProgressBlockNumber;
+    rollbackCommittedProgress(cs, newProgressBlockNumber);
     cs.processingBlockNumber = newProgressBlockNumber;
     cs.numEventsProcessed = newTotalEventsProcessed;
     return rolledBackAddresses;
@@ -838,7 +873,7 @@ function rollback(cs, rolledBackTo) {
   let rolledBackAddresses$1 = rollbackTo(forkBlock);
   cs.transactionStore.rollback(forkBlock);
   cs.blockStore.rollback(forkBlock);
-  cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, forkBlock);
+  rollbackCommittedProgress(cs, Primitive_int.min(cs.committedProgressBlockNumber, forkBlock));
   cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, forkBlock);
   return rolledBackAddresses$1;
 }
@@ -860,6 +895,7 @@ export {
   safeCheckpointTracking,
   isProgressAtHead,
   committedProgressBlockNumber,
+  committedProgressBlockTime,
   numEventsProcessed,
   pendingBudget,
   startFetchingQueries,
@@ -902,6 +938,8 @@ export {
   updateKnownHeight,
   setEndBlockToFirstEvent,
   enterReorgThreshold,
+  isInReorgThreshold,
+  shouldSaveHistory,
   advanceAfterBatch,
   applyBatchProgress,
   markReady,

package/src/ChainState.resi

@@ -23,9 +23,11 @@ let make: (
   ~addressStore: AddressStore.t,
   ~sourceManager: SourceManager.t,
   ~committedProgressBlockNumber: int,
+  ~committedProgressBlockTime: option<int>=?,
   ~safeCheckpointTracking: option<SafeCheckpointTracking.t>=?,
   ~shouldRollbackOnReorg: bool,
   ~maxReorgDepth: int,
+  ~isInReorgThreshold: bool=?,
   ~numEventsProcessed: float=?,
   ~timestampCaughtUpToHeadOrEndblock: option<Date.t>=?,
   ~isProgressAtHead: bool=?,
@@ -67,6 +69,7 @@ let getLatestValidScannedBlock: (
 let safeCheckpointTracking: t => option<SafeCheckpointTracking.t>
 let isProgressAtHead: t => bool
 let committedProgressBlockNumber: t => int
+let committedProgressBlockTime: t => option<int>
 let numEventsProcessed: t => float
 let pendingBudget: t => float
 let startFetchingQueries: (t, ~queries: array<FetchState.query>) => unit
@@ -116,7 +119,7 @@ let dispatch: (
 // Views.
 let toMetrics: t => Metrics.chainMetrics
 let toChainMetadata: t => InternalTable.Chains.metaFields
-let toChainBeforeBatch: t => Batch.chainBeforeBatch
+let toChainBeforeBatch: (t, ~isRealtime: bool) => Batch.chainBeforeBatch
 let isReadyToEnterReorgThresholdAfterBatch: (t, ~batch: Batch.t) => bool
 
 // Derived (pure).
@@ -154,6 +157,8 @@ let resetPendingQueries: t => unit
 let updateKnownHeight: (t, ~knownHeight: int) => unit
 let setEndBlockToFirstEvent: (t, ~blockNumber: int) => unit
 let enterReorgThreshold: t => unit
+let isInReorgThreshold: t => bool
+let shouldSaveHistory: t => bool
 let advanceAfterBatch: (t, ~batch: Batch.t, ~enteringReorgThreshold: bool) => unit
 let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
 let markReady: (t, ~readyAt: Date.t) => unit