@lodestar/beacon-node 1.50.0-dev.055a216e01 → 1.50.0-dev.072bf98853

package/src/chain/shufflingCache.ts

@@ -7,14 +7,12 @@ import {
   getIndexedAttestation,
 } from "@lodestar/state-transition";
 import {Attestation, CommitteeIndex, Epoch, IndexedAttestation, RootHex, Slot} from "@lodestar/types";
-import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils";
+import {LodestarError, Logger, MapDef} from "@lodestar/utils";
 import {Metrics} from "../metrics/metrics.js";
 
 /**
- * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case:
- * - when loading state bytes from disk, we need to compute shuffling for all epochs (~1s as of Sep 2023)
- * - don't have shuffling to verify attestations, need to do 1 epoch transition to add shuffling to this cache. This never happens
- * with default chain option of maxSkipSlots = 32
+ * Lookahead duties, gossip validation, and block production need epochs n-1, n, and n+1 for current epoch n.
+ * Keep the 4 highest epochs so that it works for epoch rotation.
  **/
 const MAX_EPOCHS = 4;
 
@@ -37,24 +35,29 @@ type ShufflingCacheItem = {
 type PromiseCacheItem = {
   type: CacheItemType.promise;
   timeInsertedMs: number;
-  promise: Promise<EpochShuffling>;
-  resolveFn: (shuffling: EpochShuffling) => void;
+  /** Resolves to null if the promise is cancelled */
+  promise: Promise<EpochShuffling | null>;
+  resolveFn: (shuffling: EpochShuffling | null) => void;
 };
 
 type CacheItem = ShufflingCacheItem | PromiseCacheItem;
 
+export enum ShufflingPromiseCancelReason {
+  regenError = "regen_error",
+  decisionRootMismatch = "decision_root_mismatch",
+  pruned = "pruned",
+}
+
 export type ShufflingCacheOpts = {
   maxShufflingCacheEpochs?: number;
 };
 
 /**
- * A shuffling cache to help:
- * - get committee quickly for attestation verification
- * - if a shuffling is not available (which does not happen with default chain option of maxSkipSlots = 32), track a promise to make sure we don't compute the same shuffling twice
- * - skip computing shuffling when loading state bytes from disk
+ * Cache recent epoch shufflings for lookahead duties, gossip validation, and block production.
+ * Pending promises deduplicate computation of missing shufflings.
  */
 export class ShufflingCache {
-  /** LRU cache implemented as a map, pruned every time we add an item */
+  /** Pruned to the `maxEpochs` highest epochs every time we add a shuffling */
   private readonly itemsByDecisionRootByEpoch: MapDef<Epoch, Map<RootHex, CacheItem>> = new MapDef(
     () => new Map<RootHex, CacheItem>()
   );
@@ -73,6 +76,9 @@ export class ShufflingCache {
           Array.from(this.itemsByDecisionRootByEpoch.values()).reduce((total, innerMap) => total + innerMap.size, 0)
         )
       );
+      metrics.shufflingCache.epochs.addCollect(() =>
+        metrics.shufflingCache.epochs.set(this.itemsByDecisionRootByEpoch.size)
+      );
     }
 
     this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS;
@@ -97,8 +103,8 @@ export class ShufflingCache {
         `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${epoch}, decisionRootHex: ${decisionRoot}`
       );
     }
-    let resolveFn: ((shuffling: EpochShuffling) => void) | null = null;
-    const promise = new Promise<EpochShuffling>((resolve) => {
+    let resolveFn: ((shuffling: EpochShuffling | null) => void) | null = null;
+    const promise = new Promise<EpochShuffling | null>((resolve) => {
       resolveFn = resolve;
     });
     if (resolveFn === null) {
@@ -121,7 +127,7 @@ export class ShufflingCache {
    * Return null if we don't have a shuffling for this epoch and dependentRootHex.
    */
   async get(epoch: Epoch, decisionRoot: RootHex): Promise<EpochShuffling | null> {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    const cacheItem = this.itemsByDecisionRootByEpoch.get(epoch)?.get(decisionRoot);
     if (cacheItem === undefined) {
       this.metrics?.shufflingCache.miss.inc();
       return null;
@@ -136,12 +142,10 @@ export class ShufflingCache {
   }
 
   /**
-   * Get a shuffling synchronously, return null if not present.
-   * The only time we have a promise cache item is when we regen shuffling for attestation, which never happens
-   * with default chain option.
+   * Get a shuffling synchronously, return null if not present or if it's still being computed.
    */
   getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    const cacheItem = this.itemsByDecisionRootByEpoch.get(epoch)?.get(decisionRoot);
     if (cacheItem === undefined) {
       this.metrics?.shufflingCache.miss.inc();
       return null;
@@ -161,7 +165,7 @@ export class ShufflingCache {
    * `NativeBeaconStateView`.
    */
   has(epoch: Epoch, decisionRoot: RootHex): boolean {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    const cacheItem = this.itemsByDecisionRootByEpoch.get(epoch)?.get(decisionRoot);
     return cacheItem !== undefined && isShufflingCacheItem(cacheItem);
   }
 
@@ -186,6 +190,25 @@ export class ShufflingCache {
     }
   }
 
+  /**
+   * Resolve a pending promise with null so that waiters don't hang, then remove it.
+   */
+  cancelPromise(epoch: Epoch, decisionRoot: RootHex, reason: ShufflingPromiseCancelReason): void {
+    const itemsByDecisionRoot = this.itemsByDecisionRootByEpoch.get(epoch);
+    const cacheItem = itemsByDecisionRoot?.get(decisionRoot);
+    if (itemsByDecisionRoot === undefined || cacheItem === undefined || !isPromiseCacheItem(cacheItem)) {
+      return;
+    }
+
+    cacheItem.resolveFn(null);
+    itemsByDecisionRoot.delete(decisionRoot);
+    if (itemsByDecisionRoot.size === 0) {
+      this.itemsByDecisionRootByEpoch.delete(epoch);
+    }
+    this.metrics?.shufflingCache.cancelledPromises.inc({reason});
+    this.logger?.debug("Cancelled shuffling promise", {epoch, decisionRoot, reason});
+  }
+
   getIndexedAttestation(
     epoch: number,
     decisionRoot: string,
@@ -248,8 +271,42 @@ export class ShufflingCache {
     }
     // set the shuffling
     shufflingAtEpoch.set(decisionRoot, {type: CacheItemType.shuffling, shuffling});
-    // prune the cache
-    pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs);
+    this.prune();
+  }
+
+  /**
+   * Keep the `maxEpochs` highest epochs, regardless of insertion order.
+   */
+  private prune(): void {
+    const toDelete = this.itemsByDecisionRootByEpoch.size - this.maxEpochs;
+    if (toDelete <= 0) {
+      return;
+    }
+
+    const prunedEpochs = Array.from(this.itemsByDecisionRootByEpoch.keys())
+      .sort((a, b) => a - b)
+      .slice(0, toDelete);
+    let prunedShufflings = 0;
+    for (const epoch of prunedEpochs) {
+      for (const cacheItem of this.itemsByDecisionRootByEpoch.get(epoch)?.values() ?? []) {
+        if (isPromiseCacheItem(cacheItem)) {
+          cacheItem.resolveFn(null);
+          this.metrics?.shufflingCache.cancelledPromises.inc({reason: ShufflingPromiseCancelReason.pruned});
+        } else {
+          prunedShufflings++;
+        }
+      }
+      this.itemsByDecisionRootByEpoch.delete(epoch);
+    }
+    this.metrics?.shufflingCache.prunedShufflings.inc(prunedShufflings);
+
+    this.logger?.verbose("Pruned shuffling cache", {
+      prunedEpochs: prunedEpochs.join(","),
+      prunedShufflings,
+      cachedEpochs: Array.from(this.itemsByDecisionRootByEpoch.keys())
+        .sort((a, b) => a - b)
+        .join(","),
+    });
   }
 }
 

package/src/metrics/metrics/lodestar.ts

@@ -20,6 +20,7 @@ import {InsertOutcome} from "../../chain/opPools/types.js";
 import {RegenCaller, RegenFnName} from "../../chain/regen/interface.js";
 import {ReprocessStatus} from "../../chain/reprocess.js";
 import {RejectReason} from "../../chain/seenCache/seenAttestationData.js";
+import type {ShufflingPromiseCancelReason} from "../../chain/shufflingCache.js";
 import {CacheItemType} from "../../chain/stateCache/types.js";
 import {OpSource} from "../../chain/validatorMonitor.js";
 import type {FlatFileStoreOperation} from "../../db/flatFileStore/metrics.js";
@@ -1618,6 +1619,19 @@ export function createLodestarMetrics(
         name: "lodestar_shuffling_cache_size",
         help: "Shuffling cache size",
       }),
+      epochs: register.gauge({
+        name: "lodestar_shuffling_cache_epochs",
+        help: "Number of epochs in shuffling cache",
+      }),
+      prunedShufflings: register.counter({
+        name: "lodestar_shuffling_cache_pruned_shufflings_total",
+        help: "Total number of shufflings pruned from shuffling cache",
+      }),
+      cancelledPromises: register.counter<{reason: ShufflingPromiseCancelReason}>({
+        name: "lodestar_shuffling_cache_cancelled_promises_total",
+        help: "Total number of shuffling promises resolved with null",
+        labelNames: ["reason"],
+      }),
       insertPromiseCount: register.gauge({
         name: "lodestar_shuffling_cache_insert_promise_count",
         help: "Total number of times insertPromise is called",

package/src/sync/range/batch.ts

@@ -14,7 +14,7 @@ import {IClock} from "../../util/clock.js";
 import {CustodyConfig} from "../../util/dataColumns.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {MAX_BATCH_DOWNLOAD_ATTEMPTS, MAX_BATCH_PROCESSING_ATTEMPTS} from "../constants.js";
-import {DownloadByRangeRequests, ParentPayloadCommitments} from "../utils/downloadByRange.js";
+import {DownloadByRangeRequests, ParentPayload} from "../utils/downloadByRange.js";
 import {getBatchSlotRange, hashBlocks} from "./utils/index.js";
 
 /**
@@ -175,7 +175,7 @@ export class Batch {
   private readonly clock: IClock;
   private readonly custodyConfig: CustodyConfig;
   private readonly isFirstBatchInChain: boolean;
-  private readonly latestBid: gloas.ExecutionPayloadBid | undefined;
+  private readonly parentPayload: ParentPayload | undefined;
 
   constructor(
     startEpoch: Epoch,
@@ -183,7 +183,7 @@ export class Batch {
     clock: IClock,
     custodyConfig: CustodyConfig,
     isFirstBatchInChain: boolean,
-    latestBid: gloas.ExecutionPayloadBid | undefined,
+    parentPayload: ParentPayload | undefined,
     targetSlot: Slot
   ) {
     this.config = config;
@@ -196,7 +196,7 @@ export class Batch {
     this.startSlot = startSlot;
     this.count = Math.min(count, targetSlot - startSlot + 1);
     this.isFirstBatchInChain = isFirstBatchInChain;
-    this.latestBid = latestBid;
+    this.parentPayload = parentPayload;
     this.requests = this.getRequests([]);
   }
 
@@ -208,24 +208,19 @@ export class Batch {
     }
 
     // we only know if we should download parent envelope if firstBlock is downloaded
-    if (firstBlock === undefined) return false;
-    if (this.latestBid === undefined) return false;
-    const firstBlockBidParentHash = (firstBlock.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message
-      .parentBlockHash;
-    return byteArrayEquals(firstBlockBidParentHash, this.latestBid.blockHash);
+    if (firstBlock === undefined || this.parentPayload === undefined) return false;
+    const firstBlockBid = (firstBlock.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message;
+    return (
+      byteArrayEquals(firstBlock.message.parentRoot, this.parentPayload.blockRoot) &&
+      byteArrayEquals(firstBlockBid.parentBlockHash, this.parentPayload.bid.blockHash)
+    );
   }
 
-  getParentPayloadCommitments(parentBlockRoot: Uint8Array): ParentPayloadCommitments {
-    if (this.latestBid === undefined) {
-      throw new Error(
-        `Coding error: getParentPayloadCommitments called without latestBid for parentBlockRoot=${toRootHex(parentBlockRoot)}`
-      );
+  getParentPayload(): ParentPayload {
+    if (this.parentPayload === undefined) {
+      throw new Error("Coding error: getParentPayload called without parentPayload");
     }
-    return {
-      blockRoot: parentBlockRoot,
-      blockRootHex: toRootHex(parentBlockRoot),
-      kzgCommitments: this.latestBid.blobKzgCommitments,
-    };
+    return this.parentPayload;
   }
 
   /**

package/src/sync/range/chain.ts

@@ -1,5 +1,5 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {Epoch, Root, Slot, gloas} from "@lodestar/types";
+import {Epoch, Root, Slot} from "@lodestar/types";
 import {ErrorAborted, LodestarError, Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {BlockInputErrorCode} from "../../chain/blocks/blockInput/errors.js";
@@ -16,7 +16,7 @@ import {ItTrigger} from "../../util/itTrigger.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {WarnResult, wrapError} from "../../util/wrapError.js";
 import {BATCH_BUFFER_SIZE, EPOCHS_PER_BATCH, MAX_LOOK_AHEAD_EPOCHS} from "../constants.js";
-import {DownloadByRangeError, DownloadByRangeErrorCode} from "../utils/downloadByRange.js";
+import {DownloadByRangeError, DownloadByRangeErrorCode, ParentPayload} from "../utils/downloadByRange.js";
 import {getRateLimitedUntilMs} from "../utils/rateLimit.js";
 import {RangeSyncType} from "../utils/remoteSyncType.js";
 import {Batch, BatchError, BatchErrorCode, BatchMetadata, BatchStatus} from "./batch.js";
@@ -160,7 +160,7 @@ export class SyncChain {
   private readonly clock: IClock;
   private readonly metrics: Metrics | null;
   private readonly custodyConfig: CustodyConfig;
-  private readonly latestBid: gloas.ExecutionPayloadBid | undefined;
+  private readonly parentPayload: ParentPayload | undefined;
 
   constructor(
     initialBatchEpoch: Epoch,
@@ -168,7 +168,7 @@ export class SyncChain {
     syncType: RangeSyncType,
     fns: SyncChainFns,
     modules: SyncChainModules,
-    latestBid: gloas.ExecutionPayloadBid | undefined
+    parentPayload: ParentPayload | undefined
   ) {
     const {config, clock, custodyConfig, logger, metrics} = modules;
     this.firstBatchEpoch = initialBatchEpoch;
@@ -184,7 +184,7 @@ export class SyncChain {
     this.clock = clock;
     this.metrics = metrics;
     this.custodyConfig = custodyConfig;
-    this.latestBid = latestBid;
+    this.parentPayload = parentPayload;
     this.logger = logger;
     this.logId = `${syncType}-${nextChainId++}`;
 
@@ -536,8 +536,8 @@ export class SyncChain {
       this.clock,
       this.custodyConfig,
       this.isFirstBatch,
-      // `latestBid` is only meaningful for the first batch's parent-payload check
-      this.isFirstBatch ? this.latestBid : undefined,
+      // `parentPayload` is only meaningful for the first batch
+      this.isFirstBatch ? this.parentPayload : undefined,
       this.target.slot
     );
     this.isFirstBatch = false;
@@ -586,6 +586,8 @@ export class SyncChain {
           case DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS:
           case DownloadByRangeErrorCode.PARENT_ROOT_MISMATCH:
           case DownloadByRangeErrorCode.INVALID_ENVELOPE_BEACON_BLOCK_ROOT:
+          case DownloadByRangeErrorCode.INVALID_ENVELOPE_BLOCK_HASH:
+          case DownloadByRangeErrorCode.INVALID_ENVELOPE_SLOT:
           case DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT:
           case BlobSidecarErrorCode.INCLUSION_PROOF_INVALID:
           case BlobSidecarErrorCode.INVALID_KZG_PROOF_BATCH:

package/src/sync/range/range.ts

@@ -2,8 +2,8 @@ import {EventEmitter} from "node:events";
 import {StrictEventEmitter} from "strict-event-emitter-types";
 import {BeaconConfig} from "@lodestar/config";
 import {IBeaconStateViewGloas, computeStartSlotAtEpoch, isStatePostGloas} from "@lodestar/state-transition";
-import {Epoch, Status, fulu} from "@lodestar/types";
-import {Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
+import {Epoch, Status, fulu, ssz} from "@lodestar/types";
+import {Logger, byteArrayEquals, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {PayloadError, PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {AttestationImportOpt, ImportBlockOpts} from "../../chain/blocks/index.js";
@@ -16,6 +16,7 @@ import {PeerIdStr} from "../../util/peerId.js";
 import {
   DownloadByRangeError,
   DownloadByRangeErrorCode,
+  ParentPayload,
   cacheByRangeResponses,
   downloadByRange,
 } from "../utils/downloadByRange.js";
@@ -278,15 +279,14 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
   private downloadByRange: SyncChainFns["downloadByRange"] = async (peer, batch) => {
     const batchBlocks = batch.getBlocks();
     const requests = batch.getRequestsForPeer(peer);
-    const parentRoot = requests.parentPayloadRequest?.envelopeBlockRoot ?? requests.parentPayloadRequest?.blockRoot;
-    const parentPayloadCommitments = parentRoot ? batch.getParentPayloadCommitments(parentRoot) : undefined;
+    const parentPayload = requests.parentPayloadRequest ? batch.getParentPayload() : undefined;
     const {result, warnings} = await downloadByRange({
       config: this.config,
       network: this.network,
       logger: this.logger,
       peerIdStr: peer.peerId,
       batchBlocks,
-      parentPayloadCommitments,
+      parentPayload,
       peerDasMetrics: this.chain.metrics?.peerDas,
       ...requests,
     });
@@ -299,6 +299,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       batchBlocks,
       downloadedPayloadEnvelopes,
       existingPayloadEnvelopes: batch.getPayloadEnvelopes(),
+      parentPayload,
       custodyConfig: this.chain.custodyConfig,
       seenTimestampSec: Date.now() / 1000,
     });
@@ -376,13 +377,26 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
   private addPeerOrCreateChain(startEpoch: Epoch, target: ChainTarget, peer: PeerIdStr, syncType: RangeSyncType): void {
     let syncChain = this.chains.get(syncType);
     if (!syncChain) {
-      // The first batch of a new sync chain may need to detect whether the parent block was an
-      // gloas "empty" block (no envelope produced). It does so by comparing the first
-      // downloaded block's `bid.parentBlockHash` against the head state's `latestExecutionPayloadBid.blockHash`.
+      // The first block of the first batch may build on the head's FULL variant before its payload is imported,
+      // the batch then downloads the head's payload as dangling parent and validates it against the bid
       const headState = this.chain.getHeadState();
-      const latestBid = isStatePostGloas(headState)
-        ? (headState as IBeaconStateViewGloas).latestExecutionPayloadBid
-        : undefined;
+      let parentPayload: ParentPayload | undefined;
+      if (isStatePostGloas(headState) && headState.latestBlockHeader.slot > 0) {
+        const header = ssz.phase0.BeaconBlockHeader.clone(headState.latestBlockHeader);
+        if (byteArrayEquals(header.stateRoot, ssz.Root.defaultValue())) {
+          header.stateRoot = headState.hashTreeRoot();
+        }
+        const headRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(header);
+        const headSlot = headState.latestBlockHeader.slot;
+        parentPayload = {
+          blockRoot: headRoot,
+          blockRootHex: toRootHex(headRoot),
+          slot: headSlot,
+          proposerIndex: headState.latestBlockHeader.proposerIndex,
+          forkName: this.config.getForkName(headSlot),
+          bid: (headState as IBeaconStateViewGloas).latestExecutionPayloadBid,
+        };
+      }
 
       syncChain = new SyncChain(
         startEpoch,
@@ -403,7 +417,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
           custodyConfig: this.chain.custodyConfig,
           metrics: this.metrics,
         },
-        latestBid
+        parentPayload
       );
       this.chains.set(syncType, syncChain);
 

package/src/sync/unknownBlock.ts

@@ -73,7 +73,11 @@ type AdvancePendingBlockResult =
   | "removed";
 
 class UnknownBlockRateLimitedError extends Error {
-  constructor(message: string) {
+  constructor(
+    message: string,
+    /** Earliest backoff expiry seen when the download was deferred, it is retried once that has passed */
+    readonly retryAt: number
+  ) {
     super(message);
     this.name = "UnknownBlockRateLimitedError";
   }
@@ -849,17 +853,20 @@ export class BlockInputSync {
     }
   };
 
-  private scheduleRateLimitBackoffRetry(): void {
+  /**
+   * Returns false if there is no backoff left to wait for, in which case no retry is scheduled.
+   */
+  private scheduleRateLimitBackoffRetry(): boolean {
     this.clearRateLimitBackoffTimer();
 
     if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
-      return;
+      return false;
     }
 
     const now = Date.now();
     const retryAt = this.peerBalancer.getNextRateLimitRetryAt();
     if (retryAt === null) {
-      return;
+      return false;
     }
 
     this.rateLimitBackoffTimeout = setTimeout(
@@ -870,6 +877,7 @@ export class BlockInputSync {
       },
       Math.max(0, retryAt - now)
     );
+    return true;
   }
 
   private clearRateLimitBackoffTimer(): void {
@@ -1081,7 +1089,11 @@ export class BlockInputSync {
           pendingBlock.status = PendingBlockInputStatus.pending;
         }
         this.logger.debug("Deferring unknown block download due to peer rate limit", logCtx, res.err);
-        this.scheduleRateLimitBackoffRetry();
+        if (!this.scheduleRateLimitBackoffRetry() || res.err.retryAt <= Date.now()) {
+          // The backoff this download waited for expired in the meantime, a timer is only armed if another
+          // peer is still in backoff and would retry it later than needed
+          this.triggerUnknownBlockSearch();
+        }
         return;
       }
 
@@ -1364,6 +1376,14 @@ export class BlockInputSync {
     if (!isPendingPayloadEnvelope(payload)) {
       payload.status = PendingPayloadInputStatus.pending;
     }
+    if (
+      res.err instanceof UnknownBlockRateLimitedError &&
+      (!this.scheduleRateLimitBackoffRetry() || res.err.retryAt <= Date.now())
+    ) {
+      // The backoff this download waited for expired in the meantime, a timer is only armed if another
+      // peer is still in backoff and would retry it later than needed
+      this.triggerUnknownBlockSearch();
+    }
   }
 
   private async processPayload(pendingPayload: PendingPayloadInput): Promise<void> {
@@ -1488,16 +1508,21 @@ export class BlockInputSync {
       const pendingColumns = payloadInput?.hasAllData()
         ? new Set<number>()
         : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
+      // Same `now` for both calls, a backoff expiring in between would be reported as no peer available
+      const now = Date.now();
       // prefer peers that gossiped this payload root to us (#9923)
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(
         pendingColumns,
         excludedPeers,
-        cacheItem.peerIdStrings
+        cacheItem.peerIdStrings,
+        now
       );
       if (peerMeta === null) {
-        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+        const retryAt = this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers, now);
+        if (retryAt !== null) {
           throw new UnknownBlockRateLimitedError(
-            `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+            `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`,
+            retryAt
           );
         }
 
@@ -1636,9 +1661,11 @@ export class BlockInputSync {
     );
     this.metrics?.blockInputSync.payloadFetchPeers.set({result: FetchResult.FailureMaxAttempts}, i - 1);
 
-    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
+    const retryAt = deferredByRateLimit ? this.peerBalancer.getNextRateLimitRetryAt() : null;
+    if (retryAt !== null) {
       throw new UnknownBlockRateLimitedError(
-        `Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`
+        `Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`,
+        retryAt
       );
     }
     throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
@@ -1730,16 +1757,21 @@ export class BlockInputSync {
         isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
           ? new Set(cacheItem.blockInput.getMissingSampledColumnMeta().missing)
           : defaultPendingColumns;
+      // Same `now` for both calls, a backoff expiring in between would be reported as no peer available
+      const now = Date.now();
       // prefer peers that gossiped this block root to us (#9923)
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(
         pendingColumns,
         excludedPeers,
-        cacheItem.peerIdStrings
+        cacheItem.peerIdStrings,
+        now
       );
       if (peerMeta === null) {
-        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+        const retryAt = this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers, now);
+        if (retryAt !== null) {
           throw new UnknownBlockRateLimitedError(
-            `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+            `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`,
+            retryAt
           );
         }
 
@@ -1845,8 +1877,9 @@ export class BlockInputSync {
 
     const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
 
-    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
-      throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`);
+    const retryAt = deferredByRateLimit ? this.peerBalancer.getNextRateLimitRetryAt() : null;
+    if (retryAt !== null) {
+      throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`, retryAt);
     }
 
     if (!isPendingBlockInput(cacheItem)) {
@@ -2037,8 +2070,11 @@ export class UnknownBlockPeerBalancer {
     this.rateLimitedUntilByPeer.set(peerId, rateLimitedUntilMs);
   }
 
-  getNextRateLimitRetryAt(pendingColumns?: Set<number>, excludedPeers?: Set<PeerIdStr>): number | null {
-    const now = Date.now();
+  getNextRateLimitRetryAt(
+    pendingColumns?: Set<number>,
+    excludedPeers?: Set<PeerIdStr>,
+    now = Date.now()
+  ): number | null {
     let retryAt: number | null = null;
 
     for (const [peerId, rateLimitedUntil] of this.rateLimitedUntilByPeer.entries()) {
@@ -2077,9 +2113,10 @@ export class UnknownBlockPeerBalancer {
   bestPeerForPendingColumns(
     pendingColumns: Set<number>,
     excludedPeers: Set<PeerIdStr>,
-    preferredPeers?: Set<PeerIdStr>
+    preferredPeers?: Set<PeerIdStr>,
+    now = Date.now()
   ): PeerSyncMeta | null {
-    const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers);
+    const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers, now);
     if (eligiblePeers.length === 0) {
       return null;
     }
@@ -2120,8 +2157,7 @@ export class UnknownBlockPeerBalancer {
     return totalActiveRequests;
   }
 
-  private filterPeers(pendingDataColumns: Set<number>, excludedPeers: Set<PeerIdStr>): PeerIdStr[] {
-    const now = Date.now();
+  private filterPeers(pendingDataColumns: Set<number>, excludedPeers: Set<PeerIdStr>, now: number): PeerIdStr[] {
     let maxColumnCount = 0;
     const considerPeers: {peerId: PeerIdStr; columnCount: number}[] = [];
     for (const [peerId, syncMeta] of this.peersMeta.entries()) {