@lodestar/beacon-node 1.50.0-dev.033e6b7138 → 1.50.0-dev.03c05fa09a

package/lib/sync/unknownBlock.js

@@ -11,7 +11,7 @@ import { BlockError, BlockErrorCode } from "../chain/errors/index.js";
 import { ChainEvent } from "../chain/index.js";
 import { validateGloasBlockDataColumnSidecars } from "../chain/validation/dataColumnSidecar.js";
 import { validateGossipExecutionPayloadEnvelope } from "../chain/validation/executionPayloadEnvelope.js";
-import { NetworkEvent, prettyPrintPeerIdStr } from "../network/index.js";
+import { NetworkEvent, PeerAction, prettyPrintPeerIdStr } from "../network/index.js";
 import { MAX_PEERS_PER_ROOT } from "../network/processor/constants.js";
 import { ClockEvent } from "../util/clock.js";
 import { shuffle } from "../util/shuffle.js";
@@ -34,8 +34,12 @@ const PRUNE_UNRESOLVED_SLOT_EPOCHS = 2;
  */
 export const PAYLOAD_POLL_INTERVAL_MS = 1_000;
 class UnknownBlockRateLimitedError extends Error {
-    constructor(message) {
+    retryAt;
+    constructor(message, 
+    /** Earliest backoff expiry seen when the download was deferred, it is retried once that has passed */
+    retryAt) {
         super(message);
+        this.retryAt = retryAt;
         this.name = "UnknownBlockRateLimitedError";
     }
 }
@@ -378,6 +382,10 @@ export class BlockInputSync {
         });
     };
     addByRootHex = (rootHex, peerIdStr) => {
+        if (this.knownBadBlocks.has(rootHex)) {
+            this.logger.debug("Ignoring known bad block root", { root: rootHex, peerIdStr: peerIdStr ?? "unknown peer" });
+            return false;
+        }
         let pendingBlock = this.pendingBlocks.get(rootHex);
         let added = false;
         if (!pendingBlock) {
@@ -409,6 +417,10 @@ export class BlockInputSync {
         return added;
     };
     addByBlockInput = (blockInput, peerIdStr) => {
+        if (this.knownBadBlocks.has(blockInput.blockRootHex)) {
+            this.logger.debug("Ignoring known bad block", { slot: blockInput.slot, root: blockInput.blockRootHex, peerIdStr });
+            return;
+        }
         let pendingBlock = this.pendingBlocks.get(blockInput.blockRootHex);
         // if entry is missing or was added via rootHex and now we have more complete information overwrite
         // the existing information with the more complete cache entry
@@ -441,6 +453,14 @@ export class BlockInputSync {
         }
     };
     addByPayloadRootHex = (rootHex, peerIdStr, slot) => {
+        if (this.knownBadBlocks.has(rootHex)) {
+            this.logger.debug("Ignoring payload for known bad block root", {
+                slot: slot ?? "unknown",
+                root: rootHex,
+                peerIdStr,
+            });
+            return false;
+        }
         let pendingPayload = this.pendingPayloads.get(rootHex);
         let added = false;
         if (!pendingPayload) {
@@ -499,6 +519,14 @@ export class BlockInputSync {
         return this.chain.seenPayloadEnvelopeInputCache.get(rootHex)?.slot;
     }
     addByPayloadInput = (payloadInput, peerIdStr, envelope) => {
+        if (this.knownBadBlocks.has(payloadInput.blockRootHex)) {
+            this.logger.debug("Ignoring payload input for known bad block root", {
+                slot: payloadInput.slot,
+                root: payloadInput.blockRootHex,
+                peerIdStr,
+            });
+            return;
+        }
         const pendingPayload = this.toPendingPayloadInput(payloadInput, this.pendingPayloads.get(payloadInput.blockRootHex), envelope);
         if (peerIdStr && pendingPayload.peerIdStrings.size < MAX_PEERS_PER_ROOT) {
             pendingPayload.peerIdStrings.add(peerIdStr);
@@ -552,6 +580,7 @@ export class BlockInputSync {
         if (this.chain.forkChoice.getBlockHexAndBlockHash(parentRootHex, parentBlockHashHex) !== null) {
             return { kind: "ready" };
         }
+        // The parent's payload is known but does not match `parentBlockHashHex`
         if (this.chain.forkChoice.hasPayloadHexUnsafe(parentRootHex)) {
             return { kind: "invalidParentPayload", parentRootHex, parentBlockHashHex };
         }
@@ -693,21 +722,25 @@ export class BlockInputSync {
             this.triggerUnknownBlockSearch();
         }
     };
+    /**
+     * Returns false if there is no backoff left to wait for, in which case no retry is scheduled.
+     */
     scheduleRateLimitBackoffRetry() {
         this.clearRateLimitBackoffTimer();
         if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
-            return;
+            return false;
         }
         const now = Date.now();
         const retryAt = this.peerBalancer.getNextRateLimitRetryAt();
         if (retryAt === null) {
-            return;
+            return false;
         }
         this.rateLimitBackoffTimeout = setTimeout(() => {
             this.rateLimitBackoffTimeout = undefined;
             this.triggerUnknownBlockSearch();
             this.scheduleRateLimitBackoffRetry();
         }, Math.max(0, retryAt - now));
+        return true;
     }
     clearRateLimitBackoffTimer() {
         if (this.rateLimitBackoffTimeout !== undefined) {
@@ -745,13 +778,14 @@ export class BlockInputSync {
      * Poll unknown payloads of the current slot.
      */
     async pollPayloadsAtSlot(slot, signal) {
+        const fork = this.config.getForkName(slot);
         // fixed tick count so the loop cannot outlive its slot: for mainnet, it's 6 ticks
-        const tickCount = Math.floor((this.config.SLOT_DURATION_MS - this.config.getPayloadDueMs()) / PAYLOAD_POLL_INTERVAL_MS);
+        const tickCount = Math.floor((this.config.getSlotDurationMs(fork) - this.config.getPayloadDueMs(fork)) / PAYLOAD_POLL_INTERVAL_MS);
         // a root still missing on several ticks is searched on each of them, recorded once
         const searchedRoots = new Set();
         let polls = 0;
         try {
-            await sleep(Math.max(0, this.config.getPayloadDueMs() - this.chain.clock.msFromSlot(slot)), signal);
+            await sleep(Math.max(0, this.config.getPayloadDueMs(fork) - this.chain.clock.msFromSlot(slot)), signal);
             for (polls = 0; polls < tickCount; polls++) {
                 let importedCount = 0;
                 for (const [rootHex, { slot: deferredSlot, source, peer }] of this.deferredPayloadRoots) {
@@ -882,7 +916,8 @@ export class BlockInputSync {
             }
             else if (blockSlot <= finalizedSlot) {
                 // the common ancestor of the downloading chain and canonical chain should be at least the finalized slot and
-                // we should found it through forkchoice. If not, we should penalize all peers sending us this block chain
+                // we should found it through forkchoice. If not, drop the chain without penalizing peers because pending entries are
+                // block retained on failed download, so finalization can pass the block while it is still pending
                 // 0 - 1 - ... - n - finalizedSlot
                 //                \
                 //                parent 1 - parent 2 - ... - unknownParent block
@@ -890,7 +925,7 @@ export class BlockInputSync {
                     ...logCtx2,
                     finalizedSlot,
                 });
-                this.removeAndDownScoreAllDescendants(block, DroppedItemReason.belowFinalized);
+                this.removeAllDescendants(block, DroppedItemReason.belowFinalized);
             }
             else {
                 this.onUnknownBlockRoot({ rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot });
@@ -903,7 +938,11 @@ export class BlockInputSync {
                     pendingBlock.status = PendingBlockInputStatus.pending;
                 }
                 this.logger.debug("Deferring unknown block download due to peer rate limit", logCtx, res.err);
-                this.scheduleRateLimitBackoffRetry();
+                if (!this.scheduleRateLimitBackoffRetry() || res.err.retryAt <= Date.now()) {
+                    // The backoff this download waited for expired in the meantime, a timer is only armed if another
+                    // peer is still in backoff and would retry it later than needed
+                    this.triggerUnknownBlockSearch();
+                }
                 return;
             }
             this.metrics?.blockInputSync.downloadedBlocksError.inc();
@@ -1009,14 +1048,17 @@ export class BlockInputSync {
                         this.removeAllDescendants(pendingBlock, DroppedItemReason.elError, DroppedItemReason.elError);
                         break;
                     case BlockErrorCode.EXECUTION_ENGINE_INVALID:
-                        // the peer served a bad block
+                        // Ban but do not downscore: optimistic-sync peers may forward and serve blocks whose payload is later
+                        // rejected by the execution engine. Per the spec, an INVALID execution payload SHOULD NOT cause a peer
+                        // to be down-scored or disconnected, only consensus-layer invalidity MAY.
+                        // https://github.com/ethereum/consensus-specs/blob/master/specs/bellatrix/p2p-interface.md#the-reqresp-domain
                         this.logger.debug("Execution engine rejected block from unknown parent sync", errorData, res.err);
-                        this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.elInvalid);
+                        this.removeAndBanAllDescendants(pendingBlock, DroppedItemReason.elInvalid);
                         break;
                     default:
                         // Block is not correct with respect to our chain. Log error loudly
                         this.logger.debug("Error processing block from unknown parent sync", errorData, res.err);
-                        this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.invalidBlock);
+                        this.removeAllDescendants(pendingBlock, DroppedItemReason.invalidBlock);
                 }
             }
             // Probably a queue error or something unwanted happened, mark as pending to try again latter
@@ -1138,6 +1180,12 @@ export class BlockInputSync {
         if (!isPendingPayloadEnvelope(payload)) {
             payload.status = PendingPayloadInputStatus.pending;
         }
+        if (res.err instanceof UnknownBlockRateLimitedError &&
+            (!this.scheduleRateLimitBackoffRetry() || res.err.retryAt <= Date.now())) {
+            // The backoff this download waited for expired in the meantime, a timer is only armed if another
+            // peer is still in backoff and would retry it later than needed
+            this.triggerUnknownBlockSearch();
+        }
     }
     async processPayload(pendingPayload) {
         const rootHex = pendingPayload.payloadInput.blockRootHex;
@@ -1191,18 +1239,21 @@ export class BlockInputSync {
                     this.logger.debug("Execution engine error while processing payload from unknown sync", logCtx, res.err);
                     pendingPayload.status = PendingPayloadInputStatus.downloaded;
                     break;
+                // The invalid envelope is evicted from the seen cache in the cases below, otherwise the next
+                // attempt would reuse it instead of fetching another envelope from peers
                 case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
                     this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
                     this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.elInvalid);
                     break;
                 case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
                     this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
                     this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidEnvelope);
                     break;
                 case PayloadErrorCode.INVALID_SIGNATURE:
-                    // TODO GLOAS: Decide how invalid payload inputs should eventually leave memory without
-                    // reintroducing envelope replacement / recreation flows.
                     this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
                     this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidSignature);
                     break;
                 default:
@@ -1240,11 +1291,14 @@ export class BlockInputSync {
             const pendingColumns = payloadInput?.hasAllData()
                 ? new Set()
                 : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
+            // Same `now` for both calls, a backoff expiring in between would be reported as no peer available
+            const now = Date.now();
             // prefer peers that gossiped this payload root to us (#9923)
-            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings);
+            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings, now);
             if (peerMeta === null) {
-                if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
-                    throw new UnknownBlockRateLimitedError(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`);
+                const retryAt = this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers, now);
+                if (retryAt !== null) {
+                    throw new UnknownBlockRateLimitedError(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`, retryAt);
                 }
                 const reason = pendingColumns.size > 0
                     ? `cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
@@ -1254,10 +1308,10 @@ export class BlockInputSync {
                 throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: ${reason}`);
             }
             const { peerId, client: peerClient } = peerMeta;
-            cacheItem.peerIdStrings.add(peerId);
             try {
                 if (!envelope) {
                     envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
+                    cacheItem.peerIdStrings.add(peerId);
                     const slotWasUnknown = typeof slot !== "number";
                     slot = envelope.message.payload.slotNumber;
                     if (slotWasUnknown) {
@@ -1292,6 +1346,8 @@ export class BlockInputSync {
                     const missing = pendingPayload.payloadInput.getMissingSampledColumnMeta().missing;
                     if (missing.length > 0) {
                         const columnSidecars = await this.fetchPayloadColumns(peerMeta, pendingPayload.payloadInput, missing);
+                        // toPendingPayloadInput copied cacheItem's set, so add to the object that is carried forward
+                        pendingPayload.peerIdStrings.add(peerId);
                         const seenTimestampSec = Date.now() / 1000;
                         for (const columnSidecar of columnSidecars) {
                             if (pendingPayload.payloadInput.hasColumn(columnSidecar.index)) {
@@ -1356,8 +1412,9 @@ export class BlockInputSync {
         }
         this.metrics?.blockInputSync.payloadFetchTime.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
         this.metrics?.blockInputSync.payloadFetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
-        if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
-            throw new UnknownBlockRateLimitedError(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`);
+        const retryAt = deferredByRateLimit ? this.peerBalancer.getNextRateLimitRetryAt() : null;
+        if (retryAt !== null) {
+            throw new UnknownBlockRateLimitedError(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`, retryAt);
         }
         throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
     }
@@ -1420,11 +1477,14 @@ export class BlockInputSync {
             const pendingColumns = isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
                 ? new Set(cacheItem.blockInput.getMissingSampledColumnMeta().missing)
                 : defaultPendingColumns;
+            // Same `now` for both calls, a backoff expiring in between would be reported as no peer available
+            const now = Date.now();
             // prefer peers that gossiped this block root to us (#9923)
-            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings);
+            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings, now);
             if (peerMeta === null) {
-                if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
-                    throw new UnknownBlockRateLimitedError(`Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`);
+                const retryAt = this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers, now);
+                if (retryAt !== null) {
+                    throw new UnknownBlockRateLimitedError(`Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`, retryAt);
                 }
                 // no more peer with needed columns to try, throw error
                 const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
@@ -1433,7 +1493,6 @@ export class BlockInputSync {
                 throw Error(message);
             }
             const { peerId, client: peerClient } = peerMeta;
-            cacheItem.peerIdStrings.add(peerId);
             try {
                 const downloadResult = await downloadByRoot({
                     config: this.config,
@@ -1444,6 +1503,7 @@ export class BlockInputSync {
                     cacheItem,
                 });
                 cacheItem = downloadResult.result;
+                cacheItem.peerIdStrings.add(peerId);
                 if (slot === undefined) {
                     slot = cacheItem.blockInput.slot;
                     // we were not able to observe the time into slot when starting the fetch, do it now
@@ -1520,8 +1580,9 @@ export class BlockInputSync {
             }
         } // end while loop over peers
         const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
-        if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
-            throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`);
+        const retryAt = deferredByRateLimit ? this.peerBalancer.getNextRateLimitRetryAt() : null;
+        if (retryAt !== null) {
+            throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`, retryAt);
         }
         if (!isPendingBlockInput(cacheItem)) {
             throw Error(`${message} No block and no data was found.`);
@@ -1545,41 +1606,8 @@ export class BlockInputSync {
         this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
         throw Error(message);
     }
-    /**
-     * Gets all descendant blocks of `block` recursively from `pendingBlocks`.
-     * Assumes that if a parent block does not exist or is not processable, all descendant blocks are bad too.
-     * Downscore all peers that have referenced any of this bad blocks. May report peers multiple times if they have
-     * referenced more than one bad block.
-     */
-    removeAndDownScoreAllDescendants(block, headReason) {
-        // Get all blocks that are a descendant of this one
-        const badPendingBlocks = this.removeAllDescendants(block, headReason, DroppedItemReason.invalidParent);
-        // just console log and do not penalize on pending/bad blocks for debugging
-        // console.log("removeAndDownscoreAllDescendants", {block});
-        for (const block of badPendingBlocks) {
-            //
-            // TODO(fulu): why is this commented out here?
-            //
-            //   this.knownBadBlocks.add(block.blockRootHex);
-            //   for (const peerIdStr of block.peerIdStrings) {
-            //     // TODO: Refactor peerRpcScores to work with peerIdStr only
-            //     this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot");
-            //   }
-            this.logger.debug("ignored Banning unknown block", {
-                slot: getBlockInputSyncCacheItemSlot(block),
-                root: getBlockInputSyncCacheItemRootHex(block),
-                peerIdStrings: Array.from(block.peerIdStrings)
-                    .map((id) => prettyPrintPeerIdStr(id))
-                    .join(","),
-            });
-        }
-        // Prune knownBadBlocks
-        pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
-    }
     // Once a parent payload is invalid, every descendant waiting on that payload lineage becomes unrecoverable too.
     removePendingPayloadAndDescendants(rootHex, headReason) {
-        // Keep PayloadEnvelopeInput resident in the seen cache. importBlock() owns that object and
-        // later validation/finalization logic decides when it can leave memory.
         if (this.pendingPayloads.delete(rootHex)) {
             this.metrics?.blockInputSync.removedPayloads.inc({ reason: headReason }, 1);
         }
@@ -1600,9 +1628,50 @@ export class BlockInputSync {
             });
         }
     }
+    /**
+     * Same as `removeAndBanAllDescendants`, and downscores every peer that referenced or served a banned item, once per
+     * peer. Only call for consensus-layer invalidity, never for an INVALID execution payload:
+     * https://github.com/ethereum/consensus-specs/blob/master/specs/bellatrix/p2p-interface.md#the-reqresp-domain
+     */
+    removeAndDownScoreAllDescendants(block, headReason) {
+        const peersToReport = new Set();
+        for (const bannedBlock of this.removeAndBanAllDescendants(block, headReason)) {
+            for (const peerIdStr of bannedBlock.peerIdStrings) {
+                peersToReport.add(peerIdStr);
+            }
+        }
+        for (const peerIdStr of peersToReport) {
+            this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, headReason);
+        }
+    }
+    /**
+     * Removes `block` and all its pending descendants and bans their roots in `knownBadBlocks`. Only call when the chain
+     * is provably bad: a banned root is never searched for again. Returns the banned items.
+     */
+    removeAndBanAllDescendants(block, headReason) {
+        const bannedBlocks = [];
+        for (const badBlock of this.removeAllDescendants(block, headReason, DroppedItemReason.invalidParent)) {
+            const rootHex = getBlockInputSyncCacheItemRootHex(badBlock);
+            // a gossip import may race a failing sync path, never ban a root fork choice has imported
+            if (this.chain.forkChoice.hasBlockHex(rootHex)) {
+                continue;
+            }
+            this.knownBadBlocks.add(rootHex);
+            bannedBlocks.push(badBlock);
+            this.logger.debug("Banning unknown block", {
+                slot: getBlockInputSyncCacheItemSlot(badBlock),
+                root: rootHex,
+                reason: headReason,
+                peerIdStrings: Array.from(badBlock.peerIdStrings)
+                    .map((id) => prettyPrintPeerIdStr(id))
+                    .join(","),
+            });
+        }
+        pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
+        return bannedBlocks;
+    }
     removeAllDescendants(block, headReason, descendantReason = DroppedItemReason.invalidParent) {
         const rootHex = getBlockInputSyncCacheItemRootHex(block);
-        const slot = getBlockInputSyncCacheItemSlot(block);
         // Get all blocks that are a descendant of this one (index 0 is the head item itself)
         const badPendingBlocks = [block, ...getAllDescendantBlocks(rootHex, this.pendingBlocks)];
         for (const [i, block] of badPendingBlocks.entries()) {
@@ -1618,8 +1687,8 @@ export class BlockInputSync {
             // Keep PayloadEnvelopeInput resident in the seen cache for consistency with the
             // importBlock()-owned lifecycle.
             this.logger.debug("Removing bad/unknown/incomplete BlockInputSyncCacheItem", {
-                slot,
-                blockRoot: rootHex,
+                slot: getBlockInputSyncCacheItemSlot(block),
+                root: rootHex,
             });
         }
         return badPendingBlocks;
@@ -1666,8 +1735,7 @@ export class UnknownBlockPeerBalancer {
     onRateLimited(peerId, rateLimitedUntilMs) {
         this.rateLimitedUntilByPeer.set(peerId, rateLimitedUntilMs);
     }
-    getNextRateLimitRetryAt(pendingColumns, excludedPeers) {
-        const now = Date.now();
+    getNextRateLimitRetryAt(pendingColumns, excludedPeers, now = Date.now()) {
         let retryAt = null;
         for (const [peerId, rateLimitedUntil] of this.rateLimitedUntilByPeer.entries()) {
             if (rateLimitedUntil <= now) {
@@ -1696,8 +1764,8 @@ export class UnknownBlockPeerBalancer {
      * preferredPeers are the peers NetworkProcessor told us gossiped this root (from
      *   pendingBlock/cacheItem.peerIdStrings). They are preferred but not required.
      */
-    bestPeerForPendingColumns(pendingColumns, excludedPeers, preferredPeers) {
-        const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers);
+    bestPeerForPendingColumns(pendingColumns, excludedPeers, preferredPeers, now = Date.now()) {
+        const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers, now);
         if (eligiblePeers.length === 0) {
             return null;
         }
@@ -1730,8 +1798,7 @@ export class UnknownBlockPeerBalancer {
         }
         return totalActiveRequests;
     }
-    filterPeers(pendingDataColumns, excludedPeers) {
-        const now = Date.now();
+    filterPeers(pendingDataColumns, excludedPeers, now) {
         let maxColumnCount = 0;
         const considerPeers = [];
         for (const [peerId, syncMeta] of this.peersMeta.entries()) {