@lodestar/beacon-node 1.48.0-dev.e9c9b8835d → 1.48.0-dev.fd70e5fc15

package/lib/network/processor/index.js

@@ -1,6 +1,6 @@
 import { TopicValidatorResult } from "@libp2p/gossipsub";
 import { routes } from "@lodestar/api";
-import { ForkSeq, NUMBER_OF_COLUMNS } from "@lodestar/params";
+import { ForkSeq } from "@lodestar/params";
 import { computeStartSlotAtEpoch } from "@lodestar/state-transition";
 import { MapDef, mapValues, sleep } from "@lodestar/utils";
 import { BlockInputSource } from "../../chain/blocks/blockInput/types.js";
@@ -20,28 +20,6 @@ export * from "./types.js";
  * Keep up to 3 slot of unknown roots, so we don't always emit to UnknownBlock sync.
  */
 const MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE = 3;
-/**
- * Max distinct roots we BUFFER messages for, per slot. Kept tight to avoid the OOM
- * risk. We don't support super forky condition where there are more than this many roots per slot via gossip.
- */
-export const MAX_BUFFERED_ROOTS_PER_SLOT = 5;
-/**
- * Max distinct roots we emit an unknown-root search for, per slot. Higher than
- * the buffer budget because:
- * - in the attack scenario, the genuine root may comes after the first MAX_BUFFERED_ROOTS_PER_SLOT roots
- * - if we cannot search, we'll penalize peers in UnknownBlockInput, not in NetworkProcessor
- */
-export const MAX_SEARCHED_ROOTS_PER_SLOT = 32;
-/**
- * Given the same root and topic, we'll ignore messages after the below cap. Each cap is the number of
- * legitimate messages we expect for one block, plus headroom for some malformed messages that may arrives first
- * they will be penalized by the gossip handler
- */
-export const MAX_AWAITING_MESSAGES_PER_ROOT = {
-    [GossipType.data_column_sidecar]: 2 * NUMBER_OF_COLUMNS,
-    [GossipType.execution_payload]: 2,
-    [GossipType.execution_payload_bid]: 8,
-};
 /**
  * This is respective to gossipsub seenTTL (which is 550 * 0.7 = 385s), also it's respective
  * to beacon_attestation ATTESTATION_PROPAGATION_SLOT_RANGE (32 slots).
@@ -94,14 +72,6 @@ export var ReprocessRejectReason;
      * There are too many gossip messages that have unknown block root.
      */
     ReprocessRejectReason["reached_limit"] = "reached_limit";
-    /**
-     * Too many distinct unknown roots are already tracked for the message's slot (MAX_UNKNOWN_ROOTS_PER_SLOT).
-     */
-    ReprocessRejectReason["reached_root_limit"] = "reached_root_limit";
-    /**
-     * Too many gossip messages are already buffered for this (root, topic) (MAX_AWAITING_MESSAGES_PER_ROOT).
-     */
-    ReprocessRejectReason["reached_topic_limit"] = "reached_topic_limit";
     /**
      * The awaiting gossip message is pruned per clock slot.
      */
@@ -162,16 +132,15 @@ export class NetworkProcessor {
     gossipTopicConcurrency;
     extractBlockSlotRootFns = createExtractBlockSlotRootFns();
     // we may not receive the block for messages like Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
-    // to be stored in this Map and reprocessed once the block comes. Keyed by topic per root so we can cap
-    // the number of buffered messages per (root, topic) - see MAX_AWAITING_MESSAGES_PER_ROOT.
+    // to be stored in this Map and reprocessed once the block comes
     awaitingMessagesByBlockRoot;
     awaitingBlockMessageCount = 0;
     // we may not receive the payload for messages that require the FULL payload variant to be processed,
     // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
     awaitingMessagesByPayloadBlockRoot;
     awaitingPayloadMessageCount = 0;
-    searchedRootsBySlot = new MapDef(() => new MapDef(() => ({})));
-    bufferedRootsBySlot = new MapDef(() => new Set());
+    unknownBlocksBySlot = new MapDef(() => new MapDef(() => new Set()));
+    unknownEnvelopesBySlot = new MapDef(() => new MapDef(() => new Set()));
     constructor(modules, opts) {
         this.opts = opts;
         const { chain, events, logger, metrics } = modules;
@@ -188,8 +157,8 @@ export class NetworkProcessor {
         this.chain.emitter.on(routes.events.EventType.block, this.onBlockProcessed);
         this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadEnvelopeProcessed);
         this.chain.clock.on(ClockEvent.slot, this.onClockSlot);
-        this.awaitingMessagesByBlockRoot = new MapDef(() => new MapDef(() => new Set()));
-        this.awaitingMessagesByPayloadBlockRoot = new MapDef(() => new MapDef(() => new Set()));
+        this.awaitingMessagesByBlockRoot = new MapDef(() => new Set());
+        this.awaitingMessagesByPayloadBlockRoot = new MapDef(() => new Set());
         // TODO: Implement queues and priorization for ReqResp incoming requests
         // Listens to NetworkEvent.reqRespIncomingRequest event
         if (metrics) {
@@ -241,12 +210,9 @@ export class NetworkProcessor {
         if (this.chain.seenBlock(root)) {
             return;
         }
-        const entry = this.searchedRootsBySlot.getOrDefault(slot).getOrDefault(root);
-        const alreadySearching = entry.blockPeerIds !== undefined;
-        if (entry.blockPeerIds === undefined) {
-            entry.blockPeerIds = new Set();
-        }
-        const forwardedPeers = entry.blockPeerIds;
+        const peersForRoot = this.unknownBlocksBySlot.getOrDefault(slot);
+        const alreadySearching = peersForRoot.has(root) || this.awaitingMessagesByBlockRoot.has(root);
+        const forwardedPeers = peersForRoot.getOrDefault(root);
         // brand-new search always emits (peer may be undefined)
         let shouldEmit = !alreadySearching;
         if (peer !== undefined && !forwardedPeers.has(peer) && forwardedPeers.size < MAX_PEERS_PER_ROOT) {
@@ -268,12 +234,10 @@ export class NetworkProcessor {
         if (this.chain.seenPayloadEnvelope(root)) {
             return;
         }
-        const entry = this.searchedRootsBySlot.getOrDefault(slot).getOrDefault(root);
-        const alreadySearching = entry.envelopePeerIds !== undefined;
-        if (entry.envelopePeerIds === undefined) {
-            entry.envelopePeerIds = new Set();
-        }
-        const forwardedPeers = entry.envelopePeerIds;
+        const peersForRoot = this.unknownEnvelopesBySlot.getOrDefault(slot);
+        // capture "already searching" BEFORE getOrDefault(root) creates the entry
+        const alreadySearching = peersForRoot.has(root) || this.awaitingMessagesByPayloadBlockRoot.has(root);
+        const forwardedPeers = peersForRoot.getOrDefault(root);
         let shouldEmit = !alreadySearching; // brand-new search always emits (peer may be undefined)
         if (peer !== undefined && !forwardedPeers.has(peer) && forwardedPeers.size < MAX_PEERS_PER_ROOT) {
             // First time we forward THIS peer for THIS root: emit so BlockInputSync accumulates it.
@@ -328,14 +292,13 @@ export class NetworkProcessor {
             return;
         }
         message.msgSlot = slot;
-        const peerId = message.propagationSource.toString();
         // this determines whether this message needs to wait for a Block or Envelope
-        // a message should only wait for what it voted for, hence we don't want to put it on both queues.
+        // a message should only wait for what it voted for, hence we don't want to put it on both queues
         let preprocessResult = { action: PreprocessAction.PushToQueue };
         // no need to check if root is a descendant of the current finalized block, it will be checked once we validate the message if needed
         if (root && !this.chain.forkChoice.hasBlockHexUnsafe(root)) {
             // starting from GLOAS, unknown root from data_column_sidecar also falls into this case
-            this.searchUnknownRoot({ slot, root }, true, false, peerId);
+            this.searchUnknownBlock({ slot, root }, BlockInputSource.network_processor, message.propagationSource.toString());
             // for beacon_attestation and beacon_aggregate_and_proof messages, this is only temporary.
             // if "index = 1" we need to await for the Envelope instead
             preprocessResult = { action: PreprocessAction.AwaitBlock, root };
@@ -343,32 +306,28 @@ export class NetworkProcessor {
         // 2nd extract round for some specific topics
         // we separate the search action from the await action
         // beacon_block: proactively search for parent block/envelope across all forks, but never queue.
-        // BlockInputSync handles cascading recovery if the gossip handler throws. The parent root differs from
-        // the message root and this message never awaits, so its search flushes inline here (gated by the budget).
+        // BlockInputSync handles cascading recovery if the gossip handler throws.
         if (topicType === GossipType.beacon_block) {
             const parentRoot = getParentRootFromSignedBeaconBlockSerialized(message.msg.data);
             if (parentRoot) {
-                let searchParentBlock = false;
-                let searchParentEnvelope = false;
                 if (ForkSeq[fork] >= ForkSeq.gloas) {
                     // GLOAS: also check parent envelope, same logic as execution_payload_bid
                     const parentBlockHash = getParentBlockHashFromGloasSignedBeaconBlockSerialized(message.msg.data);
                     if (parentBlockHash && !this.chain.forkChoice.getBlockHexAndBlockHash(parentRoot, parentBlockHash)) {
                         const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
                         if (protoBlock === null) {
-                            searchParentBlock = true;
+                            this.searchUnknownBlock({ slot, root: parentRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
                         }
                         else if (protoBlock.executionPayloadBlockHash &&
                             protoBlock.executionPayloadBlockHash !== parentBlockHash) {
                             // only search for the envelope by block root if we're sure there is one. Otherwise UnknownBlockSync will penalize the peer.
-                            searchParentEnvelope = true;
+                            this.searchUnknownEnvelope({ slot, root: parentRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
                         }
                     }
                 }
                 else if (!this.chain.forkChoice.hasBlockHexUnsafe(parentRoot)) {
-                    searchParentBlock = true;
+                    this.searchUnknownBlock({ slot, root: parentRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
                 }
-                this.searchUnknownRoot({ slot, root: parentRoot }, searchParentBlock, searchParentEnvelope, peerId);
             }
             preprocessResult = { action: PreprocessAction.PushToQueue };
         }
@@ -386,7 +345,7 @@ export class NetworkProcessor {
                         : getDataIndexFromSignedAggregateAndProofSerialized(message.msg.data);
                     if (attIndex === 1 && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
                         // attestation votes that the payload is available but it is not yet known
-                        this.searchUnknownRoot({ slot, root }, false, true, peerId);
+                        this.searchUnknownEnvelope({ slot, root }, BlockInputSource.network_processor, message.propagationSource.toString());
                         preprocessResult = { action: PreprocessAction.AwaitEnvelope, root };
                     }
                     break;
@@ -397,7 +356,7 @@ export class NetworkProcessor {
                     const payloadPresent = getPayloadPresentFromPayloadAttestationMessageSerialized(message.msg.data);
                     if (payloadPresent && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
                         // payload attestation votes that the payload is available but it is not yet known
-                        this.searchUnknownRoot({ slot, root }, false, true, peerId);
+                        this.searchUnknownEnvelope({ slot, root }, BlockInputSource.network_processor, message.propagationSource.toString());
                         // do not await the envelope, payload attestation processing only requires that the block is known
                         // also do not reset preprocessResult, we may already await for the block
                     }
@@ -407,7 +366,7 @@ export class NetworkProcessor {
                     if (root == null)
                         break;
                     if (!this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
-                        this.searchUnknownRoot({ slot, root }, false, true, peerId);
+                        this.searchUnknownEnvelope({ slot, root }, BlockInputSource.network_processor, message.propagationSource.toString());
                         // do not await the envelope, we can do gossip validation
                         // also do not reset preprocessResult, we may already await for the block
                     }
@@ -418,7 +377,7 @@ export class NetworkProcessor {
                     // Extract beacon_block_root directly
                     const blockRoot = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(message.msg.data);
                     if (blockRoot && !this.chain.forkChoice.hasBlockHexUnsafe(blockRoot)) {
-                        this.searchUnknownRoot({ slot, root: blockRoot }, true, false, peerId);
+                        this.searchUnknownBlock({ slot, root: blockRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
                         // We always want to await the block
                         // This allows us to properly forward the payload envelope
                         preprocessResult = { action: PreprocessAction.AwaitBlock, root: blockRoot };
@@ -435,12 +394,12 @@ export class NetworkProcessor {
                         !this.chain.forkChoice.getBlockHexAndBlockHash(parentBlockRoot, parentBlockHash)) {
                         const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentBlockRoot);
                         if (protoBlock === null) {
-                            this.searchUnknownRoot({ slot, root: parentBlockRoot }, true, false, peerId);
+                            this.searchUnknownBlock({ slot, root: parentBlockRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
                             preprocessResult = { action: PreprocessAction.AwaitBlock, root: parentBlockRoot };
                         }
                         else if (protoBlock.executionPayloadBlockHash &&
                             protoBlock.executionPayloadBlockHash !== parentBlockHash) {
-                            this.searchUnknownRoot({ slot, root: parentBlockRoot }, false, true, peerId);
+                            this.searchUnknownEnvelope({ slot, root: parentBlockRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
                             preprocessResult = { action: PreprocessAction.AwaitEnvelope, root: parentBlockRoot };
                         }
                     }
@@ -452,96 +411,37 @@ export class NetworkProcessor {
             case PreprocessAction.PushToQueue:
                 this.pushPendingGossipsubMessageToQueue(message);
                 break;
-            case PreprocessAction.AwaitBlock:
-                this.maybeAwaitBlock(preprocessResult.root, topicType, slot, message);
+            case PreprocessAction.AwaitBlock: {
+                if (this.awaitingBlockMessageCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
+                    // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+                    this.metrics?.awaitingBlockGossipMessages.reject.inc({
+                        reason: ReprocessRejectReason.reached_limit,
+                        topic: topicType,
+                    });
+                    return;
+                }
+                this.metrics?.awaitingBlockGossipMessages.queue.inc({ topic: topicType });
+                const awaitingGossipsubMessages = this.awaitingMessagesByBlockRoot.getOrDefault(preprocessResult.root);
+                awaitingGossipsubMessages.add(message);
+                this.awaitingBlockMessageCount++;
                 break;
-            case PreprocessAction.AwaitEnvelope:
-                this.maybeAwaitPayload(preprocessResult.root, topicType, slot, message);
+            }
+            case PreprocessAction.AwaitEnvelope: {
+                if (this.awaitingPayloadMessageCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
+                    this.metrics?.awaitingPayloadGossipMessages.reject.inc({
+                        reason: ReprocessRejectReason.reached_limit,
+                        topic: topicType,
+                    });
+                    return;
+                }
+                this.metrics?.awaitingPayloadGossipMessages.queue.inc({ topic: topicType });
+                const awaitingPayloadGossipsubMessages = this.awaitingMessagesByPayloadBlockRoot.getOrDefault(preprocessResult.root);
+                awaitingPayloadGossipsubMessages.add(message);
+                this.awaitingPayloadMessageCount++;
                 break;
+            }
         }
     };
-    /**
-     * Buffer a gossip message that must wait for an unknown block before it can be validated.
-     *
-     * For DOS protection, the message is ignored (reject metric incremented) when any cap is hit:
-     * - MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS: global awaiting-block message count
-     * - MAX_BUFFERED_ROOTS_PER_SLOT: distinct roots we buffer messages for, per slot
-     * - MAX_AWAITING_MESSAGES_PER_ROOT[topic]: messages buffered per (root, topic)
-     */
-    maybeAwaitBlock(root, topicType, slot, message) {
-        const metric = this.metrics?.awaitingBlockGossipMessages;
-        // global message-count cap - cheapest check first
-        if (this.awaitingBlockMessageCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
-            metric?.reject.inc({ reason: ReprocessRejectReason.reached_limit, topic: topicType });
-            return;
-        }
-        // per-(root, topic) message cap: e.g. <= NUMBER_OF_COLUMNS columns per root
-        const perRootCap = MAX_AWAITING_MESSAGES_PER_ROOT[topicType];
-        if (perRootCap !== undefined &&
-            (this.awaitingMessagesByBlockRoot.get(root)?.get(topicType)?.size ?? 0) >= perRootCap) {
-            metric?.reject.inc({ reason: ReprocessRejectReason.reached_topic_limit, topic: topicType });
-            return;
-        }
-        if (!this.tryReserveBufferedRoot(slot, root)) {
-            metric?.reject.inc({ reason: ReprocessRejectReason.reached_root_limit, topic: topicType });
-            return;
-        }
-        metric?.queue.inc({ topic: topicType });
-        this.awaitingMessagesByBlockRoot.getOrDefault(root).getOrDefault(topicType).add(message);
-        this.awaitingBlockMessageCount++;
-    }
-    /** Payload counterpart of maybeAwaitBlock */
-    maybeAwaitPayload(root, topicType, slot, message) {
-        const metric = this.metrics?.awaitingPayloadGossipMessages;
-        if (this.awaitingPayloadMessageCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
-            metric?.reject.inc({ reason: ReprocessRejectReason.reached_limit, topic: topicType });
-            return;
-        }
-        const perRootCap = MAX_AWAITING_MESSAGES_PER_ROOT[topicType];
-        if (perRootCap !== undefined &&
-            (this.awaitingMessagesByPayloadBlockRoot.get(root)?.get(topicType)?.size ?? 0) >= perRootCap) {
-            metric?.reject.inc({ reason: ReprocessRejectReason.reached_topic_limit, topic: topicType });
-            return;
-        }
-        if (!this.tryReserveBufferedRoot(slot, root)) {
-            metric?.reject.inc({ reason: ReprocessRejectReason.reached_root_limit, topic: topicType });
-            return;
-        }
-        metric?.queue.inc({ topic: topicType });
-        this.awaitingMessagesByPayloadBlockRoot.getOrDefault(root).getOrDefault(topicType).add(message);
-        this.awaitingPayloadMessageCount++;
-    }
-    /**
-     * Cap distinct roots we emit an unknown-root search for, an already-tracked root (dedup) never counts as "too many".
-     */
-    tooManySearchedRoots(slot, root) {
-        const roots = this.searchedRootsBySlot.get(slot);
-        if (roots === undefined)
-            return false;
-        return !roots.has(root) && roots.size >= MAX_SEARCHED_ROOTS_PER_SLOT;
-    }
-    /**
-     * Cap distinct roots we buffer per slot.
-     */
-    tryReserveBufferedRoot(slot, root) {
-        const roots = this.bufferedRootsBySlot.get(slot);
-        if (roots?.has(root))
-            return true;
-        if ((roots?.size ?? 0) >= MAX_BUFFERED_ROOTS_PER_SLOT)
-            return false;
-        this.bufferedRootsBySlot.getOrDefault(slot).add(root);
-        return true;
-    }
-    searchUnknownRoot(slotRoot, searchBlock, searchEnvelope, peerId) {
-        if (!searchBlock && !searchEnvelope)
-            return;
-        if (this.tooManySearchedRoots(slotRoot.slot, slotRoot.root))
-            return;
-        if (searchBlock)
-            this.searchUnknownBlock(slotRoot, BlockInputSource.network_processor, peerId);
-        if (searchEnvelope)
-            this.searchUnknownEnvelope(slotRoot, BlockInputSource.network_processor, peerId);
-    }
     pushPendingGossipsubMessageToQueue(message) {
         const topicType = message.topic.type;
         message.queueAddedMs = Date.now();
@@ -554,92 +454,66 @@ export class NetworkProcessor {
         this.executeWork();
     }
     onBlockProcessed = async ({ block: rootHex }) => {
-        const messagesByTopic = this.awaitingMessagesByBlockRoot.get(rootHex);
-        if (messagesByTopic === undefined) {
-            return;
-        }
-        let total = 0;
-        for (const messages of messagesByTopic.values()) {
-            total += messages.size;
-        }
-        if (total === 0) {
+        const waitingGossipsubMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
+        if (!waitingGossipsubMessages || waitingGossipsubMessages.size === 0) {
             return;
         }
         // Atomically remove from map and update counter before async iteration to
         // prevent double-decrement race with onClockSlot during yield points below
         if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
-            this.awaitingBlockMessageCount -= total;
+            this.awaitingBlockMessageCount -= waitingGossipsubMessages.size;
         }
         const nowSec = Date.now() / 1000;
         let count = 0;
         // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
         // cache, it may not be necessary at this time
-        for (const messages of messagesByTopic.values()) {
-            for (const message of messages) {
-                const topicType = message.topic.type;
-                this.metrics?.awaitingBlockGossipMessages.waitSecBeforeResolve.set({ topic: topicType }, nowSec - message.seenTimestampSec);
-                this.metrics?.awaitingBlockGossipMessages.resolve.inc({ topic: topicType });
-                this.pushPendingGossipsubMessageToQueue(message);
-                count++;
-                // don't want to block the event loop, worse case it'd wait for 16_084 / 1024 * 50ms = 800ms which is not a big deal
-                if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
-                    count = 0;
-                    await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
-                }
+        for (const message of waitingGossipsubMessages) {
+            const topicType = message.topic.type;
+            this.metrics?.awaitingBlockGossipMessages.waitSecBeforeResolve.set({ topic: topicType }, nowSec - message.seenTimestampSec);
+            this.metrics?.awaitingBlockGossipMessages.resolve.inc({ topic: topicType });
+            this.pushPendingGossipsubMessageToQueue(message);
+            count++;
+            // don't want to block the event loop, worse case it'd wait for 16_084 / 1024 * 50ms = 800ms which is not a big deal
+            if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
+                count = 0;
+                await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
             }
         }
     };
     onPayloadEnvelopeProcessed = async ({ blockRoot: rootHex }) => {
-        const messagesByTopic = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
-        if (messagesByTopic === undefined) {
-            return;
-        }
-        let total = 0;
-        for (const messages of messagesByTopic.values()) {
-            total += messages.size;
-        }
-        if (total === 0) {
+        const waitingGossipsubMessages = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
+        if (!waitingGossipsubMessages || waitingGossipsubMessages.size === 0) {
             return;
         }
         // Atomically remove from map and update counter before async iteration to
         // prevent double-decrement race with onClockSlot during yield points below
         if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
-            this.awaitingPayloadMessageCount -= total;
+            this.awaitingPayloadMessageCount -= waitingGossipsubMessages.size;
         }
         const nowSec = Date.now() / 1000;
         let count = 0;
-        for (const messages of messagesByTopic.values()) {
-            for (const message of messages) {
-                const topicType = message.topic.type;
-                this.metrics?.awaitingPayloadGossipMessages.waitSecBeforeResolve.set({ topic: topicType }, nowSec - message.seenTimestampSec);
-                this.metrics?.awaitingPayloadGossipMessages.resolve.inc({ topic: topicType });
-                this.pushPendingGossipsubMessageToQueue(message);
-                count++;
-                if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
-                    count = 0;
-                    await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
-                }
+        for (const message of waitingGossipsubMessages) {
+            const topicType = message.topic.type;
+            this.metrics?.awaitingPayloadGossipMessages.waitSecBeforeResolve.set({ topic: topicType }, nowSec - message.seenTimestampSec);
+            this.metrics?.awaitingPayloadGossipMessages.resolve.inc({ topic: topicType });
+            this.pushPendingGossipsubMessageToQueue(message);
+            count++;
+            if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
+                count = 0;
+                await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
             }
         }
     };
     onClockSlot = (clockSlot) => {
         const nowSec = Date.now() / 1000;
         const minSlot = clockSlot - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE;
-        for (const [slot, searchedRoots] of this.searchedRootsBySlot) {
+        for (const [slot, roots] of this.unknownBlocksBySlot) {
             if (slot > minSlot)
                 continue;
-            // expire messages awaiting an unknown block for these roots. Only roots this slot did a BLOCK search
-            // for (blockPeerIds set) have block waits keyed here; skipping the rest avoids deleting a payload wait
-            // that a newer slot created for the same root (block/envelope searches share searchedRootsBySlot).
-            for (const [rootHex, entry] of searchedRoots) {
-                if (entry.blockPeerIds === undefined)
-                    continue;
-                const messagesByTopic = this.awaitingMessagesByBlockRoot.get(rootHex);
-                if (messagesByTopic === undefined)
-                    continue;
-                let removed = 0;
-                for (const messages of messagesByTopic.values()) {
-                    for (const message of messages) {
+            for (const rootHex of roots.keys()) {
+                const gossipMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
+                if (gossipMessages !== undefined) {
+                    for (const message of gossipMessages) {
                         const topicType = message.topic.type;
                         this.metrics?.awaitingBlockGossipMessages.reject.inc({
                             topic: topicType,
@@ -648,23 +522,20 @@ export class NetworkProcessor {
                         this.metrics?.awaitingBlockGossipMessages.waitSecBeforeReject.set({ topic: topicType, reason: ReprocessRejectReason.expired }, nowSec - message.seenTimestampSec);
                         // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
                     }
-                    removed += messages.size;
-                }
-                if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
-                    this.awaitingBlockMessageCount -= removed;
+                    if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
+                        this.awaitingBlockMessageCount -= gossipMessages.size;
+                    }
                 }
             }
-            // expire messages awaiting an unknown payload envelope for these roots. Symmetric to the block pass:
-            // only roots this slot did an ENVELOPE search for (envelopePeerIds set) have payload waits keyed here.
-            for (const [rootHex, entry] of searchedRoots) {
-                if (entry.envelopePeerIds === undefined)
-                    continue;
-                const messagesByTopic = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
-                if (messagesByTopic === undefined)
-                    continue;
-                let removed = 0;
-                for (const messages of messagesByTopic.values()) {
-                    for (const message of messages) {
+            this.unknownBlocksBySlot.delete(slot);
+        }
+        for (const [slot, roots] of this.unknownEnvelopesBySlot) {
+            if (slot > minSlot)
+                continue;
+            for (const rootHex of roots.keys()) {
+                const gossipMessages = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
+                if (gossipMessages !== undefined) {
+                    for (const message of gossipMessages) {
                         const topicType = message.topic.type;
                         this.metrics?.awaitingPayloadGossipMessages.reject.inc({
                             topic: topicType,
@@ -673,17 +544,12 @@ export class NetworkProcessor {
                         this.metrics?.awaitingPayloadGossipMessages.waitSecBeforeReject.set({ topic: topicType, reason: ReprocessRejectReason.expired }, nowSec - message.seenTimestampSec);
                         // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
                     }
-                    removed += messages.size;
-                }
-                if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
-                    this.awaitingPayloadMessageCount -= removed;
+                    if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
+                        this.awaitingPayloadMessageCount -= gossipMessages.size;
+                    }
                 }
             }
-            this.searchedRootsBySlot.delete(slot);
-        }
-        for (const slot of this.bufferedRootsBySlot.keys()) {
-            if (slot <= minSlot)
-                this.bufferedRootsBySlot.delete(slot);
+            this.unknownEnvelopesBySlot.delete(slot);
         }
     };
     executeWork() {