@lodestar/beacon-node 1.42.0-dev.f6213da56d → 1.42.0-dev.fb8f8a700e

package/src/network/processor/index.ts

@@ -2,7 +2,7 @@ import {routes} from "@lodestar/api";
 import {ForkSeq} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {RootHex, Slot, SlotRootHex} from "@lodestar/types";
-import {Logger, MapDef, mapValues, pruneSetToMax, sleep} from "@lodestar/utils";
+import {Logger, MapDef, mapValues, sleep} from "@lodestar/utils";
 import {BlockInputSource} from "../../chain/blocks/blockInput/types.js";
 import {ChainEvent} from "../../chain/emitter.js";
 import {GossipErrorCode} from "../../chain/errors/gossipValidation.js";
@@ -12,6 +12,16 @@ import {Metrics} from "../../metrics/metrics.js";
 import {ClockEvent} from "../../util/clock.js";
 import {callInNextEventLoop} from "../../util/eventLoop.js";
 import {PeerIdStr} from "../../util/peerId.js";
+import {
+  getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized,
+  getDataIndexFromSignedAggregateAndProofSerialized,
+  getDataIndexFromSingleAttestationSerialized,
+  getParentBlockHashFromGloasSignedBeaconBlockSerialized,
+  getParentBlockHashFromSignedExecutionPayloadBidSerialized,
+  getParentBlockRootFromSignedExecutionPayloadBidSerialized,
+  getParentRootFromSignedBeaconBlockSerialized,
+  getPayloadPresentFromPayloadAttestationMessageSerialized,
+} from "../../util/sszBytes.js";
 import {NetworkEvent, NetworkEventBus} from "../events.js";
 import {
   GossipHandlers,
@@ -87,27 +97,29 @@ const executeGossipWorkOrder = Object.keys(executeGossipWorkOrderObj) as (keyof
 // TODO: Arbitrary constant, check metrics
 const MAX_JOBS_SUBMITTED_PER_TICK = 128;
 
-// How many attestations (aggregate + unaggregate) we keep before new ones get dropped.
+// How many gossip messages we keep before new ones get dropped.
 const MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS = 16_384;
+// TODO gloas: arbitrary constant, check metrics.
+const MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS = 1024;
 
-// We don't want to process too many attestations in a single tick
-// As seen on mainnet, attestation concurrency metric ranges from 1000 to 2000
+// We don't want to process too many gossip messages in a single tick
+// As seen on mainnet, gossip messages concurrency metric ranges from 1000 to 2000
 // so make this constant a little bit conservative
-const MAX_UNKNOWN_BLOCK_GOSSIP_OBJECTS_PER_TICK = 1024;
+const MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK = 1024;
 
 // Same motivation to JobItemQueue, we don't want to block the event loop
-const PROCESS_UNKNOWN_BLOCK_GOSSIP_OBJECTS_YIELD_EVERY_MS = 50;
+const AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS = 50;
 
 /**
  * Reprocess reject reason for metrics
  */
 export enum ReprocessRejectReason {
   /**
-   * There are too many attestations that have unknown block root.
+   * There are too many gossip messages that have unknown block root.
    */
   reached_limit = "reached_limit",
   /**
-   * The awaiting attestation is pruned per clock slot.
+   * The awaiting gossip message is pruned per clock slot.
    */
   expired = "expired",
 }
@@ -126,6 +138,20 @@ export enum CannotAcceptWorkReason {
   regen = "regen_busy",
 }
 
+/**
+ * No metrics needed here; using a number to keep it lightweight
+ */
+enum PreprocessAction {
+  AwaitBlock,
+  AwaitEnvelope,
+  PushToQueue,
+}
+
+type PreprocessResult =
+  | {action: PreprocessAction.PushToQueue}
+  | {action: PreprocessAction.AwaitBlock; root: RootHex}
+  | {action: PreprocessAction.AwaitEnvelope; root: RootHex};
+
 /**
  * Network processor handles the gossip queues and throtles processing to not overload the main thread
  * - Decides when to process work and what to process
@@ -137,7 +163,7 @@ export enum CannotAcceptWorkReason {
  *
  * ### PendingGossipsubMessage beacon_attestation example
  *
- * For attestations, processing the message includes the steps:
+ * For gossip messages, processing the message includes the steps:
  * 1. Pre shuffling sync validation
  * 2. Retrieve shuffling: async + goes into the regen queue and can be expensive
  * 3. Pre sig validation sync validation
@@ -156,11 +182,14 @@ export class NetworkProcessor {
   private readonly gossipQueues: ReturnType<typeof createGossipQueues>;
   private readonly gossipTopicConcurrency: {[K in GossipType]: number};
   private readonly extractBlockSlotRootFns = createExtractBlockSlotRootFns();
-  // we may not receive the block for Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
+  // we may not receive the block for messages like Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
   // to be stored in this Map and reprocessed once the block comes
-  private readonly awaitingGossipsubMessagesByRootBySlot: MapDef<Slot, MapDef<RootHex, Set<PendingGossipsubMessage>>>;
-  private unknownBlockGossipsubMessagesCount = 0;
-  private unknownRootsBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
+  private readonly awaitingMessagesByBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  // we may not receive the payload for messages that require the FULL payload variant to be processed,
+  // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
+  private readonly awaitingMessagesByPayloadBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private unknownBlocksBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
+  private unknownEnvelopesBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
 
   constructor(
     modules: NetworkProcessorModules,
@@ -180,13 +209,13 @@ export class NetworkProcessor {
       modules
     );
 
-    events.on(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage.bind(this));
-    this.chain.emitter.on(routes.events.EventType.block, this.onBlockProcessed.bind(this));
-    this.chain.clock.on(ClockEvent.slot, this.onClockSlot.bind(this));
+    events.on(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
+    this.chain.emitter.on(routes.events.EventType.block, this.onBlockProcessed);
+    this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadEnvelopeProcessed);
+    this.chain.clock.on(ClockEvent.slot, this.onClockSlot);
 
-    this.awaitingGossipsubMessagesByRootBySlot = new MapDef(
-      () => new MapDef<RootHex, Set<PendingGossipsubMessage>>(() => new Set())
-    );
+    this.awaitingMessagesByBlockRoot = new MapDef<RootHex, Set<PendingGossipsubMessage>>(() => new Set());
+    this.awaitingMessagesByPayloadBlockRoot = new MapDef<RootHex, Set<PendingGossipsubMessage>>(() => new Set());
 
     // TODO: Implement queues and priorization for ReqResp incoming requests
     // Listens to NetworkEvent.reqRespIncomingRequest event
@@ -198,7 +227,8 @@ export class NetworkProcessor {
           metrics.gossipValidationQueue.keySize.set({topic}, this.gossipQueues[topic].keySize);
           metrics.gossipValidationQueue.concurrency.set({topic}, this.gossipTopicConcurrency[topic]);
         }
-        metrics.reprocessGossipAttestations.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
+        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
+        metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.unknownPayloadGossipsubMessagesCount);
         // specific metric for beacon_attestation topic
         metrics.gossipValidationQueue.keyAge.reset();
         for (const ageMs of this.gossipQueues.beacon_attestation.getDataAgeMs()) {
@@ -215,6 +245,7 @@ export class NetworkProcessor {
   async stop(): Promise<void> {
     this.events.off(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
     this.chain.emitter.off(routes.events.EventType.block, this.onBlockProcessed);
+    this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadEnvelopeProcessed);
     this.chain.emitter.off(ClockEvent.slot, this.onClockSlot);
   }
 
@@ -233,72 +264,276 @@ export class NetworkProcessor {
     return queue.getAll();
   }
 
-  searchUnknownSlotRoot({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (this.chain.seenBlock(root) || this.unknownRootsBySlot.getOrDefault(slot).has(root)) {
+  /**
+   * Search block via `ChainEvent.unknownBlockRoot` event
+   * Slot is the message slot, which is not necessarily the same as the block's slot, but it can be used for a good prune strategy.
+   * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+   */
+  searchUnknownBlock({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
+    if (
+      this.chain.seenBlock(root) ||
+      this.awaitingMessagesByBlockRoot.has(root) ||
+      this.unknownBlocksBySlot.getOrDefault(slot).has(root)
+    ) {
       return;
     }
     // Search for the unknown block
-    this.unknownRootsBySlot.getOrDefault(slot).add(root);
+    this.unknownBlocksBySlot.getOrDefault(slot).add(root);
     this.chain.emitter.emit(ChainEvent.unknownBlockRoot, {rootHex: root, peer, source});
   }
 
-  private onPendingGossipsubMessage(message: PendingGossipsubMessage): void {
+  /**
+   * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` event
+   * Slot is the message slot, which is not necessarily the same as the envelope's slot, but it can be used for a good prune strategy.
+   * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+   */
+  searchUnknownEnvelope({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
+    if (
+      this.chain.seenPayloadEnvelope(root) ||
+      this.awaitingMessagesByPayloadBlockRoot.has(root) ||
+      this.unknownEnvelopesBySlot.getOrDefault(slot).has(root)
+    ) {
+      return;
+    }
+    this.unknownEnvelopesBySlot.getOrDefault(slot).add(root);
+    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, peer, source});
+  }
+
+  private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
     const topicType = message.topic.type;
     const extractBlockSlotRootFn = this.extractBlockSlotRootFns[topicType];
-    // check block root of Attestation and SignedAggregateAndProof messages
-    if (extractBlockSlotRootFn) {
-      const slotRoot = extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork);
-      // if slotRoot is null, it means the msg.data is invalid
-      // in that case message will be rejected when deserializing data in later phase (gossipValidatorFn)
-      if (slotRoot) {
-        // DOS protection: avoid processing messages that are too old
-        const {slot, root} = slotRoot;
-        const clockSlot = this.chain.clock.currentSlot;
-        const {fork} = message.topic.boundary;
-        let earliestPermissableSlot = clockSlot - DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE;
-        if (ForkSeq[fork] >= ForkSeq.deneb && topicType === GossipType.beacon_attestation) {
-          // post deneb, the attestations could be in current or previous epoch
-          earliestPermissableSlot = computeStartSlotAtEpoch(this.chain.clock.currentEpoch - 1);
+
+    // 1st extract round: make sure slot is in range and if block root is not available
+    // proactively search for it + queue the message
+    const slotRoot = extractBlockSlotRootFn
+      ? extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork)
+      : null;
+    if (slotRoot === null) {
+      // some messages don't have slot and root
+      // if the msg.data is invalid, message will be rejected when deserializing data in later phase (gossipValidatorFn)
+      this.pushPendingGossipsubMessageToQueue(message);
+      return;
+    }
+
+    // common check for all topics
+    // DOS protection: avoid processing messages that are too old
+    const {slot, root} = slotRoot;
+    const clockSlot = this.chain.clock.currentSlot;
+    const {fork} = message.topic.boundary;
+    let earliestPermissableSlot = clockSlot - DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE;
+    if (ForkSeq[fork] >= ForkSeq.deneb && topicType === GossipType.beacon_attestation) {
+      // post deneb, the attestations could be in current or previous epoch
+      earliestPermissableSlot = computeStartSlotAtEpoch(this.chain.clock.currentEpoch - 1);
+    }
+    if (slot < earliestPermissableSlot) {
+      // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+      this.metrics?.networkProcessor.gossipValidationError.inc({
+        topic: topicType,
+        error: GossipErrorCode.PAST_SLOT,
+      });
+      return;
+    }
+
+    message.msgSlot = slot;
+
+    // this determines whether this message needs to wait for a Block or Envelope
+    // a message should only wait for what it voted for, hence we don't want to put it on both queues
+    let preprocessResult: PreprocessResult = {action: PreprocessAction.PushToQueue};
+    // no need to check if root is a descendant of the current finalized block, it will be checked once we validate the message if needed
+    if (root && !this.chain.forkChoice.hasBlockHexUnsafe(root)) {
+      // starting from GLOAS, unknown root from data_column_sidecar also falls into this case
+      this.searchUnknownBlock({slot, root}, BlockInputSource.network_processor, message.propagationSource.toString());
+      // for beacon_attestation and beacon_aggregate_and_proof messages, this is only temporary.
+      // if "index = 1" we need to await for the Envelope instead
+      preprocessResult = {action: PreprocessAction.AwaitBlock, root};
+    }
+
+    // 2nd extract round for some specific topics
+    // we separate the search action from the await action
+
+    // beacon_block: proactively search for parent block/envelope across all forks, but never queue.
+    // BlockInputSync handles cascading recovery if the gossip handler throws.
+    if (topicType === GossipType.beacon_block) {
+      const parentRoot = getParentRootFromSignedBeaconBlockSerialized(message.msg.data);
+      if (parentRoot) {
+        if (ForkSeq[fork] >= ForkSeq.gloas) {
+          // GLOAS: also check parent envelope, same logic as execution_payload_bid
+          const parentBlockHash = getParentBlockHashFromGloasSignedBeaconBlockSerialized(message.msg.data);
+          if (parentBlockHash && !this.chain.forkChoice.getBlockHexAndBlockHash(parentRoot, parentBlockHash)) {
+            const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
+            if (protoBlock === null) {
+              this.searchUnknownBlock(
+                {slot, root: parentRoot},
+                BlockInputSource.network_processor,
+                message.propagationSource.toString()
+              );
+            } else if (
+              protoBlock.executionPayloadBlockHash &&
+              protoBlock.executionPayloadBlockHash !== parentBlockHash
+            ) {
+              // only search for the envelope by block root if we're sure there is one. Otherwise UnknownBlockSync will penalize the peer.
+              this.searchUnknownEnvelope(
+                {slot, root: parentRoot},
+                BlockInputSource.network_processor,
+                message.propagationSource.toString()
+              );
+            }
+          }
+        } else if (!this.chain.forkChoice.hasBlockHexUnsafe(parentRoot)) {
+          this.searchUnknownBlock(
+            {slot, root: parentRoot},
+            BlockInputSource.network_processor,
+            message.propagationSource.toString()
+          );
+        }
+      }
+      preprocessResult = {action: PreprocessAction.PushToQueue};
+    }
+
+    if (ForkSeq[fork] >= ForkSeq.gloas) {
+      // specific check for each topic
+      // note that it's supposed to NOT queue beacon_block (handled above) and execution_payload because it's not a one-off;
+      // for those topics, gossip handlers will throw and BlockInputSync will handle a tree of them instead
+      switch (topicType) {
+        case GossipType.beacon_attestation:
+        case GossipType.beacon_aggregate_and_proof: {
+          if (root == null) break;
+          const attIndex =
+            topicType === GossipType.beacon_attestation
+              ? getDataIndexFromSingleAttestationSerialized(fork, message.msg.data)
+              : getDataIndexFromSignedAggregateAndProofSerialized(message.msg.data);
+          if (attIndex === 1 && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
+            // attestation votes that the payload is available but it is not yet known
+            this.searchUnknownEnvelope(
+              {slot, root},
+              BlockInputSource.network_processor,
+              message.propagationSource.toString()
+            );
+            preprocessResult = {action: PreprocessAction.AwaitEnvelope, root};
+          }
+          break;
         }
-        if (slot < earliestPermissableSlot) {
-          // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
-          this.metrics?.networkProcessor.gossipValidationError.inc({
+        case GossipType.payload_attestation_message: {
+          if (root == null) break;
+          const payloadPresent = getPayloadPresentFromPayloadAttestationMessageSerialized(message.msg.data);
+          if (payloadPresent && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
+            // payload attestation votes that the payload is available but it is not yet known
+            this.searchUnknownEnvelope(
+              {slot, root},
+              BlockInputSource.network_processor,
+              message.propagationSource.toString()
+            );
+            // do not await the envelope, payload attestation processing only requires that the block is known
+            // also do not reset preprocessResult, we may already await for the block
+          }
+          break;
+        }
+        case GossipType.data_column_sidecar: {
+          if (root == null) break;
+          if (!this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
+            this.searchUnknownEnvelope(
+              {slot, root},
+              BlockInputSource.network_processor,
+              message.propagationSource.toString()
+            );
+            // do not await the envelope, we can do gossip validation
+            // also do not reset preprocessResult, we may already await for the block
+          }
+          break;
+        }
+        case GossipType.execution_payload: {
+          // extractBlockSlotRootFn does not return a root for this topic.
+          // Extract beacon_block_root directly and proactively trigger block sync if missing.
+          // Do NOT await the block — the handler runs immediately; BlockInputSync handles recovery.
+          const blockRoot = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(message.msg.data);
+          if (blockRoot && !this.chain.forkChoice.hasBlockHexUnsafe(blockRoot)) {
+            this.searchUnknownBlock(
+              {slot, root: blockRoot},
+              BlockInputSource.network_processor,
+              message.propagationSource.toString()
+            );
+          }
+          // do not await the block, we want UnknownBlockSync to handle it.
+          preprocessResult = {action: PreprocessAction.PushToQueue};
+          break;
+        }
+        case GossipType.execution_payload_bid: {
+          // instead of searching for the message root, this searches for the parent root
+          const parentBlockRoot = getParentBlockRootFromSignedExecutionPayloadBidSerialized(message.msg.data);
+          const parentBlockHash = getParentBlockHashFromSignedExecutionPayloadBidSerialized(message.msg.data);
+          if (
+            parentBlockRoot &&
+            parentBlockHash &&
+            !this.chain.forkChoice.getBlockHexAndBlockHash(parentBlockRoot, parentBlockHash)
+          ) {
+            const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentBlockRoot);
+            if (protoBlock === null) {
+              this.searchUnknownBlock(
+                {slot, root: parentBlockRoot},
+                BlockInputSource.network_processor,
+                message.propagationSource.toString()
+              );
+              preprocessResult = {action: PreprocessAction.AwaitBlock, root: parentBlockRoot};
+            } else if (
+              protoBlock.executionPayloadBlockHash &&
+              protoBlock.executionPayloadBlockHash !== parentBlockHash
+            ) {
+              this.searchUnknownEnvelope(
+                {slot, root: parentBlockRoot},
+                BlockInputSource.network_processor,
+                message.propagationSource.toString()
+              );
+              preprocessResult = {action: PreprocessAction.AwaitEnvelope, root: parentBlockRoot};
+            }
+          }
+          break;
+        }
+      }
+    }
+
+    switch (preprocessResult.action) {
+      case PreprocessAction.PushToQueue:
+        this.pushPendingGossipsubMessageToQueue(message);
+        break;
+      case PreprocessAction.AwaitBlock: {
+        if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
+          // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+          this.metrics?.awaitingBlockGossipMessages.reject.inc({
+            reason: ReprocessRejectReason.reached_limit,
             topic: topicType,
-            error: GossipErrorCode.PAST_SLOT,
           });
           return;
         }
-        message.msgSlot = slot;
-        // check if we processed a block with this root
-        // no need to check if root is a descendant of the current finalized block, it will be checked once we validate the message if needed
-        if (root && !this.chain.forkChoice.hasBlockHexUnsafe(root)) {
-          this.searchUnknownSlotRoot({slot, root}, BlockInputSource.gossip, message.propagationSource.toString());
-
-          if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
-            // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
-            this.metrics?.reprocessGossipAttestations.reject.inc({reason: ReprocessRejectReason.reached_limit});
-            return;
-          }
 
-          this.metrics?.reprocessGossipAttestations.total.inc();
-          const awaitingGossipsubMessagesByRoot = this.awaitingGossipsubMessagesByRootBySlot.getOrDefault(slot);
-          const awaitingGossipsubMessages = awaitingGossipsubMessagesByRoot.getOrDefault(root);
-          awaitingGossipsubMessages.add(message);
-          this.unknownBlockGossipsubMessagesCount++;
+        this.metrics?.awaitingBlockGossipMessages.queue.inc({topic: topicType});
+        const awaitingGossipsubMessages = this.awaitingMessagesByBlockRoot.getOrDefault(preprocessResult.root);
+        awaitingGossipsubMessages.add(message);
+        break;
+      }
+      case PreprocessAction.AwaitEnvelope: {
+        if (this.unknownPayloadGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
+          this.metrics?.awaitingPayloadGossipMessages.reject.inc({
+            reason: ReprocessRejectReason.reached_limit,
+            topic: topicType,
+          });
           return;
         }
+
+        this.metrics?.awaitingPayloadGossipMessages.queue.inc({topic: topicType});
+        const awaitingPayloadGossipsubMessages = this.awaitingMessagesByPayloadBlockRoot.getOrDefault(
+          preprocessResult.root
+        );
+        awaitingPayloadGossipsubMessages.add(message);
+        break;
       }
     }
-
-    // bypass the check for other messages
-    this.pushPendingGossipsubMessageToQueue(message);
-  }
+  };
 
   private pushPendingGossipsubMessageToQueue(message: PendingGossipsubMessage): void {
     const topicType = message.topic.type;
     const droppedCount = this.gossipQueues[topicType].add(message);
     if (droppedCount) {
-      // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
+      // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
       this.metrics?.gossipValidationQueue.droppedJobs.inc({topic: message.topic.type}, droppedCount);
     }
 
@@ -306,59 +541,111 @@ export class NetworkProcessor {
     this.executeWork();
   }
 
-  private async onBlockProcessed({
-    slot,
-    block: rootHex,
-  }: {
-    slot: Slot;
-    block: string;
-    executionOptimistic: boolean;
-  }): Promise<void> {
-    const byRootGossipsubMessages = this.awaitingGossipsubMessagesByRootBySlot.getOrDefault(slot);
-    const waitingGossipsubMessages = byRootGossipsubMessages.getOrDefault(rootHex);
-    if (waitingGossipsubMessages.size === 0) {
+  private onBlockProcessed = async ({block: rootHex}: {block: string; executionOptimistic: boolean}): Promise<void> => {
+    const waitingGossipsubMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
+    if (!waitingGossipsubMessages || waitingGossipsubMessages.size === 0) {
       return;
     }
 
-    this.metrics?.reprocessGossipAttestations.resolve.inc(waitingGossipsubMessages.size);
     const nowSec = Date.now() / 1000;
     let count = 0;
     // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
     // cache, it may not be necessary at this time
     for (const message of waitingGossipsubMessages) {
-      this.metrics?.reprocessGossipAttestations.waitSecBeforeResolve.set(nowSec - message.seenTimestampSec);
+      const topicType = message.topic.type;
+      this.metrics?.awaitingBlockGossipMessages.waitSecBeforeResolve.set(
+        {topic: topicType},
+        nowSec - message.seenTimestampSec
+      );
+      this.metrics?.awaitingBlockGossipMessages.resolve.inc({topic: topicType});
       this.pushPendingGossipsubMessageToQueue(message);
       count++;
       // don't want to block the event loop, worse case it'd wait for 16_084 / 1024 * 50ms = 800ms which is not a big deal
-      if (count === MAX_UNKNOWN_BLOCK_GOSSIP_OBJECTS_PER_TICK) {
+      if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
         count = 0;
-        await sleep(PROCESS_UNKNOWN_BLOCK_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+        await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
 
-    byRootGossipsubMessages.delete(rootHex);
-  }
+    this.awaitingMessagesByBlockRoot.delete(rootHex);
+  };
+
+  private onPayloadEnvelopeProcessed = async ({blockRoot: rootHex}: {blockRoot: RootHex}): Promise<void> => {
+    const waitingGossipsubMessages = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
+    if (!waitingGossipsubMessages || waitingGossipsubMessages.size === 0) {
+      return;
+    }
+
+    const nowSec = Date.now() / 1000;
+    let count = 0;
+    for (const message of waitingGossipsubMessages) {
+      const topicType = message.topic.type;
+      this.metrics?.awaitingPayloadGossipMessages.waitSecBeforeResolve.set(
+        {topic: topicType},
+        nowSec - message.seenTimestampSec
+      );
+      this.metrics?.awaitingPayloadGossipMessages.resolve.inc({topic: topicType});
+      this.pushPendingGossipsubMessageToQueue(message);
+      count++;
+      if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
+        count = 0;
+        await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+      }
+    }
+
+    this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
+  };
 
-  private onClockSlot(clockSlot: Slot): void {
+  private onClockSlot = (clockSlot: Slot): void => {
     const nowSec = Date.now() / 1000;
-    for (const [slot, gossipMessagesByRoot] of this.awaitingGossipsubMessagesByRootBySlot.entries()) {
-      if (slot < clockSlot) {
-        for (const gossipMessages of gossipMessagesByRoot.values()) {
+    const minSlot = clockSlot - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE;
+
+    for (const [slot, roots] of this.unknownBlocksBySlot) {
+      if (slot > minSlot) continue;
+      for (const rootHex of roots) {
+        const gossipMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
+        if (gossipMessages !== undefined) {
           for (const message of gossipMessages) {
-            this.metrics?.reprocessGossipAttestations.reject.inc({reason: ReprocessRejectReason.expired});
-            this.metrics?.reprocessGossipAttestations.waitSecBeforeReject.set(
-              {reason: ReprocessRejectReason.expired},
+            const topicType = message.topic.type;
+            this.metrics?.awaitingBlockGossipMessages.reject.inc({
+              topic: topicType,
+              reason: ReprocessRejectReason.expired,
+            });
+            this.metrics?.awaitingBlockGossipMessages.waitSecBeforeReject.set(
+              {topic: topicType, reason: ReprocessRejectReason.expired},
               nowSec - message.seenTimestampSec
             );
-            // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
+            // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
+          this.awaitingMessagesByBlockRoot.delete(rootHex);
         }
-        this.awaitingGossipsubMessagesByRootBySlot.delete(slot);
       }
+      this.unknownBlocksBySlot.delete(slot);
     }
-    pruneSetToMax(this.unknownRootsBySlot, MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE);
-    this.unknownBlockGossipsubMessagesCount = 0;
-  }
+
+    for (const [slot, roots] of this.unknownEnvelopesBySlot) {
+      if (slot > minSlot) continue;
+      for (const rootHex of roots) {
+        const gossipMessages = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
+        if (gossipMessages !== undefined) {
+          for (const message of gossipMessages) {
+            const topicType = message.topic.type;
+            this.metrics?.awaitingPayloadGossipMessages.reject.inc({
+              topic: topicType,
+              reason: ReprocessRejectReason.expired,
+            });
+            this.metrics?.awaitingPayloadGossipMessages.waitSecBeforeReject.set(
+              {topic: topicType, reason: ReprocessRejectReason.expired},
+              nowSec - message.seenTimestampSec
+            );
+            // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+          }
+          this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
+        }
+      }
+      this.unknownEnvelopesBySlot.delete(slot);
+    }
+  };
 
   private executeWork(): void {
     // TODO: Maybe de-bounce by timing the last time executeWork was run
@@ -496,4 +783,20 @@ export class NetworkProcessor {
 
     return null;
   }
+
+  private get unknownBlockGossipsubMessagesCount(): number {
+    let count = 0;
+    for (const messages of this.awaitingMessagesByBlockRoot.values()) {
+      count += messages.size;
+    }
+    return count;
+  }
+
+  private get unknownPayloadGossipsubMessagesCount(): number {
+    let count = 0;
+    for (const messages of this.awaitingMessagesByPayloadBlockRoot.values()) {
+      count += messages.size;
+    }
+    return count;
+  }
 }

package/src/network/reqresp/ReqRespBeaconNode.ts

@@ -297,6 +297,19 @@ export class ReqRespBeaconNode extends ReqResp {
       );
     }
 
+    if (ForkSeq[fork] >= ForkSeq.gloas) {
+      protocolsAtFork.push(
+        [
+          protocols.ExecutionPayloadEnvelopesByRoot(fork, this.config),
+          this.getHandler(ReqRespMethod.ExecutionPayloadEnvelopesByRoot),
+        ],
+        [
+          protocols.ExecutionPayloadEnvelopesByRange(fork, this.config),
+          this.getHandler(ReqRespMethod.ExecutionPayloadEnvelopesByRange),
+        ]
+      );
+    }
+
     return protocolsAtFork;
   }