@lodestar/beacon-node 1.50.0-dev.5866a397b6 → 1.50.0-dev.58d40975f9

package/src/network/processor/gossipHandlers.ts

@@ -12,9 +12,10 @@ import {
   isForkPostElectra,
   isForkPostGloas,
 } from "@lodestar/params";
-import {computeTimeAtSlot} from "@lodestar/state-transition";
+import {computeTimeAtSlot, signedBlockToSignedHeader} from "@lodestar/state-transition";
 import {
   Root,
+  RootHex,
   SignedBeaconBlock,
   SingleAttestation,
   Slot,
@@ -78,6 +79,7 @@ import {
   validateGossipSyncCommittee,
   validateGossipVoluntaryExit,
   validateSyncCommitteeGossipContributionAndProof,
+  verifyBlockProposerSignature,
 } from "../../chain/validation/index.js";
 import {validateLightClientFinalityUpdate} from "../../chain/validation/lightClientFinalityUpdate.js";
 import {validateLightClientOptimisticUpdate} from "../../chain/validation/lightClientOptimisticUpdate.js";
@@ -148,37 +150,16 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
 function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): SequentialGossipHandlers {
   const {chain, config, metrics, logger, core} = modules;
 
-  async function validateBeaconBlock(
+  /**
+   * Add a gossip block to the seen caches. The proposer signature of the block MUST be verified.
+   */
+  function cacheSignatureVerifiedBlock(
     signedBlock: SignedBeaconBlock,
+    blockRootHex: RootHex,
     fork: ForkName,
     peerIdStr: string,
     seenTimestampSec: number
-  ): Promise<IBlockInput> {
-    const slot = signedBlock.message.slot;
-    const forkTypes = config.getForkTypes(slot);
-    const blockRootHex = toRootHex(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message));
-    const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
-    const recvToValLatency = Date.now() / 1000 - seenTimestampSec;
-
-    // always set block to seen cache for all forks so that we don't need to download it
-    // TODO: validate block before adding to cache
-    // tracked in https://github.com/ChainSafe/lodestar/issues/7957
-
-    const logCtx = {
-      slot,
-      root: blockRootHex,
-      currentSlot: chain.clock.currentSlot,
-      peerId: peerIdStr,
-      delaySec,
-      recvToValLatency,
-    };
-
-    logger.debug("Received gossip block", {...logCtx});
-
-    // optimistically add gossip block to the seen cache
-    // if validation fails, we will NOT forward this gossip block to peers
-    //   - if PARENT_BLOCK_UNKNOWN error, blockInput will then be queued inside BlockInputSync. If the gossip block is really invalid, it will be pruned there
-    //   - if other validator errors, blockInput will stay in the seen cache and will be pruned on finalization
+  ): BlockInput {
     const blockInput = chain.seenBlockInputCache.getByBlock({
       block: signedBlock,
       blockRootHex,
@@ -187,8 +168,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       peerIdStr,
     });
 
-    // Optimistically seed the payload-envelope cache too, mirroring seenBlockInputCache above.
-    // This ensures we have PayloadEnvelopeInput, even through "PARENT_BLOCK_UNKNOWN" error
+    // Also seed the payload-envelope cache, so we have PayloadEnvelopeInput even through "PARENT_BLOCK_UNKNOWN" error
     // see https://github.com/ChainSafe/lodestar/issues/9475
     if (isForkPostGloas(fork)) {
       chain.seenPayloadEnvelopeInputCache.add({
@@ -202,8 +182,37 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       });
     }
 
+    return blockInput;
+  }
+
+  async function validateBeaconBlock(
+    signedBlock: SignedBeaconBlock,
+    fork: ForkName,
+    peerIdStr: string,
+    seenTimestampSec: number
+  ): Promise<IBlockInput> {
+    const slot = signedBlock.message.slot;
+    const forkTypes = config.getForkTypes(slot);
+    const blockRootHex = toRootHex(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message));
+    const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
+    const recvToValLatency = Date.now() / 1000 - seenTimestampSec;
+
+    const logCtx = {
+      slot,
+      root: blockRootHex,
+      currentSlot: chain.clock.currentSlot,
+      peerId: peerIdStr,
+      delaySec,
+      recvToValLatency,
+    };
+
+    logger.debug("Received gossip block", {...logCtx});
+
+    chain.seenBlockInputCache.markValidatingBlock(blockRootHex);
+
     try {
       const {skippedSlots} = await validateGossipBlock(config, chain, signedBlock, fork);
+      const blockInput = cacheSignatureVerifiedBlock(signedBlock, blockRootHex, fork, peerIdStr, seenTimestampSec);
 
       const blockInputMeta = blockInput.getLogMeta();
 
@@ -229,54 +238,54 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       if (e instanceof BlockGossipError) {
         logger.debug("Gossip block has error", {slot, root: blockRootHex, code: e.type.code});
         if (
-          (e.type.code === BlockErrorCode.PARENT_BLOCK_UNKNOWN ||
-            e.type.code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN) &&
-          blockInput
+          e.type.code === BlockErrorCode.PARENT_BLOCK_UNKNOWN ||
+          e.type.code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN
         ) {
+          // The block is retained for unknown parent sync without full validation. Authenticate it first, this
+          // does not need the parent state
+          const proposerIndex = signedBlock.message.proposerIndex;
+          await verifyBlockProposerSignature(chain, signedBlock, blockRootHex);
+          chain.seenBlockProposers.observeBlockRoot(
+            slot,
+            proposerIndex,
+            blockRootHex,
+            signedBlockToSignedHeader(config, signedBlock)
+          );
+
+          // Check again after the signature verification so concurrent proposals cannot both be retained
+          if (chain.seenBlockProposers.isKnown(slot, proposerIndex)) {
+            if (chain.seenBlockProposers.isRepeatProposal(slot, proposerIndex, blockRootHex)) {
+              throw new BlockGossipError(GossipAction.IGNORE, {
+                code: BlockErrorCode.REPEAT_PROPOSAL,
+                proposerIndex,
+                root: blockRootHex,
+              });
+            }
+            throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.ALREADY_KNOWN, root: blockRootHex});
+          }
+          chain.seenBlockProposers.add(slot, proposerIndex, blockRootHex);
+
+          const blockInput = cacheSignatureVerifiedBlock(signedBlock, blockRootHex, fork, peerIdStr, seenTimestampSec);
           chain.emitter.emit(ChainEvent.blockUnknownParent, {
             blockInput,
             peer: peerIdStr,
             source: BlockInputSource.gossip,
           });
-          // throw error (don't prune the blockInput)
           throw e;
         }
 
-        // IGNORE means the block is acceptable (e.g. FUTURE_SLOT, ALREADY_KNOWN), just not propagated.
-        // Keep the optimistically-added cache entries; they are pruned on finalization. Only REJECT
-        // (provably invalid), unexpected errors and repeat proposals that are not imported prune.
-        if (e.action === GossipAction.IGNORE) {
-          // Only a signature-verified sibling is imported by the beacon_block handler, any other repeat proposal
-          // is dropped from the caches and re-downloaded by sync if it ever becomes relevant. Its signature was not
-          // verified, so only its own entry is removed, never its claimed ancestors
-          if (
-            e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
-            !chain.seenBlockProposers.hasBlockRoot(slot, signedBlock.message.proposerIndex, blockRootHex)
-          ) {
-            chain.seenBlockInputCache.remove(blockRootHex);
-            if (isForkPostGloas(fork)) {
-              chain.seenPayloadEnvelopeInputCache.remove(blockRootHex);
-            }
-          }
-          throw e;
+        if (e.action === GossipAction.REJECT) {
+          chain.persistInvalidSszValue(
+            forkTypes.SignedBeaconBlock,
+            signedBlock,
+            `gossip_reject_slot_${slot}`,
+            blockRootHex
+          );
         }
-
-        chain.persistInvalidSszValue(
-          forkTypes.SignedBeaconBlock,
-          signedBlock,
-          `gossip_reject_slot_${slot}`,
-          blockRootHex
-        );
-      }
-
-      // REJECT or unexpected (non-BlockGossipError) error: drop the optimistically-added entries from
-      // both caches, keeping them consistent. The block may carry any parent root, so only its own entry is removed
-      chain.seenBlockInputCache.remove(blockRootHex);
-      if (isForkPostGloas(fork)) {
-        chain.seenPayloadEnvelopeInputCache.remove(blockRootHex);
       }
       throw e;
     } finally {
+      chain.seenBlockInputCache.unmarkValidatingBlock(blockRootHex);
       // The block received from the network may have established an equivocation, either by conflicting
       // with a previously observed block root (REPEAT_PROPOSAL) or with a root observed during validation
       const proposerIndex = signedBlock.message.proposerIndex;
@@ -752,18 +761,23 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
           e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
           // Only a signature-verified sibling recorded in the seen cache is imported. The cache holds at most two
           // roots per proposer and slot, which bounds full imports over gossip to one alternate
-          chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root)
+          chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root) &&
+          // A sibling with an unknown parent cannot be imported, sync fetches it if its branch becomes relevant
+          chain.forkChoice.getBlockHexDefaultStatus(toRootHex(signedBlock.message.parentRoot)) !== null
         ) {
-          // blockInput was optimistically seeded in validateBeaconBlock and retained on IGNORE
-          const blockInput = chain.seenBlockInputCache.get(e.type.root);
-          if (blockInput) {
-            // We're returning IGNORE (thrown below), so this block is not forwarded to peers. Unlike the
-            // happy path there is no rush to forward, so we don't need to wrap in callInNextEventLoop.
-            chain.serializedCache.set(signedBlock, serializedData);
-            // this is technically not a valid gossip block but gossip validation is a cheap subset of checks
-            // this runs the full state transition, so importing an equivocating-but-valid block here is safe.
-            handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
-          }
+          const blockInput = cacheSignatureVerifiedBlock(
+            signedBlock,
+            e.type.root,
+            topic.boundary.fork,
+            peerIdStr,
+            seenTimestampSec
+          );
+          // We're returning IGNORE (thrown below), so this block is not forwarded to peers. Unlike the
+          // happy path there is no rush to forward, so we don't need to wrap in callInNextEventLoop.
+          chain.serializedCache.set(signedBlock, serializedData);
+          // this is technically not a valid gossip block but gossip validation is a cheap subset of checks
+          // this runs the full state transition, so importing an equivocating-but-valid block here is safe.
+          handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
         }
         // rethrow so gossipValidatorFn maps IGNORE -> TopicValidatorResult.Ignore (message not forwarded)
         throw e;
@@ -1065,12 +1079,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       const {serializedData} = gossipData;
       const {fork} = topic.boundary;
       const attesterSlashing = sszDeserialize(topic, serializedData);
-      await validateGossipAttesterSlashing(chain, attesterSlashing);
+      const verifiedDomains = await validateGossipAttesterSlashing(chain, attesterSlashing);
 
       // Handler - deferred to next event loop so the validation result propagates first
       callInNextEventLoop(() => {
         try {
-          chain.opPool.insertAttesterSlashing(fork, attesterSlashing);
+          chain.opPool.insertAttesterSlashing(fork, attesterSlashing, verifiedDomains);
           chain.forkChoice.onAttesterSlashing(attesterSlashing);
           chain.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing);
         } catch (e) {
@@ -1089,12 +1103,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     }: GossipHandlerParamGeneric<GossipType.proposer_slashing>) => {
       const {serializedData} = gossipData;
       const proposerSlashing = sszDeserialize(topic, serializedData);
-      await validateGossipProposerSlashing(chain, proposerSlashing);
+      const verifiedDomain = await validateGossipProposerSlashing(chain, proposerSlashing);
 
       // Handler - deferred to next event loop so the validation result propagates first
       callInNextEventLoop(() => {
         try {
-          chain.opPool.insertProposerSlashing(proposerSlashing);
+          chain.opPool.insertProposerSlashing(proposerSlashing, verifiedDomain);
           chain.emitter.emit(routes.events.EventType.proposerSlashing, proposerSlashing);
         } catch (e) {
           logger.debug(
@@ -1341,6 +1355,9 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
                 case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
                 case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
                   core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
+                  // The builder may have signed another envelope that is valid, keeping this one would make
+                  // by-root and range sync reuse it and never import the payload
+                  chain.seenPayloadEnvelopeInputCache.removeInvalid(payloadInput);
                   // Misbehaving peer, but could highlight an issue in another client
                   logLevel = LogLevel.warn;
                   break;

package/src/network/processor/gossipValidatorFn.ts

@@ -20,6 +20,7 @@ import {
   VoluntaryExitErrorCode,
 } from "../../chain/errors/index.js";
 import {Metrics} from "../../metrics/index.js";
+import {PeerIdStr} from "../../util/peerId.js";
 import {INetworkCore} from "../core/index.js";
 import {
   BatchGossipHandlerFn,
@@ -40,6 +41,8 @@ export type ValidatorFnModules = {
   core: INetworkCore;
 };
 
+export type OnFatalPeerFn = (peer: PeerIdStr) => void;
+
 type RejectPeerActionRule = {default: PeerAction; byCode?: Record<string, PeerAction>};
 
 /**
@@ -112,7 +115,7 @@ const gossipRejectPeerAction: Record<GossipType, RejectPeerActionRule> = {
   [GossipType.light_client_optimistic_update]: {default: PeerAction.HighToleranceError},
 };
 
-function rejectPeerAction(type: GossipType, code: string): PeerAction {
+export function rejectPeerAction(type: GossipType, code: string): PeerAction {
   const rule = gossipRejectPeerAction[type];
   return rule.byCode?.[code] ?? rule.default;
 }
@@ -123,7 +126,8 @@ function rejectPeerAction(type: GossipType, code: string): PeerAction {
  */
 export function getGossipValidatorBatchFn(
   gossipHandlers: GossipHandlers,
-  modules: ValidatorFnModules
+  modules: ValidatorFnModules,
+  onFatalPeer: OnFatalPeerFn
 ): GossipValidatorBatchFn {
   const {logger, metrics, core} = modules;
 
@@ -173,6 +177,7 @@ export function getGossipValidatorBatchFn(
             metrics?.networkProcessor.gossipAttestationRejectByReason.inc({reason: e.type.code});
             const peerAction = rejectPeerAction(type, e.type.code);
             core.reportPeer(propagationSource, peerAction, e.type.code);
+            if (peerAction === PeerAction.Fatal) onFatalPeer(propagationSource);
             logger.debug(
               `Gossip validation ${type} rejected`,
               {peer: propagationSource, clientAgent, clientVersion, peerAction},
@@ -208,7 +213,11 @@ export function getGossipValidatorBatchFn(
  *
  * @see getGossipHandlers for reasoning on why GossipHandlerFn are used for gossip validation.
  */
-export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: ValidatorFnModules): GossipValidatorFn {
+export function getGossipValidatorFn(
+  gossipHandlers: GossipHandlers,
+  modules: ValidatorFnModules,
+  onFatalPeer: OnFatalPeerFn
+): GossipValidatorFn {
   const {logger, metrics, core} = modules;
 
   return async function gossipValidatorFn({
@@ -260,6 +269,7 @@ export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: Va
           metrics?.networkProcessor.gossipValidationReject.inc({topic: type});
           const peerAction = rejectPeerAction(type, e.type.code);
           core.reportPeer(propagationSource, peerAction, e.type.code);
+          if (peerAction === PeerAction.Fatal) onFatalPeer(propagationSource);
           logger.debug(
             `Gossip validation ${type} rejected`,
             {peer: propagationSource, clientAgent, clientVersion, peerAction},

package/src/network/processor/index.ts

@@ -22,6 +22,7 @@ import {
   getParentRootFromSignedBeaconBlockSerialized,
   getPayloadPresentFromPayloadAttestationMessageSerialized,
 } from "../../util/sszBytes.js";
+import {INetworkCore} from "../core/index.js";
 import {NetworkEvent, NetworkEventBus} from "../events.js";
 import {
   GossipHandlers,
@@ -34,7 +35,12 @@ import {MAX_PEERS_PER_ROOT} from "./constants.js";
 import {createExtractBlockSlotRootFns} from "./extractSlotRootFns.js";
 import {GossipHandlerOpts, ValidatorFnsModules, getGossipHandlers} from "./gossipHandlers.js";
 import {createGossipQueues} from "./gossipQueues/index.js";
-import {ValidatorFnModules, getGossipValidatorBatchFn, getGossipValidatorFn} from "./gossipValidatorFn.js";
+import {
+  ValidatorFnModules,
+  getGossipValidatorBatchFn,
+  getGossipValidatorFn,
+  rejectPeerAction,
+} from "./gossipValidatorFn.js";
 import {PendingGossipsubMessage} from "./types.js";
 
 export * from "./types.js";
@@ -102,6 +108,12 @@ type SearchedRootEntry = {
  */
 const DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE = 32;
 
+/**
+ * The network worker bans a Fatal peer only on its next PeerManager heartbeat (30s). Until then the main thread
+ * would keep validating that peer's messages, so drop them here. 3 slots (36s) outlasts one heartbeat.
+ */
+export const FATAL_PEER_EXPIRY_SLOTS = 3;
+
 type WorkOpts = {
   bypassQueue?: boolean;
 };
@@ -232,6 +244,7 @@ type SearchUnknownRootTarget =
  */
 export class NetworkProcessor {
   private readonly chain: IBeaconChain;
+  private readonly core: INetworkCore;
   private readonly events: NetworkEventBus;
   private readonly logger: Logger;
   private readonly metrics: Metrics | null;
@@ -256,23 +269,31 @@ export class NetworkProcessor {
     () => new MapDef<RootHex, SearchedRootEntry>(() => ({}))
   );
   private bufferedRootsBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
+  /** peer -> clock slot when it was Fatal'ed */
+  private readonly fatalPeers = new Map<PeerIdStr, Slot>();
 
   constructor(
     modules: NetworkProcessorModules,
     private readonly opts: NetworkProcessorOpts
   ) {
-    const {chain, events, logger, metrics} = modules;
+    const {chain, core, events, logger, metrics} = modules;
     this.chain = chain;
+    this.core = core;
     this.events = events;
     this.metrics = metrics;
     this.logger = logger;
     this.events = events;
     this.gossipQueues = createGossipQueues();
     this.gossipTopicConcurrency = mapValues(this.gossipQueues, () => 0);
-    this.gossipValidatorFn = getGossipValidatorFn(modules.gossipHandlers ?? getGossipHandlers(modules, opts), modules);
+    this.gossipValidatorFn = getGossipValidatorFn(
+      modules.gossipHandlers ?? getGossipHandlers(modules, opts),
+      modules,
+      this.onFatalPeer
+    );
     this.gossipValidatorBatchFn = getGossipValidatorBatchFn(
       modules.gossipHandlers ?? getGossipHandlers(modules, opts),
-      modules
+      modules,
+      this.onFatalPeer
     );
 
     events.on(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
@@ -406,19 +427,47 @@ export class NetworkProcessor {
     }
   }
 
+  private onFatalPeer = (peer: PeerIdStr): void => {
+    this.fatalPeers.set(peer, this.chain.clock.currentSlot);
+  };
+
   private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
     const topicType = message.topic.type;
+    if (this.fatalPeers.has(message.propagationSource)) {
+      this.metrics?.networkProcessor.gossipValidationError.inc({topic: topicType, error: GossipErrorCode.FATAL_PEER});
+      // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+      return;
+    }
+
     const extractBlockSlotRootFn = this.extractBlockSlotRootFns[topicType];
+    if (extractBlockSlotRootFn === undefined) {
+      // some messages don't have slot and root
+      this.pushPendingGossipsubMessageToQueue(message);
+      return;
+    }
 
     // 1st extract round: make sure slot is in range and if block root is not available
     // proactively search for it + queue the message
-    const slotRoot = extractBlockSlotRootFn
-      ? extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork)
-      : null;
+    const slotRoot = extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork);
     if (slotRoot === null) {
-      // some messages don't have slot and root
-      // if the msg.data is invalid, message will be rejected when deserializing data in later phase (gossipValidatorFn)
-      this.pushPendingGossipsubMessageToQueue(message);
+      // DOS protection: null means malformed bytes or slot >= 2^32, neither is forwarded by honest peers.
+      const {propagationSource, clientAgent, clientVersion} = message;
+      const code = GossipErrorCode.INVALID_SLOT;
+      const peerAction = rejectPeerAction(topicType, code);
+      this.metrics?.networkProcessor.gossipValidationReject.inc({topic: topicType});
+      this.core.reportPeer(propagationSource, peerAction, code);
+      this.logger.debug(`Gossip validation ${topicType} rejected due to null slot`, {
+        peer: propagationSource,
+        clientAgent,
+        clientVersion,
+        peerAction,
+        code,
+      });
+      this.events.emit(NetworkEvent.gossipMessageValidationResult, {
+        msgId: message.msgId,
+        propagationSource,
+        acceptance: TopicValidatorResult.Reject,
+      });
       return;
     }
 
@@ -872,6 +921,10 @@ export class NetworkProcessor {
     for (const slot of this.bufferedRootsBySlot.keys()) {
       if (slot <= minSlot) this.bufferedRootsBySlot.delete(slot);
     }
+
+    for (const [peer, slot] of this.fatalPeers) {
+      if (clockSlot - slot > FATAL_PEER_EXPIRY_SLOTS) this.fatalPeers.delete(peer);
+    }
   };
 
   private executeWork(): void {
@@ -927,6 +980,32 @@ export class NetworkProcessor {
   private async processPendingGossipsubMessage(
     messageOrArray: PendingGossipsubMessage | PendingGossipsubMessage[]
   ): Promise<void> {
+    // drop messages from peers Fatal'ed after these were queued or started awaiting a block/envelope
+    // No need to report the dropped jobs to gossip. They will be eventually pruned from the mcache
+    if (this.fatalPeers.size > 0) {
+      if (Array.isArray(messageOrArray)) {
+        const allowed: PendingGossipsubMessage[] = [];
+        for (const msg of messageOrArray) {
+          if (this.fatalPeers.has(msg.propagationSource)) {
+            this.metrics?.networkProcessor.gossipValidationError.inc({
+              topic: msg.topic.type,
+              error: GossipErrorCode.FATAL_PEER,
+            });
+          } else {
+            allowed.push(msg);
+          }
+        }
+        if (allowed.length === 0) return;
+        messageOrArray = allowed;
+      } else if (this.fatalPeers.has(messageOrArray.propagationSource)) {
+        this.metrics?.networkProcessor.gossipValidationError.inc({
+          topic: messageOrArray.topic.type,
+          error: GossipErrorCode.FATAL_PEER,
+        });
+        return;
+      }
+    }
+
     const nowSec = Date.now() / 1000;
     if (Array.isArray(messageOrArray)) {
       for (const msg of messageOrArray) {

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts

@@ -6,6 +6,7 @@ import {toRootHex} from "@lodestar/utils";
 import {EnvelopeReconstructionError} from "../../../chain/errors/index.js";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
+import {MAX_BODIES_PER_REQUEST} from "../../../util/execution.js";
 import {ExecutionPayloadEnvelopesByRootRequest} from "../../../util/types.js";
 import {prettyPrintPeerId} from "../../util.js";
 
@@ -19,10 +20,15 @@ export async function* onExecutionPayloadEnvelopesByRoot(
   // The gloas req/resp spec uses MIN_EPOCHS_FOR_BLOCK_REQUESTS to define the minimum range peers MUST serve.
   // Archival nodes may still serve older retained payloads to allow genesis sync.
 
-  // Resolve slots first so archived header envelopes are rebuilt in one EL batch, not one call per root
+  // Resolve slots first so archived envelopes can be reconstructed in EL batches.
+  // Duplicate roots are served once, since each archived envelope costs an EL fetch and a rebuild.
   const requests: {blockSlot: Slot; blockRootHex: RootHex}[] = [];
+  const seenRoots = new Set<RootHex>();
   for (const root of requestBody) {
     const rootHex = toRootHex(root);
+    if (seenRoots.has(rootHex)) continue;
+    seenRoots.add(rootHex);
+
     const block = chain.forkChoice.getBlockHexDefaultStatus(rootHex);
     // If the block is not in fork choice, it may be finalized. Attempt to find its slot in block archive
     const slot = block ? block.slot : await db.blockArchive.getSlotByRoot(root);
@@ -41,35 +47,43 @@ export async function* onExecutionPayloadEnvelopesByRoot(
     requests.push({blockSlot: slot, blockRootHex: rootHex});
   }
 
-  let envelopesBytes: (Uint8Array | null)[];
-  try {
-    // by-root allows omission, so a mismatched envelope is left out rather than failing the response
-    envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(requests, "omit");
-  } catch (e) {
-    if (e instanceof EnvelopeReconstructionError) {
-      throw new ResponseError(
-        RespStatus.RESOURCE_UNAVAILABLE,
-        `Failed to reconstruct archived envelopes: ${e.message}`
-      );
+  // Yield between EL batches so the first response does not wait for the entire request.
+  let yielded = 0;
+  for (let i = 0; i < requests.length; i += MAX_BODIES_PER_REQUEST) {
+    const batch = requests.slice(i, i + MAX_BODIES_PER_REQUEST);
+    let envelopesBytes: (Uint8Array | null)[];
+    try {
+      // By-root allows omission, so an envelope that cannot be rebuilt is left out rather than failing the response.
+      envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(batch, "omit");
+    } catch (e) {
+      if (e instanceof EnvelopeReconstructionError) {
+        // By-root permits partial responses, so an EL failure can end the response here.
+        if (yielded > 0) return;
+        throw new ResponseError(
+          RespStatus.RESOURCE_UNAVAILABLE,
+          `Failed to reconstruct archived envelopes: ${e.message}`
+        );
+      }
+      throw e;
     }
-    throw e;
-  }
 
-  for (let i = 0; i < requests.length; i++) {
-    const {blockSlot, blockRootHex} = requests[i];
-    const envelopeBytes = envelopesBytes[i];
-    if (envelopeBytes) {
-      yield {
-        data: envelopeBytes,
-        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
-      };
-    } else {
-      chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
-        slot: blockSlot,
-        root: blockRootHex,
-        peer: prettyPrintPeerId(peerId),
-        client: peerClient,
-      });
+    for (let j = 0; j < batch.length; j++) {
+      const {blockSlot, blockRootHex} = batch[j];
+      const envelopeBytes = envelopesBytes[j];
+      if (envelopeBytes) {
+        yielded++;
+        yield {
+          data: envelopeBytes,
+          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
+        };
+      } else {
+        chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
+          slot: blockSlot,
+          root: blockRootHex,
+          peer: prettyPrintPeerId(peerId),
+          client: peerClient,
+        });
+      }
     }
   }
 }