@lodestar/beacon-node 1.50.0-dev.ece0239bd3 → 1.50.0-dev.f0b11f836e

package/src/chain/errors/envelopeReconstructionError.ts

@@ -4,6 +4,8 @@ import {LodestarError} from "@lodestar/utils";
 export enum EnvelopeReconstructionErrorCode {
   /** The EL failed to answer engine_getPayloadBodiesByHashV2 (transport / EL down). Transient. */
   ENGINE_UNAVAILABLE = "ENVELOPE_RECONSTRUCTION_ERROR_ENGINE_UNAVAILABLE",
+  /** The EL does not have the block or its block access list (pruned, or not yet re-indexed after a restart). May recover. */
+  BODY_UNAVAILABLE = "ENVELOPE_RECONSTRUCTION_ERROR_BODY_UNAVAILABLE",
   /** A body returned by the EL does not hash to the root stored in the header envelope. Local DB/EL inconsistency. */
   BODY_ROOT_MISMATCH = "ENVELOPE_RECONSTRUCTION_ERROR_BODY_ROOT_MISMATCH",
   /** A by-range stream stopped short at a slot that cannot be served; the response so far is still valid */
@@ -12,6 +14,7 @@ export enum EnvelopeReconstructionErrorCode {
 
 export type EnvelopeReconstructionErrorType =
   | {code: EnvelopeReconstructionErrorCode.ENGINE_UNAVAILABLE}
+  | {code: EnvelopeReconstructionErrorCode.BODY_UNAVAILABLE; slot: Slot}
   | {
       code: EnvelopeReconstructionErrorCode.BODY_ROOT_MISMATCH;
       slot: Slot;

package/src/chain/errors/gossipValidation.ts

@@ -9,6 +9,8 @@ export enum GossipErrorCode {
   INVALID_SERIALIZED_BYTES_ERROR_CODE = "GOSSIP_ERROR_INVALID_SERIALIZED_BYTES",
   PAST_SLOT = "GOSSIP_ERROR_PAST_SLOT",
   FUTURE_SLOT = "GOSSIP_ERROR_FUTURE_SLOT",
+  FATAL_PEER = "GOSSIP_ERROR_FATAL_PEER",
+  INVALID_SLOT = "GOSSIP_ERROR_INVALID_SLOT",
 }
 
 export class GossipActionError<T extends {code: string}> extends LodestarError<T> {

package/src/chain/interface.ts

@@ -30,7 +30,7 @@ import {Metrics} from "../metrics/metrics.js";
 import {BufferPool} from "../util/bufferPool.js";
 import {IClock} from "../util/clock.js";
 import {CustodyConfig} from "../util/dataColumns.js";
-import {ReconstructMismatchPolicy} from "../util/execution.js";
+import {ReconstructMissPolicy} from "../util/execution.js";
 import {SerializedCache} from "../util/serializedCache.js";
 import {IArchiveStore} from "./archiveStore/interface.js";
 import {CheckpointBalancesCache} from "./balancesCache.js";
@@ -239,7 +239,7 @@ export interface IBeaconChain {
   getSerializedExecutionPayloadEnvelope(blockSlot: Slot, blockRootHex: string): Promise<Uint8Array | null>;
   getSerializedExecutionPayloadEnvelopes(
     requests: {blockSlot: Slot; blockRootHex: RootHex}[],
-    onMismatch?: ReconstructMismatchPolicy
+    onMiss?: ReconstructMissPolicy
   ): Promise<(Uint8Array | null)[]>;
   getExecutionPayloadEnvelope(
     blockSlot: Slot,

package/src/chain/prepareNextSlot.ts

@@ -244,6 +244,10 @@ export class PrepareNextSlotScheduler {
           parentBlockHash = preparedState.latestExecutionPayloadHeader.blockHash;
         }
 
+        // The payload_attributes event must carry the same hashes we send to the EL
+        const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice, this.logger);
+        const finalizedBlockHash = getFinalizedExecutionBlockHash(this.chain.forkChoice);
+
         let payloadAttributes: PayloadAttributes | undefined;
         // If emitPayloadAttributes is true emit a SSE payloadAttributes event for
         // every slot. Without the flag, only emit the event if we are proposing in the next slot.
@@ -256,6 +260,8 @@ export class PrepareNextSlotScheduler {
             prepareSlot,
             parentBlockRoot: fromHex(updatedHead.blockRoot),
             parentBlockHash,
+            safeBlockHash,
+            finalizedBlockHash,
             feeRecipient: feeRecipient ?? "0x0000000000000000000000000000000000000000",
           });
           this.chain.emitter.emit(routes.events.EventType.payloadAttributes, {data, version: fork});
@@ -267,9 +273,6 @@ export class PrepareNextSlotScheduler {
             computeTimeAtSlot(this.config, prepareSlot, this.chain.genesisTime) - Date.now() / 1000;
           this.metrics?.blockPayload.payloadAdvancePrepTime.observe(preparationTime);
 
-          const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice, this.logger);
-          const finalizedBlockHash = getFinalizedExecutionBlockHash(this.chain.forkChoice);
-
           // awaiting here instead of throwing an async call because there is no other task
           // left for scheduler and this gives nice semantics to catch and log errors in the
           // try/catch wrapper here.

package/src/chain/produceBlock/produceBlockBody.ts

@@ -845,6 +845,8 @@ export function getPayloadAttributesForSSE(
     prepareSlot,
     parentBlockRoot,
     parentBlockHash,
+    safeBlockHash,
+    finalizedBlockHash,
     feeRecipient,
   }: {
     /**
@@ -855,6 +857,8 @@ export function getPayloadAttributesForSSE(
     prepareSlot: Slot;
     parentBlockRoot: Root;
     parentBlockHash: Bytes32;
+    safeBlockHash: RootHex;
+    finalizedBlockHash: RootHex;
     feeRecipient: string;
   }
 ): SSEPayloadAttributes {
@@ -874,7 +878,10 @@ export function getPayloadAttributesForSSE(
     payloadAttributes,
   } as SSEPayloadAttributes;
 
-  if (!isForkPostGloas(fork)) {
+  if (isForkPostGloas(fork)) {
+    (ssePayloadAttributes as gloas.SSEPayloadAttributes).safeBlockHash = fromHex(safeBlockHash);
+    (ssePayloadAttributes as gloas.SSEPayloadAttributes).finalizedBlockHash = fromHex(finalizedBlockHash);
+  } else {
     // Removed in Gloas, builders can get the block number from the EL via the block hash if required
     (ssePayloadAttributes as bellatrix.SSEPayloadAttributes).parentBlockNumber = prepareState.payloadBlockNumber;
   }

package/src/chain/seenCache/seenPayloadEnvelopeInput.ts

@@ -264,6 +264,17 @@ export class SeenPayloadEnvelopeInput {
     }
   }
 
+  /**
+   * Removes a PayloadEnvelopeInput whose envelope failed import with a definitive invalid verdict. The envelope
+   * cannot be replaced in place, sync recreates the entry via `getOrReload()` or `add()` to fetch another one.
+   * No-op if the cached entry is a different object, it may already be the recreated one.
+   */
+  removeInvalid(payloadInput: PayloadEnvelopeInput): void {
+    if (this.payloadInputs.get(payloadInput.blockRootHex) === payloadInput) {
+      this.evictPayloadInput(payloadInput, "invalid");
+    }
+  }
+
   pruneBelowParent(parentBlock: ProtoBlock): void {
     for (const block of this.forkChoice.getAllAncestorBlocks(parentBlock.blockRoot, parentBlock.payloadStatus)) {
       // Only evict once the payload is FULL (revealed/imported) — on an EMPTY/PENDING branch we may

package/src/network/processor/gossipHandlers.ts

@@ -1341,6 +1341,9 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
                 case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
                 case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
                   core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
+                  // The builder may have signed another envelope that is valid, keeping this one would make
+                  // by-root and range sync reuse it and never import the payload
+                  chain.seenPayloadEnvelopeInputCache.removeInvalid(payloadInput);
                   // Misbehaving peer, but could highlight an issue in another client
                   logLevel = LogLevel.warn;
                   break;

package/src/network/processor/gossipValidatorFn.ts

@@ -20,6 +20,7 @@ import {
   VoluntaryExitErrorCode,
 } from "../../chain/errors/index.js";
 import {Metrics} from "../../metrics/index.js";
+import {PeerIdStr} from "../../util/peerId.js";
 import {INetworkCore} from "../core/index.js";
 import {
   BatchGossipHandlerFn,
@@ -40,6 +41,8 @@ export type ValidatorFnModules = {
   core: INetworkCore;
 };
 
+export type OnFatalPeerFn = (peer: PeerIdStr) => void;
+
 type RejectPeerActionRule = {default: PeerAction; byCode?: Record<string, PeerAction>};
 
 /**
@@ -112,7 +115,7 @@ const gossipRejectPeerAction: Record<GossipType, RejectPeerActionRule> = {
   [GossipType.light_client_optimistic_update]: {default: PeerAction.HighToleranceError},
 };
 
-function rejectPeerAction(type: GossipType, code: string): PeerAction {
+export function rejectPeerAction(type: GossipType, code: string): PeerAction {
   const rule = gossipRejectPeerAction[type];
   return rule.byCode?.[code] ?? rule.default;
 }
@@ -123,7 +126,8 @@ function rejectPeerAction(type: GossipType, code: string): PeerAction {
  */
 export function getGossipValidatorBatchFn(
   gossipHandlers: GossipHandlers,
-  modules: ValidatorFnModules
+  modules: ValidatorFnModules,
+  onFatalPeer: OnFatalPeerFn
 ): GossipValidatorBatchFn {
   const {logger, metrics, core} = modules;
 
@@ -173,6 +177,7 @@ export function getGossipValidatorBatchFn(
             metrics?.networkProcessor.gossipAttestationRejectByReason.inc({reason: e.type.code});
             const peerAction = rejectPeerAction(type, e.type.code);
             core.reportPeer(propagationSource, peerAction, e.type.code);
+            if (peerAction === PeerAction.Fatal) onFatalPeer(propagationSource);
             logger.debug(
               `Gossip validation ${type} rejected`,
               {peer: propagationSource, clientAgent, clientVersion, peerAction},
@@ -208,7 +213,11 @@ export function getGossipValidatorBatchFn(
  *
  * @see getGossipHandlers for reasoning on why GossipHandlerFn are used for gossip validation.
  */
-export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: ValidatorFnModules): GossipValidatorFn {
+export function getGossipValidatorFn(
+  gossipHandlers: GossipHandlers,
+  modules: ValidatorFnModules,
+  onFatalPeer: OnFatalPeerFn
+): GossipValidatorFn {
   const {logger, metrics, core} = modules;
 
   return async function gossipValidatorFn({
@@ -260,6 +269,7 @@ export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: Va
           metrics?.networkProcessor.gossipValidationReject.inc({topic: type});
           const peerAction = rejectPeerAction(type, e.type.code);
           core.reportPeer(propagationSource, peerAction, e.type.code);
+          if (peerAction === PeerAction.Fatal) onFatalPeer(propagationSource);
           logger.debug(
             `Gossip validation ${type} rejected`,
             {peer: propagationSource, clientAgent, clientVersion, peerAction},

package/src/network/processor/index.ts

@@ -22,6 +22,7 @@ import {
   getParentRootFromSignedBeaconBlockSerialized,
   getPayloadPresentFromPayloadAttestationMessageSerialized,
 } from "../../util/sszBytes.js";
+import {INetworkCore} from "../core/index.js";
 import {NetworkEvent, NetworkEventBus} from "../events.js";
 import {
   GossipHandlers,
@@ -34,7 +35,12 @@ import {MAX_PEERS_PER_ROOT} from "./constants.js";
 import {createExtractBlockSlotRootFns} from "./extractSlotRootFns.js";
 import {GossipHandlerOpts, ValidatorFnsModules, getGossipHandlers} from "./gossipHandlers.js";
 import {createGossipQueues} from "./gossipQueues/index.js";
-import {ValidatorFnModules, getGossipValidatorBatchFn, getGossipValidatorFn} from "./gossipValidatorFn.js";
+import {
+  ValidatorFnModules,
+  getGossipValidatorBatchFn,
+  getGossipValidatorFn,
+  rejectPeerAction,
+} from "./gossipValidatorFn.js";
 import {PendingGossipsubMessage} from "./types.js";
 
 export * from "./types.js";
@@ -102,6 +108,12 @@ type SearchedRootEntry = {
  */
 const DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE = 32;
 
+/**
+ * The network worker bans a Fatal peer only on its next PeerManager heartbeat (30s). Until then the main thread
+ * would keep validating that peer's messages, so drop them here. 3 slots (36s) outlasts one heartbeat.
+ */
+export const FATAL_PEER_EXPIRY_SLOTS = 3;
+
 type WorkOpts = {
   bypassQueue?: boolean;
 };
@@ -232,6 +244,7 @@ type SearchUnknownRootTarget =
  */
 export class NetworkProcessor {
   private readonly chain: IBeaconChain;
+  private readonly core: INetworkCore;
   private readonly events: NetworkEventBus;
   private readonly logger: Logger;
   private readonly metrics: Metrics | null;
@@ -256,23 +269,31 @@ export class NetworkProcessor {
     () => new MapDef<RootHex, SearchedRootEntry>(() => ({}))
   );
   private bufferedRootsBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
+  /** peer -> clock slot when it was Fatal'ed */
+  private readonly fatalPeers = new Map<PeerIdStr, Slot>();
 
   constructor(
     modules: NetworkProcessorModules,
     private readonly opts: NetworkProcessorOpts
   ) {
-    const {chain, events, logger, metrics} = modules;
+    const {chain, core, events, logger, metrics} = modules;
     this.chain = chain;
+    this.core = core;
     this.events = events;
     this.metrics = metrics;
     this.logger = logger;
     this.events = events;
     this.gossipQueues = createGossipQueues();
     this.gossipTopicConcurrency = mapValues(this.gossipQueues, () => 0);
-    this.gossipValidatorFn = getGossipValidatorFn(modules.gossipHandlers ?? getGossipHandlers(modules, opts), modules);
+    this.gossipValidatorFn = getGossipValidatorFn(
+      modules.gossipHandlers ?? getGossipHandlers(modules, opts),
+      modules,
+      this.onFatalPeer
+    );
     this.gossipValidatorBatchFn = getGossipValidatorBatchFn(
       modules.gossipHandlers ?? getGossipHandlers(modules, opts),
-      modules
+      modules,
+      this.onFatalPeer
     );
 
     events.on(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
@@ -406,19 +427,47 @@ export class NetworkProcessor {
     }
   }
 
+  private onFatalPeer = (peer: PeerIdStr): void => {
+    this.fatalPeers.set(peer, this.chain.clock.currentSlot);
+  };
+
   private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
     const topicType = message.topic.type;
+    if (this.fatalPeers.has(message.propagationSource)) {
+      this.metrics?.networkProcessor.gossipValidationError.inc({topic: topicType, error: GossipErrorCode.FATAL_PEER});
+      // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+      return;
+    }
+
     const extractBlockSlotRootFn = this.extractBlockSlotRootFns[topicType];
+    if (extractBlockSlotRootFn === undefined) {
+      // some messages don't have slot and root
+      this.pushPendingGossipsubMessageToQueue(message);
+      return;
+    }
 
     // 1st extract round: make sure slot is in range and if block root is not available
     // proactively search for it + queue the message
-    const slotRoot = extractBlockSlotRootFn
-      ? extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork)
-      : null;
+    const slotRoot = extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork);
     if (slotRoot === null) {
-      // some messages don't have slot and root
-      // if the msg.data is invalid, message will be rejected when deserializing data in later phase (gossipValidatorFn)
-      this.pushPendingGossipsubMessageToQueue(message);
+      // DOS protection: null means malformed bytes or slot >= 2^32, neither is forwarded by honest peers.
+      const {propagationSource, clientAgent, clientVersion} = message;
+      const code = GossipErrorCode.INVALID_SLOT;
+      const peerAction = rejectPeerAction(topicType, code);
+      this.metrics?.networkProcessor.gossipValidationReject.inc({topic: topicType});
+      this.core.reportPeer(propagationSource, peerAction, code);
+      this.logger.debug(`Gossip validation ${topicType} rejected due to null slot`, {
+        peer: propagationSource,
+        clientAgent,
+        clientVersion,
+        peerAction,
+        code,
+      });
+      this.events.emit(NetworkEvent.gossipMessageValidationResult, {
+        msgId: message.msgId,
+        propagationSource,
+        acceptance: TopicValidatorResult.Reject,
+      });
       return;
     }
 
@@ -872,6 +921,10 @@ export class NetworkProcessor {
     for (const slot of this.bufferedRootsBySlot.keys()) {
       if (slot <= minSlot) this.bufferedRootsBySlot.delete(slot);
     }
+
+    for (const [peer, slot] of this.fatalPeers) {
+      if (clockSlot - slot > FATAL_PEER_EXPIRY_SLOTS) this.fatalPeers.delete(peer);
+    }
   };
 
   private executeWork(): void {
@@ -927,6 +980,32 @@ export class NetworkProcessor {
   private async processPendingGossipsubMessage(
     messageOrArray: PendingGossipsubMessage | PendingGossipsubMessage[]
   ): Promise<void> {
+    // drop messages from peers Fatal'ed after these were queued or started awaiting a block/envelope
+    // No need to report the dropped jobs to gossip. They will be eventually pruned from the mcache
+    if (this.fatalPeers.size > 0) {
+      if (Array.isArray(messageOrArray)) {
+        const allowed: PendingGossipsubMessage[] = [];
+        for (const msg of messageOrArray) {
+          if (this.fatalPeers.has(msg.propagationSource)) {
+            this.metrics?.networkProcessor.gossipValidationError.inc({
+              topic: msg.topic.type,
+              error: GossipErrorCode.FATAL_PEER,
+            });
+          } else {
+            allowed.push(msg);
+          }
+        }
+        if (allowed.length === 0) return;
+        messageOrArray = allowed;
+      } else if (this.fatalPeers.has(messageOrArray.propagationSource)) {
+        this.metrics?.networkProcessor.gossipValidationError.inc({
+          topic: messageOrArray.topic.type,
+          error: GossipErrorCode.FATAL_PEER,
+        });
+        return;
+      }
+    }
+
     const nowSec = Date.now() / 1000;
     if (Array.isArray(messageOrArray)) {
       for (const msg of messageOrArray) {

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts

@@ -53,7 +53,7 @@ export async function* onExecutionPayloadEnvelopesByRoot(
     const batch = requests.slice(i, i + MAX_BODIES_PER_REQUEST);
     let envelopesBytes: (Uint8Array | null)[];
     try {
-      // By-root allows omission, so a mismatched envelope is left out rather than failing the response.
+      // By-root allows omission, so an envelope that cannot be rebuilt is left out rather than failing the response.
       envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(batch, "omit");
     } catch (e) {
       if (e instanceof EnvelopeReconstructionError) {

package/src/sync/range/range.ts

@@ -5,6 +5,7 @@ import {IBeaconStateViewGloas, computeStartSlotAtEpoch, isStatePostGloas} from "
 import {Epoch, Status, fulu} from "@lodestar/types";
 import {Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
+import {PayloadError, PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {AttestationImportOpt, ImportBlockOpts} from "../../chain/blocks/index.js";
 import {assertLinearChainSegment} from "../../chain/blocks/utils/chainSegment.js";
 import {BlockError} from "../../chain/errors/index.js";
@@ -207,22 +208,50 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       blsVerifyOnMainThread: false,
     };
 
-    if (this.opts?.disableProcessAsChainSegment) {
-      // Should only be used for debugging or testing
-      for (const block of blocks) {
-        await this.chain.processBlock(block, flags);
-        const payloadEnvelope = payloadEnvelopes?.get(block.slot);
-        if (payloadEnvelope?.hasPayloadEnvelope()) {
-          await this.chain.processExecutionPayload(payloadEnvelope);
+    try {
+      if (this.opts?.disableProcessAsChainSegment) {
+        // Should only be used for debugging or testing
+        for (const block of blocks) {
+          await this.chain.processBlock(block, flags);
+          const payloadEnvelope = payloadEnvelopes?.get(block.slot);
+          if (payloadEnvelope?.hasPayloadEnvelope()) {
+            await this.chain.processExecutionPayload(payloadEnvelope);
+          }
+        }
+      } else {
+        const {orphaned, skipped} = await this.chain.processChainSegment(blocks, payloadEnvelopes, flags);
+        // We log orphaned payloads to work with different clients
+        // in the future, consider applying penalties in certain conditions
+        // make sure the payload source was from range sync in that case
+        for (const {slot, payloadEnvelopeInput} of orphaned) {
+          const {peerIdStr, source} = payloadEnvelopeInput.getPayloadEnvelopeSource();
+          let client = "unknown";
+          if (peerIdStr !== undefined) {
+            try {
+              client = this.getConnectedPeerSyncMeta(peerIdStr).client;
+            } catch {
+              // peer disconnected since serving the envelope, keep "unknown"
+            }
+          }
+          this.logger.debug("Orphaned payload envelope in range sync batch", {
+            slot,
+            root: payloadEnvelopeInput.blockRootHex,
+            peer: peerIdStr ?? "unknown",
+            source,
+            client,
+            skipped,
+          });
         }
       }
-    } else {
-      const {orphaned, skipped} = await this.chain.processChainSegment(blocks, payloadEnvelopes, flags);
-      // We log orphaned payloads to work with different clients
-      // in the future, consider applying penalties in certain conditions
-      // make sure the payload source was from range sync in that case
-      for (const {slot, payloadEnvelopeInput} of orphaned) {
-        const {peerIdStr, source} = payloadEnvelopeInput.getPayloadEnvelopeSource();
+    } catch (e) {
+      if (
+        e instanceof PayloadError &&
+        (e.type.code === PayloadErrorCode.INVALID_SIGNATURE ||
+          e.type.code === PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR ||
+          e.type.code === PayloadErrorCode.EXECUTION_ENGINE_INVALID)
+      ) {
+        const {payloadInput} = e;
+        const {peerIdStr, source} = payloadInput.getPayloadEnvelopeSource();
         let client = "unknown";
         if (peerIdStr !== undefined) {
           try {
@@ -231,15 +260,18 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
             // peer disconnected since serving the envelope, keep "unknown"
           }
         }
-        this.logger.debug("Orphaned payload envelope in range sync batch", {
-          slot,
-          root: payloadEnvelopeInput.blockRootHex,
+        this.logger.debug("Removing invalid payload envelope in range sync batch", {
+          slot: payloadInput.slot,
+          root: payloadInput.blockRootHex,
           peer: peerIdStr ?? "unknown",
           source,
           client,
-          skipped,
+          code: e.type.code,
         });
+        // The batch is downloaded again, it would otherwise reuse the cached envelope and fail the same way
+        this.chain.seenPayloadEnvelopeInputCache.removeInvalid(payloadInput);
       }
+      throw e;
     }
   };
 

package/src/sync/unknownBlock.ts

@@ -1426,20 +1426,23 @@ export class BlockInputSync {
           pendingPayload.status = PendingPayloadInputStatus.downloaded;
           break;
 
+        // The invalid envelope is evicted from the seen cache in the cases below, otherwise the next
+        // attempt would reuse it instead of fetching another envelope from peers
         case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
           this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+          this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
           this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.elInvalid);
           break;
 
         case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
           this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+          this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
           this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidEnvelope);
           break;
 
         case PayloadErrorCode.INVALID_SIGNATURE:
-          // TODO GLOAS: Decide how invalid payload inputs should eventually leave memory without
-          // reintroducing envelope replacement / recreation flows.
           this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+          this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
           this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidSignature);
           break;
 
@@ -1879,8 +1882,6 @@ export class BlockInputSync {
 
   // Once a parent payload is invalid, every descendant waiting on that payload lineage becomes unrecoverable too.
   private removePendingPayloadAndDescendants(rootHex: RootHex, headReason: DroppedItemReason): void {
-    // Keep PayloadEnvelopeInput resident in the seen cache. importBlock() owns that object and
-    // later validation/finalization logic decides when it can leave memory.
     if (this.pendingPayloads.delete(rootHex)) {
       this.metrics?.blockInputSync.removedPayloads.inc({reason: headReason}, 1);
     }

package/src/util/execution.ts

@@ -266,14 +266,15 @@ type SlotEnvelopeBytes = {slot: Slot; envelopeBytes: Uint8Array};
 
 type RangeEntry = ArchivedEnvelope & {slot: Slot};
 
-/** What a body root mismatch means on a given serving path */
-export type ReconstructMismatchPolicy = "throw" | "omit";
+/** What an envelope that cannot be rebuilt (EL-unavailable body or body root mismatch) means on a given serving path */
+export type ReconstructMissPolicy = "throw" | "omit";
 
-export type RebuildMiss =
-  /** EL does not have the block, or has pruned its block access list */
-  | {slot: Slot; reason: "unavailable"}
-  /** An EL body does not hash to its stored root (local inconsistency) */
-  | {slot: Slot; reason: "mismatch"; error: EnvelopeReconstructionError};
+/**
+ * An archived envelope that could not be rebuilt. `"unavailable"`: the EL does not have the block or its
+ * block access list (BODY_UNAVAILABLE). `"mismatch"`: an EL body does not hash to its stored root, a local
+ * inconsistency (BODY_ROOT_MISMATCH). `error` carries the matching code for paths that surface it.
+ */
+export type RebuildMiss = {slot: Slot; reason: "unavailable" | "mismatch"; error: EnvelopeReconstructionError};
 
 /**
  * Stream finalized envelopes over [startSlot, endSlot) as serialized bytes, rebuilding header
@@ -405,7 +406,14 @@ async function reconstructEnvelopesBatch(
     // A zero-length block access list cannot be valid, RLP encodes an empty list as 0xc0
     if (body == null || body.withdrawals == null || body.blockAccessList == null || body.blockAccessList.length === 0) {
       metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "unavailable"});
-      return {slot, reason: "unavailable"};
+      return {
+        slot,
+        reason: "unavailable",
+        error: new EnvelopeReconstructionError(
+          {code: EnvelopeReconstructionErrorCode.BODY_UNAVAILABLE, slot},
+          `execution client cannot serve the payload body or block access list for archived envelope slot=${slot}`
+        ),
+      };
     }
     try {
       const envelope = signedHeaderEnvelopeToFull(headerEnvelope, {