@lodestar/beacon-node 1.46.0 → 1.47.0-dev.02d2dbea03

package/src/network/processor/gossipHandlers.ts

@@ -196,7 +196,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         forkName: fork,
         sampledColumns: chain.custodyConfig.sampledColumns,
         custodyColumns: chain.custodyConfig.custodyColumns,
-        timeCreatedSec: seenTimestampSec,
+        seenTimestampSec,
+        source: PayloadEnvelopeInputSource.gossip,
       });
     }
 
@@ -640,6 +641,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         metrics?.gossipBlock.elapsedTimeTillProcessed.observe(delaySec);
 
         if (isForkPostGloas(blockInput.forkName)) {
+          // we should have the payloadInput in the seen cache so no need getOrReload() here
           const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockInput.blockRootHex);
           // This payloadInput should have been created just after gossip validation
           if (!payloadInput) {
@@ -713,9 +715,32 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       const {serializedData} = gossipData;
 
       const signedBlock = sszDeserialize(topic, serializedData);
-      const blockInput = await validateBeaconBlock(signedBlock, topic.boundary.fork, peerIdStr, seenTimestampSec);
-      chain.serializedCache.set(signedBlock, serializedData);
-      handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
+      try {
+        const blockInput = await validateBeaconBlock(signedBlock, topic.boundary.fork, peerIdStr, seenTimestampSec);
+        chain.serializedCache.set(signedBlock, serializedData);
+        handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
+      } catch (e) {
+        // Spec: IGNORE the block, ie not to re-publish to peers
+        // but we should still import an equivocating (REPEAT_PROPOSAL) block into fork choice because we don't
+        // know if this block (or the 1st known block with same slot) will become canonical yet
+        if (
+          e instanceof BlockGossipError &&
+          e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
+          // this is make sure the block's proposer signature was verified, it should be true anyway
+          chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root)
+        ) {
+          // blockInput was optimistically seeded in validateBeaconBlock and retained on IGNORE
+          const blockInput = chain.seenBlockInputCache.get(e.type.root);
+          if (blockInput) {
+            chain.serializedCache.set(signedBlock, serializedData);
+            // this is technically not a valid gossip block but gossip validation is a cheap subset of checks
+            // this runs the full state transition, so importing an equivocating-but-valid block here is safe.
+            handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
+          }
+        }
+        // rethrow so gossipValidatorFn maps IGNORE -> TopicValidatorResult.Ignore (message not forwarded)
+        throw e;
+      }
     },
 
     [GossipType.blob_sidecar]: async ({
@@ -1166,6 +1191,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       chain.validatorMonitor?.registerExecutionPayloadEnvelope(OpSource.gossip, delaySec, signedEnvelope);
 
       const blockRootHex = toRootHex(envelope.beaconBlockRoot);
+      // a gossip payload cannot be `MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE` slots after the block
+      // otherwise it'll get to UnknownBlockInput flow
       const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
 
       if (!payloadInput) {
@@ -1232,11 +1259,15 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     [GossipType.payload_attestation_message]: async ({
       gossipData,
       topic,
+      seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.payload_attestation_message>) => {
       const {serializedData} = gossipData;
       const payloadAttestationMessage = sszDeserialize(topic, serializedData);
       const validationResult = await validateGossipPayloadAttestationMessage(chain, payloadAttestationMessage);
 
+      const delaySec = chain.clock.secFromSlot(payloadAttestationMessage.data.slot, seenTimestampSec);
+      metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({source: OpSource.gossip}, delaySec);
+
       try {
         const insertOutcome = chain.payloadAttestationPool.add(
           payloadAttestationMessage,
@@ -1263,11 +1294,16 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     [GossipType.execution_payload_bid]: async ({
       gossipData,
       topic,
+      seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.execution_payload_bid>) => {
       const {serializedData} = gossipData;
       const executionPayloadBid = sszDeserialize(topic, serializedData);
       const {proposerIndex} = await validateGossipExecutionPayloadBid(chain, executionPayloadBid);
 
+      // this could be negative, because it's most likely the bid of next slot comes at this clock slot
+      const elapsedSec = chain.clock.secFromSlot(executionPayloadBid.message.slot, seenTimestampSec);
+      metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({source: OpSource.gossip}, elapsedSec);
+
       // Handle valid payload bid by storing in a bid pool
       try {
         const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid);

package/src/network/reqresp/score.ts

@@ -44,17 +44,11 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod):
         // failure, so keep the stronger penalty (Fatal for Ping, as before).
         return method === ReqRespMethod.Ping ? PeerAction.Fatal : PeerAction.LowToleranceError;
       }
-      switch (method) {
-        // Ping and Status are liveness probes; their dial timeouts are dominated by transient
-        // network congestion rather than peer misbehavior, so penalize leniently to avoid
-        // self-inflicted peer starvation (https://github.com/ChainSafe/lodestar/issues/9562),
-        // while still applying some penalty so genuinely dead peers eventually free the slot.
-        case ReqRespMethod.Ping:
-        case ReqRespMethod.Status:
-          return PeerAction.HighToleranceError;
-        default:
-          return PeerAction.LowToleranceError;
-      }
+      // A dial failure means we could not open a stream. That is evidence about the transport, not
+      // about the peer's behavior, and it is dominated by transient congestion on our side, so
+      // penalize leniently for every method to avoid self-inflicted peer starvation (#9562). A
+      // genuinely dead peer still accumulates penalty and is eventually evicted, freeing the slot.
+      return PeerAction.HighToleranceError;
     // TODO: Detect SSZDecodeError and return PeerAction.Fatal
 
     case RequestErrorCode.RESP_TIMEOUT:

package/src/network/util.ts

@@ -23,11 +23,6 @@ export function getConnection(libp2p: Libp2p, peerIdStr: string): Connection | u
   return getConnectionsMap(libp2p).get(peerIdStr)?.value[0] ?? undefined;
 }
 
-// https://github.com/libp2p/js-libp2p/blob/f87cba928991736d9646b3e054c367f55cab315c/packages/gossipsub/src/gossipsub.ts#L2076
-export function isPublishToZeroPeersError(e: Error): boolean {
-  return e.message.includes("PublishError.NoPeersSubscribedToTopic");
-}
-
 // https://github.com/libp2p/js-libp2p/blob/f87cba928991736d9646b3e054c367f55cab315c/packages/gossipsub/src/gossipsub.ts#L2076
 export function isPublishDuplicateError(e: Error): boolean {
   return e.message.includes("PublishError.Duplicate");

package/src/node/deferredVoluntaryExitPublisher.ts

@@ -0,0 +1,54 @@
+import {routes} from "@lodestar/api";
+import {Logger} from "@lodestar/utils";
+import {IBeaconChain} from "../chain/index.js";
+import {INetwork} from "../network/index.js";
+import {ClockEvent} from "../util/clock.js";
+
+/**
+ * On every epoch, drain the deferred voluntary exit pool and publish the exits that have become
+ * processable. Lives at the node layer rather than inside `BeaconChain` because publishing needs
+ * `network`, which the chain does not (and should not) depend on.
+ */
+export function startDeferredVoluntaryExitPublisher({
+  chain,
+  network,
+  logger,
+  signal,
+}: {
+  chain: IBeaconChain;
+  network: INetwork;
+  logger: Logger;
+  signal: AbortSignal;
+}): void {
+  const onEpoch = async (): Promise<void> => {
+    try {
+      // Use the cached head state directly rather than getHeadStateAtCurrentEpoch, which would regen
+      // the head forward to the current epoch. At init from a stale db-finalized state that regen can
+      // be many epoch transitions. The head state is fine here: this is a best-effort drain that
+      // re-runs every epoch, so a just-processable exit is republished next tick.
+      const state = chain.getHeadState();
+      const exits = chain.deferredVoluntaryExitPool.drainProcessableExits(state);
+      for (const exit of exits) {
+        try {
+          chain.opPool.insertVoluntaryExit(exit);
+          chain.emitter.emit(routes.events.EventType.voluntaryExit, exit);
+          await network.publishVoluntaryExit(exit);
+          logger.info("Voluntary exit successfully published for validator", {
+            validatorIndex: exit.message.validatorIndex,
+          });
+        } catch (e) {
+          logger.warn(
+            "Failed to publish deferred voluntary exit",
+            {validatorIndex: exit.message.validatorIndex},
+            e as Error
+          );
+        }
+      }
+    } catch (e) {
+      logger.warn("Failed to drain deferred voluntary exit pool", {}, e as Error);
+    }
+  };
+
+  chain.clock.addListener(ClockEvent.epoch, onEpoch);
+  signal.addEventListener("abort", () => chain.clock.removeListener(ClockEvent.epoch, onEpoch), {once: true});
+}

package/src/node/nodejs.ts

@@ -1,12 +1,13 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {hasher} from "@chainsafe/persistent-merkle-tree";
 import {BeaconApiMethods} from "@lodestar/api/beacon/server";
 import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, PubkeyCache, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
 import {phase0} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
@@ -21,6 +22,7 @@ import {Network, getReqRespHandlers} from "../network/index.js";
 import {BackfillSync} from "../sync/backfill/index.js";
 import {BeaconSync, IBeaconSync} from "../sync/index.js";
 import {Clock} from "../util/clock.js";
+import {startDeferredVoluntaryExitPublisher} from "./deferredVoluntaryExitPublisher.js";
 import {runNodeNotifier} from "./notifier.js";
 import {IBeaconNodeOptions} from "./options.js";
 
@@ -334,6 +336,8 @@ export class BeaconNode {
 
     void runNodeNotifier({network, chain, sync, config, logger, signal});
 
+    startDeferredVoluntaryExitPublisher({chain, network, logger, signal});
+
     return new BeaconNode({
       opts,
       config,

package/src/sync/range/batch.ts

@@ -39,6 +39,7 @@ export enum BatchStatus {
   AwaitingValidation = "AwaitingValidation",
 }
 
+/** Identity of one download attempt*/
 export type Attempt = {
   /** The peer that made the attempt */
   peers: PeerIdStr[];
@@ -46,6 +47,16 @@ export type Attempt = {
   hash: RootHex;
 };
 
+/**
+ * A failed attempt that has been classified for blame.
+ */
+export type FailedAttempt = Attempt & {
+  /**
+   * True when the failure is evidence the peers served bad data, so they may be downscored..
+   */
+  peerAttributable: boolean;
+};
+
 type TrackedRequest = {
   /** only happen for the 1st batch in checkpoint sync */
   parentPayload: boolean;
@@ -108,10 +119,12 @@ export type BatchMetadata = {
   // Retry counters
   downloadAttempts: number;
   processingAttempts: number;
+  executionErrorAttempts: number;
 
   // Cumulative peer attribution for failed attempts (only present when non-empty)
   failedDownloadPeers?: string;
   failedProcessingPeers?: string;
+  executionErrorPeers?: string;
 };
 
 function formatRangeReq(req: {startSlot: Slot; count: number}): string {
@@ -152,10 +165,10 @@ export class Batch {
   state: BatchState = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
   /** Peers that provided good data, with column coverage for by_range requests */
   private readonly successfulDownloads = new Map<PeerIdStr, TrackedRequest>();
-  /** The `Attempts` that have been made and failed to send us this batch. */
-  readonly failedProcessingAttempts: Attempt[] = [];
-  /** The `Attempts` that have been made and failed because of execution malfunction. */
-  readonly executionErrorAttempts: Attempt[] = [];
+  /** The attempts that have been made and failed to send us this batch. */
+  readonly failedProcessingAttempts: FailedAttempt[] = [];
+  /** The attempts that have been made and failed because of execution malfunction. */
+  readonly executionErrorAttempts: FailedAttempt[] = [];
   /** The number of download retries this batch has undergone due to a failed request. */
   private readonly failedDownloadAttempts: PeerIdStr[] = [];
   private readonly config: ChainForkConfig;
@@ -419,10 +432,18 @@ export class Batch {
   }
 
   /**
-   * Gives a list of peers from which this batch has had a failed download or processing attempt.
+   * Gives a list of peers from which this batch has had a failed download attempt, or a
+   * peer-attributable failed processing/execution attempt.
    */
   getFailedPeers(): PeerIdStr[] {
-    return [...this.failedDownloadAttempts, ...this.failedProcessingAttempts.flatMap((a) => a.peers)];
+    return [...this.failedDownloadAttempts, ...this.getPeerAttributableAttempts().flatMap((a) => a.peers)];
+  }
+
+  /**
+   * Attempts whose failure is peer-attributable, across both the processing and execution error
+   */
+  getPeerAttributableAttempts(): FailedAttempt[] {
+    return [...this.failedProcessingAttempts, ...this.executionErrorAttempts].filter((a) => a.peerAttributable);
   }
 
   /**
@@ -457,6 +478,7 @@ export class Batch {
   getMetadata(): BatchMetadata {
     const {blocksRequest, blobsRequest, columnsRequest, envelopesRequest} = this.requests;
     const failedProcessingPeerList = this.failedProcessingAttempts.flatMap((a) => a.peers);
+    const executionErrorPeerList = this.executionErrorAttempts.flatMap((a) => a.peers);
     return {
       startEpoch: this.startEpoch,
       startSlot: this.startSlot,
@@ -468,12 +490,16 @@ export class Batch {
       ...(envelopesRequest && {envelopesReq: formatRangeReq(envelopesRequest)}),
       downloadAttempts: this.failedDownloadAttempts.length,
       processingAttempts: this.failedProcessingAttempts.length,
+      executionErrorAttempts: this.executionErrorAttempts.length,
       ...(this.failedDownloadAttempts.length > 0 && {
         failedDownloadPeers: this.failedDownloadAttempts.join(","),
       }),
       ...(failedProcessingPeerList.length > 0 && {
         failedProcessingPeers: failedProcessingPeerList.join(","),
       }),
+      ...(executionErrorPeerList.length > 0 && {
+        executionErrorPeers: executionErrorPeerList.join(","),
+      }),
     };
   }
 
@@ -593,7 +619,7 @@ export class Batch {
     if (allComplete) {
       const attempt: Attempt = {
         peers: this.getSuccessfulPeers(),
-        hash: hashBlocks(blocks, this.config),
+        hash: hashBlocks(blocks, payloadEnvelopes),
       };
       this.state = {status: BatchStatus.AwaitingProcessing, blocks, payloadEnvelopes: newPayloadEnvelopes, attempt};
     } else {
@@ -703,11 +729,7 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.Processing));
     }
 
-    if (isExecutionEngineError(err)) {
-      this.onExecutionEngineError(this.state.attempt);
-    } else {
-      this.onProcessingError(this.state.attempt);
-    }
+    this.routeProcessingFailure(err, this.state.attempt);
   }
 
   /**
@@ -718,10 +740,17 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingValidation));
     }
 
-    if (isExecutionEngineError(err)) {
-      this.onExecutionEngineError(this.state.attempt);
+    this.routeProcessingFailure(err, this.state.attempt);
+  }
+
+  private routeProcessingFailure(err: Error, attempt: Attempt): void {
+    const code = err instanceof BlockError || err instanceof PayloadError ? err.type.code : null;
+    const failedAttempt: FailedAttempt = {...attempt, peerAttributable: isPeerAttributableFailure(code)};
+
+    if (isExecutionEngineFailure(code)) {
+      this.onExecutionEngineError(failedAttempt);
     } else {
-      this.onProcessingError(this.state.attempt);
+      this.onProcessingError(failedAttempt);
     }
   }
 
@@ -735,7 +764,7 @@ export class Batch {
     return this.state.attempt;
   }
 
-  private onExecutionEngineError(attempt: Attempt): void {
+  private onExecutionEngineError(attempt: FailedAttempt): void {
     this.executionErrorAttempts.push(attempt);
     if (this.executionErrorAttempts.length > MAX_BATCH_PROCESSING_ATTEMPTS) {
       throw new BatchError(this.errorType({code: BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS}));
@@ -746,7 +775,7 @@ export class Batch {
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
   }
 
-  private onProcessingError(attempt: Attempt): void {
+  private onProcessingError(attempt: FailedAttempt): void {
     this.failedProcessingAttempts.push(attempt);
     if (this.failedProcessingAttempts.length > MAX_BATCH_PROCESSING_ATTEMPTS) {
       throw new BatchError(this.errorType({code: BatchErrorCode.MAX_PROCESSING_ATTEMPTS}));
@@ -789,18 +818,44 @@ type BatchErrorMetadata = {
 
 export class BatchError extends LodestarError<BatchErrorType & BatchErrorMetadata> {}
 
-function isExecutionEngineError(err: Error): boolean {
-  if (!(err instanceof BlockError)) {
-    return false;
-  }
-
-  if (err.type.code === BlockErrorCode.EXECUTION_ENGINE_ERROR) {
-    return true;
-  }
-
+/**
+ * Whether a processing failure is an execution-engine failure. Pre-gloas, all EL errors stay in
+ * BlockErrorCode, post-gloas, they stay in PayloadErrorCode.
+ */
+function isExecutionEngineFailure(code: BlockErrorCode | PayloadErrorCode | null): boolean {
   return (
-    err.type.code === BlockErrorCode.BEACON_CHAIN_ERROR &&
-    err.type.error instanceof PayloadError &&
-    err.type.error.type.code === PayloadErrorCode.EXECUTION_ENGINE_ERROR
+    code === BlockErrorCode.EXECUTION_ENGINE_INVALID ||
+    code === BlockErrorCode.EXECUTION_ENGINE_ERROR ||
+    code === PayloadErrorCode.EXECUTION_ENGINE_INVALID ||
+    code === PayloadErrorCode.EXECUTION_ENGINE_ERROR
   );
 }
+
+/**
+ * Based on an error code, return if peer is attributable for FailedAttempt.
+ */
+function isPeerAttributableFailure(code: BlockErrorCode | PayloadErrorCode | null): boolean {
+  switch (code) {
+    // EL returned a definitive INVALID verdict on the payload
+    case BlockErrorCode.EXECUTION_ENGINE_INVALID:
+    case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
+    // a block or envelope signature is invalid
+    case BlockErrorCode.INVALID_SIGNATURE:
+    case PayloadErrorCode.INVALID_SIGNATURE:
+    // the block's state transition produced an invalid state root, or failed per_slot/per_block processing
+    case BlockErrorCode.INVALID_STATE_ROOT:
+    case BlockErrorCode.PER_BLOCK_PROCESSING_ERROR:
+    // the segment this peer served is internally inconsistent (in-segment link breaks, see PR #9684)
+    case BlockErrorCode.NON_LINEAR_SLOTS:
+    case BlockErrorCode.NON_LINEAR_PARENT_ROOTS:
+    case BlockErrorCode.NON_LINEAR_PAYLOAD_ROOTS:
+    // the envelope references a mismatched block root, or fails field verification
+    case BlockErrorCode.ENVELOPE_BLOCK_ROOT_MISMATCH:
+    case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
+    // the peer served a block we have explicitly blacklisted
+    case BlockErrorCode.BLACKLISTED_BLOCK:
+      return true;
+    default:
+      return false;
+  }
+}

package/src/sync/range/chain.ts

@@ -358,9 +358,10 @@ export class SyncChain {
       // If a batch exceeds it's retry limit, maybe downscore peers.
       // shouldDownscoreOnBatchError() functions enforces that all BatchErrorCode values are covered
       if (e instanceof BatchError) {
-        const shouldReportPeer = shouldReportPeerOnBatchError(e.type.code);
+        const shouldReportPeer = shouldReportPeerOnBatchError(e.type.code, this.batches.get(e.type.startEpoch));
         if (shouldReportPeer) {
-          for (const peer of this.peerset.keys()) {
+          // Only the peers actually at fault are reported, never the whole peerset.
+          for (const peer of shouldReportPeer.peers) {
             this.reportPeer(peer, shouldReportPeer.action, shouldReportPeer.reason);
           }
         }
@@ -768,8 +769,11 @@ export class SyncChain {
 
         // The last batch attempt is right, all others are wrong. Penalize other peers
         const attemptOk = batch.validationSuccess();
-        for (const attempt of batch.failedProcessingAttempts) {
+        for (const attempt of batch.getPeerAttributableAttempts()) {
           if (attempt.hash !== attemptOk.hash) {
+            // attempt.peers contains block/column/envelope peers.
+            // And if there is one bad envelope we will downscore block and column peers to
+            // TODO: resolve this
             for (const badAttemptPeer of attempt.peers) {
               if (attemptOk.peers.find((goodPeer) => goodPeer === badAttemptPeer)) {
                 // The same peer corrected its previous attempt
@@ -830,21 +834,32 @@ export class SyncChain {
  * If peer should not be downscored, returns null.
  */
 export function shouldReportPeerOnBatchError(
-  code: BatchErrorCode
-): {action: PeerAction.LowToleranceError; reason: string} | null {
+  code: BatchErrorCode,
+  batch: Batch | undefined
+): {action: PeerAction.LowToleranceError; reason: string; peers: PeerIdStr[]} | null {
   switch (code) {
-    // A batch could not be processed after max retry limit. It's likely that all peers
-    // in this chain are sending invalid batches repeatedly so are either malicious or faulty.
-    // We drop the chain and report all peers.
-    // There are some edge cases with forks that could cause this situation, but it's unlikely.
+    //  A batch could not be processed after max retry limit. Report
+    // only the peers that served a peer-attributable failed attempt.
     case BatchErrorCode.MAX_PROCESSING_ATTEMPTS:
-      return {action: PeerAction.LowToleranceError, reason: "SyncChainMaxProcessingAttempts"};
+    case BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS: {
+      // penalize both the processing-error and execution-error peers
+      const attributablePeers = [
+        ...new Set(batch?.getPeerAttributableAttempts().flatMap((attempt) => attempt.peers) ?? []),
+      ];
+      if (attributablePeers.length === 0) {
+        return null;
+      }
+      const reason =
+        code === BatchErrorCode.MAX_PROCESSING_ATTEMPTS
+          ? "SyncChainMaxProcessingAttempts"
+          : "SyncChainMaxExecutionEngineErrorAttempts";
+      return {action: PeerAction.LowToleranceError, reason, peers: attributablePeers};
+    }
 
     // TODO: Should peers be reported for MAX_DOWNLOAD_ATTEMPTS?
     case BatchErrorCode.MAX_DOWNLOAD_ATTEMPTS:
     case BatchErrorCode.INVALID_COUNT:
     case BatchErrorCode.WRONG_STATUS:
-    case BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS:
       return null;
   }
 }

package/src/sync/range/utils/hashBlocks.ts

@@ -1,27 +1,62 @@
-import {ChainForkConfig} from "@lodestar/config";
-import {RootHex, SignedBeaconBlock} from "@lodestar/types";
-import {toRootHex} from "@lodestar/utils";
+import {digest} from "@chainsafe/as-sha256";
+import {RootHex, Slot, ssz} from "@lodestar/types";
+import {fromHex, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../../chain/blocks/blockInput/types.js";
+import {PayloadEnvelopeInput} from "../../../chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
+
+const ROOT_SIZE = 32;
+const SIGNATURE_SIZE = 96;
+/** Every block and every payload envelope contributes a fixed-size (root, signature) entry */
+const ENTRY_SIZE = ROOT_SIZE + SIGNATURE_SIZE;
+/** uint32 LE counts of block and envelope entries, so the variable-length sections are self-describing */
+const COUNT_SIZE = 4;
+const HEADER_SIZE = COUNT_SIZE * 2;
 
 /**
- * String to uniquely identify block segments. Used for peer scoring and to compare if batches are equivalent.
+ * Root uniquely identifying a batch attempt (its blocks AND payloads). Used for peer scoring and to
+ * compare if two attempts are equivalent.
+ *
+ * Signatures are part of the id for BOTH blocks and payload envelopes, not just their message roots:
+ * a peer serving a correct message with a garbage signature would otherwise produce the same id as
+ * the honest attempt and escape scoring in `SyncChain.advanceChain`. Signatures are not verified at
+ * download time, only during processChainSegment (block signatures in verifyBlocksSignatures,
+ * envelope signatures in importExecutionPayload).
+ *
+ * Entries are written as raw bytes into a single exactly-sized buffer and digested once, so an
+ * `Attempt` retains 32 bytes rather than the ~16 KB a full 32-slot gloas batch would need if the
+ * roots and signatures were concatenated as hex. sha256 (not a 64-bit hash) because a peer that
+ * could force a collision with the winning attempt would escape peer scoring.
  */
-export function hashBlocks(blocks: IBlockInput[], config: ChainForkConfig): RootHex {
-  switch (blocks.length) {
-    case 0:
-      return "0x";
-    case 1: {
-      const block0 = blocks[0].getBlock();
-      return toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0));
-    }
-    default: {
-      const block0 = blocks[0].getBlock();
-      const blockN = blocks.at(-1)?.getBlock() as SignedBeaconBlock;
-      return (
-        // TODO(fulu): should we be doing checks for presence to make sure these do not blow up?
-        toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0)) +
-        toRootHex(config.getForkTypes(blockN.message.slot).SignedBeaconBlock.hashTreeRoot(blockN))
-      );
-    }
+export function hashBlocks(blocks: IBlockInput[], payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null): RootHex {
+  // Envelopes without a payload carry no attributable data, so they are skipped entirely. A `null`
+  // map and a map whose entries all lack a payload are therefore equivalent, which is intended.
+  const envelopes =
+    payloadEnvelopes && payloadEnvelopes.size > 0
+      ? Array.from(payloadEnvelopes.entries())
+          .filter(([, envelope]) => envelope.hasPayloadEnvelope())
+          .sort(([slotA], [slotB]) => slotA - slotB)
+      : [];
+
+  const buf = Buffer.allocUnsafe(HEADER_SIZE + (blocks.length + envelopes.length) * ENTRY_SIZE);
+  buf.writeUInt32LE(blocks.length, 0);
+  buf.writeUInt32LE(envelopes.length, COUNT_SIZE);
+  let offset = HEADER_SIZE;
+
+  for (const block of blocks) {
+    buf.set(fromHex(block.blockRootHex), offset);
+    offset += ROOT_SIZE;
+    buf.set(block.getBlock().signature, offset);
+    offset += SIGNATURE_SIZE;
   }
+
+  for (const [, envelope] of envelopes) {
+    const signedEnvelope = envelope.getPayloadEnvelope();
+    // envelope's root is cached thanks to cachePermanentRootStruct, so this avoids re-hashing
+    buf.set(ssz.gloas.ExecutionPayloadEnvelope.hashTreeRoot(signedEnvelope.message), offset);
+    offset += ROOT_SIZE;
+    buf.set(signedEnvelope.signature, offset);
+    offset += SIGNATURE_SIZE;
+  }
+
+  return toRootHex(digest(buf));
 }

package/src/sync/types.ts

@@ -36,6 +36,39 @@ export enum PendingBlockInputStatus {
   processing = "processing",
 }
 
+/**
+ * Why a pending item was dropped from BlockInputSync WITHOUT completing.
+ * The happy path — a successfully imported/processed block or payload — is removed by the import
+ * flow (onPayloadImported, processReadyBlock/processPayload success) and is deliberately NOT tracked
+ * here
+ */
+export enum DroppedItemReason {
+  /** prune: item's own slot is below the finalized slot (routine cleanup), or a downloaded block landed at/below finality */
+  belowFinalized = "below_finalized",
+  /** prune: slot-less, unresolvable payload evicted after PRUNE_UNRESOLVED_SLOT_EPOCHS */
+  agedOut = "aged_out",
+  /** pruneSetToMax evicted the oldest entry because the cache hit its capacity (DoS guard) */
+  capacity = "capacity",
+  /** descendant removed because an ancestor was invalid/removed */
+  invalidParent = "invalid_parent",
+  /** transient execution-engine error removal (EXECUTION_ENGINE_ERROR) */
+  elError = "el_error",
+  /** the execution engine declared the item invalid (EXECUTION_ENGINE_INVALID) */
+  elInvalid = "el_invalid",
+  /** a block failed our own checks (not correct w.r.t. our chain) */
+  invalidBlock = "invalid_block",
+  /** a block declares a parent payload hash that conflicts with the parent's actual payload (gloas) */
+  invalidParentPayload = "invalid_parent_payload",
+  /** a payload envelope failed gossip verification (ENVELOPE_VERIFICATION_ERROR) */
+  invalidEnvelope = "invalid_envelope",
+  /** a payload had an invalid signature (INVALID_SIGNATURE) */
+  invalidSignature = "invalid_signature",
+  /** a block root was given up on after exhausting download attempts (data could not be fetched) */
+  unavailable = "unavailable",
+  /** an unexpected / unhandled error code led to removal */
+  unknown = "unknown",
+}
+
 export enum PendingPayloadInputStatus {
   pending = "pending",
   fetching = "fetching",
@@ -77,7 +110,9 @@ export type PendingPayloadInput = {
 export type PendingPayloadRootHex = {
   status: PendingPayloadInputStatus.pending | PendingPayloadInputStatus.fetching;
   rootHex: RootHex;
-  // message slot hint, may be missing when resolving a parent payload
+  // Trusted slot only (fork choice / validated data), may be missing until resolved. NOT the gossip
+  // message slot from ChainEvent.unknownEnvelopeBlockRoot, which is untrusted and not necessarily the
+  // payload/block slot. See BlockInputSync.resolvePayloadSlot.
   slot?: Slot;
   timeAddedSec: number;
   timeSyncedSec?: number;