@lodestar/beacon-node 1.46.0-dev.9be1d27b00 → 1.46.0-dev.a03686d0dc

package/src/chain/seenCache/seenBlockProposers.ts

@@ -1,21 +1,83 @@
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {Epoch, Slot, ValidatorIndex} from "@lodestar/types";
+import {Epoch, RootHex, Slot, ValidatorIndex, phase0} from "@lodestar/types";
 import {MapDef} from "@lodestar/utils";
 
+/** Two distinct block roots signed by the same proposer for the same slot are sufficient to establish an equivocation */
+const MAX_BLOCK_ROOTS_PER_PROPOSAL = 2;
+
 /**
  * Keeps a cache to filter block proposals from the same validator in the same slot.
  *
- * This cache is not bounded and for extremely long periods of non-finality it can grow a lot. However it's practically
- * limited by the possible shufflings in those epochs, and the stored data is very cheap
+ * Block roots with a signature verified against the block's proposer index are tracked separately from proposals
+ * accepted by gossip validation or block import. A root from a block signed by its declared proposer is potential
+ * equivocation evidence, but the block may still fail other validation. Such a block must not mark the proposal as
+ * known, since that would cause a later valid block for the same slot and proposer to be ignored as a repeat proposal.
+ *
+ * The signed block header of each observed root is kept so a proposer slashing can be produced from the two
+ * conflicting headers once an equivocation is established.
+ *
+ * The cache is pruned on finalization and bounds the number of roots stored per proposer and slot
  */
 export class SeenBlockProposers {
   private readonly proposerIndexesBySlot = new MapDef<Slot, Set<ValidatorIndex>>(() => new Set<ValidatorIndex>());
-  private finalizedSlot: Epoch = 0;
+  private readonly signedBlockHeadersBySlot = new MapDef<
+    Slot,
+    MapDef<ValidatorIndex, Map<RootHex, phase0.SignedBeaconBlockHeader>>
+  >(() => new MapDef<ValidatorIndex, Map<RootHex, phase0.SignedBeaconBlockHeader>>(() => new Map()));
+  private finalizedSlot: Slot = 0;
 
   isKnown(blockSlot: Slot, proposerIndex: ValidatorIndex): boolean {
     return this.proposerIndexesBySlot.get(blockSlot)?.has(proposerIndex) === true;
   }
 
+  hasBlockRoot(blockSlot: Slot, proposerIndex: ValidatorIndex, blockRoot: RootHex): boolean {
+    return this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex)?.has(blockRoot) === true;
+  }
+
+  isEquivocating(blockSlot: Slot, proposerIndex: ValidatorIndex): boolean {
+    return (
+      (this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex)?.size ?? 0) >= MAX_BLOCK_ROOTS_PER_PROPOSAL
+    );
+  }
+
+  getConflictingBlockRoots(blockSlot: Slot, proposerIndex: ValidatorIndex, blockRoot: RootHex): RootHex[] {
+    const signedBlockHeaderByRoot = this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex);
+    return signedBlockHeaderByRoot === undefined
+      ? []
+      : Array.from(signedBlockHeaderByRoot.keys()).filter((root) => root !== blockRoot);
+  }
+
+  /** Return the two signed block headers that establish an equivocation, or null if there is none */
+  getEquivocationHeaders(
+    blockSlot: Slot,
+    proposerIndex: ValidatorIndex
+  ): [phase0.SignedBeaconBlockHeader, phase0.SignedBeaconBlockHeader] | null {
+    const signedBlockHeaderByRoot = this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex);
+    if (signedBlockHeaderByRoot === undefined || signedBlockHeaderByRoot.size < MAX_BLOCK_ROOTS_PER_PROPOSAL) {
+      return null;
+    }
+    const [signedHeader1, signedHeader2] = signedBlockHeaderByRoot.values();
+    return [signedHeader1, signedHeader2];
+  }
+
+  /** Record a block only after its proposer signature has been verified */
+  observeBlockRoot(
+    blockSlot: Slot,
+    proposerIndex: ValidatorIndex,
+    blockRoot: RootHex,
+    signedBlockHeader: phase0.SignedBeaconBlockHeader
+  ): void {
+    if (blockSlot < this.finalizedSlot) {
+      throw Error(`blockSlot ${blockSlot} < finalizedSlot ${this.finalizedSlot}`);
+    }
+
+    const signedBlockHeaderByRoot = this.signedBlockHeadersBySlot.getOrDefault(blockSlot).getOrDefault(proposerIndex);
+    if (signedBlockHeaderByRoot.size < MAX_BLOCK_ROOTS_PER_PROPOSAL && !signedBlockHeaderByRoot.has(blockRoot)) {
+      signedBlockHeaderByRoot.set(blockRoot, signedBlockHeader);
+    }
+  }
+
+  /** Mark a block as known from gossip or another block import path */
   add(blockSlot: Slot, proposerIndex: ValidatorIndex): void {
     if (blockSlot < this.finalizedSlot) {
       throw Error(`blockSlot ${blockSlot} < finalizedSlot ${this.finalizedSlot}`);
@@ -31,9 +93,14 @@ export class SeenBlockProposers {
         this.proposerIndexesBySlot.delete(slot);
       }
     }
+    for (const slot of this.signedBlockHeadersBySlot.keys()) {
+      if (slot < finalizedSlot) {
+        this.signedBlockHeadersBySlot.delete(slot);
+      }
+    }
   }
 
-  seenAtEpoch(epoch: Slot, index: ValidatorIndex): boolean {
+  seenAtEpoch(epoch: Epoch, index: ValidatorIndex): boolean {
     const fromSlot = computeStartSlotAtEpoch(epoch);
     const toSlot = computeStartSlotAtEpoch(epoch + 1);
 

package/src/chain/validation/block.ts

@@ -23,8 +23,9 @@ import {
   getBlockProposerSignatureSet,
   isExecutionBlockBodyType,
   isStatePostBellatrix,
+  signedBlockToSignedHeader,
 } from "@lodestar/state-transition";
-import {SignedBeaconBlock, deneb, gloas, isGloasBeaconBlock} from "@lodestar/types";
+import {RootHex, SignedBeaconBlock, deneb, gloas, isGloasBeaconBlock, ssz} from "@lodestar/types";
 import {byteArrayEquals, sleep, toRootHex} from "@lodestar/utils";
 import {BlockErrorCode, BlockGossipError, GossipAction} from "../errors/index.js";
 import {IBeaconChain} from "../interface.js";
@@ -75,7 +76,10 @@ export async function validateGossipBlock(
   // reboot if the `observed_block_producers` cache is empty. In that case, without this
   // check, we will load the parent and state from disk only to find out later that we
   // already know this block.
-  const blockRoot = toRootHex(config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block));
+  // A block's hash tree root is identical to its header's, so the root is derived from the header
+  // which is also used as potential equivocation evidence
+  const signedBlockHeader = signedBlockToSignedHeader(config, signedBlock);
+  const blockRoot = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(signedBlockHeader.message));
   if (chain.forkChoice.getBlockHexDefaultStatus(blockRoot) !== null) {
     throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.ALREADY_KNOWN, root: blockRoot});
   }
@@ -85,7 +89,12 @@ export async function validateGossipBlock(
 
   // [IGNORE] The block is the first block with valid signature received for the proposer for the slot, signed_beacon_block.message.slot.
   const proposerIndex = block.proposerIndex;
+  const hasBlockRoot = chain.seenBlockProposers.hasBlockRoot(blockSlot, proposerIndex, blockRoot);
   if (chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
+    if (!hasBlockRoot && !chain.seenBlockProposers.isEquivocating(blockSlot, proposerIndex)) {
+      await verifyBlockProposerSignature(chain, signedBlock, blockRoot, {verifyOnMainThread: false});
+      chain.seenBlockProposers.observeBlockRoot(blockSlot, proposerIndex, blockRoot, signedBlockHeader);
+    }
     throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.REPEAT_PROPOSAL, proposerIndex});
   }
 
@@ -270,18 +279,8 @@ export async function validateGossipBlock(
   }
 
   // [REJECT] The proposer signature, signed_beacon_block.signature, is valid with respect to the proposer_index pubkey.
-  if (!chain.seenBlockInputCache.isVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature)) {
-    const signatureSet = getBlockProposerSignatureSet(chain.config, signedBlock);
-    // Don't batch so verification is not delayed
-    if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) {
-      throw new BlockGossipError(GossipAction.REJECT, {
-        code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID,
-        blockSlot,
-      });
-    }
-
-    chain.seenBlockInputCache.markVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature);
-  }
+  await verifyBlockProposerSignature(chain, signedBlock, blockRoot);
+  chain.seenBlockProposers.observeBlockRoot(blockSlot, proposerIndex, blockRoot, signedBlockHeader);
 
   // [REJECT] The block is proposed by the expected proposer_index for the block's slot in the context of the current
   // shuffling (defined by parent_root/slot). If the proposer_index cannot immediately be verified against the expected
@@ -291,11 +290,6 @@ export async function validateGossipBlock(
     throw new BlockGossipError(GossipAction.REJECT, {code: BlockErrorCode.INCORRECT_PROPOSER, proposerIndex});
   }
 
-  // Check again in case there two blocks are processed concurrently
-  if (chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
-    throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.REPEAT_PROPOSAL, proposerIndex});
-  }
-
   // Simple implementation of a pending block queue. Keeping the block here recycles the queue logic, and keeps the
   // gossip validation promise without any extra infrastructure.
   // Do the sleep at the end, since regen and signature validation can already take longer than `msToBlockSlot`.
@@ -305,7 +299,35 @@ export async function validateGossipBlock(
     await sleep(msToBlockSlot);
   }
 
+  // Check again after all async validation and the early-block delay so concurrent proposals cannot both pass
+  if (chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
+    throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.REPEAT_PROPOSAL, proposerIndex});
+  }
+
   chain.seenBlockProposers.add(blockSlot, proposerIndex);
 
   return {skippedSlots};
 }
+
+export async function verifyBlockProposerSignature(
+  chain: IBeaconChain,
+  signedBlock: SignedBeaconBlock,
+  blockRoot: RootHex,
+  opts: {verifyOnMainThread?: boolean} = {}
+): Promise<void> {
+  const blockSlot = signedBlock.message.slot;
+  if (chain.seenBlockInputCache.isVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature)) {
+    return;
+  }
+
+  const signatureSet = getBlockProposerSignatureSet(chain.config, signedBlock);
+  // Don't batch so verification is not delayed
+  if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: opts.verifyOnMainThread ?? true}))) {
+    throw new BlockGossipError(GossipAction.REJECT, {
+      code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID,
+      blockSlot,
+    });
+  }
+
+  chain.seenBlockInputCache.markVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature);
+}

package/src/chain/validation/dataColumnSidecar.ts

@@ -2,15 +2,16 @@ import {ChainConfig, ChainForkConfig} from "@lodestar/config";
 import {
   KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH,
   KZG_COMMITMENTS_SUBTREE_INDEX,
+  MIN_SEED_LOOKAHEAD,
   NUMBER_OF_COLUMNS,
+  isForkPostFulu,
 } from "@lodestar/params";
 import {
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
   getBlockHeaderProposerSignatureSetByHeaderSlot,
-  getBlockHeaderProposerSignatureSetByParentStateSlot,
 } from "@lodestar/state-transition";
-import {DataColumnSidecar, Root, Slot, SubnetID, fulu, gloas, ssz} from "@lodestar/types";
+import {DataColumnSidecar, Root, Slot, SubnetID, ValidatorIndex, fulu, gloas, ssz} from "@lodestar/types";
 import {byteArrayEquals, toRootHex, verifyMerkleBranch} from "@lodestar/utils";
 import {BeaconMetrics} from "../../metrics/metrics/beacon.js";
 import {Metrics} from "../../metrics/metrics.js";
@@ -104,19 +105,23 @@ export async function validateGossipFuluDataColumnSidecar(
     });
   }
 
-  // getBlockSlotState also checks for whether the current finalized checkpoint is an ancestor of the block.
-  // As a result, we throw an IGNORE (whereas the spec says we should REJECT for this scenario).
-  // this is something we should change this in the future to make the code airtight to the spec.
   // 7) [REJECT] The sidecar's block's parent passes validation.
-  const blockState = await chain.regen
-    .getBlockSlotState(parentBlock, blockHeader.slot, {dontTransferCache: true}, RegenCaller.validateGossipDataColumn)
-    .catch(() => {
-      throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
-        code: DataColumnSidecarErrorCode.PARENT_UNKNOWN,
-        parentRoot,
-        slot: blockHeader.slot,
-      });
-    });
+  // Post-fulu, we can use parent state to get expected proposer index thanks to proposer lookahead
+  const parentEpoch = computeEpochAtSlot(parentBlock.slot);
+  const blockEpoch = computeEpochAtSlot(blockHeader.slot);
+  const getProposerIndex = async (): Promise<ValidatorIndex> => {
+    if (isForkPostFulu(chain.config.getForkName(parentBlock.slot)) && blockEpoch - parentEpoch <= MIN_SEED_LOOKAHEAD) {
+      const parentState = await chain.regen.getState(parentBlock.stateRoot, RegenCaller.validateGossipDataColumn);
+      return parentState.getBeaconProposer(blockHeader.slot);
+    }
+    const blockState = await chain.regen.getBlockSlotState(
+      parentBlock,
+      blockHeader.slot,
+      {dontTransferCache: true},
+      RegenCaller.validateGossipDataColumn
+    );
+    return blockState.getBeaconProposer(blockHeader.slot);
+  };
 
   // 13) [REJECT] The sidecar is proposed by the expected proposer_index for the block's slot in the context of the current
   //              shuffling (defined by block_header.parent_root/block_header.slot). If the proposer_index cannot
@@ -124,7 +129,13 @@ export async function validateGossipFuluDataColumnSidecar(
   //              while proposers for the block's branch are calculated -- in such a case do not REJECT, instead IGNORE
   //              this message.
   const proposerIndex = blockHeader.proposerIndex;
-  const expectedProposerIndex = blockState.getBeaconProposer(blockHeader.slot);
+  const expectedProposerIndex = await getProposerIndex().catch(() => {
+    throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
+      code: DataColumnSidecarErrorCode.PARENT_UNKNOWN,
+      parentRoot,
+      slot: blockHeader.slot,
+    });
+  });
 
   if (proposerIndex !== expectedProposerIndex) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
@@ -137,9 +148,8 @@ export async function validateGossipFuluDataColumnSidecar(
   // 5) [REJECT] The proposer signature of sidecar.signed_block_header, is valid with respect to the block_header.proposer_index pubkey.
   const signature = dataColumnSidecar.signedBlockHeader.signature;
   if (!chain.seenBlockInputCache.isVerifiedProposerSignature(blockHeader.slot, blockRootHex, signature)) {
-    const signatureSet = getBlockHeaderProposerSignatureSetByParentStateSlot(
+    const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot(
       chain.config,
-      blockState.slot,
       dataColumnSidecar.signedBlockHeader
     );
 

package/src/metrics/metrics/lodestar.ts

@@ -1179,6 +1179,10 @@ export function createLodestarMetrics(
         name: "lodestar_oppool_proposer_slashing_pool_size",
         help: "Current size of the ProposerSlashingPool",
       }),
+      proposerSlashingsProduced: register.counter({
+        name: "lodestar_oppool_proposer_slashings_produced_total",
+        help: "Total number of proposer slashings produced from observed equivocations",
+      }),
       voluntaryExitPoolSize: register.gauge({
         name: "lodestar_oppool_voluntary_exit_pool_size",
         help: "Current size of the VoluntaryExitPool",
@@ -1708,6 +1712,41 @@ export function createLodestarMetrics(
       }),
     },
 
+    // Pre-verify builder-deposit signatures in the epochs before the Gloas fork (prepareNextSlot).
+    builderDepositPreVerify: {
+      duration: register.histogram({
+        name: "lodestar_builder_deposit_preverify_duration_seconds",
+        help: "Duration of one preVerifyBuilderDepositsPreGloas call (BLS batch verify + scan)",
+        buckets: [1, 2, 3, 4],
+      }),
+      pendingDeposits: register.gauge({
+        name: "lodestar_builder_deposit_preverify_pending_deposits",
+        help: "Current state.pendingDeposits length (backlog to drain before the fork)",
+      }),
+      cachedDeposits: register.gauge({
+        name: "lodestar_builder_deposit_preverify_cached_deposits",
+        help: "Deposit signatures verified & cached so far this pre-fork window (valid + invalid)",
+      }),
+      scannedDeposits: register.gauge({
+        name: "lodestar_builder_deposit_preverify_scanned_deposits",
+        help: "Pending deposits examined by the last pre-verify call (< pending_deposits ⇒ cap hit)",
+      }),
+      builderPubkeys: register.gauge({
+        name: "lodestar_builder_deposit_preverify_builder_pubkeys",
+        help: "Distinct builder pubkeys tracked this window (validator deposits for these are pre-verified)",
+      }),
+      validSignatures: register.counter<{type: "builder" | "validator"}>({
+        name: "lodestar_builder_deposit_preverify_valid_signatures_total",
+        help: "Cumulative deposit signatures whose BLS verification passed (VALID), by credential type",
+        labelNames: ["type"],
+      }),
+      invalidSignatures: register.counter<{type: "builder" | "validator"}>({
+        name: "lodestar_builder_deposit_preverify_invalid_signatures_total",
+        help: "Cumulative deposit signatures whose BLS verification failed (INVALID), by credential type",
+        labelNames: ["type"],
+      }),
+    },
+
     // reprocess attestations
     reprocessApiAttestations: {
       total: register.gauge({

package/src/network/core/networkCore.ts

@@ -70,6 +70,15 @@ export type BaseNetworkInit = {
   initialCustodyGroupCount: number;
 };
 
+/** Counts of what is still keeping this thread's event loop alive, e.g. `TCPWrap=3,Timeout=1` */
+function formatActiveResources(): string {
+  const counts = new Map<string, number>();
+  for (const resource of process.getActiveResourcesInfo()) {
+    counts.set(resource, (counts.get(resource) ?? 0) + 1);
+  }
+  return Array.from(counts, ([name, count]) => `${name}=${count}`).join(",");
+}
+
 /**
  * This class is meant to work both:
  * - In a libp2p worker
@@ -289,7 +298,10 @@ export class NetworkCore implements INetworkCore {
     this.attnetsService.close();
     this.syncnetsService.close();
     await this.libp2p.stop();
-    this.logger.debug("network lib2p closed");
+    // Diagnostic for the shutdown hang, this thread can spin in `Environment::CleanupHandles()` on
+    // a handle that never closes and `Worker.terminate()` then never resolves. Diffing this list
+    // between a clean and a stuck shutdown should narrow down which handle it is
+    this.logger.debug("network lib2p closed", {activeResources: formatActiveResources()});
 
     this.closed = true;
   }

package/src/network/core/networkCoreWorkerHandler.ts

@@ -10,6 +10,7 @@ import {BeaconConfig, chainConfigToJson} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp";
 import {Status} from "@lodestar/types";
+import {withTimeout} from "@lodestar/utils";
 import {Metrics} from "../../metrics/index.js";
 import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js";
 import {PeerIdStr, peerIdFromString} from "../../util/peerId.js";
@@ -58,6 +59,10 @@ type WorkerNetworkCoreModules = WorkerNetworkCoreInitModules & {
 
 const NETWORK_WORKER_EXIT_TIMEOUT_MS = 1000;
 const NETWORK_WORKER_EXIT_RETRY_COUNT = 3;
+/** `getApi().close()` is an unbounded RPC into the worker and runs before `BeaconNode.close()`
+ * archives chain state to disk, so it is bounded here. Set well above the ~2-4s a close takes on
+ * mainnet, tripping it leaves libp2p half torn down */
+const NETWORK_CORE_CLOSE_TIMEOUT_MS = 5000;
 
 /**
  * NetworkCore implementation using a Worker thread
@@ -165,15 +170,25 @@ export class WorkerNetworkCore implements INetworkCore {
 
   async close(): Promise<void> {
     this.modules.logger.debug("closing network core running in network worker");
-    await this.getApi().close();
+    try {
+      await withTimeout(async () => this.getApi().close(), NETWORK_CORE_CLOSE_TIMEOUT_MS);
+    } catch (e) {
+      this.modules.logger.debug("Error closing network core", {}, e as Error);
+    }
     this.modules.logger.debug("terminating network worker");
-    await terminateWorkerThread({
+    const terminated = await terminateWorkerThread({
       worker: this.getApi(),
       retryCount: NETWORK_WORKER_EXIT_RETRY_COUNT,
       retryMs: NETWORK_WORKER_EXIT_TIMEOUT_MS,
       logger: this.modules.logger,
     });
-    this.modules.logger.debug("terminated network worker");
+    if (terminated) {
+      this.modules.logger.debug("terminated network worker");
+    } else {
+      // Nothing more to do, `process.exit()` joins the thread regardless so the process still waits
+      // on it. What matters is returning, so the caller can archive state and close the db
+      this.modules.logger.debug("Network worker did not terminate in time, continuing shutdown without it");
+    }
   }
 
   async test(): Promise<void> {

package/src/network/network.ts

@@ -147,6 +147,7 @@ export class Network implements INetwork {
     this.chain.emitter.on(ChainEvent.updateTargetCustodyGroupCount, this.onTargetGroupCountUpdated);
     this.chain.emitter.on(ChainEvent.publishDataColumns, this.onPublishDataColumns);
     this.chain.emitter.on(ChainEvent.publishBlobSidecars, this.onPublishBlobSidecars);
+    this.chain.emitter.on(ChainEvent.publishProposerSlashing, this.onPublishProposerSlashing);
     this.chain.emitter.on(ChainEvent.updateStatus, this.onUpdateStatus);
   }
 
@@ -244,6 +245,7 @@ export class Network implements INetwork {
     this.chain.emitter.off(ChainEvent.updateTargetCustodyGroupCount, this.onTargetGroupCountUpdated);
     this.chain.emitter.off(ChainEvent.publishDataColumns, this.onPublishDataColumns);
     this.chain.emitter.off(ChainEvent.publishBlobSidecars, this.onPublishBlobSidecars);
+    this.chain.emitter.off(ChainEvent.publishProposerSlashing, this.onPublishProposerSlashing);
     this.chain.emitter.off(ChainEvent.updateStatus, this.onUpdateStatus);
     await this.core.close();
 
@@ -455,7 +457,8 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.proposer_slashing>(
       {type: GossipType.proposer_slashing, boundary},
-      proposerSlashing
+      proposerSlashing,
+      {ignoreDuplicatePublishError: true}
     );
   }
 
@@ -877,6 +880,18 @@ export class Network implements INetwork {
     return promiseAllMaybeAsync(sidecars.map((sidecar) => () => this.publishBlobSidecar(sidecar)));
   };
 
+  private onPublishProposerSlashing = async (proposerSlashing: phase0.ProposerSlashing): Promise<void> => {
+    try {
+      await this.publishProposerSlashing(proposerSlashing);
+    } catch (e) {
+      this.logger.debug(
+        "Error publishing proposer slashing",
+        {proposerIndex: proposerSlashing.signedHeader1.message.proposerIndex},
+        e as Error
+      );
+    }
+  };
+
   private onUpdateStatus = async (): Promise<void> => {
     await this.core.updateStatus(this.chain.getStatus());
   };

package/src/network/processor/gossipHandlers.ts

@@ -257,6 +257,13 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         chain.seenPayloadEnvelopeInputCache.prune(blockRootHex);
       }
       throw e;
+    } finally {
+      // The block received from the network may have established an equivocation, either by conflicting
+      // with a previously observed block root (REPEAT_PROPOSAL) or with a root observed during validation
+      const proposerIndex = signedBlock.message.proposerIndex;
+      if (chain.seenBlockProposers.isEquivocating(slot, proposerIndex)) {
+        chain.processProposerEquivocation(slot, proposerIndex);
+      }
     }
   }
 

package/src/network/reqresp/score.ts

@@ -33,9 +33,28 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod):
 
     case RequestErrorCode.DIAL_TIMEOUT:
     case RequestErrorCode.DIAL_ERROR:
-      return e.message.includes(multiStreamSelectErrorCodes.protocolSelectionFailed) && method === ReqRespMethod.Ping
-        ? PeerAction.Fatal
-        : PeerAction.LowToleranceError;
+      // `RequestError` renders `e.message` as just the error code (see `renderErrorMessage`), so the
+      // multistream "protocol selection failed" text is only on the wrapped inner error, which is
+      // available on `DIAL_ERROR` via `e.type.error` (`DIAL_TIMEOUT` carries no inner error).
+      if (
+        e.type.code === RequestErrorCode.DIAL_ERROR &&
+        e.type.error.message.includes(multiStreamSelectErrorCodes.protocolSelectionFailed)
+      ) {
+        // Peer does not support the protocol, a real incompatibility rather than a transient
+        // failure, so keep the stronger penalty (Fatal for Ping, as before).
+        return method === ReqRespMethod.Ping ? PeerAction.Fatal : PeerAction.LowToleranceError;
+      }
+      switch (method) {
+        // Ping and Status are liveness probes; their dial timeouts are dominated by transient
+        // network congestion rather than peer misbehavior, so penalize leniently to avoid
+        // self-inflicted peer starvation (https://github.com/ChainSafe/lodestar/issues/9562),
+        // while still applying some penalty so genuinely dead peers eventually free the slot.
+        case ReqRespMethod.Ping:
+        case ReqRespMethod.Status:
+          return PeerAction.HighToleranceError;
+        default:
+          return PeerAction.LowToleranceError;
+      }
     // TODO: Detect SSZDecodeError and return PeerAction.Fatal
 
     case RequestErrorCode.RESP_TIMEOUT:

package/src/sync/range/batch.ts

@@ -5,6 +5,7 @@ import {LodestarError, byteArrayEquals, prettyPrintIndices, toRootHex} from "@lo
 import {isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {isDaOutOfRange} from "../../chain/blocks/blockInput/utils.js";
+import {PayloadError, PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {PayloadEnvelopeInput} from "../../chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
 import {BlockError, BlockErrorCode} from "../../chain/errors/index.js";
 import {ZERO_HASH} from "../../constants/constants.js";
@@ -592,7 +593,7 @@ export class Batch {
     if (allComplete) {
       const attempt: Attempt = {
         peers: this.getSuccessfulPeers(),
-        hash: hashBlocks(blocks, this.config),
+        hash: hashBlocks(blocks, payloadEnvelopes),
       };
       this.state = {status: BatchStatus.AwaitingProcessing, blocks, payloadEnvelopes: newPayloadEnvelopes, attempt};
     } else {
@@ -702,7 +703,7 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.Processing));
     }
 
-    if (err instanceof BlockError && err.type.code === BlockErrorCode.EXECUTION_ENGINE_ERROR) {
+    if (isExecutionEngineError(err)) {
       this.onExecutionEngineError(this.state.attempt);
     } else {
       this.onProcessingError(this.state.attempt);
@@ -717,7 +718,7 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingValidation));
     }
 
-    if (err instanceof BlockError && err.type.code === BlockErrorCode.EXECUTION_ENGINE_ERROR) {
+    if (isExecutionEngineError(err)) {
       this.onExecutionEngineError(this.state.attempt);
     } else {
       this.onProcessingError(this.state.attempt);
@@ -787,3 +788,19 @@ type BatchErrorMetadata = {
 };
 
 export class BatchError extends LodestarError<BatchErrorType & BatchErrorMetadata> {}
+
+function isExecutionEngineError(err: Error): boolean {
+  if (!(err instanceof BlockError)) {
+    return false;
+  }
+
+  if (err.type.code === BlockErrorCode.EXECUTION_ENGINE_ERROR) {
+    return true;
+  }
+
+  return (
+    err.type.code === BlockErrorCode.BEACON_CHAIN_ERROR &&
+    err.type.error instanceof PayloadError &&
+    err.type.error.type.code === PayloadErrorCode.EXECUTION_ENGINE_ERROR
+  );
+}

package/src/sync/range/utils/hashBlocks.ts

@@ -1,27 +1,62 @@
-import {ChainForkConfig} from "@lodestar/config";
-import {RootHex, SignedBeaconBlock} from "@lodestar/types";
-import {toRootHex} from "@lodestar/utils";
+import {digest} from "@chainsafe/as-sha256";
+import {RootHex, Slot, ssz} from "@lodestar/types";
+import {fromHex, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../../chain/blocks/blockInput/types.js";
+import {PayloadEnvelopeInput} from "../../../chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
+
+const ROOT_SIZE = 32;
+const SIGNATURE_SIZE = 96;
+/** Every block and every payload envelope contributes a fixed-size (root, signature) entry */
+const ENTRY_SIZE = ROOT_SIZE + SIGNATURE_SIZE;
+/** uint32 LE counts of block and envelope entries, so the variable-length sections are self-describing */
+const COUNT_SIZE = 4;
+const HEADER_SIZE = COUNT_SIZE * 2;
 
 /**
- * String to uniquely identify block segments. Used for peer scoring and to compare if batches are equivalent.
+ * Root uniquely identifying a batch attempt (its blocks AND payloads). Used for peer scoring and to
+ * compare if two attempts are equivalent.
+ *
+ * Signatures are part of the id for BOTH blocks and payload envelopes, not just their message roots:
+ * a peer serving a correct message with a garbage signature would otherwise produce the same id as
+ * the honest attempt and escape scoring in `SyncChain.advanceChain`. Signatures are not verified at
+ * download time, only during processChainSegment (block signatures in verifyBlocksSignatures,
+ * envelope signatures in importExecutionPayload).
+ *
+ * Entries are written as raw bytes into a single exactly-sized buffer and digested once, so an
+ * `Attempt` retains 32 bytes rather than the ~16 KB a full 32-slot gloas batch would need if the
+ * roots and signatures were concatenated as hex. sha256 (not a 64-bit hash) because a peer that
+ * could force a collision with the winning attempt would escape peer scoring.
  */
-export function hashBlocks(blocks: IBlockInput[], config: ChainForkConfig): RootHex {
-  switch (blocks.length) {
-    case 0:
-      return "0x";
-    case 1: {
-      const block0 = blocks[0].getBlock();
-      return toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0));
-    }
-    default: {
-      const block0 = blocks[0].getBlock();
-      const blockN = blocks.at(-1)?.getBlock() as SignedBeaconBlock;
-      return (
-        // TODO(fulu): should we be doing checks for presence to make sure these do not blow up?
-        toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0)) +
-        toRootHex(config.getForkTypes(blockN.message.slot).SignedBeaconBlock.hashTreeRoot(blockN))
-      );
-    }
+export function hashBlocks(blocks: IBlockInput[], payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null): RootHex {
+  // Envelopes without a payload carry no attributable data, so they are skipped entirely. A `null`
+  // map and a map whose entries all lack a payload are therefore equivalent, which is intended.
+  const envelopes =
+    payloadEnvelopes && payloadEnvelopes.size > 0
+      ? Array.from(payloadEnvelopes.entries())
+          .filter(([, envelope]) => envelope.hasPayloadEnvelope())
+          .sort(([slotA], [slotB]) => slotA - slotB)
+      : [];
+
+  const buf = Buffer.allocUnsafe(HEADER_SIZE + (blocks.length + envelopes.length) * ENTRY_SIZE);
+  buf.writeUInt32LE(blocks.length, 0);
+  buf.writeUInt32LE(envelopes.length, COUNT_SIZE);
+  let offset = HEADER_SIZE;
+
+  for (const block of blocks) {
+    buf.set(fromHex(block.blockRootHex), offset);
+    offset += ROOT_SIZE;
+    buf.set(block.getBlock().signature, offset);
+    offset += SIGNATURE_SIZE;
   }
+
+  for (const [, envelope] of envelopes) {
+    const signedEnvelope = envelope.getPayloadEnvelope();
+    // envelope's root is cached thanks to cachePermanentRootStruct, so this avoids re-hashing
+    buf.set(ssz.gloas.ExecutionPayloadEnvelope.hashTreeRoot(signedEnvelope.message), offset);
+    offset += ROOT_SIZE;
+    buf.set(signedEnvelope.signature, offset);
+    offset += SIGNATURE_SIZE;
+  }
+
+  return toRootHex(digest(buf));
 }