@lodestar/beacon-node 1.44.0-dev.4ac6f45830 → 1.44.0-dev.4b91ba8c2f

package/src/chain/produceBlock/produceBlockBody.ts

@@ -108,7 +108,7 @@ export type ProduceFullGloas = {
   type: BlockType.Full;
   fork: ForkPostGloas;
   executionPayload: ExecutionPayload<ForkPostGloas>;
-  executionRequests: electra.ExecutionRequests;
+  executionRequests: gloas.ExecutionRequests;
   blobsBundle: BlobsBundle<ForkPostGloas>;
   cells: fulu.Cell[][];
   parentBlockRoot: Root;
@@ -228,7 +228,7 @@ export async function produceBlockBody<T extends BlockType>(
     );
     const parentExecutionRequests = isExtendingPayload
       ? await this.getParentExecutionRequests(parentBlock.slot, parentBlock.blockRoot)
-      : ssz.electra.ExecutionRequests.defaultValue();
+      : ssz.gloas.ExecutionRequests.defaultValue();
     executionPayloadValue = BigInt(builderBid.message.value) * GWEI_TO_WEI;
 
     const commonBlockBody = await commonBlockBodyPromise;
@@ -272,7 +272,7 @@ export async function produceBlockBody<T extends BlockType>(
 
     // Get execution payload from EL
     let parentBlockHash: Bytes32;
-    let parentExecutionRequests: electra.ExecutionRequests;
+    let parentExecutionRequests: gloas.ExecutionRequests;
     // Apply parent payload once here as it's reused by EL prep and voluntary exit filtering below
     let stateAfterParentPayload: IBeaconStateViewBellatrix = currentState;
     // Spec: should_build_on_full(store, head). `parentBlock` is the proposer's head
@@ -285,7 +285,7 @@ export async function produceBlockBody<T extends BlockType>(
       stateAfterParentPayload = currentState.withParentPayloadApplied(parentExecutionRequests);
     } else {
       parentBlockHash = currentState.latestExecutionPayloadBid.parentBlockHash;
-      parentExecutionRequests = ssz.electra.ExecutionRequests.defaultValue();
+      parentExecutionRequests = ssz.gloas.ExecutionRequests.defaultValue();
     }
     const prepareRes = await prepareExecutionPayload(
       this,
@@ -344,13 +344,13 @@ export async function produceBlockBody<T extends BlockType>(
       blockHash: executionPayload.blockHash,
       prevRandao: currentState.getRandaoMix(currentState.epoch),
       feeRecipient: executionPayload.feeRecipient,
-      gasLimit: BigInt(executionPayload.gasLimit),
+      gasLimit: executionPayload.gasLimit,
       builderIndex: BUILDER_INDEX_SELF_BUILD,
       slot: blockSlot,
       value: 0,
       executionPayment: 0,
       blobKzgCommitments: blobsBundle.commitments,
-      executionRequestsRoot: ssz.electra.ExecutionRequests.hashTreeRoot(executionRequests),
+      executionRequestsRoot: ssz.gloas.ExecutionRequests.hashTreeRoot(executionRequests as gloas.ExecutionRequests),
     };
     const signedBid: gloas.SignedExecutionPayloadBid = {
       message: bid,
@@ -375,7 +375,7 @@ export async function produceBlockBody<T extends BlockType>(
     // Store execution payload data required to construct execution payload envelope later
     const gloasResult = produceResult as ProduceFullGloas;
     gloasResult.executionPayload = executionPayload as ExecutionPayload<ForkPostGloas>;
-    gloasResult.executionRequests = executionRequests;
+    gloasResult.executionRequests = executionRequests as gloas.ExecutionRequests;
     gloasResult.blobsBundle = blobsBundle;
     gloasResult.cells = cells;
     gloasResult.parentBlockRoot = fromHex(parentBlock.blockRoot);

package/src/chain/validation/block.ts

@@ -15,12 +15,17 @@ import {BlockErrorCode, BlockGossipError, GossipAction} from "../errors/index.js
 import {IBeaconChain} from "../interface.js";
 import {RegenCaller} from "../regen/index.js";
 
+export type GossipBlockValidationResult = {
+  /** Number of skipped slots between the block and its parent (blockSlot - parentSlot - 1) */
+  skippedSlots: number;
+};
+
 export async function validateGossipBlock(
   config: ChainForkConfig,
   chain: IBeaconChain,
   signedBlock: SignedBeaconBlock,
   fork: ForkName
-): Promise<void> {
+): Promise<GossipBlockValidationResult> {
   const block = signedBlock.message;
   const blockSlot = block.slot;
   const blockEpoch = computeEpochAtSlot(blockSlot);
@@ -109,21 +114,6 @@ export async function validateGossipBlock(
     }
   }
 
-  // [IGNORE] The attestation head block is too far behind the attestation slot, causing many skip slots.
-  // This is deemed a DoS risk because we need to get the proposerShuffling. To get the shuffling we have
-  // to do a bunch of epoch transitions, the longer the distance between the parent and block,
-  // the more we have to do. epochTransitions are expensive ~750ms, so we must limit how many a
-  // single bad block can trigger
-  // Note: Ensure this check is done before calling chain.regen.getBlockSlotStat as this is the function that does various epoch transitions.
-  // Note: This validation check is not part of the spec.
-  if (chain.opts.maxSkipSlots != null && parentBlock.slot + chain.opts.maxSkipSlots < blockSlot) {
-    throw new BlockGossipError(GossipAction.IGNORE, {
-      code: BlockErrorCode.TOO_MANY_SKIPPED_SLOTS,
-      parentSlot: parentBlock.slot,
-      blockSlot,
-    });
-  }
-
   // [REJECT] The block is from a higher slot than its parent.
   if (parentBlock.slot >= blockSlot) {
     throw new BlockGossipError(GossipAction.REJECT, {
@@ -133,6 +123,10 @@ export async function validateGossipBlock(
     });
   }
 
+  // Number of skipped slots between block and parent (non-spec). Previously this gated blocks via
+  // maxSkipSlots; now the caller only observes it so legitimate post-skip blocks are no longer ignored.
+  const skippedSlots = blockSlot - parentBlock.slot - 1;
+
   // [REJECT] The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer -- i.e. validate that len(body.signed_beacon_block.message.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK
   if (isForkPostDeneb(fork) && !isForkPostGloas(fork)) {
     const blobKzgCommitmentsLen = (block as deneb.BeaconBlock).body.blobKzgCommitments.length;
@@ -247,4 +241,6 @@ export async function validateGossipBlock(
   }
 
   chain.seenBlockProposers.add(blockSlot, proposerIndex);
+
+  return {skippedSlots};
 }

package/src/chain/validation/executionPayloadBid.ts

@@ -1,4 +1,5 @@
 import {PublicKey} from "@chainsafe/blst";
+import {PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
 import {
   computeEpochAtSlot,
   createSingleSignatureSetFromComponents,
@@ -139,6 +140,17 @@ async function validateExecutionPayloadBid(
     });
   }
 
+  // [REJECT] The builder version is `PAYLOAD_BUILDER_VERSION` -- i.e.
+  // `state.builders[bid.builder_index].version == PAYLOAD_BUILDER_VERSION`.
+  if (builder.version !== PAYLOAD_BUILDER_VERSION) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.INVALID_BUILDER_VERSION,
+      builderIndex: bid.builderIndex,
+      version: builder.version,
+      expectedVersion: PAYLOAD_BUILDER_VERSION,
+    });
+  }
+
   // [REJECT] `bid.execution_payment` is zero.
   if (bid.executionPayment !== 0) {
     throw new ExecutionPayloadBidError(GossipAction.REJECT, {
@@ -175,7 +187,7 @@ async function validateExecutionPayloadBid(
   // [IGNORE] `is_gas_limit_target_compatible(parent_gas_limit, bid.gas_limit, target_gas_limit)`,
   // where `parent_gas_limit` is the `gas_limit` of the parent execution payload and
   // `target_gas_limit` is `proposer_preferences.target_gas_limit`.
-  const bidGasLimit = Number(bid.gasLimit);
+  const bidGasLimit = bid.gasLimit;
   const parentGasLimit = parentPayloadVariant.executionPayloadGasLimit;
   const targetGasLimit = proposerPreferences.message.targetGasLimit;
   if (!isGasLimitTargetCompatible(parentGasLimit, bidGasLimit, targetGasLimit)) {
@@ -232,6 +244,18 @@ async function validateExecutionPayloadBid(
     });
   }
 
+  // [REJECT] `bid.prev_randao` is the correct RANDAO mix -- i.e. validate that
+  // `bid.prev_randao == get_randao_mix(parent_state, get_current_epoch(parent_state))`.
+  const randaoMix = state.getRandaoMix(computeEpochAtSlot(state.slot));
+  if (!byteArrayEquals(bid.prevRandao, randaoMix)) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.INVALID_PREV_RANDAO,
+      builderIndex: bid.builderIndex,
+      bidPrevRandao: toHex(bid.prevRandao),
+      expectedPrevRandao: toHex(randaoMix),
+    });
+  }
+
   // [REJECT] `signed_execution_payload_bid.signature` is valid with respect to the `bid.builder_index`.
   const signatureSet = createSingleSignatureSetFromComponents(
     PublicKey.fromBytes(builder.pubkey),

package/src/chain/validation/executionPayloadEnvelope.ts

@@ -35,8 +35,6 @@ async function validateExecutionPayloadEnvelope(
   // [IGNORE] The envelope's block root `envelope.beacon_block_root` has been seen (via
   // gossip or non-gossip sources) (a client MAY queue payload for processing once
   // the block is retrieved).
-  // TODO GLOAS: Need to review this, we should queue the envelope for later
-  // processing if the block is not yet known, otherwise we would ignore it here
   const block = chain.forkChoice.getBlockDefaultStatus(envelope.beaconBlockRoot);
   if (block === null) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, {
@@ -108,7 +106,7 @@ async function validateExecutionPayloadEnvelope(
   }
 
   // [REJECT] `hash_tree_root(envelope.execution_requests) == bid.execution_requests_root`
-  const requestsRoot = ssz.electra.ExecutionRequests.hashTreeRoot(envelope.executionRequests);
+  const requestsRoot = ssz.gloas.ExecutionRequests.hashTreeRoot(envelope.executionRequests);
   if (!byteArrayEquals(requestsRoot, payloadInput.getBid().executionRequestsRoot)) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_ROOT_MISMATCH,

package/src/execution/engine/http.ts

@@ -244,7 +244,7 @@ export class ExecutionEngineHttp implements IExecutionEngine {
         if (executionRequests === undefined) {
           throw Error(`executionRequests required in notifyNewPayload for fork=${fork}`);
         }
-        const serializedExecutionRequests = serializeExecutionRequests(executionRequests);
+        const serializedExecutionRequests = serializeExecutionRequests(fork, executionRequests);
         engineRequest = {
           method: ForkSeq[fork] >= ForkSeq.gloas ? "engine_newPayloadV5" : "engine_newPayloadV4",
           params: [

package/src/execution/engine/interface.ts

@@ -1,5 +1,7 @@
 import {routes} from "@lodestar/api";
 import {
+  BUILDER_DEPOSIT_REQUEST_TYPE,
+  BUILDER_EXIT_REQUEST_TYPE,
   CONSOLIDATION_REQUEST_TYPE,
   DEPOSIT_REQUEST_TYPE,
   ForkName,
@@ -51,10 +53,18 @@ export enum ExecutionEngineState {
 export type ExecutionRequestType =
   | typeof DEPOSIT_REQUEST_TYPE
   | typeof WITHDRAWAL_REQUEST_TYPE
-  | typeof CONSOLIDATION_REQUEST_TYPE;
+  | typeof CONSOLIDATION_REQUEST_TYPE
+  | typeof BUILDER_DEPOSIT_REQUEST_TYPE
+  | typeof BUILDER_EXIT_REQUEST_TYPE;
 
 export function isExecutionRequestType(type: number): type is ExecutionRequestType {
-  return type === DEPOSIT_REQUEST_TYPE || type === WITHDRAWAL_REQUEST_TYPE || type === CONSOLIDATION_REQUEST_TYPE;
+  return (
+    type === DEPOSIT_REQUEST_TYPE ||
+    type === WITHDRAWAL_REQUEST_TYPE ||
+    type === CONSOLIDATION_REQUEST_TYPE ||
+    type === BUILDER_DEPOSIT_REQUEST_TYPE ||
+    type === BUILDER_EXIT_REQUEST_TYPE
+  );
 }
 
 export type ExecutePayloadResponse =

package/src/execution/engine/mock.ts

@@ -393,11 +393,12 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
           blobs,
           proofs,
         }),
-        executionRequests: serializeExecutionRequests({
-          deposits: ssz.electra.DepositRequests.defaultValue(),
-          withdrawals: ssz.electra.WithdrawalRequests.defaultValue(),
-          consolidations: ssz.electra.ConsolidationRequests.defaultValue(),
-        }),
+        executionRequests: serializeExecutionRequests(
+          fork,
+          ForkSeq[fork] >= ForkSeq.gloas
+            ? ssz.gloas.ExecutionRequests.defaultValue()
+            : ssz.electra.ExecutionRequests.defaultValue()
+        ),
       });
 
       // IF the payload is deemed VALID and the build process has begun

package/src/execution/engine/types.ts

@@ -1,4 +1,6 @@
 import {
+  BUILDER_DEPOSIT_REQUEST_TYPE,
+  BUILDER_EXIT_REQUEST_TYPE,
   BYTES_PER_FIELD_ELEMENT,
   BYTES_PER_LOGS_BLOOM,
   CELLS_PER_EXT_BLOB,
@@ -205,16 +207,27 @@ export type WithdrawalRpc = {
 };
 
 /**
- * ExecutionRequestsRpc only holds at most 3 elements and no repeated type:
- * - ssz'ed DepositRequests
- * - ssz'ed WithdrawalRequests
- * - ssz'ed ConsolidationRequests
+ * ExecutionRequestsRpc holds at most 5 elements and no repeated type, ordered by
+ * the single-byte request type prefix:
+ * - ssz'ed DepositRequests          (0x00)
+ * - ssz'ed WithdrawalRequests       (0x01)
+ * - ssz'ed ConsolidationRequests    (0x02)
+ * - ssz'ed BuilderDepositRequests   (0x03)
+ * - ssz'ed BuilderExitRequests      (0x04)
  */
-export type ExecutionRequestsRpc = (DepositRequestsRpc | WithdrawalRequestsRpc | ConsolidationRequestsRpc)[];
+export type ExecutionRequestsRpc = (
+  | DepositRequestsRpc
+  | WithdrawalRequestsRpc
+  | ConsolidationRequestsRpc
+  | BuilderDepositRequestsRpc
+  | BuilderExitRequestsRpc
+)[];
 
 export type DepositRequestsRpc = DATA;
 export type WithdrawalRequestsRpc = DATA;
 export type ConsolidationRequestsRpc = DATA;
+export type BuilderDepositRequestsRpc = DATA;
+export type BuilderExitRequestsRpc = DATA;
 
 export type BlobAndProofRpc = {
   blob: DATA;
@@ -339,7 +352,7 @@ export function parseExecutionPayload(
     data = response.executionPayload;
     blobsBundle = response.blobsBundle ? parseBlobsBundle(response.blobsBundle) : undefined;
     executionRequests = response.executionRequests
-      ? deserializeExecutionRequests(response.executionRequests)
+      ? deserializeExecutionRequests(fork, response.executionRequests)
       : undefined;
     shouldOverrideBuilder = response.shouldOverrideBuilder ?? false;
   } else {
@@ -526,13 +539,36 @@ function deserializeConsolidationRequests(serialized: ConsolidationRequestsRpc):
   return ssz.electra.ConsolidationRequests.deserialize(dataToBytes(serialized, null));
 }
 
+function serializeBuilderDepositRequests(
+  builderDepositRequests: gloas.BuilderDepositRequests
+): BuilderDepositRequestsRpc {
+  const requestsBytes = ssz.gloas.BuilderDepositRequests.serialize(builderDepositRequests);
+  return bytesToData(prefixRequests(requestsBytes, BUILDER_DEPOSIT_REQUEST_TYPE));
+}
+
+function deserializeBuilderDepositRequests(serialized: BuilderDepositRequestsRpc): gloas.BuilderDepositRequests {
+  return ssz.gloas.BuilderDepositRequests.deserialize(dataToBytes(serialized, null));
+}
+
+function serializeBuilderExitRequests(builderExitRequests: gloas.BuilderExitRequests): BuilderExitRequestsRpc {
+  const requestsBytes = ssz.gloas.BuilderExitRequests.serialize(builderExitRequests);
+  return bytesToData(prefixRequests(requestsBytes, BUILDER_EXIT_REQUEST_TYPE));
+}
+
+function deserializeBuilderExitRequests(serialized: BuilderExitRequestsRpc): gloas.BuilderExitRequests {
+  return ssz.gloas.BuilderExitRequests.deserialize(dataToBytes(serialized, null));
+}
+
 /**
  * This is identical to get_execution_requests_list in
  * https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.8/specs/electra/beacon-chain.md#new-get_execution_requests_list
+ *
+ * Gloas extends the list with builder deposits (0x03) and builder exits (0x04) per
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.11/specs/gloas/beacon-chain.md#modified-get_execution_requests_list
  */
-export function serializeExecutionRequests(executionRequests: ExecutionRequests): ExecutionRequestsRpc {
+export function serializeExecutionRequests(fork: ForkName, executionRequests: ExecutionRequests): ExecutionRequestsRpc {
   const {deposits, withdrawals, consolidations} = executionRequests;
-  const result = [];
+  const result: ExecutionRequestsRpc = [];
 
   if (deposits.length !== 0) {
     result.push(serializeDepositRequests(deposits));
@@ -546,15 +582,26 @@ export function serializeExecutionRequests(executionRequests: ExecutionRequests)
     result.push(serializeConsolidationRequests(consolidations));
   }
 
+  if (ForkSeq[fork] >= ForkSeq.gloas) {
+    const {builderDeposits, builderExits} = executionRequests as gloas.ExecutionRequests;
+
+    if (builderDeposits.length !== 0) {
+      result.push(serializeBuilderDepositRequests(builderDeposits));
+    }
+
+    if (builderExits.length !== 0) {
+      result.push(serializeBuilderExitRequests(builderExits));
+    }
+  }
+
   return result;
 }
 
-export function deserializeExecutionRequests(serialized: ExecutionRequestsRpc): ExecutionRequests {
-  const result: ExecutionRequests = {
-    deposits: [],
-    withdrawals: [],
-    consolidations: [],
-  };
+export function deserializeExecutionRequests(fork: ForkName, serialized: ExecutionRequestsRpc): ExecutionRequests {
+  const result: ExecutionRequests =
+    ForkSeq[fork] >= ForkSeq.gloas
+      ? {deposits: [], withdrawals: [], consolidations: [], builderDeposits: [], builderExits: []}
+      : {deposits: [], withdrawals: [], consolidations: []};
 
   if (serialized.length === 0) {
     return result;
@@ -601,6 +648,20 @@ export function deserializeExecutionRequests(serialized: ExecutionRequestsRpc):
         result.consolidations = deserializeConsolidationRequests(requests);
         break;
       }
+      case BUILDER_DEPOSIT_REQUEST_TYPE: {
+        if (ForkSeq[fork] < ForkSeq.gloas) {
+          throw Error(`Builder deposit request is not supported pre-gloas fork=${fork}`);
+        }
+        (result as gloas.ExecutionRequests).builderDeposits = deserializeBuilderDepositRequests(requests);
+        break;
+      }
+      case BUILDER_EXIT_REQUEST_TYPE: {
+        if (ForkSeq[fork] < ForkSeq.gloas) {
+          throw Error(`Builder exit request is not supported pre-gloas fork=${fork}`);
+        }
+        (result as gloas.ExecutionRequests).builderExits = deserializeBuilderExitRequests(requests);
+        break;
+      }
     }
     prevRequestType = currentRequestType;
   }

package/src/metrics/metrics/lodestar.ts

@@ -610,6 +610,11 @@ export function createLodestarMetrics(
         help: "The origination source of one of the BlockInputSync triggers",
         labelNames: ["source"],
       }),
+      payloadSource: register.counter<{source: BlockInputSource}>({
+        name: "lodestar_payload_input_sync_source_total",
+        help: "Count of payload (execution payload envelope) sync triggers, labeled by their source",
+        labelNames: ["source"],
+      }),
       pendingBlocks: register.gauge({
         name: "lodestar_sync_unknown_block_pending_blocks_size",
         help: "Current size of UnknownBlockSync pending blocks cache",
@@ -861,6 +866,12 @@ export function createLodestarMetrics(
         labelNames: ["numBlobs"],
       }),
 
+      skippedSlots: register.histogram({
+        name: "lodestar_gossip_block_skipped_slots",
+        help: "Number of skipped slots between a gossip block and its parent (blockSlot - parentSlot - 1)",
+        buckets: [0, 1, 2, 4, 8, 16, 32],
+      }),
+
       processBlockErrors: register.gauge<{error: BlockErrorCode | "NOT_BLOCK_ERROR"}>({
         name: "lodestar_gossip_block_process_block_errors",
         help: "Count of errors, by error type, while processing blocks",

package/src/network/processor/gossipHandlers.ts

@@ -185,7 +185,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       peerIdStr,
     });
     try {
-      await validateGossipBlock(config, chain, signedBlock, fork);
+      const {skippedSlots} = await validateGossipBlock(config, chain, signedBlock, fork);
 
       if (isForkPostGloas(fork)) {
         chain.seenPayloadEnvelopeInputCache.add({
@@ -205,8 +205,15 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
       metrics?.gossipBlock.gossipValidation.recvToValidation.observe(recvToValidation);
       metrics?.gossipBlock.gossipValidation.validationTime.observe(validationTime);
+      metrics?.gossipBlock.skippedSlots.observe(skippedSlots);
 
-      logger.debug("Validated gossip block", {...blockInputMeta, ...logCtx, recvToValidation, validationTime});
+      logger.debug("Validated gossip block", {
+        ...blockInputMeta,
+        ...logCtx,
+        recvToValidation,
+        validationTime,
+        skippedSlots,
+      });
 
       chain.emitter.emit(routes.events.EventType.blockGossip, {slot, block: blockRootHex});
 

package/src/network/processor/index.ts

@@ -186,9 +186,11 @@ export class NetworkProcessor {
   // we may not receive the block for messages like Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
   // to be stored in this Map and reprocessed once the block comes
   private readonly awaitingMessagesByBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private awaitingBlockMessageCount = 0;
   // we may not receive the payload for messages that require the FULL payload variant to be processed,
   // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
   private readonly awaitingMessagesByPayloadBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private awaitingPayloadMessageCount = 0;
   private unknownBlocksBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
   private unknownEnvelopesBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
 
@@ -228,8 +230,8 @@ export class NetworkProcessor {
           metrics.gossipValidationQueue.keySize.set({topic}, this.gossipQueues[topic].keySize);
           metrics.gossipValidationQueue.concurrency.set({topic}, this.gossipTopicConcurrency[topic]);
         }
-        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
-        metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.unknownPayloadGossipsubMessagesCount);
+        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.awaitingBlockMessageCount);
+        metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.awaitingPayloadMessageCount);
         // specific metric for beacon_attestation topic
         metrics.gossipValidationQueue.keyAge.reset();
         for (const ageMs of this.gossipQueues.beacon_attestation.getDataAgeMs()) {
@@ -297,7 +299,7 @@ export class NetworkProcessor {
       return;
     }
     this.unknownEnvelopesBySlot.getOrDefault(slot).add(root);
-    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, peer, source});
+    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
   }
 
   private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
@@ -497,7 +499,7 @@ export class NetworkProcessor {
         this.pushPendingGossipsubMessageToQueue(message);
         break;
       case PreprocessAction.AwaitBlock: {
-        if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
+        if (this.awaitingBlockMessageCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
           // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           this.metrics?.awaitingBlockGossipMessages.reject.inc({
             reason: ReprocessRejectReason.reached_limit,
@@ -509,10 +511,11 @@ export class NetworkProcessor {
         this.metrics?.awaitingBlockGossipMessages.queue.inc({topic: topicType});
         const awaitingGossipsubMessages = this.awaitingMessagesByBlockRoot.getOrDefault(preprocessResult.root);
         awaitingGossipsubMessages.add(message);
+        this.awaitingBlockMessageCount++;
         break;
       }
       case PreprocessAction.AwaitEnvelope: {
-        if (this.unknownPayloadGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
+        if (this.awaitingPayloadMessageCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
           this.metrics?.awaitingPayloadGossipMessages.reject.inc({
             reason: ReprocessRejectReason.reached_limit,
             topic: topicType,
@@ -525,6 +528,7 @@ export class NetworkProcessor {
           preprocessResult.root
         );
         awaitingPayloadGossipsubMessages.add(message);
+        this.awaitingPayloadMessageCount++;
         break;
       }
     }
@@ -548,6 +552,12 @@ export class NetworkProcessor {
       return;
     }
 
+    // Atomically remove from map and update counter before async iteration to
+    // prevent double-decrement race with onClockSlot during yield points below
+    if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
+      this.awaitingBlockMessageCount -= waitingGossipsubMessages.size;
+    }
+
     const nowSec = Date.now() / 1000;
     let count = 0;
     // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
@@ -567,8 +577,6 @@ export class NetworkProcessor {
         await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
-
-    this.awaitingMessagesByBlockRoot.delete(rootHex);
   };
 
   private onPayloadEnvelopeProcessed = async ({blockRoot: rootHex}: {blockRoot: RootHex}): Promise<void> => {
@@ -577,6 +585,12 @@ export class NetworkProcessor {
       return;
     }
 
+    // Atomically remove from map and update counter before async iteration to
+    // prevent double-decrement race with onClockSlot during yield points below
+    if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
+      this.awaitingPayloadMessageCount -= waitingGossipsubMessages.size;
+    }
+
     const nowSec = Date.now() / 1000;
     let count = 0;
     for (const message of waitingGossipsubMessages) {
@@ -593,8 +607,6 @@ export class NetworkProcessor {
         await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
-
-    this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
   };
 
   private onClockSlot = (clockSlot: Slot): void => {
@@ -618,7 +630,9 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          this.awaitingMessagesByBlockRoot.delete(rootHex);
+          if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
+            this.awaitingBlockMessageCount -= gossipMessages.size;
+          }
         }
       }
       this.unknownBlocksBySlot.delete(slot);
@@ -641,7 +655,9 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
+          if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
+            this.awaitingPayloadMessageCount -= gossipMessages.size;
+          }
         }
       }
       this.unknownEnvelopesBySlot.delete(slot);
@@ -784,20 +800,4 @@ export class NetworkProcessor {
 
     return null;
   }
-
-  private get unknownBlockGossipsubMessagesCount(): number {
-    let count = 0;
-    for (const messages of this.awaitingMessagesByBlockRoot.values()) {
-      count += messages.size;
-    }
-    return count;
-  }
-
-  private get unknownPayloadGossipsubMessagesCount(): number {
-    let count = 0;
-    for (const messages of this.awaitingMessagesByPayloadBlockRoot.values()) {
-      count += messages.size;
-    }
-    return count;
-  }
 }

package/src/network/reqresp/utils/dataColumnResponseValidation.ts

@@ -1,6 +1,5 @@
 import {LogData} from "@lodestar/logger";
 import {ForkSeq} from "@lodestar/params";
-import {RespStatus, ResponseError} from "@lodestar/reqresp";
 import {ColumnIndex, Slot} from "@lodestar/types";
 import {prettyBytes, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../chain/interface.js";
@@ -76,7 +75,7 @@ export async function handleColumnSidecarUnavailability({
 
 export function validateRequestedDataColumns(chain: IBeaconChain, requestedColumns: ColumnIndex[]): ColumnIndex[] {
   if (requestedColumns.length === 0) {
-    throw new ResponseError(RespStatus.INVALID_REQUEST, "dataColumnSidecar requested without column indices");
+    return [];
   }
 
   const {custodyColumns, custodyColumnsIndex} = chain.custodyConfig;

package/src/sync/types.ts

@@ -19,7 +19,14 @@ export enum PendingBlockType {
    */
   INCOMPLETE_BLOCK_INPUT = "IncompleteBlockInput",
 
-  UNKNOWN_DATA = "unknown_data",
+  /**
+   * Payload analog of UNKNOWN_BLOCK_ROOT: we have a beacon block root but not its execution payload envelope.
+   */
+  UNKNOWN_PAYLOAD_BLOCK_ROOT = "unknown_payload_block_root",
+  /**
+   * Payload analog of INCOMPLETE_BLOCK_INPUT: we have a partial payload input that did not complete in time.
+   */
+  INCOMPLETE_PAYLOAD_ENVELOPE = "incomplete_payload_envelope",
 }
 
 export enum PendingBlockInputStatus {
@@ -70,6 +77,8 @@ export type PendingPayloadInput = {
 export type PendingPayloadRootHex = {
   status: PendingPayloadInputStatus.pending | PendingPayloadInputStatus.fetching;
   rootHex: RootHex;
+  // message slot hint, may be missing when resolving a parent payload
+  slot?: Slot;
   timeAddedSec: number;
   timeSyncedSec?: number;
   peerIdStrings: Set<string>;
@@ -125,5 +134,5 @@ export function getPayloadSyncCacheItemSlot(payload: PayloadSyncCacheItem): Slot
     return payload.envelope.message.payload.slotNumber;
   }
 
-  return "unknown";
+  return payload.slot ?? "unknown";
 }