@lodestar/beacon-node 1.49.0-dev.a18e2b6354 → 1.49.0-dev.a1f8bb3877

package/src/network/processor/gossipValidatorFn.ts

@@ -1,24 +1,122 @@
 import {TopicValidatorResult} from "@libp2p/gossipsub";
 import {ChainForkConfig} from "@lodestar/config";
 import {Logger} from "@lodestar/utils";
-import {AttestationError, GossipAction, GossipActionError} from "../../chain/errors/index.js";
+import {
+  AttestationError,
+  AttestationErrorCode,
+  AttesterSlashingErrorCode,
+  BlobSidecarErrorCode,
+  BlockErrorCode,
+  BlsToExecutionChangeErrorCode,
+  DataColumnSidecarErrorCode,
+  ExecutionPayloadBidErrorCode,
+  ExecutionPayloadEnvelopeErrorCode,
+  GossipAction,
+  GossipActionError,
+  PayloadAttestationErrorCode,
+  ProposerPreferencesErrorCode,
+  ProposerSlashingErrorCode,
+  SyncCommitteeErrorCode,
+  VoluntaryExitErrorCode,
+} from "../../chain/errors/index.js";
 import {Metrics} from "../../metrics/index.js";
+import {INetworkCore} from "../core/index.js";
 import {
   BatchGossipHandlerFn,
   GossipHandlerFn,
   GossipHandlers,
   GossipMessageInfo,
+  GossipType,
   GossipValidatorBatchFn,
   GossipValidatorFn,
 } from "../gossip/interface.js";
+import {PeerAction} from "../peers/index.js";
 import {prettyPrintPeerIdStr} from "../util.js";
 
 export type ValidatorFnModules = {
   config: ChainForkConfig;
   logger: Logger;
   metrics: Metrics | null;
+  core: INetworkCore;
+};
+
+type RejectPeerActionRule = {default: PeerAction; byCode?: Record<string, PeerAction>};
+
+/**
+ * PeerAction mapping based on the topic and specific codes.
+ * Invalid signatures are Fatal on every topic.
+ */
+const gossipRejectPeerAction: Record<GossipType, RejectPeerActionRule> = {
+  [GossipType.beacon_block]: {
+    default: PeerAction.LowToleranceError,
+    byCode: {[BlockErrorCode.PROPOSAL_SIGNATURE_INVALID]: PeerAction.Fatal},
+  },
+  [GossipType.blob_sidecar]: {
+    default: PeerAction.LowToleranceError,
+    byCode: {[BlobSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID]: PeerAction.Fatal},
+  },
+  [GossipType.data_column_sidecar]: {
+    default: PeerAction.LowToleranceError,
+    byCode: {[DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID]: PeerAction.Fatal},
+  },
+  [GossipType.execution_payload]: {
+    default: PeerAction.LowToleranceError,
+    byCode: {[ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.beacon_aggregate_and_proof]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[AttestationErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.beacon_attestation]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[AttestationErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.sync_committee_contribution_and_proof]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[SyncCommitteeErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.sync_committee]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[SyncCommitteeErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.payload_attestation_message]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[PayloadAttestationErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.execution_payload_bid]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[ExecutionPayloadBidErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.proposer_preferences]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[ProposerPreferencesErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.voluntary_exit]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[VoluntaryExitErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.bls_to_execution_change]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[BlsToExecutionChangeErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.proposer_slashing]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[ProposerSlashingErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.attester_slashing]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[AttesterSlashingErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  // light client validators only throw IGNORE
+  [GossipType.light_client_finality_update]: {default: PeerAction.HighToleranceError},
+  [GossipType.light_client_optimistic_update]: {default: PeerAction.HighToleranceError},
 };
 
+function rejectPeerAction(type: GossipType, code: string): PeerAction {
+  const rule = gossipRejectPeerAction[type];
+  return rule.byCode?.[code] ?? rule.default;
+}
+
 /**
  * Similar to getGossipValidatorFn but return a function to accept a batch of beacon_attestation messages
  * with the same attestation data
@@ -27,7 +125,7 @@ export function getGossipValidatorBatchFn(
   gossipHandlers: GossipHandlers,
   modules: ValidatorFnModules
 ): GossipValidatorBatchFn {
-  const {logger, metrics} = modules;
+  const {logger, metrics, core} = modules;
 
   return async function gossipValidatorBatchFn(messageInfos: GossipMessageInfo[]) {
     // all messageInfos have same topic type
@@ -69,16 +167,19 @@ export function getGossipValidatorBatchFn(
             // only beacon_attestation topic is validated in batch
             metrics?.networkProcessor.gossipAttestationIgnoreByReason.inc({reason: e.type.code});
             return TopicValidatorResult.Ignore;
-          case GossipAction.REJECT:
+          case GossipAction.REJECT: {
             metrics?.networkProcessor.gossipValidationReject.inc({topic: type});
             // only beacon_attestation topic is validated in batch
             metrics?.networkProcessor.gossipAttestationRejectByReason.inc({reason: e.type.code});
+            const peerAction = rejectPeerAction(type, e.type.code);
+            core.reportPeer(propagationSource, peerAction, e.type.code);
             logger.debug(
               `Gossip validation ${type} rejected`,
-              {peerId: prettyPrintPeerIdStr(propagationSource), clientAgent, clientVersion},
+              {peer: propagationSource, clientAgent, clientVersion, peerAction},
               e
             );
             return TopicValidatorResult.Reject;
+          }
         }
       });
     } catch (e) {
@@ -108,7 +209,7 @@ export function getGossipValidatorBatchFn(
  * @see getGossipHandlers for reasoning on why GossipHandlerFn are used for gossip validation.
  */
 export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: ValidatorFnModules): GossipValidatorFn {
-  const {logger, metrics} = modules;
+  const {logger, metrics, core} = modules;
 
   return async function gossipValidatorFn({
     topic,
@@ -155,14 +256,17 @@ export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: Va
           metrics?.networkProcessor.gossipValidationIgnore.inc({topic: type});
           return TopicValidatorResult.Ignore;
 
-        case GossipAction.REJECT:
+        case GossipAction.REJECT: {
           metrics?.networkProcessor.gossipValidationReject.inc({topic: type});
+          const peerAction = rejectPeerAction(type, e.type.code);
+          core.reportPeer(propagationSource, peerAction, e.type.code);
           logger.debug(
             `Gossip validation ${type} rejected`,
-            {peerId: prettyPrintPeerIdStr(propagationSource), clientAgent, clientVersion},
+            {peer: propagationSource, clientAgent, clientVersion, peerAction},
             e
           );
           return TopicValidatorResult.Reject;
+        }
       }
     }
   };

package/src/network/processor/index.ts

@@ -86,8 +86,13 @@ export const MAX_AWAITING_MESSAGES_PER_ROOT: Partial<Record<GossipType, number>>
 
 /**
  * Track the forwarded peers we already emit to UnknownBlockInput sync.
+ * envelopeSlotEmitted: whether the unknownEnvelopeBlockRootSlot event was already emitted for this root.
  */
-type SearchedRootEntry = {blockPeerIds?: Set<PeerIdStr>; envelopePeerIds?: Set<PeerIdStr>};
+type SearchedRootEntry = {
+  blockPeerIds?: Set<PeerIdStr>;
+  envelopePeerIds?: Set<PeerIdStr>;
+  envelopeSlotEmitted?: boolean;
+};
 
 /**
  * This is respective to gossipsub seenTTL (which is 550 * 0.7 = 385s), also it's respective
@@ -347,12 +352,17 @@ export class NetworkProcessor {
   }
 
   /**
-   * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` event
+   * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` / `ChainEvent.unknownEnvelopeBlockRootSlot` event
    * Slot is the message slot, which is not necessarily the same as the envelope's slot, but it can be used for a good prune strategy.
    * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
    * Same peer-forwarding + dedup behavior as searchUnknownBlock.
    */
-  searchUnknownEnvelope({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
+  searchUnknownEnvelope(
+    {slot, root}: SlotRootHex,
+    source: BlockInputSource,
+    peer?: PeerIdStr,
+    slotIsPayloadSlot = false
+  ): void {
     if (this.chain.seenPayloadEnvelope(root)) {
       return;
     }
@@ -369,8 +379,17 @@ export class NetworkProcessor {
       forwardedPeers.add(peer);
       shouldEmit = true;
     }
+    // the payload-slot seed must reach the sync once, even if a slot-less trigger started the search first
+    if (slotIsPayloadSlot && !entry.envelopeSlotEmitted) {
+      shouldEmit = true;
+    }
     if (shouldEmit) {
-      this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
+      if (slotIsPayloadSlot) {
+        entry.envelopeSlotEmitted = true;
+        this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRootSlot, {rootHex: root, slot, peer, source});
+      } else {
+        this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, peer, source});
+      }
     }
   }
 
@@ -494,8 +513,9 @@ export class NetworkProcessor {
           if (root == null) break;
           const payloadPresent = getPayloadPresentFromPayloadAttestationMessageSerialized(message.msg.data);
           if (payloadPresent && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
-            // payload attestation votes that the payload is available but it is not yet known
-            this.searchUnknownRoot({slot, root}, false, true, peerId);
+            // payload attestation votes that the payload is available but it is not yet known.
+            // the PTC vote's slot is the payload's slot
+            this.searchUnknownRoot({slot, root}, false, true, peerId, true);
             // do not await the envelope, payload attestation processing only requires that the block is known
             // also do not reset preprocessResult, we may already await for the block
           }
@@ -504,7 +524,8 @@ export class NetworkProcessor {
         case GossipType.data_column_sidecar: {
           if (root == null) break;
           if (!this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
-            this.searchUnknownRoot({slot, root}, false, true, peerId);
+            // the sidecar's slot is the block's (and so the payload's) slot
+            this.searchUnknownRoot({slot, root}, false, true, peerId, true);
             // do not await the envelope, we can do gossip validation
             // also do not reset preprocessResult, we may already await for the block
           }
@@ -641,16 +662,23 @@ export class NetworkProcessor {
     return true;
   }
 
+  /**
+   * Search block/envelope given a SlotRootHex
+   * envelopeSlotIsPayloadSlot: the message slot is the payload's slot or not
+   */
   private searchUnknownRoot(
     slotRoot: SlotRootHex,
     searchBlock: boolean,
     searchEnvelope: boolean,
-    peerId: PeerIdStr
+    peerId: PeerIdStr,
+    envelopeSlotIsPayloadSlot = false
   ): void {
     if (!searchBlock && !searchEnvelope) return;
     if (this.tooManySearchedRoots(slotRoot.slot, slotRoot.root)) return;
     if (searchBlock) this.searchUnknownBlock(slotRoot, BlockInputSource.network_processor, peerId);
-    if (searchEnvelope) this.searchUnknownEnvelope(slotRoot, BlockInputSource.network_processor, peerId);
+    if (searchEnvelope) {
+      this.searchUnknownEnvelope(slotRoot, BlockInputSource.network_processor, peerId, envelopeSlotIsPayloadSlot);
+    }
   }
 
   private pushPendingGossipsubMessageToQueue(message: PendingGossipsubMessage): void {

package/src/network/reqresp/protocols.ts

@@ -1,6 +1,7 @@
 import {BeaconConfig} from "@lodestar/config";
-import {ForkName} from "@lodestar/params";
+import {ForkName, isForkPostGloas} from "@lodestar/params";
 import {ContextBytesFactory, ContextBytesType, Encoding, TypeSizes} from "@lodestar/reqresp";
+import {computeMaxGloasDataColumnSidecarSize} from "../../util/sszBytes.js";
 import {rateLimitQuotas} from "./rateLimit.js";
 import {ProtocolNoHandler, ReqRespMethod, Version, requestSszTypeByMethod, responseSszTypeByMethod} from "./types.js";
 
@@ -151,18 +152,42 @@ function toProtocol(protocol: ProtocolSummary) {
       encoding: Encoding.SSZ_SNAPPY,
       contextBytes: toContextBytes(protocol.contextBytesType, config),
       inboundRateLimits: rateLimitQuotas(fork, config)[protocol.method],
-      requestSizes: requestType === null ? null : clampTypeSizes(requestType, config),
-      responseSizes: (fork) => clampTypeSizes(responseSszTypeByMethod[protocol.method](fork, protocol.version), config),
+      requestSizes: requestType === null ? null : clampTypeSizes(requestType, config.MAX_PAYLOAD_SIZE),
+      responseSizes: (fork) =>
+        clampResponseTypeSizes(
+          responseSszTypeByMethod[protocol.method](fork, protocol.version),
+          protocol.method,
+          fork,
+          config
+        ),
     };
   };
 }
 
+function clampResponseTypeSizes(
+  type: TypeSizes,
+  method: ReqRespMethod,
+  fork: ForkName,
+  config: BeaconConfig
+): TypeSizes {
+  let maxPayloadSize = config.MAX_PAYLOAD_SIZE;
+  if (isForkPostGloas(fork)) {
+    switch (method) {
+      case ReqRespMethod.DataColumnSidecarsByRange:
+      case ReqRespMethod.DataColumnSidecarsByRoot:
+        maxPayloadSize = Math.min(maxPayloadSize, computeMaxGloasDataColumnSidecarSize(config));
+        break;
+    }
+  }
+  return clampTypeSizes(type, maxPayloadSize);
+}
+
 /**
- * Length-prefix must be within the SSZ type bounds or MAX_PAYLOAD_SIZE, whichever is smaller
+ * Length-prefix must be within the SSZ type bounds or the configured payload limit, whichever is smaller.
  * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/phase0/p2p-interface.md#encoding-strategies
  */
-function clampTypeSizes(type: TypeSizes, config: BeaconConfig): TypeSizes {
-  return {minSize: type.minSize, maxSize: Math.min(type.maxSize, config.MAX_PAYLOAD_SIZE)};
+function clampTypeSizes(type: TypeSizes, maxPayloadSize: number): TypeSizes {
+  return {minSize: type.minSize, maxSize: Math.min(type.maxSize, maxPayloadSize)};
 }
 
 function toContextBytes(type: ContextBytesType, config: BeaconConfig): ContextBytesFactory {

package/src/sync/types.ts

@@ -23,12 +23,36 @@ export enum PendingBlockType {
    * Payload analog of UNKNOWN_BLOCK_ROOT: we have a beacon block root but not its execution payload envelope.
    */
   UNKNOWN_PAYLOAD_BLOCK_ROOT = "unknown_payload_block_root",
+  /**
+   * Same as UNKNOWN_PAYLOAD_BLOCK_ROOT, but the trigger also carries the payload's slot.
+   */
+  UNKNOWN_PAYLOAD_BLOCK_ROOT_SLOT = "unknown_payload_block_root_slot",
   /**
    * Payload analog of INCOMPLETE_BLOCK_INPUT: we have a partial payload input that did not complete in time.
    */
   INCOMPLETE_PAYLOAD_ENVELOPE = "incomplete_payload_envelope",
 }
 
+export enum DownloadResult {
+  /** payload fully downloaded, ready to process */
+  Resolved = "resolved",
+  /** payload fully downloaded, but it was already imported (via gossip) while we were downloading */
+  Late = "late",
+  /** envelope downloaded, held until the block lands in fork choice */
+  WaitingForBlock = "waiting_for_block",
+  /** fetch failed, the pending entry is retained for retry */
+  Failed = "failed",
+}
+
+export enum FetchResult {
+  SuccessResolved = "success_resolved",
+  SuccessMissingParent = "success_missing_parent",
+  SuccessLate = "success_late",
+  SuccessWaitingForBlock = "success_waiting_for_block",
+  FailureTriedAllPeers = "failure_tried_all_peers",
+  FailureMaxAttempts = "failure_max_attempts",
+}
+
 export enum PendingBlockInputStatus {
   pending = "pending",
   fetching = "fetching",
@@ -110,9 +134,8 @@ export type PendingPayloadInput = {
 export type PendingPayloadRootHex = {
   status: PendingPayloadInputStatus.pending | PendingPayloadInputStatus.fetching;
   rootHex: RootHex;
-  // Trusted slot only (fork choice / validated data), may be missing until resolved. NOT the gossip
-  // message slot from ChainEvent.unknownEnvelopeBlockRoot, which is untrusted and not necessarily the
-  // payload/block slot. See BlockInputSync.resolvePayloadSlot.
+  // slot could be via fork choice or ChainEvent.unknownEnvelopeBlockRootSlot
+  // while ChainEvent.unknownEnvelopeBlockRoot does not provide one
   slot?: Slot;
   timeAddedSec: number;
   timeSyncedSec?: number;

package/src/sync/unknownBlock.ts

@@ -26,7 +26,9 @@ import {MAX_CONCURRENT_REQUESTS} from "./constants.js";
 import {SyncOptions} from "./options.js";
 import {
   BlockInputSyncCacheItem,
+  DownloadResult,
   DroppedItemReason,
+  FetchResult,
   PayloadSyncCacheItem,
   PendingBlockInput,
   PendingBlockInputStatus,
@@ -63,14 +65,6 @@ type AdvancePendingBlockResult =
   | "blocked"
   | "removed";
 
-enum FetchResult {
-  SuccessResolved = "success_resolved",
-  SuccessMissingParent = "success_missing_parent",
-  SuccessLate = "success_late",
-  FailureTriedAllPeers = "failure_tried_all_peers",
-  FailureMaxAttempts = "failure_max_attempts",
-}
-
 class UnknownBlockRateLimitedError extends Error {
   constructor(message: string) {
     super(message);
@@ -158,6 +152,7 @@ export class BlockInputSync {
       this.logger.verbose("BlockInputSync enabled.");
       this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
       this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+      this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
       this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
       this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
       this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -185,6 +180,7 @@ export class BlockInputSync {
     this.clearRateLimitBackoffTimer();
     this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
     this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+    this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
     this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
     this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -248,6 +244,19 @@ export class BlockInputSync {
     }
   };
 
+  private onUnknownEnvelopeBlockRootSlot = (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRootSlot]): void => {
+    try {
+      const isNewRoot = this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
+      if (isNewRoot) {
+        this.triggerUnknownBlockSearch();
+        this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT_SLOT});
+        this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
+      }
+    } catch (e) {
+      this.logger.debug("Error handling unknownEnvelopeBlockRootSlot event", {}, e as Error);
+    }
+  };
+
   private onIncompletePayloadEnvelope = (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]): void => {
     try {
       this.addByPayloadInput(data.payloadInput, data.peer);
@@ -1125,15 +1134,25 @@ export class BlockInputSync {
         return;
       }
       this.pendingPayloads.set(resultRootHex, pendingPayload);
+      const result = isPendingPayloadEnvelope(pendingPayload)
+        ? DownloadResult.WaitingForBlock
+        : this.chain.forkChoice.hasPayloadHexUnsafe(resultRootHex)
+          ? DownloadResult.Late
+          : DownloadResult.Resolved;
+      this.metrics?.blockInputSync.downloadedPayloadsResult.inc({result});
 
       if (isPendingPayloadEnvelope(pendingPayload)) {
         await this.reconcilePayloadEnvelope(pendingPayload);
       } else if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+        this.metrics?.blockInputSync.elapsedTimeTillPayloadReceived.observe(
+          Date.now() / 1000 - computeTimeAtSlot(this.config, pendingPayload.payloadInput.slot, this.chain.genesisTime)
+        );
         await this.processPayload(pendingPayload);
       }
       return;
     }
 
+    this.metrics?.blockInputSync.downloadedPayloadsResult.inc({result: DownloadResult.Failed});
     this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
     if (!isPendingPayloadEnvelope(payload)) {
       payload.status = PendingPayloadInputStatus.pending;
@@ -1248,6 +1267,11 @@ export class BlockInputSync {
       : await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
     let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
 
+    const fetchStartSec = Date.now() / 1000;
+    if (typeof slot === "number") {
+      this.metrics?.blockInputSync.payloadFetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
+    }
+
     let i = 0;
     let deferredByRateLimit = false;
     while (i++ < this.getMaxDownloadAttempts()) {
@@ -1271,6 +1295,11 @@ export class BlockInputSync {
           pendingColumns.size > 0
             ? `cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
             : "no peer available to download pending payload";
+        this.metrics?.blockInputSync.payloadFetchTime.observe(
+          {result: FetchResult.FailureTriedAllPeers},
+          Date.now() / 1000 - fetchStartSec
+        );
+        this.metrics?.blockInputSync.payloadFetchPeers.set({result: FetchResult.FailureTriedAllPeers}, i);
         throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: ${reason}`);
       }
 
@@ -1280,7 +1309,12 @@ export class BlockInputSync {
       try {
         if (!envelope) {
           envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
+          const slotWasUnknown = typeof slot !== "number";
           slot = envelope.message.payload.slotNumber;
+          if (slotWasUnknown) {
+            // we were not able to observe the time into slot when starting the fetch, do it now
+            this.metrics?.blockInputSync.payloadFetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
+          }
         }
 
         payloadInput ??= await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
@@ -1289,6 +1323,11 @@ export class BlockInputSync {
           // envelope and wait for the block body; reconcilePayloadEnvelope validates once the block lands.
           // payloadInput may be seeded from the block body during download, so a non-null payloadInput does not
           // imply the block is imported.
+          this.metrics?.blockInputSync.payloadFetchTime.observe(
+            {result: FetchResult.SuccessWaitingForBlock},
+            Date.now() / 1000 - fetchStartSec
+          );
+          this.metrics?.blockInputSync.payloadFetchPeers.set({result: FetchResult.SuccessWaitingForBlock}, i);
           return {
             status: PendingPayloadInputStatus.waitingForBlock,
             envelope,
@@ -1339,6 +1378,12 @@ export class BlockInputSync {
         });
 
         if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+          // late = the payload got imported (via gossip) while we were downloading, same as the block path
+          const result = this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)
+            ? FetchResult.SuccessLate
+            : FetchResult.SuccessResolved;
+          this.metrics?.blockInputSync.payloadFetchTime.observe({result}, Date.now() / 1000 - fetchStartSec);
+          this.metrics?.blockInputSync.payloadFetchPeers.set({result}, i);
           return pendingPayload;
         }
 
@@ -1373,12 +1418,17 @@ export class BlockInputSync {
       }
     }
 
+    this.metrics?.blockInputSync.payloadFetchTime.observe(
+      {result: FetchResult.FailureMaxAttempts},
+      Date.now() / 1000 - fetchStartSec
+    );
+    this.metrics?.blockInputSync.payloadFetchPeers.set({result: FetchResult.FailureMaxAttempts}, i - 1);
+
     if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
       throw new UnknownBlockRateLimitedError(
         `Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`
       );
     }
-
     throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
   }
 
@@ -1483,7 +1533,7 @@ export class BlockInputSync {
 
         // no more peer with needed columns to try, throw error
         const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
-        this.metrics?.blockInputSync.fetchTimeSec.observe(
+        this.metrics?.blockInputSync.fetchTime.observe(
           {result: FetchResult.FailureTriedAllPeers},
           Date.now() / 1000 - fetchStartSec
         );
@@ -1572,7 +1622,7 @@ export class BlockInputSync {
           : this.chain.forkChoice.hasBlockHex(cacheItem.blockInput.parentRootHex)
             ? FetchResult.SuccessResolved
             : FetchResult.SuccessMissingParent;
-        this.metrics?.blockInputSync.fetchTimeSec.observe({result}, Date.now() / 1000 - fetchStartSec);
+        this.metrics?.blockInputSync.fetchTime.observe({result}, Date.now() / 1000 - fetchStartSec);
         this.metrics?.blockInputSync.fetchPeers.set({result}, i);
         return cacheItem;
       }
@@ -1606,7 +1656,7 @@ export class BlockInputSync {
       }
     }
 
-    this.metrics?.blockInputSync.fetchTimeSec.observe(
+    this.metrics?.blockInputSync.fetchTime.observe(
       {result: FetchResult.FailureMaxAttempts},
       Date.now() / 1000 - fetchStartSec
     );

package/src/util/sszBytes.ts

@@ -1,6 +1,7 @@
 import {BitArray, deserializeUint8ArrayBitListFromBytes} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
 import {
+  BYTES_PER_CELL,
   BYTES_PER_FIELD_ELEMENT,
   FIELD_ELEMENTS_PER_BLOB,
   ForkName,
@@ -551,6 +552,23 @@ export function getBeaconBlockRootFromDataColumnSidecarSerialized(data: Uint8Arr
   return "0x" + blockRootBuf.toString("hex");
 }
 
+// beaconBlockRoot is the last fixed field in the Gloas layout
+const DATA_COLUMN_SIDECAR_GLOAS_FIXED_SIZE = BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR + ROOT_SIZE; // 56 bytes
+const KZG_PROOF_SIZE = 48; // Bytes48
+const DATA_COLUMN_SIDECAR_GLOAS_PER_BLOB_SIZE = BYTES_PER_CELL + KZG_PROOF_SIZE; // 2048 + 48 = 2096 bytes
+
+/**
+ * Gloas sidecar size bound uses the largest blob limit across the entire schedule, including future entries.
+ * https://github.com/ethereum/consensus-specs/pull/5613
+ */
+export function computeMaxGloasDataColumnSidecarSize(config: ChainForkConfig): number {
+  const maxBlobs = config.BLOB_SCHEDULE.reduce(
+    (max, entry) => Math.max(max, entry.MAX_BLOBS_PER_BLOCK),
+    config.MAX_BLOBS_PER_BLOCK_ELECTRA
+  );
+  return DATA_COLUMN_SIDECAR_GLOAS_FIXED_SIZE + maxBlobs * DATA_COLUMN_SIDECAR_GLOAS_PER_BLOB_SIZE;
+}
+
 /**
  * SignedExecutionPayloadEnvelope SSZ Layout:
  * ├─ 4 bytes: message offset (points to byte 100)