@lodestar/beacon-node 1.49.0-dev.4d35c8b82e → 1.49.0-dev.57ffc39074

package/src/chain/archiveStore/utils/pruneHistory.ts

@@ -1,7 +1,7 @@
 import {ChainConfig} from "@lodestar/config";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
+import {Logger, prettyPrintIndices} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/interface.js";
 import {Metrics} from "../../../metrics/index.js";
 
@@ -20,30 +20,40 @@ export async function pruneHistory(
     finalizedEpoch
   );
   const blockCutoffSlot = computeStartSlotAtEpoch(blockCutoffEpoch);
+  // The latest archived state is the anchor on restart and can trail finalization by a few epochs
+  const lastArchivedStateSlot = (await db.stateArchive.lastKey()) ?? 0;
+  const stateCutoffSlot = Math.min(computeStartSlotAtEpoch(finalizedEpoch), lastArchivedStateSlot);
 
   logger.debug("Preparing to prune history", {
     currentEpoch,
     finalizedEpoch,
     blockCutoffEpoch,
+    stateCutoffSlot,
   });
 
   const step0 = metrics?.pruneHistory.fetchKeys.startTimer();
-  const [blocks, states] = await Promise.all([
+  const [blocks, envelopes, states] = await Promise.all([
     db.blockArchive.keys({gte: 0, lt: blockCutoffSlot}),
-    db.stateArchive.keys({gte: 0, lt: finalizedEpoch}),
+    db.executionPayloadEnvelopeArchive.keys({gte: 0, lt: blockCutoffSlot}),
+    db.stateArchive.keys({gte: 0, lt: stateCutoffSlot}),
   ]);
   step0?.();
 
   logger.debug("Pruning history", {
     currentEpoch,
     blocksToPrune: blocks.length,
+    envelopesToPrune: envelopes.length,
     statesToPrune: states.length,
+    blockSlots: prettyPrintIndices(blocks),
+    envelopeSlots: prettyPrintIndices(envelopes),
+    stateSlots: prettyPrintIndices(states),
   });
 
   const step1 = metrics?.pruneHistory.pruneKeys.startTimer();
   await Promise.all([
     // ->
-    db.blockArchive.batchDelete(blocks),
+    db.blockArchive.batchDeleteRange(blocks),
+    db.executionPayloadEnvelopeArchive.batchDelete(envelopes),
     db.stateArchive.batchDelete(states),
   ]);
   step1?.();

package/src/chain/blocks/index.ts

@@ -104,7 +104,7 @@ export async function processBlocks(
         payloadEnvelopesToImport.delete(orphaned.slot);
         // never validated, drop it from the shared cache so it is not served to peers, by-root sync reloads the
         // entry from db if the payload is ever needed
-        this.seenPayloadEnvelopeInputCache.prune(orphaned.payloadEnvelopeInput.blockRootHex);
+        this.seenPayloadEnvelopeInputCache.remove(orphaned.payloadEnvelopeInput.blockRootHex);
       }
     }
 
@@ -278,7 +278,7 @@ async function importPayloadEnvelopesOfKnownBlocks(
       this.forkChoice.isDescendant(blockRootHex, PayloadStatus.EMPTY, head.blockRoot, head.payloadStatus)
     ) {
       // never validated, see processBlocks()
-      this.seenPayloadEnvelopeInputCache.prune(blockRootHex);
+      this.seenPayloadEnvelopeInputCache.remove(blockRootHex);
       orphaned.push({slot, payloadEnvelopeInput: payloadInput});
       continue;
     }

package/src/chain/blocks/payloadEnvelopeInput/types.ts

@@ -23,7 +23,7 @@ export enum PayloadEnvelopeInputSource {
  * - prune: explicit prune by root
  * - cap: insertion-order backstop cap (MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE)
  */
-export type PayloadEnvelopeInputPruneReason = "belowParent" | "finalized" | "prune" | "cap";
+export type PayloadEnvelopeInputPruneReason = "belowParent" | "finalized" | "remove" | "cap";
 
 export type SourceMeta = {
   source: PayloadEnvelopeInputSource;

package/src/chain/emitter.ts

@@ -72,6 +72,10 @@ export enum ChainEvent {
    * Trigger BlockInputSync to find a SignedExecutionPayloadEnvelope with specified block root.
    */
   unknownEnvelopeBlockRoot = "unknownEnvelopeBlockRoot",
+  /**
+   * Same as unknownEnvelopeBlockRoot, but with slot
+   */
+  unknownEnvelopeBlockRootSlot = "unknownEnvelopeBlockRootSlot",
   /**
    * Trigger BlockInputSync for blocks that are partially received via gossip but are not complete by time the
    * cut-off window passes for waiting on gossip
@@ -99,8 +103,8 @@ export type ChainEventData = {
     peer: PeerIdStr;
     source: BlockInputSource;
   };
-  // slot is the message slot, not necessarily the envelope's slot, but useful as a logging/prune hint
-  [ChainEvent.unknownEnvelopeBlockRoot]: {rootHex: RootHex; slot: Slot; peer?: PeerIdStr; source: BlockInputSource};
+  [ChainEvent.unknownEnvelopeBlockRoot]: {rootHex: RootHex; peer?: PeerIdStr; source: BlockInputSource};
+  [ChainEvent.unknownEnvelopeBlockRootSlot]: {rootHex: RootHex; slot: Slot; peer?: PeerIdStr; source: BlockInputSource};
 };
 
 export type IChainEvents = ApiEvents & {
@@ -126,6 +130,7 @@ export type IChainEvents = ApiEvents & {
   [ChainEvent.incompleteBlockInput]: (data: ChainEventData[ChainEvent.incompleteBlockInput]) => void;
   [ChainEvent.incompletePayloadEnvelope]: (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]) => void;
   [ChainEvent.unknownEnvelopeBlockRoot]: (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRoot]) => void;
+  [ChainEvent.unknownEnvelopeBlockRootSlot]: (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRootSlot]) => void;
 };
 
 /**

package/src/chain/errors/attesterSlashingError.ts

@@ -3,9 +3,11 @@ import {GossipActionError} from "./gossipValidation.js";
 export enum AttesterSlashingErrorCode {
   ALREADY_EXISTS = "ATTESTATION_SLASHING_ERROR_ALREADY_EXISTS",
   INVALID = "ATTESTATION_SLASHING_ERROR_INVALID",
+  INVALID_SIGNATURE = "ATTESTATION_SLASHING_ERROR_INVALID_SIGNATURE",
 }
 export type AttesterSlashingErrorType =
   | {code: AttesterSlashingErrorCode.ALREADY_EXISTS}
-  | {code: AttesterSlashingErrorCode.INVALID; error: Error};
+  | {code: AttesterSlashingErrorCode.INVALID; error: Error}
+  | {code: AttesterSlashingErrorCode.INVALID_SIGNATURE};
 
 export class AttesterSlashingError extends GossipActionError<AttesterSlashingErrorType> {}

package/src/chain/errors/payloadAttestation.ts

@@ -6,6 +6,7 @@ export enum PayloadAttestationErrorCode {
   PAYLOAD_ATTESTATION_ALREADY_KNOWN = "PAYLOAD_ATTESTATION_ERROR_PAYLOAD_ATTESTATION_ALREADY_KNOWN",
   UNKNOWN_BLOCK_ROOT = "PAYLOAD_ATTESTATION_ERROR_UNKNOWN_BLOCK_ROOT",
   INVALID_BLOCK_SLOT = "PAYLOAD_ATTESTATION_ERROR_INVALID_BLOCK_SLOT",
+  PRE_GLOAS_SLOT = "PAYLOAD_ATTESTATION_ERROR_PRE_GLOAS_SLOT",
   INVALID_BLOCK = "PAYLOAD_ATTESTATION_ERROR_INVALID_BLOCK",
   INVALID_ATTESTER = "PAYLOAD_ATTESTATION_ERROR_INVALID_ATTESTER",
   INVALID_SIGNATURE = "PAYLOAD_ATTESTATION_ERROR_INVALID_SIGNATURE",
@@ -20,6 +21,7 @@ export type PayloadAttestationErrorType =
     }
   | {code: PayloadAttestationErrorCode.UNKNOWN_BLOCK_ROOT; blockRoot: RootHex}
   | {code: PayloadAttestationErrorCode.INVALID_BLOCK_SLOT; blockRoot: RootHex; blockSlot: Slot; slot: Slot}
+  | {code: PayloadAttestationErrorCode.PRE_GLOAS_SLOT; slot: Slot}
   | {code: PayloadAttestationErrorCode.INVALID_BLOCK; blockRoot: RootHex}
   | {code: PayloadAttestationErrorCode.INVALID_ATTESTER; attesterIndex: ValidatorIndex}
   | {code: PayloadAttestationErrorCode.INVALID_SIGNATURE};

package/src/chain/errors/proposerSlashingError.ts

@@ -3,9 +3,11 @@ import {GossipActionError} from "./gossipValidation.js";
 export enum ProposerSlashingErrorCode {
   ALREADY_EXISTS = "PROPOSER_SLASHING_ERROR_ALREADY_EXISTS",
   INVALID = "PROPOSER_SLASHING_ERROR_INVALID",
+  INVALID_SIGNATURE = "PROPOSER_SLASHING_ERROR_INVALID_SIGNATURE",
 }
 export type ProposerSlashingErrorType =
   | {code: ProposerSlashingErrorCode.ALREADY_EXISTS}
-  | {code: ProposerSlashingErrorCode.INVALID; error: Error};
+  | {code: ProposerSlashingErrorCode.INVALID; error: Error}
+  | {code: ProposerSlashingErrorCode.INVALID_SIGNATURE};
 
 export class ProposerSlashingError extends GossipActionError<ProposerSlashingErrorType> {}

package/src/chain/options.ts

@@ -63,10 +63,6 @@ export type BlockProcessOpts = {
    * Will double processing times. Use only for debugging purposes.
    */
   disableBlsBatchVerify?: boolean;
-  /**
-   * Assert progressive balances the same to EpochTransitionCache
-   */
-  assertCorrectProgressiveBalances?: boolean;
   /** Used for fork_choice spec tests */
   disableOnBlockError?: boolean;
   /** Used for fork_choice spec tests */
@@ -114,7 +110,6 @@ export const defaultChainOptions: IChainOptions = {
   suggestedFeeRecipient: defaultValidatorOptions.suggestedFeeRecipient,
   graffitiAppend: true,
   serveHistoricalState: false,
-  assertCorrectProgressiveBalances: false,
   archiveStateEpochFrequency: 1024,
   archiveMode: DEFAULT_ARCHIVE_MODE,
   pruneHistory: false,

package/src/chain/produceBlock/produceBlockBody.ts

@@ -59,7 +59,18 @@ import {
   heze,
   ssz,
 } from "@lodestar/types";
-import {GWEI_TO_WEI, Logger, byteArrayEquals, fromHex, sleep, toHex, toPubkeyHex, toRootHex} from "@lodestar/utils";
+import {
+  GWEI_TO_WEI,
+  Logger,
+  byteArrayEquals,
+  fromHex,
+  prettyGweiToEth,
+  prettyWeiToEth,
+  sleep,
+  toHex,
+  toPubkeyHex,
+  toRootHex,
+} from "@lodestar/utils";
 import {ZERO_HASH_HEX} from "../../constants/index.js";
 import {numToQuantity} from "../../execution/engine/utils.js";
 import {IExecutionBuilder, IExecutionEngine, PayloadAttributes, PayloadId} from "../../execution/index.js";
@@ -259,7 +270,7 @@ export async function produceBlockBody<T extends BlockType>(
     this.logger.verbose("Produced block with builder bid", {
       slot: blockSlot,
       builderIndex: builderBid.message.builderIndex,
-      bidValue: builderBid.message.value,
+      bidValue: prettyGweiToEth(builderBid.message.value),
       parentBlockHash: toRootHex(builderBid.message.parentBlockHash),
       parentBlockRoot: toRootHex(builderBid.message.parentBlockRoot),
       blockHash: toRootHex(builderBid.message.blockHash),
@@ -410,7 +421,7 @@ export async function produceBlockBody<T extends BlockType>(
     this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
     this.logger.verbose("Produced block with self-build bid", {
       slot: blockSlot,
-      executionPayloadValue,
+      executionPayloadValue: prettyWeiToEth(executionPayloadValue),
       prepType,
       payloadId,
       fetchedTime,
@@ -490,7 +501,7 @@ export async function produceBlockBody<T extends BlockType>(
       this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
       this.logger.verbose("Fetched execution payload header from builder", {
         slot: blockSlot,
-        executionPayloadValue,
+        executionPayloadValue: prettyWeiToEth(executionPayloadValue),
         prepType,
         fetchedTime,
       });
@@ -607,7 +618,7 @@ export async function produceBlockBody<T extends BlockType>(
         this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
         this.logger.verbose("Fetched execution payload from engine", {
           slot: blockSlot,
-          executionPayloadValue,
+          executionPayloadValue: prettyWeiToEth(executionPayloadValue),
           prepType,
           payloadId,
           fetchedTime,
@@ -708,7 +719,7 @@ export async function produceBlockBody<T extends BlockType>(
     }
   }
 
-  Object.assign(logMeta, {executionPayloadValue});
+  Object.assign(logMeta, {executionPayloadValue: prettyWeiToEth(executionPayloadValue)});
   this.logger.verbose("Produced beacon block body", logMeta);
 
   return {body: blockBody as AssembledBodyType<T>, produceResult, executionPayloadValue, shouldOverrideBuilder};

package/src/chain/seenCache/seenPayloadEnvelopeInput.ts

@@ -256,10 +256,11 @@ export class SeenPayloadEnvelopeInput {
     return this.payloadInputs.size;
   }
 
-  prune(blockRootHex: RootHex): void {
+  /** Removes the single PayloadEnvelopeInput from the cache */
+  remove(blockRootHex: RootHex): void {
     const input = this.payloadInputs.get(blockRootHex);
     if (input) {
-      this.evictPayloadInput(input, "prune");
+      this.evictPayloadInput(input, "remove");
     }
   }
 

package/src/chain/validation/attesterSlashing.ts

@@ -63,8 +63,7 @@ export async function validateAttesterSlashing(
   const signatureSets = getAttesterSlashingSignatureSets(chain.config, state.slot, attesterSlashing);
   if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) {
     throw new AttesterSlashingError(GossipAction.REJECT, {
-      code: AttesterSlashingErrorCode.INVALID,
-      error: Error("Invalid signature"),
+      code: AttesterSlashingErrorCode.INVALID_SIGNATURE,
     });
   }
 }

package/src/chain/validation/payloadAttestationMessage.ts

@@ -1,4 +1,5 @@
 import {
+  computeEpochAtSlot,
   createIndexedSignatureSetFromComponents,
   getPayloadAttestationDataSigningRoot,
   isStatePostGloas,
@@ -35,6 +36,14 @@ async function validatePayloadAttestationMessage(
 ): Promise<PayloadAttestationValidationResult> {
   const {data, validatorIndex} = payloadAttestationMessage;
 
+  // [REJECT] The payload attestation slot is at or after the Gloas fork.
+  if (computeEpochAtSlot(data.slot) < chain.config.GLOAS_FORK_EPOCH) {
+    throw new PayloadAttestationError(GossipAction.REJECT, {
+      code: PayloadAttestationErrorCode.PRE_GLOAS_SLOT,
+      slot: data.slot,
+    });
+  }
+
   // [IGNORE] The message's slot is for the current slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance), i.e. `data.slot == current_slot`.
   if (!chain.clock.isCurrentSlotGivenGossipDisparity(data.slot)) {
     throw new PayloadAttestationError(GossipAction.IGNORE, {

package/src/chain/validation/proposerSlashing.ts

@@ -48,8 +48,7 @@ async function validateProposerSlashing(
   const signatureSets = getProposerSlashingSignatureSets(chain.config, state.slot, proposerSlashing);
   if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) {
     throw new ProposerSlashingError(GossipAction.REJECT, {
-      code: ProposerSlashingErrorCode.INVALID,
-      error: Error("Invalid signature"),
+      code: ProposerSlashingErrorCode.INVALID_SIGNATURE,
     });
   }
 }

package/src/chain/validatorMonitor.ts

@@ -24,7 +24,17 @@ import {
   deneb,
   gloas,
 } from "@lodestar/types";
-import {LogData, LogHandler, LogLevel, Logger, MapDef, MapDefMax, prettyPrintIndices, toRootHex} from "@lodestar/utils";
+import {
+  LogData,
+  LogHandler,
+  LogLevel,
+  Logger,
+  MapDef,
+  MapDefMax,
+  prettyGweiToEth,
+  prettyPrintIndices,
+  toRootHex,
+} from "@lodestar/utils";
 import {GENESIS_SLOT} from "../constants/constants.js";
 import {RegistryMetricCreator} from "../metrics/index.js";
 
@@ -470,7 +480,7 @@ export function createValidatorMonitor(
         src,
         builderIndex: bid.builderIndex,
         gasLimit: Number(bid.gasLimit),
-        value: bid.value.toString(),
+        value: prettyGweiToEth(bid.value),
         parentBlockRoot: toRootHex(bid.parentBlockRoot),
         parentBlockHash: toRootHex(bid.parentBlockHash),
         blockHash: toRootHex(bid.blockHash),

package/src/db/repositories/blockArchive.ts

@@ -1,8 +1,9 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Db, DbBatch, FilterOptions, KeyValue, Repository} from "@lodestar/db";
 import {Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
-import {bytesToInt, intToBytes} from "@lodestar/utils";
+import {bytesToInt, fromHex, intToBytes} from "@lodestar/utils";
 import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
+import {getParentRootFromSignedBeaconBlockSerialized} from "../../util/sszBytes.js";
 import {Bucket, getBucketNameByValue} from "../buckets.js";
 import {
   deleteParentRootIndex,
@@ -24,6 +25,10 @@ export type BlockArchiveBatchPutBinaryItem = KeyValue<Slot, Uint8Array> & {
   parentRoot: Root;
 };
 
+// The initial prune can span millions of slots. This bounds the slot index reads in flight and keeps each
+// delete batch at a few thousand keys, both tiny next to the block value batches in archiveBlocks
+const DELETE_RANGE_CHUNK_SIZE = 1000;
+
 /**
  * Stores finalized blocks. Block slot is identifier.
  */
@@ -103,6 +108,38 @@ export class BlockArchiveRepository extends Repository<Slot, SignedBeaconBlock>
     );
   }
 
+  /**
+   * Delete a contiguous range of archived blocks together with their index entries.
+   * `slots` must be every archived slot in the range, so the parent of each block is the entry before it.
+   */
+  async batchDeleteRange(slots: Slot[]): Promise<void> {
+    const sorted = [...slots].sort((a, b) => a - b);
+    let prevRoot: Root | null = null;
+
+    for (let i = 0; i < sorted.length; i += DELETE_RANGE_CHUNK_SIZE) {
+      const chunk = sorted.slice(i, i + DELETE_RANGE_CHUNK_SIZE);
+      const roots = await Promise.all(chunk.map((slot) => this.getRootBySlot(slot)));
+
+      const keys: Uint8Array[] = [];
+      for (let j = 0; j < chunk.length; j++) {
+        keys.push(this.encodeKey(chunk[j]), getSlotIndexKey(chunk[j]));
+        const root = roots[j];
+        if (root) keys.push(getRootIndexKey(root));
+        // The parent index entry pointing at this block is keyed by its parent root, which is the
+        // previous block's root unless that block is unindexed, then it is read from the block itself
+        let parentRoot = prevRoot;
+        if (parentRoot === null) {
+          const block = await this.getBinary(chunk[j]);
+          const parentRootHex = block ? getParentRootFromSignedBeaconBlockSerialized(block) : null;
+          parentRoot = parentRootHex ? fromHex(parentRootHex) : null;
+        }
+        if (parentRoot) keys.push(getParentRootIndexKey(parentRoot));
+        prevRoot = root;
+      }
+      await this.db.batchDelete(keys, this.dbReqOpts);
+    }
+  }
+
   async batch(operations: DbBatch<Slot, SignedBeaconBlock>): Promise<void> {
     await this.batchBinary(
       operations.map((operation) =>

package/src/metrics/metrics/lodestar.ts

@@ -26,7 +26,13 @@ import {ExecutionPayloadStatus} from "../../execution/index.js";
 import {GossipType} from "../../network/index.js";
 import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js";
 import {BackfillSyncMethod} from "../../sync/backfill/backfill.js";
-import {DroppedItemReason, PendingBlockType} from "../../sync/types.js";
+import {
+  DeferredPayloadResult,
+  DownloadResult,
+  DroppedItemReason,
+  FetchResult,
+  PendingBlockType,
+} from "../../sync/types.js";
 import {PeerSyncType, RangeSyncType} from "../../sync/utils/remoteSyncType.js";
 import {AllocSource} from "../../util/bufferPool.js";
 import {DataColumnReconstructionCode} from "../../util/dataColumns.js";
@@ -655,6 +661,16 @@ export function createLodestarMetrics(
         help: "Count of payload (execution payload envelope) sync triggers, labeled by their source",
         labelNames: ["source"],
       }),
+      deferredPayloadResult: register.counter<{result: DeferredPayloadResult}>({
+        name: "lodestar_sync_deferred_payload_result_total",
+        help: "Outcome of a slot's deferred optimistic payload searches, by result",
+        labelNames: ["result"],
+      }),
+      deferredPayloadPolls: register.histogram({
+        name: "lodestar_sync_deferred_payload_polls_count",
+        help: "Poll ticks after PAYLOAD_DUE until a searched payload was imported",
+        buckets: [0, 1, 2, 3, 6],
+      }),
       pendingBlocks: register.gauge({
         name: "lodestar_sync_unknown_block_pending_blocks_size",
         help: "Current size of UnknownBlockSync pending blocks cache",
@@ -709,17 +725,44 @@ export function createLodestarMetrics(
         buckets: [0, 1, 2, 4],
       }),
       // we may not have slot in case of failure, so track fetch time from start to done (either success or failure)
-      fetchTimeSec: register.histogram<{result: string}>({
+      fetchTime: register.histogram<{result: FetchResult}>({
         name: "lodestar_sync_unknown_block_fetch_time_seconds",
         help: "Fetch time from start to done (either success or failure)",
         labelNames: ["result"],
         buckets: [0, 1, 2, 4, 8],
       }),
-      fetchPeers: register.gauge<{result: string}>({
+      fetchPeers: register.gauge<{result: FetchResult}>({
         name: "lodestar_sync_unknown_block_fetch_peers_count",
         help: "Number of peers that node fetched from",
         labelNames: ["result"],
       }),
+      downloadedPayloadsResult: register.counter<{result: DownloadResult}>({
+        name: "lodestar_sync_unknown_payload_download_result_total",
+        help: "Total number of downloadPayload results in UnknownBlockSync",
+        labelNames: ["result"],
+      }),
+      elapsedTimeTillPayloadReceived: register.histogram({
+        name: "lodestar_sync_unknown_payload_elapsed_time_till_received_seconds",
+        help: "Time elapsed between payload slot time and the time payload received via unknown block sync",
+        buckets: [6, 8, 10, 12],
+      }),
+      payloadFetchBegin: register.histogram({
+        name: "lodestar_sync_unknown_payload_fetch_begin_since_slot_start_seconds",
+        help: "Time into the slot when the payload was fetched",
+        buckets: [6, 8, 10, 12],
+      }),
+      // we may not have slot in case of failure, so track fetch time from start to done (either success or failure)
+      payloadFetchTime: register.histogram<{result: FetchResult}>({
+        name: "lodestar_sync_unknown_payload_fetch_time_seconds",
+        help: "Payload fetch time from start to done (either success or failure)",
+        labelNames: ["result"],
+        buckets: [0, 1, 2, 4, 8],
+      }),
+      payloadFetchPeers: register.gauge<{result: FetchResult}>({
+        name: "lodestar_sync_unknown_payload_fetch_peers_count",
+        help: "Number of peers that node fetched the payload from",
+        labelNames: ["result"],
+      }),
       downloadByRoot: {
         success: register.gauge({
           name: "lodestar_sync_unknown_block_download_by_root_success_total",

package/src/network/gossip/encoding.ts

@@ -8,7 +8,7 @@ import {ForkName} from "@lodestar/params";
 import {intToBytes} from "@lodestar/utils";
 import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
 import {Eth2GossipsubMetrics} from "./metrics.js";
-import {GossipTopicCache, getGossipSSZMaxSize, getGossipSSZType} from "./topic.js";
+import {GossipTopicCache} from "./topic.js";
 
 // Load WASM
 const xxhash = await xxhashFactory();
@@ -97,7 +97,6 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8
 export class DataTransformSnappy implements DataTransform {
   constructor(
     private readonly gossipTopicCache: GossipTopicCache,
-    private readonly maxPayloadSize: number,
     private readonly metrics: Eth2GossipsubMetrics | null
   ) {}
 
@@ -112,12 +111,11 @@ export class DataTransformSnappy implements DataTransform {
     const uncompressedDataLength = snappyWasm.decompress_len(data);
 
     const topic = this.gossipTopicCache.getTopic(topicStr);
-    const sszType = getGossipSSZType(topic);
-    const maxSize = getGossipSSZMaxSize(topic, this.maxPayloadSize, sszType);
+    const {minSize, maxSize} = this.gossipTopicCache.getTypeSizes(topicStr);
     this.metrics?.dataTransform.inbound.inc({type: topic.type});
 
-    if (uncompressedDataLength < sszType.minSize) {
-      throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${sszType.minSize}`);
+    if (uncompressedDataLength < minSize) {
+      throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${minSize}`);
     }
     if (uncompressedDataLength > maxSize) {
       throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${maxSize}`);
@@ -136,8 +134,7 @@ export class DataTransformSnappy implements DataTransform {
    */
   outboundTransform(topicStr: string, data: Uint8Array): Uint8Array {
     const topic = this.gossipTopicCache.getTopic(topicStr);
-    const sszType = getGossipSSZType(topic);
-    const maxSize = getGossipSSZMaxSize(topic, this.maxPayloadSize, sszType);
+    const {maxSize} = this.gossipTopicCache.getTypeSizes(topicStr);
     this.metrics?.dataTransform.outbound.inc({type: topic.type});
     if (data.length > maxSize) {
       throw Error(`ssz_snappy encoded data length ${data.length} > ${maxSize}`);

package/src/network/gossip/gossipsub.ts

@@ -14,7 +14,7 @@ import {peerIdFromString} from "@libp2p/peer-id";
 import {type Multiaddr, multiaddr} from "@multiformats/multiaddr";
 import {ENR} from "@chainsafe/enr";
 import {routes} from "@lodestar/api";
-import {BeaconConfig, ForkBoundary} from "@lodestar/config";
+import {BeaconConfig, ChainConfig, ForkBoundary} from "@lodestar/config";
 import {
   ATTESTATION_SUBNET_COUNT,
   MAX_SIGNED_AGGREGATE_AND_PROOF_SIZE,
@@ -47,6 +47,22 @@ const GOSSIPSUB_HEARTBEAT_INTERVAL = 0.7 * 1000;
 
 const MAX_OUTBOUND_BUFFER_SIZE = 2 ** 24; // 16MB
 
+/**
+ * Snappy worst-case compressed length for a payload of `n` bytes
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-beta.0/specs/phase0/p2p-interface.md#max_compressed_len
+ */
+function maxCompressedLen(n: number): number {
+  return 32 + n + Math.floor(n / 6);
+}
+
+/**
+ * Max size of an inbound gossipsub RPC frame, including all bundled messages and control
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-beta.0/specs/phase0/p2p-interface.md#max_message_size
+ */
+function getMaxInboundDataLength(config: Pick<ChainConfig, "MAX_PAYLOAD_SIZE">): number {
+  return Math.max(maxCompressedLen(config.MAX_PAYLOAD_SIZE) + 1024, 1024 * 1024);
+}
+
 export type Eth2Context = {
   activeValidatorCount: number;
   currentSlot: number;
@@ -168,7 +184,7 @@ export class Eth2Gossipsub {
       fastMsgIdFn: fastMsgIdFn,
       msgIdFn: msgIdFn.bind(msgIdFn, gossipTopicCache),
       msgIdToStrFn: msgIdToStrFn,
-      dataTransform: new DataTransformSnappy(gossipTopicCache, config.MAX_PAYLOAD_SIZE, metrics),
+      dataTransform: new DataTransformSnappy(gossipTopicCache, metrics),
       metricsRegister: metricsRegister as MetricsRegister | null,
       metricsTopicStrToLabel: metricsRegister
         ? getMetricsTopicStrToLabel(networkConfig, {disableLightClientServer: opts.disableLightClientServer ?? false})
@@ -176,6 +192,7 @@ export class Eth2Gossipsub {
       asyncValidation: true,
 
       maxOutboundBufferSize: MAX_OUTBOUND_BUFFER_SIZE,
+      maxInboundDataLength: getMaxInboundDataLength(config),
       // serialize message once and send to all peers when publishing
       batchPublish: true,
       // if this is false, only publish to mesh peers. If there is not enough GOSSIP_D mesh peers,

package/src/network/gossip/topic.ts

@@ -1,5 +1,5 @@
 import {type CompositeTypeAny} from "@chainsafe/ssz";
-import {ForkDigestContext} from "@lodestar/config";
+import {BeaconConfig, ChainForkConfig, ForkDigestContext} from "@lodestar/config";
 import {
   ATTESTATION_SUBNET_COUNT,
   ForkName,
@@ -10,32 +10,47 @@ import {
   isForkPostFulu,
   isForkPostGloas,
 } from "@lodestar/params";
+import {TypeSizes} from "@lodestar/reqresp";
 import {Attestation, SingleAttestation, ssz, sszTypesFor} from "@lodestar/types";
 import {GossipAction, GossipActionError, GossipErrorCode} from "../../chain/errors/gossipValidation.js";
+import {computeMaxGloasDataColumnSidecarSize} from "../../util/sszBytes.js";
 import {NetworkConfig} from "../networkConfig.js";
 import {DEFAULT_ENCODING} from "./constants.js";
 import {GossipEncoding, GossipTopic, GossipTopicTypeMap, GossipType, SSZTypeOfGossipTopic} from "./interface.js";
 
 export interface IGossipTopicCache {
   getTopic(topicStr: string): GossipTopic;
+  getTypeSizes(topicStr: string): TypeSizes;
 }
 
 export class GossipTopicCache implements IGossipTopicCache {
   private topicsByTopicStr = new Map<string, Required<GossipTopic>>();
+  private typeSizesByTopicStr = new Map<string, TypeSizes>();
 
-  constructor(private readonly forkDigestContext: ForkDigestContext) {}
+  constructor(private readonly config: BeaconConfig) {}
 
   /** Returns cached GossipTopic, otherwise attempts to parse it from the str */
   getTopic(topicStr: string): GossipTopic {
     let topic = this.topicsByTopicStr.get(topicStr);
     if (topic === undefined) {
-      topic = parseGossipTopic(this.forkDigestContext, topicStr);
+      topic = parseGossipTopic(this.config, topicStr);
       // TODO: Consider just throwing here. We should only receive messages from known subscribed topics
       this.topicsByTopicStr.set(topicStr, topic);
     }
     return topic;
   }
 
+  getTypeSizes(topicStr: string): TypeSizes {
+    let typeSizes = this.typeSizesByTopicStr.get(topicStr);
+    if (typeSizes === undefined) {
+      const topic = this.getTopic(topicStr);
+      const sszType = getGossipSSZType(topic);
+      typeSizes = {minSize: sszType.minSize, maxSize: getGossipSSZMaxSize(topic, this.config, sszType)};
+      this.typeSizesByTopicStr.set(topicStr, typeSizes);
+    }
+    return typeSizes;
+  }
+
   /** Returns cached GossipTopic, otherwise returns undefined */
   getKnownTopic(topicStr: string): GossipTopic | undefined {
     return this.topicsByTopicStr.get(topicStr);
@@ -133,11 +148,14 @@ export function getGossipSSZType(topic: GossipTopic) {
 }
 
 /**
- * Return the maximum uncompressed SSZ byte length accepted for a gossip object, the SSZ type max size
- * or MAX_PAYLOAD_SIZE, whichever is smaller.
+ * Return the maximum uncompressed SSZ byte length allowed by the type and configured network bounds.
  */
-export function getGossipSSZMaxSize(topic: GossipTopic, maxPayloadSize: number, sszType?: CompositeTypeAny): number {
-  return Math.min((sszType ?? getGossipSSZType(topic)).maxSize, maxPayloadSize);
+export function getGossipSSZMaxSize(topic: GossipTopic, config: ChainForkConfig, sszType: CompositeTypeAny): number {
+  const maxSize = Math.min(sszType.maxSize, config.MAX_PAYLOAD_SIZE);
+  if (isForkPostGloas(topic.boundary.fork) && topic.type === GossipType.data_column_sidecar) {
+    return Math.min(maxSize, computeMaxGloasDataColumnSidecarSize(config));
+  }
+  return maxSize;
 }
 
 /**