@lodestar/beacon-node 1.49.0-dev.c9061c6942 → 1.49.0-dev.c9d60c8080

package/src/metrics/metrics/lodestar.ts

@@ -26,7 +26,13 @@ import {ExecutionPayloadStatus} from "../../execution/index.js";
 import {GossipType} from "../../network/index.js";
 import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js";
 import {BackfillSyncMethod} from "../../sync/backfill/backfill.js";
-import {DownloadResult, DroppedItemReason, FetchResult, PendingBlockType} from "../../sync/types.js";
+import {
+  DeferredPayloadResult,
+  DownloadResult,
+  DroppedItemReason,
+  FetchResult,
+  PendingBlockType,
+} from "../../sync/types.js";
 import {PeerSyncType, RangeSyncType} from "../../sync/utils/remoteSyncType.js";
 import {AllocSource} from "../../util/bufferPool.js";
 import {DataColumnReconstructionCode} from "../../util/dataColumns.js";
@@ -655,6 +661,16 @@ export function createLodestarMetrics(
         help: "Count of payload (execution payload envelope) sync triggers, labeled by their source",
         labelNames: ["source"],
       }),
+      deferredPayloadResult: register.counter<{result: DeferredPayloadResult}>({
+        name: "lodestar_sync_deferred_payload_result_total",
+        help: "Outcome of a slot's deferred optimistic payload searches, by result",
+        labelNames: ["result"],
+      }),
+      deferredPayloadPolls: register.histogram({
+        name: "lodestar_sync_deferred_payload_polls_count",
+        help: "Poll ticks after PAYLOAD_DUE until a searched payload was imported",
+        buckets: [0, 1, 2, 3, 6],
+      }),
       pendingBlocks: register.gauge({
         name: "lodestar_sync_unknown_block_pending_blocks_size",
         help: "Current size of UnknownBlockSync pending blocks cache",

package/src/network/gossip/encoding.ts

@@ -8,7 +8,7 @@ import {ForkName} from "@lodestar/params";
 import {intToBytes} from "@lodestar/utils";
 import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
 import {Eth2GossipsubMetrics} from "./metrics.js";
-import {GossipTopicCache, getGossipSSZMaxSize, getGossipSSZType} from "./topic.js";
+import {GossipTopicCache} from "./topic.js";
 
 // Load WASM
 const xxhash = await xxhashFactory();
@@ -97,7 +97,6 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8
 export class DataTransformSnappy implements DataTransform {
   constructor(
     private readonly gossipTopicCache: GossipTopicCache,
-    private readonly maxPayloadSize: number,
     private readonly metrics: Eth2GossipsubMetrics | null
   ) {}
 
@@ -112,12 +111,11 @@ export class DataTransformSnappy implements DataTransform {
     const uncompressedDataLength = snappyWasm.decompress_len(data);
 
     const topic = this.gossipTopicCache.getTopic(topicStr);
-    const sszType = getGossipSSZType(topic);
-    const maxSize = getGossipSSZMaxSize(topic, this.maxPayloadSize, sszType);
+    const {minSize, maxSize} = this.gossipTopicCache.getTypeSizes(topicStr);
     this.metrics?.dataTransform.inbound.inc({type: topic.type});
 
-    if (uncompressedDataLength < sszType.minSize) {
-      throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${sszType.minSize}`);
+    if (uncompressedDataLength < minSize) {
+      throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${minSize}`);
     }
     if (uncompressedDataLength > maxSize) {
       throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${maxSize}`);
@@ -136,8 +134,7 @@ export class DataTransformSnappy implements DataTransform {
    */
   outboundTransform(topicStr: string, data: Uint8Array): Uint8Array {
     const topic = this.gossipTopicCache.getTopic(topicStr);
-    const sszType = getGossipSSZType(topic);
-    const maxSize = getGossipSSZMaxSize(topic, this.maxPayloadSize, sszType);
+    const {maxSize} = this.gossipTopicCache.getTypeSizes(topicStr);
     this.metrics?.dataTransform.outbound.inc({type: topic.type});
     if (data.length > maxSize) {
       throw Error(`ssz_snappy encoded data length ${data.length} > ${maxSize}`);

package/src/network/gossip/gossipsub.ts

@@ -14,7 +14,7 @@ import {peerIdFromString} from "@libp2p/peer-id";
 import {type Multiaddr, multiaddr} from "@multiformats/multiaddr";
 import {ENR} from "@chainsafe/enr";
 import {routes} from "@lodestar/api";
-import {BeaconConfig, ForkBoundary} from "@lodestar/config";
+import {BeaconConfig, ChainConfig, ForkBoundary} from "@lodestar/config";
 import {
   ATTESTATION_SUBNET_COUNT,
   MAX_SIGNED_AGGREGATE_AND_PROOF_SIZE,
@@ -47,6 +47,22 @@ const GOSSIPSUB_HEARTBEAT_INTERVAL = 0.7 * 1000;
 
 const MAX_OUTBOUND_BUFFER_SIZE = 2 ** 24; // 16MB
 
+/**
+ * Snappy worst-case compressed length for a payload of `n` bytes
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-beta.0/specs/phase0/p2p-interface.md#max_compressed_len
+ */
+function maxCompressedLen(n: number): number {
+  return 32 + n + Math.floor(n / 6);
+}
+
+/**
+ * Max size of an inbound gossipsub RPC frame, including all bundled messages and control
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-beta.0/specs/phase0/p2p-interface.md#max_message_size
+ */
+function getMaxInboundDataLength(config: Pick<ChainConfig, "MAX_PAYLOAD_SIZE">): number {
+  return Math.max(maxCompressedLen(config.MAX_PAYLOAD_SIZE) + 1024, 1024 * 1024);
+}
+
 export type Eth2Context = {
   activeValidatorCount: number;
   currentSlot: number;
@@ -168,7 +184,7 @@ export class Eth2Gossipsub {
       fastMsgIdFn: fastMsgIdFn,
       msgIdFn: msgIdFn.bind(msgIdFn, gossipTopicCache),
       msgIdToStrFn: msgIdToStrFn,
-      dataTransform: new DataTransformSnappy(gossipTopicCache, config.MAX_PAYLOAD_SIZE, metrics),
+      dataTransform: new DataTransformSnappy(gossipTopicCache, metrics),
       metricsRegister: metricsRegister as MetricsRegister | null,
       metricsTopicStrToLabel: metricsRegister
         ? getMetricsTopicStrToLabel(networkConfig, {disableLightClientServer: opts.disableLightClientServer ?? false})
@@ -176,6 +192,7 @@ export class Eth2Gossipsub {
       asyncValidation: true,
 
       maxOutboundBufferSize: MAX_OUTBOUND_BUFFER_SIZE,
+      maxInboundDataLength: getMaxInboundDataLength(config),
       // serialize message once and send to all peers when publishing
       batchPublish: true,
       // if this is false, only publish to mesh peers. If there is not enough GOSSIP_D mesh peers,

package/src/network/gossip/topic.ts

@@ -1,5 +1,5 @@
 import {type CompositeTypeAny} from "@chainsafe/ssz";
-import {ForkDigestContext} from "@lodestar/config";
+import {BeaconConfig, ChainForkConfig, ForkDigestContext} from "@lodestar/config";
 import {
   ATTESTATION_SUBNET_COUNT,
   ForkName,
@@ -10,32 +10,47 @@ import {
   isForkPostFulu,
   isForkPostGloas,
 } from "@lodestar/params";
+import {TypeSizes} from "@lodestar/reqresp";
 import {Attestation, SingleAttestation, ssz, sszTypesFor} from "@lodestar/types";
 import {GossipAction, GossipActionError, GossipErrorCode} from "../../chain/errors/gossipValidation.js";
+import {computeMaxGloasDataColumnSidecarSize} from "../../util/sszBytes.js";
 import {NetworkConfig} from "../networkConfig.js";
 import {DEFAULT_ENCODING} from "./constants.js";
 import {GossipEncoding, GossipTopic, GossipTopicTypeMap, GossipType, SSZTypeOfGossipTopic} from "./interface.js";
 
 export interface IGossipTopicCache {
   getTopic(topicStr: string): GossipTopic;
+  getTypeSizes(topicStr: string): TypeSizes;
 }
 
 export class GossipTopicCache implements IGossipTopicCache {
   private topicsByTopicStr = new Map<string, Required<GossipTopic>>();
+  private typeSizesByTopicStr = new Map<string, TypeSizes>();
 
-  constructor(private readonly forkDigestContext: ForkDigestContext) {}
+  constructor(private readonly config: BeaconConfig) {}
 
   /** Returns cached GossipTopic, otherwise attempts to parse it from the str */
   getTopic(topicStr: string): GossipTopic {
     let topic = this.topicsByTopicStr.get(topicStr);
     if (topic === undefined) {
-      topic = parseGossipTopic(this.forkDigestContext, topicStr);
+      topic = parseGossipTopic(this.config, topicStr);
       // TODO: Consider just throwing here. We should only receive messages from known subscribed topics
       this.topicsByTopicStr.set(topicStr, topic);
     }
     return topic;
   }
 
+  getTypeSizes(topicStr: string): TypeSizes {
+    let typeSizes = this.typeSizesByTopicStr.get(topicStr);
+    if (typeSizes === undefined) {
+      const topic = this.getTopic(topicStr);
+      const sszType = getGossipSSZType(topic);
+      typeSizes = {minSize: sszType.minSize, maxSize: getGossipSSZMaxSize(topic, this.config, sszType)};
+      this.typeSizesByTopicStr.set(topicStr, typeSizes);
+    }
+    return typeSizes;
+  }
+
   /** Returns cached GossipTopic, otherwise returns undefined */
   getKnownTopic(topicStr: string): GossipTopic | undefined {
     return this.topicsByTopicStr.get(topicStr);
@@ -133,11 +148,14 @@ export function getGossipSSZType(topic: GossipTopic) {
 }
 
 /**
- * Return the maximum uncompressed SSZ byte length accepted for a gossip object, the SSZ type max size
- * or MAX_PAYLOAD_SIZE, whichever is smaller.
+ * Return the maximum uncompressed SSZ byte length allowed by the type and configured network bounds.
  */
-export function getGossipSSZMaxSize(topic: GossipTopic, maxPayloadSize: number, sszType?: CompositeTypeAny): number {
-  return Math.min((sszType ?? getGossipSSZType(topic)).maxSize, maxPayloadSize);
+export function getGossipSSZMaxSize(topic: GossipTopic, config: ChainForkConfig, sszType: CompositeTypeAny): number {
+  const maxSize = Math.min(sszType.maxSize, config.MAX_PAYLOAD_SIZE);
+  if (isForkPostGloas(topic.boundary.fork) && topic.type === GossipType.data_column_sidecar) {
+    return Math.min(maxSize, computeMaxGloasDataColumnSidecarSize(config));
+  }
+  return maxSize;
 }
 
 /**

package/src/network/processor/gossipHandlers.ts

@@ -244,8 +244,20 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
         // IGNORE means the block is acceptable (e.g. FUTURE_SLOT, ALREADY_KNOWN), just not propagated.
         // Keep the optimistically-added cache entries; they are pruned on finalization. Only REJECT
-        // (provably invalid) and unexpected errors prune below.
+        // (provably invalid), unexpected errors and repeat proposals that are not imported prune.
         if (e.action === GossipAction.IGNORE) {
+          // Only a signature-verified sibling is imported by the beacon_block handler, any other repeat proposal
+          // is dropped from the caches and re-downloaded by sync if it ever becomes relevant. Its signature was not
+          // verified, so only its own entry is removed, never its claimed ancestors
+          if (
+            e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
+            !chain.seenBlockProposers.hasBlockRoot(slot, signedBlock.message.proposerIndex, blockRootHex)
+          ) {
+            chain.seenBlockInputCache.remove(blockRootHex);
+            if (isForkPostGloas(fork)) {
+              chain.seenPayloadEnvelopeInputCache.remove(blockRootHex);
+            }
+          }
           throw e;
         }
 
@@ -258,10 +270,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       }
 
       // REJECT or unexpected (non-BlockGossipError) error: drop the optimistically-added entries from
-      // both caches, keeping them consistent.
-      chain.seenBlockInputCache.prune(blockRootHex);
+      // both caches, keeping them consistent. The block may carry any parent root, so only its own entry is removed
+      chain.seenBlockInputCache.remove(blockRootHex);
       if (isForkPostGloas(fork)) {
-        chain.seenPayloadEnvelopeInputCache.prune(blockRootHex);
+        chain.seenPayloadEnvelopeInputCache.remove(blockRootHex);
       }
       throw e;
     } finally {
@@ -738,7 +750,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         if (
           e instanceof BlockGossipError &&
           e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
-          // this is make sure the block's proposer signature was verified, it should be true anyway
+          // Only a signature-verified sibling recorded in the seen cache is imported. The cache holds at most two
+          // roots per proposer and slot, which bounds full imports over gossip to one alternate
           chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root)
         ) {
           // blockInput was optimistically seeded in validateBeaconBlock and retained on IGNORE

package/src/network/processor/gossipValidatorFn.ts

@@ -1,24 +1,122 @@
 import {TopicValidatorResult} from "@libp2p/gossipsub";
 import {ChainForkConfig} from "@lodestar/config";
 import {Logger} from "@lodestar/utils";
-import {AttestationError, GossipAction, GossipActionError} from "../../chain/errors/index.js";
+import {
+  AttestationError,
+  AttestationErrorCode,
+  AttesterSlashingErrorCode,
+  BlobSidecarErrorCode,
+  BlockErrorCode,
+  BlsToExecutionChangeErrorCode,
+  DataColumnSidecarErrorCode,
+  ExecutionPayloadBidErrorCode,
+  ExecutionPayloadEnvelopeErrorCode,
+  GossipAction,
+  GossipActionError,
+  PayloadAttestationErrorCode,
+  ProposerPreferencesErrorCode,
+  ProposerSlashingErrorCode,
+  SyncCommitteeErrorCode,
+  VoluntaryExitErrorCode,
+} from "../../chain/errors/index.js";
 import {Metrics} from "../../metrics/index.js";
+import {INetworkCore} from "../core/index.js";
 import {
   BatchGossipHandlerFn,
   GossipHandlerFn,
   GossipHandlers,
   GossipMessageInfo,
+  GossipType,
   GossipValidatorBatchFn,
   GossipValidatorFn,
 } from "../gossip/interface.js";
+import {PeerAction} from "../peers/index.js";
 import {prettyPrintPeerIdStr} from "../util.js";
 
 export type ValidatorFnModules = {
   config: ChainForkConfig;
   logger: Logger;
   metrics: Metrics | null;
+  core: INetworkCore;
+};
+
+type RejectPeerActionRule = {default: PeerAction; byCode?: Record<string, PeerAction>};
+
+/**
+ * PeerAction mapping based on the topic and specific codes.
+ * Invalid signatures are Fatal on every topic.
+ */
+const gossipRejectPeerAction: Record<GossipType, RejectPeerActionRule> = {
+  [GossipType.beacon_block]: {
+    default: PeerAction.LowToleranceError,
+    byCode: {[BlockErrorCode.PROPOSAL_SIGNATURE_INVALID]: PeerAction.Fatal},
+  },
+  [GossipType.blob_sidecar]: {
+    default: PeerAction.LowToleranceError,
+    byCode: {[BlobSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID]: PeerAction.Fatal},
+  },
+  [GossipType.data_column_sidecar]: {
+    default: PeerAction.LowToleranceError,
+    byCode: {[DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID]: PeerAction.Fatal},
+  },
+  [GossipType.execution_payload]: {
+    default: PeerAction.LowToleranceError,
+    byCode: {[ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.beacon_aggregate_and_proof]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[AttestationErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.beacon_attestation]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[AttestationErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.sync_committee_contribution_and_proof]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[SyncCommitteeErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.sync_committee]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[SyncCommitteeErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.payload_attestation_message]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[PayloadAttestationErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.execution_payload_bid]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[ExecutionPayloadBidErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.proposer_preferences]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[ProposerPreferencesErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.voluntary_exit]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[VoluntaryExitErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.bls_to_execution_change]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[BlsToExecutionChangeErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.proposer_slashing]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[ProposerSlashingErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  [GossipType.attester_slashing]: {
+    default: PeerAction.HighToleranceError,
+    byCode: {[AttesterSlashingErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
+  // light client validators only throw IGNORE
+  [GossipType.light_client_finality_update]: {default: PeerAction.HighToleranceError},
+  [GossipType.light_client_optimistic_update]: {default: PeerAction.HighToleranceError},
 };
 
+function rejectPeerAction(type: GossipType, code: string): PeerAction {
+  const rule = gossipRejectPeerAction[type];
+  return rule.byCode?.[code] ?? rule.default;
+}
+
 /**
  * Similar to getGossipValidatorFn but return a function to accept a batch of beacon_attestation messages
  * with the same attestation data
@@ -27,7 +125,7 @@ export function getGossipValidatorBatchFn(
   gossipHandlers: GossipHandlers,
   modules: ValidatorFnModules
 ): GossipValidatorBatchFn {
-  const {logger, metrics} = modules;
+  const {logger, metrics, core} = modules;
 
   return async function gossipValidatorBatchFn(messageInfos: GossipMessageInfo[]) {
     // all messageInfos have same topic type
@@ -69,16 +167,19 @@ export function getGossipValidatorBatchFn(
             // only beacon_attestation topic is validated in batch
             metrics?.networkProcessor.gossipAttestationIgnoreByReason.inc({reason: e.type.code});
             return TopicValidatorResult.Ignore;
-          case GossipAction.REJECT:
+          case GossipAction.REJECT: {
             metrics?.networkProcessor.gossipValidationReject.inc({topic: type});
             // only beacon_attestation topic is validated in batch
             metrics?.networkProcessor.gossipAttestationRejectByReason.inc({reason: e.type.code});
+            const peerAction = rejectPeerAction(type, e.type.code);
+            core.reportPeer(propagationSource, peerAction, e.type.code);
             logger.debug(
               `Gossip validation ${type} rejected`,
-              {peerId: prettyPrintPeerIdStr(propagationSource), clientAgent, clientVersion},
+              {peer: propagationSource, clientAgent, clientVersion, peerAction},
               e
             );
             return TopicValidatorResult.Reject;
+          }
         }
       });
     } catch (e) {
@@ -108,7 +209,7 @@ export function getGossipValidatorBatchFn(
  * @see getGossipHandlers for reasoning on why GossipHandlerFn are used for gossip validation.
  */
 export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: ValidatorFnModules): GossipValidatorFn {
-  const {logger, metrics} = modules;
+  const {logger, metrics, core} = modules;
 
   return async function gossipValidatorFn({
     topic,
@@ -155,14 +256,17 @@ export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: Va
           metrics?.networkProcessor.gossipValidationIgnore.inc({topic: type});
           return TopicValidatorResult.Ignore;
 
-        case GossipAction.REJECT:
+        case GossipAction.REJECT: {
           metrics?.networkProcessor.gossipValidationReject.inc({topic: type});
+          const peerAction = rejectPeerAction(type, e.type.code);
+          core.reportPeer(propagationSource, peerAction, e.type.code);
           logger.debug(
             `Gossip validation ${type} rejected`,
-            {peerId: prettyPrintPeerIdStr(propagationSource), clientAgent, clientVersion},
+            {peer: propagationSource, clientAgent, clientVersion, peerAction},
             e
           );
           return TopicValidatorResult.Reject;
+        }
       }
     }
   };

package/src/network/processor/index.ts

@@ -86,8 +86,13 @@ export const MAX_AWAITING_MESSAGES_PER_ROOT: Partial<Record<GossipType, number>>
 
 /**
  * Track the forwarded peers we already emit to UnknownBlockInput sync.
+ * envelopeSlotEmitted: whether the unknownEnvelopeBlockRootSlot event was already emitted for this root.
  */
-type SearchedRootEntry = {blockPeerIds?: Set<PeerIdStr>; envelopePeerIds?: Set<PeerIdStr>};
+type SearchedRootEntry = {
+  blockPeerIds?: Set<PeerIdStr>;
+  envelopePeerIds?: Set<PeerIdStr>;
+  envelopeSlotEmitted?: boolean;
+};
 
 /**
  * This is respective to gossipsub seenTTL (which is 550 * 0.7 = 385s), also it's respective
@@ -192,6 +197,19 @@ type PreprocessResult =
   | {action: PreprocessAction.AwaitBlock; root: RootHex}
   | {action: PreprocessAction.AwaitEnvelope; root: RootHex};
 
+enum SearchTarget {
+  Block,
+  PayloadEnvelope,
+}
+
+type SearchUnknownRootTarget =
+  | {target: SearchTarget.Block}
+  | {
+      target: SearchTarget.PayloadEnvelope;
+      /** The message slot is the payload's slot, so it can be used for UnknownBlockInput search */
+      slotIsPayloadSlot: boolean;
+    };
+
 /**
  * Network processor handles the gossip queues and throtles processing to not overload the main thread
  * - Decides when to process work and what to process
@@ -347,12 +365,17 @@ export class NetworkProcessor {
   }
 
   /**
-   * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` event
+   * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` / `ChainEvent.unknownEnvelopeBlockRootSlot` event
    * Slot is the message slot, which is not necessarily the same as the envelope's slot, but it can be used for a good prune strategy.
    * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
    * Same peer-forwarding + dedup behavior as searchUnknownBlock.
    */
-  searchUnknownEnvelope({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
+  searchUnknownEnvelope(
+    {slot, root}: SlotRootHex,
+    source: BlockInputSource,
+    peer?: PeerIdStr,
+    slotIsPayloadSlot = false
+  ): void {
     if (this.chain.seenPayloadEnvelope(root)) {
       return;
     }
@@ -369,8 +392,17 @@ export class NetworkProcessor {
       forwardedPeers.add(peer);
       shouldEmit = true;
     }
+    // the payload-slot seed must reach the sync once, even if a slot-less trigger started the search first
+    if (slotIsPayloadSlot && !entry.envelopeSlotEmitted) {
+      shouldEmit = true;
+    }
     if (shouldEmit) {
-      this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
+      if (slotIsPayloadSlot) {
+        entry.envelopeSlotEmitted = true;
+        this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRootSlot, {rootHex: root, slot, peer, source});
+      } else {
+        this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, peer, source});
+      }
     }
   }
 
@@ -431,7 +463,7 @@ export class NetworkProcessor {
     // no need to check if root is a descendant of the current finalized block, it will be checked once we validate the message if needed
     if (root && !this.chain.forkChoice.hasBlockHexUnsafe(root)) {
       // starting from GLOAS, unknown root from data_column_sidecar also falls into this case
-      this.searchUnknownRoot({slot, root}, true, false, peerId);
+      this.searchUnknownRoot({slot, root}, peerId, {target: SearchTarget.Block});
       // for beacon_attestation and beacon_aggregate_and_proof messages, this is only temporary.
       // if "index = 1" we need to await for the Envelope instead
       preprocessResult = {action: PreprocessAction.AwaitBlock, root};
@@ -446,27 +478,28 @@ export class NetworkProcessor {
     if (topicType === GossipType.beacon_block) {
       const parentRoot = getParentRootFromSignedBeaconBlockSerialized(message.msg.data);
       if (parentRoot) {
-        let searchParentBlock = false;
-        let searchParentEnvelope = false;
+        let search: SearchUnknownRootTarget | null = null;
         if (ForkSeq[fork] >= ForkSeq.gloas) {
           // GLOAS: also check parent envelope, same logic as execution_payload_bid
           const parentBlockHash = getParentBlockHashFromGloasSignedBeaconBlockSerialized(message.msg.data);
           if (parentBlockHash && !this.chain.forkChoice.getBlockHexAndBlockHash(parentRoot, parentBlockHash)) {
             const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
             if (protoBlock === null) {
-              searchParentBlock = true;
+              search = {target: SearchTarget.Block};
             } else if (
               protoBlock.executionPayloadBlockHash &&
               protoBlock.executionPayloadBlockHash !== parentBlockHash
             ) {
               // only search for the envelope by block root if we're sure there is one. Otherwise UnknownBlockSync will penalize the peer.
-              searchParentEnvelope = true;
+              search = {target: SearchTarget.PayloadEnvelope, slotIsPayloadSlot: false};
             }
           }
         } else if (!this.chain.forkChoice.hasBlockHexUnsafe(parentRoot)) {
-          searchParentBlock = true;
+          search = {target: SearchTarget.Block};
+        }
+        if (search !== null) {
+          this.searchUnknownRoot({slot, root: parentRoot}, peerId, search);
         }
-        this.searchUnknownRoot({slot, root: parentRoot}, searchParentBlock, searchParentEnvelope, peerId);
       }
       preprocessResult = {action: PreprocessAction.PushToQueue};
     }
@@ -485,7 +518,10 @@ export class NetworkProcessor {
               : getDataIndexFromSignedAggregateAndProofSerialized(message.msg.data);
           if (attIndex === 1 && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
             // attestation votes that the payload is available but it is not yet known
-            this.searchUnknownRoot({slot, root}, false, true, peerId);
+            this.searchUnknownRoot({slot, root}, peerId, {
+              target: SearchTarget.PayloadEnvelope,
+              slotIsPayloadSlot: false,
+            });
             preprocessResult = {action: PreprocessAction.AwaitEnvelope, root};
           }
           break;
@@ -494,8 +530,13 @@ export class NetworkProcessor {
           if (root == null) break;
           const payloadPresent = getPayloadPresentFromPayloadAttestationMessageSerialized(message.msg.data);
           if (payloadPresent && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
-            // payload attestation votes that the payload is available but it is not yet known
-            this.searchUnknownRoot({slot, root}, false, true, peerId);
+            // payload attestation votes that the payload is available but it is not yet known.
+            // this is optimistic search, the peer may not have the payload (only the ptc committee had).
+            // the PTC vote's slot is the payload's slot
+            this.searchUnknownRoot({slot, root}, undefined, {
+              target: SearchTarget.PayloadEnvelope,
+              slotIsPayloadSlot: true,
+            });
             // do not await the envelope, payload attestation processing only requires that the block is known
             // also do not reset preprocessResult, we may already await for the block
           }
@@ -504,7 +545,12 @@ export class NetworkProcessor {
         case GossipType.data_column_sidecar: {
           if (root == null) break;
           if (!this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
-            this.searchUnknownRoot({slot, root}, false, true, peerId);
+            // this is optimistic search, the peer may not have the payload.
+            // the sidecar's slot is the block's (and so the payload's) slot
+            this.searchUnknownRoot({slot, root}, undefined, {
+              target: SearchTarget.PayloadEnvelope,
+              slotIsPayloadSlot: true,
+            });
             // do not await the envelope, we can do gossip validation
             // also do not reset preprocessResult, we may already await for the block
           }
@@ -515,7 +561,7 @@ export class NetworkProcessor {
           // Extract beacon_block_root directly
           const blockRoot = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(message.msg.data);
           if (blockRoot && !this.chain.forkChoice.hasBlockHexUnsafe(blockRoot)) {
-            this.searchUnknownRoot({slot, root: blockRoot}, true, false, peerId);
+            this.searchUnknownRoot({slot, root: blockRoot}, peerId, {target: SearchTarget.Block});
             // We always want to await the block
             // This allows us to properly forward the payload envelope
             preprocessResult = {action: PreprocessAction.AwaitBlock, root: blockRoot};
@@ -534,13 +580,16 @@ export class NetworkProcessor {
           ) {
             const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentBlockRoot);
             if (protoBlock === null) {
-              this.searchUnknownRoot({slot, root: parentBlockRoot}, true, false, peerId);
+              this.searchUnknownRoot({slot, root: parentBlockRoot}, peerId, {target: SearchTarget.Block});
               preprocessResult = {action: PreprocessAction.AwaitBlock, root: parentBlockRoot};
             } else if (
               protoBlock.executionPayloadBlockHash &&
               protoBlock.executionPayloadBlockHash !== parentBlockHash
             ) {
-              this.searchUnknownRoot({slot, root: parentBlockRoot}, false, true, peerId);
+              this.searchUnknownRoot({slot, root: parentBlockRoot}, peerId, {
+                target: SearchTarget.PayloadEnvelope,
+                slotIsPayloadSlot: false,
+              });
               preprocessResult = {action: PreprocessAction.AwaitEnvelope, root: parentBlockRoot};
             }
           }
@@ -641,16 +690,24 @@ export class NetworkProcessor {
     return true;
   }
 
+  /**
+   * Search block/envelope given a SlotRootHex
+   * undefined peer id means optimistic search
+   */
   private searchUnknownRoot(
     slotRoot: SlotRootHex,
-    searchBlock: boolean,
-    searchEnvelope: boolean,
-    peerId: PeerIdStr
+    peerId: PeerIdStr | undefined,
+    search: SearchUnknownRootTarget
   ): void {
-    if (!searchBlock && !searchEnvelope) return;
     if (this.tooManySearchedRoots(slotRoot.slot, slotRoot.root)) return;
-    if (searchBlock) this.searchUnknownBlock(slotRoot, BlockInputSource.network_processor, peerId);
-    if (searchEnvelope) this.searchUnknownEnvelope(slotRoot, BlockInputSource.network_processor, peerId);
+    switch (search.target) {
+      case SearchTarget.Block:
+        this.searchUnknownBlock(slotRoot, BlockInputSource.network_processor, peerId);
+        break;
+      case SearchTarget.PayloadEnvelope:
+        this.searchUnknownEnvelope(slotRoot, BlockInputSource.network_processor, peerId, search.slotIsPayloadSlot);
+        break;
+    }
   }
 
   private pushPendingGossipsubMessageToQueue(message: PendingGossipsubMessage): void {