@lodestar/beacon-node 1.42.0-dev.7b5ce3932d → 1.42.0-dev.7bdb87e586

package/lib/network/processor/index.d.ts

@@ -27,11 +27,11 @@ export type NetworkProcessorOpts = GossipHandlerOpts & {
  */
 export declare enum ReprocessRejectReason {
     /**
-     * There are too many attestations that have unknown block root.
+     * There are too many gossip messages that have unknown block root.
      */
     reached_limit = "reached_limit",
     /**
-     * The awaiting attestation is pruned per clock slot.
+     * The awaiting gossip message is pruned per clock slot.
      */
     expired = "expired"
 }
@@ -59,7 +59,7 @@ export declare enum CannotAcceptWorkReason {
  *
  * ### PendingGossipsubMessage beacon_attestation example
  *
- * For attestations, processing the message includes the steps:
+ * For gossip messages, processing the message includes the steps:
  * 1. Pre shuffling sync validation
  * 2. Retrieve shuffling: async + goes into the regen queue and can be expensive
  * 3. Pre sig validation sync validation
@@ -79,17 +79,30 @@ export declare class NetworkProcessor {
     private readonly gossipQueues;
     private readonly gossipTopicConcurrency;
     private readonly extractBlockSlotRootFns;
-    private readonly awaitingGossipsubMessagesByRootBySlot;
-    private unknownBlockGossipsubMessagesCount;
-    private unknownRootsBySlot;
+    private readonly awaitingMessagesByBlockRoot;
+    private readonly awaitingMessagesByPayloadBlockRoot;
+    private unknownBlocksBySlot;
+    private unknownEnvelopesBySlot;
     constructor(modules: NetworkProcessorModules, opts: NetworkProcessorOpts);
     stop(): Promise<void>;
     dropAllJobs(): void;
     dumpGossipQueue(topic: GossipType): PendingGossipsubMessage[];
-    searchUnknownSlotRoot({ slot, root }: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
+    /**
+     * Search block via `ChainEvent.unknownBlockRoot` event
+     * Slot is the message slot, which is not necessarily the same as the block's slot, but it can be used for a good prune strategy.
+     * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+     */
+    searchUnknownBlock({ slot, root }: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
+    /**
+     * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` event
+     * Slot is the message slot, which is not necessarily the same as the envelope's slot, but it can be used for a good prune strategy.
+     * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+     */
+    searchUnknownEnvelope({ slot, root }: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
     private onPendingGossipsubMessage;
     private pushPendingGossipsubMessageToQueue;
     private onBlockProcessed;
+    private onPayloadEnvelopeProcessed;
     private onClockSlot;
     private executeWork;
     private processPendingGossipsubMessage;
@@ -98,5 +111,7 @@ export declare class NetworkProcessor {
      * Return null if chain can accept work, otherwise return the reason why it cannot accept work
      */
     private checkAcceptWork;
+    private get unknownBlockGossipsubMessagesCount();
+    private get unknownPayloadGossipsubMessagesCount();
 }
 //# sourceMappingURL=index.d.ts.map

package/lib/network/processor/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/network/processor/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAgB,WAAW,EAAC,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAC,MAAM,EAA0C,MAAM,iBAAiB,CAAC;AAChF,OAAO,EAAC,gBAAgB,EAAC,MAAM,wCAAwC,CAAC;AAGxE,OAAO,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAC,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAGjD,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAe,eAAe,EAAC,MAAM,cAAc,CAAC;AAC3D,OAAO,EACL,cAAc,EAEd,UAAU,EAGX,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAC,iBAAiB,EAAE,mBAAmB,EAAoB,MAAM,qBAAqB,CAAC;AAE9F,OAAO,EAAC,kBAAkB,EAAkD,MAAM,wBAAwB,CAAC;AAC3G,OAAO,EAAC,uBAAuB,EAAC,MAAM,YAAY,CAAC;AAEnD,cAAc,YAAY,CAAC;AAE3B,MAAM,MAAM,uBAAuB,GAAG,mBAAmB,GACvD,kBAAkB,GAAG;IACnB,KAAK,EAAE,YAAY,CAAC;IACpB,EAAE,EAAE,SAAS,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAExB,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC,CAAC;AAEJ,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG;IACrD,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACpC,CAAC;AAyDF;;GAEG;AACH,oBAAY,qBAAqB;IAC/B;;OAEG;IACH,aAAa,kBAAkB;IAC/B;;OAEG;IACH,OAAO,YAAY;CACpB;AAED;;GAEG;AACH,oBAAY,sBAAsB;IAChC;;OAEG;IACH,GAAG,aAAa;IAChB;;OAEG;IACH,KAAK,eAAe;CACrB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,gBAAgB;IAkBzB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAjBvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;IACtD,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAyB;IAChE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAwC;IACrE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAA8B;IACrE,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAmC;IAG3E,OAAO,CAAC,QAAQ,CAAC,qCAAqC,CAA8D;IACpH,OAAO,CAAC,kCAAkC,CAAK;IAC/C,OAAO,CAAC,kBAAkB,CAAmD;IAE7E,YACE,OAAO,EAAE,uBAAuB,EACf,IAAI,EAAE,oBAAoB,EA8C5C;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAED,WAAW,IAAI,IAAI,CAIlB;IAED,eAAe,CAAC,KAAK,EAAE,UAAU,GAAG,uBAAuB,EAAE,CAO5D;IAED,qBAAqB,CAAC,EAAC,IAAI,EAAE,IAAI,EAAC,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,IAAI,CAOjG;IAED,OAAO,CAAC,yBAAyB;IAoDjC,OAAO,CAAC,kCAAkC;YAY5B,gBAAgB;IAiC9B,OAAO,CAAC,WAAW;IAqBnB,OAAO,CAAC,WAAW;YAkDL,8BAA8B;IA0D5C,OAAO,CAAC,YAAY;IAcpB;;OAEG;IACH,OAAO,CAAC,eAAe;CAWxB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/network/processor/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAgB,WAAW,EAAC,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAC,MAAM,EAA2B,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAC,gBAAgB,EAAC,MAAM,wCAAwC,CAAC;AAGxE,OAAO,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAC,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAGjD,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAW/C,OAAO,EAAe,eAAe,EAAC,MAAM,cAAc,CAAC;AAC3D,OAAO,EACL,cAAc,EAEd,UAAU,EAGX,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAC,iBAAiB,EAAE,mBAAmB,EAAoB,MAAM,qBAAqB,CAAC;AAE9F,OAAO,EAAC,kBAAkB,EAAkD,MAAM,wBAAwB,CAAC;AAC3G,OAAO,EAAC,uBAAuB,EAAC,MAAM,YAAY,CAAC;AAEnD,cAAc,YAAY,CAAC;AAE3B,MAAM,MAAM,uBAAuB,GAAG,mBAAmB,GACvD,kBAAkB,GAAG;IACnB,KAAK,EAAE,YAAY,CAAC;IACpB,EAAE,EAAE,SAAS,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAExB,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC,CAAC;AAEJ,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG;IACrD,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACpC,CAAC;AA2DF;;GAEG;AACH,oBAAY,qBAAqB;IAC/B;;OAEG;IACH,aAAa,kBAAkB;IAC/B;;OAEG;IACH,OAAO,YAAY;CACpB;AAED;;GAEG;AACH,oBAAY,sBAAsB;IAChC;;OAEG;IACH,GAAG,aAAa;IAChB;;OAEG;IACH,KAAK,eAAe;CACrB;AAgBD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,gBAAgB;IAqBzB,OAAO,CAAC,QAAQ,CAAC,IAAI;IApBvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;IACtD,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAyB;IAChE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAwC;IACrE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAA8B;IACrE,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAmC;IAG3E,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAgD;IAG5F,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgD;IACnG,OAAO,CAAC,mBAAmB,CAAmD;IAC9E,OAAO,CAAC,sBAAsB,CAAmD;IAEjF,YACE,OAAO,EAAE,uBAAuB,EACf,IAAI,EAAE,oBAAoB,EA+C5C;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAED,WAAW,IAAI,IAAI,CAIlB;IAED,eAAe,CAAC,KAAK,EAAE,UAAU,GAAG,uBAAuB,EAAE,CAO5D;IAED;;;;OAIG;IACH,kBAAkB,CAAC,EAAC,IAAI,EAAE,IAAI,EAAC,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,IAAI,CAW9F;IAED;;;;OAIG;IACH,qBAAqB,CAAC,EAAC,IAAI,EAAE,IAAI,EAAC,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,IAAI,CAUjG;IAED,OAAO,CAAC,yBAAyB,CAoO/B;IAEF,OAAO,CAAC,kCAAkC;IAY1C,OAAO,CAAC,gBAAgB,CA2BtB;IAEF,OAAO,CAAC,0BAA0B,CAwBhC;IAEF,OAAO,CAAC,WAAW,CAiDjB;IAEF,OAAO,CAAC,WAAW;YAkDL,8BAA8B;IA0D5C,OAAO,CAAC,YAAY;IAcpB;;OAEG;IACH,OAAO,CAAC,eAAe;IAYvB,OAAO,KAAK,kCAAkC,GAM7C;IAED,OAAO,KAAK,oCAAoC,GAM/C;CACF"}

package/lib/network/processor/index.js

@@ -1,12 +1,13 @@
 import { routes } from "@lodestar/api";
 import { ForkSeq } from "@lodestar/params";
 import { computeStartSlotAtEpoch } from "@lodestar/state-transition";
-import { MapDef, mapValues, pruneSetToMax, sleep } from "@lodestar/utils";
+import { MapDef, mapValues, sleep } from "@lodestar/utils";
 import { BlockInputSource } from "../../chain/blocks/blockInput/types.js";
 import { ChainEvent } from "../../chain/emitter.js";
 import { GossipErrorCode } from "../../chain/errors/gossipValidation.js";
 import { ClockEvent } from "../../util/clock.js";
 import { callInNextEventLoop } from "../../util/eventLoop.js";
+import { getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized, getDataIndexFromSignedAggregateAndProofSerialized, getDataIndexFromSingleAttestationSerialized, getParentBlockHashFromGloasSignedBeaconBlockSerialized, getParentBlockHashFromSignedExecutionPayloadBidSerialized, getParentBlockRootFromSignedExecutionPayloadBidSerialized, getParentRootFromSignedBeaconBlockSerialized, getPayloadPresentFromPayloadAttestationMessageSerialized, } from "../../util/sszBytes.js";
 import { NetworkEvent } from "../events.js";
 import { GossipType, } from "../gossip/interface.js";
 import { createExtractBlockSlotRootFns } from "./extractSlotRootFns.js";
@@ -50,14 +51,16 @@ const executeGossipWorkOrderObj = {
 const executeGossipWorkOrder = Object.keys(executeGossipWorkOrderObj);
 // TODO: Arbitrary constant, check metrics
 const MAX_JOBS_SUBMITTED_PER_TICK = 128;
-// How many attestations (aggregate + unaggregate) we keep before new ones get dropped.
+// How many gossip messages we keep before new ones get dropped.
 const MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS = 16_384;
-// We don't want to process too many attestations in a single tick
-// As seen on mainnet, attestation concurrency metric ranges from 1000 to 2000
+// TODO gloas: arbitrary constant, check metrics.
+const MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS = 1024;
+// We don't want to process too many gossip messages in a single tick
+// As seen on mainnet, gossip messages concurrency metric ranges from 1000 to 2000
 // so make this constant a little bit conservative
-const MAX_UNKNOWN_BLOCK_GOSSIP_OBJECTS_PER_TICK = 1024;
+const MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK = 1024;
 // Same motivation to JobItemQueue, we don't want to block the event loop
-const PROCESS_UNKNOWN_BLOCK_GOSSIP_OBJECTS_YIELD_EVERY_MS = 50;
+const AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS = 50;
 export { ReprocessRejectReason };
 /**
  * Reprocess reject reason for metrics
@@ -65,11 +68,11 @@ export { ReprocessRejectReason };
 var ReprocessRejectReason;
 (function (ReprocessRejectReason) {
     /**
-     * There are too many attestations that have unknown block root.
+     * There are too many gossip messages that have unknown block root.
      */
     ReprocessRejectReason["reached_limit"] = "reached_limit";
     /**
-     * The awaiting attestation is pruned per clock slot.
+     * The awaiting gossip message is pruned per clock slot.
      */
     ReprocessRejectReason["expired"] = "expired";
 })(ReprocessRejectReason || (ReprocessRejectReason = {}));
@@ -88,6 +91,15 @@ var CannotAcceptWorkReason;
      */
     CannotAcceptWorkReason["regen"] = "regen_busy";
 })(CannotAcceptWorkReason || (CannotAcceptWorkReason = {}));
+/**
+ * No metrics needed here; using a number to keep it lightweight
+ */
+var PreprocessAction;
+(function (PreprocessAction) {
+    PreprocessAction[PreprocessAction["AwaitBlock"] = 0] = "AwaitBlock";
+    PreprocessAction[PreprocessAction["AwaitEnvelope"] = 1] = "AwaitEnvelope";
+    PreprocessAction[PreprocessAction["PushToQueue"] = 2] = "PushToQueue";
+})(PreprocessAction || (PreprocessAction = {}));
 /**
  * Network processor handles the gossip queues and throtles processing to not overload the main thread
  * - Decides when to process work and what to process
@@ -99,7 +111,7 @@ var CannotAcceptWorkReason;
  *
  * ### PendingGossipsubMessage beacon_attestation example
  *
- * For attestations, processing the message includes the steps:
+ * For gossip messages, processing the message includes the steps:
  * 1. Pre shuffling sync validation
  * 2. Retrieve shuffling: async + goes into the regen queue and can be expensive
  * 3. Pre sig validation sync validation
@@ -119,11 +131,14 @@ export class NetworkProcessor {
     gossipQueues;
     gossipTopicConcurrency;
     extractBlockSlotRootFns = createExtractBlockSlotRootFns();
-    // we may not receive the block for Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
+    // we may not receive the block for messages like Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
     // to be stored in this Map and reprocessed once the block comes
-    awaitingGossipsubMessagesByRootBySlot;
-    unknownBlockGossipsubMessagesCount = 0;
-    unknownRootsBySlot = new MapDef(() => new Set());
+    awaitingMessagesByBlockRoot;
+    // we may not receive the payload for messages that require the FULL payload variant to be processed,
+    // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
+    awaitingMessagesByPayloadBlockRoot;
+    unknownBlocksBySlot = new MapDef(() => new Set());
+    unknownEnvelopesBySlot = new MapDef(() => new Set());
     constructor(modules, opts) {
         this.opts = opts;
         const { chain, events, logger, metrics } = modules;
@@ -136,10 +151,12 @@ export class NetworkProcessor {
         this.gossipTopicConcurrency = mapValues(this.gossipQueues, () => 0);
         this.gossipValidatorFn = getGossipValidatorFn(modules.gossipHandlers ?? getGossipHandlers(modules, opts), modules);
         this.gossipValidatorBatchFn = getGossipValidatorBatchFn(modules.gossipHandlers ?? getGossipHandlers(modules, opts), modules);
-        events.on(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage.bind(this));
-        this.chain.emitter.on(routes.events.EventType.block, this.onBlockProcessed.bind(this));
-        this.chain.clock.on(ClockEvent.slot, this.onClockSlot.bind(this));
-        this.awaitingGossipsubMessagesByRootBySlot = new MapDef(() => new MapDef(() => new Set()));
+        events.on(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
+        this.chain.emitter.on(routes.events.EventType.block, this.onBlockProcessed);
+        this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadEnvelopeProcessed);
+        this.chain.clock.on(ClockEvent.slot, this.onClockSlot);
+        this.awaitingMessagesByBlockRoot = new MapDef(() => new Set());
+        this.awaitingMessagesByPayloadBlockRoot = new MapDef(() => new Set());
         // TODO: Implement queues and priorization for ReqResp incoming requests
         // Listens to NetworkEvent.reqRespIncomingRequest event
         if (metrics) {
@@ -149,7 +166,8 @@ export class NetworkProcessor {
                     metrics.gossipValidationQueue.keySize.set({ topic }, this.gossipQueues[topic].keySize);
                     metrics.gossipValidationQueue.concurrency.set({ topic }, this.gossipTopicConcurrency[topic]);
                 }
-                metrics.reprocessGossipAttestations.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
+                metrics.awaitingBlockGossipMessages.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
+                metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.unknownPayloadGossipsubMessagesCount);
                 // specific metric for beacon_attestation topic
                 metrics.gossipValidationQueue.keyAge.reset();
                 for (const ageMs of this.gossipQueues.beacon_attestation.getDataAgeMs()) {
@@ -164,6 +182,7 @@ export class NetworkProcessor {
     async stop() {
         this.events.off(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
         this.chain.emitter.off(routes.events.EventType.block, this.onBlockProcessed);
+        this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadEnvelopeProcessed);
         this.chain.emitter.off(ClockEvent.slot, this.onClockSlot);
     }
     dropAllJobs() {
@@ -178,112 +197,312 @@ export class NetworkProcessor {
         }
         return queue.getAll();
     }
-    searchUnknownSlotRoot({ slot, root }, source, peer) {
-        if (this.chain.seenBlock(root) || this.unknownRootsBySlot.getOrDefault(slot).has(root)) {
+    /**
+     * Search block via `ChainEvent.unknownBlockRoot` event
+     * Slot is the message slot, which is not necessarily the same as the block's slot, but it can be used for a good prune strategy.
+     * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+     */
+    searchUnknownBlock({ slot, root }, source, peer) {
+        if (this.chain.seenBlock(root) ||
+            this.awaitingMessagesByBlockRoot.has(root) ||
+            this.unknownBlocksBySlot.getOrDefault(slot).has(root)) {
             return;
         }
         // Search for the unknown block
-        this.unknownRootsBySlot.getOrDefault(slot).add(root);
+        this.unknownBlocksBySlot.getOrDefault(slot).add(root);
         this.chain.emitter.emit(ChainEvent.unknownBlockRoot, { rootHex: root, peer, source });
     }
-    onPendingGossipsubMessage(message) {
+    /**
+     * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` event
+     * Slot is the message slot, which is not necessarily the same as the envelope's slot, but it can be used for a good prune strategy.
+     * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+     */
+    searchUnknownEnvelope({ slot, root }, source, peer) {
+        if (this.chain.seenPayloadEnvelope(root) ||
+            this.awaitingMessagesByPayloadBlockRoot.has(root) ||
+            this.unknownEnvelopesBySlot.getOrDefault(slot).has(root)) {
+            return;
+        }
+        this.unknownEnvelopesBySlot.getOrDefault(slot).add(root);
+        this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, { rootHex: root, peer, source });
+    }
+    onPendingGossipsubMessage = (message) => {
         const topicType = message.topic.type;
         const extractBlockSlotRootFn = this.extractBlockSlotRootFns[topicType];
-        // check block root of Attestation and SignedAggregateAndProof messages
-        if (extractBlockSlotRootFn) {
-            const slotRoot = extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork);
-            // if slotRoot is null, it means the msg.data is invalid
-            // in that case message will be rejected when deserializing data in later phase (gossipValidatorFn)
-            if (slotRoot) {
-                // DOS protection: avoid processing messages that are too old
-                const { slot, root } = slotRoot;
-                const clockSlot = this.chain.clock.currentSlot;
-                const { fork } = message.topic.boundary;
-                let earliestPermissableSlot = clockSlot - DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE;
-                if (ForkSeq[fork] >= ForkSeq.deneb && topicType === GossipType.beacon_attestation) {
-                    // post deneb, the attestations could be in current or previous epoch
-                    earliestPermissableSlot = computeStartSlotAtEpoch(this.chain.clock.currentEpoch - 1);
+        // 1st extract round: make sure slot is in range and if block root is not available
+        // proactively search for it + queue the message
+        const slotRoot = extractBlockSlotRootFn
+            ? extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork)
+            : null;
+        if (slotRoot === null) {
+            // some messages don't have slot and root
+            // if the msg.data is invalid, message will be rejected when deserializing data in later phase (gossipValidatorFn)
+            this.pushPendingGossipsubMessageToQueue(message);
+            return;
+        }
+        // common check for all topics
+        // DOS protection: avoid processing messages that are too old
+        const { slot, root } = slotRoot;
+        const clockSlot = this.chain.clock.currentSlot;
+        const { fork } = message.topic.boundary;
+        let earliestPermissableSlot = clockSlot - DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE;
+        if (ForkSeq[fork] >= ForkSeq.deneb && topicType === GossipType.beacon_attestation) {
+            // post deneb, the attestations could be in current or previous epoch
+            earliestPermissableSlot = computeStartSlotAtEpoch(this.chain.clock.currentEpoch - 1);
+        }
+        if (slot < earliestPermissableSlot) {
+            // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+            this.metrics?.networkProcessor.gossipValidationError.inc({
+                topic: topicType,
+                error: GossipErrorCode.PAST_SLOT,
+            });
+            return;
+        }
+        message.msgSlot = slot;
+        // this determines whether this message needs to wait for a Block or Envelope
+        // a message should only wait for what it voted for, hence we don't want to put it on both queues
+        let preprocessResult = { action: PreprocessAction.PushToQueue };
+        // no need to check if root is a descendant of the current finalized block, it will be checked once we validate the message if needed
+        if (root && !this.chain.forkChoice.hasBlockHexUnsafe(root)) {
+            // starting from GLOAS, unknown root from data_column_sidecar also falls into this case
+            this.searchUnknownBlock({ slot, root }, BlockInputSource.network_processor, message.propagationSource.toString());
+            // for beacon_attestation and beacon_aggregate_and_proof messages, this is only temporary.
+            // if "index = 1" we need to await for the Envelope instead
+            preprocessResult = { action: PreprocessAction.AwaitBlock, root };
+        }
+        // 2nd extract round for some specific topics
+        // we separate the search action from the await action
+        // beacon_block: proactively search for parent block/envelope across all forks, but never queue.
+        // BlockInputSync handles cascading recovery if the gossip handler throws.
+        if (topicType === GossipType.beacon_block) {
+            const parentRoot = getParentRootFromSignedBeaconBlockSerialized(message.msg.data);
+            if (parentRoot) {
+                if (ForkSeq[fork] >= ForkSeq.gloas) {
+                    // GLOAS: also check parent envelope, same logic as execution_payload_bid
+                    const parentBlockHash = getParentBlockHashFromGloasSignedBeaconBlockSerialized(message.msg.data);
+                    if (parentBlockHash && !this.chain.forkChoice.getBlockHexAndBlockHash(parentRoot, parentBlockHash)) {
+                        const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
+                        if (protoBlock === null) {
+                            this.searchUnknownBlock({ slot, root: parentRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
+                        }
+                        else if (protoBlock.executionPayloadBlockHash &&
+                            protoBlock.executionPayloadBlockHash !== parentBlockHash) {
+                            // only search for the envelope by block root if we're sure there is one. Otherwise UnknownBlockSync will penalize the peer.
+                            this.searchUnknownEnvelope({ slot, root: parentRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
+                        }
+                    }
+                }
+                else if (!this.chain.forkChoice.hasBlockHexUnsafe(parentRoot)) {
+                    this.searchUnknownBlock({ slot, root: parentRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
+                }
+            }
+            preprocessResult = { action: PreprocessAction.PushToQueue };
+        }
+        if (ForkSeq[fork] >= ForkSeq.gloas) {
+            // specific check for each topic
+            // note that it's supposed to NOT queue beacon_block (handled above) and execution_payload because it's not a one-off;
+            // for those topics, gossip handlers will throw and BlockInputSync will handle a tree of them instead
+            switch (topicType) {
+                case GossipType.beacon_attestation:
+                case GossipType.beacon_aggregate_and_proof: {
+                    if (root == null)
+                        break;
+                    const attIndex = topicType === GossipType.beacon_attestation
+                        ? getDataIndexFromSingleAttestationSerialized(fork, message.msg.data)
+                        : getDataIndexFromSignedAggregateAndProofSerialized(message.msg.data);
+                    if (attIndex === 1 && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
+                        // attestation votes that the payload is available but it is not yet known
+                        this.searchUnknownEnvelope({ slot, root }, BlockInputSource.network_processor, message.propagationSource.toString());
+                        preprocessResult = { action: PreprocessAction.AwaitEnvelope, root };
+                    }
+                    break;
+                }
+                case GossipType.payload_attestation_message: {
+                    if (root == null)
+                        break;
+                    const payloadPresent = getPayloadPresentFromPayloadAttestationMessageSerialized(message.msg.data);
+                    if (payloadPresent && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
+                        // payload attestation votes that the payload is available but it is not yet known
+                        this.searchUnknownEnvelope({ slot, root }, BlockInputSource.network_processor, message.propagationSource.toString());
+                        // do not await the envelope, payload attestation processing only requires that the block is known
+                        // also do not reset preprocessResult, we may already await for the block
+                    }
+                    break;
+                }
+                case GossipType.data_column_sidecar: {
+                    if (root == null)
+                        break;
+                    if (!this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
+                        this.searchUnknownEnvelope({ slot, root }, BlockInputSource.network_processor, message.propagationSource.toString());
+                        // do not await the envelope, we can do gossip validation
+                        // also do not reset preprocessResult, we may already await for the block
+                    }
+                    break;
+                }
+                case GossipType.execution_payload: {
+                    // extractBlockSlotRootFn does not return a root for this topic.
+                    // Extract beacon_block_root directly and proactively trigger block sync if missing.
+                    // Do NOT await the block — the handler runs immediately; BlockInputSync handles recovery.
+                    const blockRoot = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(message.msg.data);
+                    if (blockRoot && !this.chain.forkChoice.hasBlockHexUnsafe(blockRoot)) {
+                        this.searchUnknownBlock({ slot, root: blockRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
+                    }
+                    // do not await the block, we want UnknownBlockSync to handle it.
+                    preprocessResult = { action: PreprocessAction.PushToQueue };
+                    break;
+                }
+                case GossipType.execution_payload_bid: {
+                    // instead of searching for the message root, this searches for the parent root
+                    const parentBlockRoot = getParentBlockRootFromSignedExecutionPayloadBidSerialized(message.msg.data);
+                    const parentBlockHash = getParentBlockHashFromSignedExecutionPayloadBidSerialized(message.msg.data);
+                    if (parentBlockRoot &&
+                        parentBlockHash &&
+                        !this.chain.forkChoice.getBlockHexAndBlockHash(parentBlockRoot, parentBlockHash)) {
+                        const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentBlockRoot);
+                        if (protoBlock === null) {
+                            this.searchUnknownBlock({ slot, root: parentBlockRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
+                            preprocessResult = { action: PreprocessAction.AwaitBlock, root: parentBlockRoot };
+                        }
+                        else if (protoBlock.executionPayloadBlockHash &&
+                            protoBlock.executionPayloadBlockHash !== parentBlockHash) {
+                            this.searchUnknownEnvelope({ slot, root: parentBlockRoot }, BlockInputSource.network_processor, message.propagationSource.toString());
+                            preprocessResult = { action: PreprocessAction.AwaitEnvelope, root: parentBlockRoot };
+                        }
+                    }
+                    break;
                 }
-                if (slot < earliestPermissableSlot) {
-                    // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
-                    this.metrics?.networkProcessor.gossipValidationError.inc({
+            }
+        }
+        switch (preprocessResult.action) {
+            case PreprocessAction.PushToQueue:
+                this.pushPendingGossipsubMessageToQueue(message);
+                break;
+            case PreprocessAction.AwaitBlock: {
+                if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
+                    // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+                    this.metrics?.awaitingBlockGossipMessages.reject.inc({
+                        reason: ReprocessRejectReason.reached_limit,
                         topic: topicType,
-                        error: GossipErrorCode.PAST_SLOT,
                     });
                     return;
                 }
-                message.msgSlot = slot;
-                // check if we processed a block with this root
-                // no need to check if root is a descendant of the current finalized block, it will be checked once we validate the message if needed
-                if (root && !this.chain.forkChoice.hasBlockHexUnsafe(root)) {
-                    this.searchUnknownSlotRoot({ slot, root }, BlockInputSource.gossip, message.propagationSource.toString());
-                    if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
-                        // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
-                        this.metrics?.reprocessGossipAttestations.reject.inc({ reason: ReprocessRejectReason.reached_limit });
-                        return;
-                    }
-                    this.metrics?.reprocessGossipAttestations.total.inc();
-                    const awaitingGossipsubMessagesByRoot = this.awaitingGossipsubMessagesByRootBySlot.getOrDefault(slot);
-                    const awaitingGossipsubMessages = awaitingGossipsubMessagesByRoot.getOrDefault(root);
-                    awaitingGossipsubMessages.add(message);
-                    this.unknownBlockGossipsubMessagesCount++;
+                this.metrics?.awaitingBlockGossipMessages.queue.inc({ topic: topicType });
+                const awaitingGossipsubMessages = this.awaitingMessagesByBlockRoot.getOrDefault(preprocessResult.root);
+                awaitingGossipsubMessages.add(message);
+                break;
+            }
+            case PreprocessAction.AwaitEnvelope: {
+                if (this.unknownPayloadGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
+                    this.metrics?.awaitingPayloadGossipMessages.reject.inc({
+                        reason: ReprocessRejectReason.reached_limit,
+                        topic: topicType,
+                    });
                     return;
                 }
+                this.metrics?.awaitingPayloadGossipMessages.queue.inc({ topic: topicType });
+                const awaitingPayloadGossipsubMessages = this.awaitingMessagesByPayloadBlockRoot.getOrDefault(preprocessResult.root);
+                awaitingPayloadGossipsubMessages.add(message);
+                break;
             }
         }
-        // bypass the check for other messages
-        this.pushPendingGossipsubMessageToQueue(message);
-    }
+    };
     pushPendingGossipsubMessageToQueue(message) {
         const topicType = message.topic.type;
         const droppedCount = this.gossipQueues[topicType].add(message);
         if (droppedCount) {
-            // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
+            // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
             this.metrics?.gossipValidationQueue.droppedJobs.inc({ topic: message.topic.type }, droppedCount);
         }
         // Tentatively perform work
         this.executeWork();
     }
-    async onBlockProcessed({ slot, block: rootHex, }) {
-        const byRootGossipsubMessages = this.awaitingGossipsubMessagesByRootBySlot.getOrDefault(slot);
-        const waitingGossipsubMessages = byRootGossipsubMessages.getOrDefault(rootHex);
-        if (waitingGossipsubMessages.size === 0) {
+    onBlockProcessed = async ({ block: rootHex }) => {
+        const waitingGossipsubMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
+        if (!waitingGossipsubMessages || waitingGossipsubMessages.size === 0) {
             return;
         }
-        this.metrics?.reprocessGossipAttestations.resolve.inc(waitingGossipsubMessages.size);
         const nowSec = Date.now() / 1000;
         let count = 0;
         // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
         // cache, it may not be necessary at this time
         for (const message of waitingGossipsubMessages) {
-            this.metrics?.reprocessGossipAttestations.waitSecBeforeResolve.set(nowSec - message.seenTimestampSec);
+            const topicType = message.topic.type;
+            this.metrics?.awaitingBlockGossipMessages.waitSecBeforeResolve.set({ topic: topicType }, nowSec - message.seenTimestampSec);
+            this.metrics?.awaitingBlockGossipMessages.resolve.inc({ topic: topicType });
             this.pushPendingGossipsubMessageToQueue(message);
             count++;
             // don't want to block the event loop, worse case it'd wait for 16_084 / 1024 * 50ms = 800ms which is not a big deal
-            if (count === MAX_UNKNOWN_BLOCK_GOSSIP_OBJECTS_PER_TICK) {
+            if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
                 count = 0;
-                await sleep(PROCESS_UNKNOWN_BLOCK_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+                await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
             }
         }
-        byRootGossipsubMessages.delete(rootHex);
-    }
-    onClockSlot(clockSlot) {
+        this.awaitingMessagesByBlockRoot.delete(rootHex);
+    };
+    onPayloadEnvelopeProcessed = async ({ blockRoot: rootHex }) => {
+        const waitingGossipsubMessages = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
+        if (!waitingGossipsubMessages || waitingGossipsubMessages.size === 0) {
+            return;
+        }
+        const nowSec = Date.now() / 1000;
+        let count = 0;
+        for (const message of waitingGossipsubMessages) {
+            const topicType = message.topic.type;
+            this.metrics?.awaitingPayloadGossipMessages.waitSecBeforeResolve.set({ topic: topicType }, nowSec - message.seenTimestampSec);
+            this.metrics?.awaitingPayloadGossipMessages.resolve.inc({ topic: topicType });
+            this.pushPendingGossipsubMessageToQueue(message);
+            count++;
+            if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
+                count = 0;
+                await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+            }
+        }
+        this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
+    };
+    onClockSlot = (clockSlot) => {
         const nowSec = Date.now() / 1000;
-        for (const [slot, gossipMessagesByRoot] of this.awaitingGossipsubMessagesByRootBySlot.entries()) {
-            if (slot < clockSlot) {
-                for (const gossipMessages of gossipMessagesByRoot.values()) {
+        const minSlot = clockSlot - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE;
+        for (const [slot, roots] of this.unknownBlocksBySlot) {
+            if (slot > minSlot)
+                continue;
+            for (const rootHex of roots) {
+                const gossipMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
+                if (gossipMessages !== undefined) {
                     for (const message of gossipMessages) {
-                        this.metrics?.reprocessGossipAttestations.reject.inc({ reason: ReprocessRejectReason.expired });
-                        this.metrics?.reprocessGossipAttestations.waitSecBeforeReject.set({ reason: ReprocessRejectReason.expired }, nowSec - message.seenTimestampSec);
-                        // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
+                        const topicType = message.topic.type;
+                        this.metrics?.awaitingBlockGossipMessages.reject.inc({
+                            topic: topicType,
+                            reason: ReprocessRejectReason.expired,
+                        });
+                        this.metrics?.awaitingBlockGossipMessages.waitSecBeforeReject.set({ topic: topicType, reason: ReprocessRejectReason.expired }, nowSec - message.seenTimestampSec);
+                        // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
                     }
+                    this.awaitingMessagesByBlockRoot.delete(rootHex);
                 }
-                this.awaitingGossipsubMessagesByRootBySlot.delete(slot);
             }
+            this.unknownBlocksBySlot.delete(slot);
         }
-        pruneSetToMax(this.unknownRootsBySlot, MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE);
-        this.unknownBlockGossipsubMessagesCount = 0;
-    }
+        for (const [slot, roots] of this.unknownEnvelopesBySlot) {
+            if (slot > minSlot)
+                continue;
+            for (const rootHex of roots) {
+                const gossipMessages = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
+                if (gossipMessages !== undefined) {
+                    for (const message of gossipMessages) {
+                        const topicType = message.topic.type;
+                        this.metrics?.awaitingPayloadGossipMessages.reject.inc({
+                            topic: topicType,
+                            reason: ReprocessRejectReason.expired,
+                        });
+                        this.metrics?.awaitingPayloadGossipMessages.waitSecBeforeReject.set({ topic: topicType, reason: ReprocessRejectReason.expired }, nowSec - message.seenTimestampSec);
+                        // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+                    }
+                    this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
+                }
+            }
+            this.unknownEnvelopesBySlot.delete(slot);
+        }
+    };
     executeWork() {
         // TODO: Maybe de-bounce by timing the last time executeWork was run
         this.metrics?.networkProcessor.executeWorkCalls.inc();
@@ -397,5 +616,19 @@ export class NetworkProcessor {
         }
         return null;
     }
+    get unknownBlockGossipsubMessagesCount() {
+        let count = 0;
+        for (const messages of this.awaitingMessagesByBlockRoot.values()) {
+            count += messages.size;
+        }
+        return count;
+    }
+    get unknownPayloadGossipsubMessagesCount() {
+        let count = 0;
+        for (const messages of this.awaitingMessagesByPayloadBlockRoot.values()) {
+            count += messages.size;
+        }
+        return count;
+    }
 }
 //# sourceMappingURL=index.js.map