@lodestar/beacon-node 1.43.0-dev.ca1fc40294 → 1.43.0-dev.dfb984e779

package/lib/sync/unknownBlock.js

@@ -1,19 +1,24 @@
+import { routes } from "@lodestar/api";
 import { ForkSeq } from "@lodestar/params";
 import { RequestError, RequestErrorCode } from "@lodestar/reqresp";
 import { computeTimeAtSlot } from "@lodestar/state-transition";
-import { prettyPrintIndices, pruneSetToMax, sleep } from "@lodestar/utils";
+import { fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex } from "@lodestar/utils";
 import { isBlockInputBlobs, isBlockInputColumns } from "../chain/blocks/blockInput/blockInput.js";
 import { BlockInputSource } from "../chain/blocks/blockInput/types.js";
+import { PayloadError, PayloadErrorCode } from "../chain/blocks/importExecutionPayload.js";
+import { PayloadEnvelopeInputSource } from "../chain/blocks/payloadEnvelopeInput/index.js";
 import { BlockError, BlockErrorCode } from "../chain/errors/index.js";
 import { ChainEvent } from "../chain/index.js";
+import { validateGloasBlockDataColumnSidecars } from "../chain/validation/dataColumnSidecar.js";
+import { validateGossipExecutionPayloadEnvelope } from "../chain/validation/executionPayloadEnvelope.js";
 import { NetworkEvent, prettyPrintPeerIdStr } from "../network/index.js";
 import { shuffle } from "../util/shuffle.js";
 import { sortBy } from "../util/sortBy.js";
 import { wrapError } from "../util/wrapError.js";
 import { MAX_CONCURRENT_REQUESTS } from "./constants.js";
-import { PendingBlockInputStatus, PendingBlockType, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, isPendingBlockInput, } from "./types.js";
+import { PendingBlockInputStatus, PendingBlockType, PendingPayloadInputStatus, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, getPayloadSyncCacheItemRootHex, getPayloadSyncCacheItemSlot, isPendingBlockInput, isPendingPayloadEnvelope, isPendingPayloadInput, } from "./types.js";
 import { DownloadByRootError, downloadByRoot } from "./utils/downloadByRoot.js";
-import { getAllDescendantBlocks, getDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
+import { getAllDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
 const MAX_ATTEMPTS_PER_BLOCK = 5;
 const MAX_KNOWN_BAD_BLOCKS = 500;
 const MAX_PENDING_BLOCKS = 100;
@@ -68,6 +73,8 @@ export class BlockInputSync {
      * block RootHex -> PendingBlock. To avoid finding same root at the same time
      */
     pendingBlocks = new Map();
+    // Payload sync is keyed by beacon block root as well, so block and payload queues can unblock each other.
+    pendingPayloads = new Map();
     knownBadBlocks = new Set();
     maxPendingBlocks;
     subscribedToNetworkEvents = false;
@@ -83,6 +90,7 @@ export class BlockInputSync {
         this.peerBalancer = new UnknownBlockPeerBalancer();
         if (metrics) {
             metrics.blockInputSync.pendingBlocks.addCollect(() => metrics.blockInputSync.pendingBlocks.set(this.pendingBlocks.size));
+            metrics.blockInputSync.pendingPayloads.addCollect(() => metrics.blockInputSync.pendingPayloads.set(this.pendingPayloads.size));
             metrics.blockInputSync.knownBadBlocks.addCollect(() => metrics.blockInputSync.knownBadBlocks.set(this.knownBadBlocks.size));
         }
     }
@@ -95,8 +103,13 @@ export class BlockInputSync {
         if (!this.subscribedToNetworkEvents) {
             this.logger.verbose("BlockInputSync enabled.");
             this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
+            this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
             this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
+            this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
             this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
+            this.chain.emitter.on(ChainEvent.envelopeUnknownBlock, this.onEnvelopeUnknownBlock);
+            this.chain.emitter.on(routes.events.EventType.block, this.onBlockImported);
+            this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
             this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
             this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
             this.subscribedToNetworkEvents = true;
@@ -105,8 +118,13 @@ export class BlockInputSync {
     unsubscribeFromNetwork() {
         this.logger.verbose("BlockInputSync disabled.");
         this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
+        this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
         this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
+        this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
         this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
+        this.chain.emitter.off(ChainEvent.envelopeUnknownBlock, this.onEnvelopeUnknownBlock);
+        this.chain.emitter.off(routes.events.EventType.block, this.onBlockImported);
+        this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadImported);
         this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
         this.network.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
         this.subscribedToNetworkEvents = false;
@@ -145,12 +163,54 @@ export class BlockInputSync {
             this.logger.debug("Error handling incompleteBlockInput event", {}, e);
         }
     };
+    onUnknownEnvelopeBlockRoot = (data) => {
+        try {
+            this.addByPayloadRootHex(data.rootHex, data.peer);
+            this.triggerUnknownBlockSearch();
+            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_DATA });
+            this.metrics?.blockInputSync.source.inc({ source: data.source });
+        }
+        catch (e) {
+            this.logger.debug("Error handling unknownEnvelopeBlockRoot event", {}, e);
+        }
+    };
+    onIncompletePayloadEnvelope = (data) => {
+        try {
+            this.addByPayloadInput(data.payloadInput, data.peer);
+            this.triggerUnknownBlockSearch();
+            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_DATA });
+            this.metrics?.blockInputSync.source.inc({ source: data.source });
+        }
+        catch (e) {
+            this.logger.debug("Error handling incompletePayloadEnvelope event", {}, e);
+        }
+    };
     /**
      * Process an unknownBlockParent event and register the block in `pendingBlocks` Map.
      */
     onUnknownParent = (data) => {
         try {
-            this.addByRootHex(data.blockInput.parentRootHex, data.peer);
+            const missingDependency = this.getMissingBlockDependency(data.blockInput);
+            if (missingDependency.kind === "invalidParentPayload") {
+                this.addByBlockInput(data.blockInput, data.peer);
+                const pendingBlock = this.pendingBlocks.get(data.blockInput.blockRootHex);
+                if (pendingBlock && isPendingBlockInput(pendingBlock)) {
+                    this.logger.debug("Ignoring block with conflicting parent payload hash", {
+                        slot: pendingBlock.blockInput.slot,
+                        root: pendingBlock.blockInput.blockRootHex,
+                        parentRoot: missingDependency.parentRootHex,
+                        parentBlockHash: missingDependency.parentBlockHashHex,
+                    });
+                    this.removeAndDownScoreAllDescendants(pendingBlock);
+                }
+                return;
+            }
+            if (missingDependency.kind === "parentPayload") {
+                this.addByPayloadRootHex(missingDependency.rootHex, data.peer);
+            }
+            else if (missingDependency.kind === "parentBlock") {
+                this.addByRootHex(missingDependency.rootHex, data.peer);
+            }
             this.addByBlockInput(data.blockInput, data.peer);
             this.triggerUnknownBlockSearch();
             this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_PARENT });
@@ -160,8 +220,31 @@ export class BlockInputSync {
             this.logger.debug("Error handling unknownParent event", {}, e);
         }
     };
+    onEnvelopeUnknownBlock = (data) => {
+        try {
+            const blockRootHex = toRootHex(data.envelope.message.beaconBlockRoot);
+            this.addByRootHex(blockRootHex, data.peer);
+            this.addByPayloadEnvelope(data.envelope, data.peer);
+            this.triggerUnknownBlockSearch();
+            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_DATA });
+            this.metrics?.blockInputSync.source.inc({ source: data.source });
+        }
+        catch (e) {
+            this.logger.debug("Error handling envelopeUnknownBlock event", {}, e);
+        }
+    };
+    onBlockImported = () => {
+        if (this.pendingPayloads.size > 0) {
+            this.triggerUnknownBlockSearch();
+        }
+    };
+    onPayloadImported = ({ blockRoot, }) => {
+        this.pendingPayloads.delete(blockRoot);
+        this.triggerUnknownBlockSearch();
+    };
     addByRootHex = (rootHex, peerIdStr) => {
         let pendingBlock = this.pendingBlocks.get(rootHex);
+        let added = false;
         if (!pendingBlock) {
             pendingBlock = {
                 status: PendingBlockInputStatus.pending,
@@ -170,6 +253,7 @@ export class BlockInputSync {
                 timeAddedSec: Date.now() / 1000,
             };
             this.pendingBlocks.set(rootHex, pendingBlock);
+            added = true;
             this.logger.verbose("Added new rootHex to BlockInputSync.pendingBlocks", {
                 root: pendingBlock.rootHex,
                 peerIdStr: peerIdStr ?? "unknown peer",
@@ -184,6 +268,7 @@ export class BlockInputSync {
         if (prunedItemCount > 0) {
             this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
         }
+        return added;
     };
     addByBlockInput = (blockInput, peerIdStr) => {
         let pendingBlock = this.pendingBlocks.get(blockInput.blockRootHex);
@@ -211,6 +296,75 @@ export class BlockInputSync {
             this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
         }
     };
+    addByPayloadRootHex = (rootHex, peerIdStr) => {
+        let pendingPayload = this.pendingPayloads.get(rootHex);
+        let added = false;
+        if (!pendingPayload) {
+            pendingPayload = {
+                status: PendingPayloadInputStatus.pending,
+                rootHex,
+                peerIdStrings: new Set(),
+                timeAddedSec: Date.now() / 1000,
+            };
+            this.pendingPayloads.set(rootHex, pendingPayload);
+            added = true;
+            this.logger.verbose("Added new payload rootHex to BlockInputSync.pendingPayloads", {
+                root: rootHex,
+                peerIdStr: peerIdStr ?? "unknown peer",
+            });
+        }
+        if (peerIdStr) {
+            pendingPayload.peerIdStrings.add(peerIdStr);
+        }
+        const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
+        if (prunedItemCount > 0) {
+            this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
+        }
+        return added;
+    };
+    addByPayloadEnvelope = (envelope, peerIdStr) => {
+        const rootHex = toRootHex(envelope.message.beaconBlockRoot);
+        const existingPendingPayload = this.pendingPayloads.get(rootHex);
+        let pendingPayload = this.pendingPayloads.get(rootHex);
+        if (!pendingPayload || !isPendingPayloadEnvelope(pendingPayload)) {
+            pendingPayload = {
+                status: PendingPayloadInputStatus.waitingForBlock,
+                envelope,
+                peerIdStrings: new Set(existingPendingPayload?.peerIdStrings ?? []),
+                timeAddedSec: existingPendingPayload?.timeAddedSec ?? Date.now() / 1000,
+            };
+            this.pendingPayloads.set(rootHex, pendingPayload);
+            this.logger.verbose("Added payload envelope to BlockInputSync.pendingPayloads", {
+                slot: envelope.message.payload.slotNumber,
+                root: rootHex,
+            });
+        }
+        else {
+            this.logger.debug("Overwriting pending payload envelope for root already waiting for block", {
+                slot: envelope.message.payload.slotNumber,
+                root: rootHex,
+            });
+            pendingPayload.envelope = envelope;
+        }
+        if (peerIdStr) {
+            pendingPayload.peerIdStrings.add(peerIdStr);
+        }
+        const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
+        if (prunedItemCount > 0) {
+            this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
+        }
+    };
+    addByPayloadInput = (payloadInput, peerIdStr, envelope) => {
+        const pendingPayload = this.toPendingPayloadInput(payloadInput, this.pendingPayloads.get(payloadInput.blockRootHex), envelope);
+        if (peerIdStr) {
+            pendingPayload.peerIdStrings.add(peerIdStr);
+        }
+        this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
+        const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
+        if (prunedItemCount > 0) {
+            this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
+        }
+    };
     onPeerConnected = (data) => {
         try {
             const peerId = data.peer;
@@ -226,47 +380,172 @@ export class BlockInputSync {
         const peerId = data.peer;
         this.peerBalancer.onPeerDisconnected(peerId);
     };
+    /**
+     * Post-gloas, a locally complete block can still be blocked on its parent's execution payload lineage.
+     * Distinguish which dependency is missing so the scheduler can enqueue the right follow-up work.
+     */
+    getMissingBlockDependency(blockInput) {
+        const parentRootHex = blockInput.parentRootHex;
+        if (!this.chain.forkChoice.hasBlockHex(parentRootHex)) {
+            return { kind: "parentBlock", rootHex: parentRootHex };
+        }
+        if (!blockInput.hasBlock()) {
+            return { kind: "block", rootHex: blockInput.blockRootHex };
+        }
+        if (this.config.getForkSeq(blockInput.slot) < ForkSeq.gloas) {
+            return { kind: "ready" };
+        }
+        const block = blockInput.getBlock();
+        const parentBlockHashHex = toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash);
+        if (this.chain.forkChoice.getBlockHexAndBlockHash(parentRootHex, parentBlockHashHex) !== null) {
+            return { kind: "ready" };
+        }
+        if (this.chain.forkChoice.hasPayloadHexUnsafe(parentRootHex)) {
+            return { kind: "invalidParentPayload", parentRootHex, parentBlockHashHex };
+        }
+        const parentPayloadInput = this.chain.seenPayloadEnvelopeInputCache.get(parentRootHex);
+        if (parentPayloadInput) {
+            if (parentPayloadInput.getBlockHashHex() === parentBlockHashHex) {
+                return { kind: "parentPayload", rootHex: parentRootHex };
+            }
+            return { kind: "invalidParentPayload", parentRootHex, parentBlockHashHex };
+        }
+        return { kind: "parentPayload", rootHex: parentRootHex };
+    }
+    advancePendingBlock(pendingBlock) {
+        const missingDependency = this.getMissingBlockDependency(pendingBlock.blockInput);
+        switch (missingDependency.kind) {
+            case "ready":
+                return "ready";
+            case "block":
+                pendingBlock.status = PendingBlockInputStatus.pending;
+                return "queued_block";
+            case "parentBlock": {
+                let added = this.addByRootHex(missingDependency.rootHex);
+                for (const peerIdStr of pendingBlock.peerIdStrings) {
+                    added = this.addByRootHex(missingDependency.rootHex, peerIdStr) || added;
+                }
+                return added ? "queued_parent_block" : "blocked";
+            }
+            case "parentPayload": {
+                let added = this.addByPayloadRootHex(missingDependency.rootHex);
+                for (const peerIdStr of pendingBlock.peerIdStrings) {
+                    added = this.addByPayloadRootHex(missingDependency.rootHex, peerIdStr) || added;
+                }
+                return added ? "queued_parent_payload" : "blocked";
+            }
+            case "invalidParentPayload":
+                this.logger.debug("Removing block with conflicting parent payload hash", {
+                    slot: pendingBlock.blockInput.slot,
+                    root: pendingBlock.blockInput.blockRootHex,
+                    parentRoot: missingDependency.parentRootHex,
+                    parentBlockHash: missingDependency.parentBlockHashHex,
+                });
+                this.removeAndDownScoreAllDescendants(pendingBlock);
+                return "removed";
+        }
+    }
+    toPendingPayloadInput(payloadInput, previous, envelope) {
+        // Normalize every payload queueing path into the same cache shape while preserving first-seen
+        // timing and peer provenance from any earlier by-root or envelope-only entry.
+        const queuedEnvelope = envelope ?? (previous && isPendingPayloadEnvelope(previous) ? previous.envelope : undefined);
+        if (queuedEnvelope && !payloadInput.hasPayloadEnvelope()) {
+            payloadInput.addPayloadEnvelope({
+                envelope: queuedEnvelope,
+                source: PayloadEnvelopeInputSource.byRoot,
+                seenTimestampSec: Date.now() / 1000,
+            });
+        }
+        return {
+            status: payloadInput.isComplete() ? PendingPayloadInputStatus.downloaded : PendingPayloadInputStatus.pending,
+            payloadInput,
+            timeAddedSec: previous?.timeAddedSec ?? Date.now() / 1000,
+            timeSyncedSec: payloadInput.isComplete() ? Date.now() / 1000 : undefined,
+            peerIdStrings: new Set(previous?.peerIdStrings ?? []),
+        };
+    }
     /**
      * Gather tip parent blocks with unknown parent and do a search for all of them
      */
     triggerUnknownBlockSearch = () => {
         // Cheap early stop to prevent calling the network.getConnectedPeers()
-        if (this.pendingBlocks.size === 0) {
+        if (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0) {
             return;
         }
-        // If the node loses all peers with pending unknown blocks, the sync will stall
+        // If the node loses all peers with pending unknown blocks or payloads, the sync will stall
         const connectedPeers = this.network.getConnectedPeers();
-        if (connectedPeers.length === 0) {
-            this.logger.debug("No connected peers, skipping unknown block search.");
-            return;
-        }
+        const hasConnectedPeers = connectedPeers.length > 0;
         const { unknowns, ancestors } = getUnknownAndAncestorBlocks(this.pendingBlocks);
-        // it's rare when there is no unknown block
-        // see https://github.com/ChainSafe/lodestar/issues/5649#issuecomment-1594213550
-        if (unknowns.length === 0) {
-            let processedBlocks = 0;
-            for (const block of ancestors) {
-                // when this happens, it's likely the block and parent block are processed by head sync
-                if (this.chain.forkChoice.hasBlockHex(block.blockInput.parentRootHex)) {
+        let processedBlocks = 0;
+        let shouldRerunBlockSearch = false;
+        for (const block of ancestors) {
+            const advanceResult = this.advancePendingBlock(block);
+            switch (advanceResult) {
+                case "ready":
                     processedBlocks++;
-                    this.processBlock(block).catch((e) => {
+                    this.processReadyBlock(block).catch((e) => {
                         this.logger.debug("Unexpected error - process old downloaded block", {}, e);
                     });
+                    break;
+                case "queued_block":
+                case "queued_parent_block":
+                    shouldRerunBlockSearch = true;
+                    break;
+                case "queued_parent_payload":
+                case "blocked":
+                case "removed":
+                    break;
+            }
+        }
+        if (unknowns.length > 0) {
+            if (!hasConnectedPeers) {
+                this.logger.debug("No connected peers, skipping unknown block download.");
+            }
+            else {
+                // Most of the time there is exactly 1 unknown block
+                for (const block of unknowns) {
+                    this.downloadBlock(block).catch((e) => {
+                        this.logger.debug("Unexpected error - downloadBlock", { root: getBlockInputSyncCacheItemRootHex(block) }, e);
+                    });
                 }
             }
+        }
+        else if (ancestors.length > 0) {
+            // It's rare when there is no unknown block
+            // see https://github.com/ChainSafe/lodestar/issues/5649#issuecomment-1594213550
             this.logger.verbose("No unknown block, process ancestor downloaded blocks", {
                 pendingBlocks: this.pendingBlocks.size,
                 ancestorBlocks: ancestors.length,
                 processedBlocks,
             });
-            return;
         }
-        // most of the time there is exactly 1 unknown block
-        for (const block of unknowns) {
-            this.downloadBlock(block).catch((e) => {
-                this.logger.debug("Unexpected error - downloadBlock", { root: getBlockInputSyncCacheItemRootHex(block) }, e);
+        // Blocks can unblock payloads and payloads can unblock blocks, so every scheduler pass services both queues.
+        for (const payload of Array.from(this.pendingPayloads.values())) {
+            if (isPendingPayloadInput(payload) && payload.status === PendingPayloadInputStatus.downloaded) {
+                this.processPayload(payload).catch((e) => {
+                    this.logger.debug("Unexpected error - process downloaded payload", {}, e);
+                });
+                continue;
+            }
+            if (isPendingPayloadEnvelope(payload)) {
+                this.reconcilePayloadEnvelope(payload).catch((e) => {
+                    this.logger.debug("Unexpected error - reconcile pending payload envelope", {}, e);
+                });
+                continue;
+            }
+            if (!hasConnectedPeers) {
+                this.logger.debug("No connected peers, skipping unknown payload download.", {
+                    root: getPayloadSyncCacheItemRootHex(payload),
+                });
+                continue;
+            }
+            this.downloadPayload(payload).catch((e) => {
+                this.logger.debug("Unexpected error - downloadPayload", { root: getPayloadSyncCacheItemRootHex(payload) }, e);
             });
         }
+        if (shouldRerunBlockSearch) {
+            this.triggerUnknownBlockSearch();
+        }
     };
     async downloadBlock(block) {
         if (block.status !== PendingBlockInputStatus.pending) {
@@ -297,10 +576,24 @@ export class BlockInputSync {
             };
             this.logger.verbose("Downloaded unknown block", logCtx2);
             if (parentInForkChoice) {
-                // Bingo! Process block. Add to pending blocks anyway for recycle the cache that prevents duplicate processing
-                this.processBlock(pending).catch((e) => {
-                    this.logger.debug("Unexpected error - process newly downloaded block", logCtx2, e);
-                });
+                // If the direct parent is already in fork choice, let the block state machine decide if
+                // the next step is block import, parent payload download, or branch removal.
+                const advanceResult = this.advancePendingBlock(pending);
+                switch (advanceResult) {
+                    case "ready":
+                        this.processReadyBlock(pending).catch((e) => {
+                            this.logger.debug("Unexpected error - process newly downloaded block", logCtx2, e);
+                        });
+                        break;
+                    case "queued_block":
+                    case "queued_parent_block":
+                    case "queued_parent_payload":
+                        this.triggerUnknownBlockSearch();
+                        break;
+                    case "blocked":
+                    case "removed":
+                        break;
+                }
             }
             else if (blockSlot <= finalizedSlot) {
                 // the common ancestor of the downloading chain and canonical chain should be at least the finalized slot and
@@ -325,26 +618,11 @@ export class BlockInputSync {
         }
     }
     /**
-     * Send block to the processor awaiting completition. If processed successfully, send all children to the processor.
-     * On error, remove and downscore all descendants.
-     * This function could run recursively for all descendant blocks
+     * Import a block that has already passed the local dependency checks in BlockInputSync.
+     * On error, remove and downscore descendants as appropriate for the failure type.
      */
-    async processBlock(pendingBlock) {
-        // pending block status is `downloaded` right after `downloadBlock`
-        // but could be `pending` if added by `onUnknownBlockParent` event and this function is called recursively
+    async processReadyBlock(pendingBlock) {
         if (pendingBlock.status !== PendingBlockInputStatus.downloaded) {
-            if (pendingBlock.status === PendingBlockInputStatus.pending) {
-                const connectedPeers = this.network.getConnectedPeers();
-                if (connectedPeers.length === 0) {
-                    this.logger.debug("No connected peers, skipping download block", {
-                        slot: pendingBlock.blockInput.slot,
-                        blockRoot: pendingBlock.blockInput.blockRootHex,
-                    });
-                    return;
-                }
-                // if the download is a success we'll call `processBlock()` for this block
-                await this.downloadBlock(pendingBlock);
-            }
             return;
         }
         pendingBlock.status = PendingBlockInputStatus.processing;
@@ -384,14 +662,9 @@ export class BlockInputSync {
         if (!res.err) {
             // no need to update status to "processed", delete anyway
             this.pendingBlocks.delete(pendingBlock.blockInput.blockRootHex);
-            // Send child blocks to the processor
-            for (const descendantBlock of getDescendantBlocks(pendingBlock.blockInput.blockRootHex, this.pendingBlocks)) {
-                if (isPendingBlockInput(descendantBlock)) {
-                    this.processBlock(descendantBlock).catch((e) => {
-                        this.logger.debug("Unexpected error - process descendant block", {}, e);
-                    });
-                }
-            }
+            // Re-enter the scheduler so descendants blocked on either parent blocks or parent payloads
+            // are advanced through the same dependency checks as every other pending item.
+            this.triggerUnknownBlockSearch();
         }
         else {
             const errorData = { slot: pendingBlock.blockInput.slot, root: pendingBlock.blockInput.blockRootHex };
@@ -406,6 +679,14 @@ export class BlockInputSync {
                         this.logger.debug("Attempted to process block but its parent was still unknown", errorData, res.err);
                         pendingBlock.status = PendingBlockInputStatus.downloaded;
                         break;
+                    case BlockErrorCode.PARENT_PAYLOAD_UNKNOWN:
+                        this.logger.error("processReadyBlock() hit unexpected parent payload dependency after readiness checks", {
+                            ...errorData,
+                            parentRoot: pendingBlock.blockInput.parentRootHex,
+                            parentBlockHash: res.err.type.parentBlockHash,
+                        }, res.err);
+                        pendingBlock.status = PendingBlockInputStatus.downloaded;
+                        break;
                     case BlockErrorCode.EXECUTION_ENGINE_ERROR:
                         // Removing the block(s) without penalizing the peers, hoping for EL to
                         // recover on a latter download + verify attempt
@@ -424,6 +705,299 @@ export class BlockInputSync {
             }
         }
     }
+    /**
+     * Reconcile an envelope-first payload entry once the block import path has seeded its
+     * PayloadEnvelopeInput. This may queue block download, validate the speculative envelope, or
+     * downgrade back to by-root fetching when the cached envelope does not match the imported block.
+     */
+    async reconcilePayloadEnvelope(pendingPayload) {
+        const rootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
+        if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+            this.pendingPayloads.delete(rootHex);
+            return;
+        }
+        const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+        if (!payloadInput) {
+            if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
+                // Column commitments live on the block body, so an envelope-only entry has to pull the block first.
+                if (!this.pendingBlocks.has(rootHex)) {
+                    this.addByRootHex(rootHex);
+                }
+                const pendingBlock = this.pendingBlocks.get(rootHex);
+                if (pendingBlock && this.network.getConnectedPeers().length > 0) {
+                    await this.downloadBlock(pendingBlock);
+                }
+            }
+            else {
+                this.logger.debug("Missing PayloadEnvelopeInput for known block while reconciling payload envelope", {
+                    root: rootHex,
+                });
+            }
+            return;
+        }
+        if (!payloadInput.hasPayloadEnvelope()) {
+            const validationResult = await wrapError(validateGossipExecutionPayloadEnvelope(this.chain, pendingPayload.envelope));
+            if (validationResult.err) {
+                this.logger.debug("Pending payload envelope failed validation after block import, refetching by root", { slot: pendingPayload.envelope.message.payload.slotNumber, root: rootHex }, validationResult.err);
+                const pendingPayloadByRoot = {
+                    status: PendingPayloadInputStatus.pending,
+                    rootHex,
+                    timeAddedSec: pendingPayload.timeAddedSec,
+                    peerIdStrings: new Set(pendingPayload.peerIdStrings),
+                };
+                this.pendingPayloads.set(rootHex, pendingPayloadByRoot);
+                if (this.network.getConnectedPeers().length > 0) {
+                    await this.downloadPayload(pendingPayloadByRoot);
+                }
+                return;
+            }
+        }
+        const upgradedPayload = this.toPendingPayloadInput(payloadInput, pendingPayload, pendingPayload.envelope);
+        this.pendingPayloads.set(rootHex, upgradedPayload);
+        if (upgradedPayload.status === PendingPayloadInputStatus.downloaded) {
+            await this.processPayload(upgradedPayload);
+            return;
+        }
+        await this.downloadPayload(upgradedPayload);
+    }
+    async downloadPayload(payload) {
+        if (isPendingPayloadEnvelope(payload)) {
+            await this.reconcilePayloadEnvelope(payload);
+            return;
+        }
+        const rootHex = getPayloadSyncCacheItemRootHex(payload);
+        if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+            this.pendingPayloads.delete(rootHex);
+            return;
+        }
+        if (payload.status !== PendingPayloadInputStatus.pending) {
+            return;
+        }
+        const logCtx = {
+            slot: getPayloadSyncCacheItemSlot(payload),
+            root: rootHex,
+            pendingPayloads: this.pendingPayloads.size,
+        };
+        this.logger.verbose("BlockInputSync.downloadPayload()", logCtx);
+        payload.status = PendingPayloadInputStatus.fetching;
+        const res = await wrapError(this.fetchPayloadInput(payload));
+        if (!res.err) {
+            const pendingPayload = res.result;
+            this.pendingPayloads.set(getPayloadSyncCacheItemRootHex(pendingPayload), pendingPayload);
+            if (isPendingPayloadEnvelope(pendingPayload)) {
+                await this.reconcilePayloadEnvelope(pendingPayload);
+            }
+            else if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+                await this.processPayload(pendingPayload);
+            }
+            return;
+        }
+        this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
+        if (!isPendingPayloadEnvelope(payload)) {
+            payload.status = PendingPayloadInputStatus.pending;
+        }
+    }
+    async processPayload(pendingPayload) {
+        const rootHex = pendingPayload.payloadInput.blockRootHex;
+        const logCtx = { slot: pendingPayload.payloadInput.slot, root: rootHex };
+        if (pendingPayload.status !== PendingPayloadInputStatus.downloaded) {
+            this.logger.debug("Skipping payload processing before payload input is downloaded", {
+                ...logCtx,
+                status: pendingPayload.status,
+            });
+            return;
+        }
+        if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+            this.logger.debug("Payload already imported while processing unknown payload", logCtx);
+            this.pendingPayloads.delete(rootHex);
+            return;
+        }
+        if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
+            this.logger.debug("Payload input is ready before its block is in fork choice", logCtx);
+            const added = this.addByRootHex(rootHex);
+            pendingPayload.status = PendingPayloadInputStatus.downloaded;
+            if (added) {
+                this.triggerUnknownBlockSearch();
+            }
+            return;
+        }
+        pendingPayload.status = PendingPayloadInputStatus.processing;
+        const res = await wrapError(this.chain.processExecutionPayload(pendingPayload.payloadInput));
+        if (!res.err) {
+            this.logger.debug("Processed payload from unknown sync", logCtx);
+            this.pendingPayloads.delete(rootHex);
+            this.triggerUnknownBlockSearch();
+            return;
+        }
+        if (res.err instanceof PayloadError) {
+            switch (res.err.type.code) {
+                case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:
+                    // Payload sync discovered the block dependency before the block queue did. Re-enqueue the
+                    // block and keep the payload ready so the scheduler can retry once the block reaches fork choice.
+                    if (this.addByRootHex(rootHex)) {
+                        this.triggerUnknownBlockSearch();
+                    }
+                    // Keep the payload out of any synchronous requeue pass; a later scheduler pass will retry it.
+                    pendingPayload.status = PendingPayloadInputStatus.downloaded;
+                    break;
+                case PayloadErrorCode.EXECUTION_ENGINE_ERROR:
+                    this.logger.debug("Execution engine error while processing payload from unknown sync", logCtx, res.err);
+                    pendingPayload.status = PendingPayloadInputStatus.downloaded;
+                    break;
+                case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
+                case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
+                case PayloadErrorCode.INVALID_SIGNATURE:
+                    // TODO GLOAS: Decide how invalid payload inputs should eventually leave memory without
+                    // reintroducing envelope replacement / recreation flows.
+                    this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.removePendingPayloadAndDescendants(rootHex);
+                    break;
+                default:
+                    this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.pendingPayloads.delete(rootHex);
+            }
+            return;
+        }
+        this.logger.debug("Unknown error processing payload from unknown sync", logCtx, res.err);
+        pendingPayload.status = PendingPayloadInputStatus.downloaded;
+    }
+    /**
+     * Download payload material keyed by beacon block root. Unlike block download, payload sync may
+     * already have a locally cached envelope or partial columns, so each attempt starts from local state
+     * and only asks peers for the remaining pieces.
+     */
+    async fetchPayloadInput(cacheItem) {
+        const rootHex = getPayloadSyncCacheItemRootHex(cacheItem);
+        const blockRoot = fromHex(rootHex);
+        const excludedPeers = new Set();
+        let slot = getPayloadSyncCacheItemSlot(cacheItem);
+        let payloadInput = isPendingPayloadInput(cacheItem)
+            ? cacheItem.payloadInput
+            : this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+        let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
+        let i = 0;
+        while (i++ < this.getMaxDownloadAttempts()) {
+            const pendingColumns = payloadInput?.hasAllData()
+                ? new Set()
+                : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
+            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
+            if (peerMeta === null) {
+                throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`);
+            }
+            const { peerId, client: peerClient } = peerMeta;
+            cacheItem.peerIdStrings.add(peerId);
+            try {
+                if (!envelope) {
+                    envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
+                    slot = envelope.message.payload.slotNumber;
+                }
+                payloadInput ??= this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+                if (!payloadInput) {
+                    if (this.chain.forkChoice.hasBlockHex(rootHex)) {
+                        throw new Error(`Missing PayloadEnvelopeInput for known block ${rootHex}`);
+                    }
+                    // Keep the validated envelope around, but wait for the block body before turning it into a full payload input.
+                    return {
+                        status: PendingPayloadInputStatus.waitingForBlock,
+                        envelope,
+                        timeAddedSec: cacheItem.timeAddedSec,
+                        peerIdStrings: cacheItem.peerIdStrings,
+                    };
+                }
+                if (!payloadInput.hasPayloadEnvelope()) {
+                    await validateGossipExecutionPayloadEnvelope(this.chain, envelope);
+                }
+                let pendingPayload = this.toPendingPayloadInput(payloadInput, cacheItem, envelope);
+                if (!pendingPayload.payloadInput.hasAllData()) {
+                    const missing = pendingPayload.payloadInput.getMissingSampledColumnMeta().missing;
+                    if (missing.length > 0) {
+                        const columnSidecars = await this.fetchPayloadColumns(peerMeta, pendingPayload.payloadInput, missing);
+                        const seenTimestampSec = Date.now() / 1000;
+                        for (const columnSidecar of columnSidecars) {
+                            if (pendingPayload.payloadInput.hasColumn(columnSidecar.index)) {
+                                continue;
+                            }
+                            pendingPayload.payloadInput.addColumn({
+                                columnSidecar,
+                                source: PayloadEnvelopeInputSource.byRoot,
+                                seenTimestampSec,
+                                peerIdStr: peerId,
+                            });
+                        }
+                        pendingPayload = this.toPendingPayloadInput(pendingPayload.payloadInput, pendingPayload);
+                    }
+                }
+                this.logger.verbose("BlockInputSync.fetchPayloadInput: successful download", {
+                    slot,
+                    rootHex,
+                    peerId,
+                    peerClient,
+                    hasPayload: pendingPayload.payloadInput.hasPayloadEnvelope(),
+                    hasAllData: pendingPayload.payloadInput.hasAllData(),
+                });
+                if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+                    return pendingPayload;
+                }
+                cacheItem = pendingPayload;
+                payloadInput = pendingPayload.payloadInput;
+            }
+            catch (e) {
+                this.logger.debug("Error downloading payload in BlockInputSync.fetchPayloadInput", { slot, rootHex, attempt: i, peer: peerId, peerClient }, e);
+                if (e instanceof RequestError) {
+                    switch (e.type.code) {
+                        case RequestErrorCode.REQUEST_RATE_LIMITED:
+                        case RequestErrorCode.REQUEST_TIMEOUT:
+                            break;
+                        default:
+                            excludedPeers.add(peerId);
+                            break;
+                    }
+                }
+                else {
+                    excludedPeers.add(peerId);
+                }
+            }
+            finally {
+                this.peerBalancer.onRequestCompleted(peerId);
+            }
+        }
+        throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
+    }
+    async fetchExecutionPayloadEnvelope(peerIdStr, blockRoot, rootHex) {
+        const response = await this.network.sendExecutionPayloadEnvelopesByRoot(peerIdStr, [blockRoot]);
+        const envelope = response.at(0);
+        if (!envelope) {
+            throw new Error(`Missing execution payload envelope for root=${rootHex}`);
+        }
+        const receivedRootHex = toRootHex(envelope.message.beaconBlockRoot);
+        if (receivedRootHex !== rootHex) {
+            throw new Error(`Execution payload envelope root mismatch requested=${rootHex} received=${receivedRootHex}`);
+        }
+        return envelope;
+    }
+    async fetchPayloadColumns(peerMeta, payloadInput, missing) {
+        const { peerId: peerIdStr } = peerMeta;
+        const peerColumns = new Set(peerMeta.custodyColumns ?? []);
+        const requestedColumns = missing.filter((columnIndex) => peerColumns.has(columnIndex));
+        if (requestedColumns.length === 0) {
+            return [];
+        }
+        const columnSidecars = (await this.network.sendDataColumnSidecarsByRoot(peerIdStr, [
+            { blockRoot: fromHex(payloadInput.blockRootHex), columns: requestedColumns },
+        ]));
+        if (columnSidecars.length === 0) {
+            throw new Error(`No data column sidecars returned for payload root=${payloadInput.blockRootHex}`);
+        }
+        const requestedColumnsSet = new Set(requestedColumns);
+        const extraColumns = columnSidecars.filter((columnSidecar) => !requestedColumnsSet.has(columnSidecar.index));
+        if (extraColumns.length > 0) {
+            throw new Error(`Received unexpected payload data columns indices=${prettyPrintIndices(extraColumns.map((column) => column.index))}`);
+        }
+        // PayloadEnvelopeInput already carries the block slot, root, and commitments, so reuse the
+        // block-based Gloas validator rather than maintaining a second payload-specific variant.
+        await validateGloasBlockDataColumnSidecars(payloadInput.slot, fromHex(payloadInput.blockRootHex), payloadInput.getBlobKzgCommitments(), columnSidecars, this.chain.metrics?.peerDas);
+        return columnSidecars;
+    }
     /**
      * From a set of shuffled peers:
      *   - fetch the block
@@ -582,6 +1156,25 @@ export class BlockInputSync {
         // Prune knownBadBlocks
         pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
     }
+    // Once a parent payload is invalid, every descendant waiting on that payload lineage becomes unrecoverable too.
+    removePendingPayloadAndDescendants(rootHex) {
+        // Keep PayloadEnvelopeInput resident in the seen cache. importBlock() owns that object and
+        // later validation/finalization logic decides when it can leave memory.
+        this.pendingPayloads.delete(rootHex);
+        const badPendingBlocks = getAllDescendantBlocks(rootHex, this.pendingBlocks);
+        this.metrics?.blockInputSync.removedBlocks.inc(badPendingBlocks.length);
+        for (const block of badPendingBlocks) {
+            const descendantRootHex = getBlockInputSyncCacheItemRootHex(block);
+            this.pendingBlocks.delete(descendantRootHex);
+            this.pendingPayloads.delete(descendantRootHex);
+            this.chain.seenBlockInputCache.prune(descendantRootHex);
+            this.logger.debug("Removing pending descendant after invalid parent payload", {
+                slot: getBlockInputSyncCacheItemSlot(block),
+                blockRoot: descendantRootHex,
+                parentPayloadRoot: rootHex,
+            });
+        }
+    }
     removeAllDescendants(block) {
         const rootHex = getBlockInputSyncCacheItemRootHex(block);
         const slot = getBlockInputSyncCacheItemSlot(block);
@@ -591,7 +1184,10 @@ export class BlockInputSync {
         for (const block of badPendingBlocks) {
             const rootHex = getBlockInputSyncCacheItemRootHex(block);
             this.pendingBlocks.delete(rootHex);
+            this.pendingPayloads.delete(rootHex);
             this.chain.seenBlockInputCache.prune(rootHex);
+            // Keep PayloadEnvelopeInput resident in the seen cache for consistency with the
+            // importBlock()-owned lifecycle.
             this.logger.debug("Removing bad/unknown/incomplete BlockInputSyncCacheItem", {
                 slot,
                 blockRoot: rootHex,