@lodestar/beacon-node 1.35.0-dev.549f58dd39 → 1.35.0-dev.56313c7299

package/lib/sync/unknownBlock.js

@@ -1,76 +1,21 @@
-import { ForkSeq } from "@lodestar/params";
-import { RequestError, RequestErrorCode } from "@lodestar/reqresp";
-import { prettyPrintIndices, pruneSetToMax, sleep } from "@lodestar/utils";
-import { isBlockInputBlobs, isBlockInputColumns } from "../chain/blocks/blockInput/blockInput.js";
-import { BlockInputSource } from "../chain/blocks/blockInput/types.js";
+import { ForkName, ForkSeq, INTERVALS_PER_SLOT } from "@lodestar/params";
+import { fromHex, pruneSetToMax, toRootHex } from "@lodestar/utils";
+import { sleep } from "@lodestar/utils";
+import { BlockInputType } from "../chain/blocks/types.js";
 import { BlockError, BlockErrorCode } from "../chain/errors/index.js";
-import { ChainEvent } from "../chain/index.js";
-import { NetworkEvent, prettyPrintPeerIdStr } from "../network/index.js";
+import { NetworkEvent } from "../network/index.js";
+import { beaconBlocksMaybeBlobsByRoot, unavailableBeaconBlobsByRoot, } from "../network/reqresp/beaconBlocksMaybeBlobsByRoot.js";
+import { byteArrayEquals } from "../util/bytes.js";
 import { shuffle } from "../util/shuffle.js";
 import { sortBy } from "../util/sortBy.js";
 import { wrapError } from "../util/wrapError.js";
 import { MAX_CONCURRENT_REQUESTS } from "./constants.js";
-import { PendingBlockInputStatus, PendingBlockType, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, isPendingBlockInput, } from "./types.js";
-import { DownloadByRootError, downloadByRoot } from "./utils/downloadByRoot.js";
+import { PendingBlockStatus, PendingBlockType } from "./interface.js";
 import { getAllDescendantBlocks, getDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
 const MAX_ATTEMPTS_PER_BLOCK = 5;
 const MAX_KNOWN_BAD_BLOCKS = 500;
 const MAX_PENDING_BLOCKS = 100;
-var FetchResult;
-(function (FetchResult) {
-    FetchResult["SuccessResolved"] = "success_resolved";
-    FetchResult["SuccessMissingParent"] = "success_missing_parent";
-    FetchResult["SuccessLate"] = "success_late";
-    FetchResult["FailureTriedAllPeers"] = "failure_tried_all_peers";
-    FetchResult["FailureMaxAttempts"] = "failure_max_attempts";
-})(FetchResult || (FetchResult = {}));
-/**
- * BlockInputSync is a class that handles ReqResp to find blocks and data related to a specific blockRoot.  The
- * blockRoot may have been found via object gossip, or the API.  Gossip objects that can trigger a search are block,
- * blobs, columns, attestations, etc.  In the case of blocks and data this is generally during the current slot but
- * can also be for items that are received late but are not fully verified and thus not in fork-choice (old blocks on
- * an unknown fork). It can also be triggered via an attestation (or sync committee message or any other item that
- * gets gossiped) that references a blockRoot that is not in fork-choice.  In rare (and realistically should not happen)
- * situations it can get triggered via the API when the validator attempts to publish a block, attestation, aggregate
- * and proof or a sync committee contribution that has unknown information included (parentRoot for instance).
- *
- * The goal of the class is to make sure that all information that is necessary for import into fork-choice is pulled
- * from peers so that the block and data can be processed, and thus the object that triggered the search can be
- * referenced and validated.
- *
- * The most common case for this search is a set of block/data that comes across gossip for the current slot, during
- * normal chain operation, but not everything was received before the gossip cutoff window happens so it is necessary
- * to pull remaining data via req/resp so that fork-choice can be updated prior to making an attestation for the
- * current slot.
- *
- * Event sources for old UnknownBlock
- *
- * - publishBlock
- * - gossipHandlers
- * - searchUnknownSlotRoot
- *    = produceSyncCommitteeContribution
- *    = validateGossipFnRetryUnknownRoot
- *        * submitPoolAttestationsV2
- *        * publishAggregateAndProofsV2
- *    = onPendingGossipsubMessage
- *        * NetworkEvent.pendingGossipsubMessage
- *            - onGossipsubMessage
- */
-export class BlockInputSync {
-    config;
-    network;
-    chain;
-    logger;
-    metrics;
-    opts;
-    /**
-     * block RootHex -> PendingBlock. To avoid finding same root at the same time
-     */
-    pendingBlocks = new Map();
-    knownBadBlocks = new Set();
-    maxPendingBlocks;
-    subscribedToNetworkEvents = false;
-    peerBalancer;
+export class UnknownBlockSync {
     constructor(config, network, chain, logger, metrics, opts) {
         this.config = config;
         this.network = network;
@@ -78,227 +23,306 @@ export class BlockInputSync {
         this.logger = logger;
         this.metrics = metrics;
         this.opts = opts;
+        /**
+         * block RootHex -> PendingBlock. To avoid finding same root at the same time
+         */
+        this.pendingBlocks = new Map();
+        this.knownBadBlocks = new Set();
+        this.subscribedToNetworkEvents = false;
+        this.engineGetBlobsCache = new Map();
+        this.blockInputsRetryTrackerCache = new Set();
+        /**
+         * Process an unknownBlock event and register the block in `pendingBlocks` Map.
+         */
+        this.onUnknownBlock = (data) => {
+            try {
+                const unknownBlockType = this.addUnknownBlock(data.rootHex, data.peer);
+                this.triggerUnknownBlockSearch();
+                this.metrics?.syncUnknownBlock.requests.inc({ type: unknownBlockType });
+            }
+            catch (e) {
+                this.logger.debug("Error handling unknownBlock event", {}, e);
+            }
+        };
+        /**
+         * Process an unknownBlockInput event and register the block in `pendingBlocks` Map.
+         */
+        this.onUnknownBlockInput = (data) => {
+            try {
+                const unknownBlockType = this.addUnknownBlock(data.blockInput, data.peer);
+                this.triggerUnknownBlockSearch();
+                this.metrics?.syncUnknownBlock.requests.inc({ type: unknownBlockType });
+            }
+            catch (e) {
+                this.logger.debug("Error handling unknownBlockInput event", {}, e);
+            }
+        };
+        /**
+         * Process an unknownBlockParent event and register the block in `pendingBlocks` Map.
+         */
+        this.onUnknownParent = (data) => {
+            try {
+                this.addUnknownParent(data.blockInput, data.peer);
+                this.triggerUnknownBlockSearch();
+                this.metrics?.syncUnknownBlock.requests.inc({ type: PendingBlockType.UNKNOWN_PARENT });
+            }
+            catch (e) {
+                this.logger.debug("Error handling unknownBlockParent event", {}, e);
+            }
+        };
+        this.onPeerConnected = (data) => {
+            try {
+                const peerId = data.peer;
+                const peerSyncMeta = this.network.getConnectedPeerSyncMeta(peerId);
+                this.peerBalancer.onPeerConnected(data.peer, peerSyncMeta);
+                this.triggerUnknownBlockSearch();
+            }
+            catch (e) {
+                this.logger.debug("Error handling peerConnected event", {}, e);
+            }
+        };
+        this.onPeerDisconnected = (data) => {
+            const peerId = data.peer;
+            this.peerBalancer.onPeerDisconnected(peerId);
+        };
+        /**
+         * Gather tip parent blocks with unknown parent and do a search for all of them
+         */
+        this.triggerUnknownBlockSearch = () => {
+            // Cheap early stop to prevent calling the network.getConnectedPeers()
+            if (this.pendingBlocks.size === 0) {
+                return;
+            }
+            // If the node loses all peers with pending unknown blocks, the sync will stall
+            const connectedPeers = this.network.getConnectedPeers();
+            if (connectedPeers.length === 0) {
+                this.logger.debug("No connected peers, skipping unknown block search.");
+                return;
+            }
+            const { unknowns, ancestors } = getUnknownAndAncestorBlocks(this.pendingBlocks);
+            // it's rare when there is no unknown block
+            // see https://github.com/ChainSafe/lodestar/issues/5649#issuecomment-1594213550
+            if (unknowns.length === 0) {
+                let processedBlocks = 0;
+                for (const block of ancestors) {
+                    // when this happens, it's likely the block and parent block are processed by head sync
+                    if (this.chain.forkChoice.hasBlockHex(block.parentBlockRootHex)) {
+                        processedBlocks++;
+                        this.processBlock(block).catch((e) => {
+                            this.logger.debug("Unexpected error - process old downloaded block", {}, e);
+                        });
+                    }
+                }
+                this.logger.verbose("No unknown block, process ancestor downloaded blocks", {
+                    pendingBlocks: this.pendingBlocks.size,
+                    ancestorBlocks: ancestors.length,
+                    processedBlocks,
+                });
+                return;
+            }
+            // most of the time there is exactly 1 unknown block
+            for (const block of unknowns) {
+                this.downloadBlock(block).catch((e) => {
+                    this.logger.debug("Unexpected error - downloadBlock", { root: block.blockRootHex }, e);
+                });
+            }
+        };
         this.maxPendingBlocks = opts?.maxPendingBlocks ?? MAX_PENDING_BLOCKS;
-        this.peerBalancer = new UnknownBlockPeerBalancer();
+        this.proposerBoostSecWindow = this.config.SECONDS_PER_SLOT / INTERVALS_PER_SLOT;
+        this.peerBalancer = new UnknownBlockPeerBalancer(this.network.custodyConfig);
         if (metrics) {
-            metrics.blockInputSync.pendingBlocks.addCollect(() => metrics.blockInputSync.pendingBlocks.set(this.pendingBlocks.size));
-            metrics.blockInputSync.knownBadBlocks.addCollect(() => metrics.blockInputSync.knownBadBlocks.set(this.knownBadBlocks.size));
+            metrics.syncUnknownBlock.pendingBlocks.addCollect(() => {
+                metrics.syncUnknownBlock.pendingBlocks.set(this.pendingBlocks.size);
+                metrics.syncUnknownBlock.knownBadBlocks.set(this.knownBadBlocks.size);
+                metrics.syncUnknownBlock.peerBalancer.peersMetaCount.set(this.peerBalancer.peersMeta.size);
+                metrics.syncUnknownBlock.peerBalancer.peersActiveRequestCount.set(this.peerBalancer.activeRequests.size);
+                metrics.syncUnknownBlock.peerBalancer.totalActiveRequests.set(this.peerBalancer.getTotalActiveRequests());
+            });
         }
     }
     subscribeToNetwork() {
-        if (this.opts?.disableBlockInputSync) {
-            this.logger.verbose("BlockInputSync disabled by disableBlockInputSync option.");
-            return;
+        if (!this.opts?.disableUnknownBlockSync) {
+            // cannot chain to the above if or the log will be incorrect
+            if (!this.subscribedToNetworkEvents) {
+                this.logger.verbose("UnknownBlockSync enabled.");
+                this.network.events.on(NetworkEvent.unknownBlock, this.onUnknownBlock);
+                this.network.events.on(NetworkEvent.unknownBlockInput, this.onUnknownBlockInput);
+                this.network.events.on(NetworkEvent.unknownBlockParent, this.onUnknownParent);
+                this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
+                this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
+                this.subscribedToNetworkEvents = true;
+            }
         }
-        // cannot chain to the above if or the log will be incorrect
-        if (!this.subscribedToNetworkEvents) {
-            this.logger.verbose("BlockInputSync enabled.");
-            this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
-            this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
-            this.chain.emitter.on(ChainEvent.unknownParent, this.onUnknownParent);
-            this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
-            this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
-            this.subscribedToNetworkEvents = true;
+        else {
+            this.logger.verbose("UnknownBlockSync disabled by disableUnknownBlockSync option.");
         }
     }
     unsubscribeFromNetwork() {
-        this.logger.verbose("BlockInputSync disabled.");
-        this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
-        this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
-        this.chain.emitter.off(ChainEvent.unknownParent, this.onUnknownParent);
+        this.logger.verbose("UnknownBlockSync disabled.");
+        this.network.events.off(NetworkEvent.unknownBlock, this.onUnknownBlock);
+        this.network.events.off(NetworkEvent.unknownBlockInput, this.onUnknownBlockInput);
+        this.network.events.off(NetworkEvent.unknownBlockParent, this.onUnknownParent);
         this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
         this.network.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
         this.subscribedToNetworkEvents = false;
     }
     close() {
         this.unsubscribeFromNetwork();
+        // add more in the future if needed
     }
     isSubscribedToNetwork() {
         return this.subscribedToNetworkEvents;
     }
     /**
-     * Process an unknownBlock event and register the block in `pendingBlocks` Map.
+     * When a blockInput comes with  an unknown parent:
+     * - add the block to pendingBlocks with status downloaded or pending blockRootHex as key. This is similar to
+     * an `onUnknownBlock` event, but the blocks is downloaded.
+     * - add the parent root to pendingBlocks with status pending, parentBlockRootHex as key. This is
+     * the same to an `onUnknownBlock` event with parentBlockRootHex as root.
      */
-    onUnknownBlockRoot = (data) => {
-        try {
-            this.addByRootHex(data.rootHex, data.peer);
-            this.triggerUnknownBlockSearch();
-            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_BLOCK_ROOT });
-            this.metrics?.blockInputSync.source.inc({ source: data.source });
-        }
-        catch (e) {
-            this.logger.debug("Error handling unknownBlockRoot event", {}, e);
-        }
-    };
-    /**
-     * Process an unknownBlockInput event and register the block in `pendingBlocks` Map.
-     */
-    onIncompleteBlockInput = (data) => {
-        try {
-            this.addByBlockInput(data.blockInput, data.peer);
-            this.triggerUnknownBlockSearch();
-            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.INCOMPLETE_BLOCK_INPUT });
-            this.metrics?.blockInputSync.source.inc({ source: data.source });
-        }
-        catch (e) {
-            this.logger.debug("Error handling incompleteBlockInput event", {}, e);
-        }
-    };
-    /**
-     * Process an unknownBlockParent event and register the block in `pendingBlocks` Map.
-     */
-    onUnknownParent = (data) => {
-        try {
-            this.addByRootHex(data.blockInput.parentRootHex, data.peer);
-            this.addByBlockInput(data.blockInput, data.peer);
-            this.triggerUnknownBlockSearch();
-            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_PARENT });
-            this.metrics?.blockInputSync.source.inc({ source: data.source });
-        }
-        catch (e) {
-            this.logger.debug("Error handling unknownParent event", {}, e);
-        }
-    };
-    addByRootHex = (rootHex, peerIdStr) => {
-        let pendingBlock = this.pendingBlocks.get(rootHex);
+    addUnknownParent(blockInput, peerIdStr) {
+        const block = blockInput.block.message;
+        const blockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block);
+        const blockRootHex = toRootHex(blockRoot);
+        const parentBlockRootHex = toRootHex(block.parentRoot);
+        // add 1 pending block with status downloaded
+        let pendingBlock = this.pendingBlocks.get(blockRootHex);
         if (!pendingBlock) {
-            pendingBlock = {
-                status: PendingBlockInputStatus.pending,
-                rootHex: rootHex,
-                peerIdStrings: new Set(),
-                timeAddedSec: Date.now() / 1000,
-            };
-            this.pendingBlocks.set(rootHex, pendingBlock);
-            this.logger.verbose("Added new rootHex to BlockInputSync.pendingBlocks", {
-                root: pendingBlock.rootHex,
-                peerIdStr: peerIdStr ?? "unknown peer",
+            pendingBlock =
+                blockInput.type === BlockInputType.dataPromise
+                    ? {
+                        unknownBlockType: PendingBlockType.UNKNOWN_DATA,
+                        blockRootHex,
+                        // this will be set after we download block
+                        parentBlockRootHex: null,
+                        blockInput,
+                        peerIdStrs: new Set(),
+                        status: PendingBlockStatus.pending,
+                        downloadAttempts: 0,
+                    }
+                    : {
+                        blockRootHex,
+                        parentBlockRootHex,
+                        blockInput,
+                        peerIdStrs: new Set(),
+                        status: PendingBlockStatus.downloaded,
+                        downloadAttempts: 0,
+                    };
+            this.pendingBlocks.set(blockRootHex, pendingBlock);
+            this.logger.verbose("Added unknown block parent to pendingBlocks", {
+                root: blockRootHex,
+                parent: parentBlockRootHex,
             });
         }
-        if (peerIdStr) {
-            pendingBlock.peerIdStrings.add(peerIdStr);
+        pendingBlock.peerIdStrs.add(peerIdStr);
+        // add 1 pending block with status pending
+        this.addUnknownBlock(parentBlockRootHex, peerIdStr);
+    }
+    addUnknownBlock(blockInputOrRootHex, peerIdStr) {
+        let blockRootHex;
+        let blockInput;
+        let unknownBlockType;
+        if (typeof blockInputOrRootHex === "string") {
+            blockRootHex = blockInputOrRootHex;
+            blockInput = null;
+            unknownBlockType = PendingBlockType.UNKNOWN_BLOCK;
         }
-        // TODO: check this prune methodology
-        // Limit pending blocks to prevent DOS attacks that cause OOM
-        const prunedItemCount = pruneSetToMax(this.pendingBlocks, this.maxPendingBlocks);
-        if (prunedItemCount > 0) {
-            this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
-        }
-    };
-    addByBlockInput = (blockInput, peerIdStr) => {
-        let pendingBlock = this.pendingBlocks.get(blockInput.blockRootHex);
-        // if entry is missing or was added via rootHex and now we have more complete information overwrite
-        // the existing information with the more complete cache entry
-        if (!pendingBlock || !isPendingBlockInput(pendingBlock)) {
+        else {
+            if (blockInputOrRootHex.block !== null) {
+                const { block } = blockInputOrRootHex;
+                blockRootHex = toRootHex(this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message));
+                unknownBlockType = PendingBlockType.UNKNOWN_DATA;
+            }
+            else {
+                unknownBlockType = PendingBlockType.UNKNOWN_BLOCKINPUT;
+                blockRootHex = blockInputOrRootHex.blockRootHex;
+            }
+            blockInput = blockInputOrRootHex;
+        }
+        let pendingBlock = this.pendingBlocks.get(blockRootHex);
+        if (!pendingBlock) {
             pendingBlock = {
-                // can be added via unknown parent and we may already have full block input. need to check and set correctly
-                // so we pull the data if its missing or handle the block correctly in getIncompleteAndAncestorBlocks
-                status: blockInput.hasBlockAndAllData() ? PendingBlockInputStatus.downloaded : PendingBlockInputStatus.pending,
+                unknownBlockType,
+                blockRootHex,
+                // this will be set after we download block
+                parentBlockRootHex: null,
                 blockInput,
-                peerIdStrings: new Set(),
-                timeAddedSec: Date.now() / 1000,
+                peerIdStrs: new Set(),
+                status: PendingBlockStatus.pending,
+                downloadAttempts: 0,
             };
-            this.pendingBlocks.set(blockInput.blockRootHex, pendingBlock);
-            this.logger.verbose("Added blockInput to BlockInputSync.pendingBlocks", pendingBlock.blockInput.getLogMeta());
+            this.pendingBlocks.set(blockRootHex, pendingBlock);
+            this.logger.verbose("Added unknown block to pendingBlocks", {
+                unknownBlockType,
+                root: blockRootHex,
+                slot: blockInput?.block?.message.slot ?? "unknown",
+            });
         }
         if (peerIdStr) {
-            pendingBlock.peerIdStrings.add(peerIdStr);
+            pendingBlock.peerIdStrs.add(peerIdStr);
         }
-        // TODO: check this prune methodology
         // Limit pending blocks to prevent DOS attacks that cause OOM
         const prunedItemCount = pruneSetToMax(this.pendingBlocks, this.maxPendingBlocks);
         if (prunedItemCount > 0) {
-            this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
-        }
-    };
-    onPeerConnected = (data) => {
-        try {
-            const peerId = data.peer;
-            const peerSyncMeta = this.network.getConnectedPeerSyncMeta(peerId);
-            this.peerBalancer.onPeerConnected(data.peer, peerSyncMeta);
-            this.triggerUnknownBlockSearch();
-        }
-        catch (e) {
-            this.logger.debug("Error handling peerConnected event", {}, e);
-        }
-    };
-    onPeerDisconnected = (data) => {
-        const peerId = data.peer;
-        this.peerBalancer.onPeerDisconnected(peerId);
-    };
-    /**
-     * Gather tip parent blocks with unknown parent and do a search for all of them
-     */
-    triggerUnknownBlockSearch = () => {
-        // Cheap early stop to prevent calling the network.getConnectedPeers()
-        if (this.pendingBlocks.size === 0) {
-            return;
+            this.logger.warn(`Pruned ${prunedItemCount} pending blocks from UnknownBlockSync`);
         }
-        // If the node loses all peers with pending unknown blocks, the sync will stall
-        const connectedPeers = this.network.getConnectedPeers();
-        if (connectedPeers.length === 0) {
-            this.logger.debug("No connected peers, skipping unknown block search.");
-            return;
-        }
-        const { unknowns, ancestors } = getUnknownAndAncestorBlocks(this.pendingBlocks);
-        // it's rare when there is no unknown block
-        // see https://github.com/ChainSafe/lodestar/issues/5649#issuecomment-1594213550
-        if (unknowns.length === 0) {
-            let processedBlocks = 0;
-            for (const block of ancestors) {
-                // when this happens, it's likely the block and parent block are processed by head sync
-                if (this.chain.forkChoice.hasBlockHex(block.blockInput.parentRootHex)) {
-                    processedBlocks++;
-                    this.processBlock(block).catch((e) => {
-                        this.logger.debug("Unexpected error - process old downloaded block", {}, e);
-                    });
-                }
-            }
-            this.logger.verbose("No unknown block, process ancestor downloaded blocks", {
-                pendingBlocks: this.pendingBlocks.size,
-                ancestorBlocks: ancestors.length,
-                processedBlocks,
-            });
-            return;
-        }
-        // most of the time there is exactly 1 unknown block
-        for (const block of unknowns) {
-            this.downloadBlock(block).catch((e) => {
-                this.logger.debug("Unexpected error - downloadBlock", { root: getBlockInputSyncCacheItemRootHex(block) }, e);
-            });
-        }
-    };
+        return unknownBlockType;
+    }
     async downloadBlock(block) {
-        if (block.status !== PendingBlockInputStatus.pending) {
+        if (block.status !== PendingBlockStatus.pending) {
             return;
         }
-        const rootHex = getBlockInputSyncCacheItemRootHex(block);
+        const unknownBlockType = block.unknownBlockType;
         const logCtx = {
-            slot: getBlockInputSyncCacheItemSlot(block),
-            root: rootHex,
+            root: block.blockRootHex,
             pendingBlocks: this.pendingBlocks.size,
+            slot: block.blockInput?.block?.message.slot ?? "unknown",
+            unknownBlockType,
         };
-        this.logger.verbose("BlockInputSync.downloadBlock()", logCtx);
-        block.status = PendingBlockInputStatus.fetching;
-        const res = await wrapError(this.fetchBlockInput(block));
+        this.logger.verbose("Downloading unknown block", logCtx);
+        block.status = PendingBlockStatus.fetching;
+        let res;
+        if (block.blockInput === null) {
+            // we only have block root, and nothing else
+            res = await wrapError(this.fetchUnknownBlockRoot(fromHex(block.blockRootHex)));
+        }
+        else {
+            res = await wrapError(this.fetchUnavailableBlockInput(block.blockInput));
+        }
+        if (res.err)
+            this.metrics?.syncUnknownBlock.downloadedBlocksError.inc();
+        else
+            this.metrics?.syncUnknownBlock.downloadedBlocksSuccess.inc();
         if (!res.err) {
-            this.metrics?.blockInputSync.downloadedBlocksSuccess.inc();
-            const pending = res.result;
-            this.pendingBlocks.set(pending.blockInput.blockRootHex, pending);
-            const blockSlot = pending.blockInput.slot;
+            const { blockInput, peerIdStr } = res.result;
+            // fetchUnknownBlockRoot and fetchUnavailableBlockInput should return available data BlockInput, throw error if not
+            if (blockInput.type === BlockInputType.dataPromise) {
+                // if there were any peers who would have had the missing datacolumns, it would have resulted in err
+                throw Error(`Expected BlockInput to be available, got dataPromise for ${block.blockRootHex}`);
+            }
+            block = {
+                ...block,
+                status: PendingBlockStatus.downloaded,
+                blockInput,
+                parentBlockRootHex: toRootHex(blockInput.block.message.parentRoot),
+            };
+            this.pendingBlocks.set(block.blockRootHex, block);
+            const blockSlot = blockInput.block.message.slot;
             const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
             const delaySec = Date.now() / 1000 - (this.chain.genesisTime + blockSlot * this.config.SECONDS_PER_SLOT);
-            this.metrics?.blockInputSync.elapsedTimeTillReceived.observe(delaySec);
-            const parentInForkChoice = this.chain.forkChoice.hasBlockHex(pending.blockInput.parentRootHex);
-            const logCtx2 = {
-                ...logCtx,
-                slot: blockSlot,
-                parentInForkChoice,
-            };
-            this.logger.verbose("Downloaded unknown block", logCtx2);
-            if (parentInForkChoice) {
+            this.metrics?.syncUnknownBlock.elapsedTimeTillReceived.observe(delaySec);
+            const parentInForkchoice = this.chain.forkChoice.hasBlock(blockInput.block.message.parentRoot);
+            this.logger.verbose("Downloaded unknown block", {
+                root: block.blockRootHex,
+                pendingBlocks: this.pendingBlocks.size,
+                parentInForkchoice,
+                blockInputType: blockInput.type,
+                unknownBlockType,
+            });
+            if (parentInForkchoice) {
                 // Bingo! Process block. Add to pending blocks anyway for recycle the cache that prevents duplicate processing
-                this.processBlock(pending).catch((e) => {
-                    this.logger.debug("Unexpected error - process newly downloaded block", logCtx2, e);
+                this.processBlock(block).catch((e) => {
+                    this.logger.debug("Unexpected error - process newly downloaded block", {}, e);
                 });
             }
             else if (blockSlot <= finalizedSlot) {
@@ -307,20 +331,34 @@ export class BlockInputSync {
                 // 0 - 1 - ... - n - finalizedSlot
                 //                \
                 //                parent 1 - parent 2 - ... - unknownParent block
+                const blockRoot = this.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(blockInput.block.message);
                 this.logger.debug("Downloaded block is before finalized slot", {
-                    ...logCtx2,
                     finalizedSlot,
+                    blockSlot,
+                    parentRoot: toRootHex(blockRoot),
+                    unknownBlockType,
                 });
-                this.removeAndDownScoreAllDescendants(block);
+                this.removeAndDownscoreAllDescendants(block);
             }
             else {
-                this.onUnknownBlockRoot({ rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot });
+                this.onUnknownParent({ blockInput, peer: peerIdStr });
             }
         }
         else {
-            this.metrics?.blockInputSync.downloadedBlocksError.inc();
-            this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
-            this.removeAndDownScoreAllDescendants(block);
+            // block download has error, this allows to retry the download of the block
+            block.status = PendingBlockStatus.pending;
+            // parentSlot > finalizedSlot, continue downloading parent of parent
+            block.downloadAttempts++;
+            const errorData = { root: block.blockRootHex, attempts: block.downloadAttempts, unknownBlockType };
+            if (block.downloadAttempts > MAX_ATTEMPTS_PER_BLOCK) {
+                // Give up on this block and assume it does not exist, penalizing all peers as if it was a bad block
+                this.logger.debug("Ignoring unknown block root after many failed downloads", errorData, res.err);
+                this.removeAndDownscoreAllDescendants(block);
+            }
+            else {
+                // Try again when a new peer connects, its status changes, or a new unknownBlockParent event happens
+                this.logger.debug("Error downloading unknown block root", errorData, res.err);
+            }
         }
     }
     /**
@@ -331,14 +369,11 @@ export class BlockInputSync {
     async processBlock(pendingBlock) {
         // pending block status is `downloaded` right after `downloadBlock`
         // but could be `pending` if added by `onUnknownBlockParent` event and this function is called recursively
-        if (pendingBlock.status !== PendingBlockInputStatus.downloaded) {
-            if (pendingBlock.status === PendingBlockInputStatus.pending) {
+        if (pendingBlock.status !== PendingBlockStatus.downloaded) {
+            if (pendingBlock.status === PendingBlockStatus.pending) {
                 const connectedPeers = this.network.getConnectedPeers();
                 if (connectedPeers.length === 0) {
-                    this.logger.debug("No connected peers, skipping download block", {
-                        slot: pendingBlock.blockInput.slot,
-                        blockRoot: pendingBlock.blockInput.blockRootHex,
-                    });
+                    this.logger.debug("No connected peers, skipping download block", { blockRoot: pendingBlock.blockRootHex });
                     return;
                 }
                 // if the download is a success we'll call `processBlock()` for this block
@@ -346,22 +381,23 @@ export class BlockInputSync {
             }
             return;
         }
-        pendingBlock.status = PendingBlockInputStatus.processing;
+        pendingBlock.status = PendingBlockStatus.processing;
         // this prevents unbundling attack
         // see https://lighthouse-blog.sigmaprime.io/mev-unbundling-rpc.html
-        const { slot: blockSlot, proposerIndex } = pendingBlock.blockInput.getBlock().message;
-        const fork = this.config.getForkName(blockSlot);
-        const proposerBoostWindowMs = this.config.getAttestationDueMs(fork);
-        if (this.chain.clock.msFromSlot(blockSlot) < proposerBoostWindowMs &&
+        const { slot: blockSlot, proposerIndex } = pendingBlock.blockInput.block.message;
+        if (this.chain.clock.secFromSlot(blockSlot) < this.proposerBoostSecWindow &&
             this.chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
             // proposer is known by a gossip block already, wait a bit to make sure this block is not
             // eligible for proposer boost to prevent unbundling attack
+            const blockRoot = this.config
+                .getForkTypes(blockSlot)
+                .BeaconBlock.hashTreeRoot(pendingBlock.blockInput.block.message);
             this.logger.verbose("Avoid proposer boost for this block of known proposer", {
-                slot: blockSlot,
-                root: pendingBlock.blockInput.blockRootHex,
+                blockSlot,
+                blockRoot: toRootHex(blockRoot),
                 proposerIndex,
             });
-            await sleep(proposerBoostWindowMs);
+            await sleep(this.proposerBoostSecWindow * 1000);
         }
         // At gossip time, it's critical to keep a good number of mesh peers.
         // To do that, the Gossip Job Wait Time should be consistently <3s to avoid the behavior penalties in gossip
@@ -379,24 +415,21 @@ export class BlockInputSync {
             eagerPersistBlock: true,
         }));
         if (res.err)
-            this.metrics?.blockInputSync.processedBlocksError.inc();
+            this.metrics?.syncUnknownBlock.processedBlocksError.inc();
         else
-            this.metrics?.blockInputSync.processedBlocksSuccess.inc();
+            this.metrics?.syncUnknownBlock.processedBlocksSuccess.inc();
         if (!res.err) {
             // no need to update status to "processed", delete anyway
-            this.pendingBlocks.delete(pendingBlock.blockInput.blockRootHex);
-            this.chain.seenBlockInputCache.prune(pendingBlock.blockInput.blockRootHex);
+            this.pendingBlocks.delete(pendingBlock.blockRootHex);
             // Send child blocks to the processor
-            for (const descendantBlock of getDescendantBlocks(pendingBlock.blockInput.blockRootHex, this.pendingBlocks)) {
-                if (isPendingBlockInput(descendantBlock)) {
-                    this.processBlock(descendantBlock).catch((e) => {
-                        this.logger.debug("Unexpected error - process descendant block", {}, e);
-                    });
-                }
+            for (const descendantBlock of getDescendantBlocks(pendingBlock.blockRootHex, this.pendingBlocks)) {
+                this.processBlock(descendantBlock).catch((e) => {
+                    this.logger.debug("Unexpected error - process descendant block", {}, e);
+                });
             }
         }
         else {
-            const errorData = { slot: pendingBlock.blockInput.slot, root: pendingBlock.blockInput.blockRootHex };
+            const errorData = { root: pendingBlock.blockRootHex, slot: pendingBlock.blockInput.block.message.slot };
             if (res.err instanceof BlockError) {
                 switch (res.err.type.code) {
                     // This cases are already handled with `{ignoreIfKnown: true}`
@@ -406,7 +439,7 @@ export class BlockInputSync {
                     case BlockErrorCode.PRESTATE_MISSING:
                         // Should not happen, mark as downloaded to try again latter
                         this.logger.debug("Attempted to process block but its parent was still unknown", errorData, res.err);
-                        pendingBlock.status = PendingBlockInputStatus.downloaded;
+                        pendingBlock.status = PendingBlockStatus.downloaded;
                         break;
                     case BlockErrorCode.EXECUTION_ENGINE_ERROR:
                         // Removing the block(s) without penalizing the peers, hoping for EL to
@@ -416,13 +449,13 @@ export class BlockInputSync {
                     default:
                         // Block is not correct with respect to our chain. Log error loudly
                         this.logger.debug("Error processing block from unknown parent sync", errorData, res.err);
-                        this.removeAndDownScoreAllDescendants(pendingBlock);
+                        this.removeAndDownscoreAllDescendants(pendingBlock);
                 }
             }
             // Probably a queue error or something unwanted happened, mark as pending to try again latter
             else {
                 this.logger.debug("Unknown error processing block from unknown block sync", errorData, res.err);
-                pendingBlock.status = PendingBlockInputStatus.downloaded;
+                pendingBlock.status = PendingBlockStatus.downloaded;
             }
         }
     }
@@ -435,129 +468,144 @@ export class BlockInputSync {
      * prefulu, will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers, postfulu we may attempt more as defined in `getMaxDownloadAttempts()` function
      * Also verifies the received block root + returns the peer that provided the block for future downscoring.
      */
-    async fetchBlockInput(cacheItem) {
-        const rootHex = getBlockInputSyncCacheItemRootHex(cacheItem);
+    async fetchUnknownBlockRoot(blockRoot) {
+        const blockRootHex = toRootHex(blockRoot);
         const excludedPeers = new Set();
+        let partialDownload = null;
         const defaultPendingColumns = this.config.getForkSeq(this.chain.clock.currentSlot) >= ForkSeq.fulu
-            ? new Set(this.network.custodyConfig.sampledColumns)
+            ? new Set(this.network.custodyConfig.sampleGroups)
             : null;
-        const fetchStartSec = Date.now() / 1000;
-        let slot = isPendingBlockInput(cacheItem) ? cacheItem.blockInput.slot : undefined;
-        if (slot !== undefined) {
-            this.metrics?.blockInputSync.fetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
-        }
+        let lastError = null;
         let i = 0;
         while (i++ < this.getMaxDownloadAttempts()) {
-            const pendingColumns = isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
-                ? new Set(cacheItem.blockInput.getMissingSampledColumnMeta().missing)
-                : defaultPendingColumns;
-            // pendingDataColumns is null pre-fulu
-            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
-            if (peerMeta === null) {
+            // pendingDataColumns is null prefulu
+            const peer = this.peerBalancer.bestPeerForPendingColumns(partialDownload ? new Set(partialDownload.pendingDataColumns) : defaultPendingColumns, excludedPeers);
+            if (peer === null) {
                 // no more peer with needed columns to try, throw error
-                let message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer`;
-                if (pendingColumns) {
-                    message += ` with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
-                }
-                this.metrics?.blockInputSync.fetchTimeSec.observe({ result: FetchResult.FailureTriedAllPeers }, Date.now() / 1000 - fetchStartSec);
-                this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureTriedAllPeers }, i);
-                throw Error(message);
+                throw Error(`Error fetching UnknownBlockRoot after ${i}: cannot find peer with needed columns ${partialDownload?.pendingDataColumns.join(", ")}`);
             }
-            const { peerId, client: peerClient } = peerMeta;
-            cacheItem.peerIdStrings.add(peerId);
+            const { peerId, client: peerClient } = peer;
+            excludedPeers.add(peerId);
             try {
-                const downloadResult = await downloadByRoot({
-                    config: this.config,
-                    network: this.network,
-                    seenCache: this.chain.seenBlockInputCache,
-                    emitter: this.chain.emitter,
-                    peerMeta,
-                    cacheItem,
-                });
-                cacheItem = downloadResult.result;
-                if (slot === undefined) {
-                    slot = cacheItem.blockInput.slot;
-                    // we were not able to observe the time into slot when starting the fetch, do it now
-                    this.metrics?.blockInputSync.fetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
+                const { blocks: [blockInput], pendingDataColumns, } = await beaconBlocksMaybeBlobsByRoot(this.config, this.network, peerId, [blockRoot], partialDownload, peerClient, this.metrics, this.logger);
+                // Peer does not have the block, try with next peer
+                if (blockInput === undefined) {
+                    continue;
                 }
-                const logCtx = { slot, rootHex, peerId, peerClient };
-                this.logger.verbose("BlockInputSync.fetchBlockInput: successful download", logCtx);
-                this.metrics?.blockInputSync.downloadByRoot.success.inc();
-                const warnings = downloadResult.warnings;
-                if (warnings) {
-                    for (const warning of warnings) {
-                        this.logger.debug("BlockInputSync.fetchBlockInput: downloaded with warning", logCtx, warning);
-                        this.metrics?.blockInputSync.downloadByRoot.warn.inc({ code: warning.type.code, client: peerClient });
-                    }
-                    // TODO: penalize peer?
+                if (pendingDataColumns !== null) {
+                    partialDownload = { blocks: [blockInput], pendingDataColumns };
+                    continue;
                 }
+                // data is available, verify block root is correct
+                const block = blockInput.block.message;
+                const receivedBlockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block);
+                if (!byteArrayEquals(receivedBlockRoot, blockRoot)) {
+                    throw Error(`Wrong block received by peer, got ${toRootHex(receivedBlockRoot)} expected ${blockRootHex}`);
+                }
+                return { blockInput, peerIdStr: peerId };
             }
             catch (e) {
-                this.logger.debug("Error downloading in BlockInputSync.fetchBlockInput", { slot, rootHex, attempt: i, peer: peerId, peerClient }, e);
-                const downloadByRootMetrics = this.metrics?.blockInputSync.downloadByRoot;
-                // TODO: penalize peer?
-                if (e instanceof DownloadByRootError) {
-                    const errorCode = e.type.code;
-                    downloadByRootMetrics?.error.inc({ code: errorCode, client: peerClient });
-                    excludedPeers.add(peerId);
+                this.logger.debug("Error fetching UnknownBlockRoot", { attempt: i, blockRootHex, peer: peerId }, e);
+                lastError = e;
+            }
+            finally {
+                this.peerBalancer.onRequestCompleted(peerId);
+            }
+        }
+        if (lastError) {
+            lastError.message = `Error fetching UnknownBlockRoot after ${i} attempts: ${lastError.message}`;
+            throw lastError;
+        }
+        throw Error(`Error fetching UnknownBlockRoot after ${i}: cannot download all blobs or data columns for block ${blockRootHex}`);
+    }
+    /**
+     * We have partial block input:
+     * - we have block but not have all blobs (deneb) or needed columns (fulu)
+     * - we don't have block and have some blobs (deneb) or some columns (fulu)
+     * Fetches missing block/data columns/block for the blockinput. This function returns either preData or availableData BlockInput.
+     */
+    async fetchUnavailableBlockInput(unavailableBlockInput) {
+        if (unavailableBlockInput.block !== null && unavailableBlockInput.type !== BlockInputType.dataPromise) {
+            return { blockInput: unavailableBlockInput, peerIdStr: "" };
+        }
+        let blockRootHex;
+        let blobKzgCommitmentsLen;
+        let blockRoot;
+        const dataMeta = {};
+        let sampledColumns = [];
+        if (unavailableBlockInput.block === null) {
+            blockRootHex = unavailableBlockInput.blockRootHex;
+            blockRoot = fromHex(blockRootHex);
+        }
+        else {
+            const { cachedData, block: unavailableBlock } = unavailableBlockInput;
+            blockRoot = this.config
+                .getForkTypes(unavailableBlock.message.slot)
+                .BeaconBlock.hashTreeRoot(unavailableBlock.message);
+            blockRootHex = toRootHex(blockRoot);
+            blobKzgCommitmentsLen = unavailableBlock.message.body.blobKzgCommitments.length;
+            if (cachedData.fork === ForkName.deneb || cachedData.fork === ForkName.electra) {
+                const pendingBlobs = blobKzgCommitmentsLen - cachedData.blobsCache.size;
+                Object.assign(dataMeta, { pendingBlobs });
+            }
+            else if (cachedData.fork === ForkName.fulu) {
+                sampledColumns = this.network.custodyConfig.sampledColumns;
+                const pendingColumns = sampledColumns.length - cachedData.dataColumnsCache.size;
+                Object.assign(dataMeta, { pendingColumns });
+            }
+        }
+        let lastError = null;
+        let i = 0;
+        const excludedPeers = new Set();
+        while (i++ < this.getMaxDownloadAttempts()) {
+            const bestPeer = this.peerBalancer.bestPeerForBlockInput(unavailableBlockInput, excludedPeers);
+            if (bestPeer === null) {
+                // no more peer to try, throw error
+                throw Error(`Error fetching UnavailableBlockInput after ${i}: cannot find peer with needed columns ${sampledColumns.join(", ")}`);
+            }
+            const { peerId, client: peerClient } = bestPeer;
+            excludedPeers.add(peerId);
+            try {
+                const blockInput = await unavailableBeaconBlobsByRoot(this.config, this.network, peerId, peerClient, unavailableBlockInput, {
+                    metrics: this.metrics,
+                    logger: this.logger,
+                    executionEngine: this.chain.executionEngine,
+                    emitter: this.chain.emitter,
+                    blockInputsRetryTrackerCache: this.blockInputsRetryTrackerCache,
+                    engineGetBlobsCache: this.engineGetBlobsCache,
+                });
+                if (unavailableBlockInput.block !== null && blockInput.type === BlockInputType.dataPromise) {
+                    // all datacolumns were not downloaded we can continue with other peers
+                    // as unavailableBlockInput.block's dataColumnsCache would be updated
+                    continue;
                 }
-                else if (e instanceof RequestError) {
-                    // should look into req_resp metrics in this case
-                    downloadByRootMetrics?.error.inc({ code: "req_resp", client: peerClient });
-                    switch (e.type.code) {
-                        case RequestErrorCode.REQUEST_RATE_LIMITED:
-                        case RequestErrorCode.REQUEST_TIMEOUT:
-                            // do not exclude peer for these errors
-                            break;
-                        default:
-                            excludedPeers.add(peerId);
-                            break;
-                    }
+                // data is available, verify block root is correct
+                const block = blockInput.block.message;
+                const receivedBlockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block);
+                if (!byteArrayEquals(receivedBlockRoot, blockRoot)) {
+                    throw Error(`Wrong block received by peer, got ${toRootHex(receivedBlockRoot)} expected ${blockRootHex}`);
+                }
+                if (unavailableBlockInput.block === null) {
+                    this.logger.debug("Fetched  NullBlockInput", { attempts: i, blockRootHex });
                 }
                 else {
-                    // investigate if this happens
-                    downloadByRootMetrics?.error.inc({ code: "unknown", client: peerClient });
-                    excludedPeers.add(peerId);
+                    this.logger.debug("Fetched UnavailableBlockInput", { attempts: i, ...dataMeta, blobKzgCommitmentsLen });
                 }
+                return { blockInput, peerIdStr: peerId };
+            }
+            catch (e) {
+                this.logger.debug("Error fetching UnavailableBlockInput", { attempt: i, blockRootHex, peer: peerId }, e);
+                lastError = e;
             }
             finally {
                 this.peerBalancer.onRequestCompleted(peerId);
             }
-            this.pendingBlocks.set(getBlockInputSyncCacheItemRootHex(cacheItem), cacheItem);
-            if (cacheItem.status === PendingBlockInputStatus.downloaded) {
-                // download was successful, no need to go with another peer, return
-                const result = this.chain.forkChoice.hasBlockHex(cacheItem.blockInput.blockRootHex)
-                    ? FetchResult.SuccessLate
-                    : this.chain.forkChoice.hasBlockHex(cacheItem.blockInput.parentRootHex)
-                        ? FetchResult.SuccessResolved
-                        : FetchResult.SuccessMissingParent;
-                this.metrics?.blockInputSync.fetchTimeSec.observe({ result }, Date.now() / 1000 - fetchStartSec);
-                this.metrics?.blockInputSync.fetchPeers.set({ result }, i);
-                return cacheItem;
-            }
-        } // end while loop over peers
-        const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
-        if (!isPendingBlockInput(cacheItem)) {
-            throw Error(`${message} No block and no data was found.`);
-        }
-        if (!cacheItem.blockInput.hasBlock()) {
-            throw new Error(`${message} Block was not found.`);
-        }
-        if (isBlockInputBlobs(cacheItem.blockInput)) {
-            const missing = cacheItem.blockInput.getMissingBlobMeta().map((b) => b.index);
-            if (missing.length) {
-                throw new Error(`${message} Missing blob indices=${prettyPrintIndices(missing)}.`);
-            }
-        }
-        if (isBlockInputColumns(cacheItem.blockInput)) {
-            const missing = cacheItem.blockInput.getMissingSampledColumnMeta().missing;
-            if (missing.length) {
-                throw new Error(`${message} Missing column indices=${prettyPrintIndices(missing)}.`);
-            }
-        }
-        this.metrics?.blockInputSync.fetchTimeSec.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
-        this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
-        throw Error(message);
+        }
+        if (lastError) {
+            lastError.message = `Error fetching UnavailableBlockInput after ${i} attempts: ${lastError.message}`;
+            throw lastError;
+        }
+        throw Error(`Error fetching UnavailableBlockInput after ${i}: unknown error`);
     }
     /**
      * Gets all descendant blocks of `block` recursively from `pendingBlocks`.
@@ -565,44 +613,33 @@ export class BlockInputSync {
      * Downscore all peers that have referenced any of this bad blocks. May report peers multiple times if they have
      * referenced more than one bad block.
      */
-    removeAndDownScoreAllDescendants(block) {
+    removeAndDownscoreAllDescendants(block) {
         // Get all blocks that are a descendant of this one
         const badPendingBlocks = this.removeAllDescendants(block);
         // just console log and do not penalize on pending/bad blocks for debugging
         // console.log("removeAndDownscoreAllDescendants", {block});
         for (const block of badPendingBlocks) {
-            //
-            // TODO(fulu): why is this commented out here?
-            //
             //   this.knownBadBlocks.add(block.blockRootHex);
             //   for (const peerIdStr of block.peerIdStrs) {
             //     // TODO: Refactor peerRpcScores to work with peerIdStr only
             //     this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot");
             //   }
             this.logger.debug("ignored Banning unknown block", {
-                slot: getBlockInputSyncCacheItemSlot(block),
-                root: getBlockInputSyncCacheItemRootHex(block),
-                peerIdStrings: Array.from(block.peerIdStrings)
-                    .map((id) => prettyPrintPeerIdStr(id))
-                    .join(","),
+                root: block.blockRootHex,
+                peerIdStrs: Array.from(block.peerIdStrs).join(","),
             });
         }
         // Prune knownBadBlocks
         pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
     }
     removeAllDescendants(block) {
-        const rootHex = getBlockInputSyncCacheItemRootHex(block);
-        const slot = getBlockInputSyncCacheItemSlot(block);
         // Get all blocks that are a descendant of this one
-        const badPendingBlocks = [block, ...getAllDescendantBlocks(rootHex, this.pendingBlocks)];
-        this.metrics?.blockInputSync.removedBlocks.inc(badPendingBlocks.length);
+        const badPendingBlocks = [block, ...getAllDescendantBlocks(block.blockRootHex, this.pendingBlocks)];
+        this.metrics?.syncUnknownBlock.removedBlocks.inc(badPendingBlocks.length);
         for (const block of badPendingBlocks) {
-            const rootHex = getBlockInputSyncCacheItemRootHex(block);
-            this.pendingBlocks.delete(rootHex);
-            this.chain.seenBlockInputCache.prune(rootHex);
-            this.logger.debug("Removing bad/unknown/incomplete BlockInputSyncCacheItem", {
-                slot,
-                blockRoot: rootHex,
+            this.pendingBlocks.delete(block.blockRootHex);
+            this.logger.debug("Removing unknown parent block", {
+                root: block.blockRootHex,
             });
         }
         return badPendingBlocks;
@@ -620,11 +657,10 @@ export class BlockInputSync {
  * Class to track active byRoots requests and balance them across eligible peers.
  */
 export class UnknownBlockPeerBalancer {
-    peersMeta;
-    activeRequests;
-    constructor() {
+    constructor(custodyConfig) {
         this.peersMeta = new Map();
         this.activeRequests = new Map();
+        this.custodyConfig = custodyConfig;
     }
     /** Trigger on each peer re-status */
     onPeerConnected(peerId, syncMeta) {
@@ -658,16 +694,36 @@ export class UnknownBlockPeerBalancer {
      * called from fetchUnavailableBlockInput() where we have either BlockInput or NullBlockInput
      * excludedPeers are the peers that we requested already so we don't want to try again
      */
-    bestPeerForBlockInput(blockInput, excludedPeers) {
+    bestPeerForBlockInput(unavailableBlockInput, excludedPeers) {
+        let cachedData = undefined;
+        if (unavailableBlockInput.block === null) {
+            // NullBlockInput
+            cachedData = unavailableBlockInput.cachedData;
+        }
+        else {
+            // BlockInput
+            if (unavailableBlockInput.type !== BlockInputType.dataPromise) {
+                throw Error(`bestPeerForBlockInput called with BlockInput type ${unavailableBlockInput.type}, expected dataPromise`);
+            }
+            cachedData = unavailableBlockInput.cachedData;
+        }
         const eligiblePeers = [];
-        if (isBlockInputColumns(blockInput)) {
-            const pendingDataColumns = new Set(blockInput.getMissingSampledColumnMeta().missing);
-            // there could be no pending column in case when block is still missing
+        if (cachedData.fork === ForkName.fulu) {
+            // cached data is CachedDataColumns
+            const { dataColumnsCache } = cachedData;
+            const pendingDataColumns = new Set();
+            for (const column of this.custodyConfig.sampledColumns) {
+                if (!dataColumnsCache.has(column)) {
+                    pendingDataColumns.add(column);
+                }
+            }
+            // there could be no pending column in case of NullBlockInput
             eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers));
         }
         else {
             // prefulu
-            eligiblePeers.push(...this.filterPeers(null, excludedPeers));
+            const pendingDataColumns = null;
+            eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers));
         }
         if (eligiblePeers.length === 0) {
             return null;
@@ -719,7 +775,7 @@ export class UnknownBlockPeerBalancer {
                 continue;
             }
             // postfulu, find peers that have custody columns that we need
-            const { custodyColumns: peerColumns } = syncMeta;
+            const { custodyGroups: peerColumns } = syncMeta;
             // check if the peer has all needed columns
             // get match
             const columns = peerColumns.reduce((acc, elem) => {