@lodestar/beacon-node 1.34.0-dev.0c7dfe25ea → 1.34.0-dev.11ca515d52

package/lib/sync/unknownBlock.js

@@ -1,4 +1,4 @@
-import { ForkName, INTERVALS_PER_SLOT, NUMBER_OF_COLUMNS } from "@lodestar/params";
+import { ForkName, ForkSeq, INTERVALS_PER_SLOT } from "@lodestar/params";
 import { fromHex, pruneSetToMax, toRootHex } from "@lodestar/utils";
 import { sleep } from "@lodestar/utils";
 import { BlockInputType } from "../chain/blocks/types.js";
@@ -7,7 +7,9 @@ import { NetworkEvent } from "../network/index.js";
 import { beaconBlocksMaybeBlobsByRoot, unavailableBeaconBlobsByRoot, } from "../network/reqresp/beaconBlocksMaybeBlobsByRoot.js";
 import { byteArrayEquals } from "../util/bytes.js";
 import { shuffle } from "../util/shuffle.js";
+import { sortBy } from "../util/sortBy.js";
 import { wrapError } from "../util/wrapError.js";
+import { MAX_CONCURRENT_REQUESTS } from "./constants.js";
 import { PendingBlockStatus, PendingBlockType } from "./interface.js";
 import { getAllDescendantBlocks, getDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
 const MAX_ATTEMPTS_PER_BLOCK = 5;
@@ -68,6 +70,21 @@ export class UnknownBlockSync {
                 this.logger.debug("Error handling unknownBlockParent event", {}, e);
             }
         };
+        this.onPeerConnected = (data) => {
+            try {
+                const peerId = data.peer;
+                const peerSyncMeta = this.network.getConnectedPeerSyncMeta(peerId);
+                this.peerBalancer.onPeerConnected(data.peer, peerSyncMeta);
+                this.triggerUnknownBlockSearch();
+            }
+            catch (e) {
+                this.logger.debug("Error handling peerConnected event", {}, e);
+            }
+        };
+        this.onPeerDisconnected = (data) => {
+            const peerId = data.peer;
+            this.peerBalancer.onPeerDisconnected(peerId);
+        };
         /**
          * Gather tip parent blocks with unknown parent and do a search for all of them
          */
@@ -105,16 +122,22 @@ export class UnknownBlockSync {
             }
             // most of the time there is exactly 1 unknown block
             for (const block of unknowns) {
-                this.downloadBlock(block, connectedPeers).catch((e) => {
+                this.downloadBlock(block).catch((e) => {
                     this.logger.debug("Unexpected error - downloadBlock", { root: block.blockRootHex }, e);
                 });
             }
         };
         this.maxPendingBlocks = opts?.maxPendingBlocks ?? MAX_PENDING_BLOCKS;
         this.proposerBoostSecWindow = this.config.SECONDS_PER_SLOT / INTERVALS_PER_SLOT;
+        this.peerBalancer = new UnknownBlockPeerBalancer(this.network.custodyConfig);
         if (metrics) {
-            metrics.syncUnknownBlock.pendingBlocks.addCollect(() => metrics.syncUnknownBlock.pendingBlocks.set(this.pendingBlocks.size));
-            metrics.syncUnknownBlock.knownBadBlocks.addCollect(() => metrics.syncUnknownBlock.knownBadBlocks.set(this.knownBadBlocks.size));
+            metrics.syncUnknownBlock.pendingBlocks.addCollect(() => {
+                metrics.syncUnknownBlock.pendingBlocks.set(this.pendingBlocks.size);
+                metrics.syncUnknownBlock.knownBadBlocks.set(this.knownBadBlocks.size);
+                metrics.syncUnknownBlock.peerBalancer.peersMetaCount.set(this.peerBalancer.peersMeta.size);
+                metrics.syncUnknownBlock.peerBalancer.peersActiveRequestCount.set(this.peerBalancer.activeRequests.size);
+                metrics.syncUnknownBlock.peerBalancer.totalActiveRequests.set(this.peerBalancer.getTotalActiveRequests());
+            });
         }
     }
     subscribeToNetwork() {
@@ -125,7 +148,8 @@ export class UnknownBlockSync {
                 this.network.events.on(NetworkEvent.unknownBlock, this.onUnknownBlock);
                 this.network.events.on(NetworkEvent.unknownBlockInput, this.onUnknownBlockInput);
                 this.network.events.on(NetworkEvent.unknownBlockParent, this.onUnknownParent);
-                this.network.events.on(NetworkEvent.peerConnected, this.triggerUnknownBlockSearch);
+                this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
+                this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
                 this.subscribedToNetworkEvents = true;
             }
         }
@@ -138,7 +162,8 @@ export class UnknownBlockSync {
         this.network.events.off(NetworkEvent.unknownBlock, this.onUnknownBlock);
         this.network.events.off(NetworkEvent.unknownBlockInput, this.onUnknownBlockInput);
         this.network.events.off(NetworkEvent.unknownBlockParent, this.onUnknownParent);
-        this.network.events.off(NetworkEvent.peerConnected, this.triggerUnknownBlockSearch);
+        this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
+        this.network.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
         this.subscribedToNetworkEvents = false;
     }
     close() {
@@ -150,7 +175,7 @@ export class UnknownBlockSync {
     }
     /**
      * When a blockInput comes with  an unknown parent:
-     * - add the block to pendingBlocks with status downloaded, blockRootHex as key. This is similar to
+     * - add the block to pendingBlocks with status downloaded or pending blockRootHex as key. This is similar to
      * an `onUnknownBlock` event, but the blocks is downloaded.
      * - add the parent root to pendingBlocks with status pending, parentBlockRootHex as key. This is
      * the same to an `onUnknownBlock` event with parentBlockRootHex as root.
@@ -163,14 +188,26 @@ export class UnknownBlockSync {
         // add 1 pending block with status downloaded
         let pendingBlock = this.pendingBlocks.get(blockRootHex);
         if (!pendingBlock) {
-            pendingBlock = {
-                blockRootHex,
-                parentBlockRootHex,
-                blockInput,
-                peerIdStrs: new Set(),
-                status: PendingBlockStatus.downloaded,
-                downloadAttempts: 0,
-            };
+            pendingBlock =
+                blockInput.type === BlockInputType.dataPromise
+                    ? {
+                        unknownBlockType: PendingBlockType.UNKNOWN_DATA,
+                        blockRootHex,
+                        // this will be set after we download block
+                        parentBlockRootHex: null,
+                        blockInput,
+                        peerIdStrs: new Set(),
+                        status: PendingBlockStatus.pending,
+                        downloadAttempts: 0,
+                    }
+                    : {
+                        blockRootHex,
+                        parentBlockRootHex,
+                        blockInput,
+                        peerIdStrs: new Set(),
+                        status: PendingBlockStatus.downloaded,
+                        downloadAttempts: 0,
+                    };
             this.pendingBlocks.set(blockRootHex, pendingBlock);
             this.logger.verbose("Added unknown block parent to pendingBlocks", {
                 root: blockRootHex,
@@ -207,6 +244,7 @@ export class UnknownBlockSync {
             pendingBlock = {
                 unknownBlockType,
                 blockRootHex,
+                // this will be set after we download block
                 parentBlockRootHex: null,
                 blockInput,
                 peerIdStrs: new Set(),
@@ -230,7 +268,7 @@ export class UnknownBlockSync {
         }
         return unknownBlockType;
     }
-    async downloadBlock(block, allPeers) {
+    async downloadBlock(block) {
         if (block.status !== PendingBlockStatus.pending) {
             return;
         }
@@ -244,57 +282,12 @@ export class UnknownBlockSync {
         this.logger.verbose("Downloading unknown block", logCtx);
         block.status = PendingBlockStatus.fetching;
         let res;
-        let connectedPeers;
         if (block.blockInput === null) {
-            connectedPeers = allPeers;
             // we only have block root, and nothing else
-            res = await wrapError(this.fetchUnknownBlockRoot(fromHex(block.blockRootHex), connectedPeers));
+            res = await wrapError(this.fetchUnknownBlockRoot(fromHex(block.blockRootHex)));
         }
         else {
-            const { cachedData } = block.blockInput;
-            if (cachedData.fork === ForkName.fulu) {
-                const { dataColumnsCache } = cachedData;
-                const sampledColumns = this.network.custodyConfig.sampledColumns;
-                const neededColumns = sampledColumns.reduce((acc, elem) => {
-                    if (dataColumnsCache.get(elem) === undefined) {
-                        acc.push(elem);
-                    }
-                    return acc;
-                }, []);
-                connectedPeers =
-                    neededColumns.length <= 0
-                        ? allPeers
-                        : allPeers.filter((peer) => {
-                            const { custodyGroups: peerColumns } = this.network.getConnectedPeerSyncMeta(peer);
-                            const columns = peerColumns.reduce((acc, elem) => {
-                                if (neededColumns.includes(elem)) {
-                                    acc.push(elem);
-                                }
-                                return acc;
-                            }, []);
-                            return columns.length > 0;
-                        });
-                if (connectedPeers.length > 0) {
-                    this.logger.debug("Filtered peers to those having relevant columns for downloading data", {
-                        ...logCtx,
-                        allPeers: allPeers.length,
-                        connectedPeers: connectedPeers.length,
-                    });
-                }
-                else {
-                    this.logger.debug("Skipping download as no filtered peers having relevant data", {
-                        ...logCtx,
-                        allPeers: allPeers.length,
-                        connectedPeers: connectedPeers.length,
-                        neededColumns: neededColumns.join(" "),
-                    });
-                    return;
-                }
-            }
-            else {
-                connectedPeers = allPeers;
-            }
-            res = await wrapError(this.fetchUnavailableBlockInput(block.blockInput, connectedPeers));
+            res = await wrapError(this.fetchUnavailableBlockInput(block.blockInput));
         }
         if (res.err)
             this.metrics?.syncUnknownBlock.downloadedBlocksError.inc();
@@ -302,77 +295,57 @@ export class UnknownBlockSync {
             this.metrics?.syncUnknownBlock.downloadedBlocksSuccess.inc();
         if (!res.err) {
             const { blockInput, peerIdStr } = res.result;
+            // fetchUnknownBlockRoot and fetchUnavailableBlockInput should return available data BlockInput, throw error if not
             if (blockInput.type === BlockInputType.dataPromise) {
                 // if there were any peers who would have had the missing datacolumns, it would have resulted in err
-                block = {
-                    ...block,
-                    blockInput,
-                    unknownBlockType: PendingBlockType.UNKNOWN_DATA,
-                };
-                block.blockInput = blockInput;
-                this.pendingBlocks.set(block.blockRootHex, block);
-                block.status = PendingBlockStatus.pending;
-                // parentSlot > finalizedSlot, continue downloading parent of parent
-                block.downloadAttempts += this.config.CUSTODY_REQUIREMENT / NUMBER_OF_COLUMNS;
-                const errorData = { root: block.blockRootHex, attempts: block.downloadAttempts, unknownBlockType };
-                if (block.downloadAttempts > MAX_ATTEMPTS_PER_BLOCK) {
-                    // Give up on this block and assume it does not exist, penalizing all peers as if it was a bad block
-                    this.logger.debug("Ignoring unknown block after many failed downloads", errorData);
-                    this.removeAndDownscoreAllDescendants(block);
-                }
-                else {
-                    // Try again when a new peer connects, its status changes, or a new unknownBlockParent event happens
-                    this.logger.debug("Error downloading full unknown block", errorData);
-                }
+                throw Error(`Expected BlockInput to be available, got dataPromise for ${block.blockRootHex}`);
             }
-            else {
-                block = {
-                    ...block,
-                    status: PendingBlockStatus.downloaded,
-                    blockInput,
-                    parentBlockRootHex: toRootHex(blockInput.block.message.parentRoot),
-                };
-                this.pendingBlocks.set(block.blockRootHex, block);
-                const blockSlot = blockInput.block.message.slot;
-                const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
-                const delaySec = Date.now() / 1000 - (this.chain.genesisTime + blockSlot * this.config.SECONDS_PER_SLOT);
-                this.metrics?.syncUnknownBlock.elapsedTimeTillReceived.observe(delaySec);
-                const parentInForkchoice = this.chain.forkChoice.hasBlock(blockInput.block.message.parentRoot);
-                this.logger.verbose("Downloaded unknown block", {
-                    root: block.blockRootHex,
-                    pendingBlocks: this.pendingBlocks.size,
-                    parentInForkchoice,
-                    blockInputType: blockInput.type,
+            block = {
+                ...block,
+                status: PendingBlockStatus.downloaded,
+                blockInput,
+                parentBlockRootHex: toRootHex(blockInput.block.message.parentRoot),
+            };
+            this.pendingBlocks.set(block.blockRootHex, block);
+            const blockSlot = blockInput.block.message.slot;
+            const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
+            const delaySec = Date.now() / 1000 - (this.chain.genesisTime + blockSlot * this.config.SECONDS_PER_SLOT);
+            this.metrics?.syncUnknownBlock.elapsedTimeTillReceived.observe(delaySec);
+            const parentInForkchoice = this.chain.forkChoice.hasBlock(blockInput.block.message.parentRoot);
+            this.logger.verbose("Downloaded unknown block", {
+                root: block.blockRootHex,
+                pendingBlocks: this.pendingBlocks.size,
+                parentInForkchoice,
+                blockInputType: blockInput.type,
+                unknownBlockType,
+            });
+            if (parentInForkchoice) {
+                // Bingo! Process block. Add to pending blocks anyway for recycle the cache that prevents duplicate processing
+                this.processBlock(block).catch((e) => {
+                    this.logger.debug("Unexpected error - process newly downloaded block", {}, e);
+                });
+            }
+            else if (blockSlot <= finalizedSlot) {
+                // the common ancestor of the downloading chain and canonical chain should be at least the finalized slot and
+                // we should found it through forkchoice. If not, we should penalize all peers sending us this block chain
+                // 0 - 1 - ... - n - finalizedSlot
+                //                \
+                //                parent 1 - parent 2 - ... - unknownParent block
+                const blockRoot = this.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(blockInput.block.message);
+                this.logger.debug("Downloaded block is before finalized slot", {
+                    finalizedSlot,
+                    blockSlot,
+                    parentRoot: toRootHex(blockRoot),
                     unknownBlockType,
                 });
-                if (parentInForkchoice) {
-                    // Bingo! Process block. Add to pending blocks anyway for recycle the cache that prevents duplicate processing
-                    this.processBlock(block).catch((e) => {
-                        this.logger.debug("Unexpected error - process newly downloaded block", {}, e);
-                    });
-                }
-                else if (blockSlot <= finalizedSlot) {
-                    // the common ancestor of the downloading chain and canonical chain should be at least the finalized slot and
-                    // we should found it through forkchoice. If not, we should penalize all peers sending us this block chain
-                    // 0 - 1 - ... - n - finalizedSlot
-                    //                \
-                    //                parent 1 - parent 2 - ... - unknownParent block
-                    const blockRoot = this.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(blockInput.block.message);
-                    this.logger.debug("Downloaded block is before finalized slot", {
-                        finalizedSlot,
-                        blockSlot,
-                        parentRoot: toRootHex(blockRoot),
-                        unknownBlockType,
-                    });
-                    this.removeAndDownscoreAllDescendants(block);
-                }
-                else {
-                    this.onUnknownParent({ blockInput, peer: peerIdStr });
-                }
+                this.removeAndDownscoreAllDescendants(block);
+            }
+            else {
+                this.onUnknownParent({ blockInput, peer: peerIdStr });
             }
         }
         else {
-            // this allows to retry the download of the block
+            // block download has error, this allows to retry the download of the block
             block.status = PendingBlockStatus.pending;
             // parentSlot > finalizedSlot, continue downloading parent of parent
             block.downloadAttempts++;
@@ -391,9 +364,21 @@ export class UnknownBlockSync {
     /**
      * Send block to the processor awaiting completition. If processed successfully, send all children to the processor.
      * On error, remove and downscore all descendants.
+     * This function could run recursively for all descendant blocks
      */
     async processBlock(pendingBlock) {
+        // pending block status is `downloaded` right after `downloadBlock`
+        // but could be `pending` if added by `onUnknownBlockParent` event and this function is called recursively
         if (pendingBlock.status !== PendingBlockStatus.downloaded) {
+            if (pendingBlock.status === PendingBlockStatus.pending) {
+                const connectedPeers = this.network.getConnectedPeers();
+                if (connectedPeers.length === 0) {
+                    this.logger.debug("No connected peers, skipping download block", { blockRoot: pendingBlock.blockRootHex });
+                    return;
+                }
+                // if the download is a success we'll call `processBlock()` for this block
+                await this.downloadBlock(pendingBlock);
+            }
             return;
         }
         pendingBlock.status = PendingBlockStatus.processing;
@@ -480,44 +465,27 @@ export class UnknownBlockSync {
      *   - from deneb, fetch all missing blobs
      *   - from peerDAS, fetch sampled colmns
      * TODO: this means we only have block root, and nothing else. Consider to reflect this in the function name
-     * Will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers if connectPeers.length > MAX_ATTEMPTS_PER_BLOCK.
+     * prefulu, will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers, postfulu we may attempt more as defined in `getMaxDownloadAttempts()` function
      * Also verifies the received block root + returns the peer that provided the block for future downscoring.
      */
-    async fetchUnknownBlockRoot(blockRoot, connectedPeers) {
-        const shuffledPeers = shuffle(connectedPeers);
+    async fetchUnknownBlockRoot(blockRoot) {
         const blockRootHex = toRootHex(blockRoot);
-        let lastError = null;
+        const excludedPeers = new Set();
         let partialDownload = null;
-        let fetchedPeerId = null;
-        for (let i = 0; i < MAX_ATTEMPTS_PER_BLOCK; i++) {
-            const peerId = shuffledPeers[i % shuffledPeers.length];
-            const { custodyGroups: peerColumns, client: peerClient } = this.network.getConnectedPeerSyncMeta(peerId);
-            if (partialDownload !== null) {
-                const [prevBlockInput] = partialDownload.blocks;
-                if (prevBlockInput === undefined || prevBlockInput.type !== BlockInputType.dataPromise) {
-                    throw Error(`prevBlockInput=${prevBlockInput?.type} in partialDownload`);
-                }
-                const { cachedData } = prevBlockInput;
-                if (cachedData.fork === ForkName.fulu) {
-                    const { dataColumnsCache } = cachedData;
-                    const sampledColumns = this.network.custodyConfig.sampledColumns;
-                    const neededColumns = sampledColumns.reduce((acc, elem) => {
-                        if (dataColumnsCache.get(elem) === undefined) {
-                            acc.push(elem);
-                        }
-                        return acc;
-                    }, []);
-                    const columns = peerColumns.reduce((acc, elem) => {
-                        if (neededColumns.includes(elem)) {
-                            acc.push(elem);
-                        }
-                        return acc;
-                    }, []);
-                    if (columns.length === 0) {
-                        continue;
-                    }
-                }
+        const defaultPendingColumns = this.config.getForkSeq(this.chain.clock.currentSlot) >= ForkSeq.fulu
+            ? new Set(this.network.custodyConfig.sampleGroups)
+            : null;
+        let lastError = null;
+        let i = 0;
+        while (i++ < this.getMaxDownloadAttempts()) {
+            // pendingDataColumns is null prefulu
+            const peer = this.peerBalancer.bestPeerForPendingColumns(partialDownload ? new Set(partialDownload.pendingDataColumns) : defaultPendingColumns, excludedPeers);
+            if (peer === null) {
+                // no more peer with needed columns to try, throw error
+                throw Error(`Error fetching UnknownBlockRoot after ${i}: cannot find peer with needed columns ${partialDownload?.pendingDataColumns.join(", ")}`);
             }
+            const { peerId, client: peerClient } = peer;
+            excludedPeers.add(peerId);
             try {
                 const { blocks: [blockInput], pendingDataColumns, } = await beaconBlocksMaybeBlobsByRoot(this.config, this.network, peerId, [blockRoot], partialDownload, peerClient, this.metrics, this.logger);
                 // Peer does not have the block, try with next peer
@@ -526,10 +494,9 @@ export class UnknownBlockSync {
                 }
                 if (pendingDataColumns !== null) {
                     partialDownload = { blocks: [blockInput], pendingDataColumns };
-                    fetchedPeerId = peerId;
                     continue;
                 }
-                // Verify block root is correct
+                // data is available, verify block root is correct
                 const block = blockInput.block.message;
                 const receivedBlockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block);
                 if (!byteArrayEquals(receivedBlockRoot, blockRoot)) {
@@ -541,29 +508,26 @@ export class UnknownBlockSync {
                 this.logger.debug("Error fetching UnknownBlockRoot", { attempt: i, blockRootHex, peer: peerId }, e);
                 lastError = e;
             }
+            finally {
+                this.peerBalancer.onRequestCompleted(peerId);
+            }
         }
         if (lastError) {
-            lastError.message = `Error fetching UnknownBlockRoot after ${MAX_ATTEMPTS_PER_BLOCK} attempts: ${lastError.message}`;
+            lastError.message = `Error fetching UnknownBlockRoot after ${i} attempts: ${lastError.message}`;
             throw lastError;
         }
-        if (partialDownload !== null && fetchedPeerId !== null) {
-            const { blocks: [blockInput], } = partialDownload;
-            return { blockInput, peerIdStr: fetchedPeerId };
-        }
-        throw Error(`Error fetching UnknownBlockRoot after ${MAX_ATTEMPTS_PER_BLOCK}: unknown error because either partialDownload is null=${partialDownload === null} or fetchedPeerId is null=${fetchedPeerId === null} `);
+        throw Error(`Error fetching UnknownBlockRoot after ${i}: cannot download all blobs or data columns for block ${blockRootHex}`);
     }
     /**
      * We have partial block input:
      * - we have block but not have all blobs (deneb) or needed columns (fulu)
      * - we don't have block and have some blobs (deneb) or some columns (fulu)
-     * Fetches missing blobs for the blockinput, in future can also pull block is thats also missing
-     * along with the blobs (i.e. only some blobs are available)
+     * Fetches missing block/data columns/block for the blockinput. This function returns either preData or availableData BlockInput.
      */
-    async fetchUnavailableBlockInput(unavailableBlockInput, connectedPeers) {
+    async fetchUnavailableBlockInput(unavailableBlockInput) {
         if (unavailableBlockInput.block !== null && unavailableBlockInput.type !== BlockInputType.dataPromise) {
             return { blockInput: unavailableBlockInput, peerIdStr: "" };
         }
-        const shuffledPeers = shuffle(connectedPeers);
         let blockRootHex;
         let blobKzgCommitmentsLen;
         let blockRoot;
@@ -591,30 +555,16 @@ export class UnknownBlockSync {
             }
         }
         let lastError = null;
-        for (let i = 0; i < MAX_ATTEMPTS_PER_BLOCK; i++) {
-            const peerId = shuffledPeers[i % shuffledPeers.length];
-            const { custodyGroups: peerColumns, client: peerClient } = this.network.getConnectedPeerSyncMeta(peerId);
-            if (unavailableBlockInput.block !== null) {
-                const { cachedData } = unavailableBlockInput;
-                if (cachedData.fork === ForkName.fulu) {
-                    const { dataColumnsCache } = cachedData;
-                    const neededColumns = sampledColumns.reduce((acc, elem) => {
-                        if (dataColumnsCache.get(elem) === undefined) {
-                            acc.push(elem);
-                        }
-                        return acc;
-                    }, []);
-                    const columns = peerColumns.reduce((acc, elem) => {
-                        if (neededColumns.includes(elem)) {
-                            acc.push(elem);
-                        }
-                        return acc;
-                    }, []);
-                    if (columns.length === 0) {
-                        continue;
-                    }
-                }
+        let i = 0;
+        const excludedPeers = new Set();
+        while (i++ < this.getMaxDownloadAttempts()) {
+            const bestPeer = this.peerBalancer.bestPeerForBlockInput(unavailableBlockInput, excludedPeers);
+            if (bestPeer === null) {
+                // no more peer to try, throw error
+                throw Error(`Error fetching UnavailableBlockInput after ${i}: cannot find peer with needed columns ${sampledColumns.join(", ")}`);
             }
+            const { peerId, client: peerClient } = bestPeer;
+            excludedPeers.add(peerId);
             try {
                 const blockInput = await unavailableBeaconBlobsByRoot(this.config, this.network, peerId, peerClient, unavailableBlockInput, {
                     metrics: this.metrics,
@@ -624,16 +574,12 @@ export class UnknownBlockSync {
                     blockInputsRetryTrackerCache: this.blockInputsRetryTrackerCache,
                     engineGetBlobsCache: this.engineGetBlobsCache,
                 });
-                // Peer does not have the block, try with next peer
-                if (blockInput === undefined) {
-                    continue;
-                }
                 if (unavailableBlockInput.block !== null && blockInput.type === BlockInputType.dataPromise) {
                     // all datacolumns were not downloaded we can continue with other peers
                     // as unavailableBlockInput.block's dataColumnsCache would be updated
                     continue;
                 }
-                // Verify block root is correct
+                // data is available, verify block root is correct
                 const block = blockInput.block.message;
                 const receivedBlockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block);
                 if (!byteArrayEquals(receivedBlockRoot, blockRoot)) {
@@ -651,12 +597,15 @@ export class UnknownBlockSync {
                 this.logger.debug("Error fetching UnavailableBlockInput", { attempt: i, blockRootHex, peer: peerId }, e);
                 lastError = e;
             }
+            finally {
+                this.peerBalancer.onRequestCompleted(peerId);
+            }
         }
         if (lastError) {
-            lastError.message = `Error fetching UnavailableBlockInput after ${MAX_ATTEMPTS_PER_BLOCK} attempts: ${lastError.message}`;
+            lastError.message = `Error fetching UnavailableBlockInput after ${i} attempts: ${lastError.message}`;
             throw lastError;
         }
-        throw Error(`Error fetching UnavailableBlockInput after ${MAX_ATTEMPTS_PER_BLOCK}: unknown error`);
+        throw Error(`Error fetching UnavailableBlockInput after ${i}: unknown error`);
     }
     /**
      * Gets all descendant blocks of `block` recursively from `pendingBlocks`.
@@ -695,5 +644,160 @@ export class UnknownBlockSync {
         }
         return badPendingBlocks;
     }
+    getMaxDownloadAttempts() {
+        if (this.config.getForkSeq(this.chain.clock.currentSlot) < ForkSeq.fulu) {
+            return MAX_ATTEMPTS_PER_BLOCK;
+        }
+        // TODO: I consider max 20 downloads per block for a supernode is enough for devnets
+        // review this computation for public testnets or mainnet
+        return Math.min(20, (MAX_ATTEMPTS_PER_BLOCK * this.network.custodyConfig.sampleGroups.length) / this.config.SAMPLES_PER_SLOT);
+    }
+}
+/**
+ * Class to track active byRoots requests and balance them across eligible peers.
+ */
+export class UnknownBlockPeerBalancer {
+    constructor(custodyConfig) {
+        this.peersMeta = new Map();
+        this.activeRequests = new Map();
+        this.custodyConfig = custodyConfig;
+    }
+    /** Trigger on each peer re-status */
+    onPeerConnected(peerId, syncMeta) {
+        this.peersMeta.set(peerId, syncMeta);
+        if (!this.activeRequests.has(peerId)) {
+            this.activeRequests.set(peerId, 0);
+        }
+    }
+    onPeerDisconnected(peerId) {
+        this.peersMeta.delete(peerId);
+        this.activeRequests.delete(peerId);
+    }
+    /**
+     * called from fetchUnknownBlockRoot() where we only have block root and nothing else
+     * excludedPeers are the peers that we requested already so we don't want to try again
+     * pendingColumns is empty for prefulu, or the 1st time we we download a block by root
+     */
+    bestPeerForPendingColumns(pendingColumns, excludedPeers) {
+        const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers);
+        if (eligiblePeers.length === 0) {
+            return null;
+        }
+        const sortedEligiblePeers = sortBy(shuffle(eligiblePeers), 
+        // prefer peers with least active req
+        (peerId) => this.activeRequests.get(peerId) ?? 0);
+        const bestPeerId = sortedEligiblePeers[0];
+        this.onRequest(bestPeerId);
+        return this.peersMeta.get(bestPeerId) ?? null;
+    }
+    /**
+     * called from fetchUnavailableBlockInput() where we have either BlockInput or NullBlockInput
+     * excludedPeers are the peers that we requested already so we don't want to try again
+     */
+    bestPeerForBlockInput(unavailableBlockInput, excludedPeers) {
+        let cachedData = undefined;
+        if (unavailableBlockInput.block === null) {
+            // NullBlockInput
+            cachedData = unavailableBlockInput.cachedData;
+        }
+        else {
+            // BlockInput
+            if (unavailableBlockInput.type !== BlockInputType.dataPromise) {
+                throw Error(`bestPeerForBlockInput called with BlockInput type ${unavailableBlockInput.type}, expected dataPromise`);
+            }
+            cachedData = unavailableBlockInput.cachedData;
+        }
+        const eligiblePeers = [];
+        if (cachedData.fork === ForkName.fulu) {
+            // cached data is CachedDataColumns
+            const { dataColumnsCache } = cachedData;
+            const pendingDataColumns = new Set();
+            for (const column of this.custodyConfig.sampledColumns) {
+                if (!dataColumnsCache.has(column)) {
+                    pendingDataColumns.add(column);
+                }
+            }
+            // there could be no pending column in case of NullBlockInput
+            eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers));
+        }
+        else {
+            // prefulu
+            const pendingDataColumns = null;
+            eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers));
+        }
+        if (eligiblePeers.length === 0) {
+            return null;
+        }
+        const sortedEligiblePeers = sortBy(shuffle(eligiblePeers), 
+        // prefer peers with least active req
+        (peerId) => this.activeRequests.get(peerId) ?? 0);
+        const bestPeerId = sortedEligiblePeers[0];
+        this.onRequest(bestPeerId);
+        return this.peersMeta.get(bestPeerId) ?? null;
+    }
+    /**
+     * Consumers don't need to call this method directly, it is called internally by bestPeer*() methods
+     * make this public for testing
+     */
+    onRequest(peerId) {
+        this.activeRequests.set(peerId, (this.activeRequests.get(peerId) ?? 0) + 1);
+    }
+    /**
+     * Consumers should call this method when a request is completed for a peer.
+     */
+    onRequestCompleted(peerId) {
+        this.activeRequests.set(peerId, Math.max(0, (this.activeRequests.get(peerId) ?? 1) - 1));
+    }
+    getTotalActiveRequests() {
+        let totalActiveRequests = 0;
+        for (const count of this.activeRequests.values()) {
+            totalActiveRequests += count;
+        }
+        return totalActiveRequests;
+    }
+    // pendingDataColumns could be null for prefulu
+    filterPeers(pendingDataColumns, excludedPeers) {
+        let maxColumnCount = 0;
+        const considerPeers = [];
+        for (const [peerId, syncMeta] of this.peersMeta.entries()) {
+            if (excludedPeers.has(peerId)) {
+                // made request to this peer already
+                continue;
+            }
+            const activeRequests = this.activeRequests.get(peerId) ?? 0;
+            if (activeRequests >= MAX_CONCURRENT_REQUESTS) {
+                // should return peer with no more than MAX_CONCURRENT_REQUESTS active requests
+                continue;
+            }
+            if (pendingDataColumns === null || pendingDataColumns.size === 0) {
+                // prefulu, no pending columns
+                considerPeers.push({ peerId, columnCount: 0 });
+                continue;
+            }
+            // postfulu, find peers that have custody columns that we need
+            const { custodyGroups: peerColumns } = syncMeta;
+            // check if the peer has all needed columns
+            // get match
+            const columns = peerColumns.reduce((acc, elem) => {
+                if (pendingDataColumns.has(elem)) {
+                    acc.push(elem);
+                }
+                return acc;
+            }, []);
+            if (columns.length > 0) {
+                if (columns.length > maxColumnCount) {
+                    maxColumnCount = columns.length;
+                }
+                considerPeers.push({ peerId, columnCount: columns.length });
+            }
+        } // end for
+        const eligiblePeers = [];
+        for (const { peerId, columnCount } of considerPeers) {
+            if (columnCount === maxColumnCount) {
+                eligiblePeers.push(peerId);
+            }
+        }
+        return eligiblePeers;
+    }
 }
 //# sourceMappingURL=unknownBlock.js.map