@lodestar/beacon-node 1.34.0-dev.d5a4e7a09c → 1.34.0-dev.d9fc7bb103

package/lib/sync/unknownBlock.js

@@ -1,13 +1,15 @@
-import { INTERVALS_PER_SLOT } from "@lodestar/params";
+import { ForkName, ForkSeq, INTERVALS_PER_SLOT } from "@lodestar/params";
 import { fromHex, pruneSetToMax, toRootHex } from "@lodestar/utils";
 import { sleep } from "@lodestar/utils";
 import { BlockInputType } from "../chain/blocks/types.js";
 import { BlockError, BlockErrorCode } from "../chain/errors/index.js";
-import { NetworkEvent, PeerAction } from "../network/index.js";
+import { NetworkEvent } from "../network/index.js";
 import { beaconBlocksMaybeBlobsByRoot, unavailableBeaconBlobsByRoot, } from "../network/reqresp/beaconBlocksMaybeBlobsByRoot.js";
 import { byteArrayEquals } from "../util/bytes.js";
 import { shuffle } from "../util/shuffle.js";
+import { sortBy } from "../util/sortBy.js";
 import { wrapError } from "../util/wrapError.js";
+import { MAX_CONCURRENT_REQUESTS } from "./constants.js";
 import { PendingBlockStatus, PendingBlockType } from "./interface.js";
 import { getAllDescendantBlocks, getDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
 const MAX_ATTEMPTS_PER_BLOCK = 5;
@@ -68,6 +70,21 @@ export class UnknownBlockSync {
                 this.logger.debug("Error handling unknownBlockParent event", {}, e);
             }
         };
+        this.onPeerConnected = (data) => {
+            try {
+                const peerId = data.peer;
+                const peerSyncMeta = this.network.getConnectedPeerSyncMeta(peerId);
+                this.peerBalancer.onPeerConnected(data.peer, peerSyncMeta);
+                this.triggerUnknownBlockSearch();
+            }
+            catch (e) {
+                this.logger.debug("Error handling peerConnected event", {}, e);
+            }
+        };
+        this.onPeerDisconnected = (data) => {
+            const peerId = data.peer;
+            this.peerBalancer.onPeerDisconnected(peerId);
+        };
         /**
          * Gather tip parent blocks with unknown parent and do a search for all of them
          */
@@ -105,16 +122,22 @@ export class UnknownBlockSync {
             }
             // most of the time there is exactly 1 unknown block
             for (const block of unknowns) {
-                this.downloadBlock(block, connectedPeers).catch((e) => {
+                this.downloadBlock(block).catch((e) => {
                     this.logger.debug("Unexpected error - downloadBlock", { root: block.blockRootHex }, e);
                 });
             }
         };
         this.maxPendingBlocks = opts?.maxPendingBlocks ?? MAX_PENDING_BLOCKS;
         this.proposerBoostSecWindow = this.config.SECONDS_PER_SLOT / INTERVALS_PER_SLOT;
+        this.peerBalancer = new UnknownBlockPeerBalancer(this.network.custodyConfig);
         if (metrics) {
-            metrics.syncUnknownBlock.pendingBlocks.addCollect(() => metrics.syncUnknownBlock.pendingBlocks.set(this.pendingBlocks.size));
-            metrics.syncUnknownBlock.knownBadBlocks.addCollect(() => metrics.syncUnknownBlock.knownBadBlocks.set(this.knownBadBlocks.size));
+            metrics.syncUnknownBlock.pendingBlocks.addCollect(() => {
+                metrics.syncUnknownBlock.pendingBlocks.set(this.pendingBlocks.size);
+                metrics.syncUnknownBlock.knownBadBlocks.set(this.knownBadBlocks.size);
+                metrics.syncUnknownBlock.peerBalancer.peersMetaCount.set(this.peerBalancer.peersMeta.size);
+                metrics.syncUnknownBlock.peerBalancer.peersActiveRequestCount.set(this.peerBalancer.activeRequests.size);
+                metrics.syncUnknownBlock.peerBalancer.totalActiveRequests.set(this.peerBalancer.getTotalActiveRequests());
+            });
         }
     }
     subscribeToNetwork() {
@@ -125,7 +148,8 @@ export class UnknownBlockSync {
                 this.network.events.on(NetworkEvent.unknownBlock, this.onUnknownBlock);
                 this.network.events.on(NetworkEvent.unknownBlockInput, this.onUnknownBlockInput);
                 this.network.events.on(NetworkEvent.unknownBlockParent, this.onUnknownParent);
-                this.network.events.on(NetworkEvent.peerConnected, this.triggerUnknownBlockSearch);
+                this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
+                this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
                 this.subscribedToNetworkEvents = true;
             }
         }
@@ -138,7 +162,8 @@ export class UnknownBlockSync {
         this.network.events.off(NetworkEvent.unknownBlock, this.onUnknownBlock);
         this.network.events.off(NetworkEvent.unknownBlockInput, this.onUnknownBlockInput);
         this.network.events.off(NetworkEvent.unknownBlockParent, this.onUnknownParent);
-        this.network.events.off(NetworkEvent.peerConnected, this.triggerUnknownBlockSearch);
+        this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
+        this.network.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
         this.subscribedToNetworkEvents = false;
     }
     close() {
@@ -150,7 +175,7 @@ export class UnknownBlockSync {
     }
     /**
      * When a blockInput comes with  an unknown parent:
-     * - add the block to pendingBlocks with status downloaded, blockRootHex as key. This is similar to
+     * - add the block to pendingBlocks with status downloaded or pending blockRootHex as key. This is similar to
      * an `onUnknownBlock` event, but the blocks is downloaded.
      * - add the parent root to pendingBlocks with status pending, parentBlockRootHex as key. This is
      * the same to an `onUnknownBlock` event with parentBlockRootHex as root.
@@ -163,14 +188,26 @@ export class UnknownBlockSync {
         // add 1 pending block with status downloaded
         let pendingBlock = this.pendingBlocks.get(blockRootHex);
         if (!pendingBlock) {
-            pendingBlock = {
-                blockRootHex,
-                parentBlockRootHex,
-                blockInput,
-                peerIdStrs: new Set(),
-                status: PendingBlockStatus.downloaded,
-                downloadAttempts: 0,
-            };
+            pendingBlock =
+                blockInput.type === BlockInputType.dataPromise
+                    ? {
+                        unknownBlockType: PendingBlockType.UNKNOWN_DATA,
+                        blockRootHex,
+                        // this will be set after we download block
+                        parentBlockRootHex: null,
+                        blockInput,
+                        peerIdStrs: new Set(),
+                        status: PendingBlockStatus.pending,
+                        downloadAttempts: 0,
+                    }
+                    : {
+                        blockRootHex,
+                        parentBlockRootHex,
+                        blockInput,
+                        peerIdStrs: new Set(),
+                        status: PendingBlockStatus.downloaded,
+                        downloadAttempts: 0,
+                    };
             this.pendingBlocks.set(blockRootHex, pendingBlock);
             this.logger.verbose("Added unknown block parent to pendingBlocks", {
                 root: blockRootHex,
@@ -194,7 +231,7 @@ export class UnknownBlockSync {
             if (blockInputOrRootHex.block !== null) {
                 const { block } = blockInputOrRootHex;
                 blockRootHex = toRootHex(this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message));
-                unknownBlockType = PendingBlockType.UNKNOWN_BLOBS;
+                unknownBlockType = PendingBlockType.UNKNOWN_DATA;
             }
             else {
                 unknownBlockType = PendingBlockType.UNKNOWN_BLOCKINPUT;
@@ -207,6 +244,7 @@ export class UnknownBlockSync {
             pendingBlock = {
                 unknownBlockType,
                 blockRootHex,
+                // this will be set after we download block
                 parentBlockRootHex: null,
                 blockInput,
                 peerIdStrs: new Set(),
@@ -230,24 +268,26 @@ export class UnknownBlockSync {
         }
         return unknownBlockType;
     }
-    async downloadBlock(block, connectedPeers) {
+    async downloadBlock(block) {
         if (block.status !== PendingBlockStatus.pending) {
             return;
         }
         const unknownBlockType = block.unknownBlockType;
-        this.logger.verbose("Downloading unknown block", {
+        const logCtx = {
             root: block.blockRootHex,
             pendingBlocks: this.pendingBlocks.size,
             slot: block.blockInput?.block?.message.slot ?? "unknown",
             unknownBlockType,
-        });
+        };
+        this.logger.verbose("Downloading unknown block", logCtx);
         block.status = PendingBlockStatus.fetching;
         let res;
         if (block.blockInput === null) {
-            res = await wrapError(this.fetchUnknownBlockRoot(fromHex(block.blockRootHex), connectedPeers));
+            // we only have block root, and nothing else
+            res = await wrapError(this.fetchUnknownBlockRoot(fromHex(block.blockRootHex)));
         }
         else {
-            res = await wrapError(this.fetchUnavailableBlockInput(block.blockInput, connectedPeers));
+            res = await wrapError(this.fetchUnavailableBlockInput(block.blockInput));
         }
         if (res.err)
             this.metrics?.syncUnknownBlock.downloadedBlocksError.inc();
@@ -255,6 +295,11 @@ export class UnknownBlockSync {
             this.metrics?.syncUnknownBlock.downloadedBlocksSuccess.inc();
         if (!res.err) {
             const { blockInput, peerIdStr } = res.result;
+            // fetchUnknownBlockRoot and fetchUnavailableBlockInput should return available data BlockInput, throw error if not
+            if (blockInput.type === BlockInputType.dataPromise) {
+                // if there were any peers who would have had the missing datacolumns, it would have resulted in err
+                throw Error(`Expected BlockInput to be available, got dataPromise for ${block.blockRootHex}`);
+            }
             block = {
                 ...block,
                 status: PendingBlockStatus.downloaded,
@@ -300,7 +345,7 @@ export class UnknownBlockSync {
             }
         }
         else {
-            // this allows to retry the download of the block
+            // block download has error, this allows to retry the download of the block
             block.status = PendingBlockStatus.pending;
             // parentSlot > finalizedSlot, continue downloading parent of parent
             block.downloadAttempts++;
@@ -319,9 +364,21 @@ export class UnknownBlockSync {
     /**
      * Send block to the processor awaiting completition. If processed successfully, send all children to the processor.
      * On error, remove and downscore all descendants.
+     * This function could run recursively for all descendant blocks
      */
     async processBlock(pendingBlock) {
+        // pending block status is `downloaded` right after `downloadBlock`
+        // but could be `pending` if added by `onUnknownBlockParent` event and this function is called recursively
         if (pendingBlock.status !== PendingBlockStatus.downloaded) {
+            if (pendingBlock.status === PendingBlockStatus.pending) {
+                const connectedPeers = this.network.getConnectedPeers();
+                if (connectedPeers.length === 0) {
+                    this.logger.debug("No connected peers, skipping download block", { blockRoot: pendingBlock.blockRootHex });
+                    return;
+                }
+                // if the download is a success we'll call `processBlock()` for this block
+                await this.downloadBlock(pendingBlock);
+            }
             return;
         }
         pendingBlock.status = PendingBlockStatus.processing;
@@ -403,83 +460,126 @@ export class UnknownBlockSync {
         }
     }
     /**
-     * Fetches the parent of a block by root from a set of shuffled peers.
-     * Will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers if connectPeers.length > MAX_ATTEMPTS_PER_BLOCK.
+     * From a set of shuffled peers:
+     *   - fetch the block
+     *   - from deneb, fetch all missing blobs
+     *   - from peerDAS, fetch sampled colmns
+     * TODO: this means we only have block root, and nothing else. Consider to reflect this in the function name
+     * prefulu, will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers, postfulu we may attempt more as defined in `getMaxDownloadAttempts()` function
      * Also verifies the received block root + returns the peer that provided the block for future downscoring.
      */
-    async fetchUnknownBlockRoot(blockRoot, connectedPeers) {
-        const shuffledPeers = shuffle(connectedPeers);
+    async fetchUnknownBlockRoot(blockRoot) {
         const blockRootHex = toRootHex(blockRoot);
+        const excludedPeers = new Set();
+        let partialDownload = null;
+        const defaultPendingColumns = this.config.getForkSeq(this.chain.clock.currentSlot) >= ForkSeq.fulu
+            ? new Set(this.network.custodyConfig.sampleGroups)
+            : null;
         let lastError = null;
-        for (let i = 0; i < MAX_ATTEMPTS_PER_BLOCK; i++) {
-            const peer = shuffledPeers[i % shuffledPeers.length];
+        let i = 0;
+        while (i++ < this.getMaxDownloadAttempts()) {
+            // pendingDataColumns is null prefulu
+            const peer = this.peerBalancer.bestPeerForPendingColumns(partialDownload ? new Set(partialDownload.pendingDataColumns) : defaultPendingColumns, excludedPeers);
+            if (peer === null) {
+                // no more peer with needed columns to try, throw error
+                throw Error(`Error fetching UnknownBlockRoot after ${i}: cannot find peer with needed columns ${partialDownload?.pendingDataColumns.join(", ")}`);
+            }
+            const { peerId, client: peerClient } = peer;
+            excludedPeers.add(peerId);
             try {
-                const [blockInput] = await beaconBlocksMaybeBlobsByRoot(this.config, this.network, peer, [blockRoot]);
+                const { blocks: [blockInput], pendingDataColumns, } = await beaconBlocksMaybeBlobsByRoot(this.config, this.network, peerId, [blockRoot], partialDownload, peerClient, this.metrics, this.logger);
                 // Peer does not have the block, try with next peer
                 if (blockInput === undefined) {
                     continue;
                 }
-                // Verify block root is correct
+                if (pendingDataColumns !== null) {
+                    partialDownload = { blocks: [blockInput], pendingDataColumns };
+                    continue;
+                }
+                // data is available, verify block root is correct
                 const block = blockInput.block.message;
                 const receivedBlockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block);
                 if (!byteArrayEquals(receivedBlockRoot, blockRoot)) {
                     throw Error(`Wrong block received by peer, got ${toRootHex(receivedBlockRoot)} expected ${blockRootHex}`);
                 }
-                return { blockInput, peerIdStr: peer };
+                return { blockInput, peerIdStr: peerId };
             }
             catch (e) {
-                this.logger.debug("Error fetching UnknownBlockRoot", { attempt: i, blockRootHex, peer }, e);
+                this.logger.debug("Error fetching UnknownBlockRoot", { attempt: i, blockRootHex, peer: peerId }, e);
                 lastError = e;
             }
+            finally {
+                this.peerBalancer.onRequestCompleted(peerId);
+            }
         }
         if (lastError) {
-            lastError.message = `Error fetching UnknownBlockRoot after ${MAX_ATTEMPTS_PER_BLOCK} attempts: ${lastError.message}`;
+            lastError.message = `Error fetching UnknownBlockRoot after ${i} attempts: ${lastError.message}`;
             throw lastError;
         }
-        throw Error(`Error fetching UnknownBlockRoot after ${MAX_ATTEMPTS_PER_BLOCK}: unknown error`);
+        throw Error(`Error fetching UnknownBlockRoot after ${i}: cannot download all blobs or data columns for block ${blockRootHex}`);
     }
     /**
-     * Fetches missing blobs for the blockinput, in future can also pull block is thats also missing
-     * along with the blobs (i.e. only some blobs are available)
+     * We have partial block input:
+     * - we have block but not have all blobs (deneb) or needed columns (fulu)
+     * - we don't have block and have some blobs (deneb) or some columns (fulu)
+     * Fetches missing block/data columns/block for the blockinput. This function returns either preData or availableData BlockInput.
      */
-    async fetchUnavailableBlockInput(unavailableBlockInput, connectedPeers) {
+    async fetchUnavailableBlockInput(unavailableBlockInput) {
         if (unavailableBlockInput.block !== null && unavailableBlockInput.type !== BlockInputType.dataPromise) {
             return { blockInput: unavailableBlockInput, peerIdStr: "" };
         }
-        const shuffledPeers = shuffle(connectedPeers);
         let blockRootHex;
-        let pendingBlobs;
         let blobKzgCommitmentsLen;
         let blockRoot;
+        const dataMeta = {};
+        let sampledColumns = [];
         if (unavailableBlockInput.block === null) {
             blockRootHex = unavailableBlockInput.blockRootHex;
             blockRoot = fromHex(blockRootHex);
         }
         else {
-            const unavailableBlock = unavailableBlockInput.block;
+            const { cachedData, block: unavailableBlock } = unavailableBlockInput;
             blockRoot = this.config
                 .getForkTypes(unavailableBlock.message.slot)
                 .BeaconBlock.hashTreeRoot(unavailableBlock.message);
             blockRootHex = toRootHex(blockRoot);
             blobKzgCommitmentsLen = unavailableBlock.message.body.blobKzgCommitments.length;
-            pendingBlobs = blobKzgCommitmentsLen - unavailableBlockInput.cachedData.blobsCache.size;
+            if (cachedData.fork === ForkName.deneb || cachedData.fork === ForkName.electra) {
+                const pendingBlobs = blobKzgCommitmentsLen - cachedData.blobsCache.size;
+                Object.assign(dataMeta, { pendingBlobs });
+            }
+            else if (cachedData.fork === ForkName.fulu) {
+                sampledColumns = this.network.custodyConfig.sampledColumns;
+                const pendingColumns = sampledColumns.length - cachedData.dataColumnsCache.size;
+                Object.assign(dataMeta, { pendingColumns });
+            }
         }
         let lastError = null;
-        for (let i = 0; i < MAX_ATTEMPTS_PER_BLOCK; i++) {
-            const peer = shuffledPeers[i % shuffledPeers.length];
+        let i = 0;
+        const excludedPeers = new Set();
+        while (i++ < this.getMaxDownloadAttempts()) {
+            const bestPeer = this.peerBalancer.bestPeerForBlockInput(unavailableBlockInput, excludedPeers);
+            if (bestPeer === null) {
+                // no more peer to try, throw error
+                throw Error(`Error fetching UnavailableBlockInput after ${i}: cannot find peer with needed columns ${sampledColumns.join(", ")}`);
+            }
+            const { peerId, client: peerClient } = bestPeer;
+            excludedPeers.add(peerId);
             try {
-                const blockInput = await unavailableBeaconBlobsByRoot(this.config, this.network, peer, unavailableBlockInput, {
+                const blockInput = await unavailableBeaconBlobsByRoot(this.config, this.network, peerId, peerClient, unavailableBlockInput, {
                     metrics: this.metrics,
-                    emitter: this.chain.emitter,
+                    logger: this.logger,
                     executionEngine: this.chain.executionEngine,
-                    engineGetBlobsCache: this.engineGetBlobsCache,
+                    emitter: this.chain.emitter,
                     blockInputsRetryTrackerCache: this.blockInputsRetryTrackerCache,
+                    engineGetBlobsCache: this.engineGetBlobsCache,
                 });
-                // Peer does not have the block, try with next peer
-                if (blockInput === undefined) {
+                if (unavailableBlockInput.block !== null && blockInput.type === BlockInputType.dataPromise) {
+                    // all datacolumns were not downloaded we can continue with other peers
+                    // as unavailableBlockInput.block's dataColumnsCache would be updated
                     continue;
                 }
-                // Verify block root is correct
+                // data is available, verify block root is correct
                 const block = blockInput.block.message;
                 const receivedBlockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block);
                 if (!byteArrayEquals(receivedBlockRoot, blockRoot)) {
@@ -489,20 +589,23 @@ export class UnknownBlockSync {
                     this.logger.debug("Fetched  NullBlockInput", { attempts: i, blockRootHex });
                 }
                 else {
-                    this.logger.debug("Fetched UnavailableBlockInput", { attempts: i, pendingBlobs, blobKzgCommitmentsLen });
+                    this.logger.debug("Fetched UnavailableBlockInput", { attempts: i, ...dataMeta, blobKzgCommitmentsLen });
                 }
-                return { blockInput, peerIdStr: peer };
+                return { blockInput, peerIdStr: peerId };
             }
             catch (e) {
-                this.logger.debug("Error fetching UnavailableBlockInput", { attempt: i, blockRootHex, peer }, e);
+                this.logger.debug("Error fetching UnavailableBlockInput", { attempt: i, blockRootHex, peer: peerId }, e);
                 lastError = e;
             }
+            finally {
+                this.peerBalancer.onRequestCompleted(peerId);
+            }
         }
         if (lastError) {
-            lastError.message = `Error fetching UnavailableBlockInput after ${MAX_ATTEMPTS_PER_BLOCK} attempts: ${lastError.message}`;
+            lastError.message = `Error fetching UnavailableBlockInput after ${i} attempts: ${lastError.message}`;
             throw lastError;
         }
-        throw Error(`Error fetching UnavailableBlockInput after ${MAX_ATTEMPTS_PER_BLOCK}: unknown error`);
+        throw Error(`Error fetching UnavailableBlockInput after ${i}: unknown error`);
     }
     /**
      * Gets all descendant blocks of `block` recursively from `pendingBlocks`.
@@ -513,13 +616,15 @@ export class UnknownBlockSync {
     removeAndDownscoreAllDescendants(block) {
         // Get all blocks that are a descendant of this one
         const badPendingBlocks = this.removeAllDescendants(block);
+        // just console log and do not penalize on pending/bad blocks for debugging
+        // console.log("removeAndDownscoreAllDescendants", {block});
         for (const block of badPendingBlocks) {
-            this.knownBadBlocks.add(block.blockRootHex);
-            for (const peerIdStr of block.peerIdStrs) {
-                // TODO: Refactor peerRpcScores to work with peerIdStr only
-                this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot");
-            }
-            this.logger.debug("Banning unknown block", {
+            //   this.knownBadBlocks.add(block.blockRootHex);
+            //   for (const peerIdStr of block.peerIdStrs) {
+            //     // TODO: Refactor peerRpcScores to work with peerIdStr only
+            //     this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot");
+            //   }
+            this.logger.debug("ignored Banning unknown block", {
                 root: block.blockRootHex,
                 peerIdStrs: Array.from(block.peerIdStrs).join(","),
             });
@@ -539,5 +644,160 @@ export class UnknownBlockSync {
         }
         return badPendingBlocks;
     }
+    getMaxDownloadAttempts() {
+        if (this.config.getForkSeq(this.chain.clock.currentSlot) < ForkSeq.fulu) {
+            return MAX_ATTEMPTS_PER_BLOCK;
+        }
+        // TODO: I consider max 20 downloads per block for a supernode is enough for devnets
+        // review this computation for public testnets or mainnet
+        return Math.min(20, (MAX_ATTEMPTS_PER_BLOCK * this.network.custodyConfig.sampleGroups.length) / this.config.SAMPLES_PER_SLOT);
+    }
+}
+/**
+ * Class to track active byRoots requests and balance them across eligible peers.
+ */
+export class UnknownBlockPeerBalancer {
+    constructor(custodyConfig) {
+        this.peersMeta = new Map();
+        this.activeRequests = new Map();
+        this.custodyConfig = custodyConfig;
+    }
+    /** Trigger on each peer re-status */
+    onPeerConnected(peerId, syncMeta) {
+        this.peersMeta.set(peerId, syncMeta);
+        if (!this.activeRequests.has(peerId)) {
+            this.activeRequests.set(peerId, 0);
+        }
+    }
+    onPeerDisconnected(peerId) {
+        this.peersMeta.delete(peerId);
+        this.activeRequests.delete(peerId);
+    }
+    /**
+     * called from fetchUnknownBlockRoot() where we only have block root and nothing else
+     * excludedPeers are the peers that we requested already so we don't want to try again
+     * pendingColumns is empty for prefulu, or the 1st time we we download a block by root
+     */
+    bestPeerForPendingColumns(pendingColumns, excludedPeers) {
+        const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers);
+        if (eligiblePeers.length === 0) {
+            return null;
+        }
+        const sortedEligiblePeers = sortBy(shuffle(eligiblePeers), 
+        // prefer peers with least active req
+        (peerId) => this.activeRequests.get(peerId) ?? 0);
+        const bestPeerId = sortedEligiblePeers[0];
+        this.onRequest(bestPeerId);
+        return this.peersMeta.get(bestPeerId) ?? null;
+    }
+    /**
+     * called from fetchUnavailableBlockInput() where we have either BlockInput or NullBlockInput
+     * excludedPeers are the peers that we requested already so we don't want to try again
+     */
+    bestPeerForBlockInput(unavailableBlockInput, excludedPeers) {
+        let cachedData = undefined;
+        if (unavailableBlockInput.block === null) {
+            // NullBlockInput
+            cachedData = unavailableBlockInput.cachedData;
+        }
+        else {
+            // BlockInput
+            if (unavailableBlockInput.type !== BlockInputType.dataPromise) {
+                throw Error(`bestPeerForBlockInput called with BlockInput type ${unavailableBlockInput.type}, expected dataPromise`);
+            }
+            cachedData = unavailableBlockInput.cachedData;
+        }
+        const eligiblePeers = [];
+        if (cachedData.fork === ForkName.fulu) {
+            // cached data is CachedDataColumns
+            const { dataColumnsCache } = cachedData;
+            const pendingDataColumns = new Set();
+            for (const column of this.custodyConfig.sampledColumns) {
+                if (!dataColumnsCache.has(column)) {
+                    pendingDataColumns.add(column);
+                }
+            }
+            // there could be no pending column in case of NullBlockInput
+            eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers));
+        }
+        else {
+            // prefulu
+            const pendingDataColumns = null;
+            eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers));
+        }
+        if (eligiblePeers.length === 0) {
+            return null;
+        }
+        const sortedEligiblePeers = sortBy(shuffle(eligiblePeers), 
+        // prefer peers with least active req
+        (peerId) => this.activeRequests.get(peerId) ?? 0);
+        const bestPeerId = sortedEligiblePeers[0];
+        this.onRequest(bestPeerId);
+        return this.peersMeta.get(bestPeerId) ?? null;
+    }
+    /**
+     * Consumers don't need to call this method directly, it is called internally by bestPeer*() methods
+     * make this public for testing
+     */
+    onRequest(peerId) {
+        this.activeRequests.set(peerId, (this.activeRequests.get(peerId) ?? 0) + 1);
+    }
+    /**
+     * Consumers should call this method when a request is completed for a peer.
+     */
+    onRequestCompleted(peerId) {
+        this.activeRequests.set(peerId, Math.max(0, (this.activeRequests.get(peerId) ?? 1) - 1));
+    }
+    getTotalActiveRequests() {
+        let totalActiveRequests = 0;
+        for (const count of this.activeRequests.values()) {
+            totalActiveRequests += count;
+        }
+        return totalActiveRequests;
+    }
+    // pendingDataColumns could be null for prefulu
+    filterPeers(pendingDataColumns, excludedPeers) {
+        let maxColumnCount = 0;
+        const considerPeers = [];
+        for (const [peerId, syncMeta] of this.peersMeta.entries()) {
+            if (excludedPeers.has(peerId)) {
+                // made request to this peer already
+                continue;
+            }
+            const activeRequests = this.activeRequests.get(peerId) ?? 0;
+            if (activeRequests >= MAX_CONCURRENT_REQUESTS) {
+                // should return peer with no more than MAX_CONCURRENT_REQUESTS active requests
+                continue;
+            }
+            if (pendingDataColumns === null || pendingDataColumns.size === 0) {
+                // prefulu, no pending columns
+                considerPeers.push({ peerId, columnCount: 0 });
+                continue;
+            }
+            // postfulu, find peers that have custody columns that we need
+            const { custodyGroups: peerColumns } = syncMeta;
+            // check if the peer has all needed columns
+            // get match
+            const columns = peerColumns.reduce((acc, elem) => {
+                if (pendingDataColumns.has(elem)) {
+                    acc.push(elem);
+                }
+                return acc;
+            }, []);
+            if (columns.length > 0) {
+                if (columns.length > maxColumnCount) {
+                    maxColumnCount = columns.length;
+                }
+                considerPeers.push({ peerId, columnCount: columns.length });
+            }
+        } // end for
+        const eligiblePeers = [];
+        for (const { peerId, columnCount } of considerPeers) {
+            if (columnCount === maxColumnCount) {
+                eligiblePeers.push(peerId);
+            }
+        }
+        return eligiblePeers;
+    }
 }
 //# sourceMappingURL=unknownBlock.js.map