@aztec/p2p 0.0.1-commit.fcb71a6 → 0.0.1-commit.ff7989d6c

package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js

@@ -0,0 +1,562 @@
+import { chunkWrapAround } from '@aztec/foundation/collection';
+import { TimeoutError } from '@aztec/foundation/error';
+import { createLogger } from '@aztec/foundation/log';
+import { FifoMemoryQueue, Semaphore } from '@aztec/foundation/queue';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
+import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
+import { TxHash } from '@aztec/stdlib/tx';
+import { peerIdFromString } from '@libp2p/peer-id';
+import { ReqRespSubProtocol } from '.././interface.js';
+import { BlockTxsRequest, BlockTxsResponse } from '.././protocols/index.js';
+import { ReqRespStatus } from '.././status.js';
+import { DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD, DEFAULT_BATCH_TX_REQUESTER_DUMB_PARALLEL_WORKER_COUNT, DEFAULT_BATCH_TX_REQUESTER_SMART_PARALLEL_WORKER_COUNT, DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE } from './config.js';
+import { MissingTxMetadataCollection } from './missing_txs.js';
+import { PeerCollection } from './peer_collection.js';
+import { BatchRequestTxValidator } from './tx_validator.js';
+/*
+ * Tries to fetch all missing transaction until deadline is hit.
+ * Transactions are yield by calling run*() method
+ *
+ * We have a couple of peer types:
+ * - Pinned peer is the one who sent us the block proposal
+ * - Dumb peer:
+ *    - We query this peer blindly because we don't know which txs it has
+ *    - We hope it might have some of the transactions we asked for
+ *    - When this peer sends response it might become Smart peer
+ * - Smart peer:
+ *    - Initially there are no smart peers, all are considered "dumb"
+ *    - Peer becomes smart when in response it tels us exactly which transactions it has
+ *      AND we are missing some of those transactions
+ * - Bad peer:
+ *    - Is the peer which was unable to send us successful response N times in a row
+ * */ export class BatchTxRequester {
+    blockTxsSource;
+    pinnedPeer;
+    timeoutMs;
+    p2pService;
+    logger;
+    dateProvider;
+    opts;
+    peers;
+    txsMetadata;
+    deadline;
+    smartRequesterSemaphore;
+    txQueue;
+    txValidator;
+    smartParallelWorkerCount;
+    dumbParallelWorkerCount;
+    txBatchSize;
+    constructor(missingTxsTracker, blockTxsSource, pinnedPeer, timeoutMs, p2pService, logger, dateProvider, opts){
+        this.blockTxsSource = blockTxsSource;
+        this.pinnedPeer = pinnedPeer;
+        this.timeoutMs = timeoutMs;
+        this.p2pService = p2pService;
+        this.logger = logger ?? createLogger('p2p:reqresp_batch');
+        this.dateProvider = dateProvider ?? new DateProvider();
+        this.opts = opts ?? {};
+        this.smartParallelWorkerCount = this.opts.smartParallelWorkerCount ?? DEFAULT_BATCH_TX_REQUESTER_SMART_PARALLEL_WORKER_COUNT;
+        this.dumbParallelWorkerCount = this.opts.dumbParallelWorkerCount ?? DEFAULT_BATCH_TX_REQUESTER_DUMB_PARALLEL_WORKER_COUNT;
+        this.txBatchSize = this.opts.txBatchSize ?? DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE;
+        this.deadline = this.dateProvider.now() + this.timeoutMs;
+        this.txQueue = new FifoMemoryQueue(this.logger);
+        this.txValidator = this.opts.txValidator ?? new BatchRequestTxValidator(this.p2pService.txValidatorConfig);
+        if (this.opts.peerCollection) {
+            this.peers = this.opts.peerCollection;
+        } else {
+            const initialPeers = this.p2pService.connectionSampler.getPeerListSortedByConnectionCountAsc();
+            const badPeerThreshold = this.opts.badPeerThreshold ?? DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD;
+            this.peers = new PeerCollection(initialPeers, this.pinnedPeer, this.dateProvider, badPeerThreshold, this.p2pService.peerScoring);
+        }
+        this.txsMetadata = new MissingTxMetadataCollection(missingTxsTracker, this.txBatchSize);
+        this.smartRequesterSemaphore = this.opts.semaphore ?? new Semaphore(0);
+    }
+    /*
+   * Fetches all missing transactions and yields them  one by one
+   * */ async *run() {
+        // Our timeout is represented in milliseconds but queue expects seconds
+        // We also want to make sure we wait at least 1 second in case of very low timeouts
+        const timeoutQueueAfter = Math.max(Math.ceil(this.timeoutMs / 1_000), 1);
+        try {
+            if (this.txsMetadata.getMissingTxHashes().size === 0) {
+                return undefined;
+            }
+            // Start workers in background
+            const workersPromise = executeTimeout(()=>Promise.allSettled([
+                    this.smartRequester(),
+                    this.dumbRequester(),
+                    this.pinnedPeerRequester()
+                ]), this.timeoutMs).finally(()=>{
+                this.txQueue.end();
+            });
+            while(true){
+                const tx = await this.txQueue.get(timeoutQueueAfter);
+                // null indicates that the queue has ended
+                if (tx === null) {
+                    break;
+                }
+                yield tx;
+                if (this.shouldStop()) {
+                    // Drain queue before ending
+                    let remaining;
+                    while((remaining = this.txQueue.getImmediate()) !== undefined){
+                        yield remaining;
+                    }
+                    break;
+                }
+            }
+            this.unlockSmartRequesterSemaphores();
+            await workersPromise;
+        } catch (e) {
+            this.logger.error(`Batch tx requester failed with error: ${e.message}`, {
+                error: e
+            });
+        } finally{
+            this.txQueue.end();
+            this.unlockSmartRequesterSemaphores();
+        }
+    }
+    /*
+   * Fetches all missing transactions
+   * @returns Collection of fetched transactions */ static async collectAllTxs(generator) {
+        const txs = [];
+        for await (const tx of generator){
+            if (tx === undefined) {
+                break;
+            }
+            txs.push(tx);
+        }
+        return txs;
+    }
+    /*
+   * Handles so-called pinned peer
+   * The pinned peer is the one who sent us block proposal
+   * We expect pinned peer to have all transactions from the proposal at some point
+   * This holds because they them selves have to attest to proposal and thus fetch all missing transactions
+   *
+   * Given the reasoning above - we query pinned peer separately from dumb/smart peers
+   * */ async pinnedPeerRequester() {
+        if (!this.pinnedPeer) {
+            this.logger.debug('No pinned peer to request from');
+            return;
+        }
+        while(!this.shouldStop()){
+            // We've hit rate limits on the pinned peer - wait until rate limit expires (clamped to deadline)
+            const rateLimitDelay = this.peers.getPeerRateLimitDelayMs(this.pinnedPeer);
+            const pinnedPeerIsRateLimited = rateLimitDelay !== undefined;
+            if (pinnedPeerIsRateLimited) {
+                await this.sleepClampedToDeadline(rateLimitDelay);
+                continue;
+            }
+            const pinnedPeerWentBad = this.peers.getBadPeers().has(this.pinnedPeer.toString());
+            if (pinnedPeerWentBad) {
+                return;
+            }
+            // From pinned peer we always request transactions so that we first request the least requested and not in flight
+            // This makes sense since pinned peer should have ALL transactions,
+            // Thus if it has all it is best to ask pinned first for the transactions we have trouble getting from other peers
+            const txs = this.txsMetadata.getTxsToRequestFromThePeer(this.pinnedPeer);
+            if (txs.length === 0) {
+                this.logger.debug(`Pinned peer ${this.pinnedPeer.toString()} has no txs to request`);
+                return;
+            }
+            const request = BlockTxsRequest.fromTxsSourceAndMissingTxs(this.blockTxsSource, txs);
+            if (!request) {
+                return;
+            }
+            txs.forEach((tx)=>{
+                this.txsMetadata.markRequested(tx);
+                this.txsMetadata.markInFlightBySmartPeer(tx);
+            });
+            await this.requestTxBatch(this.pinnedPeer, request);
+            txs.forEach((tx)=>{
+                this.txsMetadata.markNotInFlightBySmartPeer(tx);
+            });
+        }
+    }
+    /*
+   * Starts dumb worker loops
+   * */ async dumbRequester() {
+        const nextPeerIndex = this.makeRoundRobinIndexer();
+        const nextBatchIndex = this.makeRoundRobinIndexer();
+        // Chunk missing tx hashes into batches of txBatchSize, wrapping around to ensure no peer gets less than txBatchSize
+        const txChunks = ()=>{
+            const missingHashes = Array.from(this.txsMetadata.getMissingTxHashes());
+            return chunkWrapAround(missingHashes, this.txBatchSize);
+        };
+        const makeRequest = (_pid)=>{
+            const chunks = txChunks();
+            const idx = nextBatchIndex(()=>chunks.length);
+            const noMoreTxsToRequest = idx === undefined;
+            if (noMoreTxsToRequest) {
+                return undefined;
+            }
+            const txs = chunks[idx].map((t)=>TxHash.fromString(t));
+            // If peer is dumb peer, we don't know yet if they received full blockProposal
+            // there is solid chance that peer didn't receive proposal yet, thus we must send full hashes
+            const includeFullHashesInRequestNotJustIndices = true;
+            const blockRequest = BlockTxsRequest.fromTxsSourceAndMissingTxs(this.blockTxsSource, txs, includeFullHashesInRequestNotJustIndices);
+            const blockRequestHasNoMissingTxsFromTheProposal = !blockRequest;
+            if (blockRequestHasNoMissingTxsFromTheProposal) {
+                return undefined;
+            }
+            return {
+                blockRequest,
+                txs
+            };
+        };
+        const nextPeer = ()=>{
+            const peers = this.peers.getDumbPeersToQuery();
+            const idx = nextPeerIndex(()=>peers.length);
+            return idx === undefined ? undefined : peerIdFromString(peers[idx]);
+        };
+        const workerCount = Math.min(this.dumbParallelWorkerCount, this.peers.getAllPeers().size);
+        const workers = Array.from({
+            length: workerCount
+        }, (_, index)=>this.dumbWorkerLoop(nextPeer, makeRequest, index + 1));
+        await Promise.allSettled(workers);
+    }
+    /*
+   * Dumb worker loop.
+   * It fetches next available dumb peer and builds request for that peer
+   * Loops until shouldStop condition is not met or there are no more dubm peers to query
+   *   This can happen if e.g. all "dumb" peers transition to "smart" or e.g. become "bad"
+   * */ async dumbWorkerLoop(pickNextPeer, request, workerIndex) {
+        try {
+            this.logger.debug(`Dumb worker ${workerIndex} started`);
+            while(!this.shouldStop()){
+                const peerId = pickNextPeer();
+                const weRanOutOfPeersToQuery = peerId === undefined;
+                if (weRanOutOfPeersToQuery) {
+                    const nextDumbPeerDelay = this.peers.getNextDumbPeerAvailabilityDelayMs();
+                    const thereAreSomeRateLimitedDumbPeers = nextDumbPeerDelay !== undefined;
+                    if (thereAreSomeRateLimitedDumbPeers) {
+                        // There are still some dumb peers to query but they have been rate limited
+                        // Sleep until the earliest one gets unblocked (clamped to deadline)
+                        await this.sleepClampedToDeadline(nextDumbPeerDelay);
+                        continue;
+                    }
+                    this.logger.debug(`Worker loop dumb: No more peers to query`);
+                    break;
+                }
+                const nextBatchTxRequest = request(peerId);
+                if (!nextBatchTxRequest) {
+                    this.logger.debug(`Worker loop dumb: no txs to request, exiting`);
+                    break;
+                }
+                const { blockRequest, txs } = nextBatchTxRequest;
+                this.logger.debug(`Worker type dumb: Requesting txs from peer ${peerId.toString()}: ${txs.map((tx)=>tx.toString()).join(', ')}`);
+                await this.requestTxBatch(peerId, blockRequest);
+            }
+        } catch (err) {
+            this.logger.error(`Dumb worker ${workerIndex} encountered an error: ${err}`);
+        } finally{
+            this.logger.debug(`Dumb worker ${workerIndex} finished`);
+        }
+    }
+    /*
+   * Starts smart worker loops
+   * */ async smartRequester() {
+        const nextPeerIndex = this.makeRoundRobinIndexer();
+        const nextPeer = ()=>{
+            const peers = this.peers.getSmartPeersToQuery();
+            const idx = nextPeerIndex(()=>peers.length);
+            return idx === undefined ? undefined : peerIdFromString(peers[idx]);
+        };
+        const makeRequest = (pid)=>{
+            const txs = this.txsMetadata.getTxsToRequestFromThePeer(pid);
+            const blockRequest = BlockTxsRequest.fromTxsSourceAndMissingTxs(this.blockTxsSource, txs);
+            if (!blockRequest) {
+                return undefined;
+            }
+            return {
+                blockRequest,
+                txs
+            };
+        };
+        const workers = Array.from({
+            length: Math.min(this.smartParallelWorkerCount, this.peers.getAllPeers().size)
+        }, (_, index)=>this.smartWorkerLoop(nextPeer, makeRequest, index + 1));
+        await Promise.allSettled(workers);
+    }
+    /*
+   * Smart worker loop.
+   * It fetches next available smart peer and builds request for that peer
+   * Loops until shouldStop condition is not met
+   *
+   * Notes:
+   * - We don't start worker loop immediately, but block on semaphore
+   *   until some dumb peer transactions to smart peer
+   * - We might run out of smart peers, because:
+   *    - they "went bad"
+   *    - there are less smart peers than worker loops
+   *   In such scenario we either wait for next dumb peer to become smart or kill the worker loop
+   * */ async smartWorkerLoop(pickNextPeer, request, workerIndex) {
+        try {
+            this.logger.trace(`Smart worker ${workerIndex} started`);
+            await executeTimeout((_)=>this.smartRequesterSemaphore.acquire(), this.timeoutMs);
+            this.logger.trace(`Smart worker ${workerIndex} acquired semaphore`);
+            while(!this.shouldStop()){
+                const peerId = pickNextPeer();
+                const weRanOutOfPeersToQuery = peerId === undefined;
+                if (weRanOutOfPeersToQuery) {
+                    this.logger.debug(`Worker loop smart: No more peers to query`);
+                    // If there are no more dumb peers to query then none of our peers can become smart,
+                    // thus we can simply exit this worker
+                    const noMoreDumbPeersToQuery = this.peers.getDumbPeersToQuery().length === 0;
+                    if (noMoreDumbPeersToQuery) {
+                        // These might be either smart peers that will get unblocked after _some time_
+                        const nextSmartPeerDelay = this.peers.getNextSmartPeerAvailabilityDelayMs();
+                        const thereAreSomeRateLimitedSmartPeers = nextSmartPeerDelay !== undefined;
+                        if (thereAreSomeRateLimitedSmartPeers) {
+                            await this.sleepClampedToDeadline(nextSmartPeerDelay);
+                            continue;
+                        }
+                        this.logger.debug(`Worker loop smart: No more smart peers to query killing ${workerIndex}`);
+                        break;
+                    }
+                    // Otherwise there are still some dumb peers that could become smart.
+                    // We end up here when all known smart peers became temporarily unavailable via combination of
+                    // (bad, in-flight, or rate-limited) or in some weird scenario all current smart peers turn bad which is permanent
+                    // but dumb peers still exist that could become smart.
+                    //
+                    // When a dumb peer responds with valid txIndices, it gets
+                    // promoted to smart and releases the semaphore, waking this worker.
+                    await executeTimeout((_)=>this.smartRequesterSemaphore.acquire(), this.timeoutMs);
+                    this.logger.debug(`Worker loop smart: acquired next smart peer`);
+                    continue;
+                }
+                const nextBatchTxRequest = request(peerId);
+                if (!nextBatchTxRequest) {
+                    this.logger.debug(`Worker loop smart: no txs to request, exiting`);
+                    break;
+                }
+                const { blockRequest, txs } = nextBatchTxRequest;
+                // We only mark transactions as in flight if queried by Smart peer
+                // Because asking them from dumb peer is shot in the dark (there is a good chance they won't have it)
+                // So we don't gain anything if we mark txs in-flight for dumb peers
+                txs.forEach((tx)=>{
+                    this.txsMetadata.markRequested(tx);
+                    this.txsMetadata.markInFlightBySmartPeer(tx);
+                });
+                await this.requestTxBatch(peerId, blockRequest);
+                txs.forEach((tx)=>{
+                    this.txsMetadata.markNotInFlightBySmartPeer(tx);
+                });
+            }
+        } catch (err) {
+            if (err instanceof TimeoutError) {
+                this.logger.debug(`Smart worker ${workerIndex} timed out waiting for semaphore`);
+            } else {
+                this.logger.error(`Smart worker ${workerIndex} encountered an error: ${err}`);
+            }
+        } finally{
+            this.logger.debug(`Smart worker ${workerIndex} finished`);
+        }
+    }
+    /*
+   * Sends actual request to the peer and handles response
+   *
+   * @param peerId - the peer to send request to
+   * @param request - the actual request
+   */ async requestTxBatch(peerId, request) {
+        try {
+            this.peers.markPeerInFlight(peerId);
+            const response = await this.p2pService.reqResp.sendRequestToPeer(peerId, ReqRespSubProtocol.BLOCK_TXS, request.toBuffer());
+            if (response.status !== ReqRespStatus.SUCCESS) {
+                this.logger.debug(`Peer ${peerId.toString()} failed to respond with status: ${response.status}`);
+                this.handleFailResponseFromPeer(peerId, response.status);
+                return;
+            }
+            const blockResponse = BlockTxsResponse.fromBuffer(response.data);
+            await this.handleSuccessResponseFromPeer(peerId, blockResponse);
+        } catch (err) {
+            this.logger.error(`Failed to get valid response from peer ${peerId.toString()}: ${err.message}`, {
+                peerId,
+                error: err
+            });
+            this.handleFailResponseFromPeer(peerId, ReqRespStatus.UNKNOWN);
+        } finally{
+            this.peers.unMarkPeerInFlight(peerId);
+        }
+    }
+    /*
+   * Handles failed response form the peer
+   * There are 3 scenarios
+   * - RATE_LIMIT_EXCEEDED: We mark this and don't query this peer again for some_time
+   * - FAILURE and UNKNOWN: We penalise this, if peer has been penalised this way N times they are not queried again
+   *   this implies we will query these peers couple of more times and give them a chance to "redeem" themselves before completely ignoring them
+   */ handleFailResponseFromPeer(peerId, responseStatus) {
+        //TODO: Should we ban these peers?
+        if (responseStatus === ReqRespStatus.FAILURE || responseStatus === ReqRespStatus.UNKNOWN) {
+            this.peers.penalisePeer(peerId, PeerErrorSeverity.HighToleranceError);
+            return;
+        }
+        if (responseStatus === ReqRespStatus.RATE_LIMIT_EXCEEDED) {
+            this.peers.markPeerRateLimitExceeded(peerId);
+        }
+    }
+    /*
+   * Handles successful response form the peer, this includes
+   * - Handling received transactions
+   * - Deciding if the peer is "smart" or not
+   * */ async handleSuccessResponseFromPeer(peerId, response) {
+        this.logger.debug(`Received txs: ${response.txs.length} from peer ${peerId.toString()} `);
+        await this.handleReceivedTxs(peerId, response.txs);
+        this.decideIfPeerIsSmart(peerId, response);
+    }
+    /*
+   * Handles received txs.
+   * Transactions are validated and then put on async queue
+   * to be yielded by main running loop
+   * */ async handleReceivedTxs(peerId, txs) {
+        const newTxs = txs.filter((tx)=>!this.txsMetadata.alreadyFetched(tx.txHash));
+        if (newTxs.length === 0) {
+            return;
+        }
+        //TODO: this validation can be slow, maybe spawn worker just for validation
+        // We could use the async queue for communication.
+        const validationResults = await Promise.allSettled(newTxs.map(async (tx)=>({
+                tx,
+                isValid: (await this.txValidator.validateRequestedTx(tx)).result === 'valid'
+            })));
+        let hasInvalidTx = false;
+        validationResults.forEach((result)=>{
+            if (result.status === 'fulfilled' && result.value.isValid) {
+                if (this.txsMetadata.markFetched(peerId, result.value.tx)) {
+                    this.txQueue.put(result.value.tx);
+                }
+            } else {
+                hasInvalidTx = true;
+            }
+        });
+        if (hasInvalidTx) {
+            this.peers.penalisePeer(peerId, PeerErrorSeverity.LowToleranceError);
+        } else {
+            // If we have received successful response from the peer, they have "redeemed" themselves and not considered bad anymore
+            this.peers.unMarkPeerAsBad(peerId);
+        }
+        const missingTxHashes = this.txsMetadata.getMissingTxHashes();
+        if (missingTxHashes.size === 0) {
+            // wake sleepers so they can see shouldStop() and exit before waiting on timeout
+            this.unlockSmartRequesterSemaphores();
+        } else {
+            this.logger.trace(`Missing txs: ${Array.from(this.txsMetadata.getMissingTxHashes()).map((tx)=>tx.toString()).join(', ')}`);
+        }
+    }
+    /*
+   * Peer is smart if:
+   * - They are not pinned peer
+   * - They have sent successful response indicating which txs from Block proposal they have
+   * - They have transactions we are missing
+   */ decideIfPeerIsSmart(peerId, response) {
+        const pinnedPeerShouldNeverBeMarkedAsSmart = this.pinnedPeer && peerId.toString() === this.pinnedPeer.toString();
+        if (pinnedPeerShouldNeverBeMarkedAsSmart) {
+            return;
+        }
+        const smartPeersAreDisabled = this.smartParallelWorkerCount === 0;
+        if (smartPeersAreDisabled) {
+            return;
+        }
+        // If block response is invalid we still want to query this peer in the future
+        // Because they sent successful response, so they might become smart peer in the future
+        // Or send us needed txs
+        if (!this.isBlockResponseValid(response)) {
+            return;
+        }
+        // We mark peer as "smart" only if they have some txs we are missing
+        // Otherwise we keep them as "dumb" in hope they'll receive some new txs we are missing in the future
+        if (!this.peerHasSomeTxsWeAreMissing(peerId, response)) {
+            this.logger.debug(`${peerId.toString()} has no txs we are missing, skipping`);
+            return;
+        }
+        this.peers.markPeerSmart(peerId);
+        this.markTxsPeerHas(peerId, response);
+        // Unblock smart workers
+        if (this.peers.getSmartPeersToQuery().length <= this.smartParallelWorkerCount) {
+            this.smartRequesterSemaphore.release();
+        }
+    }
+    isBlockResponseValid(response) {
+        const archiveRootsMatch = this.blockTxsSource.archive.toString() === response.archiveRoot.toString();
+        const peerHasSomeTxsFromProposal = !response.txIndices.isEmpty();
+        return archiveRootsMatch && peerHasSomeTxsFromProposal;
+    }
+    peerHasSomeTxsWeAreMissing(_peerId, response) {
+        const txsPeerHas = new Set(this.extractHashesPeerHasFromResponse(response).map((h)=>h.toString()));
+        return this.txsMetadata.getMissingTxHashes().intersection(txsPeerHas).size > 0;
+    }
+    markTxsPeerHas(peerId, response) {
+        const txsPeerHas = this.extractHashesPeerHasFromResponse(response);
+        this.logger.debug(`${peerId.toString()} has txs: ${txsPeerHas.map((tx)=>tx.toString()).join(', ')}`);
+        this.txsMetadata.markPeerHas(peerId, txsPeerHas);
+    }
+    extractHashesPeerHasFromResponse(response) {
+        const hashes = [];
+        const indicesOfHashesPeerHas = new Set(response.txIndices.getTrueIndices());
+        this.blockTxsSource.txHashes.forEach((hash, idx)=>{
+            if (indicesOfHashesPeerHas.has(idx)) {
+                hashes.push(hash);
+            }
+        });
+        return hashes;
+    }
+    /*
+   * Helper function to crate round robin indexer -
+   * i.e. the "thing" which returns next index/number in round robin fashion
+   **/ makeRoundRobinIndexer(start = 0) {
+        let i = start;
+        /*
+     * Function to calculate next round-robin number
+     * Idea is that we pass in an array size and based on it and previous state we call next
+     * Array size can change between calls thus it is passed as function
+     *
+     * @returns next index or undefined if size is 0
+     */ return (size)=>{
+            const length = size();
+            if (length === 0) {
+                return undefined;
+            }
+            const current = i % length;
+            i = (current + 1) % length;
+            return current;
+        };
+    }
+    /*
+   * @returns true if all missing txs have been fetched */ fetchedAllTxs() {
+        return this.txsMetadata.getMissingTxHashes().size == 0;
+    }
+    /*
+   * Checks if the BatchTxRequester should stop fetching missing txs
+   * Conditions for stopping are:
+   * - There have been no missing transactions to start with
+   * - All transactions have been fetched
+   * - The deadline has been hit (no more time to fetch)
+   * - This process has been cancelled via abortSignal
+   *
+   * @returns true if BatchTxRequester should stop, otherwise false*/ shouldStop() {
+        const aborted = this.opts.abortSignal?.aborted ?? false;
+        if (aborted) {
+            this.unlockSmartRequesterSemaphores();
+        }
+        return aborted || this.fetchedAllTxs() || this.dateProvider.now() > this.deadline;
+    }
+    /*
+   * Helper function which unlocks all smart requester semaphores
+   * @note This is needed otherwise they will block forever
+   * */ unlockSmartRequesterSemaphores() {
+        for(let i = 0; i < this.smartParallelWorkerCount; i++){
+            this.smartRequesterSemaphore.release();
+        }
+    }
+    /*
+   * Sleeps for the given duration, but clamped to the deadline.
+   * This ensures we don't sleep past the deadline.
+   * */ async sleepClampedToDeadline(durationMs) {
+        const remaining = this.deadline - this.dateProvider.now();
+        const thereIsTimeRemaining = remaining > 0;
+        if (thereIsTimeRemaining) {
+            await sleep(Math.min(durationMs, remaining));
+        }
+    }
+}

package/dest/services/reqresp/batch-tx-requester/config.d.ts

@@ -0,0 +1,17 @@
+import { type ConfigMappingsType } from '@aztec/foundation/config';
+export declare const DEFAULT_BATCH_TX_REQUESTER_SMART_PARALLEL_WORKER_COUNT = 10;
+export declare const DEFAULT_BATCH_TX_REQUESTER_DUMB_PARALLEL_WORKER_COUNT = 10;
+export declare const DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE = 8;
+export declare const DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD = 2;
+export interface BatchTxRequesterConfig {
+    /** Max concurrent requests to smart peers. */
+    batchTxRequesterSmartParallelWorkerCount: number;
+    /** Max concurrent requests to dumb peers. */
+    batchTxRequesterDumbParallelWorkerCount: number;
+    /** Max transactions per request / chunk size. */
+    batchTxRequesterTxBatchSize: number;
+    /** Failures before a peer is considered bad (see > threshold logic). */
+    batchTxRequesterBadPeerThreshold: number;
+}
+export declare const batchTxRequesterConfigMappings: ConfigMappingsType<BatchTxRequesterConfig>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi9zcmMvc2VydmljZXMvcmVxcmVzcC9iYXRjaC10eC1yZXF1ZXN0ZXIvY29uZmlnLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUFzQixNQUFNLDBCQUEwQixDQUFDO0FBRXZGLGVBQU8sTUFBTSxzREFBc0QsS0FBSyxDQUFDO0FBQ3pFLGVBQU8sTUFBTSxxREFBcUQsS0FBSyxDQUFDO0FBQ3hFLGVBQU8sTUFBTSx3Q0FBd0MsSUFBSSxDQUFDO0FBQzFELGVBQU8sTUFBTSw2Q0FBNkMsSUFBSSxDQUFDO0FBRS9ELE1BQU0sV0FBVyxzQkFBc0I7SUFDckMsOENBQThDO0lBQzlDLHdDQUF3QyxFQUFFLE1BQU0sQ0FBQztJQUNqRCw2Q0FBNkM7SUFDN0MsdUNBQXVDLEVBQUUsTUFBTSxDQUFDO0lBQ2hELGlEQUFpRDtJQUNqRCwyQkFBMkIsRUFBRSxNQUFNLENBQUM7SUFDcEMsd0VBQXdFO0lBQ3hFLGdDQUFnQyxFQUFFLE1BQU0sQ0FBQztDQUMxQztBQUVELGVBQU8sTUFBTSw4QkFBOEIsRUFBRSxrQkFBa0IsQ0FBQyxzQkFBc0IsQ0FxQnJGLENBQUMifQ==

package/dest/services/reqresp/batch-tx-requester/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../src/services/reqresp/batch-tx-requester/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAsB,MAAM,0BAA0B,CAAC;AAEvF,eAAO,MAAM,sDAAsD,KAAK,CAAC;AACzE,eAAO,MAAM,qDAAqD,KAAK,CAAC;AACxE,eAAO,MAAM,wCAAwC,IAAI,CAAC;AAC1D,eAAO,MAAM,6CAA6C,IAAI,CAAC;AAE/D,MAAM,WAAW,sBAAsB;IACrC,8CAA8C;IAC9C,wCAAwC,EAAE,MAAM,CAAC;IACjD,6CAA6C;IAC7C,uCAAuC,EAAE,MAAM,CAAC;IAChD,iDAAiD;IACjD,2BAA2B,EAAE,MAAM,CAAC;IACpC,wEAAwE;IACxE,gCAAgC,EAAE,MAAM,CAAC;CAC1C;AAED,eAAO,MAAM,8BAA8B,EAAE,kBAAkB,CAAC,sBAAsB,CAqBrF,CAAC"}

package/dest/services/reqresp/batch-tx-requester/config.js

@@ -0,0 +1,27 @@
+import { numberConfigHelper } from '@aztec/foundation/config';
+export const DEFAULT_BATCH_TX_REQUESTER_SMART_PARALLEL_WORKER_COUNT = 10;
+export const DEFAULT_BATCH_TX_REQUESTER_DUMB_PARALLEL_WORKER_COUNT = 10;
+export const DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE = 8;
+export const DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD = 2;
+export const batchTxRequesterConfigMappings = {
+    batchTxRequesterSmartParallelWorkerCount: {
+        env: 'P2P_BATCH_TX_REQUESTER_SMART_PARALLEL_WORKER_COUNT',
+        description: 'Max concurrent requests to smart peers for batch tx requester.',
+        ...numberConfigHelper(DEFAULT_BATCH_TX_REQUESTER_SMART_PARALLEL_WORKER_COUNT)
+    },
+    batchTxRequesterDumbParallelWorkerCount: {
+        env: 'P2P_BATCH_TX_REQUESTER_DUMB_PARALLEL_WORKER_COUNT',
+        description: 'Max concurrent requests to dumb peers for batch tx requester.',
+        ...numberConfigHelper(DEFAULT_BATCH_TX_REQUESTER_DUMB_PARALLEL_WORKER_COUNT)
+    },
+    batchTxRequesterTxBatchSize: {
+        env: 'P2P_BATCH_TX_REQUESTER_TX_BATCH_SIZE',
+        description: 'Max transactions per request / chunk size for batch tx requester.',
+        ...numberConfigHelper(DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE)
+    },
+    batchTxRequesterBadPeerThreshold: {
+        env: 'P2P_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD',
+        description: 'Failures before a peer is considered bad (see > threshold logic).',
+        ...numberConfigHelper(DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD)
+    }
+};

package/dest/services/reqresp/batch-tx-requester/interface.d.ts

@@ -0,0 +1,46 @@
+import type { ISemaphore } from '@aztec/foundation/queue';
+import type { PeerErrorSeverity } from '@aztec/stdlib/p2p';
+import type { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { PeerId } from '@libp2p/interface';
+import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
+import type { ReqRespInterface } from '../interface.js';
+import type { IPeerCollection } from './peer_collection.js';
+import type { BatchRequestTxValidatorConfig, IBatchRequestTxValidator } from './tx_validator.js';
+export interface IPeerPenalizer {
+    penalizePeer(peerId: PeerId, penalty: PeerErrorSeverity): void;
+}
+export interface ITxMetadataCollection {
+    getMissingTxHashes(): Set<string>;
+    markFetched(peerId: PeerId, tx: Tx): boolean;
+    getTxsToRequestFromThePeer(peer: PeerId): TxHash[];
+    markRequested(txHash: TxHash): void;
+    markInFlightBySmartPeer(txHash: TxHash): void;
+    markNotInFlightBySmartPeer(txHash: TxHash): void;
+    alreadyFetched(txHash: TxHash): boolean;
+    markPeerHas(peerId: PeerId, txHashes: TxHash[]): void;
+}
+/**
+ * Interface for BatchTxRequester dependencies that can be injected from upstream
+ */
+export interface BatchTxRequesterLibP2PService {
+    /** ReqResp interface for sending requests to peers */
+    reqResp: Pick<ReqRespInterface, 'sendBatchRequest' | 'sendRequestToPeer'>;
+    /** Connection sampler for getting peer lists */
+    connectionSampler: Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'>;
+    /** Configuration needed for transaction validation */
+    txValidatorConfig: BatchRequestTxValidatorConfig;
+    /** Peer scoring for penalizing peers */
+    peerScoring: IPeerPenalizer;
+}
+export interface BatchTxRequesterOptions {
+    smartParallelWorkerCount?: number;
+    dumbParallelWorkerCount?: number;
+    txBatchSize?: number;
+    badPeerThreshold?: number;
+    semaphore?: ISemaphore;
+    peerCollection?: IPeerCollection;
+    abortSignal?: AbortSignal;
+    /** Optional tx validator for testing - if not provided, one is created from p2pService.txValidatorConfig */
+    txValidator?: IBatchRequestTxValidator;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW50ZXJmYWNlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi9zcmMvc2VydmljZXMvcmVxcmVzcC9iYXRjaC10eC1yZXF1ZXN0ZXIvaW50ZXJmYWNlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLFVBQVUsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDM0QsT0FBTyxLQUFLLEVBQUUsRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRW5ELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBRWhELE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLE1BQU0sNkNBQTZDLENBQUM7QUFDckYsT0FBTyxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUN4RCxPQUFPLEtBQUssRUFBRSxlQUFlLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUM1RCxPQUFPLEtBQUssRUFBRSw2QkFBNkIsRUFBRSx3QkFBd0IsRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBRWpHLE1BQU0sV0FBVyxjQUFjO0lBQzdCLFlBQVksQ0FBQyxNQUFNLEVBQUUsTUFBTSxFQUFFLE9BQU8sRUFBRSxpQkFBaUIsR0FBRyxJQUFJLENBQUM7Q0FDaEU7QUFFRCxNQUFNLFdBQVcscUJBQXFCO0lBQ3BDLGtCQUFrQixJQUFJLEdBQUcsQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUNsQyxXQUFXLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxFQUFFLEVBQUUsRUFBRSxHQUFHLE9BQU8sQ0FBQztJQUM3QywwQkFBMEIsQ0FBQyxJQUFJLEVBQUUsTUFBTSxHQUFHLE1BQU0sRUFBRSxDQUFDO0lBQ25ELGFBQWEsQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLElBQUksQ0FBQztJQUNwQyx1QkFBdUIsQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLElBQUksQ0FBQztJQUM5QywwQkFBMEIsQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLElBQUksQ0FBQztJQUNqRCxjQUFjLENBQUMsTUFBTSxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUM7SUFFeEMsV0FBVyxDQUFDLE1BQU0sRUFBRSxNQUFNLEVBQUUsUUFBUSxFQUFFLE1BQU0sRUFBRSxHQUFHLElBQUksQ0FBQztDQUN2RDtBQUVEOztHQUVHO0FBQ0gsTUFBTSxXQUFXLDZCQUE2QjtJQUM1QyxzREFBc0Q7SUFDdEQsT0FBTyxFQUFFLElBQUksQ0FBQyxnQkFBZ0IsRUFBRSxrQkFBa0IsR0FBRyxtQkFBbUIsQ0FBQyxDQUFDO0lBQzFFLGdEQUFnRDtJQUNoRCxpQkFBaUIsRUFBRSxJQUFJLENBQUMsaUJBQWlCLEVBQUUsdUNBQXVDLENBQUMsQ0FBQztJQUNwRixzREFBc0Q7SUFDdEQsaUJBQWlCLEVBQUUsNkJBQTZCLENBQUM7SUFDakQsd0NBQXdDO0lBQ3hDLFdBQVcsRUFBRSxjQUFjLENBQUM7Q0FDN0I7QUFFRCxNQUFNLFdBQVcsdUJBQXVCO0lBQ3RDLHdCQUF3QixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2xDLHVCQUF1QixDQUFDLEVBQUUsTUFBTSxDQUFDO0lBQ2pDLFdBQVcsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNyQixnQkFBZ0IsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUUxQixTQUFTLENBQUMsRUFBRSxVQUFVLENBQUM7SUFDdkIsY0FBYyxDQUFDLEVBQUUsZUFBZSxDQUFDO0lBQ2pDLFdBQVcsQ0FBQyxFQUFFLFdBQVcsQ0FBQztJQUMxQiw0R0FBNEc7SUFDNUcsV0FBVyxDQUFDLEVBQUUsd0JBQXdCLENBQUM7Q0FDeEMifQ==

package/dest/services/reqresp/batch-tx-requester/interface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../../../../src/services/reqresp/batch-tx-requester/interface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6CAA6C,CAAC;AACrF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,6BAA6B,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAEjG,MAAM,WAAW,cAAc;IAC7B,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,IAAI,CAAC;CAChE;AAED,MAAM,WAAW,qBAAqB;IACpC,kBAAkB,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;IAC7C,0BAA0B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACnD,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9C,0BAA0B,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACjD,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAExC,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;CACvD;AAED;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC5C,sDAAsD;IACtD,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,GAAG,mBAAmB,CAAC,CAAC;IAC1E,gDAAgD;IAChD,iBAAiB,EAAE,IAAI,CAAC,iBAAiB,EAAE,uCAAuC,CAAC,CAAC;IACpF,sDAAsD;IACtD,iBAAiB,EAAE,6BAA6B,CAAC;IACjD,wCAAwC;IACxC,WAAW,EAAE,cAAc,CAAC;CAC7B;AAED,MAAM,WAAW,uBAAuB;IACtC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,cAAc,CAAC,EAAE,eAAe,CAAC;IACjC,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,4GAA4G;IAC5G,WAAW,CAAC,EAAE,wBAAwB,CAAC;CACxC"}

package/dest/services/reqresp/batch-tx-requester/interface.js

@@ -0,0 +1 @@
+export { };