@aztec/p2p 0.0.1-commit.e61ad554 → 0.0.1-commit.ec5f612

package/src/services/reqresp/batch-tx-requester/config.ts

@@ -0,0 +1,40 @@
+import { type ConfigMappingsType, numberConfigHelper } from '@aztec/foundation/config';
+
+export const DEFAULT_BATCH_TX_REQUESTER_SMART_PARALLEL_WORKER_COUNT = 10;
+export const DEFAULT_BATCH_TX_REQUESTER_DUMB_PARALLEL_WORKER_COUNT = 10;
+export const DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE = 8;
+export const DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD = 2;
+
+export interface BatchTxRequesterConfig {
+  /** Max concurrent requests to smart peers. */
+  batchTxRequesterSmartParallelWorkerCount: number;
+  /** Max concurrent requests to dumb peers. */
+  batchTxRequesterDumbParallelWorkerCount: number;
+  /** Max transactions per request / chunk size. */
+  batchTxRequesterTxBatchSize: number;
+  /** Failures before a peer is considered bad (see > threshold logic). */
+  batchTxRequesterBadPeerThreshold: number;
+}
+
+export const batchTxRequesterConfigMappings: ConfigMappingsType<BatchTxRequesterConfig> = {
+  batchTxRequesterSmartParallelWorkerCount: {
+    env: 'P2P_BATCH_TX_REQUESTER_SMART_PARALLEL_WORKER_COUNT',
+    description: 'Max concurrent requests to smart peers for batch tx requester.',
+    ...numberConfigHelper(DEFAULT_BATCH_TX_REQUESTER_SMART_PARALLEL_WORKER_COUNT),
+  },
+  batchTxRequesterDumbParallelWorkerCount: {
+    env: 'P2P_BATCH_TX_REQUESTER_DUMB_PARALLEL_WORKER_COUNT',
+    description: 'Max concurrent requests to dumb peers for batch tx requester.',
+    ...numberConfigHelper(DEFAULT_BATCH_TX_REQUESTER_DUMB_PARALLEL_WORKER_COUNT),
+  },
+  batchTxRequesterTxBatchSize: {
+    env: 'P2P_BATCH_TX_REQUESTER_TX_BATCH_SIZE',
+    description: 'Max transactions per request / chunk size for batch tx requester.',
+    ...numberConfigHelper(DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE),
+  },
+  batchTxRequesterBadPeerThreshold: {
+    env: 'P2P_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD',
+    description: 'Failures before a peer is considered bad (see > threshold logic).',
+    ...numberConfigHelper(DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD),
+  },
+};

package/src/services/reqresp/batch-tx-requester/interface.ts

@@ -0,0 +1,53 @@
+import type { ISemaphore } from '@aztec/foundation/queue';
+import type { PeerErrorSeverity } from '@aztec/stdlib/p2p';
+import type { Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { PeerId } from '@libp2p/interface';
+
+import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
+import type { ReqRespInterface } from '../interface.js';
+import type { IPeerCollection } from './peer_collection.js';
+import type { BatchRequestTxValidatorConfig, IBatchRequestTxValidator } from './tx_validator.js';
+
+export interface IPeerPenalizer {
+  penalizePeer(peerId: PeerId, penalty: PeerErrorSeverity): void;
+}
+
+export interface ITxMetadataCollection {
+  getMissingTxHashes(): Set<string>;
+  markFetched(peerId: PeerId, tx: Tx): boolean;
+  getTxsToRequestFromThePeer(peer: PeerId): TxHash[];
+  markRequested(txHash: TxHash): void;
+  markInFlightBySmartPeer(txHash: TxHash): void;
+  markNotInFlightBySmartPeer(txHash: TxHash): void;
+  alreadyFetched(txHash: TxHash): boolean;
+  // Returns true if tx was marked as fetched, false if it was already marked as fetched
+  markPeerHas(peerId: PeerId, txHashes: TxHash[]): void;
+}
+
+/**
+ * Interface for BatchTxRequester dependencies that can be injected from upstream
+ */
+export interface BatchTxRequesterLibP2PService {
+  /** ReqResp interface for sending requests to peers */
+  reqResp: Pick<ReqRespInterface, 'sendBatchRequest' | 'sendRequestToPeer'>;
+  /** Connection sampler for getting peer lists */
+  connectionSampler: Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'>;
+  /** Configuration needed for transaction validation */
+  txValidatorConfig: BatchRequestTxValidatorConfig;
+  /** Peer scoring for penalizing peers */
+  peerScoring: IPeerPenalizer;
+}
+
+export interface BatchTxRequesterOptions {
+  smartParallelWorkerCount?: number;
+  dumbParallelWorkerCount?: number;
+  txBatchSize?: number;
+  badPeerThreshold?: number;
+  //Injectable for testing purposes
+  semaphore?: ISemaphore;
+  peerCollection?: IPeerCollection;
+  abortSignal?: AbortSignal;
+  /** Optional tx validator for testing - if not provided, one is created from p2pService.txValidatorConfig */
+  txValidator?: IBatchRequestTxValidator;
+}

package/src/services/reqresp/batch-tx-requester/missing_txs.ts

@@ -0,0 +1,161 @@
+import { type Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { PeerId } from '@libp2p/interface';
+
+import type { IMissingTxsTracker } from '../../tx_collection/missing_txs_tracker.js';
+import { DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE } from './config.js';
+import type { ITxMetadataCollection } from './interface.js';
+
+class MissingTxMetadata {
+  constructor(
+    public readonly txHash: string,
+    public requestedCount = 0,
+    public inFlightCount = 0,
+    public tx: Tx | undefined = undefined,
+    public readonly peers = new Set<string>(),
+  ) {}
+
+  public markAsRequested() {
+    this.requestedCount++;
+  }
+
+  public markInFlight() {
+    this.inFlightCount++;
+  }
+
+  public markNotInFlight() {
+    this.inFlightCount = Math.max(--this.inFlightCount, 0);
+  }
+
+  public isInFlight(): boolean {
+    return this.inFlightCount > 0;
+  }
+}
+
+/*
+ * Single source or truth for transactions we are fetching
+ * This could be better optimized but given expected count of missing txs (N < 100)
+ * At the moment there is no need for it. And benefit is that we have everything in single store
+ * */
+export class MissingTxMetadataCollection implements ITxMetadataCollection {
+  private txMetadata = new Map<string, MissingTxMetadata>();
+
+  constructor(
+    private missingTxsTracker: IMissingTxsTracker,
+    private readonly txBatchSize: number = DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE,
+  ) {
+    missingTxsTracker.missingTxHashes.forEach(hash => this.txMetadata.set(hash, new MissingTxMetadata(hash)));
+  }
+
+  public getPrioritizingNotInFlightAndLowerRequestCount(txs: string[]): MissingTxMetadata[] {
+    const filtered = Array.from(this.txMetadata.values()).filter(txMeta => txs.includes(txMeta.txHash.toString()));
+
+    const [notInFlight, inFlight] = filtered.reduce<[MissingTxMetadata[], MissingTxMetadata[]]>(
+      (buckets, tx) => {
+        tx.isInFlight() ? buckets[1].push(tx) : buckets[0].push(tx);
+        return buckets;
+      },
+      [[], []],
+    );
+
+    notInFlight.sort((a, b) => a.requestedCount - b.requestedCount);
+    inFlight.sort((a, b) => a.inFlightCount - b.inFlightCount);
+
+    return [...notInFlight, ...inFlight];
+  }
+
+  public getMissingTxHashes(): Set<string> {
+    return this.missingTxsTracker.missingTxHashes;
+  }
+
+  public getTxsPeerHas(peer: PeerId): Set<string> {
+    const peerIdStr = peer.toString();
+    const txsPeerHas = new Set<string>();
+
+    this.txMetadata.values().forEach(txMeta => {
+      if (txMeta.peers.has(peerIdStr)) {
+        txsPeerHas.add(txMeta.txHash.toString());
+      }
+    });
+
+    return txsPeerHas;
+  }
+
+  public getTxsToRequestFromThePeer(peer: PeerId): TxHash[] {
+    const txsPeerHas = this.getTxsPeerHas(peer);
+    const missingTxHashes = this.getMissingTxHashes();
+
+    const txsToRequest = txsPeerHas.intersection(missingTxHashes);
+
+    if (txsToRequest.size >= this.txBatchSize) {
+      return this.getPrioritizingNotInFlightAndLowerRequestCount(Array.from(txsToRequest))
+        .map(t => TxHash.fromString(t.txHash))
+        .slice(0, this.txBatchSize);
+    }
+
+    // Otherwise fill the txs to request till txBatchSize with random txs we are missing
+    // Who knows, maybe we get lucky and peer received these txs in the meantime
+
+    const countToFill = this.txBatchSize - txsToRequest.size;
+    const txsToFill = this.getPrioritizingNotInFlightAndLowerRequestCount(
+      Array.from(this.getMissingTxHashes().difference(txsToRequest)),
+    )
+      .slice(0, countToFill)
+      .map(t => TxHash.fromString(t.txHash));
+
+    return [...Array.from(txsToRequest).map(t => TxHash.fromString(t)), ...txsToFill];
+  }
+
+  public markRequested(txHash: TxHash) {
+    this.txMetadata.get(txHash.toString())?.markAsRequested();
+  }
+
+  /*
+   * This should be called only when requesting tx from smart peer
+   * Because the smart peer should return this tx, whereas
+   * "dumb" peer might return it, or might not - we don't know
+   * */
+  public markInFlightBySmartPeer(txHash: TxHash) {
+    this.txMetadata.get(txHash.toString())?.markInFlight();
+  }
+
+  /*
+   * This should be called only when requesting tx from smart peer
+   * Because the smart peer should return this tx, whereas
+   * "dumb" peer might return it, or might not - we don't know*/
+  public markNotInFlightBySmartPeer(txHash: TxHash) {
+    this.txMetadata.get(txHash.toString())?.markNotInFlight();
+  }
+
+  public alreadyFetched(txHash: TxHash): boolean {
+    return !this.missingTxsTracker.isMissing(txHash.toString());
+  }
+
+  public markFetched(peerId: PeerId, tx: Tx): boolean {
+    const txHashStr = tx.txHash.toString();
+    const txMeta = this.txMetadata.get(txHashStr);
+    if (!txMeta) {
+      //TODO: what to do about peer which sent txs we didn't request?
+      // 1. don't request from it in the scope of this batch request
+      // 2. ban it immediately?
+      // 3. track it and ban it?
+      //
+      return false;
+    }
+
+    txMeta.peers.add(peerId.toString());
+    return this.missingTxsTracker.markFetched(tx);
+  }
+
+  public markPeerHas(peerId: PeerId, txHash: TxHash[]) {
+    const peerIdStr = peerId.toString();
+    txHash
+      .map(t => t.toString())
+      .forEach(txh => {
+        const txMeta = this.txMetadata.get(txh);
+        if (txMeta) {
+          txMeta.peers.add(peerIdStr);
+        }
+      });
+  }
+}

package/src/services/reqresp/batch-tx-requester/peer_collection.ts

@@ -0,0 +1,244 @@
+import type { DateProvider } from '@aztec/foundation/timer';
+import type { PeerErrorSeverity } from '@aztec/stdlib/p2p';
+
+import type { PeerId } from '@libp2p/interface';
+import { peerIdFromString } from '@libp2p/peer-id';
+
+import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
+import { DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD } from './config.js';
+import type { IPeerPenalizer } from './interface.js';
+
+export const RATE_LIMIT_EXCEEDED_PEER_CACHE_TTL = 1000; // 1s
+
+export interface IPeerCollection {
+  markPeerSmart(peerId: PeerId): void;
+
+  /** Sample next peer in round-robin fashion. No smart peers if returns undefined */
+  nextSmartPeerToQuery(): PeerId | undefined;
+  /** Sample next peer in round-robin fashion. No dumb peers if returns undefined */
+  nextDumbPeerToQuery(): PeerId | undefined;
+
+  thereAreSomeDumbRatelimitExceededPeers(): boolean;
+  penalisePeer(peerId: PeerId, severity: PeerErrorSeverity): void;
+  unMarkPeerAsBad(peerId: PeerId): void;
+  getBadPeers(): Set<string>;
+  markPeerInFlight(peerId: PeerId): void;
+  unMarkPeerInFlight(peerId: PeerId): void;
+  markPeerRateLimitExceeded(peerId: PeerId): void;
+  getRateLimitExceededPeers(): Set<string>;
+  getPeerRateLimitDelayMs(peerId: PeerId): number | undefined;
+  getNextDumbPeerAvailabilityDelayMs(): number | undefined;
+  getNextSmartPeerAvailabilityDelayMs(): number | undefined;
+}
+
+export class PeerCollection implements IPeerCollection {
+  private readonly smartPeers = new Set<string>();
+  private readonly inFlightPeers = new Set<string>();
+  private readonly rateLimitExceededPeers = new Map<string, number>();
+  private readonly peerPenaltyCounters = new Map<string, number>();
+  private readonly badPeers = new Set<string>();
+
+  constructor(
+    private readonly connectionSampler: Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'>,
+    private readonly pinnedPeerId: PeerId | undefined,
+    private readonly dateProvider: DateProvider,
+    private readonly badPeerThreshold: number = DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD,
+    private readonly peerPenalizer?: IPeerPenalizer,
+  ) {
+    // Pinned peer is treated specially, always mark it as in-flight
+    // and never return it as part of smart/dumb peers
+    if (this.pinnedPeerId) {
+      const peerIdStr = this.pinnedPeerId.toString();
+      this.inFlightPeers.add(peerIdStr);
+    }
+  }
+
+  public markPeerSmart(peerId: PeerId): void {
+    this.smartPeers.add(peerId.toString());
+  }
+
+  // We keep track of all peers that are queried for peer sampling algorithm
+  private queriedSmartPeers: Set<string> = new Set<string>();
+  private queriedDumbPeers: Set<string> = new Set<string>();
+
+  private static nextPeer(allPeers: Set<string>, queried: Set<string>): PeerId | undefined {
+    if (allPeers.size === 0) {
+      return undefined;
+    }
+    const availablePeers = allPeers.difference(queried);
+    let [first] = availablePeers;
+    if (first === undefined) {
+      // We queried all peers. Start over
+      [first] = allPeers;
+      queried.clear();
+    }
+    queried.add(first);
+    return peerIdFromString(first);
+  }
+
+  public nextSmartPeerToQuery(): PeerId | undefined {
+    return PeerCollection.nextPeer(this.availableSmartPeers, this.queriedSmartPeers);
+  }
+
+  public nextDumbPeerToQuery(): PeerId | undefined {
+    return PeerCollection.nextPeer(this.availableDumbPeers, this.queriedDumbPeers);
+  }
+
+  private get availableSmartPeers(): Set<string> {
+    return this.peers.intersection(
+      this.smartPeers.difference(this.getBadPeers().union(this.inFlightPeers).union(this.getRateLimitExceededPeers())),
+    );
+  }
+
+  private get availableDumbPeers(): Set<string> {
+    return this.peers.difference(
+      this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers()),
+    );
+  }
+
+  public thereAreSomeDumbRatelimitExceededPeers(): boolean {
+    return (
+      this.getRateLimitExceededPeers().difference(this.smartPeers.union(this.badPeers).union(this.inFlightPeers)).size >
+      0
+    );
+  }
+
+  public markPeerInFlight(peerId: PeerId) {
+    this.inFlightPeers.add(peerId.toString());
+  }
+
+  public unMarkPeerInFlight(peerId: PeerId) {
+    // Never unmark the pinned peer as in-flight
+    if (this.pinnedPeerId && this.pinnedPeerId.toString() === peerId.toString()) {
+      return;
+    }
+    this.inFlightPeers.delete(peerId.toString());
+  }
+
+  public markPeerRateLimitExceeded(peerId: PeerId) {
+    const ttl = this.dateProvider.now() + RATE_LIMIT_EXCEEDED_PEER_CACHE_TTL;
+    this.rateLimitExceededPeers.set(peerId.toString(), ttl);
+  }
+
+  public getRateLimitExceededPeers(): Set<string> {
+    const now = this.dateProvider.now();
+    const rateLimitedPeers = new Set<string>();
+
+    for (const [peerId, expirationTime] of this.rateLimitExceededPeers) {
+      if (expirationTime <= now) {
+        this.rateLimitExceededPeers.delete(peerId);
+      } else {
+        rateLimitedPeers.add(peerId);
+      }
+    }
+
+    return rateLimitedPeers;
+  }
+
+  public penalisePeer(peerId: PeerId, severity: PeerErrorSeverity): void {
+    const key = peerId.toString();
+    const newPenaltyCount = (this.peerPenaltyCounters.get(key) ?? 0) + 1;
+    this.peerPenaltyCounters.set(key, newPenaltyCount);
+    this.peerPenalizer?.penalizePeer(peerId, severity);
+    if (newPenaltyCount > this.badPeerThreshold) {
+      this.badPeers.add(key);
+    }
+  }
+
+  public unMarkPeerAsBad(peerId: PeerId) {
+    const key = peerId.toString();
+    this.badPeers.delete(key);
+    this.peerPenaltyCounters.delete(key);
+  }
+
+  public getBadPeers(): Set<string> {
+    return new Set(this.badPeers);
+  }
+
+  public getPeerRateLimitDelayMs(peerId: PeerId): number | undefined {
+    const key = peerId.toString();
+    const expiry = this.rateLimitExceededPeers.get(key);
+    const peerIsNotRateLimited = expiry === undefined;
+    if (peerIsNotRateLimited) {
+      return undefined;
+    }
+
+    const now = this.dateProvider.now();
+    const rateLimitHasExpired = expiry <= now;
+    if (rateLimitHasExpired) {
+      this.rateLimitExceededPeers.delete(key);
+      return undefined;
+    }
+    return expiry - now;
+  }
+
+  public getNextDumbPeerAvailabilityDelayMs(): number | undefined {
+    // Note: this _is_ suboptimal
+    // (we could've tracked rate limits ) per dumb/smart peers - different collections
+    // but everything is in memory and small scale so this, wile suboptimal is not slow
+    return this.getNextRateLimitDelayMs(
+      peerIdStr =>
+        !this.smartPeers.has(peerIdStr) &&
+        !this.getBadPeers().has(peerIdStr) &&
+        !this.inFlightPeers.has(peerIdStr) &&
+        this.peers.has(peerIdStr),
+    );
+  }
+
+  public getNextSmartPeerAvailabilityDelayMs(): number | undefined {
+    return this.getNextRateLimitDelayMs(
+      peerIdStr =>
+        this.smartPeers.has(peerIdStr) && !this.getBadPeers().has(peerIdStr) && !this.inFlightPeers.has(peerIdStr),
+    );
+  }
+
+  private getNextRateLimitDelayMs(filter: (peerIdStr: string) => boolean): number | undefined {
+    const now = this.dateProvider.now();
+    let minExpiry: number | undefined;
+
+    for (const [peerIdStr, expiry] of this.rateLimitExceededPeers) {
+      const rateLimitHasExpired = expiry <= now;
+      if (rateLimitHasExpired) {
+        this.rateLimitExceededPeers.delete(peerIdStr);
+        continue;
+      }
+
+      const peerDoesNotMatchFilter = !filter(peerIdStr);
+      if (peerDoesNotMatchFilter) {
+        continue;
+      }
+
+      minExpiry = minExpiry === undefined ? expiry : Math.min(minExpiry, expiry);
+    }
+
+    const noRateLimitedPeersMatchFilter = minExpiry === undefined;
+    if (noRateLimitedPeersMatchFilter) {
+      return undefined;
+    }
+
+    return minExpiry! - now;
+  }
+
+  private orderedPeers: Set<string> = new Set();
+
+  private get peers(): Set<string> {
+    const pinnedStr = this.pinnedPeerId?.toString();
+    const currentlyConnected = new Set(
+      this.connectionSampler
+        .getPeerListSortedByConnectionCountAsc()
+        .map(p => p.toString())
+        .filter(p => p !== pinnedStr),
+    );
+
+    // Remove disconnected peers, preserving order of the rest.
+    this.orderedPeers = this.orderedPeers.intersection(currentlyConnected);
+
+    // Append newly connected peers at the end (lowest priority).
+    for (const peer of currentlyConnected) {
+      if (!this.orderedPeers.has(peer)) {
+        this.orderedPeers.add(peer);
+      }
+    }
+    return this.orderedPeers;
+  }
+}

package/src/services/reqresp/batch-tx-requester/tx_validator.ts

@@ -0,0 +1,37 @@
+import type { ClientProtocolCircuitVerifier } from '@aztec/stdlib/interfaces/server';
+import { Tx, type TxValidationResult, type TxValidator } from '@aztec/stdlib/tx';
+
+import { createTxValidatorForReqResponseReceivedTxs } from '../../../msg_validators/index.js';
+
+export interface BatchRequestTxValidatorConfig {
+  l1ChainId: number;
+  rollupVersion: number;
+  proofVerifier: ClientProtocolCircuitVerifier;
+}
+
+export interface IBatchRequestTxValidator {
+  validateRequestedTx(tx: Tx): Promise<TxValidationResult>;
+  validateRequestedTxs(txs: Tx[]): Promise<TxValidationResult[]>;
+}
+
+export class BatchRequestTxValidator implements IBatchRequestTxValidator {
+  readonly txValidator: TxValidator;
+  constructor(private readonly config: BatchRequestTxValidatorConfig) {
+    this.txValidator = BatchRequestTxValidator.createRequestedTxValidator(this.config);
+  }
+
+  public async validateRequestedTx(tx: Tx): Promise<TxValidationResult> {
+    return await this.txValidator.validateTx(tx);
+  }
+
+  public async validateRequestedTxs(txs: Tx[]): Promise<TxValidationResult[]> {
+    return await Promise.all(txs.map(tx => this.validateRequestedTx(tx)));
+  }
+
+  static createRequestedTxValidator(config: BatchRequestTxValidatorConfig): TxValidator {
+    return createTxValidatorForReqResponseReceivedTxs(config.proofVerifier, {
+      l1ChainId: config.l1ChainId,
+      rollupVersion: config.rollupVersion,
+    });
+  }
+}

package/src/services/reqresp/connection-sampler/batch_connection_sampler.ts

@@ -20,6 +20,8 @@ import type { ConnectionSampler } from './connection_sampler.js';
 export class BatchConnectionSampler {
   private readonly batch: PeerId[] = [];
   private readonly requestsPerPeer: number;
+  /** Tracks peer-index combinations that returned empty/invalid responses */
+  private readonly failedPeerIndices: Map<string, Set<number>> = new Map();
 
   constructor(
     private readonly connectionSampler: ConnectionSampler,
@@ -44,10 +46,12 @@ export class BatchConnectionSampler {
   }
 
   /**
-   * Gets the peer responsible for handling a specific request index
+   * Gets the peer responsible for handling a specific request index.
+   * If the primary peer has previously failed for this index, tries other peers.
+   * If all batch peers have failed, attempts to sample a new peer.
    *
    * @param index - The request index
-   * @returns The peer assigned to handle this request
+   * @returns The peer assigned to handle this request, or undefined if no peer available
    */
   getPeerForRequest(index: number): PeerId | undefined {
     if (this.batch.length === 0) {
@@ -55,8 +59,65 @@ export class BatchConnectionSampler {
     }
 
     // Calculate which peer bucket this index belongs to
-    const peerIndex = Math.floor(index / this.requestsPerPeer) % this.batch.length;
-    return this.batch[peerIndex];
+    const primaryPeerIndex = Math.floor(index / this.requestsPerPeer) % this.batch.length;
+
+    // Try peers starting from primary, wrapping around
+    for (let offset = 0; offset < this.batch.length; offset++) {
+      const peerIndex = (primaryPeerIndex + offset) % this.batch.length;
+      const peer = this.batch[peerIndex];
+      const peerKey = peer.toString();
+
+      const failedIndices = this.failedPeerIndices.get(peerKey);
+      if (!failedIndices || !failedIndices.has(index)) {
+        return peer;
+      }
+    }
+
+    // All batch peers have failed for this index - try to sample a new peer
+    const newPeer = this.sampleNewPeer();
+    if (newPeer) {
+      return newPeer;
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Attempts to sample a new peer that isn't already in the batch.
+   * If successful, adds the peer to the batch.
+   *
+   * @returns The new peer if one was sampled, undefined otherwise
+   */
+  private sampleNewPeer(): PeerId | undefined {
+    // Exclude all current batch peers
+    const excluding = new Map(this.batch.map(p => [p.toString(), true] as const));
+    const newPeer = this.connectionSampler.getPeer(excluding);
+
+    if (newPeer) {
+      this.batch.push(newPeer);
+      this.logger.trace('Sampled new peer for exhausted index', { newPeer: newPeer.toString() });
+      return newPeer;
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Marks that a peer returned an empty/invalid response for a specific request index.
+   * The peer will not be assigned this index again.
+   *
+   * @param peerId - The peer that failed
+   * @param index - The request index that failed
+   */
+  markPeerFailedForIndex(peerId: PeerId, index: number): void {
+    const peerKey = peerId.toString();
+    let failedIndices = this.failedPeerIndices.get(peerKey);
+    if (!failedIndices) {
+      failedIndices = new Set();
+      this.failedPeerIndices.set(peerKey, failedIndices);
+    }
+    failedIndices.add(index);
+    this.logger.trace('Marked peer failed for index', { peerId: peerKey, index });
   }
 
   /**

package/src/services/reqresp/connection-sampler/connection_sampler.ts

@@ -125,6 +125,22 @@ export class ConnectionSampler {
     return { peer: lastPeer, sampledPeers };
   }
 
+  /*
+   * Returns all peers sorted by connection count ascending,
+   * meaning that the peers with the least number of active connections are earlier in an array
+   *
+   * @param: excluding - peers to exclude
+   * @return: list of peer ids
+   * */
+  public getPeerListSortedByConnectionCountAsc(excluding?: Set<string>): PeerId[] {
+    return this.libp2p
+      .getPeers()
+      .filter(id => !excluding?.has(id.toString()))
+      .map(id => ({ id, count: this.activeConnectionsCount.get(id.toString()) ?? 0 }))
+      .sort((a, b) => a.count - b.count)
+      .map(p => p.id);
+  }
+
   /**
    * Samples a batch of unique peers from the libp2p node, prioritizing peers without active connections
    *