@aztec/p2p 0.0.1-commit.e2b2873ed → 0.0.1-commit.e304674f1

package/src/services/reqresp/batch-tx-requester/peer_collection.ts

@@ -2,18 +2,23 @@ import type { DateProvider } from '@aztec/foundation/timer';
 import type { PeerErrorSeverity } from '@aztec/stdlib/p2p';
 
 import type { PeerId } from '@libp2p/interface';
+import { peerIdFromString } from '@libp2p/peer-id';
 
+import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
 import { DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD } from './config.js';
 import type { IPeerPenalizer } from './interface.js';
 
 export const RATE_LIMIT_EXCEEDED_PEER_CACHE_TTL = 1000; // 1s
 
 export interface IPeerCollection {
-  getAllPeers(): Set<string>;
-  getSmartPeers(): Set<string>;
   markPeerSmart(peerId: PeerId): void;
-  getSmartPeersToQuery(): Array<string>;
-  getDumbPeersToQuery(): Array<string>;
+  markPeerDumb(peerId: PeerId): void;
+
+  /** Sample next peer in round-robin fashion. No smart peers if returns undefined */
+  nextSmartPeerToQuery(): PeerId | undefined;
+  /** Sample next peer in round-robin fashion. No dumb peers if returns undefined */
+  nextDumbPeerToQuery(): PeerId | undefined;
+
   thereAreSomeDumbRatelimitExceededPeers(): boolean;
   penalisePeer(peerId: PeerId, severity: PeerErrorSeverity): void;
   unMarkPeerAsBad(peerId: PeerId): void;
@@ -28,8 +33,6 @@ export interface IPeerCollection {
 }
 
 export class PeerCollection implements IPeerCollection {
-  private readonly peers;
-
   private readonly smartPeers = new Set<string>();
   private readonly inFlightPeers = new Set<string>();
   private readonly rateLimitExceededPeers = new Map<string, number>();
@@ -37,46 +40,64 @@ export class PeerCollection implements IPeerCollection {
   private readonly badPeers = new Set<string>();
 
   constructor(
-    initialPeers: PeerId[],
+    private readonly connectionSampler: Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'>,
     private readonly pinnedPeerId: PeerId | undefined,
     private readonly dateProvider: DateProvider,
     private readonly badPeerThreshold: number = DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD,
     private readonly peerPenalizer?: IPeerPenalizer,
   ) {
-    this.peers = new Set(initialPeers.map(peer => peer.toString()));
-
-    // Pinned peer is treaded specially, always mark it as in-flight
+    // Pinned peer is treated specially, always mark it as in-flight
     // and never return it as part of smart/dumb peers
     if (this.pinnedPeerId) {
       const peerIdStr = this.pinnedPeerId.toString();
       this.inFlightPeers.add(peerIdStr);
-      this.peers.delete(peerIdStr);
     }
   }
 
-  public getAllPeers(): Set<string> {
-    return this.peers;
+  public markPeerSmart(peerId: PeerId): void {
+    this.smartPeers.add(peerId.toString());
   }
 
-  public getSmartPeers(): Set<string> {
-    return this.smartPeers;
+  public markPeerDumb(peerId: PeerId): void {
+    this.smartPeers.delete(peerId.toString());
   }
 
-  public markPeerSmart(peerId: PeerId): void {
-    this.smartPeers.add(peerId.toString());
+  // We keep track of all peers that are queried for peer sampling algorithm
+  private queriedSmartPeers: Set<string> = new Set<string>();
+  private queriedDumbPeers: Set<string> = new Set<string>();
+
+  private static nextPeer(allPeers: Set<string>, queried: Set<string>): PeerId | undefined {
+    if (allPeers.size === 0) {
+      return undefined;
+    }
+    const availablePeers = allPeers.difference(queried);
+    let [first] = availablePeers;
+    if (first === undefined) {
+      // We queried all peers. Start over
+      [first] = allPeers;
+      queried.clear();
+    }
+    queried.add(first);
+    return peerIdFromString(first);
   }
 
-  public getSmartPeersToQuery(): Array<string> {
-    return Array.from(
+  public nextSmartPeerToQuery(): PeerId | undefined {
+    return PeerCollection.nextPeer(this.availableSmartPeers, this.queriedSmartPeers);
+  }
+
+  public nextDumbPeerToQuery(): PeerId | undefined {
+    return PeerCollection.nextPeer(this.availableDumbPeers, this.queriedDumbPeers);
+  }
+
+  private get availableSmartPeers(): Set<string> {
+    return this.peers.intersection(
       this.smartPeers.difference(this.getBadPeers().union(this.inFlightPeers).union(this.getRateLimitExceededPeers())),
     );
   }
 
-  public getDumbPeersToQuery(): Array<string> {
-    return Array.from(
-      this.peers.difference(
-        this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers()),
-      ),
+  private get availableDumbPeers(): Set<string> {
+    return this.peers.difference(
+      this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers()),
     );
   }
 
@@ -202,4 +223,27 @@ export class PeerCollection implements IPeerCollection {
 
     return minExpiry! - now;
   }
+
+  private orderedPeers: Set<string> = new Set();
+
+  private get peers(): Set<string> {
+    const pinnedStr = this.pinnedPeerId?.toString();
+    const currentlyConnected = new Set(
+      this.connectionSampler
+        .getPeerListSortedByConnectionCountAsc()
+        .map(p => p.toString())
+        .filter(p => p !== pinnedStr),
+    );
+
+    // Remove disconnected peers, preserving order of the rest.
+    this.orderedPeers = this.orderedPeers.intersection(currentlyConnected);
+
+    // Append newly connected peers at the end (lowest priority).
+    for (const peer of currentlyConnected) {
+      if (!this.orderedPeers.has(peer)) {
+        this.orderedPeers.add(peer);
+      }
+    }
+    return this.orderedPeers;
+  }
 }

package/src/services/reqresp/batch-tx-requester/tx_validator.ts

@@ -1,7 +1,7 @@
 import type { ClientProtocolCircuitVerifier } from '@aztec/stdlib/interfaces/server';
 import { Tx, type TxValidationResult, type TxValidator } from '@aztec/stdlib/tx';
 
-import { createTxReqRespValidator } from '../../../msg_validators/tx_validator/factory.js';
+import { createTxValidatorForReqResponseReceivedTxs } from '../../../msg_validators/index.js';
 
 export interface BatchRequestTxValidatorConfig {
   l1ChainId: number;
@@ -29,7 +29,7 @@ export class BatchRequestTxValidator implements IBatchRequestTxValidator {
   }
 
   static createRequestedTxValidator(config: BatchRequestTxValidatorConfig): TxValidator {
-    return createTxReqRespValidator(config.proofVerifier, {
+    return createTxValidatorForReqResponseReceivedTxs(config.proofVerifier, {
       l1ChainId: config.l1ChainId,
       rollupVersion: config.rollupVersion,
     });

package/src/services/reqresp/interface.ts

@@ -1,6 +1,3 @@
-import { Fr } from '@aztec/foundation/curves/bn254';
-import { L2Block } from '@aztec/stdlib/block';
-import { MAX_L2_BLOCK_SIZE_KB } from '@aztec/stdlib/p2p';
 import { TxArray, TxHashArray } from '@aztec/stdlib/tx';
 
 import type { PeerId } from '@libp2p/interface';
@@ -24,7 +21,6 @@ export const PING_PROTOCOL = '/aztec/req/ping/1.0.0';
 export const STATUS_PROTOCOL = '/aztec/req/status/1.0.0';
 export const GOODBYE_PROTOCOL = '/aztec/req/goodbye/1.0.0';
 export const TX_REQ_PROTOCOL = '/aztec/req/tx/1.0.0';
-export const BLOCK_REQ_PROTOCOL = '/aztec/req/block/1.0.0';
 export const AUTH_PROTOCOL = '/aztec/req/auth/1.0.0';
 export const BLOCK_TXS_REQ_PROTOCOL = '/aztec/req/block_txs/1.0.0';
 
@@ -33,7 +29,6 @@ export enum ReqRespSubProtocol {
   STATUS = STATUS_PROTOCOL,
   GOODBYE = GOODBYE_PROTOCOL,
   TX = TX_REQ_PROTOCOL,
-  BLOCK = BLOCK_REQ_PROTOCOL,
   AUTH = AUTH_PROTOCOL,
   BLOCK_TXS = BLOCK_TXS_REQ_PROTOCOL,
 }
@@ -105,7 +100,6 @@ export const DEFAULT_SUB_PROTOCOL_VALIDATORS: ReqRespSubProtocolValidators = {
   [ReqRespSubProtocol.STATUS]: noopValidator,
   [ReqRespSubProtocol.TX]: noopValidator,
   [ReqRespSubProtocol.GOODBYE]: noopValidator,
-  [ReqRespSubProtocol.BLOCK]: noopValidator,
   [ReqRespSubProtocol.AUTH]: noopValidator,
   [ReqRespSubProtocol.BLOCK_TXS]: noopValidator,
 };
@@ -203,10 +197,6 @@ export const subProtocolMap = {
     request: RequestableBuffer,
     response: RequestableBuffer,
   },
-  [ReqRespSubProtocol.BLOCK]: {
-    request: Fr, // block number
-    response: L2Block,
-  },
   [ReqRespSubProtocol.AUTH]: {
     request: AuthRequest,
     response: AuthResponse,
@@ -229,7 +219,6 @@ export type ExpectedResponseSizeCalculator = (requestBuffer: Buffer) => number;
 export const subProtocolSizeCalculators: Record<ReqRespSubProtocol, ExpectedResponseSizeCalculator> = {
   [ReqRespSubProtocol.TX]: calculateTxResponseSize,
   [ReqRespSubProtocol.BLOCK_TXS]: calculateBlockTxsResponseSize,
-  [ReqRespSubProtocol.BLOCK]: () => MAX_L2_BLOCK_SIZE_KB,
   [ReqRespSubProtocol.STATUS]: () => 1,
   [ReqRespSubProtocol.PING]: () => 1,
   [ReqRespSubProtocol.AUTH]: () => 1,

package/src/services/reqresp/metrics.ts

@@ -26,7 +26,6 @@ export class ReqRespMetrics {
         ReqRespSubProtocol.STATUS,
         ReqRespSubProtocol.GOODBYE,
         ReqRespSubProtocol.TX,
-        ReqRespSubProtocol.BLOCK,
         ReqRespSubProtocol.AUTH,
         ReqRespSubProtocol.BLOCK_TXS,
       ],

package/src/services/reqresp/protocols/index.ts

@@ -5,6 +5,5 @@ export * from './ping.js';
 export * from './status.js';
 export * from './tx.js';
 export * from './goodbye.js';
-export * from './block.js';
 export * from './auth.js';
 export * from './block_txs/index.js';

package/src/services/reqresp/protocols/tx.ts

@@ -51,9 +51,7 @@ export function reqRespTxHandler(mempools: MemPools): ReqRespSubProtocolHandler
  *  Per: https://github.com/AztecProtocol/aztec-packages/issues/15149#issuecomment-2999054485
  *  we define Q as max number of transactions per batch, the comment explains why we use 8.
  */
-//TODO: (mralj) chunk size should by default be 8, this is just temporary until the protocol is implemented correctly
-//more info:  https://github.com/AztecProtocol/aztec-packages/pull/15516#pullrequestreview-2995474321
-export function chunkTxHashesRequest(hashes: TxHash[], chunkSize = 1): Array<TxHashArray> {
+export function chunkTxHashesRequest(hashes: TxHash[], chunkSize = 8): Array<TxHashArray> {
   return chunk(hashes, chunkSize).map(chunk => new TxHashArray(...chunk));
 }
 

package/src/services/reqresp/rate-limiter/rate_limiter.ts

@@ -97,9 +97,10 @@ export function prettyPrintRateLimitStatus(status: RateLimitStatus) {
  * 2. Individual rate limits for each peer.
  *
  * How it works:
- * - When a request comes in, it first checks against the global rate limit.
- * - If the global limit allows, it then checks against the specific peer's rate limit.
- * - The request is only allowed if both the global and peer-specific limits allow it.
+ * - When a request comes in, it first checks against the peer's individual rate limit.
+ * - If the peer limit allows, it then checks against the global rate limit.
+ * - The request is only allowed if both the peer-specific and global limits allow it.
+ * - Checking peer limit first ensures a rate-limited peer cannot exhaust the global quota.
  * - It automatically creates and manages rate limiters for new peers as they make requests.
  * - It periodically cleans up rate limiters for inactive peers to conserve memory.
  *
@@ -119,10 +120,6 @@ export class SubProtocolRateLimiter {
   }
 
   allow(peerId: PeerId): RateLimitStatus {
-    if (!this.globalLimiter.allow()) {
-      return RateLimitStatus.DeniedGlobal;
-    }
-
     const peerIdStr = peerId.toString();
     let peerLimiter: PeerRateLimiter | undefined = this.peerLimiters.get(peerIdStr);
     if (!peerLimiter) {
@@ -135,10 +132,17 @@ export class SubProtocolRateLimiter {
     } else {
       peerLimiter.lastAccess = Date.now();
     }
-    const peerLimitAllowed = peerLimiter.limiter.allow();
-    if (!peerLimitAllowed) {
+
+    // Check peer limit first: a rate-limited peer must not consume global quota,
+    // otherwise one spamming peer can starve all others by exhausting the global bucket.
+    if (!peerLimiter.limiter.allow()) {
       return RateLimitStatus.DeniedPeer;
     }
+
+    if (!this.globalLimiter.allow()) {
+      return RateLimitStatus.DeniedGlobal;
+    }
+
     return RateLimitStatus.Allowed;
   }
 

package/src/services/reqresp/rate-limiter/rate_limits.ts

@@ -42,16 +42,6 @@ export const DEFAULT_RATE_LIMITS: ReqRespSubProtocolRateLimits = {
       quotaCount: 200,
     },
   },
-  [ReqRespSubProtocol.BLOCK]: {
-    peerLimit: {
-      quotaTimeMs: 1000,
-      quotaCount: 2,
-    },
-    globalLimit: {
-      quotaTimeMs: 1000,
-      quotaCount: 5,
-    },
-  },
   [ReqRespSubProtocol.GOODBYE]: {
     peerLimit: {
       quotaTimeMs: 1000,

package/src/services/reqresp/reqresp.ts

@@ -16,7 +16,7 @@ import {
   IndividualReqRespTimeoutError,
   InvalidResponseError,
 } from '../../errors/reqresp.error.js';
-import { SnappyTransform } from '../encoding.js';
+import { OversizedSnappyResponseError, SnappyTransform } from '../encoding.js';
 import type { PeerScoring } from '../peer-manager/peer_scoring.js';
 import {
   DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS,
@@ -320,7 +320,7 @@ export class ReqResp implements ReqRespInterface {
               };
 
               for (const index of indices) {
-                this.logger.info(`Sending request ${index} to peer ${peerAsString}`);
+                this.logger.trace(`Sending request ${index} to peer ${peerAsString}`);
                 const response = await this.sendRequestToPeer(peer, subProtocol, requestBuffers[index]);
 
                 // Check the status of the response buffer
@@ -462,7 +462,7 @@ export class ReqResp implements ReqRespInterface {
       );
       return resp;
     } catch (e: any) {
-      this.logger.warn(`SUBPROTOCOL: ${subProtocol}\n`, e);
+      this.logger.debug(`SUBPROTOCOL: ${subProtocol}\n`, e);
       // On error we immediately abort the stream, this is preferred way,
       // because it signals to the sender that error happened, whereas
       // closing the stream only closes our side and is much slower
@@ -553,16 +553,10 @@ export class ReqResp implements ReqRespInterface {
         data: message,
       };
     } catch (e: any) {
+      // All errors (invalid status bytes, oversized snappy responses, corrupt data, etc.)
+      // are re-thrown so the caller can penalize the peer via handleResponseError.
       this.logger.debug(`Reading message failed: ${e.message}`);
-
-      let status = ReqRespStatus.UNKNOWN;
-      if (e instanceof ReqRespStatusError) {
-        status = e.status;
-      }
-
-      return {
-        status,
-      };
+      throw e;
     }
   }
 
@@ -627,7 +621,9 @@ export class ReqResp implements ReqRespInterface {
         // and that this stream should be dropped
         const isMessageToNotWarn =
           err instanceof Error &&
-          ['stream reset', 'Cannot push value onto an ended pushable'].some(msg => err.message.includes(msg));
+          ['stream reset', 'Cannot push value onto an ended pushable', 'read ECONNRESET'].some(msg =>
+            err.message.includes(msg),
+          );
         const level = isMessageToNotWarn ? 'debug' : 'warn';
         this.logger[level]('Unknown stream error while handling the stream, aborting', {
           protocol,
@@ -778,6 +774,20 @@ export class ReqResp implements ReqRespInterface {
       return undefined;
     }
 
+    // Invalid status byte: the peer sent a status byte that doesn't match any known status code.
+    // This is a protocol violation, penalize harshly.
+    if (e instanceof ReqRespStatusError) {
+      this.logger.warn(`Invalid status byte from peer ${peerId.toString()} in ${subProtocol}: ${e.message}`, logTags);
+      return PeerErrorSeverity.LowToleranceError;
+    }
+
+    // Oversized snappy response: the peer is sending data that exceeds the allowed size.
+    // This is a protocol violation that wastes bandwidth, so penalize harshly.
+    if (e instanceof OversizedSnappyResponseError) {
+      this.logger.warn(`Oversized response from peer ${peerId.toString()} in ${subProtocol}: ${e.message}`, logTags);
+      return PeerErrorSeverity.LowToleranceError;
+    }
+
     return this.categorizeConnectionErrors(e, peerId, subProtocol);
   }
 

package/src/services/service.ts

@@ -1,7 +1,13 @@
 import type { SlotNumber } from '@aztec/foundation/branded-types';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { PeerInfo } from '@aztec/stdlib/interfaces/server';
-import type { BlockProposal, CheckpointAttestation, CheckpointProposalCore, Gossipable } from '@aztec/stdlib/p2p';
+import type {
+  BlockProposal,
+  CheckpointAttestation,
+  CheckpointProposalCore,
+  Gossipable,
+  TopicType,
+} from '@aztec/stdlib/p2p';
 import type { Tx } from '@aztec/stdlib/tx';
 
 import type { PeerId } from '@libp2p/interface';
@@ -111,7 +117,12 @@ export interface P2PService {
   // Leaky abstraction: fix https://github.com/AztecProtocol/aztec-packages/issues/7963
   registerBlockReceivedCallback(callback: P2PBlockReceivedCallback): void;
 
-  registerCheckpointReceivedCallback(callback: P2PCheckpointReceivedCallback): void;
+  registerValidatorCheckpointReceivedCallback(callback: P2PCheckpointReceivedCallback): void;
+
+  registerAllNodesCheckpointReceivedCallback(callback: P2PCheckpointReceivedCallback): void;
+
+  /** Fires the all-nodes checkpoint callback for our own proposal (gossipsub doesn't deliver own messages). */
+  notifyOwnCheckpointProposal(checkpoint: CheckpointProposalCore): Promise<void>;
 
   /**
    * Registers a callback invoked when a duplicate proposal is detected (equivocation).
@@ -130,7 +141,10 @@ export interface P2PService {
 
   getPeers(includePending?: boolean): PeerInfo[];
 
-  validate(txs: Tx[]): Promise<void>;
+  /** Returns the number of peers in the GossipSub mesh for a given topic type. */
+  getGossipMeshPeerCount(topicType: TopicType): number;
+
+  validateTxsReceivedInBlockProposal(txs: Tx[]): Promise<void>;
 
   addReqRespSubProtocol(
     subProtocol: ReqRespSubProtocol,

package/src/services/tx_collection/config.ts

@@ -37,6 +37,18 @@ export type TxCollectionConfig = {
   txCollectionFileStoreSlowDelayMs: number;
   /** Delay in ms before file store collection starts after fast collection is triggered */
   txCollectionFileStoreFastDelayMs: number;
+  /** Number of concurrent workers for fast file store collection */
+  txCollectionFileStoreFastWorkerCount: number;
+  /** Number of concurrent workers for slow file store collection */
+  txCollectionFileStoreSlowWorkerCount: number;
+  /** Base backoff time in ms for fast file store collection retries */
+  txCollectionFileStoreFastBackoffBaseMs: number;
+  /** Base backoff time in ms for slow file store collection retries */
+  txCollectionFileStoreSlowBackoffBaseMs: number;
+  /** Max backoff time in ms for fast file store collection retries */
+  txCollectionFileStoreFastBackoffMaxMs: number;
+  /** Max backoff time in ms for slow file store collection retries */
+  txCollectionFileStoreSlowBackoffMaxMs: number;
 };
 
 export const txCollectionConfigMappings: ConfigMappingsType<TxCollectionConfig> = {
@@ -121,4 +133,34 @@ export const txCollectionConfigMappings: ConfigMappingsType<TxCollectionConfig>
     description: 'Delay before file store collection starts after fast collection',
     ...numberConfigHelper(2_000),
   },
+  txCollectionFileStoreFastWorkerCount: {
+    env: 'TX_COLLECTION_FILE_STORE_FAST_WORKER_COUNT',
+    description: 'Number of concurrent workers for fast file store collection',
+    ...numberConfigHelper(5),
+  },
+  txCollectionFileStoreSlowWorkerCount: {
+    env: 'TX_COLLECTION_FILE_STORE_SLOW_WORKER_COUNT',
+    description: 'Number of concurrent workers for slow file store collection',
+    ...numberConfigHelper(2),
+  },
+  txCollectionFileStoreFastBackoffBaseMs: {
+    env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_BASE_MS',
+    description: 'Base backoff time in ms for fast file store collection retries',
+    ...numberConfigHelper(1_000),
+  },
+  txCollectionFileStoreSlowBackoffBaseMs: {
+    env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_BASE_MS',
+    description: 'Base backoff time in ms for slow file store collection retries',
+    ...numberConfigHelper(5_000),
+  },
+  txCollectionFileStoreFastBackoffMaxMs: {
+    env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_MAX_MS',
+    description: 'Max backoff time in ms for fast file store collection retries',
+    ...numberConfigHelper(5_000),
+  },
+  txCollectionFileStoreSlowBackoffMaxMs: {
+    env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_MAX_MS',
+    description: 'Max backoff time in ms for slow file store collection retries',
+    ...numberConfigHelper(30_000),
+  },
 };