@aztec/p2p 0.0.1-commit.2448fdb → 0.0.1-commit.2606882

package/src/services/peer-manager/metrics.ts

@@ -18,6 +18,7 @@ export class PeerManagerMetrics {
   private sentGoodbyes: UpDownCounter;
   private receivedGoodbyes: UpDownCounter;
   private peerCount: Gauge;
+  private healthyPeerCount: Gauge;
   private lowScoreDisconnects: UpDownCounter;
   private peerConnectionDuration: Histogram;
 
@@ -49,6 +50,7 @@ export class PeerManagerMetrics {
       goodbyeReasonAttrs,
     );
     this.peerCount = meter.createGauge(Metrics.PEER_MANAGER_PEER_COUNT);
+    this.healthyPeerCount = meter.createGauge(Metrics.PEER_MANAGER_HEALTHY_PEER_COUNT);
     this.lowScoreDisconnects = createUpDownCounterWithDefault(meter, Metrics.PEER_MANAGER_LOW_SCORE_DISCONNECTS, {
       [Attributes.P2P_PEER_SCORE_STATE]: ['Banned', 'Disconnect'],
     });
@@ -67,6 +69,10 @@ export class PeerManagerMetrics {
     this.peerCount.record(count);
   }
 
+  public recordHealthyPeerCount(count: number) {
+    this.healthyPeerCount.record(count);
+  }
+
   public recordLowScoreDisconnect(scoreState: 'Banned' | 'Disconnect') {
     this.lowScoreDisconnects.add(1, { [Attributes.P2P_PEER_SCORE_STATE]: scoreState });
   }
@@ -79,6 +85,7 @@ export class PeerManagerMetrics {
     const connectedAt = this.peerConnectedAt.get(id.toString());
     if (connectedAt) {
       this.peerConnectionDuration.record(Date.now() - connectedAt);
+      this.peerConnectedAt.delete(id.toString());
     }
   }
 }

package/src/services/peer-manager/peer_manager.ts

@@ -32,7 +32,7 @@ import { PeerScoreState, type PeerScoring } from './peer_scoring.js';
 const MAX_DIAL_ATTEMPTS = 3;
 const MAX_CACHED_PEERS = 100;
 const MAX_CACHED_PEER_AGE_MS = 5 * 60 * 1000; // 5 minutes
-const FAILED_PEER_BAN_TIME_MS = 5 * 60 * 1000; // 5 minutes timeout after failing MAX_DIAL_ATTEMPTS
+const DEFAULT_FAILED_PEER_BAN_TIME_MS = 5 * 60 * 1000; // 5 minutes timeout after failing MAX_DIAL_ATTEMPTS
 const GOODBYE_DIAL_TIMEOUT_MS = 1000;
 const FAILED_AUTH_HANDSHAKE_EXPIRY_MS = 60 * 60 * 1000; // 1 hour
 
@@ -226,20 +226,30 @@ export class PeerManager implements PeerManagerInterface {
   }
 
   /**
-   * Cleans up expired timeouts.
+   * Cleans up expired timeouts and stale failed-auth-handshake entries.
    *
    * When peers fail to dial after a number of retries, they are temporarily timed out.
    * This function removes any peers that have been in the timed out state for too long.
    * To give them a chance to reconnect.
+   *
+   * Also evicts entries from the failed-auth-handshake map whose expiry window has passed.
+   * Without this, peers that probe once and never reconnect would leave their entries in the
+   * map forever, causing an unbounded memory leak.
    */
   private cleanupExpiredTimeouts() {
-    // Clean up expired timeouts
     const now = this.dateProvider.now();
+
     for (const [peerId, timedOutPeer] of this.timedOutPeers.entries()) {
       if (now >= timedOutPeer.timeoutUntilMs) {
         this.timedOutPeers.delete(peerId);
       }
     }
+
+    for (const [id, entry] of this.failedAuthHandshakes.entries()) {
+      if (now - entry.lastFailureTimestamp > FAILED_AUTH_HANDSHAKE_EXPIRY_MS) {
+        this.failedAuthHandshakes.delete(id);
+      }
+    }
   }
 
   /**
@@ -303,15 +313,20 @@ export class PeerManager implements PeerManagerInterface {
    */
   private handleDisconnectedPeerEvent(e: CustomEvent<PeerId>) {
     const peerId = e.detail;
+    const peerIdStr = peerId.toString();
     this.metrics.peerDisconnected(peerId);
-    this.logger.verbose(`Disconnected from peer ${peerId.toString()}`);
-    const validatorAddress = this.authenticatedPeerIdToValidatorAddress.get(peerId.toString());
+    this.logger.verbose(`Disconnected from peer ${peerIdStr}`);
+    const validatorAddress = this.authenticatedPeerIdToValidatorAddress.get(peerIdStr);
     if (validatorAddress !== undefined) {
       this.logger.info(
-        `Removing authentication for validator ${validatorAddress} at peer id ${peerId.toString()} due to disconnection`,
+        `Removing authentication for validator ${validatorAddress} at peer id ${peerIdStr} due to disconnection`,
       );
       this.authenticatedValidatorAddressToPeerId.delete(validatorAddress.toString());
-      this.authenticatedPeerIdToValidatorAddress.delete(peerId.toString());
+      this.authenticatedPeerIdToValidatorAddress.delete(peerIdStr);
+    }
+
+    if (this.peerScoring.getScoreState(peerIdStr) === PeerScoreState.Healthy) {
+      this.peerScoring.removePeer(peerIdStr);
     }
   }
 
@@ -512,10 +527,12 @@ export class PeerManager implements PeerManagerInterface {
       protectedConnections: protectedPeerCount,
       maxPeerCount: this.config.maxPeerCount,
       cachedPeers: this.cachedPeers.size,
+      discv5KadPeers: this.peerDiscoveryService.getKadValues().length,
       ...this.peerScoring.getStats(),
     });
 
-    this.metrics.recordPeerCount(healthyConnections.length);
+    this.metrics.recordPeerCount(connections.length);
+    this.metrics.recordHealthyPeerCount(healthyConnections.length);
 
     // Exit if no peers to connect
     if (peersToConnect <= 0) {
@@ -712,6 +729,12 @@ export class PeerManager implements PeerManagerInterface {
       return;
     }
 
+    // Don't dial peers that have exceeded the auth failure threshold
+    if (!this.isNodeAllowedToConnect(peerId)) {
+      this.logger.trace(`Skipping peer ${peerId} due to failed auth handshake attempts`);
+      return;
+    }
+
     const [multiaddrTcp] = await Promise.all([enr.getFullMultiaddr('tcp')]);
 
     this.logger.trace(`Handling discovered peer ${peerId} at ${multiaddrTcp?.toString() ?? 'undefined address'}`);
@@ -775,7 +798,8 @@ export class PeerManager implements PeerManagerInterface {
         // Add to timed out peers
         this.timedOutPeers.set(id, {
           peerId: id,
-          timeoutUntilMs: this.dateProvider.now() + FAILED_PEER_BAN_TIME_MS,
+          timeoutUntilMs:
+            this.dateProvider.now() + (this.config.peerFailedBanTimeMs ?? DEFAULT_FAILED_PEER_BAN_TIME_MS),
         });
       }
     }
@@ -937,6 +961,8 @@ export class PeerManager implements PeerManagerInterface {
           `Received auth for validator ${sender.toString()} from peer ${peerIdString}, but this validator is already authenticated to peer ${peerForAddress.toString()}`,
           { ...logData, address: sender.toString() },
         );
+        this.markAuthHandshakeFailed(peerId);
+        this.markPeerForDisconnect(peerId);
         return;
       }
 
@@ -966,14 +992,14 @@ export class PeerManager implements PeerManagerInterface {
     const peerIdStr = peerId.toString();
 
     const existingEntry = this.failedAuthHandshakes.get(peerIdStr);
+    const failureCount = (existingEntry?.count || 0) + 1;
     this.failedAuthHandshakes.set(peerIdStr, {
-      count: (existingEntry?.count || 0) + 1,
+      count: failureCount,
       lastFailureTimestamp: now,
     });
 
     const connections = this.libP2PNode.getConnections(peerId);
     connections.forEach(conn => {
-      // We mark the IP address
       const address = conn.remoteAddr.nodeAddress().address;
       const existingAddressEntry = this.failedAuthHandshakes.get(address);
       this.failedAuthHandshakes.set(address, {
@@ -981,6 +1007,15 @@ export class PeerManager implements PeerManagerInterface {
         lastFailureTimestamp: now,
       });
     });
+
+    // Ban the peer from being re-dialed for a cooldown period (exponential backoff)
+    const banTimeMs = this.config.peerFailedBanTimeMs ?? DEFAULT_FAILED_PEER_BAN_TIME_MS;
+    const backoffMs = banTimeMs * Math.pow(2, Math.min(failureCount - 1, 5));
+    this.timedOutPeers.set(peerIdStr, {
+      peerId: peerIdStr,
+      timeoutUntilMs: now + backoffMs,
+    });
+    this.cachedPeers.delete(peerIdStr);
   }
 
   /*

package/src/services/peer-manager/peer_scoring.ts

@@ -1,5 +1,6 @@
 import { median } from '@aztec/foundation/collection';
 import { createLogger } from '@aztec/foundation/log';
+import { DateProvider } from '@aztec/foundation/timer';
 import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
 import {
   Attributes,
@@ -54,6 +55,7 @@ export enum PeerScoreState {
 // TODO: move into config / constants
 const MIN_SCORE_BEFORE_BAN = -100;
 const MIN_SCORE_BEFORE_DISCONNECT = -50;
+const SCORE_CLEANUP_THRESHOLD = 0.1;
 
 export class PeerScoring {
   private logger = createLogger('p2p:peer-scoring');
@@ -65,7 +67,11 @@ export class PeerScoring {
 
   private peerStateCounter: UpDownCounter;
 
-  constructor(config: P2PConfig, telemetry: TelemetryClient = getTelemetryClient()) {
+  constructor(
+    config: P2PConfig,
+    telemetry: TelemetryClient = getTelemetryClient(),
+    private readonly dateProvider: DateProvider = new DateProvider(),
+  ) {
     const orderedValues = config.peerPenaltyValues?.sort((a, b) => a - b);
     this.peerPenalties = {
       [PeerErrorSeverity.HighToleranceError]:
@@ -92,7 +98,7 @@ export class PeerScoring {
   }
 
   updateScore(peerId: string, scoreDelta: number): number {
-    const currentTime = Date.now();
+    const currentTime = this.dateProvider.now();
     const lastUpdate = this.lastUpdateTime.get(peerId) || currentTime;
     const timePassed = currentTime - lastUpdate;
     const decayPeriods = Math.floor(timePassed / this.decayInterval);
@@ -111,19 +117,35 @@ export class PeerScoring {
   }
 
   decayAllScores(): void {
-    const currentTime = Date.now();
+    const currentTime = this.dateProvider.now();
     for (const [peerId, lastUpdate] of this.lastUpdateTime.entries()) {
       const timePassed = currentTime - lastUpdate;
       const decayPeriods = Math.floor(timePassed / this.decayInterval);
       if (decayPeriods > 0) {
         let score = this.scores.get(peerId) || 0;
         score *= Math.pow(this.decayFactor, decayPeriods);
-        this.scores.set(peerId, score);
-        this.lastUpdateTime.set(peerId, currentTime);
+        if (Math.abs(score) < SCORE_CLEANUP_THRESHOLD) {
+          this.scores.delete(peerId);
+          this.lastUpdateTime.delete(peerId);
+        } else {
+          this.scores.set(peerId, score);
+          this.lastUpdateTime.set(peerId, currentTime);
+        }
       }
     }
   }
 
+  /** Resets all peer scores. Useful for benchmarks to prevent cross-case contamination. */
+  resetAllScores(): void {
+    this.scores.clear();
+    this.lastUpdateTime.clear();
+  }
+
+  removePeer(peerId: string): void {
+    this.scores.delete(peerId);
+    this.lastUpdateTime.delete(peerId);
+  }
+
   getScore(peerId: string): number {
     return this.scores.get(peerId) || 0;
   }

package/src/services/reqresp/README.md

@@ -46,7 +46,6 @@ Per-protocol size limits checked via preamble before decompression.
 | Error Type | Severity |
 |------------|----------|
 | GOODBYE subprotocol errors | None |
-| `CollectiveReqRespTimeoutError` / `InvalidResponseError` | None |
 | `AbortError` / connection close / muxer closed | None |
 | `ECONNRESET` / `EPIPE` / `ECONNREFUSED` / `ERR_UNEXPECTED_EOF` | HighToleranceError |
 | `ERR_UNSUPPORTED_PROTOCOL` | HighToleranceError |
@@ -183,19 +182,6 @@ Protected peers (private/trusted/preferred) are always considered "authenticated
 
 Conditional registration: BLOCK_TXS handler only registered when `config.disableTransactions` is false. Otherwise peers get `ERR_UNSUPPORTED_PROTOCOL`.
 
-**Requester side via `sendBatchRequest`** (Snappy limit: `max(N, 1) * 512 + 1` KB):
-
-| Rule | Consequence | File |
-|------|-------------|------|
-| Archive root must match request | MidToleranceError | `libp2p_service.ts` (`validateRequestedBlockTxs`) |
-| BitVector length must match request | MidToleranceError | same |
-| No duplicate tx hashes | MidToleranceError | same |
-| Tx count within bounds | MidToleranceError | same |
-| Local block proposal must exist for archive root | Rejected (no penalty) | same |
-| All tx hashes must be in proposal's tx list at allowed indices | LowToleranceError | same |
-| Txs in strictly increasing index order | LowToleranceError | same |
-| Each tx passes well-formedness (Metadata [4 fields], Size, Data, Proof) | LowToleranceError | same |
-
 **Requester side via `BatchTxRequester`** (separate validation path):
 
 | Rule | Consequence | File |

package/src/services/reqresp/batch-tx-requester/README.md

@@ -170,6 +170,37 @@ class BlockTxsResponse {
 
 The `BitVector` is a compact representation where each bit corresponds to a transaction index in the block proposal. This allows efficient capability advertisement without repeating full hashes.
 
+## Cancellation
+
+All cancellation is managed by a single `RequestTracker` instance, shared across the entire collection
+flow. The `RequestTracker` owns the deadline, tracks which txs are still missing, and exposes a
+`cancellationToken` promise that resolves when the request should stop (deadline hit, all txs fetched,
+or external `cancel()` call).
+
+Cancellation propagates from the deepest stack level upward:
+
+```
+RequestTracker.finish()
+  ├── resolves cancellationToken promise
+  │
+  ├── BatchTxRequester workers (deepest)
+  │     ├── shouldStop() checks requestTracker.cancelled → exit loop
+  │     ├── sleepClampedToDeadline races sleep vs cancellationToken → wakes
+  │     └── semaphore.acquire races vs cancellationToken → wakes
+  │           │
+  │           ▼ workers settle → txQueue.end() → generator returns
+  │
+  ├── Node collection loops
+  │     ├── notFinished() checks requestTracker.cancelled → exit loop
+  │     └── inter-retry sleep races vs cancellationToken → wakes
+  │           │
+  │           ▼ all node loops settle
+  │
+  └── collectFast (outermost)
+        awaits Promise.allSettled([reqresp, nodes]) → settles after inner tasks
+        finally: requestTracker.cancel() (idempotent), cleanup
+```
+
 ## Key Files
 
 | File | Description |
@@ -179,15 +210,16 @@ The `BitVector` is a compact representation where each bit corresponds to a tran
 | `peer_collection.ts` | Manages peer classification (dumb/smart/bad) and rate limiting |
 | `interface.ts` | Type definitions for dependencies |
 | `../protocols/block_txs/` | Wire protocol definitions (`BlockTxsRequest`, `BlockTxsResponse`, `BitVector`) |
+| `../../tx_collection/request_tracker.ts` | Centralized deadline, missing tx tracking, and cancellation signal |
 
 ## Stopping Conditions
 
-The `BatchTxRequester` stops when any of these conditions are met:
+The `BatchTxRequester` stops when any of these conditions are met, all managed by the `RequestTracker`:
 
-1. **All transactions fetched** - Success!
-2. **Deadline exceeded** - Timeout configured by caller
-3. **Abort signal** - External cancellation
-4. **No transactions to fetch** - Nothing was missing
+1. **All transactions fetched** - `markFetched()` removes the last missing tx, triggering `finish()`
+2. **Deadline exceeded** - `setTimeout` in `RequestTracker` fires, triggering `finish()`
+3. **External cancellation** - `RequestTracker.cancel()` called (e.g., from `stop()`, `stopCollectingForBlocksUpTo`)
+4. **No transactions to fetch** - Empty hash set at construction, `RequestTracker` finishes immediately
 
 ## Configuration
 
@@ -228,11 +260,15 @@ Request to peer fails
 ## Usage Example
 
 ```typescript
+const requestTracker = RequestTracker.create(
+  missingTxHashes,                          // TxHash[] - what we need
+  new Date(Date.now() + 5_000),             // deadline
+);
+
 const requester = new BatchTxRequester(
-  missingTxHashes,      // TxHash[] - what we need
+  requestTracker,       // IRequestTracker - tracks missing txs, deadline, and cancellation
   blockTxsSource,       // BlockTxsSource - the proposal or block we need txs for
   pinnedPeer,           // PeerId | undefined - peer expected to have the txs
-  timeoutMs,            // number - how long to try
   p2pService,           // BatchTxRequesterLibP2PService
 );
 
@@ -273,6 +309,8 @@ const txs = await BatchTxRequester.collectAllTxs(requester.run());
 │  1. Try RPC nodes first (fast)    │   │  Periodic polling of RPC nodes      │
 │  2. Fall back to BatchTxRequester │   │  and peers for missing txs          │
 │                                   │   │                                     │
+│  Creates RequestTracker per       │   │                                     │
+│  request with deadline            │   │                                     │
 └───────────────────┬───────────────┘   └─────────────────────────────────────┘
                     │
                     │ For 'proposal' and 'block' requests
@@ -281,6 +319,7 @@ const txs = await BatchTxRequester.collectAllTxs(requester.run());
 │                           BatchTxRequester                                  │
 │                                                                             │
 │  Aggressive parallel fetching from multiple peers                           │
+│  Shares RequestTracker with FastTxCollection for unified cancellation       │
 │  Uses BLOCK_TXS sub-protocol for efficient batching                         │
 └───────────────────┬─────────────────────────────────────────────────────────┘
                     │