@aztec/p2p 0.0.1-commit.f2ce05ee → 0.0.1-commit.f5a9928

package/src/services/peer-manager/metrics.ts

@@ -18,6 +18,7 @@ export class PeerManagerMetrics {
   private sentGoodbyes: UpDownCounter;
   private receivedGoodbyes: UpDownCounter;
   private peerCount: Gauge;
+  private healthyPeerCount: Gauge;
   private lowScoreDisconnects: UpDownCounter;
   private peerConnectionDuration: Histogram;
 
@@ -49,6 +50,7 @@ export class PeerManagerMetrics {
       goodbyeReasonAttrs,
     );
     this.peerCount = meter.createGauge(Metrics.PEER_MANAGER_PEER_COUNT);
+    this.healthyPeerCount = meter.createGauge(Metrics.PEER_MANAGER_HEALTHY_PEER_COUNT);
     this.lowScoreDisconnects = createUpDownCounterWithDefault(meter, Metrics.PEER_MANAGER_LOW_SCORE_DISCONNECTS, {
       [Attributes.P2P_PEER_SCORE_STATE]: ['Banned', 'Disconnect'],
     });
@@ -67,6 +69,10 @@ export class PeerManagerMetrics {
     this.peerCount.record(count);
   }
 
+  public recordHealthyPeerCount(count: number) {
+    this.healthyPeerCount.record(count);
+  }
+
   public recordLowScoreDisconnect(scoreState: 'Banned' | 'Disconnect') {
     this.lowScoreDisconnects.add(1, { [Attributes.P2P_PEER_SCORE_STATE]: scoreState });
   }
@@ -79,6 +85,7 @@ export class PeerManagerMetrics {
     const connectedAt = this.peerConnectedAt.get(id.toString());
     if (connectedAt) {
       this.peerConnectionDuration.record(Date.now() - connectedAt);
+      this.peerConnectedAt.delete(id.toString());
     }
   }
 }

package/src/services/peer-manager/peer_manager.ts

@@ -1,4 +1,5 @@
 import type { EpochCacheInterface } from '@aztec/epoch-cache';
+import { compactArray } from '@aztec/foundation/collection';
 import { makeEthSignDigest, tryRecoverAddress } from '@aztec/foundation/crypto/secp256k1-signer';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import type { EthAddress } from '@aztec/foundation/eth-address';
@@ -32,9 +33,11 @@ import { PeerScoreState, type PeerScoring } from './peer_scoring.js';
 const MAX_DIAL_ATTEMPTS = 3;
 const MAX_CACHED_PEERS = 100;
 const MAX_CACHED_PEER_AGE_MS = 5 * 60 * 1000; // 5 minutes
-const FAILED_PEER_BAN_TIME_MS = 5 * 60 * 1000; // 5 minutes timeout after failing MAX_DIAL_ATTEMPTS
+const DEFAULT_FAILED_PEER_BAN_TIME_MS = 5 * 60 * 1000; // 5 minutes timeout after failing MAX_DIAL_ATTEMPTS
 const GOODBYE_DIAL_TIMEOUT_MS = 1000;
 const FAILED_AUTH_HANDSHAKE_EXPIRY_MS = 60 * 60 * 1000; // 1 hour
+const ZERO_PEER_WARNING_THRESHOLD_HEARTBEATS = 3; // tolerate transient dips, such as at startup
+const ZERO_PEER_WARNING_REPEAT_INTERVAL_MS = 60 * 1000;
 
 type CachedPeer = {
   peerId: PeerId;
@@ -70,6 +73,10 @@ export class PeerManager implements PeerManagerInterface {
   private failedAuthHandshakes: Map<string, FailedAuthHandshakeEntry> = new Map();
   private validatorAddresses: EthAddress[] = [];
   private initializedPreferredPeers: boolean = false;
+  private consecutiveZeroPeerHeartbeats: number = 0;
+  private zeroPeerWarningHeartbeats: number = 0;
+  private zeroPeerWarningIssued: boolean = false;
+  private lastConnectedPeerAtMs: number | undefined;
 
   private metrics: PeerManagerMetrics;
   private handlers: {
@@ -114,6 +121,10 @@ export class PeerManager implements PeerManagerInterface {
 
     // Display peer counts every 60 seconds
     this.displayPeerCountsPeerHeartbeat = Math.floor(60_000 / this.config.peerCheckIntervalMS);
+    this.zeroPeerWarningHeartbeats = Math.max(
+      1,
+      Math.floor(ZERO_PEER_WARNING_REPEAT_INTERVAL_MS / this.config.peerCheckIntervalMS),
+    );
   }
   /**
    * Initializes the trusted peers.
@@ -164,6 +175,7 @@ export class PeerManager implements PeerManagerInterface {
   public async heartbeat() {
     this.heartbeatCounter++;
     this.peerScoring.decayAllScores();
+    this.peerScoring.pruneExpiredBans();
     this.cleanupExpiredTimeouts();
 
     await this.setupDirectPeersIfValidator();
@@ -171,6 +183,45 @@ export class PeerManager implements PeerManagerInterface {
     await this.processScheduledDisconnects();
 
     this.discover();
+    this.checkPeerConnectivity();
+  }
+
+  /**
+   * Flags a node that has been left without any connected peers, since it can neither gossip nor propagate txs.
+   *
+   * Warnings only start after a few consecutive zero-peer heartbeats so that startup or a transient dip stays quiet,
+   * and then repeat roughly once a minute instead of on every heartbeat. Recovery is logged once.
+   */
+  private checkPeerConnectivity() {
+    const connectedPeerCount = this.libP2PNode.getPeers().length;
+    const now = this.dateProvider.now();
+
+    if (connectedPeerCount > 0) {
+      if (this.zeroPeerWarningIssued) {
+        this.logger.info(`Peer connectivity restored with ${connectedPeerCount} connected peers`, {
+          connectedPeerCount,
+          zeroPeerHeartbeats: this.consecutiveZeroPeerHeartbeats,
+        });
+      }
+      this.consecutiveZeroPeerHeartbeats = 0;
+      this.zeroPeerWarningIssued = false;
+      this.lastConnectedPeerAtMs = now;
+      return;
+    }
+
+    this.consecutiveZeroPeerHeartbeats++;
+    const heartbeatsSinceThreshold = this.consecutiveZeroPeerHeartbeats - ZERO_PEER_WARNING_THRESHOLD_HEARTBEATS;
+    if (heartbeatsSinceThreshold < 0 || heartbeatsSinceThreshold % this.zeroPeerWarningHeartbeats !== 0) {
+      return;
+    }
+
+    this.zeroPeerWarningIssued = true;
+    this.logger.warn('Node has no connected peers; gossip and tx propagation are unavailable', {
+      zeroPeerHeartbeats: this.consecutiveZeroPeerHeartbeats,
+      peerCheckIntervalMS: this.config.peerCheckIntervalMS,
+      msSinceLastConnectedPeer: this.lastConnectedPeerAtMs === undefined ? undefined : now - this.lastConnectedPeerAtMs,
+      cachedPeers: this.cachedPeers.size,
+    });
   }
 
   /*
@@ -198,21 +249,27 @@ export class PeerManager implements PeerManagerInterface {
       .then(peerIds => peerIds.forEach(peerId => this.preferredPeers.add(peerId.toString())))
       .catch(e => this.logger.error('Error initializing preferred peers', e));
 
-    const directPeers = (
+    const directPeers = compactArray(
       await Promise.all(
         preferredPeersEnrs.map(async enr => {
-          const peerId = await enr.peerId();
-          const address = enr.getLocationMultiaddr('tcp');
-          if (address === undefined) {
-            throw new Error(`Direct peer ${peerId.toString()} has no TCP address, ENR: ${enr.encodeTxt()}`);
+          try {
+            const peerId = await enr.peerId();
+            const address = enr.getLocationMultiaddr('tcp');
+            if (address === undefined) {
+              throw new Error(`Direct peer ${peerId.toString()} has no TCP address, ENR: ${enr.encodeTxt()}`);
+            }
+            return {
+              id: peerId,
+              addrs: [address],
+            };
+          } catch (err) {
+            // A malformed configured ENR shouldn't abort preferred-peer setup — skip it and log.
+            this.logger.warn(`Skipping preferred peer with invalid ENR`, { err });
+            return undefined;
           }
-          return {
-            id: peerId,
-            addrs: [address],
-          };
         }),
-      )
-    ).filter(peer => peer !== undefined);
+      ),
+    );
 
     await Promise.all(
       directPeers.map(peer => {
@@ -226,20 +283,30 @@ export class PeerManager implements PeerManagerInterface {
   }
 
   /**
-   * Cleans up expired timeouts.
+   * Cleans up expired timeouts and stale failed-auth-handshake entries.
    *
    * When peers fail to dial after a number of retries, they are temporarily timed out.
    * This function removes any peers that have been in the timed out state for too long.
    * To give them a chance to reconnect.
+   *
+   * Also evicts entries from the failed-auth-handshake map whose expiry window has passed.
+   * Without this, peers that probe once and never reconnect would leave their entries in the
+   * map forever, causing an unbounded memory leak.
    */
   private cleanupExpiredTimeouts() {
-    // Clean up expired timeouts
     const now = this.dateProvider.now();
+
     for (const [peerId, timedOutPeer] of this.timedOutPeers.entries()) {
       if (now >= timedOutPeer.timeoutUntilMs) {
         this.timedOutPeers.delete(peerId);
       }
     }
+
+    for (const [id, entry] of this.failedAuthHandshakes.entries()) {
+      if (now - entry.lastFailureTimestamp > FAILED_AUTH_HANDSHAKE_EXPIRY_MS) {
+        this.failedAuthHandshakes.delete(id);
+      }
+    }
   }
 
   /**
@@ -303,15 +370,20 @@ export class PeerManager implements PeerManagerInterface {
    */
   private handleDisconnectedPeerEvent(e: CustomEvent<PeerId>) {
     const peerId = e.detail;
+    const peerIdStr = peerId.toString();
     this.metrics.peerDisconnected(peerId);
-    this.logger.verbose(`Disconnected from peer ${peerId.toString()}`);
-    const validatorAddress = this.authenticatedPeerIdToValidatorAddress.get(peerId.toString());
+    this.logger.verbose(`Disconnected from peer ${peerIdStr}`);
+    const validatorAddress = this.authenticatedPeerIdToValidatorAddress.get(peerIdStr);
     if (validatorAddress !== undefined) {
       this.logger.info(
-        `Removing authentication for validator ${validatorAddress} at peer id ${peerId.toString()} due to disconnection`,
+        `Removing authentication for validator ${validatorAddress} at peer id ${peerIdStr} due to disconnection`,
       );
       this.authenticatedValidatorAddressToPeerId.delete(validatorAddress.toString());
-      this.authenticatedPeerIdToValidatorAddress.delete(peerId.toString());
+      this.authenticatedPeerIdToValidatorAddress.delete(peerIdStr);
+    }
+
+    if (this.peerScoring.getScoreState(peerIdStr) === PeerScoreState.Healthy) {
+      this.peerScoring.removePeer(peerIdStr);
     }
   }
 
@@ -479,15 +551,38 @@ export class PeerManager implements PeerManagerInterface {
    *
    * @returns: True if node is allowed to connect, otherwise false
    * */
-  public isNodeAllowedToConnect(id: string | PeerId): boolean {
-    const entry = this.failedAuthHandshakes.get(id.toString());
+  /**
+   * Whether a peer is allowed to connect, given its peer id. Rejects peers serving an active ban and
+   * peers that have exceeded the failed auth-handshake limit. Use this once the peer id is known —
+   * i.e. for the encrypted-inbound gater and when dialing.
+   */
+  public isPeerAllowedToConnect(peerId: string | PeerId): boolean {
+    const id = peerId.toString();
+    if (this.peerScoring.getScoreState(id) === PeerScoreState.Banned) {
+      return false;
+    }
+    return this.isWithinFailedAuthLimit(id);
+  }
+
+  /**
+   * Whether a connection from an address is allowed. Bans are keyed by peer id, which isn't known at
+   * the raw-inbound layer, so only the failed auth-handshake limit (also tracked per address) is
+   * enforced here; the ban is applied once the peer id is known via {@link isPeerAllowedToConnect}.
+   */
+  public isAddressAllowedToConnect(address: string): boolean {
+    return this.isWithinFailedAuthLimit(address);
+  }
+
+  /** Whether the failed auth-handshake count for a peer id or address is below the configured limit. */
+  private isWithinFailedAuthLimit(key: string): boolean {
+    const entry = this.failedAuthHandshakes.get(key);
     if (!entry) {
       return true;
     }
 
     // In case entry is too old, remove it and allow connection
     if (this.dateProvider.now() - entry.lastFailureTimestamp > FAILED_AUTH_HANDSHAKE_EXPIRY_MS) {
-      this.failedAuthHandshakes.delete(id.toString());
+      this.failedAuthHandshakes.delete(key);
       return true;
     }
 
@@ -512,10 +607,12 @@ export class PeerManager implements PeerManagerInterface {
       protectedConnections: protectedPeerCount,
       maxPeerCount: this.config.maxPeerCount,
       cachedPeers: this.cachedPeers.size,
+      discv5KadPeers: this.peerDiscoveryService.getKadValues().length,
       ...this.peerScoring.getStats(),
     });
 
-    this.metrics.recordPeerCount(healthyConnections.length);
+    this.metrics.recordPeerCount(connections.length);
+    this.metrics.recordHealthyPeerCount(healthyConnections.length);
 
     // Exit if no peers to connect
     if (peersToConnect <= 0) {
@@ -712,6 +809,12 @@ export class PeerManager implements PeerManagerInterface {
       return;
     }
 
+    // Don't dial banned peers or those that have exceeded the auth failure threshold
+    if (!this.isPeerAllowedToConnect(peerId)) {
+      this.logger.trace(`Skipping peer ${peerId} due to ban or failed auth handshake attempts`);
+      return;
+    }
+
     const [multiaddrTcp] = await Promise.all([enr.getFullMultiaddr('tcp')]);
 
     this.logger.trace(`Handling discovered peer ${peerId} at ${multiaddrTcp?.toString() ?? 'undefined address'}`);
@@ -775,7 +878,8 @@ export class PeerManager implements PeerManagerInterface {
         // Add to timed out peers
         this.timedOutPeers.set(id, {
           peerId: id,
-          timeoutUntilMs: this.dateProvider.now() + FAILED_PEER_BAN_TIME_MS,
+          timeoutUntilMs:
+            this.dateProvider.now() + (this.config.peerFailedBanTimeMs ?? DEFAULT_FAILED_PEER_BAN_TIME_MS),
         });
       }
     }
@@ -937,6 +1041,8 @@ export class PeerManager implements PeerManagerInterface {
           `Received auth for validator ${sender.toString()} from peer ${peerIdString}, but this validator is already authenticated to peer ${peerForAddress.toString()}`,
           { ...logData, address: sender.toString() },
         );
+        this.markAuthHandshakeFailed(peerId);
+        this.markPeerForDisconnect(peerId);
         return;
       }
 
@@ -966,14 +1072,14 @@ export class PeerManager implements PeerManagerInterface {
     const peerIdStr = peerId.toString();
 
     const existingEntry = this.failedAuthHandshakes.get(peerIdStr);
+    const failureCount = (existingEntry?.count || 0) + 1;
     this.failedAuthHandshakes.set(peerIdStr, {
-      count: (existingEntry?.count || 0) + 1,
+      count: failureCount,
       lastFailureTimestamp: now,
     });
 
     const connections = this.libP2PNode.getConnections(peerId);
     connections.forEach(conn => {
-      // We mark the IP address
       const address = conn.remoteAddr.nodeAddress().address;
       const existingAddressEntry = this.failedAuthHandshakes.get(address);
       this.failedAuthHandshakes.set(address, {
@@ -981,6 +1087,15 @@ export class PeerManager implements PeerManagerInterface {
         lastFailureTimestamp: now,
       });
     });
+
+    // Ban the peer from being re-dialed for a cooldown period (exponential backoff)
+    const banTimeMs = this.config.peerFailedBanTimeMs ?? DEFAULT_FAILED_PEER_BAN_TIME_MS;
+    const backoffMs = banTimeMs * Math.pow(2, Math.min(failureCount - 1, 5));
+    this.timedOutPeers.set(peerIdStr, {
+      peerId: peerIdStr,
+      timeoutUntilMs: now + backoffMs,
+    });
+    this.cachedPeers.delete(peerIdStr);
   }
 
   /*

package/src/services/peer-manager/peer_scoring.ts

@@ -1,5 +1,6 @@
 import { median } from '@aztec/foundation/collection';
 import { createLogger } from '@aztec/foundation/log';
+import { DateProvider } from '@aztec/foundation/timer';
 import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
 import {
   Attributes,
@@ -54,6 +55,10 @@ export enum PeerScoreState {
 // TODO: move into config / constants
 const MIN_SCORE_BEFORE_BAN = -100;
 const MIN_SCORE_BEFORE_DISCONNECT = -50;
+const SCORE_CLEANUP_THRESHOLD = 0.1;
+
+/** An active ban: the score the peer held when banned, and the timestamp at which the ban lifts. */
+type BanRecord = { score: number; expiry: number };
 
 export class PeerScoring {
   private logger = createLogger('p2p:peer-scoring');
@@ -65,7 +70,21 @@ export class PeerScoring {
 
   private peerStateCounter: UpDownCounter;
 
-  constructor(config: P2PConfig, telemetry: TelemetryClient = getTelemetryClient()) {
+  /** Active bans, keyed by peer id. Held in memory only, so they are cleared on restart. */
+  private bannedPeers: Map<string, BanRecord> = new Map();
+  /**
+   * How long a peer remains banned once its score crosses MIN_SCORE_BEFORE_BAN. While banned, the
+   * peer's ban score is returned by getScore regardless of decay, so it cannot recover its way out
+   * of the ban early. After the ban expires the live (decayed) score takes over again.
+   */
+  private readonly banDurationMs: number;
+
+  constructor(
+    config: P2PConfig,
+    telemetry: TelemetryClient = getTelemetryClient(),
+    private readonly dateProvider: DateProvider = new DateProvider(),
+  ) {
+    this.banDurationMs = config.peerBanDurationSeconds * 1000;
     const orderedValues = config.peerPenaltyValues?.sort((a, b) => a - b);
     this.peerPenalties = {
       [PeerErrorSeverity.HighToleranceError]:
@@ -92,7 +111,7 @@ export class PeerScoring {
   }
 
   updateScore(peerId: string, scoreDelta: number): number {
-    const currentTime = Date.now();
+    const currentTime = this.dateProvider.now();
     const lastUpdate = this.lastUpdateTime.get(peerId) || currentTime;
     const timePassed = currentTime - lastUpdate;
     const decayPeriods = Math.floor(timePassed / this.decayInterval);
@@ -107,29 +126,102 @@ export class PeerScoring {
 
     this.scores.set(peerId, currentScore);
     this.lastUpdateTime.set(peerId, currentTime);
+
+    this.maybeBanPeer(peerId, currentScore);
+
     return currentScore;
   }
 
+  /**
+   * Records a ban for a peer whose score has crossed the ban threshold, with an expiry banDurationMs
+   * in the future. No-op if the score is above the threshold or the peer is already serving an active
+   * ban (an existing ban is not extended; the original window stands). A previously expired ban does
+   * not block a fresh one — getActiveBanScore prunes it first.
+   */
+  private maybeBanPeer(peerId: string, score: number): void {
+    if (score >= MIN_SCORE_BEFORE_BAN || this.getActiveBanScore(peerId) !== undefined) {
+      return;
+    }
+    const record: BanRecord = { score, expiry: this.dateProvider.now() + this.banDurationMs };
+    this.bannedPeers.set(peerId, record);
+    this.logger.verbose(`Banning peer ${peerId} until ${new Date(record.expiry).toISOString()}`, {
+      peerId,
+      score,
+      expiry: record.expiry,
+    });
+  }
+
   decayAllScores(): void {
-    const currentTime = Date.now();
+    const currentTime = this.dateProvider.now();
     for (const [peerId, lastUpdate] of this.lastUpdateTime.entries()) {
       const timePassed = currentTime - lastUpdate;
       const decayPeriods = Math.floor(timePassed / this.decayInterval);
       if (decayPeriods > 0) {
         let score = this.scores.get(peerId) || 0;
         score *= Math.pow(this.decayFactor, decayPeriods);
-        this.scores.set(peerId, score);
-        this.lastUpdateTime.set(peerId, currentTime);
+        if (Math.abs(score) < SCORE_CLEANUP_THRESHOLD) {
+          this.scores.delete(peerId);
+          this.lastUpdateTime.delete(peerId);
+        } else {
+          this.scores.set(peerId, score);
+          this.lastUpdateTime.set(peerId, currentTime);
+        }
       }
     }
   }
 
+  /**
+   * Removes bans whose window has elapsed. Expired bans are otherwise only pruned lazily when their
+   * peer's score is next queried, so a banned peer that disconnects and is never queried again would
+   * linger in the map. Called periodically (per heartbeat) to bound the ban map's size.
+   */
+  pruneExpiredBans(): void {
+    const now = this.dateProvider.now();
+    const expired = [...this.bannedPeers.entries()].filter(([, ban]) => ban.expiry <= now).map(([peerId]) => peerId);
+    for (const peerId of expired) {
+      this.bannedPeers.delete(peerId);
+    }
+  }
+
+  /** Resets all peer scores. Useful for benchmarks to prevent cross-case contamination. */
+  resetAllScores(): void {
+    this.scores.clear();
+    this.lastUpdateTime.clear();
+    this.bannedPeers.clear();
+  }
+
+  removePeer(peerId: string): void {
+    this.scores.delete(peerId);
+    this.lastUpdateTime.delete(peerId);
+  }
+
+  /**
+   * The single source of truth for whether a peer is banned. Returns the ban score while the ban is
+   * active, or undefined if the peer is not banned. A ban that has expired is lazily lifted before
+   * returning undefined, so callers never see a stale ban.
+   */
+  private getActiveBanScore(peerId: string): number | undefined {
+    const ban = this.bannedPeers.get(peerId);
+    if (ban === undefined) {
+      return undefined;
+    }
+    if (ban.expiry > this.dateProvider.now()) {
+      return ban.score;
+    }
+    // Ban expired: lift it so the peer can recover.
+    this.bannedPeers.delete(peerId);
+    return undefined;
+  }
+
   getScore(peerId: string): number {
-    return this.scores.get(peerId) || 0;
+    // While a ban is active its ban score is returned regardless of how the live score has decayed,
+    // so the peer stays banned for the full duration.
+    return this.getActiveBanScore(peerId) ?? this.scores.get(peerId) ?? 0;
   }
 
   public getScoreState(peerId: string): PeerScoreState {
-    // TODO(#11329): permanently store banned peers?
+    // A banned peer stays banned for the full ban duration regardless of score decay (see getScore /
+    // maybeBanPeer), rather than silently recovering once its decayed score is cleaned up.
     const score = this.getScore(peerId);
     if (score < MIN_SCORE_BEFORE_BAN) {
       return PeerScoreState.Banned;
@@ -143,7 +235,9 @@ export class PeerScoring {
   getStats(): { medianScore: number; healthyCount: number; disconnectCount: number; bannedCount: number } {
     const stateCounts = { healthy: 0, disconnect: 0, banned: 0 };
 
-    for (const peerId of this.scores.keys()) {
+    // Include banned peers whose live score may have been decayed away but whose ban is still active.
+    const peerIds = new Set([...this.scores.keys(), ...this.bannedPeers.keys()]);
+    for (const peerId of peerIds) {
       const state = this.getScoreState(peerId);
       switch (state) {
         case PeerScoreState.Healthy: