@aztec/p2p 0.68.0 → 0.68.2

package/src/services/discv5/discV5_service.ts

@@ -11,26 +11,16 @@ import EventEmitter from 'events';
 import type { P2PConfig } from '../../config.js';
 import { convertToMultiaddr } from '../../util.js';
 import { type PeerDiscoveryService, PeerDiscoveryState } from '../service.js';
-
-export const AZTEC_ENR_KEY = 'aztec_network';
+import { AZTEC_ENR_KEY, AZTEC_NET, Discv5Event, PeerEvent } from '../types.js';
 
 const delayBeforeStart = 2000; // 2sec
 
-export enum AztecENR {
-  devnet = 0x01,
-  testnet = 0x02,
-  mainnet = 0x03,
-}
-
-// TODO: Make this an env var
-export const AZTEC_NET = AztecENR.devnet;
-
 /**
  * Peer discovery service using Discv5.
  */
 export class DiscV5Service extends EventEmitter implements PeerDiscoveryService {
   /** The Discv5 instance */
-  private discv5: Discv5;
+  private discv5: Discv5 & Discv5EventEmitter;
 
   /** This instance's ENR */
   private enr: SignableENR;
@@ -88,13 +78,8 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
       metricsRegistry,
     });
 
-    (this.discv5 as Discv5EventEmitter).on('discovered', (enr: ENR) => this.onDiscovered(enr));
-    (this.discv5 as Discv5EventEmitter).on('enrAdded', async (enr: ENR) => {
-      const multiAddrTcp = await enr.getFullMultiaddr('tcp');
-      const multiAddrUdp = await enr.getFullMultiaddr('udp');
-      this.logger.debug(`Added ENR ${enr.encodeTxt()}`, { multiAddrTcp, multiAddrUdp, nodeId: enr.nodeId });
-      this.onDiscovered(enr);
-    });
+    this.discv5.on(Discv5Event.DISCOVERED, this.onDiscovered.bind(this));
+    this.discv5.on(Discv5Event.ENR_ADDED, this.onEnrAdded.bind(this));
   }
 
   public async start(): Promise<void> {
@@ -168,10 +153,21 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
   }
 
   public async stop(): Promise<void> {
+    await this.discv5.off(Discv5Event.DISCOVERED, this.onDiscovered);
+    await this.discv5.off(Discv5Event.ENR_ADDED, this.onEnrAdded);
+
     await this.discv5.stop();
+
     this.currentState = PeerDiscoveryState.STOPPED;
   }
 
+  private async onEnrAdded(enr: ENR) {
+    const multiAddrTcp = await enr.getFullMultiaddr('tcp');
+    const multiAddrUdp = await enr.getFullMultiaddr('udp');
+    this.logger.debug(`Added ENR ${enr.encodeTxt()}`, { multiAddrTcp, multiAddrUdp, nodeId: enr.nodeId });
+    this.onDiscovered(enr);
+  }
+
   private onDiscovered(enr: ENR) {
     // check the peer is an aztec peer
     const value = enr.kvs.get(AZTEC_ENR_KEY);
@@ -179,7 +175,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
       const network = value[0];
       // check if the peer is on the same network
       if (network === AZTEC_NET) {
-        this.emit('peer:discovered', enr);
+        this.emit(PeerEvent.DISCOVERED, enr);
       }
     }
   }

package/src/services/libp2p/libp2p_service.ts

@@ -26,7 +26,12 @@ import type { AztecKVStore } from '@aztec/kv-store';
 import { Attributes, OtelMetricsAdapter, type TelemetryClient, WithTracer, trackSpan } from '@aztec/telemetry-client';
 
 import { type ENR } from '@chainsafe/enr';
-import { type GossipSub, type GossipSubComponents, gossipsub } from '@chainsafe/libp2p-gossipsub';
+import {
+  type GossipSub,
+  type GossipSubComponents,
+  type GossipsubMessage,
+  gossipsub,
+} from '@chainsafe/libp2p-gossipsub';
 import { createPeerScoreParams, createTopicScoreParams } from '@chainsafe/libp2p-gossipsub/score';
 import { noise } from '@chainsafe/libp2p-noise';
 import { yamux } from '@chainsafe/libp2p-yamux';
@@ -64,6 +69,7 @@ import {
 } from '../reqresp/interface.js';
 import { ReqResp } from '../reqresp/reqresp.js';
 import type { P2PService, PeerDiscoveryService } from '../service.js';
+import { GossipSubEvent } from '../types.js';
 
 interface MessageValidator {
   validator: {
@@ -119,7 +125,7 @@ export class LibP2PService<T extends P2PClientType> extends WithTracer implement
   ) {
     super(telemetry, 'LibP2PService');
 
-    this.peerManager = new PeerManager(node, peerDiscoveryService, config, this.tracer, logger);
+    this.peerManager = new PeerManager(node, peerDiscoveryService, config, telemetry, logger);
     this.node.services.pubsub.score.params.appSpecificScore = (peerId: string) => {
       return this.peerManager.getPeerScore(peerId);
     };
@@ -179,12 +185,7 @@ export class LibP2PService<T extends P2PClientType> extends WithTracer implement
     }
 
     // add GossipSub listener
-    this.node.services.pubsub.addEventListener('gossipsub:message', async e => {
-      const { msg } = e.detail;
-      this.logger.trace(`Received PUBSUB message.`);
-
-      await this.jobQueue.put(() => this.handleNewGossipMessage(msg));
-    });
+    this.node.services.pubsub.addEventListener(GossipSubEvent.MESSAGE, this.handleGossipSubEvent.bind(this));
 
     // Start running promise for peer discovery
     this.discoveryRunningPromise = new RunningPromise(
@@ -212,6 +213,13 @@ export class LibP2PService<T extends P2PClientType> extends WithTracer implement
    * @returns An empty promise.
    */
   public async stop() {
+    // Remove gossip sub listener
+    this.node.services.pubsub.removeEventListener(GossipSubEvent.MESSAGE, this.handleGossipSubEvent.bind(this));
+
+    // Stop peer manager
+    this.logger.debug('Stopping peer manager...');
+    this.peerManager.stop();
+
     this.logger.debug('Stopping job queue...');
     await this.jobQueue.end();
     this.logger.debug('Stopping running promise...');
@@ -365,6 +373,13 @@ export class LibP2PService<T extends P2PClientType> extends WithTracer implement
     return this.peerManager.getPeers(includePending);
   }
 
+  private async handleGossipSubEvent(e: CustomEvent<GossipsubMessage>) {
+    const { msg } = e.detail;
+    this.logger.trace(`Received PUBSUB message.`);
+
+    await this.jobQueue.put(() => this.handleNewGossipMessage(msg));
+  }
+
   /**
    * Send Request via the ReqResp service
    * The subprotocol defined will determine the request and response types

package/src/services/peer-scoring/peer_scoring.ts

@@ -9,6 +9,16 @@ const DefaultPeerPenalties = {
   [PeerErrorSeverity.HighToleranceError]: 2,
 };
 
+export enum PeerScoreState {
+  Banned,
+  Disconnect,
+  Healthy,
+}
+
+// TODO: move into config / constants
+const MIN_SCORE_BEFORE_BAN = -100;
+const MIN_SCORE_BEFORE_DISCONNECT = -50;
+
 export class PeerScoring {
   private scores: Map<string, number> = new Map();
   private lastUpdateTime: Map<string, number> = new Map();
@@ -65,6 +75,17 @@ export class PeerScoring {
     return this.scores.get(peerId) || 0;
   }
 
+  getScoreState(peerId: string) {
+    // TODO: permanently store banned peers???
+    const score = this.getScore(peerId);
+    if (score < MIN_SCORE_BEFORE_BAN) {
+      return PeerScoreState.Banned;
+    } else if (score < MIN_SCORE_BEFORE_DISCONNECT) {
+      return PeerScoreState.Disconnect;
+    }
+    return PeerScoreState.Healthy;
+  }
+
   getStats(): { medianScore: number } {
     return { medianScore: median(Array.from(this.scores.values())) ?? 0 };
   }

package/src/services/peer_manager.ts

@@ -1,71 +1,116 @@
 import { type PeerErrorSeverity, type PeerInfo } from '@aztec/circuit-types';
 import { createLogger } from '@aztec/foundation/log';
-import { type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
+import { type TelemetryClient, WithTracer, trackSpan } from '@aztec/telemetry-client';
 
 import { type ENR } from '@chainsafe/enr';
-import { type PeerId } from '@libp2p/interface';
+import { type Connection, type PeerId } from '@libp2p/interface';
 import { type Multiaddr } from '@multiformats/multiaddr';
 import { inspect } from 'util';
 
 import { type P2PConfig } from '../config.js';
 import { type PubSubLibp2p } from '../util.js';
-import { PeerScoring } from './peer-scoring/peer_scoring.js';
+import { PeerScoreState, PeerScoring } from './peer-scoring/peer_scoring.js';
 import { type PeerDiscoveryService } from './service.js';
+import { PeerEvent } from './types.js';
 
 const MAX_DIAL_ATTEMPTS = 3;
 const MAX_CACHED_PEERS = 100;
+const MAX_CACHED_PEER_AGE_MS = 5 * 60 * 1000; // 5 minutes
+const FAILED_PEER_BAN_TIME_MS = 5 * 60 * 1000; // 5 minutes timeout after failing MAX_DIAL_ATTEMPTS
 
 type CachedPeer = {
   peerId: PeerId;
   enr: ENR;
   multiaddrTcp: Multiaddr;
   dialAttempts: number;
+  addedUnixMs: number;
 };
 
-export class PeerManager implements Traceable {
+type TimedOutPeer = {
+  peerId: string;
+  timeoutUntilMs: number;
+};
+
+export class PeerManager extends WithTracer {
   private cachedPeers: Map<string, CachedPeer> = new Map();
   private peerScoring: PeerScoring;
   private heartbeatCounter: number = 0;
+  private displayPeerCountsPeerHeartbeat: number = 0;
+  private timedOutPeers: Map<string, TimedOutPeer> = new Map();
 
   constructor(
     private libP2PNode: PubSubLibp2p,
     private peerDiscoveryService: PeerDiscoveryService,
     private config: P2PConfig,
-    public readonly tracer: Tracer,
+    telemetryClient: TelemetryClient,
     private logger = createLogger('p2p:peer-manager'),
   ) {
+    super(telemetryClient, 'PeerManager');
+
     this.peerScoring = new PeerScoring(config);
     // Handle new established connections
-    this.libP2PNode.addEventListener('peer:connect', evt => {
-      const peerId = evt.detail;
-      if (this.peerDiscoveryService.isBootstrapPeer(peerId)) {
-        this.logger.verbose(`Connected to bootstrap peer ${peerId.toString()}`);
-      } else {
-        this.logger.verbose(`Connected to transaction peer ${peerId.toString()}`);
-      }
-    });
-
+    this.libP2PNode.addEventListener(PeerEvent.CONNECTED, this.handleConnectedPeerEvent.bind(this));
     // Handle lost connections
-    this.libP2PNode.addEventListener('peer:disconnect', evt => {
-      const peerId = evt.detail;
-      if (this.peerDiscoveryService.isBootstrapPeer(peerId)) {
-        this.logger.verbose(`Disconnected from bootstrap peer ${peerId.toString()}`);
-      } else {
-        this.logger.verbose(`Disconnected from transaction peer ${peerId.toString()}`);
-      }
-    });
+    this.libP2PNode.addEventListener(PeerEvent.DISCONNECTED, this.handleDisconnectedPeerEvent.bind(this));
 
     // Handle Discovered peers
-    this.peerDiscoveryService.on('peer:discovered', async (enr: ENR) => {
-      await this.handleDiscoveredPeer(enr);
-    });
+    this.peerDiscoveryService.on(PeerEvent.DISCOVERED, this.handleDiscoveredPeer.bind(this));
+
+    // Display peer counts every 60 seconds
+    this.displayPeerCountsPeerHeartbeat = Math.floor(60_000 / this.config.peerCheckIntervalMS);
   }
 
   @trackSpan('PeerManager.heartbeat')
   public heartbeat() {
     this.heartbeatCounter++;
-    this.discover();
     this.peerScoring.decayAllScores();
+
+    this.cleanupExpiredTimeouts();
+
+    this.discover();
+  }
+
+  /**
+   * Cleans up expired timeouts.
+   *
+   * When peers fail to dial after a number of retries, they are temporarily timed out.
+   * This function removes any peers that have been in the timed out state for too long.
+   * To give them a chance to reconnect.
+   */
+  private cleanupExpiredTimeouts() {
+    // Clean up expired timeouts
+    const now = Date.now();
+    for (const [peerId, timedOutPeer] of this.timedOutPeers.entries()) {
+      if (now >= timedOutPeer.timeoutUntilMs) {
+        this.timedOutPeers.delete(peerId);
+      }
+    }
+  }
+
+  /**
+   * Simply logs the type of connected peer.
+   * @param e - The connected peer event.
+   */
+  private handleConnectedPeerEvent(e: CustomEvent<PeerId>) {
+    const peerId = e.detail;
+    if (this.peerDiscoveryService.isBootstrapPeer(peerId)) {
+      this.logger.verbose(`Connected to bootstrap peer ${peerId.toString()}`);
+    } else {
+      this.logger.verbose(`Connected to transaction peer ${peerId.toString()}`);
+    }
+  }
+
+  /**
+   * Simply logs the type of disconnected peer.
+   * @param e - The disconnected peer event.
+   */
+  private handleDisconnectedPeerEvent(e: CustomEvent<PeerId>) {
+    const peerId = e.detail;
+    if (this.peerDiscoveryService.isBootstrapPeer(peerId)) {
+      this.logger.verbose(`Disconnected from bootstrap peer ${peerId.toString()}`);
+    } else {
+      this.logger.verbose(`Disconnected from transaction peer ${peerId.toString()}`);
+    }
   }
 
   public penalizePeer(peerId: PeerId, penalty: PeerErrorSeverity) {
@@ -116,13 +161,14 @@ export class PeerManager implements Traceable {
    * Discovers peers.
    */
   private discover() {
-    // Get current connections
     const connections = this.libP2PNode.getConnections();
 
+    const healthyConnections = this.pruneUnhealthyPeers(connections);
+
     // Calculate how many connections we're looking to make
-    const peersToConnect = this.config.maxPeerCount - connections.length;
+    const peersToConnect = this.config.maxPeerCount - healthyConnections.length;
 
-    const logLevel = this.heartbeatCounter % 60 === 0 ? 'info' : 'debug';
+    const logLevel = this.heartbeatCounter % this.displayPeerCountsPeerHeartbeat === 0 ? 'info' : 'debug';
     this.logger[logLevel](`Connected to ${connections.length} peers`, {
       connections: connections.length,
       maxPeerCount: this.config.maxPeerCount,
@@ -146,7 +192,12 @@ export class PeerManager implements Traceable {
 
     for (const [id, peerData] of this.cachedPeers.entries()) {
       // if already dialling or connected to, remove from cache
-      if (pendingDials.has(id) || connections.some(conn => conn.remotePeer.equals(peerData.peerId))) {
+      if (
+        pendingDials.has(id) ||
+        healthyConnections.some(conn => conn.remotePeer.equals(peerData.peerId)) ||
+        // if peer has been in cache for the max cache age, remove from cache
+        Date.now() - peerData.addedUnixMs > MAX_CACHED_PEER_AGE_MS
+      ) {
         this.cachedPeers.delete(id);
       } else {
         // cachedPeersToDial.set(id, enr);
@@ -158,6 +209,7 @@ export class PeerManager implements Traceable {
     cachedPeersToDial.reverse();
 
     for (const peer of cachedPeersToDial) {
+      // We remove from the cache before, as dialling will add it back if it fails
       this.cachedPeers.delete(peer.peerId.toString());
       void this.dialPeer(peer);
     }
@@ -169,35 +221,74 @@ export class PeerManager implements Traceable {
     }
   }
 
+  private pruneUnhealthyPeers(connections: Connection[]): Connection[] {
+    const connectedHealthyPeers: Connection[] = [];
+
+    for (const peer of connections) {
+      const score = this.peerScoring.getScoreState(peer.remotePeer.toString());
+      switch (score) {
+        // TODO: add goodbye and give reasons
+        case PeerScoreState.Banned:
+        case PeerScoreState.Disconnect:
+          void this.disconnectPeer(peer.remotePeer);
+          break;
+        case PeerScoreState.Healthy:
+          connectedHealthyPeers.push(peer);
+      }
+    }
+
+    return connectedHealthyPeers;
+  }
+
+  // TODO: send a goodbye with a reason to the peer
+  private async disconnectPeer(peer: PeerId) {
+    this.logger.debug(`Disconnecting peer ${peer.toString()}`);
+    await this.libP2PNode.hangUp(peer);
+  }
+
   /**
    *  Handles a discovered peer.
    * @param enr - The discovered peer's ENR.
    */
   private async handleDiscoveredPeer(enr: ENR) {
-    // TODO: Will be handling peer scoring here
+    // Check that the peer has not already been banned
+    const peerId = await enr.peerId();
+    const peerIdString = peerId.toString();
+
+    // Check if peer is temporarily timed out
+    const timedOutPeer = this.timedOutPeers.get(peerIdString);
+    if (timedOutPeer) {
+      if (Date.now() < timedOutPeer.timeoutUntilMs) {
+        this.logger.trace(`Skipping timed out peer ${peerId}`);
+        return;
+      }
+      // Timeout period expired, remove from timed out peers
+      this.timedOutPeers.delete(peerIdString);
+    }
 
-    // check if peer is already connected
-    const [peerId, multiaddrTcp] = await Promise.all([enr.peerId(), enr.getFullMultiaddr('tcp')]);
+    if (this.peerScoring.getScoreState(peerIdString) != PeerScoreState.Healthy) {
+      return;
+    }
 
-    this.logger.trace(
-      `Handling discovered peer ${peerId.toString()} at ${multiaddrTcp?.toString() ?? 'undefined address'}`,
-    );
+    const [multiaddrTcp] = await Promise.all([enr.getFullMultiaddr('tcp')]);
+
+    this.logger.trace(`Handling discovered peer ${peerId} at ${multiaddrTcp?.toString() ?? 'undefined address'}`);
 
-    // throw if no tcp addr in multiaddr
+    // stop if no tcp addr in multiaddr
     if (!multiaddrTcp) {
       this.logger.debug(`No TCP address in discovered node's multiaddr ${enr.encodeTxt()}`);
       return;
     }
+    // check if peer is already connected
     const connections = this.libP2PNode.getConnections();
     if (connections.some(conn => conn.remotePeer.equals(peerId))) {
-      this.logger.trace(`Already connected to peer ${peerId.toString()}`);
+      this.logger.trace(`Already connected to peer ${peerId}`);
       return;
     }
 
     // check if peer is already in cache
-    const id = peerId.toString();
-    if (this.cachedPeers.has(id)) {
-      this.logger.trace(`Peer already in cache ${id}`);
+    if (this.cachedPeers.has(peerIdString)) {
+      this.logger.trace(`Peer already in cache ${peerIdString}`);
       return;
     }
 
@@ -207,14 +298,15 @@ export class PeerManager implements Traceable {
       enr,
       multiaddrTcp,
       dialAttempts: 0,
+      addedUnixMs: Date.now(),
     };
 
     // Determine if we should dial immediately or not
     if (this.shouldDialPeer()) {
       void this.dialPeer(cachedPeer);
     } else {
-      this.logger.trace(`Caching peer ${id}`);
-      this.cachedPeers.set(id, cachedPeer);
+      this.logger.trace(`Caching peer ${peerIdString}`);
+      this.cachedPeers.set(peerIdString, cachedPeer);
       // Prune set of cached peers
       this.pruneCachedPeers();
     }
@@ -222,6 +314,8 @@ export class PeerManager implements Traceable {
 
   private async dialPeer(peer: CachedPeer) {
     const id = peer.peerId.toString();
+
+    // Add to the address book before dialing
     await this.libP2PNode.peerStore.merge(peer.peerId, { multiaddrs: [peer.multiaddrTcp] });
 
     this.logger.trace(`Dialing peer ${id}`);
@@ -236,6 +330,11 @@ export class PeerManager implements Traceable {
         formatLibp2pDialError(error as Error);
         this.logger.debug(`Failed to dial peer ${id} (dropping)`, { error: inspect(error) });
         this.cachedPeers.delete(id);
+        // Add to timed out peers
+        this.timedOutPeers.set(id, {
+          peerId: id,
+          timeoutUntilMs: Date.now() + FAILED_PEER_BAN_TIME_MS,
+        });
       }
     }
   }
@@ -267,6 +366,16 @@ export class PeerManager implements Traceable {
       }
     }
   }
+
+  /**
+   * Stops the peer manager.
+   * Removing all event listeners.
+   */
+  public stop() {
+    this.libP2PNode.removeEventListener(PeerEvent.CONNECTED, this.handleConnectedPeerEvent);
+    this.libP2PNode.removeEventListener(PeerEvent.DISCONNECTED, this.handleDisconnectedPeerEvent);
+    this.peerDiscoveryService.off(PeerEvent.DISCOVERED, this.handleDiscoveredPeer);
+  }
 }
 
 /**

package/src/services/reqresp/reqresp.ts

@@ -1,7 +1,7 @@
 // @attribution: lodestar impl for inspiration
 import { PeerErrorSeverity } from '@aztec/circuit-types';
 import { type Logger, createLogger } from '@aztec/foundation/log';
-import { executeTimeoutWithCustomError } from '@aztec/foundation/timer';
+import { executeTimeout } from '@aztec/foundation/timer';
 
 import { type IncomingStreamData, type PeerId, type Stream } from '@libp2p/interface';
 import { pipe } from 'it-pipe';
@@ -159,7 +159,7 @@ export class ReqResp {
     };
 
     try {
-      return await executeTimeoutWithCustomError<InstanceType<SubProtocolMap[SubProtocol]['response']> | undefined>(
+      return await executeTimeout<InstanceType<SubProtocolMap[SubProtocol]['response']> | undefined>(
         requestFunction,
         this.overallRequestTimeoutMs,
         () => new CollectiveReqRespTimeoutError(),
@@ -205,7 +205,7 @@ export class ReqResp {
       this.logger.trace(`Stream opened with ${peerId.toString()} for ${subProtocol}`);
 
       // Open the stream with a timeout
-      const result = await executeTimeoutWithCustomError<Buffer>(
+      const result = await executeTimeout<Buffer>(
         (): Promise<Buffer> => pipe([payload], stream!, this.readMessage.bind(this)),
         this.individualRequestTimeoutMs,
         () => new IndividualReqRespTimeoutError(),

package/src/services/types.ts

@@ -0,0 +1,44 @@
+/***************************************************
+ *                    Events
+ ***************************************************/
+
+/**
+ * Events emitted from the libp2p node.
+ */
+export enum PeerEvent {
+  DISCOVERED = 'peer:discovered',
+  CONNECTED = 'peer:connect',
+  DISCONNECTED = 'peer:disconnect',
+}
+
+/**
+ * Events emitted from the Discv5 service.
+ */
+export enum Discv5Event {
+  DISCOVERED = 'discovered',
+  ENR_ADDED = 'enrAdded',
+}
+
+/**
+ * Events emitted from the GossipSub protocol.
+ */
+export enum GossipSubEvent {
+  MESSAGE = 'gossipsub:message',
+}
+
+/***************************************************
+ *                    Types
+ ***************************************************/
+/**
+ * Aztec network specific types
+ */
+export const AZTEC_ENR_KEY = 'aztec_network';
+
+export enum AztecENR {
+  devnet = 0x01,
+  testnet = 0x02,
+  mainnet = 0x03,
+}
+
+// TODO: Make this an env var
+export const AZTEC_NET = AztecENR.devnet;