@aztec/p2p 0.0.0-test.1 → 0.0.1-commit.5476d83

package/src/services/peer-manager/peer_scoring.ts

@@ -1,6 +1,14 @@
 import { median } from '@aztec/foundation/collection';
 import { createLogger } from '@aztec/foundation/log';
 import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
+import {
+  Attributes,
+  Metrics,
+  type TelemetryClient,
+  type UpDownCounter,
+  ValueType,
+  getTelemetryClient,
+} from '@aztec/telemetry-client';
 
 import type { PeerId } from '@libp2p/interface';
 
@@ -30,7 +38,9 @@ export class PeerScoring {
   private decayFactor = 0.9;
   peerPenalties: { [key in PeerErrorSeverity]: number };
 
-  constructor(config: P2PConfig) {
+  private peerStateCounter: UpDownCounter;
+
+  constructor(config: P2PConfig, telemetry: TelemetryClient = getTelemetryClient()) {
     const orderedValues = config.peerPenaltyValues?.sort((a, b) => a - b);
     this.peerPenalties = {
       [PeerErrorSeverity.HighToleranceError]:
@@ -40,6 +50,13 @@ export class PeerScoring {
       [PeerErrorSeverity.LowToleranceError]:
         orderedValues?.[2] ?? DefaultPeerPenalties[PeerErrorSeverity.LowToleranceError],
     };
+
+    const meter = telemetry.getMeter('PeerScoring');
+
+    this.peerStateCounter = meter.createUpDownCounter(Metrics.P2P_PEER_STATE_COUNT, {
+      description: 'Count of peers by state (Healthy, Disconnect, Banned)',
+      valueType: ValueType.INT,
+    });
   }
 
   public penalizePeer(peerId: PeerId, penalty: PeerErrorSeverity) {
@@ -99,7 +116,33 @@ export class PeerScoring {
     return PeerScoreState.Healthy;
   }
 
-  getStats(): { medianScore: number } {
-    return { medianScore: median(Array.from(this.scores.values())) ?? 0 };
+  getStats(): { medianScore: number; healthyCount: number; disconnectCount: number; bannedCount: number } {
+    const stateCounts = { healthy: 0, disconnect: 0, banned: 0 };
+
+    for (const peerId of this.scores.keys()) {
+      const state = this.getScoreState(peerId);
+      switch (state) {
+        case PeerScoreState.Healthy:
+          stateCounts.healthy++;
+          break;
+        case PeerScoreState.Disconnect:
+          stateCounts.disconnect++;
+          break;
+        case PeerScoreState.Banned:
+          stateCounts.banned++;
+          break;
+      }
+    }
+
+    this.peerStateCounter.add(stateCounts.healthy, { [Attributes.P2P_PEER_SCORE_STATE]: 'Healthy' });
+    this.peerStateCounter.add(stateCounts.disconnect, { [Attributes.P2P_PEER_SCORE_STATE]: 'Disconnect' });
+    this.peerStateCounter.add(stateCounts.banned, { [Attributes.P2P_PEER_SCORE_STATE]: 'Banned' });
+
+    return {
+      medianScore: median(Array.from(this.scores.values())) ?? 0,
+      healthyCount: stateCounts.healthy,
+      disconnectCount: stateCounts.disconnect,
+      bannedCount: stateCounts.banned,
+    };
   }
 }

package/src/services/reqresp/config.ts

@@ -1,24 +1,30 @@
-import { type ConfigMapping, numberConfigHelper } from '@aztec/foundation/config';
+import { type ConfigMapping, booleanConfigHelper, numberConfigHelper } from '@aztec/foundation/config';
 
-export const DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS = 2000;
-export const DEFAULT_OVERALL_REQUEST_TIMEOUT_MS = 4000;
+export const DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS = 10_000;
+export const DEFAULT_OVERALL_REQUEST_TIMEOUT_MS = 10_000; // Not currently used
+export const DEFAULT_REQRESP_DIAL_TIMEOUT_MS = 5_000;
+export const DEFAULT_OPTIMISTIC_NEGOTIATION = false;
 
 // For use in tests.
 export const DEFAULT_P2P_REQRESP_CONFIG: P2PReqRespConfig = {
   overallRequestTimeoutMs: DEFAULT_OVERALL_REQUEST_TIMEOUT_MS,
   individualRequestTimeoutMs: DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS,
+  dialTimeoutMs: DEFAULT_REQRESP_DIAL_TIMEOUT_MS,
+  p2pOptimisticNegotiation: DEFAULT_OPTIMISTIC_NEGOTIATION,
 };
 
 export interface P2PReqRespConfig {
-  /**
-   * The overall timeout for a request response operation.
-   */
+  /** The overall timeout for a request response operation. */
   overallRequestTimeoutMs: number;
 
-  /**
-   * The timeout for an individual request response peer interaction.
-   */
+  /** The timeout for an individual request response peer interaction. */
   individualRequestTimeoutMs: number;
+
+  /** Whether to use optimistic protocol negotiation when dialing to another peer (opposite of `negotiateFully`). */
+  p2pOptimisticNegotiation: boolean;
+
+  /** How long to wait for the dial protocol to establish a connection */
+  dialTimeoutMs: number;
 }
 
 export const p2pReqRespConfigMappings: Record<keyof P2PReqRespConfig, ConfigMapping> = {
@@ -32,4 +38,15 @@ export const p2pReqRespConfigMappings: Record<keyof P2PReqRespConfig, ConfigMapp
     description: 'The timeout for an individual request response peer interaction.',
     ...numberConfigHelper(DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS),
   },
+  dialTimeoutMs: {
+    env: 'P2P_REQRESP_DIAL_TIMEOUT_MS',
+    description: 'How long to wait for the dial protocol to establish a connection',
+    ...numberConfigHelper(DEFAULT_REQRESP_DIAL_TIMEOUT_MS),
+  },
+  p2pOptimisticNegotiation: {
+    env: 'P2P_REQRESP_OPTIMISTIC_NEGOTIATION',
+    description:
+      'Whether to use optimistic protocol negotiation when dialing to another peer (opposite of `negotiateFully`).',
+    ...booleanConfigHelper(DEFAULT_OPTIMISTIC_NEGOTIATION),
+  },
 };

package/src/services/reqresp/connection-sampler/batch_connection_sampler.ts

@@ -18,11 +18,16 @@ import type { ConnectionSampler } from './connection_sampler.js';
  * If a peer fails, it is replaced while maintaining the same bucket.
  */
 export class BatchConnectionSampler {
-  private readonly logger = createLogger('p2p:reqresp:batch-connection-sampler');
   private readonly batch: PeerId[] = [];
   private readonly requestsPerPeer: number;
 
-  constructor(private readonly connectionSampler: ConnectionSampler, batchSize: number, maxPeers: number) {
+  constructor(
+    private readonly connectionSampler: ConnectionSampler,
+    batchSize: number,
+    maxPeers: number,
+    exclude?: PeerId[],
+    private readonly logger = createLogger('p2p:reqresp:batch-connection-sampler'),
+  ) {
     if (maxPeers <= 0) {
       throw new Error('Max peers cannot be 0');
     }
@@ -34,7 +39,8 @@ export class BatchConnectionSampler {
     this.requestsPerPeer = Math.max(1, Math.floor(batchSize / maxPeers));
 
     // Sample initial peers
-    this.batch = this.connectionSampler.samplePeersBatch(maxPeers);
+    const excluding = exclude && new Map(exclude.map(peerId => [peerId.toString(), true] as const));
+    this.batch = this.connectionSampler.samplePeersBatch(maxPeers, excluding);
   }
 
   /**
@@ -66,15 +72,15 @@ export class BatchConnectionSampler {
     }
 
     const excluding = new Map([[peerId.toString(), true]]);
-    const newPeer = this.connectionSampler.getPeer(excluding);
+    const newPeer = this.connectionSampler.getPeer(excluding); // Q: Shouldn't we accumulate all excluded peers? Otherwise the sampler could return us a previously excluded peer?
 
     if (newPeer) {
       this.batch[index] = newPeer;
-      this.logger.trace(`Replaced peer ${peerId} with ${newPeer}`, { peerId, newPeer });
+      this.logger.trace('Replaced peer', { peerId, newPeer });
     } else {
       // If we couldn't get a replacement, remove the peer and compact the array
       this.batch.splice(index, 1);
-      this.logger.trace(`Removed peer ${peerId}`, { peerId });
+      this.logger.trace('Removed peer', { peerId });
     }
   }
 

package/src/services/reqresp/connection-sampler/connection_sampler.ts

@@ -1,15 +1,10 @@
+import { AbortError } from '@aztec/foundation/error';
 import { createLogger } from '@aztec/foundation/log';
-import { SerialQueue } from '@aztec/foundation/queue';
 
 import type { Libp2p, PeerId, Stream } from '@libp2p/interface';
 
 const MAX_SAMPLE_ATTEMPTS = 4;
 
-interface StreamAndPeerId {
-  stream: Stream;
-  peerId: PeerId;
-}
-
 export class RandomSampler {
   random(max: number) {
     return Math.floor(Math.random() * max);
@@ -17,31 +12,33 @@ export class RandomSampler {
 }
 
 /**
- * A class that samples peers from the libp2p node and returns a peer that we don't already have a connection open to.
+ * A class that samples peers from the libp2p node and returns a peer that we don't already have a reqresp connection open to.
  * If we already have a connection open, we try to sample a different peer.
  * We do this MAX_SAMPLE_ATTEMPTS times, if we still don't find a peer we just go for it.
  *
  * @dev Close must always be called on connections, else memory leak
  */
 export class ConnectionSampler {
-  private readonly logger = createLogger('p2p:reqresp:connection-sampler');
   private cleanupInterval: NodeJS.Timeout;
-  private abortController: AbortController = new AbortController();
 
-  private readonly activeConnectionsCount: Map<PeerId, number> = new Map();
-  private readonly streams: Map<string, StreamAndPeerId> = new Map();
+  // Map from stringified peer id to number of active connections
+  protected readonly activeConnectionsCount: Map<string, number> = new Map();
+
+  // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+  protected readonly streams: Set<Stream> = new Set();
 
-  // Serial queue to ensure that we only dial one peer at a time
-  private dialQueue: SerialQueue = new SerialQueue();
+  private abortOnStop: AbortController = new AbortController();
 
   constructor(
     private readonly libp2p: Libp2p,
-    private readonly cleanupIntervalMs: number = 60000, // Default to 1 minute
-    private readonly sampler: RandomSampler = new RandomSampler(), // Allow randomness to be mocked for testing
+    private readonly sampler: RandomSampler,
+    private readonly logger = createLogger('p2p:reqresp:connection-sampler'),
+    private readonly opts: { cleanupIntervalMs?: number; p2pOptimisticNegotiation?: boolean } = {},
   ) {
-    this.cleanupInterval = setInterval(() => void this.cleanupStaleConnections(), this.cleanupIntervalMs);
-
-    this.dialQueue.start();
+    this.cleanupInterval = setInterval(
+      () => void this.cleanupStaleConnections(),
+      this.opts.cleanupIntervalMs ?? 60_000,
+    );
   }
 
   /**
@@ -49,13 +46,11 @@ export class ConnectionSampler {
    */
   async stop() {
     this.logger.info('Stopping connection sampler');
+    this.abortOnStop.abort(new AbortError('Connection sampler stopped'));
     clearInterval(this.cleanupInterval);
 
-    this.abortController.abort();
-    await this.dialQueue.end();
-
     // Close all active streams
-    const closePromises = Array.from(this.streams.keys()).map(streamId => this.close(streamId));
+    const closePromises = Array.from(this.streams.values()).map(stream => this.close(stream));
     await Promise.all(closePromises);
     this.logger.info('Connection sampler stopped');
   }
@@ -69,70 +64,112 @@ export class ConnectionSampler {
   getPeer(excluding?: Map<string, boolean>): PeerId | undefined {
     // In libp2p getPeers performs a shallow copy, so this array can be sliced from safetly
     const peers = this.libp2p.getPeers();
+    const { peer } = this.getPeerFromList(peers, excluding);
+    return peer;
+  }
 
+  /**
+   * Samples a peer from a list of peers, excluding those that have active (reqresp) connections or are in the exclusion list
+   *
+   * @param peers - The list of peers to sample from
+   * @param excluding - The peers to exclude from the sampling
+   * @returns - A peer from the list, or undefined if no peers are available,
+   *          - a boolean indicating if the peer has active connections, and
+   *          - all sampled peers - to enable optional resampling
+   *
+   * @dev The provided list peers, should be mutated by this function. This allows batch sampling
+   *      to be performed without making extra copies of the list.
+   */
+  getPeerFromList(
+    peers: PeerId[],
+    excluding?: Map<string, boolean>,
+  ): {
+    peer: PeerId | undefined;
+    sampledPeers: PeerId[];
+  } {
     if (peers.length === 0) {
-      return undefined;
+      return { peer: undefined, sampledPeers: [] };
     }
 
-    let randomIndex = this.sampler.random(peers.length);
-    let attempts = 0;
-
-    // Keep sampling while:
-    // - we haven't exceeded max attempts AND
-    // - either the peer has active connections OR is in the exclusion list
-    while (
-      attempts < MAX_SAMPLE_ATTEMPTS &&
-      ((this.activeConnectionsCount.get(peers[randomIndex]) ?? 0) > 0 ||
-        (excluding?.get(peers[randomIndex]?.toString()) ?? false))
-    ) {
+    const sampledPeers: PeerId[] = [];
+    // Try to find a peer that has no active connections and is not in the exclusion list
+    for (let attempts = 0; attempts < MAX_SAMPLE_ATTEMPTS && peers.length > 0; attempts++) {
+      const randomIndex = this.sampler.random(peers.length);
+      const peer = peers[randomIndex];
+      const hasActiveConnections = (this.activeConnectionsCount.get(peer.toString()) ?? 0) > 0;
+      const isExcluded = excluding?.get(peer.toString()) ?? false;
+
+      // Remove this peer from consideration
       peers.splice(randomIndex, 1);
-      randomIndex = this.sampler.random(peers.length);
-      attempts++;
+
+      // If peer is suitable (no active connections and not excluded), return it
+      if (!hasActiveConnections && !isExcluded) {
+        this.logger.trace('Sampled peer', {
+          attempts,
+          peer,
+        });
+        return { peer, sampledPeers };
+      }
+
+      // Keep track of peers that have active reqresp channels, batch sampling will use these to resample
+      sampledPeers.push(peer);
     }
 
-    this.logger.trace(`Sampled peer in ${attempts} attempts`, {
-      attempts,
-      peer: peers[randomIndex]?.toString(),
+    // If we've exhausted our attempts or peers list is empty, return the last peer if available
+    const lastPeer = peers.length > 0 ? peers[this.sampler.random(peers.length)] : undefined;
+
+    this.logger.trace('Sampled peer', {
+      attempts: MAX_SAMPLE_ATTEMPTS,
+      peer: lastPeer?.toString(),
     });
-    return peers[randomIndex];
+    return { peer: lastPeer, sampledPeers };
   }
 
   /**
    * Samples a batch of unique peers from the libp2p node, prioritizing peers without active connections
    *
    * @param numberToSample - The number of peers to sample
+   * @param excluding - The peers to exclude from the sampling
    * @returns Array of unique sampled peers, prioritizing those without active connections
    */
-  samplePeersBatch(numberToSample: number): PeerId[] {
+  samplePeersBatch(numberToSample: number, excluding?: Map<string, boolean>): PeerId[] {
     const peers = this.libp2p.getPeers();
-    const sampledPeers: PeerId[] = [];
-    const peersWithConnections: PeerId[] = []; // Hold onto peers with active connections incase we need to sample more
-
-    for (const peer of peers) {
-      const activeConnections = this.activeConnectionsCount.get(peer) ?? 0;
-      if (activeConnections === 0) {
-        if (sampledPeers.push(peer) === numberToSample) {
-          return sampledPeers;
-        }
-      } else {
-        peersWithConnections.push(peer);
+    this.logger.debug('Sampling peers batch', { numberToSample, peers });
+
+    // Only sample as many peers as we have available
+    numberToSample = Math.min(numberToSample, peers.length);
+
+    const batch: PeerId[] = [];
+    // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+    const withActiveConnections: Set<PeerId> = new Set();
+    for (let i = 0; i < numberToSample; i++) {
+      const { peer, sampledPeers } = this.getPeerFromList(peers, excluding);
+      if (peer) {
+        batch.push(peer);
+      }
+      if (sampledPeers.length > 0) {
+        sampledPeers.forEach(peer => withActiveConnections.add(peer));
       }
     }
+    const lengthWithoutConnections = batch.length;
 
     // If we still need more peers, sample from those with connections
-    while (sampledPeers.length < numberToSample && peersWithConnections.length > 0) {
-      const randomIndex = this.sampler.random(peersWithConnections.length);
-      const [peer] = peersWithConnections.splice(randomIndex, 1);
-      sampledPeers.push(peer);
+    while (batch.length < numberToSample && withActiveConnections.size > 0) {
+      const randomIndex = this.sampler.random(withActiveConnections.size);
+
+      const peer = Array.from(withActiveConnections)[randomIndex];
+      withActiveConnections.delete(peer);
+      batch.push(peer);
     }
 
-    this.logger.trace(`Batch sampled ${sampledPeers.length} unique peers`, {
-      peers: sampledPeers,
-      withoutConnections: sampledPeers.length - peersWithConnections.length,
-      withConnections: peersWithConnections.length,
+    this.logger.trace('Batch sampled peers', {
+      length: batch.length,
+      peers: batch,
+      withoutConnections: lengthWithoutConnections,
+      withConnections: numberToSample - lengthWithoutConnections,
     });
 
-    return sampledPeers;
+    return batch;
   }
 
   // Set of passthrough functions to keep track of active connections
@@ -142,22 +179,27 @@ export class ConnectionSampler {
    *
    * @param peerId - The peer id
    * @param protocol - The protocol
+   * @param timeout - Abort connection if it takes too long
    * @returns The stream
    */
-  async dialProtocol(peerId: PeerId, protocol: string): Promise<Stream> {
-    // Dialling at the same time can cause race conditions where two different streams
-    // end up with the same id, hence a serial queue
-    const stream = await this.dialQueue.put(() =>
-      this.libp2p.dialProtocol(peerId, protocol, { signal: this.abortController.signal }),
-    );
+  async dialProtocol(peerId: PeerId, protocol: string, timeout?: number): Promise<Stream> {
+    const stream = await this.libp2p.dialProtocol(peerId, protocol, {
+      signal: AbortSignal.any(
+        timeout ? [this.abortOnStop.signal, AbortSignal.timeout(timeout!)] : [this.abortOnStop.signal],
+      ),
+      negotiateFully: !this.opts.p2pOptimisticNegotiation,
+    });
+    stream.metadata.peerId = peerId;
+    this.streams.add(stream);
 
-    this.streams.set(stream.id, { stream, peerId });
-    const updatedActiveConnectionsCount = (this.activeConnectionsCount.get(peerId) ?? 0) + 1;
-    this.activeConnectionsCount.set(peerId, updatedActiveConnectionsCount);
+    const peerIdString = peerId.toString();
+    const updatedActiveConnectionsCount = (this.activeConnectionsCount.get(peerIdString) ?? 0) + 1;
+    this.activeConnectionsCount.set(peerIdString, updatedActiveConnectionsCount);
 
-    this.logger.trace(`Dialed protocol ${protocol} with peer ${peerId.toString()}`, {
+    this.logger.trace('Dialed protocol', {
       streamId: stream.id,
-      peerId: peerId.toString(),
+      protocol,
+      peerId: peerIdString,
       activeConnectionsCount: updatedActiveConnectionsCount,
     });
     return stream;
@@ -165,34 +207,39 @@ export class ConnectionSampler {
 
   /**
    * Closes a stream and updates the active connections count
-   *
-   * @param streamId - The stream id
    */
-  async close(streamId: string): Promise<void> {
-    try {
-      const streamAndPeerId = this.streams.get(streamId);
-      if (!streamAndPeerId) {
-        this.logger.warn(`Stream ${streamId} not found`);
-        return;
-      }
+  async close(stream: Stream): Promise<void> {
+    let peerId: PeerId | undefined = undefined;
 
-      const { stream, peerId } = streamAndPeerId;
+    try {
+      peerId = stream.metadata.peerId;
+      let updatedActiveConnectionsCount = undefined;
 
-      const updatedActiveConnectionsCount = (this.activeConnectionsCount.get(peerId) ?? 1) - 1;
-      this.activeConnectionsCount.set(peerId, updatedActiveConnectionsCount);
+      if (!peerId) {
+        this.logger.warn(`Stream ${stream.id} does not have a peerId set`);
+      } else {
+        updatedActiveConnectionsCount = (this.activeConnectionsCount.get(peerId.toString()) ?? 1) - 1;
+        this.activeConnectionsCount.set(peerId.toString(), updatedActiveConnectionsCount);
+      }
 
-      this.logger.trace(`Closing connection to peer ${peerId.toString()}`, {
-        streamId,
-        peerId: peerId.toString(),
+      this.logger.trace('Closing connection', {
+        streamId: stream.id,
+        peerId: peerId?.toString(),
         protocol: stream.protocol,
         activeConnectionsCount: updatedActiveConnectionsCount,
       });
 
-      await stream?.close();
+      if (!this.streams.has(stream)) {
+        this.logger.debug(`Stream ${stream.id} is not in the active streams set`);
+      }
+
+      await stream.close();
     } catch (error) {
-      this.logger.error(`Failed to close connection to peer with stream id ${streamId}`, error);
+      this.logger.error(`Failed to close connection to peer ${peerId ?? 'unknown'} with stream id ${stream.id}`, error);
+      // graceful close failed, abort the stream
+      stream.abort(new AbortError('Failed to close stream gracefully'));
     } finally {
-      this.streams.delete(streamId);
+      this.streams.delete(stream);
     }
   }
 
@@ -202,15 +249,21 @@ export class ConnectionSampler {
   private async cleanupStaleConnections() {
     // Look for streams without anything in the activeConnectionsCount
     // If we find anything, close the stream
-    for (const [streamId, { peerId }] of this.streams.entries()) {
+    for (const stream of this.streams.values()) {
       try {
         // Check if we have lost track of accounting
-        if (this.activeConnectionsCount.get(peerId) === 0) {
-          await this.close(streamId);
-          this.logger.debug(`Cleaned up stale connection ${streamId} to peer ${peerId.toString()}`);
+        const peerId: PeerId = stream.metadata.peerId;
+        if (!peerId) {
+          this.logger.warn(`Stream ${stream.id} does not have a peerId set`);
+        } else if (this.activeConnectionsCount.get(peerId.toString()) === 0) {
+          await this.close(stream);
+          this.logger.debug('Cleaned up stale connection', { streamId: stream.id, peerId: peerId.toString() });
         }
       } catch (error) {
-        this.logger.error(`Error cleaning up stale connection ${streamId}`, { error });
+        this.logger.error(
+          `Error cleaning up stale connection to peer ${stream.metadata.peerId?.toString() ?? 'unknown'} stream ${stream.id}`,
+          { error },
+        );
       }
     }
   }

package/src/services/reqresp/index.ts

@@ -2,3 +2,5 @@
  * Request Response protocol allows nodes to ask their peers for data
  * that they missed via the traditional gossip protocol.
  */
+export * from './protocols/index.js';
+export * from './interface.js';