@aztec/p2p 0.0.0-test.0

package/src/services/reqresp/config.ts

@@ -0,0 +1,35 @@
+import { type ConfigMapping, numberConfigHelper } from '@aztec/foundation/config';
+
+export const DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS = 2000;
+export const DEFAULT_OVERALL_REQUEST_TIMEOUT_MS = 4000;
+
+// For use in tests.
+export const DEFAULT_P2P_REQRESP_CONFIG: P2PReqRespConfig = {
+  overallRequestTimeoutMs: DEFAULT_OVERALL_REQUEST_TIMEOUT_MS,
+  individualRequestTimeoutMs: DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS,
+};
+
+export interface P2PReqRespConfig {
+  /**
+   * The overall timeout for a request response operation.
+   */
+  overallRequestTimeoutMs: number;
+
+  /**
+   * The timeout for an individual request response peer interaction.
+   */
+  individualRequestTimeoutMs: number;
+}
+
+export const p2pReqRespConfigMappings: Record<keyof P2PReqRespConfig, ConfigMapping> = {
+  overallRequestTimeoutMs: {
+    env: 'P2P_REQRESP_OVERALL_REQUEST_TIMEOUT_MS',
+    description: 'The overall timeout for a request response operation.',
+    ...numberConfigHelper(DEFAULT_OVERALL_REQUEST_TIMEOUT_MS),
+  },
+  individualRequestTimeoutMs: {
+    env: 'P2P_REQRESP_INDIVIDUAL_REQUEST_TIMEOUT_MS',
+    description: 'The timeout for an individual request response peer interaction.',
+    ...numberConfigHelper(DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS),
+  },
+};

package/src/services/reqresp/connection-sampler/batch_connection_sampler.ts

@@ -0,0 +1,94 @@
+import { createLogger } from '@aztec/foundation/log';
+
+import type { PeerId } from '@libp2p/interface';
+
+import type { ConnectionSampler } from './connection_sampler.js';
+
+/**
+ * Manages batches of peers for parallel request processing.
+ * Tracks active peers and provides deterministic peer assignment for requests.
+ *
+ * Example with 3 peers and 10 requests:
+ *
+ * Peers:    [P1]      [P2]     [P3]
+ *           ↓ ↓ ↓ ↓   ↓ ↓ ↓   ↓ ↓ ↓
+ * Requests: 0,1,2,9 | 3,4,5 | 6,7,8
+ *
+ * Each peer handles a bucket of consecutive requests.
+ * If a peer fails, it is replaced while maintaining the same bucket.
+ */
+export class BatchConnectionSampler {
+  private readonly logger = createLogger('p2p:reqresp:batch-connection-sampler');
+  private readonly batch: PeerId[] = [];
+  private readonly requestsPerPeer: number;
+
+  constructor(private readonly connectionSampler: ConnectionSampler, batchSize: number, maxPeers: number) {
+    if (maxPeers <= 0) {
+      throw new Error('Max peers cannot be 0');
+    }
+    if (batchSize <= 0) {
+      throw new Error('Batch size cannot be 0');
+    }
+
+    // Calculate how many requests each peer should handle, cannot be 0
+    this.requestsPerPeer = Math.max(1, Math.floor(batchSize / maxPeers));
+
+    // Sample initial peers
+    this.batch = this.connectionSampler.samplePeersBatch(maxPeers);
+  }
+
+  /**
+   * Gets the peer responsible for handling a specific request index
+   *
+   * @param index - The request index
+   * @returns The peer assigned to handle this request
+   */
+  getPeerForRequest(index: number): PeerId | undefined {
+    if (this.batch.length === 0) {
+      return undefined;
+    }
+
+    // Calculate which peer bucket this index belongs to
+    const peerIndex = Math.floor(index / this.requestsPerPeer) % this.batch.length;
+    return this.batch[peerIndex];
+  }
+
+  /**
+   * Removes a peer and replaces it with a new one, maintaining the same position
+   * in the batch array to keep request distribution consistent
+   *
+   * @param peerId - The peer to remove and replace
+   */
+  removePeerAndReplace(peerId: PeerId): void {
+    const index = this.batch.findIndex(p => p === peerId);
+    if (index === -1) {
+      return;
+    }
+
+    const excluding = new Map([[peerId.toString(), true]]);
+    const newPeer = this.connectionSampler.getPeer(excluding);
+
+    if (newPeer) {
+      this.batch[index] = newPeer;
+      this.logger.trace(`Replaced peer ${peerId} with ${newPeer}`, { peerId, newPeer });
+    } else {
+      // If we couldn't get a replacement, remove the peer and compact the array
+      this.batch.splice(index, 1);
+      this.logger.trace(`Removed peer ${peerId}`, { peerId });
+    }
+  }
+
+  /**
+   * Gets the number of active peers
+   */
+  get activePeerCount(): number {
+    return this.batch.length;
+  }
+
+  /**
+   * Gets the number of requests each peer is assigned to handle
+   */
+  get requestsPerBucket(): number {
+    return this.requestsPerPeer;
+  }
+}

package/src/services/reqresp/connection-sampler/connection_sampler.ts

@@ -0,0 +1,217 @@
+import { createLogger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
+
+import type { Libp2p, PeerId, Stream } from '@libp2p/interface';
+
+const MAX_SAMPLE_ATTEMPTS = 4;
+
+interface StreamAndPeerId {
+  stream: Stream;
+  peerId: PeerId;
+}
+
+export class RandomSampler {
+  random(max: number) {
+    return Math.floor(Math.random() * max);
+  }
+}
+
+/**
+ * A class that samples peers from the libp2p node and returns a peer that we don't already have a connection open to.
+ * If we already have a connection open, we try to sample a different peer.
+ * We do this MAX_SAMPLE_ATTEMPTS times, if we still don't find a peer we just go for it.
+ *
+ * @dev Close must always be called on connections, else memory leak
+ */
+export class ConnectionSampler {
+  private readonly logger = createLogger('p2p:reqresp:connection-sampler');
+  private cleanupInterval: NodeJS.Timeout;
+  private abortController: AbortController = new AbortController();
+
+  private readonly activeConnectionsCount: Map<PeerId, number> = new Map();
+  private readonly streams: Map<string, StreamAndPeerId> = new Map();
+
+  // Serial queue to ensure that we only dial one peer at a time
+  private dialQueue: SerialQueue = new SerialQueue();
+
+  constructor(
+    private readonly libp2p: Libp2p,
+    private readonly cleanupIntervalMs: number = 60000, // Default to 1 minute
+    private readonly sampler: RandomSampler = new RandomSampler(), // Allow randomness to be mocked for testing
+  ) {
+    this.cleanupInterval = setInterval(() => void this.cleanupStaleConnections(), this.cleanupIntervalMs);
+
+    this.dialQueue.start();
+  }
+
+  /**
+   * Stops the cleanup job and closes all active connections
+   */
+  async stop() {
+    this.logger.info('Stopping connection sampler');
+    clearInterval(this.cleanupInterval);
+
+    this.abortController.abort();
+    await this.dialQueue.end();
+
+    // Close all active streams
+    const closePromises = Array.from(this.streams.keys()).map(streamId => this.close(streamId));
+    await Promise.all(closePromises);
+    this.logger.info('Connection sampler stopped');
+  }
+
+  /**
+   *
+   * @param excluding - The peers to exclude from the sampling
+   *        This is to prevent sampling with replacement
+   * @returns
+   */
+  getPeer(excluding?: Map<string, boolean>): PeerId | undefined {
+    // In libp2p getPeers performs a shallow copy, so this array can be sliced from safetly
+    const peers = this.libp2p.getPeers();
+
+    if (peers.length === 0) {
+      return undefined;
+    }
+
+    let randomIndex = this.sampler.random(peers.length);
+    let attempts = 0;
+
+    // Keep sampling while:
+    // - we haven't exceeded max attempts AND
+    // - either the peer has active connections OR is in the exclusion list
+    while (
+      attempts < MAX_SAMPLE_ATTEMPTS &&
+      ((this.activeConnectionsCount.get(peers[randomIndex]) ?? 0) > 0 ||
+        (excluding?.get(peers[randomIndex]?.toString()) ?? false))
+    ) {
+      peers.splice(randomIndex, 1);
+      randomIndex = this.sampler.random(peers.length);
+      attempts++;
+    }
+
+    this.logger.trace(`Sampled peer in ${attempts} attempts`, {
+      attempts,
+      peer: peers[randomIndex]?.toString(),
+    });
+    return peers[randomIndex];
+  }
+
+  /**
+   * Samples a batch of unique peers from the libp2p node, prioritizing peers without active connections
+   *
+   * @param numberToSample - The number of peers to sample
+   * @returns Array of unique sampled peers, prioritizing those without active connections
+   */
+  samplePeersBatch(numberToSample: number): PeerId[] {
+    const peers = this.libp2p.getPeers();
+    const sampledPeers: PeerId[] = [];
+    const peersWithConnections: PeerId[] = []; // Hold onto peers with active connections incase we need to sample more
+
+    for (const peer of peers) {
+      const activeConnections = this.activeConnectionsCount.get(peer) ?? 0;
+      if (activeConnections === 0) {
+        if (sampledPeers.push(peer) === numberToSample) {
+          return sampledPeers;
+        }
+      } else {
+        peersWithConnections.push(peer);
+      }
+    }
+
+    // If we still need more peers, sample from those with connections
+    while (sampledPeers.length < numberToSample && peersWithConnections.length > 0) {
+      const randomIndex = this.sampler.random(peersWithConnections.length);
+      const [peer] = peersWithConnections.splice(randomIndex, 1);
+      sampledPeers.push(peer);
+    }
+
+    this.logger.trace(`Batch sampled ${sampledPeers.length} unique peers`, {
+      peers: sampledPeers,
+      withoutConnections: sampledPeers.length - peersWithConnections.length,
+      withConnections: peersWithConnections.length,
+    });
+
+    return sampledPeers;
+  }
+
+  // Set of passthrough functions to keep track of active connections
+
+  /**
+   * Dials a protocol and returns the stream
+   *
+   * @param peerId - The peer id
+   * @param protocol - The protocol
+   * @returns The stream
+   */
+  async dialProtocol(peerId: PeerId, protocol: string): Promise<Stream> {
+    // Dialling at the same time can cause race conditions where two different streams
+    // end up with the same id, hence a serial queue
+    const stream = await this.dialQueue.put(() =>
+      this.libp2p.dialProtocol(peerId, protocol, { signal: this.abortController.signal }),
+    );
+
+    this.streams.set(stream.id, { stream, peerId });
+    const updatedActiveConnectionsCount = (this.activeConnectionsCount.get(peerId) ?? 0) + 1;
+    this.activeConnectionsCount.set(peerId, updatedActiveConnectionsCount);
+
+    this.logger.trace(`Dialed protocol ${protocol} with peer ${peerId.toString()}`, {
+      streamId: stream.id,
+      peerId: peerId.toString(),
+      activeConnectionsCount: updatedActiveConnectionsCount,
+    });
+    return stream;
+  }
+
+  /**
+   * Closes a stream and updates the active connections count
+   *
+   * @param streamId - The stream id
+   */
+  async close(streamId: string): Promise<void> {
+    try {
+      const streamAndPeerId = this.streams.get(streamId);
+      if (!streamAndPeerId) {
+        this.logger.warn(`Stream ${streamId} not found`);
+        return;
+      }
+
+      const { stream, peerId } = streamAndPeerId;
+
+      const updatedActiveConnectionsCount = (this.activeConnectionsCount.get(peerId) ?? 1) - 1;
+      this.activeConnectionsCount.set(peerId, updatedActiveConnectionsCount);
+
+      this.logger.trace(`Closing connection to peer ${peerId.toString()}`, {
+        streamId,
+        peerId: peerId.toString(),
+        protocol: stream.protocol,
+        activeConnectionsCount: updatedActiveConnectionsCount,
+      });
+
+      await stream?.close();
+    } catch (error) {
+      this.logger.error(`Failed to close connection to peer with stream id ${streamId}`, error);
+    } finally {
+      this.streams.delete(streamId);
+    }
+  }
+
+  /**
+   * Cleans up stale connections that we have lost accounting for
+   */
+  private async cleanupStaleConnections() {
+    // Look for streams without anything in the activeConnectionsCount
+    // If we find anything, close the stream
+    for (const [streamId, { peerId }] of this.streams.entries()) {
+      try {
+        // Check if we have lost track of accounting
+        if (this.activeConnectionsCount.get(peerId) === 0) {
+          await this.close(streamId);
+          this.logger.debug(`Cleaned up stale connection ${streamId} to peer ${peerId.toString()}`);
+        }
+      } catch (error) {
+        this.logger.error(`Error cleaning up stale connection ${streamId}`, { error });
+      }
+    }
+  }
+}

package/src/services/reqresp/index.ts

@@ -0,0 +1,4 @@
+/**
+ * Request Response protocol allows nodes to ask their peers for data
+ * that they missed via the traditional gossip protocol.
+ */

package/src/services/reqresp/interface.ts

@@ -0,0 +1,185 @@
+import { Fr } from '@aztec/foundation/fields';
+import { L2Block } from '@aztec/stdlib/block';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { PeerId } from '@libp2p/interface';
+
+import type { ReqRespStatus } from './status.js';
+
+/*
+ * Request Response Sub Protocols
+ */
+export const PING_PROTOCOL = '/aztec/req/ping/0.1.0';
+export const STATUS_PROTOCOL = '/aztec/req/status/0.1.0';
+export const GOODBYE_PROTOCOL = '/aztec/req/goodbye/0.1.0';
+export const TX_REQ_PROTOCOL = '/aztec/req/tx/0.1.0';
+export const BLOCK_REQ_PROTOCOL = '/aztec/req/block/0.1.0';
+
+export enum ReqRespSubProtocol {
+  PING = PING_PROTOCOL,
+  STATUS = STATUS_PROTOCOL,
+  GOODBYE = GOODBYE_PROTOCOL,
+  TX = TX_REQ_PROTOCOL,
+  BLOCK = BLOCK_REQ_PROTOCOL,
+}
+
+/**
+ * A handler for a sub protocol
+ * The message will arrive as a buffer, and the handler must return a buffer
+ */
+export type ReqRespSubProtocolHandler = (peerId: PeerId, msg: Buffer) => Promise<Buffer>;
+
+/**
+ * A type mapping from supprotocol to it's rate limits
+ */
+export type ReqRespSubProtocolRateLimits = Record<ReqRespSubProtocol, ProtocolRateLimitQuota>;
+
+/**
+ * The response from the ReqResp protocol
+ * Consists of a status (Error code) and data
+ */
+export interface ReqRespResponse {
+  status: ReqRespStatus;
+  data: Buffer;
+}
+
+/**
+ * A rate limit quota
+ */
+export interface RateLimitQuota {
+  /**
+   * The time window in ms
+   */
+  quotaTimeMs: number;
+  /**
+   * The number of requests allowed within the time window
+   */
+  quotaCount: number;
+}
+
+export interface ProtocolRateLimitQuota {
+  /**
+   * The rate limit quota for a single peer
+   */
+  peerLimit: RateLimitQuota;
+  /**
+   * The rate limit quota for the global peer set
+   */
+  globalLimit: RateLimitQuota;
+}
+
+export const noopValidator = () => Promise.resolve(true);
+
+/**
+ * A type mapping from supprotocol to it's handling function
+ */
+export type ReqRespSubProtocolHandlers = Record<ReqRespSubProtocol, ReqRespSubProtocolHandler>;
+
+type ResponseValidator<RequestIdentifier, Response> = (
+  request: RequestIdentifier,
+  response: Response,
+  peerId: PeerId,
+) => Promise<boolean>;
+
+export type ReqRespSubProtocolValidators = {
+  [S in ReqRespSubProtocol]: ResponseValidator<any, any>;
+};
+
+export const DEFAULT_SUB_PROTOCOL_VALIDATORS: ReqRespSubProtocolValidators = {
+  [ReqRespSubProtocol.PING]: noopValidator,
+  [ReqRespSubProtocol.STATUS]: noopValidator,
+  [ReqRespSubProtocol.TX]: noopValidator,
+  [ReqRespSubProtocol.GOODBYE]: noopValidator,
+  [ReqRespSubProtocol.BLOCK]: noopValidator,
+};
+
+/**
+ * Sub protocol map determines the request and response types for each
+ * Req Resp protocol
+ */
+export type SubProtocolMap = {
+  [S in ReqRespSubProtocol]: RequestResponsePair<any, any>;
+};
+
+/**
+ * Default handler for unimplemented sub protocols, this SHOULD be overwritten
+ * by the service, but is provided as a fallback
+ */
+const defaultHandler = (_msg: any): Promise<Buffer> => {
+  return Promise.resolve(Buffer.from('unimplemented'));
+};
+
+/**
+ * Default sub protocol handlers - this SHOULD be overwritten by the service,
+ */
+export const DEFAULT_SUB_PROTOCOL_HANDLERS: ReqRespSubProtocolHandlers = {
+  [ReqRespSubProtocol.PING]: defaultHandler,
+  [ReqRespSubProtocol.STATUS]: defaultHandler,
+  [ReqRespSubProtocol.TX]: defaultHandler,
+  [ReqRespSubProtocol.GOODBYE]: defaultHandler,
+  [ReqRespSubProtocol.BLOCK]: defaultHandler,
+};
+
+/**
+ * The Request Response Pair interface defines the methods that each
+ * request response pair must implement
+ */
+interface RequestResponsePair<Req extends { toBuffer(): Buffer }, Res> {
+  /**
+   * The request must implement the toBuffer method (generic serialisation)
+   */
+  request: new (...args: any[]) => Req;
+  /**
+   * The response must implement the static fromBuffer method (generic serialisation)
+   */
+  response: {
+    new (...args: any[]): Res;
+    fromBuffer(buffer: Buffer): Res;
+  };
+}
+
+/**
+ * RequestableBuffer is a wrapper around a buffer that allows it to be
+ * used in generic request response protocols
+ *
+ * An instance of the RequestResponsePair defined above
+ */
+export class RequestableBuffer {
+  constructor(public buffer: Buffer) {}
+
+  toBuffer() {
+    return this.buffer;
+  }
+
+  static fromBuffer(buffer: Buffer) {
+    return new RequestableBuffer(buffer);
+  }
+}
+
+/**
+ * A mapping from each protocol to their request and response types
+ * This defines the request and response types for each sub protocol, used primarily
+ * as a type rather than an object
+ */
+export const subProtocolMap: SubProtocolMap = {
+  [ReqRespSubProtocol.PING]: {
+    request: RequestableBuffer,
+    response: RequestableBuffer,
+  },
+  [ReqRespSubProtocol.STATUS]: {
+    request: RequestableBuffer,
+    response: RequestableBuffer,
+  },
+  [ReqRespSubProtocol.TX]: {
+    request: TxHash,
+    response: Tx,
+  },
+  [ReqRespSubProtocol.GOODBYE]: {
+    request: RequestableBuffer,
+    response: RequestableBuffer,
+  },
+  [ReqRespSubProtocol.BLOCK]: {
+    request: Fr, // block number
+    response: L2Block,
+  },
+};

package/src/services/reqresp/metrics.ts

@@ -0,0 +1,57 @@
+// Request response metrics
+import { Attributes, Metrics, ValueType } from '@aztec/telemetry-client';
+import type { TelemetryClient, Tracer, UpDownCounter } from '@aztec/telemetry-client';
+
+export class ReqRespMetrics {
+  public readonly tracer: Tracer;
+
+  private readonly sentRequests: UpDownCounter;
+  private readonly receivedRequests: UpDownCounter;
+
+  private readonly failedOutboundRequests: UpDownCounter;
+  private readonly failedInboundRequests: UpDownCounter;
+
+  constructor(readonly telemetryClient: TelemetryClient, name = 'ReqResp') {
+    this.tracer = telemetryClient.getTracer(name);
+
+    const meter = telemetryClient.getMeter(name);
+    this.sentRequests = meter.createUpDownCounter(Metrics.P2P_REQ_RESP_SENT_REQUESTS, {
+      description: 'Number of requests sent to peers',
+      unit: 'requests',
+      valueType: ValueType.INT,
+    });
+    this.receivedRequests = meter.createUpDownCounter(Metrics.P2P_REQ_RESP_RECEIVED_REQUESTS, {
+      description: 'Number of requests received from peers',
+      unit: 'requests',
+      valueType: ValueType.INT,
+    });
+
+    this.failedOutboundRequests = meter.createUpDownCounter(Metrics.P2P_REQ_RESP_FAILED_OUTBOUND_REQUESTS, {
+      description: 'Number of failed outbound requests - nodes not getting valid responses',
+      unit: 'requests',
+      valueType: ValueType.INT,
+    });
+
+    this.failedInboundRequests = meter.createUpDownCounter(Metrics.P2P_REQ_RESP_FAILED_INBOUND_REQUESTS, {
+      description: 'Number of failed inbound requests - node failing to respond to requests',
+      unit: 'requests',
+      valueType: ValueType.INT,
+    });
+  }
+
+  public recordRequestSent(protocol: string) {
+    this.sentRequests.add(1, { [Attributes.P2P_REQ_RESP_PROTOCOL]: protocol });
+  }
+
+  public recordRequestReceived(protocol: string) {
+    this.receivedRequests.add(1, { [Attributes.P2P_REQ_RESP_PROTOCOL]: protocol });
+  }
+
+  public recordRequestError(protocol: string) {
+    this.failedOutboundRequests.add(1, { [Attributes.P2P_REQ_RESP_PROTOCOL]: protocol });
+  }
+
+  public recordResponseError(protocol: string) {
+    this.failedInboundRequests.add(1, { [Attributes.P2P_REQ_RESP_PROTOCOL]: protocol });
+  }
+}

package/src/services/reqresp/protocols/block.ts

@@ -0,0 +1,15 @@
+import { Fr } from '@aztec/foundation/fields';
+import type { L2BlockSource } from '@aztec/stdlib/block';
+
+import type { PeerId } from '@libp2p/interface';
+
+import type { ReqRespSubProtocolHandler } from '../interface.js';
+
+export function reqRespBlockHandler(l2BlockSource: L2BlockSource): ReqRespSubProtocolHandler {
+  return async (_peerId: PeerId, msg: Buffer) => {
+    const blockNumber = Fr.fromBuffer(msg);
+
+    const foundBlock = await l2BlockSource.getBlock(Number(blockNumber));
+    return foundBlock ? foundBlock.toBuffer() : Buffer.alloc(0);
+  };
+}