@aztec/p2p 0.0.1-commit.3469e52 → 0.0.1-commit.350e0a4d

package/src/services/reqresp/batch-tx-requester/interface.ts

@@ -0,0 +1,61 @@
+import type { ISemaphore } from '@aztec/foundation/queue';
+import type { PeerErrorSeverity } from '@aztec/stdlib/p2p';
+import type { Tx, TxHash, TxValidator } from '@aztec/stdlib/tx';
+
+import type { PeerId } from '@libp2p/interface';
+
+import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
+import type { BlockTxsRequest, BlockTxsResponse } from '../index.js';
+import type { ReqRespInterface } from '../interface.js';
+import type { IPeerCollection } from './peer_collection.js';
+import type { BatchRequestTxValidatorConfig } from './tx_validator.js';
+
+export interface IPeerPenalizer {
+  penalizePeer(peerId: PeerId, penalty: PeerErrorSeverity): void;
+}
+
+export interface ITxMetadataCollection {
+  getMissingTxHashes(): Set<string>;
+  markFetched(peerId: PeerId, tx: Tx): boolean;
+  getTxsToRequestFromThePeer(peer: PeerId): TxHash[];
+  markRequested(txHash: TxHash): void;
+  markInFlightBySmartPeer(txHash: TxHash): void;
+  markNotInFlightBySmartPeer(txHash: TxHash): void;
+  alreadyFetched(txHash: TxHash): boolean;
+  // Returns true if tx was marked as fetched, false if it was already marked as fetched
+  markPeerHas(peerId: PeerId, txHashes: TxHash[]): void;
+  /** Remove all tx metadata associations for a peer (e.g. on demotion from smart to dumb). */
+  clearPeerData(peerId: PeerId): void;
+}
+
+/**
+ * Interface for BatchTxRequester dependencies that can be injected from upstream
+ */
+export interface BatchTxRequesterLibP2PService {
+  /** ReqResp interface for sending requests to peers */
+  reqResp: Pick<ReqRespInterface, 'sendRequestToPeer'>;
+  /** Connection sampler for getting peer lists */
+  connectionSampler: Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'>;
+  /** Configuration needed for transaction validation */
+  txValidatorConfig: BatchRequestTxValidatorConfig;
+  /** Peer scoring for penalizing peers */
+  peerScoring: IPeerPenalizer;
+  /** Validate the requested block transactions request-response consistency */
+  validateRequestedBlockTxsConsistency: (
+    request: BlockTxsRequest,
+    response: BlockTxsResponse,
+    peerId: PeerId,
+  ) => Promise<boolean>;
+}
+
+export interface BatchTxRequesterOptions {
+  smartParallelWorkerCount?: number;
+  dumbParallelWorkerCount?: number;
+  txBatchSize?: number;
+  badPeerThreshold?: number;
+  //Injectable for testing purposes
+  semaphore?: ISemaphore;
+  peerCollection?: IPeerCollection;
+  /** Optional tx validator for testing - if not provided, one is created from p2pService.txValidatorConfig */
+  txValidator?: TxValidator;
+}

package/src/services/reqresp/batch-tx-requester/missing_txs.ts

@@ -0,0 +1,168 @@
+import { type Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { PeerId } from '@libp2p/interface';
+
+import type { IRequestTracker } from '../../tx_collection/request_tracker.js';
+import { DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE } from './config.js';
+import type { ITxMetadataCollection } from './interface.js';
+
+class MissingTxMetadata {
+  constructor(
+    public readonly txHash: string,
+    public requestedCount = 0,
+    public inFlightCount = 0,
+    public tx: Tx | undefined = undefined,
+    public readonly peers = new Set<string>(),
+  ) {}
+
+  public markAsRequested() {
+    this.requestedCount++;
+  }
+
+  public markInFlight() {
+    this.inFlightCount++;
+  }
+
+  public markNotInFlight() {
+    this.inFlightCount = Math.max(--this.inFlightCount, 0);
+  }
+
+  public isInFlight(): boolean {
+    return this.inFlightCount > 0;
+  }
+}
+
+/*
+ * Single source or truth for transactions we are fetching
+ * This could be better optimized but given expected count of missing txs (N < 100)
+ * At the moment there is no need for it. And benefit is that we have everything in single store
+ * */
+export class MissingTxMetadataCollection implements ITxMetadataCollection {
+  private txMetadata = new Map<string, MissingTxMetadata>();
+
+  constructor(
+    private requestTracker: IRequestTracker,
+    private readonly txBatchSize: number = DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE,
+  ) {
+    requestTracker.missingTxHashes.forEach(hash => this.txMetadata.set(hash, new MissingTxMetadata(hash)));
+  }
+
+  public getPrioritizingNotInFlightAndLowerRequestCount(txs: string[]): MissingTxMetadata[] {
+    const filtered = Array.from(this.txMetadata.values()).filter(txMeta => txs.includes(txMeta.txHash.toString()));
+
+    const [notInFlight, inFlight] = filtered.reduce<[MissingTxMetadata[], MissingTxMetadata[]]>(
+      (buckets, tx) => {
+        tx.isInFlight() ? buckets[1].push(tx) : buckets[0].push(tx);
+        return buckets;
+      },
+      [[], []],
+    );
+
+    notInFlight.sort((a, b) => a.requestedCount - b.requestedCount);
+    inFlight.sort((a, b) => a.inFlightCount - b.inFlightCount);
+
+    return [...notInFlight, ...inFlight];
+  }
+
+  public getMissingTxHashes(): Set<string> {
+    return this.requestTracker.missingTxHashes;
+  }
+
+  public getTxsPeerHas(peer: PeerId): Set<string> {
+    const peerIdStr = peer.toString();
+    const txsPeerHas = new Set<string>();
+
+    this.txMetadata.values().forEach(txMeta => {
+      if (txMeta.peers.has(peerIdStr)) {
+        txsPeerHas.add(txMeta.txHash.toString());
+      }
+    });
+
+    return txsPeerHas;
+  }
+
+  public getTxsToRequestFromThePeer(peer: PeerId): TxHash[] {
+    const txsPeerHas = this.getTxsPeerHas(peer);
+    const missingTxHashes = this.getMissingTxHashes();
+
+    const txsToRequest = txsPeerHas.intersection(missingTxHashes);
+
+    if (txsToRequest.size >= this.txBatchSize) {
+      return this.getPrioritizingNotInFlightAndLowerRequestCount(Array.from(txsToRequest))
+        .map(t => TxHash.fromString(t.txHash))
+        .slice(0, this.txBatchSize);
+    }
+
+    // Otherwise fill the txs to request till txBatchSize with random txs we are missing
+    // Who knows, maybe we get lucky and peer received these txs in the meantime
+
+    const countToFill = this.txBatchSize - txsToRequest.size;
+    const txsToFill = this.getPrioritizingNotInFlightAndLowerRequestCount(
+      Array.from(this.getMissingTxHashes().difference(txsToRequest)),
+    )
+      .slice(0, countToFill)
+      .map(t => TxHash.fromString(t.txHash));
+
+    return [...Array.from(txsToRequest).map(t => TxHash.fromString(t)), ...txsToFill];
+  }
+
+  public markRequested(txHash: TxHash) {
+    this.txMetadata.get(txHash.toString())?.markAsRequested();
+  }
+
+  /*
+   * This should be called only when requesting tx from smart peer
+   * Because the smart peer should return this tx, whereas
+   * "dumb" peer might return it, or might not - we don't know
+   * */
+  public markInFlightBySmartPeer(txHash: TxHash) {
+    this.txMetadata.get(txHash.toString())?.markInFlight();
+  }
+
+  /*
+   * This should be called only when requesting tx from smart peer
+   * Because the smart peer should return this tx, whereas
+   * "dumb" peer might return it, or might not - we don't know*/
+  public markNotInFlightBySmartPeer(txHash: TxHash) {
+    this.txMetadata.get(txHash.toString())?.markNotInFlight();
+  }
+
+  public alreadyFetched(txHash: TxHash): boolean {
+    return !this.requestTracker.isMissing(txHash.toString());
+  }
+
+  public markFetched(peerId: PeerId, tx: Tx): boolean {
+    const txHashStr = tx.txHash.toString();
+    const txMeta = this.txMetadata.get(txHashStr);
+    if (!txMeta) {
+      //TODO: what to do about peer which sent txs we didn't request?
+      // 1. don't request from it in the scope of this batch request
+      // 2. ban it immediately?
+      // 3. track it and ban it?
+      //
+      return false;
+    }
+
+    txMeta.peers.add(peerId.toString());
+    return this.requestTracker.markFetched(tx);
+  }
+
+  public markPeerHas(peerId: PeerId, txHash: TxHash[]) {
+    const peerIdStr = peerId.toString();
+    txHash
+      .map(t => t.toString())
+      .forEach(txh => {
+        const txMeta = this.txMetadata.get(txh);
+        if (txMeta) {
+          txMeta.peers.add(peerIdStr);
+        }
+      });
+  }
+
+  public clearPeerData(peerId: PeerId) {
+    const peerIdStr = peerId.toString();
+    for (const txMeta of this.txMetadata.values()) {
+      txMeta.peers.delete(peerIdStr);
+    }
+  }
+}

package/src/services/reqresp/batch-tx-requester/peer_collection.ts

@@ -0,0 +1,249 @@
+import type { DateProvider } from '@aztec/foundation/timer';
+import type { PeerErrorSeverity } from '@aztec/stdlib/p2p';
+
+import type { PeerId } from '@libp2p/interface';
+import { peerIdFromString } from '@libp2p/peer-id';
+
+import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
+import { DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD } from './config.js';
+import type { IPeerPenalizer } from './interface.js';
+
+export const RATE_LIMIT_EXCEEDED_PEER_CACHE_TTL = 1000; // 1s
+
+export interface IPeerCollection {
+  markPeerSmart(peerId: PeerId): void;
+  markPeerDumb(peerId: PeerId): void;
+
+  /** Sample next peer in round-robin fashion. No smart peers if returns undefined */
+  nextSmartPeerToQuery(): PeerId | undefined;
+  /** Sample next peer in round-robin fashion. No dumb peers if returns undefined */
+  nextDumbPeerToQuery(): PeerId | undefined;
+
+  thereAreSomeDumbRatelimitExceededPeers(): boolean;
+  penalisePeer(peerId: PeerId, severity: PeerErrorSeverity): void;
+  unMarkPeerAsBad(peerId: PeerId): void;
+  getBadPeers(): Set<string>;
+  markPeerInFlight(peerId: PeerId): void;
+  unMarkPeerInFlight(peerId: PeerId): void;
+  markPeerRateLimitExceeded(peerId: PeerId): void;
+  getRateLimitExceededPeers(): Set<string>;
+  getPeerRateLimitDelayMs(peerId: PeerId): number | undefined;
+  getNextDumbPeerAvailabilityDelayMs(): number | undefined;
+  getNextSmartPeerAvailabilityDelayMs(): number | undefined;
+}
+
+export class PeerCollection implements IPeerCollection {
+  private readonly smartPeers = new Set<string>();
+  private readonly inFlightPeers = new Set<string>();
+  private readonly rateLimitExceededPeers = new Map<string, number>();
+  private readonly peerPenaltyCounters = new Map<string, number>();
+  private readonly badPeers = new Set<string>();
+
+  constructor(
+    private readonly connectionSampler: Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'>,
+    private readonly pinnedPeerId: PeerId | undefined,
+    private readonly dateProvider: DateProvider,
+    private readonly badPeerThreshold: number = DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD,
+    private readonly peerPenalizer?: IPeerPenalizer,
+  ) {
+    // Pinned peer is treated specially, always mark it as in-flight
+    // and never return it as part of smart/dumb peers
+    if (this.pinnedPeerId) {
+      const peerIdStr = this.pinnedPeerId.toString();
+      this.inFlightPeers.add(peerIdStr);
+    }
+  }
+
+  public markPeerSmart(peerId: PeerId): void {
+    this.smartPeers.add(peerId.toString());
+  }
+
+  public markPeerDumb(peerId: PeerId): void {
+    this.smartPeers.delete(peerId.toString());
+  }
+
+  // We keep track of all peers that are queried for peer sampling algorithm
+  private queriedSmartPeers: Set<string> = new Set<string>();
+  private queriedDumbPeers: Set<string> = new Set<string>();
+
+  private static nextPeer(allPeers: Set<string>, queried: Set<string>): PeerId | undefined {
+    if (allPeers.size === 0) {
+      return undefined;
+    }
+    const availablePeers = allPeers.difference(queried);
+    let [first] = availablePeers;
+    if (first === undefined) {
+      // We queried all peers. Start over
+      [first] = allPeers;
+      queried.clear();
+    }
+    queried.add(first);
+    return peerIdFromString(first);
+  }
+
+  public nextSmartPeerToQuery(): PeerId | undefined {
+    return PeerCollection.nextPeer(this.availableSmartPeers, this.queriedSmartPeers);
+  }
+
+  public nextDumbPeerToQuery(): PeerId | undefined {
+    return PeerCollection.nextPeer(this.availableDumbPeers, this.queriedDumbPeers);
+  }
+
+  private get availableSmartPeers(): Set<string> {
+    return this.peers.intersection(
+      this.smartPeers.difference(this.getBadPeers().union(this.inFlightPeers).union(this.getRateLimitExceededPeers())),
+    );
+  }
+
+  private get availableDumbPeers(): Set<string> {
+    return this.peers.difference(
+      this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers()),
+    );
+  }
+
+  public thereAreSomeDumbRatelimitExceededPeers(): boolean {
+    return (
+      this.getRateLimitExceededPeers().difference(this.smartPeers.union(this.badPeers).union(this.inFlightPeers)).size >
+      0
+    );
+  }
+
+  public markPeerInFlight(peerId: PeerId) {
+    this.inFlightPeers.add(peerId.toString());
+  }
+
+  public unMarkPeerInFlight(peerId: PeerId) {
+    // Never unmark the pinned peer as in-flight
+    if (this.pinnedPeerId && this.pinnedPeerId.toString() === peerId.toString()) {
+      return;
+    }
+    this.inFlightPeers.delete(peerId.toString());
+  }
+
+  public markPeerRateLimitExceeded(peerId: PeerId) {
+    const ttl = this.dateProvider.now() + RATE_LIMIT_EXCEEDED_PEER_CACHE_TTL;
+    this.rateLimitExceededPeers.set(peerId.toString(), ttl);
+  }
+
+  public getRateLimitExceededPeers(): Set<string> {
+    const now = this.dateProvider.now();
+    const rateLimitedPeers = new Set<string>();
+
+    for (const [peerId, expirationTime] of this.rateLimitExceededPeers) {
+      if (expirationTime <= now) {
+        this.rateLimitExceededPeers.delete(peerId);
+      } else {
+        rateLimitedPeers.add(peerId);
+      }
+    }
+
+    return rateLimitedPeers;
+  }
+
+  public penalisePeer(peerId: PeerId, severity: PeerErrorSeverity): void {
+    const key = peerId.toString();
+    const newPenaltyCount = (this.peerPenaltyCounters.get(key) ?? 0) + 1;
+    this.peerPenaltyCounters.set(key, newPenaltyCount);
+    this.peerPenalizer?.penalizePeer(peerId, severity);
+    if (newPenaltyCount > this.badPeerThreshold) {
+      this.badPeers.add(key);
+    }
+  }
+
+  public unMarkPeerAsBad(peerId: PeerId) {
+    const key = peerId.toString();
+    this.badPeers.delete(key);
+    this.peerPenaltyCounters.delete(key);
+  }
+
+  public getBadPeers(): Set<string> {
+    return new Set(this.badPeers);
+  }
+
+  public getPeerRateLimitDelayMs(peerId: PeerId): number | undefined {
+    const key = peerId.toString();
+    const expiry = this.rateLimitExceededPeers.get(key);
+    const peerIsNotRateLimited = expiry === undefined;
+    if (peerIsNotRateLimited) {
+      return undefined;
+    }
+
+    const now = this.dateProvider.now();
+    const rateLimitHasExpired = expiry <= now;
+    if (rateLimitHasExpired) {
+      this.rateLimitExceededPeers.delete(key);
+      return undefined;
+    }
+    return expiry - now;
+  }
+
+  public getNextDumbPeerAvailabilityDelayMs(): number | undefined {
+    // Note: this _is_ suboptimal
+    // (we could've tracked rate limits ) per dumb/smart peers - different collections
+    // but everything is in memory and small scale so this, wile suboptimal is not slow
+    return this.getNextRateLimitDelayMs(
+      peerIdStr =>
+        !this.smartPeers.has(peerIdStr) &&
+        !this.getBadPeers().has(peerIdStr) &&
+        !this.inFlightPeers.has(peerIdStr) &&
+        this.peers.has(peerIdStr),
+    );
+  }
+
+  public getNextSmartPeerAvailabilityDelayMs(): number | undefined {
+    return this.getNextRateLimitDelayMs(
+      peerIdStr =>
+        this.smartPeers.has(peerIdStr) && !this.getBadPeers().has(peerIdStr) && !this.inFlightPeers.has(peerIdStr),
+    );
+  }
+
+  private getNextRateLimitDelayMs(filter: (peerIdStr: string) => boolean): number | undefined {
+    const now = this.dateProvider.now();
+    let minExpiry: number | undefined;
+
+    for (const [peerIdStr, expiry] of this.rateLimitExceededPeers) {
+      const rateLimitHasExpired = expiry <= now;
+      if (rateLimitHasExpired) {
+        this.rateLimitExceededPeers.delete(peerIdStr);
+        continue;
+      }
+
+      const peerDoesNotMatchFilter = !filter(peerIdStr);
+      if (peerDoesNotMatchFilter) {
+        continue;
+      }
+
+      minExpiry = minExpiry === undefined ? expiry : Math.min(minExpiry, expiry);
+    }
+
+    const noRateLimitedPeersMatchFilter = minExpiry === undefined;
+    if (noRateLimitedPeersMatchFilter) {
+      return undefined;
+    }
+
+    return minExpiry! - now;
+  }
+
+  private orderedPeers: Set<string> = new Set();
+
+  private get peers(): Set<string> {
+    const pinnedStr = this.pinnedPeerId?.toString();
+    const currentlyConnected = new Set(
+      this.connectionSampler
+        .getPeerListSortedByConnectionCountAsc()
+        .map(p => p.toString())
+        .filter(p => p !== pinnedStr),
+    );
+
+    // Remove disconnected peers, preserving order of the rest.
+    this.orderedPeers = this.orderedPeers.intersection(currentlyConnected);
+
+    // Append newly connected peers at the end (lowest priority).
+    for (const peer of currentlyConnected) {
+      if (!this.orderedPeers.has(peer)) {
+        this.orderedPeers.add(peer);
+      }
+    }
+    return this.orderedPeers;
+  }
+}

package/src/services/reqresp/batch-tx-requester/tx_validator.ts

@@ -0,0 +1,24 @@
+import type { ClientProtocolCircuitVerifier } from '@aztec/stdlib/interfaces/server';
+import type { TxValidator } from '@aztec/stdlib/tx';
+
+import { createTxValidatorForOnDemandReceivedTxs } from '../../../msg_validators/index.js';
+import type { TxValidationCache } from '../../../msg_validators/tx_validator/tx_validation_cache.js';
+
+export interface BatchRequestTxValidatorConfig {
+  l1ChainId: number;
+  rollupVersion: number;
+  proofVerifier: ClientProtocolCircuitVerifier;
+  txValidationCache?: TxValidationCache;
+}
+
+export function createBatchRequestTxValidator(config: BatchRequestTxValidatorConfig): TxValidator {
+  return createTxValidatorForOnDemandReceivedTxs(
+    config.proofVerifier,
+    {
+      l1ChainId: config.l1ChainId,
+      rollupVersion: config.rollupVersion,
+    },
+    /*bindings=*/ undefined,
+    config.txValidationCache,
+  );
+}

package/src/services/reqresp/config.ts

@@ -1,10 +1,27 @@
-import { type ConfigMapping, booleanConfigHelper, numberConfigHelper } from '@aztec/foundation/config';
+import { type ConfigMappingsType, booleanConfigHelper, numberConfigHelper } from '@aztec/foundation/config';
 
 export const DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS = 10_000;
 export const DEFAULT_OVERALL_REQUEST_TIMEOUT_MS = 10_000; // Not currently used
 export const DEFAULT_REQRESP_DIAL_TIMEOUT_MS = 5_000;
 export const DEFAULT_OPTIMISTIC_NEGOTIATION = false;
 
+/**
+ * Max size of a yamux frame, header included. This is the library default, pinned explicitly when configuring the
+ * muxer in the libp2p service so a dependency upgrade cannot silently change it.
+ */
+export const YAMUX_MAX_MESSAGE_SIZE_BYTES = 64 * 1024;
+
+/** Size of a yamux data frame header. */
+const YAMUX_HEADER_LENGTH_BYTES = 12;
+
+/**
+ * Max size of a reqresp request payload. A request must fit in a single muxer frame: the responder never reassembles
+ * a request from multiple chunks (see `ReqResp.processStream`), and yamux splits writes larger than one frame into
+ * multiple frames, each of which arrives as a separate chunk. mplex only splits at 1 MiB, so yamux is the binding
+ * constraint.
+ */
+export const MAX_REQRESP_REQUEST_SIZE_BYTES = YAMUX_MAX_MESSAGE_SIZE_BYTES - YAMUX_HEADER_LENGTH_BYTES;
+
 // For use in tests.
 export const DEFAULT_P2P_REQRESP_CONFIG: P2PReqRespConfig = {
   overallRequestTimeoutMs: DEFAULT_OVERALL_REQUEST_TIMEOUT_MS,
@@ -27,7 +44,7 @@ export interface P2PReqRespConfig {
   dialTimeoutMs: number;
 }
 
-export const p2pReqRespConfigMappings: Record<keyof P2PReqRespConfig, ConfigMapping> = {
+export const p2pReqRespConfigMappings: ConfigMappingsType<P2PReqRespConfig> = {
   overallRequestTimeoutMs: {
     env: 'P2P_REQRESP_OVERALL_REQUEST_TIMEOUT_MS',
     description: 'The overall timeout for a request response operation.',

package/src/services/reqresp/connection-sampler/connection_sampler.ts

@@ -125,6 +125,22 @@ export class ConnectionSampler {
     return { peer: lastPeer, sampledPeers };
   }
 
+  /*
+   * Returns all peers sorted by connection count ascending,
+   * meaning that the peers with the least number of active connections are earlier in an array
+   *
+   * @param: excluding - peers to exclude
+   * @return: list of peer ids
+   * */
+  public getPeerListSortedByConnectionCountAsc(excluding?: Set<string>): PeerId[] {
+    return this.libp2p
+      .getPeers()
+      .filter(id => !excluding?.has(id.toString()))
+      .map(id => ({ id, count: this.activeConnectionsCount.get(id.toString()) ?? 0 }))
+      .sort((a, b) => a.count - b.count)
+      .map(p => p.id);
+  }
+
   /**
    * Samples a batch of unique peers from the libp2p node, prioritizing peers without active connections
    *