@aztec/p2p 0.0.1-commit.96dac018d → 0.0.1-commit.993d52e

package/src/services/libp2p/libp2p_service.ts

@@ -16,13 +16,12 @@ import {
   CheckpointProposal,
   type CheckpointProposalCore,
   type Gossipable,
-  P2PClientType,
   P2PMessage,
   type ValidationResult as P2PValidationResult,
   PeerErrorSeverity,
   TopicType,
   createTopicString,
-  getTopicsForClientAndConfig,
+  getTopicsForConfig,
   metricsTopicStrToLabels,
 } from '@aztec/stdlib/p2p';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
@@ -135,7 +134,7 @@ type ReceivedMessageValidationResult<T, M = undefined> =
 /**
  * Lib P2P implementation of the P2PService interface.
  */
-export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends WithTracer implements P2PService {
+export class LibP2PService extends WithTracer implements P2PService {
   private discoveryRunningPromise?: RunningPromise;
   private msgIdSeenValidators: Record<TopicType, MessageSeenValidator> = {} as Record<TopicType, MessageSeenValidator>;
 
@@ -182,7 +181,6 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
   protected logger: Logger;
 
   constructor(
-    private clientType: T,
     private config: P2PConfig,
     protected node: PubSubLibp2p,
     private peerDiscoveryService: PeerDiscoveryService,
@@ -224,9 +222,13 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
       this.protocolVersion,
     );
 
-    this.blockProposalValidator = new BlockProposalValidator(epochCache, { txsPermitted: !config.disableTransactions });
+    this.blockProposalValidator = new BlockProposalValidator(epochCache, {
+      txsPermitted: !config.disableTransactions,
+      maxTxsPerBlock: config.maxTxsPerBlock,
+    });
     this.checkpointProposalValidator = new CheckpointProposalValidator(epochCache, {
       txsPermitted: !config.disableTransactions,
+      maxTxsPerBlock: config.maxTxsPerBlock,
     });
     this.checkpointAttestationValidator = config.fishermanMode
       ? new FishermanAttestationValidator(epochCache, mempools.attestationPool, telemetry)
@@ -262,8 +264,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
    * @param txPool - The transaction pool to be accessed by the service.
    * @returns The new service.
    */
-  public static async new<T extends P2PClientType>(
-    clientType: T,
+  public static async new(
     config: P2PConfig,
     peerId: PeerId,
     deps: {
@@ -475,7 +476,6 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
       peerManager.shouldDisableP2PGossip(peerId) ? -Infinity : peerManager.getPeerScore(peerId);
 
     return new LibP2PService(
-      clientType,
       config,
       node,
       peerDiscoveryService,
@@ -549,7 +549,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
     await this.node.start();
 
     // Subscribe to standard GossipSub topics by default
-    for (const topic of getTopicsForClientAndConfig(this.clientType, this.config.disableTransactions)) {
+    for (const topic of getTopicsForConfig(this.config.disableTransactions)) {
       this.subscribeToTopic(this.topicStrings[topic]);
     }
 
@@ -818,9 +818,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
       if (msg.topic === this.topicStrings[TopicType.tx]) {
         await this.handleGossipedTx(p2pMessage.payload, msgId, source);
       } else if (msg.topic === this.topicStrings[TopicType.checkpoint_attestation]) {
-        if (this.clientType === P2PClientType.Full) {
-          await this.processCheckpointAttestationFromPeer(p2pMessage.payload, msgId, source);
-        }
+        await this.processCheckpointAttestationFromPeer(p2pMessage.payload, msgId, source);
       } else if (msg.topic === this.topicStrings[TopicType.block_proposal]) {
         await this.processBlockFromPeer(p2pMessage.payload, msgId, source);
       } else if (msg.topic === this.topicStrings[TopicType.checkpoint_proposal]) {

package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts

@@ -8,6 +8,7 @@ import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
 import { Tx, TxArray, TxHash } from '@aztec/stdlib/tx';
 
 import type { PeerId } from '@libp2p/interface';
+import { peerIdFromString } from '@libp2p/peer-id';
 
 import type { IMissingTxsTracker } from '../../tx_collection/missing_txs_tracker.js';
 import { ReqRespSubProtocol } from '.././interface.js';
@@ -89,9 +90,10 @@ export class BatchTxRequester {
     if (this.opts.peerCollection) {
       this.peers = this.opts.peerCollection;
     } else {
+      const initialPeers = this.p2pService.connectionSampler.getPeerListSortedByConnectionCountAsc();
       const badPeerThreshold = this.opts.badPeerThreshold ?? DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD;
       this.peers = new PeerCollection(
-        this.p2pService.connectionSampler,
+        initialPeers,
         this.pinnedPeer,
         this.dateProvider,
         badPeerThreshold,
@@ -225,6 +227,7 @@ export class BatchTxRequester {
    * Starts dumb worker loops
    * */
   private async dumbRequester() {
+    const nextPeerIndex = this.makeRoundRobinIndexer();
     const nextBatchIndex = this.makeRoundRobinIndexer();
 
     // Chunk missing tx hashes into batches of txBatchSize, wrapping around to ensure no peer gets less than txBatchSize
@@ -260,9 +263,15 @@ export class BatchTxRequester {
       return { blockRequest, txs };
     };
 
-    const workerCount = this.dumbParallelWorkerCount;
+    const nextPeer = () => {
+      const peers = this.peers.getDumbPeersToQuery();
+      const idx = nextPeerIndex(() => peers.length);
+      return idx === undefined ? undefined : peerIdFromString(peers[idx]);
+    };
+
+    const workerCount = Math.min(this.dumbParallelWorkerCount, this.peers.getAllPeers().size);
     const workers = Array.from({ length: workerCount }, (_, index) =>
-      this.dumbWorkerLoop(this.peers.nextDumbPeerToQuery.bind(this.peers), makeRequest, index + 1),
+      this.dumbWorkerLoop(nextPeer, makeRequest, index + 1),
     );
 
     await Promise.allSettled(workers);
@@ -323,6 +332,14 @@ export class BatchTxRequester {
    * Starts smart worker loops
    * */
   private async smartRequester() {
+    const nextPeerIndex = this.makeRoundRobinIndexer();
+
+    const nextPeer = () => {
+      const peers = this.peers.getSmartPeersToQuery();
+      const idx = nextPeerIndex(() => peers.length);
+      return idx === undefined ? undefined : peerIdFromString(peers[idx]);
+    };
+
     const makeRequest = (pid: PeerId) => {
       const txs = this.txsMetadata.getTxsToRequestFromThePeer(pid);
       const blockRequest = BlockTxsRequest.fromTxsSourceAndMissingTxs(this.blockTxsSource, txs);
@@ -333,8 +350,9 @@ export class BatchTxRequester {
       return { blockRequest, txs };
     };
 
-    const workers = Array.from({ length: this.smartParallelWorkerCount }, (_, index) =>
-      this.smartWorkerLoop(this.peers.nextSmartPeerToQuery.bind(this.peers), makeRequest, index + 1),
+    const workers = Array.from(
+      { length: Math.min(this.smartParallelWorkerCount, this.peers.getAllPeers().size) },
+      (_, index) => this.smartWorkerLoop(nextPeer, makeRequest, index + 1),
     );
 
     await Promise.allSettled(workers);
@@ -369,18 +387,26 @@ export class BatchTxRequester {
         if (weRanOutOfPeersToQuery) {
           this.logger.debug(`Worker loop smart: No more peers to query`);
 
-          // If we have rate limited peers wait for them.
-          const nextSmartPeerDelay = this.peers.getNextSmartPeerAvailabilityDelayMs();
-          const thereAreSomeRateLimitedSmartPeers = nextSmartPeerDelay !== undefined;
-          if (thereAreSomeRateLimitedSmartPeers) {
-            await this.sleepClampedToDeadline(nextSmartPeerDelay);
-            continue;
+          // If there are no more dumb peers to query then none of our peers can become smart,
+          // thus we can simply exit this worker
+          const noMoreDumbPeersToQuery = this.peers.getDumbPeersToQuery().length === 0;
+          if (noMoreDumbPeersToQuery) {
+            // These might be either smart peers that will get unblocked after _some time_
+            const nextSmartPeerDelay = this.peers.getNextSmartPeerAvailabilityDelayMs();
+            const thereAreSomeRateLimitedSmartPeers = nextSmartPeerDelay !== undefined;
+            if (thereAreSomeRateLimitedSmartPeers) {
+              await this.sleepClampedToDeadline(nextSmartPeerDelay);
+              continue;
+            }
+
+            this.logger.debug(`Worker loop smart: No more smart peers to query killing ${workerIndex}`);
+            break;
           }
 
+          // Otherwise there are still some dumb peers that could become smart.
           // We end up here when all known smart peers became temporarily unavailable via combination of
           // (bad, in-flight, or rate-limited) or in some weird scenario all current smart peers turn bad which is permanent
-          // but there are dumb peers that could be promoted
-          // or new peer can join as dumb and be promoted later
+          // but dumb peers still exist that could become smart.
           //
           // When a dumb peer responds with valid txIndices, it gets
           // promoted to smart and releases the semaphore, waking this worker.
@@ -573,7 +599,9 @@ export class BatchTxRequester {
     this.markTxsPeerHas(peerId, response);
 
     // Unblock smart workers
-    this.smartRequesterSemaphore.release();
+    if (this.peers.getSmartPeersToQuery().length <= this.smartParallelWorkerCount) {
+      this.smartRequesterSemaphore.release();
+    }
   }
 
   private isBlockResponseValid(response: BlockTxsResponse): boolean {

package/src/services/reqresp/batch-tx-requester/peer_collection.ts

@@ -2,22 +2,18 @@ import type { DateProvider } from '@aztec/foundation/timer';
 import type { PeerErrorSeverity } from '@aztec/stdlib/p2p';
 
 import type { PeerId } from '@libp2p/interface';
-import { peerIdFromString } from '@libp2p/peer-id';
 
-import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
 import { DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD } from './config.js';
 import type { IPeerPenalizer } from './interface.js';
 
 export const RATE_LIMIT_EXCEEDED_PEER_CACHE_TTL = 1000; // 1s
 
 export interface IPeerCollection {
+  getAllPeers(): Set<string>;
+  getSmartPeers(): Set<string>;
   markPeerSmart(peerId: PeerId): void;
-
-  /** Sample next peer in round-robin fashion. No smart peers if returns undefined */
-  nextSmartPeerToQuery(): PeerId | undefined;
-  /** Sample next peer in round-robin fashion. No dumb peers if returns undefined */
-  nextDumbPeerToQuery(): PeerId | undefined;
-
+  getSmartPeersToQuery(): Array<string>;
+  getDumbPeersToQuery(): Array<string>;
   thereAreSomeDumbRatelimitExceededPeers(): boolean;
   penalisePeer(peerId: PeerId, severity: PeerErrorSeverity): void;
   unMarkPeerAsBad(peerId: PeerId): void;
@@ -32,6 +28,8 @@ export interface IPeerCollection {
 }
 
 export class PeerCollection implements IPeerCollection {
+  private readonly peers;
+
   private readonly smartPeers = new Set<string>();
   private readonly inFlightPeers = new Set<string>();
   private readonly rateLimitExceededPeers = new Map<string, number>();
@@ -39,60 +37,46 @@ export class PeerCollection implements IPeerCollection {
   private readonly badPeers = new Set<string>();
 
   constructor(
-    private readonly connectionSampler: Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'>,
+    initialPeers: PeerId[],
     private readonly pinnedPeerId: PeerId | undefined,
     private readonly dateProvider: DateProvider,
     private readonly badPeerThreshold: number = DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD,
     private readonly peerPenalizer?: IPeerPenalizer,
   ) {
-    // Pinned peer is treated specially, always mark it as in-flight
+    this.peers = new Set(initialPeers.map(peer => peer.toString()));
+
+    // Pinned peer is treaded specially, always mark it as in-flight
     // and never return it as part of smart/dumb peers
     if (this.pinnedPeerId) {
       const peerIdStr = this.pinnedPeerId.toString();
       this.inFlightPeers.add(peerIdStr);
+      this.peers.delete(peerIdStr);
     }
   }
 
-  public markPeerSmart(peerId: PeerId): void {
-    this.smartPeers.add(peerId.toString());
-  }
-
-  // We keep track of all peers that are queried for peer sampling algorithm
-  private queriedSmartPeers: Set<string> = new Set<string>();
-  private queriedDumbPeers: Set<string> = new Set<string>();
-
-  private static nextPeer(allPeers: Set<string>, queried: Set<string>): PeerId | undefined {
-    if (allPeers.size === 0) {
-      return undefined;
-    }
-    const availablePeers = allPeers.difference(queried);
-    let [first] = availablePeers;
-    if (first === undefined) {
-      // We queried all peers. Start over
-      [first] = allPeers;
-      queried.clear();
-    }
-    queried.add(first);
-    return peerIdFromString(first);
+  public getAllPeers(): Set<string> {
+    return this.peers;
   }
 
-  public nextSmartPeerToQuery(): PeerId | undefined {
-    return PeerCollection.nextPeer(this.availableSmartPeers, this.queriedSmartPeers);
+  public getSmartPeers(): Set<string> {
+    return this.smartPeers;
   }
 
-  public nextDumbPeerToQuery(): PeerId | undefined {
-    return PeerCollection.nextPeer(this.availableDumbPeers, this.queriedDumbPeers);
+  public markPeerSmart(peerId: PeerId): void {
+    this.smartPeers.add(peerId.toString());
   }
 
-  private get availableSmartPeers(): Set<string> {
-    return this.peers.intersection(
+  public getSmartPeersToQuery(): Array<string> {
+    return Array.from(
       this.smartPeers.difference(this.getBadPeers().union(this.inFlightPeers).union(this.getRateLimitExceededPeers())),
     );
   }
 
-  private get availableDumbPeers(): Set<string> {
-    return this.peers.difference(
-      this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers()),
+  public getDumbPeersToQuery(): Array<string> {
+    return Array.from(
+      this.peers.difference(
+        this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers()),
+      ),
     );
   }
 
@@ -218,27 +202,4 @@ export class PeerCollection implements IPeerCollection {
 
     return minExpiry! - now;
   }
-
-  private orderedPeers: Set<string> = new Set();
-
-  private get peers(): Set<string> {
-    const pinnedStr = this.pinnedPeerId?.toString();
-    const currentlyConnected = new Set(
-      this.connectionSampler
-        .getPeerListSortedByConnectionCountAsc()
-        .map(p => p.toString())
-        .filter(p => p !== pinnedStr),
-    );
-
-    // Remove disconnected peers, preserving order of the rest.
-    this.orderedPeers = this.orderedPeers.intersection(currentlyConnected);
-
-    // Append newly connected peers at the end (lowest priority).
-    for (const peer of currentlyConnected) {
-      if (!this.orderedPeers.has(peer)) {
-        this.orderedPeers.add(peer);
-      }
-    }
-    return this.orderedPeers;
-  }
 }

package/src/test-helpers/make-test-p2p-clients.ts

@@ -7,7 +7,6 @@ import { sleep } from '@aztec/foundation/sleep';
 import type { DataStoreConfig } from '@aztec/kv-store/config';
 import { openTmpStore } from '@aztec/kv-store/lmdb-v2';
 import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
-import { P2PClientType } from '@aztec/stdlib/p2p';
 
 import { createP2PClient } from '../client/index.js';
 import type { P2PClient } from '../client/p2p_client.js';
@@ -98,7 +97,6 @@ export async function makeTestP2PClient(
   const kvStore = await openTmpStore('test');
 
   const client = await createP2PClient(
-    P2PClientType.Full,
     config,
     l2BlockSource,
     proofVerifier,

package/src/test-helpers/mock-pubsub.ts

@@ -4,7 +4,6 @@ import type { AztecAsyncKVStore } from '@aztec/kv-store';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import type { ContractDataSource } from '@aztec/stdlib/contract';
 import type { ClientProtocolCircuitVerifier, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
-import { P2PClientType } from '@aztec/stdlib/p2p';
 import type { TelemetryClient } from '@aztec/telemetry-client';
 
 import type { GossipsubEvents, GossipsubMessage } from '@chainsafe/libp2p-gossipsub';
@@ -42,11 +41,10 @@ type GossipSubService = PubSubLibp2p['services']['pubsub'];
  * Given a mock gossip sub network, returns a factory function that creates an instance LibP2PService connected to it.
  * Designed to be used in tests in P2PClientDeps.p2pServiceFactory.
  */
-export function getMockPubSubP2PServiceFactory<T extends P2PClientType>(
+export function getMockPubSubP2PServiceFactory(
   network: MockGossipSubNetwork,
-): (...args: Parameters<(typeof LibP2PService<T>)['new']>) => Promise<LibP2PService<T>> {
+): (...args: Parameters<(typeof LibP2PService)['new']>) => Promise<LibP2PService> {
   return (
-    clientType: P2PClientType,
     config: P2PConfig,
     peerId: PeerId,
     deps: {
@@ -66,8 +64,7 @@ export function getMockPubSubP2PServiceFactory<T extends P2PClientType>(
     const peerManager = new DummyPeerManager(peerId, network);
     const reqresp: ReqRespInterface = new MockReqResp(peerId, network);
     const peerDiscoveryService = new DummyPeerDiscoveryService();
-    const service = new LibP2PService<T>(
-      clientType as T,
+    const service = new LibP2PService(
       config,
       libp2p,
       peerDiscoveryService,

package/src/test-helpers/reqresp-nodes.ts

@@ -12,7 +12,6 @@ import type {
   IVCProofVerificationResult,
   WorldStateSynchronizer,
 } from '@aztec/stdlib/interfaces/server';
-import type { P2PClientType } from '@aztec/stdlib/p2p';
 import type { Tx } from '@aztec/stdlib/tx';
 import { compressComponentVersions } from '@aztec/stdlib/versioning';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
@@ -107,8 +106,7 @@ export async function createLibp2pNode(
  *
  *
  */
-export async function createTestLibP2PService<T extends P2PClientType>(
-  clientType: T,
+export async function createTestLibP2PService(
   boostrapAddrs: string[] = [],
   archiver: L2BlockSource & ContractDataSource,
   worldStateSynchronizer: WorldStateSynchronizer,
@@ -159,8 +157,7 @@ export async function createTestLibP2PService<T extends P2PClientType>(
   p2pNode.services.pubsub.score.params.appSpecificScore = (peerId: string) =>
     peerManager.shouldDisableP2PGossip(peerId) ? -Infinity : peerManager.getPeerScore(peerId);
 
-  return new LibP2PService<T>(
-    clientType,
+  return new LibP2PService(
     config,
     p2pNode as PubSubLibp2p,
     discoveryService,

package/src/testbench/p2p_client_testbench_worker.ts

@@ -19,7 +19,7 @@ import { protocolContractsHash } from '@aztec/protocol-contracts';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import type { ContractDataSource } from '@aztec/stdlib/contract';
 import type { ClientProtocolCircuitVerifier, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
-import { type BlockProposal, P2PClientType, P2PMessage } from '@aztec/stdlib/p2p';
+import { type BlockProposal, P2PMessage } from '@aztec/stdlib/p2p';
 import { ChonkProof } from '@aztec/stdlib/proofs';
 import { makeAztecAddress, makeBlockHeader, makeBlockProposal, mockTx } from '@aztec/stdlib/testing';
 import { Tx, TxHash, type TxValidationResult } from '@aztec/stdlib/tx';
@@ -86,12 +86,11 @@ export interface BenchReadyMessage {
 }
 const txCache = new Map<number, Tx[]>();
 
-class TestLibP2PService<T extends P2PClientType = P2PClientType.Full> extends LibP2PService<T> {
+class TestLibP2PService extends LibP2PService {
   private disableTxValidation: boolean;
   private gossipMessageCount = 0;
 
   constructor(
-    clientType: T,
     config: P2PConfig,
     node: PubSubLibp2p,
     peerDiscoveryService: PeerDiscoveryService,
@@ -107,7 +106,6 @@ class TestLibP2PService<T extends P2PClientType = P2PClientType.Full> extends Li
     disableTxValidation = true,
   ) {
     super(
-      clientType,
       config,
       node,
       peerDiscoveryService,
@@ -365,7 +363,6 @@ process.on('message', async msg => {
       };
 
       const client = await createP2PClient(
-        P2PClientType.Full,
         config as P2PConfig & DataStoreConfig,
         l2BlockSource,
         proofVerifier as ClientProtocolCircuitVerifier,
@@ -378,7 +375,6 @@ process.on('message', async msg => {
       );
 
       const testService = new TestLibP2PService(
-        P2PClientType.Full,
         config,
         (client as any).p2pService.node,
         (client as any).p2pService.peerDiscoveryService,