@aztec/p2p 0.0.1-commit.8afd444 → 0.0.1-commit.9117c5f5a

package/src/services/tx_collection/tx_source.ts

@@ -6,9 +6,11 @@ import type { Tx, TxHash } from '@aztec/stdlib/tx';
 import { type ComponentsVersions, getComponentsVersionsFromConfig } from '@aztec/stdlib/versioning';
 import { makeTracedFetch } from '@aztec/telemetry-client';
 
+export type TxSourceCollectionResult = { validTxs: Tx[]; invalidTxHashes: string[] };
+
 export interface TxSource {
   getInfo(): string;
-  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+  getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
 }
 
 export class NodeRpcTxSource implements TxSource {
@@ -26,8 +28,25 @@ export class NodeRpcTxSource implements TxSource {
     return this.info;
   }
 
-  public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
-    return this.client.getTxsByHash(txHashes);
+  public async getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult> {
+    return this.verifyTxs(await this.client.getTxsByHash(txHashes));
+  }
+
+  private async verifyTxs(txs: Tx[]): Promise<TxSourceCollectionResult> {
+    // Validate tx hashes for all collected txs from external sources
+    const validTxs: Tx[] = [];
+    const invalidTxHashes: string[] = [];
+    await Promise.all(
+      txs.map(async tx => {
+        const isValid = await tx.validateTxHash();
+        if (isValid) {
+          validTxs.push(tx);
+        } else {
+          invalidTxHashes.push(tx.getTxHash().toString());
+        }
+      }),
+    );
+    return { validTxs: validTxs, invalidTxHashes: invalidTxHashes };
   }
 }
 

package/src/services/tx_file_store/config.ts

@@ -6,8 +6,6 @@ import { type ConfigMappingsType, booleanConfigHelper, numberConfigHelper } from
 export type TxFileStoreConfig = {
   /** URL for uploading txs to file storage (s3://, gs://, file://) */
   txFileStoreUrl?: string;
-  /** URL for downloading txs from file storage */
-  txFileStoreDownloadUrl?: string;
   /** Max concurrent uploads */
   txFileStoreUploadConcurrency: number;
   /** Max queue size to prevent unbounded memory growth */
@@ -21,10 +19,6 @@ export const txFileStoreConfigMappings: ConfigMappingsType<TxFileStoreConfig> =
     env: 'TX_FILE_STORE_URL',
     description: 'URL for uploading txs to file storage (s3://, gs://, file://)',
   },
-  txFileStoreDownloadUrl: {
-    env: 'TX_FILE_STORE_DOWNLOAD_URL',
-    description: 'URL for downloading txs from file storage',
-  },
   txFileStoreUploadConcurrency: {
     env: 'TX_FILE_STORE_UPLOAD_CONCURRENCY',
     description: 'Maximum number of concurrent tx uploads',

package/src/services/tx_file_store/tx_file_store.ts

@@ -6,7 +6,7 @@ import { type FileStore, createFileStore } from '@aztec/stdlib/file-store';
 import type { Tx } from '@aztec/stdlib/tx';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
-import type { TxPool, TxPoolEvents } from '../../mem_pools/tx_pool/index.js';
+import type { TxPoolV2 } from '../../mem_pools/index.js';
 import type { TxFileStoreConfig } from './config.js';
 import { TxFileStoreInstrumentation } from './instrumentation.js';
 
@@ -18,7 +18,7 @@ export class TxFileStore {
   private uploadQueue: Tx[] = [];
   private activeUploads = 0;
   private readonly queueProcessor: RunningPromise;
-  private readonly handleTxsAdded: TxPoolEvents['txs-added'];
+  private readonly handleTxsAdded: (args: { txs: Tx[]; source?: string }) => void;
 
   /** Recently uploaded tx hashes for deduplication. */
   private recentUploads: Set<string> = new Set();
@@ -27,10 +27,11 @@ export class TxFileStore {
 
   private constructor(
     private readonly fileStore: FileStore,
-    private readonly txPool: TxPool,
+    private readonly txPool: TxPoolV2,
     private readonly config: TxFileStoreConfig,
     private readonly instrumentation: TxFileStoreInstrumentation,
     private readonly log: Logger,
+    private readonly basePath: string,
   ) {
     this.handleTxsAdded = (args: { txs: Tx[]; source?: string }) => {
       this.enqueueTxs(args.txs);
@@ -48,8 +49,9 @@ export class TxFileStore {
    * @returns The file store instance, or undefined if not configured/enabled.
    */
   static async create(
-    txPool: TxPool,
+    txPool: TxPoolV2,
     config: TxFileStoreConfig,
+    basePath: string,
     log: Logger = createLogger('p2p:tx_file_store'),
     telemetry: TelemetryClient = getTelemetryClient(),
     fileStoreOverride?: FileStore,
@@ -71,8 +73,8 @@ export class TxFileStore {
     }
 
     const instrumentation = new TxFileStoreInstrumentation(telemetry, 'TxFileStore');
-    log.info('Created tx file store', { url: config.txFileStoreUrl });
-    return new TxFileStore(fileStore, txPool, config, instrumentation, log);
+    log.info('Created tx file store', { url: config.txFileStoreUrl, basePath });
+    return new TxFileStore(fileStore, txPool, config, instrumentation, log, basePath);
   }
 
   /** Starts listening to TxPool events and uploading txs. */
@@ -122,7 +124,7 @@ export class TxFileStore {
 
   private async uploadTx(tx: Tx): Promise<void> {
     const txHash = tx.getTxHash().toString();
-    const path = `txs/${txHash}.bin`;
+    const path = `${this.basePath}/txs/${txHash}.bin`;
     const timer = new Timer();
 
     if (this.recentUploads.has(txHash)) {
@@ -144,7 +146,7 @@ export class TxFileStore {
       }
 
       await retry(
-        () => this.fileStore.save(path, tx.toBuffer(), { compress: false }),
+        () => this.fileStore.save(path, tx.toBuffer(), { compress: true }),
         `Uploading tx ${txHash}`,
         makeBackoff([0.1, 0.5, 2]),
         this.log,

package/src/services/tx_provider.ts

@@ -5,13 +5,13 @@ import { elapsed } from '@aztec/foundation/timer';
 import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
 import type { ITxProvider } from '@aztec/stdlib/interfaces/server';
 import type { BlockProposal } from '@aztec/stdlib/p2p';
-import { Tx, TxHash } from '@aztec/stdlib/tx';
+import { type BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import type { PeerId } from '@libp2p/interface';
 
 import type { P2PClient } from '../client/p2p_client.js';
-import type { TxPool } from '../mem_pools/index.js';
+import type { TxPoolV2 } from '../mem_pools/tx_pool_v2/interfaces.js';
 import type { FastCollectionRequestInput, TxCollection } from './tx_collection/tx_collection.js';
 import { TxProviderInstrumentation } from './tx_provider_instrumentation.js';
 
@@ -24,8 +24,8 @@ export class TxProvider implements ITxProvider {
 
   constructor(
     private txCollection: TxCollection,
-    private txPool: TxPool,
-    private txValidator: Pick<P2PClient, 'validate'>,
+    private txPool: TxPoolV2,
+    private blockProposalTransactionValidator: Pick<P2PClient, 'validateTxsReceivedInBlockProposal'>,
     private log: Logger = createLogger('p2p:tx-collector'),
     client: TelemetryClient = getTelemetryClient(),
   ) {
@@ -144,6 +144,7 @@ export class TxProvider implements ITxProvider {
     // Take txs from the proposal body if there are any
     // Note that we still have to validate these txs, but we do it in parallel with tx collection
     const proposal = request.type === 'proposal' ? request.blockProposal : undefined;
+    const proposalBlockHeader = proposal?.blockHeader;
     const txsFromProposal = this.extractFromProposal(proposal, [...missingTxHashes]);
     if (txsFromProposal.length > 0) {
       this.instrumentation.incTxsFromProposals(txsFromProposal.length);
@@ -155,7 +156,7 @@ export class TxProvider implements ITxProvider {
     }
 
     if (missingTxHashes.size === 0) {
-      await this.processProposalTxs(txsFromProposal);
+      await this.processProposalTxs(txsFromProposal, proposalBlockHeader!);
       this.instrumentation.incTxsFromP2P(0, txHashes.length);
       return { txsFromMempool, txsFromProposal };
     }
@@ -163,7 +164,7 @@ export class TxProvider implements ITxProvider {
     // Start tx collection from the network if needed, while we validate the txs taken from the proposal in parallel
     const [txsFromNetwork] = await Promise.all([
       this.collectFromP2P(request, [...missingTxHashes], opts),
-      this.processProposalTxs(txsFromProposal),
+      this.processProposalTxs(txsFromProposal, proposalBlockHeader!),
     ] as const);
 
     if (txsFromNetwork.length > 0) {
@@ -222,11 +223,11 @@ export class TxProvider implements ITxProvider {
     return compactArray(proposal.txs ?? []).filter(tx => missingTxHashes.includes(tx.getTxHash().toString()));
   }
 
-  private async processProposalTxs(txs: Tx[]): Promise<void> {
+  private async processProposalTxs(txs: Tx[], blockHeader: BlockHeader): Promise<void> {
     if (txs.length === 0) {
       return;
     }
-    await this.txValidator.validate(txs);
-    await this.txPool.addTxs(txs);
+    await this.blockProposalTransactionValidator.validateTxsReceivedInBlockProposal(txs);
+    await this.txPool.addProtectedTxs(txs, blockHeader);
   }
 }

package/src/test-helpers/make-test-p2p-clients.ts

@@ -7,13 +7,12 @@ import { sleep } from '@aztec/foundation/sleep';
 import type { DataStoreConfig } from '@aztec/kv-store/config';
 import { openTmpStore } from '@aztec/kv-store/lmdb-v2';
 import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
-import { P2PClientType } from '@aztec/stdlib/p2p';
 
 import { createP2PClient } from '../client/index.js';
 import type { P2PClient } from '../client/p2p_client.js';
 import type { P2PConfig } from '../config.js';
 import type { AttestationPool } from '../mem_pools/attestation_pool/attestation_pool.js';
-import type { TxPool } from '../mem_pools/tx_pool/index.js';
+import type { TxPoolV2 } from '../mem_pools/tx_pool_v2/interfaces.js';
 import { generatePeerIdPrivateKeys } from '../test-helpers/generate-peer-id-private-keys.js';
 import { getPorts } from './get-ports.js';
 import { makeEnrs } from './make-enrs.js';
@@ -22,7 +21,7 @@ import { AlwaysFalseCircuitVerifier, AlwaysTrueCircuitVerifier } from './reqresp
 
 export interface MakeTestP2PClientOptions {
   mockAttestationPool: AttestationPool;
-  mockTxPool: TxPool;
+  mockTxPool: TxPoolV2;
   mockEpochCache: EpochCache;
   mockWorldState: WorldStateSynchronizer;
   alwaysTrueVerifier?: boolean;
@@ -98,7 +97,6 @@ export async function makeTestP2PClient(
   const kvStore = await openTmpStore('test');
 
   const client = await createP2PClient(
-    P2PClientType.Full,
     config,
     l2BlockSource,
     proofVerifier,
@@ -108,7 +106,7 @@ export async function makeTestP2PClient(
     undefined,
     undefined,
     {
-      txPool: mockTxPool as unknown as TxPool,
+      txPool: mockTxPool as unknown as TxPoolV2,
       attestationPool: mockAttestationPool as unknown as AttestationPool,
       store: kvStore,
       logger,

package/src/test-helpers/mock-pubsub.ts

@@ -4,7 +4,6 @@ import type { AztecAsyncKVStore } from '@aztec/kv-store';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import type { ContractDataSource } from '@aztec/stdlib/contract';
 import type { ClientProtocolCircuitVerifier, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
-import { P2PClientType } from '@aztec/stdlib/p2p';
 import type { TelemetryClient } from '@aztec/telemetry-client';
 
 import type { GossipsubEvents, GossipsubMessage } from '@chainsafe/libp2p-gossipsub';
@@ -19,8 +18,20 @@ import {
 
 import type { P2PConfig } from '../config.js';
 import type { MemPools } from '../mem_pools/interface.js';
-import { DummyPeerDiscoveryService, DummyPeerManager, DummyReqResp, LibP2PService } from '../services/index.js';
-import type { ReqRespInterface } from '../services/reqresp/interface.js';
+import { DummyPeerDiscoveryService, DummyPeerManager, LibP2PService } from '../services/index.js';
+import type { P2PReqRespConfig } from '../services/reqresp/config.js';
+import type { ConnectionSampler } from '../services/reqresp/connection-sampler/connection_sampler.js';
+import {
+  type ReqRespInterface,
+  type ReqRespResponse,
+  type ReqRespSubProtocol,
+  type ReqRespSubProtocolHandler,
+  type ReqRespSubProtocolHandlers,
+  type ReqRespSubProtocolValidators,
+  type SubProtocolMap,
+  responseFromBuffer,
+} from '../services/reqresp/interface.js';
+import { ReqRespStatus } from '../services/reqresp/status.js';
 import { GossipSubEvent } from '../types/index.js';
 import type { PubSubLibp2p } from '../util.js';
 
@@ -30,11 +41,10 @@ type GossipSubService = PubSubLibp2p['services']['pubsub'];
  * Given a mock gossip sub network, returns a factory function that creates an instance LibP2PService connected to it.
  * Designed to be used in tests in P2PClientDeps.p2pServiceFactory.
  */
-export function getMockPubSubP2PServiceFactory<T extends P2PClientType>(
+export function getMockPubSubP2PServiceFactory(
   network: MockGossipSubNetwork,
-): (...args: Parameters<(typeof LibP2PService<T>)['new']>) => Promise<LibP2PService<T>> {
+): (...args: Parameters<(typeof LibP2PService)['new']>) => Promise<LibP2PService> {
   return (
-    clientType: P2PClientType,
     config: P2PConfig,
     peerId: PeerId,
     deps: {
@@ -52,10 +62,9 @@ export function getMockPubSubP2PServiceFactory<T extends P2PClientType>(
     deps.logger.verbose('Creating mock PubSub service');
     const libp2p = new MockPubSub(peerId, network);
     const peerManager = new DummyPeerManager(peerId, network);
-    const reqresp: ReqRespInterface = new DummyReqResp();
+    const reqresp: ReqRespInterface = new MockReqResp(peerId, network);
     const peerDiscoveryService = new DummyPeerDiscoveryService();
-    const service = new LibP2PService<T>(
-      clientType as T,
+    const service = new LibP2PService(
       config,
       libp2p,
       peerDiscoveryService,
@@ -74,6 +83,115 @@ export function getMockPubSubP2PServiceFactory<T extends P2PClientType>(
   };
 }
 
+/**
+ * Mock implementation of ReqRespInterface that routes requests to other peers' handlers through the mock network.
+ * When a peer calls sendBatchRequest, the mock iterates over network peers and invokes their registered handler
+ * for the sub-protocol, simulating the request-response protocol without actual libp2p streams.
+ */
+class MockReqResp implements ReqRespInterface {
+  private handlers: Partial<ReqRespSubProtocolHandlers> = {};
+  private logger = createLogger('p2p:test:mock-reqresp');
+
+  constructor(
+    private peerId: PeerId,
+    private network: MockGossipSubNetwork,
+  ) {
+    network.registerReqRespPeer(this);
+  }
+
+  updateConfig(_config: Partial<P2PReqRespConfig>): void {}
+
+  start(
+    subProtocolHandlers: Partial<ReqRespSubProtocolHandlers>,
+    _subProtocolValidators: ReqRespSubProtocolValidators,
+  ): Promise<void> {
+    Object.assign(this.handlers, subProtocolHandlers);
+    return Promise.resolve();
+  }
+
+  addSubProtocol(
+    subProtocol: ReqRespSubProtocol,
+    handler: ReqRespSubProtocolHandler,
+    _validator?: ReqRespSubProtocolValidators[ReqRespSubProtocol],
+  ): Promise<void> {
+    this.handlers[subProtocol] = handler;
+    return Promise.resolve();
+  }
+
+  stop(): Promise<void> {
+    this.handlers = {};
+    return Promise.resolve();
+  }
+
+  getHandler(subProtocol: ReqRespSubProtocol): ReqRespSubProtocolHandler | undefined {
+    return this.handlers[subProtocol];
+  }
+
+  async sendBatchRequest<SubProtocol extends ReqRespSubProtocol>(
+    subProtocol: SubProtocol,
+    requests: InstanceType<SubProtocolMap[SubProtocol]['request']>[],
+    pinnedPeer: PeerId | undefined,
+    _timeoutMs?: number,
+    _maxPeers?: number,
+    _maxRetryAttempts?: number,
+  ): Promise<InstanceType<SubProtocolMap[SubProtocol]['response']>[]> {
+    const responses: InstanceType<SubProtocolMap[SubProtocol]['response']>[] = [];
+    const peers = this.network.getReqRespPeers().filter(p => !p.peerId.equals(this.peerId));
+    const targetPeers = pinnedPeer ? peers.filter(p => p.peerId.equals(pinnedPeer)) : peers;
+
+    for (const request of requests) {
+      const requestBuffer = request.toBuffer();
+      for (const peer of targetPeers) {
+        const handler = peer.getHandler(subProtocol);
+        if (!handler) {
+          continue;
+        }
+        try {
+          const responseBuffer = await handler(this.peerId, requestBuffer);
+          if (responseBuffer.length > 0) {
+            const response = responseFromBuffer(subProtocol, responseBuffer);
+            responses.push(response as InstanceType<SubProtocolMap[SubProtocol]['response']>);
+            break;
+          }
+        } catch (err) {
+          this.logger.debug(`Mock reqresp handler error from peer ${peer.peerId}`, { err });
+        }
+      }
+    }
+
+    return responses;
+  }
+
+  async sendRequestToPeer(
+    peerId: PeerId,
+    subProtocol: ReqRespSubProtocol,
+    payload: Buffer,
+    _dialTimeout?: number,
+  ): Promise<ReqRespResponse> {
+    const peer = this.network.getReqRespPeers().find(p => p.peerId.equals(peerId));
+    const handler = peer?.getHandler(subProtocol);
+    if (!handler) {
+      return { status: ReqRespStatus.SUCCESS, data: Buffer.from([]) };
+    }
+    try {
+      const data = await handler(this.peerId, payload);
+      return { status: ReqRespStatus.SUCCESS, data };
+    } catch {
+      return { status: ReqRespStatus.FAILURE };
+    }
+  }
+
+  getConnectionSampler(): Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'> {
+    return {
+      getPeerListSortedByConnectionCountAsc: () =>
+        this.network
+          .getReqRespPeers()
+          .filter(p => !p.peerId.equals(this.peerId))
+          .map(p => p.peerId),
+    };
+  }
+}
+
 /**
  * Implementation of PubSub services that relies on a mock gossip sub network.
  * This is used in tests to simulate a gossip sub network without needing a real P2P network.
@@ -149,6 +267,16 @@ class MockGossipSubService extends TypedEventEmitter<GossipsubEvents> implements
       { msgId, propagationSource, acceptance },
     );
   }
+
+  getMeshPeers(topic?: TopicStr): PeerIdStr[] {
+    if (topic && !this.subscribedTopics.has(topic)) {
+      return [];
+    }
+    return this.network
+      .getPeers()
+      .filter(peer => !this.peerId.equals(peer))
+      .map(peer => peer.toString());
+  }
 }
 
 /**
@@ -157,6 +285,7 @@ class MockGossipSubService extends TypedEventEmitter<GossipsubEvents> implements
  */
 export class MockGossipSubNetwork {
   private peers: MockGossipSubService[] = [];
+  private reqRespPeers: MockReqResp[] = [];
   private nextMsgId = 0;
 
   private logger = createLogger('p2p:test:mock-gossipsub-network');
@@ -169,6 +298,14 @@ export class MockGossipSubNetwork {
     this.peers.push(peer);
   }
 
+  public registerReqRespPeer(peer: MockReqResp): void {
+    this.reqRespPeers.push(peer);
+  }
+
+  public getReqRespPeers(): MockReqResp[] {
+    return this.reqRespPeers;
+  }
+
   public publishToPeers(topic: TopicStr, data: Uint8Array, sender: PeerId): void {
     const msgId = (this.nextMsgId++).toString();
     this.logger.debug(`Network is distributing message on topic ${topic}`, {

package/src/test-helpers/reqresp-nodes.ts

@@ -12,7 +12,6 @@ import type {
   IVCProofVerificationResult,
   WorldStateSynchronizer,
 } from '@aztec/stdlib/interfaces/server';
-import type { P2PClientType } from '@aztec/stdlib/p2p';
 import type { Tx } from '@aztec/stdlib/tx';
 import { compressComponentVersions } from '@aztec/stdlib/versioning';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
@@ -34,6 +33,7 @@ import { BootstrapNode } from '../bootstrap/bootstrap.js';
 import type { BootnodeConfig, P2PConfig } from '../config.js';
 import type { MemPools } from '../mem_pools/interface.js';
 import { DiscV5Service } from '../services/discv5/discV5_service.js';
+import { APP_SPECIFIC_WEIGHT } from '../services/gossipsub/scoring.js';
 import { LibP2PService } from '../services/libp2p/libp2p_service.js';
 import { PeerManager } from '../services/peer-manager/peer_manager.js';
 import { PeerScoring } from '../services/peer-manager/peer_scoring.js';
@@ -106,8 +106,7 @@ export async function createLibp2pNode(
  *
  *
  */
-export async function createTestLibP2PService<T extends P2PClientType>(
-  clientType: T,
+export async function createTestLibP2PService(
   boostrapAddrs: string[] = [],
   archiver: L2BlockSource & ContractDataSource,
   worldStateSynchronizer: WorldStateSynchronizer,
@@ -154,12 +153,11 @@ export async function createTestLibP2PService<T extends P2PClientType>(
     epochCache,
   );
 
-  p2pNode.services.pubsub.score.params.appSpecificWeight = 10;
+  p2pNode.services.pubsub.score.params.appSpecificWeight = APP_SPECIFIC_WEIGHT;
   p2pNode.services.pubsub.score.params.appSpecificScore = (peerId: string) =>
     peerManager.shouldDisableP2PGossip(peerId) ? -Infinity : peerManager.getPeerScore(peerId);
 
-  return new LibP2PService<T>(
-    clientType,
+  return new LibP2PService(
     config,
     p2pNode as PubSubLibp2p,
     discoveryService,