@aztec/p2p 0.0.1-commit.8f9871590 → 0.0.1-commit.934299a21

package/src/services/tx_collection/slow_tx_collection.ts

@@ -8,8 +8,7 @@ import type { L2Block } from '@aztec/stdlib/block';
 import { type L1RollupConstants, getEpochAtSlot, getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
 import { type Tx, TxHash } from '@aztec/stdlib/tx';
 
-import { type ReqRespInterface, ReqRespSubProtocol } from '../reqresp/interface.js';
-import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+import { type ReqRespInterface, ReqRespSubProtocol, chunkTxHashesRequest } from '../reqresp/index.js';
 import type { TxCollectionConfig } from './config.js';
 import type { FastTxCollection } from './fast_tx_collection.js';
 import type { MissingTxInfo } from './tx_collection.js';
@@ -120,8 +119,8 @@ export class SlowTxCollection {
       const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
       for (const batch of chunk(txHashes, this.config.txCollectionNodeRpcMaxBatchSize)) {
         await this.txCollectionSink.collect(
-          hashes => node.getTxsByHash(hashes),
-          batch,
+          () => node.getTxsByHash(batch),
+          batch.map(h => h.toString()),
           {
             description: `node ${node.getInfo()}`,
             node: node.getInfo(),
@@ -166,18 +165,18 @@ export class SlowTxCollection {
       const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
       const maxPeers = boundInclusive(Math.ceil(txHashes.length / 3), 4, 16);
       await this.txCollectionSink.collect(
-        async hashes => {
+        async () => {
           const txs = await this.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
             ReqRespSubProtocol.TX,
-            chunkTxHashesRequest(hashes),
+            chunkTxHashesRequest(txHashes),
             pinnedPeer,
             timeoutMs,
             maxPeers,
             maxRetryAttempts,
           );
-          return txs.flat();
+          return { validTxs: txs.flat(), invalidTxHashes: [] };
         },
-        txHashes,
+        txHashes.map(h => h.toString()),
         { description: 'slow reqresp', timeoutMs, method: 'slow-req-resp' },
         { type: 'mined', block },
       );
@@ -197,7 +196,7 @@ export class SlowTxCollection {
     // from mined unproven blocks it has seen in the past.
     const fastRequests = this.fastCollection.getFastCollectionRequests();
     const fastCollectionTxs: Set<string> = new Set(
-      ...Array.from(fastRequests.values()).flatMap(r => r.missingTxHashes),
+      fastRequests.values().flatMap(r => Array.from(r.missingTxTracker.missingTxHashes)),
     );
 
     // Return all missing txs that are not in fastCollectionTxs and are ready for reqresp if requested

package/src/services/tx_collection/tx_collection.ts

@@ -7,7 +7,8 @@ import { DateProvider } from '@aztec/foundation/timer';
 import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
 import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
 import type { BlockProposal } from '@aztec/stdlib/p2p';
-import { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { Tx } from '@aztec/stdlib/tx';
+import { TxHash } from '@aztec/stdlib/tx';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import type { PeerId } from '@libp2p/interface';
@@ -18,6 +19,7 @@ import type { TxCollectionConfig } from './config.js';
 import { FastTxCollection } from './fast_tx_collection.js';
 import { FileStoreTxCollection } from './file_store_tx_collection.js';
 import type { FileStoreTxSource } from './file_store_tx_source.js';
+import type { IMissingTxsTracker } from './missing_txs_tracker.js';
 import { SlowTxCollection, getProofDeadlineForSlot } from './slow_tx_collection.js';
 import { type TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
 import type { TxSource } from './tx_source.js';
@@ -31,11 +33,10 @@ export type FastCollectionRequestInput =
   | { type: 'proposal'; blockProposal: BlockProposal; blockNumber: BlockNumber };
 
 export type FastCollectionRequest = FastCollectionRequestInput & {
-  missingTxHashes: Set<string>;
+  missingTxTracker: IMissingTxsTracker;
   deadline: Date;
   blockInfo: L2BlockInfo;
   promise: PromiseWithResolvers<void>;
-  foundTxs: Map<string, Tx>;
 };
 
 /**

package/src/services/tx_collection/tx_collection_sink.ts

@@ -2,14 +2,15 @@ import type { Logger } from '@aztec/foundation/log';
 import { elapsed } from '@aztec/foundation/timer';
 import type { TypedEventEmitter } from '@aztec/foundation/types';
 import type { L2Block } from '@aztec/stdlib/block';
-import type { BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx';
+import type { BlockHeader, Tx } from '@aztec/stdlib/tx';
 import type { TelemetryClient } from '@aztec/telemetry-client';
 
 import EventEmitter from 'node:events';
 
-import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interfaces.js';
+import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/index.js';
 import { TxCollectionInstrumentation } from './instrumentation.js';
 import type { CollectionMethod } from './tx_collection.js';
+import type { TxSourceCollectionResult } from './tx_source.js';
 
 /** Context determining how collected txs should be added to the pool. */
 export type TxAddContext = { type: 'proposal'; blockHeader: BlockHeader } | { type: 'mined'; block: L2Block };
@@ -31,52 +32,37 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
   }
 
   public async collect(
-    collectValidTxsFn: (txHashes: TxHash[]) => Promise<(Tx | undefined)[]>,
-    requested: TxHash[],
+    collectValidTxsFn: () => Promise<TxSourceCollectionResult>,
+    requested: string[],
     info: Record<string, any> & { description: string; method: CollectionMethod },
     context: TxAddContext,
   ) {
     this.log.trace(`Requesting ${requested.length} txs via ${info.description}`, {
       ...info,
-      requestedTxs: requested.map(t => t.toString()),
+      requestedTxs: requested,
     });
 
     // Execute collection function and measure the time taken, catching any errors.
-    const [duration, txs] = await elapsed(async () => {
+    const [duration, { validTxs, invalidTxHashes }] = await elapsed(async () => {
       try {
-        const response = await collectValidTxsFn(requested);
-        return response.filter(tx => tx !== undefined);
+        return await collectValidTxsFn();
       } catch (err) {
         this.log.error(`Error collecting txs via ${info.description}`, err, {
           ...info,
-          requestedTxs: requested.map(hash => hash.toString()),
+          requestedTxs: requested,
         });
-        return [] as Tx[];
+        return { validTxs: [] as Tx[], invalidTxHashes: [] as string[] };
       }
     });
 
-    if (txs.length === 0) {
+    if (validTxs.length === 0 && invalidTxHashes.length === 0) {
       this.log.trace(`No txs found via ${info.description}`, {
         ...info,
-        requestedTxs: requested.map(t => t.toString()),
+        requestedTxs: requested,
       });
-      return { txs, requested, duration };
+      return { txs: validTxs, requested, duration };
     }
 
-    // Validate tx hashes for all collected txs from external sources
-    const validTxs: Tx[] = [];
-    const invalidTxHashes: string[] = [];
-    await Promise.all(
-      txs.map(async tx => {
-        const isValid = await tx.validateTxHash();
-        if (isValid) {
-          validTxs.push(tx);
-        } else {
-          invalidTxHashes.push(tx.getTxHash().toString());
-        }
-      }),
-    );
-
     if (invalidTxHashes.length > 0) {
       this.log.warn(`Rejecting ${invalidTxHashes.length} txs with invalid hashes from ${info.description}`, {
         ...info,
@@ -87,7 +73,7 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
     if (validTxs.length === 0) {
       this.log.trace(`No valid txs found via ${info.description} after validation`, {
         ...info,
-        requestedTxs: requested.map(t => t.toString()),
+        requestedTxs: requested,
         invalidTxHashes,
       });
       return { txs: [], requested, duration };
@@ -99,7 +85,7 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
         ...info,
         duration,
         txs: validTxs.map(t => t.getTxHash().toString()),
-        requestedTxs: requested.map(t => t.toString()),
+        requestedTxs: requested,
         rejectedCount: invalidTxHashes.length,
       },
     );

package/src/services/tx_collection/tx_source.ts

@@ -6,9 +6,11 @@ import type { Tx, TxHash } from '@aztec/stdlib/tx';
 import { type ComponentsVersions, getComponentsVersionsFromConfig } from '@aztec/stdlib/versioning';
 import { makeTracedFetch } from '@aztec/telemetry-client';
 
+export type TxSourceCollectionResult = { validTxs: Tx[]; invalidTxHashes: string[] };
+
 export interface TxSource {
   getInfo(): string;
-  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+  getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
 }
 
 export class NodeRpcTxSource implements TxSource {
@@ -26,8 +28,25 @@ export class NodeRpcTxSource implements TxSource {
     return this.info;
   }
 
-  public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
-    return this.client.getTxsByHash(txHashes);
+  public async getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult> {
+    return this.verifyTxs(await this.client.getTxsByHash(txHashes));
+  }
+
+  private async verifyTxs(txs: Tx[]): Promise<TxSourceCollectionResult> {
+    // Validate tx hashes for all collected txs from external sources
+    const validTxs: Tx[] = [];
+    const invalidTxHashes: string[] = [];
+    await Promise.all(
+      txs.map(async tx => {
+        const isValid = await tx.validateTxHash();
+        if (isValid) {
+          validTxs.push(tx);
+        } else {
+          invalidTxHashes.push(tx.getTxHash().toString());
+        }
+      }),
+    );
+    return { validTxs: validTxs, invalidTxHashes: invalidTxHashes };
   }
 }
 

package/src/services/tx_file_store/tx_file_store.ts

@@ -146,7 +146,7 @@ export class TxFileStore {
       }
 
       await retry(
-        () => this.fileStore.save(path, tx.toBuffer(), { compress: false }),
+        () => this.fileStore.save(path, tx.toBuffer(), { compress: true }),
         `Uploading tx ${txHash}`,
         makeBackoff([0.1, 0.5, 2]),
         this.log,

package/src/services/tx_provider.ts

@@ -25,7 +25,7 @@ export class TxProvider implements ITxProvider {
   constructor(
     private txCollection: TxCollection,
     private txPool: TxPoolV2,
-    private txValidator: Pick<P2PClient, 'validate'>,
+    private blockProposalTransactionValidator: Pick<P2PClient, 'validateTxsReceivedInBlockProposal'>,
     private log: Logger = createLogger('p2p:tx-collector'),
     client: TelemetryClient = getTelemetryClient(),
   ) {
@@ -227,7 +227,7 @@ export class TxProvider implements ITxProvider {
     if (txs.length === 0) {
       return;
     }
-    await this.txValidator.validate(txs);
+    await this.blockProposalTransactionValidator.validateTxsReceivedInBlockProposal(txs);
     await this.txPool.addProtectedTxs(txs, blockHeader);
   }
 }

package/src/test-helpers/mock-pubsub.ts

@@ -270,6 +270,16 @@ class MockGossipSubService extends TypedEventEmitter<GossipsubEvents> implements
       { msgId, propagationSource, acceptance },
     );
   }
+
+  getMeshPeers(topic?: TopicStr): PeerIdStr[] {
+    if (topic && !this.subscribedTopics.has(topic)) {
+      return [];
+    }
+    return this.network
+      .getPeers()
+      .filter(peer => !this.peerId.equals(peer))
+      .map(peer => peer.toString());
+  }
 }
 
 /**

package/src/test-helpers/testbench-utils.ts

@@ -59,7 +59,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
 
   // === Core Operations (TxPoolV2) ===
 
-  addPendingTxs(txs: Tx[], opts?: { source?: string }): Promise<AddTxsResult> {
+  addPendingTxs(txs: Tx[], opts?: { source?: string; feeComparisonOnly?: boolean }): Promise<AddTxsResult> {
     const accepted: TxHash[] = [];
     const newTxs: Tx[] = [];
     for (const tx of txs) {
@@ -76,7 +76,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
     return Promise.resolve({ accepted, ignored: [], rejected: [] });
   }
 
-  canAddPendingTx(tx: Tx): Promise<'accepted' | 'ignored' | 'rejected'> {
+  canAddPendingTx(tx: Tx): Promise<'accepted' | 'ignored'> {
     const key = tx.getTxHash().toString();
     if (this.txsByHash.has(key)) {
       return Promise.resolve('ignored');
@@ -123,7 +123,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
     return Promise.resolve();
   }
 
-  handlePrunedBlocks(_latestBlock: L2BlockId): Promise<void> {
+  handlePrunedBlocks(_latestBlock: L2BlockId, _options?: { deleteAllTxs?: boolean }): Promise<void> {
     return Promise.resolve();
   }
 

package/src/testbench/p2p_client_testbench_worker.ts

@@ -29,22 +29,19 @@ import type { Message, PeerId } from '@libp2p/interface';
 import { TopicValidatorResult } from '@libp2p/interface';
 import { peerIdFromString } from '@libp2p/peer-id';
 
-import type { P2PClient } from '../client/p2p_client.js';
+import type { P2PClient } from '../client/index.js';
 import type { P2PConfig } from '../config.js';
 import { createP2PClient } from '../index.js';
-import type { MemPools } from '../mem_pools/interface.js';
-import { LibP2PService } from '../services/libp2p/libp2p_service.js';
+import type { MemPools } from '../mem_pools/index.js';
+import { BatchTxRequesterCollector, LibP2PService, SendBatchRequestCollector } from '../services/index.js';
 import type { PeerManager } from '../services/peer-manager/peer_manager.js';
 import type { BatchTxRequesterLibP2PService } from '../services/reqresp/batch-tx-requester/interface.js';
 import type { IBatchRequestTxValidator } from '../services/reqresp/batch-tx-requester/tx_validator.js';
 import { RateLimitStatus } from '../services/reqresp/rate-limiter/rate_limiter.js';
 import type { ReqResp } from '../services/reqresp/reqresp.js';
 import type { PeerDiscoveryService } from '../services/service.js';
-import {
-  BatchTxRequesterCollector,
-  SendBatchRequestCollector,
-} from '../services/tx_collection/proposal_tx_collector.js';
-import { AlwaysTrueCircuitVerifier } from '../test-helpers/reqresp-nodes.js';
+import { MissingTxsTracker } from '../services/tx_collection/missing_txs_tracker.js';
+import { AlwaysTrueCircuitVerifier } from '../test-helpers/index.js';
 import {
   BENCHMARK_CONSTANTS,
   type CollectorType,
@@ -55,7 +52,7 @@ import {
   createMockEpochCache,
   createMockWorldStateSynchronizer,
   filterTxsByDistribution,
-} from '../test-helpers/testbench-utils.js';
+} from '../test-helpers/index.js';
 import type { PubSubLibp2p } from '../util.js';
 
 export type { DistributionPattern, CollectorType } from '../test-helpers/testbench-utils.js';
@@ -277,7 +274,12 @@ async function runAggregatorBenchmark(
         new DateProvider(),
         noopTxValidator,
       );
-      const fetchedTxs = await collector.collectTxs(txHashes, blockProposal, pinnedPeer, timeoutMs);
+      const fetchedTxs = await collector.collectTxs(
+        MissingTxsTracker.fromArray(txHashes),
+        blockProposal,
+        pinnedPeer,
+        timeoutMs,
+      );
       const durationMs = timer.ms();
       return {
         type: 'BENCH_RESULT',
@@ -292,7 +294,12 @@ async function runAggregatorBenchmark(
       BENCHMARK_CONSTANTS.FIXED_MAX_PEERS,
       BENCHMARK_CONSTANTS.FIXED_MAX_RETRY_ATTEMPTS,
     );
-    const fetchedTxs = await collector.collectTxs(txHashes, blockProposal, pinnedPeer, timeoutMs);
+    const fetchedTxs = await collector.collectTxs(
+      MissingTxsTracker.fromArray(txHashes),
+      blockProposal,
+      pinnedPeer,
+      timeoutMs,
+    );
     const durationMs = timer.ms();
     return {
       type: 'BENCH_RESULT',

package/src/util.ts

@@ -23,7 +23,13 @@ export interface PubSubLibp2p extends Pick<Libp2p, 'status' | 'start' | 'stop' |
   services: {
     pubsub: Pick<
       GossipSub,
-      'addEventListener' | 'removeEventListener' | 'publish' | 'subscribe' | 'reportMessageValidationResult' | 'direct'
+      | 'addEventListener'
+      | 'removeEventListener'
+      | 'publish'
+      | 'subscribe'
+      | 'reportMessageValidationResult'
+      | 'direct'
+      | 'getMeshPeers'
     > & { score: Pick<GossipSub['score'], 'score'> };
   };
 }