@aztec/p2p 4.0.0-devnet.1-patch.1 → 4.0.0-devnet.2-patch.0

package/src/services/tx_collection/tx_collection.ts

@@ -7,7 +7,8 @@ import { DateProvider } from '@aztec/foundation/timer';
 import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
 import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
 import type { BlockProposal } from '@aztec/stdlib/p2p';
-import { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { Tx } from '@aztec/stdlib/tx';
+import { TxHash } from '@aztec/stdlib/tx';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import type { PeerId } from '@libp2p/interface';
@@ -18,7 +19,8 @@ import type { TxCollectionConfig } from './config.js';
 import { FastTxCollection } from './fast_tx_collection.js';
 import { FileStoreTxCollection } from './file_store_tx_collection.js';
 import type { FileStoreTxSource } from './file_store_tx_source.js';
-import { SlowTxCollection } from './slow_tx_collection.js';
+import type { IMissingTxsTracker } from './missing_txs_tracker.js';
+import { SlowTxCollection, getProofDeadlineForSlot } from './slow_tx_collection.js';
 import { type TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
 import type { TxSource } from './tx_source.js';
 
@@ -31,11 +33,10 @@ export type FastCollectionRequestInput =
   | { type: 'proposal'; blockProposal: BlockProposal; blockNumber: BlockNumber };
 
 export type FastCollectionRequest = FastCollectionRequestInput & {
-  missingTxHashes: Set<string>;
+  missingTxTracker: IMissingTxsTracker;
   deadline: Date;
   blockInfo: L2BlockInfo;
   promise: PromiseWithResolvers<void>;
-  foundTxs: Map<string, Tx>;
 };
 
 /**
@@ -56,8 +57,11 @@ export class TxCollection {
   /** Fast collection methods */
   protected readonly fastCollection: FastTxCollection;
 
-  /** File store collection */
-  protected readonly fileStoreCollection: FileStoreTxCollection;
+  /** File store collection for slow (mined block) path */
+  protected readonly fileStoreSlowCollection: FileStoreTxCollection;
+
+  /** File store collection for fast (proposal/proving) path */
+  protected readonly fileStoreFastCollection: FileStoreTxCollection;
 
   /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */
   private readonly reconcileFoundTxsLoop: RunningPromise;
@@ -111,7 +115,28 @@ export class TxCollection {
     );
 
     this.hasFileStoreSources = fileStoreSources.length > 0;
-    this.fileStoreCollection = new FileStoreTxCollection(fileStoreSources, this.txCollectionSink, this.log);
+    this.fileStoreSlowCollection = new FileStoreTxCollection(
+      fileStoreSources,
+      this.txCollectionSink,
+      {
+        workerCount: config.txCollectionFileStoreSlowWorkerCount,
+        backoffBaseMs: config.txCollectionFileStoreSlowBackoffBaseMs,
+        backoffMaxMs: config.txCollectionFileStoreSlowBackoffMaxMs,
+      },
+      this.dateProvider,
+      this.log,
+    );
+    this.fileStoreFastCollection = new FileStoreTxCollection(
+      fileStoreSources,
+      this.txCollectionSink,
+      {
+        workerCount: config.txCollectionFileStoreFastWorkerCount,
+        backoffBaseMs: config.txCollectionFileStoreFastBackoffBaseMs,
+        backoffMaxMs: config.txCollectionFileStoreFastBackoffMaxMs,
+      },
+      this.dateProvider,
+      this.log,
+    );
 
     this.reconcileFoundTxsLoop = new RunningPromise(
       () => this.reconcileFoundTxsWithPool(),
@@ -137,7 +162,8 @@ export class TxCollection {
   public start(): Promise<void> {
     this.started = true;
     this.slowCollection.start();
-    this.fileStoreCollection.start();
+    this.fileStoreSlowCollection.start();
+    this.fileStoreFastCollection.start();
     this.reconcileFoundTxsLoop.start();
 
     // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup
@@ -150,7 +176,8 @@ export class TxCollection {
     await Promise.all([
       this.slowCollection.stop(),
       this.fastCollection.stop(),
-      this.fileStoreCollection.stop(),
+      this.fileStoreSlowCollection.stop(),
+      this.fileStoreFastCollection.stop(),
       this.reconcileFoundTxsLoop.stop(),
     ]);
 
@@ -175,6 +202,7 @@ export class TxCollection {
     // Delay file store collection to give P2P methods time to find txs first
     if (this.hasFileStoreSources) {
       const context: TxAddContext = { type: 'mined', block };
+      const deadline = getProofDeadlineForSlot(block.header.getSlot(), this.constants);
       sleep(this.config.txCollectionFileStoreSlowDelayMs)
         .then(() => {
           if (this.started) {
@@ -182,7 +210,7 @@ export class TxCollection {
             const stillMissing = new Set(this.slowCollection.getMissingTxHashes().map(h => h.toString()));
             const remaining = txHashes.filter(h => stillMissing.has(h.toString()));
             if (remaining.length > 0) {
-              this.fileStoreCollection.startCollecting(remaining, context);
+              this.fileStoreSlowCollection.startCollecting(remaining, context, deadline);
             }
           }
         })
@@ -223,7 +251,7 @@ export class TxCollection {
       sleep(this.config.txCollectionFileStoreFastDelayMs)
         .then(() => {
           if (this.started) {
-            this.fileStoreCollection.startCollecting(hashes, context);
+            this.fileStoreFastCollection.startCollecting(hashes, context, opts.deadline);
           }
         })
         .catch(err => this.log.error('Error in file store fast delay', err));
@@ -245,7 +273,8 @@ export class TxCollection {
   private foundTxs(txs: Tx[]) {
     this.slowCollection.foundTxs(txs);
     this.fastCollection.foundTxs(txs);
-    this.fileStoreCollection.foundTxs(txs);
+    this.fileStoreSlowCollection.foundTxs(txs);
+    this.fileStoreFastCollection.foundTxs(txs);
   }
 
   /**
@@ -255,7 +284,8 @@ export class TxCollection {
   public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void {
     this.slowCollection.stopCollectingForBlocksUpTo(blockNumber);
     this.fastCollection.stopCollectingForBlocksUpTo(blockNumber);
-    this.fileStoreCollection.clearPending();
+    this.fileStoreSlowCollection.clearPending();
+    this.fileStoreFastCollection.clearPending();
   }
 
   /**
@@ -265,7 +295,8 @@ export class TxCollection {
   public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void {
     this.slowCollection.stopCollectingForBlocksAfter(blockNumber);
     this.fastCollection.stopCollectingForBlocksAfter(blockNumber);
-    this.fileStoreCollection.clearPending();
+    this.fileStoreSlowCollection.clearPending();
+    this.fileStoreFastCollection.clearPending();
   }
 
   /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */

package/src/services/tx_collection/tx_collection_sink.ts

@@ -2,14 +2,15 @@ import type { Logger } from '@aztec/foundation/log';
 import { elapsed } from '@aztec/foundation/timer';
 import type { TypedEventEmitter } from '@aztec/foundation/types';
 import type { L2Block } from '@aztec/stdlib/block';
-import type { BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx';
+import type { BlockHeader, Tx } from '@aztec/stdlib/tx';
 import type { TelemetryClient } from '@aztec/telemetry-client';
 
 import EventEmitter from 'node:events';
 
-import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interfaces.js';
+import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/index.js';
 import { TxCollectionInstrumentation } from './instrumentation.js';
 import type { CollectionMethod } from './tx_collection.js';
+import type { TxSourceCollectionResult } from './tx_source.js';
 
 /** Context determining how collected txs should be added to the pool. */
 export type TxAddContext = { type: 'proposal'; blockHeader: BlockHeader } | { type: 'mined'; block: L2Block };
@@ -31,52 +32,37 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
   }
 
   public async collect(
-    collectValidTxsFn: (txHashes: TxHash[]) => Promise<(Tx | undefined)[]>,
-    requested: TxHash[],
+    collectValidTxsFn: () => Promise<TxSourceCollectionResult>,
+    requested: string[],
     info: Record<string, any> & { description: string; method: CollectionMethod },
     context: TxAddContext,
   ) {
     this.log.trace(`Requesting ${requested.length} txs via ${info.description}`, {
       ...info,
-      requestedTxs: requested.map(t => t.toString()),
+      requestedTxs: requested,
     });
 
     // Execute collection function and measure the time taken, catching any errors.
-    const [duration, txs] = await elapsed(async () => {
+    const [duration, { validTxs, invalidTxHashes }] = await elapsed(async () => {
       try {
-        const response = await collectValidTxsFn(requested);
-        return response.filter(tx => tx !== undefined);
+        return await collectValidTxsFn();
       } catch (err) {
         this.log.error(`Error collecting txs via ${info.description}`, err, {
           ...info,
-          requestedTxs: requested.map(hash => hash.toString()),
+          requestedTxs: requested,
         });
-        return [] as Tx[];
+        return { validTxs: [] as Tx[], invalidTxHashes: [] as string[] };
       }
     });
 
-    if (txs.length === 0) {
+    if (validTxs.length === 0 && invalidTxHashes.length === 0) {
       this.log.trace(`No txs found via ${info.description}`, {
         ...info,
-        requestedTxs: requested.map(t => t.toString()),
+        requestedTxs: requested,
       });
-      return { txs, requested, duration };
+      return { txs: validTxs, requested, duration };
     }
 
-    // Validate tx hashes for all collected txs from external sources
-    const validTxs: Tx[] = [];
-    const invalidTxHashes: string[] = [];
-    await Promise.all(
-      txs.map(async tx => {
-        const isValid = await tx.validateTxHash();
-        if (isValid) {
-          validTxs.push(tx);
-        } else {
-          invalidTxHashes.push(tx.getTxHash().toString());
-        }
-      }),
-    );
-
     if (invalidTxHashes.length > 0) {
       this.log.warn(`Rejecting ${invalidTxHashes.length} txs with invalid hashes from ${info.description}`, {
         ...info,
@@ -87,7 +73,7 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
     if (validTxs.length === 0) {
       this.log.trace(`No valid txs found via ${info.description} after validation`, {
         ...info,
-        requestedTxs: requested.map(t => t.toString()),
+        requestedTxs: requested,
         invalidTxHashes,
       });
       return { txs: [], requested, duration };
@@ -99,7 +85,7 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
         ...info,
         duration,
         txs: validTxs.map(t => t.getTxHash().toString()),
-        requestedTxs: requested.map(t => t.toString()),
+        requestedTxs: requested,
         rejectedCount: invalidTxHashes.length,
       },
     );

package/src/services/tx_collection/tx_source.ts

@@ -6,9 +6,11 @@ import type { Tx, TxHash } from '@aztec/stdlib/tx';
 import { type ComponentsVersions, getComponentsVersionsFromConfig } from '@aztec/stdlib/versioning';
 import { makeTracedFetch } from '@aztec/telemetry-client';
 
+export type TxSourceCollectionResult = { validTxs: Tx[]; invalidTxHashes: string[] };
+
 export interface TxSource {
   getInfo(): string;
-  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+  getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
 }
 
 export class NodeRpcTxSource implements TxSource {
@@ -26,8 +28,25 @@ export class NodeRpcTxSource implements TxSource {
     return this.info;
   }
 
-  public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
-    return this.client.getTxsByHash(txHashes);
+  public async getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult> {
+    return this.verifyTxs(await this.client.getTxsByHash(txHashes));
+  }
+
+  private async verifyTxs(txs: Tx[]): Promise<TxSourceCollectionResult> {
+    // Validate tx hashes for all collected txs from external sources
+    const validTxs: Tx[] = [];
+    const invalidTxHashes: string[] = [];
+    await Promise.all(
+      txs.map(async tx => {
+        const isValid = await tx.validateTxHash();
+        if (isValid) {
+          validTxs.push(tx);
+        } else {
+          invalidTxHashes.push(tx.getTxHash().toString());
+        }
+      }),
+    );
+    return { validTxs: validTxs, invalidTxHashes: invalidTxHashes };
   }
 }
 

package/src/services/tx_file_store/tx_file_store.ts

@@ -31,6 +31,7 @@ export class TxFileStore {
     private readonly config: TxFileStoreConfig,
     private readonly instrumentation: TxFileStoreInstrumentation,
     private readonly log: Logger,
+    private readonly basePath: string,
   ) {
     this.handleTxsAdded = (args: { txs: Tx[]; source?: string }) => {
       this.enqueueTxs(args.txs);
@@ -50,6 +51,7 @@ export class TxFileStore {
   static async create(
     txPool: TxPoolV2,
     config: TxFileStoreConfig,
+    basePath: string,
     log: Logger = createLogger('p2p:tx_file_store'),
     telemetry: TelemetryClient = getTelemetryClient(),
     fileStoreOverride?: FileStore,
@@ -71,8 +73,8 @@ export class TxFileStore {
     }
 
     const instrumentation = new TxFileStoreInstrumentation(telemetry, 'TxFileStore');
-    log.info('Created tx file store', { url: config.txFileStoreUrl });
-    return new TxFileStore(fileStore, txPool, config, instrumentation, log);
+    log.info('Created tx file store', { url: config.txFileStoreUrl, basePath });
+    return new TxFileStore(fileStore, txPool, config, instrumentation, log, basePath);
   }
 
   /** Starts listening to TxPool events and uploading txs. */
@@ -122,7 +124,7 @@ export class TxFileStore {
 
   private async uploadTx(tx: Tx): Promise<void> {
     const txHash = tx.getTxHash().toString();
-    const path = `txs/${txHash}.bin`;
+    const path = `${this.basePath}/txs/${txHash}.bin`;
     const timer = new Timer();
 
     if (this.recentUploads.has(txHash)) {
@@ -144,7 +146,7 @@ export class TxFileStore {
       }
 
       await retry(
-        () => this.fileStore.save(path, tx.toBuffer(), { compress: false }),
+        () => this.fileStore.save(path, tx.toBuffer(), { compress: true }),
         `Uploading tx ${txHash}`,
         makeBackoff([0.1, 0.5, 2]),
         this.log,

package/src/test-helpers/testbench-utils.ts

@@ -59,7 +59,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
 
   // === Core Operations (TxPoolV2) ===
 
-  addPendingTxs(txs: Tx[], opts?: { source?: string }): Promise<AddTxsResult> {
+  addPendingTxs(txs: Tx[], opts?: { source?: string; feeComparisonOnly?: boolean }): Promise<AddTxsResult> {
     const accepted: TxHash[] = [];
     const newTxs: Tx[] = [];
     for (const tx of txs) {
@@ -123,7 +123,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
     return Promise.resolve();
   }
 
-  handlePrunedBlocks(_latestBlock: L2BlockId): Promise<void> {
+  handlePrunedBlocks(_latestBlock: L2BlockId, _options?: { deleteAllTxs?: boolean }): Promise<void> {
     return Promise.resolve();
   }
 
@@ -163,6 +163,10 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
     return Promise.resolve([...this.txsByHash.keys()].map(key => TxHash.fromString(key)));
   }
 
+  getEligiblePendingTxHashes(): Promise<TxHash[]> {
+    return this.getPendingTxHashes();
+  }
+
   getPendingTxCount(): Promise<number> {
     return Promise.resolve(this.txsByHash.size);
   }
@@ -252,6 +256,10 @@ export class InMemoryAttestationPool {
     return Promise.resolve({ added: true, alreadyExists: false, count: 1 });
   }
 
+  hasBlockProposalsForSlot(_slot: SlotNumber): Promise<boolean> {
+    return Promise.resolve(false);
+  }
+
   isEmpty(): Promise<boolean> {
     return Promise.resolve(this.proposals.size === 0);
   }

package/src/testbench/p2p_client_testbench_worker.ts

@@ -29,22 +29,19 @@ import type { Message, PeerId } from '@libp2p/interface';
 import { TopicValidatorResult } from '@libp2p/interface';
 import { peerIdFromString } from '@libp2p/peer-id';
 
-import type { P2PClient } from '../client/p2p_client.js';
+import type { P2PClient } from '../client/index.js';
 import type { P2PConfig } from '../config.js';
 import { createP2PClient } from '../index.js';
-import type { MemPools } from '../mem_pools/interface.js';
-import { LibP2PService } from '../services/libp2p/libp2p_service.js';
+import type { MemPools } from '../mem_pools/index.js';
+import { BatchTxRequesterCollector, LibP2PService, SendBatchRequestCollector } from '../services/index.js';
 import type { PeerManager } from '../services/peer-manager/peer_manager.js';
 import type { BatchTxRequesterLibP2PService } from '../services/reqresp/batch-tx-requester/interface.js';
 import type { IBatchRequestTxValidator } from '../services/reqresp/batch-tx-requester/tx_validator.js';
 import { RateLimitStatus } from '../services/reqresp/rate-limiter/rate_limiter.js';
 import type { ReqResp } from '../services/reqresp/reqresp.js';
 import type { PeerDiscoveryService } from '../services/service.js';
-import {
-  BatchTxRequesterCollector,
-  SendBatchRequestCollector,
-} from '../services/tx_collection/proposal_tx_collector.js';
-import { AlwaysTrueCircuitVerifier } from '../test-helpers/reqresp-nodes.js';
+import { MissingTxsTracker } from '../services/tx_collection/missing_txs_tracker.js';
+import { AlwaysTrueCircuitVerifier } from '../test-helpers/index.js';
 import {
   BENCHMARK_CONSTANTS,
   type CollectorType,
@@ -55,7 +52,7 @@ import {
   createMockEpochCache,
   createMockWorldStateSynchronizer,
   filterTxsByDistribution,
-} from '../test-helpers/testbench-utils.js';
+} from '../test-helpers/index.js';
 import type { PubSubLibp2p } from '../util.js';
 
 export type { DistributionPattern, CollectorType } from '../test-helpers/testbench-utils.js';
@@ -166,7 +163,7 @@ async function generateDeterministicTxs(txCount: number, seed: number, config: P
     return cached.slice(0, txCount);
   }
 
-  const includeByTimestampBase = BigInt(seed);
+  const expirationTimestampBase = BigInt(seed);
   for (let i = cached.length; i < txCount; i++) {
     const txSeed = seed * 10000 + i;
     const tx = await mockTx(txSeed, {
@@ -182,7 +179,7 @@ async function generateDeterministicTxs(txCount: number, seed: number, config: P
       hasPublicTeardownCallRequest: false,
       publicCalldataSize: 0,
     });
-    tx.data.includeByTimestamp = includeByTimestampBase + BigInt(i);
+    tx.data.expirationTimestamp = expirationTimestampBase + BigInt(i);
     await tx.recomputeHash();
     cached.push(tx);
   }
@@ -277,7 +274,12 @@ async function runAggregatorBenchmark(
         new DateProvider(),
         noopTxValidator,
       );
-      const fetchedTxs = await collector.collectTxs(txHashes, blockProposal, pinnedPeer, timeoutMs);
+      const fetchedTxs = await collector.collectTxs(
+        MissingTxsTracker.fromArray(txHashes),
+        blockProposal,
+        pinnedPeer,
+        timeoutMs,
+      );
       const durationMs = timer.ms();
       return {
         type: 'BENCH_RESULT',
@@ -292,7 +294,12 @@ async function runAggregatorBenchmark(
       BENCHMARK_CONSTANTS.FIXED_MAX_PEERS,
       BENCHMARK_CONSTANTS.FIXED_MAX_RETRY_ATTEMPTS,
     );
-    const fetchedTxs = await collector.collectTxs(txHashes, blockProposal, pinnedPeer, timeoutMs);
+    const fetchedTxs = await collector.collectTxs(
+      MissingTxsTracker.fromArray(txHashes),
+      blockProposal,
+      pinnedPeer,
+      timeoutMs,
+    );
     const durationMs = timer.ms();
     return {
       type: 'BENCH_RESULT',