@aztec/p2p 0.0.1-commit.0208eb9 → 0.0.1-commit.031e54b

package/src/services/tx_collection/tx_collection_sink.ts

@@ -1,24 +1,29 @@
 import type { Logger } from '@aztec/foundation/log';
 import { elapsed } from '@aztec/foundation/timer';
 import type { TypedEventEmitter } from '@aztec/foundation/types';
-import { Tx, type TxHash } from '@aztec/stdlib/tx';
+import type { L2Block } from '@aztec/stdlib/block';
+import type { BlockHeader, Tx } from '@aztec/stdlib/tx';
 import type { TelemetryClient } from '@aztec/telemetry-client';
 
 import EventEmitter from 'node:events';
 
-import type { TxPool, TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
+import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/index.js';
 import { TxCollectionInstrumentation } from './instrumentation.js';
 import type { CollectionMethod } from './tx_collection.js';
+import type { TxSourceCollectionResult } from './tx_source.js';
+
+/** Context determining how collected txs should be added to the pool. */
+export type TxAddContext = { type: 'proposal'; blockHeader: BlockHeader } | { type: 'mined'; block: L2Block };
 
 /**
- * Executes collection requests from the fast and slow collection loops, and handles collected txs
+ * Executes collection requests from fast collection paths, and handles collected txs
  * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
  */
-export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitter<TxPoolEvents>) {
+export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitter<TxPoolV2Events>) {
   private readonly instrumentation: TxCollectionInstrumentation;
 
   constructor(
-    private readonly txPool: TxPool,
+    private readonly txPool: TxPoolV2,
     telemetryClient: TelemetryClient,
     private readonly log: Logger,
   ) {
@@ -27,51 +32,37 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
   }
 
   public async collect(
-    collectValidTxsFn: (txHashes: TxHash[]) => Promise<(Tx | undefined)[]>,
-    requested: TxHash[],
+    collectValidTxsFn: () => Promise<TxSourceCollectionResult>,
+    requested: string[],
     info: Record<string, any> & { description: string; method: CollectionMethod },
+    context: TxAddContext,
   ) {
     this.log.trace(`Requesting ${requested.length} txs via ${info.description}`, {
       ...info,
-      requestedTxs: requested.map(t => t.toString()),
+      requestedTxs: requested,
     });
 
     // Execute collection function and measure the time taken, catching any errors.
-    const [duration, txs] = await elapsed(async () => {
+    const [duration, { validTxs, invalidTxHashes }] = await elapsed(async () => {
       try {
-        const response = await collectValidTxsFn(requested);
-        return response.filter(tx => tx !== undefined);
+        return await collectValidTxsFn();
       } catch (err) {
         this.log.error(`Error collecting txs via ${info.description}`, err, {
           ...info,
-          requestedTxs: requested.map(hash => hash.toString()),
+          requestedTxs: requested,
         });
-        return [] as Tx[];
+        return { validTxs: [] as Tx[], invalidTxHashes: [] as string[] };
       }
     });
 
-    if (txs.length === 0) {
+    if (validTxs.length === 0 && invalidTxHashes.length === 0) {
       this.log.trace(`No txs found via ${info.description}`, {
         ...info,
-        requestedTxs: requested.map(t => t.toString()),
+        requestedTxs: requested,
       });
-      return { txs, requested, duration };
+      return { txs: validTxs, requested, duration };
     }
 
-    // Validate tx hashes for all collected txs from external sources
-    const validTxs: Tx[] = [];
-    const invalidTxHashes: string[] = [];
-    await Promise.all(
-      txs.map(async tx => {
-        const isValid = await tx.validateTxHash();
-        if (isValid) {
-          validTxs.push(tx);
-        } else {
-          invalidTxHashes.push(tx.getTxHash().toString());
-        }
-      }),
-    );
-
     if (invalidTxHashes.length > 0) {
       this.log.warn(`Rejecting ${invalidTxHashes.length} txs with invalid hashes from ${info.description}`, {
         ...info,
@@ -82,7 +73,7 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
     if (validTxs.length === 0) {
       this.log.trace(`No valid txs found via ${info.description} after validation`, {
         ...info,
-        requestedTxs: requested.map(t => t.toString()),
+        requestedTxs: requested,
         invalidTxHashes,
       });
       return { txs: [], requested, duration };
@@ -94,27 +85,32 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
         ...info,
         duration,
         txs: validTxs.map(t => t.getTxHash().toString()),
-        requestedTxs: requested.map(t => t.toString()),
+        requestedTxs: requested,
         rejectedCount: invalidTxHashes.length,
       },
     );
 
-    return await this.foundTxs(validTxs, { ...info, duration });
+    return await this.foundTxs(validTxs, { ...info, duration }, context);
   }
 
   private async foundTxs(
     txs: Tx[],
     info: Record<string, any> & { description: string; method: CollectionMethod; duration: number },
+    context: TxAddContext,
   ) {
     // Report metrics for the collection
     this.instrumentation.increaseTxsFor(info.method, txs.length, info.duration);
 
-    // Mark txs as found in the slow missing txs set and all fast requests
+    // Mark txs as found in all in-flight collection requests
     this.emit('txs-added', { txs });
 
-    // Add the txs to the tx pool (should not fail, but we catch it just in case)
+    // Add the txs to the tx pool using the appropriate method based on context
     try {
-      await this.txPool.addTxs(txs, { source: `tx-collection` });
+      if (context.type === 'mined') {
+        await this.txPool.addMinedTxs(txs, context.block.header, { source: 'tx-collection' });
+      } else {
+        await this.txPool.addProtectedTxs(txs, context.blockHeader, { source: 'tx-collection' });
+      }
     } catch (err) {
       this.log.error(`Error adding txs to the pool via ${info.description}`, err, {
         ...info,

package/src/services/tx_collection/tx_source.ts

@@ -2,36 +2,56 @@ import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
 import { protocolContractsHash } from '@aztec/protocol-contracts';
 import type { ChainConfig } from '@aztec/stdlib/config';
 import { type AztecNode, createAztecNodeClient } from '@aztec/stdlib/interfaces/client';
-import type { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { Tx, TxHash, TxValidator } from '@aztec/stdlib/tx';
 import { type ComponentsVersions, getComponentsVersionsFromConfig } from '@aztec/stdlib/versioning';
 import { makeTracedFetch } from '@aztec/telemetry-client';
 
+export type TxSourceCollectionResult = { validTxs: Tx[]; invalidTxHashes: string[] };
+
 export interface TxSource {
   getInfo(): string;
-  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+  getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
 }
 
 export class NodeRpcTxSource implements TxSource {
   constructor(
     private readonly client: Pick<AztecNode, 'getTxsByHash'>,
+    private readonly txValidator: TxValidator,
     private readonly info: string,
   ) {}
 
-  public static fromUrl(nodeUrl: string, versions: ComponentsVersions): NodeRpcTxSource {
+  public static fromUrl(nodeUrl: string, txValidator: TxValidator, versions: ComponentsVersions): NodeRpcTxSource {
     const client = createAztecNodeClient(nodeUrl, versions, makeTracedFetch([1, 2, 3], false));
-    return new NodeRpcTxSource(client, nodeUrl);
+    return new NodeRpcTxSource(client, txValidator, nodeUrl);
   }
 
   public getInfo() {
     return this.info;
   }
 
-  public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
-    return this.client.getTxsByHash(txHashes);
+  public async getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult> {
+    return this.verifyTxs(await this.client.getTxsByHash(txHashes));
+  }
+
+  private async verifyTxs(txs: Tx[]): Promise<TxSourceCollectionResult> {
+    // Validate tx hashes for all collected txs from external sources
+    const validTxs: Tx[] = [];
+    const invalidTxHashes: string[] = [];
+    await Promise.all(
+      txs.map(async tx => {
+        const validation = await this.txValidator.validateTx(tx);
+        if (validation.result === 'valid') {
+          validTxs.push(tx);
+        } else {
+          invalidTxHashes.push(tx.getTxHash().toString());
+        }
+      }),
+    );
+    return { validTxs: validTxs, invalidTxHashes: invalidTxHashes };
   }
 }
 
-export function createNodeRpcTxSources(urls: string[], chainConfig: ChainConfig) {
+export function createNodeRpcTxSources(urls: string[], txValidator: TxValidator, chainConfig: ChainConfig) {
   const versions = getComponentsVersionsFromConfig(chainConfig, protocolContractsHash, getVKTreeRoot());
-  return urls.map(url => NodeRpcTxSource.fromUrl(url, versions));
+  return urls.map(url => NodeRpcTxSource.fromUrl(url, txValidator, versions));
 }

package/src/services/tx_file_store/config.ts

@@ -6,8 +6,6 @@ import { type ConfigMappingsType, booleanConfigHelper, numberConfigHelper } from
 export type TxFileStoreConfig = {
   /** URL for uploading txs to file storage (s3://, gs://, file://) */
   txFileStoreUrl?: string;
-  /** URL for downloading txs from file storage */
-  txFileStoreDownloadUrl?: string;
   /** Max concurrent uploads */
   txFileStoreUploadConcurrency: number;
   /** Max queue size to prevent unbounded memory growth */
@@ -21,10 +19,6 @@ export const txFileStoreConfigMappings: ConfigMappingsType<TxFileStoreConfig> =
     env: 'TX_FILE_STORE_URL',
     description: 'URL for uploading txs to file storage (s3://, gs://, file://)',
   },
-  txFileStoreDownloadUrl: {
-    env: 'TX_FILE_STORE_DOWNLOAD_URL',
-    description: 'URL for downloading txs from file storage',
-  },
   txFileStoreUploadConcurrency: {
     env: 'TX_FILE_STORE_UPLOAD_CONCURRENCY',
     description: 'Maximum number of concurrent tx uploads',

package/src/services/tx_file_store/tx_file_store.ts

@@ -1,3 +1,4 @@
+import { FifoSet } from '@aztec/foundation/fifo-set';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/promise';
 import { makeBackoff, retry } from '@aztec/foundation/retry';
@@ -6,10 +7,12 @@ import { type FileStore, createFileStore } from '@aztec/stdlib/file-store';
 import type { Tx } from '@aztec/stdlib/tx';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
-import type { TxPool, TxPoolEvents } from '../../mem_pools/tx_pool/index.js';
+import type { TxPoolV2 } from '../../mem_pools/index.js';
 import type { TxFileStoreConfig } from './config.js';
 import { TxFileStoreInstrumentation } from './instrumentation.js';
 
+const MAX_RECENT_UPLOADS = 1000;
+
 /**
  * Uploads validated transactions to a file store as a fallback retrieval mechanism.
  * Listens to TxPool txs-added events and uploads txs asynchronously with bounded concurrency.
@@ -18,19 +21,18 @@ export class TxFileStore {
   private uploadQueue: Tx[] = [];
   private activeUploads = 0;
   private readonly queueProcessor: RunningPromise;
-  private readonly handleTxsAdded: TxPoolEvents['txs-added'];
+  private readonly handleTxsAdded: (args: { txs: Tx[]; source?: string }) => void;
 
   /** Recently uploaded tx hashes for deduplication. */
-  private recentUploads: Set<string> = new Set();
-  private recentUploadsOrder: string[] = [];
-  private readonly maxRecentUploads = 1000;
+  private recentUploads = FifoSet.withLimit<string>(MAX_RECENT_UPLOADS);
 
   private constructor(
     private readonly fileStore: FileStore,
-    private readonly txPool: TxPool,
+    private readonly txPool: TxPoolV2,
     private readonly config: TxFileStoreConfig,
     private readonly instrumentation: TxFileStoreInstrumentation,
     private readonly log: Logger,
+    private readonly basePath: string,
   ) {
     this.handleTxsAdded = (args: { txs: Tx[]; source?: string }) => {
       this.enqueueTxs(args.txs);
@@ -48,8 +50,9 @@ export class TxFileStore {
    * @returns The file store instance, or undefined if not configured/enabled.
    */
   static async create(
-    txPool: TxPool,
+    txPool: TxPoolV2,
     config: TxFileStoreConfig,
+    basePath: string,
     log: Logger = createLogger('p2p:tx_file_store'),
     telemetry: TelemetryClient = getTelemetryClient(),
     fileStoreOverride?: FileStore,
@@ -71,8 +74,8 @@ export class TxFileStore {
     }
 
     const instrumentation = new TxFileStoreInstrumentation(telemetry, 'TxFileStore');
-    log.info('Created tx file store', { url: config.txFileStoreUrl });
-    return new TxFileStore(fileStore, txPool, config, instrumentation, log);
+    log.info('Created tx file store', { url: config.txFileStoreUrl, basePath });
+    return new TxFileStore(fileStore, txPool, config, instrumentation, log, basePath);
   }
 
   /** Starts listening to TxPool events and uploading txs. */
@@ -122,29 +125,16 @@ export class TxFileStore {
 
   private async uploadTx(tx: Tx): Promise<void> {
     const txHash = tx.getTxHash().toString();
-    const path = `txs/${txHash}.bin`;
+    const path = `${this.basePath}/txs/${txHash}.bin`;
     const timer = new Timer();
 
-    if (this.recentUploads.has(txHash)) {
+    if (!this.recentUploads.addIfAbsent(txHash)) {
       return;
     }
 
     try {
-      this.recentUploads.add(txHash);
-      this.recentUploadsOrder.push(txHash);
-
-      if (this.recentUploadsOrder.length > this.maxRecentUploads) {
-        // delete old entries in recentUploads
-        for (const txHashToRemove of this.recentUploadsOrder.splice(
-          0,
-          this.recentUploadsOrder.length - this.maxRecentUploads,
-        )) {
-          this.recentUploads.delete(txHashToRemove);
-        }
-      }
-
       await retry(
-        () => this.fileStore.save(path, tx.toBuffer(), { compress: false }),
+        () => this.fileStore.save(path, tx.toBuffer(), { compress: true }),
         `Uploading tx ${txHash}`,
         makeBackoff([0.1, 0.5, 2]),
         this.log,

package/src/services/tx_provider.ts

@@ -5,13 +5,13 @@ import { elapsed } from '@aztec/foundation/timer';
 import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
 import type { ITxProvider } from '@aztec/stdlib/interfaces/server';
 import type { BlockProposal } from '@aztec/stdlib/p2p';
-import { Tx, TxHash } from '@aztec/stdlib/tx';
+import { type BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import type { PeerId } from '@libp2p/interface';
 
 import type { P2PClient } from '../client/p2p_client.js';
-import type { TxPool } from '../mem_pools/index.js';
+import type { TxPoolV2 } from '../mem_pools/tx_pool_v2/interfaces.js';
 import type { FastCollectionRequestInput, TxCollection } from './tx_collection/tx_collection.js';
 import { TxProviderInstrumentation } from './tx_provider_instrumentation.js';
 
@@ -24,14 +24,19 @@ export class TxProvider implements ITxProvider {
 
   constructor(
     private txCollection: TxCollection,
-    private txPool: TxPool,
-    private txValidator: Pick<P2PClient, 'validate'>,
+    private txPool: TxPoolV2,
+    private blockProposalTransactionValidator: Pick<P2PClient, 'validateTxsReceivedInBlockProposal'>,
     private log: Logger = createLogger('p2p:tx-collector'),
     client: TelemetryClient = getTelemetryClient(),
   ) {
     this.instrumentation = new TxProviderInstrumentation(client, 'TxProvider');
   }
 
+  /** Returns whether each tx hash is currently in the local tx pool. */
+  public hasTxs(txHashes: TxHash[]): Promise<boolean[]> {
+    return this.txPool.hasTxs(txHashes);
+  }
+
   /** Returns txs from the tx pool given their hashes.*/
   public async getAvailableTxs(txHashes: TxHash[]): Promise<{ txs: Tx[]; missingTxs: TxHash[] }> {
     const response = await this.txPool.getTxsByHash(txHashes);
@@ -144,6 +149,7 @@ export class TxProvider implements ITxProvider {
     // Take txs from the proposal body if there are any
     // Note that we still have to validate these txs, but we do it in parallel with tx collection
     const proposal = request.type === 'proposal' ? request.blockProposal : undefined;
+    const proposalBlockHeader = proposal?.blockHeader;
     const txsFromProposal = this.extractFromProposal(proposal, [...missingTxHashes]);
     if (txsFromProposal.length > 0) {
       this.instrumentation.incTxsFromProposals(txsFromProposal.length);
@@ -155,7 +161,7 @@ export class TxProvider implements ITxProvider {
     }
 
     if (missingTxHashes.size === 0) {
-      await this.processProposalTxs(txsFromProposal);
+      await this.processProposalTxs(txsFromProposal, proposalBlockHeader!);
       this.instrumentation.incTxsFromP2P(0, txHashes.length);
       return { txsFromMempool, txsFromProposal };
     }
@@ -163,7 +169,7 @@ export class TxProvider implements ITxProvider {
     // Start tx collection from the network if needed, while we validate the txs taken from the proposal in parallel
     const [txsFromNetwork] = await Promise.all([
       this.collectFromP2P(request, [...missingTxHashes], opts),
-      this.processProposalTxs(txsFromProposal),
+      this.processProposalTxs(txsFromProposal, proposalBlockHeader!),
     ] as const);
 
     if (txsFromNetwork.length > 0) {
@@ -222,11 +228,11 @@ export class TxProvider implements ITxProvider {
     return compactArray(proposal.txs ?? []).filter(tx => missingTxHashes.includes(tx.getTxHash().toString()));
   }
 
-  private async processProposalTxs(txs: Tx[]): Promise<void> {
+  private async processProposalTxs(txs: Tx[], blockHeader: BlockHeader): Promise<void> {
     if (txs.length === 0) {
       return;
     }
-    await this.txValidator.validate(txs);
-    await this.txPool.addTxs(txs);
+    await this.blockProposalTransactionValidator.validateTxsReceivedInBlockProposal(txs);
+    await this.txPool.addProtectedTxs(txs, blockHeader);
   }
 }

package/src/test-helpers/make-test-p2p-clients.ts

@@ -4,16 +4,16 @@ import { SecretValue } from '@aztec/foundation/config';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
-import type { DataStoreConfig } from '@aztec/kv-store/config';
 import { openTmpStore } from '@aztec/kv-store/lmdb-v2';
+import { GasFees } from '@aztec/stdlib/gas';
 import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
-import { P2PClientType } from '@aztec/stdlib/p2p';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
 
 import { createP2PClient } from '../client/index.js';
 import type { P2PClient } from '../client/p2p_client.js';
 import type { P2PConfig } from '../config.js';
 import type { AttestationPool } from '../mem_pools/attestation_pool/attestation_pool.js';
-import type { TxPool } from '../mem_pools/tx_pool/index.js';
+import type { TxPoolV2 } from '../mem_pools/tx_pool_v2/interfaces.js';
 import { generatePeerIdPrivateKeys } from '../test-helpers/generate-peer-id-private-keys.js';
 import { getPorts } from './get-ports.js';
 import { makeEnrs } from './make-enrs.js';
@@ -22,7 +22,7 @@ import { AlwaysFalseCircuitVerifier, AlwaysTrueCircuitVerifier } from './reqresp
 
 export interface MakeTestP2PClientOptions {
   mockAttestationPool: AttestationPool;
-  mockTxPool: TxPool;
+  mockTxPool: TxPoolV2;
   mockEpochCache: EpochCache;
   mockWorldState: WorldStateSynchronizer;
   alwaysTrueVerifier?: boolean;
@@ -98,22 +98,23 @@ export async function makeTestP2PClient(
   const kvStore = await openTmpStore('test');
 
   const client = await createP2PClient(
-    P2PClientType.Full,
     config,
     l2BlockSource,
     proofVerifier,
     mockWorldState,
     mockEpochCache,
+    { getCurrentMinFees: () => Promise.resolve(GasFees.empty()) },
     'test-p2p-client',
     undefined,
     undefined,
     {
-      txPool: mockTxPool as unknown as TxPool,
+      txPool: mockTxPool as unknown as TxPoolV2,
       attestationPool: mockAttestationPool as unknown as AttestationPool,
       store: kvStore,
       logger,
       p2pServiceFactory: mockGossipSubNetwork && getMockPubSubP2PServiceFactory(mockGossipSubNetwork),
     },
+    await l2BlockSource.getInitialHeader().hash(),
   );
 
   return client;