@aztec/p2p 0.0.1-commit.e61ad554 → 0.0.1-commit.f146247c

package/dest/testbench/worker_client_manager.js

@@ -1,15 +1,19 @@
-import { SecretValue } from '@aztec/foundation/config';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { sleep } from '@aztec/foundation/sleep';
 import { fork } from 'child_process';
+import { existsSync } from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import { getP2PDefaultConfig } from '../config.js';
 import { generatePeerIdPrivateKeys } from '../test-helpers/generate-peer-id-private-keys.js';
 import { getPorts } from '../test-helpers/get-ports.js';
 import { makeEnr, makeEnrs } from '../test-helpers/make-enrs.js';
+import { BENCHMARK_CONSTANTS } from '../test-helpers/testbench-utils.js';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const workerPath = path.join(__dirname, '../../dest/testbench/p2p_client_testbench_worker.js');
+const p2pRoot = path.resolve(__dirname, '../..');
+const workerTsPath = path.join(__dirname, 'p2p_client_testbench_worker.ts');
+const workerJsPath = path.join(p2pRoot, 'dest/testbench/p2p_client_testbench_worker.js');
+const tsconfigPath = path.join(p2pRoot, 'tsconfig.json');
 const testChainConfig = {
     l1ChainId: 31337,
     rollupVersion: 1,
@@ -22,6 +26,7 @@ class WorkerClientManager {
     peerIdPrivateKeys = [];
     peerEnrs = [];
     ports = [];
+    peerIds = [];
     p2pConfig;
     logger;
     messageReceivedByClient = [];
@@ -36,12 +41,15 @@ class WorkerClientManager {
         });
     }
     /**
-   * Creates a client configuration object
+   * Creates a client configuration object for IPC.
+   * Note: We send the raw peerIdPrivateKey string instead of SecretValue
+   * because SecretValue.toJSON() returns '[Redacted]', losing the value.
+   * The worker must re-wrap it in SecretValue.
    */ createClientConfig(clientIndex, port, otherNodes) {
         return {
             ...getP2PDefaultConfig(),
             p2pEnabled: true,
-            peerIdPrivateKey: new SecretValue(this.peerIdPrivateKeys[clientIndex]),
+            peerIdPrivateKey: this.peerIdPrivateKeys[clientIndex],
             listenAddress: '127.0.0.1',
             p2pIp: '127.0.0.1',
             p2pPort: port,
@@ -52,17 +60,29 @@ class WorkerClientManager {
         };
     }
     /**
-   * Spawns a worker process and returns a promise that resolves when the worker is ready
+   * Spawns a worker process and returns a promise that resolves when the worker is ready.
+   * Config uses raw string for peerIdPrivateKey (not SecretValue) for IPC serialization.
    */ spawnWorkerProcess(config, clientIndex) {
-        const childProcess = fork(workerPath);
-        // Extract the raw peerIdPrivateKey value since SecretValue can't be serialized via IPC
-        const serializedConfig = {
-            ...config,
-            peerIdPrivateKey: config.peerIdPrivateKey?.getValue()
+        const useCompiled = existsSync(workerJsPath);
+        const workerPath = useCompiled ? workerJsPath : workerTsPath;
+        const execArgv = [
+            ...process.execArgv
+        ];
+        if (!useCompiled && !execArgv.includes('ts-node/esm')) {
+            execArgv.push('--loader', 'ts-node/esm');
+        }
+        const env = {
+            ...process.env,
+            TS_NODE_PROJECT: tsconfigPath
         };
+        const childProcess = fork(workerPath, {
+            cwd: p2pRoot,
+            execArgv,
+            env
+        });
         childProcess.send({
             type: 'START',
-            config: serializedConfig,
+            config,
             clientIndex
         });
         // Handle unexpected child process exit
@@ -78,29 +98,54 @@ class WorkerClientManager {
         });
         // Create ready signal promise
         const readySignal = new Promise((resolve, reject)=>{
-            // Set a timeout to avoid hanging indefinitely
+            let resolved = false;
             const timeout = setTimeout(()=>{
+                if (resolved) {
+                    return;
+                }
+                resolved = true;
+                childProcess.off('message', messageHandler);
+                childProcess.off('exit', exitHandler);
                 reject(new Error(`Timeout waiting for worker ${clientIndex} to be ready`));
-            }, 30000); // 30 second timeout
-            childProcess.once('message', (msg)=>{
-                clearTimeout(timeout);
+            }, BENCHMARK_CONSTANTS.WORKER_READY_TIMEOUT_MS);
+            const messageHandler = (msg)=>{
+                if (resolved) {
+                    return;
+                }
+                // Only handle READY or ERROR messages; ignore others (e.g., GOSSIP_RECEIVED)
                 if (msg.type === 'READY') {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    childProcess.off('message', messageHandler);
+                    childProcess.off('exit', exitHandler);
+                    if (typeof msg.peerId === 'string') {
+                        this.peerIds[clientIndex] = msg.peerId;
+                    }
                     resolve();
-                }
-                // For future use
-                if (msg.type === 'ERROR') {
+                } else if (msg.type === 'ERROR') {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    childProcess.off('message', messageHandler);
+                    childProcess.off('exit', exitHandler);
                     reject(new Error(msg.error));
                 }
-            });
-            // Also resolve/reject if process exits before sending message
-            childProcess.once('exit', (code)=>{
+            // Ignore other message types and keep waiting for READY/ERROR
+            };
+            const exitHandler = (code)=>{
+                if (resolved) {
+                    return;
+                }
+                resolved = true;
                 clearTimeout(timeout);
+                childProcess.off('message', messageHandler);
                 if (code === 0) {
                     resolve();
                 } else {
                     reject(new Error(`Worker ${clientIndex} exited with code ${code} before becoming ready`));
                 }
-            });
+            };
+            childProcess.on('message', messageHandler);
+            childProcess.once('exit', exitHandler);
         });
         return [
             childProcess,
@@ -108,34 +153,46 @@ class WorkerClientManager {
         ];
     }
     /**
-   * Creates a number of worker clients in separate processes
-   * All are configured to connect to each other and overrided with the test specific config
+   * Creates a number of worker clients in separate processes.
+   * All are configured to connect to each other and overridden with the test-specific config.
    *
    * @param numberOfClients - The number of clients to create
+   * @param options - Optional overrides for seeding/bootstrapping strategy
    * @returns The ENRs of the created clients
-   */ async makeWorkerClients(numberOfClients) {
+   */ async makeWorkerClients(numberOfClients, options = {}) {
         try {
+            const bootstrapMode = options.bootstrapMode ?? (this.p2pConfig.bootstrapNodesAsFullPeers ? 'all' : 'subset');
+            const seedPeerLimit = options.seedPeerLimit ?? 10;
+            const batchSize = options.batchSize ?? (bootstrapMode === 'all' ? Math.min(5, numberOfClients) : numberOfClients);
+            const batchDelayMs = options.batchDelayMs ?? (bootstrapMode === 'all' ? 500 : 0);
             this.messageReceivedByClient = new Array(numberOfClients).fill(0);
             this.peerIdPrivateKeys = generatePeerIdPrivateKeys(numberOfClients);
             this.ports = await getPorts(numberOfClients);
             this.peerEnrs = await makeEnrs(this.peerIdPrivateKeys, this.ports, testChainConfig);
+            this.peerIds = new Array(numberOfClients);
             this.processes = [];
             const readySignals = [];
-            for(let i = 0; i < numberOfClients; i++){
-                this.logger.info(`Creating client ${i}`);
-                // Maximum seed with 10 other peers to allow peer discovery to connect them at a smoother rate
-                const otherNodes = this.peerEnrs.filter((_, ind)=>ind < Math.min(i, 10));
-                const config = this.createClientConfig(i, this.ports[i], otherNodes);
-                const [childProcess, readySignal] = this.spawnWorkerProcess(config, i);
-                readySignals.push(readySignal);
-                this.processes.push(childProcess);
+            for(let batchStart = 0; batchStart < numberOfClients; batchStart += batchSize){
+                const batchEnd = Math.min(batchStart + batchSize, numberOfClients);
+                const batchPromises = [];
+                for(let i = batchStart; i < batchEnd; i++){
+                    this.logger.info(`Creating client ${i}`);
+                    const otherNodes = bootstrapMode === 'all' ? this.peerEnrs.filter((_, ind)=>ind !== i) : this.peerEnrs.filter((_, ind)=>ind < Math.min(i, seedPeerLimit));
+                    const config = this.createClientConfig(i, this.ports[i], otherNodes);
+                    const [childProcess, readySignal] = this.spawnWorkerProcess(config, i);
+                    readySignals.push(readySignal);
+                    batchPromises.push(readySignal);
+                    this.processes.push(childProcess);
+                }
+                await Promise.all(batchPromises);
+                if (batchEnd < numberOfClients && batchDelayMs > 0) {
+                    await sleep(batchDelayMs);
+                }
             }
-            // Wait for peers to all connect with each other
-            await sleep(10000);
-            // Wait for all peers to be booted up with timeout
+            await sleep(BENCHMARK_CONSTANTS.PEER_DISCOVERY_WAIT_MS);
             await Promise.race([
                 Promise.all(readySignals),
-                new Promise((_, reject)=>setTimeout(()=>reject(new Error('Timeout waiting for all workers to be ready')), 30000))
+                new Promise((_, reject)=>setTimeout(()=>reject(new Error('Timeout waiting for all workers to be ready')), BENCHMARK_CONSTANTS.WORKER_READY_TIMEOUT_MS))
             ]);
             return this.peerEnrs;
         } catch (error) {
@@ -177,10 +234,9 @@ class WorkerClientManager {
             const config = this.createClientConfig(clientIndex, newPort, otherNodes);
             const [childProcess, readySignal] = this.spawnWorkerProcess(config, clientIndex);
             this.processes[clientIndex] = childProcess;
-            // Wait for the process to be ready with a timeout
             await Promise.race([
                 readySignal,
-                new Promise((_, reject)=>setTimeout(()=>reject(new Error(`Timeout waiting for client ${clientIndex} to be ready`)), 30000))
+                new Promise((_, reject)=>setTimeout(()=>reject(new Error(`Timeout waiting for client ${clientIndex} to be ready`)), BENCHMARK_CONSTANTS.WORKER_READY_TIMEOUT_MS))
             ]);
         } catch (error) {
             this.logger.error(`Error during changePort for client ${clientIndex}:`, error);
@@ -196,15 +252,14 @@ class WorkerClientManager {
             return Promise.resolve();
         }
         return new Promise((resolve)=>{
-            // Set a timeout for the graceful exit
             const forceKillTimeout = setTimeout(()=>{
                 this.logger.warn(`Process ${index} didn't exit gracefully, force killing...`);
                 try {
-                    process1.kill('SIGKILL'); // Force kill
+                    process1.kill('SIGKILL');
                 } catch (e) {
                     this.logger.error(`Error force killing process ${index}:`, e);
                 }
-            }, 5000); // 5 second timeout for graceful exit
+            }, BENCHMARK_CONSTANTS.GRACEFUL_SHUTDOWN_TIMEOUT_MS);
             // Listen for process exit
             process1.once('exit', ()=>{
                 clearTimeout(forceKillTimeout);
@@ -233,7 +288,6 @@ class WorkerClientManager {
         this.logger.info(`Cleaning up ${this.processes.length} worker processes`);
         // Create array of promises for each process termination
         const terminationPromises = this.processes.map((process1, index)=>this.terminateProcess(process1, index));
-        // Wait for all processes to terminate with a timeout
         try {
             await Promise.race([
                 Promise.all(terminationPromises),
@@ -250,14 +304,142 @@ class WorkerClientManager {
                             }
                         });
                         resolve();
-                    }, 10000); // 10 second timeout for all processes
+                    }, BENCHMARK_CONSTANTS.CLEANUP_TIMEOUT_MS);
                 })
             ]);
         } catch (error) {
             this.logger.error('Error during cleanup:', error);
         }
         this.processes = [];
+        this.peerIds = [];
         this.logger.info('All worker processes cleaned up');
     }
+    /**
+   * Run a req/resp benchmark across all worker clients.
+   *
+   * This sends a BENCH_REQRESP command to all workers:
+   * - Aggregator (client 0) runs the collector and returns timing results
+   * - Responders (clients 1..N) populate their tx pools based on distribution
+   *
+   * All workers generate the same txs deterministically from a shared seed,
+   * then filter based on their peerIndex and distribution pattern.
+   */ async runReqRespBenchmark(config) {
+        const peerCount = this.processes.length;
+        if (peerCount < 2) {
+            throw new Error('Need at least 2 peers to run req/resp benchmark');
+        }
+        const seed = config.seed ?? Date.now();
+        const blockNumber = config.blockNumber ?? 1;
+        const pinnedPeerId = config.pinnedPeerIndex !== undefined ? this.peerIds[config.pinnedPeerIndex] : undefined;
+        this.logger.info(`Starting req/resp benchmark: txCount=${config.txCount}, distribution=${config.distribution}, collector=${config.collectorType}`);
+        const readyPromises = [];
+        for(let i = 1; i < peerCount; i++){
+            const cmd = {
+                type: 'BENCH_REQRESP',
+                txCount: config.txCount,
+                peerCount,
+                distribution: config.distribution,
+                collectorType: config.collectorType,
+                timeoutMs: config.timeoutMs,
+                isAggregator: false,
+                peerIndex: i,
+                pinnedPeerIndex: config.pinnedPeerIndex,
+                pinnedPeerId,
+                blockNumber,
+                seed
+            };
+            this.processes[i].send(cmd);
+            readyPromises.push(this.waitForBenchReady(i, 30000));
+        }
+        await Promise.all(readyPromises);
+        this.logger.info('All responder peers ready, starting aggregator benchmark...');
+        const aggregatorCmd = {
+            type: 'BENCH_REQRESP',
+            txCount: config.txCount,
+            peerCount,
+            distribution: config.distribution,
+            collectorType: config.collectorType,
+            timeoutMs: config.timeoutMs,
+            isAggregator: true,
+            peerIndex: 0,
+            pinnedPeerIndex: config.pinnedPeerIndex,
+            pinnedPeerId,
+            blockNumber,
+            seed
+        };
+        this.processes[0].send(aggregatorCmd);
+        const result = await this.waitForBenchResult(0, config.timeoutMs + 30000);
+        this.logger.info(`Benchmark complete: fetched=${result.fetchedCount}/${config.txCount}, duration=${result.durationMs.toFixed(0)}ms, success=${result.success}`);
+        return {
+            txCount: config.txCount,
+            distribution: config.distribution,
+            collector: config.collectorType,
+            durationMs: result.durationMs,
+            fetchedCount: result.fetchedCount,
+            success: result.success,
+            error: result.error
+        };
+    }
+    waitForBenchReady(clientIndex, timeoutMs) {
+        return new Promise((resolve, reject)=>{
+            let resolved = false;
+            const handler = (msg)=>{
+                if (resolved) {
+                    return;
+                }
+                if (msg.type === 'BENCH_READY') {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    this.processes[clientIndex].off('message', handler);
+                    resolve();
+                } else if (msg.type === 'ERROR') {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    this.processes[clientIndex].off('message', handler);
+                    reject(new Error(`Client ${clientIndex} error: ${msg.error}`));
+                }
+            };
+            const timeout = setTimeout(()=>{
+                if (resolved) {
+                    return;
+                }
+                resolved = true;
+                this.processes[clientIndex].off('message', handler);
+                reject(new Error(`Timeout waiting for BENCH_READY from client ${clientIndex}`));
+            }, timeoutMs);
+            this.processes[clientIndex].on('message', handler);
+        });
+    }
+    waitForBenchResult(clientIndex, timeoutMs) {
+        return new Promise((resolve, reject)=>{
+            let resolved = false;
+            const handler = (msg)=>{
+                if (resolved) {
+                    return;
+                }
+                if (msg.type === 'BENCH_RESULT') {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    this.processes[clientIndex].off('message', handler);
+                    resolve(msg);
+                } else if (msg.type === 'ERROR') {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    this.processes[clientIndex].off('message', handler);
+                    reject(new Error(`Client ${clientIndex} error: ${msg.error}`));
+                }
+            };
+            const timeout = setTimeout(()=>{
+                if (resolved) {
+                    return;
+                }
+                resolved = true;
+                this.processes[clientIndex].off('message', handler);
+                reject(new Error(`Timeout waiting for BENCH_RESULT from client ${clientIndex}`));
+            }, timeoutMs);
+            this.processes[clientIndex].on('message', handler);
+        });
+    }
 }
 export { WorkerClientManager, testChainConfig };
+export { COLLECTOR_DISPLAY_NAMES } from './p2p_client_testbench_worker.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/p2p",
-  "version": "0.0.1-commit.e61ad554",
+  "version": "0.0.1-commit.f146247c",
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
@@ -67,17 +67,17 @@
     ]
   },
   "dependencies": {
-    "@aztec/constants": "0.0.1-commit.e61ad554",
-    "@aztec/epoch-cache": "0.0.1-commit.e61ad554",
-    "@aztec/ethereum": "0.0.1-commit.e61ad554",
-    "@aztec/foundation": "0.0.1-commit.e61ad554",
-    "@aztec/kv-store": "0.0.1-commit.e61ad554",
-    "@aztec/noir-contracts.js": "0.0.1-commit.e61ad554",
-    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.e61ad554",
-    "@aztec/protocol-contracts": "0.0.1-commit.e61ad554",
-    "@aztec/simulator": "0.0.1-commit.e61ad554",
-    "@aztec/stdlib": "0.0.1-commit.e61ad554",
-    "@aztec/telemetry-client": "0.0.1-commit.e61ad554",
+    "@aztec/constants": "0.0.1-commit.f146247c",
+    "@aztec/epoch-cache": "0.0.1-commit.f146247c",
+    "@aztec/ethereum": "0.0.1-commit.f146247c",
+    "@aztec/foundation": "0.0.1-commit.f146247c",
+    "@aztec/kv-store": "0.0.1-commit.f146247c",
+    "@aztec/noir-contracts.js": "0.0.1-commit.f146247c",
+    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.f146247c",
+    "@aztec/protocol-contracts": "0.0.1-commit.f146247c",
+    "@aztec/simulator": "0.0.1-commit.f146247c",
+    "@aztec/stdlib": "0.0.1-commit.f146247c",
+    "@aztec/telemetry-client": "0.0.1-commit.f146247c",
     "@chainsafe/libp2p-gossipsub": "13.0.0",
     "@chainsafe/libp2p-noise": "^15.0.0",
     "@chainsafe/libp2p-yamux": "^6.0.2",
@@ -104,8 +104,8 @@
     "xxhash-wasm": "^1.1.0"
   },
   "devDependencies": {
-    "@aztec/archiver": "0.0.1-commit.e61ad554",
-    "@aztec/world-state": "0.0.1-commit.e61ad554",
+    "@aztec/archiver": "0.0.1-commit.f146247c",
+    "@aztec/world-state": "0.0.1-commit.f146247c",
     "@jest/globals": "^30.0.0",
     "@types/jest": "^30.0.0",
     "@types/node": "^22.15.17",

package/src/bootstrap/bootstrap.ts

@@ -1,4 +1,4 @@
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import type { AztecAsyncKVStore } from '@aztec/kv-store';
 import type { P2PBootstrapApi } from '@aztec/stdlib/interfaces/server';
 import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client';
@@ -18,12 +18,15 @@ import { convertToMultiaddr, getPeerIdPrivateKey, getPublicIp } from '../util.js
 export class BootstrapNode implements P2PBootstrapApi {
   private node?: Discv5EventEmitter = undefined;
   private peerId?: PeerId;
+  private logger: Logger;
 
   constructor(
     private store: AztecAsyncKVStore,
     private telemetry: TelemetryClient,
-    private logger = createLogger('p2p:bootstrap'),
-  ) {}
+    bindings?: LoggerBindings,
+  ) {
+    this.logger = createLogger('p2p:bootstrap', bindings);
+  }
 
   /**
    * Starts the bootstrap node.
@@ -65,7 +68,7 @@ export class BootstrapNode implements P2PBootstrapApi {
 
     this.logger.debug(`Starting bootstrap node ${peerId} listening on ${listenAddrUdp.toString()}`);
 
-    const metricsRegistry = new OtelMetricsAdapter(this.telemetry);
+    const metricsRegistry = new OtelMetricsAdapter(this.telemetry, this.logger.getBindings());
     this.node = Discv5.create({
       enr: ourEnr,
       peerId,

package/src/client/factory.ts

@@ -62,15 +62,11 @@ export async function createP2PClient<T extends P2PClientType>(
     );
   }
 
-  const store = deps.store ?? (await createStore(P2P_STORE_NAME, 2, config, createLogger('p2p:lmdb-v2')));
-  const archive = await createStore(P2P_ARCHIVE_STORE_NAME, 1, config, createLogger('p2p-archive:lmdb-v2'));
-  const peerStore = await createStore(P2P_PEER_STORE_NAME, 1, config, createLogger('p2p-peer:lmdb-v2'));
-  const attestationStore = await createStore(
-    P2P_ATTESTATION_STORE_NAME,
-    1,
-    config,
-    createLogger('p2p-attestation:lmdb-v2'),
-  );
+  const bindings = logger.getBindings();
+  const store = deps.store ?? (await createStore(P2P_STORE_NAME, 2, config, bindings));
+  const archive = await createStore(P2P_ARCHIVE_STORE_NAME, 1, config, bindings);
+  const peerStore = await createStore(P2P_PEER_STORE_NAME, 1, config, bindings);
+  const attestationStore = await createStore(P2P_ATTESTATION_STORE_NAME, 1, config, bindings);
   const l1Constants = await archiver.getL1Constants();
 
   const mempools: MemPools = {
@@ -110,7 +106,7 @@ export async function createP2PClient<T extends P2PClientType>(
   }
 
   const txCollection = new TxCollection(
-    p2pService,
+    p2pService.getBatchTxRequesterService(),
     nodeSources,
     l1Constants,
     mempools.txPool,

package/src/client/p2p_client.ts

@@ -326,8 +326,10 @@ export class P2PClient<T extends P2PClientType = P2PClientType.Full>
     [Attributes.BLOCK_ARCHIVE]: proposal.archive.toString(),
     [Attributes.P2P_ID]: (await proposal.p2pMessageLoggingIdentifier()).toString(),
   }))
-  public broadcastProposal(proposal: BlockProposal): Promise<void> {
+  public async broadcastProposal(proposal: BlockProposal): Promise<void> {
     this.log.verbose(`Broadcasting proposal for slot ${proposal.slotNumber} to peers`);
+    // Store our own proposal so we can respond to req/resp requests for it
+    await this.attestationPool.addBlockProposal(proposal);
     return this.p2pService.propagate(proposal);
   }
 
@@ -336,8 +338,13 @@ export class P2PClient<T extends P2PClientType = P2PClientType.Full>
     [Attributes.BLOCK_ARCHIVE]: proposal.archive.toString(),
     [Attributes.P2P_ID]: (await proposal.p2pMessageLoggingIdentifier()).toString(),
   }))
-  public broadcastCheckpointProposal(proposal: CheckpointProposal): Promise<void> {
+  public async broadcastCheckpointProposal(proposal: CheckpointProposal): Promise<void> {
     this.log.verbose(`Broadcasting checkpoint proposal for slot ${proposal.slotNumber} to peers`);
+    const blockProposal = proposal.getBlockProposal();
+    if (blockProposal) {
+      // Store our own last-block proposal so we can respond to req/resp requests for it.
+      await this.attestationPool.addBlockProposal(blockProposal);
+    }
     return this.p2pService.propagate(proposal);
   }