@aztec/p2p 0.0.1-commit.7d4e6cd → 0.0.1-commit.87a0206

package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts

@@ -0,0 +1,336 @@
+import { MockL2BlockSource } from '@aztec/archiver/test';
+import { SecretValue } from '@aztec/foundation/config';
+import { createLogger } from '@aztec/foundation/log';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider, Timer, executeTimeout } from '@aztec/foundation/timer';
+import type { DataStoreConfig } from '@aztec/kv-store/config';
+import { openTmpStore } from '@aztec/kv-store/lmdb-v2';
+import type { L2BlockSource } from '@aztec/stdlib/block';
+import type { ContractDataSource } from '@aztec/stdlib/contract';
+import type { ClientProtocolCircuitVerifier } from '@aztec/stdlib/interfaces/server';
+import { P2PClientType, PeerErrorSeverity } from '@aztec/stdlib/p2p';
+import type { Tx, TxValidationResult } from '@aztec/stdlib/tx';
+import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
+
+import type { PeerId } from '@libp2p/interface';
+import { peerIdFromString } from '@libp2p/peer-id';
+
+import type { P2PConfig } from '../../../config.js';
+import type { IBatchRequestTxValidator } from '../../../services/reqresp/batch-tx-requester/tx_validator.js';
+import { RateLimitStatus } from '../../../services/reqresp/rate-limiter/rate_limiter.js';
+import {
+  BatchTxRequesterCollector,
+  SendBatchRequestCollector,
+} from '../../../services/tx_collection/proposal_tx_collector.js';
+import { AlwaysTrueCircuitVerifier } from '../../../test-helpers/reqresp-nodes.js';
+import {
+  BENCHMARK_CONSTANTS,
+  InMemoryAttestationPool,
+  InMemoryTxPool,
+  UNLIMITED_RATE_LIMIT_QUOTA,
+  calculateInternalTimeout,
+  createMockEpochCache,
+  createMockWorldStateSynchronizer,
+} from '../../../test-helpers/testbench-utils.js';
+import { createP2PClient } from '../../index.js';
+import type { P2PClient } from '../../p2p_client.js';
+import {
+  type WorkerCommand,
+  type WorkerResponse,
+  deserializeBlockProposal,
+  deserializeTx,
+  deserializeTxHash,
+} from './proposal_tx_collector_worker_protocol.js';
+
+let client: P2PClient | undefined;
+let txPool: InMemoryTxPool | undefined;
+let attestationPool: InMemoryAttestationPool | undefined;
+let logger = createLogger('p2p:proposal-bench');
+let kvStore: Awaited<ReturnType<typeof openTmpStore>> | undefined;
+let ipcDisconnected = false;
+
+function ensureClient(): P2PClient {
+  if (!client || !txPool) {
+    throw new Error('Worker client not started');
+  }
+  return client;
+}
+
+function isIpcDisconnectError(err: unknown): boolean {
+  const code = (err as NodeJS.ErrnoException | undefined)?.code;
+  return code === 'EPIPE' || code === 'ERR_IPC_CHANNEL_CLOSED';
+}
+
+function sendMessage(message: WorkerResponse): Promise<void> {
+  const send = process.send;
+  if (!send || !process.connected || ipcDisconnected) {
+    return Promise.resolve();
+  }
+
+  return new Promise(resolve => {
+    const fallbackTimeout = setTimeout(() => resolve(), 2000);
+    try {
+      send.call(process, message, undefined, undefined, err => {
+        clearTimeout(fallbackTimeout);
+        if (!err) {
+          resolve();
+          return;
+        }
+        if (isIpcDisconnectError(err)) {
+          ipcDisconnected = true;
+          resolve();
+          return;
+        }
+        logger.warn('Failed to send IPC message', { error: err?.message ?? String(err) });
+        resolve();
+      });
+    } catch (err: any) {
+      clearTimeout(fallbackTimeout);
+      if (isIpcDisconnectError(err)) {
+        ipcDisconnected = true;
+        resolve();
+        return;
+      }
+      logger.warn('Failed to send IPC message', { error: err?.message ?? String(err) });
+      resolve();
+    }
+  });
+}
+
+async function startClient(config: P2PConfig, clientIndex: number) {
+  txPool = new InMemoryTxPool();
+  attestationPool = new InMemoryAttestationPool();
+  const epochCache = createMockEpochCache();
+  const worldState = createMockWorldStateSynchronizer();
+  const l2BlockSource = new MockL2BlockSource();
+  const proofVerifier = new AlwaysTrueCircuitVerifier();
+  kvStore = await openTmpStore(`proposal-bench-${clientIndex}`, true, BENCHMARK_CONSTANTS.KV_STORE_MAP_SIZE_KB);
+  logger = createLogger(`p2p:proposal-bench:${clientIndex}`);
+
+  const telemetry = getTelemetryClient();
+  const deps = {
+    txPool,
+    attestationPool,
+    store: kvStore,
+    logger,
+  };
+
+  client = await createP2PClient(
+    P2PClientType.Full,
+    config as P2PConfig & DataStoreConfig,
+    l2BlockSource as L2BlockSource & ContractDataSource,
+    proofVerifier as ClientProtocolCircuitVerifier,
+    worldState,
+    epochCache,
+    'proposal-tx-collector-bench-worker',
+    new DateProvider(),
+    telemetry as TelemetryClient,
+    deps,
+  );
+
+  await client.start();
+  installUnlimitedRateLimits();
+
+  for (let i = 0; i < 120; i++) {
+    if (client.isReady()) {
+      return;
+    }
+    await sleep(500);
+  }
+
+  throw new Error('Timed out waiting for P2P client readiness');
+}
+
+function installSamplerOverrides(peerList: ReturnType<typeof peerIdFromString>[]) {
+  const reqResp = (ensureClient() as any).p2pService.reqresp as any;
+  const sampler = reqResp.connectionSampler as any;
+
+  sampler.getPeerListSortedByConnectionCountAsc = (excluding?: Set<string>) => {
+    if (!excluding || excluding.size === 0) {
+      return peerList;
+    }
+    return peerList.filter(peerId => !excluding.has(peerId.toString()));
+  };
+  sampler.samplePeersBatch = (numberToSample: number, excluding?: Map<string, boolean>) => {
+    const filtered = peerList.filter(peerId => !excluding?.has(peerId.toString()));
+    return filtered.slice(0, Math.min(numberToSample, filtered.length));
+  };
+  sampler.getPeer = (excluding?: Map<string, boolean>) => {
+    const filtered = peerList.filter(peerId => !excluding?.has(peerId.toString()));
+    return filtered[0];
+  };
+}
+
+function installUnlimitedRateLimits() {
+  const reqResp = (ensureClient() as any).p2pService.reqresp as any;
+  const rateLimiter = reqResp.rateLimiter as any;
+
+  rateLimiter.getRateLimits = () => UNLIMITED_RATE_LIMIT_QUOTA;
+  rateLimiter.allow = () => RateLimitStatus.Allowed;
+}
+
+async function runCollector(cmd: Extract<WorkerCommand, { type: 'RUN_COLLECTOR' }>) {
+  const { collectorType, txHashes, blockProposal, pinnedPeerId, peerIds, timeoutMs } = cmd;
+  const reqResp = (ensureClient() as any).p2pService.reqresp as any;
+  const peerList = peerIds.map(peerId => peerIdFromString(peerId));
+
+  installSamplerOverrides(peerList);
+  installUnlimitedRateLimits();
+
+  const p2pService = {
+    reqResp,
+    connectionSampler: {
+      getPeerListSortedByConnectionCountAsc: () => peerList,
+    },
+    txValidatorConfig: {
+      l1ChainId: 1,
+      rollupVersion: 1,
+      proofVerifier: {
+        verifyProof: () => Promise.resolve({ valid: true, durationMs: 0, totalDurationMs: 0 }),
+        stop: () => Promise.resolve(),
+      },
+    },
+    peerScoring: {
+      penalizePeer: (_peerId: PeerId, _penalty: PeerErrorSeverity) => {},
+    },
+  };
+
+  const parsedTxHashes = txHashes.map(deserializeTxHash);
+  const parsedProposal = deserializeBlockProposal(blockProposal);
+  const pinnedPeer = pinnedPeerId ? peerIdFromString(pinnedPeerId) : undefined;
+
+  const timer = new Timer();
+  let fetchedCount = 0;
+
+  const internalTimeoutMs = calculateInternalTimeout(timeoutMs);
+
+  const noopTxValidator: IBatchRequestTxValidator = {
+    validateRequestedTx: (_tx: Tx): Promise<TxValidationResult> => Promise.resolve({ result: 'valid' }),
+    validateRequestedTxs: (txs: Tx[]): Promise<TxValidationResult[]> =>
+      Promise.resolve(txs.map(() => ({ result: 'valid' }))),
+  };
+
+  try {
+    if (collectorType === 'batch-requester') {
+      const collector = new BatchTxRequesterCollector(p2pService, logger, new DateProvider(), noopTxValidator);
+      const fetched = await executeTimeout(
+        (_signal: AbortSignal) => collector.collectTxs(parsedTxHashes, parsedProposal, pinnedPeer, internalTimeoutMs),
+        timeoutMs,
+        () => new Error(`Collector timed out after ${timeoutMs}ms`),
+      );
+      fetchedCount = fetched.length;
+    } else {
+      const collector = new SendBatchRequestCollector(
+        p2pService,
+        BENCHMARK_CONSTANTS.FIXED_MAX_PEERS,
+        BENCHMARK_CONSTANTS.FIXED_MAX_RETRY_ATTEMPTS,
+      );
+      const fetched = await executeTimeout(
+        (_signal: AbortSignal) => collector.collectTxs(parsedTxHashes, parsedProposal, pinnedPeer, internalTimeoutMs),
+        timeoutMs,
+        () => new Error(`Collector timed out after ${timeoutMs}ms`),
+      );
+      fetchedCount = fetched.length;
+    }
+  } catch (err: any) {
+    logger.warn(`Collector error: ${err?.message ?? String(err)}`);
+  }
+
+  return { durationMs: timer.ms(), fetchedCount };
+}
+
+async function stopClient() {
+  if (!client) {
+    return;
+  }
+  await client.stop();
+  if (kvStore?.close) {
+    await kvStore.close();
+  }
+  client = undefined;
+  txPool = undefined;
+  attestationPool = undefined;
+}
+
+process.on('disconnect', () => {
+  ipcDisconnected = true;
+  void stopClient().finally(() => process.exit(0));
+});
+
+process.on('error', err => {
+  if (isIpcDisconnectError(err)) {
+    ipcDisconnected = true;
+    return;
+  }
+  logger.warn('Worker process error', { error: err?.message ?? String(err) });
+});
+
+process.on('message', (msg: WorkerCommand) => {
+  void (async () => {
+    if (!msg || typeof msg !== 'object') {
+      return;
+    }
+
+    const requestId = msg.requestId;
+
+    try {
+      switch (msg.type) {
+        case 'START': {
+          const rawConfig = msg.config;
+          const config: P2PConfig = {
+            ...rawConfig,
+            peerIdPrivateKey: rawConfig.peerIdPrivateKey ? new SecretValue(rawConfig.peerIdPrivateKey) : undefined,
+          } as P2PConfig;
+
+          await startClient(config, msg.clientIndex);
+          const peerId = (ensureClient() as any).p2pService.node.peerId.toString();
+          await sendMessage({ type: 'READY', requestId, peerId });
+          break;
+        }
+        case 'SET_TXS': {
+          if (!txPool) {
+            throw new Error('Tx pool not initialized');
+          }
+          const txs = msg.txs.map(deserializeTx);
+          const count = msg.mode === 'append' ? txPool.appendTxs(txs) : txPool.setTxs(txs);
+          await sendMessage({ type: 'TXS_SET', requestId, count });
+          break;
+        }
+        case 'SET_BLOCK_PROPOSAL': {
+          if (!attestationPool) {
+            throw new Error('Attestation pool not initialized');
+          }
+          const proposal = deserializeBlockProposal(msg.blockProposal);
+          await attestationPool.tryAddBlockProposal(proposal);
+          await sendMessage({ type: 'BLOCK_PROPOSAL_SET', requestId, archiveRoot: proposal.archive.toString() });
+          break;
+        }
+        case 'RUN_COLLECTOR': {
+          const { durationMs, fetchedCount } = await runCollector(msg);
+          await sendMessage({ type: 'COLLECTOR_RESULT', requestId, durationMs, fetchedCount });
+          break;
+        }
+        case 'GET_PEER_COUNT': {
+          const peers = await ensureClient().getPeers();
+          await sendMessage({ type: 'PEER_COUNT', requestId, count: peers.length });
+          break;
+        }
+        case 'STOP': {
+          await stopClient();
+          await sendMessage({ type: 'STOPPED', requestId });
+          process.exit(0);
+          break;
+        }
+        default: {
+          const _exhaustive: never = msg;
+          throw new Error(`Unknown command: ${(msg as { type?: string }).type}`);
+        }
+      }
+    } catch (err: any) {
+      await sendMessage({ type: 'ERROR', requestId, error: err?.message ?? String(err) });
+      if (msg.type === 'START') {
+        process.exit(1);
+      }
+    }
+  })();
+});

package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.ts

@@ -0,0 +1,43 @@
+import { BlockProposal } from '@aztec/stdlib/p2p';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { P2PConfig } from '../../../config.js';
+
+export type SerializedP2PConfig = Omit<P2PConfig, 'peerIdPrivateKey'> & { peerIdPrivateKey?: string };
+
+export type CollectorType = 'batch-requester' | 'send-batch-request';
+
+export type WorkerCommand =
+  | { type: 'START'; requestId: string; clientIndex: number; config: SerializedP2PConfig }
+  | { type: 'SET_TXS'; requestId: string; txs: string[]; mode?: 'replace' | 'append' }
+  | { type: 'SET_BLOCK_PROPOSAL'; requestId: string; blockProposal: string }
+  | {
+      type: 'RUN_COLLECTOR';
+      requestId: string;
+      collectorType: CollectorType;
+      txHashes: string[];
+      blockProposal: string;
+      pinnedPeerId?: string;
+      peerIds: string[];
+      timeoutMs: number;
+    }
+  | { type: 'GET_PEER_COUNT'; requestId: string }
+  | { type: 'STOP'; requestId: string };
+
+export type WorkerResponse =
+  | { type: 'READY'; requestId: string; peerId: string }
+  | { type: 'TXS_SET'; requestId: string; count: number }
+  | { type: 'BLOCK_PROPOSAL_SET'; requestId: string; archiveRoot: string }
+  | { type: 'COLLECTOR_RESULT'; requestId: string; durationMs: number; fetchedCount: number }
+  | { type: 'PEER_COUNT'; requestId: string; count: number }
+  | { type: 'STOPPED'; requestId: string }
+  | { type: 'ERROR'; requestId: string; error: string };
+
+export const serializeTx = (tx: Tx) => tx.toBuffer().toString('hex');
+export const deserializeTx = (hex: string) => Tx.fromBuffer(Buffer.from(hex, 'hex'));
+
+export const serializeTxHash = (txHash: TxHash) => txHash.toString();
+export const deserializeTxHash = (hex: string) => TxHash.fromString(hex);
+
+export const serializeBlockProposal = (proposal: BlockProposal) => proposal.toBuffer().toString('hex');
+export const deserializeBlockProposal = (hex: string) => BlockProposal.fromBuffer(Buffer.from(hex, 'hex'));

package/src/config.ts

@@ -2,10 +2,10 @@ import {
   type ConfigMappingsType,
   SecretValue,
   booleanConfigHelper,
-  floatConfigHelper,
   getConfigFromMappings,
   getDefaultConfig,
   numberConfigHelper,
+  percentageConfigHelper,
   pickConfigMappings,
   secretStringConfigHelper,
 } from '@aztec/foundation/config';
@@ -15,13 +15,23 @@ import { FunctionSelector } from '@aztec/stdlib/abi/function-selector';
 import { AztecAddress } from '@aztec/stdlib/aztec-address';
 import { type AllowedElement, type ChainConfig, chainConfigMappings } from '@aztec/stdlib/config';
 
+import {
+  type BatchTxRequesterConfig,
+  batchTxRequesterConfigMappings,
+} from './services/reqresp/batch-tx-requester/config.js';
 import { type P2PReqRespConfig, p2pReqRespConfigMappings } from './services/reqresp/config.js';
 import { type TxCollectionConfig, txCollectionConfigMappings } from './services/tx_collection/config.js';
+import { type TxFileStoreConfig, txFileStoreConfigMappings } from './services/tx_file_store/config.js';
 
 /**
  * P2P client configuration values.
  */
-export interface P2PConfig extends P2PReqRespConfig, ChainConfig, TxCollectionConfig {
+export interface P2PConfig
+  extends P2PReqRespConfig,
+    BatchTxRequesterConfig,
+    ChainConfig,
+    TxCollectionConfig,
+    TxFileStoreConfig {
   /** A flag dictating whether the P2P subsystem should be enabled. */
   p2pEnabled: boolean;
 
@@ -407,7 +417,7 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
   dropTransactionsProbability: {
     env: 'P2P_DROP_TX_CHANCE',
     description: 'The probability that a transaction is discarded (0 - 1). - For testing purposes only',
-    ...floatConfigHelper(0),
+    ...percentageConfigHelper(0),
   },
   disableTransactions: {
     env: 'TRANSACTIONS_DISABLED',
@@ -432,8 +442,10 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
     ...booleanConfigHelper(false),
   },
   ...p2pReqRespConfigMappings,
+  ...batchTxRequesterConfigMappings,
   ...chainConfigMappings,
   ...txCollectionConfigMappings,
+  ...txFileStoreConfigMappings,
 };
 
 /**