@aztec/p2p 0.0.1-commit.c7c42ec → 0.0.1-commit.c80b6263

package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts

@@ -0,0 +1,336 @@
+import { MockL2BlockSource } from '@aztec/archiver/test';
+import { SecretValue } from '@aztec/foundation/config';
+import { createLogger } from '@aztec/foundation/log';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider, Timer, executeTimeout } from '@aztec/foundation/timer';
+import type { DataStoreConfig } from '@aztec/kv-store/config';
+import { openTmpStore } from '@aztec/kv-store/lmdb-v2';
+import type { L2BlockSource } from '@aztec/stdlib/block';
+import type { ContractDataSource } from '@aztec/stdlib/contract';
+import type { ClientProtocolCircuitVerifier } from '@aztec/stdlib/interfaces/server';
+import { P2PClientType, PeerErrorSeverity } from '@aztec/stdlib/p2p';
+import type { Tx, TxValidationResult } from '@aztec/stdlib/tx';
+import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
+
+import type { PeerId } from '@libp2p/interface';
+import { peerIdFromString } from '@libp2p/peer-id';
+
+import type { P2PConfig } from '../../../config.js';
+import type { IBatchRequestTxValidator } from '../../../services/reqresp/batch-tx-requester/tx_validator.js';
+import { RateLimitStatus } from '../../../services/reqresp/rate-limiter/rate_limiter.js';
+import {
+  BatchTxRequesterCollector,
+  SendBatchRequestCollector,
+} from '../../../services/tx_collection/proposal_tx_collector.js';
+import { AlwaysTrueCircuitVerifier } from '../../../test-helpers/reqresp-nodes.js';
+import {
+  BENCHMARK_CONSTANTS,
+  InMemoryAttestationPool,
+  InMemoryTxPool,
+  UNLIMITED_RATE_LIMIT_QUOTA,
+  calculateInternalTimeout,
+  createMockEpochCache,
+  createMockWorldStateSynchronizer,
+} from '../../../test-helpers/testbench-utils.js';
+import { createP2PClient } from '../../index.js';
+import type { P2PClient } from '../../p2p_client.js';
+import {
+  type WorkerCommand,
+  type WorkerResponse,
+  deserializeBlockProposal,
+  deserializeTx,
+  deserializeTxHash,
+} from './proposal_tx_collector_worker_protocol.js';
+
+let client: P2PClient | undefined;
+let txPool: InMemoryTxPool | undefined;
+let attestationPool: InMemoryAttestationPool | undefined;
+let logger = createLogger('p2p:proposal-bench');
+let kvStore: Awaited<ReturnType<typeof openTmpStore>> | undefined;
+let ipcDisconnected = false;
+
+function ensureClient(): P2PClient {
+  if (!client || !txPool) {
+    throw new Error('Worker client not started');
+  }
+  return client;
+}
+
+function isIpcDisconnectError(err: unknown): boolean {
+  const code = (err as NodeJS.ErrnoException | undefined)?.code;
+  return code === 'EPIPE' || code === 'ERR_IPC_CHANNEL_CLOSED';
+}
+
+function sendMessage(message: WorkerResponse): Promise<void> {
+  const send = process.send;
+  if (!send || !process.connected || ipcDisconnected) {
+    return Promise.resolve();
+  }
+
+  return new Promise(resolve => {
+    const fallbackTimeout = setTimeout(() => resolve(), 2000);
+    try {
+      send.call(process, message, undefined, undefined, err => {
+        clearTimeout(fallbackTimeout);
+        if (!err) {
+          resolve();
+          return;
+        }
+        if (isIpcDisconnectError(err)) {
+          ipcDisconnected = true;
+          resolve();
+          return;
+        }
+        logger.warn('Failed to send IPC message', { error: err?.message ?? String(err) });
+        resolve();
+      });
+    } catch (err: any) {
+      clearTimeout(fallbackTimeout);
+      if (isIpcDisconnectError(err)) {
+        ipcDisconnected = true;
+        resolve();
+        return;
+      }
+      logger.warn('Failed to send IPC message', { error: err?.message ?? String(err) });
+      resolve();
+    }
+  });
+}
+
+async function startClient(config: P2PConfig, clientIndex: number) {
+  txPool = new InMemoryTxPool();
+  attestationPool = new InMemoryAttestationPool();
+  const epochCache = createMockEpochCache();
+  const worldState = createMockWorldStateSynchronizer();
+  const l2BlockSource = new MockL2BlockSource();
+  const proofVerifier = new AlwaysTrueCircuitVerifier();
+  kvStore = await openTmpStore(`proposal-bench-${clientIndex}`);
+  logger = createLogger(`p2p:proposal-bench:${clientIndex}`);
+
+  const telemetry = getTelemetryClient();
+  const deps = {
+    txPool,
+    attestationPool,
+    store: kvStore,
+    logger,
+  };
+
+  client = await createP2PClient(
+    P2PClientType.Full,
+    config as P2PConfig & DataStoreConfig,
+    l2BlockSource as L2BlockSource & ContractDataSource,
+    proofVerifier as ClientProtocolCircuitVerifier,
+    worldState,
+    epochCache,
+    'proposal-tx-collector-bench-worker',
+    new DateProvider(),
+    telemetry as TelemetryClient,
+    deps,
+  );
+
+  await client.start();
+  installUnlimitedRateLimits();
+
+  for (let i = 0; i < 120; i++) {
+    if (client.isReady()) {
+      return;
+    }
+    await sleep(500);
+  }
+
+  throw new Error('Timed out waiting for P2P client readiness');
+}
+
+function installSamplerOverrides(peerList: ReturnType<typeof peerIdFromString>[]) {
+  const reqResp = (ensureClient() as any).p2pService.reqresp as any;
+  const sampler = reqResp.connectionSampler as any;
+
+  sampler.getPeerListSortedByConnectionCountAsc = (excluding?: Set<string>) => {
+    if (!excluding || excluding.size === 0) {
+      return peerList;
+    }
+    return peerList.filter(peerId => !excluding.has(peerId.toString()));
+  };
+  sampler.samplePeersBatch = (numberToSample: number, excluding?: Map<string, boolean>) => {
+    const filtered = peerList.filter(peerId => !excluding?.has(peerId.toString()));
+    return filtered.slice(0, Math.min(numberToSample, filtered.length));
+  };
+  sampler.getPeer = (excluding?: Map<string, boolean>) => {
+    const filtered = peerList.filter(peerId => !excluding?.has(peerId.toString()));
+    return filtered[0];
+  };
+}
+
+function installUnlimitedRateLimits() {
+  const reqResp = (ensureClient() as any).p2pService.reqresp as any;
+  const rateLimiter = reqResp.rateLimiter as any;
+
+  rateLimiter.getRateLimits = () => UNLIMITED_RATE_LIMIT_QUOTA;
+  rateLimiter.allow = () => RateLimitStatus.Allowed;
+}
+
+async function runCollector(cmd: Extract<WorkerCommand, { type: 'RUN_COLLECTOR' }>) {
+  const { collectorType, txHashes, blockProposal, pinnedPeerId, peerIds, timeoutMs } = cmd;
+  const reqResp = (ensureClient() as any).p2pService.reqresp as any;
+  const peerList = peerIds.map(peerId => peerIdFromString(peerId));
+
+  installSamplerOverrides(peerList);
+  installUnlimitedRateLimits();
+
+  const p2pService = {
+    reqResp,
+    connectionSampler: {
+      getPeerListSortedByConnectionCountAsc: () => peerList,
+    },
+    txValidatorConfig: {
+      l1ChainId: 1,
+      rollupVersion: 1,
+      proofVerifier: {
+        verifyProof: () => Promise.resolve({ valid: true, durationMs: 0, totalDurationMs: 0 }),
+        stop: () => Promise.resolve(),
+      },
+    },
+    peerScoring: {
+      penalizePeer: (_peerId: PeerId, _penalty: PeerErrorSeverity) => {},
+    },
+  };
+
+  const parsedTxHashes = txHashes.map(deserializeTxHash);
+  const parsedProposal = deserializeBlockProposal(blockProposal);
+  const pinnedPeer = pinnedPeerId ? peerIdFromString(pinnedPeerId) : undefined;
+
+  const timer = new Timer();
+  let fetchedCount = 0;
+
+  const internalTimeoutMs = calculateInternalTimeout(timeoutMs);
+
+  const noopTxValidator: IBatchRequestTxValidator = {
+    validateRequestedTx: (_tx: Tx): Promise<TxValidationResult> => Promise.resolve({ result: 'valid' }),
+    validateRequestedTxs: (txs: Tx[]): Promise<TxValidationResult[]> =>
+      Promise.resolve(txs.map(() => ({ result: 'valid' }))),
+  };
+
+  try {
+    if (collectorType === 'batch-requester') {
+      const collector = new BatchTxRequesterCollector(p2pService, logger, new DateProvider(), noopTxValidator);
+      const fetched = await executeTimeout(
+        (_signal: AbortSignal) => collector.collectTxs(parsedTxHashes, parsedProposal, pinnedPeer, internalTimeoutMs),
+        timeoutMs,
+        () => new Error(`Collector timed out after ${timeoutMs}ms`),
+      );
+      fetchedCount = fetched.length;
+    } else {
+      const collector = new SendBatchRequestCollector(
+        p2pService,
+        BENCHMARK_CONSTANTS.FIXED_MAX_PEERS,
+        BENCHMARK_CONSTANTS.FIXED_MAX_RETRY_ATTEMPTS,
+      );
+      const fetched = await executeTimeout(
+        (_signal: AbortSignal) => collector.collectTxs(parsedTxHashes, parsedProposal, pinnedPeer, internalTimeoutMs),
+        timeoutMs,
+        () => new Error(`Collector timed out after ${timeoutMs}ms`),
+      );
+      fetchedCount = fetched.length;
+    }
+  } catch (err: any) {
+    logger.warn(`Collector error: ${err?.message ?? String(err)}`);
+  }
+
+  return { durationMs: timer.ms(), fetchedCount };
+}
+
+async function stopClient() {
+  if (!client) {
+    return;
+  }
+  await client.stop();
+  if (kvStore?.close) {
+    await kvStore.close();
+  }
+  client = undefined;
+  txPool = undefined;
+  attestationPool = undefined;
+}
+
+process.on('disconnect', () => {
+  ipcDisconnected = true;
+  void stopClient().finally(() => process.exit(0));
+});
+
+process.on('error', err => {
+  if (isIpcDisconnectError(err)) {
+    ipcDisconnected = true;
+    return;
+  }
+  logger.warn('Worker process error', { error: err?.message ?? String(err) });
+});
+
+process.on('message', (msg: WorkerCommand) => {
+  void (async () => {
+    if (!msg || typeof msg !== 'object') {
+      return;
+    }
+
+    const requestId = msg.requestId;
+
+    try {
+      switch (msg.type) {
+        case 'START': {
+          const rawConfig = msg.config;
+          const config: P2PConfig = {
+            ...rawConfig,
+            peerIdPrivateKey: rawConfig.peerIdPrivateKey ? new SecretValue(rawConfig.peerIdPrivateKey) : undefined,
+          } as P2PConfig;
+
+          await startClient(config, msg.clientIndex);
+          const peerId = (ensureClient() as any).p2pService.node.peerId.toString();
+          await sendMessage({ type: 'READY', requestId, peerId });
+          break;
+        }
+        case 'SET_TXS': {
+          if (!txPool) {
+            throw new Error('Tx pool not initialized');
+          }
+          const txs = msg.txs.map(deserializeTx);
+          const count = msg.mode === 'append' ? txPool.appendTxs(txs) : txPool.setTxs(txs);
+          await sendMessage({ type: 'TXS_SET', requestId, count });
+          break;
+        }
+        case 'SET_BLOCK_PROPOSAL': {
+          if (!attestationPool) {
+            throw new Error('Attestation pool not initialized');
+          }
+          const proposal = deserializeBlockProposal(msg.blockProposal);
+          await attestationPool.addBlockProposal(proposal);
+          await sendMessage({ type: 'BLOCK_PROPOSAL_SET', requestId, archiveRoot: proposal.archive.toString() });
+          break;
+        }
+        case 'RUN_COLLECTOR': {
+          const { durationMs, fetchedCount } = await runCollector(msg);
+          await sendMessage({ type: 'COLLECTOR_RESULT', requestId, durationMs, fetchedCount });
+          break;
+        }
+        case 'GET_PEER_COUNT': {
+          const peers = await ensureClient().getPeers();
+          await sendMessage({ type: 'PEER_COUNT', requestId, count: peers.length });
+          break;
+        }
+        case 'STOP': {
+          await stopClient();
+          await sendMessage({ type: 'STOPPED', requestId });
+          process.exit(0);
+          break;
+        }
+        default: {
+          const _exhaustive: never = msg;
+          throw new Error(`Unknown command: ${(msg as { type?: string }).type}`);
+        }
+      }
+    } catch (err: any) {
+      await sendMessage({ type: 'ERROR', requestId, error: err?.message ?? String(err) });
+      if (msg.type === 'START') {
+        process.exit(1);
+      }
+    }
+  })();
+});

package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.ts

@@ -0,0 +1,43 @@
+import { BlockProposal } from '@aztec/stdlib/p2p';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { P2PConfig } from '../../../config.js';
+
+export type SerializedP2PConfig = Omit<P2PConfig, 'peerIdPrivateKey'> & { peerIdPrivateKey?: string };
+
+export type CollectorType = 'batch-requester' | 'send-batch-request';
+
+export type WorkerCommand =
+  | { type: 'START'; requestId: string; clientIndex: number; config: SerializedP2PConfig }
+  | { type: 'SET_TXS'; requestId: string; txs: string[]; mode?: 'replace' | 'append' }
+  | { type: 'SET_BLOCK_PROPOSAL'; requestId: string; blockProposal: string }
+  | {
+      type: 'RUN_COLLECTOR';
+      requestId: string;
+      collectorType: CollectorType;
+      txHashes: string[];
+      blockProposal: string;
+      pinnedPeerId?: string;
+      peerIds: string[];
+      timeoutMs: number;
+    }
+  | { type: 'GET_PEER_COUNT'; requestId: string }
+  | { type: 'STOP'; requestId: string };
+
+export type WorkerResponse =
+  | { type: 'READY'; requestId: string; peerId: string }
+  | { type: 'TXS_SET'; requestId: string; count: number }
+  | { type: 'BLOCK_PROPOSAL_SET'; requestId: string; archiveRoot: string }
+  | { type: 'COLLECTOR_RESULT'; requestId: string; durationMs: number; fetchedCount: number }
+  | { type: 'PEER_COUNT'; requestId: string; count: number }
+  | { type: 'STOPPED'; requestId: string }
+  | { type: 'ERROR'; requestId: string; error: string };
+
+export const serializeTx = (tx: Tx) => tx.toBuffer().toString('hex');
+export const deserializeTx = (hex: string) => Tx.fromBuffer(Buffer.from(hex, 'hex'));
+
+export const serializeTxHash = (txHash: TxHash) => txHash.toString();
+export const deserializeTxHash = (hex: string) => TxHash.fromString(hex);
+
+export const serializeBlockProposal = (proposal: BlockProposal) => proposal.toBuffer().toString('hex');
+export const deserializeBlockProposal = (hex: string) => BlockProposal.fromBuffer(Buffer.from(hex, 'hex'));

package/src/config.ts

@@ -2,10 +2,10 @@ import {
   type ConfigMappingsType,
   SecretValue,
   booleanConfigHelper,
-  floatConfigHelper,
   getConfigFromMappings,
   getDefaultConfig,
   numberConfigHelper,
+  percentageConfigHelper,
   pickConfigMappings,
   secretStringConfigHelper,
 } from '@aztec/foundation/config';
@@ -15,13 +15,17 @@ import { FunctionSelector } from '@aztec/stdlib/abi/function-selector';
 import { AztecAddress } from '@aztec/stdlib/aztec-address';
 import { type AllowedElement, type ChainConfig, chainConfigMappings } from '@aztec/stdlib/config';
 
+import {
+  type BatchTxRequesterConfig,
+  batchTxRequesterConfigMappings,
+} from './services/reqresp/batch-tx-requester/config.js';
 import { type P2PReqRespConfig, p2pReqRespConfigMappings } from './services/reqresp/config.js';
 import { type TxCollectionConfig, txCollectionConfigMappings } from './services/tx_collection/config.js';
 
 /**
  * P2P client configuration values.
  */
-export interface P2PConfig extends P2PReqRespConfig, ChainConfig, TxCollectionConfig {
+export interface P2PConfig extends P2PReqRespConfig, BatchTxRequesterConfig, ChainConfig, TxCollectionConfig {
   /** A flag dictating whether the P2P subsystem should be enabled. */
   p2pEnabled: boolean;
 
@@ -407,7 +411,7 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
   dropTransactionsProbability: {
     env: 'P2P_DROP_TX_CHANCE',
     description: 'The probability that a transaction is discarded (0 - 1). - For testing purposes only',
-    ...floatConfigHelper(0),
+    ...percentageConfigHelper(0),
   },
   disableTransactions: {
     env: 'TRANSACTIONS_DISABLED',
@@ -432,6 +436,7 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
     ...booleanConfigHelper(false),
   },
   ...p2pReqRespConfigMappings,
+  ...batchTxRequesterConfigMappings,
   ...chainConfigMappings,
   ...txCollectionConfigMappings,
 };

package/src/mem_pools/attestation_pool/attestation_pool.ts

@@ -1,5 +1,10 @@
 import type { SlotNumber } from '@aztec/foundation/branded-types';
-import type { BlockAttestation, BlockProposal } from '@aztec/stdlib/p2p';
+import type {
+  BlockProposal,
+  CheckpointAttestation,
+  CheckpointProposal,
+  CheckpointProposalCore,
+} from '@aztec/stdlib/p2p';
 
 /**
  * An Attestation Pool contains attestations collected by a validator
@@ -32,75 +37,74 @@ export interface AttestationPool {
   hasBlockProposal(idOrProposal: string | BlockProposal): Promise<boolean>;
 
   /**
-   * AddAttestations
+   * Adds a checkpoint proposal to the pool.
    *
-   * @param attestations - Attestations to add into the pool
+   * If the proposal contains a lastBlock, the BlockProposal is automatically extracted
+   * and stored separately via addBlockProposal. The checkpoint proposal is then stored
+   * without the lastBlock info (as CheckpointProposalCore).
+   *
+   * @param proposal - The checkpoint proposal to add
+   * @throws ProposalSlotCapExceededError if the slot has reached the maximum number of proposals
    */
-  addAttestations(attestations: BlockAttestation[]): Promise<void>;
+  addCheckpointProposal(proposal: CheckpointProposal): Promise<void>;
 
   /**
-   * DeleteAttestation
+   * Get checkpoint proposal by its ID.
+   *
+   * Returns a CheckpointProposalCore (without lastBlock info) since the lastBlock
+   * is extracted and stored separately as a BlockProposal when added.
    *
-   * @param attestations - Attestations to remove from the pool
+   * @param id - The ID of the checkpoint proposal to retrieve (proposal.archive)
+   * @return The checkpoint proposal core if it exists, otherwise undefined.
    */
-  deleteAttestations(attestations: BlockAttestation[]): Promise<void>;
+  getCheckpointProposal(id: string): Promise<CheckpointProposalCore | undefined>;
 
   /**
-   * Delete Attestations with a slot number smaller than the given slot
-   *
-   * Removes all attestations associated with a slot
+   * Check if a checkpoint proposal exists in the pool
    *
-   * @param slot - The oldest slot to keep.
+   * @param idOrProposal - The ID of the checkpoint proposal or the proposal itself
+   * @return True if the proposal exists, false otherwise.
    */
-  deleteAttestationsOlderThan(slot: SlotNumber): Promise<void>;
+  hasCheckpointProposal(idOrProposal: string | CheckpointProposal): Promise<boolean>;
 
   /**
-   * Delete Attestations for slot
-   *
-   * Removes all attestations associated with a slot
+   * Add checkpoint attestations to the pool
    *
-   * @param slot - The slot to delete.
+   * @param attestations - Checkpoint attestations to add into the pool
    */
-  deleteAttestationsForSlot(slot: SlotNumber): Promise<void>;
+  addCheckpointAttestations(attestations: CheckpointAttestation[]): Promise<void>;
 
   /**
-   * Delete Attestations for slot and proposal
-   *
-   * Removes all attestations associated with a slot and proposal
+   * Delete checkpoint attestations older than the given slot
    *
-   * @param slot - The slot to delete.
-   * @param proposalId - The proposal to delete.
+   * @param slot - The oldest slot to keep.
    */
-  deleteAttestationsForSlotAndProposal(slot: SlotNumber, proposalId: string): Promise<void>;
+  deleteCheckpointAttestationsOlderThan(slot: SlotNumber): Promise<void>;
 
   /**
-   * Get all Attestations for all proposals for a given slot
-   *
-   * Retrieve all of the attestations observed pertaining to a given slot
+   * Get all checkpoint attestations for a given slot
    *
    * @param slot - The slot to query
-   * @return BlockAttestations
+   * @return CheckpointAttestations
    */
-  getAttestationsForSlot(slot: SlotNumber): Promise<BlockAttestation[]>;
+  getCheckpointAttestationsForSlot(slot: SlotNumber): Promise<CheckpointAttestation[]>;
 
   /**
-   * Get Attestations for slot and given proposal
-   *
-   * Retrieve all of the attestations observed pertaining to a given slot
+   * Get checkpoint attestations for slot and given proposal
    *
    * @param slot - The slot to query
    * @param proposalId - The proposal to query
-   * @return BlockAttestations
+   * @return CheckpointAttestations
    */
-  getAttestationsForSlotAndProposal(slot: SlotNumber, proposalId: string): Promise<BlockAttestation[]>;
+  getCheckpointAttestationsForSlotAndProposal(slot: SlotNumber, proposalId: string): Promise<CheckpointAttestation[]>;
 
   /**
-   * Check if a specific attestation exists in the pool
+   * Check if a specific checkpoint attestation exists in the pool
    *
    * @param attestation - The attestation to check
    * @return True if the attestation exists, false otherwise
    */
-  hasAttestation(attestation: BlockAttestation): Promise<boolean>;
+  hasCheckpointAttestation(attestation: CheckpointAttestation): Promise<boolean>;
 
   /**
    * Returns whether adding this proposal is permitted at current capacity:
@@ -114,16 +118,39 @@ export interface AttestationPool {
   canAddProposal(block: BlockProposal): Promise<boolean>;
 
   /**
-   * Returns whether an attestation would be accepted for (slot, proposalId):
-   * - True if the attestation already exists for this sender.
-   * - True if the attestation cap for (slot, proposalId) has not been reached.
-   * - False if the cap is reached and this attestation would be a new unique entry.
+   * Returns whether adding this checkpoint proposal is permitted at current capacity.
+   *
+   * @param proposal - The checkpoint proposal to check
+   * @returns True if the proposal can be added, false otherwise.
+   */
+  canAddCheckpointProposal(proposal: CheckpointProposal): Promise<boolean>;
+
+  /**
+   * Returns whether a checkpoint attestation would be accepted for (slot, proposalId).
    *
    * @param attestation - The attestation to check
-   * @param committeeSize - Committee size for the attestation's slot, implementation may add a small buffer
+   * @param committeeSize - Committee size for the attestation's slot
    * @returns True if the attestation can be added, false otherwise.
    */
-  canAddAttestation(attestation: BlockAttestation, committeeSize: number): Promise<boolean>;
+  canAddCheckpointAttestation(attestation: CheckpointAttestation, committeeSize: number): Promise<boolean>;
+
+  /**
+   * Returns whether the checkpoint proposal cap for the given slot has been reached.
+   *
+   * @param slot - The slot to check
+   * @returns True if the cap has been reached, false otherwise.
+   */
+  hasReachedCheckpointProposalCap(slot: SlotNumber): Promise<boolean>;
+
+  /**
+   * Returns whether the checkpoint attestation cap for the given slot and proposal has been reached.
+   *
+   * @param slot - The slot to check
+   * @param proposalId - The proposal to check
+   * @param committeeSize - Committee size for the slot
+   * @returns True if the cap has been reached, false otherwise.
+   */
+  hasReachedCheckpointAttestationCap(slot: SlotNumber, proposalId: string, committeeSize: number): Promise<boolean>;
 
   /** Returns whether the pool is empty. */
   isEmpty(): Promise<boolean>;