@aztec/p2p 0.0.0-test.0 → 0.0.1-commit.03f7ef2

package/src/services/tx_provider.ts

@@ -0,0 +1,229 @@
+import { BlockNumber } from '@aztec/foundation/branded-types';
+import { compactArray } from '@aztec/foundation/collection';
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { elapsed } from '@aztec/foundation/timer';
+import type { L2BlockInfo, L2BlockNew } from '@aztec/stdlib/block';
+import type { ITxProvider } from '@aztec/stdlib/interfaces/server';
+import type { BlockProposal } from '@aztec/stdlib/p2p';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
+
+import type { PeerId } from '@libp2p/interface';
+
+import type { P2PClient } from '../client/p2p_client.js';
+import type { TxPool } from '../mem_pools/index.js';
+import type { FastCollectionRequestInput, TxCollection } from './tx_collection/tx_collection.js';
+import { TxProviderInstrumentation } from './tx_provider_instrumentation.js';
+
+/**
+ * Gathers and returns txs given a block proposal, block, or their hashes.
+ * Loads available txs from the tx pool, and  relies on a TxCollection service to collect txs from the network and other nodes.
+ */
+export class TxProvider implements ITxProvider {
+  protected instrumentation: TxProviderInstrumentation;
+
+  constructor(
+    private txCollection: TxCollection,
+    private txPool: TxPool,
+    private txValidator: Pick<P2PClient, 'validate'>,
+    private log: Logger = createLogger('p2p:tx-collector'),
+    client: TelemetryClient = getTelemetryClient(),
+  ) {
+    this.instrumentation = new TxProviderInstrumentation(client, 'TxProvider');
+  }
+
+  /** Returns txs from the tx pool given their hashes.*/
+  public async getAvailableTxs(txHashes: TxHash[]): Promise<{ txs: Tx[]; missingTxs: TxHash[] }> {
+    const response = await this.txPool.getTxsByHash(txHashes);
+    if (response.length !== txHashes.length) {
+      throw new Error(`Unexpected response size from tx pool: expected ${txHashes.length} but got ${response.length}`);
+    }
+    const txs: Tx[] = [];
+    const missingTxs: TxHash[] = [];
+
+    for (let i = 0; i < txHashes.length; i++) {
+      const tx = response[i];
+      if (tx === undefined) {
+        missingTxs.push(txHashes[i]);
+      } else {
+        txs.push(tx);
+      }
+    }
+
+    return { txs, missingTxs };
+  }
+
+  /** Gathers txs from the tx pool, proposal body, remote rpc nodes, and reqresp. */
+  public getTxsForBlockProposal(
+    blockProposal: BlockProposal,
+    blockNumber: BlockNumber,
+    opts: { pinnedPeer: PeerId | undefined; deadline: Date },
+  ): Promise<{ txs: Tx[]; missingTxs: TxHash[] }> {
+    return this.getOrderedTxsFromAllSources(
+      { type: 'proposal', blockProposal, blockNumber },
+      { ...blockProposal.toBlockInfo(), blockNumber },
+      blockProposal.txHashes,
+      { ...opts, pinnedPeer: opts.pinnedPeer },
+    );
+  }
+
+  /** Gathers txs from the tx pool, remote rpc nodes, and reqresp. */
+  public getTxsForBlock(block: L2BlockNew, opts: { deadline: Date }): Promise<{ txs: Tx[]; missingTxs: TxHash[] }> {
+    return this.getOrderedTxsFromAllSources(
+      { type: 'block', block },
+      block.toBlockInfo(),
+      block.body.txEffects.map(tx => tx.txHash),
+      { ...opts, pinnedPeer: undefined },
+    );
+  }
+
+  private async getOrderedTxsFromAllSources(
+    request: FastCollectionRequestInput,
+    blockInfo: L2BlockInfo,
+    txHashes: TxHash[],
+    opts: { pinnedPeer: PeerId | undefined; deadline: Date },
+  ) {
+    const [durationMs, result] = await elapsed(() => this.getTxsFromAllSources(request, blockInfo, txHashes, opts));
+    const { missingTxHashes, txsFromMempool, txsFromNetwork, txsFromProposal } = result;
+
+    const txs = [...(txsFromMempool ?? []), ...(txsFromProposal ?? []), ...(txsFromNetwork ?? [])];
+    const missingTxs = missingTxHashes?.length ?? 0;
+
+    const level = missingTxs === 0 ? 'verbose' : 'warn';
+    this.log[level](`Retrieved ${txs.length} out of ${blockInfo.txCount} txs for ${request.type}`, {
+      ...blockInfo,
+      txsFromProposal: txsFromProposal?.length,
+      txsFromMempool: txsFromMempool?.length,
+      txsFromNetwork: txsFromNetwork?.length,
+      missingTxs,
+      durationMs,
+    });
+
+    const orderedTxs = this.orderTxs(txs, txHashes);
+    if (orderedTxs.length + missingTxs !== txHashes.length) {
+      throw new Error(
+        `Error collecting txs for ${request.type} with ${txHashes.length} txs: found ${orderedTxs.length} and flagged ${missingTxs} as missing`,
+      );
+    }
+
+    return { txs: orderedTxs, missingTxs: (missingTxHashes ?? []).map(TxHash.fromString), durationMs };
+  }
+
+  private orderTxs(txs: Tx[], order: TxHash[]): Tx[] {
+    const txsMap = new Map(txs.map(tx => [tx.txHash.toString(), tx]));
+    return order.map(hash => txsMap.get(hash.toString())!).filter(tx => tx !== undefined);
+  }
+
+  private async getTxsFromAllSources(
+    request: FastCollectionRequestInput,
+    blockInfo: L2BlockInfo,
+    txHashes: TxHash[],
+    opts: { pinnedPeer: PeerId | undefined; deadline: Date },
+  ) {
+    const missingTxHashes = new Set(txHashes.map(txHash => txHash.toString()));
+    if (missingTxHashes.size === 0) {
+      this.log.debug(`Received request with no transactions`, blockInfo);
+      return {};
+    }
+
+    // First go to our tx pool and fetch whatever txs we have there
+    // We go to the mempool first since those txs are already validated
+    const txsFromMempool = compactArray(await this.txPool.getTxsByHash(txHashes));
+    txsFromMempool.forEach(tx => missingTxHashes.delete(tx.getTxHash().toString()));
+    this.instrumentation.incTxsFromMempool(txsFromMempool.length);
+    this.log.debug(
+      `Retrieved ${txsFromMempool.length} txs from mempool for block proposal (${missingTxHashes.size} pending)`,
+      { ...blockInfo, missingTxHashes: [...missingTxHashes] },
+    );
+
+    if (missingTxHashes.size === 0) {
+      return { txsFromMempool };
+    }
+
+    // Take txs from the proposal body if there are any
+    // Note that we still have to validate these txs, but we do it in parallel with tx collection
+    const proposal = request.type === 'proposal' ? request.blockProposal : undefined;
+    const txsFromProposal = this.extractFromProposal(proposal, [...missingTxHashes]);
+    if (txsFromProposal.length > 0) {
+      this.instrumentation.incTxsFromProposals(txsFromProposal.length);
+      txsFromProposal.forEach(tx => missingTxHashes.delete(tx.txHash.toString()));
+      this.log.debug(`Retrieved ${txsFromProposal.length} txs from proposal body (${missingTxHashes.size} pending)`, {
+        ...blockInfo,
+        missingTxHashes: [...missingTxHashes],
+      });
+    }
+
+    if (missingTxHashes.size === 0) {
+      await this.processProposalTxs(txsFromProposal);
+      return { txsFromMempool, txsFromProposal };
+    }
+
+    // Start tx collection from the network if needed, while we validate the txs taken from the proposal in parallel
+    const [txsFromNetwork] = await Promise.all([
+      this.collectFromP2P(request, [...missingTxHashes], opts),
+      this.processProposalTxs(txsFromProposal),
+    ] as const);
+
+    if (txsFromNetwork.length > 0) {
+      txsFromNetwork.forEach(tx => missingTxHashes.delete(tx.txHash.toString()));
+      this.instrumentation.incTxsFromP2P(txsFromNetwork.length, txHashes.length);
+      this.log.debug(
+        `Retrieved ${txsFromNetwork.length} txs from network for block proposal (${missingTxHashes.size} pending)`,
+        { ...blockInfo, missingTxHashes: [...missingTxHashes] },
+      );
+    }
+
+    if (missingTxHashes.size === 0) {
+      return { txsFromNetwork, txsFromMempool, txsFromProposal };
+    }
+
+    // We are still missing txs, make one last attempt to collect them from our pool, in case they showed up somehow else
+    const moreTxsFromPool = compactArray(await this.txPool.getTxsByHash([...missingTxHashes].map(TxHash.fromString)));
+
+    if (moreTxsFromPool.length > 0) {
+      this.instrumentation.incTxsFromMempool(moreTxsFromPool.length);
+      this.log.debug(
+        `Retrieved ${moreTxsFromPool.length} txs from pool retry for block proposal (${missingTxHashes.size} pending)`,
+        { ...blockInfo, missingTxHashes: [...missingTxHashes] },
+      );
+    }
+
+    if (missingTxHashes.size > 0) {
+      this.instrumentation.incMissingTxs(missingTxHashes.size);
+    }
+
+    return {
+      txsFromNetwork,
+      txsFromMempool: [...txsFromMempool, ...moreTxsFromPool],
+      txsFromProposal,
+      missingTxHashes: [...missingTxHashes],
+    };
+  }
+
+  private async collectFromP2P(
+    input: FastCollectionRequestInput,
+    txHashes: TxHash[] | string[],
+    opts: { deadline: Date; pinnedPeer?: PeerId },
+  ): Promise<Tx[]> {
+    const requestedAt = Date.now();
+    const result = await this.txCollection.collectFastFor(input, txHashes, opts);
+    const requestProcessedAt = Date.now();
+    this.instrumentation.recordTxsRequestDelay(requestProcessedAt - requestedAt);
+    return result;
+  }
+
+  private extractFromProposal(proposal: BlockProposal | undefined, missingTxHashes: string[]): Tx[] {
+    if (!proposal) {
+      return [];
+    }
+    return compactArray(proposal.txs ?? []).filter(tx => missingTxHashes.includes(tx.getTxHash().toString()));
+  }
+
+  private async processProposalTxs(txs: Tx[]): Promise<void> {
+    if (txs.length === 0) {
+      return;
+    }
+    await this.txValidator.validate(txs);
+    await this.txPool.addTxs(txs);
+  }
+}

package/src/services/tx_provider_instrumentation.ts

@@ -0,0 +1,61 @@
+import { type Histogram, Metrics, type TelemetryClient, type UpDownCounter } from '@aztec/telemetry-client';
+
+export class TxProviderInstrumentation {
+  private txFromProposalCount: UpDownCounter;
+  private txFromMempoolCount: UpDownCounter;
+  private txFromP2PCount: UpDownCounter;
+  private missingTxsCount: UpDownCounter;
+
+  private fractionOfTxsRequestedFromP2P: Histogram;
+  private txsRequestDelay: Histogram;
+
+  constructor(client: TelemetryClient, name: string) {
+    const meter = client.getMeter(name);
+
+    this.txFromProposalCount = meter.createUpDownCounter(Metrics.TX_PROVIDER_TXS_FROM_PROPOSALS_COUNT, {
+      description: 'The number of txs taken from block proposals',
+    });
+
+    this.txFromMempoolCount = meter.createUpDownCounter(Metrics.TX_PROVIDER_TXS_FROM_MEMPOOL_COUNT, {
+      description: 'The number of txs taken from the local mempool',
+    });
+
+    this.txFromP2PCount = meter.createUpDownCounter(Metrics.TX_PROVIDER_TXS_FROM_P2P_COUNT, {
+      description: 'The number of txs taken from the p2p network',
+    });
+
+    this.missingTxsCount = meter.createUpDownCounter(Metrics.TX_PROVIDER_MISSING_TXS_COUNT, {
+      description: 'The number of txs not found anywhere',
+    });
+
+    this.fractionOfTxsRequestedFromP2P = meter.createHistogram(Metrics.TX_PROVIDER_P2P_TXS_REQUESTED_FRACTION, {
+      description: 'The fraction of transaction requested from peers',
+    });
+
+    this.txsRequestDelay = meter.createHistogram(Metrics.TX_PROVIDER_P2P_TXS_REQUEST_DELAY, {
+      unit: 'ms',
+      description: 'The time it took to request missing transactions from p2p',
+    });
+  }
+
+  incTxsFromProposals(count: number) {
+    this.txFromProposalCount.add(count);
+  }
+
+  incTxsFromMempool(count: number) {
+    this.txFromMempoolCount.add(count);
+  }
+
+  incTxsFromP2P(count: number, total: number) {
+    this.txFromP2PCount.add(count);
+    this.fractionOfTxsRequestedFromP2P.record(count / total);
+  }
+
+  recordTxsRequestDelay(delay: number) {
+    this.txsRequestDelay.record(delay);
+  }
+
+  incMissingTxs(count: number) {
+    this.missingTxsCount.add(count);
+  }
+}

package/src/test-helpers/index.ts

@@ -3,3 +3,4 @@ export * from './get-ports.js';
 export * from './make-enrs.js';
 export * from './make-test-p2p-clients.js';
 export * from './reqresp-nodes.js';
+export * from './mock-pubsub.js';

package/src/test-helpers/make-enrs.ts

@@ -1,7 +1,7 @@
 import type { ChainConfig } from '@aztec/stdlib/config';
 
-import { SignableENR } from '@chainsafe/enr';
 import { multiaddr } from '@multiformats/multiaddr';
+import { SignableENR } from '@nethermindeth/enr';
 
 import { convertToMultiaddr, createLibP2PPeerIdFromPrivateKey } from '../util.js';
 import { setAztecEnrKey } from '../versioning.js';
@@ -30,10 +30,9 @@ export async function makeEnr(p2pPrivateKey: string, port: number, config: Chain
   const peerId = await createLibP2PPeerIdFromPrivateKey(p2pPrivateKey);
   const enr = SignableENR.createFromPeerId(peerId);
 
-  const udpAnnounceAddress = `127.0.0.1:${port}`;
-  const tcpAnnounceAddress = `127.0.0.1:${port}`;
-  const udpPublicAddr = multiaddr(convertToMultiaddr(udpAnnounceAddress, 'udp'));
-  const tcpPublicAddr = multiaddr(convertToMultiaddr(tcpAnnounceAddress, 'tcp'));
+  const p2pIp = `127.0.0.1`;
+  const udpPublicAddr = multiaddr(convertToMultiaddr(p2pIp, port, 'udp'));
+  const tcpPublicAddr = multiaddr(convertToMultiaddr(p2pIp, port, 'tcp'));
 
   // ENRS must include the network and a discoverable address (udp for discv5)
   setAztecEnrKey(enr, config);

package/src/test-helpers/make-test-p2p-clients.ts

@@ -1,6 +1,9 @@
 import { MockL2BlockSource } from '@aztec/archiver/test';
 import type { EpochCache } from '@aztec/epoch-cache';
+import { SecretValue } from '@aztec/foundation/config';
 import { type Logger, createLogger } from '@aztec/foundation/log';
+import { retryUntil } from '@aztec/foundation/retry';
+import { sleep } from '@aztec/foundation/sleep';
 import type { DataStoreConfig } from '@aztec/kv-store/config';
 import { openTmpStore } from '@aztec/kv-store/lmdb-v2';
 import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
@@ -14,9 +17,10 @@ import type { TxPool } from '../mem_pools/tx_pool/index.js';
 import { generatePeerIdPrivateKeys } from '../test-helpers/generate-peer-id-private-keys.js';
 import { getPorts } from './get-ports.js';
 import { makeEnrs } from './make-enrs.js';
+import { type MockGossipSubNetwork, getMockPubSubP2PServiceFactory } from './mock-pubsub.js';
 import { AlwaysFalseCircuitVerifier, AlwaysTrueCircuitVerifier } from './reqresp-nodes.js';
 
-interface MakeTestP2PClientOptions {
+export interface MakeTestP2PClientOptions {
   mockAttestationPool: AttestationPool;
   mockTxPool: TxPool;
   mockEpochCache: EpochCache;
@@ -25,6 +29,27 @@ interface MakeTestP2PClientOptions {
   p2pBaseConfig: P2PConfig;
   p2pConfigOverrides?: Partial<P2PConfig>;
   logger?: Logger;
+  mockGossipSubNetwork?: MockGossipSubNetwork;
+}
+
+/**
+ * Creates a single P2P client and immediately starts it for testing purposes.
+ * @param peerIdPrivateKey - The private key of the peer.
+ * @param port - The port to run the client on.
+ * @param peers - The peers to connect to.
+ * @param options - The options for the client.
+ * @returns The created and already started client.
+ */
+export async function makeAndStartTestP2PClient(
+  peerIdPrivateKey: string,
+  port: number,
+  peers: string[],
+  options: MakeTestP2PClientOptions,
+) {
+  const client = await makeTestP2PClient(peerIdPrivateKey, port, peers, options);
+
+  await client.start();
+  return client;
 }
 
 /**
@@ -47,22 +72,18 @@ export async function makeTestP2PClient(
     mockTxPool,
     mockEpochCache,
     mockWorldState,
+    mockGossipSubNetwork,
     logger = createLogger('p2p-test-client'),
   }: MakeTestP2PClientOptions,
 ) {
-  const addr = `127.0.0.1:${port}`;
-  const listenAddr = `0.0.0.0:${port}`;
-
   // Filter nodes so that we only dial active peers
-
   const config: P2PConfig & DataStoreConfig = {
     ...p2pBaseConfig,
     p2pEnabled: true,
-    peerIdPrivateKey,
-    tcpListenAddress: listenAddr, // run on port 0
-    udpListenAddress: listenAddr,
-    tcpAnnounceAddress: addr,
-    udpAnnounceAddress: addr,
+    peerIdPrivateKey: new SecretValue(peerIdPrivateKey),
+    p2pIp: `127.0.0.1`,
+    listenAddress: `127.0.0.1`,
+    p2pPort: port,
     bootstrapNodes: peers,
     peerCheckIntervalMS: 1000,
     maxPeerCount: 10,
@@ -75,12 +96,7 @@ export async function makeTestP2PClient(
 
   const proofVerifier = alwaysTrueVerifier ? new AlwaysTrueCircuitVerifier() : new AlwaysFalseCircuitVerifier();
   const kvStore = await openTmpStore('test');
-  const deps = {
-    txPool: mockTxPool as unknown as TxPool,
-    attestationPool: mockAttestationPool as unknown as AttestationPool,
-    store: kvStore,
-    logger,
-  };
+
   const client = await createP2PClient(
     P2PClientType.Full,
     config,
@@ -88,14 +104,68 @@ export async function makeTestP2PClient(
     proofVerifier,
     mockWorldState,
     mockEpochCache,
+    'test-p2p-client',
     undefined,
-    deps,
+    undefined,
+    {
+      txPool: mockTxPool as unknown as TxPool,
+      attestationPool: mockAttestationPool as unknown as AttestationPool,
+      store: kvStore,
+      logger,
+      p2pServiceFactory: mockGossipSubNetwork && getMockPubSubP2PServiceFactory(mockGossipSubNetwork),
+    },
   );
-  await client.start();
 
   return client;
 }
 
+/**
+ * Creates a number of P2P clients and immediately starts them for testing purposes.
+ * @param numberOfPeers - The number of clients to create.
+ * @param options - The options for the clients.
+ * @returns The created and started clients.
+ */
+export async function makeAndStartTestP2PClients(numberOfPeers: number, testConfig: MakeTestP2PClientOptions) {
+  const clients: P2PClient[] = [];
+  const peerIdPrivateKeys = generatePeerIdPrivateKeys(numberOfPeers);
+
+  let ports = [];
+  while (true) {
+    try {
+      ports = await getPorts(numberOfPeers);
+      break;
+    } catch {
+      await sleep(1000);
+    }
+  }
+
+  const peerEnrs = await makeEnrs(peerIdPrivateKeys, ports, testConfig.p2pBaseConfig);
+
+  for (let i = 0; i < numberOfPeers; i++) {
+    const client = await makeAndStartTestP2PClient(peerIdPrivateKeys[i], ports[i], peerEnrs, {
+      ...testConfig,
+      logger: createLogger(`p2p:${i}`),
+    });
+    clients.push(client);
+  }
+
+  await retryUntil(() => clients.every(c => c.isReady()), 'p2p clients started', 10, 0.5);
+  testConfig.logger?.info(`Created and started ${clients.length} P2P clients at ports ${ports.join(',')}`, {
+    ports,
+    peerEnrs,
+    peerIdPrivateKeys,
+  });
+
+  return clients.map((client, index) => {
+    return {
+      client,
+      peerPrivateKey: peerIdPrivateKeys[index],
+      port: ports[index],
+      enr: peerEnrs[index],
+    };
+  });
+}
+
 /**
  * Creates a number of P2P clients for testing purposes.
  * @param numberOfPeers - The number of clients to create.
@@ -106,7 +176,16 @@ export async function makeTestP2PClients(numberOfPeers: number, testConfig: Make
   const clients: P2PClient[] = [];
   const peerIdPrivateKeys = generatePeerIdPrivateKeys(numberOfPeers);
 
-  const ports = await getPorts(numberOfPeers);
+  let ports = [];
+  while (true) {
+    try {
+      ports = await getPorts(numberOfPeers);
+      break;
+    } catch {
+      await sleep(1000);
+    }
+  }
+
   const peerEnrs = await makeEnrs(peerIdPrivateKeys, ports, testConfig.p2pBaseConfig);
 
   for (let i = 0; i < numberOfPeers; i++) {
@@ -117,6 +196,17 @@ export async function makeTestP2PClients(numberOfPeers: number, testConfig: Make
     clients.push(client);
   }
 
-  await Promise.all(clients.map(client => client.isReady()));
-  return clients;
+  return clients.map((client, index) => {
+    return {
+      client,
+      peerPrivateKey: peerIdPrivateKeys[index],
+      port: ports[index],
+      enr: peerEnrs[index],
+    };
+  });
+}
+
+export async function startTestP2PClients(clients: P2PClient[]) {
+  await Promise.all(clients.map(c => c.start()));
+  await retryUntil(() => clients.every(c => c.isReady()), 'p2p clients started', 10, 0.5);
 }