@aztec/p2p 0.0.1-commit.7b97ef96e → 0.0.1-commit.7cbc774

package/src/services/tx_collection/fast_tx_collection.ts

@@ -1,387 +0,0 @@
-import { BlockNumber } from '@aztec/foundation/branded-types';
-import { times } from '@aztec/foundation/collection';
-import { AbortError, TimeoutError } from '@aztec/foundation/error';
-import { type Logger, createLogger } from '@aztec/foundation/log';
-import { promiseWithResolvers } from '@aztec/foundation/promise';
-import { sleep } from '@aztec/foundation/sleep';
-import { DateProvider, elapsed } from '@aztec/foundation/timer';
-import type { L2BlockInfo } from '@aztec/stdlib/block';
-import type { BlockProposal } from '@aztec/stdlib/p2p';
-import { type Tx, TxHash } from '@aztec/stdlib/tx';
-
-import type { PeerId } from '@libp2p/interface';
-
-import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
-import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
-import type { TxCollectionConfig } from './config.js';
-import { MissingTxsTracker } from './missing_txs_tracker.js';
-import {
-  BatchTxRequesterCollector,
-  type MissingTxsCollector,
-  SendBatchRequestCollector,
-} from './proposal_tx_collector.js';
-import type { FastCollectionRequest, FastCollectionRequestInput } from './tx_collection.js';
-import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
-import type { TxSource } from './tx_source.js';
-
-export class FastTxCollection {
-  // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
-  protected requests: Set<FastCollectionRequest> = new Set();
-  private missingTxsCollector: MissingTxsCollector;
-
-  constructor(
-    p2pService: BatchTxRequesterLibP2PService,
-    private nodes: TxSource[],
-    private txCollectionSink: TxCollectionSink,
-    private config: TxCollectionConfig,
-    private dateProvider: DateProvider = new DateProvider(),
-    private log: Logger = createLogger('p2p:tx_collection_service'),
-    missingTxsCollector?: MissingTxsCollector,
-  ) {
-    const batchTxRequesterConfig = this.config as Partial<BatchTxRequesterConfig>;
-    const missingTxsCollectorType = this.config.txCollectionMissingTxsCollectorType;
-    this.missingTxsCollector =
-      missingTxsCollector ??
-      (missingTxsCollectorType === 'old'
-        ? new SendBatchRequestCollector(p2pService)
-        : new BatchTxRequesterCollector(p2pService, log, dateProvider, undefined, batchTxRequesterConfig));
-  }
-
-  public async stop() {
-    this.requests.forEach(request => request.promise.reject(new AbortError(`Stopped collection service`)));
-    await Promise.resolve();
-  }
-
-  public getFastCollectionRequests() {
-    return this.requests;
-  }
-
-  public async collectFastFor(
-    input: FastCollectionRequestInput,
-    txHashes: TxHash[] | string[],
-    opts: { deadline: Date; pinnedPeer?: PeerId },
-  ) {
-    const timeout = opts.deadline.getTime() - this.dateProvider.now();
-    if (timeout <= 0) {
-      this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, {
-        deadline: opts.deadline.getTime(),
-        now: this.dateProvider.now(),
-      });
-      return [];
-    }
-
-    const blockInfo: L2BlockInfo =
-      input.type === 'proposal'
-        ? { ...input.blockProposal.toBlockInfo(), blockNumber: input.blockNumber }
-        : { ...input.block.toBlockInfo() };
-
-    // This promise is used to await for the collection to finish during the main collectFast method.
-    // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
-    const promise = promiseWithResolvers<void>();
-    const timeoutTimer = setTimeout(() => promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
-
-    const request: FastCollectionRequest = {
-      ...input,
-      blockInfo,
-      promise,
-      missingTxTracker: MissingTxsTracker.fromArray(txHashes),
-      deadline: opts.deadline,
-    };
-
-    const [duration] = await elapsed(() => this.collectFast(request, { ...opts }));
-    clearTimeout(timeoutTimer);
-
-    this.log.verbose(
-      `Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
-      {
-        ...blockInfo,
-        duration,
-        requestType: input.type,
-        missingTxs: [...request.missingTxTracker.missingTxHashes],
-      },
-    );
-    return request.missingTxTracker.collectedTxs;
-  }
-
-  protected async collectFast(
-    request: FastCollectionRequest,
-    opts: { proposal?: BlockProposal; deadline: Date; pinnedPeer?: PeerId },
-  ) {
-    this.requests.add(request);
-    const { blockInfo } = request;
-
-    this.log.debug(
-      `Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
-      { ...blockInfo, requestType: request.type, deadline: opts.deadline },
-    );
-
-    try {
-      // Start blasting all nodes for the txs. We give them a little time to respond before we start reqresp.
-      // And keep an eye on the request promise to ensure we don't wait longer than the deadline or return as soon
-      // as we have collected all txs, whatever the source.
-      const nodeCollectionPromise = this.collectFastFromNodes(request, opts);
-      const waitBeforeReqResp = sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs);
-      await Promise.race([request.promise.promise, waitBeforeReqResp]);
-
-      // If we have collected all txs, we can stop here
-      if (request.missingTxTracker.allFetched()) {
-        this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
-        return;
-      }
-
-      // Start blasting reqresp for the remaining txs. Note that node collection keeps running in parallel.
-      // We stop when we have collected all txs, timed out, or both node collection and reqresp have given up.
-      const collectionPromise = Promise.allSettled([this.collectFastViaReqResp(request, opts), nodeCollectionPromise]);
-      await Promise.race([collectionPromise, request.promise.promise]);
-    } catch (err) {
-      // Log and swallow all errors
-      const logCtx = {
-        ...blockInfo,
-        errorMessage: err instanceof Error ? err.message : undefined,
-        missingTxs: request.missingTxTracker.missingTxHashes.values().map(txHash => txHash.toString()),
-      };
-      if (err instanceof Error && err.name === 'TimeoutError') {
-        this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
-      } else if (err instanceof Error && err.name === 'AbortError') {
-        this.log.warn(`Aborted collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
-      } else {
-        this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, logCtx);
-      }
-    } finally {
-      // Ensure no unresolved promises and remove the request from the set
-      request.promise.resolve();
-      this.requests.delete(request);
-    }
-  }
-
-  /**
-   * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests
-   * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
-   * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
-   * tx is requested more than once at the same time to the same node.
-   */
-  private async collectFastFromNodes(request: FastCollectionRequest, opts: { deadline: Date }): Promise<void> {
-    if (this.nodes.length === 0) {
-      return;
-    }
-
-    // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
-    const attemptsPerTx = [...request.missingTxTracker.missingTxHashes].map(txHash => ({
-      txHash,
-      attempts: 0,
-      found: false,
-    }));
-
-    // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
-    await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx, opts)));
-  }
-
-  private async collectFastFromNode(
-    request: FastCollectionRequest,
-    node: TxSource,
-    attemptsPerTx: { txHash: string; attempts: number; found: boolean }[],
-    opts: { deadline: Date },
-  ) {
-    const notFinished = () =>
-      this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
-
-    const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
-    const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
-    const activeRequestsToThisNode = new Set<string>(); // Track the txs being actively requested to this node
-
-    const processBatch = async () => {
-      while (notFinished()) {
-        // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts,
-        // so we prioritize txs that have been requested less often.
-        const batch = [];
-        let index = 0;
-        while (batch.length < maxBatchSize) {
-          const txToRequest = attemptsPerTx[index++];
-          if (!txToRequest) {
-            // No more txs to process
-            break;
-          } else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
-            // Mark as found if it was found somewhere else, we'll then remove it from the array.
-            // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
-            txToRequest.found = true;
-          } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) {
-            // If the tx is not alredy being requested to this node, add it to the current batch and increase attempts.
-            // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that
-            // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting
-            // the same txs at the same time.
-            batch.push(txToRequest);
-            activeRequestsToThisNode.add(txToRequest.txHash);
-            txToRequest.attempts++;
-          }
-        }
-
-        // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end.
-        attemptsPerTx.sort((a, b) =>
-          a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found),
-        );
-        const firstFoundTxIndex = attemptsPerTx.findIndex(tx => tx.found);
-        if (firstFoundTxIndex !== -1) {
-          attemptsPerTx.length = firstFoundTxIndex;
-        }
-
-        // If we see no more txs to request, we can stop this "process" loop
-        if (batch.length === 0) {
-          return;
-        }
-
-        const txHashes = batch.map(({ txHash }) => txHash);
-        // Collect this batch from the node
-        await this.txCollectionSink.collect(
-          async () => {
-            const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
-            for (const tx of result.validTxs) {
-              request.missingTxTracker.markFetched(tx);
-            }
-            return result;
-          },
-          txHashes,
-          {
-            description: `fast ${node.getInfo()}`,
-            node: node.getInfo(),
-            method: 'fast-node-rpc',
-            ...request.blockInfo,
-          },
-          this.getAddContext(request),
-        );
-
-        // Clear from the active requests the txs we just requested
-        for (const requestedTx of batch) {
-          activeRequestsToThisNode.delete(requestedTx.txHash);
-        }
-
-        // Sleep a bit until hitting the node again (or not, depending on config)
-        if (notFinished()) {
-          await sleep(this.config.txCollectionFastNodeIntervalMs);
-        }
-      }
-    };
-
-    // Kick off N parallel requests to the node, up to the maxParallelRequests limit
-    await Promise.all(times(maxParallelRequests, processBatch));
-  }
-
-  private async collectFastViaReqResp(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) {
-    const timeoutMs = +request.deadline - this.dateProvider.now();
-    const pinnedPeer = opts.pinnedPeer;
-    const blockInfo = request.blockInfo;
-    const slotNumber = blockInfo.slotNumber;
-    if (timeoutMs < 100) {
-      this.log.warn(
-        `Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`,
-        { timeoutMs, ...blockInfo },
-      );
-      return;
-    }
-
-    this.log.debug(
-      `Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
-      { ...blockInfo, timeoutMs, pinnedPeer },
-    );
-
-    try {
-      await this.txCollectionSink.collect(
-        async () => {
-          let result: Tx[];
-          if (request.type === 'proposal') {
-            result = await this.missingTxsCollector.collectTxs(
-              request.missingTxTracker,
-              request.blockProposal,
-              pinnedPeer,
-              timeoutMs,
-            );
-          } else if (request.type === 'block') {
-            const blockTxsSource = {
-              txHashes: request.block.body.txEffects.map(e => e.txHash),
-              archive: request.block.archive.root,
-            };
-            result = await this.missingTxsCollector.collectTxs(
-              request.missingTxTracker,
-              blockTxsSource,
-              pinnedPeer,
-              timeoutMs,
-            );
-          } else {
-            throw new Error(`Unknown request type: ${(request as any).type}`);
-          }
-          return { validTxs: result, invalidTxHashes: [] };
-        },
-        Array.from(request.missingTxTracker.missingTxHashes),
-        { description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo },
-        this.getAddContext(request),
-      );
-    } catch (err) {
-      this.log.error(`Error sending fast reqresp request for txs`, err, {
-        txs: [...request.missingTxTracker.missingTxHashes],
-        ...blockInfo,
-      });
-    }
-  }
-
-  /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */
-  private getAddContext(request: FastCollectionRequest): TxAddContext {
-    if (request.type === 'proposal') {
-      return { type: 'proposal', blockHeader: request.blockProposal.blockHeader };
-    } else {
-      return { type: 'mined', block: request.block };
-    }
-  }
-
-  /**
-   * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
-   * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
-   */
-  public foundTxs(txs: Tx[]) {
-    for (const request of this.requests) {
-      for (const tx of txs) {
-        const txHash = tx.txHash.toString();
-        // Remove the tx hash from the missing set, and add it to the found set.
-        if (request.missingTxTracker.markFetched(tx)) {
-          this.log.trace(`Found tx ${txHash} for fast collection request`, {
-            ...request.blockInfo,
-            txHash: tx.txHash.toString(),
-            type: request.type,
-          });
-        }
-        // If we found all txs for this request, we resolve the promise
-        if (request.missingTxTracker.allFetched()) {
-          this.log.trace(`All txs found for fast collection request`, {
-            ...request.blockInfo,
-            type: request.type,
-          });
-          request.promise.resolve();
-        }
-      }
-    }
-  }
-
-  /**
-   * Stop collecting all txs for blocks less than or requal to the block number specified.
-   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
-   */
-  public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void {
-    for (const request of this.requests) {
-      if (request.blockInfo.blockNumber <= blockNumber) {
-        request.promise.reject(new AbortError(`Stopped collecting txs up to block ${blockNumber}`));
-        this.requests.delete(request);
-      }
-    }
-  }
-
-  /**
-   * Stop collecting all txs for blocks greater than the block number specified.
-   * To be called when there is a chain prune and previously mined txs are no longer relevant.
-   */
-  public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void {
-    for (const request of this.requests) {
-      if (request.blockInfo.blockNumber > blockNumber) {
-        request.promise.reject(new AbortError(`Stopped collecting txs after block ${blockNumber}`));
-        this.requests.delete(request);
-      }
-    }
-  }
-}

package/src/services/tx_collection/missing_txs_tracker.ts

@@ -1,52 +0,0 @@
-import { TxHash } from '@aztec/stdlib/tx';
-import type { Tx } from '@aztec/stdlib/tx';
-
-/**
- * Tracks which transactions are still missing and need to be fetched.
- * Allows external code to mark transactions as fetched, enabling coordination
- * between multiple fetching mechanisms (e.g., BatchTxRequester and Rpc Node requests).
- */
-export interface IMissingTxsTracker {
-  /** Returns the set of transaction hashes that are still missing. */
-  get missingTxHashes(): Set<string>;
-  /** Size of this.missingTxHashes */
-  get numberOfMissingTxs(): number;
-  /** Are all requested txs are fetched */
-  allFetched(): boolean;
-  /** Checks that transaction is still missing */
-  isMissing(txHash: string): boolean;
-  /** Marks a transaction as fetched. Returns true if it was previously missing. */
-  markFetched(tx: Tx): boolean;
-  /** Get list of collected txs */
-  get collectedTxs(): Tx[];
-}
-
-export class MissingTxsTracker implements IMissingTxsTracker {
-  public readonly collectedTxs: Tx[] = [];
-
-  private constructor(public readonly missingTxHashes: Set<string>) {}
-
-  public static fromArray(hashes: TxHash[] | string[]) {
-    return new MissingTxsTracker(new Set(hashes.map(hash => hash.toString())));
-  }
-
-  markFetched(tx: Tx): boolean {
-    if (this.missingTxHashes.delete(tx.txHash.toString())) {
-      this.collectedTxs.push(tx);
-      return true;
-    }
-    return false;
-  }
-
-  get numberOfMissingTxs(): number {
-    return this.missingTxHashes.size;
-  }
-
-  allFetched(): boolean {
-    return this.numberOfMissingTxs === 0;
-  }
-
-  isMissing(txHash: string): boolean {
-    return this.missingTxHashes.has(txHash.toString());
-  }
-}

package/src/services/tx_collection/proposal_tx_collector.ts

@@ -1,113 +0,0 @@
-import type { Logger } from '@aztec/foundation/log';
-import type { DateProvider } from '@aztec/foundation/timer';
-import { type Tx, TxHash } from '@aztec/stdlib/tx';
-
-import type { PeerId } from '@libp2p/interface';
-
-import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js';
-import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
-import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
-import type { IBatchRequestTxValidator } from '../reqresp/batch-tx-requester/tx_validator.js';
-import { type BlockTxsSource, ReqRespSubProtocol, chunkTxHashesRequest } from '../reqresp/index.js';
-import type { IMissingTxsTracker } from './missing_txs_tracker.js';
-
-/**
- * Strategy interface for collecting missing transactions for a block or proposal.
- * Allows swapping between different tx collection implementations for benchmarking.
- */
-export interface MissingTxsCollector {
-  /**
-   * Collect missing transactions for a block or proposal.
-   * @param missingTxsTracker - The missing transactions tracker
-   * @param blockTxsSource - The block or proposal containing the transactions
-   * @param pinnedPeer - Optional peer expected to have the transactions
-   * @param timeoutMs - Timeout in milliseconds
-   * @returns The collected transactions
-   */
-  collectTxs(
-    missingTxsTracker: IMissingTxsTracker,
-    blockTxsSource: BlockTxsSource,
-    pinnedPeer: PeerId | undefined,
-    timeoutMs: number,
-  ): Promise<Tx[]>;
-}
-
-/**
- * Collects transactions using the BatchTxRequester implementation.
- * This uses a smart/dumb peer strategy with parallel workers.
- */
-export class BatchTxRequesterCollector implements MissingTxsCollector {
-  constructor(
-    private p2pService: BatchTxRequesterLibP2PService,
-    private log: Logger,
-    private dateProvider: DateProvider,
-    private txValidator?: IBatchRequestTxValidator,
-    private batchTxRequesterConfig?: Partial<BatchTxRequesterConfig>,
-  ) {}
-
-  async collectTxs(
-    missingTxsTracker: IMissingTxsTracker,
-    blockTxsSource: BlockTxsSource,
-    pinnedPeer: PeerId | undefined,
-    timeoutMs: number,
-  ): Promise<Tx[]> {
-    const {
-      batchTxRequesterSmartParallelWorkerCount: smartParallelWorkerCount,
-      batchTxRequesterDumbParallelWorkerCount: dumbParallelWorkerCount,
-      batchTxRequesterTxBatchSize: txBatchSize,
-      batchTxRequesterBadPeerThreshold: badPeerThreshold,
-    } = this.batchTxRequesterConfig ?? {};
-
-    const batchRequester = new BatchTxRequester(
-      missingTxsTracker,
-      blockTxsSource,
-      pinnedPeer,
-      timeoutMs,
-      this.p2pService,
-      this.log,
-      this.dateProvider,
-      {
-        smartParallelWorkerCount,
-        dumbParallelWorkerCount,
-        txBatchSize,
-        badPeerThreshold,
-        txValidator: this.txValidator,
-      },
-    );
-
-    return await BatchTxRequester.collectAllTxs(batchRequester.run());
-  }
-}
-
-const DEFAULT_MAX_PEERS = 10;
-const DEFAULT_MAX_RETRY_ATTEMPTS = 3;
-
-/**
- * Collects transactions using the sendBatchRequest implementation from ReqResp.
- * This is the original implementation that balances requests across peers.
- */
-export class SendBatchRequestCollector implements MissingTxsCollector {
-  constructor(
-    private p2pService: BatchTxRequesterLibP2PService,
-    private maxPeers: number = DEFAULT_MAX_PEERS,
-    private maxRetryAttempts: number = DEFAULT_MAX_RETRY_ATTEMPTS,
-  ) {}
-
-  async collectTxs(
-    missingTxsTracker: IMissingTxsTracker,
-    _blockTxsSource: BlockTxsSource,
-    pinnedPeer: PeerId | undefined,
-    timeoutMs: number,
-  ): Promise<Tx[]> {
-    const txs = await this.p2pService.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
-      ReqRespSubProtocol.TX,
-      chunkTxHashesRequest(Array.from(missingTxsTracker.missingTxHashes).map(TxHash.fromString)),
-      pinnedPeer,
-      timeoutMs,
-      this.maxPeers,
-      this.maxRetryAttempts,
-    );
-
-    return txs.flat();
-  }
-}