@aztec/p2p 0.0.0-test.0 → 0.0.1-commit.023c3e5

package/src/services/tx_collection/proposal_tx_collector.ts

@@ -0,0 +1,114 @@
+import type { Logger } from '@aztec/foundation/log';
+import type { DateProvider } from '@aztec/foundation/timer';
+import type { BlockProposal } from '@aztec/stdlib/p2p';
+import type { Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { PeerId } from '@libp2p/interface';
+
+import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js';
+import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
+import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
+import type { IBatchRequestTxValidator } from '../reqresp/batch-tx-requester/tx_validator.js';
+import { ReqRespSubProtocol } from '../reqresp/interface.js';
+import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+
+/**
+ * Strategy interface for collecting transactions for block proposals.
+ * Allows swapping between different tx collection implementations for benchmarking.
+ */
+export interface ProposalTxCollector {
+  /**
+   * Collect transactions for a block proposal.
+   * @param txHashes - The transaction hashes to collect
+   * @param blockProposal - The block proposal containing the transactions
+   * @param pinnedPeer - Optional peer that sent the proposal (expected to have all txs)
+   * @param timeoutMs - Timeout in milliseconds
+   * @returns The collected transactions
+   */
+  collectTxs(
+    txHashes: TxHash[],
+    blockProposal: BlockProposal,
+    pinnedPeer: PeerId | undefined,
+    timeoutMs: number,
+  ): Promise<Tx[]>;
+}
+
+/**
+ * Collects transactions using the BatchTxRequester implementation.
+ * This uses a smart/dumb peer strategy with parallel workers.
+ */
+export class BatchTxRequesterCollector implements ProposalTxCollector {
+  constructor(
+    private p2pService: BatchTxRequesterLibP2PService,
+    private log: Logger,
+    private dateProvider: DateProvider,
+    private txValidator?: IBatchRequestTxValidator,
+    private batchTxRequesterConfig?: Partial<BatchTxRequesterConfig>,
+  ) {}
+
+  async collectTxs(
+    txHashes: TxHash[],
+    blockProposal: BlockProposal,
+    pinnedPeer: PeerId | undefined,
+    timeoutMs: number,
+  ): Promise<Tx[]> {
+    const {
+      batchTxRequesterSmartParallelWorkerCount: smartParallelWorkerCount,
+      batchTxRequesterDumbParallelWorkerCount: dumbParallelWorkerCount,
+      batchTxRequesterTxBatchSize: txBatchSize,
+      batchTxRequesterBadPeerThreshold: badPeerThreshold,
+    } = this.batchTxRequesterConfig ?? {};
+
+    const batchRequester = new BatchTxRequester(
+      txHashes,
+      blockProposal,
+      pinnedPeer,
+      timeoutMs,
+      this.p2pService,
+      this.log,
+      this.dateProvider,
+      {
+        smartParallelWorkerCount,
+        dumbParallelWorkerCount,
+        txBatchSize,
+        badPeerThreshold,
+        txValidator: this.txValidator,
+      },
+    );
+
+    return await BatchTxRequester.collectAllTxs(batchRequester.run());
+  }
+}
+
+const DEFAULT_MAX_PEERS = 10;
+const DEFAULT_MAX_RETRY_ATTEMPTS = 3;
+
+/**
+ * Collects transactions using the sendBatchRequest implementation from ReqResp.
+ * This is the original implementation that balances requests across peers.
+ */
+export class SendBatchRequestCollector implements ProposalTxCollector {
+  constructor(
+    private p2pService: BatchTxRequesterLibP2PService,
+    private maxPeers: number = DEFAULT_MAX_PEERS,
+    private maxRetryAttempts: number = DEFAULT_MAX_RETRY_ATTEMPTS,
+  ) {}
+
+  async collectTxs(
+    txHashes: TxHash[],
+    _blockProposal: BlockProposal,
+    pinnedPeer: PeerId | undefined,
+    timeoutMs: number,
+  ): Promise<Tx[]> {
+    const txs = await this.p2pService.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
+      ReqRespSubProtocol.TX,
+      chunkTxHashesRequest(txHashes),
+      pinnedPeer,
+      timeoutMs,
+      this.maxPeers,
+      this.maxRetryAttempts,
+    );
+
+    return txs.flat();
+  }
+}

package/src/services/tx_collection/slow_tx_collection.ts

@@ -0,0 +1,233 @@
+import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { chunk } from '@aztec/foundation/collection';
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { boundInclusive } from '@aztec/foundation/number';
+import { RunningPromise } from '@aztec/foundation/promise';
+import { DateProvider } from '@aztec/foundation/timer';
+import type { L2Block } from '@aztec/stdlib/block';
+import { type L1RollupConstants, getEpochAtSlot, getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
+import { type Tx, TxHash } from '@aztec/stdlib/tx';
+
+import { type ReqRespInterface, ReqRespSubProtocol } from '../reqresp/interface.js';
+import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+import type { TxCollectionConfig } from './config.js';
+import type { FastTxCollection } from './fast_tx_collection.js';
+import type { MissingTxInfo } from './tx_collection.js';
+import type { TxCollectionSink } from './tx_collection_sink.js';
+import type { TxSource } from './tx_source.js';
+
+export class SlowTxCollection {
+  /** Map from tx hash to missing tx info to collect via slow loop */
+  private missingTxs: Map<string, MissingTxInfo> = new Map();
+
+  /** One slow collection loop for each node tx source we have */
+  private nodesSlowCollectionLoops: RunningPromise[];
+
+  /** Slow collection loop for reqresp which collects txs from peers */
+  private reqrespSlowCollectionLoop: RunningPromise;
+
+  constructor(
+    private reqResp: Pick<ReqRespInterface, 'sendBatchRequest'>,
+    private nodes: TxSource[],
+    private txCollectionSink: TxCollectionSink,
+    private fastCollection: Pick<FastTxCollection, 'getFastCollectionRequests'>,
+    private constants: L1RollupConstants,
+    private config: TxCollectionConfig,
+    private dateProvider: DateProvider = new DateProvider(),
+    private log: Logger = createLogger('p2p:tx_collection_service'),
+  ) {
+    this.nodesSlowCollectionLoops = this.nodes.map(
+      node =>
+        new RunningPromise(
+          () => this.collectMissingTxsFromNode(node),
+          this.log,
+          this.config.txCollectionSlowNodesIntervalMs,
+        ),
+    );
+
+    this.reqrespSlowCollectionLoop = new RunningPromise(
+      () => this.collectMissingTxsViaReqResp(),
+      this.log,
+      this.config.txCollectionSlowReqRespIntervalMs,
+    );
+  }
+
+  public getMissingTxHashes() {
+    return Array.from(this.missingTxs.keys()).map(TxHash.fromString);
+  }
+
+  public start() {
+    this.nodesSlowCollectionLoops.forEach(loop => loop.start());
+    this.reqrespSlowCollectionLoop.start();
+  }
+
+  public async stop() {
+    await Promise.all([
+      this.reqrespSlowCollectionLoop.stop(),
+      ...this.nodesSlowCollectionLoops.map(loop => loop.stop()),
+    ]);
+  }
+
+  public async trigger() {
+    await Promise.all([
+      this.reqrespSlowCollectionLoop.trigger(),
+      ...this.nodesSlowCollectionLoops.map(loop => loop.trigger()),
+    ]);
+  }
+
+  /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
+  public startCollecting(block: L2Block, txHashes: TxHash[]) {
+    const slot = block.header.getSlot();
+    const deadline = this.getDeadlineForSlot(slot);
+    if (+deadline < this.dateProvider.now()) {
+      this.log.debug(`Skipping collection of txs for block ${block.number} at slot ${slot} as it is already expired`, {
+        blockNumber: block.number,
+        slot: slot.toString(),
+        txHashes: txHashes.map(txHash => txHash.toString()),
+        deadline: +deadline,
+        now: this.dateProvider.now(),
+      });
+    }
+
+    for (const txHash of txHashes) {
+      this.missingTxs.set(txHash.toString(), {
+        blockNumber: block.number,
+        deadline: this.getDeadlineForSlot(block.header.getSlot()),
+        readyForReqResp: this.nodes.length === 0, // If we have no nodes, we can start reqresp immediately
+      });
+    }
+  }
+
+  /** Entrypoint for the node slow collection loop */
+  private async collectMissingTxsFromNode(node: TxSource) {
+    // If we have any fast requests, we skip the slow collection for this node if configured to do so
+    const requests = this.fastCollection.getFastCollectionRequests();
+    if (this.config.txCollectionDisableSlowDuringFastRequests && requests.size > 0) {
+      this.log.trace(`Skipping node slow collection due to active fast requests`);
+      return;
+    }
+
+    // Gather all missing txs that are not in fast collection and request them from the node
+    const missingTxs = this.getMissingTxsForSlowCollection();
+    const missingTxHashes = missingTxs.map(([txHash]) => txHash).map(TxHash.fromString);
+    if (missingTxHashes.length === 0) {
+      return;
+    }
+
+    // Request in chunks to avoid hitting RPC limits
+    for (const batch of chunk(missingTxHashes, this.config.txCollectionNodeRpcMaxBatchSize)) {
+      await this.txCollectionSink.collect(txHashes => node.getTxsByHash(txHashes), batch, {
+        description: `node ${node.getInfo()}`,
+        node: node.getInfo(),
+        method: 'slow-node-rpc',
+      });
+    }
+
+    // Mark every tx that is still missing as ready for reqresp.
+    // Note that we can just mark all requested txs as ready for reqresp, without filtering out the ones
+    // we retrieved, since the ones already found will have been removed from `missingTxs`.
+    missingTxs.forEach(([_txHash, info]) => {
+      info.readyForReqResp = true;
+    });
+  }
+
+  /** Entrypoint for the reqresp slow collection loop */
+  private async collectMissingTxsViaReqResp() {
+    // If we have any fast requests, we skip the slow collection if configured to do so
+    const requests = this.fastCollection.getFastCollectionRequests();
+    if (this.config.txCollectionDisableSlowDuringFastRequests && requests.size > 0) {
+      this.log.trace(`Skipping reqresp slow collection due to active fast requests`);
+      return;
+    }
+
+    // Gather all missing txs that are not in fast collection and are ready for reqresp
+    // A tx is flagged as ready for reqresp if it has been requested from a node at least once
+    const missingTxs = this.getMissingTxsForSlowCollection({ onlyReqRespReady: true });
+    if (missingTxs.length === 0) {
+      return;
+    }
+
+    const pinnedPeer = undefined;
+    const timeoutMs = this.config.txCollectionSlowReqRespTimeoutMs;
+    const maxPeers = boundInclusive(Math.ceil(missingTxs.length / 3), 4, 16);
+    const maxRetryAttempts = 3;
+    // Send a batch request via reqresp for the missing txs
+    await this.txCollectionSink.collect(
+      async txHashes => {
+        const txs = await this.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
+          ReqRespSubProtocol.TX,
+          chunkTxHashesRequest(txHashes),
+          pinnedPeer,
+          timeoutMs,
+          maxPeers,
+          maxRetryAttempts,
+        );
+
+        return txs.flat();
+      },
+      missingTxs.map(([txHash]) => TxHash.fromString(txHash)),
+      { description: 'slow reqresp', timeoutMs, method: 'slow-req-resp' },
+    );
+  }
+
+  /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */
+  private getMissingTxsForSlowCollection(opts: { onlyReqRespReady?: boolean } = {}): [string, MissingTxInfo][] {
+    // Remove expired txs from missingTxs
+    const now = this.dateProvider.now();
+    const expiredTxs = Array.from(this.missingTxs.entries()).filter(([_, value]) => +value.deadline < now);
+    expiredTxs.forEach(([txHash]) => this.missingTxs.delete(txHash));
+
+    // Gather all txs that are marked for fast collection, we do not want to collect them via slow collection.
+    // There are some situations where a tx is in both slow and fast collection, for example when a prover node
+    // is fast-collecting missing txs for proving an epoch, and still has the tx in the slow collection loops
+    // from mined unproven blocks it has seen in the past.
+    const fastRequests = this.fastCollection.getFastCollectionRequests();
+    const fastCollectionTxs: Set<string> = new Set(
+      ...Array.from(fastRequests.values()).flatMap(r => r.missingTxHashes),
+    );
+
+    // Return all missing txs that are not in fastCollectionTxs and are ready for reqresp if requested
+    return Array.from(this.missingTxs.entries())
+      .filter(([txHash, _]) => !fastCollectionTxs.has(txHash))
+      .filter(([_, value]) => !opts.onlyReqRespReady || value.readyForReqResp);
+  }
+
+  /** Stop collecting the given txs since we have found them. Called whenever tx pool emits a tx-added event. */
+  public foundTxs(txs: Tx[]): void {
+    for (const txHash of txs.map(tx => tx.txHash)) {
+      this.missingTxs.delete(txHash.toString());
+    }
+  }
+
+  /**
+   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+   */
+  public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void {
+    for (const [txHash, info] of this.missingTxs.entries()) {
+      if (info.blockNumber <= blockNumber) {
+        this.missingTxs.delete(txHash);
+      }
+    }
+  }
+
+  /**
+   * Stop collecting all txs for blocks greater than the block number specified.
+   * To be called when there is a chain prune and previously mined txs are no longer relevant.
+   */
+  public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void {
+    for (const [txHash, info] of this.missingTxs.entries()) {
+      if (info.blockNumber > blockNumber) {
+        this.missingTxs.delete(txHash);
+      }
+    }
+  }
+
+  /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */
+  private getDeadlineForSlot(slotNumber: SlotNumber): Date {
+    const epoch = getEpochAtSlot(slotNumber, this.constants);
+    const submissionEndEpoch = EpochNumber(epoch + this.constants.proofSubmissionEpochs);
+    const submissionEndTimestamp = getTimestampRangeForEpoch(submissionEndEpoch, this.constants)[1];
+    return new Date(Number(submissionEndTimestamp) * 1000);
+  }
+}

package/src/services/tx_collection/tx_collection.ts

@@ -0,0 +1,216 @@
+import { BlockNumber } from '@aztec/foundation/branded-types';
+import { compactArray } from '@aztec/foundation/collection';
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type PromiseWithResolvers, RunningPromise } from '@aztec/foundation/promise';
+import { DateProvider } from '@aztec/foundation/timer';
+import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
+import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
+import type { BlockProposal } from '@aztec/stdlib/p2p';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
+
+import type { PeerId } from '@libp2p/interface';
+
+import type { TxPool } from '../../mem_pools/index.js';
+import type { TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
+import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
+import type { TxCollectionConfig } from './config.js';
+import { FastTxCollection } from './fast_tx_collection.js';
+import { SlowTxCollection } from './slow_tx_collection.js';
+import { TxCollectionSink } from './tx_collection_sink.js';
+import type { TxSource } from './tx_source.js';
+
+export type CollectionMethod = 'fast-req-resp' | 'fast-node-rpc' | 'slow-req-resp' | 'slow-node-rpc';
+
+export type MissingTxInfo = { blockNumber: BlockNumber; deadline: Date; readyForReqResp: boolean };
+
+export type FastCollectionRequestInput =
+  | { type: 'block'; block: L2Block }
+  | { type: 'proposal'; blockProposal: BlockProposal; blockNumber: BlockNumber };
+
+export type FastCollectionRequest = FastCollectionRequestInput & {
+  missingTxHashes: Set<string>;
+  deadline: Date;
+  blockInfo: L2BlockInfo;
+  promise: PromiseWithResolvers<void>;
+  foundTxs: Map<string, Tx>;
+};
+
+/**
+ * Coordinates tx collection from remote RPC nodes and reqresp.
+ *
+ * The slow collection loops are used for periodically collecting missing txs from mined blocks,
+ * from remote RPC nodes and via reqresp. The fast collection methods are used to quickly gather
+ * txs, usually for attesting to block proposals or preparing to prove an epoch. The slow and fast
+ * collection instances both send their txs to the collection sink, which handles metrics and adds
+ * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is notified
+ * via events and notifies the slow and fast collection loops to stop collecting that tx, so that we don't
+ * collect the same tx multiple times.
+ */
+export class TxCollection {
+  /** Slow collection background loops */
+  protected readonly slowCollection: SlowTxCollection;
+
+  /** Fast collection methods */
+  protected readonly fastCollection: FastTxCollection;
+
+  /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */
+  private readonly reconcileFoundTxsLoop: RunningPromise;
+
+  /** Handles txs found by the slow and fast collection loops */
+  private readonly txCollectionSink: TxCollectionSink;
+
+  /** Handler for the txs-added event from the tx pool */
+  protected readonly handleTxsAddedToPool: TxPoolEvents['txs-added'];
+
+  /** Handler for the txs-added event from the tx collection sink */
+  protected readonly handleTxsFound: TxPoolEvents['txs-added'];
+
+  constructor(
+    private readonly p2pService: BatchTxRequesterLibP2PService,
+    private readonly nodes: TxSource[],
+    private readonly constants: L1RollupConstants,
+    private readonly txPool: TxPool,
+    private readonly config: TxCollectionConfig,
+    private readonly dateProvider: DateProvider = new DateProvider(),
+    telemetryClient: TelemetryClient = getTelemetryClient(),
+    private readonly log: Logger = createLogger('p2p:tx_collection_service'),
+  ) {
+    this.txCollectionSink = new TxCollectionSink(this.txPool, telemetryClient, this.log);
+
+    this.fastCollection = new FastTxCollection(
+      this.p2pService,
+      this.nodes,
+      this.txCollectionSink,
+      this.config,
+      this.dateProvider,
+      this.log,
+    );
+
+    this.slowCollection = new SlowTxCollection(
+      this.p2pService.reqResp,
+      this.nodes,
+      this.txCollectionSink,
+      this.fastCollection,
+      constants,
+      this.config,
+      this.dateProvider,
+      this.log,
+    );
+
+    this.reconcileFoundTxsLoop = new RunningPromise(
+      () => this.reconcileFoundTxsWithPool(),
+      this.log,
+      this.config.txCollectionReconcileIntervalMs,
+    );
+
+    this.handleTxsFound = (args: Parameters<TxPoolEvents['txs-added']>[0]) => {
+      this.foundTxs(args.txs);
+    };
+    this.txCollectionSink.on('txs-added', this.handleTxsFound);
+
+    this.handleTxsAddedToPool = (args: Parameters<TxPoolEvents['txs-added']>[0]) => {
+      const { txs, source } = args;
+      if (source !== 'tx-collection') {
+        this.foundTxs(txs);
+      }
+    };
+    this.txPool.on('txs-added', this.handleTxsAddedToPool);
+  }
+
+  /** Starts all collection loops. */
+  public start(): Promise<void> {
+    this.slowCollection.start();
+    this.reconcileFoundTxsLoop.start();
+
+    // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup
+    return Promise.resolve();
+  }
+
+  /** Stops all activity. */
+  public async stop() {
+    await Promise.all([this.slowCollection.stop(), this.fastCollection.stop(), this.reconcileFoundTxsLoop.stop()]);
+
+    this.txPool.removeListener('txs-added', this.handleTxsAddedToPool);
+    this.txCollectionSink.removeListener('txs-added', this.handleTxsFound);
+  }
+
+  /** Force trigger the slow collection and reconciliation loops */
+  public async trigger() {
+    await Promise.all([this.reconcileFoundTxsLoop.trigger(), this.slowCollection.trigger()]);
+  }
+
+  /** Returns L1 rollup constants. */
+  public getConstants(): L1RollupConstants {
+    return this.constants;
+  }
+
+  /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
+  public startCollecting(block: L2Block, txHashes: TxHash[]) {
+    return this.slowCollection.startCollecting(block, txHashes);
+  }
+
+  /** Collects the set of txs for the given block proposal as fast as possible */
+  public collectFastForProposal(
+    blockProposal: BlockProposal,
+    blockNumber: BlockNumber,
+    txHashes: TxHash[] | string[],
+    opts: { deadline: Date; pinnedPeer?: PeerId },
+  ) {
+    return this.collectFastFor({ type: 'proposal', blockProposal, blockNumber }, txHashes, opts);
+  }
+
+  /** Collects the set of txs for the given mined block as fast as possible */
+  public collectFastForBlock(
+    block: L2Block,
+    txHashes: TxHash[] | string[],
+    opts: { deadline: Date; pinnedPeer?: PeerId },
+  ) {
+    return this.collectFastFor({ type: 'block', block }, txHashes, opts);
+  }
+
+  /** Collects the set of txs for the given proposal or block as fast as possible */
+  public collectFastFor(
+    input: FastCollectionRequestInput,
+    txHashes: TxHash[] | string[],
+    opts: { deadline: Date; pinnedPeer?: PeerId },
+  ) {
+    return this.fastCollection.collectFastFor(input, txHashes, opts);
+  }
+
+  /** Mark the given txs as found. Stops collecting them. */
+  private foundTxs(txs: Tx[]) {
+    this.slowCollection.foundTxs(txs);
+    this.fastCollection.foundTxs(txs);
+  }
+
+  /**
+   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+   */
+  public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void {
+    this.slowCollection.stopCollectingForBlocksUpTo(blockNumber);
+    this.fastCollection.stopCollectingForBlocksUpTo(blockNumber);
+  }
+
+  /**
+   * Stop collecting all txs for blocks greater than the block number specified.
+   * To be called when there is a chain prune and previously mined txs are no longer relevant.
+   */
+  public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void {
+    this.slowCollection.stopCollectingForBlocksAfter(blockNumber);
+    this.fastCollection.stopCollectingForBlocksAfter(blockNumber);
+  }
+
+  /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */
+  private async reconcileFoundTxsWithPool() {
+    const missingTxHashes = this.slowCollection.getMissingTxHashes();
+    const foundTxs = compactArray(await this.txPool.getTxsByHash(missingTxHashes));
+    if (foundTxs.length > 0) {
+      this.log.verbose(`Found ${foundTxs.length} txs in the pool during reconciliation`, {
+        foundTxs: foundTxs.map(t => t.getTxHash().toString()),
+      });
+      this.foundTxs(foundTxs);
+    }
+  }
+}

package/src/services/tx_collection/tx_collection_sink.ts

@@ -0,0 +1,129 @@
+import type { Logger } from '@aztec/foundation/log';
+import { elapsed } from '@aztec/foundation/timer';
+import type { TypedEventEmitter } from '@aztec/foundation/types';
+import { Tx, type TxHash } from '@aztec/stdlib/tx';
+import type { TelemetryClient } from '@aztec/telemetry-client';
+
+import EventEmitter from 'node:events';
+
+import type { TxPool, TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
+import { TxCollectionInstrumentation } from './instrumentation.js';
+import type { CollectionMethod } from './tx_collection.js';
+
+/**
+ * Executes collection requests from the fast and slow collection loops, and handles collected txs
+ * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
+ */
+export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitter<TxPoolEvents>) {
+  private readonly instrumentation: TxCollectionInstrumentation;
+
+  constructor(
+    private readonly txPool: TxPool,
+    telemetryClient: TelemetryClient,
+    private readonly log: Logger,
+  ) {
+    super();
+    this.instrumentation = new TxCollectionInstrumentation(telemetryClient, 'TxCollection');
+  }
+
+  public async collect(
+    collectValidTxsFn: (txHashes: TxHash[]) => Promise<(Tx | undefined)[]>,
+    requested: TxHash[],
+    info: Record<string, any> & { description: string; method: CollectionMethod },
+  ) {
+    this.log.trace(`Requesting ${requested.length} txs via ${info.description}`, {
+      ...info,
+      requestedTxs: requested.map(t => t.toString()),
+    });
+
+    // Execute collection function and measure the time taken, catching any errors.
+    const [duration, txs] = await elapsed(async () => {
+      try {
+        const response = await collectValidTxsFn(requested);
+        return response.filter(tx => tx !== undefined);
+      } catch (err) {
+        this.log.error(`Error collecting txs via ${info.description}`, err, {
+          ...info,
+          requestedTxs: requested.map(hash => hash.toString()),
+        });
+        return [] as Tx[];
+      }
+    });
+
+    if (txs.length === 0) {
+      this.log.trace(`No txs found via ${info.description}`, {
+        ...info,
+        requestedTxs: requested.map(t => t.toString()),
+      });
+      return { txs, requested, duration };
+    }
+
+    // Validate tx hashes for all collected txs from external sources
+    const validTxs: Tx[] = [];
+    const invalidTxHashes: string[] = [];
+    await Promise.all(
+      txs.map(async tx => {
+        const isValid = await tx.validateTxHash();
+        if (isValid) {
+          validTxs.push(tx);
+        } else {
+          invalidTxHashes.push(tx.getTxHash().toString());
+        }
+      }),
+    );
+
+    if (invalidTxHashes.length > 0) {
+      this.log.warn(`Rejecting ${invalidTxHashes.length} txs with invalid hashes from ${info.description}`, {
+        ...info,
+        invalidTxHashes,
+      });
+    }
+
+    if (validTxs.length === 0) {
+      this.log.trace(`No valid txs found via ${info.description} after validation`, {
+        ...info,
+        requestedTxs: requested.map(t => t.toString()),
+        invalidTxHashes,
+      });
+      return { txs: [], requested, duration };
+    }
+
+    this.log.verbose(
+      `Collected ${validTxs.length} txs out of ${requested.length} requested via ${info.description} in ${duration}ms`,
+      {
+        ...info,
+        duration,
+        txs: validTxs.map(t => t.getTxHash().toString()),
+        requestedTxs: requested.map(t => t.toString()),
+        rejectedCount: invalidTxHashes.length,
+      },
+    );
+
+    return await this.foundTxs(validTxs, { ...info, duration });
+  }
+
+  private async foundTxs(
+    txs: Tx[],
+    info: Record<string, any> & { description: string; method: CollectionMethod; duration: number },
+  ) {
+    // Report metrics for the collection
+    this.instrumentation.increaseTxsFor(info.method, txs.length, info.duration);
+
+    // Mark txs as found in the slow missing txs set and all fast requests
+    this.emit('txs-added', { txs });
+
+    // Add the txs to the tx pool (should not fail, but we catch it just in case)
+    try {
+      await this.txPool.addTxs(txs, { source: `tx-collection` });
+    } catch (err) {
+      this.log.error(`Error adding txs to the pool via ${info.description}`, err, {
+        ...info,
+        txs: txs.map(tx => tx.txHash.toString()),
+      });
+      // Return no txs since none have been added
+      return { txs: [] };
+    }
+
+    return { txs };
+  }
+}