@aztec/p2p 0.0.1-commit.4d79d1f2d → 0.0.1-commit.4d9804df

package/dest/services/tx_collection/tx_collection.js

@@ -1,24 +1,21 @@
-import { compactArray } from '@aztec/foundation/collection';
+import { times } from '@aztec/foundation/collection';
 import { createLogger } from '@aztec/foundation/log';
-import { RunningPromise } from '@aztec/foundation/promise';
 import { sleep } from '@aztec/foundation/sleep';
-import { DateProvider } from '@aztec/foundation/timer';
+import { DateProvider, elapsed } from '@aztec/foundation/timer';
 import { TxHash } from '@aztec/stdlib/tx';
 import { getTelemetryClient } from '@aztec/telemetry-client';
-import { FastTxCollection } from './fast_tx_collection.js';
+import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js';
 import { FileStoreTxCollection } from './file_store_tx_collection.js';
-import { SlowTxCollection } from './slow_tx_collection.js';
+import { RequestTracker } from './request_tracker.js';
 import { TxCollectionSink } from './tx_collection_sink.js';
 /**
- * Coordinates tx collection from remote RPC nodes and reqresp.
+ * Coordinates tx collection from remote RPC nodes, reqresp, and file store.
  *
- * The slow collection loops are used for periodically collecting missing txs from mined blocks,
- * from remote RPC nodes and via reqresp. The fast collection methods are used to quickly gather
- * txs, usually for attesting to block proposals or preparing to prove an epoch. The slow and fast
- * collection instances both send their txs to the collection sink, which handles metrics and adds
- * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is notified
- * via events and notifies the slow and fast collection loops to stop collecting that tx, so that we don't
- * collect the same tx multiple times.
+ * Runs a sequential pipeline: node RPC → reqresp → file store. Node collection starts immediately,
+ * reqresp starts after a configured delay, and file store (if configured) starts after a further
+ * delay. All paths send txs to the collection sink, which handles metrics and adds them to the
+ * tx pool. Whenever a tx is added to the sink or the pool, this service is notified and stops
+ * collecting that tx across all in-flight requests.
  */ export class TxCollection {
     p2pService;
     nodes;
@@ -27,15 +24,13 @@ import { TxCollectionSink } from './tx_collection_sink.js';
     config;
     dateProvider;
     log;
-    /** Slow collection background loops */ slowCollection;
-    /** Fast collection methods */ fastCollection;
-    /** File store collection */ fileStoreCollection;
-    /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */ reconcileFoundTxsLoop;
-    /** Handles txs found by the slow and fast collection loops */ txCollectionSink;
+    // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+    requests;
+    /** The collector for txs via reqresp */ reqRespTxsCollector;
+    /** File store collection for the fast (proposal/proving) path */ fileStoreFastCollection;
+    /** Handles txs found by collection paths before adding to the pool */ txCollectionSink;
     /** Handler for the txs-added event from the tx pool */ handleTxsAddedToPool;
     /** Handler for the txs-added event from the tx collection sink */ handleTxsFound;
-    /** Whether the service has been started. */ started;
-    /** Whether file store sources are configured. */ hasFileStoreSources;
     constructor(p2pService, nodes, constants, txPool, config, fileStoreSources = [], dateProvider = new DateProvider(), telemetryClient = getTelemetryClient(), log = createLogger('p2p:tx_collection_service')){
         this.p2pService = p2pService;
         this.nodes = nodes;
@@ -44,13 +39,14 @@ import { TxCollectionSink } from './tx_collection_sink.js';
         this.config = config;
         this.dateProvider = dateProvider;
         this.log = log;
-        this.started = false;
+        this.requests = new Set();
         this.txCollectionSink = new TxCollectionSink(this.txPool, telemetryClient, this.log);
-        this.fastCollection = new FastTxCollection(this.p2pService, this.nodes, this.txCollectionSink, this.config, this.dateProvider, this.log);
-        this.slowCollection = new SlowTxCollection(this.p2pService.reqResp, this.nodes, this.txCollectionSink, this.fastCollection, constants, this.config, this.dateProvider, this.log);
-        this.hasFileStoreSources = fileStoreSources.length > 0;
-        this.fileStoreCollection = new FileStoreTxCollection(fileStoreSources, this.txCollectionSink, this.log);
-        this.reconcileFoundTxsLoop = new RunningPromise(()=>this.reconcileFoundTxsWithPool(), this.log, this.config.txCollectionReconcileIntervalMs);
+        this.reqRespTxsCollector = (requestTracker, blockTxsSource, pinnedPeer)=>BatchTxRequester.collectAllTxs(new BatchTxRequester(requestTracker, blockTxsSource, pinnedPeer, this.p2pService, this.log, this.dateProvider).run());
+        this.fileStoreFastCollection = new FileStoreTxCollection(fileStoreSources, this.txCollectionSink, {
+            workerCount: config.txCollectionFileStoreFastWorkerCount,
+            backoffBaseMs: config.txCollectionFileStoreFastBackoffBaseMs,
+            backoffMaxMs: config.txCollectionFileStoreFastBackoffMaxMs
+        }, this.dateProvider, this.log);
         this.handleTxsFound = (args)=>{
             this.foundTxs(args.txs);
         };
@@ -63,122 +59,298 @@ import { TxCollectionSink } from './tx_collection_sink.js';
         };
         this.txPool.on('txs-added', this.handleTxsAddedToPool);
     }
-    /** Starts all collection loops. */ start() {
-        this.started = true;
-        this.slowCollection.start();
-        this.fileStoreCollection.start();
-        this.reconcileFoundTxsLoop.start();
-        // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup
-        return Promise.resolve();
-    }
-    /** Stops all activity. */ async stop() {
-        this.started = false;
-        await Promise.all([
-            this.slowCollection.stop(),
-            this.fastCollection.stop(),
-            this.fileStoreCollection.stop(),
-            this.reconcileFoundTxsLoop.stop()
-        ]);
+    /** Stops all activity. Cancels in-flight requests; file store workers self-terminate. */ stop() {
+        this.requests.forEach((request)=>{
+            request.requestTracker.cancel();
+        });
         this.txPool.removeListener('txs-added', this.handleTxsAddedToPool);
         this.txCollectionSink.removeListener('txs-added', this.handleTxsFound);
     }
-    /** Force trigger the slow collection and reconciliation loops */ async trigger() {
-        await Promise.all([
-            this.reconcileFoundTxsLoop.trigger(),
-            this.slowCollection.trigger()
-        ]);
-    }
     /** Returns L1 rollup constants. */ getConstants() {
         return this.constants;
     }
-    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */ startCollecting(block, txHashes) {
-        this.slowCollection.startCollecting(block, txHashes);
-        // Delay file store collection to give P2P methods time to find txs first
-        if (this.hasFileStoreSources) {
-            const context = {
-                type: 'mined',
-                block
-            };
-            sleep(this.config.txCollectionFileStoreSlowDelayMs).then(()=>{
-                if (this.started) {
-                    // Only queue txs that are still missing after the delay
-                    const stillMissing = new Set(this.slowCollection.getMissingTxHashes().map((h)=>h.toString()));
-                    const remaining = txHashes.filter((h)=>stillMissing.has(h.toString()));
-                    if (remaining.length > 0) {
-                        this.fileStoreCollection.startCollecting(remaining, context);
-                    }
-                }
-            }).catch((err)=>this.log.error('Error in file store slow delay', err));
-        }
-    }
-    /** Collects the set of txs for the given block proposal as fast as possible */ collectFastForProposal(blockProposal, blockNumber, txHashes, opts) {
-        return this.collectFastFor({
-            type: 'proposal',
-            blockProposal,
-            blockNumber
-        }, txHashes, opts);
-    }
     /** Collects the set of txs for the given mined block as fast as possible */ collectFastForBlock(block, txHashes, opts) {
         return this.collectFastFor({
             type: 'block',
             block
         }, txHashes, opts);
     }
-    /** Collects the set of txs for the given proposal or block as fast as possible */ collectFastFor(input, txHashes, opts) {
+    /** Collects the set of txs for the given proposal or block as fast as possible */ async collectFastFor(input, txHashes, opts) {
+        const timeout = opts.deadline.getTime() - this.dateProvider.now();
+        if (timeout <= 0) {
+            this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, {
+                deadline: opts.deadline.getTime(),
+                now: this.dateProvider.now()
+            });
+            return [];
+        }
         const hashes = txHashes.map((h)=>typeof h === 'string' ? TxHash.fromString(h) : h);
-        // Delay file store collection to give P2P methods time to find txs first
-        if (this.hasFileStoreSources) {
-            const context = this.getAddContextForInput(input);
-            sleep(this.config.txCollectionFileStoreFastDelayMs).then(()=>{
-                if (this.started) {
-                    this.fileStoreCollection.startCollecting(hashes, context);
+        const blockInfo = input.type === 'proposal' ? {
+            ...input.blockProposal.toBlockInfo(),
+            blockNumber: input.blockNumber
+        } : {
+            ...input.block.toBlockInfo()
+        };
+        const request = {
+            ...input,
+            blockInfo,
+            requestTracker: RequestTracker.create(hashes, opts.deadline, this.dateProvider)
+        };
+        const [duration] = await elapsed(()=>this.collectFast(request, {
+                pinnedPeer: opts.pinnedPeer
+            }));
+        this.log.verbose(`Collected ${request.requestTracker.collectedTxs.length} txs out of ${hashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            duration,
+            requestType: input.type,
+            missingTxs: [
+                ...request.requestTracker.missingTxHashes
+            ]
+        });
+        return request.requestTracker.collectedTxs;
+    }
+    async collectFast(request, opts) {
+        this.requests.add(request);
+        const { blockInfo } = request;
+        this.log.debug(`Starting fast collection of ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            requestType: request.type,
+            deadline: request.requestTracker.deadline
+        });
+        try {
+            // 1. Start node collection in the background.
+            // Note: this will be a noop if no nodes are configured.
+            const nodeCollectionPromise = this.collectFastFromNodes(request);
+            // 2. Wait before starting reqresp, interruptible by cancellation or node exhaustion.
+            await Promise.race([
+                request.requestTracker.cancellationToken,
+                sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs),
+                nodeCollectionPromise
+            ]);
+            // 3. Start reqresp in the background (runs in parallel with node collection).
+            // Note: this will be a noop if all TXs were already found.
+            const reqRespPromise = this.collectFastViaReqResp(request, opts);
+            // 4. Wait before starting file store, interruptible by cancellation.
+            await Promise.race([
+                request.requestTracker.cancellationToken,
+                sleep(this.config.txCollectionFileStoreFastDelayMs),
+                reqRespPromise
+            ]);
+            // 5. Start file store collection in the background. Self-terminates on tracker cancel / all-found.
+            // Note: this will be a noop if all TXs were already found.
+            const fileStorePromise = this.fileStoreFastCollection.startCollecting(request.requestTracker, this.getAddContext(request));
+            // 6. Wait for all paths to settle.
+            // NOTE: The request will automatically be cancelled after `opt.deadline` is reached.
+            await Promise.allSettled([
+                reqRespPromise,
+                nodeCollectionPromise,
+                fileStorePromise
+            ]);
+        } catch (err) {
+            this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, {
+                ...blockInfo,
+                missingTxs: request.requestTracker.missingTxHashes.values().map((txHash)=>txHash.toString())
+            });
+        } finally{
+            request.requestTracker.cancel();
+            this.requests.delete(request);
+        }
+    }
+    /**
+   * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests
+   * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
+   * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
+   * tx is requested more than once at the same time to the same node.
+   */ async collectFastFromNodes(request) {
+        if (this.nodes.length === 0) {
+            return;
+        }
+        // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
+        const attemptsPerTx = [
+            ...request.requestTracker.missingTxHashes
+        ].map((txHash)=>({
+                txHash,
+                attempts: 0,
+                found: false
+            }));
+        // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
+        await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx)));
+    }
+    async collectFastFromNode(request, node, attemptsPerTx) {
+        const notFinished = ()=>!request.requestTracker.checkCancelled();
+        const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
+        const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
+        const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
+        const processBatch = async ()=>{
+            while(notFinished()){
+                // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts,
+                // so we prioritize txs that have been requested less often.
+                const batch = [];
+                let index = 0;
+                while(batch.length < maxBatchSize){
+                    const txToRequest = attemptsPerTx[index++];
+                    if (!txToRequest) {
+                        break;
+                    } else if (!request.requestTracker.isMissing(txToRequest.txHash)) {
+                        // Mark as found if it was found somewhere else, we'll then remove it from the array.
+                        // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
+                        txToRequest.found = true;
+                    } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) {
+                        // If the tx is not already being requested to this node, add it to the current batch and increase attempts.
+                        // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that
+                        // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting
+                        // the same txs at the same time.
+                        batch.push(txToRequest);
+                        activeRequestsToThisNode.add(txToRequest.txHash);
+                        txToRequest.attempts++;
+                    }
                 }
-            }).catch((err)=>this.log.error('Error in file store fast delay', err));
+                // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end.
+                attemptsPerTx.sort((a, b)=>a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found));
+                const firstFoundTxIndex = attemptsPerTx.findIndex((tx)=>tx.found);
+                if (firstFoundTxIndex !== -1) {
+                    attemptsPerTx.length = firstFoundTxIndex;
+                }
+                // If we see no more txs to request, we can stop this "process" loop
+                if (batch.length === 0) {
+                    return;
+                }
+                const txHashes = batch.map(({ txHash })=>txHash);
+                // Collect this batch from the node
+                await this.txCollectionSink.collect(async ()=>{
+                    const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
+                    for (const tx of result.validTxs){
+                        request.requestTracker.markFetched(tx);
+                    }
+                    return result;
+                }, txHashes, {
+                    description: `fast ${node.getInfo()}`,
+                    node: node.getInfo(),
+                    method: 'fast-node-rpc',
+                    ...request.blockInfo
+                }, this.getAddContext(request));
+                // Clear from the active requests the txs we just requested
+                for (const requestedTx of batch){
+                    activeRequestsToThisNode.delete(requestedTx.txHash);
+                }
+                // Sleep a bit until hitting the node again, but wake up immediately on cancellation
+                if (notFinished()) {
+                    await Promise.race([
+                        sleep(this.config.txCollectionFastNodeIntervalMs),
+                        request.requestTracker.cancellationToken
+                    ]);
+                }
+            }
+        };
+        // Kick off N parallel requests to the node, up to the maxParallelRequests limit
+        await Promise.all(times(maxParallelRequests, processBatch));
+    }
+    async collectFastViaReqResp(request, opts) {
+        const pinnedPeer = opts.pinnedPeer;
+        const blockInfo = request.blockInfo;
+        const slotNumber = blockInfo.slotNumber;
+        if (request.requestTracker.timeoutMs < 100) {
+            this.log.warn(`Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`, {
+                timeoutMs: request.requestTracker.timeoutMs,
+                ...blockInfo
+            });
+            return;
+        }
+        if (request.requestTracker.checkCancelled()) {
+            this.log.debug(`No txs to collect via reqresp for ${request.type} at slot ${blockInfo.slotNumber}`, {
+                ...blockInfo
+            });
+            return;
+        }
+        this.log.debug(`Starting fast reqresp for ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            timeoutMs: request.requestTracker.timeoutMs,
+            pinnedPeer
+        });
+        try {
+            await this.txCollectionSink.collect(async ()=>{
+                let blockTxsSource;
+                if (request.type === 'proposal') {
+                    blockTxsSource = request.blockProposal;
+                } else if (request.type === 'block') {
+                    blockTxsSource = {
+                        txHashes: request.block.body.txEffects.map((e)=>e.txHash),
+                        archive: request.block.archive.root
+                    };
+                } else {
+                    throw new Error(`Unknown request type: ${request.type}`);
+                }
+                const result = await this.reqRespTxsCollector(request.requestTracker, blockTxsSource, pinnedPeer);
+                return {
+                    validTxs: result,
+                    invalidTxHashes: []
+                };
+            }, Array.from(request.requestTracker.missingTxHashes), {
+                description: `reqresp for slot ${slotNumber}`,
+                method: 'fast-req-resp',
+                ...opts,
+                ...request.blockInfo
+            }, this.getAddContext(request));
+        } catch (err) {
+            this.log.error(`Error sending fast reqresp request for txs`, err, {
+                txs: [
+                    ...request.requestTracker.missingTxHashes
+                ],
+                ...blockInfo
+            });
         }
-        return this.fastCollection.collectFastFor(input, txHashes, opts);
     }
-    /** Returns the TxAddContext for the given fast collection request input */ getAddContextForInput(input) {
-        if (input.type === 'proposal') {
+    /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */ getAddContext(request) {
+        if (request.type === 'proposal') {
             return {
                 type: 'proposal',
-                blockHeader: input.blockProposal.blockHeader
+                blockHeader: request.blockProposal.blockHeader
             };
         } else {
             return {
                 type: 'mined',
-                block: input.block
+                block: request.block
             };
         }
     }
-    /** Mark the given txs as found. Stops collecting them. */ foundTxs(txs) {
-        this.slowCollection.foundTxs(txs);
-        this.fastCollection.foundTxs(txs);
-        this.fileStoreCollection.foundTxs(txs);
+    /** Mark the given txs as found. Stops collecting them across all in-flight requests. */ foundTxs(txs) {
+        for (const request of this.requests){
+            for (const tx of txs){
+                if (request.requestTracker.markFetched(tx)) {
+                    this.log.trace(`Found tx ${tx.txHash.toString()} for fast collection request`, {
+                        ...request.blockInfo,
+                        txHash: tx.txHash.toString(),
+                        type: request.type
+                    });
+                    if (request.requestTracker.allFetched()) {
+                        this.log.trace(`All txs found for fast collection request`, {
+                            ...request.blockInfo,
+                            type: request.type
+                        });
+                        break;
+                    }
+                }
+            }
+        }
     }
     /**
-   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * Stop collecting all txs for blocks less than or equal to the block number specified.
    * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
    */ stopCollectingForBlocksUpTo(blockNumber) {
-        this.slowCollection.stopCollectingForBlocksUpTo(blockNumber);
-        this.fastCollection.stopCollectingForBlocksUpTo(blockNumber);
-        this.fileStoreCollection.clearPending();
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber <= blockNumber) {
+                request.requestTracker.cancel();
+            }
+        }
     }
     /**
    * Stop collecting all txs for blocks greater than the block number specified.
    * To be called when there is a chain prune and previously mined txs are no longer relevant.
    */ stopCollectingForBlocksAfter(blockNumber) {
-        this.slowCollection.stopCollectingForBlocksAfter(blockNumber);
-        this.fastCollection.stopCollectingForBlocksAfter(blockNumber);
-        this.fileStoreCollection.clearPending();
-    }
-    /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */ async reconcileFoundTxsWithPool() {
-        const missingTxHashes = this.slowCollection.getMissingTxHashes();
-        const foundTxs = compactArray(await this.txPool.getTxsByHash(missingTxHashes));
-        if (foundTxs.length > 0) {
-            this.log.verbose(`Found ${foundTxs.length} txs in the pool during reconciliation`, {
-                foundTxs: foundTxs.map((t)=>t.getTxHash().toString())
-            });
-            this.foundTxs(foundTxs);
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber > blockNumber) {
+                request.requestTracker.cancel();
+            }
         }
     }
 }

package/dest/services/tx_collection/tx_collection_sink.d.ts

@@ -1,10 +1,11 @@
 import type { Logger } from '@aztec/foundation/log';
 import type { TypedEventEmitter } from '@aztec/foundation/types';
 import type { L2Block } from '@aztec/stdlib/block';
-import type { BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx';
+import type { BlockHeader, Tx } from '@aztec/stdlib/tx';
 import type { TelemetryClient } from '@aztec/telemetry-client';
-import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interfaces.js';
+import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/index.js';
 import type { CollectionMethod } from './tx_collection.js';
+import type { TxSourceCollectionResult } from './tx_source.js';
 /** Context determining how collected txs should be added to the pool. */
 export type TxAddContext = {
     type: 'proposal';
@@ -15,7 +16,7 @@ export type TxAddContext = {
 };
 declare const TxCollectionSink_base: new () => TypedEventEmitter<TxPoolV2Events>;
 /**
- * Executes collection requests from the fast and slow collection loops, and handles collected txs
+ * Executes collection requests from fast collection paths, and handles collected txs
  * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
  */
 export declare class TxCollectionSink extends TxCollectionSink_base {
@@ -23,12 +24,12 @@ export declare class TxCollectionSink extends TxCollectionSink_base {
     private readonly log;
     private readonly instrumentation;
     constructor(txPool: TxPoolV2, telemetryClient: TelemetryClient, log: Logger);
-    collect(collectValidTxsFn: (txHashes: TxHash[]) => Promise<(Tx | undefined)[]>, requested: TxHash[], info: Record<string, any> & {
+    collect(collectValidTxsFn: () => Promise<TxSourceCollectionResult>, requested: string[], info: Record<string, any> & {
         description: string;
         method: CollectionMethod;
     }, context: TxAddContext): Promise<{
         txs: Tx[];
-        requested: TxHash[];
+        requested: string[];
         duration: number;
     } | {
         txs: Tx[];
@@ -36,4 +37,4 @@ export declare class TxCollectionSink extends TxCollectionSink_base {
     private foundTxs;
 }
 export {};
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfY29sbGVjdGlvbl9zaW5rLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi90eF9jb2xsZWN0aW9uX3NpbmsudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFFcEQsT0FBTyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUNqRSxPQUFPLEtBQUssRUFBRSxPQUFPLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUNuRCxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLGVBQWUsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBSS9ELE9BQU8sS0FBSyxFQUFFLFFBQVEsRUFBRSxjQUFjLEVBQUUsTUFBTSwwQ0FBMEMsQ0FBQztBQUV6RixPQUFPLEtBQUssRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBRTNELHlFQUF5RTtBQUN6RSxNQUFNLE1BQU0sWUFBWSxHQUFHO0lBQUUsSUFBSSxFQUFFLFVBQVUsQ0FBQztJQUFDLFdBQVcsRUFBRSxXQUFXLENBQUE7Q0FBRSxHQUFHO0lBQUUsSUFBSSxFQUFFLE9BQU8sQ0FBQztJQUFDLEtBQUssRUFBRSxPQUFPLENBQUE7Q0FBRSxDQUFDOztBQUU5Rzs7O0dBR0c7QUFDSCxxQkFBYSxnQkFBaUIsU0FBUSxxQkFBNkQ7SUFJL0YsT0FBTyxDQUFDLFFBQVEsQ0FBQyxNQUFNO0lBRXZCLE9BQU8sQ0FBQyxRQUFRLENBQUMsR0FBRztJQUx0QixPQUFPLENBQUMsUUFBUSxDQUFDLGVBQWUsQ0FBOEI7SUFFOUQsWUFDbUIsTUFBTSxFQUFFLFFBQVEsRUFDakMsZUFBZSxFQUFFLGVBQWUsRUFDZixHQUFHLEVBQUUsTUFBTSxFQUk3QjtJQUVZLE9BQU8sQ0FDbEIsaUJBQWlCLEVBQUUsQ0FBQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEtBQUssT0FBTyxDQUFDLENBQUMsRUFBRSxHQUFHLFNBQVMsQ0FBQyxFQUFFLENBQUMsRUFDdEUsU0FBUyxFQUFFLE1BQU0sRUFBRSxFQUNuQixJQUFJLEVBQUUsTUFBTSxDQUFDLE1BQU0sRUFBRSxHQUFHLENBQUMsR0FBRztRQUFFLFdBQVcsRUFBRSxNQUFNLENBQUM7UUFBQyxNQUFNLEVBQUUsZ0JBQWdCLENBQUE7S0FBRSxFQUM3RSxPQUFPLEVBQUUsWUFBWTs7Ozs7O09BdUV0QjtZQUVhLFFBQVE7Q0E2QnZCIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfY29sbGVjdGlvbl9zaW5rLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi90eF9jb2xsZWN0aW9uX3NpbmsudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFFcEQsT0FBTyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUNqRSxPQUFPLEtBQUssRUFBRSxPQUFPLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUNuRCxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsRUFBRSxFQUFFLE1BQU0sa0JBQWtCLENBQUM7QUFDeEQsT0FBTyxLQUFLLEVBQUUsZUFBZSxFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFJL0QsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLGNBQWMsRUFBRSxNQUFNLDBCQUEwQixDQUFDO0FBRXpFLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDM0QsT0FBTyxLQUFLLEVBQUUsd0JBQXdCLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUUvRCx5RUFBeUU7QUFDekUsTUFBTSxNQUFNLFlBQVksR0FBRztJQUFFLElBQUksRUFBRSxVQUFVLENBQUM7SUFBQyxXQUFXLEVBQUUsV0FBVyxDQUFBO0NBQUUsR0FBRztJQUFFLElBQUksRUFBRSxPQUFPLENBQUM7SUFBQyxLQUFLLEVBQUUsT0FBTyxDQUFBO0NBQUUsQ0FBQzs7QUFFOUc7OztHQUdHO0FBQ0gscUJBQWEsZ0JBQWlCLFNBQVEscUJBQTZEO0lBSS9GLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQUV2QixPQUFPLENBQUMsUUFBUSxDQUFDLEdBQUc7SUFMdEIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxlQUFlLENBQThCO0lBRTlELFlBQ21CLE1BQU0sRUFBRSxRQUFRLEVBQ2pDLGVBQWUsRUFBRSxlQUFlLEVBQ2YsR0FBRyxFQUFFLE1BQU0sRUFJN0I7SUFFWSxPQUFPLENBQ2xCLGlCQUFpQixFQUFFLE1BQU0sT0FBTyxDQUFDLHdCQUF3QixDQUFDLEVBQzFELFNBQVMsRUFBRSxNQUFNLEVBQUUsRUFDbkIsSUFBSSxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsR0FBRyxDQUFDLEdBQUc7UUFBRSxXQUFXLEVBQUUsTUFBTSxDQUFDO1FBQUMsTUFBTSxFQUFFLGdCQUFnQixDQUFBO0tBQUUsRUFDN0UsT0FBTyxFQUFFLFlBQVk7Ozs7OztPQXdEdEI7WUFFYSxRQUFRO0NBNkJ2QiJ9

package/dest/services/tx_collection/tx_collection_sink.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tx_collection_sink.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_collection_sink.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEpD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAChE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAI/D,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,0CAA0C,CAAC;AAEzF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,yEAAyE;AACzE,MAAM,MAAM,YAAY,GAAG;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,WAAW,EAAE,WAAW,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;;AAE9G;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,qBAA6D;IAI/F,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,GAAG;IALtB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA8B;IAE9D,YACmB,MAAM,EAAE,QAAQ,EACjC,eAAe,EAAE,eAAe,EACf,GAAG,EAAE,MAAM,EAI7B;IAEY,OAAO,CAClB,iBAAiB,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,EACtE,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,gBAAgB,CAAA;KAAE,EAC7E,OAAO,EAAE,YAAY;;;;;;OAuEtB;YAEa,QAAQ;CA6BvB"}
+{"version":3,"file":"tx_collection_sink.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_collection_sink.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEpD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAI/D,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAEzE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAE/D,yEAAyE;AACzE,MAAM,MAAM,YAAY,GAAG;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,WAAW,EAAE,WAAW,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;;AAE9G;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,qBAA6D;IAI/F,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,GAAG;IALtB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA8B;IAE9D,YACmB,MAAM,EAAE,QAAQ,EACjC,eAAe,EAAE,eAAe,EACf,GAAG,EAAE,MAAM,EAI7B;IAEY,OAAO,CAClB,iBAAiB,EAAE,MAAM,OAAO,CAAC,wBAAwB,CAAC,EAC1D,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,gBAAgB,CAAA;KAAE,EAC7E,OAAO,EAAE,YAAY;;;;;;OAwDtB;YAEa,QAAQ;CA6BvB"}

package/dest/services/tx_collection/tx_collection_sink.js

@@ -2,7 +2,7 @@ import { elapsed } from '@aztec/foundation/timer';
 import EventEmitter from 'node:events';
 import { TxCollectionInstrumentation } from './instrumentation.js';
 /**
- * Executes collection requests from the fast and slow collection loops, and handles collected txs
+ * Executes collection requests from fast collection paths, and handles collected txs
  * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
  */ export class TxCollectionSink extends EventEmitter {
     txPool;
@@ -15,43 +15,34 @@ import { TxCollectionInstrumentation } from './instrumentation.js';
     async collect(collectValidTxsFn, requested, info, context) {
         this.log.trace(`Requesting ${requested.length} txs via ${info.description}`, {
             ...info,
-            requestedTxs: requested.map((t)=>t.toString())
+            requestedTxs: requested
         });
         // Execute collection function and measure the time taken, catching any errors.
-        const [duration, txs] = await elapsed(async ()=>{
+        const [duration, { validTxs, invalidTxHashes }] = await elapsed(async ()=>{
             try {
-                const response = await collectValidTxsFn(requested);
-                return response.filter((tx)=>tx !== undefined);
+                return await collectValidTxsFn();
             } catch (err) {
                 this.log.error(`Error collecting txs via ${info.description}`, err, {
                     ...info,
-                    requestedTxs: requested.map((hash)=>hash.toString())
+                    requestedTxs: requested
                 });
-                return [];
+                return {
+                    validTxs: [],
+                    invalidTxHashes: []
+                };
             }
         });
-        if (txs.length === 0) {
+        if (validTxs.length === 0 && invalidTxHashes.length === 0) {
             this.log.trace(`No txs found via ${info.description}`, {
                 ...info,
-                requestedTxs: requested.map((t)=>t.toString())
+                requestedTxs: requested
             });
             return {
-                txs,
+                txs: validTxs,
                 requested,
                 duration
             };
         }
-        // Validate tx hashes for all collected txs from external sources
-        const validTxs = [];
-        const invalidTxHashes = [];
-        await Promise.all(txs.map(async (tx)=>{
-            const isValid = await tx.validateTxHash();
-            if (isValid) {
-                validTxs.push(tx);
-            } else {
-                invalidTxHashes.push(tx.getTxHash().toString());
-            }
-        }));
         if (invalidTxHashes.length > 0) {
             this.log.warn(`Rejecting ${invalidTxHashes.length} txs with invalid hashes from ${info.description}`, {
                 ...info,
@@ -61,7 +52,7 @@ import { TxCollectionInstrumentation } from './instrumentation.js';
         if (validTxs.length === 0) {
             this.log.trace(`No valid txs found via ${info.description} after validation`, {
                 ...info,
-                requestedTxs: requested.map((t)=>t.toString()),
+                requestedTxs: requested,
                 invalidTxHashes
             });
             return {
@@ -74,7 +65,7 @@ import { TxCollectionInstrumentation } from './instrumentation.js';
             ...info,
             duration,
             txs: validTxs.map((t)=>t.getTxHash().toString()),
-            requestedTxs: requested.map((t)=>t.toString()),
+            requestedTxs: requested,
             rejectedCount: invalidTxHashes.length
         });
         return await this.foundTxs(validTxs, {
@@ -85,7 +76,7 @@ import { TxCollectionInstrumentation } from './instrumentation.js';
     async foundTxs(txs, info, context) {
         // Report metrics for the collection
         this.instrumentation.increaseTxsFor(info.method, txs.length, info.duration);
-        // Mark txs as found in the slow missing txs set and all fast requests
+        // Mark txs as found in all in-flight collection requests
         this.emit('txs-added', {
             txs
         });

package/dest/services/tx_collection/tx_source.d.ts

@@ -1,18 +1,24 @@
 import type { ChainConfig } from '@aztec/stdlib/config';
 import { type AztecNode } from '@aztec/stdlib/interfaces/client';
-import type { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { Tx, TxHash, TxValidator } from '@aztec/stdlib/tx';
 import { type ComponentsVersions } from '@aztec/stdlib/versioning';
+export type TxSourceCollectionResult = {
+    validTxs: Tx[];
+    invalidTxHashes: string[];
+};
 export interface TxSource {
     getInfo(): string;
-    getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+    getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
 }
 export declare class NodeRpcTxSource implements TxSource {
     private readonly client;
+    private readonly txValidator;
     private readonly info;
-    constructor(client: Pick<AztecNode, 'getTxsByHash'>, info: string);
-    static fromUrl(nodeUrl: string, versions: ComponentsVersions): NodeRpcTxSource;
+    constructor(client: Pick<AztecNode, 'getTxsByHash'>, txValidator: TxValidator, info: string);
+    static fromUrl(nodeUrl: string, txValidator: TxValidator, versions: ComponentsVersions): NodeRpcTxSource;
     getInfo(): string;
-    getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+    getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
+    private verifyTxs;
 }
-export declare function createNodeRpcTxSources(urls: string[], chainConfig: ChainConfig): NodeRpcTxSource[];
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfc291cmNlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi90eF9zb3VyY2UudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBRUEsT0FBTyxLQUFLLEVBQUUsV0FBVyxFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDeEQsT0FBTyxFQUFFLEtBQUssU0FBUyxFQUF5QixNQUFNLGlDQUFpQyxDQUFDO0FBQ3hGLE9BQU8sS0FBSyxFQUFFLEVBQUUsRUFBRSxNQUFNLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUNuRCxPQUFPLEVBQUUsS0FBSyxrQkFBa0IsRUFBbUMsTUFBTSwwQkFBMEIsQ0FBQztBQUdwRyxNQUFNLFdBQVcsUUFBUTtJQUN2QixPQUFPLElBQUksTUFBTSxDQUFDO0lBQ2xCLFlBQVksQ0FBQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsT0FBTyxDQUFDLENBQUMsRUFBRSxHQUFHLFNBQVMsQ0FBQyxFQUFFLENBQUMsQ0FBQztDQUMvRDtBQUVELHFCQUFhLGVBQWdCLFlBQVcsUUFBUTtJQUU1QyxPQUFPLENBQUMsUUFBUSxDQUFDLE1BQU07SUFDdkIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxJQUFJO0lBRnZCLFlBQ21CLE1BQU0sRUFBRSxJQUFJLENBQUMsU0FBUyxFQUFFLGNBQWMsQ0FBQyxFQUN2QyxJQUFJLEVBQUUsTUFBTSxFQUMzQjtJQUVKLE9BQWMsT0FBTyxDQUFDLE9BQU8sRUFBRSxNQUFNLEVBQUUsUUFBUSxFQUFFLGtCQUFrQixHQUFHLGVBQWUsQ0FHcEY7SUFFTSxPQUFPLFdBRWI7SUFFTSxZQUFZLENBQUMsUUFBUSxFQUFFLE1BQU0sRUFBRSxHQUFHLE9BQU8sQ0FBQyxDQUFDLEVBQUUsR0FBRyxTQUFTLENBQUMsRUFBRSxDQUFDLENBRW5FO0NBQ0Y7QUFFRCx3QkFBZ0Isc0JBQXNCLENBQUMsSUFBSSxFQUFFLE1BQU0sRUFBRSxFQUFFLFdBQVcsRUFBRSxXQUFXLHFCQUc5RSJ9
+export declare function createNodeRpcTxSources(urls: string[], txValidator: TxValidator, chainConfig: ChainConfig): NodeRpcTxSource[];
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfc291cmNlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi90eF9zb3VyY2UudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBRUEsT0FBTyxLQUFLLEVBQUUsV0FBVyxFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDeEQsT0FBTyxFQUFFLEtBQUssU0FBUyxFQUF5QixNQUFNLGlDQUFpQyxDQUFDO0FBQ3hGLE9BQU8sS0FBSyxFQUFFLEVBQUUsRUFBRSxNQUFNLEVBQUUsV0FBVyxFQUFFLE1BQU0sa0JBQWtCLENBQUM7QUFDaEUsT0FBTyxFQUFFLEtBQUssa0JBQWtCLEVBQW1DLE1BQU0sMEJBQTBCLENBQUM7QUFHcEcsTUFBTSxNQUFNLHdCQUF3QixHQUFHO0lBQUUsUUFBUSxFQUFFLEVBQUUsRUFBRSxDQUFDO0lBQUMsZUFBZSxFQUFFLE1BQU0sRUFBRSxDQUFBO0NBQUUsQ0FBQztBQUVyRixNQUFNLFdBQVcsUUFBUTtJQUN2QixPQUFPLElBQUksTUFBTSxDQUFDO0lBQ2xCLFlBQVksQ0FBQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsT0FBTyxDQUFDLHdCQUF3QixDQUFDLENBQUM7Q0FDckU7QUFFRCxxQkFBYSxlQUFnQixZQUFXLFFBQVE7SUFFNUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxNQUFNO0lBQ3ZCLE9BQU8sQ0FBQyxRQUFRLENBQUMsV0FBVztJQUM1QixPQUFPLENBQUMsUUFBUSxDQUFDLElBQUk7SUFIdkIsWUFDbUIsTUFBTSxFQUFFLElBQUksQ0FBQyxTQUFTLEVBQUUsY0FBYyxDQUFDLEVBQ3ZDLFdBQVcsRUFBRSxXQUFXLEVBQ3hCLElBQUksRUFBRSxNQUFNLEVBQzNCO0lBRUosT0FBYyxPQUFPLENBQUMsT0FBTyxFQUFFLE1BQU0sRUFBRSxXQUFXLEVBQUUsV0FBVyxFQUFFLFFBQVEsRUFBRSxrQkFBa0IsR0FBRyxlQUFlLENBRzlHO0lBRU0sT0FBTyxXQUViO0lBRVksWUFBWSxDQUFDLFFBQVEsRUFBRSxNQUFNLEVBQUUsR0FBRyxPQUFPLENBQUMsd0JBQXdCLENBQUMsQ0FHL0U7WUFFYSxTQUFTO0NBZ0J4QjtBQUVELHdCQUFnQixzQkFBc0IsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsV0FBVyxFQUFFLFdBQVcsRUFBRSxXQUFXLEVBQUUsV0FBVyxxQkFHeEcifQ==