@aztec-labs/p2p 6.0.0-nightly.20260829

package/dest/services/tx_collection/tx_collection.js

@@ -0,0 +1,356 @@
+import { times } from '@aztec-labs/foundation/collection';
+import { createLogger } from '@aztec-labs/foundation/log';
+import { sleep } from '@aztec-labs/foundation/sleep';
+import { DateProvider, elapsed } from '@aztec-labs/foundation/timer';
+import { TxHash } from '@aztec-labs/stdlib/tx';
+import { getTelemetryClient } from '@aztec-labs/telemetry-client';
+import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js';
+import { FileStoreTxCollection } from './file_store_tx_collection.js';
+import { RequestTracker } from './request_tracker.js';
+import { TxCollectionSink } from './tx_collection_sink.js';
+/**
+ * Coordinates tx collection from remote RPC nodes, reqresp, and file store.
+ *
+ * Runs a sequential pipeline: node RPC → reqresp → file store. Node collection starts immediately,
+ * reqresp starts after a configured delay, and file store (if configured) starts after a further
+ * delay. All paths send txs to the collection sink, which handles metrics and adds them to the
+ * tx pool. Whenever a tx is added to the sink or the pool, this service is notified and stops
+ * collecting that tx across all in-flight requests.
+ */ export class TxCollection {
+    p2pService;
+    nodes;
+    constants;
+    txPool;
+    config;
+    dateProvider;
+    log;
+    // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+    requests;
+    /** The collector for txs via reqresp */ reqRespTxsCollector;
+    /** File store collection for the fast (proposal/proving) path */ fileStoreFastCollection;
+    /** Handles txs found by collection paths before adding to the pool */ txCollectionSink;
+    /** Handler for the txs-added event from the tx pool */ handleTxsAddedToPool;
+    /** Handler for the txs-added event from the tx collection sink */ handleTxsFound;
+    constructor(p2pService, nodes, constants, txPool, config, fileStoreSources = [], dateProvider = new DateProvider(), telemetryClient = getTelemetryClient(), log = createLogger('p2p:tx_collection_service')){
+        this.p2pService = p2pService;
+        this.nodes = nodes;
+        this.constants = constants;
+        this.txPool = txPool;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+        this.requests = new Set();
+        this.txCollectionSink = new TxCollectionSink(this.txPool, telemetryClient, this.log);
+        this.reqRespTxsCollector = (requestTracker, blockTxsSource, pinnedPeer)=>BatchTxRequester.collectAllTxs(new BatchTxRequester(requestTracker, blockTxsSource, pinnedPeer, this.p2pService, this.log, this.dateProvider).run());
+        this.fileStoreFastCollection = new FileStoreTxCollection(fileStoreSources, this.txCollectionSink, {
+            workerCount: config.txCollectionFileStoreFastWorkerCount,
+            backoffBaseMs: config.txCollectionFileStoreFastBackoffBaseMs,
+            backoffMaxMs: config.txCollectionFileStoreFastBackoffMaxMs
+        }, this.dateProvider, this.log);
+        this.handleTxsFound = (args)=>{
+            this.foundTxs(args.txs);
+        };
+        this.txCollectionSink.on('txs-added', this.handleTxsFound);
+        this.handleTxsAddedToPool = (args)=>{
+            const { txs, source } = args;
+            if (source !== 'tx-collection') {
+                this.foundTxs(txs);
+            }
+        };
+        this.txPool.on('txs-added', this.handleTxsAddedToPool);
+    }
+    /** Stops all activity. Cancels in-flight requests; file store workers self-terminate. */ stop() {
+        this.requests.forEach((request)=>{
+            request.requestTracker.cancel();
+        });
+        this.txPool.removeListener('txs-added', this.handleTxsAddedToPool);
+        this.txCollectionSink.removeListener('txs-added', this.handleTxsFound);
+    }
+    /** Returns L1 rollup constants. */ getConstants() {
+        return this.constants;
+    }
+    /** Collects the set of txs for the given mined block as fast as possible */ collectFastForBlock(block, txHashes, opts) {
+        return this.collectFastFor({
+            type: 'block',
+            block
+        }, txHashes, opts);
+    }
+    /** Collects the set of txs for the given proposal or block as fast as possible */ async collectFastFor(input, txHashes, opts) {
+        const timeout = opts.deadline.getTime() - this.dateProvider.now();
+        if (timeout <= 0) {
+            this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, {
+                deadline: opts.deadline.getTime(),
+                now: this.dateProvider.now()
+            });
+            return [];
+        }
+        const hashes = txHashes.map((h)=>typeof h === 'string' ? TxHash.fromString(h) : h);
+        const blockInfo = input.type === 'proposal' ? {
+            ...input.blockProposal.toBlockInfo(),
+            blockNumber: input.blockNumber
+        } : {
+            ...input.block.toBlockInfo()
+        };
+        const request = {
+            ...input,
+            blockInfo,
+            requestTracker: RequestTracker.create(hashes, opts.deadline, this.dateProvider)
+        };
+        const [duration] = await elapsed(()=>this.collectFast(request, {
+                pinnedPeer: opts.pinnedPeer
+            }));
+        this.log.verbose(`Collected ${request.requestTracker.collectedTxs.length} txs out of ${hashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            duration,
+            requestType: input.type,
+            missingTxs: [
+                ...request.requestTracker.missingTxHashes
+            ]
+        });
+        return request.requestTracker.collectedTxs;
+    }
+    async collectFast(request, opts) {
+        this.requests.add(request);
+        const { blockInfo } = request;
+        this.log.debug(`Starting fast collection of ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            requestType: request.type,
+            deadline: request.requestTracker.deadline
+        });
+        try {
+            // 1. Start node collection in the background.
+            // Note: this will be a noop if no nodes are configured.
+            const nodeCollectionPromise = this.collectFastFromNodes(request);
+            // 2. Wait before starting reqresp, interruptible by cancellation or node exhaustion.
+            await Promise.race([
+                request.requestTracker.cancellationToken,
+                sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs),
+                nodeCollectionPromise
+            ]);
+            // 3. Start reqresp in the background (runs in parallel with node collection).
+            // Note: this will be a noop if all TXs were already found.
+            const reqRespPromise = this.collectFastViaReqResp(request, opts);
+            // 4. Wait before starting file store, interruptible by cancellation.
+            await Promise.race([
+                request.requestTracker.cancellationToken,
+                sleep(this.config.txCollectionFileStoreFastDelayMs),
+                reqRespPromise
+            ]);
+            // 5. Start file store collection in the background. Self-terminates on tracker cancel / all-found.
+            // Note: this will be a noop if all TXs were already found.
+            const fileStorePromise = this.fileStoreFastCollection.startCollecting(request.requestTracker, this.getAddContext(request));
+            // 6. Wait for all paths to settle.
+            // NOTE: The request will automatically be cancelled after `opt.deadline` is reached.
+            await Promise.allSettled([
+                reqRespPromise,
+                nodeCollectionPromise,
+                fileStorePromise
+            ]);
+        } catch (err) {
+            this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, {
+                ...blockInfo,
+                missingTxs: request.requestTracker.missingTxHashes.values().map((txHash)=>txHash.toString())
+            });
+        } finally{
+            request.requestTracker.cancel();
+            this.requests.delete(request);
+        }
+    }
+    /**
+   * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests
+   * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
+   * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
+   * tx is requested more than once at the same time to the same node.
+   */ async collectFastFromNodes(request) {
+        if (this.nodes.length === 0) {
+            return;
+        }
+        // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
+        const attemptsPerTx = [
+            ...request.requestTracker.missingTxHashes
+        ].map((txHash)=>({
+                txHash,
+                attempts: 0,
+                found: false
+            }));
+        // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
+        await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx)));
+    }
+    async collectFastFromNode(request, node, attemptsPerTx) {
+        const notFinished = ()=>!request.requestTracker.checkCancelled();
+        const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
+        const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
+        const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
+        const processBatch = async ()=>{
+            while(notFinished()){
+                // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts,
+                // so we prioritize txs that have been requested less often.
+                const batch = [];
+                let index = 0;
+                while(batch.length < maxBatchSize){
+                    const txToRequest = attemptsPerTx[index++];
+                    if (!txToRequest) {
+                        break;
+                    } else if (!request.requestTracker.isMissing(txToRequest.txHash)) {
+                        // Mark as found if it was found somewhere else, we'll then remove it from the array.
+                        // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
+                        txToRequest.found = true;
+                    } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) {
+                        // If the tx is not already being requested to this node, add it to the current batch and increase attempts.
+                        // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that
+                        // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting
+                        // the same txs at the same time.
+                        batch.push(txToRequest);
+                        activeRequestsToThisNode.add(txToRequest.txHash);
+                        txToRequest.attempts++;
+                    }
+                }
+                // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end.
+                attemptsPerTx.sort((a, b)=>a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found));
+                const firstFoundTxIndex = attemptsPerTx.findIndex((tx)=>tx.found);
+                if (firstFoundTxIndex !== -1) {
+                    attemptsPerTx.length = firstFoundTxIndex;
+                }
+                // If we see no more txs to request, we can stop this "process" loop
+                if (batch.length === 0) {
+                    return;
+                }
+                const txHashes = batch.map(({ txHash })=>txHash);
+                // Collect this batch from the node
+                await this.txCollectionSink.collect(async ()=>{
+                    const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
+                    for (const tx of result.validTxs){
+                        request.requestTracker.markFetched(tx);
+                    }
+                    return result;
+                }, txHashes, {
+                    description: `fast ${node.getInfo()}`,
+                    node: node.getInfo(),
+                    method: 'fast-node-rpc',
+                    ...request.blockInfo
+                }, this.getAddContext(request));
+                // Clear from the active requests the txs we just requested
+                for (const requestedTx of batch){
+                    activeRequestsToThisNode.delete(requestedTx.txHash);
+                }
+                // Sleep a bit until hitting the node again, but wake up immediately on cancellation
+                if (notFinished()) {
+                    await Promise.race([
+                        sleep(this.config.txCollectionFastNodeIntervalMs),
+                        request.requestTracker.cancellationToken
+                    ]);
+                }
+            }
+        };
+        // Kick off N parallel requests to the node, up to the maxParallelRequests limit
+        await Promise.all(times(maxParallelRequests, processBatch));
+    }
+    async collectFastViaReqResp(request, opts) {
+        const pinnedPeer = opts.pinnedPeer;
+        const blockInfo = request.blockInfo;
+        const slotNumber = blockInfo.slotNumber;
+        if (request.requestTracker.timeoutMs < 100) {
+            this.log.warn(`Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`, {
+                timeoutMs: request.requestTracker.timeoutMs,
+                ...blockInfo
+            });
+            return;
+        }
+        if (request.requestTracker.checkCancelled()) {
+            this.log.debug(`No txs to collect via reqresp for ${request.type} at slot ${blockInfo.slotNumber}`, {
+                ...blockInfo
+            });
+            return;
+        }
+        this.log.debug(`Starting fast reqresp for ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            timeoutMs: request.requestTracker.timeoutMs,
+            pinnedPeer
+        });
+        try {
+            await this.txCollectionSink.collect(async ()=>{
+                let blockTxsSource;
+                if (request.type === 'proposal') {
+                    blockTxsSource = request.blockProposal;
+                } else if (request.type === 'block') {
+                    blockTxsSource = {
+                        txHashes: request.block.body.txEffects.map((e)=>e.txHash),
+                        archive: request.block.archive.root
+                    };
+                } else {
+                    throw new Error(`Unknown request type: ${request.type}`);
+                }
+                const result = await this.reqRespTxsCollector(request.requestTracker, blockTxsSource, pinnedPeer);
+                return {
+                    validTxs: result,
+                    invalidTxHashes: []
+                };
+            }, Array.from(request.requestTracker.missingTxHashes), {
+                description: `reqresp for slot ${slotNumber}`,
+                method: 'fast-req-resp',
+                ...opts,
+                ...request.blockInfo
+            }, this.getAddContext(request));
+        } catch (err) {
+            this.log.error(`Error sending fast reqresp request for txs`, err, {
+                txs: [
+                    ...request.requestTracker.missingTxHashes
+                ],
+                ...blockInfo
+            });
+        }
+    }
+    /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */ getAddContext(request) {
+        if (request.type === 'proposal') {
+            return {
+                type: 'proposal',
+                blockHeader: request.blockProposal.blockHeader
+            };
+        } else {
+            return {
+                type: 'mined',
+                block: request.block
+            };
+        }
+    }
+    /** Mark the given txs as found. Stops collecting them across all in-flight requests. */ foundTxs(txs) {
+        for (const request of this.requests){
+            for (const tx of txs){
+                if (request.requestTracker.markFetched(tx)) {
+                    this.log.trace(`Found tx ${tx.txHash.toString()} for fast collection request`, {
+                        ...request.blockInfo,
+                        txHash: tx.txHash.toString(),
+                        type: request.type
+                    });
+                    if (request.requestTracker.allFetched()) {
+                        this.log.trace(`All txs found for fast collection request`, {
+                            ...request.blockInfo,
+                            type: request.type
+                        });
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks less than or equal to the block number specified.
+   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+   */ stopCollectingForBlocksUpTo(blockNumber) {
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber <= blockNumber) {
+                request.requestTracker.cancel();
+            }
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks greater than the block number specified.
+   * To be called when there is a chain prune and previously mined txs are no longer relevant.
+   */ stopCollectingForBlocksAfter(blockNumber) {
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber > blockNumber) {
+                request.requestTracker.cancel();
+            }
+        }
+    }
+}

package/dest/services/tx_collection/tx_collection_sink.d.ts

@@ -0,0 +1,40 @@
+import type { Logger } from '@aztec-labs/foundation/log';
+import type { TypedEventEmitter } from '@aztec-labs/foundation/types';
+import type { L2Block } from '@aztec-labs/stdlib/block';
+import type { BlockHeader, Tx } from '@aztec-labs/stdlib/tx';
+import type { TelemetryClient } from '@aztec-labs/telemetry-client';
+import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/index.js';
+import type { CollectionMethod } from './tx_collection.js';
+import type { TxSourceCollectionResult } from './tx_source.js';
+/** Context determining how collected txs should be added to the pool. */
+export type TxAddContext = {
+    type: 'proposal';
+    blockHeader: BlockHeader;
+} | {
+    type: 'mined';
+    block: L2Block;
+};
+declare const TxCollectionSink_base: new () => TypedEventEmitter<TxPoolV2Events>;
+/**
+ * Executes collection requests from fast collection paths, and handles collected txs
+ * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
+ */
+export declare class TxCollectionSink extends TxCollectionSink_base {
+    private readonly txPool;
+    private readonly log;
+    private readonly instrumentation;
+    constructor(txPool: TxPoolV2, telemetryClient: TelemetryClient, log: Logger);
+    collect(collectValidTxsFn: () => Promise<TxSourceCollectionResult>, requested: string[], info: Record<string, any> & {
+        description: string;
+        method: CollectionMethod;
+    }, context: TxAddContext): Promise<{
+        txs: Tx[];
+        requested: string[];
+        duration: number;
+    } | {
+        txs: Tx[];
+    }>;
+    private foundTxs;
+}
+export {};
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfY29sbGVjdGlvbl9zaW5rLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi90eF9jb2xsZWN0aW9uX3NpbmsudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sNEJBQTRCLENBQUM7QUFFekQsT0FBTyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSw4QkFBOEIsQ0FBQztBQUN0RSxPQUFPLEtBQUssRUFBRSxPQUFPLEVBQUUsTUFBTSwwQkFBMEIsQ0FBQztBQUN4RCxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsRUFBRSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDN0QsT0FBTyxLQUFLLEVBQUUsZUFBZSxFQUFFLE1BQU0sOEJBQThCLENBQUM7QUFHcEUsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLGNBQWMsRUFBRSxNQUFNLDBCQUEwQixDQUFDO0FBRXpFLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDM0QsT0FBTyxLQUFLLEVBQUUsd0JBQXdCLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUUvRCx5RUFBeUU7QUFDekUsTUFBTSxNQUFNLFlBQVksR0FBRztJQUFFLElBQUksRUFBRSxVQUFVLENBQUM7SUFBQyxXQUFXLEVBQUUsV0FBVyxDQUFBO0NBQUUsR0FBRztJQUFFLElBQUksRUFBRSxPQUFPLENBQUM7SUFBQyxLQUFLLEVBQUUsT0FBTyxDQUFBO0NBQUUsQ0FBQzs7QUFFOUc7OztHQUdHO0FBQ0gscUJBQWEsZ0JBQWlCLFNBQVEscUJBQTZEO0lBSS9GLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQUV2QixPQUFPLENBQUMsUUFBUSxDQUFDLEdBQUc7SUFMdEIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxlQUFlLENBQThCO0lBRTlELFlBQ21CLE1BQU0sRUFBRSxRQUFRLEVBQ2pDLGVBQWUsRUFBRSxlQUFlLEVBQ2YsR0FBRyxFQUFFLE1BQU0sRUFJN0I7SUFFWSxPQUFPLENBQ2xCLGlCQUFpQixFQUFFLE1BQU0sT0FBTyxDQUFDLHdCQUF3QixDQUFDLEVBQzFELFNBQVMsRUFBRSxNQUFNLEVBQUUsRUFDbkIsSUFBSSxFQUFFLE1BQU0sQ0FBQyxNQUFNLEVBQUUsR0FBRyxDQUFDLEdBQUc7UUFBRSxXQUFXLEVBQUUsTUFBTSxDQUFDO1FBQUMsTUFBTSxFQUFFLGdCQUFnQixDQUFBO0tBQUUsRUFDN0UsT0FBTyxFQUFFLFlBQVk7Ozs7OztPQXdEdEI7WUFFYSxRQUFRO0NBNkJ2QiJ9

package/dest/services/tx_collection/tx_collection_sink.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx_collection_sink.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_collection_sink.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEzD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAGpE,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAEzE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAE/D,yEAAyE;AACzE,MAAM,MAAM,YAAY,GAAG;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,WAAW,EAAE,WAAW,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;;AAE9G;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,qBAA6D;IAI/F,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,GAAG;IALtB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA8B;IAE9D,YACmB,MAAM,EAAE,QAAQ,EACjC,eAAe,EAAE,eAAe,EACf,GAAG,EAAE,MAAM,EAI7B;IAEY,OAAO,CAClB,iBAAiB,EAAE,MAAM,OAAO,CAAC,wBAAwB,CAAC,EAC1D,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,gBAAgB,CAAA;KAAE,EAC7E,OAAO,EAAE,YAAY;;;;;;OAwDtB;YAEa,QAAQ;CA6BvB"}

package/dest/services/tx_collection/tx_collection_sink.js

@@ -0,0 +1,108 @@
+import { elapsed } from '@aztec-labs/foundation/timer';
+import EventEmitter from 'node:events';
+import { TxCollectionInstrumentation } from './instrumentation.js';
+/**
+ * Executes collection requests from fast collection paths, and handles collected txs
+ * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
+ */ export class TxCollectionSink extends EventEmitter {
+    txPool;
+    log;
+    instrumentation;
+    constructor(txPool, telemetryClient, log){
+        super(), this.txPool = txPool, this.log = log;
+        this.instrumentation = new TxCollectionInstrumentation(telemetryClient, 'TxCollection');
+    }
+    async collect(collectValidTxsFn, requested, info, context) {
+        this.log.trace(`Requesting ${requested.length} txs via ${info.description}`, {
+            ...info,
+            requestedTxs: requested
+        });
+        // Execute collection function and measure the time taken, catching any errors.
+        const [duration, { validTxs, invalidTxHashes }] = await elapsed(async ()=>{
+            try {
+                return await collectValidTxsFn();
+            } catch (err) {
+                this.log.error(`Error collecting txs via ${info.description}`, err, {
+                    ...info,
+                    requestedTxs: requested
+                });
+                return {
+                    validTxs: [],
+                    invalidTxHashes: []
+                };
+            }
+        });
+        if (validTxs.length === 0 && invalidTxHashes.length === 0) {
+            this.log.trace(`No txs found via ${info.description}`, {
+                ...info,
+                requestedTxs: requested
+            });
+            return {
+                txs: validTxs,
+                requested,
+                duration
+            };
+        }
+        if (invalidTxHashes.length > 0) {
+            this.log.warn(`Rejecting ${invalidTxHashes.length} txs with invalid hashes from ${info.description}`, {
+                ...info,
+                invalidTxHashes
+            });
+        }
+        if (validTxs.length === 0) {
+            this.log.trace(`No valid txs found via ${info.description} after validation`, {
+                ...info,
+                requestedTxs: requested,
+                invalidTxHashes
+            });
+            return {
+                txs: [],
+                requested,
+                duration
+            };
+        }
+        this.log.verbose(`Collected ${validTxs.length} txs out of ${requested.length} requested via ${info.description} in ${duration}ms`, {
+            ...info,
+            duration,
+            txs: validTxs.map((t)=>t.getTxHash().toString()),
+            requestedTxs: requested,
+            rejectedCount: invalidTxHashes.length
+        });
+        return await this.foundTxs(validTxs, {
+            ...info,
+            duration
+        }, context);
+    }
+    async foundTxs(txs, info, context) {
+        // Report metrics for the collection
+        this.instrumentation.increaseTxsFor(info.method, txs.length, info.duration);
+        // Mark txs as found in all in-flight collection requests
+        this.emit('txs-added', {
+            txs
+        });
+        // Add the txs to the tx pool using the appropriate method based on context
+        try {
+            if (context.type === 'mined') {
+                await this.txPool.addMinedTxs(txs, context.block.header, {
+                    source: 'tx-collection'
+                });
+            } else {
+                await this.txPool.addProtectedTxs(txs, context.blockHeader, {
+                    source: 'tx-collection'
+                });
+            }
+        } catch (err) {
+            this.log.error(`Error adding txs to the pool via ${info.description}`, err, {
+                ...info,
+                txs: txs.map((tx)=>tx.txHash.toString())
+            });
+            // Return no txs since none have been added
+            return {
+                txs: []
+            };
+        }
+        return {
+            txs
+        };
+    }
+}

package/dest/services/tx_collection/tx_source.d.ts

@@ -0,0 +1,24 @@
+import type { ChainConfig } from '@aztec-labs/stdlib/config';
+import { type AztecNode } from '@aztec-labs/stdlib/interfaces/client';
+import type { Tx, TxHash, TxValidator } from '@aztec-labs/stdlib/tx';
+import { type ComponentsVersions } from '@aztec-labs/stdlib/versioning';
+export type TxSourceCollectionResult = {
+    validTxs: Tx[];
+    invalidTxHashes: string[];
+};
+export interface TxSource {
+    getInfo(): string;
+    getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
+}
+export declare class NodeRpcTxSource implements TxSource {
+    private readonly client;
+    private readonly txValidator;
+    private readonly info;
+    constructor(client: Pick<AztecNode, 'getTxsByHash'>, txValidator: TxValidator, info: string);
+    static fromUrl(nodeUrl: string, txValidator: TxValidator, versions: ComponentsVersions): NodeRpcTxSource;
+    getInfo(): string;
+    getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
+    private verifyTxs;
+}
+export declare function createNodeRpcTxSources(urls: string[], txValidator: TxValidator, chainConfig: ChainConfig): NodeRpcTxSource[];
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfc291cmNlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi90eF9zb3VyY2UudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBRUEsT0FBTyxLQUFLLEVBQUUsV0FBVyxFQUFFLE1BQU0sMkJBQTJCLENBQUM7QUFDN0QsT0FBTyxFQUFFLEtBQUssU0FBUyxFQUF5QixNQUFNLHNDQUFzQyxDQUFDO0FBQzdGLE9BQU8sS0FBSyxFQUFFLEVBQUUsRUFBRSxNQUFNLEVBQUUsV0FBVyxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDckUsT0FBTyxFQUFFLEtBQUssa0JBQWtCLEVBQW1DLE1BQU0sK0JBQStCLENBQUM7QUFHekcsTUFBTSxNQUFNLHdCQUF3QixHQUFHO0lBQUUsUUFBUSxFQUFFLEVBQUUsRUFBRSxDQUFDO0lBQUMsZUFBZSxFQUFFLE1BQU0sRUFBRSxDQUFBO0NBQUUsQ0FBQztBQUVyRixNQUFNLFdBQVcsUUFBUTtJQUN2QixPQUFPLElBQUksTUFBTSxDQUFDO0lBQ2xCLFlBQVksQ0FBQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsT0FBTyxDQUFDLHdCQUF3QixDQUFDLENBQUM7Q0FDckU7QUFFRCxxQkFBYSxlQUFnQixZQUFXLFFBQVE7SUFFNUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxNQUFNO0lBQ3ZCLE9BQU8sQ0FBQyxRQUFRLENBQUMsV0FBVztJQUM1QixPQUFPLENBQUMsUUFBUSxDQUFDLElBQUk7SUFIdkIsWUFDbUIsTUFBTSxFQUFFLElBQUksQ0FBQyxTQUFTLEVBQUUsY0FBYyxDQUFDLEVBQ3ZDLFdBQVcsRUFBRSxXQUFXLEVBQ3hCLElBQUksRUFBRSxNQUFNLEVBQzNCO0lBRUosT0FBYyxPQUFPLENBQUMsT0FBTyxFQUFFLE1BQU0sRUFBRSxXQUFXLEVBQUUsV0FBVyxFQUFFLFFBQVEsRUFBRSxrQkFBa0IsR0FBRyxlQUFlLENBTTlHO0lBRU0sT0FBTyxXQUViO0lBRVksWUFBWSxDQUFDLFFBQVEsRUFBRSxNQUFNLEVBQUUsR0FBRyxPQUFPLENBQUMsd0JBQXdCLENBQUMsQ0FHL0U7WUFFYSxTQUFTO0NBZ0J4QjtBQUVELHdCQUFnQixzQkFBc0IsQ0FBQyxJQUFJLEVBQUUsTUFBTSxFQUFFLEVBQUUsV0FBVyxFQUFFLFdBQVcsRUFBRSxXQUFXLEVBQUUsV0FBVyxxQkFHeEcifQ==

package/dest/services/tx_collection/tx_source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx_source.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_source.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,KAAK,SAAS,EAAyB,MAAM,sCAAsC,CAAC;AAC7F,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,EAAE,KAAK,kBAAkB,EAAmC,MAAM,+BAA+B,CAAC;AAGzG,MAAM,MAAM,wBAAwB,GAAG;IAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAAC,eAAe,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC;AAErF,MAAM,WAAW,QAAQ;IACvB,OAAO,IAAI,MAAM,CAAC;IAClB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACrE;AAED,qBAAa,eAAgB,YAAW,QAAQ;IAE5C,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAHvB,YACmB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EACvC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EAC3B;IAEJ,OAAc,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,kBAAkB,GAAG,eAAe,CAM9G;IAEM,OAAO,WAEb;IAEY,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAG/E;YAEa,SAAS;CAgBxB;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,qBAGxG"}

package/dest/services/tx_collection/tx_source.js

@@ -0,0 +1,56 @@
+import { getVKTreeRoot } from '@aztec-labs/noir-protocol-circuits-types/vk-tree';
+import { protocolContractsHash } from '@aztec-labs/protocol-contracts';
+import { createAztecNodeClient } from '@aztec-labs/stdlib/interfaces/client';
+import { getComponentsVersionsFromConfig } from '@aztec-labs/stdlib/versioning';
+import { makeTracedFetch } from '@aztec-labs/telemetry-client';
+export class NodeRpcTxSource {
+    client;
+    txValidator;
+    info;
+    constructor(client, txValidator, info){
+        this.client = client;
+        this.txValidator = txValidator;
+        this.info = info;
+    }
+    static fromUrl(nodeUrl, txValidator, versions) {
+        const client = createAztecNodeClient(nodeUrl, {
+            versions,
+            fetch: makeTracedFetch([
+                1,
+                2,
+                3
+            ], false)
+        });
+        return new NodeRpcTxSource(client, txValidator, nodeUrl);
+    }
+    getInfo() {
+        return this.info;
+    }
+    async getTxsByHash(txHashes) {
+        // Collected txs feed block validation and proving, so we need their proofs.
+        return this.verifyTxs(await this.client.getTxsByHash(txHashes, {
+            includeProof: true
+        }));
+    }
+    async verifyTxs(txs) {
+        // Validate tx hashes for all collected txs from external sources
+        const validTxs = [];
+        const invalidTxHashes = [];
+        await Promise.all(txs.map(async (tx)=>{
+            const validation = await this.txValidator.validateTx(tx);
+            if (validation.result === 'valid') {
+                validTxs.push(tx);
+            } else {
+                invalidTxHashes.push(tx.getTxHash().toString());
+            }
+        }));
+        return {
+            validTxs: validTxs,
+            invalidTxHashes: invalidTxHashes
+        };
+    }
+}
+export function createNodeRpcTxSources(urls, txValidator, chainConfig) {
+    const versions = getComponentsVersionsFromConfig(chainConfig, protocolContractsHash, getVKTreeRoot());
+    return urls.map((url)=>NodeRpcTxSource.fromUrl(url, txValidator, versions));
+}

package/dest/services/tx_file_store/config.d.ts

@@ -0,0 +1,16 @@
+import { type ConfigMappingsType } from '@aztec-labs/foundation/config';
+/**
+ * Configuration for the TxFileStore service.
+ */
+export type TxFileStoreConfig = {
+    /** URL for uploading txs to file storage (s3://, gs://, file://) */
+    txFileStoreUrl?: string;
+    /** Max concurrent uploads */
+    txFileStoreUploadConcurrency: number;
+    /** Max queue size to prevent unbounded memory growth */
+    txFileStoreMaxQueueSize: number;
+    /** Enable tx file store upload */
+    txFileStoreEnabled: boolean;
+};
+export declare const txFileStoreConfigMappings: ConfigMappingsType<TxFileStoreConfig>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfZmlsZV9zdG9yZS9jb25maWcudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLEtBQUssa0JBQWtCLEVBQTJDLE1BQU0sK0JBQStCLENBQUM7QUFFakg7O0dBRUc7QUFDSCxNQUFNLE1BQU0saUJBQWlCLEdBQUc7SUFDOUIsb0VBQW9FO0lBQ3BFLGNBQWMsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUN4Qiw2QkFBNkI7SUFDN0IsNEJBQTRCLEVBQUUsTUFBTSxDQUFDO0lBQ3JDLHdEQUF3RDtJQUN4RCx1QkFBdUIsRUFBRSxNQUFNLENBQUM7SUFDaEMsa0NBQWtDO0lBQ2xDLGtCQUFrQixFQUFFLE9BQU8sQ0FBQztDQUM3QixDQUFDO0FBRUYsZUFBTyxNQUFNLHlCQUF5QixFQUFFLGtCQUFrQixDQUFDLGlCQUFpQixDQW9CM0UsQ0FBQyJ9

package/dest/services/tx_file_store/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../src/services/tx_file_store/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAA2C,MAAM,+BAA+B,CAAC;AAEjH;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6BAA6B;IAC7B,4BAA4B,EAAE,MAAM,CAAC;IACrC,wDAAwD;IACxD,uBAAuB,EAAE,MAAM,CAAC;IAChC,kCAAkC;IAClC,kBAAkB,EAAE,OAAO,CAAC;CAC7B,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,kBAAkB,CAAC,iBAAiB,CAoB3E,CAAC"}

package/dest/services/tx_file_store/config.js

@@ -0,0 +1,22 @@
+import { booleanConfigHelper, numberConfigHelper } from '@aztec-labs/foundation/config';
+export const txFileStoreConfigMappings = {
+    txFileStoreUrl: {
+        env: 'TX_FILE_STORE_URL',
+        description: 'URL for uploading txs to file storage (s3://, gs://, file://)'
+    },
+    txFileStoreUploadConcurrency: {
+        env: 'TX_FILE_STORE_UPLOAD_CONCURRENCY',
+        description: 'Maximum number of concurrent tx uploads',
+        ...numberConfigHelper(10)
+    },
+    txFileStoreMaxQueueSize: {
+        env: 'TX_FILE_STORE_MAX_QUEUE_SIZE',
+        description: 'Maximum queue size for pending uploads (oldest dropped when exceeded)',
+        ...numberConfigHelper(1000)
+    },
+    txFileStoreEnabled: {
+        env: 'TX_FILE_STORE_ENABLED',
+        description: 'Enable uploading transactions to file storage',
+        ...booleanConfigHelper(false)
+    }
+};

package/dest/services/tx_file_store/index.d.ts

@@ -0,0 +1,4 @@
+export { TxFileStore } from './tx_file_store.js';
+export { TxFileStoreInstrumentation } from './instrumentation.js';
+export { type TxFileStoreConfig, txFileStoreConfigMappings } from './config.js';
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zZXJ2aWNlcy90eF9maWxlX3N0b3JlL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUNqRCxPQUFPLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUNsRSxPQUFPLEVBQUUsS0FBSyxpQkFBaUIsRUFBRSx5QkFBeUIsRUFBRSxNQUFNLGFBQWEsQ0FBQyJ9

package/dest/services/tx_file_store/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/tx_file_store/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,KAAK,iBAAiB,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC"}

package/dest/services/tx_file_store/index.js

@@ -0,0 +1,3 @@
+export { TxFileStore } from './tx_file_store.js';
+export { TxFileStoreInstrumentation } from './instrumentation.js';
+export { txFileStoreConfigMappings } from './config.js';