@aztec/p2p 0.0.0-test.0 → 0.0.1-commit.023c3e5

package/dest/services/tx_collection/fast_tx_collection.js

@@ -0,0 +1,311 @@
+import { times } from '@aztec/foundation/collection';
+import { AbortError, TimeoutError } from '@aztec/foundation/error';
+import { createLogger } from '@aztec/foundation/log';
+import { boundInclusive } from '@aztec/foundation/number';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider, elapsed } from '@aztec/foundation/timer';
+import { TxHash } from '@aztec/stdlib/tx';
+import { ReqRespSubProtocol } from '../reqresp/interface.js';
+import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+import { BatchTxRequesterCollector, SendBatchRequestCollector } from './proposal_tx_collector.js';
+export class FastTxCollection {
+    p2pService;
+    nodes;
+    txCollectionSink;
+    config;
+    dateProvider;
+    log;
+    // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+    requests;
+    proposalTxCollector;
+    constructor(p2pService, nodes, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service'), proposalTxCollector){
+        this.p2pService = p2pService;
+        this.nodes = nodes;
+        this.txCollectionSink = txCollectionSink;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+        this.requests = new Set();
+        const batchTxRequesterConfig = this.config;
+        const proposalCollectorType = this.config.txCollectionProposalTxCollectorType;
+        this.proposalTxCollector = proposalTxCollector ?? (proposalCollectorType === 'old' ? new SendBatchRequestCollector(p2pService) : new BatchTxRequesterCollector(p2pService, log, dateProvider, undefined, batchTxRequesterConfig));
+    }
+    async stop() {
+        this.requests.forEach((request)=>request.promise.reject(new AbortError(`Stopped collection service`)));
+        await Promise.resolve();
+    }
+    getFastCollectionRequests() {
+        return this.requests;
+    }
+    async collectFastFor(input, txHashes, opts) {
+        const timeout = opts.deadline.getTime() - this.dateProvider.now();
+        if (timeout <= 0) {
+            this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, {
+                deadline: opts.deadline.getTime(),
+                now: this.dateProvider.now()
+            });
+            return [];
+        }
+        const blockInfo = input.type === 'proposal' ? {
+            ...input.blockProposal.toBlockInfo(),
+            blockNumber: input.blockNumber
+        } : {
+            ...input.block.toBlockInfo()
+        };
+        // This promise is used to await for the collection to finish during the main collectFast method.
+        // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
+        const promise = promiseWithResolvers();
+        setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
+        const request = {
+            ...input,
+            blockInfo,
+            promise,
+            foundTxs: new Map(),
+            missingTxHashes: new Set(txHashes.map((t)=>t.toString())),
+            deadline: opts.deadline
+        };
+        const [duration] = await elapsed(()=>this.collectFast(request, {
+                ...opts
+            }));
+        this.log.verbose(`Collected ${request.foundTxs.size} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            duration,
+            requestType: input.type,
+            missingTxs: [
+                ...request.missingTxHashes
+            ]
+        });
+        return [
+            ...request.foundTxs.values()
+        ];
+    }
+    async collectFast(request, opts) {
+        this.requests.add(request);
+        const { blockInfo } = request;
+        this.log.debug(`Starting fast collection of ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            requestType: request.type,
+            deadline: opts.deadline
+        });
+        try {
+            // Start blasting all nodes for the txs. We give them a little time to respond before we start reqresp.
+            // And keep an eye on the request promise to ensure we don't wait longer than the deadline or return as soon
+            // as we have collected all txs, whatever the source.
+            const nodeCollectionPromise = this.collectFastFromNodes(request, opts);
+            const waitBeforeReqResp = sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs);
+            await Promise.race([
+                request.promise.promise,
+                waitBeforeReqResp
+            ]);
+            // If we have collected all txs, we can stop here
+            if (request.missingTxHashes.size === 0) {
+                this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
+                return;
+            }
+            // Start blasting reqresp for the remaining txs. Note that node collection keeps running in parallel.
+            // We stop when we have collected all txs, timed out, or both node collection and reqresp have given up.
+            const collectionPromise = Promise.allSettled([
+                this.collectFastViaReqResp(request, opts),
+                nodeCollectionPromise
+            ]);
+            await Promise.race([
+                collectionPromise,
+                request.promise.promise
+            ]);
+        } catch (err) {
+            // Log and swallow all errors
+            const logCtx = {
+                ...blockInfo,
+                errorMessage: err instanceof Error ? err.message : undefined,
+                missingTxs: [
+                    ...request.missingTxHashes
+                ].map((txHash)=>txHash.toString())
+            };
+            if (err instanceof Error && err.name === 'TimeoutError') {
+                this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
+            } else if (err instanceof Error && err.name === 'AbortError') {
+                this.log.warn(`Aborted collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
+            } else {
+                this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, logCtx);
+            }
+        } finally{
+            // Ensure no unresolved promises and remove the request from the set
+            request.promise.resolve();
+            this.requests.delete(request);
+        }
+    }
+    /**
+   * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests
+   * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
+   * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
+   * tx is requested more than once at the same time to the same node.
+   */ async collectFastFromNodes(request, opts) {
+        if (this.nodes.length === 0) {
+            return;
+        }
+        // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
+        const attemptsPerTx = [
+            ...request.missingTxHashes
+        ].map((txHash)=>({
+                txHash,
+                attempts: 0,
+                found: false
+            }));
+        // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
+        await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx, opts)));
+    }
+    async collectFastFromNode(request, node, attemptsPerTx, opts) {
+        const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && request.missingTxHashes.size > 0 && this.requests.has(request);
+        const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
+        const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
+        const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
+        const processBatch = async ()=>{
+            while(notFinished()){
+                // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts,
+                // so we prioritize txs that have been requested less often.
+                const batch = [];
+                let index = 0;
+                while(batch.length < maxBatchSize){
+                    const txToRequest = attemptsPerTx[index++];
+                    if (!txToRequest) {
+                        break;
+                    } else if (!request.missingTxHashes.has(txToRequest.txHash)) {
+                        // Mark as found if it was found somewhere else, we'll then remove it from the array.
+                        // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
+                        txToRequest.found = true;
+                    } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) {
+                        // If the tx is not alredy being requested to this node, add it to the current batch and increase attempts.
+                        // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that
+                        // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting
+                        // the same txs at the same time.
+                        batch.push(txToRequest);
+                        activeRequestsToThisNode.add(txToRequest.txHash);
+                        txToRequest.attempts++;
+                    }
+                }
+                // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end.
+                attemptsPerTx.sort((a, b)=>a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found));
+                const firstFoundTxIndex = attemptsPerTx.findIndex((tx)=>tx.found);
+                if (firstFoundTxIndex !== -1) {
+                    attemptsPerTx.length = firstFoundTxIndex;
+                }
+                // If we see no more txs to request, we can stop this "process" loop
+                if (batch.length === 0) {
+                    return;
+                }
+                // Collect this batch from the node
+                await this.txCollectionSink.collect((txHashes)=>node.getTxsByHash(txHashes), batch.map(({ txHash })=>TxHash.fromString(txHash)), {
+                    description: `fast ${node.getInfo()}`,
+                    node: node.getInfo(),
+                    method: 'fast-node-rpc',
+                    ...request.blockInfo
+                });
+                // Clear from the active requests the txs we just requested
+                for (const requestedTx of batch){
+                    activeRequestsToThisNode.delete(requestedTx.txHash);
+                }
+                // Sleep a bit until hitting the node again (or not, depending on config)
+                if (notFinished()) {
+                    await sleep(this.config.txCollectionFastNodeIntervalMs);
+                }
+            }
+        };
+        // Kick off N parallel requests to the node, up to the maxParallelRequests limit
+        await Promise.all(times(maxParallelRequests, processBatch));
+    }
+    async collectFastViaReqResp(request, opts) {
+        const timeoutMs = +request.deadline - this.dateProvider.now();
+        const pinnedPeer = opts.pinnedPeer;
+        const maxPeers = boundInclusive(Math.ceil(request.missingTxHashes.size / 2), 8, 32);
+        const maxRetryAttempts = 5;
+        const blockInfo = request.blockInfo;
+        const slotNumber = blockInfo.slotNumber;
+        if (timeoutMs < 100) {
+            this.log.warn(`Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`, {
+                timeoutMs,
+                ...blockInfo
+            });
+            return;
+        }
+        this.log.debug(`Starting fast reqresp for ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            timeoutMs,
+            pinnedPeer
+        });
+        try {
+            await this.txCollectionSink.collect(async (txHashes)=>{
+                if (request.type === 'proposal') {
+                    return await this.proposalTxCollector.collectTxs(txHashes, request.blockProposal, pinnedPeer, timeoutMs);
+                } else if (request.type === 'block') {
+                    const txs = await this.p2pService.reqResp.sendBatchRequest(ReqRespSubProtocol.TX, chunkTxHashesRequest(txHashes), pinnedPeer, timeoutMs, maxPeers, maxRetryAttempts);
+                    return txs.flat();
+                } else {
+                    throw new Error(`Unknown request type: ${request.type}`);
+                }
+            }, Array.from(request.missingTxHashes).map((txHash)=>TxHash.fromString(txHash)), {
+                description: `reqresp for slot ${slotNumber}`,
+                method: 'fast-req-resp',
+                ...opts,
+                ...request.blockInfo
+            });
+        } catch (err) {
+            this.log.error(`Error sending fast reqresp request for txs`, err, {
+                txs: [
+                    ...request.missingTxHashes
+                ],
+                ...blockInfo
+            });
+        }
+    }
+    /**
+   * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
+   * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
+   */ foundTxs(txs) {
+        for (const request of this.requests){
+            for (const tx of txs){
+                const txHash = tx.txHash.toString();
+                // Remove the tx hash from the missing set, and add it to the found set.
+                if (request.missingTxHashes.has(txHash)) {
+                    request.missingTxHashes.delete(txHash);
+                    request.foundTxs.set(txHash, tx);
+                    this.log.trace(`Found tx ${txHash} for fast collection request`, {
+                        ...request.blockInfo,
+                        txHash: tx.txHash.toString(),
+                        type: request.type
+                    });
+                    // If we found all txs for this request, we resolve the promise
+                    if (request.missingTxHashes.size === 0) {
+                        this.log.trace(`All txs found for fast collection request`, {
+                            ...request.blockInfo,
+                            type: request.type
+                        });
+                        request.promise.resolve();
+                    }
+                }
+            }
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+   */ stopCollectingForBlocksUpTo(blockNumber) {
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber <= blockNumber) {
+                request.promise.reject(new AbortError(`Stopped collecting txs up to block ${blockNumber}`));
+                this.requests.delete(request);
+            }
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks greater than the block number specified.
+   * To be called when there is a chain prune and previously mined txs are no longer relevant.
+   */ stopCollectingForBlocksAfter(blockNumber) {
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber > blockNumber) {
+                request.promise.reject(new AbortError(`Stopped collecting txs after block ${blockNumber}`));
+                this.requests.delete(request);
+            }
+        }
+    }
+}

package/dest/services/tx_collection/index.d.ts

@@ -0,0 +1,4 @@
+export { TxCollection, type FastCollectionRequestInput } from './tx_collection.js';
+export { type TxSource, createNodeRpcTxSources, NodeRpcTxSource } from './tx_source.js';
+export { type ProposalTxCollector, BatchTxRequesterCollector, SendBatchRequestCollector, } from './proposal_tx_collector.js';
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zZXJ2aWNlcy90eF9jb2xsZWN0aW9uL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxZQUFZLEVBQUUsS0FBSywwQkFBMEIsRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBQ25GLE9BQU8sRUFBRSxLQUFLLFFBQVEsRUFBRSxzQkFBc0IsRUFBRSxlQUFlLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUN4RixPQUFPLEVBQ0wsS0FBSyxtQkFBbUIsRUFDeEIseUJBQXlCLEVBQ3pCLHlCQUF5QixHQUMxQixNQUFNLDRCQUE0QixDQUFDIn0=

package/dest/services/tx_collection/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,KAAK,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AACnF,OAAO,EAAE,KAAK,QAAQ,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACxF,OAAO,EACL,KAAK,mBAAmB,EACxB,yBAAyB,EACzB,yBAAyB,GAC1B,MAAM,4BAA4B,CAAC"}

package/dest/services/tx_collection/index.js

@@ -0,0 +1,3 @@
+export { TxCollection } from './tx_collection.js';
+export { createNodeRpcTxSources, NodeRpcTxSource } from './tx_source.js';
+export { BatchTxRequesterCollector, SendBatchRequestCollector } from './proposal_tx_collector.js';

package/dest/services/tx_collection/instrumentation.d.ts

@@ -0,0 +1,10 @@
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import type { CollectionMethod } from './tx_collection.js';
+export declare class TxCollectionInstrumentation {
+    private txsCollected;
+    private collectionDurationPerTx;
+    private collectionDurationPerRequest;
+    constructor(client: TelemetryClient, name: string);
+    increaseTxsFor(what: CollectionMethod, count: number, duration: number): void;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5zdHJ1bWVudGF0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9pbnN0cnVtZW50YXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUlMLEtBQUssZUFBZSxFQUdyQixNQUFNLHlCQUF5QixDQUFDO0FBRWpDLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFFM0QscUJBQWEsMkJBQTJCO0lBQ3RDLE9BQU8sQ0FBQyxZQUFZLENBQWdCO0lBQ3BDLE9BQU8sQ0FBQyx1QkFBdUIsQ0FBWTtJQUMzQyxPQUFPLENBQUMsNEJBQTRCLENBQVk7SUFFaEQsWUFBWSxNQUFNLEVBQUUsZUFBZSxFQUFFLElBQUksRUFBRSxNQUFNLEVBVWhEO0lBRUQsY0FBYyxDQUFDLElBQUksRUFBRSxnQkFBZ0IsRUFBRSxLQUFLLEVBQUUsTUFBTSxFQUFFLFFBQVEsRUFBRSxNQUFNLFFBS3JFO0NBQ0YifQ==

package/dest/services/tx_collection/instrumentation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"instrumentation.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,qBAAa,2BAA2B;IACtC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,uBAAuB,CAAY;IAC3C,OAAO,CAAC,4BAA4B,CAAY;IAEhD,YAAY,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,EAUhD;IAED,cAAc,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,QAKrE;CACF"}

package/dest/services/tx_collection/instrumentation.js

@@ -0,0 +1,31 @@
+import { Attributes, Metrics, createUpDownCounterWithDefault } from '@aztec/telemetry-client';
+export class TxCollectionInstrumentation {
+    txsCollected;
+    collectionDurationPerTx;
+    collectionDurationPerRequest;
+    constructor(client, name){
+        const meter = client.getMeter(name);
+        this.txsCollected = createUpDownCounterWithDefault(meter, Metrics.TX_COLLECTOR_COUNT, {
+            [Attributes.TX_COLLECTION_METHOD]: [
+                'fast-req-resp',
+                'fast-node-rpc',
+                'slow-req-resp',
+                'slow-node-rpc'
+            ]
+        });
+        this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX);
+        this.collectionDurationPerRequest = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_REQUEST);
+    }
+    increaseTxsFor(what, count, duration) {
+        const durationPerTx = Math.ceil(duration / count);
+        this.collectionDurationPerTx.record(durationPerTx, {
+            [Attributes.TX_COLLECTION_METHOD]: what
+        });
+        this.collectionDurationPerRequest.record(Math.ceil(duration), {
+            [Attributes.TX_COLLECTION_METHOD]: what
+        });
+        this.txsCollected.add(count, {
+            [Attributes.TX_COLLECTION_METHOD]: what
+        });
+    }
+}

package/dest/services/tx_collection/proposal_tx_collector.d.ts

@@ -0,0 +1,48 @@
+import type { Logger } from '@aztec/foundation/log';
+import type { DateProvider } from '@aztec/foundation/timer';
+import type { BlockProposal } from '@aztec/stdlib/p2p';
+import type { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { PeerId } from '@libp2p/interface';
+import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
+import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
+import type { IBatchRequestTxValidator } from '../reqresp/batch-tx-requester/tx_validator.js';
+/**
+ * Strategy interface for collecting transactions for block proposals.
+ * Allows swapping between different tx collection implementations for benchmarking.
+ */
+export interface ProposalTxCollector {
+    /**
+     * Collect transactions for a block proposal.
+     * @param txHashes - The transaction hashes to collect
+     * @param blockProposal - The block proposal containing the transactions
+     * @param pinnedPeer - Optional peer that sent the proposal (expected to have all txs)
+     * @param timeoutMs - Timeout in milliseconds
+     * @returns The collected transactions
+     */
+    collectTxs(txHashes: TxHash[], blockProposal: BlockProposal, pinnedPeer: PeerId | undefined, timeoutMs: number): Promise<Tx[]>;
+}
+/**
+ * Collects transactions using the BatchTxRequester implementation.
+ * This uses a smart/dumb peer strategy with parallel workers.
+ */
+export declare class BatchTxRequesterCollector implements ProposalTxCollector {
+    private p2pService;
+    private log;
+    private dateProvider;
+    private txValidator?;
+    private batchTxRequesterConfig?;
+    constructor(p2pService: BatchTxRequesterLibP2PService, log: Logger, dateProvider: DateProvider, txValidator?: IBatchRequestTxValidator | undefined, batchTxRequesterConfig?: Partial<BatchTxRequesterConfig> | undefined);
+    collectTxs(txHashes: TxHash[], blockProposal: BlockProposal, pinnedPeer: PeerId | undefined, timeoutMs: number): Promise<Tx[]>;
+}
+/**
+ * Collects transactions using the sendBatchRequest implementation from ReqResp.
+ * This is the original implementation that balances requests across peers.
+ */
+export declare class SendBatchRequestCollector implements ProposalTxCollector {
+    private p2pService;
+    private maxPeers;
+    private maxRetryAttempts;
+    constructor(p2pService: BatchTxRequesterLibP2PService, maxPeers?: number, maxRetryAttempts?: number);
+    collectTxs(txHashes: TxHash[], _blockProposal: BlockProposal, pinnedPeer: PeerId | undefined, timeoutMs: number): Promise<Tx[]>;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvcG9zYWxfdHhfY29sbGVjdG9yLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9wcm9wb3NhbF90eF9jb2xsZWN0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDcEQsT0FBTyxLQUFLLEVBQUUsWUFBWSxFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFDNUQsT0FBTyxLQUFLLEVBQUUsYUFBYSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDdkQsT0FBTyxLQUFLLEVBQUUsRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRW5ELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBR2hELE9BQU8sS0FBSyxFQUFFLHNCQUFzQixFQUFFLE1BQU0seUNBQXlDLENBQUM7QUFDdEYsT0FBTyxLQUFLLEVBQUUsNkJBQTZCLEVBQUUsTUFBTSw0Q0FBNEMsQ0FBQztBQUNoRyxPQUFPLEtBQUssRUFBRSx3QkFBd0IsRUFBRSxNQUFNLCtDQUErQyxDQUFDO0FBSTlGOzs7R0FHRztBQUNILE1BQU0sV0FBVyxtQkFBbUI7SUFDbEM7Ozs7Ozs7T0FPRztJQUNILFVBQVUsQ0FDUixRQUFRLEVBQUUsTUFBTSxFQUFFLEVBQ2xCLGFBQWEsRUFBRSxhQUFhLEVBQzVCLFVBQVUsRUFBRSxNQUFNLEdBQUcsU0FBUyxFQUM5QixTQUFTLEVBQUUsTUFBTSxHQUNoQixPQUFPLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQztDQUNsQjtBQUVEOzs7R0FHRztBQUNILHFCQUFhLHlCQUEwQixZQUFXLG1CQUFtQjtJQUVqRSxPQUFPLENBQUMsVUFBVTtJQUNsQixPQUFPLENBQUMsR0FBRztJQUNYLE9BQU8sQ0FBQyxZQUFZO0lBQ3BCLE9BQU8sQ0FBQyxXQUFXLENBQUM7SUFDcEIsT0FBTyxDQUFDLHNCQUFzQixDQUFDO0lBTGpDLFlBQ1UsVUFBVSxFQUFFLDZCQUE2QixFQUN6QyxHQUFHLEVBQUUsTUFBTSxFQUNYLFlBQVksRUFBRSxZQUFZLEVBQzFCLFdBQVcsQ0FBQyxzQ0FBMEIsRUFDdEMsc0JBQXNCLENBQUMsNkNBQWlDLEVBQzlEO0lBRUUsVUFBVSxDQUNkLFFBQVEsRUFBRSxNQUFNLEVBQUUsRUFDbEIsYUFBYSxFQUFFLGFBQWEsRUFDNUIsVUFBVSxFQUFFLE1BQU0sR0FBRyxTQUFTLEVBQzlCLFNBQVMsRUFBRSxNQUFNLEdBQ2hCLE9BQU8sQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQTBCZjtDQUNGO0FBS0Q7OztHQUdHO0FBQ0gscUJBQWEseUJBQTBCLFlBQVcsbUJBQW1CO0lBRWpFLE9BQU8sQ0FBQyxVQUFVO0lBQ2xCLE9BQU8sQ0FBQyxRQUFRO0lBQ2hCLE9BQU8sQ0FBQyxnQkFBZ0I7SUFIMUIsWUFDVSxVQUFVLEVBQUUsNkJBQTZCLEVBQ3pDLFFBQVEsR0FBRSxNQUEwQixFQUNwQyxnQkFBZ0IsR0FBRSxNQUFtQyxFQUMzRDtJQUVFLFVBQVUsQ0FDZCxRQUFRLEVBQUUsTUFBTSxFQUFFLEVBQ2xCLGNBQWMsRUFBRSxhQUFhLEVBQzdCLFVBQVUsRUFBRSxNQUFNLEdBQUcsU0FBUyxFQUM5QixTQUFTLEVBQUUsTUFBTSxHQUNoQixPQUFPLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FXZjtDQUNGIn0=

package/dest/services/tx_collection/proposal_tx_collector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proposal_tx_collector.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/proposal_tx_collector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACtF,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAChG,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,+CAA+C,CAAC;AAI9F;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;;;OAOG;IACH,UAAU,CACR,QAAQ,EAAE,MAAM,EAAE,EAClB,aAAa,EAAE,aAAa,EAC5B,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;CAClB;AAED;;;GAGG;AACH,qBAAa,yBAA0B,YAAW,mBAAmB;IAEjE,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,GAAG;IACX,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,WAAW,CAAC;IACpB,OAAO,CAAC,sBAAsB,CAAC;IALjC,YACU,UAAU,EAAE,6BAA6B,EACzC,GAAG,EAAE,MAAM,EACX,YAAY,EAAE,YAAY,EAC1B,WAAW,CAAC,sCAA0B,EACtC,sBAAsB,CAAC,6CAAiC,EAC9D;IAEE,UAAU,CACd,QAAQ,EAAE,MAAM,EAAE,EAClB,aAAa,EAAE,aAAa,EAC5B,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,EAAE,EAAE,CAAC,CA0Bf;CACF;AAKD;;;GAGG;AACH,qBAAa,yBAA0B,YAAW,mBAAmB;IAEjE,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,gBAAgB;IAH1B,YACU,UAAU,EAAE,6BAA6B,EACzC,QAAQ,GAAE,MAA0B,EACpC,gBAAgB,GAAE,MAAmC,EAC3D;IAEE,UAAU,CACd,QAAQ,EAAE,MAAM,EAAE,EAClB,cAAc,EAAE,aAAa,EAC7B,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,EAAE,EAAE,CAAC,CAWf;CACF"}

package/dest/services/tx_collection/proposal_tx_collector.js

@@ -0,0 +1,50 @@
+import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js';
+import { ReqRespSubProtocol } from '../reqresp/interface.js';
+import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+/**
+ * Collects transactions using the BatchTxRequester implementation.
+ * This uses a smart/dumb peer strategy with parallel workers.
+ */ export class BatchTxRequesterCollector {
+    p2pService;
+    log;
+    dateProvider;
+    txValidator;
+    batchTxRequesterConfig;
+    constructor(p2pService, log, dateProvider, txValidator, batchTxRequesterConfig){
+        this.p2pService = p2pService;
+        this.log = log;
+        this.dateProvider = dateProvider;
+        this.txValidator = txValidator;
+        this.batchTxRequesterConfig = batchTxRequesterConfig;
+    }
+    async collectTxs(txHashes, blockProposal, pinnedPeer, timeoutMs) {
+        const { batchTxRequesterSmartParallelWorkerCount: smartParallelWorkerCount, batchTxRequesterDumbParallelWorkerCount: dumbParallelWorkerCount, batchTxRequesterTxBatchSize: txBatchSize, batchTxRequesterBadPeerThreshold: badPeerThreshold } = this.batchTxRequesterConfig ?? {};
+        const batchRequester = new BatchTxRequester(txHashes, blockProposal, pinnedPeer, timeoutMs, this.p2pService, this.log, this.dateProvider, {
+            smartParallelWorkerCount,
+            dumbParallelWorkerCount,
+            txBatchSize,
+            badPeerThreshold,
+            txValidator: this.txValidator
+        });
+        return await BatchTxRequester.collectAllTxs(batchRequester.run());
+    }
+}
+const DEFAULT_MAX_PEERS = 10;
+const DEFAULT_MAX_RETRY_ATTEMPTS = 3;
+/**
+ * Collects transactions using the sendBatchRequest implementation from ReqResp.
+ * This is the original implementation that balances requests across peers.
+ */ export class SendBatchRequestCollector {
+    p2pService;
+    maxPeers;
+    maxRetryAttempts;
+    constructor(p2pService, maxPeers = DEFAULT_MAX_PEERS, maxRetryAttempts = DEFAULT_MAX_RETRY_ATTEMPTS){
+        this.p2pService = p2pService;
+        this.maxPeers = maxPeers;
+        this.maxRetryAttempts = maxRetryAttempts;
+    }
+    async collectTxs(txHashes, _blockProposal, pinnedPeer, timeoutMs) {
+        const txs = await this.p2pService.reqResp.sendBatchRequest(ReqRespSubProtocol.TX, chunkTxHashesRequest(txHashes), pinnedPeer, timeoutMs, this.maxPeers, this.maxRetryAttempts);
+        return txs.flat();
+    }
+}

package/dest/services/tx_collection/slow_tx_collection.d.ts

@@ -0,0 +1,53 @@
+import { BlockNumber } from '@aztec/foundation/branded-types';
+import { type Logger } from '@aztec/foundation/log';
+import { DateProvider } from '@aztec/foundation/timer';
+import type { L2Block } from '@aztec/stdlib/block';
+import { type L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
+import { type Tx, TxHash } from '@aztec/stdlib/tx';
+import { type ReqRespInterface } from '../reqresp/interface.js';
+import type { TxCollectionConfig } from './config.js';
+import type { FastTxCollection } from './fast_tx_collection.js';
+import type { TxCollectionSink } from './tx_collection_sink.js';
+import type { TxSource } from './tx_source.js';
+export declare class SlowTxCollection {
+    private reqResp;
+    private nodes;
+    private txCollectionSink;
+    private fastCollection;
+    private constants;
+    private config;
+    private dateProvider;
+    private log;
+    /** Map from tx hash to missing tx info to collect via slow loop */
+    private missingTxs;
+    /** One slow collection loop for each node tx source we have */
+    private nodesSlowCollectionLoops;
+    /** Slow collection loop for reqresp which collects txs from peers */
+    private reqrespSlowCollectionLoop;
+    constructor(reqResp: Pick<ReqRespInterface, 'sendBatchRequest'>, nodes: TxSource[], txCollectionSink: TxCollectionSink, fastCollection: Pick<FastTxCollection, 'getFastCollectionRequests'>, constants: L1RollupConstants, config: TxCollectionConfig, dateProvider?: DateProvider, log?: Logger);
+    getMissingTxHashes(): TxHash[];
+    start(): void;
+    stop(): Promise<void>;
+    trigger(): Promise<void>;
+    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
+    startCollecting(block: L2Block, txHashes: TxHash[]): void;
+    private collectMissingTxsFromNode;
+    private collectMissingTxsViaReqResp;
+    /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */
+    private getMissingTxsForSlowCollection;
+    /** Stop collecting the given txs since we have found them. Called whenever tx pool emits a tx-added event. */
+    foundTxs(txs: Tx[]): void;
+    /**
+     * Stop collecting all txs for blocks less than or requal to the block number specified.
+     * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+     */
+    stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void;
+    /**
+     * Stop collecting all txs for blocks greater than the block number specified.
+     * To be called when there is a chain prune and previously mined txs are no longer relevant.
+     */
+    stopCollectingForBlocksAfter(blockNumber: BlockNumber): void;
+    /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */
+    private getDeadlineForSlot;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2xvd190eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9zbG93X3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBMkIsTUFBTSxpQ0FBaUMsQ0FBQztBQUV2RixPQUFPLEVBQUUsS0FBSyxNQUFNLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFHbEUsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ3ZELE9BQU8sS0FBSyxFQUFFLE9BQU8sRUFBRSxNQUFNLHFCQUFxQixDQUFDO0FBQ25ELE9BQU8sRUFBRSxLQUFLLGlCQUFpQixFQUE2QyxNQUFNLDZCQUE2QixDQUFDO0FBQ2hILE9BQU8sRUFBRSxLQUFLLEVBQUUsRUFBRSxNQUFNLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUVuRCxPQUFPLEVBQUUsS0FBSyxnQkFBZ0IsRUFBc0IsTUFBTSx5QkFBeUIsQ0FBQztBQUVwRixPQUFPLEtBQUssRUFBRSxrQkFBa0IsRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUN0RCxPQUFPLEtBQUssRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBRWhFLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFDaEUsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFL0MscUJBQWEsZ0JBQWdCO0lBV3pCLE9BQU8sQ0FBQyxPQUFPO0lBQ2YsT0FBTyxDQUFDLEtBQUs7SUFDYixPQUFPLENBQUMsZ0JBQWdCO0lBQ3hCLE9BQU8sQ0FBQyxjQUFjO0lBQ3RCLE9BQU8sQ0FBQyxTQUFTO0lBQ2pCLE9BQU8sQ0FBQyxNQUFNO0lBQ2QsT0FBTyxDQUFDLFlBQVk7SUFDcEIsT0FBTyxDQUFDLEdBQUc7SUFqQmIsbUVBQW1FO0lBQ25FLE9BQU8sQ0FBQyxVQUFVLENBQXlDO0lBRTNELCtEQUErRDtJQUMvRCxPQUFPLENBQUMsd0JBQXdCLENBQW1CO0lBRW5ELHFFQUFxRTtJQUNyRSxPQUFPLENBQUMseUJBQXlCLENBQWlCO0lBRWxELFlBQ1UsT0FBTyxFQUFFLElBQUksQ0FBQyxnQkFBZ0IsRUFBRSxrQkFBa0IsQ0FBQyxFQUNuRCxLQUFLLEVBQUUsUUFBUSxFQUFFLEVBQ2pCLGdCQUFnQixFQUFFLGdCQUFnQixFQUNsQyxjQUFjLEVBQUUsSUFBSSxDQUFDLGdCQUFnQixFQUFFLDJCQUEyQixDQUFDLEVBQ25FLFNBQVMsRUFBRSxpQkFBaUIsRUFDNUIsTUFBTSxFQUFFLGtCQUFrQixFQUMxQixZQUFZLEdBQUUsWUFBaUMsRUFDL0MsR0FBRyxHQUFFLE1BQWtELEVBZ0JoRTtJQUVNLGtCQUFrQixhQUV4QjtJQUVNLEtBQUssU0FHWDtJQUVZLElBQUksa0JBS2hCO0lBRVksT0FBTyxrQkFLbkI7SUFFRCxtRkFBbUY7SUFDNUUsZUFBZSxDQUFDLEtBQUssRUFBRSxPQUFPLEVBQUUsUUFBUSxFQUFFLE1BQU0sRUFBRSxRQW9CeEQ7WUFHYSx5QkFBeUI7WUFpQ3pCLDJCQUEyQjtJQXNDekMsK0hBQStIO0lBQy9ILE9BQU8sQ0FBQyw4QkFBOEI7SUFxQnRDLDhHQUE4RztJQUN2RyxRQUFRLENBQUMsR0FBRyxFQUFFLEVBQUUsRUFBRSxHQUFHLElBQUksQ0FJL0I7SUFFRDs7O09BR0c7SUFDSSwyQkFBMkIsQ0FBQyxXQUFXLEVBQUUsV0FBVyxHQUFHLElBQUksQ0FNakU7SUFFRDs7O09BR0c7SUFDSSw0QkFBNEIsQ0FBQyxXQUFXLEVBQUUsV0FBVyxHQUFHLElBQUksQ0FNbEU7SUFFRCxvSUFBb0k7SUFDcEksT0FBTyxDQUFDLGtCQUFrQjtDQU0zQiJ9

package/dest/services/tx_collection/slow_tx_collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"slow_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/slow_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAA2B,MAAM,iCAAiC,CAAC;AAEvF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,KAAK,iBAAiB,EAA6C,MAAM,6BAA6B,CAAC;AAChH,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,KAAK,gBAAgB,EAAsB,MAAM,yBAAyB,CAAC;AAEpF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAWzB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IAjBb,mEAAmE;IACnE,OAAO,CAAC,UAAU,CAAyC;IAE3D,+DAA+D;IAC/D,OAAO,CAAC,wBAAwB,CAAmB;IAEnD,qEAAqE;IACrE,OAAO,CAAC,yBAAyB,CAAiB;IAElD,YACU,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,EACnD,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,cAAc,EAAE,IAAI,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,EACnE,SAAS,EAAE,iBAAiB,EAC5B,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD,EAgBhE;IAEM,kBAAkB,aAExB;IAEM,KAAK,SAGX;IAEY,IAAI,kBAKhB;IAEY,OAAO,kBAKnB;IAED,mFAAmF;IAC5E,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAoBxD;YAGa,yBAAyB;YAiCzB,2BAA2B;IAsCzC,+HAA+H;IAC/H,OAAO,CAAC,8BAA8B;IAqBtC,8GAA8G;IACvG,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,IAAI,CAI/B;IAED;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAMjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAMlE;IAED,oIAAoI;IACpI,OAAO,CAAC,kBAAkB;CAM3B"}