@aztec/p2p 0.0.0-test.1 → 0.0.1-commit.5476d83

package/dest/services/tx_collection/instrumentation.d.ts

@@ -0,0 +1,10 @@
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import type { CollectionMethod } from './tx_collection.js';
+export declare class TxCollectionInstrumentation {
+    private txsCollected;
+    private collectionDurationPerTx;
+    private collectionDurationPerRequest;
+    constructor(client: TelemetryClient, name: string);
+    increaseTxsFor(what: CollectionMethod, count: number, duration: number): void;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5zdHJ1bWVudGF0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9pbnN0cnVtZW50YXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUlMLEtBQUssZUFBZSxFQUdyQixNQUFNLHlCQUF5QixDQUFDO0FBRWpDLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFFM0QscUJBQWEsMkJBQTJCO0lBQ3RDLE9BQU8sQ0FBQyxZQUFZLENBQWdCO0lBQ3BDLE9BQU8sQ0FBQyx1QkFBdUIsQ0FBWTtJQUMzQyxPQUFPLENBQUMsNEJBQTRCLENBQVk7SUFFaEQsWUFBWSxNQUFNLEVBQUUsZUFBZSxFQUFFLElBQUksRUFBRSxNQUFNLEVBa0JoRDtJQUVELGNBQWMsQ0FBQyxJQUFJLEVBQUUsZ0JBQWdCLEVBQUUsS0FBSyxFQUFFLE1BQU0sRUFBRSxRQUFRLEVBQUUsTUFBTSxRQUtyRTtDQUNGIn0=

package/dest/services/tx_collection/instrumentation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"instrumentation.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,qBAAa,2BAA2B;IACtC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,uBAAuB,CAAY;IAC3C,OAAO,CAAC,4BAA4B,CAAY;IAEhD,YAAY,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,EAkBhD;IAED,cAAc,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,QAKrE;CACF"}

package/dest/services/tx_collection/instrumentation.js

@@ -0,0 +1,34 @@
+import { Attributes, Metrics, ValueType } from '@aztec/telemetry-client';
+export class TxCollectionInstrumentation {
+    txsCollected;
+    collectionDurationPerTx;
+    collectionDurationPerRequest;
+    constructor(client, name){
+        const meter = client.getMeter(name);
+        this.txsCollected = meter.createUpDownCounter(Metrics.TX_COLLECTOR_COUNT, {
+            description: 'The number of txs collected'
+        });
+        this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX, {
+            unit: 'ms',
+            description: 'Average duration per tx of an individual tx collection request',
+            valueType: ValueType.INT
+        });
+        this.collectionDurationPerRequest = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_REQUEST, {
+            unit: 'ms',
+            description: 'Total duration of an individual tx collection request',
+            valueType: ValueType.INT
+        });
+    }
+    increaseTxsFor(what, count, duration) {
+        const durationPerTx = Math.ceil(duration / count);
+        this.collectionDurationPerTx.record(durationPerTx, {
+            [Attributes.TX_COLLECTION_METHOD]: what
+        });
+        this.collectionDurationPerRequest.record(Math.ceil(duration), {
+            [Attributes.TX_COLLECTION_METHOD]: what
+        });
+        this.txsCollected.add(count, {
+            [Attributes.TX_COLLECTION_METHOD]: what
+        });
+    }
+}

package/dest/services/tx_collection/slow_tx_collection.d.ts

@@ -0,0 +1,52 @@
+import { type Logger } from '@aztec/foundation/log';
+import { DateProvider } from '@aztec/foundation/timer';
+import type { L2Block } from '@aztec/stdlib/block';
+import { type L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
+import { type Tx, TxHash } from '@aztec/stdlib/tx';
+import { type ReqRespInterface } from '../reqresp/interface.js';
+import type { TxCollectionConfig } from './config.js';
+import type { FastTxCollection } from './fast_tx_collection.js';
+import type { TxCollectionSink } from './tx_collection_sink.js';
+import type { TxSource } from './tx_source.js';
+export declare class SlowTxCollection {
+    private reqResp;
+    private nodes;
+    private txCollectionSink;
+    private fastCollection;
+    private constants;
+    private config;
+    private dateProvider;
+    private log;
+    /** Map from tx hash to missing tx info to collect via slow loop */
+    private missingTxs;
+    /** One slow collection loop for each node tx source we have */
+    private nodesSlowCollectionLoops;
+    /** Slow collection loop for reqresp which collects txs from peers */
+    private reqrespSlowCollectionLoop;
+    constructor(reqResp: Pick<ReqRespInterface, 'sendBatchRequest'>, nodes: TxSource[], txCollectionSink: TxCollectionSink, fastCollection: Pick<FastTxCollection, 'getFastCollectionRequests'>, constants: L1RollupConstants, config: TxCollectionConfig, dateProvider?: DateProvider, log?: Logger);
+    getMissingTxHashes(): TxHash[];
+    start(): void;
+    stop(): Promise<void>;
+    trigger(): Promise<void>;
+    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
+    startCollecting(block: L2Block, txHashes: TxHash[]): void;
+    private collectMissingTxsFromNode;
+    private collectMissingTxsViaReqResp;
+    /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */
+    private getMissingTxsForSlowCollection;
+    /** Stop collecting the given txs since we have found them. Called whenever tx pool emits a tx-added event. */
+    foundTxs(txs: Tx[]): void;
+    /**
+     * Stop collecting all txs for blocks less than or requal to the block number specified.
+     * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+     */
+    stopCollectingForBlocksUpTo(blockNumber: number): void;
+    /**
+     * Stop collecting all txs for blocks greater than the block number specified.
+     * To be called when there is a chain prune and previously mined txs are no longer relevant.
+     */
+    stopCollectingForBlocksAfter(blockNumber: number): void;
+    /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */
+    private getDeadlineForSlot;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2xvd190eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9zbG93X3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBRUEsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBR2xFLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUN2RCxPQUFPLEtBQUssRUFBRSxPQUFPLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUNuRCxPQUFPLEVBQUUsS0FBSyxpQkFBaUIsRUFBNkMsTUFBTSw2QkFBNkIsQ0FBQztBQUNoSCxPQUFPLEVBQUUsS0FBSyxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sa0JBQWtCLENBQUM7QUFFbkQsT0FBTyxFQUFFLEtBQUssZ0JBQWdCLEVBQXNCLE1BQU0seUJBQXlCLENBQUM7QUFFcEYsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFDdEQsT0FBTyxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUVoRSxPQUFPLEtBQUssRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLFFBQVEsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBRS9DLHFCQUFhLGdCQUFnQjtJQVd6QixPQUFPLENBQUMsT0FBTztJQUNmLE9BQU8sQ0FBQyxLQUFLO0lBQ2IsT0FBTyxDQUFDLGdCQUFnQjtJQUN4QixPQUFPLENBQUMsY0FBYztJQUN0QixPQUFPLENBQUMsU0FBUztJQUNqQixPQUFPLENBQUMsTUFBTTtJQUNkLE9BQU8sQ0FBQyxZQUFZO0lBQ3BCLE9BQU8sQ0FBQyxHQUFHO0lBakJiLG1FQUFtRTtJQUNuRSxPQUFPLENBQUMsVUFBVSxDQUF5QztJQUUzRCwrREFBK0Q7SUFDL0QsT0FBTyxDQUFDLHdCQUF3QixDQUFtQjtJQUVuRCxxRUFBcUU7SUFDckUsT0FBTyxDQUFDLHlCQUF5QixDQUFpQjtJQUVsRCxZQUNVLE9BQU8sRUFBRSxJQUFJLENBQUMsZ0JBQWdCLEVBQUUsa0JBQWtCLENBQUMsRUFDbkQsS0FBSyxFQUFFLFFBQVEsRUFBRSxFQUNqQixnQkFBZ0IsRUFBRSxnQkFBZ0IsRUFDbEMsY0FBYyxFQUFFLElBQUksQ0FBQyxnQkFBZ0IsRUFBRSwyQkFBMkIsQ0FBQyxFQUNuRSxTQUFTLEVBQUUsaUJBQWlCLEVBQzVCLE1BQU0sRUFBRSxrQkFBa0IsRUFDMUIsWUFBWSxHQUFFLFlBQWlDLEVBQy9DLEdBQUcsR0FBRSxNQUFrRCxFQWdCaEU7SUFFTSxrQkFBa0IsYUFFeEI7SUFFTSxLQUFLLFNBR1g7SUFFWSxJQUFJLGtCQUtoQjtJQUVZLE9BQU8sa0JBS25CO0lBRUQsbUZBQW1GO0lBQzVFLGVBQWUsQ0FBQyxLQUFLLEVBQUUsT0FBTyxFQUFFLFFBQVEsRUFBRSxNQUFNLEVBQUUsUUFvQnhEO1lBR2EseUJBQXlCO1lBaUN6QiwyQkFBMkI7SUFzQ3pDLCtIQUErSDtJQUMvSCxPQUFPLENBQUMsOEJBQThCO0lBcUJ0Qyw4R0FBOEc7SUFDdkcsUUFBUSxDQUFDLEdBQUcsRUFBRSxFQUFFLEVBQUUsR0FBRyxJQUFJLENBSS9CO0lBRUQ7OztPQUdHO0lBQ0ksMkJBQTJCLENBQUMsV0FBVyxFQUFFLE1BQU0sR0FBRyxJQUFJLENBTTVEO0lBRUQ7OztPQUdHO0lBQ0ksNEJBQTRCLENBQUMsV0FBVyxFQUFFLE1BQU0sR0FBRyxJQUFJLENBTTdEO0lBRUQsb0lBQW9JO0lBQ3BJLE9BQU8sQ0FBQyxrQkFBa0I7Q0FNM0IifQ==

package/dest/services/tx_collection/slow_tx_collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"slow_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/slow_tx_collection.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,KAAK,iBAAiB,EAA6C,MAAM,6BAA6B,CAAC;AAChH,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,KAAK,gBAAgB,EAAsB,MAAM,yBAAyB,CAAC;AAEpF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAWzB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IAjBb,mEAAmE;IACnE,OAAO,CAAC,UAAU,CAAyC;IAE3D,+DAA+D;IAC/D,OAAO,CAAC,wBAAwB,CAAmB;IAEnD,qEAAqE;IACrE,OAAO,CAAC,yBAAyB,CAAiB;IAElD,YACU,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,EACnD,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,cAAc,EAAE,IAAI,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,EACnE,SAAS,EAAE,iBAAiB,EAC5B,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD,EAgBhE;IAEM,kBAAkB,aAExB;IAEM,KAAK,SAGX;IAEY,IAAI,kBAKhB;IAEY,OAAO,kBAKnB;IAED,mFAAmF;IAC5E,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAoBxD;YAGa,yBAAyB;YAiCzB,2BAA2B;IAsCzC,+HAA+H;IAC/H,OAAO,CAAC,8BAA8B;IAqBtC,8GAA8G;IACvG,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,IAAI,CAI/B;IAED;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAM5D;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAM7D;IAED,oIAAoI;IACpI,OAAO,CAAC,kBAAkB;CAM3B"}

package/dest/services/tx_collection/slow_tx_collection.js

@@ -0,0 +1,177 @@
+import { EpochNumber } from '@aztec/foundation/branded-types';
+import { chunk } from '@aztec/foundation/collection';
+import { createLogger } from '@aztec/foundation/log';
+import { boundInclusive } from '@aztec/foundation/number';
+import { RunningPromise } from '@aztec/foundation/promise';
+import { DateProvider } from '@aztec/foundation/timer';
+import { getEpochAtSlot, getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
+import { TxHash } from '@aztec/stdlib/tx';
+import { ReqRespSubProtocol } from '../reqresp/interface.js';
+import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+export class SlowTxCollection {
+    reqResp;
+    nodes;
+    txCollectionSink;
+    fastCollection;
+    constants;
+    config;
+    dateProvider;
+    log;
+    /** Map from tx hash to missing tx info to collect via slow loop */ missingTxs;
+    /** One slow collection loop for each node tx source we have */ nodesSlowCollectionLoops;
+    /** Slow collection loop for reqresp which collects txs from peers */ reqrespSlowCollectionLoop;
+    constructor(reqResp, nodes, txCollectionSink, fastCollection, constants, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service')){
+        this.reqResp = reqResp;
+        this.nodes = nodes;
+        this.txCollectionSink = txCollectionSink;
+        this.fastCollection = fastCollection;
+        this.constants = constants;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+        this.missingTxs = new Map();
+        this.nodesSlowCollectionLoops = this.nodes.map((node)=>new RunningPromise(()=>this.collectMissingTxsFromNode(node), this.log, this.config.txCollectionSlowNodesIntervalMs));
+        this.reqrespSlowCollectionLoop = new RunningPromise(()=>this.collectMissingTxsViaReqResp(), this.log, this.config.txCollectionSlowReqRespIntervalMs);
+    }
+    getMissingTxHashes() {
+        return Array.from(this.missingTxs.keys()).map(TxHash.fromString);
+    }
+    start() {
+        this.nodesSlowCollectionLoops.forEach((loop)=>loop.start());
+        this.reqrespSlowCollectionLoop.start();
+    }
+    async stop() {
+        await Promise.all([
+            this.reqrespSlowCollectionLoop.stop(),
+            ...this.nodesSlowCollectionLoops.map((loop)=>loop.stop())
+        ]);
+    }
+    async trigger() {
+        await Promise.all([
+            this.reqrespSlowCollectionLoop.trigger(),
+            ...this.nodesSlowCollectionLoops.map((loop)=>loop.trigger())
+        ]);
+    }
+    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */ startCollecting(block, txHashes) {
+        const slot = block.header.getSlot();
+        const deadline = this.getDeadlineForSlot(slot);
+        if (+deadline < this.dateProvider.now()) {
+            this.log.debug(`Skipping collection of txs for block ${block.number} at slot ${slot} as it is already expired`, {
+                blockNumber: block.number,
+                slot: slot.toString(),
+                txHashes: txHashes.map((txHash)=>txHash.toString()),
+                deadline: +deadline,
+                now: this.dateProvider.now()
+            });
+        }
+        for (const txHash of txHashes){
+            this.missingTxs.set(txHash.toString(), {
+                blockNumber: block.number,
+                deadline: this.getDeadlineForSlot(block.header.getSlot()),
+                readyForReqResp: this.nodes.length === 0
+            });
+        }
+    }
+    /** Entrypoint for the node slow collection loop */ async collectMissingTxsFromNode(node) {
+        // If we have any fast requests, we skip the slow collection for this node if configured to do so
+        const requests = this.fastCollection.getFastCollectionRequests();
+        if (this.config.txCollectionDisableSlowDuringFastRequests && requests.size > 0) {
+            this.log.trace(`Skipping node slow collection due to active fast requests`);
+            return;
+        }
+        // Gather all missing txs that are not in fast collection and request them from the node
+        const missingTxs = this.getMissingTxsForSlowCollection();
+        const missingTxHashes = missingTxs.map(([txHash])=>txHash).map(TxHash.fromString);
+        if (missingTxHashes.length === 0) {
+            return;
+        }
+        // Request in chunks to avoid hitting RPC limits
+        for (const batch of chunk(missingTxHashes, this.config.txCollectionNodeRpcMaxBatchSize)){
+            await this.txCollectionSink.collect((txHashes)=>node.getTxsByHash(txHashes), batch, {
+                description: `node ${node.getInfo()}`,
+                node: node.getInfo(),
+                method: 'slow-node-rpc'
+            });
+        }
+        // Mark every tx that is still missing as ready for reqresp.
+        // Note that we can just mark all requested txs as ready for reqresp, without filtering out the ones
+        // we retrieved, since the ones already found will have been removed from `missingTxs`.
+        missingTxs.forEach(([_txHash, info])=>{
+            info.readyForReqResp = true;
+        });
+    }
+    /** Entrypoint for the reqresp slow collection loop */ async collectMissingTxsViaReqResp() {
+        // If we have any fast requests, we skip the slow collection if configured to do so
+        const requests = this.fastCollection.getFastCollectionRequests();
+        if (this.config.txCollectionDisableSlowDuringFastRequests && requests.size > 0) {
+            this.log.trace(`Skipping reqresp slow collection due to active fast requests`);
+            return;
+        }
+        // Gather all missing txs that are not in fast collection and are ready for reqresp
+        // A tx is flagged as ready for reqresp if it has been requested from a node at least once
+        const missingTxs = this.getMissingTxsForSlowCollection({
+            onlyReqRespReady: true
+        });
+        if (missingTxs.length === 0) {
+            return;
+        }
+        const pinnedPeer = undefined;
+        const timeoutMs = this.config.txCollectionSlowReqRespTimeoutMs;
+        const maxPeers = boundInclusive(Math.ceil(missingTxs.length / 3), 4, 16);
+        const maxRetryAttempts = 3;
+        // Send a batch request via reqresp for the missing txs
+        await this.txCollectionSink.collect(async (txHashes)=>{
+            const txs = await this.reqResp.sendBatchRequest(ReqRespSubProtocol.TX, chunkTxHashesRequest(txHashes), pinnedPeer, timeoutMs, maxPeers, maxRetryAttempts);
+            return txs.flat();
+        }, missingTxs.map(([txHash])=>TxHash.fromString(txHash)), {
+            description: 'slow reqresp',
+            timeoutMs,
+            method: 'slow-req-resp'
+        });
+    }
+    /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */ getMissingTxsForSlowCollection(opts = {}) {
+        // Remove expired txs from missingTxs
+        const now = this.dateProvider.now();
+        const expiredTxs = Array.from(this.missingTxs.entries()).filter(([_, value])=>+value.deadline < now);
+        expiredTxs.forEach(([txHash])=>this.missingTxs.delete(txHash));
+        // Gather all txs that are marked for fast collection, we do not want to collect them via slow collection.
+        // There are some situations where a tx is in both slow and fast collection, for example when a prover node
+        // is fast-collecting missing txs for proving an epoch, and still has the tx in the slow collection loops
+        // from mined unproven blocks it has seen in the past.
+        const fastRequests = this.fastCollection.getFastCollectionRequests();
+        const fastCollectionTxs = new Set(...Array.from(fastRequests.values()).flatMap((r)=>r.missingTxHashes));
+        // Return all missing txs that are not in fastCollectionTxs and are ready for reqresp if requested
+        return Array.from(this.missingTxs.entries()).filter(([txHash, _])=>!fastCollectionTxs.has(txHash)).filter(([_, value])=>!opts.onlyReqRespReady || value.readyForReqResp);
+    }
+    /** Stop collecting the given txs since we have found them. Called whenever tx pool emits a tx-added event. */ foundTxs(txs) {
+        for (const txHash of txs.map((tx)=>tx.txHash)){
+            this.missingTxs.delete(txHash.toString());
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+   */ stopCollectingForBlocksUpTo(blockNumber) {
+        for (const [txHash, info] of this.missingTxs.entries()){
+            if (info.blockNumber <= blockNumber) {
+                this.missingTxs.delete(txHash);
+            }
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks greater than the block number specified.
+   * To be called when there is a chain prune and previously mined txs are no longer relevant.
+   */ stopCollectingForBlocksAfter(blockNumber) {
+        for (const [txHash, info] of this.missingTxs.entries()){
+            if (info.blockNumber > blockNumber) {
+                this.missingTxs.delete(txHash);
+            }
+        }
+    }
+    /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */ getDeadlineForSlot(slotNumber) {
+        const epoch = getEpochAtSlot(slotNumber, this.constants);
+        const submissionEndEpoch = EpochNumber(epoch + this.constants.proofSubmissionEpochs);
+        const submissionEndTimestamp = getTimestampRangeForEpoch(submissionEndEpoch, this.constants)[1];
+        return new Date(Number(submissionEndTimestamp) * 1000);
+    }
+}

package/dest/services/tx_collection/tx_collection.d.ts

@@ -0,0 +1,109 @@
+import { type Logger } from '@aztec/foundation/log';
+import { type PromiseWithResolvers } from '@aztec/foundation/promise';
+import { DateProvider } from '@aztec/foundation/timer';
+import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
+import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
+import type { BlockProposal } from '@aztec/stdlib/p2p';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import type { PeerId } from '@libp2p/interface';
+import type { TxPool } from '../../mem_pools/index.js';
+import type { TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
+import type { ReqRespInterface } from '../reqresp/interface.js';
+import type { TxCollectionConfig } from './config.js';
+import { FastTxCollection } from './fast_tx_collection.js';
+import { SlowTxCollection } from './slow_tx_collection.js';
+import type { TxSource } from './tx_source.js';
+export type CollectionMethod = 'fast-req-resp' | 'fast-node-rpc' | 'slow-req-resp' | 'slow-node-rpc';
+export type MissingTxInfo = {
+    blockNumber: number;
+    deadline: Date;
+    readyForReqResp: boolean;
+};
+export type FastCollectionRequestInput = {
+    type: 'block';
+    block: L2Block;
+} | {
+    type: 'proposal';
+    blockProposal: BlockProposal;
+    blockNumber: number;
+};
+export type FastCollectionRequest = FastCollectionRequestInput & {
+    missingTxHashes: Set<string>;
+    deadline: Date;
+    blockInfo: L2BlockInfo;
+    promise: PromiseWithResolvers<void>;
+    foundTxs: Map<string, Tx>;
+};
+/**
+ * Coordinates tx collection from remote RPC nodes and reqresp.
+ *
+ * The slow collection loops are used for periodically collecting missing txs from mined blocks,
+ * from remote RPC nodes and via reqresp. The fast collection methods are used to quickly gather
+ * txs, usually for attesting to block proposals or preparing to prove an epoch. The slow and fast
+ * collection instances both send their txs to the collection sink, which handles metrics and adds
+ * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is notified
+ * via events and notifies the slow and fast collection loops to stop collecting that tx, so that we don't
+ * collect the same tx multiple times.
+ */
+export declare class TxCollection {
+    private readonly reqResp;
+    private readonly nodes;
+    private readonly constants;
+    private readonly txPool;
+    private readonly config;
+    private readonly dateProvider;
+    private readonly log;
+    /** Slow collection background loops */
+    protected readonly slowCollection: SlowTxCollection;
+    /** Fast collection methods */
+    protected readonly fastCollection: FastTxCollection;
+    /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */
+    private readonly reconcileFoundTxsLoop;
+    /** Handles txs found by the slow and fast collection loops */
+    private readonly txCollectionSink;
+    /** Handler for the txs-added event from the tx pool */
+    protected readonly handleTxsAddedToPool: TxPoolEvents['txs-added'];
+    /** Handler for the txs-added event from the tx collection sink */
+    protected readonly handleTxsFound: TxPoolEvents['txs-added'];
+    constructor(reqResp: Pick<ReqRespInterface, 'sendBatchRequest'>, nodes: TxSource[], constants: L1RollupConstants, txPool: TxPool, config: TxCollectionConfig, dateProvider?: DateProvider, telemetryClient?: TelemetryClient, log?: Logger);
+    /** Starts all collection loops. */
+    start(): Promise<void>;
+    /** Stops all activity. */
+    stop(): Promise<void>;
+    /** Force trigger the slow collection and reconciliation loops */
+    trigger(): Promise<void>;
+    /** Returns L1 rollup constants. */
+    getConstants(): L1RollupConstants;
+    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
+    startCollecting(block: L2Block, txHashes: TxHash[]): void;
+    /** Collects the set of txs for the given block proposal as fast as possible */
+    collectFastForProposal(blockProposal: BlockProposal, blockNumber: number, txHashes: TxHash[] | string[], opts: {
+        deadline: Date;
+        pinnedPeer?: PeerId;
+    }): Promise<Tx[]>;
+    /** Collects the set of txs for the given mined block as fast as possible */
+    collectFastForBlock(block: L2Block, txHashes: TxHash[] | string[], opts: {
+        deadline: Date;
+        pinnedPeer?: PeerId;
+    }): Promise<Tx[]>;
+    /** Collects the set of txs for the given proposal or block as fast as possible */
+    collectFastFor(input: FastCollectionRequestInput, txHashes: TxHash[] | string[], opts: {
+        deadline: Date;
+        pinnedPeer?: PeerId;
+    }): Promise<Tx[]>;
+    /** Mark the given txs as found. Stops collecting them. */
+    private foundTxs;
+    /**
+     * Stop collecting all txs for blocks less than or requal to the block number specified.
+     * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+     */
+    stopCollectingForBlocksUpTo(blockNumber: number): void;
+    /**
+     * Stop collecting all txs for blocks greater than the block number specified.
+     * To be called when there is a chain prune and previously mined txs are no longer relevant.
+     */
+    stopCollectingForBlocksAfter(blockNumber: number): void;
+    private reconcileFoundTxsWithPool;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfY29sbGVjdGlvbi5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NlcnZpY2VzL3R4X2NvbGxlY3Rpb24vdHhfY29sbGVjdGlvbi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxPQUFPLEVBQUUsS0FBSyxNQUFNLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFDbEUsT0FBTyxFQUFFLEtBQUssb0JBQW9CLEVBQWtCLE1BQU0sMkJBQTJCLENBQUM7QUFDdEYsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ3ZELE9BQU8sS0FBSyxFQUFFLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUNoRSxPQUFPLEtBQUssRUFBRSxpQkFBaUIsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQ3JFLE9BQU8sS0FBSyxFQUFFLGFBQWEsRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBQ3ZELE9BQU8sRUFBRSxFQUFFLEVBQUUsTUFBTSxFQUFFLE1BQU0sa0JBQWtCLENBQUM7QUFDOUMsT0FBTyxFQUFFLEtBQUssZUFBZSxFQUFzQixNQUFNLHlCQUF5QixDQUFDO0FBRW5GLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBRWhELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLDBCQUEwQixDQUFDO0FBQ3ZELE9BQU8sS0FBSyxFQUFFLFlBQVksRUFBRSxNQUFNLG9DQUFvQyxDQUFDO0FBQ3ZFLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFDaEUsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFDdEQsT0FBTyxFQUFFLGdCQUFnQixFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFDM0QsT0FBTyxFQUFFLGdCQUFnQixFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFFM0QsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFL0MsTUFBTSxNQUFNLGdCQUFnQixHQUFHLGVBQWUsR0FBRyxlQUFlLEdBQUcsZUFBZSxHQUFHLGVBQWUsQ0FBQztBQUVyRyxNQUFNLE1BQU0sYUFBYSxHQUFHO0lBQUUsV0FBVyxFQUFFLE1BQU0sQ0FBQztJQUFDLFFBQVEsRUFBRSxJQUFJLENBQUM7SUFBQyxlQUFlLEVBQUUsT0FBTyxDQUFBO0NBQUUsQ0FBQztBQUU5RixNQUFNLE1BQU0sMEJBQTBCLEdBQ2xDO0lBQUUsSUFBSSxFQUFFLE9BQU8sQ0FBQztJQUFDLEtBQUssRUFBRSxPQUFPLENBQUE7Q0FBRSxHQUNqQztJQUFFLElBQUksRUFBRSxVQUFVLENBQUM7SUFBQyxhQUFhLEVBQUUsYUFBYSxDQUFDO0lBQUMsV0FBVyxFQUFFLE1BQU0sQ0FBQTtDQUFFLENBQUM7QUFFNUUsTUFBTSxNQUFNLHFCQUFxQixHQUFHLDBCQUEwQixHQUFHO0lBQy9ELGVBQWUsRUFBRSxHQUFHLENBQUMsTUFBTSxDQUFDLENBQUM7SUFDN0IsUUFBUSxFQUFFLElBQUksQ0FBQztJQUNmLFNBQVMsRUFBRSxXQUFXLENBQUM7SUFDdkIsT0FBTyxFQUFFLG9CQUFvQixDQUFDLElBQUksQ0FBQyxDQUFDO0lBQ3BDLFFBQVEsRUFBRSxHQUFHLENBQUMsTUFBTSxFQUFFLEVBQUUsQ0FBQyxDQUFDO0NBQzNCLENBQUM7QUFFRjs7Ozs7Ozs7OztHQVVHO0FBQ0gscUJBQWEsWUFBWTtJQW9CckIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxPQUFPO0lBQ3hCLE9BQU8sQ0FBQyxRQUFRLENBQUMsS0FBSztJQUN0QixPQUFPLENBQUMsUUFBUSxDQUFDLFNBQVM7SUFDMUIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxNQUFNO0lBQ3ZCLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQUN2QixPQUFPLENBQUMsUUFBUSxDQUFDLFlBQVk7SUFFN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHO0lBMUJ0Qix1Q0FBdUM7SUFDdkMsU0FBUyxDQUFDLFFBQVEsQ0FBQyxjQUFjLEVBQUUsZ0JBQWdCLENBQUM7SUFFcEQsOEJBQThCO0lBQzlCLFNBQVMsQ0FBQyxRQUFRLENBQUMsY0FBYyxFQUFFLGdCQUFnQixDQUFDO0lBRXBELG1HQUFtRztJQUNuRyxPQUFPLENBQUMsUUFBUSxDQUFDLHFCQUFxQixDQUFpQjtJQUV2RCw4REFBOEQ7SUFDOUQsT0FBTyxDQUFDLFFBQVEsQ0FBQyxnQkFBZ0IsQ0FBbUI7SUFFcEQsdURBQXVEO0lBQ3ZELFNBQVMsQ0FBQyxRQUFRLENBQUMsb0JBQW9CLEVBQUUsWUFBWSxDQUFDLFdBQVcsQ0FBQyxDQUFDO0lBRW5FLGtFQUFrRTtJQUNsRSxTQUFTLENBQUMsUUFBUSxDQUFDLGNBQWMsRUFBRSxZQUFZLENBQUMsV0FBVyxDQUFDLENBQUM7SUFFN0QsWUFDbUIsT0FBTyxFQUFFLElBQUksQ0FBQyxnQkFBZ0IsRUFBRSxrQkFBa0IsQ0FBQyxFQUNuRCxLQUFLLEVBQUUsUUFBUSxFQUFFLEVBQ2pCLFNBQVMsRUFBRSxpQkFBaUIsRUFDNUIsTUFBTSxFQUFFLE1BQU0sRUFDZCxNQUFNLEVBQUUsa0JBQWtCLEVBQzFCLFlBQVksR0FBRSxZQUFpQyxFQUNoRSxlQUFlLEdBQUUsZUFBc0MsRUFDdEMsR0FBRyxHQUFFLE1BQWtELEVBMEN6RTtJQUVELG1DQUFtQztJQUM1QixLQUFLLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQU01QjtJQUVELDBCQUEwQjtJQUNiLElBQUksa0JBS2hCO0lBRUQsaUVBQWlFO0lBQ3BELE9BQU8sa0JBRW5CO0lBRUQsbUNBQW1DO0lBQzVCLFlBQVksSUFBSSxpQkFBaUIsQ0FFdkM7SUFFRCxtRkFBbUY7SUFDNUUsZUFBZSxDQUFDLEtBQUssRUFBRSxPQUFPLEVBQUUsUUFBUSxFQUFFLE1BQU0sRUFBRSxRQUV4RDtJQUVELCtFQUErRTtJQUN4RSxzQkFBc0IsQ0FDM0IsYUFBYSxFQUFFLGFBQWEsRUFDNUIsV0FBVyxFQUFFLE1BQU0sRUFDbkIsUUFBUSxFQUFFLE1BQU0sRUFBRSxHQUFHLE1BQU0sRUFBRSxFQUM3QixJQUFJLEVBQUU7UUFBRSxRQUFRLEVBQUUsSUFBSSxDQUFDO1FBQUMsVUFBVSxDQUFDLEVBQUUsTUFBTSxDQUFBO0tBQUUsaUJBRzlDO0lBRUQsNEVBQTRFO0lBQ3JFLG1CQUFtQixDQUN4QixLQUFLLEVBQUUsT0FBTyxFQUNkLFFBQVEsRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLEVBQUUsRUFDN0IsSUFBSSxFQUFFO1FBQUUsUUFBUSxFQUFFLElBQUksQ0FBQztRQUFDLFVBQVUsQ0FBQyxFQUFFLE1BQU0sQ0FBQTtLQUFFLGlCQUc5QztJQUVELGtGQUFrRjtJQUMzRSxjQUFjLENBQ25CLEtBQUssRUFBRSwwQkFBMEIsRUFDakMsUUFBUSxFQUFFLE1BQU0sRUFBRSxHQUFHLE1BQU0sRUFBRSxFQUM3QixJQUFJLEVBQUU7UUFBRSxRQUFRLEVBQUUsSUFBSSxDQUFDO1FBQUMsVUFBVSxDQUFDLEVBQUUsTUFBTSxDQUFBO0tBQUUsaUJBRzlDO0lBRUQsMERBQTBEO0lBQzFELE9BQU8sQ0FBQyxRQUFRO0lBS2hCOzs7T0FHRztJQUNJLDJCQUEyQixDQUFDLFdBQVcsRUFBRSxNQUFNLEdBQUcsSUFBSSxDQUc1RDtJQUVEOzs7T0FHRztJQUNJLDRCQUE0QixDQUFDLFdBQVcsRUFBRSxNQUFNLEdBQUcsSUFBSSxDQUc3RDtZQUdhLHlCQUF5QjtDQVV4QyJ9

package/dest/services/tx_collection/tx_collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_collection.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,KAAK,oBAAoB,EAAkB,MAAM,2BAA2B,CAAC;AACtF,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAEnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,MAAM,MAAM,gBAAgB,GAAG,eAAe,GAAG,eAAe,GAAG,eAAe,GAAG,eAAe,CAAC;AAErG,MAAM,MAAM,aAAa,GAAG;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,IAAI,CAAC;IAAC,eAAe,EAAE,OAAO,CAAA;CAAE,CAAC;AAE9F,MAAM,MAAM,0BAA0B,GAClC;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,aAAa,EAAE,aAAa,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E,MAAM,MAAM,qBAAqB,GAAG,0BAA0B,GAAG;IAC/D,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,EAAE,IAAI,CAAC;IACf,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,oBAAoB,CAAC,IAAI,CAAC,CAAC;IACpC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CAC3B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,YAAY;IAoBrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IA1BtB,uCAAuC;IACvC,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAEpD,8BAA8B;IAC9B,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAEpD,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAiB;IAEvD,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IAEpD,uDAAuD;IACvD,SAAS,CAAC,QAAQ,CAAC,oBAAoB,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;IAEnE,kEAAkE;IAClE,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;IAE7D,YACmB,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,EACnD,KAAK,EAAE,QAAQ,EAAE,EACjB,SAAS,EAAE,iBAAiB,EAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAChE,eAAe,GAAE,eAAsC,EACtC,GAAG,GAAE,MAAkD,EA0CzE;IAED,mCAAmC;IAC5B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAM5B;IAED,0BAA0B;IACb,IAAI,kBAKhB;IAED,iEAAiE;IACpD,OAAO,kBAEnB;IAED,mCAAmC;IAC5B,YAAY,IAAI,iBAAiB,CAEvC;IAED,mFAAmF;IAC5E,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAExD;IAED,+EAA+E;IACxE,sBAAsB,CAC3B,aAAa,EAAE,aAAa,EAC5B,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBAG9C;IAED,4EAA4E;IACrE,mBAAmB,CACxB,KAAK,EAAE,OAAO,EACd,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBAG9C;IAED,kFAAkF;IAC3E,cAAc,CACnB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBAG9C;IAED,0DAA0D;IAC1D,OAAO,CAAC,QAAQ;IAKhB;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAG5D;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAG7D;YAGa,yBAAyB;CAUxC"}

package/dest/services/tx_collection/tx_collection.js

@@ -0,0 +1,128 @@
+import { compactArray } from '@aztec/foundation/collection';
+import { createLogger } from '@aztec/foundation/log';
+import { RunningPromise } from '@aztec/foundation/promise';
+import { DateProvider } from '@aztec/foundation/timer';
+import { getTelemetryClient } from '@aztec/telemetry-client';
+import { FastTxCollection } from './fast_tx_collection.js';
+import { SlowTxCollection } from './slow_tx_collection.js';
+import { TxCollectionSink } from './tx_collection_sink.js';
+/**
+ * Coordinates tx collection from remote RPC nodes and reqresp.
+ *
+ * The slow collection loops are used for periodically collecting missing txs from mined blocks,
+ * from remote RPC nodes and via reqresp. The fast collection methods are used to quickly gather
+ * txs, usually for attesting to block proposals or preparing to prove an epoch. The slow and fast
+ * collection instances both send their txs to the collection sink, which handles metrics and adds
+ * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is notified
+ * via events and notifies the slow and fast collection loops to stop collecting that tx, so that we don't
+ * collect the same tx multiple times.
+ */ export class TxCollection {
+    reqResp;
+    nodes;
+    constants;
+    txPool;
+    config;
+    dateProvider;
+    log;
+    /** Slow collection background loops */ slowCollection;
+    /** Fast collection methods */ fastCollection;
+    /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */ reconcileFoundTxsLoop;
+    /** Handles txs found by the slow and fast collection loops */ txCollectionSink;
+    /** Handler for the txs-added event from the tx pool */ handleTxsAddedToPool;
+    /** Handler for the txs-added event from the tx collection sink */ handleTxsFound;
+    constructor(reqResp, nodes, constants, txPool, config, dateProvider = new DateProvider(), telemetryClient = getTelemetryClient(), log = createLogger('p2p:tx_collection_service')){
+        this.reqResp = reqResp;
+        this.nodes = nodes;
+        this.constants = constants;
+        this.txPool = txPool;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+        this.txCollectionSink = new TxCollectionSink(this.txPool, telemetryClient, this.log);
+        this.fastCollection = new FastTxCollection(this.reqResp, this.nodes, this.txCollectionSink, this.config, this.dateProvider, this.log);
+        this.slowCollection = new SlowTxCollection(this.reqResp, this.nodes, this.txCollectionSink, this.fastCollection, constants, this.config, this.dateProvider, this.log);
+        this.reconcileFoundTxsLoop = new RunningPromise(()=>this.reconcileFoundTxsWithPool(), this.log, this.config.txCollectionReconcileIntervalMs);
+        this.handleTxsFound = (args)=>{
+            this.foundTxs(args.txs);
+        };
+        this.txCollectionSink.on('txs-added', this.handleTxsFound);
+        this.handleTxsAddedToPool = (args)=>{
+            const { txs, source } = args;
+            if (source !== 'tx-collection') {
+                this.foundTxs(txs);
+            }
+        };
+        this.txPool.on('txs-added', this.handleTxsAddedToPool);
+    }
+    /** Starts all collection loops. */ start() {
+        this.slowCollection.start();
+        this.reconcileFoundTxsLoop.start();
+        // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup
+        return Promise.resolve();
+    }
+    /** Stops all activity. */ async stop() {
+        await Promise.all([
+            this.slowCollection.stop(),
+            this.fastCollection.stop(),
+            this.reconcileFoundTxsLoop.stop()
+        ]);
+        this.txPool.removeListener('txs-added', this.handleTxsAddedToPool);
+        this.txCollectionSink.removeListener('txs-added', this.handleTxsFound);
+    }
+    /** Force trigger the slow collection and reconciliation loops */ async trigger() {
+        await Promise.all([
+            this.reconcileFoundTxsLoop.trigger(),
+            this.slowCollection.trigger()
+        ]);
+    }
+    /** Returns L1 rollup constants. */ getConstants() {
+        return this.constants;
+    }
+    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */ startCollecting(block, txHashes) {
+        return this.slowCollection.startCollecting(block, txHashes);
+    }
+    /** Collects the set of txs for the given block proposal as fast as possible */ collectFastForProposal(blockProposal, blockNumber, txHashes, opts) {
+        return this.collectFastFor({
+            type: 'proposal',
+            blockProposal,
+            blockNumber
+        }, txHashes, opts);
+    }
+    /** Collects the set of txs for the given mined block as fast as possible */ collectFastForBlock(block, txHashes, opts) {
+        return this.collectFastFor({
+            type: 'block',
+            block
+        }, txHashes, opts);
+    }
+    /** Collects the set of txs for the given proposal or block as fast as possible */ collectFastFor(input, txHashes, opts) {
+        return this.fastCollection.collectFastFor(input, txHashes, opts);
+    }
+    /** Mark the given txs as found. Stops collecting them. */ foundTxs(txs) {
+        this.slowCollection.foundTxs(txs);
+        this.fastCollection.foundTxs(txs);
+    }
+    /**
+   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+   */ stopCollectingForBlocksUpTo(blockNumber) {
+        this.slowCollection.stopCollectingForBlocksUpTo(blockNumber);
+        this.fastCollection.stopCollectingForBlocksUpTo(blockNumber);
+    }
+    /**
+   * Stop collecting all txs for blocks greater than the block number specified.
+   * To be called when there is a chain prune and previously mined txs are no longer relevant.
+   */ stopCollectingForBlocksAfter(blockNumber) {
+        this.slowCollection.stopCollectingForBlocksAfter(blockNumber);
+        this.fastCollection.stopCollectingForBlocksAfter(blockNumber);
+    }
+    /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */ async reconcileFoundTxsWithPool() {
+        const missingTxHashes = this.slowCollection.getMissingTxHashes();
+        const foundTxs = compactArray(await this.txPool.getTxsByHash(missingTxHashes));
+        if (foundTxs.length > 0) {
+            this.log.verbose(`Found ${foundTxs.length} txs in the pool during reconciliation`, {
+                foundTxs: foundTxs.map((t)=>t.getTxHash().toString())
+            });
+            this.foundTxs(foundTxs);
+        }
+    }
+}

package/dest/services/tx_collection/tx_collection_sink.d.ts

@@ -0,0 +1,30 @@
+import type { Logger } from '@aztec/foundation/log';
+import type { TypedEventEmitter } from '@aztec/foundation/types';
+import { Tx, type TxHash } from '@aztec/stdlib/tx';
+import type { TelemetryClient } from '@aztec/telemetry-client';
+import type { TxPool, TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
+import type { CollectionMethod } from './tx_collection.js';
+declare const TxCollectionSink_base: new () => TypedEventEmitter<TxPoolEvents>;
+/**
+ * Executes collection requests from the fast and slow collection loops, and handles collected txs
+ * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
+ */
+export declare class TxCollectionSink extends TxCollectionSink_base {
+    private readonly txPool;
+    private readonly log;
+    private readonly instrumentation;
+    constructor(txPool: TxPool, telemetryClient: TelemetryClient, log: Logger);
+    collect(collectValidTxsFn: (txHashes: TxHash[]) => Promise<(Tx | undefined)[]>, requested: TxHash[], info: Record<string, any> & {
+        description: string;
+        method: CollectionMethod;
+    }): Promise<{
+        txs: Tx[];
+        requested: TxHash[];
+        duration: number;
+    } | {
+        txs: Tx[];
+    }>;
+    private foundTxs;
+}
+export {};
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfY29sbGVjdGlvbl9zaW5rLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi90eF9jb2xsZWN0aW9uX3NpbmsudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFFcEQsT0FBTyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUNqRSxPQUFPLEVBQUUsRUFBRSxFQUFFLEtBQUssTUFBTSxFQUFFLE1BQU0sa0JBQWtCLENBQUM7QUFDbkQsT0FBTyxLQUFLLEVBQUUsZUFBZSxFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFJL0QsT0FBTyxLQUFLLEVBQUUsTUFBTSxFQUFFLFlBQVksRUFBRSxNQUFNLG9DQUFvQyxDQUFDO0FBRS9FLE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sb0JBQW9CLENBQUM7O0FBRTNEOzs7R0FHRztBQUNILHFCQUFhLGdCQUFpQixTQUFRLHFCQUEyRDtJQUk3RixPQUFPLENBQUMsUUFBUSxDQUFDLE1BQU07SUFFdkIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHO0lBTHRCLE9BQU8sQ0FBQyxRQUFRLENBQUMsZUFBZSxDQUE4QjtJQUU5RCxZQUNtQixNQUFNLEVBQUUsTUFBTSxFQUMvQixlQUFlLEVBQUUsZUFBZSxFQUNmLEdBQUcsRUFBRSxNQUFNLEVBSTdCO0lBRVksT0FBTyxDQUNsQixpQkFBaUIsRUFBRSxDQUFDLFFBQVEsRUFBRSxNQUFNLEVBQUUsS0FBSyxPQUFPLENBQUMsQ0FBQyxFQUFFLEdBQUcsU0FBUyxDQUFDLEVBQUUsQ0FBQyxFQUN0RSxTQUFTLEVBQUUsTUFBTSxFQUFFLEVBQ25CLElBQUksRUFBRSxNQUFNLENBQUMsTUFBTSxFQUFFLEdBQUcsQ0FBQyxHQUFHO1FBQUUsV0FBVyxFQUFFLE1BQU0sQ0FBQztRQUFDLE1BQU0sRUFBRSxnQkFBZ0IsQ0FBQTtLQUFFOzs7Ozs7T0F1RTlFO1lBRWEsUUFBUTtDQXdCdkIifQ==

package/dest/services/tx_collection/tx_collection_sink.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx_collection_sink.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_collection_sink.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEpD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,EAAE,EAAE,KAAK,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAI/D,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAE/E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;;AAE3D;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,qBAA2D;IAI7F,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,GAAG;IALtB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA8B;IAE9D,YACmB,MAAM,EAAE,MAAM,EAC/B,eAAe,EAAE,eAAe,EACf,GAAG,EAAE,MAAM,EAI7B;IAEY,OAAO,CAClB,iBAAiB,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,EACtE,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,gBAAgB,CAAA;KAAE;;;;;;OAuE9E;YAEa,QAAQ;CAwBvB"}