@aztec/p2p 1.2.1 → 2.0.0-nightly.20250813

package/dest/services/tx_collection/slow_tx_collection.js

@@ -0,0 +1,176 @@
+import { chunk } from '@aztec/foundation/collection';
+import { createLogger } from '@aztec/foundation/log';
+import { boundInclusive } from '@aztec/foundation/number';
+import { RunningPromise } from '@aztec/foundation/promise';
+import { DateProvider } from '@aztec/foundation/timer';
+import { getEpochAtSlot, getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
+import { TxHash } from '@aztec/stdlib/tx';
+import { ReqRespSubProtocol } from '../reqresp/interface.js';
+import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+export class SlowTxCollection {
+    reqResp;
+    nodes;
+    txCollectionSink;
+    fastCollection;
+    constants;
+    config;
+    dateProvider;
+    log;
+    /** Map from tx hash to missing tx info to collect via slow loop */ missingTxs;
+    /** One slow collection loop for each node tx source we have */ nodesSlowCollectionLoops;
+    /** Slow collection loop for reqresp which collects txs from peers */ reqrespSlowCollectionLoop;
+    constructor(reqResp, nodes, txCollectionSink, fastCollection, constants, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service')){
+        this.reqResp = reqResp;
+        this.nodes = nodes;
+        this.txCollectionSink = txCollectionSink;
+        this.fastCollection = fastCollection;
+        this.constants = constants;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+        this.missingTxs = new Map();
+        this.nodesSlowCollectionLoops = this.nodes.map((node)=>new RunningPromise(()=>this.collectMissingTxsFromNode(node), this.log, this.config.txCollectionSlowNodesIntervalMs));
+        this.reqrespSlowCollectionLoop = new RunningPromise(()=>this.collectMissingTxsViaReqResp(), this.log, this.config.txCollectionSlowReqRespIntervalMs);
+    }
+    getMissingTxHashes() {
+        return Array.from(this.missingTxs.keys()).map(TxHash.fromString);
+    }
+    start() {
+        this.nodesSlowCollectionLoops.forEach((loop)=>loop.start());
+        this.reqrespSlowCollectionLoop.start();
+    }
+    async stop() {
+        await Promise.all([
+            this.reqrespSlowCollectionLoop.stop(),
+            ...this.nodesSlowCollectionLoops.map((loop)=>loop.stop())
+        ]);
+    }
+    async trigger() {
+        await Promise.all([
+            this.reqrespSlowCollectionLoop.trigger(),
+            ...this.nodesSlowCollectionLoops.map((loop)=>loop.trigger())
+        ]);
+    }
+    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */ startCollecting(block, txHashes) {
+        const slot = block.header.getSlot();
+        const deadline = this.getDeadlineForSlot(slot);
+        if (+deadline < this.dateProvider.now()) {
+            this.log.debug(`Skipping collection of txs for block ${block.number} at slot ${slot} as it is already expired`, {
+                blockNumber: block.number,
+                slot: slot.toString(),
+                txHashes: txHashes.map((txHash)=>txHash.toString()),
+                deadline: +deadline,
+                now: this.dateProvider.now()
+            });
+        }
+        for (const txHash of txHashes){
+            this.missingTxs.set(txHash.toString(), {
+                blockNumber: block.number,
+                deadline: this.getDeadlineForSlot(block.header.getSlot()),
+                readyForReqResp: this.nodes.length === 0
+            });
+        }
+    }
+    /** Entrypoint for the node slow collection loop */ async collectMissingTxsFromNode(node) {
+        // If we have any fast requests, we skip the slow collection for this node if configured to do so
+        const requests = this.fastCollection.getFastCollectionRequests();
+        if (this.config.txCollectionDisableSlowDuringFastRequests && requests.size > 0) {
+            this.log.trace(`Skipping node slow collection due to active fast requests`);
+            return;
+        }
+        // Gather all missing txs that are not in fast collection and request them from the node
+        const missingTxs = this.getMissingTxsForSlowCollection();
+        const missingTxHashes = missingTxs.map(([txHash])=>txHash).map(TxHash.fromString);
+        if (missingTxHashes.length === 0) {
+            return;
+        }
+        // Request in chunks to avoid hitting RPC limits
+        for (const batch of chunk(missingTxHashes, this.config.txCollectionNodeRpcMaxBatchSize)){
+            await this.txCollectionSink.collect((txHashes)=>node.getTxsByHash(txHashes), batch, {
+                description: `node ${node.getInfo()}`,
+                node: node.getInfo(),
+                method: 'slow-node-rpc'
+            });
+        }
+        // Mark every tx that is still missing as ready for reqresp.
+        // Note that we can just mark all requested txs as ready for reqresp, without filtering out the ones
+        // we retrieved, since the ones already found will have been removed from `missingTxs`.
+        missingTxs.forEach(([_txHash, info])=>{
+            info.readyForReqResp = true;
+        });
+    }
+    /** Entrypoint for the reqresp slow collection loop */ async collectMissingTxsViaReqResp() {
+        // If we have any fast requests, we skip the slow collection if configured to do so
+        const requests = this.fastCollection.getFastCollectionRequests();
+        if (this.config.txCollectionDisableSlowDuringFastRequests && requests.size > 0) {
+            this.log.trace(`Skipping reqresp slow collection due to active fast requests`);
+            return;
+        }
+        // Gather all missing txs that are not in fast collection and are ready for reqresp
+        // A tx is flagged as ready for reqresp if it has been requested from a node at least once
+        const missingTxs = this.getMissingTxsForSlowCollection({
+            onlyReqRespReady: true
+        });
+        if (missingTxs.length === 0) {
+            return;
+        }
+        const pinnedPeer = undefined;
+        const timeoutMs = this.config.txCollectionSlowReqRespTimeoutMs;
+        const maxPeers = boundInclusive(Math.ceil(missingTxs.length / 3), 4, 16);
+        const maxRetryAttempts = 3;
+        // Send a batch request via reqresp for the missing txs
+        await this.txCollectionSink.collect(async (txHashes)=>{
+            const txs = await this.reqResp.sendBatchRequest(ReqRespSubProtocol.TX, chunkTxHashesRequest(txHashes), pinnedPeer, timeoutMs, maxPeers, maxRetryAttempts);
+            return txs.flat();
+        }, missingTxs.map(([txHash])=>TxHash.fromString(txHash)), {
+            description: 'slow reqresp',
+            timeoutMs,
+            method: 'slow-req-resp'
+        });
+    }
+    /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */ getMissingTxsForSlowCollection(opts = {}) {
+        // Remove expired txs from missingTxs
+        const now = this.dateProvider.now();
+        const expiredTxs = Array.from(this.missingTxs.entries()).filter(([_, value])=>+value.deadline < now);
+        expiredTxs.forEach(([txHash])=>this.missingTxs.delete(txHash));
+        // Gather all txs that are marked for fast collection, we do not want to collect them via slow collection.
+        // There are some situations where a tx is in both slow and fast collection, for example when a prover node
+        // is fast-collecting missing txs for proving an epoch, and still has the tx in the slow collection loops
+        // from mined unproven blocks it has seen in the past.
+        const fastRequests = this.fastCollection.getFastCollectionRequests();
+        const fastCollectionTxs = new Set(...Array.from(fastRequests.values()).flatMap((r)=>r.missingTxHashes));
+        // Return all missing txs that are not in fastCollectionTxs and are ready for reqresp if requested
+        return Array.from(this.missingTxs.entries()).filter(([txHash, _])=>!fastCollectionTxs.has(txHash)).filter(([_, value])=>!opts.onlyReqRespReady || value.readyForReqResp);
+    }
+    /** Stop collecting the given txs since we have found them. Called whenever tx pool emits a tx-added event. */ foundTxs(txs) {
+        for (const txHash of txs.map((tx)=>tx.txHash)){
+            this.missingTxs.delete(txHash.toString());
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+   */ stopCollectingForBlocksUpTo(blockNumber) {
+        for (const [txHash, info] of this.missingTxs.entries()){
+            if (info.blockNumber <= blockNumber) {
+                this.missingTxs.delete(txHash);
+            }
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks greater than the block number specified.
+   * To be called when there is a chain prune and previously mined txs are no longer relevant.
+   */ stopCollectingForBlocksAfter(blockNumber) {
+        for (const [txHash, info] of this.missingTxs.entries()){
+            if (info.blockNumber > blockNumber) {
+                this.missingTxs.delete(txHash);
+            }
+        }
+    }
+    /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */ getDeadlineForSlot(slotNumber) {
+        const epoch = getEpochAtSlot(slotNumber, this.constants);
+        const submissionEndEpoch = epoch + BigInt(this.constants.proofSubmissionEpochs);
+        const submissionEndTimestamp = getTimestampRangeForEpoch(submissionEndEpoch, this.constants)[1];
+        return new Date(Number(submissionEndTimestamp) * 1000);
+    }
+}

package/dest/services/tx_collection/tx_collection.d.ts

@@ -0,0 +1,109 @@
+import { type Logger } from '@aztec/foundation/log';
+import { type PromiseWithResolvers } from '@aztec/foundation/promise';
+import { DateProvider } from '@aztec/foundation/timer';
+import type { BlockInfo, L2Block } from '@aztec/stdlib/block';
+import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
+import type { BlockProposal } from '@aztec/stdlib/p2p';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import type { PeerId } from '@libp2p/interface';
+import type { TxPool } from '../../mem_pools/index.js';
+import type { TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
+import type { ReqRespInterface } from '../reqresp/interface.js';
+import type { TxCollectionConfig } from './config.js';
+import { FastTxCollection } from './fast_tx_collection.js';
+import { SlowTxCollection } from './slow_tx_collection.js';
+import type { TxSource } from './tx_source.js';
+export type CollectionMethod = 'fast-req-resp' | 'fast-node-rpc' | 'slow-req-resp' | 'slow-node-rpc';
+export type MissingTxInfo = {
+    blockNumber: number;
+    deadline: Date;
+    readyForReqResp: boolean;
+};
+export type FastCollectionRequestInput = {
+    type: 'block';
+    block: L2Block;
+} | {
+    type: 'proposal';
+    blockProposal: BlockProposal;
+};
+export type FastCollectionRequest = FastCollectionRequestInput & {
+    missingTxHashes: Set<string>;
+    deadline: Date;
+    blockInfo: BlockInfo;
+    promise: PromiseWithResolvers<void>;
+    foundTxs: Map<string, Tx>;
+};
+/**
+ * Coordinates tx collection from remote RPC nodes and reqresp.
+ *
+ * The slow collection loops are used for periodically collecting missing txs from mined blocks,
+ * from remote RPC nodes and via reqresp. The fast collection methods are used to quickly gather
+ * txs, usually for attesting to block proposals or preparing to prove an epoch. The slow and fast
+ * collection instances both send their txs to the collection sink, which handles metrics and adds
+ * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is notified
+ * via events and notifies the slow and fast collection loops to stop collecting that tx, so that we don't
+ * collect the same tx multiple times.
+ */
+export declare class TxCollection {
+    private readonly reqResp;
+    private readonly nodes;
+    private readonly constants;
+    private readonly txPool;
+    private readonly config;
+    private readonly dateProvider;
+    private readonly log;
+    /** Slow collection background loops */
+    protected readonly slowCollection: SlowTxCollection;
+    /** Fast collection methods */
+    protected readonly fastCollection: FastTxCollection;
+    /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */
+    private readonly reconcileFoundTxsLoop;
+    /** Handles txs found by the slow and fast collection loops */
+    private readonly txCollectionSink;
+    /** Handler for the txs-added event from the tx pool */
+    protected readonly handleTxsAddedToPool: TxPoolEvents['txs-added'];
+    /** Handler for the txs-added event from the tx collection sink */
+    protected readonly handleTxsFound: TxPoolEvents['txs-added'];
+    constructor(reqResp: Pick<ReqRespInterface, 'sendBatchRequest'>, nodes: TxSource[], constants: L1RollupConstants, txPool: TxPool, config: TxCollectionConfig, dateProvider?: DateProvider, telemetryClient?: TelemetryClient, log?: Logger);
+    /** Starts all collection loops. */
+    start(): Promise<void>;
+    /** Stops all activity. */
+    stop(): Promise<void>;
+    /** Force trigger the slow collection and reconciliation loops */
+    trigger(): Promise<void>;
+    /** Returns L1 rollup constants. */
+    getConstants(): L1RollupConstants;
+    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
+    startCollecting(block: L2Block, txHashes: TxHash[]): void;
+    /** Collects the set of txs for the given block proposal as fast as possible */
+    collectFastForProposal(blockProposal: BlockProposal, txHashes: TxHash[] | string[], opts: {
+        deadline: Date;
+        pinnedPeer?: PeerId;
+    }): Promise<Tx[]>;
+    /** Collects the set of txs for the given mined block as fast as possible */
+    collectFastForBlock(block: L2Block, txHashes: TxHash[] | string[], opts: {
+        deadline: Date;
+        pinnedPeer?: PeerId;
+    }): Promise<Tx[]>;
+    /** Collects the set of txs for the given proposal or block as fast as possible */
+    collectFastFor(input: FastCollectionRequestInput, txHashes: TxHash[] | string[], opts: {
+        deadline: Date;
+        pinnedPeer?: PeerId;
+    }): Promise<Tx[]>;
+    /** Mark the given txs as found. Stops collecting them. */
+    private foundTxs;
+    /**
+     * Stop collecting all txs for blocks less than or requal to the block number specified.
+     * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+     */
+    stopCollectingForBlocksUpTo(blockNumber: number): void;
+    /**
+     * Stop collecting all txs for blocks greater than the block number specified.
+     * To be called when there is a chain prune and previously mined txs are no longer relevant.
+     */
+    stopCollectingForBlocksAfter(blockNumber: number): void;
+    /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */
+    private reconcileFoundTxsWithPool;
+}
+//# sourceMappingURL=tx_collection.d.ts.map

package/dest/services/tx_collection/tx_collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_collection.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,KAAK,oBAAoB,EAAkB,MAAM,2BAA2B,CAAC;AACtF,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAEnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,MAAM,MAAM,gBAAgB,GAAG,eAAe,GAAG,eAAe,GAAG,eAAe,GAAG,eAAe,CAAC;AAErG,MAAM,MAAM,aAAa,GAAG;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,IAAI,CAAC;IAAC,eAAe,EAAE,OAAO,CAAA;CAAE,CAAC;AAE9F,MAAM,MAAM,0BAA0B,GAClC;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,aAAa,EAAE,aAAa,CAAA;CAAE,CAAC;AAEvD,MAAM,MAAM,qBAAqB,GAAG,0BAA0B,GAAG;IAC/D,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,EAAE,IAAI,CAAC;IACf,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,EAAE,oBAAoB,CAAC,IAAI,CAAC,CAAC;IACpC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CAC3B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,YAAY;IAoBrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IA1BtB,uCAAuC;IACvC,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAEpD,8BAA8B;IAC9B,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAEpD,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAiB;IAEvD,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IAEpD,uDAAuD;IACvD,SAAS,CAAC,QAAQ,CAAC,oBAAoB,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;IAEnE,kEAAkE;IAClE,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;gBAG1C,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,EACnD,KAAK,EAAE,QAAQ,EAAE,EACjB,SAAS,EAAE,iBAAiB,EAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAChE,eAAe,GAAE,eAAsC,EACtC,GAAG,GAAE,MAAkD;IA4C1E,mCAAmC;IAC5B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ7B,0BAA0B;IACb,IAAI;IAOjB,iEAAiE;IACpD,OAAO;IAIpB,mCAAmC;IAC5B,YAAY,IAAI,iBAAiB;IAIxC,mFAAmF;IAC5E,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE;IAIzD,+EAA+E;IACxE,sBAAsB,CAC3B,aAAa,EAAE,aAAa,EAC5B,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE;IAK/C,4EAA4E;IACrE,mBAAmB,CACxB,KAAK,EAAE,OAAO,EACd,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE;IAK/C,kFAAkF;IAC3E,cAAc,CACnB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE;IAK/C,0DAA0D;IAC1D,OAAO,CAAC,QAAQ;IAKhB;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAK7D;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAK9D,8HAA8H;YAChH,yBAAyB;CAUxC"}

package/dest/services/tx_collection/tx_collection.js

@@ -0,0 +1,127 @@
+import { compactArray } from '@aztec/foundation/collection';
+import { createLogger } from '@aztec/foundation/log';
+import { RunningPromise } from '@aztec/foundation/promise';
+import { DateProvider } from '@aztec/foundation/timer';
+import { getTelemetryClient } from '@aztec/telemetry-client';
+import { FastTxCollection } from './fast_tx_collection.js';
+import { SlowTxCollection } from './slow_tx_collection.js';
+import { TxCollectionSink } from './tx_collection_sink.js';
+/**
+ * Coordinates tx collection from remote RPC nodes and reqresp.
+ *
+ * The slow collection loops are used for periodically collecting missing txs from mined blocks,
+ * from remote RPC nodes and via reqresp. The fast collection methods are used to quickly gather
+ * txs, usually for attesting to block proposals or preparing to prove an epoch. The slow and fast
+ * collection instances both send their txs to the collection sink, which handles metrics and adds
+ * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is notified
+ * via events and notifies the slow and fast collection loops to stop collecting that tx, so that we don't
+ * collect the same tx multiple times.
+ */ export class TxCollection {
+    reqResp;
+    nodes;
+    constants;
+    txPool;
+    config;
+    dateProvider;
+    log;
+    /** Slow collection background loops */ slowCollection;
+    /** Fast collection methods */ fastCollection;
+    /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */ reconcileFoundTxsLoop;
+    /** Handles txs found by the slow and fast collection loops */ txCollectionSink;
+    /** Handler for the txs-added event from the tx pool */ handleTxsAddedToPool;
+    /** Handler for the txs-added event from the tx collection sink */ handleTxsFound;
+    constructor(reqResp, nodes, constants, txPool, config, dateProvider = new DateProvider(), telemetryClient = getTelemetryClient(), log = createLogger('p2p:tx_collection_service')){
+        this.reqResp = reqResp;
+        this.nodes = nodes;
+        this.constants = constants;
+        this.txPool = txPool;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+        this.txCollectionSink = new TxCollectionSink(this.txPool, telemetryClient, this.log);
+        this.fastCollection = new FastTxCollection(this.reqResp, this.nodes, this.txCollectionSink, this.config, this.dateProvider, this.log);
+        this.slowCollection = new SlowTxCollection(this.reqResp, this.nodes, this.txCollectionSink, this.fastCollection, constants, this.config, this.dateProvider, this.log);
+        this.reconcileFoundTxsLoop = new RunningPromise(()=>this.reconcileFoundTxsWithPool(), this.log, this.config.txCollectionReconcileIntervalMs);
+        this.handleTxsFound = (args)=>{
+            this.foundTxs(args.txs);
+        };
+        this.txCollectionSink.on('txs-added', this.handleTxsFound);
+        this.handleTxsAddedToPool = (args)=>{
+            const { txs, source } = args;
+            if (source !== 'tx-collection') {
+                this.foundTxs(txs);
+            }
+        };
+        this.txPool.on('txs-added', this.handleTxsAddedToPool);
+    }
+    /** Starts all collection loops. */ start() {
+        this.slowCollection.start();
+        this.reconcileFoundTxsLoop.start();
+        // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup
+        return Promise.resolve();
+    }
+    /** Stops all activity. */ async stop() {
+        await Promise.all([
+            this.slowCollection.stop(),
+            this.fastCollection.stop(),
+            this.reconcileFoundTxsLoop.stop()
+        ]);
+        this.txPool.removeListener('txs-added', this.handleTxsAddedToPool);
+        this.txCollectionSink.removeListener('txs-added', this.handleTxsFound);
+    }
+    /** Force trigger the slow collection and reconciliation loops */ async trigger() {
+        await Promise.all([
+            this.reconcileFoundTxsLoop.trigger(),
+            this.slowCollection.trigger()
+        ]);
+    }
+    /** Returns L1 rollup constants. */ getConstants() {
+        return this.constants;
+    }
+    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */ startCollecting(block, txHashes) {
+        return this.slowCollection.startCollecting(block, txHashes);
+    }
+    /** Collects the set of txs for the given block proposal as fast as possible */ collectFastForProposal(blockProposal, txHashes, opts) {
+        return this.collectFastFor({
+            type: 'proposal',
+            blockProposal
+        }, txHashes, opts);
+    }
+    /** Collects the set of txs for the given mined block as fast as possible */ collectFastForBlock(block, txHashes, opts) {
+        return this.collectFastFor({
+            type: 'block',
+            block
+        }, txHashes, opts);
+    }
+    /** Collects the set of txs for the given proposal or block as fast as possible */ collectFastFor(input, txHashes, opts) {
+        return this.fastCollection.collectFastFor(input, txHashes, opts);
+    }
+    /** Mark the given txs as found. Stops collecting them. */ foundTxs(txs) {
+        this.slowCollection.foundTxs(txs);
+        this.fastCollection.foundTxs(txs);
+    }
+    /**
+   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+   */ stopCollectingForBlocksUpTo(blockNumber) {
+        this.slowCollection.stopCollectingForBlocksUpTo(blockNumber);
+        this.fastCollection.stopCollectingForBlocksUpTo(blockNumber);
+    }
+    /**
+   * Stop collecting all txs for blocks greater than the block number specified.
+   * To be called when there is a chain prune and previously mined txs are no longer relevant.
+   */ stopCollectingForBlocksAfter(blockNumber) {
+        this.slowCollection.stopCollectingForBlocksAfter(blockNumber);
+        this.fastCollection.stopCollectingForBlocksAfter(blockNumber);
+    }
+    /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */ async reconcileFoundTxsWithPool() {
+        const missingTxHashes = this.slowCollection.getMissingTxHashes();
+        const foundTxs = compactArray(await this.txPool.getTxsByHash(missingTxHashes));
+        if (foundTxs.length > 0) {
+            this.log.verbose(`Found ${foundTxs.length} txs in the pool during reconciliation`, {
+                foundTxs: foundTxs.map((t)=>t.getTxHash().toString())
+            });
+            this.foundTxs(foundTxs);
+        }
+    }
+}

package/dest/services/tx_collection/tx_collection_sink.d.ts

@@ -0,0 +1,30 @@
+import type { Logger } from '@aztec/foundation/log';
+import type { TypedEventEmitter } from '@aztec/foundation/types';
+import { Tx, type TxHash } from '@aztec/stdlib/tx';
+import type { TelemetryClient } from '@aztec/telemetry-client';
+import type { TxPool, TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
+import type { CollectionMethod } from './tx_collection.js';
+declare const TxCollectionSink_base: new () => TypedEventEmitter<TxPoolEvents>;
+/**
+ * Executes collection requests from the fast and slow collection loops, and handles collected txs
+ * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
+ */
+export declare class TxCollectionSink extends TxCollectionSink_base {
+    private readonly txPool;
+    private readonly log;
+    private readonly instrumentation;
+    constructor(txPool: TxPool, telemetryClient: TelemetryClient, log: Logger);
+    collect(collectValidTxsFn: (txHashes: TxHash[]) => Promise<(Tx | undefined)[]>, requested: TxHash[], info: Record<string, any> & {
+        description: string;
+        method: CollectionMethod;
+    }): Promise<{
+        txs: Tx[];
+    } | {
+        txs: Tx[];
+        requested: TxHash[];
+        duration: number;
+    }>;
+    private foundTxs;
+}
+export {};
+//# sourceMappingURL=tx_collection_sink.d.ts.map

package/dest/services/tx_collection/tx_collection_sink.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx_collection_sink.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_collection_sink.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEpD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,EAAE,EAAE,KAAK,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAI/D,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAE/E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;qCAMJ,UAAU,iBAAiB,CAAC,YAAY,CAAC;AAJhG;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,qBAA2D;IAI7F,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,GAAG;IALtB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA8B;gBAG3C,MAAM,EAAE,MAAM,EAC/B,eAAe,EAAE,eAAe,EACf,GAAG,EAAE,MAAM;IAMjB,OAAO,CAClB,iBAAiB,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,EACtE,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,gBAAgB,CAAA;KAAE;;;;;;;YA0CjE,QAAQ;CAwBvB"}

package/dest/services/tx_collection/tx_collection_sink.js

@@ -0,0 +1,81 @@
+import { elapsed } from '@aztec/foundation/timer';
+import EventEmitter from 'node:events';
+import { TxCollectionInstrumentation } from './instrumentation.js';
+/**
+ * Executes collection requests from the fast and slow collection loops, and handles collected txs
+ * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
+ */ export class TxCollectionSink extends EventEmitter {
+    txPool;
+    log;
+    instrumentation;
+    constructor(txPool, telemetryClient, log){
+        super(), this.txPool = txPool, this.log = log;
+        this.instrumentation = new TxCollectionInstrumentation(telemetryClient, 'TxCollection');
+    }
+    async collect(collectValidTxsFn, requested, info) {
+        this.log.trace(`Requesting ${requested.length} txs via ${info.description}`, {
+            ...info,
+            requestedTxs: requested.map((t)=>t.toString())
+        });
+        // Execute collection function and measure the time taken, catching any errors.
+        const [duration, txs] = await elapsed(async ()=>{
+            try {
+                const response = await collectValidTxsFn(requested);
+                return response.filter((tx)=>tx !== undefined);
+            } catch (err) {
+                this.log.error(`Error collecting txs via ${info.description}`, err, {
+                    ...info,
+                    requestedTxs: requested.map((hash)=>hash.toString())
+                });
+                return [];
+            }
+        });
+        if (txs.length === 0) {
+            this.log.trace(`No txs found via ${info.description}`, {
+                ...info,
+                requestedTxs: requested.map((t)=>t.toString())
+            });
+            return {
+                txs,
+                requested,
+                duration
+            };
+        }
+        this.log.verbose(`Collected ${txs.length} txs out of ${requested.length} requested via ${info.description} in ${duration}ms`, {
+            ...info,
+            duration,
+            txs: txs.map((t)=>t.getTxHash().toString()),
+            requestedTxs: requested.map((t)=>t.toString())
+        });
+        return await this.foundTxs(txs, {
+            ...info,
+            duration
+        });
+    }
+    async foundTxs(txs, info) {
+        // Report metrics for the collection
+        this.instrumentation.increaseTxsFor(info.method, txs.length, info.duration);
+        // Mark txs as found in the slow missing txs set and all fast requests
+        this.emit('txs-added', {
+            txs
+        });
+        // Add the txs to the tx pool (should not fail, but we catch it just in case)
+        try {
+            await this.txPool.addTxs(txs, {
+                source: `tx-collection`
+            });
+        } catch (err) {
+            this.log.error(`Error adding txs to the pool via ${info.description}`, err, {
+                ...info,
+                txs: txs.map((tx)=>tx.txHash.toString())
+            });
+            // Return no txs since none have been added
+            return {
+                txs: []
+            };
+        }
+        return {
+            txs
+        };
+    }
+}

package/dest/services/tx_collection/tx_source.d.ts

@@ -0,0 +1,18 @@
+import type { ChainConfig } from '@aztec/stdlib/config';
+import { type AztecNode } from '@aztec/stdlib/interfaces/client';
+import type { Tx, TxHash } from '@aztec/stdlib/tx';
+import { type ComponentsVersions } from '@aztec/stdlib/versioning';
+export interface TxSource {
+    getInfo(): string;
+    getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+}
+export declare class NodeRpcTxSource implements TxSource {
+    private readonly client;
+    private readonly info;
+    constructor(client: Pick<AztecNode, 'getTxsByHash'>, info: string);
+    static fromUrl(nodeUrl: string, versions: ComponentsVersions): NodeRpcTxSource;
+    getInfo(): string;
+    getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+}
+export declare function createNodeRpcTxSources(urls: string[], chainConfig: ChainConfig): NodeRpcTxSource[];
+//# sourceMappingURL=tx_source.d.ts.map

package/dest/services/tx_collection/tx_source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx_source.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_source.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,KAAK,SAAS,EAAyB,MAAM,iCAAiC,CAAC;AACxF,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,KAAK,kBAAkB,EAAmC,MAAM,0BAA0B,CAAC;AAGpG,MAAM,WAAW,QAAQ;IACvB,OAAO,IAAI,MAAM,CAAC;IAClB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;CAC/D;AAED,qBAAa,eAAgB,YAAW,QAAQ;IAE5C,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;gBADJ,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EACvC,IAAI,EAAE,MAAM;WAGjB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,kBAAkB,GAAG,eAAe;IAK9E,OAAO;IAIP,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC;CAGrE;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,WAAW,qBAG9E"}

package/dest/services/tx_collection/tx_source.js

@@ -0,0 +1,31 @@
+import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
+import { protocolContractTreeRoot } from '@aztec/protocol-contracts';
+import { createAztecNodeClient } from '@aztec/stdlib/interfaces/client';
+import { getComponentsVersionsFromConfig } from '@aztec/stdlib/versioning';
+import { makeTracedFetch } from '@aztec/telemetry-client';
+export class NodeRpcTxSource {
+    client;
+    info;
+    constructor(client, info){
+        this.client = client;
+        this.info = info;
+    }
+    static fromUrl(nodeUrl, versions) {
+        const client = createAztecNodeClient(nodeUrl, versions, makeTracedFetch([
+            1,
+            2,
+            3
+        ], false));
+        return new NodeRpcTxSource(client, nodeUrl);
+    }
+    getInfo() {
+        return this.info;
+    }
+    getTxsByHash(txHashes) {
+        return this.client.getTxsByHash(txHashes);
+    }
+}
+export function createNodeRpcTxSources(urls, chainConfig) {
+    const versions = getComponentsVersionsFromConfig(chainConfig, protocolContractTreeRoot, getVKTreeRoot());
+    return urls.map((url)=>NodeRpcTxSource.fromUrl(url, versions));
+}

package/dest/services/tx_provider.d.ts

@@ -0,0 +1,49 @@
+import { type Logger } from '@aztec/foundation/log';
+import type { L2Block } from '@aztec/stdlib/block';
+import type { ITxProvider } from '@aztec/stdlib/interfaces/server';
+import type { BlockProposal } from '@aztec/stdlib/p2p';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import type { PeerId } from '@libp2p/interface';
+import type { P2PClient } from '../client/p2p_client.js';
+import type { TxPool } from '../mem_pools/index.js';
+import type { TxCollection } from './tx_collection/tx_collection.js';
+import { TxProviderInstrumentation } from './tx_provider_instrumentation.js';
+/**
+ * Gathers and returns txs given a block proposal, block, or their hashes.
+ * Loads available txs from the tx pool, and  relies on a TxCollection service to collect txs from the network and other nodes.
+ */
+export declare class TxProvider implements ITxProvider {
+    private txCollection;
+    private txPool;
+    private txValidator;
+    private log;
+    protected instrumentation: TxProviderInstrumentation;
+    constructor(txCollection: TxCollection, txPool: TxPool, txValidator: Pick<P2PClient, 'validate'>, log?: Logger, client?: TelemetryClient);
+    /** Returns txs from the tx pool given their hashes.*/
+    getAvailableTxs(txHashes: TxHash[]): Promise<{
+        txs: Tx[];
+        missingTxs: TxHash[];
+    }>;
+    /** Gathers txs from the tx pool, proposal body, remote rpc nodes, and reqresp. */
+    getTxsForBlockProposal(blockProposal: BlockProposal, opts: {
+        pinnedPeer: PeerId | undefined;
+        deadline: Date;
+    }): Promise<{
+        txs: Tx[];
+        missingTxs: TxHash[];
+    }>;
+    /** Gathers txs from the tx pool, remote rpc nodes, and reqresp. */
+    getTxsForBlock(block: L2Block, opts: {
+        deadline: Date;
+    }): Promise<{
+        txs: Tx[];
+        missingTxs: TxHash[];
+    }>;
+    private getOrderedTxsFromAllSources;
+    private orderTxs;
+    private getTxsFromAllSources;
+    private extractFromProposal;
+    private processProposalTxs;
+}
+//# sourceMappingURL=tx_provider.d.ts.map

package/dest/services/tx_provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx_provider.d.ts","sourceRoot":"","sources":["../../src/services/tx_provider.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAElE,OAAO,KAAK,EAAa,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAEnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EAA8B,YAAY,EAAE,MAAM,kCAAkC,CAAC;AACjG,OAAO,EAAE,yBAAyB,EAAE,MAAM,kCAAkC,CAAC;AAE7E;;;GAGG;AACH,qBAAa,UAAW,YAAW,WAAW;IAI1C,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,GAAG;IANb,SAAS,CAAC,eAAe,EAAE,yBAAyB,CAAC;gBAG3C,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,EACxC,GAAG,GAAE,MAAyC,EACtD,MAAM,GAAE,eAAsC;IAKhD,sDAAsD;IACzC,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAoB9F,kFAAkF;IAC3E,sBAAsB,CAC3B,aAAa,EAAE,aAAa,EAC5B,IAAI,EAAE;QAAE,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;QAAC,QAAQ,EAAE,IAAI,CAAA;KAAE,GACvD,OAAO,CAAC;QAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAS/C,mEAAmE;IAC5D,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;YAS/F,2BAA2B;IAgCzC,OAAO,CAAC,QAAQ;YAKF,oBAAoB;IAsFlC,OAAO,CAAC,mBAAmB;YAOb,kBAAkB;CAOjC"}