@aztec/p2p 0.0.1-commit.96bb3f7 → 0.0.1-commit.993d240

package/dest/services/tx_collection/tx_collection.d.ts

@@ -1,90 +1,77 @@
 import { BlockNumber } from '@aztec/foundation/branded-types';
 import { type Logger } from '@aztec/foundation/log';
-import { type PromiseWithResolvers } from '@aztec/foundation/promise';
 import { DateProvider } from '@aztec/foundation/timer';
-import type { L2BlockInfo, L2BlockNew } from '@aztec/stdlib/block';
+import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
 import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
 import type { BlockProposal } from '@aztec/stdlib/p2p';
-import { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { Tx } from '@aztec/stdlib/tx';
+import { TxHash } from '@aztec/stdlib/tx';
 import { type TelemetryClient } from '@aztec/telemetry-client';
 import type { PeerId } from '@libp2p/interface';
-import type { TxPool } from '../../mem_pools/index.js';
-import type { TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
-import type { ReqRespInterface } from '../reqresp/interface.js';
+import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interfaces.js';
+import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
+import type { BlockTxsSource } from '../reqresp/index.js';
 import type { TxCollectionConfig } from './config.js';
-import { FastTxCollection } from './fast_tx_collection.js';
-import { SlowTxCollection } from './slow_tx_collection.js';
+import { FileStoreTxCollection } from './file_store_tx_collection.js';
+import type { FileStoreTxSource } from './file_store_tx_source.js';
+import { type IRequestTracker } from './request_tracker.js';
 import type { TxSource } from './tx_source.js';
-export type CollectionMethod = 'fast-req-resp' | 'fast-node-rpc' | 'slow-req-resp' | 'slow-node-rpc';
-export type MissingTxInfo = {
-    blockNumber: BlockNumber;
-    deadline: Date;
-    readyForReqResp: boolean;
-};
+export type CollectionMethod = 'fast-req-resp' | 'fast-node-rpc' | 'file-store';
 export type FastCollectionRequestInput = {
     type: 'block';
-    block: L2BlockNew;
+    block: L2Block;
 } | {
     type: 'proposal';
     blockProposal: BlockProposal;
     blockNumber: BlockNumber;
 };
 export type FastCollectionRequest = FastCollectionRequestInput & {
-    missingTxHashes: Set<string>;
-    deadline: Date;
+    requestTracker: IRequestTracker;
     blockInfo: L2BlockInfo;
-    promise: PromiseWithResolvers<void>;
-    foundTxs: Map<string, Tx>;
 };
 /**
- * Coordinates tx collection from remote RPC nodes and reqresp.
+ * Collect missing transactions for a block or proposal via reqresp.
+ * @param requestTracker - The missing transactions tracker
+ * @param blockTxsSource - The block or proposal containing the transactions
+ * @param pinnedPeer - Optional peer expected to have the transactions
+ * @returns The collected transactions
+ */
+export type IReqRespTxsCollector = (requestTracker: IRequestTracker, blockTxsSource: BlockTxsSource, pinnedPeer: PeerId | undefined) => Promise<Tx[]>;
+/**
+ * Coordinates tx collection from remote RPC nodes, reqresp, and file store.
  *
- * The slow collection loops are used for periodically collecting missing txs from mined blocks,
- * from remote RPC nodes and via reqresp. The fast collection methods are used to quickly gather
- * txs, usually for attesting to block proposals or preparing to prove an epoch. The slow and fast
- * collection instances both send their txs to the collection sink, which handles metrics and adds
- * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is notified
- * via events and notifies the slow and fast collection loops to stop collecting that tx, so that we don't
- * collect the same tx multiple times.
+ * Runs a sequential pipeline: node RPC → reqresp → file store. Node collection starts immediately,
+ * reqresp starts after a configured delay, and file store (if configured) starts after a further
+ * delay. All paths send txs to the collection sink, which handles metrics and adds them to the
+ * tx pool. Whenever a tx is added to the sink or the pool, this service is notified and stops
+ * collecting that tx across all in-flight requests.
  */
 export declare class TxCollection {
-    private readonly reqResp;
+    private readonly p2pService;
     private readonly nodes;
     private readonly constants;
     private readonly txPool;
     private readonly config;
     private readonly dateProvider;
     private readonly log;
-    /** Slow collection background loops */
-    protected readonly slowCollection: SlowTxCollection;
-    /** Fast collection methods */
-    protected readonly fastCollection: FastTxCollection;
-    /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */
-    private readonly reconcileFoundTxsLoop;
-    /** Handles txs found by the slow and fast collection loops */
+    protected requests: Set<FastCollectionRequest>;
+    /** The collector for txs via reqresp */
+    protected reqRespTxsCollector?: IReqRespTxsCollector;
+    /** File store collection for the fast (proposal/proving) path */
+    protected readonly fileStoreFastCollection: FileStoreTxCollection;
+    /** Handles txs found by collection paths before adding to the pool */
     private readonly txCollectionSink;
     /** Handler for the txs-added event from the tx pool */
-    protected readonly handleTxsAddedToPool: TxPoolEvents['txs-added'];
+    protected readonly handleTxsAddedToPool: TxPoolV2Events['txs-added'];
     /** Handler for the txs-added event from the tx collection sink */
-    protected readonly handleTxsFound: TxPoolEvents['txs-added'];
-    constructor(reqResp: Pick<ReqRespInterface, 'sendBatchRequest'>, nodes: TxSource[], constants: L1RollupConstants, txPool: TxPool, config: TxCollectionConfig, dateProvider?: DateProvider, telemetryClient?: TelemetryClient, log?: Logger);
-    /** Starts all collection loops. */
-    start(): Promise<void>;
-    /** Stops all activity. */
-    stop(): Promise<void>;
-    /** Force trigger the slow collection and reconciliation loops */
-    trigger(): Promise<void>;
+    protected readonly handleTxsFound: TxPoolV2Events['txs-added'];
+    constructor(p2pService: BatchTxRequesterLibP2PService, nodes: TxSource[], constants: L1RollupConstants, txPool: TxPoolV2, config: TxCollectionConfig, fileStoreSources?: FileStoreTxSource[], dateProvider?: DateProvider, telemetryClient?: TelemetryClient, log?: Logger);
+    /** Stops all activity. Cancels in-flight requests; file store workers self-terminate. */
+    stop(): void;
     /** Returns L1 rollup constants. */
     getConstants(): L1RollupConstants;
-    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
-    startCollecting(block: L2BlockNew, txHashes: TxHash[]): void;
-    /** Collects the set of txs for the given block proposal as fast as possible */
-    collectFastForProposal(blockProposal: BlockProposal, blockNumber: BlockNumber, txHashes: TxHash[] | string[], opts: {
-        deadline: Date;
-        pinnedPeer?: PeerId;
-    }): Promise<Tx[]>;
     /** Collects the set of txs for the given mined block as fast as possible */
-    collectFastForBlock(block: L2BlockNew, txHashes: TxHash[] | string[], opts: {
+    collectFastForBlock(block: L2Block, txHashes: TxHash[] | string[], opts: {
         deadline: Date;
         pinnedPeer?: PeerId;
     }): Promise<Tx[]>;
@@ -93,10 +80,18 @@ export declare class TxCollection {
         deadline: Date;
         pinnedPeer?: PeerId;
     }): Promise<Tx[]>;
-    /** Mark the given txs as found. Stops collecting them. */
+    protected collectFast(request: FastCollectionRequest, opts: {
+        pinnedPeer?: PeerId;
+    }): Promise<void>;
+    private collectFastFromNodes;
+    private collectFastFromNode;
+    private collectFastViaReqResp;
+    /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */
+    private getAddContext;
+    /** Mark the given txs as found. Stops collecting them across all in-flight requests. */
     private foundTxs;
     /**
-     * Stop collecting all txs for blocks less than or requal to the block number specified.
+     * Stop collecting all txs for blocks less than or equal to the block number specified.
      * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
      */
     stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void;
@@ -105,6 +100,5 @@ export declare class TxCollection {
      * To be called when there is a chain prune and previously mined txs are no longer relevant.
      */
     stopCollectingForBlocksAfter(blockNumber: BlockNumber): void;
-    private reconcileFoundTxsWithPool;
 }
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfY29sbGVjdGlvbi5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NlcnZpY2VzL3R4X2NvbGxlY3Rpb24vdHhfY29sbGVjdGlvbi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFFOUQsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBQ2xFLE9BQU8sRUFBRSxLQUFLLG9CQUFvQixFQUFrQixNQUFNLDJCQUEyQixDQUFDO0FBQ3RGLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUN2RCxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsVUFBVSxFQUFFLE1BQU0scUJBQXFCLENBQUM7QUFDbkUsT0FBTyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUNyRSxPQUFPLEtBQUssRUFBRSxhQUFhLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQztBQUN2RCxPQUFPLEVBQUUsRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBQzlDLE9BQU8sRUFBRSxLQUFLLGVBQWUsRUFBc0IsTUFBTSx5QkFBeUIsQ0FBQztBQUVuRixPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQztBQUVoRCxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSwwQkFBMEIsQ0FBQztBQUN2RCxPQUFPLEtBQUssRUFBRSxZQUFZLEVBQUUsTUFBTSxvQ0FBb0MsQ0FBQztBQUN2RSxPQUFPLEtBQUssRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sYUFBYSxDQUFDO0FBQ3RELE9BQU8sRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQzNELE9BQU8sRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBRTNELE9BQU8sS0FBSyxFQUFFLFFBQVEsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBRS9DLE1BQU0sTUFBTSxnQkFBZ0IsR0FBRyxlQUFlLEdBQUcsZUFBZSxHQUFHLGVBQWUsR0FBRyxlQUFlLENBQUM7QUFFckcsTUFBTSxNQUFNLGFBQWEsR0FBRztJQUFFLFdBQVcsRUFBRSxXQUFXLENBQUM7SUFBQyxRQUFRLEVBQUUsSUFBSSxDQUFDO0lBQUMsZUFBZSxFQUFFLE9BQU8sQ0FBQTtDQUFFLENBQUM7QUFFbkcsTUFBTSxNQUFNLDBCQUEwQixHQUNsQztJQUFFLElBQUksRUFBRSxPQUFPLENBQUM7SUFBQyxLQUFLLEVBQUUsVUFBVSxDQUFBO0NBQUUsR0FDcEM7SUFBRSxJQUFJLEVBQUUsVUFBVSxDQUFDO0lBQUMsYUFBYSxFQUFFLGFBQWEsQ0FBQztJQUFDLFdBQVcsRUFBRSxXQUFXLENBQUE7Q0FBRSxDQUFDO0FBRWpGLE1BQU0sTUFBTSxxQkFBcUIsR0FBRywwQkFBMEIsR0FBRztJQUMvRCxlQUFlLEVBQUUsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDO0lBQzdCLFFBQVEsRUFBRSxJQUFJLENBQUM7SUFDZixTQUFTLEVBQUUsV0FBVyxDQUFDO0lBQ3ZCLE9BQU8sRUFBRSxvQkFBb0IsQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUNwQyxRQUFRLEVBQUUsR0FBRyxDQUFDLE1BQU0sRUFBRSxFQUFFLENBQUMsQ0FBQztDQUMzQixDQUFDO0FBRUY7Ozs7Ozs7Ozs7R0FVRztBQUNILHFCQUFhLFlBQVk7SUFvQnJCLE9BQU8sQ0FBQyxRQUFRLENBQUMsT0FBTztJQUN4QixPQUFPLENBQUMsUUFBUSxDQUFDLEtBQUs7SUFDdEIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTO0lBQzFCLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQUN2QixPQUFPLENBQUMsUUFBUSxDQUFDLE1BQU07SUFDdkIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxZQUFZO0lBRTdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsR0FBRztJQTFCdEIsdUNBQXVDO0lBQ3ZDLFNBQVMsQ0FBQyxRQUFRLENBQUMsY0FBYyxFQUFFLGdCQUFnQixDQUFDO0lBRXBELDhCQUE4QjtJQUM5QixTQUFTLENBQUMsUUFBUSxDQUFDLGNBQWMsRUFBRSxnQkFBZ0IsQ0FBQztJQUVwRCxtR0FBbUc7SUFDbkcsT0FBTyxDQUFDLFFBQVEsQ0FBQyxxQkFBcUIsQ0FBaUI7SUFFdkQsOERBQThEO0lBQzlELE9BQU8sQ0FBQyxRQUFRLENBQUMsZ0JBQWdCLENBQW1CO0lBRXBELHVEQUF1RDtJQUN2RCxTQUFTLENBQUMsUUFBUSxDQUFDLG9CQUFvQixFQUFFLFlBQVksQ0FBQyxXQUFXLENBQUMsQ0FBQztJQUVuRSxrRUFBa0U7SUFDbEUsU0FBUyxDQUFDLFFBQVEsQ0FBQyxjQUFjLEVBQUUsWUFBWSxDQUFDLFdBQVcsQ0FBQyxDQUFDO0lBRTdELFlBQ21CLE9BQU8sRUFBRSxJQUFJLENBQUMsZ0JBQWdCLEVBQUUsa0JBQWtCLENBQUMsRUFDbkQsS0FBSyxFQUFFLFFBQVEsRUFBRSxFQUNqQixTQUFTLEVBQUUsaUJBQWlCLEVBQzVCLE1BQU0sRUFBRSxNQUFNLEVBQ2QsTUFBTSxFQUFFLGtCQUFrQixFQUMxQixZQUFZLEdBQUUsWUFBaUMsRUFDaEUsZUFBZSxHQUFFLGVBQXNDLEVBQ3RDLEdBQUcsR0FBRSxNQUFrRCxFQTBDekU7SUFFRCxtQ0FBbUM7SUFDNUIsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FNNUI7SUFFRCwwQkFBMEI7SUFDYixJQUFJLGtCQUtoQjtJQUVELGlFQUFpRTtJQUNwRCxPQUFPLGtCQUVuQjtJQUVELG1DQUFtQztJQUM1QixZQUFZLElBQUksaUJBQWlCLENBRXZDO0lBRUQsbUZBQW1GO0lBQzVFLGVBQWUsQ0FBQyxLQUFLLEVBQUUsVUFBVSxFQUFFLFFBQVEsRUFBRSxNQUFNLEVBQUUsUUFFM0Q7SUFFRCwrRUFBK0U7SUFDeEUsc0JBQXNCLENBQzNCLGFBQWEsRUFBRSxhQUFhLEVBQzVCLFdBQVcsRUFBRSxXQUFXLEVBQ3hCLFFBQVEsRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLEVBQUUsRUFDN0IsSUFBSSxFQUFFO1FBQUUsUUFBUSxFQUFFLElBQUksQ0FBQztRQUFDLFVBQVUsQ0FBQyxFQUFFLE1BQU0sQ0FBQTtLQUFFLGlCQUc5QztJQUVELDRFQUE0RTtJQUNyRSxtQkFBbUIsQ0FDeEIsS0FBSyxFQUFFLFVBQVUsRUFDakIsUUFBUSxFQUFFLE1BQU0sRUFBRSxHQUFHLE1BQU0sRUFBRSxFQUM3QixJQUFJLEVBQUU7UUFBRSxRQUFRLEVBQUUsSUFBSSxDQUFDO1FBQUMsVUFBVSxDQUFDLEVBQUUsTUFBTSxDQUFBO0tBQUUsaUJBRzlDO0lBRUQsa0ZBQWtGO0lBQzNFLGNBQWMsQ0FDbkIsS0FBSyxFQUFFLDBCQUEwQixFQUNqQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsTUFBTSxFQUFFLEVBQzdCLElBQUksRUFBRTtRQUFFLFFBQVEsRUFBRSxJQUFJLENBQUM7UUFBQyxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUE7S0FBRSxpQkFHOUM7SUFFRCwwREFBMEQ7SUFDMUQsT0FBTyxDQUFDLFFBQVE7SUFLaEI7OztPQUdHO0lBQ0ksMkJBQTJCLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxJQUFJLENBR2pFO0lBRUQ7OztPQUdHO0lBQ0ksNEJBQTRCLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxJQUFJLENBR2xFO1lBR2EseUJBQXlCO0NBVXhDIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHhfY29sbGVjdGlvbi5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NlcnZpY2VzL3R4X2NvbGxlY3Rpb24vdHhfY29sbGVjdGlvbi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFFOUQsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBRWxFLE9BQU8sRUFBRSxZQUFZLEVBQVcsTUFBTSx5QkFBeUIsQ0FBQztBQUNoRSxPQUFPLEtBQUssRUFBRSxPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0scUJBQXFCLENBQUM7QUFDaEUsT0FBTyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUNyRSxPQUFPLEtBQUssRUFBRSxhQUFhLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQztBQUN2RCxPQUFPLEtBQUssRUFBRSxFQUFFLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUMzQyxPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sa0JBQWtCLENBQUM7QUFDMUMsT0FBTyxFQUFFLEtBQUssZUFBZSxFQUFzQixNQUFNLHlCQUF5QixDQUFDO0FBRW5GLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBRWhELE9BQU8sS0FBSyxFQUFFLFFBQVEsRUFBRSxjQUFjLEVBQUUsTUFBTSwwQ0FBMEMsQ0FBQztBQUV6RixPQUFPLEtBQUssRUFBRSw2QkFBNkIsRUFBRSxNQUFNLDRDQUE0QyxDQUFDO0FBQ2hHLE9BQU8sS0FBSyxFQUFFLGNBQWMsRUFBRSxNQUFNLHFCQUFxQixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sYUFBYSxDQUFDO0FBQ3RELE9BQU8sRUFBRSxxQkFBcUIsRUFBRSxNQUFNLCtCQUErQixDQUFDO0FBQ3RFLE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLE1BQU0sMkJBQTJCLENBQUM7QUFDbkUsT0FBTyxFQUFFLEtBQUssZUFBZSxFQUFrQixNQUFNLHNCQUFzQixDQUFDO0FBRTVFLE9BQU8sS0FBSyxFQUFFLFFBQVEsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBRS9DLE1BQU0sTUFBTSxnQkFBZ0IsR0FBRyxlQUFlLEdBQUcsZUFBZSxHQUFHLFlBQVksQ0FBQztBQUVoRixNQUFNLE1BQU0sMEJBQTBCLEdBQ2xDO0lBQUUsSUFBSSxFQUFFLE9BQU8sQ0FBQztJQUFDLEtBQUssRUFBRSxPQUFPLENBQUE7Q0FBRSxHQUNqQztJQUFFLElBQUksRUFBRSxVQUFVLENBQUM7SUFBQyxhQUFhLEVBQUUsYUFBYSxDQUFDO0lBQUMsV0FBVyxFQUFFLFdBQVcsQ0FBQTtDQUFFLENBQUM7QUFFakYsTUFBTSxNQUFNLHFCQUFxQixHQUFHLDBCQUEwQixHQUFHO0lBQy9ELGNBQWMsRUFBRSxlQUFlLENBQUM7SUFDaEMsU0FBUyxFQUFFLFdBQVcsQ0FBQztDQUN4QixDQUFDO0FBRUY7Ozs7OztHQU1HO0FBQ0gsTUFBTSxNQUFNLG9CQUFvQixHQUFHLENBQ2pDLGNBQWMsRUFBRSxlQUFlLEVBQy9CLGNBQWMsRUFBRSxjQUFjLEVBQzlCLFVBQVUsRUFBRSxNQUFNLEdBQUcsU0FBUyxLQUMzQixPQUFPLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQztBQUVuQjs7Ozs7Ozs7R0FRRztBQUNILHFCQUFhLFlBQVk7SUFvQnJCLE9BQU8sQ0FBQyxRQUFRLENBQUMsVUFBVTtJQUMzQixPQUFPLENBQUMsUUFBUSxDQUFDLEtBQUs7SUFDdEIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTO0lBQzFCLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQUN2QixPQUFPLENBQUMsUUFBUSxDQUFDLE1BQU07SUFFdkIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxZQUFZO0lBRTdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsR0FBRztJQTFCdEIsU0FBUyxDQUFDLFFBQVEsRUFBRSxHQUFHLENBQUMscUJBQXFCLENBQUMsQ0FBYTtJQUUzRCx3Q0FBd0M7SUFDeEMsU0FBUyxDQUFDLG1CQUFtQixDQUFDLEVBQUUsb0JBQW9CLENBQUM7SUFFckQsaUVBQWlFO0lBQ2pFLFNBQVMsQ0FBQyxRQUFRLENBQUMsdUJBQXVCLEVBQUUscUJBQXFCLENBQUM7SUFFbEUsc0VBQXNFO0lBQ3RFLE9BQU8sQ0FBQyxRQUFRLENBQUMsZ0JBQWdCLENBQW1CO0lBRXBELHVEQUF1RDtJQUN2RCxTQUFTLENBQUMsUUFBUSxDQUFDLG9CQUFvQixFQUFFLGNBQWMsQ0FBQyxXQUFXLENBQUMsQ0FBQztJQUVyRSxrRUFBa0U7SUFDbEUsU0FBUyxDQUFDLFFBQVEsQ0FBQyxjQUFjLEVBQUUsY0FBYyxDQUFDLFdBQVcsQ0FBQyxDQUFDO0lBRS9ELFlBQ21CLFVBQVUsRUFBRSw2QkFBNkIsRUFDekMsS0FBSyxFQUFFLFFBQVEsRUFBRSxFQUNqQixTQUFTLEVBQUUsaUJBQWlCLEVBQzVCLE1BQU0sRUFBRSxRQUFRLEVBQ2hCLE1BQU0sRUFBRSxrQkFBa0IsRUFDM0MsZ0JBQWdCLEdBQUUsaUJBQWlCLEVBQU8sRUFDekIsWUFBWSxHQUFFLFlBQWlDLEVBQ2hFLGVBQWUsR0FBRSxlQUFzQyxFQUN0QyxHQUFHLEdBQUUsTUFBa0QsRUF3Q3pFO0lBRUQseUZBQXlGO0lBQ2xGLElBQUksU0FPVjtJQUVELG1DQUFtQztJQUM1QixZQUFZLElBQUksaUJBQWlCLENBRXZDO0lBRUQsNEVBQTRFO0lBQ3JFLG1CQUFtQixDQUN4QixLQUFLLEVBQUUsT0FBTyxFQUNkLFFBQVEsRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLEVBQUUsRUFDN0IsSUFBSSxFQUFFO1FBQUUsUUFBUSxFQUFFLElBQUksQ0FBQztRQUFDLFVBQVUsQ0FBQyxFQUFFLE1BQU0sQ0FBQTtLQUFFLGlCQUc5QztJQUVELGtGQUFrRjtJQUNyRSxjQUFjLENBQ3pCLEtBQUssRUFBRSwwQkFBMEIsRUFDakMsUUFBUSxFQUFFLE1BQU0sRUFBRSxHQUFHLE1BQU0sRUFBRSxFQUM3QixJQUFJLEVBQUU7UUFBRSxRQUFRLEVBQUUsSUFBSSxDQUFDO1FBQUMsVUFBVSxDQUFDLEVBQUUsTUFBTSxDQUFBO0tBQUUsaUJBb0M5QztJQUVELFVBQWdCLFdBQVcsQ0FBQyxPQUFPLEVBQUUscUJBQXFCLEVBQUUsSUFBSSxFQUFFO1FBQUUsVUFBVSxDQUFDLEVBQUUsTUFBTSxDQUFBO0tBQUUsaUJBbUR4RjtZQVFhLG9CQUFvQjtZQWdCcEIsbUJBQW1CO1lBMEZuQixxQkFBcUI7SUFzRG5DLHlHQUF5RztJQUN6RyxPQUFPLENBQUMsYUFBYTtJQVFyQix3RkFBd0Y7SUFDeEYsT0FBTyxDQUFDLFFBQVE7SUFxQmhCOzs7T0FHRztJQUNJLDJCQUEyQixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU1qRTtJQUVEOzs7T0FHRztJQUNJLDRCQUE0QixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU1sRTtDQUNGIn0=

package/dest/services/tx_collection/tx_collection.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,KAAK,oBAAoB,EAAkB,MAAM,2BAA2B,CAAC;AACtF,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAEnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AACvE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,MAAM,MAAM,gBAAgB,GAAG,eAAe,GAAG,eAAe,GAAG,eAAe,GAAG,eAAe,CAAC;AAErG,MAAM,MAAM,aAAa,GAAG;IAAE,WAAW,EAAE,WAAW,CAAC;IAAC,QAAQ,EAAE,IAAI,CAAC;IAAC,eAAe,EAAE,OAAO,CAAA;CAAE,CAAC;AAEnG,MAAM,MAAM,0BAA0B,GAClC;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,aAAa,EAAE,aAAa,CAAC;IAAC,WAAW,EAAE,WAAW,CAAA;CAAE,CAAC;AAEjF,MAAM,MAAM,qBAAqB,GAAG,0BAA0B,GAAG;IAC/D,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,EAAE,IAAI,CAAC;IACf,SAAS,EAAE,WAAW,CAAC;IACvB,OAAO,EAAE,oBAAoB,CAAC,IAAI,CAAC,CAAC;IACpC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CAC3B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,YAAY;IAoBrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IA1BtB,uCAAuC;IACvC,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAEpD,8BAA8B;IAC9B,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAEpD,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAiB;IAEvD,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IAEpD,uDAAuD;IACvD,SAAS,CAAC,QAAQ,CAAC,oBAAoB,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;IAEnE,kEAAkE;IAClE,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;IAE7D,YACmB,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,EACnD,KAAK,EAAE,QAAQ,EAAE,EACjB,SAAS,EAAE,iBAAiB,EAC5B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAChE,eAAe,GAAE,eAAsC,EACtC,GAAG,GAAE,MAAkD,EA0CzE;IAED,mCAAmC;IAC5B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAM5B;IAED,0BAA0B;IACb,IAAI,kBAKhB;IAED,iEAAiE;IACpD,OAAO,kBAEnB;IAED,mCAAmC;IAC5B,YAAY,IAAI,iBAAiB,CAEvC;IAED,mFAAmF;IAC5E,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,QAE3D;IAED,+EAA+E;IACxE,sBAAsB,CAC3B,aAAa,EAAE,aAAa,EAC5B,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBAG9C;IAED,4EAA4E;IACrE,mBAAmB,CACxB,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBAG9C;IAED,kFAAkF;IAC3E,cAAc,CACnB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBAG9C;IAED,0DAA0D;IAC1D,OAAO,CAAC,QAAQ;IAKhB;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAGjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAGlE;YAGa,yBAAyB;CAUxC"}
+{"version":3,"file":"tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAElE,OAAO,EAAE,YAAY,EAAW,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,yBAAyB,CAAC;AAEnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,0CAA0C,CAAC;AAEzF,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAChG,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAAE,KAAK,eAAe,EAAkB,MAAM,sBAAsB,CAAC;AAE5E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,MAAM,MAAM,gBAAgB,GAAG,eAAe,GAAG,eAAe,GAAG,YAAY,CAAC;AAEhF,MAAM,MAAM,0BAA0B,GAClC;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,aAAa,EAAE,aAAa,CAAC;IAAC,WAAW,EAAE,WAAW,CAAA;CAAE,CAAC;AAEjF,MAAM,MAAM,qBAAqB,GAAG,0BAA0B,GAAG;IAC/D,cAAc,EAAE,eAAe,CAAC;IAChC,SAAS,EAAE,WAAW,CAAC;CACxB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,oBAAoB,GAAG,CACjC,cAAc,EAAE,eAAe,EAC/B,cAAc,EAAE,cAAc,EAC9B,UAAU,EAAE,MAAM,GAAG,SAAS,KAC3B,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;AAEnB;;;;;;;;GAQG;AACH,qBAAa,YAAY;IAoBrB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IA1BtB,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAa;IAE3D,wCAAwC;IACxC,SAAS,CAAC,mBAAmB,CAAC,EAAE,oBAAoB,CAAC;IAErD,iEAAiE;IACjE,SAAS,CAAC,QAAQ,CAAC,uBAAuB,EAAE,qBAAqB,CAAC;IAElE,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IAEpD,uDAAuD;IACvD,SAAS,CAAC,QAAQ,CAAC,oBAAoB,EAAE,cAAc,CAAC,WAAW,CAAC,CAAC;IAErE,kEAAkE;IAClE,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC,WAAW,CAAC,CAAC;IAE/D,YACmB,UAAU,EAAE,6BAA6B,EACzC,KAAK,EAAE,QAAQ,EAAE,EACjB,SAAS,EAAE,iBAAiB,EAC5B,MAAM,EAAE,QAAQ,EAChB,MAAM,EAAE,kBAAkB,EAC3C,gBAAgB,GAAE,iBAAiB,EAAO,EACzB,YAAY,GAAE,YAAiC,EAChE,eAAe,GAAE,eAAsC,EACtC,GAAG,GAAE,MAAkD,EAwCzE;IAED,yFAAyF;IAClF,IAAI,SAOV;IAED,mCAAmC;IAC5B,YAAY,IAAI,iBAAiB,CAEvC;IAED,4EAA4E;IACrE,mBAAmB,CACxB,KAAK,EAAE,OAAO,EACd,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBAG9C;IAED,kFAAkF;IACrE,cAAc,CACzB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBAoC9C;IAED,UAAgB,WAAW,CAAC,OAAO,EAAE,qBAAqB,EAAE,IAAI,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBAmDxF;YAQa,oBAAoB;YAgBpB,mBAAmB;YA0FnB,qBAAqB;IAsDnC,yGAAyG;IACzG,OAAO,CAAC,aAAa;IAQrB,wFAAwF;IACxF,OAAO,CAAC,QAAQ;IAqBhB;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAMjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAMlE;CACF"}

package/dest/services/tx_collection/tx_collection.js

@@ -1,47 +1,52 @@
-import { compactArray } from '@aztec/foundation/collection';
+import { times } from '@aztec/foundation/collection';
 import { createLogger } from '@aztec/foundation/log';
-import { RunningPromise } from '@aztec/foundation/promise';
-import { DateProvider } from '@aztec/foundation/timer';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider, elapsed } from '@aztec/foundation/timer';
+import { TxHash } from '@aztec/stdlib/tx';
 import { getTelemetryClient } from '@aztec/telemetry-client';
-import { FastTxCollection } from './fast_tx_collection.js';
-import { SlowTxCollection } from './slow_tx_collection.js';
+import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js';
+import { FileStoreTxCollection } from './file_store_tx_collection.js';
+import { RequestTracker } from './request_tracker.js';
 import { TxCollectionSink } from './tx_collection_sink.js';
 /**
- * Coordinates tx collection from remote RPC nodes and reqresp.
+ * Coordinates tx collection from remote RPC nodes, reqresp, and file store.
  *
- * The slow collection loops are used for periodically collecting missing txs from mined blocks,
- * from remote RPC nodes and via reqresp. The fast collection methods are used to quickly gather
- * txs, usually for attesting to block proposals or preparing to prove an epoch. The slow and fast
- * collection instances both send their txs to the collection sink, which handles metrics and adds
- * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is notified
- * via events and notifies the slow and fast collection loops to stop collecting that tx, so that we don't
- * collect the same tx multiple times.
+ * Runs a sequential pipeline: node RPC → reqresp → file store. Node collection starts immediately,
+ * reqresp starts after a configured delay, and file store (if configured) starts after a further
+ * delay. All paths send txs to the collection sink, which handles metrics and adds them to the
+ * tx pool. Whenever a tx is added to the sink or the pool, this service is notified and stops
+ * collecting that tx across all in-flight requests.
  */ export class TxCollection {
-    reqResp;
+    p2pService;
     nodes;
     constants;
     txPool;
     config;
     dateProvider;
     log;
-    /** Slow collection background loops */ slowCollection;
-    /** Fast collection methods */ fastCollection;
-    /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */ reconcileFoundTxsLoop;
-    /** Handles txs found by the slow and fast collection loops */ txCollectionSink;
+    // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+    requests;
+    /** The collector for txs via reqresp */ reqRespTxsCollector;
+    /** File store collection for the fast (proposal/proving) path */ fileStoreFastCollection;
+    /** Handles txs found by collection paths before adding to the pool */ txCollectionSink;
     /** Handler for the txs-added event from the tx pool */ handleTxsAddedToPool;
     /** Handler for the txs-added event from the tx collection sink */ handleTxsFound;
-    constructor(reqResp, nodes, constants, txPool, config, dateProvider = new DateProvider(), telemetryClient = getTelemetryClient(), log = createLogger('p2p:tx_collection_service')){
-        this.reqResp = reqResp;
+    constructor(p2pService, nodes, constants, txPool, config, fileStoreSources = [], dateProvider = new DateProvider(), telemetryClient = getTelemetryClient(), log = createLogger('p2p:tx_collection_service')){
+        this.p2pService = p2pService;
         this.nodes = nodes;
         this.constants = constants;
         this.txPool = txPool;
         this.config = config;
         this.dateProvider = dateProvider;
         this.log = log;
+        this.requests = new Set();
         this.txCollectionSink = new TxCollectionSink(this.txPool, telemetryClient, this.log);
-        this.fastCollection = new FastTxCollection(this.reqResp, this.nodes, this.txCollectionSink, this.config, this.dateProvider, this.log);
-        this.slowCollection = new SlowTxCollection(this.reqResp, this.nodes, this.txCollectionSink, this.fastCollection, constants, this.config, this.dateProvider, this.log);
-        this.reconcileFoundTxsLoop = new RunningPromise(()=>this.reconcileFoundTxsWithPool(), this.log, this.config.txCollectionReconcileIntervalMs);
+        this.reqRespTxsCollector = (requestTracker, blockTxsSource, pinnedPeer)=>BatchTxRequester.collectAllTxs(new BatchTxRequester(requestTracker, blockTxsSource, pinnedPeer, this.p2pService, this.log, this.dateProvider).run());
+        this.fileStoreFastCollection = new FileStoreTxCollection(fileStoreSources, this.txCollectionSink, {
+            workerCount: config.txCollectionFileStoreFastWorkerCount,
+            backoffBaseMs: config.txCollectionFileStoreFastBackoffBaseMs,
+            backoffMaxMs: config.txCollectionFileStoreFastBackoffMaxMs
+        }, this.dateProvider, this.log);
         this.handleTxsFound = (args)=>{
             this.foundTxs(args.txs);
         };
@@ -54,75 +59,298 @@ import { TxCollectionSink } from './tx_collection_sink.js';
         };
         this.txPool.on('txs-added', this.handleTxsAddedToPool);
     }
-    /** Starts all collection loops. */ start() {
-        this.slowCollection.start();
-        this.reconcileFoundTxsLoop.start();
-        // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup
-        return Promise.resolve();
-    }
-    /** Stops all activity. */ async stop() {
-        await Promise.all([
-            this.slowCollection.stop(),
-            this.fastCollection.stop(),
-            this.reconcileFoundTxsLoop.stop()
-        ]);
+    /** Stops all activity. Cancels in-flight requests; file store workers self-terminate. */ stop() {
+        this.requests.forEach((request)=>{
+            request.requestTracker.cancel();
+        });
         this.txPool.removeListener('txs-added', this.handleTxsAddedToPool);
         this.txCollectionSink.removeListener('txs-added', this.handleTxsFound);
     }
-    /** Force trigger the slow collection and reconciliation loops */ async trigger() {
-        await Promise.all([
-            this.reconcileFoundTxsLoop.trigger(),
-            this.slowCollection.trigger()
-        ]);
-    }
     /** Returns L1 rollup constants. */ getConstants() {
         return this.constants;
     }
-    /** Starts collecting the given tx hashes for the given L2Block in the slow loop */ startCollecting(block, txHashes) {
-        return this.slowCollection.startCollecting(block, txHashes);
-    }
-    /** Collects the set of txs for the given block proposal as fast as possible */ collectFastForProposal(blockProposal, blockNumber, txHashes, opts) {
-        return this.collectFastFor({
-            type: 'proposal',
-            blockProposal,
-            blockNumber
-        }, txHashes, opts);
-    }
     /** Collects the set of txs for the given mined block as fast as possible */ collectFastForBlock(block, txHashes, opts) {
         return this.collectFastFor({
             type: 'block',
             block
         }, txHashes, opts);
     }
-    /** Collects the set of txs for the given proposal or block as fast as possible */ collectFastFor(input, txHashes, opts) {
-        return this.fastCollection.collectFastFor(input, txHashes, opts);
+    /** Collects the set of txs for the given proposal or block as fast as possible */ async collectFastFor(input, txHashes, opts) {
+        const timeout = opts.deadline.getTime() - this.dateProvider.now();
+        if (timeout <= 0) {
+            this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, {
+                deadline: opts.deadline.getTime(),
+                now: this.dateProvider.now()
+            });
+            return [];
+        }
+        const hashes = txHashes.map((h)=>typeof h === 'string' ? TxHash.fromString(h) : h);
+        const blockInfo = input.type === 'proposal' ? {
+            ...input.blockProposal.toBlockInfo(),
+            blockNumber: input.blockNumber
+        } : {
+            ...input.block.toBlockInfo()
+        };
+        const request = {
+            ...input,
+            blockInfo,
+            requestTracker: RequestTracker.create(hashes, opts.deadline, this.dateProvider)
+        };
+        const [duration] = await elapsed(()=>this.collectFast(request, {
+                pinnedPeer: opts.pinnedPeer
+            }));
+        this.log.verbose(`Collected ${request.requestTracker.collectedTxs.length} txs out of ${hashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            duration,
+            requestType: input.type,
+            missingTxs: [
+                ...request.requestTracker.missingTxHashes
+            ]
+        });
+        return request.requestTracker.collectedTxs;
     }
-    /** Mark the given txs as found. Stops collecting them. */ foundTxs(txs) {
-        this.slowCollection.foundTxs(txs);
-        this.fastCollection.foundTxs(txs);
+    async collectFast(request, opts) {
+        this.requests.add(request);
+        const { blockInfo } = request;
+        this.log.debug(`Starting fast collection of ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            requestType: request.type,
+            deadline: request.requestTracker.deadline
+        });
+        try {
+            // 1. Start node collection in the background.
+            // Note: this will be a noop if no nodes are configured.
+            const nodeCollectionPromise = this.collectFastFromNodes(request);
+            // 2. Wait before starting reqresp, interruptible by cancellation or node exhaustion.
+            await Promise.race([
+                request.requestTracker.cancellationToken,
+                sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs),
+                nodeCollectionPromise
+            ]);
+            // 3. Start reqresp in the background (runs in parallel with node collection).
+            // Note: this will be a noop if all TXs were already found.
+            const reqRespPromise = this.collectFastViaReqResp(request, opts);
+            // 4. Wait before starting file store, interruptible by cancellation.
+            await Promise.race([
+                request.requestTracker.cancellationToken,
+                sleep(this.config.txCollectionFileStoreFastDelayMs),
+                reqRespPromise
+            ]);
+            // 5. Start file store collection in the background. Self-terminates on tracker cancel / all-found.
+            // Note: this will be a noop if all TXs were already found.
+            const fileStorePromise = this.fileStoreFastCollection.startCollecting(request.requestTracker, this.getAddContext(request));
+            // 6. Wait for all paths to settle.
+            // NOTE: The request will automatically be cancelled after `opt.deadline` is reached.
+            await Promise.allSettled([
+                reqRespPromise,
+                nodeCollectionPromise,
+                fileStorePromise
+            ]);
+        } catch (err) {
+            this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, {
+                ...blockInfo,
+                missingTxs: request.requestTracker.missingTxHashes.values().map((txHash)=>txHash.toString())
+            });
+        } finally{
+            request.requestTracker.cancel();
+            this.requests.delete(request);
+        }
     }
     /**
-   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests
+   * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
+   * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
+   * tx is requested more than once at the same time to the same node.
+   */ async collectFastFromNodes(request) {
+        if (this.nodes.length === 0) {
+            return;
+        }
+        // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
+        const attemptsPerTx = [
+            ...request.requestTracker.missingTxHashes
+        ].map((txHash)=>({
+                txHash,
+                attempts: 0,
+                found: false
+            }));
+        // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
+        await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx)));
+    }
+    async collectFastFromNode(request, node, attemptsPerTx) {
+        const notFinished = ()=>!request.requestTracker.checkCancelled();
+        const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
+        const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
+        const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
+        const processBatch = async ()=>{
+            while(notFinished()){
+                // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts,
+                // so we prioritize txs that have been requested less often.
+                const batch = [];
+                let index = 0;
+                while(batch.length < maxBatchSize){
+                    const txToRequest = attemptsPerTx[index++];
+                    if (!txToRequest) {
+                        break;
+                    } else if (!request.requestTracker.isMissing(txToRequest.txHash)) {
+                        // Mark as found if it was found somewhere else, we'll then remove it from the array.
+                        // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
+                        txToRequest.found = true;
+                    } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) {
+                        // If the tx is not already being requested to this node, add it to the current batch and increase attempts.
+                        // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that
+                        // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting
+                        // the same txs at the same time.
+                        batch.push(txToRequest);
+                        activeRequestsToThisNode.add(txToRequest.txHash);
+                        txToRequest.attempts++;
+                    }
+                }
+                // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end.
+                attemptsPerTx.sort((a, b)=>a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found));
+                const firstFoundTxIndex = attemptsPerTx.findIndex((tx)=>tx.found);
+                if (firstFoundTxIndex !== -1) {
+                    attemptsPerTx.length = firstFoundTxIndex;
+                }
+                // If we see no more txs to request, we can stop this "process" loop
+                if (batch.length === 0) {
+                    return;
+                }
+                const txHashes = batch.map(({ txHash })=>txHash);
+                // Collect this batch from the node
+                await this.txCollectionSink.collect(async ()=>{
+                    const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
+                    for (const tx of result.validTxs){
+                        request.requestTracker.markFetched(tx);
+                    }
+                    return result;
+                }, txHashes, {
+                    description: `fast ${node.getInfo()}`,
+                    node: node.getInfo(),
+                    method: 'fast-node-rpc',
+                    ...request.blockInfo
+                }, this.getAddContext(request));
+                // Clear from the active requests the txs we just requested
+                for (const requestedTx of batch){
+                    activeRequestsToThisNode.delete(requestedTx.txHash);
+                }
+                // Sleep a bit until hitting the node again, but wake up immediately on cancellation
+                if (notFinished()) {
+                    await Promise.race([
+                        sleep(this.config.txCollectionFastNodeIntervalMs),
+                        request.requestTracker.cancellationToken
+                    ]);
+                }
+            }
+        };
+        // Kick off N parallel requests to the node, up to the maxParallelRequests limit
+        await Promise.all(times(maxParallelRequests, processBatch));
+    }
+    async collectFastViaReqResp(request, opts) {
+        const pinnedPeer = opts.pinnedPeer;
+        const blockInfo = request.blockInfo;
+        const slotNumber = blockInfo.slotNumber;
+        if (request.requestTracker.timeoutMs < 100) {
+            this.log.warn(`Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`, {
+                timeoutMs: request.requestTracker.timeoutMs,
+                ...blockInfo
+            });
+            return;
+        }
+        if (request.requestTracker.checkCancelled()) {
+            this.log.debug(`No txs to collect via reqresp for ${request.type} at slot ${blockInfo.slotNumber}`, {
+                ...blockInfo
+            });
+            return;
+        }
+        this.log.debug(`Starting fast reqresp for ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            timeoutMs: request.requestTracker.timeoutMs,
+            pinnedPeer
+        });
+        try {
+            await this.txCollectionSink.collect(async ()=>{
+                let blockTxsSource;
+                if (request.type === 'proposal') {
+                    blockTxsSource = request.blockProposal;
+                } else if (request.type === 'block') {
+                    blockTxsSource = {
+                        txHashes: request.block.body.txEffects.map((e)=>e.txHash),
+                        archive: request.block.archive.root
+                    };
+                } else {
+                    throw new Error(`Unknown request type: ${request.type}`);
+                }
+                const result = await this.reqRespTxsCollector(request.requestTracker, blockTxsSource, pinnedPeer);
+                return {
+                    validTxs: result,
+                    invalidTxHashes: []
+                };
+            }, Array.from(request.requestTracker.missingTxHashes), {
+                description: `reqresp for slot ${slotNumber}`,
+                method: 'fast-req-resp',
+                ...opts,
+                ...request.blockInfo
+            }, this.getAddContext(request));
+        } catch (err) {
+            this.log.error(`Error sending fast reqresp request for txs`, err, {
+                txs: [
+                    ...request.requestTracker.missingTxHashes
+                ],
+                ...blockInfo
+            });
+        }
+    }
+    /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */ getAddContext(request) {
+        if (request.type === 'proposal') {
+            return {
+                type: 'proposal',
+                blockHeader: request.blockProposal.blockHeader
+            };
+        } else {
+            return {
+                type: 'mined',
+                block: request.block
+            };
+        }
+    }
+    /** Mark the given txs as found. Stops collecting them across all in-flight requests. */ foundTxs(txs) {
+        for (const request of this.requests){
+            for (const tx of txs){
+                if (request.requestTracker.markFetched(tx)) {
+                    this.log.trace(`Found tx ${tx.txHash.toString()} for fast collection request`, {
+                        ...request.blockInfo,
+                        txHash: tx.txHash.toString(),
+                        type: request.type
+                    });
+                    if (request.requestTracker.allFetched()) {
+                        this.log.trace(`All txs found for fast collection request`, {
+                            ...request.blockInfo,
+                            type: request.type
+                        });
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks less than or equal to the block number specified.
    * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
    */ stopCollectingForBlocksUpTo(blockNumber) {
-        this.slowCollection.stopCollectingForBlocksUpTo(blockNumber);
-        this.fastCollection.stopCollectingForBlocksUpTo(blockNumber);
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber <= blockNumber) {
+                request.requestTracker.cancel();
+            }
+        }
     }
     /**
    * Stop collecting all txs for blocks greater than the block number specified.
    * To be called when there is a chain prune and previously mined txs are no longer relevant.
    */ stopCollectingForBlocksAfter(blockNumber) {
-        this.slowCollection.stopCollectingForBlocksAfter(blockNumber);
-        this.fastCollection.stopCollectingForBlocksAfter(blockNumber);
-    }
-    /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */ async reconcileFoundTxsWithPool() {
-        const missingTxHashes = this.slowCollection.getMissingTxHashes();
-        const foundTxs = compactArray(await this.txPool.getTxsByHash(missingTxHashes));
-        if (foundTxs.length > 0) {
-            this.log.verbose(`Found ${foundTxs.length} txs in the pool during reconciliation`, {
-                foundTxs: foundTxs.map((t)=>t.getTxHash().toString())
-            });
-            this.foundTxs(foundTxs);
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber > blockNumber) {
+                request.requestTracker.cancel();
+            }
         }
     }
 }