@aztec/p2p 0.0.1-commit.8afd444 → 0.0.1-commit.934299a21

package/dest/services/reqresp/reqresp.js

@@ -384,7 +384,7 @@ import { SnappyTransform } from '../encoding.js';
 import { DEFAULT_INDIVIDUAL_REQUEST_TIMEOUT_MS, DEFAULT_REQRESP_DIAL_TIMEOUT_MS } from './config.js';
 import { BatchConnectionSampler } from './connection-sampler/batch_connection_sampler.js';
 import { ConnectionSampler, RandomSampler } from './connection-sampler/connection_sampler.js';
-import { DEFAULT_SUB_PROTOCOL_VALIDATORS, ReqRespSubProtocol, responseFromBuffer } from './interface.js';
+import { DEFAULT_SUB_PROTOCOL_VALIDATORS, ReqRespSubProtocol, responseFromBuffer, subProtocolSizeCalculators } from './interface.js';
 import { ReqRespMetrics } from './metrics.js';
 import { RateLimitStatus, RequestResponseRateLimiter, prettyPrintRateLimitStatus } from './rate-limiter/rate_limiter.js';
 import { ReqRespStatus, ReqRespStatusError, parseStatusChunk, prettyPrintReqRespStatus } from './status.js';
@@ -709,17 +709,21 @@ _dec = trackSpan('ReqResp.sendBatchRequest', (subProtocol, requests)=>({
         let stream;
         try {
             this.metrics.recordRequestSent(subProtocol);
+            // Calculate expected response size based on the request payload
+            const expectedSizeKb = subProtocolSizeCalculators[subProtocol](payload);
             this.logger.trace(`Sending request to peer ${peerId.toString()} on sub protocol ${subProtocol}`);
             stream = await this.connectionSampler.dialProtocol(peerId, subProtocol, dialTimeout);
             this.logger.trace(`Opened stream ${stream.id} for sending request to peer ${peerId.toString()} on sub protocol ${subProtocol}`);
             const timeoutErr = new IndividualReqRespTimeoutError();
+            // Create a wrapper to pass the expected size to readMessage
+            const readMessageWithSizeLimit = (source)=>this.readMessage(source, expectedSizeKb);
             const [_, resp] = await executeTimeout((signal)=>Promise.all([
                     pipeline([
                         payload
                     ], stream.sink, {
                         signal
                     }),
-                    pipeline(stream.source, this.readMessage.bind(this), {
+                    pipeline(stream.source, readMessageWithSizeLimit, {
                         signal
                     })
                 ]), this.individualRequestTimeoutMs, ()=>timeoutErr);
@@ -773,7 +777,10 @@ _dec = trackSpan('ReqResp.sendBatchRequest', (subProtocol, requests)=>({
    * The message is split into two components
    * - The first chunk should contain a control byte, indicating the status of the response see `ReqRespStatus`
    * - The second chunk should contain the response data
-   */ async readMessage(source) {
+   *
+   * @param source - The async iterable source of data chunks
+   * @param maxSizeKb - Optional maximum expected size in KB for the decompressed response
+   */ async readMessage(source, maxSizeKb) {
         let status;
         const chunks = [];
         try {
@@ -794,7 +801,7 @@ _dec = trackSpan('ReqResp.sendBatchRequest', (subProtocol, requests)=>({
                 }
             }
             const messageData = Buffer.concat(chunks);
-            const message = this.snappyTransform.inboundTransformData(messageData);
+            const message = this.snappyTransform.inboundTransformData(messageData, undefined, maxSizeKb);
             return {
                 status: status ?? ReqRespStatus.UNKNOWN,
                 data: message

package/dest/services/service.d.ts

@@ -1,6 +1,7 @@
+import type { SlotNumber } from '@aztec/foundation/branded-types';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { PeerInfo } from '@aztec/stdlib/interfaces/server';
-import type { BlockProposal, CheckpointAttestation, CheckpointProposalCore, Gossipable } from '@aztec/stdlib/p2p';
+import type { BlockProposal, CheckpointAttestation, CheckpointProposalCore, Gossipable, TopicType } from '@aztec/stdlib/p2p';
 import type { Tx } from '@aztec/stdlib/tx';
 import type { PeerId } from '@libp2p/interface';
 import type { ENR } from '@nethermindeth/enr';
@@ -28,6 +29,28 @@ export type P2PBlockReceivedCallback = (block: BlockProposal, sender: PeerId) =>
  */
 export type P2PCheckpointReceivedCallback = (checkpoint: CheckpointProposalCore, sender: PeerId) => Promise<CheckpointAttestation[] | undefined>;
 export type AuthReceivedCallback = (peerId: PeerId, authRequest: AuthRequest) => Promise<AuthResponse | undefined>;
+/** Minimal info passed to the duplicate proposal callback. */
+export type DuplicateProposalInfo = {
+    slot: SlotNumber;
+    proposer: EthAddress;
+    type: 'checkpoint' | 'block';
+};
+/**
+ * Callback for when a duplicate proposal is detected (equivocation).
+ * Invoked on the first duplicate (when count goes from 1 to 2).
+ */
+export type P2PDuplicateProposalCallback = (info: DuplicateProposalInfo) => void;
+/** Minimal info passed to the duplicate attestation callback. */
+export type DuplicateAttestationInfo = {
+    slot: SlotNumber;
+    attester: EthAddress;
+};
+/**
+ * Callback for when a duplicate attestation is detected (equivocation).
+ * A validator signing attestations for different proposals at the same slot.
+ * Invoked on the first duplicate (when count goes from 1 to 2).
+ */
+export type P2PDuplicateAttestationCallback = (info: DuplicateAttestationInfo) => void;
 /**
  * The interface for a P2P service implementation.
  */
@@ -57,9 +80,22 @@ export interface P2PService {
     sendBatchRequest<Protocol extends ReqRespSubProtocol>(protocol: Protocol, requests: InstanceType<SubProtocolMap[Protocol]['request']>[], pinnedPeerId?: PeerId, timeoutMs?: number, maxPeers?: number, maxRetryAttempts?: number): Promise<InstanceType<SubProtocolMap[Protocol]['response']>[]>;
     registerBlockReceivedCallback(callback: P2PBlockReceivedCallback): void;
     registerCheckpointReceivedCallback(callback: P2PCheckpointReceivedCallback): void;
+    /**
+     * Registers a callback invoked when a duplicate proposal is detected (equivocation).
+     * The callback is triggered on the first duplicate (when count goes from 1 to 2).
+     */
+    registerDuplicateProposalCallback(callback: P2PDuplicateProposalCallback): void;
+    /**
+     * Registers a callback invoked when a duplicate attestation is detected (equivocation).
+     * A validator signing attestations for different proposals at the same slot.
+     * The callback is triggered on the first duplicate (when count goes from 1 to 2).
+     */
+    registerDuplicateAttestationCallback(callback: P2PDuplicateAttestationCallback): void;
     getEnr(): ENR | undefined;
     getPeers(includePending?: boolean): PeerInfo[];
-    validate(txs: Tx[]): Promise<void>;
+    /** Returns the number of peers in the GossipSub mesh for a given topic type. */
+    getGossipMeshPeerCount(topicType: TopicType): number;
+    validateTxsReceivedInBlockProposal(txs: Tx[]): Promise<void>;
     addReqRespSubProtocol(subProtocol: ReqRespSubProtocol, handler: ReqRespSubProtocolHandler, validator?: ReqRespSubProtocolValidators[ReqRespSubProtocol]): Promise<void>;
     handleAuthRequestFromPeer(authRequest: AuthRequest, peerId: PeerId): Promise<StatusMessage>;
     updateConfig(config: Partial<P2PReqRespConfig>): void;
@@ -104,4 +140,4 @@ export interface PeerDiscoveryService extends EventEmitter {
     getEnr(): ENR | undefined;
     bootstrapNodeEnrs: ENR[];
 }
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2VydmljZS5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3NlcnZpY2VzL3NlcnZpY2UudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFDaEUsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFDaEUsT0FBTyxLQUFLLEVBQUUsYUFBYSxFQUFFLHFCQUFxQixFQUFFLHNCQUFzQixFQUFFLFVBQVUsRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBQ2xILE9BQU8sS0FBSyxFQUFFLEVBQUUsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRTNDLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBQ2hELE9BQU8sS0FBSyxFQUFFLEdBQUcsRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBQzlDLE9BQU8sS0FBSyxZQUFZLE1BQU0sUUFBUSxDQUFDO0FBRXZDLE9BQU8sS0FBSyxFQUFFLDZCQUE2QixFQUFFLE1BQU0sMkNBQTJDLENBQUM7QUFDL0YsT0FBTyxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUM1RCxPQUFPLEtBQUssRUFBRSxhQUFhLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUN4RCxPQUFPLEtBQUssRUFDVixrQkFBa0IsRUFDbEIseUJBQXlCLEVBQ3pCLDRCQUE0QixFQUM1QixjQUFjLEVBQ2YsTUFBTSx3QkFBd0IsQ0FBQztBQUNoQyxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsWUFBWSxFQUFFLE1BQU0sNkJBQTZCLENBQUM7QUFFN0Usb0JBQVksa0JBQWtCO0lBQzVCLE9BQU8sWUFBWTtJQUNuQixPQUFPLFlBQVk7Q0FDcEI7QUFFRDs7OztHQUlHO0FBQ0gsTUFBTSxNQUFNLHdCQUF3QixHQUFHLENBQUMsS0FBSyxFQUFFLGFBQWEsRUFBRSxNQUFNLEVBQUUsTUFBTSxLQUFLLE9BQU8sQ0FBQyxPQUFPLENBQUMsQ0FBQztBQUVsRzs7Ozs7R0FLRztBQUNILE1BQU0sTUFBTSw2QkFBNkIsR0FBRyxDQUMxQyxVQUFVLEVBQUUsc0JBQXNCLEVBQ2xDLE1BQU0sRUFBRSxNQUFNLEtBQ1gsT0FBTyxDQUFDLHFCQUFxQixFQUFFLEdBQUcsU0FBUyxDQUFDLENBQUM7QUFFbEQsTUFBTSxNQUFNLG9CQUFvQixHQUFHLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxXQUFXLEVBQUUsV0FBVyxLQUFLLE9BQU8sQ0FBQyxZQUFZLEdBQUcsU0FBUyxDQUFDLENBQUM7QUFFbkg7O0dBRUc7QUFDSCxNQUFNLFdBQVcsVUFBVTtJQUN6Qjs7O09BR0c7SUFDSCxLQUFLLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRXZCOzs7T0FHRztJQUNILElBQUksSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFFdEI7OztPQUdHO0lBQ0gsU0FBUyxDQUFDLENBQUMsU0FBUyxVQUFVLEVBQUUsT0FBTyxFQUFFLENBQUMsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFFM0Q7Ozs7OztPQU1HO0lBQ0gsZ0JBQWdCLENBQUMsUUFBUSxTQUFTLGtCQUFrQixFQUNsRCxRQUFRLEVBQUUsUUFBUSxFQUNsQixRQUFRLEVBQUUsWUFBWSxDQUFDLGNBQWMsQ0FBQyxRQUFRLENBQUMsQ0FBQyxTQUFTLENBQUMsQ0FBQyxFQUFFLEVBQzdELFlBQVksQ0FBQyxFQUFFLE1BQU0sRUFDckIsU0FBUyxDQUFDLEVBQUUsTUFBTSxFQUNsQixRQUFRLENBQUMsRUFBRSxNQUFNLEVBQ2pCLGdCQUFnQixDQUFDLEVBQUUsTUFBTSxHQUN4QixPQUFPLENBQUMsWUFBWSxDQUFDLGNBQWMsQ0FBQyxRQUFRLENBQUMsQ0FBQyxVQUFVLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUdqRSw2QkFBNkIsQ0FBQyxRQUFRLEVBQUUsd0JBQXdCLEdBQUcsSUFBSSxDQUFDO0lBRXhFLGtDQUFrQyxDQUFDLFFBQVEsRUFBRSw2QkFBNkIsR0FBRyxJQUFJLENBQUM7SUFFbEYsTUFBTSxJQUFJLEdBQUcsR0FBRyxTQUFTLENBQUM7SUFFMUIsUUFBUSxDQUFDLGNBQWMsQ0FBQyxFQUFFLE9BQU8sR0FBRyxRQUFRLEVBQUUsQ0FBQztJQUUvQyxRQUFRLENBQUMsR0FBRyxFQUFFLEVBQUUsRUFBRSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUVuQyxxQkFBcUIsQ0FDbkIsV0FBVyxFQUFFLGtCQUFrQixFQUMvQixPQUFPLEVBQUUseUJBQXlCLEVBQ2xDLFNBQVMsQ0FBQyxFQUFFLDRCQUE0QixDQUFDLGtCQUFrQixDQUFDLEdBQzNELE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUVqQix5QkFBeUIsQ0FBQyxXQUFXLEVBQUUsV0FBVyxFQUFFLE1BQU0sRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUFDLGFBQWEsQ0FBQyxDQUFDO0lBRTVGLFlBQVksQ0FBQyxNQUFNLEVBQUUsT0FBTyxDQUFDLGdCQUFnQixDQUFDLEdBQUcsSUFBSSxDQUFDO0lBRXRELDRGQUE0RjtJQUM1Riw4QkFBOEIsQ0FBQyxPQUFPLEVBQUUsVUFBVSxFQUFFLEdBQUcsSUFBSSxDQUFDO0lBRTVELGdEQUFnRDtJQUNoRCwwQkFBMEIsSUFBSSw2QkFBNkIsQ0FBQztDQUM3RDtBQUVEOztHQUVHO0FBQ0gsTUFBTSxXQUFXLG9CQUFxQixTQUFRLFlBQVk7SUFDeEQ7O1NBRUs7SUFDTCxLQUFLLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRXZCOztTQUVLO0lBQ0wsSUFBSSxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUV0Qjs7O09BR0c7SUFDSCxZQUFZLElBQUksR0FBRyxFQUFFLENBQUM7SUFFdEI7O09BRUc7SUFDSCxtQkFBbUIsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFFckM7Ozs7T0FJRztJQUNILGVBQWUsQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FBQztJQUV6Qzs7T0FFRztJQUNILEVBQUUsQ0FBQyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsUUFBUSxFQUFFLENBQUMsR0FBRyxFQUFFLEdBQUcsS0FBSyxJQUFJLEdBQUcsSUFBSSxDQUFDO0lBQ2pFLElBQUksQ0FBQyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsR0FBRyxFQUFFLEdBQUcsR0FBRyxPQUFPLENBQUM7SUFFbEQsU0FBUyxJQUFJLGtCQUFrQixDQUFDO0lBRWhDLE1BQU0sSUFBSSxHQUFHLEdBQUcsU0FBUyxDQUFDO0lBRTFCLGlCQUFpQixFQUFFLEdBQUcsRUFBRSxDQUFDO0NBQzFCIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2VydmljZS5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3NlcnZpY2VzL3NlcnZpY2UudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFDbEUsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFDaEUsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFDaEUsT0FBTyxLQUFLLEVBQ1YsYUFBYSxFQUNiLHFCQUFxQixFQUNyQixzQkFBc0IsRUFDdEIsVUFBVSxFQUNWLFNBQVMsRUFDVixNQUFNLG1CQUFtQixDQUFDO0FBQzNCLE9BQU8sS0FBSyxFQUFFLEVBQUUsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRTNDLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBQ2hELE9BQU8sS0FBSyxFQUFFLEdBQUcsRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBQzlDLE9BQU8sS0FBSyxZQUFZLE1BQU0sUUFBUSxDQUFDO0FBRXZDLE9BQU8sS0FBSyxFQUFFLDZCQUE2QixFQUFFLE1BQU0sMkNBQTJDLENBQUM7QUFDL0YsT0FBTyxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUM1RCxPQUFPLEtBQUssRUFBRSxhQUFhLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUN4RCxPQUFPLEtBQUssRUFDVixrQkFBa0IsRUFDbEIseUJBQXlCLEVBQ3pCLDRCQUE0QixFQUM1QixjQUFjLEVBQ2YsTUFBTSx3QkFBd0IsQ0FBQztBQUNoQyxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsWUFBWSxFQUFFLE1BQU0sNkJBQTZCLENBQUM7QUFFN0Usb0JBQVksa0JBQWtCO0lBQzVCLE9BQU8sWUFBWTtJQUNuQixPQUFPLFlBQVk7Q0FDcEI7QUFFRDs7OztHQUlHO0FBQ0gsTUFBTSxNQUFNLHdCQUF3QixHQUFHLENBQUMsS0FBSyxFQUFFLGFBQWEsRUFBRSxNQUFNLEVBQUUsTUFBTSxLQUFLLE9BQU8sQ0FBQyxPQUFPLENBQUMsQ0FBQztBQUVsRzs7Ozs7R0FLRztBQUNILE1BQU0sTUFBTSw2QkFBNkIsR0FBRyxDQUMxQyxVQUFVLEVBQUUsc0JBQXNCLEVBQ2xDLE1BQU0sRUFBRSxNQUFNLEtBQ1gsT0FBTyxDQUFDLHFCQUFxQixFQUFFLEdBQUcsU0FBUyxDQUFDLENBQUM7QUFFbEQsTUFBTSxNQUFNLG9CQUFvQixHQUFHLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxXQUFXLEVBQUUsV0FBVyxLQUFLLE9BQU8sQ0FBQyxZQUFZLEdBQUcsU0FBUyxDQUFDLENBQUM7QUFFbkgsOERBQThEO0FBQzlELE1BQU0sTUFBTSxxQkFBcUIsR0FBRztJQUNsQyxJQUFJLEVBQUUsVUFBVSxDQUFDO0lBQ2pCLFFBQVEsRUFBRSxVQUFVLENBQUM7SUFDckIsSUFBSSxFQUFFLFlBQVksR0FBRyxPQUFPLENBQUM7Q0FDOUIsQ0FBQztBQUVGOzs7R0FHRztBQUNILE1BQU0sTUFBTSw0QkFBNEIsR0FBRyxDQUFDLElBQUksRUFBRSxxQkFBcUIsS0FBSyxJQUFJLENBQUM7QUFFakYsaUVBQWlFO0FBQ2pFLE1BQU0sTUFBTSx3QkFBd0IsR0FBRztJQUNyQyxJQUFJLEVBQUUsVUFBVSxDQUFDO0lBQ2pCLFFBQVEsRUFBRSxVQUFVLENBQUM7Q0FDdEIsQ0FBQztBQUVGOzs7O0dBSUc7QUFDSCxNQUFNLE1BQU0sK0JBQStCLEdBQUcsQ0FBQyxJQUFJLEVBQUUsd0JBQXdCLEtBQUssSUFBSSxDQUFDO0FBRXZGOztHQUVHO0FBQ0gsTUFBTSxXQUFXLFVBQVU7SUFDekI7OztPQUdHO0lBQ0gsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUV2Qjs7O09BR0c7SUFDSCxJQUFJLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRXRCOzs7T0FHRztJQUNILFNBQVMsQ0FBQyxDQUFDLFNBQVMsVUFBVSxFQUFFLE9BQU8sRUFBRSxDQUFDLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRTNEOzs7Ozs7T0FNRztJQUNILGdCQUFnQixDQUFDLFFBQVEsU0FBUyxrQkFBa0IsRUFDbEQsUUFBUSxFQUFFLFFBQVEsRUFDbEIsUUFBUSxFQUFFLFlBQVksQ0FBQyxjQUFjLENBQUMsUUFBUSxDQUFDLENBQUMsU0FBUyxDQUFDLENBQUMsRUFBRSxFQUM3RCxZQUFZLENBQUMsRUFBRSxNQUFNLEVBQ3JCLFNBQVMsQ0FBQyxFQUFFLE1BQU0sRUFDbEIsUUFBUSxDQUFDLEVBQUUsTUFBTSxFQUNqQixnQkFBZ0IsQ0FBQyxFQUFFLE1BQU0sR0FDeEIsT0FBTyxDQUFDLFlBQVksQ0FBQyxjQUFjLENBQUMsUUFBUSxDQUFDLENBQUMsVUFBVSxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUM7SUFHakUsNkJBQTZCLENBQUMsUUFBUSxFQUFFLHdCQUF3QixHQUFHLElBQUksQ0FBQztJQUV4RSxrQ0FBa0MsQ0FBQyxRQUFRLEVBQUUsNkJBQTZCLEdBQUcsSUFBSSxDQUFDO0lBRWxGOzs7T0FHRztJQUNILGlDQUFpQyxDQUFDLFFBQVEsRUFBRSw0QkFBNEIsR0FBRyxJQUFJLENBQUM7SUFFaEY7Ozs7T0FJRztJQUNILG9DQUFvQyxDQUFDLFFBQVEsRUFBRSwrQkFBK0IsR0FBRyxJQUFJLENBQUM7SUFFdEYsTUFBTSxJQUFJLEdBQUcsR0FBRyxTQUFTLENBQUM7SUFFMUIsUUFBUSxDQUFDLGNBQWMsQ0FBQyxFQUFFLE9BQU8sR0FBRyxRQUFRLEVBQUUsQ0FBQztJQUUvQyxnRkFBZ0Y7SUFDaEYsc0JBQXNCLENBQUMsU0FBUyxFQUFFLFNBQVMsR0FBRyxNQUFNLENBQUM7SUFFckQsa0NBQWtDLENBQUMsR0FBRyxFQUFFLEVBQUUsRUFBRSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUU3RCxxQkFBcUIsQ0FDbkIsV0FBVyxFQUFFLGtCQUFrQixFQUMvQixPQUFPLEVBQUUseUJBQXlCLEVBQ2xDLFNBQVMsQ0FBQyxFQUFFLDRCQUE0QixDQUFDLGtCQUFrQixDQUFDLEdBQzNELE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUVqQix5QkFBeUIsQ0FBQyxXQUFXLEVBQUUsV0FBVyxFQUFFLE1BQU0sRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUFDLGFBQWEsQ0FBQyxDQUFDO0lBRTVGLFlBQVksQ0FBQyxNQUFNLEVBQUUsT0FBTyxDQUFDLGdCQUFnQixDQUFDLEdBQUcsSUFBSSxDQUFDO0lBRXRELDRGQUE0RjtJQUM1Riw4QkFBOEIsQ0FBQyxPQUFPLEVBQUUsVUFBVSxFQUFFLEdBQUcsSUFBSSxDQUFDO0lBRTVELGdEQUFnRDtJQUNoRCwwQkFBMEIsSUFBSSw2QkFBNkIsQ0FBQztDQUM3RDtBQUVEOztHQUVHO0FBQ0gsTUFBTSxXQUFXLG9CQUFxQixTQUFRLFlBQVk7SUFDeEQ7O1NBRUs7SUFDTCxLQUFLLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRXZCOztTQUVLO0lBQ0wsSUFBSSxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUV0Qjs7O09BR0c7SUFDSCxZQUFZLElBQUksR0FBRyxFQUFFLENBQUM7SUFFdEI7O09BRUc7SUFDSCxtQkFBbUIsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFFckM7Ozs7T0FJRztJQUNILGVBQWUsQ0FBQyxNQUFNLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FBQztJQUV6Qzs7T0FFRztJQUNILEVBQUUsQ0FBQyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsUUFBUSxFQUFFLENBQUMsR0FBRyxFQUFFLEdBQUcsS0FBSyxJQUFJLEdBQUcsSUFBSSxDQUFDO0lBQ2pFLElBQUksQ0FBQyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsR0FBRyxFQUFFLEdBQUcsR0FBRyxPQUFPLENBQUM7SUFFbEQsU0FBUyxJQUFJLGtCQUFrQixDQUFDO0lBRWhDLE1BQU0sSUFBSSxHQUFHLEdBQUcsU0FBUyxDQUFDO0lBRTFCLGlCQUFpQixFQUFFLEdBQUcsRUFBRSxDQUFDO0NBQzFCIn0=

package/dest/services/service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../src/services/service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAE,aAAa,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAClH,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,YAAY,MAAM,QAAQ,CAAC;AAEvC,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,2CAA2C,CAAC;AAC/F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EACV,kBAAkB,EAClB,yBAAyB,EACzB,4BAA4B,EAC5B,cAAc,EACf,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE7E,oBAAY,kBAAkB;IAC5B,OAAO,YAAY;IACnB,OAAO,YAAY;CACpB;AAED;;;;GAIG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAElG;;;;;GAKG;AACH,MAAM,MAAM,6BAA6B,GAAG,CAC1C,UAAU,EAAE,sBAAsB,EAClC,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,qBAAqB,EAAE,GAAG,SAAS,CAAC,CAAC;AAElD,MAAM,MAAM,oBAAoB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,KAAK,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;AAEnH;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;;OAGG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB;;;OAGG;IACH,SAAS,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;OAMG;IACH,gBAAgB,CAAC,QAAQ,SAAS,kBAAkB,EAClD,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAC7D,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,EACjB,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC;IAGjE,6BAA6B,CAAC,QAAQ,EAAE,wBAAwB,GAAG,IAAI,CAAC;IAExE,kCAAkC,CAAC,QAAQ,EAAE,6BAA6B,GAAG,IAAI,CAAC;IAElF,MAAM,IAAI,GAAG,GAAG,SAAS,CAAC;IAE1B,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,GAAG,QAAQ,EAAE,CAAC;IAE/C,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnC,qBAAqB,CACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,yBAAyB,EAClC,SAAS,CAAC,EAAE,4BAA4B,CAAC,kBAAkB,CAAC,GAC3D,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,yBAAyB,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE5F,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IAEtD,4FAA4F;IAC5F,8BAA8B,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAE5D,gDAAgD;IAChD,0BAA0B,IAAI,6BAA6B,CAAC;CAC7D;AAED;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD;;SAEK;IACL,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;SAEK;IACL,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB;;;OAGG;IACH,YAAY,IAAI,GAAG,EAAE,CAAC;IAEtB;;OAEG;IACH,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;;;OAIG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAEzC;;OAEG;IACH,EAAE,CAAC,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,GAAG,IAAI,CAAC;IACjE,IAAI,CAAC,KAAK,EAAE,iBAAiB,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC;IAElD,SAAS,IAAI,kBAAkB,CAAC;IAEhC,MAAM,IAAI,GAAG,GAAG,SAAS,CAAC;IAE1B,iBAAiB,EAAE,GAAG,EAAE,CAAC;CAC1B"}
+{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../src/services/service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EACV,aAAa,EACb,qBAAqB,EACrB,sBAAsB,EACtB,UAAU,EACV,SAAS,EACV,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,YAAY,MAAM,QAAQ,CAAC;AAEvC,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,2CAA2C,CAAC;AAC/F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EACV,kBAAkB,EAClB,yBAAyB,EACzB,4BAA4B,EAC5B,cAAc,EACf,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE7E,oBAAY,kBAAkB;IAC5B,OAAO,YAAY;IACnB,OAAO,YAAY;CACpB;AAED;;;;GAIG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAElG;;;;;GAKG;AACH,MAAM,MAAM,6BAA6B,GAAG,CAC1C,UAAU,EAAE,sBAAsB,EAClC,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,qBAAqB,EAAE,GAAG,SAAS,CAAC,CAAC;AAElD,MAAM,MAAM,oBAAoB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,KAAK,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;AAEnH,8DAA8D;AAC9D,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,UAAU,CAAC;IACrB,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC;CAC9B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,4BAA4B,GAAG,CAAC,IAAI,EAAE,qBAAqB,KAAK,IAAI,CAAC;AAEjF,iEAAiE;AACjE,MAAM,MAAM,wBAAwB,GAAG;IACrC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,+BAA+B,GAAG,CAAC,IAAI,EAAE,wBAAwB,KAAK,IAAI,CAAC;AAEvF;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;;OAGG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB;;;OAGG;IACH,SAAS,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;OAMG;IACH,gBAAgB,CAAC,QAAQ,SAAS,kBAAkB,EAClD,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAC7D,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,EACjB,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC;IAGjE,6BAA6B,CAAC,QAAQ,EAAE,wBAAwB,GAAG,IAAI,CAAC;IAExE,kCAAkC,CAAC,QAAQ,EAAE,6BAA6B,GAAG,IAAI,CAAC;IAElF;;;OAGG;IACH,iCAAiC,CAAC,QAAQ,EAAE,4BAA4B,GAAG,IAAI,CAAC;IAEhF;;;;OAIG;IACH,oCAAoC,CAAC,QAAQ,EAAE,+BAA+B,GAAG,IAAI,CAAC;IAEtF,MAAM,IAAI,GAAG,GAAG,SAAS,CAAC;IAE1B,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,GAAG,QAAQ,EAAE,CAAC;IAE/C,gFAAgF;IAChF,sBAAsB,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,CAAC;IAErD,kCAAkC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D,qBAAqB,CACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,yBAAyB,EAClC,SAAS,CAAC,EAAE,4BAA4B,CAAC,kBAAkB,CAAC,GAC3D,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,yBAAyB,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE5F,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IAEtD,4FAA4F;IAC5F,8BAA8B,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAE5D,gDAAgD;IAChD,0BAA0B,IAAI,6BAA6B,CAAC;CAC7D;AAED;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD;;SAEK;IACL,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;SAEK;IACL,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB;;;OAGG;IACH,YAAY,IAAI,GAAG,EAAE,CAAC;IAEtB;;OAEG;IACH,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;;;OAIG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAEzC;;OAEG;IACH,EAAE,CAAC,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,GAAG,IAAI,CAAC;IACjE,IAAI,CAAC,KAAK,EAAE,iBAAiB,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC;IAElD,SAAS,IAAI,kBAAkB,CAAC;IAEhC,MAAM,IAAI,GAAG,GAAG,SAAS,CAAC;IAE1B,iBAAiB,EAAE,GAAG,EAAE,CAAC;CAC1B"}

package/dest/services/tx_collection/config.d.ts

@@ -1,5 +1,5 @@
 import { type ConfigMappingsType } from '@aztec/foundation/config';
-export type ProposalTxCollectorType = 'new' | 'old';
+export type MissingTxsCollectorType = 'new' | 'old';
 export type TxCollectionConfig = {
     /** How long to wait before starting reqresp for fast collection  */
     txCollectionFastNodesTimeoutBeforeReqRespMs: number;
@@ -21,8 +21,26 @@ export type TxCollectionConfig = {
     txCollectionFastMaxParallelRequestsPerNode: number;
     /** Maximum number of transactions to request from a node in a single batch */
     txCollectionNodeRpcMaxBatchSize: number;
-    /** Which collector implementation to use for proposal tx collection */
-    txCollectionProposalTxCollectorType: ProposalTxCollectorType;
+    /** Which collector implementation to use for missing txs collection */
+    txCollectionMissingTxsCollectorType: MissingTxsCollectorType;
+    /** A comma-separated list of file store URLs (s3://, gs://, file://, http://) for tx collection */
+    txCollectionFileStoreUrls: string[];
+    /** Delay in ms before file store collection starts after slow collection is triggered */
+    txCollectionFileStoreSlowDelayMs: number;
+    /** Delay in ms before file store collection starts after fast collection is triggered */
+    txCollectionFileStoreFastDelayMs: number;
+    /** Number of concurrent workers for fast file store collection */
+    txCollectionFileStoreFastWorkerCount: number;
+    /** Number of concurrent workers for slow file store collection */
+    txCollectionFileStoreSlowWorkerCount: number;
+    /** Base backoff time in ms for fast file store collection retries */
+    txCollectionFileStoreFastBackoffBaseMs: number;
+    /** Base backoff time in ms for slow file store collection retries */
+    txCollectionFileStoreSlowBackoffBaseMs: number;
+    /** Max backoff time in ms for fast file store collection retries */
+    txCollectionFileStoreFastBackoffMaxMs: number;
+    /** Max backoff time in ms for slow file store collection retries */
+    txCollectionFileStoreSlowBackoffMaxMs: number;
 };
 export declare const txCollectionConfigMappings: ConfigMappingsType<TxCollectionConfig>;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9jb25maWcudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUNMLEtBQUssa0JBQWtCLEVBSXhCLE1BQU0sMEJBQTBCLENBQUM7QUFHbEMsTUFBTSxNQUFNLHVCQUF1QixHQUFHLEtBQUssR0FBRyxLQUFLLENBQUM7QUFFcEQsTUFBTSxNQUFNLGtCQUFrQixHQUFHO0lBQy9CLG9FQUFvRTtJQUNwRSwyQ0FBMkMsRUFBRSxNQUFNLENBQUM7SUFDcEQsaURBQWlEO0lBQ2pELCtCQUErQixFQUFFLE1BQU0sQ0FBQztJQUN4QyxzQ0FBc0M7SUFDdEMsaUNBQWlDLEVBQUUsTUFBTSxDQUFDO0lBQzFDLHFFQUFxRTtJQUNyRSxnQ0FBZ0MsRUFBRSxNQUFNLENBQUM7SUFDekMsd0RBQXdEO0lBQ3hELCtCQUErQixFQUFFLE1BQU0sQ0FBQztJQUN4QyxnR0FBZ0c7SUFDaEcseUNBQXlDLEVBQUUsT0FBTyxDQUFDO0lBQ25ELDJGQUEyRjtJQUMzRiw4QkFBOEIsRUFBRSxNQUFNLENBQUM7SUFDdkMsNkVBQTZFO0lBQzdFLHVCQUF1QixFQUFFLE1BQU0sRUFBRSxDQUFDO0lBQ2xDLG1GQUFtRjtJQUNuRiwwQ0FBMEMsRUFBRSxNQUFNLENBQUM7SUFDbkQsOEVBQThFO0lBQzlFLCtCQUErQixFQUFFLE1BQU0sQ0FBQztJQUN4Qyx1RUFBdUU7SUFDdkUsbUNBQW1DLEVBQUUsdUJBQXVCLENBQUM7Q0FDOUQsQ0FBQztBQUVGLGVBQU8sTUFBTSwwQkFBMEIsRUFBRSxrQkFBa0IsQ0FBQyxrQkFBa0IsQ0E4RDdFLENBQUMifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9jb25maWcudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUNMLEtBQUssa0JBQWtCLEVBSXhCLE1BQU0sMEJBQTBCLENBQUM7QUFHbEMsTUFBTSxNQUFNLHVCQUF1QixHQUFHLEtBQUssR0FBRyxLQUFLLENBQUM7QUFFcEQsTUFBTSxNQUFNLGtCQUFrQixHQUFHO0lBQy9CLG9FQUFvRTtJQUNwRSwyQ0FBMkMsRUFBRSxNQUFNLENBQUM7SUFDcEQsaURBQWlEO0lBQ2pELCtCQUErQixFQUFFLE1BQU0sQ0FBQztJQUN4QyxzQ0FBc0M7SUFDdEMsaUNBQWlDLEVBQUUsTUFBTSxDQUFDO0lBQzFDLHFFQUFxRTtJQUNyRSxnQ0FBZ0MsRUFBRSxNQUFNLENBQUM7SUFDekMsd0RBQXdEO0lBQ3hELCtCQUErQixFQUFFLE1BQU0sQ0FBQztJQUN4QyxnR0FBZ0c7SUFDaEcseUNBQXlDLEVBQUUsT0FBTyxDQUFDO0lBQ25ELDJGQUEyRjtJQUMzRiw4QkFBOEIsRUFBRSxNQUFNLENBQUM7SUFDdkMsNkVBQTZFO0lBQzdFLHVCQUF1QixFQUFFLE1BQU0sRUFBRSxDQUFDO0lBQ2xDLG1GQUFtRjtJQUNuRiwwQ0FBMEMsRUFBRSxNQUFNLENBQUM7SUFDbkQsOEVBQThFO0lBQzlFLCtCQUErQixFQUFFLE1BQU0sQ0FBQztJQUN4Qyx1RUFBdUU7SUFDdkUsbUNBQW1DLEVBQUUsdUJBQXVCLENBQUM7SUFDN0QsbUdBQW1HO0lBQ25HLHlCQUF5QixFQUFFLE1BQU0sRUFBRSxDQUFDO0lBQ3BDLHlGQUF5RjtJQUN6RixnQ0FBZ0MsRUFBRSxNQUFNLENBQUM7SUFDekMseUZBQXlGO0lBQ3pGLGdDQUFnQyxFQUFFLE1BQU0sQ0FBQztJQUN6QyxrRUFBa0U7SUFDbEUsb0NBQW9DLEVBQUUsTUFBTSxDQUFDO0lBQzdDLGtFQUFrRTtJQUNsRSxvQ0FBb0MsRUFBRSxNQUFNLENBQUM7SUFDN0MscUVBQXFFO0lBQ3JFLHNDQUFzQyxFQUFFLE1BQU0sQ0FBQztJQUMvQyxxRUFBcUU7SUFDckUsc0NBQXNDLEVBQUUsTUFBTSxDQUFDO0lBQy9DLG9FQUFvRTtJQUNwRSxxQ0FBcUMsRUFBRSxNQUFNLENBQUM7SUFDOUMsb0VBQW9FO0lBQ3BFLHFDQUFxQyxFQUFFLE1BQU0sQ0FBQztDQUMvQyxDQUFDO0FBRUYsZUFBTyxNQUFNLDBCQUEwQixFQUFFLGtCQUFrQixDQUFDLGtCQUFrQixDQWdIN0UsQ0FBQyJ9

package/dest/services/tx_collection/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/config.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,kBAAkB,EAIxB,MAAM,0BAA0B,CAAC;AAGlC,MAAM,MAAM,uBAAuB,GAAG,KAAK,GAAG,KAAK,CAAC;AAEpD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,oEAAoE;IACpE,2CAA2C,EAAE,MAAM,CAAC;IACpD,iDAAiD;IACjD,+BAA+B,EAAE,MAAM,CAAC;IACxC,sCAAsC;IACtC,iCAAiC,EAAE,MAAM,CAAC;IAC1C,qEAAqE;IACrE,gCAAgC,EAAE,MAAM,CAAC;IACzC,wDAAwD;IACxD,+BAA+B,EAAE,MAAM,CAAC;IACxC,gGAAgG;IAChG,yCAAyC,EAAE,OAAO,CAAC;IACnD,2FAA2F;IAC3F,8BAA8B,EAAE,MAAM,CAAC;IACvC,6EAA6E;IAC7E,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,mFAAmF;IACnF,0CAA0C,EAAE,MAAM,CAAC;IACnD,8EAA8E;IAC9E,+BAA+B,EAAE,MAAM,CAAC;IACxC,uEAAuE;IACvE,mCAAmC,EAAE,uBAAuB,CAAC;CAC9D,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,kBAAkB,CAAC,kBAAkB,CA8D7E,CAAC"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/config.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,kBAAkB,EAIxB,MAAM,0BAA0B,CAAC;AAGlC,MAAM,MAAM,uBAAuB,GAAG,KAAK,GAAG,KAAK,CAAC;AAEpD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,oEAAoE;IACpE,2CAA2C,EAAE,MAAM,CAAC;IACpD,iDAAiD;IACjD,+BAA+B,EAAE,MAAM,CAAC;IACxC,sCAAsC;IACtC,iCAAiC,EAAE,MAAM,CAAC;IAC1C,qEAAqE;IACrE,gCAAgC,EAAE,MAAM,CAAC;IACzC,wDAAwD;IACxD,+BAA+B,EAAE,MAAM,CAAC;IACxC,gGAAgG;IAChG,yCAAyC,EAAE,OAAO,CAAC;IACnD,2FAA2F;IAC3F,8BAA8B,EAAE,MAAM,CAAC;IACvC,6EAA6E;IAC7E,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,mFAAmF;IACnF,0CAA0C,EAAE,MAAM,CAAC;IACnD,8EAA8E;IAC9E,+BAA+B,EAAE,MAAM,CAAC;IACxC,uEAAuE;IACvE,mCAAmC,EAAE,uBAAuB,CAAC;IAC7D,mGAAmG;IACnG,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,yFAAyF;IACzF,gCAAgC,EAAE,MAAM,CAAC;IACzC,yFAAyF;IACzF,gCAAgC,EAAE,MAAM,CAAC;IACzC,kEAAkE;IAClE,oCAAoC,EAAE,MAAM,CAAC;IAC7C,kEAAkE;IAClE,oCAAoC,EAAE,MAAM,CAAC;IAC7C,qEAAqE;IACrE,sCAAsC,EAAE,MAAM,CAAC;IAC/C,qEAAqE;IACrE,sCAAsC,EAAE,MAAM,CAAC;IAC/C,oEAAoE;IACpE,qCAAqC,EAAE,MAAM,CAAC;IAC9C,oEAAoE;IACpE,qCAAqC,EAAE,MAAM,CAAC;CAC/C,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,kBAAkB,CAAC,kBAAkB,CAgH7E,CAAC"}

package/dest/services/tx_collection/config.js

@@ -55,12 +55,58 @@ export const txCollectionConfigMappings = {
         description: 'Maximum number of transactions to request from a node in a single batch',
         ...numberConfigHelper(MAX_RPC_TXS_LEN)
     },
-    txCollectionProposalTxCollectorType: {
-        env: 'TX_COLLECTION_PROPOSAL_TX_COLLECTOR_TYPE',
-        description: 'Which collector implementation to use for proposal tx collection (new or old)',
+    txCollectionMissingTxsCollectorType: {
+        env: 'TX_COLLECTION_MISSING_TXS_COLLECTOR_TYPE',
+        description: 'Which collector implementation to use for missing txs collection (new or old)',
         ...enumConfigHelper([
             'new',
             'old'
         ], 'new')
+    },
+    txCollectionFileStoreUrls: {
+        env: 'TX_COLLECTION_FILE_STORE_URLS',
+        description: 'A comma-separated list of file store URLs (s3://, gs://, file://, http://) for tx collection',
+        parseEnv: (val)=>val.split(',').map((url)=>url.trim()).filter((url)=>url.length > 0),
+        defaultValue: []
+    },
+    txCollectionFileStoreSlowDelayMs: {
+        env: 'TX_COLLECTION_FILE_STORE_SLOW_DELAY_MS',
+        description: 'Delay before file store collection starts after slow collection',
+        ...numberConfigHelper(24_000)
+    },
+    txCollectionFileStoreFastDelayMs: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_DELAY_MS',
+        description: 'Delay before file store collection starts after fast collection',
+        ...numberConfigHelper(2_000)
+    },
+    txCollectionFileStoreFastWorkerCount: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_WORKER_COUNT',
+        description: 'Number of concurrent workers for fast file store collection',
+        ...numberConfigHelper(5)
+    },
+    txCollectionFileStoreSlowWorkerCount: {
+        env: 'TX_COLLECTION_FILE_STORE_SLOW_WORKER_COUNT',
+        description: 'Number of concurrent workers for slow file store collection',
+        ...numberConfigHelper(2)
+    },
+    txCollectionFileStoreFastBackoffBaseMs: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_BASE_MS',
+        description: 'Base backoff time in ms for fast file store collection retries',
+        ...numberConfigHelper(1_000)
+    },
+    txCollectionFileStoreSlowBackoffBaseMs: {
+        env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_BASE_MS',
+        description: 'Base backoff time in ms for slow file store collection retries',
+        ...numberConfigHelper(5_000)
+    },
+    txCollectionFileStoreFastBackoffMaxMs: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_MAX_MS',
+        description: 'Max backoff time in ms for fast file store collection retries',
+        ...numberConfigHelper(5_000)
+    },
+    txCollectionFileStoreSlowBackoffMaxMs: {
+        env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_MAX_MS',
+        description: 'Max backoff time in ms for slow file store collection retries',
+        ...numberConfigHelper(30_000)
     }
 };

package/dest/services/tx_collection/fast_tx_collection.d.ts

@@ -6,20 +6,19 @@ import { type Tx, TxHash } from '@aztec/stdlib/tx';
 import type { PeerId } from '@libp2p/interface';
 import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
 import type { TxCollectionConfig } from './config.js';
-import { type ProposalTxCollector } from './proposal_tx_collector.js';
+import { type MissingTxsCollector } from './proposal_tx_collector.js';
 import type { FastCollectionRequest, FastCollectionRequestInput } from './tx_collection.js';
 import type { TxCollectionSink } from './tx_collection_sink.js';
 import type { TxSource } from './tx_source.js';
 export declare class FastTxCollection {
-    private p2pService;
     private nodes;
     private txCollectionSink;
     private config;
     private dateProvider;
     private log;
     protected requests: Set<FastCollectionRequest>;
-    private proposalTxCollector;
-    constructor(p2pService: BatchTxRequesterLibP2PService, nodes: TxSource[], txCollectionSink: TxCollectionSink, config: TxCollectionConfig, dateProvider?: DateProvider, log?: Logger, proposalTxCollector?: ProposalTxCollector);
+    private missingTxsCollector;
+    constructor(p2pService: BatchTxRequesterLibP2PService, nodes: TxSource[], txCollectionSink: TxCollectionSink, config: TxCollectionConfig, dateProvider?: DateProvider, log?: Logger, missingTxsCollector?: MissingTxsCollector);
     stop(): Promise<void>;
     getFastCollectionRequests(): Set<FastCollectionRequest>;
     collectFastFor(input: FastCollectionRequestInput, txHashes: TxHash[] | string[], opts: {
@@ -34,6 +33,8 @@ export declare class FastTxCollection {
     private collectFastFromNodes;
     private collectFastFromNode;
     private collectFastViaReqResp;
+    /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */
+    private getAddContext;
     /**
      * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
      * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
@@ -50,4 +51,4 @@ export declare class FastTxCollection {
      */
     stopCollectingForBlocksAfter(blockNumber: BlockNumber): void;
 }
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFzdF90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9mYXN0X3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRzlELE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUlsRSxPQUFPLEVBQUUsWUFBWSxFQUFXLE1BQU0seUJBQXlCLENBQUM7QUFFaEUsT0FBTyxLQUFLLEVBQUUsYUFBYSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDdkQsT0FBTyxFQUFFLEtBQUssRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRW5ELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBR2hELE9BQU8sS0FBSyxFQUFFLDZCQUE2QixFQUFFLE1BQU0sNENBQTRDLENBQUM7QUFHaEcsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFDdEQsT0FBTyxFQUVMLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0sNEJBQTRCLENBQUM7QUFDcEMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUM1RixPQUFPLEtBQUssRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLFFBQVEsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBRS9DLHFCQUFhLGdCQUFnQjtJQU16QixPQUFPLENBQUMsVUFBVTtJQUNsQixPQUFPLENBQUMsS0FBSztJQUNiLE9BQU8sQ0FBQyxnQkFBZ0I7SUFDeEIsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsWUFBWTtJQUNwQixPQUFPLENBQUMsR0FBRztJQVRiLFNBQVMsQ0FBQyxRQUFRLEVBQUUsR0FBRyxDQUFDLHFCQUFxQixDQUFDLENBQWE7SUFDM0QsT0FBTyxDQUFDLG1CQUFtQixDQUFzQjtJQUVqRCxZQUNVLFVBQVUsRUFBRSw2QkFBNkIsRUFDekMsS0FBSyxFQUFFLFFBQVEsRUFBRSxFQUNqQixnQkFBZ0IsRUFBRSxnQkFBZ0IsRUFDbEMsTUFBTSxFQUFFLGtCQUFrQixFQUMxQixZQUFZLEdBQUUsWUFBaUMsRUFDL0MsR0FBRyxHQUFFLE1BQWtELEVBQy9ELG1CQUFtQixDQUFDLEVBQUUsbUJBQW1CLEVBUzFDO0lBRVksSUFBSSxrQkFHaEI7SUFFTSx5QkFBeUIsK0JBRS9CO0lBRVksY0FBYyxDQUN6QixLQUFLLEVBQUUsMEJBQTBCLEVBQ2pDLFFBQVEsRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLEVBQUUsRUFDN0IsSUFBSSxFQUFFO1FBQUUsUUFBUSxFQUFFLElBQUksQ0FBQztRQUFDLFVBQVUsQ0FBQyxFQUFFLE1BQU0sQ0FBQTtLQUFFLGlCQTBDOUM7SUFFRCxVQUFnQixXQUFXLENBQ3pCLE9BQU8sRUFBRSxxQkFBcUIsRUFDOUIsSUFBSSxFQUFFO1FBQUUsUUFBUSxDQUFDLEVBQUUsYUFBYSxDQUFDO1FBQUMsUUFBUSxFQUFFLElBQUksQ0FBQztRQUFDLFVBQVUsQ0FBQyxFQUFFLE1BQU0sQ0FBQTtLQUFFLGlCQStDeEU7WUFRYSxvQkFBb0I7WUFZcEIsbUJBQW1CO1lBaUZuQixxQkFBcUI7SUFtRG5DOzs7T0FHRztJQUNJLFFBQVEsQ0FBQyxHQUFHLEVBQUUsRUFBRSxFQUFFLFFBd0J4QjtJQUVEOzs7T0FHRztJQUNJLDJCQUEyQixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9qRTtJQUVEOzs7T0FHRztJQUNJLDRCQUE0QixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9sRTtDQUNGIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFzdF90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9mYXN0X3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRzlELE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQUUsWUFBWSxFQUFXLE1BQU0seUJBQXlCLENBQUM7QUFFaEUsT0FBTyxLQUFLLEVBQUUsYUFBYSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDdkQsT0FBTyxFQUFFLEtBQUssRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRW5ELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBR2hELE9BQU8sS0FBSyxFQUFFLDZCQUE2QixFQUFFLE1BQU0sNENBQTRDLENBQUM7QUFDaEcsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFFdEQsT0FBTyxFQUVMLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0sNEJBQTRCLENBQUM7QUFDcEMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUM1RixPQUFPLEtBQUssRUFBZ0IsZ0JBQWdCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUM5RSxPQUFPLEtBQUssRUFBRSxRQUFRLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUUvQyxxQkFBYSxnQkFBZ0I7SUFPekIsT0FBTyxDQUFDLEtBQUs7SUFDYixPQUFPLENBQUMsZ0JBQWdCO0lBQ3hCLE9BQU8sQ0FBQyxNQUFNO0lBQ2QsT0FBTyxDQUFDLFlBQVk7SUFDcEIsT0FBTyxDQUFDLEdBQUc7SUFUYixTQUFTLENBQUMsUUFBUSxFQUFFLEdBQUcsQ0FBQyxxQkFBcUIsQ0FBQyxDQUFhO0lBQzNELE9BQU8sQ0FBQyxtQkFBbUIsQ0FBc0I7SUFFakQsWUFDRSxVQUFVLEVBQUUsNkJBQTZCLEVBQ2pDLEtBQUssRUFBRSxRQUFRLEVBQUUsRUFDakIsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLE1BQU0sRUFBRSxrQkFBa0IsRUFDMUIsWUFBWSxHQUFFLFlBQWlDLEVBQy9DLEdBQUcsR0FBRSxNQUFrRCxFQUMvRCxtQkFBbUIsQ0FBQyxFQUFFLG1CQUFtQixFQVMxQztJQUVZLElBQUksa0JBR2hCO0lBRU0seUJBQXlCLCtCQUUvQjtJQUVZLGNBQWMsQ0FDekIsS0FBSyxFQUFFLDBCQUEwQixFQUNqQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsTUFBTSxFQUFFLEVBQzdCLElBQUksRUFBRTtRQUFFLFFBQVEsRUFBRSxJQUFJLENBQUM7UUFBQyxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUE7S0FBRSxpQkEwQzlDO0lBRUQsVUFBZ0IsV0FBVyxDQUN6QixPQUFPLEVBQUUscUJBQXFCLEVBQzlCLElBQUksRUFBRTtRQUFFLFFBQVEsQ0FBQyxFQUFFLGFBQWEsQ0FBQztRQUFDLFFBQVEsRUFBRSxJQUFJLENBQUM7UUFBQyxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUE7S0FBRSxpQkErQ3hFO1lBUWEsb0JBQW9CO1lBZ0JwQixtQkFBbUI7WUF5Rm5CLHFCQUFxQjtJQXlEbkMseUdBQXlHO0lBQ3pHLE9BQU8sQ0FBQyxhQUFhO0lBUXJCOzs7T0FHRztJQUNJLFFBQVEsQ0FBQyxHQUFHLEVBQUUsRUFBRSxFQUFFLFFBc0J4QjtJQUVEOzs7T0FHRztJQUNJLDJCQUEyQixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9qRTtJQUVEOzs7T0FHRztJQUNJLDRCQUE0QixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9sRTtDQUNGIn0=

package/dest/services/tx_collection/fast_tx_collection.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fast_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/fast_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAW,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAGhG,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAEL,KAAK,mBAAmB,EAEzB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAMzB,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IATb,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAa;IAC3D,OAAO,CAAC,mBAAmB,CAAsB;IAEjD,YACU,UAAU,EAAE,6BAA6B,EACzC,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD,EAC/D,mBAAmB,CAAC,EAAE,mBAAmB,EAS1C;IAEY,IAAI,kBAGhB;IAEM,yBAAyB,+BAE/B;IAEY,cAAc,CACzB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA0C9C;IAED,UAAgB,WAAW,CACzB,OAAO,EAAE,qBAAqB,EAC9B,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA+CxE;YAQa,oBAAoB;YAYpB,mBAAmB;YAiFnB,qBAAqB;IAmDnC;;;OAGG;IACI,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,QAwBxB;IAED;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOlE;CACF"}
+{"version":3,"file":"fast_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/fast_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAW,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAChG,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAEL,KAAK,mBAAmB,EAEzB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,KAAK,EAAgB,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAOzB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IATb,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAa;IAC3D,OAAO,CAAC,mBAAmB,CAAsB;IAEjD,YACE,UAAU,EAAE,6BAA6B,EACjC,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD,EAC/D,mBAAmB,CAAC,EAAE,mBAAmB,EAS1C;IAEY,IAAI,kBAGhB;IAEM,yBAAyB,+BAE/B;IAEY,cAAc,CACzB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA0C9C;IAED,UAAgB,WAAW,CACzB,OAAO,EAAE,qBAAqB,EAC9B,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA+CxE;YAQa,oBAAoB;YAgBpB,mBAAmB;YAyFnB,qBAAqB;IAyDnC,yGAAyG;IACzG,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACI,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,QAsBxB;IAED;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOlE;CACF"}

package/dest/services/tx_collection/fast_tx_collection.js

@@ -1,16 +1,13 @@
 import { times } from '@aztec/foundation/collection';
 import { AbortError, TimeoutError } from '@aztec/foundation/error';
 import { createLogger } from '@aztec/foundation/log';
-import { boundInclusive } from '@aztec/foundation/number';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { sleep } from '@aztec/foundation/sleep';
 import { DateProvider, elapsed } from '@aztec/foundation/timer';
 import { TxHash } from '@aztec/stdlib/tx';
-import { ReqRespSubProtocol } from '../reqresp/interface.js';
-import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+import { MissingTxsTracker } from './missing_txs_tracker.js';
 import { BatchTxRequesterCollector, SendBatchRequestCollector } from './proposal_tx_collector.js';
 export class FastTxCollection {
-    p2pService;
     nodes;
     txCollectionSink;
     config;
@@ -18,9 +15,8 @@ export class FastTxCollection {
     log;
     // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
     requests;
-    proposalTxCollector;
-    constructor(p2pService, nodes, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service'), proposalTxCollector){
-        this.p2pService = p2pService;
+    missingTxsCollector;
+    constructor(p2pService, nodes, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service'), missingTxsCollector){
         this.nodes = nodes;
         this.txCollectionSink = txCollectionSink;
         this.config = config;
@@ -28,8 +24,8 @@ export class FastTxCollection {
         this.log = log;
         this.requests = new Set();
         const batchTxRequesterConfig = this.config;
-        const proposalCollectorType = this.config.txCollectionProposalTxCollectorType;
-        this.proposalTxCollector = proposalTxCollector ?? (proposalCollectorType === 'old' ? new SendBatchRequestCollector(p2pService) : new BatchTxRequesterCollector(p2pService, log, dateProvider, undefined, batchTxRequesterConfig));
+        const missingTxsCollectorType = this.config.txCollectionMissingTxsCollectorType;
+        this.missingTxsCollector = missingTxsCollector ?? (missingTxsCollectorType === 'old' ? new SendBatchRequestCollector(p2pService) : new BatchTxRequesterCollector(p2pService, log, dateProvider, undefined, batchTxRequesterConfig));
     }
     async stop() {
         this.requests.forEach((request)=>request.promise.reject(new AbortError(`Stopped collection service`)));
@@ -56,34 +52,32 @@ export class FastTxCollection {
         // This promise is used to await for the collection to finish during the main collectFast method.
         // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
         const promise = promiseWithResolvers();
-        setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
+        const timeoutTimer = setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
         const request = {
             ...input,
             blockInfo,
             promise,
-            foundTxs: new Map(),
-            missingTxHashes: new Set(txHashes.map((t)=>t.toString())),
+            missingTxTracker: MissingTxsTracker.fromArray(txHashes),
             deadline: opts.deadline
         };
         const [duration] = await elapsed(()=>this.collectFast(request, {
                 ...opts
             }));
-        this.log.verbose(`Collected ${request.foundTxs.size} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
+        clearTimeout(timeoutTimer);
+        this.log.verbose(`Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
             ...blockInfo,
             duration,
             requestType: input.type,
             missingTxs: [
-                ...request.missingTxHashes
+                ...request.missingTxTracker.missingTxHashes
             ]
         });
-        return [
-            ...request.foundTxs.values()
-        ];
+        return request.missingTxTracker.collectedTxs;
     }
     async collectFast(request, opts) {
         this.requests.add(request);
         const { blockInfo } = request;
-        this.log.debug(`Starting fast collection of ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+        this.log.debug(`Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
             ...blockInfo,
             requestType: request.type,
             deadline: opts.deadline
@@ -99,7 +93,7 @@ export class FastTxCollection {
                 waitBeforeReqResp
             ]);
             // If we have collected all txs, we can stop here
-            if (request.missingTxHashes.size === 0) {
+            if (request.missingTxTracker.allFetched()) {
                 this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
                 return;
             }
@@ -118,9 +112,7 @@ export class FastTxCollection {
             const logCtx = {
                 ...blockInfo,
                 errorMessage: err instanceof Error ? err.message : undefined,
-                missingTxs: [
-                    ...request.missingTxHashes
-                ].map((txHash)=>txHash.toString())
+                missingTxs: request.missingTxTracker.missingTxHashes.values().map((txHash)=>txHash.toString())
             };
             if (err instanceof Error && err.name === 'TimeoutError') {
                 this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
@@ -146,7 +138,7 @@ export class FastTxCollection {
         }
         // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
         const attemptsPerTx = [
-            ...request.missingTxHashes
+            ...request.missingTxTracker.missingTxHashes
         ].map((txHash)=>({
                 txHash,
                 attempts: 0,
@@ -156,7 +148,7 @@ export class FastTxCollection {
         await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx, opts)));
     }
     async collectFastFromNode(request, node, attemptsPerTx, opts) {
-        const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && request.missingTxHashes.size > 0 && this.requests.has(request);
+        const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
         const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
         const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
         const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
@@ -170,7 +162,7 @@ export class FastTxCollection {
                     const txToRequest = attemptsPerTx[index++];
                     if (!txToRequest) {
                         break;
-                    } else if (!request.missingTxHashes.has(txToRequest.txHash)) {
+                    } else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
                         // Mark as found if it was found somewhere else, we'll then remove it from the array.
                         // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
                         txToRequest.found = true;
@@ -194,13 +186,20 @@ export class FastTxCollection {
                 if (batch.length === 0) {
                     return;
                 }
+                const txHashes = batch.map(({ txHash })=>txHash);
                 // Collect this batch from the node
-                await this.txCollectionSink.collect((txHashes)=>node.getTxsByHash(txHashes), batch.map(({ txHash })=>TxHash.fromString(txHash)), {
+                await this.txCollectionSink.collect(async ()=>{
+                    const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
+                    for (const tx of result.validTxs){
+                        request.missingTxTracker.markFetched(tx);
+                    }
+                    return result;
+                }, txHashes, {
                     description: `fast ${node.getInfo()}`,
                     node: node.getInfo(),
                     method: 'fast-node-rpc',
                     ...request.blockInfo
-                });
+                }, this.getAddContext(request));
                 // Clear from the active requests the txs we just requested
                 for (const requestedTx of batch){
                     activeRequestsToThisNode.delete(requestedTx.txHash);
@@ -217,8 +216,6 @@ export class FastTxCollection {
     async collectFastViaReqResp(request, opts) {
         const timeoutMs = +request.deadline - this.dateProvider.now();
         const pinnedPeer = opts.pinnedPeer;
-        const maxPeers = boundInclusive(Math.ceil(request.missingTxHashes.size / 2), 8, 32);
-        const maxRetryAttempts = 5;
         const blockInfo = request.blockInfo;
         const slotNumber = blockInfo.slotNumber;
         if (timeoutMs < 100) {
@@ -228,36 +225,57 @@ export class FastTxCollection {
             });
             return;
         }
-        this.log.debug(`Starting fast reqresp for ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+        this.log.debug(`Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
             ...blockInfo,
             timeoutMs,
             pinnedPeer
         });
         try {
-            await this.txCollectionSink.collect(async (txHashes)=>{
+            await this.txCollectionSink.collect(async ()=>{
+                let result;
                 if (request.type === 'proposal') {
-                    return await this.proposalTxCollector.collectTxs(txHashes, request.blockProposal, pinnedPeer, timeoutMs);
+                    result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, request.blockProposal, pinnedPeer, timeoutMs);
                 } else if (request.type === 'block') {
-                    const txs = await this.p2pService.reqResp.sendBatchRequest(ReqRespSubProtocol.TX, chunkTxHashesRequest(txHashes), pinnedPeer, timeoutMs, maxPeers, maxRetryAttempts);
-                    return txs.flat();
+                    const blockTxsSource = {
+                        txHashes: request.block.body.txEffects.map((e)=>e.txHash),
+                        archive: request.block.archive.root
+                    };
+                    result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, blockTxsSource, pinnedPeer, timeoutMs);
                 } else {
                     throw new Error(`Unknown request type: ${request.type}`);
                 }
-            }, Array.from(request.missingTxHashes).map((txHash)=>TxHash.fromString(txHash)), {
+                return {
+                    validTxs: result,
+                    invalidTxHashes: []
+                };
+            }, Array.from(request.missingTxTracker.missingTxHashes), {
                 description: `reqresp for slot ${slotNumber}`,
                 method: 'fast-req-resp',
                 ...opts,
                 ...request.blockInfo
-            });
+            }, this.getAddContext(request));
         } catch (err) {
             this.log.error(`Error sending fast reqresp request for txs`, err, {
                 txs: [
-                    ...request.missingTxHashes
+                    ...request.missingTxTracker.missingTxHashes
                 ],
                 ...blockInfo
             });
         }
     }
+    /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */ getAddContext(request) {
+        if (request.type === 'proposal') {
+            return {
+                type: 'proposal',
+                blockHeader: request.blockProposal.blockHeader
+            };
+        } else {
+            return {
+                type: 'mined',
+                block: request.block
+            };
+        }
+    }
     /**
    * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
    * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
@@ -266,22 +284,20 @@ export class FastTxCollection {
             for (const tx of txs){
                 const txHash = tx.txHash.toString();
                 // Remove the tx hash from the missing set, and add it to the found set.
-                if (request.missingTxHashes.has(txHash)) {
-                    request.missingTxHashes.delete(txHash);
-                    request.foundTxs.set(txHash, tx);
+                if (request.missingTxTracker.markFetched(tx)) {
                     this.log.trace(`Found tx ${txHash} for fast collection request`, {
                         ...request.blockInfo,
                         txHash: tx.txHash.toString(),
                         type: request.type
                     });
-                    // If we found all txs for this request, we resolve the promise
-                    if (request.missingTxHashes.size === 0) {
-                        this.log.trace(`All txs found for fast collection request`, {
-                            ...request.blockInfo,
-                            type: request.type
-                        });
-                        request.promise.resolve();
-                    }
+                }
+                // If we found all txs for this request, we resolve the promise
+                if (request.missingTxTracker.allFetched()) {
+                    this.log.trace(`All txs found for fast collection request`, {
+                        ...request.blockInfo,
+                        type: request.type
+                    });
+                    request.promise.resolve();
                 }
             }
         }