@aztec/p2p 0.0.1-commit.c80b6263 → 0.0.1-commit.cd76b27

package/dest/services/tx_collection/config.js

@@ -55,12 +55,58 @@ export const txCollectionConfigMappings = {
         description: 'Maximum number of transactions to request from a node in a single batch',
         ...numberConfigHelper(MAX_RPC_TXS_LEN)
     },
-    txCollectionProposalTxCollectorType: {
-        env: 'TX_COLLECTION_PROPOSAL_TX_COLLECTOR_TYPE',
-        description: 'Which collector implementation to use for proposal tx collection (new or old)',
+    txCollectionMissingTxsCollectorType: {
+        env: 'TX_COLLECTION_MISSING_TXS_COLLECTOR_TYPE',
+        description: 'Which collector implementation to use for missing txs collection (new or old)',
         ...enumConfigHelper([
             'new',
             'old'
         ], 'new')
+    },
+    txCollectionFileStoreUrls: {
+        env: 'TX_COLLECTION_FILE_STORE_URLS',
+        description: 'A comma-separated list of file store URLs (s3://, gs://, file://, http://) for tx collection',
+        parseEnv: (val)=>val.split(',').map((url)=>url.trim()).filter((url)=>url.length > 0),
+        defaultValue: []
+    },
+    txCollectionFileStoreSlowDelayMs: {
+        env: 'TX_COLLECTION_FILE_STORE_SLOW_DELAY_MS',
+        description: 'Delay before file store collection starts after slow collection',
+        ...numberConfigHelper(24_000)
+    },
+    txCollectionFileStoreFastDelayMs: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_DELAY_MS',
+        description: 'Delay before file store collection starts after fast collection',
+        ...numberConfigHelper(2_000)
+    },
+    txCollectionFileStoreFastWorkerCount: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_WORKER_COUNT',
+        description: 'Number of concurrent workers for fast file store collection',
+        ...numberConfigHelper(5)
+    },
+    txCollectionFileStoreSlowWorkerCount: {
+        env: 'TX_COLLECTION_FILE_STORE_SLOW_WORKER_COUNT',
+        description: 'Number of concurrent workers for slow file store collection',
+        ...numberConfigHelper(2)
+    },
+    txCollectionFileStoreFastBackoffBaseMs: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_BASE_MS',
+        description: 'Base backoff time in ms for fast file store collection retries',
+        ...numberConfigHelper(1_000)
+    },
+    txCollectionFileStoreSlowBackoffBaseMs: {
+        env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_BASE_MS',
+        description: 'Base backoff time in ms for slow file store collection retries',
+        ...numberConfigHelper(5_000)
+    },
+    txCollectionFileStoreFastBackoffMaxMs: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_MAX_MS',
+        description: 'Max backoff time in ms for fast file store collection retries',
+        ...numberConfigHelper(5_000)
+    },
+    txCollectionFileStoreSlowBackoffMaxMs: {
+        env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_MAX_MS',
+        description: 'Max backoff time in ms for slow file store collection retries',
+        ...numberConfigHelper(30_000)
     }
 };

package/dest/services/tx_collection/fast_tx_collection.d.ts

@@ -6,20 +6,19 @@ import { type Tx, TxHash } from '@aztec/stdlib/tx';
 import type { PeerId } from '@libp2p/interface';
 import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
 import type { TxCollectionConfig } from './config.js';
-import { type ProposalTxCollector } from './proposal_tx_collector.js';
+import { type MissingTxsCollector } from './proposal_tx_collector.js';
 import type { FastCollectionRequest, FastCollectionRequestInput } from './tx_collection.js';
 import type { TxCollectionSink } from './tx_collection_sink.js';
 import type { TxSource } from './tx_source.js';
 export declare class FastTxCollection {
-    private p2pService;
     private nodes;
     private txCollectionSink;
     private config;
     private dateProvider;
     private log;
     protected requests: Set<FastCollectionRequest>;
-    private proposalTxCollector;
-    constructor(p2pService: BatchTxRequesterLibP2PService, nodes: TxSource[], txCollectionSink: TxCollectionSink, config: TxCollectionConfig, dateProvider?: DateProvider, log?: Logger, proposalTxCollector?: ProposalTxCollector);
+    private missingTxsCollector;
+    constructor(p2pService: BatchTxRequesterLibP2PService, nodes: TxSource[], txCollectionSink: TxCollectionSink, config: TxCollectionConfig, dateProvider?: DateProvider, log?: Logger, missingTxsCollector?: MissingTxsCollector);
     stop(): Promise<void>;
     getFastCollectionRequests(): Set<FastCollectionRequest>;
     collectFastFor(input: FastCollectionRequestInput, txHashes: TxHash[] | string[], opts: {
@@ -34,6 +33,8 @@ export declare class FastTxCollection {
     private collectFastFromNodes;
     private collectFastFromNode;
     private collectFastViaReqResp;
+    /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */
+    private getAddContext;
     /**
      * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
      * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
@@ -50,4 +51,4 @@ export declare class FastTxCollection {
      */
     stopCollectingForBlocksAfter(blockNumber: BlockNumber): void;
 }
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFzdF90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9mYXN0X3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRzlELE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUlsRSxPQUFPLEVBQUUsWUFBWSxFQUFXLE1BQU0seUJBQXlCLENBQUM7QUFFaEUsT0FBTyxLQUFLLEVBQUUsYUFBYSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDdkQsT0FBTyxFQUFFLEtBQUssRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRW5ELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBR2hELE9BQU8sS0FBSyxFQUFFLDZCQUE2QixFQUFFLE1BQU0sNENBQTRDLENBQUM7QUFHaEcsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFDdEQsT0FBTyxFQUVMLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0sNEJBQTRCLENBQUM7QUFDcEMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUM1RixPQUFPLEtBQUssRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLFFBQVEsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBRS9DLHFCQUFhLGdCQUFnQjtJQU16QixPQUFPLENBQUMsVUFBVTtJQUNsQixPQUFPLENBQUMsS0FBSztJQUNiLE9BQU8sQ0FBQyxnQkFBZ0I7SUFDeEIsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsWUFBWTtJQUNwQixPQUFPLENBQUMsR0FBRztJQVRiLFNBQVMsQ0FBQyxRQUFRLEVBQUUsR0FBRyxDQUFDLHFCQUFxQixDQUFDLENBQWE7SUFDM0QsT0FBTyxDQUFDLG1CQUFtQixDQUFzQjtJQUVqRCxZQUNVLFVBQVUsRUFBRSw2QkFBNkIsRUFDekMsS0FBSyxFQUFFLFFBQVEsRUFBRSxFQUNqQixnQkFBZ0IsRUFBRSxnQkFBZ0IsRUFDbEMsTUFBTSxFQUFFLGtCQUFrQixFQUMxQixZQUFZLEdBQUUsWUFBaUMsRUFDL0MsR0FBRyxHQUFFLE1BQWtELEVBQy9ELG1CQUFtQixDQUFDLEVBQUUsbUJBQW1CLEVBUzFDO0lBRVksSUFBSSxrQkFHaEI7SUFFTSx5QkFBeUIsK0JBRS9CO0lBRVksY0FBYyxDQUN6QixLQUFLLEVBQUUsMEJBQTBCLEVBQ2pDLFFBQVEsRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLEVBQUUsRUFDN0IsSUFBSSxFQUFFO1FBQUUsUUFBUSxFQUFFLElBQUksQ0FBQztRQUFDLFVBQVUsQ0FBQyxFQUFFLE1BQU0sQ0FBQTtLQUFFLGlCQTBDOUM7SUFFRCxVQUFnQixXQUFXLENBQ3pCLE9BQU8sRUFBRSxxQkFBcUIsRUFDOUIsSUFBSSxFQUFFO1FBQUUsUUFBUSxDQUFDLEVBQUUsYUFBYSxDQUFDO1FBQUMsUUFBUSxFQUFFLElBQUksQ0FBQztRQUFDLFVBQVUsQ0FBQyxFQUFFLE1BQU0sQ0FBQTtLQUFFLGlCQStDeEU7WUFRYSxvQkFBb0I7WUFZcEIsbUJBQW1CO1lBaUZuQixxQkFBcUI7SUFtRG5DOzs7T0FHRztJQUNJLFFBQVEsQ0FBQyxHQUFHLEVBQUUsRUFBRSxFQUFFLFFBd0J4QjtJQUVEOzs7T0FHRztJQUNJLDJCQUEyQixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9qRTtJQUVEOzs7T0FHRztJQUNJLDRCQUE0QixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9sRTtDQUNGIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFzdF90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9mYXN0X3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRzlELE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQUUsWUFBWSxFQUFXLE1BQU0seUJBQXlCLENBQUM7QUFFaEUsT0FBTyxLQUFLLEVBQUUsYUFBYSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDdkQsT0FBTyxFQUFFLEtBQUssRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRW5ELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBR2hELE9BQU8sS0FBSyxFQUFFLDZCQUE2QixFQUFFLE1BQU0sNENBQTRDLENBQUM7QUFDaEcsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFFdEQsT0FBTyxFQUVMLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0sNEJBQTRCLENBQUM7QUFDcEMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUM1RixPQUFPLEtBQUssRUFBZ0IsZ0JBQWdCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUM5RSxPQUFPLEtBQUssRUFBRSxRQUFRLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUUvQyxxQkFBYSxnQkFBZ0I7SUFPekIsT0FBTyxDQUFDLEtBQUs7SUFDYixPQUFPLENBQUMsZ0JBQWdCO0lBQ3hCLE9BQU8sQ0FBQyxNQUFNO0lBQ2QsT0FBTyxDQUFDLFlBQVk7SUFDcEIsT0FBTyxDQUFDLEdBQUc7SUFUYixTQUFTLENBQUMsUUFBUSxFQUFFLEdBQUcsQ0FBQyxxQkFBcUIsQ0FBQyxDQUFhO0lBQzNELE9BQU8sQ0FBQyxtQkFBbUIsQ0FBc0I7SUFFakQsWUFDRSxVQUFVLEVBQUUsNkJBQTZCLEVBQ2pDLEtBQUssRUFBRSxRQUFRLEVBQUUsRUFDakIsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLE1BQU0sRUFBRSxrQkFBa0IsRUFDMUIsWUFBWSxHQUFFLFlBQWlDLEVBQy9DLEdBQUcsR0FBRSxNQUFrRCxFQUMvRCxtQkFBbUIsQ0FBQyxFQUFFLG1CQUFtQixFQVMxQztJQUVZLElBQUksa0JBR2hCO0lBRU0seUJBQXlCLCtCQUUvQjtJQUVZLGNBQWMsQ0FDekIsS0FBSyxFQUFFLDBCQUEwQixFQUNqQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsTUFBTSxFQUFFLEVBQzdCLElBQUksRUFBRTtRQUFFLFFBQVEsRUFBRSxJQUFJLENBQUM7UUFBQyxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUE7S0FBRSxpQkEwQzlDO0lBRUQsVUFBZ0IsV0FBVyxDQUN6QixPQUFPLEVBQUUscUJBQXFCLEVBQzlCLElBQUksRUFBRTtRQUFFLFFBQVEsQ0FBQyxFQUFFLGFBQWEsQ0FBQztRQUFDLFFBQVEsRUFBRSxJQUFJLENBQUM7UUFBQyxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUE7S0FBRSxpQkErQ3hFO1lBUWEsb0JBQW9CO1lBZ0JwQixtQkFBbUI7WUF5Rm5CLHFCQUFxQjtJQXlEbkMseUdBQXlHO0lBQ3pHLE9BQU8sQ0FBQyxhQUFhO0lBUXJCOzs7T0FHRztJQUNJLFFBQVEsQ0FBQyxHQUFHLEVBQUUsRUFBRSxFQUFFLFFBc0J4QjtJQUVEOzs7T0FHRztJQUNJLDJCQUEyQixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9qRTtJQUVEOzs7T0FHRztJQUNJLDRCQUE0QixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9sRTtDQUNGIn0=

package/dest/services/tx_collection/fast_tx_collection.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fast_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/fast_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAW,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAGhG,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAEL,KAAK,mBAAmB,EAEzB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAMzB,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IATb,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAa;IAC3D,OAAO,CAAC,mBAAmB,CAAsB;IAEjD,YACU,UAAU,EAAE,6BAA6B,EACzC,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD,EAC/D,mBAAmB,CAAC,EAAE,mBAAmB,EAS1C;IAEY,IAAI,kBAGhB;IAEM,yBAAyB,+BAE/B;IAEY,cAAc,CACzB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA0C9C;IAED,UAAgB,WAAW,CACzB,OAAO,EAAE,qBAAqB,EAC9B,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA+CxE;YAQa,oBAAoB;YAYpB,mBAAmB;YAiFnB,qBAAqB;IAmDnC;;;OAGG;IACI,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,QAwBxB;IAED;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOlE;CACF"}
+{"version":3,"file":"fast_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/fast_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAW,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAChG,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAEL,KAAK,mBAAmB,EAEzB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,KAAK,EAAgB,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAOzB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IATb,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAa;IAC3D,OAAO,CAAC,mBAAmB,CAAsB;IAEjD,YACE,UAAU,EAAE,6BAA6B,EACjC,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD,EAC/D,mBAAmB,CAAC,EAAE,mBAAmB,EAS1C;IAEY,IAAI,kBAGhB;IAEM,yBAAyB,+BAE/B;IAEY,cAAc,CACzB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA0C9C;IAED,UAAgB,WAAW,CACzB,OAAO,EAAE,qBAAqB,EAC9B,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA+CxE;YAQa,oBAAoB;YAgBpB,mBAAmB;YAyFnB,qBAAqB;IAyDnC,yGAAyG;IACzG,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACI,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,QAsBxB;IAED;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOlE;CACF"}

package/dest/services/tx_collection/fast_tx_collection.js

@@ -1,16 +1,13 @@
 import { times } from '@aztec/foundation/collection';
 import { AbortError, TimeoutError } from '@aztec/foundation/error';
 import { createLogger } from '@aztec/foundation/log';
-import { boundInclusive } from '@aztec/foundation/number';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { sleep } from '@aztec/foundation/sleep';
 import { DateProvider, elapsed } from '@aztec/foundation/timer';
 import { TxHash } from '@aztec/stdlib/tx';
-import { ReqRespSubProtocol } from '../reqresp/interface.js';
-import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+import { MissingTxsTracker } from './missing_txs_tracker.js';
 import { BatchTxRequesterCollector, SendBatchRequestCollector } from './proposal_tx_collector.js';
 export class FastTxCollection {
-    p2pService;
     nodes;
     txCollectionSink;
     config;
@@ -18,9 +15,8 @@ export class FastTxCollection {
     log;
     // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
     requests;
-    proposalTxCollector;
-    constructor(p2pService, nodes, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service'), proposalTxCollector){
-        this.p2pService = p2pService;
+    missingTxsCollector;
+    constructor(p2pService, nodes, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service'), missingTxsCollector){
         this.nodes = nodes;
         this.txCollectionSink = txCollectionSink;
         this.config = config;
@@ -28,8 +24,8 @@ export class FastTxCollection {
         this.log = log;
         this.requests = new Set();
         const batchTxRequesterConfig = this.config;
-        const proposalCollectorType = this.config.txCollectionProposalTxCollectorType;
-        this.proposalTxCollector = proposalTxCollector ?? (proposalCollectorType === 'old' ? new SendBatchRequestCollector(p2pService) : new BatchTxRequesterCollector(p2pService, log, dateProvider, undefined, batchTxRequesterConfig));
+        const missingTxsCollectorType = this.config.txCollectionMissingTxsCollectorType;
+        this.missingTxsCollector = missingTxsCollector ?? (missingTxsCollectorType === 'old' ? new SendBatchRequestCollector(p2pService) : new BatchTxRequesterCollector(p2pService, log, dateProvider, undefined, batchTxRequesterConfig));
     }
     async stop() {
         this.requests.forEach((request)=>request.promise.reject(new AbortError(`Stopped collection service`)));
@@ -56,34 +52,32 @@ export class FastTxCollection {
         // This promise is used to await for the collection to finish during the main collectFast method.
         // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
         const promise = promiseWithResolvers();
-        setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
+        const timeoutTimer = setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
         const request = {
             ...input,
             blockInfo,
             promise,
-            foundTxs: new Map(),
-            missingTxHashes: new Set(txHashes.map((t)=>t.toString())),
+            missingTxTracker: MissingTxsTracker.fromArray(txHashes),
             deadline: opts.deadline
         };
         const [duration] = await elapsed(()=>this.collectFast(request, {
                 ...opts
             }));
-        this.log.verbose(`Collected ${request.foundTxs.size} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
+        clearTimeout(timeoutTimer);
+        this.log.verbose(`Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
             ...blockInfo,
             duration,
             requestType: input.type,
             missingTxs: [
-                ...request.missingTxHashes
+                ...request.missingTxTracker.missingTxHashes
             ]
         });
-        return [
-            ...request.foundTxs.values()
-        ];
+        return request.missingTxTracker.collectedTxs;
     }
     async collectFast(request, opts) {
         this.requests.add(request);
         const { blockInfo } = request;
-        this.log.debug(`Starting fast collection of ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+        this.log.debug(`Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
             ...blockInfo,
             requestType: request.type,
             deadline: opts.deadline
@@ -99,7 +93,7 @@ export class FastTxCollection {
                 waitBeforeReqResp
             ]);
             // If we have collected all txs, we can stop here
-            if (request.missingTxHashes.size === 0) {
+            if (request.missingTxTracker.allFetched()) {
                 this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
                 return;
             }
@@ -118,9 +112,7 @@ export class FastTxCollection {
             const logCtx = {
                 ...blockInfo,
                 errorMessage: err instanceof Error ? err.message : undefined,
-                missingTxs: [
-                    ...request.missingTxHashes
-                ].map((txHash)=>txHash.toString())
+                missingTxs: request.missingTxTracker.missingTxHashes.values().map((txHash)=>txHash.toString())
             };
             if (err instanceof Error && err.name === 'TimeoutError') {
                 this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
@@ -146,7 +138,7 @@ export class FastTxCollection {
         }
         // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
         const attemptsPerTx = [
-            ...request.missingTxHashes
+            ...request.missingTxTracker.missingTxHashes
         ].map((txHash)=>({
                 txHash,
                 attempts: 0,
@@ -156,7 +148,7 @@ export class FastTxCollection {
         await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx, opts)));
     }
     async collectFastFromNode(request, node, attemptsPerTx, opts) {
-        const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && request.missingTxHashes.size > 0 && this.requests.has(request);
+        const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
         const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
         const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
         const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
@@ -170,7 +162,7 @@ export class FastTxCollection {
                     const txToRequest = attemptsPerTx[index++];
                     if (!txToRequest) {
                         break;
-                    } else if (!request.missingTxHashes.has(txToRequest.txHash)) {
+                    } else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
                         // Mark as found if it was found somewhere else, we'll then remove it from the array.
                         // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
                         txToRequest.found = true;
@@ -194,13 +186,20 @@ export class FastTxCollection {
                 if (batch.length === 0) {
                     return;
                 }
+                const txHashes = batch.map(({ txHash })=>txHash);
                 // Collect this batch from the node
-                await this.txCollectionSink.collect((txHashes)=>node.getTxsByHash(txHashes), batch.map(({ txHash })=>TxHash.fromString(txHash)), {
+                await this.txCollectionSink.collect(async ()=>{
+                    const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
+                    for (const tx of result.validTxs){
+                        request.missingTxTracker.markFetched(tx);
+                    }
+                    return result;
+                }, txHashes, {
                     description: `fast ${node.getInfo()}`,
                     node: node.getInfo(),
                     method: 'fast-node-rpc',
                     ...request.blockInfo
-                });
+                }, this.getAddContext(request));
                 // Clear from the active requests the txs we just requested
                 for (const requestedTx of batch){
                     activeRequestsToThisNode.delete(requestedTx.txHash);
@@ -217,8 +216,6 @@ export class FastTxCollection {
     async collectFastViaReqResp(request, opts) {
         const timeoutMs = +request.deadline - this.dateProvider.now();
         const pinnedPeer = opts.pinnedPeer;
-        const maxPeers = boundInclusive(Math.ceil(request.missingTxHashes.size / 2), 8, 32);
-        const maxRetryAttempts = 5;
         const blockInfo = request.blockInfo;
         const slotNumber = blockInfo.slotNumber;
         if (timeoutMs < 100) {
@@ -228,36 +225,57 @@ export class FastTxCollection {
             });
             return;
         }
-        this.log.debug(`Starting fast reqresp for ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+        this.log.debug(`Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
             ...blockInfo,
             timeoutMs,
             pinnedPeer
         });
         try {
-            await this.txCollectionSink.collect(async (txHashes)=>{
+            await this.txCollectionSink.collect(async ()=>{
+                let result;
                 if (request.type === 'proposal') {
-                    return await this.proposalTxCollector.collectTxs(txHashes, request.blockProposal, pinnedPeer, timeoutMs);
+                    result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, request.blockProposal, pinnedPeer, timeoutMs);
                 } else if (request.type === 'block') {
-                    const txs = await this.p2pService.reqResp.sendBatchRequest(ReqRespSubProtocol.TX, chunkTxHashesRequest(txHashes), pinnedPeer, timeoutMs, maxPeers, maxRetryAttempts);
-                    return txs.flat();
+                    const blockTxsSource = {
+                        txHashes: request.block.body.txEffects.map((e)=>e.txHash),
+                        archive: request.block.archive.root
+                    };
+                    result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, blockTxsSource, pinnedPeer, timeoutMs);
                 } else {
                     throw new Error(`Unknown request type: ${request.type}`);
                 }
-            }, Array.from(request.missingTxHashes).map((txHash)=>TxHash.fromString(txHash)), {
+                return {
+                    validTxs: result,
+                    invalidTxHashes: []
+                };
+            }, Array.from(request.missingTxTracker.missingTxHashes), {
                 description: `reqresp for slot ${slotNumber}`,
                 method: 'fast-req-resp',
                 ...opts,
                 ...request.blockInfo
-            });
+            }, this.getAddContext(request));
         } catch (err) {
             this.log.error(`Error sending fast reqresp request for txs`, err, {
                 txs: [
-                    ...request.missingTxHashes
+                    ...request.missingTxTracker.missingTxHashes
                 ],
                 ...blockInfo
             });
         }
     }
+    /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */ getAddContext(request) {
+        if (request.type === 'proposal') {
+            return {
+                type: 'proposal',
+                blockHeader: request.blockProposal.blockHeader
+            };
+        } else {
+            return {
+                type: 'mined',
+                block: request.block
+            };
+        }
+    }
     /**
    * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
    * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
@@ -266,22 +284,20 @@ export class FastTxCollection {
             for (const tx of txs){
                 const txHash = tx.txHash.toString();
                 // Remove the tx hash from the missing set, and add it to the found set.
-                if (request.missingTxHashes.has(txHash)) {
-                    request.missingTxHashes.delete(txHash);
-                    request.foundTxs.set(txHash, tx);
+                if (request.missingTxTracker.markFetched(tx)) {
                     this.log.trace(`Found tx ${txHash} for fast collection request`, {
                         ...request.blockInfo,
                         txHash: tx.txHash.toString(),
                         type: request.type
                     });
-                    // If we found all txs for this request, we resolve the promise
-                    if (request.missingTxHashes.size === 0) {
-                        this.log.trace(`All txs found for fast collection request`, {
-                            ...request.blockInfo,
-                            type: request.type
-                        });
-                        request.promise.resolve();
-                    }
+                }
+                // If we found all txs for this request, we resolve the promise
+                if (request.missingTxTracker.allFetched()) {
+                    this.log.trace(`All txs found for fast collection request`, {
+                        ...request.blockInfo,
+                        type: request.type
+                    });
+                    request.promise.resolve();
                 }
             }
         }

package/dest/services/tx_collection/file_store_tx_collection.d.ts

@@ -0,0 +1,53 @@
+import { type Logger } from '@aztec/foundation/log';
+import { DateProvider } from '@aztec/foundation/timer';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { FileStoreTxSource } from './file_store_tx_source.js';
+import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
+/** Configuration for a FileStoreTxCollection instance. */
+export type FileStoreCollectionConfig = {
+    workerCount: number;
+    backoffBaseMs: number;
+    backoffMaxMs: number;
+};
+/**
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
+ * Uses a shared worker pool that pulls entries with priority (fewest attempts first),
+ * retries with round-robin across sources, and applies exponential backoff between
+ * full cycles through all sources.
+ */
+export declare class FileStoreTxCollection {
+    private readonly sources;
+    private readonly txCollectionSink;
+    private readonly config;
+    private readonly dateProvider;
+    private readonly log;
+    /** Map from tx hash string to entry for all pending downloads. */
+    private entries;
+    /** Worker promises for the shared worker pool. */
+    private workers;
+    /** Whether the worker pool is running. */
+    private running;
+    /** Signal used to wake sleeping workers when new entries arrive or stop is called. */
+    private wakeSignal;
+    constructor(sources: FileStoreTxSource[], txCollectionSink: TxCollectionSink, config: FileStoreCollectionConfig, dateProvider?: DateProvider, log?: Logger);
+    /** Starts the shared worker pool. */
+    start(): void;
+    /** Stops all workers and clears state. */
+    stop(): Promise<void>;
+    /** Adds entries to the shared map and wakes workers. */
+    startCollecting(txHashes: TxHash[], context: TxAddContext, deadline: Date): void;
+    /** Removes entries for txs that have been found elsewhere. */
+    foundTxs(txs: Tx[]): void;
+    /** Clears all pending entries. */
+    clearPending(): void;
+    private workerLoop;
+    /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */
+    private getNextAction;
+    /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */
+    private getBackoffMs;
+    /** Resolves the current wake signal and creates a new one. */
+    private wake;
+    private waitForWake;
+    private sleepOrWake;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZV9zdG9yZV90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9maWxlX3N0b3JlX3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBR2xFLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUN2RCxPQUFPLEVBQUUsRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRTlDLE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLE1BQU0sMkJBQTJCLENBQUM7QUFDbkUsT0FBTyxLQUFLLEVBQUUsWUFBWSxFQUFFLGdCQUFnQixFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFFOUUsMERBQTBEO0FBQzFELE1BQU0sTUFBTSx5QkFBeUIsR0FBRztJQUN0QyxXQUFXLEVBQUUsTUFBTSxDQUFDO0lBQ3BCLGFBQWEsRUFBRSxNQUFNLENBQUM7SUFDdEIsWUFBWSxFQUFFLE1BQU0sQ0FBQztDQUN0QixDQUFDO0FBV0Y7Ozs7O0dBS0c7QUFDSCxxQkFBYSxxQkFBcUI7SUFjOUIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxPQUFPO0lBQ3hCLE9BQU8sQ0FBQyxRQUFRLENBQUMsZ0JBQWdCO0lBQ2pDLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQUN2QixPQUFPLENBQUMsUUFBUSxDQUFDLFlBQVk7SUFDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHO0lBakJ0QixrRUFBa0U7SUFDbEUsT0FBTyxDQUFDLE9BQU8sQ0FBdUM7SUFFdEQsa0RBQWtEO0lBQ2xELE9BQU8sQ0FBQyxPQUFPLENBQXVCO0lBRXRDLDBDQUEwQztJQUMxQyxPQUFPLENBQUMsT0FBTyxDQUFTO0lBRXhCLHNGQUFzRjtJQUN0RixPQUFPLENBQUMsVUFBVSxDQUE2QjtJQUUvQyxZQUNtQixPQUFPLEVBQUUsaUJBQWlCLEVBQUUsRUFDNUIsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLE1BQU0sRUFBRSx5QkFBeUIsRUFDakMsWUFBWSxHQUFFLFlBQWlDLEVBQy9DLEdBQUcsR0FBRSxNQUFxRCxFQUc1RTtJQUVELHFDQUFxQztJQUM5QixLQUFLLElBQUksSUFBSSxDQVNuQjtJQUVELDBDQUEwQztJQUM3QixJQUFJLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQU1qQztJQUVELHdEQUF3RDtJQUNqRCxlQUFlLENBQUMsUUFBUSxFQUFFLE1BQU0sRUFBRSxFQUFFLE9BQU8sRUFBRSxZQUFZLEVBQUUsUUFBUSxFQUFFLElBQUksR0FBRyxJQUFJLENBc0J0RjtJQUVELDhEQUE4RDtJQUN2RCxRQUFRLENBQUMsR0FBRyxFQUFFLEVBQUUsRUFBRSxHQUFHLElBQUksQ0FJL0I7SUFFRCxrQ0FBa0M7SUFDM0IsWUFBWSxJQUFJLElBQUksQ0FFMUI7WUFFYSxVQUFVO0lBa0N4QixxR0FBcUc7SUFDckcsT0FBTyxDQUFDLGFBQWE7SUE4QnJCLDZGQUE2RjtJQUM3RixPQUFPLENBQUMsWUFBWTtJQVFwQiw4REFBOEQ7SUFDOUQsT0FBTyxDQUFDLElBQUk7WUFNRSxXQUFXO1lBS1gsV0FBVztDQUcxQiJ9

package/dest/services/tx_collection/file_store_tx_collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file_store_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE9E,0DAA0D;AAC1D,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAWF;;;;;GAKG;AACH,qBAAa,qBAAqB;IAc9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IAjBtB,kEAAkE;IAClE,OAAO,CAAC,OAAO,CAAuC;IAEtD,kDAAkD;IAClD,OAAO,CAAC,OAAO,CAAuB;IAEtC,0CAA0C;IAC1C,OAAO,CAAC,OAAO,CAAS;IAExB,sFAAsF;IACtF,OAAO,CAAC,UAAU,CAA6B;IAE/C,YACmB,OAAO,EAAE,iBAAiB,EAAE,EAC5B,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,yBAAyB,EACjC,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAqD,EAG5E;IAED,qCAAqC;IAC9B,KAAK,IAAI,IAAI,CASnB;IAED,0CAA0C;IAC7B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAMjC;IAED,wDAAwD;IACjD,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,GAAG,IAAI,CAsBtF;IAED,8DAA8D;IACvD,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,IAAI,CAI/B;IAED,kCAAkC;IAC3B,YAAY,IAAI,IAAI,CAE1B;YAEa,UAAU;IAkCxB,qGAAqG;IACrG,OAAO,CAAC,aAAa;IA8BrB,6FAA6F;IAC7F,OAAO,CAAC,YAAY;IAQpB,8DAA8D;IAC9D,OAAO,CAAC,IAAI;YAME,WAAW;YAKX,WAAW;CAG1B"}

package/dest/services/tx_collection/file_store_tx_collection.js

@@ -0,0 +1,167 @@
+import { createLogger } from '@aztec/foundation/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider } from '@aztec/foundation/timer';
+import { TxHash } from '@aztec/stdlib/tx';
+/**
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
+ * Uses a shared worker pool that pulls entries with priority (fewest attempts first),
+ * retries with round-robin across sources, and applies exponential backoff between
+ * full cycles through all sources.
+ */ export class FileStoreTxCollection {
+    sources;
+    txCollectionSink;
+    config;
+    dateProvider;
+    log;
+    /** Map from tx hash string to entry for all pending downloads. */ entries;
+    /** Worker promises for the shared worker pool. */ workers;
+    /** Whether the worker pool is running. */ running;
+    /** Signal used to wake sleeping workers when new entries arrive or stop is called. */ wakeSignal;
+    constructor(sources, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:file_store_tx_collection')){
+        this.sources = sources;
+        this.txCollectionSink = txCollectionSink;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+        this.entries = new Map();
+        this.workers = [];
+        this.running = false;
+        this.wakeSignal = promiseWithResolvers();
+    }
+    /** Starts the shared worker pool. */ start() {
+        if (this.sources.length === 0) {
+            this.log.debug('No file store sources configured');
+            return;
+        }
+        this.running = true;
+        for(let i = 0; i < this.config.workerCount; i++){
+            this.workers.push(this.workerLoop());
+        }
+    }
+    /** Stops all workers and clears state. */ async stop() {
+        this.running = false;
+        this.wake();
+        await Promise.all(this.workers);
+        this.workers = [];
+        this.entries.clear();
+    }
+    /** Adds entries to the shared map and wakes workers. */ startCollecting(txHashes, context, deadline) {
+        if (this.sources.length === 0 || txHashes.length === 0) {
+            return;
+        }
+        if (+deadline <= this.dateProvider.now()) {
+            return;
+        }
+        for (const txHash of txHashes){
+            const hashStr = txHash.toString();
+            if (!this.entries.has(hashStr)) {
+                this.entries.set(hashStr, {
+                    txHash: hashStr,
+                    context,
+                    deadline,
+                    attempts: 0,
+                    lastAttemptTime: 0,
+                    nextSourceIndex: Math.floor(Math.random() * this.sources.length)
+                });
+            }
+        }
+        this.wake();
+    }
+    /** Removes entries for txs that have been found elsewhere. */ foundTxs(txs) {
+        for (const tx of txs){
+            this.entries.delete(tx.getTxHash().toString());
+        }
+    }
+    /** Clears all pending entries. */ clearPending() {
+        this.entries.clear();
+    }
+    async workerLoop() {
+        while(this.running){
+            const action = this.getNextAction();
+            if (action.type === 'sleep') {
+                await action.promise;
+                continue;
+            }
+            const entry = action.entry;
+            const source = this.sources[entry.nextSourceIndex % this.sources.length];
+            entry.nextSourceIndex++;
+            entry.attempts++;
+            entry.lastAttemptTime = this.dateProvider.now();
+            try {
+                const result = await this.txCollectionSink.collect(()=>source.getTxsByHash([
+                        TxHash.fromString(entry.txHash)
+                    ]), [
+                    entry.txHash
+                ], {
+                    description: `file-store ${source.getInfo()}`,
+                    method: 'file-store',
+                    fileStore: source.getInfo()
+                }, entry.context);
+                if (result.txs.length > 0) {
+                    this.entries.delete(entry.txHash);
+                }
+            } catch (err) {
+                this.log.trace(`Error downloading tx ${entry.txHash} from ${source.getInfo()}`, {
+                    err
+                });
+            }
+        }
+    }
+    /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */ getNextAction() {
+        const now = this.dateProvider.now();
+        let best;
+        let earliestReadyAt = Infinity;
+        for (const [key, entry] of this.entries){
+            if (+entry.deadline <= now) {
+                this.entries.delete(key);
+                continue;
+            }
+            const backoffMs = this.getBackoffMs(entry);
+            const readyAt = entry.lastAttemptTime + backoffMs;
+            if (readyAt > now) {
+                earliestReadyAt = Math.min(earliestReadyAt, readyAt);
+                continue;
+            }
+            if (!best || entry.attempts < best.attempts) {
+                best = entry;
+            }
+        }
+        if (best) {
+            return {
+                type: 'process',
+                entry: best
+            };
+        }
+        if (earliestReadyAt < Infinity) {
+            return {
+                type: 'sleep',
+                promise: this.sleepOrWake(earliestReadyAt - now)
+            };
+        }
+        return {
+            type: 'sleep',
+            promise: this.waitForWake()
+        };
+    }
+    /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */ getBackoffMs(entry) {
+        const fullCycles = Math.floor(entry.attempts / this.sources.length);
+        if (fullCycles === 0) {
+            return 0;
+        }
+        return Math.min(this.config.backoffBaseMs * Math.pow(2, fullCycles - 1), this.config.backoffMaxMs);
+    }
+    /** Resolves the current wake signal and creates a new one. */ wake() {
+        this.wakeSignal.resolve();
+        this.wakeSignal = promiseWithResolvers();
+    }
+    /** Waits until the wake signal is resolved. */ async waitForWake() {
+        await this.wakeSignal.promise;
+    }
+    /** Sleeps for the given duration or until the wake signal is resolved. */ async sleepOrWake(ms) {
+        await Promise.race([
+            sleep(ms),
+            this.wakeSignal.promise
+        ]);
+    }
+}

package/dest/services/tx_collection/file_store_tx_source.d.ts

@@ -0,0 +1,37 @@
+import { type Logger } from '@aztec/foundation/log';
+import { type TxHash } from '@aztec/stdlib/tx';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import type { TxSource, TxSourceCollectionResult } from './tx_source.js';
+/** TxSource implementation that downloads txs from a file store. */
+export declare class FileStoreTxSource implements TxSource {
+    private readonly fileStore;
+    private readonly baseUrl;
+    private readonly basePath;
+    private readonly log;
+    private downloadsSuccess;
+    private downloadsFailed;
+    private downloadDuration;
+    private downloadSize;
+    private constructor();
+    /**
+     * Creates a FileStoreTxSource from a URL.
+     * @param url - The file store URL (s3://, gs://, file://, http://, https://).
+     * @param basePath - Base path for tx files within the store.
+     * @param log - Optional logger.
+     * @param telemetry - Optional telemetry client.
+     * @returns The FileStoreTxSource instance, or undefined if creation fails.
+     */
+    static create(url: string, basePath: string, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource | undefined>;
+    getInfo(): string;
+    getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
+}
+/**
+ * Creates FileStoreTxSource instances from URLs.
+ * @param urls - Array of file store URLs.
+ * @param basePath - Base path for tx files within each store.
+ * @param log - Optional logger.
+ * @param telemetry - Optional telemetry client.
+ * @returns Array of successfully created FileStoreTxSource instances.
+ */
+export declare function createFileStoreTxSources(urls: string[], basePath: string, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource[]>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZV9zdG9yZV90eF9zb3VyY2UuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zZXJ2aWNlcy90eF9jb2xsZWN0aW9uL2ZpbGVfc3RvcmVfdHhfc291cmNlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQU0sS0FBSyxNQUFNLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUNuRCxPQUFPLEVBR0wsS0FBSyxlQUFlLEVBR3JCLE1BQU0seUJBQXlCLENBQUM7QUFFakMsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLHdCQUF3QixFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFekUsb0VBQW9FO0FBQ3BFLHFCQUFhLGlCQUFrQixZQUFXLFFBQVE7SUFPOUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTO0lBQzFCLE9BQU8sQ0FBQyxRQUFRLENBQUMsT0FBTztJQUN4QixPQUFPLENBQUMsUUFBUSxDQUFDLFFBQVE7SUFDekIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHO0lBVHRCLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBZ0I7SUFDeEMsT0FBTyxDQUFDLGVBQWUsQ0FBZ0I7SUFDdkMsT0FBTyxDQUFDLGdCQUFnQixDQUFZO0lBQ3BDLE9BQU8sQ0FBQyxZQUFZLENBQVk7SUFFaEMsT0FBTyxlQVlOO0lBRUQ7Ozs7Ozs7T0FPRztJQUNILE9BQW9CLE1BQU0sQ0FDeEIsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsTUFBTSxFQUNoQixHQUFHLEdBQUUsTUFBaUQsRUFDdEQsU0FBUyxHQUFFLGVBQXNDLEdBQ2hELE9BQU8sQ0FBQyxpQkFBaUIsR0FBRyxTQUFTLENBQUMsQ0FZeEM7SUFFTSxPQUFPLElBQUksTUFBTSxDQUV2QjtJQUVZLFlBQVksQ0FBQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsT0FBTyxDQUFDLHdCQUF3QixDQUFDLENBK0IvRTtDQUNGO0FBRUQ7Ozs7Ozs7R0FPRztBQUNILHdCQUFzQix3QkFBd0IsQ0FDNUMsSUFBSSxFQUFFLE1BQU0sRUFBRSxFQUNkLFFBQVEsRUFBRSxNQUFNLEVBQ2hCLEdBQUcsR0FBRSxNQUFpRCxFQUN0RCxTQUFTLEdBQUUsZUFBc0MsR0FDaEQsT0FBTyxDQUFDLGlCQUFpQixFQUFFLENBQUMsQ0FHOUIifQ==

package/dest/services/tx_collection/file_store_tx_source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file_store_tx_source.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAM,KAAK,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAGL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,QAAQ,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAEzE,oEAAoE;AACpE,qBAAa,iBAAkB,YAAW,QAAQ;IAO9C,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,GAAG;IATtB,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,gBAAgB,CAAY;IACpC,OAAO,CAAC,YAAY,CAAY;IAEhC,OAAO,eAYN;IAED;;;;;;;OAOG;IACH,OAAoB,MAAM,CACxB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAYxC;IAEM,OAAO,IAAI,MAAM,CAEvB;IAEY,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CA+B/E;CACF;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,MAAM,EAChB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAG9B"}