@aztec/p2p 0.0.1-commit.86469d5 → 0.0.1-commit.8655d4a

package/dest/services/tx_collection/config.js

@@ -1,4 +1,4 @@
-import { booleanConfigHelper, enumConfigHelper, numberConfigHelper } from '@aztec/foundation/config';
+import { numberConfigHelper } from '@aztec/foundation/config';
 import { MAX_RPC_TXS_LEN } from '@aztec/stdlib/interfaces/api-limit';
 export const txCollectionConfigMappings = {
     txCollectionFastNodesTimeoutBeforeReqRespMs: {
@@ -6,31 +6,6 @@ export const txCollectionConfigMappings = {
         description: 'How long to wait before starting reqresp for fast collection',
         ...numberConfigHelper(200)
     },
-    txCollectionSlowNodesIntervalMs: {
-        env: 'TX_COLLECTION_SLOW_NODES_INTERVAL_MS',
-        description: 'How often to collect from configured nodes in the slow collection loop',
-        ...numberConfigHelper(12_000)
-    },
-    txCollectionSlowReqRespIntervalMs: {
-        env: 'TX_COLLECTION_SLOW_REQ_RESP_INTERVAL_MS',
-        description: 'How often to collect from peers via reqresp in the slow collection loop',
-        ...numberConfigHelper(12_000)
-    },
-    txCollectionSlowReqRespTimeoutMs: {
-        env: 'TX_COLLECTION_SLOW_REQ_RESP_TIMEOUT_MS',
-        description: 'How long to wait for a reqresp response during slow collection',
-        ...numberConfigHelper(20_000)
-    },
-    txCollectionReconcileIntervalMs: {
-        env: 'TX_COLLECTION_RECONCILE_INTERVAL_MS',
-        description: 'How often to reconcile found txs from the tx pool',
-        ...numberConfigHelper(60_000)
-    },
-    txCollectionDisableSlowDuringFastRequests: {
-        env: 'TX_COLLECTION_DISABLE_SLOW_DURING_FAST_REQUESTS',
-        description: 'Whether to disable the slow collection loop if we are dealing with any immediate requests',
-        ...booleanConfigHelper(true)
-    },
     txCollectionFastNodeIntervalMs: {
         env: 'TX_COLLECTION_FAST_NODE_INTERVAL_MS',
         description: 'How many ms to wait between retried request to a node via RPC during fast collection',
@@ -55,12 +30,30 @@ export const txCollectionConfigMappings = {
         description: 'Maximum number of transactions to request from a node in a single batch',
         ...numberConfigHelper(MAX_RPC_TXS_LEN)
     },
-    txCollectionProposalTxCollectorType: {
-        env: 'TX_COLLECTION_PROPOSAL_TX_COLLECTOR_TYPE',
-        description: 'Which collector implementation to use for proposal tx collection (new or old)',
-        ...enumConfigHelper([
-            'new',
-            'old'
-        ], 'new')
+    txCollectionFileStoreUrls: {
+        env: 'TX_COLLECTION_FILE_STORE_URLS',
+        description: 'A comma-separated list of file store URLs (s3://, gs://, file://, http://) for tx collection',
+        parseEnv: (val)=>val.split(',').map((url)=>url.trim()).filter((url)=>url.length > 0),
+        defaultValue: []
+    },
+    txCollectionFileStoreFastDelayMs: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_DELAY_MS',
+        description: 'Delay in ms from reqresp start before file store collection begins',
+        ...numberConfigHelper(2_000)
+    },
+    txCollectionFileStoreFastWorkerCount: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_WORKER_COUNT',
+        description: 'Number of concurrent workers for fast file store collection',
+        ...numberConfigHelper(5)
+    },
+    txCollectionFileStoreFastBackoffBaseMs: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_BASE_MS',
+        description: 'Base backoff time in ms for fast file store collection retries',
+        ...numberConfigHelper(1_000)
+    },
+    txCollectionFileStoreFastBackoffMaxMs: {
+        env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_MAX_MS',
+        description: 'Max backoff time in ms for fast file store collection retries',
+        ...numberConfigHelper(5_000)
     }
 };

package/dest/services/tx_collection/file_store_tx_collection.d.ts

@@ -0,0 +1,37 @@
+import { type Logger } from '@aztec/foundation/log';
+import { DateProvider } from '@aztec/foundation/timer';
+import type { FileStoreTxSource } from './file_store_tx_source.js';
+import type { IRequestTracker } from './request_tracker.js';
+import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
+/** Configuration for a FileStoreTxCollection instance. */
+export type FileStoreCollectionConfig = {
+    workerCount: number;
+    backoffBaseMs: number;
+    backoffMaxMs: number;
+};
+/**
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
+ * Each call to startCollecting spins up its own worker pool which pulls entries with priority
+ * (fewest attempts first), retries with round-robin across sources, and applies exponential
+ * backoff between full cycles through all sources. Workers self-terminate when the request
+ * tracker is cancelled (deadline / all-fetched / external) or when there is nothing left to do.
+ */
+export declare class FileStoreTxCollection {
+    private readonly sources;
+    private readonly txCollectionSink;
+    private readonly config;
+    private readonly dateProvider;
+    private readonly log;
+    constructor(sources: FileStoreTxSource[], txCollectionSink: TxCollectionSink, config: FileStoreCollectionConfig, dateProvider?: DateProvider, log?: Logger);
+    /**
+     * Spins up workers to download all txs still missing from the tracker, racing across the
+     * configured file store sources. Resolves once all workers settle.
+     */
+    startCollecting(requestTracker: IRequestTracker, context: TxAddContext): Promise<void>;
+    private workerLoop;
+    /** Single-pass scan: removes stale entries, finds the best ready entry, or computes sleep time. */
+    private getNextAction;
+    /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */
+    private getBackoffMs;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZV9zdG9yZV90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9maWxlX3N0b3JlX3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBRWxFLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUd2RCxPQUFPLEtBQUssRUFBRSxpQkFBaUIsRUFBRSxNQUFNLDJCQUEyQixDQUFDO0FBQ25FLE9BQU8sS0FBSyxFQUFFLGVBQWUsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQzVELE9BQU8sS0FBSyxFQUFFLFlBQVksRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBRTlFLDBEQUEwRDtBQUMxRCxNQUFNLE1BQU0seUJBQXlCLEdBQUc7SUFDdEMsV0FBVyxFQUFFLE1BQU0sQ0FBQztJQUNwQixhQUFhLEVBQUUsTUFBTSxDQUFDO0lBQ3RCLFlBQVksRUFBRSxNQUFNLENBQUM7Q0FDdEIsQ0FBQztBQVNGOzs7Ozs7R0FNRztBQUNILHFCQUFhLHFCQUFxQjtJQUU5QixPQUFPLENBQUMsUUFBUSxDQUFDLE9BQU87SUFDeEIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxnQkFBZ0I7SUFDakMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxNQUFNO0lBQ3ZCLE9BQU8sQ0FBQyxRQUFRLENBQUMsWUFBWTtJQUM3QixPQUFPLENBQUMsUUFBUSxDQUFDLEdBQUc7SUFMdEIsWUFDbUIsT0FBTyxFQUFFLGlCQUFpQixFQUFFLEVBQzVCLGdCQUFnQixFQUFFLGdCQUFnQixFQUNsQyxNQUFNLEVBQUUseUJBQXlCLEVBQ2pDLFlBQVksR0FBRSxZQUFpQyxFQUMvQyxHQUFHLEdBQUUsTUFBcUQsRUFDekU7SUFFSjs7O09BR0c7SUFDVSxlQUFlLENBQUMsY0FBYyxFQUFFLGVBQWUsRUFBRSxPQUFPLEVBQUUsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0F3QmxHO1lBRWEsVUFBVTtJQXVDeEIsbUdBQW1HO0lBQ25HLE9BQU8sQ0FBQyxhQUFhO0lBa0NyQiw2RkFBNkY7SUFDN0YsT0FBTyxDQUFDLFlBQVk7Q0FPckIifQ==

package/dest/services/tx_collection/file_store_tx_collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file_store_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_collection.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAElE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE9E,0DAA0D;AAC1D,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AASF;;;;;;GAMG;AACH,qBAAa,qBAAqB;IAE9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IALtB,YACmB,OAAO,EAAE,iBAAiB,EAAE,EAC5B,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,yBAAyB,EACjC,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAqD,EACzE;IAEJ;;;OAGG;IACU,eAAe,CAAC,cAAc,EAAE,eAAe,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAwBlG;YAEa,UAAU;IAuCxB,mGAAmG;IACnG,OAAO,CAAC,aAAa;IAkCrB,6FAA6F;IAC7F,OAAO,CAAC,YAAY;CAOrB"}

package/dest/services/tx_collection/file_store_tx_collection.js

@@ -0,0 +1,127 @@
+import { times } from '@aztec/foundation/collection';
+import { createLogger } from '@aztec/foundation/log';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider } from '@aztec/foundation/timer';
+import { TxHash } from '@aztec/stdlib/tx';
+/**
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
+ * Each call to startCollecting spins up its own worker pool which pulls entries with priority
+ * (fewest attempts first), retries with round-robin across sources, and applies exponential
+ * backoff between full cycles through all sources. Workers self-terminate when the request
+ * tracker is cancelled (deadline / all-fetched / external) or when there is nothing left to do.
+ */ export class FileStoreTxCollection {
+    sources;
+    txCollectionSink;
+    config;
+    dateProvider;
+    log;
+    constructor(sources, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:file_store_tx_collection')){
+        this.sources = sources;
+        this.txCollectionSink = txCollectionSink;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+    }
+    /**
+   * Spins up workers to download all txs still missing from the tracker, racing across the
+   * configured file store sources. Resolves once all workers settle.
+   */ async startCollecting(requestTracker, context) {
+        if (this.sources.length === 0 || requestTracker.checkCancelled()) {
+            return;
+        }
+        // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+        const entries = new Set();
+        for (const hashStr of requestTracker.missingTxHashes){
+            entries.add({
+                txHash: hashStr,
+                attempts: 0,
+                lastAttemptTime: 0,
+                nextSourceIndex: Math.floor(Math.random() * this.sources.length)
+            });
+        }
+        // Yield before spawning so the synchronous caller can finish any follow-up (eg. marking a tx
+        // as fetched on the tracker, or cancelling it) before workers begin scanning entries.
+        await Promise.resolve();
+        if (requestTracker.checkCancelled()) {
+            return;
+        }
+        await Promise.allSettled(times(this.config.workerCount, ()=>this.workerLoop(entries, requestTracker, context)));
+    }
+    async workerLoop(// eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+    entries, requestTracker, context) {
+        while(!requestTracker.checkCancelled() && entries.size > 0){
+            const action = this.getNextAction(entries, requestTracker);
+            if (action.type === 'sleep') {
+                await Promise.race([
+                    sleep(action.ms),
+                    requestTracker.cancellationToken
+                ]);
+                continue;
+            }
+            const entry = action.entry;
+            const source = this.sources[entry.nextSourceIndex % this.sources.length];
+            entry.nextSourceIndex++;
+            entry.attempts++;
+            entry.lastAttemptTime = this.dateProvider.now();
+            try {
+                const result = await this.txCollectionSink.collect(()=>source.getTxsByHash([
+                        TxHash.fromString(entry.txHash)
+                    ]), [
+                    entry.txHash
+                ], {
+                    description: `file-store ${source.getInfo()}`,
+                    method: 'file-store',
+                    fileStore: source.getInfo()
+                }, context);
+                if (result.txs.length > 0) {
+                    entries.delete(entry);
+                }
+            } catch (err) {
+                this.log.trace(`Error downloading tx ${entry.txHash} from ${source.getInfo()}`, {
+                    err
+                });
+            }
+        }
+    }
+    /** Single-pass scan: removes stale entries, finds the best ready entry, or computes sleep time. */ getNextAction(// eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+    entries, requestTracker) {
+        const now = this.dateProvider.now();
+        let best;
+        let earliestReadyAt = Infinity;
+        for (const entry of entries){
+            // Drop entries whose tx was already found via another collection path.
+            if (!requestTracker.isMissing(entry.txHash)) {
+                entries.delete(entry);
+                continue;
+            }
+            const backoffMs = this.getBackoffMs(entry);
+            const readyAt = entry.lastAttemptTime + backoffMs;
+            if (readyAt > now) {
+                earliestReadyAt = Math.min(earliestReadyAt, readyAt);
+                continue;
+            }
+            if (!best || entry.attempts < best.attempts) {
+                best = entry;
+            }
+        }
+        if (best) {
+            return {
+                type: 'process',
+                entry: best
+            };
+        }
+        // earliestReadyAt is finite whenever there are surviving entries; if entries became empty,
+        // the outer worker loop will exit on its next iteration via entries.size === 0.
+        return {
+            type: 'sleep',
+            ms: earliestReadyAt === Infinity ? 0 : earliestReadyAt - now
+        };
+    }
+    /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */ getBackoffMs(entry) {
+        const fullCycles = Math.floor(entry.attempts / this.sources.length);
+        if (fullCycles === 0) {
+            return 0;
+        }
+        return Math.min(this.config.backoffBaseMs * Math.pow(2, fullCycles - 1), this.config.backoffMaxMs);
+    }
+}

package/dest/services/tx_collection/file_store_tx_source.d.ts

@@ -0,0 +1,38 @@
+import { type Logger } from '@aztec/foundation/log';
+import { type TxHash, type TxValidator } from '@aztec/stdlib/tx';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import type { TxSource, TxSourceCollectionResult } from './tx_source.js';
+/** TxSource implementation that downloads txs from a file store. */
+export declare class FileStoreTxSource implements TxSource {
+    private readonly fileStore;
+    private readonly baseUrl;
+    private readonly basePath;
+    private readonly txValidator;
+    private readonly log;
+    private downloadsSuccess;
+    private downloadsFailed;
+    private downloadDuration;
+    private downloadSize;
+    private constructor();
+    /**
+     * Creates a FileStoreTxSource from a URL.
+     * @param url - The file store URL (s3://, gs://, file://, http://, https://).
+     * @param basePath - Base path for tx files within the store.
+     * @param log - Optional logger.
+     * @param telemetry - Optional telemetry client.
+     * @returns The FileStoreTxSource instance, or undefined if creation fails.
+     */
+    static create(url: string, basePath: string, txValidator: TxValidator, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource | undefined>;
+    getInfo(): string;
+    getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
+}
+/**
+ * Creates FileStoreTxSource instances from URLs.
+ * @param urls - Array of file store URLs.
+ * @param basePath - Base path for tx files within each store.
+ * @param log - Optional logger.
+ * @param telemetry - Optional telemetry client.
+ * @returns Array of successfully created FileStoreTxSource instances.
+ */
+export declare function createFileStoreTxSources(urls: string[], basePath: string, txValidator: TxValidator, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource[]>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZV9zdG9yZV90eF9zb3VyY2UuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zZXJ2aWNlcy90eF9jb2xsZWN0aW9uL2ZpbGVfc3RvcmVfdHhfc291cmNlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQU0sS0FBSyxNQUFNLEVBQUUsS0FBSyxXQUFXLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUNyRSxPQUFPLEVBR0wsS0FBSyxlQUFlLEVBR3JCLE1BQU0seUJBQXlCLENBQUM7QUFFakMsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLHdCQUF3QixFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFekUsb0VBQW9FO0FBQ3BFLHFCQUFhLGlCQUFrQixZQUFXLFFBQVE7SUFPOUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTO0lBQzFCLE9BQU8sQ0FBQyxRQUFRLENBQUMsT0FBTztJQUN4QixPQUFPLENBQUMsUUFBUSxDQUFDLFFBQVE7SUFDekIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxXQUFXO0lBQzVCLE9BQU8sQ0FBQyxRQUFRLENBQUMsR0FBRztJQVZ0QixPQUFPLENBQUMsZ0JBQWdCLENBQWdCO0lBQ3hDLE9BQU8sQ0FBQyxlQUFlLENBQWdCO0lBQ3ZDLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBWTtJQUNwQyxPQUFPLENBQUMsWUFBWSxDQUFZO0lBRWhDLE9BQU8sZUFhTjtJQUVEOzs7Ozs7O09BT0c7SUFDSCxPQUFvQixNQUFNLENBQ3hCLEdBQUcsRUFBRSxNQUFNLEVBQ1gsUUFBUSxFQUFFLE1BQU0sRUFDaEIsV0FBVyxFQUFFLFdBQVcsRUFDeEIsR0FBRyxHQUFFLE1BQWlELEVBQ3RELFNBQVMsR0FBRSxlQUFzQyxHQUNoRCxPQUFPLENBQUMsaUJBQWlCLEdBQUcsU0FBUyxDQUFDLENBWXhDO0lBRU0sT0FBTyxJQUFJLE1BQU0sQ0FFdkI7SUFFWSxZQUFZLENBQUMsUUFBUSxFQUFFLE1BQU0sRUFBRSxHQUFHLE9BQU8sQ0FBQyx3QkFBd0IsQ0FBQyxDQXFDL0U7Q0FDRjtBQUVEOzs7Ozs7O0dBT0c7QUFDSCx3QkFBc0Isd0JBQXdCLENBQzVDLElBQUksRUFBRSxNQUFNLEVBQUUsRUFDZCxRQUFRLEVBQUUsTUFBTSxFQUNoQixXQUFXLEVBQUUsV0FBVyxFQUN4QixHQUFHLEdBQUUsTUFBaUQsRUFDdEQsU0FBUyxHQUFFLGVBQXNDLEdBQ2hELE9BQU8sQ0FBQyxpQkFBaUIsRUFBRSxDQUFDLENBSzlCIn0=

package/dest/services/tx_collection/file_store_tx_source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file_store_tx_source.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_source.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAM,KAAK,MAAM,EAAE,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAGL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,QAAQ,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAEzE,oEAAoE;AACpE,qBAAa,iBAAkB,YAAW,QAAQ;IAO9C,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,GAAG;IAVtB,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,gBAAgB,CAAY;IACpC,OAAO,CAAC,YAAY,CAAY;IAEhC,OAAO,eAaN;IAED;;;;;;;OAOG;IACH,OAAoB,MAAM,CACxB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,EACxB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAYxC;IAEM,OAAO,IAAI,MAAM,CAEvB;IAEY,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqC/E;CACF;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,EACxB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAK9B"}

package/dest/services/tx_collection/file_store_tx_source.js

@@ -0,0 +1,100 @@
+import { partitionAsync } from '@aztec/foundation/collection';
+import { createLogger } from '@aztec/foundation/log';
+import { Timer } from '@aztec/foundation/timer';
+import { createReadOnlyFileStore } from '@aztec/stdlib/file-store';
+import { Tx } from '@aztec/stdlib/tx';
+import { Metrics, getTelemetryClient } from '@aztec/telemetry-client';
+/** TxSource implementation that downloads txs from a file store. */ export class FileStoreTxSource {
+    fileStore;
+    baseUrl;
+    basePath;
+    txValidator;
+    log;
+    downloadsSuccess;
+    downloadsFailed;
+    downloadDuration;
+    downloadSize;
+    constructor(fileStore, baseUrl, basePath, txValidator, log, telemetry){
+        this.fileStore = fileStore;
+        this.baseUrl = baseUrl;
+        this.basePath = basePath;
+        this.txValidator = txValidator;
+        this.log = log;
+        const meter = telemetry.getMeter('file-store-tx-source');
+        this.downloadsSuccess = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_SUCCESS);
+        this.downloadsFailed = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_FAILED);
+        this.downloadDuration = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_DURATION);
+        this.downloadSize = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_SIZE);
+    }
+    /**
+   * Creates a FileStoreTxSource from a URL.
+   * @param url - The file store URL (s3://, gs://, file://, http://, https://).
+   * @param basePath - Base path for tx files within the store.
+   * @param log - Optional logger.
+   * @param telemetry - Optional telemetry client.
+   * @returns The FileStoreTxSource instance, or undefined if creation fails.
+   */ static async create(url, basePath, txValidator, log = createLogger('p2p:file_store_tx_source'), telemetry = getTelemetryClient()) {
+        try {
+            const fileStore = await createReadOnlyFileStore(url, log);
+            if (!fileStore) {
+                log.warn(`Failed to create file store for URL: ${url}`);
+                return undefined;
+            }
+            return new FileStoreTxSource(fileStore, url, basePath, txValidator, log, telemetry);
+        } catch (err) {
+            log.warn(`Error creating file store for URL: ${url}`, {
+                error: err
+            });
+            return undefined;
+        }
+    }
+    getInfo() {
+        return `file-store:${this.baseUrl}`;
+    }
+    async getTxsByHash(txHashes) {
+        const results = await Promise.all(txHashes.map(async (txHash)=>{
+            const path = `${this.basePath}/txs/${txHash.toString()}.bin`;
+            const timer = new Timer();
+            try {
+                const buffer = await this.fileStore.read(path);
+                const tx = Tx.fromBuffer(buffer);
+                return {
+                    tx,
+                    downloadDuration: timer.ms(),
+                    downloadSize: buffer.length
+                };
+            } catch  {
+                this.downloadsFailed.add(1);
+                return undefined;
+            }
+        }));
+        const txs = results.filter((tx)=>tx !== undefined);
+        const [validTxs, invalidTxs] = await partitionAsync(txs, async ({ tx, downloadDuration, downloadSize })=>{
+            const valid = await this.txValidator.validateTx(tx);
+            if (valid.result === 'valid') {
+                this.downloadsSuccess.add(1);
+                this.downloadDuration.record(Math.ceil(downloadDuration));
+                this.downloadSize.record(downloadSize);
+                return true;
+            } else {
+                this.downloadsFailed.add(1);
+                return false;
+            }
+        });
+        return {
+            validTxs: validTxs.map(({ tx })=>tx),
+            invalidTxHashes: invalidTxs.map(({ tx })=>tx.getTxHash().toString())
+        };
+    }
+}
+/**
+ * Creates FileStoreTxSource instances from URLs.
+ * @param urls - Array of file store URLs.
+ * @param basePath - Base path for tx files within each store.
+ * @param log - Optional logger.
+ * @param telemetry - Optional telemetry client.
+ * @returns Array of successfully created FileStoreTxSource instances.
+ */ export async function createFileStoreTxSources(urls, basePath, txValidator, log = createLogger('p2p:file_store_tx_source'), telemetry = getTelemetryClient()) {
+    const sources = await Promise.all(urls.map((url)=>FileStoreTxSource.create(url, basePath, txValidator, log, telemetry)));
+    return sources.filter((s)=>s !== undefined);
+}

package/dest/services/tx_collection/index.d.ts

@@ -1,4 +1,4 @@
-export { TxCollection, type FastCollectionRequestInput } from './tx_collection.js';
+export { TxCollection, type FastCollectionRequestInput, type IReqRespTxsCollector } from './tx_collection.js';
 export { type TxSource, createNodeRpcTxSources, NodeRpcTxSource } from './tx_source.js';
-export { type ProposalTxCollector, BatchTxRequesterCollector, SendBatchRequestCollector, } from './proposal_tx_collector.js';
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zZXJ2aWNlcy90eF9jb2xsZWN0aW9uL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxZQUFZLEVBQUUsS0FBSywwQkFBMEIsRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBQ25GLE9BQU8sRUFBRSxLQUFLLFFBQVEsRUFBRSxzQkFBc0IsRUFBRSxlQUFlLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUN4RixPQUFPLEVBQ0wsS0FBSyxtQkFBbUIsRUFDeEIseUJBQXlCLEVBQ3pCLHlCQUF5QixHQUMxQixNQUFNLDRCQUE0QixDQUFDIn0=
+export { FileStoreTxSource, createFileStoreTxSources } from './file_store_tx_source.js';
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zZXJ2aWNlcy90eF9jb2xsZWN0aW9uL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxZQUFZLEVBQUUsS0FBSywwQkFBMEIsRUFBRSxLQUFLLG9CQUFvQixFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDOUcsT0FBTyxFQUFFLEtBQUssUUFBUSxFQUFFLHNCQUFzQixFQUFFLGVBQWUsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBQ3hGLE9BQU8sRUFBRSxpQkFBaUIsRUFBRSx3QkFBd0IsRUFBRSxNQUFNLDJCQUEyQixDQUFDIn0=

package/dest/services/tx_collection/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,KAAK,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AACnF,OAAO,EAAE,KAAK,QAAQ,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACxF,OAAO,EACL,KAAK,mBAAmB,EACxB,yBAAyB,EACzB,yBAAyB,GAC1B,MAAM,4BAA4B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,KAAK,0BAA0B,EAAE,KAAK,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9G,OAAO,EAAE,KAAK,QAAQ,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACxF,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC"}

package/dest/services/tx_collection/index.js

@@ -1,3 +1,3 @@
 export { TxCollection } from './tx_collection.js';
 export { createNodeRpcTxSources, NodeRpcTxSource } from './tx_source.js';
-export { BatchTxRequesterCollector, SendBatchRequestCollector } from './proposal_tx_collector.js';
+export { FileStoreTxSource, createFileStoreTxSources } from './file_store_tx_source.js';

package/dest/services/tx_collection/instrumentation.js

@@ -9,8 +9,7 @@ export class TxCollectionInstrumentation {
             [Attributes.TX_COLLECTION_METHOD]: [
                 'fast-req-resp',
                 'fast-node-rpc',
-                'slow-req-resp',
-                'slow-node-rpc'
+                'file-store'
             ]
         });
         this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX);

package/dest/services/tx_collection/request_tracker.d.ts

@@ -0,0 +1,53 @@
+import type { DateProvider } from '@aztec/foundation/timer';
+import { TxHash } from '@aztec/stdlib/tx';
+import type { Tx } from '@aztec/stdlib/tx';
+/**
+ * Tracks which transactions are still missing and need to be fetched.
+ * Manages the request deadline and serves as the sole source of cancellation signal.
+ * The request is cancelled when all txs are fetched or the deadline expires.
+ */
+export interface IRequestTracker {
+    /** Returns the set of transaction hashes that are still missing. */
+    get missingTxHashes(): Set<string>;
+    /** Size of this.missingTxHashes */
+    get numberOfMissingTxs(): number;
+    /** Are all requested txs fetched */
+    allFetched(): boolean;
+    /** Checks that transaction is still missing */
+    isMissing(txHash: string): boolean;
+    /** Marks a transaction as fetched. Returns true if it was previously missing. */
+    markFetched(tx: Tx): boolean;
+    /** Get list of collected txs */
+    get collectedTxs(): Tx[];
+    /** The deadline for this request. */
+    get deadline(): Date;
+    /** Remaining time in milliseconds until deadline. Returns 0 if already past. */
+    get timeoutMs(): number;
+    /** Checks whether the request is cancelled (deadline expired or all fetched). May trigger cancellation if deadline has passed. */
+    checkCancelled(): boolean;
+    /** Resolves when deadline expires or all txs are fetched. */
+    get cancellationToken(): Promise<void>;
+    /** Externally cancel the request. */
+    cancel(): void;
+}
+export declare class RequestTracker implements IRequestTracker {
+    readonly missingTxHashes: Set<string>;
+    readonly deadline: Date;
+    private readonly dateProvider?;
+    readonly collectedTxs: Tx[];
+    private done;
+    private readonly cancellationTokenPromise;
+    private readonly deadlineTimer;
+    private constructor();
+    static create(hashes: TxHash[] | string[], deadline: Date, dateProvider?: DateProvider): RequestTracker;
+    markFetched(tx: Tx): boolean;
+    get numberOfMissingTxs(): number;
+    allFetched(): boolean;
+    isMissing(txHash: string): boolean;
+    get timeoutMs(): number;
+    checkCancelled(): boolean;
+    get cancellationToken(): Promise<void>;
+    cancel(): void;
+    private finish;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicmVxdWVzdF90cmFja2VyLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9yZXF1ZXN0X3RyYWNrZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxLQUFLLEVBQUUsWUFBWSxFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFDNUQsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBQzFDLE9BQU8sS0FBSyxFQUFFLEVBQUUsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRTNDOzs7O0dBSUc7QUFDSCxNQUFNLFdBQVcsZUFBZTtJQUM5QixvRUFBb0U7SUFDcEUsSUFBSSxlQUFlLElBQUksR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDO0lBQ25DLG1DQUFtQztJQUNuQyxJQUFJLGtCQUFrQixJQUFJLE1BQU0sQ0FBQztJQUNqQyxvQ0FBb0M7SUFDcEMsVUFBVSxJQUFJLE9BQU8sQ0FBQztJQUN0QiwrQ0FBK0M7SUFDL0MsU0FBUyxDQUFDLE1BQU0sRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUFDO0lBQ25DLGlGQUFpRjtJQUNqRixXQUFXLENBQUMsRUFBRSxFQUFFLEVBQUUsR0FBRyxPQUFPLENBQUM7SUFDN0IsZ0NBQWdDO0lBQ2hDLElBQUksWUFBWSxJQUFJLEVBQUUsRUFBRSxDQUFDO0lBQ3pCLHFDQUFxQztJQUNyQyxJQUFJLFFBQVEsSUFBSSxJQUFJLENBQUM7SUFDckIsZ0ZBQWdGO0lBQ2hGLElBQUksU0FBUyxJQUFJLE1BQU0sQ0FBQztJQUN4QixrSUFBa0k7SUFDbEksY0FBYyxJQUFJLE9BQU8sQ0FBQztJQUMxQiw2REFBNkQ7SUFDN0QsSUFBSSxpQkFBaUIsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDdkMscUNBQXFDO0lBQ3JDLE1BQU0sSUFBSSxJQUFJLENBQUM7Q0FDaEI7QUFFRCxxQkFBYSxjQUFlLFlBQVcsZUFBZTthQU9sQyxlQUFlLEVBQUUsR0FBRyxDQUFDLE1BQU0sQ0FBQzthQUM1QixRQUFRLEVBQUUsSUFBSTtJQUM5QixPQUFPLENBQUMsUUFBUSxDQUFDLFlBQVksQ0FBQztJQVJoQyxTQUFnQixZQUFZLEVBQUUsRUFBRSxFQUFFLENBQU07SUFDeEMsT0FBTyxDQUFDLElBQUksQ0FBUztJQUNyQixPQUFPLENBQUMsUUFBUSxDQUFDLHdCQUF3QixDQUE2QjtJQUN0RSxPQUFPLENBQUMsUUFBUSxDQUFDLGFBQWEsQ0FBNEM7SUFFMUUsT0FBTyxlQW1CTjtJQUVELE9BQWMsTUFBTSxDQUFDLE1BQU0sRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLEVBQUUsRUFBRSxRQUFRLEVBQUUsSUFBSSxFQUFFLFlBQVksQ0FBQyxFQUFFLFlBQVksa0JBRTVGO0lBRUQsV0FBVyxDQUFDLEVBQUUsRUFBRSxFQUFFLEdBQUcsT0FBTyxDQVMzQjtJQUVELElBQUksa0JBQWtCLElBQUksTUFBTSxDQUUvQjtJQUVELFVBQVUsSUFBSSxPQUFPLENBRXBCO0lBRUQsU0FBUyxDQUFDLE1BQU0sRUFBRSxNQUFNLEdBQUcsT0FBTyxDQUVqQztJQUVELElBQUksU0FBUyxJQUFJLE1BQU0sQ0FHdEI7SUFFRCxjQUFjLElBQUksT0FBTyxDQVl4QjtJQUVELElBQUksaUJBQWlCLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUVyQztJQUVELE1BQU0sSUFBSSxJQUFJLENBRWI7SUFFRCxPQUFPLENBQUMsTUFBTTtDQVVmIn0=

package/dest/services/tx_collection/request_tracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request_tracker.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/request_tracker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAE3C;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,oEAAoE;IACpE,IAAI,eAAe,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,mCAAmC;IACnC,IAAI,kBAAkB,IAAI,MAAM,CAAC;IACjC,oCAAoC;IACpC,UAAU,IAAI,OAAO,CAAC;IACtB,+CAA+C;IAC/C,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IACnC,iFAAiF;IACjF,WAAW,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;IAC7B,gCAAgC;IAChC,IAAI,YAAY,IAAI,EAAE,EAAE,CAAC;IACzB,qCAAqC;IACrC,IAAI,QAAQ,IAAI,IAAI,CAAC;IACrB,gFAAgF;IAChF,IAAI,SAAS,IAAI,MAAM,CAAC;IACxB,kIAAkI;IAClI,cAAc,IAAI,OAAO,CAAC;IAC1B,6DAA6D;IAC7D,IAAI,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,qCAAqC;IACrC,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,qBAAa,cAAe,YAAW,eAAe;aAOlC,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC;aAC5B,QAAQ,EAAE,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;IARhC,SAAgB,YAAY,EAAE,EAAE,EAAE,CAAM;IACxC,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAA6B;IACtE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA4C;IAE1E,OAAO,eAmBN;IAED,OAAc,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,EAAE,YAAY,kBAE5F;IAED,WAAW,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,CAS3B;IAED,IAAI,kBAAkB,IAAI,MAAM,CAE/B;IAED,UAAU,IAAI,OAAO,CAEpB;IAED,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEjC;IAED,IAAI,SAAS,IAAI,MAAM,CAGtB;IAED,cAAc,IAAI,OAAO,CAYxB;IAED,IAAI,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAErC;IAED,MAAM,IAAI,IAAI,CAEb;IAED,OAAO,CAAC,MAAM;CAUf"}

package/dest/services/tx_collection/request_tracker.js

@@ -0,0 +1,84 @@
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+export class RequestTracker {
+    missingTxHashes;
+    deadline;
+    dateProvider;
+    collectedTxs;
+    done;
+    cancellationTokenPromise;
+    deadlineTimer;
+    constructor(missingTxHashes, deadline, dateProvider){
+        this.missingTxHashes = missingTxHashes;
+        this.deadline = deadline;
+        this.dateProvider = dateProvider;
+        this.collectedTxs = [];
+        this.done = false;
+        this.cancellationTokenPromise = promiseWithResolvers();
+        if (missingTxHashes.size === 0) {
+            this.finish();
+            return;
+        }
+        const now = this.dateProvider?.now() ?? Date.now();
+        const remaining = deadline.getTime() - now;
+        if (remaining <= 0) {
+            this.finish();
+        } else {
+            this.deadlineTimer = setTimeout(()=>this.finish(), remaining);
+        }
+    }
+    static create(hashes, deadline, dateProvider) {
+        return new RequestTracker(new Set(hashes.map((hash)=>hash.toString())), deadline, dateProvider);
+    }
+    markFetched(tx) {
+        if (this.missingTxHashes.delete(tx.txHash.toString())) {
+            this.collectedTxs.push(tx);
+            if (this.allFetched()) {
+                this.finish();
+            }
+            return true;
+        }
+        return false;
+    }
+    get numberOfMissingTxs() {
+        return this.missingTxHashes.size;
+    }
+    allFetched() {
+        return this.numberOfMissingTxs === 0;
+    }
+    isMissing(txHash) {
+        return this.missingTxHashes.has(txHash.toString());
+    }
+    get timeoutMs() {
+        const now = this.dateProvider?.now() ?? Date.now();
+        return Math.max(0, this.deadline.getTime() - now);
+    }
+    checkCancelled() {
+        if (this.done) {
+            return true;
+        }
+        // Synchronous fallback: check deadline even if setTimeout hasn't fired yet.
+        // This prevents macrotask starvation in tight async loops from blocking cancellation.
+        const now = this.dateProvider?.now() ?? Date.now();
+        if (now >= this.deadline.getTime()) {
+            this.finish();
+            return true;
+        }
+        return false;
+    }
+    get cancellationToken() {
+        return this.cancellationTokenPromise.promise;
+    }
+    cancel() {
+        this.finish();
+    }
+    finish() {
+        if (this.done) {
+            return;
+        }
+        this.done = true;
+        if (this.deadlineTimer) {
+            clearTimeout(this.deadlineTimer);
+        }
+        this.cancellationTokenPromise.resolve();
+    }
+}