@aztec/p2p 0.0.0-test.0 → 0.0.1-commit.001888fc

package/dest/services/tx_collection/fast_tx_collection.js

@@ -0,0 +1,327 @@
+import { times } from '@aztec/foundation/collection';
+import { AbortError, TimeoutError } from '@aztec/foundation/error';
+import { createLogger } from '@aztec/foundation/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider, elapsed } from '@aztec/foundation/timer';
+import { TxHash } from '@aztec/stdlib/tx';
+import { MissingTxsTracker } from './missing_txs_tracker.js';
+import { BatchTxRequesterCollector, SendBatchRequestCollector } from './proposal_tx_collector.js';
+export class FastTxCollection {
+    nodes;
+    txCollectionSink;
+    config;
+    dateProvider;
+    log;
+    // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
+    requests;
+    missingTxsCollector;
+    constructor(p2pService, nodes, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service'), missingTxsCollector){
+        this.nodes = nodes;
+        this.txCollectionSink = txCollectionSink;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+        this.requests = new Set();
+        const batchTxRequesterConfig = this.config;
+        const missingTxsCollectorType = this.config.txCollectionMissingTxsCollectorType;
+        this.missingTxsCollector = missingTxsCollector ?? (missingTxsCollectorType === 'old' ? new SendBatchRequestCollector(p2pService) : new BatchTxRequesterCollector(p2pService, log, dateProvider, undefined, batchTxRequesterConfig));
+    }
+    async stop() {
+        this.requests.forEach((request)=>request.promise.reject(new AbortError(`Stopped collection service`)));
+        await Promise.resolve();
+    }
+    getFastCollectionRequests() {
+        return this.requests;
+    }
+    async collectFastFor(input, txHashes, opts) {
+        const timeout = opts.deadline.getTime() - this.dateProvider.now();
+        if (timeout <= 0) {
+            this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, {
+                deadline: opts.deadline.getTime(),
+                now: this.dateProvider.now()
+            });
+            return [];
+        }
+        const blockInfo = input.type === 'proposal' ? {
+            ...input.blockProposal.toBlockInfo(),
+            blockNumber: input.blockNumber
+        } : {
+            ...input.block.toBlockInfo()
+        };
+        // This promise is used to await for the collection to finish during the main collectFast method.
+        // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
+        const promise = promiseWithResolvers();
+        const timeoutTimer = setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
+        const request = {
+            ...input,
+            blockInfo,
+            promise,
+            missingTxTracker: MissingTxsTracker.fromArray(txHashes),
+            deadline: opts.deadline
+        };
+        const [duration] = await elapsed(()=>this.collectFast(request, {
+                ...opts
+            }));
+        clearTimeout(timeoutTimer);
+        this.log.verbose(`Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            duration,
+            requestType: input.type,
+            missingTxs: [
+                ...request.missingTxTracker.missingTxHashes
+            ]
+        });
+        return request.missingTxTracker.collectedTxs;
+    }
+    async collectFast(request, opts) {
+        this.requests.add(request);
+        const { blockInfo } = request;
+        this.log.debug(`Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            requestType: request.type,
+            deadline: opts.deadline
+        });
+        try {
+            // Start blasting all nodes for the txs. We give them a little time to respond before we start reqresp.
+            // And keep an eye on the request promise to ensure we don't wait longer than the deadline or return as soon
+            // as we have collected all txs, whatever the source.
+            const nodeCollectionPromise = this.collectFastFromNodes(request, opts);
+            const waitBeforeReqResp = sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs);
+            await Promise.race([
+                request.promise.promise,
+                waitBeforeReqResp
+            ]);
+            // If we have collected all txs, we can stop here
+            if (request.missingTxTracker.allFetched()) {
+                this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
+                return;
+            }
+            // Start blasting reqresp for the remaining txs. Note that node collection keeps running in parallel.
+            // We stop when we have collected all txs, timed out, or both node collection and reqresp have given up.
+            const collectionPromise = Promise.allSettled([
+                this.collectFastViaReqResp(request, opts),
+                nodeCollectionPromise
+            ]);
+            await Promise.race([
+                collectionPromise,
+                request.promise.promise
+            ]);
+        } catch (err) {
+            // Log and swallow all errors
+            const logCtx = {
+                ...blockInfo,
+                errorMessage: err instanceof Error ? err.message : undefined,
+                missingTxs: request.missingTxTracker.missingTxHashes.values().map((txHash)=>txHash.toString())
+            };
+            if (err instanceof Error && err.name === 'TimeoutError') {
+                this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
+            } else if (err instanceof Error && err.name === 'AbortError') {
+                this.log.warn(`Aborted collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
+            } else {
+                this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, logCtx);
+            }
+        } finally{
+            // Ensure no unresolved promises and remove the request from the set
+            request.promise.resolve();
+            this.requests.delete(request);
+        }
+    }
+    /**
+   * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests
+   * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
+   * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
+   * tx is requested more than once at the same time to the same node.
+   */ async collectFastFromNodes(request, opts) {
+        if (this.nodes.length === 0) {
+            return;
+        }
+        // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
+        const attemptsPerTx = [
+            ...request.missingTxTracker.missingTxHashes
+        ].map((txHash)=>({
+                txHash,
+                attempts: 0,
+                found: false
+            }));
+        // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
+        await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx, opts)));
+    }
+    async collectFastFromNode(request, node, attemptsPerTx, opts) {
+        const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
+        const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
+        const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
+        const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
+        const processBatch = async ()=>{
+            while(notFinished()){
+                // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts,
+                // so we prioritize txs that have been requested less often.
+                const batch = [];
+                let index = 0;
+                while(batch.length < maxBatchSize){
+                    const txToRequest = attemptsPerTx[index++];
+                    if (!txToRequest) {
+                        break;
+                    } else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
+                        // Mark as found if it was found somewhere else, we'll then remove it from the array.
+                        // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
+                        txToRequest.found = true;
+                    } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) {
+                        // If the tx is not alredy being requested to this node, add it to the current batch and increase attempts.
+                        // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that
+                        // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting
+                        // the same txs at the same time.
+                        batch.push(txToRequest);
+                        activeRequestsToThisNode.add(txToRequest.txHash);
+                        txToRequest.attempts++;
+                    }
+                }
+                // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end.
+                attemptsPerTx.sort((a, b)=>a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found));
+                const firstFoundTxIndex = attemptsPerTx.findIndex((tx)=>tx.found);
+                if (firstFoundTxIndex !== -1) {
+                    attemptsPerTx.length = firstFoundTxIndex;
+                }
+                // If we see no more txs to request, we can stop this "process" loop
+                if (batch.length === 0) {
+                    return;
+                }
+                const txHashes = batch.map(({ txHash })=>txHash);
+                // Collect this batch from the node
+                await this.txCollectionSink.collect(async ()=>{
+                    const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
+                    for (const tx of result.validTxs){
+                        request.missingTxTracker.markFetched(tx);
+                    }
+                    return result;
+                }, txHashes, {
+                    description: `fast ${node.getInfo()}`,
+                    node: node.getInfo(),
+                    method: 'fast-node-rpc',
+                    ...request.blockInfo
+                }, this.getAddContext(request));
+                // Clear from the active requests the txs we just requested
+                for (const requestedTx of batch){
+                    activeRequestsToThisNode.delete(requestedTx.txHash);
+                }
+                // Sleep a bit until hitting the node again (or not, depending on config)
+                if (notFinished()) {
+                    await sleep(this.config.txCollectionFastNodeIntervalMs);
+                }
+            }
+        };
+        // Kick off N parallel requests to the node, up to the maxParallelRequests limit
+        await Promise.all(times(maxParallelRequests, processBatch));
+    }
+    async collectFastViaReqResp(request, opts) {
+        const timeoutMs = +request.deadline - this.dateProvider.now();
+        const pinnedPeer = opts.pinnedPeer;
+        const blockInfo = request.blockInfo;
+        const slotNumber = blockInfo.slotNumber;
+        if (timeoutMs < 100) {
+            this.log.warn(`Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`, {
+                timeoutMs,
+                ...blockInfo
+            });
+            return;
+        }
+        this.log.debug(`Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
+            ...blockInfo,
+            timeoutMs,
+            pinnedPeer
+        });
+        try {
+            await this.txCollectionSink.collect(async ()=>{
+                let result;
+                if (request.type === 'proposal') {
+                    result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, request.blockProposal, pinnedPeer, timeoutMs);
+                } else if (request.type === 'block') {
+                    const blockTxsSource = {
+                        txHashes: request.block.body.txEffects.map((e)=>e.txHash),
+                        archive: request.block.archive.root
+                    };
+                    result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, blockTxsSource, pinnedPeer, timeoutMs);
+                } else {
+                    throw new Error(`Unknown request type: ${request.type}`);
+                }
+                return {
+                    validTxs: result,
+                    invalidTxHashes: []
+                };
+            }, Array.from(request.missingTxTracker.missingTxHashes), {
+                description: `reqresp for slot ${slotNumber}`,
+                method: 'fast-req-resp',
+                ...opts,
+                ...request.blockInfo
+            }, this.getAddContext(request));
+        } catch (err) {
+            this.log.error(`Error sending fast reqresp request for txs`, err, {
+                txs: [
+                    ...request.missingTxTracker.missingTxHashes
+                ],
+                ...blockInfo
+            });
+        }
+    }
+    /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */ getAddContext(request) {
+        if (request.type === 'proposal') {
+            return {
+                type: 'proposal',
+                blockHeader: request.blockProposal.blockHeader
+            };
+        } else {
+            return {
+                type: 'mined',
+                block: request.block
+            };
+        }
+    }
+    /**
+   * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
+   * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
+   */ foundTxs(txs) {
+        for (const request of this.requests){
+            for (const tx of txs){
+                const txHash = tx.txHash.toString();
+                // Remove the tx hash from the missing set, and add it to the found set.
+                if (request.missingTxTracker.markFetched(tx)) {
+                    this.log.trace(`Found tx ${txHash} for fast collection request`, {
+                        ...request.blockInfo,
+                        txHash: tx.txHash.toString(),
+                        type: request.type
+                    });
+                }
+                // If we found all txs for this request, we resolve the promise
+                if (request.missingTxTracker.allFetched()) {
+                    this.log.trace(`All txs found for fast collection request`, {
+                        ...request.blockInfo,
+                        type: request.type
+                    });
+                    request.promise.resolve();
+                }
+            }
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks less than or requal to the block number specified.
+   * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
+   */ stopCollectingForBlocksUpTo(blockNumber) {
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber <= blockNumber) {
+                request.promise.reject(new AbortError(`Stopped collecting txs up to block ${blockNumber}`));
+                this.requests.delete(request);
+            }
+        }
+    }
+    /**
+   * Stop collecting all txs for blocks greater than the block number specified.
+   * To be called when there is a chain prune and previously mined txs are no longer relevant.
+   */ stopCollectingForBlocksAfter(blockNumber) {
+        for (const request of this.requests){
+            if (request.blockInfo.blockNumber > blockNumber) {
+                request.promise.reject(new AbortError(`Stopped collecting txs after block ${blockNumber}`));
+                this.requests.delete(request);
+            }
+        }
+    }
+}

package/dest/services/tx_collection/file_store_tx_collection.d.ts

@@ -0,0 +1,53 @@
+import { type Logger } from '@aztec/foundation/log';
+import { DateProvider } from '@aztec/foundation/timer';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+import type { FileStoreTxSource } from './file_store_tx_source.js';
+import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
+/** Configuration for a FileStoreTxCollection instance. */
+export type FileStoreCollectionConfig = {
+    workerCount: number;
+    backoffBaseMs: number;
+    backoffMaxMs: number;
+};
+/**
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
+ * Uses a shared worker pool that pulls entries with priority (fewest attempts first),
+ * retries with round-robin across sources, and applies exponential backoff between
+ * full cycles through all sources.
+ */
+export declare class FileStoreTxCollection {
+    private readonly sources;
+    private readonly txCollectionSink;
+    private readonly config;
+    private readonly dateProvider;
+    private readonly log;
+    /** Map from tx hash string to entry for all pending downloads. */
+    private entries;
+    /** Worker promises for the shared worker pool. */
+    private workers;
+    /** Whether the worker pool is running. */
+    private running;
+    /** Signal used to wake sleeping workers when new entries arrive or stop is called. */
+    private wakeSignal;
+    constructor(sources: FileStoreTxSource[], txCollectionSink: TxCollectionSink, config: FileStoreCollectionConfig, dateProvider?: DateProvider, log?: Logger);
+    /** Starts the shared worker pool. */
+    start(): void;
+    /** Stops all workers and clears state. */
+    stop(): Promise<void>;
+    /** Adds entries to the shared map and wakes workers. */
+    startCollecting(txHashes: TxHash[], context: TxAddContext, deadline: Date): void;
+    /** Removes entries for txs that have been found elsewhere. */
+    foundTxs(txs: Tx[]): void;
+    /** Clears all pending entries. */
+    clearPending(): void;
+    private workerLoop;
+    /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */
+    private getNextAction;
+    /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */
+    private getBackoffMs;
+    /** Resolves the current wake signal and creates a new one. */
+    private wake;
+    private waitForWake;
+    private sleepOrWake;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZV9zdG9yZV90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9maWxlX3N0b3JlX3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBR2xFLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUN2RCxPQUFPLEVBQUUsRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRTlDLE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLE1BQU0sMkJBQTJCLENBQUM7QUFDbkUsT0FBTyxLQUFLLEVBQUUsWUFBWSxFQUFFLGdCQUFnQixFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFFOUUsMERBQTBEO0FBQzFELE1BQU0sTUFBTSx5QkFBeUIsR0FBRztJQUN0QyxXQUFXLEVBQUUsTUFBTSxDQUFDO0lBQ3BCLGFBQWEsRUFBRSxNQUFNLENBQUM7SUFDdEIsWUFBWSxFQUFFLE1BQU0sQ0FBQztDQUN0QixDQUFDO0FBV0Y7Ozs7O0dBS0c7QUFDSCxxQkFBYSxxQkFBcUI7SUFjOUIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxPQUFPO0lBQ3hCLE9BQU8sQ0FBQyxRQUFRLENBQUMsZ0JBQWdCO0lBQ2pDLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQUN2QixPQUFPLENBQUMsUUFBUSxDQUFDLFlBQVk7SUFDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHO0lBakJ0QixrRUFBa0U7SUFDbEUsT0FBTyxDQUFDLE9BQU8sQ0FBdUM7SUFFdEQsa0RBQWtEO0lBQ2xELE9BQU8sQ0FBQyxPQUFPLENBQXVCO0lBRXRDLDBDQUEwQztJQUMxQyxPQUFPLENBQUMsT0FBTyxDQUFTO0lBRXhCLHNGQUFzRjtJQUN0RixPQUFPLENBQUMsVUFBVSxDQUE2QjtJQUUvQyxZQUNtQixPQUFPLEVBQUUsaUJBQWlCLEVBQUUsRUFDNUIsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLE1BQU0sRUFBRSx5QkFBeUIsRUFDakMsWUFBWSxHQUFFLFlBQWlDLEVBQy9DLEdBQUcsR0FBRSxNQUFxRCxFQUc1RTtJQUVELHFDQUFxQztJQUM5QixLQUFLLElBQUksSUFBSSxDQVNuQjtJQUVELDBDQUEwQztJQUM3QixJQUFJLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQU1qQztJQUVELHdEQUF3RDtJQUNqRCxlQUFlLENBQUMsUUFBUSxFQUFFLE1BQU0sRUFBRSxFQUFFLE9BQU8sRUFBRSxZQUFZLEVBQUUsUUFBUSxFQUFFLElBQUksR0FBRyxJQUFJLENBc0J0RjtJQUVELDhEQUE4RDtJQUN2RCxRQUFRLENBQUMsR0FBRyxFQUFFLEVBQUUsRUFBRSxHQUFHLElBQUksQ0FJL0I7SUFFRCxrQ0FBa0M7SUFDM0IsWUFBWSxJQUFJLElBQUksQ0FFMUI7WUFFYSxVQUFVO0lBa0N4QixxR0FBcUc7SUFDckcsT0FBTyxDQUFDLGFBQWE7SUE4QnJCLDZGQUE2RjtJQUM3RixPQUFPLENBQUMsWUFBWTtJQVFwQiw4REFBOEQ7SUFDOUQsT0FBTyxDQUFDLElBQUk7WUFNRSxXQUFXO1lBS1gsV0FBVztDQUcxQiJ9

package/dest/services/tx_collection/file_store_tx_collection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file_store_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE9E,0DAA0D;AAC1D,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAWF;;;;;GAKG;AACH,qBAAa,qBAAqB;IAc9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IAjBtB,kEAAkE;IAClE,OAAO,CAAC,OAAO,CAAuC;IAEtD,kDAAkD;IAClD,OAAO,CAAC,OAAO,CAAuB;IAEtC,0CAA0C;IAC1C,OAAO,CAAC,OAAO,CAAS;IAExB,sFAAsF;IACtF,OAAO,CAAC,UAAU,CAA6B;IAE/C,YACmB,OAAO,EAAE,iBAAiB,EAAE,EAC5B,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,yBAAyB,EACjC,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAqD,EAG5E;IAED,qCAAqC;IAC9B,KAAK,IAAI,IAAI,CASnB;IAED,0CAA0C;IAC7B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAMjC;IAED,wDAAwD;IACjD,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,GAAG,IAAI,CAsBtF;IAED,8DAA8D;IACvD,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,IAAI,CAI/B;IAED,kCAAkC;IAC3B,YAAY,IAAI,IAAI,CAE1B;YAEa,UAAU;IAkCxB,qGAAqG;IACrG,OAAO,CAAC,aAAa;IA8BrB,6FAA6F;IAC7F,OAAO,CAAC,YAAY;IAQpB,8DAA8D;IAC9D,OAAO,CAAC,IAAI;YAME,WAAW;YAKX,WAAW;CAG1B"}

package/dest/services/tx_collection/file_store_tx_collection.js

@@ -0,0 +1,167 @@
+import { createLogger } from '@aztec/foundation/log';
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider } from '@aztec/foundation/timer';
+import { TxHash } from '@aztec/stdlib/tx';
+/**
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
+ * Uses a shared worker pool that pulls entries with priority (fewest attempts first),
+ * retries with round-robin across sources, and applies exponential backoff between
+ * full cycles through all sources.
+ */ export class FileStoreTxCollection {
+    sources;
+    txCollectionSink;
+    config;
+    dateProvider;
+    log;
+    /** Map from tx hash string to entry for all pending downloads. */ entries;
+    /** Worker promises for the shared worker pool. */ workers;
+    /** Whether the worker pool is running. */ running;
+    /** Signal used to wake sleeping workers when new entries arrive or stop is called. */ wakeSignal;
+    constructor(sources, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:file_store_tx_collection')){
+        this.sources = sources;
+        this.txCollectionSink = txCollectionSink;
+        this.config = config;
+        this.dateProvider = dateProvider;
+        this.log = log;
+        this.entries = new Map();
+        this.workers = [];
+        this.running = false;
+        this.wakeSignal = promiseWithResolvers();
+    }
+    /** Starts the shared worker pool. */ start() {
+        if (this.sources.length === 0) {
+            this.log.debug('No file store sources configured');
+            return;
+        }
+        this.running = true;
+        for(let i = 0; i < this.config.workerCount; i++){
+            this.workers.push(this.workerLoop());
+        }
+    }
+    /** Stops all workers and clears state. */ async stop() {
+        this.running = false;
+        this.wake();
+        await Promise.all(this.workers);
+        this.workers = [];
+        this.entries.clear();
+    }
+    /** Adds entries to the shared map and wakes workers. */ startCollecting(txHashes, context, deadline) {
+        if (this.sources.length === 0 || txHashes.length === 0) {
+            return;
+        }
+        if (+deadline <= this.dateProvider.now()) {
+            return;
+        }
+        for (const txHash of txHashes){
+            const hashStr = txHash.toString();
+            if (!this.entries.has(hashStr)) {
+                this.entries.set(hashStr, {
+                    txHash: hashStr,
+                    context,
+                    deadline,
+                    attempts: 0,
+                    lastAttemptTime: 0,
+                    nextSourceIndex: Math.floor(Math.random() * this.sources.length)
+                });
+            }
+        }
+        this.wake();
+    }
+    /** Removes entries for txs that have been found elsewhere. */ foundTxs(txs) {
+        for (const tx of txs){
+            this.entries.delete(tx.getTxHash().toString());
+        }
+    }
+    /** Clears all pending entries. */ clearPending() {
+        this.entries.clear();
+    }
+    async workerLoop() {
+        while(this.running){
+            const action = this.getNextAction();
+            if (action.type === 'sleep') {
+                await action.promise;
+                continue;
+            }
+            const entry = action.entry;
+            const source = this.sources[entry.nextSourceIndex % this.sources.length];
+            entry.nextSourceIndex++;
+            entry.attempts++;
+            entry.lastAttemptTime = this.dateProvider.now();
+            try {
+                const result = await this.txCollectionSink.collect(()=>source.getTxsByHash([
+                        TxHash.fromString(entry.txHash)
+                    ]), [
+                    entry.txHash
+                ], {
+                    description: `file-store ${source.getInfo()}`,
+                    method: 'file-store',
+                    fileStore: source.getInfo()
+                }, entry.context);
+                if (result.txs.length > 0) {
+                    this.entries.delete(entry.txHash);
+                }
+            } catch (err) {
+                this.log.trace(`Error downloading tx ${entry.txHash} from ${source.getInfo()}`, {
+                    err
+                });
+            }
+        }
+    }
+    /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */ getNextAction() {
+        const now = this.dateProvider.now();
+        let best;
+        let earliestReadyAt = Infinity;
+        for (const [key, entry] of this.entries){
+            if (+entry.deadline <= now) {
+                this.entries.delete(key);
+                continue;
+            }
+            const backoffMs = this.getBackoffMs(entry);
+            const readyAt = entry.lastAttemptTime + backoffMs;
+            if (readyAt > now) {
+                earliestReadyAt = Math.min(earliestReadyAt, readyAt);
+                continue;
+            }
+            if (!best || entry.attempts < best.attempts) {
+                best = entry;
+            }
+        }
+        if (best) {
+            return {
+                type: 'process',
+                entry: best
+            };
+        }
+        if (earliestReadyAt < Infinity) {
+            return {
+                type: 'sleep',
+                promise: this.sleepOrWake(earliestReadyAt - now)
+            };
+        }
+        return {
+            type: 'sleep',
+            promise: this.waitForWake()
+        };
+    }
+    /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */ getBackoffMs(entry) {
+        const fullCycles = Math.floor(entry.attempts / this.sources.length);
+        if (fullCycles === 0) {
+            return 0;
+        }
+        return Math.min(this.config.backoffBaseMs * Math.pow(2, fullCycles - 1), this.config.backoffMaxMs);
+    }
+    /** Resolves the current wake signal and creates a new one. */ wake() {
+        this.wakeSignal.resolve();
+        this.wakeSignal = promiseWithResolvers();
+    }
+    /** Waits until the wake signal is resolved. */ async waitForWake() {
+        await this.wakeSignal.promise;
+    }
+    /** Sleeps for the given duration or until the wake signal is resolved. */ async sleepOrWake(ms) {
+        await Promise.race([
+            sleep(ms),
+            this.wakeSignal.promise
+        ]);
+    }
+}

package/dest/services/tx_collection/file_store_tx_source.d.ts

@@ -0,0 +1,37 @@
+import { type Logger } from '@aztec/foundation/log';
+import { type TxHash } from '@aztec/stdlib/tx';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import type { TxSource, TxSourceCollectionResult } from './tx_source.js';
+/** TxSource implementation that downloads txs from a file store. */
+export declare class FileStoreTxSource implements TxSource {
+    private readonly fileStore;
+    private readonly baseUrl;
+    private readonly basePath;
+    private readonly log;
+    private downloadsSuccess;
+    private downloadsFailed;
+    private downloadDuration;
+    private downloadSize;
+    private constructor();
+    /**
+     * Creates a FileStoreTxSource from a URL.
+     * @param url - The file store URL (s3://, gs://, file://, http://, https://).
+     * @param basePath - Base path for tx files within the store.
+     * @param log - Optional logger.
+     * @param telemetry - Optional telemetry client.
+     * @returns The FileStoreTxSource instance, or undefined if creation fails.
+     */
+    static create(url: string, basePath: string, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource | undefined>;
+    getInfo(): string;
+    getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
+}
+/**
+ * Creates FileStoreTxSource instances from URLs.
+ * @param urls - Array of file store URLs.
+ * @param basePath - Base path for tx files within each store.
+ * @param log - Optional logger.
+ * @param telemetry - Optional telemetry client.
+ * @returns Array of successfully created FileStoreTxSource instances.
+ */
+export declare function createFileStoreTxSources(urls: string[], basePath: string, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource[]>;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZV9zdG9yZV90eF9zb3VyY2UuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zZXJ2aWNlcy90eF9jb2xsZWN0aW9uL2ZpbGVfc3RvcmVfdHhfc291cmNlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQU0sS0FBSyxNQUFNLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUNuRCxPQUFPLEVBR0wsS0FBSyxlQUFlLEVBR3JCLE1BQU0seUJBQXlCLENBQUM7QUFFakMsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLHdCQUF3QixFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFekUsb0VBQW9FO0FBQ3BFLHFCQUFhLGlCQUFrQixZQUFXLFFBQVE7SUFPOUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTO0lBQzFCLE9BQU8sQ0FBQyxRQUFRLENBQUMsT0FBTztJQUN4QixPQUFPLENBQUMsUUFBUSxDQUFDLFFBQVE7SUFDekIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHO0lBVHRCLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBZ0I7SUFDeEMsT0FBTyxDQUFDLGVBQWUsQ0FBZ0I7SUFDdkMsT0FBTyxDQUFDLGdCQUFnQixDQUFZO0lBQ3BDLE9BQU8sQ0FBQyxZQUFZLENBQVk7SUFFaEMsT0FBTyxlQVlOO0lBRUQ7Ozs7Ozs7T0FPRztJQUNILE9BQW9CLE1BQU0sQ0FDeEIsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsTUFBTSxFQUNoQixHQUFHLEdBQUUsTUFBaUQsRUFDdEQsU0FBUyxHQUFFLGVBQXNDLEdBQ2hELE9BQU8sQ0FBQyxpQkFBaUIsR0FBRyxTQUFTLENBQUMsQ0FZeEM7SUFFTSxPQUFPLElBQUksTUFBTSxDQUV2QjtJQUVZLFlBQVksQ0FBQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsT0FBTyxDQUFDLHdCQUF3QixDQUFDLENBK0IvRTtDQUNGO0FBRUQ7Ozs7Ozs7R0FPRztBQUNILHdCQUFzQix3QkFBd0IsQ0FDNUMsSUFBSSxFQUFFLE1BQU0sRUFBRSxFQUNkLFFBQVEsRUFBRSxNQUFNLEVBQ2hCLEdBQUcsR0FBRSxNQUFpRCxFQUN0RCxTQUFTLEdBQUUsZUFBc0MsR0FDaEQsT0FBTyxDQUFDLGlCQUFpQixFQUFFLENBQUMsQ0FHOUIifQ==

package/dest/services/tx_collection/file_store_tx_source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file_store_tx_source.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAM,KAAK,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAGL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,QAAQ,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAEzE,oEAAoE;AACpE,qBAAa,iBAAkB,YAAW,QAAQ;IAO9C,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,GAAG;IATtB,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,gBAAgB,CAAY;IACpC,OAAO,CAAC,YAAY,CAAY;IAEhC,OAAO,eAYN;IAED;;;;;;;OAOG;IACH,OAAoB,MAAM,CACxB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAYxC;IAEM,OAAO,IAAI,MAAM,CAEvB;IAEY,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CA+B/E;CACF;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,MAAM,EAChB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAG9B"}