@aztec/p2p 0.0.1-commit.f2ce05ee → 0.0.1-commit.f504929

package/src/services/reqresp/reqresp.ts

@@ -36,6 +36,7 @@ import {
   type ReqRespSubProtocolValidators,
   type SubProtocolMap,
   responseFromBuffer,
+  subProtocolSizeCalculators,
 } from './interface.js';
 import { ReqRespMetrics } from './metrics.js';
 import {
@@ -437,6 +438,9 @@ export class ReqResp implements ReqRespInterface {
     try {
       this.metrics.recordRequestSent(subProtocol);
 
+      // Calculate expected response size based on the request payload
+      const expectedSizeKb = subProtocolSizeCalculators[subProtocol](payload);
+
       this.logger.trace(`Sending request to peer ${peerId.toString()} on sub protocol ${subProtocol}`);
       stream = await this.connectionSampler.dialProtocol(peerId, subProtocol, dialTimeout);
       this.logger.trace(
@@ -444,11 +448,14 @@ export class ReqResp implements ReqRespInterface {
       );
 
       const timeoutErr = new IndividualReqRespTimeoutError();
+      // Create a wrapper to pass the expected size to readMessage
+      const readMessageWithSizeLimit = (source: AsyncIterable<Uint8ArrayList>) =>
+        this.readMessage(source, expectedSizeKb);
       const [_, resp] = await executeTimeout(
         signal =>
           Promise.all([
             pipeline([payload], stream!.sink, { signal }),
-            pipeline(stream!.source, this.readMessage.bind(this), { signal }),
+            pipeline(stream!.source, readMessageWithSizeLimit, { signal }),
           ]),
         this.individualRequestTimeoutMs,
         () => timeoutErr,
@@ -510,8 +517,11 @@ export class ReqResp implements ReqRespInterface {
    * The message is split into two components
    * - The first chunk should contain a control byte, indicating the status of the response see `ReqRespStatus`
    * - The second chunk should contain the response data
+   *
+   * @param source - The async iterable source of data chunks
+   * @param maxSizeKb - Optional maximum expected size in KB for the decompressed response
    */
-  private async readMessage(source: AsyncIterable<Uint8ArrayList>): Promise<ReqRespResponse> {
+  private async readMessage(source: AsyncIterable<Uint8ArrayList>, maxSizeKb?: number): Promise<ReqRespResponse> {
     let status: ReqRespStatus | undefined;
     const chunks: Uint8Array[] = [];
 
@@ -536,7 +546,7 @@ export class ReqResp implements ReqRespInterface {
       }
 
       const messageData = Buffer.concat(chunks);
-      const message: Buffer = this.snappyTransform.inboundTransformData(messageData);
+      const message: Buffer = this.snappyTransform.inboundTransformData(messageData, undefined, maxSizeKb);
 
       return {
         status: status ?? ReqRespStatus.UNKNOWN,
@@ -617,7 +627,9 @@ export class ReqResp implements ReqRespInterface {
         // and that this stream should be dropped
         const isMessageToNotWarn =
           err instanceof Error &&
-          ['stream reset', 'Cannot push value onto an ended pushable'].some(msg => err.message.includes(msg));
+          ['stream reset', 'Cannot push value onto an ended pushable', 'read ECONNRESET'].some(msg =>
+            err.message.includes(msg),
+          );
         const level = isMessageToNotWarn ? 'debug' : 'warn';
         this.logger[level]('Unknown stream error while handling the stream, aborting', {
           protocol,

package/src/services/service.ts

@@ -1,7 +1,13 @@
 import type { SlotNumber } from '@aztec/foundation/branded-types';
 import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { PeerInfo } from '@aztec/stdlib/interfaces/server';
-import type { BlockProposal, CheckpointAttestation, CheckpointProposalCore, Gossipable } from '@aztec/stdlib/p2p';
+import type {
+  BlockProposal,
+  CheckpointAttestation,
+  CheckpointProposalCore,
+  Gossipable,
+  TopicType,
+} from '@aztec/stdlib/p2p';
 import type { Tx } from '@aztec/stdlib/tx';
 
 import type { PeerId } from '@libp2p/interface';
@@ -57,6 +63,19 @@ export type DuplicateProposalInfo = {
  */
 export type P2PDuplicateProposalCallback = (info: DuplicateProposalInfo) => void;
 
+/** Minimal info passed to the duplicate attestation callback. */
+export type DuplicateAttestationInfo = {
+  slot: SlotNumber;
+  attester: EthAddress;
+};
+
+/**
+ * Callback for when a duplicate attestation is detected (equivocation).
+ * A validator signing attestations for different proposals at the same slot.
+ * Invoked on the first duplicate (when count goes from 1 to 2).
+ */
+export type P2PDuplicateAttestationCallback = (info: DuplicateAttestationInfo) => void;
+
 /**
  * The interface for a P2P service implementation.
  */
@@ -106,11 +125,21 @@ export interface P2PService {
    */
   registerDuplicateProposalCallback(callback: P2PDuplicateProposalCallback): void;
 
+  /**
+   * Registers a callback invoked when a duplicate attestation is detected (equivocation).
+   * A validator signing attestations for different proposals at the same slot.
+   * The callback is triggered on the first duplicate (when count goes from 1 to 2).
+   */
+  registerDuplicateAttestationCallback(callback: P2PDuplicateAttestationCallback): void;
+
   getEnr(): ENR | undefined;
 
   getPeers(includePending?: boolean): PeerInfo[];
 
-  validate(txs: Tx[]): Promise<void>;
+  /** Returns the number of peers in the GossipSub mesh for a given topic type. */
+  getGossipMeshPeerCount(topicType: TopicType): number;
+
+  validateTxsReceivedInBlockProposal(txs: Tx[]): Promise<void>;
 
   addReqRespSubProtocol(
     subProtocol: ReqRespSubProtocol,

package/src/services/tx_collection/config.ts

@@ -31,6 +31,24 @@ export type TxCollectionConfig = {
   txCollectionNodeRpcMaxBatchSize: number;
   /** Which collector implementation to use for missing txs collection */
   txCollectionMissingTxsCollectorType: MissingTxsCollectorType;
+  /** A comma-separated list of file store URLs (s3://, gs://, file://, http://) for tx collection */
+  txCollectionFileStoreUrls: string[];
+  /** Delay in ms before file store collection starts after slow collection is triggered */
+  txCollectionFileStoreSlowDelayMs: number;
+  /** Delay in ms before file store collection starts after fast collection is triggered */
+  txCollectionFileStoreFastDelayMs: number;
+  /** Number of concurrent workers for fast file store collection */
+  txCollectionFileStoreFastWorkerCount: number;
+  /** Number of concurrent workers for slow file store collection */
+  txCollectionFileStoreSlowWorkerCount: number;
+  /** Base backoff time in ms for fast file store collection retries */
+  txCollectionFileStoreFastBackoffBaseMs: number;
+  /** Base backoff time in ms for slow file store collection retries */
+  txCollectionFileStoreSlowBackoffBaseMs: number;
+  /** Max backoff time in ms for fast file store collection retries */
+  txCollectionFileStoreFastBackoffMaxMs: number;
+  /** Max backoff time in ms for slow file store collection retries */
+  txCollectionFileStoreSlowBackoffMaxMs: number;
 };
 
 export const txCollectionConfigMappings: ConfigMappingsType<TxCollectionConfig> = {
@@ -95,4 +113,54 @@ export const txCollectionConfigMappings: ConfigMappingsType<TxCollectionConfig>
     description: 'Which collector implementation to use for missing txs collection (new or old)',
     ...enumConfigHelper(['new', 'old'] as const, 'new'),
   },
+  txCollectionFileStoreUrls: {
+    env: 'TX_COLLECTION_FILE_STORE_URLS',
+    description: 'A comma-separated list of file store URLs (s3://, gs://, file://, http://) for tx collection',
+    parseEnv: (val: string) =>
+      val
+        .split(',')
+        .map(url => url.trim())
+        .filter(url => url.length > 0),
+    defaultValue: [],
+  },
+  txCollectionFileStoreSlowDelayMs: {
+    env: 'TX_COLLECTION_FILE_STORE_SLOW_DELAY_MS',
+    description: 'Delay before file store collection starts after slow collection',
+    ...numberConfigHelper(24_000),
+  },
+  txCollectionFileStoreFastDelayMs: {
+    env: 'TX_COLLECTION_FILE_STORE_FAST_DELAY_MS',
+    description: 'Delay before file store collection starts after fast collection',
+    ...numberConfigHelper(2_000),
+  },
+  txCollectionFileStoreFastWorkerCount: {
+    env: 'TX_COLLECTION_FILE_STORE_FAST_WORKER_COUNT',
+    description: 'Number of concurrent workers for fast file store collection',
+    ...numberConfigHelper(5),
+  },
+  txCollectionFileStoreSlowWorkerCount: {
+    env: 'TX_COLLECTION_FILE_STORE_SLOW_WORKER_COUNT',
+    description: 'Number of concurrent workers for slow file store collection',
+    ...numberConfigHelper(2),
+  },
+  txCollectionFileStoreFastBackoffBaseMs: {
+    env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_BASE_MS',
+    description: 'Base backoff time in ms for fast file store collection retries',
+    ...numberConfigHelper(1_000),
+  },
+  txCollectionFileStoreSlowBackoffBaseMs: {
+    env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_BASE_MS',
+    description: 'Base backoff time in ms for slow file store collection retries',
+    ...numberConfigHelper(5_000),
+  },
+  txCollectionFileStoreFastBackoffMaxMs: {
+    env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_MAX_MS',
+    description: 'Max backoff time in ms for fast file store collection retries',
+    ...numberConfigHelper(5_000),
+  },
+  txCollectionFileStoreSlowBackoffMaxMs: {
+    env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_MAX_MS',
+    description: 'Max backoff time in ms for slow file store collection retries',
+    ...numberConfigHelper(30_000),
+  },
 };

package/src/services/tx_collection/fast_tx_collection.ts

@@ -14,13 +14,14 @@ import type { PeerId } from '@libp2p/interface';
 import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
 import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
 import type { TxCollectionConfig } from './config.js';
+import { MissingTxsTracker } from './missing_txs_tracker.js';
 import {
   BatchTxRequesterCollector,
   type MissingTxsCollector,
   SendBatchRequestCollector,
 } from './proposal_tx_collector.js';
 import type { FastCollectionRequest, FastCollectionRequestInput } from './tx_collection.js';
-import type { TxCollectionSink } from './tx_collection_sink.js';
+import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
 import type { TxSource } from './tx_source.js';
 
 export class FastTxCollection {
@@ -77,29 +78,29 @@ export class FastTxCollection {
     // This promise is used to await for the collection to finish during the main collectFast method.
     // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
     const promise = promiseWithResolvers<void>();
-    setTimeout(() => promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
+    const timeoutTimer = setTimeout(() => promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
 
     const request: FastCollectionRequest = {
       ...input,
       blockInfo,
       promise,
-      foundTxs: new Map<string, Tx>(),
-      missingTxHashes: new Set(txHashes.map(t => t.toString())),
+      missingTxTracker: MissingTxsTracker.fromArray(txHashes),
       deadline: opts.deadline,
     };
 
     const [duration] = await elapsed(() => this.collectFast(request, { ...opts }));
+    clearTimeout(timeoutTimer);
 
     this.log.verbose(
-      `Collected ${request.foundTxs.size} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
+      `Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
       {
         ...blockInfo,
         duration,
         requestType: input.type,
-        missingTxs: [...request.missingTxHashes],
+        missingTxs: [...request.missingTxTracker.missingTxHashes],
       },
     );
-    return [...request.foundTxs.values()];
+    return request.missingTxTracker.collectedTxs;
   }
 
   protected async collectFast(
@@ -110,7 +111,7 @@ export class FastTxCollection {
     const { blockInfo } = request;
 
     this.log.debug(
-      `Starting fast collection of ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
+      `Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
       { ...blockInfo, requestType: request.type, deadline: opts.deadline },
     );
 
@@ -123,7 +124,7 @@ export class FastTxCollection {
       await Promise.race([request.promise.promise, waitBeforeReqResp]);
 
       // If we have collected all txs, we can stop here
-      if (request.missingTxHashes.size === 0) {
+      if (request.missingTxTracker.allFetched()) {
         this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
         return;
       }
@@ -137,7 +138,7 @@ export class FastTxCollection {
       const logCtx = {
         ...blockInfo,
         errorMessage: err instanceof Error ? err.message : undefined,
-        missingTxs: [...request.missingTxHashes].map(txHash => txHash.toString()),
+        missingTxs: request.missingTxTracker.missingTxHashes.values().map(txHash => txHash.toString()),
       };
       if (err instanceof Error && err.name === 'TimeoutError') {
         this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
@@ -165,7 +166,11 @@ export class FastTxCollection {
     }
 
     // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
-    const attemptsPerTx = [...request.missingTxHashes].map(txHash => ({ txHash, attempts: 0, found: false }));
+    const attemptsPerTx = [...request.missingTxTracker.missingTxHashes].map(txHash => ({
+      txHash,
+      attempts: 0,
+      found: false,
+    }));
 
     // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
     await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx, opts)));
@@ -178,7 +183,7 @@ export class FastTxCollection {
     opts: { deadline: Date },
   ) {
     const notFinished = () =>
-      this.dateProvider.now() <= +opts.deadline && request.missingTxHashes.size > 0 && this.requests.has(request);
+      this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
 
     const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
     const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
@@ -195,7 +200,7 @@ export class FastTxCollection {
           if (!txToRequest) {
             // No more txs to process
             break;
-          } else if (!request.missingTxHashes.has(txToRequest.txHash)) {
+          } else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
             // Mark as found if it was found somewhere else, we'll then remove it from the array.
             // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
             txToRequest.found = true;
@@ -224,16 +229,24 @@ export class FastTxCollection {
           return;
         }
 
+        const txHashes = batch.map(({ txHash }) => txHash);
         // Collect this batch from the node
         await this.txCollectionSink.collect(
-          txHashes => node.getTxsByHash(txHashes),
-          batch.map(({ txHash }) => TxHash.fromString(txHash)),
+          async () => {
+            const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
+            for (const tx of result.validTxs) {
+              request.missingTxTracker.markFetched(tx);
+            }
+            return result;
+          },
+          txHashes,
           {
             description: `fast ${node.getInfo()}`,
             node: node.getInfo(),
             method: 'fast-node-rpc',
             ...request.blockInfo,
           },
+          this.getAddContext(request),
         );
 
         // Clear from the active requests the txs we just requested
@@ -266,36 +279,58 @@ export class FastTxCollection {
     }
 
     this.log.debug(
-      `Starting fast reqresp for ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
+      `Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
       { ...blockInfo, timeoutMs, pinnedPeer },
     );
 
     try {
       await this.txCollectionSink.collect(
-        async txHashes => {
+        async () => {
+          let result: Tx[];
           if (request.type === 'proposal') {
-            return await this.missingTxsCollector.collectTxs(txHashes, request.blockProposal, pinnedPeer, timeoutMs);
+            result = await this.missingTxsCollector.collectTxs(
+              request.missingTxTracker,
+              request.blockProposal,
+              pinnedPeer,
+              timeoutMs,
+            );
           } else if (request.type === 'block') {
             const blockTxsSource = {
               txHashes: request.block.body.txEffects.map(e => e.txHash),
               archive: request.block.archive.root,
             };
-            return await this.missingTxsCollector.collectTxs(txHashes, blockTxsSource, pinnedPeer, timeoutMs);
+            result = await this.missingTxsCollector.collectTxs(
+              request.missingTxTracker,
+              blockTxsSource,
+              pinnedPeer,
+              timeoutMs,
+            );
           } else {
             throw new Error(`Unknown request type: ${(request as any).type}`);
           }
+          return { validTxs: result, invalidTxHashes: [] };
         },
-        Array.from(request.missingTxHashes).map(txHash => TxHash.fromString(txHash)),
+        Array.from(request.missingTxTracker.missingTxHashes),
         { description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo },
+        this.getAddContext(request),
       );
     } catch (err) {
       this.log.error(`Error sending fast reqresp request for txs`, err, {
-        txs: [...request.missingTxHashes],
+        txs: [...request.missingTxTracker.missingTxHashes],
         ...blockInfo,
       });
     }
   }
 
+  /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */
+  private getAddContext(request: FastCollectionRequest): TxAddContext {
+    if (request.type === 'proposal') {
+      return { type: 'proposal', blockHeader: request.blockProposal.blockHeader };
+    } else {
+      return { type: 'mined', block: request.block };
+    }
+  }
+
   /**
    * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
    * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
@@ -305,22 +340,20 @@ export class FastTxCollection {
       for (const tx of txs) {
         const txHash = tx.txHash.toString();
         // Remove the tx hash from the missing set, and add it to the found set.
-        if (request.missingTxHashes.has(txHash)) {
-          request.missingTxHashes.delete(txHash);
-          request.foundTxs.set(txHash, tx);
+        if (request.missingTxTracker.markFetched(tx)) {
           this.log.trace(`Found tx ${txHash} for fast collection request`, {
             ...request.blockInfo,
             txHash: tx.txHash.toString(),
             type: request.type,
           });
-          // If we found all txs for this request, we resolve the promise
-          if (request.missingTxHashes.size === 0) {
-            this.log.trace(`All txs found for fast collection request`, {
-              ...request.blockInfo,
-              type: request.type,
-            });
-            request.promise.resolve();
-          }
+        }
+        // If we found all txs for this request, we resolve the promise
+        if (request.missingTxTracker.allFetched()) {
+          this.log.trace(`All txs found for fast collection request`, {
+            ...request.blockInfo,
+            type: request.type,
+          });
+          request.promise.resolve();
         }
       }
     }

package/src/services/tx_collection/file_store_tx_collection.ts

@@ -0,0 +1,202 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
+import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider } from '@aztec/foundation/timer';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { FileStoreTxSource } from './file_store_tx_source.js';
+import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
+
+/** Configuration for a FileStoreTxCollection instance. */
+export type FileStoreCollectionConfig = {
+  workerCount: number;
+  backoffBaseMs: number;
+  backoffMaxMs: number;
+};
+
+type FileStoreTxEntry = {
+  txHash: string;
+  context: TxAddContext;
+  deadline: Date;
+  attempts: number;
+  lastAttemptTime: number;
+  nextSourceIndex: number;
+};
+
+/**
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
+ * Uses a shared worker pool that pulls entries with priority (fewest attempts first),
+ * retries with round-robin across sources, and applies exponential backoff between
+ * full cycles through all sources.
+ */
+export class FileStoreTxCollection {
+  /** Map from tx hash string to entry for all pending downloads. */
+  private entries = new Map<string, FileStoreTxEntry>();
+
+  /** Worker promises for the shared worker pool. */
+  private workers: Promise<void>[] = [];
+
+  /** Whether the worker pool is running. */
+  private running = false;
+
+  /** Signal used to wake sleeping workers when new entries arrive or stop is called. */
+  private wakeSignal: PromiseWithResolvers<void>;
+
+  constructor(
+    private readonly sources: FileStoreTxSource[],
+    private readonly txCollectionSink: TxCollectionSink,
+    private readonly config: FileStoreCollectionConfig,
+    private readonly dateProvider: DateProvider = new DateProvider(),
+    private readonly log: Logger = createLogger('p2p:file_store_tx_collection'),
+  ) {
+    this.wakeSignal = promiseWithResolvers<void>();
+  }
+
+  /** Starts the shared worker pool. */
+  public start(): void {
+    if (this.sources.length === 0) {
+      this.log.debug('No file store sources configured');
+      return;
+    }
+    this.running = true;
+    for (let i = 0; i < this.config.workerCount; i++) {
+      this.workers.push(this.workerLoop());
+    }
+  }
+
+  /** Stops all workers and clears state. */
+  public async stop(): Promise<void> {
+    this.running = false;
+    this.wake();
+    await Promise.all(this.workers);
+    this.workers = [];
+    this.entries.clear();
+  }
+
+  /** Adds entries to the shared map and wakes workers. */
+  public startCollecting(txHashes: TxHash[], context: TxAddContext, deadline: Date): void {
+    if (this.sources.length === 0 || txHashes.length === 0) {
+      return;
+    }
+    if (+deadline <= this.dateProvider.now()) {
+      return;
+    }
+
+    for (const txHash of txHashes) {
+      const hashStr = txHash.toString();
+      if (!this.entries.has(hashStr)) {
+        this.entries.set(hashStr, {
+          txHash: hashStr,
+          context,
+          deadline,
+          attempts: 0,
+          lastAttemptTime: 0,
+          nextSourceIndex: Math.floor(Math.random() * this.sources.length),
+        });
+      }
+    }
+    this.wake();
+  }
+
+  /** Removes entries for txs that have been found elsewhere. */
+  public foundTxs(txs: Tx[]): void {
+    for (const tx of txs) {
+      this.entries.delete(tx.getTxHash().toString());
+    }
+  }
+
+  /** Clears all pending entries. */
+  public clearPending(): void {
+    this.entries.clear();
+  }
+
+  private async workerLoop(): Promise<void> {
+    while (this.running) {
+      const action = this.getNextAction();
+      if (action.type === 'sleep') {
+        await action.promise;
+        continue;
+      }
+
+      const entry = action.entry;
+      const source = this.sources[entry.nextSourceIndex % this.sources.length];
+      entry.nextSourceIndex++;
+      entry.attempts++;
+      entry.lastAttemptTime = this.dateProvider.now();
+
+      try {
+        const result = await this.txCollectionSink.collect(
+          () => source.getTxsByHash([TxHash.fromString(entry.txHash)]),
+          [entry.txHash],
+          {
+            description: `file-store ${source.getInfo()}`,
+            method: 'file-store',
+            fileStore: source.getInfo(),
+          },
+          entry.context,
+        );
+        if (result.txs.length > 0) {
+          this.entries.delete(entry.txHash);
+        }
+      } catch (err) {
+        this.log.trace(`Error downloading tx ${entry.txHash} from ${source.getInfo()}`, { err });
+      }
+    }
+  }
+
+  /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */
+  private getNextAction(): { type: 'process'; entry: FileStoreTxEntry } | { type: 'sleep'; promise: Promise<void> } {
+    const now = this.dateProvider.now();
+    let best: FileStoreTxEntry | undefined;
+    let earliestReadyAt = Infinity;
+
+    for (const [key, entry] of this.entries) {
+      if (+entry.deadline <= now) {
+        this.entries.delete(key);
+        continue;
+      }
+      const backoffMs = this.getBackoffMs(entry);
+      const readyAt = entry.lastAttemptTime + backoffMs;
+      if (readyAt > now) {
+        earliestReadyAt = Math.min(earliestReadyAt, readyAt);
+        continue;
+      }
+      if (!best || entry.attempts < best.attempts) {
+        best = entry;
+      }
+    }
+
+    if (best) {
+      return { type: 'process', entry: best };
+    }
+    if (earliestReadyAt < Infinity) {
+      return { type: 'sleep', promise: this.sleepOrWake(earliestReadyAt - now) };
+    }
+    return { type: 'sleep', promise: this.waitForWake() };
+  }
+
+  /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */
+  private getBackoffMs(entry: FileStoreTxEntry): number {
+    const fullCycles = Math.floor(entry.attempts / this.sources.length);
+    if (fullCycles === 0) {
+      return 0;
+    }
+    return Math.min(this.config.backoffBaseMs * Math.pow(2, fullCycles - 1), this.config.backoffMaxMs);
+  }
+
+  /** Resolves the current wake signal and creates a new one. */
+  private wake(): void {
+    this.wakeSignal.resolve();
+    this.wakeSignal = promiseWithResolvers<void>();
+  }
+
+  /** Waits until the wake signal is resolved. */
+  private async waitForWake(): Promise<void> {
+    await this.wakeSignal.promise;
+  }
+
+  /** Sleeps for the given duration or until the wake signal is resolved. */
+  private async sleepOrWake(ms: number): Promise<void> {
+    await Promise.race([sleep(ms), this.wakeSignal.promise]);
+  }
+}