@aztec/p2p 0.0.1-commit.43c09e3f → 0.0.1-commit.4d79d1f2d

package/src/services/tx_collection/fast_tx_collection.ts

@@ -2,7 +2,6 @@ import { BlockNumber } from '@aztec/foundation/branded-types';
 import { times } from '@aztec/foundation/collection';
 import { AbortError, TimeoutError } from '@aztec/foundation/error';
 import { type Logger, createLogger } from '@aztec/foundation/log';
-import { boundInclusive } from '@aztec/foundation/number';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { sleep } from '@aztec/foundation/sleep';
 import { DateProvider, elapsed } from '@aztec/foundation/timer';
@@ -14,37 +13,35 @@ import type { PeerId } from '@libp2p/interface';
 
 import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
 import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
-import { ReqRespSubProtocol } from '../reqresp/interface.js';
-import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
 import type { TxCollectionConfig } from './config.js';
 import {
   BatchTxRequesterCollector,
-  type ProposalTxCollector,
+  type MissingTxsCollector,
   SendBatchRequestCollector,
 } from './proposal_tx_collector.js';
 import type { FastCollectionRequest, FastCollectionRequestInput } from './tx_collection.js';
-import type { TxCollectionSink } from './tx_collection_sink.js';
+import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
 import type { TxSource } from './tx_source.js';
 
 export class FastTxCollection {
   // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
   protected requests: Set<FastCollectionRequest> = new Set();
-  private proposalTxCollector: ProposalTxCollector;
+  private missingTxsCollector: MissingTxsCollector;
 
   constructor(
-    private p2pService: BatchTxRequesterLibP2PService,
+    p2pService: BatchTxRequesterLibP2PService,
     private nodes: TxSource[],
     private txCollectionSink: TxCollectionSink,
     private config: TxCollectionConfig,
     private dateProvider: DateProvider = new DateProvider(),
     private log: Logger = createLogger('p2p:tx_collection_service'),
-    proposalTxCollector?: ProposalTxCollector,
+    missingTxsCollector?: MissingTxsCollector,
   ) {
     const batchTxRequesterConfig = this.config as Partial<BatchTxRequesterConfig>;
-    const proposalCollectorType = this.config.txCollectionProposalTxCollectorType;
-    this.proposalTxCollector =
-      proposalTxCollector ??
-      (proposalCollectorType === 'old'
+    const missingTxsCollectorType = this.config.txCollectionMissingTxsCollectorType;
+    this.missingTxsCollector =
+      missingTxsCollector ??
+      (missingTxsCollectorType === 'old'
         ? new SendBatchRequestCollector(p2pService)
         : new BatchTxRequesterCollector(p2pService, log, dateProvider, undefined, batchTxRequesterConfig));
   }
@@ -80,7 +77,7 @@ export class FastTxCollection {
     // This promise is used to await for the collection to finish during the main collectFast method.
     // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
     const promise = promiseWithResolvers<void>();
-    setTimeout(() => promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
+    const timeoutTimer = setTimeout(() => promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
 
     const request: FastCollectionRequest = {
       ...input,
@@ -92,6 +89,7 @@ export class FastTxCollection {
     };
 
     const [duration] = await elapsed(() => this.collectFast(request, { ...opts }));
+    clearTimeout(timeoutTimer);
 
     this.log.verbose(
       `Collected ${request.foundTxs.size} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
@@ -237,6 +235,7 @@ export class FastTxCollection {
             method: 'fast-node-rpc',
             ...request.blockInfo,
           },
+          this.getAddContext(request),
         );
 
         // Clear from the active requests the txs we just requested
@@ -258,8 +257,6 @@ export class FastTxCollection {
   private async collectFastViaReqResp(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) {
     const timeoutMs = +request.deadline - this.dateProvider.now();
     const pinnedPeer = opts.pinnedPeer;
-    const maxPeers = boundInclusive(Math.ceil(request.missingTxHashes.size / 2), 8, 32);
-    const maxRetryAttempts = 5;
     const blockInfo = request.blockInfo;
     const slotNumber = blockInfo.slotNumber;
     if (timeoutMs < 100) {
@@ -279,24 +276,20 @@ export class FastTxCollection {
       await this.txCollectionSink.collect(
         async txHashes => {
           if (request.type === 'proposal') {
-            return await this.proposalTxCollector.collectTxs(txHashes, request.blockProposal, pinnedPeer, timeoutMs);
+            return await this.missingTxsCollector.collectTxs(txHashes, request.blockProposal, pinnedPeer, timeoutMs);
           } else if (request.type === 'block') {
-            const txs = await this.p2pService.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
-              ReqRespSubProtocol.TX,
-              chunkTxHashesRequest(txHashes),
-              pinnedPeer,
-              timeoutMs,
-              maxPeers,
-              maxRetryAttempts,
-            );
-
-            return txs.flat();
+            const blockTxsSource = {
+              txHashes: request.block.body.txEffects.map(e => e.txHash),
+              archive: request.block.archive.root,
+            };
+            return await this.missingTxsCollector.collectTxs(txHashes, blockTxsSource, pinnedPeer, timeoutMs);
           } else {
             throw new Error(`Unknown request type: ${(request as any).type}`);
           }
         },
         Array.from(request.missingTxHashes).map(txHash => TxHash.fromString(txHash)),
         { description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo },
+        this.getAddContext(request),
       );
     } catch (err) {
       this.log.error(`Error sending fast reqresp request for txs`, err, {
@@ -306,6 +299,15 @@ export class FastTxCollection {
     }
   }
 
+  /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */
+  private getAddContext(request: FastCollectionRequest): TxAddContext {
+    if (request.type === 'proposal') {
+      return { type: 'proposal', blockHeader: request.blockProposal.blockHeader };
+    } else {
+      return { type: 'mined', block: request.block };
+    }
+  }
+
   /**
    * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
    * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.

package/src/services/tx_collection/file_store_tx_collection.ts

@@ -0,0 +1,152 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { FifoMemoryQueue } from '@aztec/foundation/queue';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+
+import type { FileStoreTxSource } from './file_store_tx_source.js';
+import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
+
+// Internal constants (not configurable by node operators)
+const FILE_STORE_DOWNLOAD_CONCURRENCY = 5; // Max concurrent downloads
+
+/**
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
+ * Runs in parallel to slow/fast collection. The delay before starting file store
+ * collection is managed by the TxCollection orchestrator, not this class.
+ */
+export class FileStoreTxCollection {
+  /** Map from tx hash to add context for txs queued for download. */
+  private pendingTxs = new Map<string, TxAddContext>();
+
+  /**
+   * Tracks tx hashes found elsewhere, even before startCollecting is called.
+   * Needed because the orchestrator delays startCollecting via a real sleep, but foundTxs
+   * may arrive during that delay — before the hashes are added to pendingTxs.
+   */
+  private foundTxHashes = new Set<string>();
+
+  /** Queue of tx hashes to be downloaded. */
+  private downloadQueue = new FifoMemoryQueue<TxHash>();
+
+  /** Worker promises for concurrent downloads. */
+  private workers: Promise<void>[] = [];
+
+  /** Whether the collection has been started. */
+  private started = false;
+
+  constructor(
+    private readonly fileStoreSources: FileStoreTxSource[],
+    private readonly txCollectionSink: TxCollectionSink,
+    private readonly log: Logger = createLogger('p2p:file_store_tx_collection'),
+  ) {}
+
+  /** Starts the file store collection workers. */
+  public start() {
+    if (this.fileStoreSources.length === 0) {
+      this.log.debug('No file store sources configured, skipping file store collection');
+      return;
+    }
+
+    this.started = true;
+    this.downloadQueue = new FifoMemoryQueue<TxHash>();
+
+    // Start concurrent download workers
+    for (let i = 0; i < FILE_STORE_DOWNLOAD_CONCURRENCY; i++) {
+      this.workers.push(this.downloadQueue.process(txHash => this.processDownload(txHash)));
+    }
+
+    this.log.info(`Started file store tx collection with ${this.fileStoreSources.length} sources`, {
+      sources: this.fileStoreSources.map(s => s.getInfo()),
+      concurrency: FILE_STORE_DOWNLOAD_CONCURRENCY,
+    });
+  }
+
+  /** Stops all collection activity. */
+  public async stop() {
+    if (!this.started) {
+      return;
+    }
+    this.started = false;
+    this.downloadQueue.end();
+    await Promise.all(this.workers);
+    this.workers = [];
+    this.pendingTxs.clear();
+    this.foundTxHashes.clear();
+  }
+
+  /** Remove the given tx hashes from pending. */
+  public stopCollecting(txHashes: TxHash[]) {
+    for (const txHash of txHashes) {
+      const hashStr = txHash.toString();
+      this.pendingTxs.delete(hashStr);
+    }
+  }
+
+  /** Clears all pending state. Items already in the download queue will still be processed but won't be re-queued. */
+  public clearPending() {
+    this.pendingTxs.clear();
+    this.foundTxHashes.clear();
+  }
+
+  /** Queue the given tx hashes for file store collection. */
+  public startCollecting(txHashes: TxHash[], context: TxAddContext) {
+    for (const txHash of txHashes) {
+      const hashStr = txHash.toString();
+      if (!this.pendingTxs.has(hashStr) && !this.foundTxHashes.has(hashStr)) {
+        this.pendingTxs.set(hashStr, context);
+        this.downloadQueue.put(txHash);
+      }
+    }
+  }
+
+  /** Stop tracking txs that were found elsewhere. */
+  public foundTxs(txs: Tx[]) {
+    for (const tx of txs) {
+      const hashStr = tx.getTxHash().toString();
+      this.pendingTxs.delete(hashStr);
+      this.foundTxHashes.add(hashStr);
+    }
+  }
+
+  /** Processes a single tx hash from the download queue. */
+  private async processDownload(txHash: TxHash) {
+    const hashStr = txHash.toString();
+    const context = this.pendingTxs.get(hashStr);
+
+    // Skip if already found by another method
+    if (!context) {
+      return;
+    }
+
+    await this.downloadTx(txHash, context);
+    this.pendingTxs.delete(hashStr);
+  }
+
+  /** Attempt to download a tx from file stores (round-robin). */
+  private async downloadTx(txHash: TxHash, context: TxAddContext) {
+    const startIndex = Math.floor(Math.random() * this.fileStoreSources.length);
+    for (let i = startIndex; i < startIndex + this.fileStoreSources.length; i++) {
+      const source = this.fileStoreSources[i % this.fileStoreSources.length];
+
+      try {
+        const result = await this.txCollectionSink.collect(
+          hashes => source.getTxsByHash(hashes),
+          [txHash],
+          {
+            description: `file-store ${source.getInfo()}`,
+            method: 'file-store',
+            fileStore: source.getInfo(),
+          },
+          context,
+        );
+
+        if (result.txs.length > 0) {
+          return;
+        }
+      } catch (err) {
+        this.log.trace(`Failed to download tx ${txHash} from ${source.getInfo()}`, { err });
+      }
+    }
+
+    this.log.trace(`Tx ${txHash} not found in any file store`);
+  }
+}

package/src/services/tx_collection/file_store_tx_source.ts

@@ -0,0 +1,70 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store';
+import { Tx, type TxHash } from '@aztec/stdlib/tx';
+
+import type { TxSource } from './tx_source.js';
+
+/** TxSource implementation that downloads txs from a file store. */
+export class FileStoreTxSource implements TxSource {
+  private constructor(
+    private readonly fileStore: ReadOnlyFileStore,
+    private readonly baseUrl: string,
+    private readonly log: Logger,
+  ) {}
+
+  /**
+   * Creates a FileStoreTxSource from a URL.
+   * @param url - The file store URL (s3://, gs://, file://, http://, https://).
+   * @param log - Optional logger.
+   * @returns The FileStoreTxSource instance, or undefined if creation fails.
+   */
+  public static async create(
+    url: string,
+    log: Logger = createLogger('p2p:file_store_tx_source'),
+  ): Promise<FileStoreTxSource | undefined> {
+    try {
+      const fileStore = await createReadOnlyFileStore(url, log);
+      if (!fileStore) {
+        log.warn(`Failed to create file store for URL: ${url}`);
+        return undefined;
+      }
+      return new FileStoreTxSource(fileStore, url, log);
+    } catch (err) {
+      log.warn(`Error creating file store for URL: ${url}`, { error: err });
+      return undefined;
+    }
+  }
+
+  public getInfo(): string {
+    return `file-store:${this.baseUrl}`;
+  }
+
+  public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return Promise.all(
+      txHashes.map(async txHash => {
+        const path = `txs/${txHash.toString()}.bin`;
+        try {
+          const buffer = await this.fileStore.read(path);
+          return Tx.fromBuffer(buffer);
+        } catch {
+          // Tx not found or error reading - return undefined
+          return undefined;
+        }
+      }),
+    );
+  }
+}
+
+/**
+ * Creates FileStoreTxSource instances from URLs.
+ * @param urls - Array of file store URLs.
+ * @param log - Optional logger.
+ * @returns Array of successfully created FileStoreTxSource instances.
+ */
+export async function createFileStoreTxSources(
+  urls: string[],
+  log: Logger = createLogger('p2p:file_store_tx_source'),
+): Promise<FileStoreTxSource[]> {
+  const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, log)));
+  return sources.filter((s): s is FileStoreTxSource => s !== undefined);
+}

package/src/services/tx_collection/index.ts

@@ -1,7 +1,8 @@
 export { TxCollection, type FastCollectionRequestInput } from './tx_collection.js';
 export { type TxSource, createNodeRpcTxSources, NodeRpcTxSource } from './tx_source.js';
 export {
-  type ProposalTxCollector,
+  type MissingTxsCollector,
   BatchTxRequesterCollector,
   SendBatchRequestCollector,
 } from './proposal_tx_collector.js';
+export { FileStoreTxSource, createFileStoreTxSources } from './file_store_tx_source.js';

package/src/services/tx_collection/proposal_tx_collector.ts

@@ -1,6 +1,5 @@
 import type { Logger } from '@aztec/foundation/log';
 import type { DateProvider } from '@aztec/foundation/timer';
-import type { BlockProposal } from '@aztec/stdlib/p2p';
 import type { Tx, TxHash } from '@aztec/stdlib/tx';
 
 import type { PeerId } from '@libp2p/interface';
@@ -9,25 +8,24 @@ import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_request
 import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
 import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
 import type { IBatchRequestTxValidator } from '../reqresp/batch-tx-requester/tx_validator.js';
-import { ReqRespSubProtocol } from '../reqresp/interface.js';
-import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
+import { type BlockTxsSource, ReqRespSubProtocol, chunkTxHashesRequest } from '../reqresp/index.js';
 
 /**
- * Strategy interface for collecting transactions for block proposals.
+ * Strategy interface for collecting missing transactions for a block or proposal.
  * Allows swapping between different tx collection implementations for benchmarking.
  */
-export interface ProposalTxCollector {
+export interface MissingTxsCollector {
   /**
-   * Collect transactions for a block proposal.
+   * Collect missing transactions for a block or proposal.
    * @param txHashes - The transaction hashes to collect
-   * @param blockProposal - The block proposal containing the transactions
-   * @param pinnedPeer - Optional peer that sent the proposal (expected to have all txs)
+   * @param blockTxsSource - The block or proposal containing the transactions
+   * @param pinnedPeer - Optional peer expected to have the transactions
    * @param timeoutMs - Timeout in milliseconds
    * @returns The collected transactions
    */
   collectTxs(
     txHashes: TxHash[],
-    blockProposal: BlockProposal,
+    blockTxsSource: BlockTxsSource,
     pinnedPeer: PeerId | undefined,
     timeoutMs: number,
   ): Promise<Tx[]>;
@@ -37,7 +35,7 @@ export interface ProposalTxCollector {
  * Collects transactions using the BatchTxRequester implementation.
  * This uses a smart/dumb peer strategy with parallel workers.
  */
-export class BatchTxRequesterCollector implements ProposalTxCollector {
+export class BatchTxRequesterCollector implements MissingTxsCollector {
   constructor(
     private p2pService: BatchTxRequesterLibP2PService,
     private log: Logger,
@@ -48,7 +46,7 @@ export class BatchTxRequesterCollector implements ProposalTxCollector {
 
   async collectTxs(
     txHashes: TxHash[],
-    blockProposal: BlockProposal,
+    blockTxsSource: BlockTxsSource,
     pinnedPeer: PeerId | undefined,
     timeoutMs: number,
   ): Promise<Tx[]> {
@@ -61,7 +59,7 @@ export class BatchTxRequesterCollector implements ProposalTxCollector {
 
     const batchRequester = new BatchTxRequester(
       txHashes,
-      blockProposal,
+      blockTxsSource,
       pinnedPeer,
       timeoutMs,
       this.p2pService,
@@ -87,7 +85,7 @@ const DEFAULT_MAX_RETRY_ATTEMPTS = 3;
  * Collects transactions using the sendBatchRequest implementation from ReqResp.
  * This is the original implementation that balances requests across peers.
  */
-export class SendBatchRequestCollector implements ProposalTxCollector {
+export class SendBatchRequestCollector implements MissingTxsCollector {
   constructor(
     private p2pService: BatchTxRequesterLibP2PService,
     private maxPeers: number = DEFAULT_MAX_PEERS,
@@ -96,7 +94,7 @@ export class SendBatchRequestCollector implements ProposalTxCollector {
 
   async collectTxs(
     txHashes: TxHash[],
-    _blockProposal: BlockProposal,
+    _blockTxsSource: BlockTxsSource,
     pinnedPeer: PeerId | undefined,
     timeoutMs: number,
   ): Promise<Tx[]> {

package/src/services/tx_collection/slow_tx_collection.ts

@@ -91,6 +91,7 @@ export class SlowTxCollection {
 
     for (const txHash of txHashes) {
       this.missingTxs.set(txHash.toString(), {
+        block,
         blockNumber: block.number,
         deadline: this.getDeadlineForSlot(block.header.getSlot()),
         readyForReqResp: this.nodes.length === 0, // If we have no nodes, we can start reqresp immediately
@@ -109,18 +110,26 @@ export class SlowTxCollection {
 
     // Gather all missing txs that are not in fast collection and request them from the node
     const missingTxs = this.getMissingTxsForSlowCollection();
-    const missingTxHashes = missingTxs.map(([txHash]) => txHash).map(TxHash.fromString);
-    if (missingTxHashes.length === 0) {
+    if (missingTxs.length === 0) {
       return;
     }
 
-    // Request in chunks to avoid hitting RPC limits
-    for (const batch of chunk(missingTxHashes, this.config.txCollectionNodeRpcMaxBatchSize)) {
-      await this.txCollectionSink.collect(txHashes => node.getTxsByHash(txHashes), batch, {
-        description: `node ${node.getInfo()}`,
-        node: node.getInfo(),
-        method: 'slow-node-rpc',
-      });
+    // Group by block so we pass the correct mined context to the sink
+    for (const entries of this.groupByBlock(missingTxs)) {
+      const block = entries[0][1].block;
+      const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
+      for (const batch of chunk(txHashes, this.config.txCollectionNodeRpcMaxBatchSize)) {
+        await this.txCollectionSink.collect(
+          hashes => node.getTxsByHash(hashes),
+          batch,
+          {
+            description: `node ${node.getInfo()}`,
+            node: node.getInfo(),
+            method: 'slow-node-rpc',
+          },
+          { type: 'mined', block },
+        );
+      }
     }
 
     // Mark every tx that is still missing as ready for reqresp.
@@ -149,25 +158,30 @@ export class SlowTxCollection {
 
     const pinnedPeer = undefined;
     const timeoutMs = this.config.txCollectionSlowReqRespTimeoutMs;
-    const maxPeers = boundInclusive(Math.ceil(missingTxs.length / 3), 4, 16);
     const maxRetryAttempts = 3;
-    // Send a batch request via reqresp for the missing txs
-    await this.txCollectionSink.collect(
-      async txHashes => {
-        const txs = await this.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
-          ReqRespSubProtocol.TX,
-          chunkTxHashesRequest(txHashes),
-          pinnedPeer,
-          timeoutMs,
-          maxPeers,
-          maxRetryAttempts,
-        );
 
-        return txs.flat();
-      },
-      missingTxs.map(([txHash]) => TxHash.fromString(txHash)),
-      { description: 'slow reqresp', timeoutMs, method: 'slow-req-resp' },
-    );
+    // Group by block so we pass the correct mined context to the sink
+    for (const entries of this.groupByBlock(missingTxs)) {
+      const block = entries[0][1].block;
+      const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
+      const maxPeers = boundInclusive(Math.ceil(txHashes.length / 3), 4, 16);
+      await this.txCollectionSink.collect(
+        async hashes => {
+          const txs = await this.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
+            ReqRespSubProtocol.TX,
+            chunkTxHashesRequest(hashes),
+            pinnedPeer,
+            timeoutMs,
+            maxPeers,
+            maxRetryAttempts,
+          );
+          return txs.flat();
+        },
+        txHashes,
+        { description: 'slow reqresp', timeoutMs, method: 'slow-req-resp' },
+        { type: 'mined', block },
+      );
+    }
   }
 
   /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */
@@ -223,6 +237,21 @@ export class SlowTxCollection {
     }
   }
 
+  /** Groups missing tx entries by block number. */
+  private groupByBlock(entries: [string, MissingTxInfo][]): [string, MissingTxInfo][][] {
+    const groups = new Map<number, [string, MissingTxInfo][]>();
+    for (const entry of entries) {
+      const bn = +entry[1].blockNumber;
+      let group = groups.get(bn);
+      if (!group) {
+        group = [];
+        groups.set(bn, group);
+      }
+      group.push(entry);
+    }
+    return [...groups.values()];
+  }
+
   /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */
   private getDeadlineForSlot(slotNumber: SlotNumber): Date {
     const epoch = getEpochAtSlot(slotNumber, this.constants);