@aztec/p2p 0.0.1-commit.c7c42ec → 0.0.1-commit.c949de6bc

package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts

@@ -0,0 +1,444 @@
+import { SlotNumber } from '@aztec/foundation/branded-types';
+import type { L2BlockId } from '@aztec/stdlib/block';
+
+import { type TxMetaData, type TxState, compareFee, compareTxHash, txHashFromBigInt } from './tx_metadata.js';
+
+/**
+ * Manages in-memory indices for the transaction pool.
+ *
+ * Tracks transaction metadata and maintains several indices for efficient querying:
+ * - Metadata by txHash (primary store)
+ * - Nullifier to txHash mapping (pending txs only)
+ * - Fee payer to txHashes mapping (pending txs only)
+ * - Priority-ordered pending txs
+ * - Protected transaction tracking
+ *
+ * Key invariant: Only pending txs appear in nullifier/feePayer/priority indices.
+ */
+export class TxPoolIndices {
+  /** Primary metadata store: txHash -> TxMetaData */
+  #metadata: Map<string, TxMetaData> = new Map();
+  /** Nullifier to txHash index (pending txs only) */
+  #nullifierToTxHash: Map<string, string> = new Map();
+  /** Fee payer to txHashes index (pending txs only) */
+  #feePayerToTxHashes: Map<string, Set<string>> = new Map();
+  /** Pending txHash bigints grouped by priority fee */
+  #pendingByPriority: Map<bigint, Set<bigint>> = new Map();
+  /** Protected transactions: txHash -> slotNumber */
+  #protectedTransactions: Map<string, SlotNumber> = new Map();
+
+  // ============================================================================
+  // STATE QUERIES
+  // ============================================================================
+
+  /**
+   * Derives the transaction state from its metadata and protection status.
+   * A transaction is:
+   * - 'mined' if it has a minedL2BlockId
+   * - 'protected' if it's in the protectedTransactions map (but not mined)
+   * - 'pending' otherwise
+   */
+  getTxState(meta: TxMetaData): TxState {
+    if (meta.minedL2BlockId !== undefined) {
+      return 'mined';
+    } else if (this.#protectedTransactions.has(meta.txHash)) {
+      return 'protected';
+    } else {
+      return 'pending';
+    }
+  }
+
+  getMetadata(txHash: string): TxMetaData | undefined {
+    return this.#metadata.get(txHash);
+  }
+
+  has(txHash: string): boolean {
+    return this.#metadata.has(txHash);
+  }
+
+  isEmpty(): boolean {
+    return this.#metadata.size === 0;
+  }
+
+  getTxCount(): number {
+    return this.#metadata.size;
+  }
+
+  // ============================================================================
+  // ITERATION
+  // ============================================================================
+
+  /**
+   * Iterates pending transaction hashes in priority order.
+   * @param order - 'desc' for highest priority first, 'asc' for lowest priority first
+   */
+  *iteratePendingByPriority(order: 'asc' | 'desc', filter?: (hash: string) => boolean): Generator<string> {
+    const feeCompareFn = order === 'desc' ? (a: bigint, b: bigint) => compareFee(b, a) : compareFee;
+    const hashCompareFn =
+      order === 'desc' ? (a: bigint, b: bigint) => compareTxHash(b, a) : (a: bigint, b: bigint) => compareTxHash(a, b);
+
+    const sortedFees = [...this.#pendingByPriority.keys()].sort(feeCompareFn);
+
+    for (const fee of sortedFees) {
+      const hashesAtFee = this.#pendingByPriority.get(fee)!;
+      const sortedHashes = [...hashesAtFee].sort(hashCompareFn);
+      for (const hashBigInt of sortedHashes) {
+        const hash = txHashFromBigInt(hashBigInt);
+        if (filter === undefined || filter(hash)) {
+          yield hash;
+        }
+      }
+    }
+  }
+
+  /**
+   * Iterates pending transaction hashes in priority order, skipping txs received after maxReceivedAt.
+   * @param order - 'desc' for highest priority first, 'asc' for lowest priority first
+   * @param maxReceivedAt - Only yield txs with receivedAt <= this value
+   */
+  *iterateEligiblePendingByPriority(order: 'asc' | 'desc', maxReceivedAt: number): Generator<string> {
+    const filter = (hash: string) => {
+      const meta = this.#metadata.get(hash);
+      return meta !== undefined && meta.receivedAt <= maxReceivedAt;
+    };
+
+    yield* this.iteratePendingByPriority(order, filter);
+  }
+
+  /** Iterates all metadata entries */
+  *iterateMetadata(): Generator<[string, TxMetaData]> {
+    yield* this.#metadata;
+  }
+
+  // ============================================================================
+  // INDEX MODIFICATIONS
+  // ============================================================================
+
+  /** Adds a new pending transaction to all indices */
+  addPending(meta: TxMetaData): void {
+    this.#metadata.set(meta.txHash, meta);
+    this.#addToPendingIndices(meta);
+  }
+
+  /** Adds a new protected transaction (not added to pending indices) */
+  addProtected(meta: TxMetaData, slot: SlotNumber): void {
+    this.#metadata.set(meta.txHash, meta);
+    this.#protectedTransactions.set(meta.txHash, slot);
+  }
+
+  /** Adds a new mined transaction (not added to pending indices) */
+  addMined(meta: TxMetaData): void {
+    this.#metadata.set(meta.txHash, meta);
+  }
+
+  /** Marks an existing transaction as mined and removes from pending indices */
+  markAsMined(meta: TxMetaData, blockId: L2BlockId): void {
+    meta.minedL2BlockId = blockId;
+    // Safe to call unconditionally - removeFromPendingIndices is idempotent
+    this.#removeFromPendingIndices(meta);
+  }
+
+  /** Clears the mined status from a transaction */
+  markAsUnmined(meta: TxMetaData): void {
+    meta.minedL2BlockId = undefined;
+  }
+
+  /**
+   * Updates protection status for an existing transaction.
+   * Removes from pending indices if transitioning from pending to protected.
+   */
+  updateProtection(txHash: string, slotNumber: SlotNumber): void {
+    const currentSlot = this.#protectedTransactions.get(txHash);
+
+    // Only update if not already protected at an equal or later slot
+    if (currentSlot !== undefined && currentSlot >= slotNumber) {
+      return;
+    }
+
+    // Remove from pending indices if transitioning from pending to protected
+    if (currentSlot === undefined) {
+      const meta = this.#metadata.get(txHash);
+      if (meta) {
+        this.#removeFromPendingIndices(meta);
+      }
+    }
+
+    this.#protectedTransactions.set(txHash, slotNumber);
+  }
+
+  /** Sets protection for a txHash that may not have metadata yet */
+  setProtection(txHash: string, slotNumber: SlotNumber): void {
+    this.#protectedTransactions.set(txHash, slotNumber);
+  }
+
+  /** Gets the protection slot for a txHash, if protected */
+  getProtectionSlot(txHash: string): SlotNumber | undefined {
+    return this.#protectedTransactions.get(txHash);
+  }
+
+  /** Removes protection from tx hashes */
+  clearProtection(txHashes: string[]): void {
+    for (const txHash of txHashes) {
+      this.#protectedTransactions.delete(txHash);
+    }
+  }
+
+  /** Removes a transaction from all indices */
+  remove(txHash: string): void {
+    const meta = this.#metadata.get(txHash);
+    if (!meta) {
+      return;
+    }
+
+    this.#metadata.delete(txHash);
+    this.#protectedTransactions.delete(txHash);
+    this.#removeFromPendingIndices(meta);
+  }
+
+  /** Removes a transaction from pending indices only (not metadata) */
+  removeFromPendingIndices(meta: TxMetaData): void {
+    this.#removeFromPendingIndices(meta);
+  }
+
+  /** Adds a transaction to pending indices (used during conflict resolution) */
+  addToPendingIndices(meta: TxMetaData): void {
+    this.#addToPendingIndices(meta);
+  }
+
+  // ============================================================================
+  // QUERIES FOR EVICTION RULES
+  // ============================================================================
+
+  /** Gets all pending transactions for a given fee payer */
+  getFeePayerPendingTxs(feePayer: string): TxMetaData[] {
+    const txHashes = this.#feePayerToTxHashes.get(feePayer);
+    if (!txHashes) {
+      return [];
+    }
+    const result: TxMetaData[] = [];
+    for (const txHash of txHashes) {
+      const meta = this.#metadata.get(txHash);
+      if (meta && this.getTxState(meta) === 'pending') {
+        result.push(meta);
+      }
+    }
+    return result;
+  }
+
+  /** Gets the count of pending transactions */
+  getPendingTxCount(): number {
+    let count = 0;
+    for (const hashes of this.#pendingByPriority.values()) {
+      count += hashes.size;
+    }
+    return count;
+  }
+
+  /** Gets the lowest priority pending transaction hashes (up to limit) */
+  getLowestPriorityPending(limit: number): string[] {
+    if (limit <= 0) {
+      return [];
+    }
+
+    const result: string[] = [];
+    for (const hash of this.iteratePendingByPriority('asc')) {
+      result.push(hash);
+      if (result.length >= limit) {
+        break;
+      }
+    }
+    return result;
+  }
+
+  /** Gets the lowest priority pending transaction */
+  getLowestPriorityPendingTx(): TxMetaData | undefined {
+    for (const txHash of this.iteratePendingByPriority('asc')) {
+      const meta = this.#metadata.get(txHash);
+      if (meta) {
+        return meta;
+      }
+    }
+    return undefined;
+  }
+
+  /** Gets all pending transactions */
+  getPendingTxs(): TxMetaData[] {
+    const result: TxMetaData[] = [];
+    for (const hashSet of this.#pendingByPriority.values()) {
+      for (const txHashBigInt of hashSet) {
+        const meta = this.#metadata.get(txHashFromBigInt(txHashBigInt));
+        if (meta) {
+          result.push(meta);
+        }
+      }
+    }
+    return result;
+  }
+
+  /** Gets all fee payers with pending transactions */
+  getPendingFeePayers(): string[] {
+    return Array.from(this.#feePayerToTxHashes.keys());
+  }
+
+  /** Gets the txHash that uses a given nullifier (pending txs only) */
+  getTxHashByNullifier(nullifier: string): string | undefined {
+    return this.#nullifierToTxHash.get(nullifier);
+  }
+
+  /** Gets txHashes for a fee payer */
+  getTxHashesByFeePayer(feePayer: string): Set<string> | undefined {
+    return this.#feePayerToTxHashes.get(feePayer);
+  }
+
+  // ============================================================================
+  // FIND/FILTER OPERATIONS
+  // ============================================================================
+
+  /** Finds all transactions mined in blocks after the given block number */
+  findTxsMinedAfter(blockNumber: number): TxMetaData[] {
+    const result: TxMetaData[] = [];
+    for (const meta of this.#metadata.values()) {
+      if (meta.minedL2BlockId !== undefined && meta.minedL2BlockId.number > blockNumber) {
+        result.push(meta);
+      }
+    }
+    return result;
+  }
+
+  /** Finds tx hashes mined at or before the given block number */
+  findTxsMinedAtOrBefore(blockNumber: number): string[] {
+    const result: string[] = [];
+    for (const [txHash, meta] of this.#metadata) {
+      if (meta.minedL2BlockId !== undefined && meta.minedL2BlockId.number <= blockNumber) {
+        result.push(txHash);
+      }
+    }
+    return result;
+  }
+
+  /** Finds protected tx hashes from slots earlier than the given slot number */
+  findExpiredProtectedTxs(slotNumber: SlotNumber): string[] {
+    const result: string[] = [];
+    for (const [txHash, protectedSlot] of this.#protectedTransactions) {
+      if (protectedSlot < slotNumber) {
+        result.push(txHash);
+      }
+    }
+    return result;
+  }
+
+  /** Filters out transactions that are currently protected */
+  filterUnprotected(txs: TxMetaData[]): TxMetaData[] {
+    return txs.filter(meta => !this.#protectedTransactions.has(meta.txHash));
+  }
+
+  /** Filters to transactions that have metadata and are not mined */
+  filterRestorable(txHashes: string[]): TxMetaData[] {
+    const result: TxMetaData[] = [];
+    for (const txHash of txHashes) {
+      const meta = this.#metadata.get(txHash);
+      if (meta && meta.minedL2BlockId === undefined) {
+        result.push(meta);
+      }
+    }
+    return result;
+  }
+
+  // ============================================================================
+  // METRICS
+  // ============================================================================
+
+  /** Counts transactions by state and estimates total metadata memory usage */
+  countTxs(): { pending: number; protected: number; mined: number; totalMetadataBytes: number } {
+    let pending = 0;
+    let protected_ = 0;
+    let mined = 0;
+    let totalMetadataBytes = 0;
+
+    for (const meta of this.#metadata.values()) {
+      totalMetadataBytes += meta.estimatedSizeBytes;
+      const state = this.getTxState(meta);
+      if (state === 'pending') {
+        pending++;
+      } else if (state === 'protected') {
+        protected_++;
+      } else if (state === 'mined') {
+        mined++;
+      }
+    }
+
+    return { pending, protected: protected_, mined, totalMetadataBytes };
+  }
+
+  /** Returns the estimated total memory consumed by all metadata objects */
+  getTotalMetadataBytes(): number {
+    let total = 0;
+    for (const meta of this.#metadata.values()) {
+      total += meta.estimatedSizeBytes;
+    }
+    return total;
+  }
+
+  /** Gets all mined transactions with their block IDs */
+  getMinedTxs(): [string, L2BlockId][] {
+    const result: [string, L2BlockId][] = [];
+    for (const [txHash, meta] of this.#metadata) {
+      if (meta.minedL2BlockId !== undefined) {
+        result.push([txHash, meta.minedL2BlockId]);
+      }
+    }
+    return result;
+  }
+
+  // ============================================================================
+  // PRIVATE HELPERS
+  // ============================================================================
+
+  #addToPendingIndices(meta: TxMetaData): void {
+    // Add to nullifier index
+    for (const nullifier of meta.nullifiers) {
+      this.#nullifierToTxHash.set(nullifier, meta.txHash);
+    }
+
+    // Add to fee payer index
+    let feePayerSet = this.#feePayerToTxHashes.get(meta.feePayer);
+    if (!feePayerSet) {
+      feePayerSet = new Set();
+      this.#feePayerToTxHashes.set(meta.feePayer, feePayerSet);
+    }
+    feePayerSet.add(meta.txHash);
+
+    // Add to priority bucket
+    let prioritySet = this.#pendingByPriority.get(meta.priorityFee);
+    if (!prioritySet) {
+      prioritySet = new Set();
+      this.#pendingByPriority.set(meta.priorityFee, prioritySet);
+    }
+    prioritySet.add(meta.txHashBigInt);
+  }
+
+  #removeFromPendingIndices(meta: TxMetaData): void {
+    // Remove from nullifier index
+    for (const nullifier of meta.nullifiers) {
+      this.#nullifierToTxHash.delete(nullifier);
+    }
+
+    // Remove from fee payer index
+    const feePayerSet = this.#feePayerToTxHashes.get(meta.feePayer);
+    if (feePayerSet) {
+      feePayerSet.delete(meta.txHash);
+      if (feePayerSet.size === 0) {
+        this.#feePayerToTxHashes.delete(meta.feePayer);
+      }
+    }
+
+    // Remove from priority map
+    const hashSet = this.#pendingByPriority.get(meta.priorityFee);
+    if (hashSet) {
+      hashSet.delete(meta.txHashBigInt);
+      if (hashSet.size === 0) {
+        this.#pendingByPriority.delete(meta.priorityFee);
+      }
+    }
+  }
+}

package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts

@@ -0,0 +1,223 @@
+import { SlotNumber } from '@aztec/foundation/branded-types';
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
+import { DateProvider } from '@aztec/foundation/timer';
+import type { TypedEventEmitter } from '@aztec/foundation/types';
+import type { AztecAsyncKVStore } from '@aztec/kv-store';
+import type { L2Block, L2BlockId } from '@aztec/stdlib/block';
+import { BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx';
+import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
+
+import EventEmitter from 'node:events';
+
+import { PoolInstrumentation, PoolName } from '../instrumentation.js';
+import type { AddTxsResult, TxPoolV2, TxPoolV2Config, TxPoolV2Dependencies, TxPoolV2Events } from './interfaces.js';
+import type { TxState } from './tx_metadata.js';
+import { TxPoolV2Impl } from './tx_pool_v2_impl.js';
+
+/**
+ * Implementation of TxPoolV2 with explicit state management.
+ *
+ * This class is a thin wrapper that manages the serial queue and delegates
+ * all operations to TxPoolV2Impl.
+ */
+export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitter<TxPoolV2Events>) implements TxPoolV2 {
+  #queue: SerialQueue;
+  #impl: TxPoolV2Impl;
+  #metrics?: PoolInstrumentation<Tx>;
+  #store: AztecAsyncKVStore;
+  #telemetry: TelemetryClient;
+  #log: Logger;
+  #started = false;
+
+  constructor(
+    store: AztecAsyncKVStore,
+    archiveStore: AztecAsyncKVStore,
+    deps: TxPoolV2Dependencies,
+    telemetry: TelemetryClient = getTelemetryClient(),
+    config: Partial<TxPoolV2Config> = {},
+    dateProvider: DateProvider = new DateProvider(),
+    log = createLogger('p2p:tx_pool_v2'),
+  ) {
+    super();
+
+    this.#store = store;
+    this.#telemetry = telemetry;
+    this.#log = log;
+    this.#queue = new SerialQueue();
+
+    // Create callbacks that the impl uses to notify us about events and metrics
+    const callbacks = {
+      onTxsAdded: (txs: Tx[], opts: { source?: string }) => {
+        this.#metrics?.transactionsAdded(txs);
+        this.emit('txs-added', { txs, ...opts });
+      },
+      onTxsRemoved: (txHashes: string[] | bigint[]) => {
+        this.#metrics?.transactionsRemoved(txHashes);
+        // Convert to TxHash objects for the event
+        const hashes = txHashes.map(h => (typeof h === 'string' ? TxHash.fromString(h) : TxHash.fromBigInt(h)));
+        this.emit('txs-removed', { txHashes: hashes });
+      },
+    };
+
+    // Create the implementation
+    this.#impl = new TxPoolV2Impl(store, archiveStore, deps, callbacks, telemetry, config, dateProvider, log);
+  }
+
+  // ============================================================================
+  // PUBLIC API - All methods queue to the implementation
+  // ============================================================================
+
+  // === Core Operations ===
+
+  addPendingTxs(txs: Tx[], opts: { source?: string; feeComparisonOnly?: boolean } = {}): Promise<AddTxsResult> {
+    return this.#queue.put(() => this.#impl.addPendingTxs(txs, opts));
+  }
+
+  canAddPendingTx(tx: Tx): Promise<'accepted' | 'ignored'> {
+    return this.#queue.put(() => this.#impl.canAddPendingTx(tx));
+  }
+
+  addProtectedTxs(txs: Tx[], block: BlockHeader, opts: { source?: string } = {}): Promise<void> {
+    return this.#queue.put(() => this.#impl.addProtectedTxs(txs, block, opts));
+  }
+
+  protectTxs(txHashes: TxHash[], block: BlockHeader): Promise<TxHash[]> {
+    return this.#queue.put(() => this.#impl.protectTxs(txHashes, block));
+  }
+
+  addMinedTxs(txs: Tx[], block: BlockHeader, opts: { source?: string } = {}): Promise<void> {
+    return this.#queue.put(() => this.#impl.addMinedTxs(txs, block, opts));
+  }
+
+  // === State Transition Handlers ===
+
+  handleMinedBlock(block: L2Block): Promise<void> {
+    return this.#queue.put(() => this.#impl.handleMinedBlock(block));
+  }
+
+  prepareForSlot(slotNumber: SlotNumber): Promise<void> {
+    return this.#queue.put(() => this.#impl.prepareForSlot(slotNumber));
+  }
+
+  handlePrunedBlocks(latestBlock: L2BlockId, options?: { deleteAllTxs?: boolean }): Promise<void> {
+    return this.#queue.put(() => this.#impl.handlePrunedBlocks(latestBlock, options));
+  }
+
+  handleFailedExecution(txHashes: TxHash[]): Promise<void> {
+    return this.#queue.put(() => this.#impl.handleFailedExecution(txHashes));
+  }
+
+  handleFinalizedBlock(block: BlockHeader): Promise<void> {
+    return this.#queue.put(() => this.#impl.handleFinalizedBlock(block));
+  }
+
+  // === Queries ===
+
+  getTxByHash(txHash: TxHash): Promise<Tx | undefined> {
+    return this.#queue.put(() => this.#impl.getTxByHash(txHash));
+  }
+
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
+    return this.#queue.put(() => this.#impl.getTxsByHash(txHashes));
+  }
+
+  hasTxs(txHashes: TxHash[]): Promise<boolean[]> {
+    return this.#queue.put(() => this.#impl.hasTxs(txHashes));
+  }
+
+  getTxStatus(txHash: TxHash): Promise<TxState | 'deleted' | undefined> {
+    return this.#queue.put(() => Promise.resolve(this.#impl.getTxStatus(txHash)));
+  }
+
+  getPendingTxHashes(): Promise<TxHash[]> {
+    return this.#queue.put(() => Promise.resolve(this.#impl.getPendingTxHashes()));
+  }
+
+  getEligiblePendingTxHashes(): Promise<TxHash[]> {
+    return this.#queue.put(() => Promise.resolve(this.#impl.getEligiblePendingTxHashes()));
+  }
+
+  getPendingTxCount(): Promise<number> {
+    return this.#queue.put(() => Promise.resolve(this.#impl.getPendingTxCount()));
+  }
+
+  getMinedTxHashes(): Promise<[TxHash, L2BlockId][]> {
+    return this.#queue.put(() => Promise.resolve(this.#impl.getMinedTxHashes()));
+  }
+
+  getMinedTxCount(): Promise<number> {
+    return this.#queue.put(() => Promise.resolve(this.#impl.getMinedTxCount()));
+  }
+
+  isEmpty(): Promise<boolean> {
+    return this.#queue.put(() => Promise.resolve(this.#impl.isEmpty()));
+  }
+
+  getArchivedTxByHash(txHash: TxHash): Promise<Tx | undefined> {
+    return this.#queue.put(() => this.#impl.getArchivedTxByHash(txHash));
+  }
+
+  getLowestPriorityPending(limit: number): Promise<TxHash[]> {
+    return this.#queue.put(() => Promise.resolve(this.#impl.getLowestPriorityPending(limit)));
+  }
+
+  // === Configuration ===
+
+  updateConfig(config: Partial<TxPoolV2Config>): Promise<void> {
+    return this.#queue.put(() => {
+      this.#impl.updateConfig(config);
+      return Promise.resolve();
+    });
+  }
+
+  // === Lifecycle ===
+
+  /**
+   * Starts the pool and initializes state from persistence.
+   */
+  async start(): Promise<void> {
+    if (this.#started) {
+      return;
+    }
+
+    this.#log.info('Starting transaction pool...');
+
+    // Start the serial queue
+    this.#queue.start();
+    this.#started = true;
+
+    // Setup metrics - created after queue is started so callbacks can safely queue
+    this.#metrics = new PoolInstrumentation(
+      this.#telemetry,
+      PoolName.TX_POOL,
+      () =>
+        this.#queue.put(() => {
+          const counts = this.#impl.countTxs();
+          return Promise.resolve({
+            itemCount: {
+              pending: counts.pending,
+              protected: counts.protected,
+              mined: counts.mined,
+              softDeleted: counts.softDeleted,
+            },
+          });
+        }),
+      () => this.#store.estimateSize(),
+    );
+
+    // Hydrate state from persistence (runs in queue)
+    await this.#queue.put(() => this.#impl.hydrateFromDatabase());
+
+    this.#log.info(
+      `Transaction pool started with ${this.#impl.getTxCount()} transactions (${this.#impl.getPendingTxCount()} pending, ${this.#impl.getMinedTxCount()} mined)`,
+    );
+  }
+
+  async stop(): Promise<void> {
+    if (this.#started) {
+      await this.#queue.end();
+      this.#started = false;
+    }
+  }
+}