@aztec/p2p 0.0.1-commit.fce3e4f → 0.0.1-commit.ff7989d6c

package/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts

@@ -0,0 +1,91 @@
+import { createLogger } from '@aztec/foundation/log';
+
+import type { EvictionConfig, EvictionContext, EvictionResult, EvictionRule, PoolOperations } from './interfaces.js';
+import { EvictionEvent } from './interfaces.js';
+
+/**
+ * Eviction rule that removes low-priority transactions when the pool exceeds configured limits.
+ * Triggers on TXS_ADDED and CHAIN_PRUNED events.
+ */
+export class LowPriorityEvictionRule implements EvictionRule {
+  public readonly name = 'LowPriorityEviction';
+
+  private log = createLogger('p2p:tx_pool_v2:low_priority_eviction_rule');
+  private maxPoolSize: number;
+
+  constructor(config: { maxPoolSize: number }) {
+    this.maxPoolSize = config.maxPoolSize;
+  }
+
+  async evict(context: EvictionContext, pool: PoolOperations): Promise<EvictionResult> {
+    if (context.event !== EvictionEvent.TXS_ADDED && context.event !== EvictionEvent.CHAIN_PRUNED) {
+      return {
+        reason: 'low_priority',
+        success: true,
+        txsEvicted: [],
+      };
+    }
+
+    if (this.maxPoolSize === 0) {
+      return {
+        reason: 'low_priority',
+        success: true,
+        txsEvicted: [],
+      };
+    }
+
+    try {
+      const currentTxCount = pool.getPendingTxCount();
+
+      if (currentTxCount <= this.maxPoolSize) {
+        this.log.trace(
+          `Not evicting low priority txs. Pending tx count below limit ${currentTxCount} <= ${this.maxPoolSize}`,
+        );
+        return {
+          reason: 'low_priority',
+          success: true,
+          txsEvicted: [],
+        };
+      }
+
+      this.log.info(`Evicting low priority txs. Pending tx count above limit: ${currentTxCount} > ${this.maxPoolSize}`);
+      const numberToEvict = currentTxCount - this.maxPoolSize;
+      const txsToEvict = pool.getLowestPriorityPending(numberToEvict);
+
+      if (txsToEvict.length > 0) {
+        if (context.event === EvictionEvent.TXS_ADDED) {
+          const toEvictSet = new Set(txsToEvict);
+          const numNewTxsEvicted = context.newTxHashes.filter(newTxHash => toEvictSet.has(newTxHash)).length;
+          this.log.info(`Evicted ${txsToEvict.length} low priority txs, including ${numNewTxsEvicted} newly added txs`);
+        } else {
+          this.log.info(`Evicted ${txsToEvict.length} low priority txs after chain prune`);
+        }
+        await pool.deleteTxs(txsToEvict, this.name);
+      }
+
+      this.log.debug(`Evicted ${txsToEvict.length} low priority txs`, {
+        txHashes: txsToEvict,
+      });
+
+      return {
+        reason: 'low_priority',
+        success: true,
+        txsEvicted: txsToEvict,
+      };
+    } catch (err) {
+      this.log.error('Failed to evict low priority transactions', { err });
+      return {
+        reason: 'low_priority',
+        success: false,
+        txsEvicted: [],
+        error: new Error('Failed to evict low priority txs', { cause: err }),
+      };
+    }
+  }
+
+  updateConfig(config: EvictionConfig): void {
+    if (config.maxPendingTxCount !== undefined) {
+      this.maxPoolSize = config.maxPendingTxCount;
+    }
+  }
+}

package/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts

@@ -0,0 +1,90 @@
+import { createLogger } from '@aztec/foundation/log';
+
+import { type TxMetaData, comparePriority } from '../tx_metadata.js';
+import {
+  type EvictionConfig,
+  type PreAddContext,
+  type PreAddPoolAccess,
+  type PreAddResult,
+  type PreAddRule,
+  TxPoolRejectionCode,
+} from './interfaces.js';
+
+/**
+ * Pre-add rule that checks if the pool is at capacity and handles low-priority eviction.
+ *
+ * When the pool is at capacity:
+ * - If incoming tx has higher priority than the lowest priority tx, evict the lowest and accept incoming
+ * - If incoming tx has equal or lower priority than the lowest, ignore incoming (it would be evicted anyway)
+ */
+export class LowPriorityPreAddRule implements PreAddRule {
+  public readonly name = 'LowPriorityPreAdd';
+
+  private log = createLogger('p2p:tx_pool_v2:low_priority_pre_add_rule');
+  private maxPoolSize: number;
+
+  constructor(config: { maxPoolSize: number }) {
+    this.maxPoolSize = config.maxPoolSize;
+  }
+
+  check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess, context?: PreAddContext): Promise<PreAddResult> {
+    // Skip if max pool size is disabled (0 = unlimited)
+    if (this.maxPoolSize === 0) {
+      return Promise.resolve({ shouldIgnore: false, txHashesToEvict: [] });
+    }
+
+    const currentCount = poolAccess.getPendingTxCount();
+
+    // If pool is not at capacity, accept the tx
+    if (currentCount < this.maxPoolSize) {
+      return Promise.resolve({ shouldIgnore: false, txHashesToEvict: [] });
+    }
+
+    // Pool is at capacity - need to compare priorities
+    const lowestPriorityMeta = poolAccess.getLowestPriorityPendingTx();
+    if (!lowestPriorityMeta) {
+      // No pending txs (shouldn't happen if count > 0, but handle gracefully)
+      return Promise.resolve({ shouldIgnore: false, txHashesToEvict: [] });
+    }
+
+    // Compare incoming tx against lowest priority tx.
+    // feeOnly mode (RPC): use strict fee comparison only — avoids churn from hash ordering
+    // Default (gossip): use full comparePriority (fee + tx hash tiebreaker) for determinism
+    const isHigherPriority = context?.feeComparisonOnly
+      ? incomingMeta.priorityFee > lowestPriorityMeta.priorityFee
+      : comparePriority(incomingMeta, lowestPriorityMeta) > 0;
+
+    if (isHigherPriority) {
+      this.log.debug(
+        `Pool at capacity (${currentCount}/${this.maxPoolSize}), evicting ${lowestPriorityMeta.txHash} ` +
+          `(priority ${lowestPriorityMeta.priorityFee}) for ${incomingMeta.txHash} (priority ${incomingMeta.priorityFee})`,
+      );
+      return Promise.resolve({
+        shouldIgnore: false,
+        txHashesToEvict: [lowestPriorityMeta.txHash],
+      });
+    }
+
+    // Incoming tx has equal or lower priority - ignore it (it would be evicted anyway)
+    this.log.debug(
+      `Pool at capacity (${currentCount}/${this.maxPoolSize}), ignoring ${incomingMeta.txHash} ` +
+        `(priority ${incomingMeta.priorityFee}) - lower than existing minimum (priority ${lowestPriorityMeta.priorityFee})`,
+    );
+    return Promise.resolve({
+      shouldIgnore: true,
+      txHashesToEvict: [],
+      reason: {
+        code: TxPoolRejectionCode.LOW_PRIORITY_FEE,
+        message: `Tx does not meet minimum priority fee. Required: ${lowestPriorityMeta.priorityFee + 1n}, got: ${incomingMeta.priorityFee}`,
+        minimumPriorityFee: lowestPriorityMeta.priorityFee + 1n,
+        txPriorityFee: incomingMeta.priorityFee,
+      },
+    });
+  }
+
+  updateConfig(config: EvictionConfig): void {
+    if (config.maxPendingTxCount !== undefined) {
+      this.maxPoolSize = config.maxPendingTxCount;
+    }
+  }
+}

package/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts

@@ -0,0 +1,31 @@
+import { createLogger } from '@aztec/foundation/log';
+
+import { type TxMetaData, checkNullifierConflict } from '../tx_metadata.js';
+import type { PreAddContext, PreAddPoolAccess, PreAddResult, PreAddRule } from './interfaces.js';
+
+/**
+ * Pre-add rule that checks for nullifier conflicts between incoming and existing transactions.
+ *
+ * When an incoming tx shares nullifiers with existing pending txs:
+ * - If the incoming tx has strictly higher priority, evict all conflicting txs
+ * - If any conflicting tx has equal or higher priority, ignore the incoming tx
+ */
+export class NullifierConflictRule implements PreAddRule {
+  public readonly name = 'NullifierConflict';
+
+  private log = createLogger('p2p:tx_pool_v2:nullifier_conflict_rule');
+
+  check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess, _context?: PreAddContext): Promise<PreAddResult> {
+    const result = checkNullifierConflict(
+      incomingMeta,
+      nullifier => poolAccess.getTxHashByNullifier(nullifier),
+      txHash => poolAccess.getMetadata(txHash),
+    );
+
+    if (result.shouldIgnore) {
+      this.log.debug(`Ignoring tx ${incomingMeta.txHash}: ${result.reason?.message}`);
+    }
+
+    return Promise.resolve(result);
+  }
+}

package/src/mem_pools/tx_pool_v2/index.ts

@@ -0,0 +1,12 @@
+export { AztecKVTxPoolV2 } from './tx_pool_v2.js';
+export {
+  type TxPoolV2,
+  type TxPoolV2Config,
+  type TxPoolV2Events,
+  type AddTxsResult,
+  type PoolReadAccess,
+  DEFAULT_TX_POOL_V2_CONFIG,
+} from './interfaces.js';
+export { type TxMetaData, type TxState, buildTxMetaData, comparePriority } from './tx_metadata.js';
+export { TxArchive } from './archive/index.js';
+export { DeletedPool } from './deleted_pool.js';

package/src/mem_pools/tx_pool_v2/instrumentation.ts

@@ -0,0 +1,69 @@
+import {
+  Attributes,
+  type Meter,
+  Metrics,
+  type ObservableGauge,
+  type ObservableResult,
+  type TelemetryClient,
+  type UpDownCounter,
+  createUpDownCounterWithDefault,
+} from '@aztec/telemetry-client';
+
+/** Callback that returns the current estimated metadata memory in bytes. */
+export type MetadataMemoryCallback = () => number;
+
+/** Instrumentation for TxPoolV2Impl internal operations. */
+export class TxPoolV2Instrumentation {
+  #evictedCounter: UpDownCounter;
+  #ignoredCounter: UpDownCounter;
+  #rejectedCounter: UpDownCounter;
+  #softDeletedHitsCounter: UpDownCounter;
+  #missingOnProtectCounter: UpDownCounter;
+  #missingPreviouslyEvictedCounter: UpDownCounter;
+  #metadataMemoryGauge: ObservableGauge;
+
+  constructor(telemetry: TelemetryClient, metadataMemoryCallback: MetadataMemoryCallback) {
+    const meter: Meter = telemetry.getMeter('TxPoolV2Impl');
+
+    this.#evictedCounter = createUpDownCounterWithDefault(meter, Metrics.MEMPOOL_TX_POOL_V2_EVICTED_COUNT);
+    this.#ignoredCounter = createUpDownCounterWithDefault(meter, Metrics.MEMPOOL_TX_POOL_V2_IGNORED_COUNT);
+    this.#rejectedCounter = createUpDownCounterWithDefault(meter, Metrics.MEMPOOL_TX_POOL_V2_REJECTED_COUNT);
+    this.#softDeletedHitsCounter = createUpDownCounterWithDefault(meter, Metrics.MEMPOOL_TX_POOL_V2_SOFT_DELETED_HITS);
+    this.#missingOnProtectCounter = createUpDownCounterWithDefault(
+      meter,
+      Metrics.MEMPOOL_TX_POOL_V2_MISSING_ON_PROTECT,
+    );
+    this.#missingPreviouslyEvictedCounter = createUpDownCounterWithDefault(
+      meter,
+      Metrics.MEMPOOL_TX_POOL_V2_MISSING_PREVIOUSLY_EVICTED,
+    );
+    this.#metadataMemoryGauge = meter.createObservableGauge(Metrics.MEMPOOL_TX_POOL_V2_METADATA_MEMORY);
+    this.#metadataMemoryGauge.addCallback((result: ObservableResult) => {
+      result.observe(metadataMemoryCallback());
+    });
+  }
+
+  recordEvictions(count: number, reason: string) {
+    this.#evictedCounter.add(count, { [Attributes.TX_POOL_EVICTION_REASON]: reason });
+  }
+
+  recordIgnored(count: number) {
+    this.#ignoredCounter.add(count);
+  }
+
+  recordRejected(count: number) {
+    this.#rejectedCounter.add(count);
+  }
+
+  recordSoftDeletedHits(count: number) {
+    this.#softDeletedHitsCounter.add(count);
+  }
+
+  recordMissingOnProtect(count: number) {
+    this.#missingOnProtectCounter.add(count);
+  }
+
+  recordMissingPreviouslyEvicted(count: number) {
+    this.#missingPreviouslyEvictedCounter.add(count);
+  }
+}

package/src/mem_pools/tx_pool_v2/interfaces.ts

@@ -0,0 +1,242 @@
+import type { SlotNumber } from '@aztec/foundation/branded-types';
+import type { TypedEventEmitter } from '@aztec/foundation/types';
+import type { L2Block, L2BlockId, L2BlockSource } from '@aztec/stdlib/block';
+import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
+import type { BlockHeader, Tx, TxHash, TxValidator } from '@aztec/stdlib/tx';
+
+import type { TxPoolRejectionError } from './eviction/interfaces.js';
+import type { TxMetaData, TxState } from './tx_metadata.js';
+
+/**
+ * Result of adding transactions to the pending pool.
+ * Categorizes transactions by their outcome.
+ */
+export type AddTxsResult = {
+  /** Transactions successfully added to the pool */
+  accepted: TxHash[];
+  /** Transactions ignored because they're valid but undesirable (e.g., duplicate, lower priority nullifier conflict) */
+  ignored: TxHash[];
+  /** Transactions rejected because they failed validation (e.g., invalid proof, expired timestamp) */
+  rejected: TxHash[];
+  /** Optional rejection errors, only present when there are rejections with structured errors. */
+  errors?: Map<string, TxPoolRejectionError>;
+};
+
+/**
+ * Events emitted by TxPoolV2.
+ */
+export type TxPoolV2Events = {
+  /** Emitted when transactions are successfully added to the pool */
+  'txs-added': (args: { txs: Tx[]; source?: string }) => void;
+  /** Emitted when transactions are removed from the pool */
+  'txs-removed': (args: { txHashes: TxHash[] }) => void;
+};
+
+/**
+ * Configuration options for TxPoolV2.
+ */
+export type TxPoolV2Config = {
+  /** Maximum number of pending transactions before low-priority eviction */
+  maxPendingTxCount: number;
+  /** Maximum number of archived transactions to retain (0 = disabled) */
+  archivedTxLimit: number;
+  /** Minimum age (ms) a transaction must have been in the pool before it's eligible for block building */
+  minTxPoolAgeMs: number;
+  /** Maximum number of evicted tx hashes to remember for metrics tracking */
+  evictedTxCacheSize: number;
+  /** The probability (0-1) that a transaction is discarded. 0 disables dropping. For testing purposes only. */
+  dropTransactionsProbability: number;
+};
+
+/**
+ * Default configuration values for TxPoolV2.
+ */
+export const DEFAULT_TX_POOL_V2_CONFIG: TxPoolV2Config = {
+  maxPendingTxCount: 0, // 0 = disabled
+  archivedTxLimit: 0, // 0 = disabled
+  minTxPoolAgeMs: 2_000,
+  evictedTxCacheSize: 10_000,
+  dropTransactionsProbability: 0,
+};
+
+/**
+ * Dependencies required by TxPoolV2.
+ */
+export type TxPoolV2Dependencies = {
+  /** Block source (Archiver) for checking mined status and verifying pruned blocks */
+  l2BlockSource: L2BlockSource;
+  /** World state synchronizer for validating transactions after chain prunes */
+  worldStateSynchronizer: WorldStateSynchronizer;
+  /** Factory that creates a validator for re-validating pool transactions using metadata */
+  createTxValidator: () => Promise<TxValidator<TxMetaData>>;
+};
+
+/**
+ * Read-only access to pool state for pre-add checks.
+ * Used by eviction rules to inspect pool state during transaction addition.
+ */
+export interface PoolReadAccess {
+  /** Get metadata for a transaction by its hash (as string) */
+  getMetadata(txHash: string): TxMetaData | undefined;
+  /** Get the transaction hash that uses a specific nullifier (as string) */
+  getTxHashByNullifier(nullifier: string): string | undefined;
+  /** Get all transaction hashes for a fee payer (as string) */
+  getTxHashesByFeePayer(feePayer: string): Set<string> | undefined;
+  /** Get the current pending transaction count */
+  getPendingTxCount(): number;
+}
+
+/**
+ *
+ * The pool manages transactions through a state machine:
+ * - Pending: Transaction is awaiting inclusion in a block
+ * - Protected: Transaction is being considered for a block proposal
+ * - Mined: Transaction has been included in a block
+ * - Deleted: Transaction has been removed from the pool
+ *
+ * All state-mutating operations are serialized through a handler queue
+ * to prevent race conditions.
+ */
+export interface TxPoolV2 extends TypedEventEmitter<TxPoolV2Events> {
+  // === Core Operations ===
+
+  /**
+   * Adds transactions to the pending pool with challenge and validation.
+   * Handles nullifier conflicts via the challenge mechanism.
+   * @param txs - Transactions to add
+   * @param opts - Optional metadata (e.g., source for logging)
+   * @returns Result categorizing each transaction as accepted, rejected, or ignored
+   */
+  addPendingTxs(txs: Tx[], opts?: { source?: string; feeComparisonOnly?: boolean }): Promise<AddTxsResult>;
+
+  /**
+   * Checks if a transaction can be added without modifying the pool.
+   * Performs the same validation as addPendingTxs but doesn't persist changes.
+   * @param tx - Transaction to check
+   * @returns Result: 'accepted', 'ignored' (if already in pool or undesirable), or 'rejected' (if validation fails)
+   */
+  canAddPendingTx(tx: Tx): Promise<'accepted' | 'ignored' | 'rejected'>;
+
+  /**
+   * Adds transactions as immediately protected for a given slot.
+   * Used when receiving transactions from a block proposal we're validating.
+   * @param txs - Transactions to add as protected
+   * @param block - Block header providing slot context
+   * @param opts - Optional metadata (e.g., source for logging)
+   */
+  addProtectedTxs(txs: Tx[], block: BlockHeader, opts?: { source?: string }): Promise<void>;
+
+  /**
+   * Protects existing transactions by hash for a given slot.
+   * Returns hashes of transactions that weren't found in the pool.
+   * Records unknown hashes for automatic protection when received via gossip.
+   * @param txHashes - Hashes of transactions to protect
+   * @param block - Block header providing slot context
+   * @returns Hashes of transactions not found in the pool
+   */
+  protectTxs(txHashes: TxHash[], block: BlockHeader): Promise<TxHash[]>;
+
+  /**
+   * Adds transactions as already mined.
+   * Used by prover nodes fetching transactions via request/response.
+   * @param txs - Transactions to add as mined
+   * @param block - Block header the transactions were mined in
+   * @param opts - Optional metadata (e.g., source for logging)
+   */
+  addMinedTxs(txs: Tx[], block: BlockHeader, opts?: { source?: string }): Promise<void>;
+
+  // === State Transition Handlers ===
+
+  /**
+   * Handles a mined block - marks transactions as mined and evicts conflicting pending txs.
+   * Uses nullifiers directly from the block to evict pending transactions with conflicts.
+   * @param block - The complete mined block
+   */
+  handleMinedBlock(block: L2Block): Promise<void>;
+
+  /**
+   * Prepares the pool for a new slot.
+   * Unprotects transactions from earlier slots and validates them before
+   * returning to pending state.
+   * @param slotNumber - The slot number to prepare for
+   */
+  prepareForSlot(slotNumber: SlotNumber): Promise<void>;
+
+  /**
+   * Handles pruned blocks during a reorg.
+   * Un-mines all transactions mined in blocks beyond the given latest block
+   * and validates them before returning to pending.
+   * @param latestBlock - The latest valid block ID after the prune
+   */
+  handlePrunedBlocks(latestBlock: L2BlockId, options?: { deleteAllTxs?: boolean }): Promise<void>;
+
+  /**
+   * Handles failed transaction execution.
+   * Deletes transactions that failed during block building.
+   * @param txHashes - Hashes of transactions that failed
+   */
+  handleFailedExecution(txHashes: TxHash[]): Promise<void>;
+
+  /**
+   * Handles a finalized block.
+   * Permanently deletes mined transactions and optionally archives them.
+   * @param block - Header of the finalized block
+   */
+  handleFinalizedBlock(block: BlockHeader): Promise<void>;
+
+  /** Gets a transaction by its hash */
+  getTxByHash(txHash: TxHash): Promise<Tx | undefined>;
+
+  /** Gets multiple transactions by their hashes */
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+
+  /** Checks if transactions exist in the pool */
+  hasTxs(txHashes: TxHash[]): Promise<boolean[]>;
+
+  /** Gets the status of a transaction */
+  getTxStatus(txHash: TxHash): Promise<TxState | 'deleted' | undefined>;
+
+  /** Gets pending transaction hashes sorted by priority (highest first) */
+  getPendingTxHashes(): Promise<TxHash[]>;
+
+  /** Gets pending transaction hashes that have been in the pool long enough per minTxPoolAgeMs, sorted by priority (highest first) */
+  getEligiblePendingTxHashes(): Promise<TxHash[]>;
+
+  /** Gets the count of pending transactions */
+  getPendingTxCount(): Promise<number>;
+
+  /** Gets mined transaction hashes with their block IDs */
+  getMinedTxHashes(): Promise<[TxHash, L2BlockId][]>;
+
+  /** Gets the count of mined transactions */
+  getMinedTxCount(): Promise<number>;
+
+  /** Checks if the pool is empty */
+  isEmpty(): Promise<boolean>;
+
+  /** Gets an archived transaction by its hash */
+  getArchivedTxByHash(txHash: TxHash): Promise<Tx | undefined>;
+
+  /** Gets the lowest priority pending transactions */
+  getLowestPriorityPending(limit: number): Promise<TxHash[]>;
+
+  // === Configuration ===
+
+  /** Updates the pool configuration */
+  updateConfig(config: Partial<TxPoolV2Config>): Promise<void>;
+
+  // === Lifecycle ===
+
+  /**
+   * Starts the pool and initializes state from persistence.
+   * Must be called before other operations.
+   * - Reads all transactions from the database
+   * - Checks each against the Archiver to determine mined status
+   * - Validates all non-mined transactions
+   * - Populates in-memory indices
+   */
+  start(): Promise<void>;
+
+  /** Stops the pool and releases resources */
+  stop(): Promise<void>;
+}