@aztec/p2p 0.0.1-commit.e3c1de76 → 0.0.1-commit.e558bd1c

package/src/mem_pools/tx_pool_v2/interfaces.ts

@@ -0,0 +1,227 @@
+import type { SlotNumber } from '@aztec/foundation/branded-types';
+import type { TypedEventEmitter } from '@aztec/foundation/types';
+import type { L2Block, L2BlockId, L2BlockSource } from '@aztec/stdlib/block';
+import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
+import type { BlockHeader, Tx, TxHash, TxValidator } from '@aztec/stdlib/tx';
+
+import type { TxMetaData, TxState } from './tx_metadata.js';
+
+/**
+ * Result of adding transactions to the pending pool.
+ * Categorizes transactions by their outcome.
+ */
+export type AddTxsResult = {
+  /** Transactions successfully added to the pool */
+  accepted: TxHash[];
+  /** Transactions ignored because they're valid but undesirable (e.g., duplicate, lower priority nullifier conflict) */
+  ignored: TxHash[];
+  /** Transactions rejected because they failed validation (e.g., invalid proof, expired timestamp) */
+  rejected: TxHash[];
+};
+
+/**
+ * Events emitted by TxPoolV2.
+ */
+export type TxPoolV2Events = {
+  /** Emitted when transactions are successfully added to the pool */
+  'txs-added': (args: { txs: Tx[]; source?: string }) => void;
+  /** Emitted when transactions are removed from the pool */
+  'txs-removed': (args: { txHashes: TxHash[] }) => void;
+};
+
+/**
+ * Configuration options for TxPoolV2.
+ */
+export type TxPoolV2Config = {
+  /** Maximum number of pending transactions before low-priority eviction */
+  maxPendingTxCount: number;
+  /** Maximum number of archived transactions to retain (0 = disabled) */
+  archivedTxLimit: number;
+};
+
+/**
+ * Default configuration values for TxPoolV2.
+ */
+export const DEFAULT_TX_POOL_V2_CONFIG: TxPoolV2Config = {
+  maxPendingTxCount: 0, // 0 = disabled
+  archivedTxLimit: 0, // 0 = disabled
+};
+
+/**
+ * Dependencies required by TxPoolV2.
+ */
+export type TxPoolV2Dependencies = {
+  /** Block source (Archiver) for checking mined status and verifying pruned blocks */
+  l2BlockSource: L2BlockSource;
+  /** World state synchronizer for validating transactions after chain prunes */
+  worldStateSynchronizer: WorldStateSynchronizer;
+  /** Validator for transactions entering the pending pool */
+  pendingTxValidator: TxValidator<Tx>;
+};
+
+/**
+ * Read-only access to pool state for pre-add checks.
+ * Used by eviction rules to inspect pool state during transaction addition.
+ */
+export interface PoolReadAccess {
+  /** Get metadata for a transaction by its hash (as string) */
+  getMetadata(txHash: string): TxMetaData | undefined;
+  /** Get the transaction hash that uses a specific nullifier (as string) */
+  getTxHashByNullifier(nullifier: string): string | undefined;
+  /** Get all transaction hashes for a fee payer (as string) */
+  getTxHashesByFeePayer(feePayer: string): Set<string> | undefined;
+  /** Get the current pending transaction count */
+  getPendingTxCount(): number;
+}
+
+/**
+ *
+ * The pool manages transactions through a state machine:
+ * - Pending: Transaction is awaiting inclusion in a block
+ * - Protected: Transaction is being considered for a block proposal
+ * - Mined: Transaction has been included in a block
+ * - Deleted: Transaction has been removed from the pool
+ *
+ * All state-mutating operations are serialized through a handler queue
+ * to prevent race conditions.
+ */
+export interface TxPoolV2 extends TypedEventEmitter<TxPoolV2Events> {
+  // === Core Operations ===
+
+  /**
+   * Adds transactions to the pending pool with challenge and validation.
+   * Handles nullifier conflicts via the challenge mechanism.
+   * @param txs - Transactions to add
+   * @param opts - Optional metadata (e.g., source for logging)
+   * @returns Result categorizing each transaction as accepted, rejected, or ignored
+   */
+  addPendingTxs(txs: Tx[], opts?: { source?: string }): Promise<AddTxsResult>;
+
+  /**
+   * Checks if a transaction can be added without modifying the pool.
+   * Performs the same validation as addPendingTxs but doesn't persist changes.
+   * @param tx - Transaction to check
+   * @returns Result: 'accepted', 'ignored' (if already in pool or undesirable), or 'rejected' (if validation fails)
+   */
+  canAddPendingTx(tx: Tx): Promise<'accepted' | 'ignored' | 'rejected'>;
+
+  /**
+   * Adds transactions as immediately protected for a given slot.
+   * Used when receiving transactions from a block proposal we're validating.
+   * @param txs - Transactions to add as protected
+   * @param block - Block header providing slot context
+   * @param opts - Optional metadata (e.g., source for logging)
+   */
+  addProtectedTxs(txs: Tx[], block: BlockHeader, opts?: { source?: string }): Promise<void>;
+
+  /**
+   * Protects existing transactions by hash for a given slot.
+   * Returns hashes of transactions that weren't found in the pool.
+   * Records unknown hashes for automatic protection when received via gossip.
+   * @param txHashes - Hashes of transactions to protect
+   * @param block - Block header providing slot context
+   * @returns Hashes of transactions not found in the pool
+   */
+  protectTxs(txHashes: TxHash[], block: BlockHeader): Promise<TxHash[]>;
+
+  /**
+   * Adds transactions as already mined.
+   * Used by prover nodes fetching transactions via request/response.
+   * @param txs - Transactions to add as mined
+   * @param block - Block header the transactions were mined in
+   * @param opts - Optional metadata (e.g., source for logging)
+   */
+  addMinedTxs(txs: Tx[], block: BlockHeader, opts?: { source?: string }): Promise<void>;
+
+  // === State Transition Handlers ===
+
+  /**
+   * Handles a mined block - marks transactions as mined and evicts conflicting pending txs.
+   * Uses nullifiers directly from the block to evict pending transactions with conflicts.
+   * @param block - The complete mined block
+   */
+  handleMinedBlock(block: L2Block): Promise<void>;
+
+  /**
+   * Prepares the pool for a new slot.
+   * Unprotects transactions from earlier slots and validates them before
+   * returning to pending state.
+   * @param slotNumber - The slot number to prepare for
+   */
+  prepareForSlot(slotNumber: SlotNumber): Promise<void>;
+
+  /**
+   * Handles pruned blocks during a reorg.
+   * Un-mines all transactions mined in blocks beyond the given latest block
+   * and validates them before returning to pending.
+   * @param latestBlock - The latest valid block ID after the prune
+   */
+  handlePrunedBlocks(latestBlock: L2BlockId): Promise<void>;
+
+  /**
+   * Handles failed transaction execution.
+   * Deletes transactions that failed during block building.
+   * @param txHashes - Hashes of transactions that failed
+   */
+  handleFailedExecution(txHashes: TxHash[]): Promise<void>;
+
+  /**
+   * Handles a finalized block.
+   * Permanently deletes mined transactions and optionally archives them.
+   * @param block - Header of the finalized block
+   */
+  handleFinalizedBlock(block: BlockHeader): Promise<void>;
+
+  /** Gets a transaction by its hash */
+  getTxByHash(txHash: TxHash): Promise<Tx | undefined>;
+
+  /** Gets multiple transactions by their hashes */
+  getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
+
+  /** Checks if transactions exist in the pool */
+  hasTxs(txHashes: TxHash[]): Promise<boolean[]>;
+
+  /** Gets the status of a transaction */
+  getTxStatus(txHash: TxHash): Promise<TxState | 'deleted' | undefined>;
+
+  /** Gets pending transaction hashes sorted by priority (highest first) */
+  getPendingTxHashes(): Promise<TxHash[]>;
+
+  /** Gets the count of pending transactions */
+  getPendingTxCount(): Promise<number>;
+
+  /** Gets mined transaction hashes with their block IDs */
+  getMinedTxHashes(): Promise<[TxHash, L2BlockId][]>;
+
+  /** Gets the count of mined transactions */
+  getMinedTxCount(): Promise<number>;
+
+  /** Checks if the pool is empty */
+  isEmpty(): Promise<boolean>;
+
+  /** Gets an archived transaction by its hash */
+  getArchivedTxByHash(txHash: TxHash): Promise<Tx | undefined>;
+
+  /** Gets the lowest priority pending transactions */
+  getLowestPriorityPending(limit: number): Promise<TxHash[]>;
+
+  // === Configuration ===
+
+  /** Updates the pool configuration */
+  updateConfig(config: Partial<TxPoolV2Config>): Promise<void>;
+
+  // === Lifecycle ===
+
+  /**
+   * Starts the pool and initializes state from persistence.
+   * Must be called before other operations.
+   * - Reads all transactions from the database
+   * - Checks each against the Archiver to determine mined status
+   * - Validates all non-mined transactions
+   * - Populates in-memory indices
+   */
+  start(): Promise<void>;
+
+  /** Stops the pool and releases resources */
+  stop(): Promise<void>;
+}

package/src/mem_pools/tx_pool_v2/tx_metadata.ts

@@ -0,0 +1,161 @@
+import { Fr } from '@aztec/foundation/curves/bn254';
+import { ProtocolContractAddress } from '@aztec/protocol-contracts';
+import type { L2BlockId } from '@aztec/stdlib/block';
+import type { Tx } from '@aztec/stdlib/tx';
+
+import { getFeePayerBalanceDelta } from '../../msg_validators/tx_validator/fee_payer_balance.js';
+import { getTxPriorityFee } from '../tx_pool/priority.js';
+import type { PreAddResult } from './eviction/interfaces.js';
+
+/**
+ * Lightweight in-memory representation of a transaction.
+ * Stored for every tx in the pool to enable efficient queries and challenges
+ * without deserializing full transaction data.
+ *
+ * Uses strings for txHash and nullifiers to enable fast Map/Set lookups
+ * without repeated .toString() conversions.
+ */
+export type TxMetaData = {
+  /** The transaction hash as hex string */
+  readonly txHash: string;
+
+  /** Block ID (number and hash) in which the transaction was mined (undefined if not mined) */
+  minedL2BlockId?: L2BlockId;
+
+  /** Hash of the block header the transaction uses as its anchor (hex string) */
+  readonly anchorBlockHeaderHash: string;
+
+  /** The total priority fee (used for ordering and challenges) */
+  readonly priorityFee: bigint;
+
+  /** The fee payer address as hex string */
+  readonly feePayer: string;
+
+  /** The claim amount for the fee payer */
+  readonly claimAmount: bigint;
+
+  /** The fee limit */
+  readonly feeLimit: bigint;
+
+  /** Non-empty nullifiers emitted by the transaction (hex strings) */
+  readonly nullifiers: readonly string[];
+
+  /** Timestamp by which the transaction must be included (for expiration checks) */
+  readonly includeByTimestamp: bigint;
+};
+
+/** Transaction state derived from TxMetaData fields and pool protection status */
+export type TxState = 'pending' | 'protected' | 'mined';
+
+/**
+ * Builds TxMetaData from a full Tx object.
+ * Extracts all relevant fields for efficient in-memory storage and querying.
+ */
+export async function buildTxMetaData(tx: Tx): Promise<TxMetaData> {
+  const txHash = tx.getTxHash().toString();
+  const anchorBlockHeaderHash = (await tx.data.constants.anchorBlockHeader.hash()).toString();
+  const priorityFee = getTxPriorityFee(tx);
+  const feePayer = tx.data.feePayer.toString();
+  const nullifiers = tx.data.getNonEmptyNullifiers().map(n => n.toString());
+  const includeByTimestamp = tx.data.includeByTimestamp;
+
+  const { feeLimit, claimAmount } = await getFeePayerBalanceDelta(tx, ProtocolContractAddress.FeeJuice);
+
+  return {
+    txHash,
+    anchorBlockHeaderHash,
+    priorityFee,
+    feePayer,
+    claimAmount,
+    feeLimit,
+    nullifiers,
+    includeByTimestamp,
+  };
+}
+
+/** Minimal fields required for priority comparison. */
+type PriorityComparable = Pick<TxMetaData, 'txHash' | 'priorityFee'>;
+
+/**
+ * Compares two priority fees in ascending order.
+ * Returns negative if a < b, positive if a > b, 0 if equal.
+ */
+export function compareFee(a: bigint, b: bigint): number {
+  return a < b ? -1 : a > b ? 1 : 0;
+}
+
+/**
+ * Compares two tx hashes in ascending order.
+ * Uses field element comparison for deterministic ordering.
+ * Returns negative if a < b, positive if a > b, 0 if equal.
+ */
+export function compareTxHash(a: string, b: string): number {
+  const fieldA = Fr.fromHexString(a);
+  const fieldB = Fr.fromHexString(b);
+  return fieldA.cmp(fieldB);
+}
+
+/**
+ * Compares two transactions by priority fee, with txHash as tiebreaker.
+ * Returns negative if a < b, positive if a > b, 0 if equal.
+ * Use with sort() for ascending order, or negate/reverse for descending.
+ */
+export function comparePriority(a: PriorityComparable, b: PriorityComparable): number {
+  const feeComparison = compareFee(a.priorityFee, b.priorityFee);
+  if (feeComparison !== 0) {
+    return feeComparison;
+  }
+  return compareTxHash(a.txHash, b.txHash);
+}
+
+/**
+ * Checks for nullifier conflicts between an incoming transaction and existing pool state.
+ *
+ * When the incoming tx shares nullifiers with existing pending txs:
+ * - If the incoming tx has strictly higher priority, mark conflicting txs for eviction
+ * - If any conflicting tx has equal or higher priority, ignore the incoming tx
+ *
+ * @param incomingMeta - Metadata for the incoming transaction
+ * @param getTxHashByNullifier - Accessor to find which tx uses a nullifier
+ * @param getMetadata - Accessor to get metadata for a tx hash
+ */
+export function checkNullifierConflict(
+  incomingMeta: TxMetaData,
+  getTxHashByNullifier: (nullifier: string) => string | undefined,
+  getMetadata: (txHash: string) => TxMetaData | undefined,
+): PreAddResult {
+  const txHashesToEvict: string[] = [];
+
+  for (const nullifier of incomingMeta.nullifiers) {
+    const conflictingHashStr = getTxHashByNullifier(nullifier);
+
+    if (!conflictingHashStr || conflictingHashStr === incomingMeta.txHash) {
+      continue;
+    }
+
+    // Skip if already marked for eviction
+    if (txHashesToEvict.includes(conflictingHashStr)) {
+      continue;
+    }
+
+    const conflictingMeta = getMetadata(conflictingHashStr);
+    if (!conflictingMeta) {
+      continue;
+    }
+
+    // If incoming tx has strictly higher priority, mark for eviction
+    // Otherwise, ignore incoming tx (ties go to existing tx)
+    // Use comparePriority for deterministic ordering (includes txHash as tiebreaker)
+    if (comparePriority(incomingMeta, conflictingMeta) > 0) {
+      txHashesToEvict.push(conflictingHashStr);
+    } else {
+      return {
+        shouldIgnore: true,
+        txHashesToEvict: [],
+        reason: `nullifier conflict with ${conflictingHashStr}`,
+      };
+    }
+  }
+
+  return { shouldIgnore: false, txHashesToEvict };
+}

package/src/mem_pools/tx_pool_v2/tx_pool_bench_metrics.ts

@@ -0,0 +1,77 @@
+/**
+ * Metrics types for tx pool benchmarks.
+ */
+
+export enum TxPoolOperation {
+  ADD_PENDING_TXS = 'addPendingTxs',
+  CAN_ADD_PENDING_TX = 'canAddPendingTx',
+  GET_PENDING_TX_HASHES = 'getPendingTxHashes',
+  GET_TX_BY_HASH = 'getTxByHash',
+  GET_TXS_BY_HASH = 'getTxsByHash',
+  HAS_TXS = 'hasTxs',
+  HANDLE_MINED_BLOCK = 'handleMinedBlock',
+  PREPARE_FOR_SLOT = 'prepareForSlot',
+  HANDLE_PRUNED_BLOCKS = 'handlePrunedBlocks',
+  GET_LOWEST_PRIORITY_PENDING = 'getLowestPriorityPending',
+  HYDRATE_FROM_DATABASE = 'hydrateFromDatabase',
+}
+
+type OperationMetric = {
+  operation: TxPoolOperation;
+  poolSize: number;
+  batchSize: number;
+  value: number;
+};
+
+/**
+ * Collects and formats tx pool benchmark metrics.
+ */
+export class TxPoolBenchMetrics {
+  private metrics: OperationMetric[] = [];
+
+  public addMetric(operation: TxPoolOperation, poolSize: number, batchSize: number, value: number) {
+    this.metrics.push({ operation, poolSize, batchSize, value });
+  }
+
+  public toPrettyString(): string {
+    let pretty = 'TxPool Benchmark Metrics:\n';
+    pretty += '='.repeat(60) + '\n';
+
+    // Group by operation
+    const byOperation = new Map<TxPoolOperation, OperationMetric[]>();
+    for (const metric of this.metrics) {
+      if (!byOperation.has(metric.operation)) {
+        byOperation.set(metric.operation, []);
+      }
+      byOperation.get(metric.operation)!.push(metric);
+    }
+
+    for (const [operation, opMetrics] of byOperation) {
+      pretty += `\n${operation}:\n`;
+      for (const metric of opMetrics) {
+        const poolSizeStr = metric.poolSize > 0 ? `pool=${metric.poolSize}` : '';
+        const batchSizeStr = metric.batchSize > 0 ? `batch=${metric.batchSize}` : '';
+        const params = [poolSizeStr, batchSizeStr].filter(Boolean).join(', ');
+        pretty += `  ${params ? `(${params})` : ''}: ${metric.value.toFixed(3)} ms\n`;
+      }
+    }
+
+    return pretty;
+  }
+
+  public toGithubActionBenchmarkJSON(indent = 2): string {
+    const data = this.metrics.map(metric => {
+      const poolSizeStr = metric.poolSize > 0 ? `${metric.poolSize} txs in pool` : '';
+      const batchSizeStr = metric.batchSize > 0 ? `batch of ${metric.batchSize}` : '';
+      const params = [poolSizeStr, batchSizeStr].filter(Boolean).join('/');
+
+      return {
+        name: `TxPool/${metric.operation}${params ? `/${params}` : ''}`,
+        value: metric.value,
+        unit: 'ms',
+      };
+    });
+
+    return JSON.stringify(data, null, indent);
+  }
+}