@aztec/p2p 0.0.1-commit.a072138 → 0.0.1-commit.aada20e3

package/src/mem_pools/tx_pool_v2/README.md

@@ -0,0 +1,188 @@
+# TxPoolV2
+
+Transaction pool implementation with explicit state management and pluggable eviction rules.
+
+## Overview
+
+TxPoolV2 manages transactions through a state machine with clear transitions:
+
+```
+              addPendingTxs()
+                    │
+                    ▼
+┌─────────────────────────────────────┐
+│              PENDING                │◄──────────────────┐
+│   (awaiting block inclusion)        │                   │
+└─────────────────────────────────────┘                   │
+        │                                                 │
+        │ protectTxs() / addProtectedTxs()               │
+        ▼                                                 │
+┌─────────────────────────────────────┐                   │
+│            PROTECTED                │───────────────────┘
+│   (in a block proposal)             │  prepareForSlot()
+└─────────────────────────────────────┘  (slot passed without mining)
+        │                                                 │
+        │ handleMinedBlock()                              │
+        ▼                                                 │
+┌─────────────────────────────────────┐                   │
+│              MINED                  │───────────────────┘
+│   (included in a block)             │  handlePrunedBlocks()
+└─────────────────────────────────────┘  (reorg)
+        │
+        │ handleFinalizedBlock()
+        ▼
+┌─────────────────────────────────────┐
+│             DELETED                 │
+│   (optionally archived)             │
+└─────────────────────────────────────┘
+```
+
+## Key Components
+
+### TxPoolV2 (`tx_pool_v2.ts`)
+
+The public API wrapper that serializes all operations through a queue to prevent race conditions. Delegates to `TxPoolV2Impl` for actual logic.
+
+### TxPoolV2Impl (`tx_pool_v2_impl.ts`)
+
+Core implementation containing:
+- KV store persistence for transactions and metadata
+- In-memory indices for fast lookups (by nullifier, fee payer, priority)
+- State transition logic
+- Pre-add rule execution
+- Post-event eviction rule execution
+
+### TxMetaData (`tx_metadata.ts`)
+
+Lightweight metadata stored alongside each transaction:
+- `txHash`: Transaction identifier
+- `state`: Current state (pending, protected, mined)
+- `priorityFee`: For priority ordering
+- `feePayer`: For balance-based eviction
+- `nullifiers`: For conflict detection
+- `estimatedTxFee`: For balance calculations
+- `blockId`: Block info when mined
+- `protectedSlot`: Slot number when protected
+
+## Eviction Rules
+
+The pool uses a pluggable rule system for managing transactions.
+
+### Pre-Add Rules
+
+Checked before adding a transaction to the pending pool:
+
+| Rule | Purpose |
+|------|---------|
+| `NullifierConflictRule` | Handles transactions with conflicting nullifiers. Higher priority tx wins. |
+| `FeePayerBalancePreAddRule` | Ensures fee payer has sufficient balance for all their pending txs. |
+| `LowPriorityPreAddRule` | Rejects txs when pool is full and new tx has lowest priority. |
+
+### Post-Event Eviction Rules
+
+Run after events to clean up the pool:
+
+| Rule | Trigger | Purpose |
+|------|---------|---------|
+| `LowPriorityEvictionRule` | `txs_added` | Evicts lowest priority txs when pool exceeds limit. |
+| `FeePayerBalanceEvictionRule` | `block_mined` | Evicts txs when fee payer balance decreases. |
+| `InvalidTxsAfterMiningRule` | `block_mined` | Evicts pending txs with nullifiers that were just mined. |
+| `InvalidTxsAfterReorgRule` | `chain_pruned` | Evicts txs with invalid anchor blocks after reorg. |
+
+## Usage
+
+### Creating a Pool
+
+```typescript
+import { AztecKVTxPoolV2 } from './tx_pool_v2.js';
+
+const pool = new AztecKVTxPoolV2(txStore, archiveStore, {
+  l2BlockSource: archiver,
+  worldStateSynchronizer: worldState,
+  pendingTxValidator: validator,
+});
+
+await pool.start();
+```
+
+### Adding Transactions
+
+```typescript
+// Add to pending pool (validates and runs pre-add rules)
+const result = await pool.addPendingTxs(txs, { source: 'gossip' });
+// result: { accepted: TxHash[], ignored: TxHash[], rejected: TxHash[] }
+
+// Check without modifying pool
+const canAdd = await pool.canAddPendingTx(tx);
+// canAdd: 'accepted' | 'ignored' | 'rejected'
+```
+
+### Block Building Flow
+
+```typescript
+// 1. Proposer protects txs for their block
+await pool.protectTxs(selectedTxHashes, blockHeader);
+
+// 2. On successful mining
+await pool.handleMinedBlock(minedTxHashes, blockHeader);
+
+// 3. If slot passes without mining
+await pool.prepareForSlot(nextSlotNumber);
+
+// 4. On finalization
+await pool.handleFinalizedBlock(finalizedBlockHeader);
+```
+
+### Handling Reorgs
+
+```typescript
+// When blocks are pruned, un-mine affected transactions
+await pool.handlePrunedBlocks(latestValidBlockId);
+```
+
+## Configuration
+
+```typescript
+await pool.updateConfig({
+  maxPendingTxCount: 10000,  // 0 = unlimited
+  archivedTxLimit: 1000,     // 0 = disabled
+});
+```
+
+## Return Values
+
+### AddTxsResult
+
+When adding pending transactions, each tx is categorized:
+
+| Status | Meaning |
+|--------|---------|
+| `accepted` | Successfully added to the pool |
+| `ignored` | Valid but not added (duplicate, lost nullifier conflict, insufficient balance) |
+| `rejected` | Failed validation (invalid proof, expired, etc.) |
+
+## Archive
+
+Finalized transactions can optionally be archived for historical queries:
+
+```typescript
+const archivedTx = await pool.getArchivedTxByHash(txHash);
+```
+
+The archive uses FIFO eviction when `archivedTxLimit` is reached.
+
+## Testing
+
+```bash
+# Unit tests (131 tests)
+yarn test src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts
+
+# Compatibility tests (25 tests)
+yarn test src/mem_pools/tx_pool_v2/tx_pool_v2.compat.test.ts
+
+# Eviction rule tests
+yarn test src/mem_pools/tx_pool_v2/eviction/
+
+# Benchmarks
+yarn test src/mem_pools/tx_pool_v2/tx_pool_v2_bench.test.ts
+```

package/src/mem_pools/tx_pool_v2/archive/index.ts

@@ -0,0 +1 @@
+export { TxArchive } from './tx_archive.js';

package/src/mem_pools/tx_pool_v2/archive/tx_archive.ts

@@ -0,0 +1,120 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
+import { ChonkProof } from '@aztec/stdlib/proofs';
+import { Tx, TxHash } from '@aztec/stdlib/tx';
+
+/**
+ * Manages archived transactions with FIFO eviction.
+ * Archived transactions have their proofs stripped to save space.
+ */
+export class TxArchive {
+  #store: AztecAsyncKVStore;
+  #txs: AztecAsyncMap<string, Buffer>;
+  #indices: AztecAsyncMap<number, string>;
+  #limit: number;
+  #log: Logger;
+
+  constructor(store: AztecAsyncKVStore, limit: number, log?: Logger) {
+    this.#store = store;
+    this.#txs = store.openMap('archivedTxs');
+    this.#indices = store.openMap('archivedTxIndices');
+    this.#limit = limit;
+    this.#log = log ?? createLogger('p2p:tx_pool_v2:archive');
+  }
+
+  /**
+   * Updates the maximum number of archived transactions.
+   */
+  updateLimit(limit: number): void {
+    this.#limit = limit;
+  }
+
+  /**
+   * Gets the current archive limit.
+   */
+  getLimit(): number {
+    return this.#limit;
+  }
+
+  /**
+   * Checks if archiving is enabled.
+   */
+  isEnabled(): boolean {
+    return this.#limit > 0;
+  }
+
+  /**
+   * Archives transactions, stripping their proofs.
+   * Evicts oldest transactions if the limit is exceeded.
+   */
+  async archiveTxs(txs: Tx[]): Promise<void> {
+    if (!this.isEnabled() || txs.length === 0) {
+      return;
+    }
+
+    try {
+      await this.#store.transactionAsync(async () => {
+        // Get current head and tail indices
+        let headIdx = await this.getHeadIndex();
+        let tailIdx = await this.getTailIndex();
+
+        for (const tx of txs) {
+          // Evict oldest entries if at capacity
+          while (headIdx - tailIdx >= this.#limit) {
+            const txHashToEvict = await this.#indices.getAsync(tailIdx);
+            if (txHashToEvict) {
+              await this.#txs.delete(txHashToEvict);
+              await this.#indices.delete(tailIdx);
+            }
+            tailIdx++;
+          }
+
+          // Archive the transaction with stripped proof
+          const archivedTx = this.stripProof(tx);
+          const txHash = tx.getTxHash().toString();
+          await this.#txs.set(txHash, archivedTx.toBuffer());
+          await this.#indices.set(headIdx, txHash);
+          headIdx++;
+        }
+
+        this.#log.debug(`Archived ${txs.length} txs, total: ${headIdx - tailIdx}`);
+      });
+    } catch (error) {
+      this.#log.error('Error archiving transactions', { error });
+    }
+  }
+
+  /**
+   * Retrieves an archived transaction by its hash.
+   */
+  async getTxByHash(txHash: TxHash): Promise<Tx | undefined> {
+    const buffer = await this.#txs.getAsync(txHash.toString());
+    return buffer ? Tx.fromBuffer(buffer) : undefined;
+  }
+
+  /**
+   * Gets the current count of archived transactions.
+   */
+  async getCount(): Promise<number> {
+    const head = await this.getHeadIndex();
+    const tail = await this.getTailIndex();
+    return head - tail;
+  }
+
+  /**
+   * Strips the proof from a transaction for archival.
+   */
+  private stripProof(tx: Tx): Tx {
+    return new Tx(tx.txHash, tx.data, ChonkProof.empty(), tx.contractClassLogFields, tx.publicFunctionCalldata);
+  }
+
+  private async getHeadIndex(): Promise<number> {
+    const entry = await this.#indices.entriesAsync({ limit: 1, reverse: true }).next();
+    return (entry.value?.[0] ?? -1) + 1;
+  }
+
+  private async getTailIndex(): Promise<number> {
+    const entry = await this.#indices.entriesAsync({ limit: 1 }).next();
+    return entry.value?.[0] ?? 0;
+  }
+}

package/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts

@@ -0,0 +1,147 @@
+import type { BlockNumber } from '@aztec/foundation/branded-types';
+import type { Logger } from '@aztec/foundation/log';
+import type { BlockHeader } from '@aztec/stdlib/tx';
+
+import type { TxMetaData } from '../tx_metadata.js';
+import {
+  type EvictionConfig,
+  type EvictionContext,
+  EvictionEvent,
+  type EvictionRule,
+  type PoolOperations,
+  type PreAddPoolAccess,
+  type PreAddResult,
+  type PreAddRule,
+} from './interfaces.js';
+
+/**
+ * Manages eviction rules for the transaction pool.
+ * Coordinates pre-add rules (run during addPendingTxs) and post-event rules (run after events).
+ */
+export class EvictionManager {
+  private preAddRules: PreAddRule[] = [];
+  private postEventRules: EvictionRule[] = [];
+
+  constructor(
+    private pool: PoolOperations,
+    private log: Logger,
+  ) {}
+
+  /**
+   * Registers a pre-add rule that runs during transaction addition.
+   */
+  registerPreAddRule(rule: PreAddRule): void {
+    this.preAddRules.push(rule);
+    this.log.debug(`Registered pre-add rule: ${rule.name}`);
+  }
+
+  /**
+   * Registers a post-event eviction rule that runs after events.
+   */
+  registerRule(rule: EvictionRule): void {
+    this.postEventRules.push(rule);
+    this.log.debug(`Registered eviction rule: ${rule.name}`);
+  }
+
+  /**
+   * Runs all pre-add rules for an incoming transaction.
+   * Returns combined result of all rules.
+   */
+  async runPreAddRules(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess): Promise<PreAddResult> {
+    const allTxHashesToEvict: string[] = [];
+
+    for (const rule of this.preAddRules) {
+      try {
+        const result = await rule.check(incomingMeta, poolAccess);
+
+        if (result.shouldIgnore) {
+          return result;
+        }
+
+        // Collect txs to evict from all rules
+        for (const txHash of result.txHashesToEvict) {
+          if (!allTxHashesToEvict.includes(txHash)) {
+            allTxHashesToEvict.push(txHash);
+          }
+        }
+      } catch (err) {
+        this.log.error(`Error running pre-add rule ${rule.name}`, { err, txHash: incomingMeta.txHash });
+        // On error, ignore the transaction to be safe
+        return {
+          shouldIgnore: true,
+          txHashesToEvict: [],
+          reason: `pre-add rule ${rule.name} error: ${err}`,
+        };
+      }
+    }
+
+    return {
+      shouldIgnore: false,
+      txHashesToEvict: allTxHashesToEvict,
+    };
+  }
+
+  /**
+   * Runs post-event eviction after new transactions are added.
+   */
+  async evictAfterNewTxs(newTxHashes: string[], feePayers: string[]): Promise<void> {
+    const context: EvictionContext = {
+      event: EvictionEvent.TXS_ADDED,
+      newTxHashes,
+      feePayers,
+    };
+
+    await this.runPostEventRules(context);
+  }
+
+  /**
+   * Runs post-event eviction after a block is mined.
+   */
+  async evictAfterNewBlock(block: BlockHeader, newNullifiers: string[], feePayers: string[]): Promise<void> {
+    const context: EvictionContext = {
+      event: EvictionEvent.BLOCK_MINED,
+      block,
+      newNullifiers,
+      feePayers,
+    };
+
+    await this.runPostEventRules(context);
+  }
+
+  /**
+   * Runs post-event eviction after a chain prune (reorg).
+   */
+  async evictAfterChainPrune(blockNumber: BlockNumber): Promise<void> {
+    const context: EvictionContext = {
+      event: EvictionEvent.CHAIN_PRUNED,
+      blockNumber,
+    };
+
+    await this.runPostEventRules(context);
+  }
+
+  /**
+   * Updates configuration for all rules.
+   */
+  updateConfig(config: EvictionConfig): void {
+    for (const rule of this.preAddRules) {
+      rule.updateConfig?.(config);
+    }
+    for (const rule of this.postEventRules) {
+      rule.updateConfig?.(config);
+    }
+  }
+
+  private async runPostEventRules(context: EvictionContext): Promise<void> {
+    for (const rule of this.postEventRules) {
+      try {
+        const result = await rule.evict(context, this.pool);
+        if (!result.success) {
+          this.log.warn(`Eviction rule ${rule.name} failed`, { error: result.error });
+        }
+      } catch (err) {
+        this.log.error(`Error running eviction rule ${rule.name}`, { err });
+      }
+    }
+  }
+}

package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts

@@ -0,0 +1,118 @@
+import { createLogger } from '@aztec/foundation/log';
+import { ProtocolContractAddress } from '@aztec/protocol-contracts';
+import { computeFeePayerBalanceStorageSlot } from '@aztec/protocol-contracts/fee-juice';
+import { AztecAddress } from '@aztec/stdlib/aztec-address';
+import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
+import { DatabasePublicStateSource, type MerkleTreeReadOperations } from '@aztec/stdlib/trees';
+
+import { type TxMetaData, comparePriority } from '../tx_metadata.js';
+import type { EvictionContext, EvictionResult, EvictionRule, PoolOperations } from './interfaces.js';
+import { EvictionEvent } from './interfaces.js';
+
+/**
+ * Eviction rule that removes transactions when fee payers have insufficient balance.
+ * Triggers on TXS_ADDED, BLOCK_MINED, and CHAIN_PRUNED events.
+ */
+export class FeePayerBalanceEvictionRule implements EvictionRule {
+  public readonly name = 'FeePayerBalanceEviction';
+  public readonly reason = 'fee_payer_balance';
+
+  private log = createLogger('p2p:tx_pool_v2:fee_payer_balance_eviction_rule');
+
+  constructor(private worldState: WorldStateSynchronizer) {}
+
+  async evict(context: EvictionContext, pool: PoolOperations): Promise<EvictionResult> {
+    try {
+      if (context.event === EvictionEvent.TXS_ADDED) {
+        return await this.evictForFeePayers(context.feePayers, this.worldState.getCommitted(), pool);
+      }
+
+      if (context.event === EvictionEvent.BLOCK_MINED) {
+        const blockNumber = context.block.getBlockNumber();
+        await this.worldState.syncImmediate(blockNumber);
+        return await this.evictForFeePayers(context.feePayers, this.worldState.getSnapshot(blockNumber), pool);
+      }
+
+      if (context.event === EvictionEvent.CHAIN_PRUNED) {
+        await this.worldState.syncImmediate(context.blockNumber);
+        const feePayers = pool.getPendingFeePayers();
+        return await this.evictForFeePayers(feePayers, this.worldState.getSnapshot(context.blockNumber), pool);
+      }
+
+      return {
+        reason: this.reason,
+        success: true,
+        txsEvicted: [],
+      };
+    } catch (err) {
+      this.log.error('Failed to evict txs due to fee payer balance', { err });
+      return {
+        reason: this.reason,
+        success: false,
+        txsEvicted: [],
+        error: new Error('Failed to evict txs due to fee payer balance', { cause: err }),
+      };
+    }
+  }
+
+  private async evictForFeePayers(
+    feePayers: string[],
+    db: MerkleTreeReadOperations,
+    pool: PoolOperations,
+  ): Promise<EvictionResult> {
+    const publicStateSource = new DatabasePublicStateSource(db);
+
+    const txsToEvict = (
+      await Promise.all(feePayers.map(feePayer => this.getEvictionsForFeePayer(feePayer, publicStateSource, pool)))
+    ).flat();
+
+    if (txsToEvict.length > 0) {
+      await pool.deleteTxs(txsToEvict);
+    }
+
+    return {
+      reason: this.reason,
+      success: true,
+      txsEvicted: txsToEvict,
+    };
+  }
+
+  private async getEvictionsForFeePayer(
+    feePayerStr: string,
+    publicStateSource: DatabasePublicStateSource,
+    pool: PoolOperations,
+  ): Promise<string[]> {
+    const feePayer = AztecAddress.fromString(feePayerStr);
+    const initialBalance = (
+      await publicStateSource.storageRead(
+        ProtocolContractAddress.FeeJuice,
+        await computeFeePayerBalanceStorageSlot(feePayer),
+      )
+    ).toBigInt();
+
+    const txs: TxMetaData[] = pool.getFeePayerPendingTxs(feePayerStr);
+
+    if (txs.length === 0) {
+      return [];
+    }
+
+    const txsToEvict: string[] = [];
+    let balance = initialBalance;
+
+    // Sort by priority descending (highest first), with hash as tiebreaker
+    txs.sort((a, b) => comparePriority(b, a));
+
+    for (const tx of txs) {
+      const available = balance + tx.claimAmount;
+      if (available >= tx.feeLimit) {
+        balance = available - tx.feeLimit;
+        continue;
+      }
+
+      // This tx cannot be covered - mark for eviction
+      txsToEvict.push(tx.txHash);
+    }
+
+    return txsToEvict;
+  }
+}

package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts

@@ -0,0 +1,111 @@
+import { createLogger } from '@aztec/foundation/log';
+
+import { type TxMetaData, comparePriority } from '../tx_metadata.js';
+import type { PreAddPoolAccess, PreAddResult, PreAddRule } from './interfaces.js';
+
+/**
+ * Pre-add rule that checks if a fee payer has sufficient balance to cover the incoming transaction.
+ *
+ * When an incoming tx is added:
+ * - Get the fee payer's on-chain balance
+ * - Get all existing pending txs for this fee payer
+ * - Insert incoming tx in priority order
+ * - Walk through in priority order, tracking running balance
+ * - If incoming tx can be covered: accept, mark lower-priority txs for eviction if needed
+ * - If incoming tx cannot be covered: ignore it
+ */
+export class FeePayerBalancePreAddRule implements PreAddRule {
+  public readonly name = 'FeePayerBalancePreAdd';
+
+  private log = createLogger('p2p:tx_pool_v2:fee_payer_balance_pre_add_rule');
+
+  async check(incomingMeta: TxMetaData, poolAccess: PreAddPoolAccess): Promise<PreAddResult> {
+    // Get fee payer's on-chain balance
+    const initialBalance = await poolAccess.getFeePayerBalance(incomingMeta.feePayer);
+
+    // Get existing pending txs for this fee payer
+    const existingTxs = poolAccess.getFeePayerPendingTxs(incomingMeta.feePayer);
+
+    // Create combined list with incoming tx
+    const allTxs: Array<{
+      txHash: string;
+      priorityFee: bigint;
+      feeLimit: bigint;
+      claimAmount: bigint;
+      isIncoming: boolean;
+    }> = [
+      ...existingTxs.map(t => ({
+        txHash: t.txHash,
+        priorityFee: t.priorityFee,
+        feeLimit: t.feeLimit,
+        claimAmount: t.claimAmount,
+        isIncoming: false,
+      })),
+      {
+        txHash: incomingMeta.txHash,
+        priorityFee: incomingMeta.priorityFee,
+        feeLimit: incomingMeta.feeLimit,
+        claimAmount: incomingMeta.claimAmount,
+        isIncoming: true,
+      },
+    ];
+
+    // Sort by priority descending (highest first), with hash as tiebreaker
+    allTxs.sort((a, b) => comparePriority(b, a));
+
+    // Walk through in priority order, tracking balance
+    let balance = initialBalance;
+    let incomingTxCovered = false;
+    const txsToEvict: string[] = [];
+
+    for (const txInfo of allTxs) {
+      const available = balance + txInfo.claimAmount;
+
+      if (available >= txInfo.feeLimit) {
+        // This tx can be covered
+        balance = available - txInfo.feeLimit;
+        if (txInfo.isIncoming) {
+          incomingTxCovered = true;
+        }
+      } else {
+        // This tx cannot be covered
+        if (txInfo.isIncoming) {
+          // Incoming tx cannot be covered - ignore it
+          this.log.debug(
+            `Ignoring tx ${incomingMeta.txHash}: fee payer ${incomingMeta.feePayer} has insufficient balance`,
+            { balance: initialBalance, feeLimit: incomingMeta.feeLimit, claimAmount: incomingMeta.claimAmount },
+          );
+          return {
+            shouldIgnore: true,
+            txHashesToEvict: [],
+            reason: `fee payer ${incomingMeta.feePayer} has insufficient balance`,
+          };
+        } else {
+          // Existing tx cannot be covered after adding incoming - mark for eviction
+          txsToEvict.push(txInfo.txHash);
+        }
+      }
+    }
+
+    if (!incomingTxCovered) {
+      // This shouldn't happen if the logic above is correct, but just in case
+      this.log.warn(`Incoming tx ${incomingMeta.txHash} was not covered but also not ignored - this is a bug`);
+      return {
+        shouldIgnore: true,
+        txHashesToEvict: [],
+        reason: 'internal error: tx coverage not determined',
+      };
+    }
+
+    if (txsToEvict.length > 0) {
+      this.log.debug(
+        `Accepting tx ${incomingMeta.txHash}, evicting ${txsToEvict.length} lower-priority txs due to fee payer balance`,
+      );
+    }
+
+    return {
+      shouldIgnore: false,
+      txHashesToEvict: txsToEvict,
+    };
+  }
+}