@aztec/p2p 0.69.1 → 0.71.0

package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts

@@ -1,14 +1,16 @@
 import { Tx, TxHash } from '@aztec/circuit-types';
 import { type TxAddedToPoolStats } from '@aztec/circuit-types/stats';
+import { ClientIvcProof } from '@aztec/circuits.js';
 import { type Logger, createLogger } from '@aztec/foundation/log';
-import { type AztecKVStore, type AztecMap, type AztecSet } from '@aztec/kv-store';
-import { type TelemetryClient } from '@aztec/telemetry-client';
+import { type AztecKVStore, type AztecMap, type AztecMultiMap } from '@aztec/kv-store';
+import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import { PoolInstrumentation, PoolName } from '../instrumentation.js';
+import { getPendingTxPriority } from './priority.js';
 import { type TxPool } from './tx_pool.js';
 
 /**
- * In-memory implementation of the Transaction Pool.
+ * KV implementation of the Transaction Pool.
  */
 export class AztecKVTxPool implements TxPool {
   #store: AztecKVStore;
@@ -16,43 +18,77 @@ export class AztecKVTxPool implements TxPool {
   /** Our tx pool, stored as a Map, with K: tx hash and V: the transaction. */
   #txs: AztecMap<string, Buffer>;
 
-  /** Index for pending txs. */
-  #pendingTxs: AztecSet<string>;
-  /** Index for mined txs. */
-  #minedTxs: AztecMap<string, number>;
+  /** Index from tx hash to the block number in which they were mined, filtered by mined txs. */
+  #minedTxHashToBlock: AztecMap<string, number>;
+
+  /** Index from tx priority (stored as hex) to its tx hash, filtered by pending txs. */
+  #pendingTxPriorityToHash: AztecMultiMap<string, string>;
+
+  /** KV store for archived txs. */
+  #archive: AztecKVStore;
+
+  /** Archived txs map for future lookup. */
+  #archivedTxs: AztecMap<string, Buffer>;
+
+  /** Indexes of the archived txs by insertion order. */
+  #archivedTxIndices: AztecMap<number, string>;
+
+  /** Number of txs to archive. */
+  #archivedTxLimit: number;
 
   #log: Logger;
 
   #metrics: PoolInstrumentation<Tx>;
 
   /**
-   * Class constructor for in-memory TxPool. Initiates our transaction pool as a JS Map.
-   * @param store - A KV store.
+   * Class constructor for KV TxPool. Initiates our transaction pool as an AztecMap.
+   * @param store - A KV store for live txs in the pool.
+   * @param archive - A KV store for archived txs.
+   * @param telemetry - A telemetry client.
+   * @param archivedTxLimit - The number of txs to archive.
    * @param log - A logger.
    */
-  constructor(store: AztecKVStore, telemetry: TelemetryClient, log = createLogger('p2p:tx_pool')) {
+  constructor(
+    store: AztecKVStore,
+    archive: AztecKVStore,
+    telemetry: TelemetryClient = getTelemetryClient(),
+    archivedTxLimit: number = 0,
+    log = createLogger('p2p:tx_pool'),
+  ) {
     this.#txs = store.openMap('txs');
-    this.#minedTxs = store.openMap('minedTxs');
-    this.#pendingTxs = store.openSet('pendingTxs');
+    this.#minedTxHashToBlock = store.openMap('txHashToBlockMined');
+    this.#pendingTxPriorityToHash = store.openMultiMap('pendingTxFeeToHash');
+
+    this.#archivedTxs = archive.openMap('archivedTxs');
+    this.#archivedTxIndices = archive.openMap('archivedTxIndices');
+    this.#archivedTxLimit = archivedTxLimit;
 
     this.#store = store;
+    this.#archive = archive;
     this.#log = log;
     this.#metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, () => store.estimateSize());
   }
 
   public markAsMined(txHashes: TxHash[], blockNumber: number): Promise<void> {
+    if (txHashes.length === 0) {
+      return Promise.resolve();
+    }
+
+    let deletedPending = 0;
     return this.#store.transaction(() => {
-      let deleted = 0;
       for (const hash of txHashes) {
         const key = hash.toString();
-        void this.#minedTxs.set(key, blockNumber);
-        if (this.#pendingTxs.has(key)) {
-          deleted++;
-          void this.#pendingTxs.delete(key);
+        void this.#minedTxHashToBlock.set(key, blockNumber);
+
+        const tx = this.getTxByHash(hash);
+        if (tx) {
+          deletedPending++;
+          const fee = getPendingTxPriority(tx);
+          void this.#pendingTxPriorityToHash.deleteValue(fee, key);
         }
       }
-      this.#metrics.recordRemovedObjects(deleted, 'pending');
       this.#metrics.recordAddedObjects(txHashes.length, 'mined');
+      this.#metrics.recordRemovedObjects(deletedPending, 'pending');
     });
   }
 
@@ -61,33 +97,30 @@ export class AztecKVTxPool implements TxPool {
       return Promise.resolve();
     }
 
+    let markedAsPending = 0;
     return this.#store.transaction(() => {
-      let deleted = 0;
-      let added = 0;
       for (const hash of txHashes) {
         const key = hash.toString();
-        if (this.#minedTxs.has(key)) {
-          deleted++;
-          void this.#minedTxs.delete(key);
-        }
+        void this.#minedTxHashToBlock.delete(key);
 
-        if (this.#txs.has(key)) {
-          added++;
-          void this.#pendingTxs.add(key);
+        const tx = this.getTxByHash(hash);
+        if (tx) {
+          void this.#pendingTxPriorityToHash.set(getPendingTxPriority(tx), key);
+          markedAsPending++;
         }
       }
 
-      this.#metrics.recordRemovedObjects(deleted, 'mined');
-      this.#metrics.recordAddedObjects(added, 'pending');
+      this.#metrics.recordAddedObjects(markedAsPending, 'pending');
+      this.#metrics.recordRemovedObjects(markedAsPending, 'mined');
     });
   }
 
   public getPendingTxHashes(): TxHash[] {
-    return Array.from(this.#pendingTxs.entries()).map(x => TxHash.fromString(x));
+    return Array.from(this.#pendingTxPriorityToHash.values({ reverse: true })).map(x => TxHash.fromString(x));
   }
 
   public getMinedTxHashes(): [TxHash, number][] {
-    return Array.from(this.#minedTxs.entries()).map(([txHash, blockNumber]) => [
+    return Array.from(this.#minedTxHashToBlock.entries()).map(([txHash, blockNumber]) => [
       TxHash.fromString(txHash),
       blockNumber,
     ]);
@@ -95,10 +128,10 @@ export class AztecKVTxPool implements TxPool {
 
   public getTxStatus(txHash: TxHash): 'pending' | 'mined' | undefined {
     const key = txHash.toString();
-    if (this.#pendingTxs.has(key)) {
-      return 'pending';
-    } else if (this.#minedTxs.has(key)) {
+    if (this.#minedTxHashToBlock.has(key)) {
       return 'mined';
+    } else if (this.#txs.has(key)) {
+      return 'pending';
     } else {
       return undefined;
     }
@@ -111,7 +144,27 @@ export class AztecKVTxPool implements TxPool {
    */
   public getTxByHash(txHash: TxHash): Tx | undefined {
     const buffer = this.#txs.get(txHash.toString());
-    return buffer ? Tx.fromBuffer(buffer) : undefined;
+    if (buffer) {
+      const tx = Tx.fromBuffer(buffer);
+      tx.setTxHash(txHash);
+      return tx;
+    }
+    return undefined;
+  }
+
+  /**
+   * Checks if an archived tx exists and returns it.
+   * @param txHash - The tx hash.
+   * @returns The transaction metadata, if found, 'undefined' otherwise.
+   */
+  public getArchivedTxByHash(txHash: TxHash): Tx | undefined {
+    const buffer = this.#archivedTxs.get(txHash.toString());
+    if (buffer) {
+      const tx = Tx.fromBuffer(buffer);
+      tx.setTxHash(txHash);
+      return tx;
+    }
+    return undefined;
   }
 
   /**
@@ -120,11 +173,10 @@ export class AztecKVTxPool implements TxPool {
    * @returns Empty promise.
    */
   public addTxs(txs: Tx[]): Promise<void> {
-    const txHashes = txs.map(tx => tx.getTxHash());
     return this.#store.transaction(() => {
       let pendingCount = 0;
-      for (const [i, tx] of txs.entries()) {
-        const txHash = txHashes[i];
+      for (const tx of txs) {
+        const txHash = tx.getTxHash();
         this.#log.verbose(`Adding tx ${txHash.toString()} to pool`, {
           eventName: 'tx-added-to-pool',
           ...tx.getStats(),
@@ -132,10 +184,11 @@ export class AztecKVTxPool implements TxPool {
 
         const key = txHash.toString();
         void this.#txs.set(key, tx.toBuffer());
-        if (!this.#minedTxs.has(key)) {
+
+        if (!this.#minedTxHashToBlock.has(key)) {
           pendingCount++;
           // REFACTOR: Use an lmdb conditional write to avoid race conditions with this write tx
-          void this.#pendingTxs.add(key);
+          void this.#pendingTxPriorityToHash.set(getPendingTxPriority(tx), key);
           this.#metrics.recordSize(tx);
         }
       }
@@ -147,29 +200,43 @@ export class AztecKVTxPool implements TxPool {
   /**
    * Deletes transactions from the pool. Tx hashes that are not present are ignored.
    * @param txHashes - An array of tx hashes to be removed from the tx pool.
-   * @returns The number of transactions that was deleted from the pool.
+   * @returns Empty promise.
    */
   public deleteTxs(txHashes: TxHash[]): Promise<void> {
-    return this.#store.transaction(() => {
-      let pendingDeleted = 0;
-      let minedDeleted = 0;
+    let pendingDeleted = 0;
+    let minedDeleted = 0;
+
+    const deletedTxs: Tx[] = [];
+    const poolDbTx = this.#store.transaction(() => {
       for (const hash of txHashes) {
         const key = hash.toString();
-        void this.#txs.delete(key);
-        if (this.#pendingTxs.has(key)) {
-          pendingDeleted++;
-          void this.#pendingTxs.delete(key);
-        }
+        const tx = this.getTxByHash(hash);
 
-        if (this.#minedTxs.has(key)) {
-          minedDeleted++;
-          void this.#minedTxs.delete(key);
+        if (tx) {
+          const fee = getPendingTxPriority(tx);
+          void this.#pendingTxPriorityToHash.deleteValue(fee, key);
+
+          const isMined = this.#minedTxHashToBlock.has(key);
+          if (isMined) {
+            minedDeleted++;
+          } else {
+            pendingDeleted++;
+          }
+
+          if (this.#archivedTxLimit) {
+            deletedTxs.push(tx);
+          }
+
+          void this.#txs.delete(key);
+          void this.#minedTxHashToBlock.delete(key);
         }
       }
 
       this.#metrics.recordRemovedObjects(pendingDeleted, 'pending');
       this.#metrics.recordRemovedObjects(minedDeleted, 'mined');
     });
+
+    return this.#archivedTxLimit ? poolDbTx.then(() => this.archiveTxs(deletedTxs)) : poolDbTx;
   }
 
   /**
@@ -177,7 +244,11 @@ export class AztecKVTxPool implements TxPool {
    * @returns Array of tx objects in the order they were added to the pool.
    */
   public getAllTxs(): Tx[] {
-    return Array.from(this.#txs.values()).map(buffer => Tx.fromBuffer(buffer));
+    return Array.from(this.#txs.entries()).map(([hash, buffer]) => {
+      const tx = Tx.fromBuffer(buffer);
+      tx.setTxHash(TxHash.fromString(hash));
+      return tx;
+    });
   }
 
   /**
@@ -187,4 +258,41 @@ export class AztecKVTxPool implements TxPool {
   public getAllTxHashes(): TxHash[] {
     return Array.from(this.#txs.keys()).map(x => TxHash.fromString(x));
   }
+
+  /**
+   * Archives a list of txs for future reference. The number of archived txs is limited by the specified archivedTxLimit.
+   * @param txs - The list of transactions to archive.
+   * @returns Empty promise.
+   */
+  private archiveTxs(txs: Tx[]): Promise<void> {
+    return this.#archive.transaction(() => {
+      // calcualte the head and tail indices of the archived txs by insertion order.
+      let headIdx = (this.#archivedTxIndices.entries({ limit: 1, reverse: true }).next().value?.[0] ?? -1) + 1;
+      let tailIdx = this.#archivedTxIndices.entries({ limit: 1 }).next().value?.[0] ?? 0;
+
+      for (const tx of txs) {
+        while (headIdx - tailIdx >= this.#archivedTxLimit) {
+          const txHash = this.#archivedTxIndices.get(tailIdx);
+          if (txHash) {
+            void this.#archivedTxs.delete(txHash);
+            void this.#archivedTxIndices.delete(tailIdx);
+          }
+          tailIdx++;
+        }
+
+        const archivedTx: Tx = new Tx(
+          tx.data,
+          ClientIvcProof.empty(),
+          tx.unencryptedLogs,
+          tx.contractClassLogs,
+          tx.enqueuedPublicFunctionCalls,
+          tx.publicTeardownFunctionCall,
+        );
+        const txHash = tx.getTxHash().toString();
+        void this.#archivedTxs.set(txHash, archivedTx.toBuffer());
+        void this.#archivedTxIndices.set(headIdx, txHash);
+        headIdx++;
+      }
+    });
+  }
 }

package/src/mem_pools/tx_pool/memory_tx_pool.ts

@@ -1,9 +1,10 @@
 import { Tx, TxHash } from '@aztec/circuit-types';
 import { type TxAddedToPoolStats } from '@aztec/circuit-types/stats';
 import { createLogger } from '@aztec/foundation/log';
-import { type TelemetryClient } from '@aztec/telemetry-client';
+import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import { PoolInstrumentation, PoolName } from '../instrumentation.js';
+import { getPendingTxPriority } from './priority.js';
 import { type TxPool } from './tx_pool.js';
 
 /**
@@ -23,7 +24,7 @@ export class InMemoryTxPool implements TxPool {
    * Class constructor for in-memory TxPool. Initiates our transaction pool as a JS Map.
    * @param log - A logger.
    */
-  constructor(telemetry: TelemetryClient, private log = createLogger('p2p:tx_pool')) {
+  constructor(telemetry: TelemetryClient = getTelemetryClient(), private log = createLogger('p2p:tx_pool')) {
     this.txs = new Map<bigint, Tx>();
     this.minedTxs = new Map();
     this.pendingTxs = new Set();
@@ -68,7 +69,10 @@ export class InMemoryTxPool implements TxPool {
   }
 
   public getPendingTxHashes(): TxHash[] {
-    return Array.from(this.pendingTxs).map(x => TxHash.fromBigInt(x));
+    return this.getAllTxs()
+      .sort((tx1, tx2) => -getPendingTxPriority(tx1).localeCompare(getPendingTxPriority(tx2)))
+      .map(tx => tx.getTxHash())
+      .filter(txHash => this.pendingTxs.has(txHash.toBigInt()));
   }
 
   public getMinedTxHashes(): [TxHash, number][] {
@@ -96,6 +100,10 @@ export class InMemoryTxPool implements TxPool {
     return result === undefined ? undefined : Tx.clone(result);
   }
 
+  public getArchivedTxByHash(): Tx | undefined {
+    return undefined;
+  }
+
   /**
    * Adds a list of transactions to the pool. Duplicates are ignored.
    * @param txs - An array of txs to be added to the pool.

package/src/mem_pools/tx_pool/priority.ts

@@ -0,0 +1,13 @@
+import { type Tx } from '@aztec/circuit-types';
+import { Buffer32 } from '@aztec/foundation/buffer';
+
+/**
+ * Returns a string representing the priority of a tx.
+ * Txs with a higher priority value are returned first when retrieving pending tx hashes.
+ * We currently use the sum of the priority fees for the tx for this value, represented as hex.
+ */
+export function getPendingTxPriority(tx: Tx): string {
+  const priorityFees = tx.getGasSettings().maxPriorityFeesPerGas;
+  const totalFees = priorityFees.feePerDaGas.toBigInt() + priorityFees.feePerL2Gas.toBigInt();
+  return Buffer32.fromBigInt(totalFees).toString();
+}

package/src/mem_pools/tx_pool/tx_pool.ts

@@ -17,6 +17,13 @@ export interface TxPool {
    */
   getTxByHash(txHash: TxHash): Tx | undefined;
 
+  /**
+   * Checks if an archived transaction exists in the pool and returns it.
+   * @param txHash - The hash of the transaction, used as an ID.
+   * @returns The transaction, if found, 'undefined' otherwise.
+   */
+  getArchivedTxByHash(txHash: TxHash): Tx | undefined;
+
   /**
    * Marks the set of txs as mined, as opposed to pending.
    * @param txHashes - Hashes of the txs to flag as mined.
@@ -49,7 +56,7 @@ export interface TxPool {
   getAllTxHashes(): TxHash[];
 
   /**
-   * Gets the hashes of pending transactions currently in the tx pool.
+   * Gets the hashes of pending transactions currently in the tx pool sorted by priority (see getPendingTxPriority).
    * @returns An array of pending transaction hashes found in the tx pool.
    */
   getPendingTxHashes(): TxHash[];

package/src/mem_pools/tx_pool/tx_pool_test_suite.ts

@@ -1,4 +1,6 @@
-import { mockTx } from '@aztec/circuit-types';
+import { type Tx, mockTx } from '@aztec/circuit-types';
+import { GasFees } from '@aztec/circuits.js';
+import { unfreeze } from '@aztec/foundation/types';
 
 import { type TxPool } from './tx_pool.js';
 
@@ -67,10 +69,12 @@ export function describeTxPool(getTxPool: () => TxPool) {
     await pool.addTxs([tx1]);
     // this peer knows that tx2 was mined, but it does not have the tx object
     await pool.markAsMined([tx1.getTxHash(), someTxHashThatThisPeerDidNotSee], 1);
-    expect(pool.getMinedTxHashes()).toEqual([
-      [tx1.getTxHash(), 1],
-      [someTxHashThatThisPeerDidNotSee, 1],
-    ]);
+    expect(new Set(pool.getMinedTxHashes())).toEqual(
+      new Set([
+        [tx1.getTxHash(), 1],
+        [someTxHashThatThisPeerDidNotSee, 1],
+      ]),
+    );
 
     // reorg: both txs should now become available again
     await pool.markMinedAsPending([tx1.getTxHash(), someTxHashThatThisPeerDidNotSee]);
@@ -101,4 +105,22 @@ export function describeTxPool(getTxPool: () => TxPool) {
     expect(poolTxHashes).toHaveLength(3);
     expect(poolTxHashes).toEqual(expect.arrayContaining([tx1.getTxHash(), tx2.getTxHash(), tx3.getTxHash()]));
   });
+
+  it('Returns pending tx hashes sorted by priority', async () => {
+    const withPriorityFee = (tx: Tx, fee: number) => {
+      unfreeze(tx.data.constants.txContext.gasSettings).maxPriorityFeesPerGas = new GasFees(fee, fee);
+      return tx;
+    };
+
+    const tx1 = withPriorityFee(mockTx(0), 1000);
+    const tx2 = withPriorityFee(mockTx(1), 100);
+    const tx3 = withPriorityFee(mockTx(2), 200);
+    const tx4 = withPriorityFee(mockTx(3), 3000);
+
+    await pool.addTxs([tx1, tx2, tx3, tx4]);
+
+    const poolTxHashes = pool.getPendingTxHashes();
+    expect(poolTxHashes).toHaveLength(4);
+    expect(poolTxHashes).toEqual([tx4, tx1, tx3, tx2].map(tx => tx.getTxHash()));
+  });
 }

package/src/mocks/index.ts

@@ -8,8 +8,7 @@ import {
 import { type EpochCache } from '@aztec/epoch-cache';
 import { type DataStoreConfig } from '@aztec/kv-store/config';
 import { openTmpStore } from '@aztec/kv-store/lmdb';
-import { type TelemetryClient } from '@aztec/telemetry-client';
-import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
+import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import { gossipsub } from '@chainsafe/libp2p-gossipsub';
 import { noise } from '@chainsafe/libp2p-noise';
@@ -247,7 +246,7 @@ export function createBootstrapNodeConfig(privateKey: string, port: number): Boo
 export function createBootstrapNodeFromPrivateKey(
   privateKey: string,
   port: number,
-  telemetry: TelemetryClient = new NoopTelemetryClient(),
+  telemetry: TelemetryClient = getTelemetryClient(),
 ): Promise<BootstrapNode> {
   const config = createBootstrapNodeConfig(privateKey, port);
   return startBootstrapNode(config, telemetry);
@@ -255,7 +254,7 @@ export function createBootstrapNodeFromPrivateKey(
 
 export async function createBootstrapNode(
   port: number,
-  telemetry: TelemetryClient = new NoopTelemetryClient(),
+  telemetry: TelemetryClient = getTelemetryClient(),
 ): Promise<BootstrapNode> {
   const peerId = await createSecp256k1PeerId();
   const config = createBootstrapNodeConfig(Buffer.from(peerId.privateKey!).toString('hex'), port);

package/src/msg_validators/tx_validator/block_header_validator.ts

@@ -0,0 +1,25 @@
+import { type AnyTx, Tx, type TxValidationResult, type TxValidator } from '@aztec/circuit-types';
+import { type Fr } from '@aztec/circuits.js';
+import { createLogger } from '@aztec/foundation/log';
+
+export interface ArchiveSource {
+  getArchiveIndices: (archives: Fr[]) => Promise<(bigint | undefined)[]>;
+}
+
+export class BlockHeaderTxValidator<T extends AnyTx> implements TxValidator<T> {
+  #log = createLogger('p2p:tx_validator:tx_block_header');
+  #archiveSource: ArchiveSource;
+
+  constructor(archiveSource: ArchiveSource) {
+    this.#archiveSource = archiveSource;
+  }
+
+  async validateTx(tx: T): Promise<TxValidationResult> {
+    const [index] = await this.#archiveSource.getArchiveIndices([tx.data.constants.historicalHeader.hash()]);
+    if (index === undefined) {
+      this.#log.warn(`Rejecting tx ${Tx.getHash(tx)} for referencing an unknown block header`);
+      return { result: 'invalid', reason: ['Block header not found'] };
+    }
+    return { result: 'valid' };
+  }
+}

package/src/msg_validators/tx_validator/index.ts

@@ -3,3 +3,4 @@ export * from './data_validator.js';
 export * from './double_spend_validator.js';
 export * from './metadata_validator.js';
 export * from './tx_proof_validator.js';
+export * from './block_header_validator.js';

package/src/services/discv5/discV5_service.ts

@@ -1,6 +1,6 @@
 import { createLogger } from '@aztec/foundation/log';
 import { sleep } from '@aztec/foundation/sleep';
-import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client';
+import { OtelMetricsAdapter, type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import { Discv5, type Discv5EventEmitter } from '@chainsafe/discv5';
 import { ENR, SignableENR } from '@chainsafe/enr';
@@ -38,7 +38,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
   constructor(
     private peerId: PeerId,
     config: P2PConfig,
-    telemetry: TelemetryClient,
+    telemetry: TelemetryClient = getTelemetryClient(),
     private logger = createLogger('p2p:discv5_service'),
   ) {
     super();