@aztec/p2p 0.84.0 → 0.85.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dest/bootstrap/bootstrap.js +1 -1
  2. package/dest/client/factory.d.ts.map +1 -1
  3. package/dest/client/factory.js +5 -2
  4. package/dest/client/p2p_client.d.ts +2 -0
  5. package/dest/client/p2p_client.d.ts.map +1 -1
  6. package/dest/client/p2p_client.js +4 -1
  7. package/dest/config.d.ts +15 -3
  8. package/dest/config.d.ts.map +1 -1
  9. package/dest/config.js +12 -1
  10. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts +56 -2
  11. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts.map +1 -1
  12. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.js +229 -16
  13. package/dest/msg_validators/tx_validator/archive_cache.d.ts +14 -0
  14. package/dest/msg_validators/tx_validator/archive_cache.d.ts.map +1 -0
  15. package/dest/msg_validators/tx_validator/archive_cache.js +22 -0
  16. package/dest/msg_validators/tx_validator/block_header_validator.d.ts.map +1 -1
  17. package/dest/msg_validators/tx_validator/block_header_validator.js +2 -2
  18. package/dest/msg_validators/tx_validator/data_validator.d.ts.map +1 -1
  19. package/dest/msg_validators/tx_validator/data_validator.js +8 -8
  20. package/dest/msg_validators/tx_validator/double_spend_validator.d.ts.map +1 -1
  21. package/dest/msg_validators/tx_validator/double_spend_validator.js +3 -3
  22. package/dest/msg_validators/tx_validator/gas_validator.d.ts +2 -1
  23. package/dest/msg_validators/tx_validator/gas_validator.d.ts.map +1 -1
  24. package/dest/msg_validators/tx_validator/gas_validator.js +32 -5
  25. package/dest/msg_validators/tx_validator/index.d.ts +1 -0
  26. package/dest/msg_validators/tx_validator/index.d.ts.map +1 -1
  27. package/dest/msg_validators/tx_validator/index.js +1 -0
  28. package/dest/msg_validators/tx_validator/metadata_validator.d.ts.map +1 -1
  29. package/dest/msg_validators/tx_validator/metadata_validator.js +4 -4
  30. package/dest/msg_validators/tx_validator/phases_validator.d.ts.map +1 -1
  31. package/dest/msg_validators/tx_validator/phases_validator.js +10 -2
  32. package/dest/msg_validators/tx_validator/tx_proof_validator.d.ts.map +1 -1
  33. package/dest/msg_validators/tx_validator/tx_proof_validator.js +2 -2
  34. package/dest/services/discv5/discV5_service.d.ts +1 -2
  35. package/dest/services/discv5/discV5_service.d.ts.map +1 -1
  36. package/dest/services/discv5/discV5_service.js +6 -8
  37. package/dest/services/dummy_service.d.ts +2 -1
  38. package/dest/services/dummy_service.d.ts.map +1 -1
  39. package/dest/services/dummy_service.js +1 -1
  40. package/dest/services/libp2p/libp2p_service.d.ts +14 -8
  41. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  42. package/dest/services/libp2p/libp2p_service.js +2 -2
  43. package/dest/services/peer-manager/peer_manager.d.ts.map +1 -1
  44. package/dest/services/peer-manager/peer_manager.js +0 -1
  45. package/dest/services/service.d.ts +1 -1
  46. package/dest/services/service.d.ts.map +1 -1
  47. package/dest/testbench/p2p_client_testbench_worker.js +46 -16
  48. package/dest/testbench/parse_log_file.js +4 -4
  49. package/dest/testbench/testbench.js +1 -1
  50. package/dest/testbench/worker_client_manager.d.ts.map +1 -1
  51. package/dest/testbench/worker_client_manager.js +3 -2
  52. package/dest/util.d.ts +7 -3
  53. package/dest/util.d.ts.map +1 -1
  54. package/dest/util.js +44 -7
  55. package/package.json +12 -12
  56. package/src/bootstrap/bootstrap.ts +1 -1
  57. package/src/client/factory.ts +7 -2
  58. package/src/client/p2p_client.ts +6 -1
  59. package/src/config.ts +26 -2
  60. package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts +312 -27
  61. package/src/msg_validators/tx_validator/archive_cache.ts +28 -0
  62. package/src/msg_validators/tx_validator/block_header_validator.ts +2 -2
  63. package/src/msg_validators/tx_validator/data_validator.ts +19 -8
  64. package/src/msg_validators/tx_validator/double_spend_validator.ts +10 -3
  65. package/src/msg_validators/tx_validator/gas_validator.ts +36 -6
  66. package/src/msg_validators/tx_validator/index.ts +1 -0
  67. package/src/msg_validators/tx_validator/metadata_validator.ts +12 -4
  68. package/src/msg_validators/tx_validator/phases_validator.ts +6 -1
  69. package/src/msg_validators/tx_validator/tx_proof_validator.ts +2 -2
  70. package/src/services/discv5/discV5_service.ts +10 -8
  71. package/src/services/dummy_service.ts +2 -1
  72. package/src/services/libp2p/libp2p_service.ts +9 -9
  73. package/src/services/peer-manager/peer_manager.ts +1 -1
  74. package/src/services/service.ts +1 -1
  75. package/src/testbench/p2p_client_testbench_worker.ts +97 -16
  76. package/src/testbench/parse_log_file.ts +4 -4
  77. package/src/testbench/testbench.ts +1 -1
  78. package/src/testbench/worker_client_manager.ts +4 -2
  79. package/src/util.ts +57 -8
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aztec/p2p",
3
- "version": "0.84.0",
3
+ "version": "0.85.0",
4
4
  "type": "module",
5
5
  "exports": {
6
6
  ".": "./dest/index.js",
@@ -65,16 +65,16 @@
65
65
  ]
66
66
  },
67
67
  "dependencies": {
68
- "@aztec/constants": "0.84.0",
69
- "@aztec/epoch-cache": "0.84.0",
70
- "@aztec/foundation": "0.84.0",
71
- "@aztec/kv-store": "0.84.0",
72
- "@aztec/noir-contracts.js": "0.84.0",
73
- "@aztec/noir-protocol-circuits-types": "0.84.0",
74
- "@aztec/protocol-contracts": "0.84.0",
75
- "@aztec/simulator": "0.84.0",
76
- "@aztec/stdlib": "0.84.0",
77
- "@aztec/telemetry-client": "0.84.0",
68
+ "@aztec/constants": "0.85.0",
69
+ "@aztec/epoch-cache": "0.85.0",
70
+ "@aztec/foundation": "0.85.0",
71
+ "@aztec/kv-store": "0.85.0",
72
+ "@aztec/noir-contracts.js": "0.85.0",
73
+ "@aztec/noir-protocol-circuits-types": "0.85.0",
74
+ "@aztec/protocol-contracts": "0.85.0",
75
+ "@aztec/simulator": "0.85.0",
76
+ "@aztec/stdlib": "0.85.0",
77
+ "@aztec/telemetry-client": "0.85.0",
78
78
  "@chainsafe/discv5": "9.0.0",
79
79
  "@chainsafe/enr": "3.0.0",
80
80
  "@chainsafe/libp2p-gossipsub": "13.0.0",
@@ -103,7 +103,7 @@
103
103
  "xxhash-wasm": "^1.1.0"
104
104
  },
105
105
  "devDependencies": {
106
- "@aztec/archiver": "0.84.0",
106
+ "@aztec/archiver": "0.85.0",
107
107
  "@jest/globals": "^29.5.0",
108
108
  "@types/jest": "^29.5.0",
109
109
  "@types/node": "^18.14.6",
@@ -38,7 +38,7 @@ export class BootstrapNode implements P2PBootstrapApi {
38
38
  throw new Error('You need to provide a P2P IP address.');
39
39
  }
40
40
 
41
- const peerIdPrivateKey = await getPeerIdPrivateKey(config, this.store);
41
+ const peerIdPrivateKey = await getPeerIdPrivateKey(config, this.store, this.logger);
42
42
 
43
43
  const { enr: ourEnr, peerId } = await createBootnodeENRandPeerId(
44
44
  peerIdPrivateKey,
@@ -43,7 +43,12 @@ export const createP2PClient = async <T extends P2PClientType>(
43
43
  const archive = await createStore('p2p-archive', 1, config, createLogger('p2p-archive:lmdb-v2'));
44
44
 
45
45
  const mempools: MemPools<T> = {
46
- txPool: deps.txPool ?? new AztecKVTxPool(store, archive, telemetry, config.archivedTxLimit),
46
+ txPool:
47
+ deps.txPool ??
48
+ new AztecKVTxPool(store, archive, worldStateSynchronizer, telemetry, {
49
+ maxTxPoolSize: config.maxTxPoolSize,
50
+ archivedTxLimit: config.archivedTxLimit,
51
+ }),
47
52
  attestationPool:
48
53
  clientType === P2PClientType.Full
49
54
  ? ((deps.attestationPool ?? new InMemoryAttestationPool(telemetry)) as T extends P2PClientType.Full
@@ -59,7 +64,7 @@ export const createP2PClient = async <T extends P2PClientType>(
59
64
  config = await configureP2PClientAddresses(_config);
60
65
 
61
66
  // Create peer discovery service
62
- const peerIdPrivateKey = await getPeerIdPrivateKey(config, store);
67
+ const peerIdPrivateKey = await getPeerIdPrivateKey(config, store, logger);
63
68
  const peerId = await createLibP2PPeerIdFromPrivateKey(peerIdPrivateKey);
64
69
  const discoveryService = new DiscV5Service(
65
70
  peerId,
@@ -372,6 +372,11 @@ export class P2PClient<T extends P2PClientType = P2PClientType.Full>
372
372
  this.log.info('P2P client stopped.');
373
373
  }
374
374
 
375
+ /** Triggers a sync to the archiver. Used for testing. */
376
+ public async sync() {
377
+ await this.blockStream.sync();
378
+ }
379
+
375
380
  @trackSpan('p2pClient.broadcastProposal', async proposal => ({
376
381
  [Attributes.BLOCK_NUMBER]: proposal.blockNumber.toNumber(),
377
382
  [Attributes.SLOT_NUMBER]: proposal.slotNumber.toNumber(),
@@ -738,7 +743,7 @@ export class P2PClient<T extends P2PClientType = P2PClientType.Full>
738
743
  this.log.info(
739
744
  `Detected chain prune. Removing invalid txs count=${
740
745
  txsToDelete.length
741
- } newLatestBlock=${latestBlock} previousLatestBlock=${this.getSyncedLatestBlockNum()}`,
746
+ } newLatestBlock=${latestBlock} previousLatestBlock=${await this.getSyncedLatestBlockNum()}`,
742
747
  );
743
748
 
744
749
  // delete invalid txs (both pending and mined)
package/src/config.ts CHANGED
@@ -68,6 +68,11 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig {
68
68
  */
69
69
  peerIdPrivateKey?: string;
70
70
 
71
+ /**
72
+ * An optional path to store generated peer id private keys. If blank, will default to storing any generated keys in the data directory.
73
+ */
74
+ peerIdPrivateKeyPath?: string;
75
+
71
76
  /**
72
77
  * A list of bootstrap peers to connect to.
73
78
  */
@@ -160,7 +165,9 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig {
160
165
  */
161
166
  peerPenaltyValues: number[];
162
167
 
163
- /** Limit of transactions to archive in the tx pool. Once the archived tx limit is reached, the oldest archived txs will be purged. */
168
+ /**
169
+ * Limit of transactions to archive in the tx pool. Once the archived tx limit is reached, the oldest archived txs will be purged.
170
+ */
164
171
  archivedTxLimit: number;
165
172
 
166
173
  /**
@@ -175,6 +182,11 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig {
175
182
 
176
183
  /** Which calls are allowed in the public setup phase of a tx. */
177
184
  txPublicSetupAllowList: AllowedElement[];
185
+
186
+ /**
187
+ * The maximum cumulative tx size (in bytes) of pending txs before evicting lower priority txs.
188
+ */
189
+ maxTxPoolSize: number;
178
190
  }
179
191
 
180
192
  export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
@@ -221,10 +233,16 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
221
233
  env: 'PEER_ID_PRIVATE_KEY',
222
234
  description: 'An optional peer id private key. If blank, will generate a random key.',
223
235
  },
236
+ peerIdPrivateKeyPath: {
237
+ env: 'PEER_ID_PRIVATE_KEY_PATH',
238
+ description:
239
+ 'An optional path to store generated peer id private keys. If blank, will default to storing any generated keys in the root of the data directory.',
240
+ },
224
241
  bootstrapNodes: {
225
242
  env: 'BOOTSTRAP_NODES',
226
243
  parseEnv: (val: string) => val.split(','),
227
244
  description: 'A list of bootstrap peer ENRs to connect to. Separated by commas.',
245
+ defaultValue: [],
228
246
  },
229
247
  bootstrapNodeEnrVersionCheck: {
230
248
  env: 'P2P_BOOTSTRAP_NODE_ENR_VERSION_CHECK',
@@ -354,6 +372,11 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
354
372
  printDefault: () =>
355
373
  'AuthRegistry, FeeJuice.increase_public_balance, Token.increase_public_balance, FPC.prepare_fee',
356
374
  },
375
+ maxTxPoolSize: {
376
+ env: 'P2P_MAX_TX_POOL_SIZE',
377
+ description: 'The maximum cumulative tx size of pending txs (in bytes) before evicting lower priority txs.',
378
+ ...numberConfigHelper(100_000_000), // 100MB
379
+ },
357
380
  ...p2pReqRespConfigMappings,
358
381
  ...chainConfigMappings,
359
382
  };
@@ -375,7 +398,7 @@ export function getP2PDefaultConfig(): P2PConfig {
375
398
  */
376
399
  export type BootnodeConfig = Pick<
377
400
  P2PConfig,
378
- 'p2pIp' | 'p2pPort' | 'peerIdPrivateKey' | 'bootstrapNodes' | 'listenAddress'
401
+ 'p2pIp' | 'p2pPort' | 'peerIdPrivateKey' | 'peerIdPrivateKeyPath' | 'bootstrapNodes' | 'listenAddress'
379
402
  > &
380
403
  Required<Pick<P2PConfig, 'p2pIp' | 'p2pPort'>> &
381
404
  Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKB'> &
@@ -386,6 +409,7 @@ const bootnodeConfigKeys: (keyof BootnodeConfig)[] = [
386
409
  'p2pPort',
387
410
  'listenAddress',
388
411
  'peerIdPrivateKey',
412
+ 'peerIdPrivateKeyPath',
389
413
  'dataDirectory',
390
414
  'dataStoreMapSizeKB',
391
415
  'bootstrapNodes',
@@ -1,11 +1,18 @@
1
+ import { Fr } from '@aztec/foundation/fields';
1
2
  import { toArray } from '@aztec/foundation/iterable';
2
3
  import { type Logger, createLogger } from '@aztec/foundation/log';
3
- import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap } from '@aztec/kv-store';
4
+ import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap, AztecAsyncSingleton } from '@aztec/kv-store';
5
+ import { ProtocolContractAddress } from '@aztec/protocol-contracts';
6
+ import { GasFees } from '@aztec/stdlib/gas';
7
+ import type { MerkleTreeReadOperations, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
4
8
  import { ClientIvcProof } from '@aztec/stdlib/proofs';
5
9
  import type { TxAddedToPoolStats } from '@aztec/stdlib/stats';
10
+ import { DatabasePublicStateSource } from '@aztec/stdlib/trees';
6
11
  import { Tx, TxHash } from '@aztec/stdlib/tx';
7
12
  import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
8
13
 
14
+ import { ArchiveCache } from '../../msg_validators/tx_validator/archive_cache.js';
15
+ import { GasTxValidator } from '../../msg_validators/tx_validator/gas_validator.js';
9
16
  import { PoolInstrumentation, PoolName } from '../instrumentation.js';
10
17
  import { getPendingTxPriority } from './priority.js';
11
18
  import type { TxPool } from './tx_pool.js';
@@ -19,12 +26,27 @@ export class AztecKVTxPool implements TxPool {
19
26
  /** Our tx pool, stored as a Map, with K: tx hash and V: the transaction. */
20
27
  #txs: AztecAsyncMap<string, Buffer>;
21
28
 
29
+ /** The maximum cumulative tx size that the pending txs in the pool take up. */
30
+ #maxTxPoolSize: number | undefined;
31
+
22
32
  /** Index from tx hash to the block number in which they were mined, filtered by mined txs. */
23
33
  #minedTxHashToBlock: AztecAsyncMap<string, number>;
24
34
 
25
35
  /** Index from tx priority (stored as hex) to its tx hash, filtered by pending txs. */
26
36
  #pendingTxPriorityToHash: AztecAsyncMultiMap<string, string>;
27
37
 
38
+ /** Index from tx hash to its tx size (in bytes), filtered by pending txs. */
39
+ #pendingTxHashToSize: AztecAsyncMap<string, number>;
40
+
41
+ /** Index from tx hash to its header hash, filtered by pending txs. */
42
+ #pendingTxHashToHeaderHash: AztecAsyncMap<string, string>;
43
+
44
+ /** The cumulative tx size in bytes that the pending txs in the pool take up. */
45
+ #pendingTxSize: AztecAsyncSingleton<number>;
46
+
47
+ /** In-memory mapping of pending tx hashes to the hydrated pending tx in the pool. */
48
+ #pendingTxs: Map<string, Tx>;
49
+
28
50
  /** KV store for archived txs. */
29
51
  #archive: AztecAsyncKVStore;
30
52
 
@@ -37,6 +59,9 @@ export class AztecKVTxPool implements TxPool {
37
59
  /** Number of txs to archive. */
38
60
  #archivedTxLimit: number;
39
61
 
62
+ /** The world state synchronizer used in the node. */
63
+ #worldStateSynchronizer: WorldStateSynchronizer;
64
+
40
65
  #log: Logger;
41
66
 
42
67
  #metrics: PoolInstrumentation<Tx>;
@@ -52,20 +77,30 @@ export class AztecKVTxPool implements TxPool {
52
77
  constructor(
53
78
  store: AztecAsyncKVStore,
54
79
  archive: AztecAsyncKVStore,
80
+ worldStateSynchronizer: WorldStateSynchronizer,
55
81
  telemetry: TelemetryClient = getTelemetryClient(),
56
- archivedTxLimit: number = 0,
82
+ config: {
83
+ maxTxPoolSize?: number;
84
+ archivedTxLimit?: number;
85
+ } = {},
57
86
  log = createLogger('p2p:tx_pool'),
58
87
  ) {
59
88
  this.#txs = store.openMap('txs');
60
89
  this.#minedTxHashToBlock = store.openMap('txHashToBlockMined');
61
90
  this.#pendingTxPriorityToHash = store.openMultiMap('pendingTxFeeToHash');
91
+ this.#pendingTxHashToSize = store.openMap('pendingTxHashToSize');
92
+ this.#pendingTxHashToHeaderHash = store.openMap('pendingTxHashToHeaderHash');
93
+ this.#pendingTxSize = store.openSingleton('pendingTxSize');
94
+ this.#maxTxPoolSize = config.maxTxPoolSize;
95
+ this.#pendingTxs = new Map<string, Tx>();
62
96
 
63
97
  this.#archivedTxs = archive.openMap('archivedTxs');
64
98
  this.#archivedTxIndices = archive.openMap('archivedTxIndices');
65
- this.#archivedTxLimit = archivedTxLimit;
99
+ this.#archivedTxLimit = config.archivedTxLimit ?? 0;
66
100
 
67
101
  this.#store = store;
68
102
  this.#archive = archive;
103
+ this.#worldStateSynchronizer = worldStateSynchronizer;
69
104
  this.#log = log;
70
105
  this.#metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, () => store.estimateSize());
71
106
  }
@@ -76,20 +111,35 @@ export class AztecKVTxPool implements TxPool {
76
111
  }
77
112
 
78
113
  let deletedPending = 0;
114
+ const minedNullifiers = new Set<string>();
115
+ const minedFeePayers = new Set<string>();
79
116
  return this.#store.transactionAsync(async () => {
117
+ let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
80
118
  for (const hash of txHashes) {
81
119
  const key = hash.toString();
82
120
  await this.#minedTxHashToBlock.set(key, blockNumber);
83
121
 
84
- const tx = await this.getTxByHash(hash);
122
+ const tx = await this.getPendingTxByHash(hash);
85
123
  if (tx) {
124
+ const nullifiers = tx.data.getNonEmptyNullifiers();
125
+ nullifiers.forEach(nullifier => minedNullifiers.add(nullifier.toString()));
126
+ minedFeePayers.add(tx.data.feePayer.toString());
127
+
86
128
  deletedPending++;
87
- const fee = getPendingTxPriority(tx);
88
- await this.#pendingTxPriorityToHash.deleteValue(fee, key);
129
+ pendingTxSize -= tx.getSize();
130
+ await this.removePendingTxIndices(tx, key);
89
131
  }
90
132
  }
91
133
  this.#metrics.recordAddedObjects(txHashes.length, 'mined');
92
- this.#metrics.recordRemovedObjects(deletedPending, 'pending');
134
+ await this.#pendingTxSize.set(pendingTxSize);
135
+
136
+ const numTxsEvicted = await this.evictInvalidTxsAfterMining(
137
+ txHashes,
138
+ blockNumber,
139
+ minedNullifiers,
140
+ minedFeePayers,
141
+ );
142
+ this.#metrics.recordRemovedObjects(deletedPending + numTxsEvicted, 'pending');
93
143
  });
94
144
  }
95
145
 
@@ -99,21 +149,35 @@ export class AztecKVTxPool implements TxPool {
99
149
  }
100
150
 
101
151
  let markedAsPending = 0;
102
- return this.#store.transactionAsync(async () => {
103
- for (const hash of txHashes) {
104
- const key = hash.toString();
105
- await this.#minedTxHashToBlock.delete(key);
152
+ return this.#store
153
+ .transactionAsync(async () => {
154
+ let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
155
+ for (const hash of txHashes) {
156
+ const key = hash.toString();
157
+ await this.#minedTxHashToBlock.delete(key);
106
158
 
107
- const tx = await this.getTxByHash(hash);
108
- if (tx) {
109
- await this.#pendingTxPriorityToHash.set(getPendingTxPriority(tx), key);
110
- markedAsPending++;
159
+ // Rehydrate the tx in the in-memory pending txs mapping
160
+ const tx = await this.getPendingTxByHash(hash);
161
+ if (tx) {
162
+ await this.addPendingTxIndices(tx, key);
163
+ pendingTxSize += tx.getSize();
164
+ markedAsPending++;
165
+ }
111
166
  }
112
- }
113
167
 
114
- this.#metrics.recordAddedObjects(markedAsPending, 'pending');
115
- this.#metrics.recordRemovedObjects(markedAsPending, 'mined');
116
- });
168
+ await this.#pendingTxSize.set(pendingTxSize);
169
+ })
170
+ .then(async () => {
171
+ const numInvalidTxsEvicted = await this.evictInvalidTxsAfterReorg(txHashes);
172
+ const { numLowPriorityTxsEvicted, numNewTxsEvicted } = await this.evictLowPriorityTxs(txHashes);
173
+
174
+ this.#metrics.recordAddedObjects(markedAsPending - numNewTxsEvicted, 'pending');
175
+ this.#metrics.recordRemovedObjects(
176
+ numInvalidTxsEvicted + numLowPriorityTxsEvicted - numNewTxsEvicted,
177
+ 'pending',
178
+ );
179
+ this.#metrics.recordRemovedObjects(markedAsPending, 'mined');
180
+ });
117
181
  }
118
182
 
119
183
  public async getPendingTxHashes(): Promise<TxHash[]> {
@@ -180,6 +244,7 @@ export class AztecKVTxPool implements TxPool {
180
244
  );
181
245
  await this.#store.transactionAsync(async () => {
182
246
  let pendingCount = 0;
247
+ let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
183
248
  await Promise.all(
184
249
  txs.map(async (tx, i) => {
185
250
  const { txHash, txStats } = hashesAndStats[i];
@@ -193,14 +258,21 @@ export class AztecKVTxPool implements TxPool {
193
258
 
194
259
  if (!(await this.#minedTxHashToBlock.hasAsync(key))) {
195
260
  pendingCount++;
196
- // REFACTOR: Use an lmdb conditional write to avoid race conditions with this write tx
197
- await this.#pendingTxPriorityToHash.set(getPendingTxPriority(tx), key);
261
+ pendingTxSize += tx.getSize();
262
+ await this.addPendingTxIndices(tx, key);
198
263
  this.#metrics.recordSize(tx);
199
264
  }
200
265
  }),
201
266
  );
202
267
 
203
- this.#metrics.recordAddedObjects(pendingCount, 'pending');
268
+ await this.#pendingTxSize.set(pendingTxSize);
269
+
270
+ const { numLowPriorityTxsEvicted, numNewTxsEvicted } = await this.evictLowPriorityTxs(
271
+ hashesAndStats.map(({ txHash }) => txHash),
272
+ );
273
+
274
+ this.#metrics.recordAddedObjects(pendingCount - numNewTxsEvicted, 'pending');
275
+ this.#metrics.recordRemovedObjects(numLowPriorityTxsEvicted - numNewTxsEvicted, 'pending');
204
276
  });
205
277
  }
206
278
 
@@ -209,28 +281,28 @@ export class AztecKVTxPool implements TxPool {
209
281
  * @param txHashes - An array of tx hashes to be removed from the tx pool.
210
282
  * @returns Empty promise.
211
283
  */
212
- public deleteTxs(txHashes: TxHash[]): Promise<void> {
284
+ public deleteTxs(txHashes: TxHash[], eviction = false): Promise<void> {
213
285
  let pendingDeleted = 0;
214
286
  let minedDeleted = 0;
215
287
 
216
288
  const deletedTxs: Tx[] = [];
217
289
  const poolDbTx = this.#store.transactionAsync(async () => {
290
+ let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
218
291
  for (const hash of txHashes) {
219
292
  const key = hash.toString();
220
293
  const tx = await this.getTxByHash(hash);
221
294
 
222
295
  if (tx) {
223
- const fee = getPendingTxPriority(tx);
224
- await this.#pendingTxPriorityToHash.deleteValue(fee, key);
225
-
226
296
  const isMined = await this.#minedTxHashToBlock.hasAsync(key);
227
297
  if (isMined) {
228
298
  minedDeleted++;
229
299
  } else {
230
300
  pendingDeleted++;
301
+ pendingTxSize -= tx.getSize();
302
+ await this.removePendingTxIndices(tx, key);
231
303
  }
232
304
 
233
- if (this.#archivedTxLimit) {
305
+ if (!eviction && this.#archivedTxLimit) {
234
306
  deletedTxs.push(tx);
235
307
  }
236
308
 
@@ -239,6 +311,7 @@ export class AztecKVTxPool implements TxPool {
239
311
  }
240
312
  }
241
313
 
314
+ await this.#pendingTxSize.set(pendingTxSize);
242
315
  this.#metrics.recordRemovedObjects(pendingDeleted, 'pending');
243
316
  this.#metrics.recordRemovedObjects(minedDeleted, 'mined');
244
317
  });
@@ -268,6 +341,50 @@ export class AztecKVTxPool implements TxPool {
268
341
  return vals.map(x => TxHash.fromString(x));
269
342
  }
270
343
 
344
+ /**
345
+ * Creates a GasTxValidator instance.
346
+ * @param db - DB for the validator to use
347
+ * @returns A GasTxValidator instance
348
+ */
349
+ protected createGasTxValidator(db: MerkleTreeReadOperations): GasTxValidator {
350
+ return new GasTxValidator(new DatabasePublicStateSource(db), ProtocolContractAddress.FeeJuice, GasFees.empty());
351
+ }
352
+
353
+ /**
354
+ * Creates an ArchiveCache instance.
355
+ * @param db - DB for the cache to use
356
+ * @returns An ArchiveCache instance
357
+ */
358
+ protected createArchiveCache(db: MerkleTreeReadOperations): ArchiveCache {
359
+ return new ArchiveCache(db);
360
+ }
361
+
362
+ /**
363
+ * Checks if a cached transaction exists in the in-memory pending tx pool and returns it.
364
+ * Otherwise, it checks the tx pool, updates the pending tx pool, and returns the tx.
365
+ * @param txHash - The generated tx hash.
366
+ * @returns The transaction, if found, 'undefined' otherwise.
367
+ */
368
+ private async getPendingTxByHash(txHash: TxHash | string): Promise<Tx | undefined> {
369
+ let key;
370
+ if (typeof txHash === 'string') {
371
+ key = txHash;
372
+ txHash = TxHash.fromString(txHash);
373
+ } else {
374
+ key = txHash.toString();
375
+ }
376
+
377
+ if (this.#pendingTxs.has(key)) {
378
+ return this.#pendingTxs.get(key);
379
+ }
380
+ const tx = await this.getTxByHash(txHash);
381
+ if (tx) {
382
+ this.#pendingTxs.set(key, tx);
383
+ return tx;
384
+ }
385
+ return undefined;
386
+ }
387
+
271
388
  /**
272
389
  * Archives a list of txs for future reference. The number of archived txs is limited by the specified archivedTxLimit.
273
390
  * @param txs - The list of transactions to archive.
@@ -300,4 +417,172 @@ export class AztecKVTxPool implements TxPool {
300
417
  }
301
418
  });
302
419
  }
420
+
421
+ /**
422
+ * Evicts pending txs with the lowest priority fees from the pool to accomodate the max tx count and cumulative max tx size
423
+ * after new txs are added.
424
+ *
425
+ * @param newTxHashes - The tx hashes of the new txs added to the pool.
426
+ * @returns The total number of txs evicted from the pool and the number of new txs that were evicted.
427
+ */
428
+ private async evictLowPriorityTxs(
429
+ newTxHashes: TxHash[],
430
+ ): Promise<{ numLowPriorityTxsEvicted: number; numNewTxsEvicted: number }> {
431
+ if (this.#maxTxPoolSize === undefined) {
432
+ return { numLowPriorityTxsEvicted: 0, numNewTxsEvicted: 0 };
433
+ }
434
+
435
+ let numNewTxsEvicted = 0;
436
+ const txsToEvict: TxHash[] = [];
437
+
438
+ let pendingTxsSize = (await this.#pendingTxSize.getAsync()) ?? 0;
439
+ if (pendingTxsSize > this.#maxTxPoolSize) {
440
+ for await (const txHash of this.#pendingTxPriorityToHash.valuesAsync()) {
441
+ this.#log.verbose(`Evicting tx ${txHash} from pool due to low priority to satisfy max tx size limit`);
442
+ txsToEvict.push(TxHash.fromString(txHash));
443
+
444
+ const txSize =
445
+ (await this.#pendingTxHashToSize.getAsync(txHash.toString())) ??
446
+ (await this.getPendingTxByHash(txHash))?.getSize();
447
+ if (txSize) {
448
+ pendingTxsSize -= txSize;
449
+ if (pendingTxsSize <= this.#maxTxPoolSize) {
450
+ break;
451
+ }
452
+ }
453
+ }
454
+ numNewTxsEvicted += newTxHashes.filter(txHash => txsToEvict.includes(txHash)).length;
455
+ }
456
+
457
+ if (txsToEvict.length > 0) {
458
+ await this.deleteTxs(txsToEvict, true);
459
+ }
460
+ return {
461
+ numLowPriorityTxsEvicted: txsToEvict.length,
462
+ numNewTxsEvicted,
463
+ };
464
+ }
465
+
466
+ /**
467
+ * Evicts invalid pending txs from the pool after a txs from a block are mined.
468
+ * Eviction criteria includes:
469
+ * - txs with nullifiers that are already included in the mined block
470
+ * - txs with an insufficient fee payer balance
471
+ * - txs with a max block number lower than the mined block
472
+ *
473
+ * @param minedTxHashes - The tx hashes of the txs mined in the block.
474
+ * @param blockNumber - The block number of the mined block.
475
+ * @returns The total number of txs evicted from the pool.
476
+ */
477
+ private async evictInvalidTxsAfterMining(
478
+ minedTxHashes: TxHash[],
479
+ blockNumber: number,
480
+ minedNullifiers: Set<string>,
481
+ minedFeePayers: Set<string>,
482
+ ): Promise<number> {
483
+ if (minedTxHashes.length === 0) {
484
+ return 0;
485
+ }
486
+
487
+ // Wait for world state to be synced to at least the mined block number
488
+ await this.#worldStateSynchronizer.syncImmediate(blockNumber);
489
+
490
+ const db = this.#worldStateSynchronizer.getCommitted();
491
+ const gasTxValidator = this.createGasTxValidator(db);
492
+
493
+ const txsToEvict: TxHash[] = [];
494
+ for await (const txHash of this.#pendingTxPriorityToHash.valuesAsync()) {
495
+ const tx = await this.getPendingTxByHash(txHash);
496
+ if (!tx) {
497
+ continue;
498
+ }
499
+
500
+ // Evict pending txs that share nullifiers with mined txs
501
+ const txNullifiers = tx.data.getNonEmptyNullifiers();
502
+ if (txNullifiers.some(nullifier => minedNullifiers.has(nullifier.toString()))) {
503
+ this.#log.verbose(`Evicting tx ${txHash} from pool due to a duplicate nullifier with a mined tx`);
504
+ txsToEvict.push(TxHash.fromString(txHash));
505
+ continue;
506
+ }
507
+
508
+ // Evict pending txs with an insufficient fee payer balance
509
+ if (
510
+ minedFeePayers.has(tx.data.feePayer.toString()) &&
511
+ (await gasTxValidator.validateTxFee(tx)).result === 'invalid'
512
+ ) {
513
+ this.#log.verbose(`Evicting tx ${txHash} from pool due to an insufficient fee payer balance`);
514
+ txsToEvict.push(TxHash.fromString(txHash));
515
+ continue;
516
+ }
517
+
518
+ // Evict pending txs with a max block number less than or equal to the mined block
519
+ const maxBlockNumber = tx.data.rollupValidationRequests.maxBlockNumber;
520
+ if (maxBlockNumber.isSome && maxBlockNumber.value.toNumber() <= blockNumber) {
521
+ this.#log.verbose(`Evicting tx ${txHash} from pool due to an invalid max block number`);
522
+ txsToEvict.push(TxHash.fromString(txHash));
523
+ continue;
524
+ }
525
+ }
526
+
527
+ if (txsToEvict.length > 0) {
528
+ await this.deleteTxs(txsToEvict, true);
529
+ }
530
+ return txsToEvict.length;
531
+ }
532
+
533
+ /**
534
+ * Evicts pending txs that no longer have valid archive roots or fee payer balances from the pool after a reorg.
535
+ *
536
+ * @param txHashes - The tx hashes of the txs that were moved from mined to pending.
537
+ * @returns The total number of txs evicted from the pool.
538
+ */
539
+ private async evictInvalidTxsAfterReorg(txHashes: TxHash[]): Promise<number> {
540
+ if (txHashes.length === 0) {
541
+ return 0;
542
+ }
543
+
544
+ await this.#worldStateSynchronizer.syncImmediate();
545
+ const db = this.#worldStateSynchronizer.getCommitted();
546
+ const archiveCache = this.createArchiveCache(db);
547
+ const gasTxValidator = this.createGasTxValidator(db);
548
+
549
+ const txsToEvict: TxHash[] = [];
550
+
551
+ for await (const [txHash, headerHash] of this.#pendingTxHashToHeaderHash.entriesAsync()) {
552
+ const tx = await this.getPendingTxByHash(txHash);
553
+ if (!tx) {
554
+ continue;
555
+ }
556
+
557
+ const [index] = await archiveCache.getArchiveIndices([Fr.fromString(headerHash)]);
558
+ if (index === undefined) {
559
+ this.#log.verbose(`Evicting tx ${txHash} from pool due to an invalid archive root`);
560
+ txsToEvict.push(TxHash.fromString(txHash));
561
+ continue;
562
+ }
563
+
564
+ if ((await gasTxValidator.validateTxFee(tx)).result === 'invalid') {
565
+ this.#log.verbose(`Evicting tx ${txHash} from pool due to an insufficient fee payer balance`);
566
+ txsToEvict.push(TxHash.fromString(txHash));
567
+ }
568
+ }
569
+
570
+ if (txsToEvict.length > 0) {
571
+ await this.deleteTxs(txsToEvict, true);
572
+ }
573
+ return txsToEvict.length;
574
+ }
575
+
576
+ private async addPendingTxIndices(tx: Tx, txHash: string): Promise<void> {
577
+ await this.#pendingTxPriorityToHash.set(getPendingTxPriority(tx), txHash);
578
+ await this.#pendingTxHashToSize.set(txHash, tx.getSize());
579
+ await this.#pendingTxHashToHeaderHash.set(txHash, (await tx.data.constants.historicalHeader.hash()).toString());
580
+ }
581
+
582
+ private async removePendingTxIndices(tx: Tx, txHash: string): Promise<void> {
583
+ await this.#pendingTxPriorityToHash.deleteValue(getPendingTxPriority(tx), txHash);
584
+ await this.#pendingTxHashToSize.delete(txHash);
585
+ await this.#pendingTxHashToHeaderHash.delete(txHash);
586
+ this.#pendingTxs.delete(txHash);
587
+ }
303
588
  }
@@ -0,0 +1,28 @@
1
+ import type { Fr } from '@aztec/foundation/fields';
2
+ import type { ArchiveSource } from '@aztec/p2p';
3
+ import type { MerkleTreeReadOperations } from '@aztec/stdlib/interfaces/server';
4
+ import { MerkleTreeId } from '@aztec/stdlib/trees';
5
+
6
+ /**
7
+ * Implements an archive source by checking a DB and an in-memory collection.
8
+ * Intended for validating transactions as they are added to a block.
9
+ */
10
+ export class ArchiveCache implements ArchiveSource {
11
+ archives: Map<string, bigint>;
12
+
13
+ constructor(private db: MerkleTreeReadOperations) {
14
+ this.archives = new Map<string, bigint>();
15
+ }
16
+
17
+ public async getArchiveIndices(archives: Fr[]): Promise<(bigint | undefined)[]> {
18
+ const toCheckDb = archives.filter(n => !this.archives.has(n.toString()));
19
+ const dbHits = await this.db.findLeafIndices(MerkleTreeId.ARCHIVE, toCheckDb);
20
+ dbHits.forEach((x, index) => {
21
+ if (x !== undefined) {
22
+ this.archives.set(toCheckDb[index].toString(), x);
23
+ }
24
+ });
25
+
26
+ return archives.map(n => this.archives.get(n.toString()));
27
+ }
28
+ }