@aztec/p2p 3.0.0-rc.5 → 4.0.0-nightly.20260107

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/dest/client/factory.d.ts +2 -2
  2. package/dest/client/factory.d.ts.map +1 -1
  3. package/dest/client/factory.js +2 -3
  4. package/dest/client/p2p_client.d.ts +2 -2
  5. package/dest/client/p2p_client.d.ts.map +1 -1
  6. package/dest/client/p2p_client.js +395 -21
  7. package/dest/config.d.ts +4 -7
  8. package/dest/config.d.ts.map +1 -1
  9. package/dest/config.js +6 -9
  10. package/dest/mem_pools/instrumentation.d.ts +7 -1
  11. package/dest/mem_pools/instrumentation.d.ts.map +1 -1
  12. package/dest/mem_pools/instrumentation.js +29 -2
  13. package/dest/mem_pools/interface.d.ts +3 -4
  14. package/dest/mem_pools/interface.d.ts.map +1 -1
  15. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts +28 -24
  16. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts.map +1 -1
  17. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.js +261 -323
  18. package/dest/mem_pools/tx_pool/eviction/eviction_manager.d.ts +18 -0
  19. package/dest/mem_pools/tx_pool/eviction/eviction_manager.d.ts.map +1 -0
  20. package/dest/mem_pools/tx_pool/eviction/eviction_manager.js +56 -0
  21. package/dest/mem_pools/tx_pool/eviction/eviction_strategy.d.ts +83 -0
  22. package/dest/mem_pools/tx_pool/eviction/eviction_strategy.d.ts.map +1 -0
  23. package/dest/mem_pools/tx_pool/eviction/eviction_strategy.js +5 -0
  24. package/dest/mem_pools/tx_pool/eviction/insufficient_fee_payer_balance_rule.d.ts +15 -0
  25. package/dest/mem_pools/tx_pool/eviction/insufficient_fee_payer_balance_rule.d.ts.map +1 -0
  26. package/dest/mem_pools/tx_pool/eviction/insufficient_fee_payer_balance_rule.js +88 -0
  27. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.d.ts +17 -0
  28. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.d.ts.map +1 -0
  29. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.js +84 -0
  30. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_reorg_rule.d.ts +19 -0
  31. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_reorg_rule.d.ts.map +1 -0
  32. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_reorg_rule.js +76 -0
  33. package/dest/mem_pools/tx_pool/eviction/low_priority_eviction_rule.d.ts +26 -0
  34. package/dest/mem_pools/tx_pool/eviction/low_priority_eviction_rule.d.ts.map +1 -0
  35. package/dest/mem_pools/tx_pool/eviction/low_priority_eviction_rule.js +84 -0
  36. package/dest/mem_pools/tx_pool/index.d.ts +1 -2
  37. package/dest/mem_pools/tx_pool/index.d.ts.map +1 -1
  38. package/dest/mem_pools/tx_pool/index.js +0 -1
  39. package/dest/mem_pools/tx_pool/priority.d.ts +5 -1
  40. package/dest/mem_pools/tx_pool/priority.d.ts.map +1 -1
  41. package/dest/mem_pools/tx_pool/priority.js +6 -1
  42. package/dest/mem_pools/tx_pool/tx_pool.d.ts +8 -4
  43. package/dest/mem_pools/tx_pool/tx_pool.d.ts.map +1 -1
  44. package/dest/mem_pools/tx_pool/tx_pool_test_suite.d.ts +1 -1
  45. package/dest/mem_pools/tx_pool/tx_pool_test_suite.d.ts.map +1 -1
  46. package/dest/mem_pools/tx_pool/tx_pool_test_suite.js +25 -20
  47. package/dest/services/libp2p/libp2p_service.d.ts +4 -4
  48. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  49. package/dest/services/libp2p/libp2p_service.js +447 -64
  50. package/dest/services/peer-manager/metrics.d.ts +6 -1
  51. package/dest/services/peer-manager/metrics.d.ts.map +1 -1
  52. package/dest/services/peer-manager/metrics.js +17 -0
  53. package/dest/services/peer-manager/peer_manager.d.ts +1 -1
  54. package/dest/services/peer-manager/peer_manager.d.ts.map +1 -1
  55. package/dest/services/peer-manager/peer_manager.js +385 -9
  56. package/dest/services/reqresp/protocols/tx.d.ts +2 -3
  57. package/dest/services/reqresp/protocols/tx.d.ts.map +1 -1
  58. package/dest/services/reqresp/reqresp.js +402 -24
  59. package/dest/services/tx_provider.d.ts +2 -1
  60. package/dest/services/tx_provider.d.ts.map +1 -1
  61. package/dest/services/tx_provider.js +11 -2
  62. package/dest/services/tx_provider_instrumentation.d.ts +5 -2
  63. package/dest/services/tx_provider_instrumentation.d.ts.map +1 -1
  64. package/dest/services/tx_provider_instrumentation.js +14 -1
  65. package/dest/test-helpers/reqresp-nodes.d.ts +2 -2
  66. package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
  67. package/dest/testbench/p2p_client_testbench_worker.js +1 -0
  68. package/package.json +14 -14
  69. package/src/client/factory.ts +5 -10
  70. package/src/client/p2p_client.ts +12 -17
  71. package/src/config.ts +8 -14
  72. package/src/mem_pools/instrumentation.ts +33 -0
  73. package/src/mem_pools/interface.ts +2 -4
  74. package/src/mem_pools/tx_pool/README.md +255 -0
  75. package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts +308 -368
  76. package/src/mem_pools/tx_pool/eviction/eviction_manager.ts +71 -0
  77. package/src/mem_pools/tx_pool/eviction/eviction_strategy.ts +93 -0
  78. package/src/mem_pools/tx_pool/eviction/insufficient_fee_payer_balance_rule.ts +108 -0
  79. package/src/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.ts +104 -0
  80. package/src/mem_pools/tx_pool/eviction/invalid_txs_after_reorg_rule.ts +91 -0
  81. package/src/mem_pools/tx_pool/eviction/low_priority_eviction_rule.ts +106 -0
  82. package/src/mem_pools/tx_pool/index.ts +0 -1
  83. package/src/mem_pools/tx_pool/priority.ts +8 -1
  84. package/src/mem_pools/tx_pool/tx_pool.ts +8 -3
  85. package/src/mem_pools/tx_pool/tx_pool_test_suite.ts +18 -13
  86. package/src/services/libp2p/libp2p_service.ts +12 -17
  87. package/src/services/peer-manager/metrics.ts +22 -0
  88. package/src/services/peer-manager/peer_manager.ts +2 -0
  89. package/src/services/reqresp/protocols/tx.ts +1 -2
  90. package/src/services/tx_provider.ts +17 -2
  91. package/src/services/tx_provider_instrumentation.ts +19 -2
  92. package/src/test-helpers/mock-pubsub.ts +1 -1
  93. package/src/test-helpers/reqresp-nodes.ts +1 -1
  94. package/src/testbench/p2p_client_testbench_worker.ts +2 -1
  95. package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts +0 -81
  96. package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts.map +0 -1
  97. package/dest/mem_pools/tx_pool/memory_tx_pool.js +0 -239
  98. package/src/mem_pools/tx_pool/memory_tx_pool.ts +0 -285
@@ -1,39 +1,38 @@
1
+ import { insertIntoSortedArray } from '@aztec/foundation/array';
1
2
  import { Fr } from '@aztec/foundation/curves/bn254';
2
3
  import { toArray } from '@aztec/foundation/iterable';
3
4
  import { createLogger } from '@aztec/foundation/log';
4
- import { ProtocolContractAddress } from '@aztec/protocol-contracts';
5
- import { GasFees } from '@aztec/stdlib/gas';
6
5
  import { ChonkProof } from '@aztec/stdlib/proofs';
7
- import { DatabasePublicStateSource } from '@aztec/stdlib/trees';
8
6
  import { Tx, TxHash } from '@aztec/stdlib/tx';
9
7
  import { getTelemetryClient } from '@aztec/telemetry-client';
10
8
  import assert from 'assert';
11
9
  import EventEmitter from 'node:events';
12
10
  import { ArchiveCache } from '../../msg_validators/tx_validator/archive_cache.js';
13
- import { GasTxValidator } from '../../msg_validators/tx_validator/gas_validator.js';
14
11
  import { PoolInstrumentation, PoolName } from '../instrumentation.js';
12
+ import { EvictionManager } from './eviction/eviction_manager.js';
13
+ import { InsufficientFeePayerBalanceRule } from './eviction/insufficient_fee_payer_balance_rule.js';
14
+ import { InvalidTxsAfterMiningRule } from './eviction/invalid_txs_after_mining_rule.js';
15
+ import { InvalidTxsAfterReorgRule } from './eviction/invalid_txs_after_reorg_rule.js';
16
+ import { LowPriorityEvictionRule } from './eviction/low_priority_eviction_rule.js';
15
17
  import { getPendingTxPriority } from './priority.js';
16
18
  /**
17
19
  * KV implementation of the Transaction Pool.
18
20
  */ export class AztecKVTxPool extends EventEmitter {
19
21
  #store;
20
22
  /** Our tx pool, stored as a Map, with K: tx hash and V: the transaction. */ #txs;
21
- /** The maximum cumulative tx size that the pending txs in the pool take up. */ #maxTxPoolSize = 0;
22
- /** The tx evicion logic will kick after pool size is greater than maxTxPoolSize * txPoolOverflowFactor */ txPoolOverflowFactor = 1;
23
+ /** Holds the historical block for each tx */ #pendingTxHashToHistoricalBlockHeaderHash;
23
24
  /** Index from tx hash to the block number in which they were mined, filtered by mined txs. */ #minedTxHashToBlock;
24
25
  /** Index from tx priority (stored as hex) to its tx hash, filtered by pending txs. */ #pendingTxPriorityToHash;
25
- /** Index from tx hash to its tx size (in bytes), filtered by pending txs. */ #pendingTxHashToSize;
26
- /** Index from tx hash to its header hash, filtered by pending txs. */ #pendingTxHashToHeaderHash;
27
26
  /** Map from tx hash to the block number it was originally mined in (for soft-deleted txs). */ #deletedMinedTxHashes;
28
27
  /** MultiMap from block number to deleted mined tx hashes for efficient cleanup. */ #blockToDeletedMinedTxHash;
29
- /** The cumulative tx size in bytes that the pending txs in the pool take up. */ #pendingTxSize;
30
- /** In-memory mapping of pending tx hashes to the hydrated pending tx in the pool. */ #pendingTxs;
28
+ #historicalHeaderToTxHash;
29
+ #feePayerToTxHash;
31
30
  /** In-memory set of txs that should not be evicted from the pool. */ #nonEvictableTxs;
32
31
  /** KV store for archived txs. */ #archive;
33
32
  /** Archived txs map for future lookup. */ #archivedTxs;
34
33
  /** Indexes of the archived txs by insertion order. */ #archivedTxIndices;
35
34
  /** Number of txs to archive. */ #archivedTxLimit = 0;
36
- /** The world state synchronizer used in the node. */ #worldStateSynchronizer;
35
+ #evictionManager;
37
36
  #log;
38
37
  #metrics;
39
38
  /**
@@ -43,25 +42,31 @@ import { getPendingTxPriority } from './priority.js';
43
42
  * @param telemetry - A telemetry client.
44
43
  * @param archivedTxLimit - The number of txs to archive.
45
44
  * @param log - A logger.
46
- */ constructor(store, archive, worldStateSynchronizer, telemetry = getTelemetryClient(), config = {}, log = createLogger('p2p:tx_pool')){
45
+ */ constructor(store, archive, worldState, telemetry = getTelemetryClient(), config = {}, log = createLogger('p2p:tx_pool')){
47
46
  super();
48
47
  this.#log = log;
48
+ this.#evictionManager = new EvictionManager(this);
49
+ this.#evictionManager.registerRule(new InvalidTxsAfterMiningRule());
50
+ this.#evictionManager.registerRule(new InvalidTxsAfterReorgRule(worldState));
51
+ this.#evictionManager.registerRule(new InsufficientFeePayerBalanceRule(worldState));
52
+ this.#evictionManager.registerRule(new LowPriorityEvictionRule({
53
+ //NOTE: 0 effectively disables low priority eviction
54
+ maxPoolSize: config.maxPendingTxCount ?? 0
55
+ }));
49
56
  this.updateConfig(config);
50
57
  this.#txs = store.openMap('txs');
51
58
  this.#minedTxHashToBlock = store.openMap('txHashToBlockMined');
52
59
  this.#pendingTxPriorityToHash = store.openMultiMap('pendingTxFeeToHash');
53
- this.#pendingTxHashToSize = store.openMap('pendingTxHashToSize');
54
- this.#pendingTxHashToHeaderHash = store.openMap('pendingTxHashToHeaderHash');
55
- this.#pendingTxSize = store.openSingleton('pendingTxSize');
56
60
  this.#deletedMinedTxHashes = store.openMap('deletedMinedTxHashes');
57
61
  this.#blockToDeletedMinedTxHash = store.openMultiMap('blockToDeletedMinedTxHash');
58
- this.#pendingTxs = new Map();
62
+ this.#pendingTxHashToHistoricalBlockHeaderHash = store.openMap('txHistoricalBlock');
63
+ this.#historicalHeaderToTxHash = store.openMultiMap('historicalHeaderToPendingTxHash');
64
+ this.#feePayerToTxHash = store.openMultiMap('feePayerToPendingTxHash');
59
65
  this.#nonEvictableTxs = new Set();
60
66
  this.#archivedTxs = archive.openMap('archivedTxs');
61
67
  this.#archivedTxIndices = archive.openMap('archivedTxIndices');
62
68
  this.#store = store;
63
69
  this.#archive = archive;
64
- this.#worldStateSynchronizer = worldStateSynchronizer;
65
70
  this.#metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, this.countTxs, ()=>store.estimateSize());
66
71
  }
67
72
  countTxs = async ()=>{
@@ -92,59 +97,61 @@ import { getPendingTxPriority } from './priority.js';
92
97
  if (txHashes.length === 0) {
93
98
  return Promise.resolve();
94
99
  }
95
- const minedNullifiers = new Set();
96
- const minedFeePayers = new Set();
97
- await this.#store.transactionAsync(async ()=>{
98
- let pendingTxSize = await this.#pendingTxSize.getAsync() ?? 0;
99
- for (const hash of txHashes){
100
- const key = hash.toString();
101
- // If this tx was previously soft-deleted, remove it from the deleted sets
102
- if (await this.#deletedMinedTxHashes.hasAsync(key)) {
103
- const originalBlock = await this.#deletedMinedTxHashes.getAsync(key);
104
- await this.#deletedMinedTxHashes.delete(key);
105
- // Remove from block-to-hash mapping
106
- if (originalBlock !== undefined) {
107
- await this.#blockToDeletedMinedTxHash.deleteValue(originalBlock, key);
100
+ const uniqueMinedNullifiers = [];
101
+ const uniqueMinedFeePayers = [];
102
+ try {
103
+ await this.#store.transactionAsync(async ()=>{
104
+ for (const hash of txHashes){
105
+ const key = hash.toString();
106
+ await this.#minedTxHashToBlock.set(key, blockHeader.globalVariables.blockNumber);
107
+ const tx = await this.getPendingTxByHash(hash);
108
+ if (tx) {
109
+ const nullifiers = tx.data.getNonEmptyNullifiers();
110
+ nullifiers.forEach((nullifier)=>insertIntoSortedArray(uniqueMinedNullifiers, nullifier, Fr.cmp, false));
111
+ insertIntoSortedArray(uniqueMinedFeePayers, tx.data.feePayer, (a, b)=>a.toField().cmp(b.toField()), false);
112
+ await this.removePendingTxIndicesInDbTx(tx, key);
113
+ }
114
+ // If this tx was previously soft-deleted, remove it from the deleted sets
115
+ if (await this.#deletedMinedTxHashes.hasAsync(key)) {
116
+ const originalBlock = await this.#deletedMinedTxHashes.getAsync(key);
117
+ await this.#deletedMinedTxHashes.delete(key);
118
+ // Remove from block-to-hash mapping
119
+ if (originalBlock !== undefined) {
120
+ await this.#blockToDeletedMinedTxHash.deleteValue(originalBlock, key);
121
+ }
108
122
  }
109
123
  }
110
- await this.#minedTxHashToBlock.set(key, blockHeader.globalVariables.blockNumber);
111
- const tx = await this.getPendingTxByHash(hash);
112
- if (tx) {
113
- const nullifiers = tx.data.getNonEmptyNullifiers();
114
- nullifiers.forEach((nullifier)=>minedNullifiers.add(nullifier.toString()));
115
- minedFeePayers.add(tx.data.feePayer.toString());
116
- pendingTxSize -= tx.getSize();
117
- await this.removePendingTxIndices(tx, key);
118
- }
119
- }
120
- await this.#pendingTxSize.set(pendingTxSize);
121
- await this.evictInvalidTxsAfterMining(txHashes, blockHeader, minedNullifiers, minedFeePayers);
122
- });
123
- // We update this after the transaction above. This ensures that the non-evictable transactions are not evicted
124
- // until any that have been mined are marked as such.
125
- // The non-evictable set is not considered when evicting transactions that are invalid after a block is mined.
126
- this.#nonEvictableTxs.clear();
124
+ });
125
+ await this.#evictionManager.evictAfterNewBlock(blockHeader, uniqueMinedNullifiers, uniqueMinedFeePayers);
126
+ this.#metrics.transactionsRemoved(txHashes.map((hash)=>hash.toBigInt()));
127
+ } catch (err) {
128
+ this.#log.warn('Unexpected error when marking txs as mined', {
129
+ err
130
+ });
131
+ }
127
132
  }
128
- async markMinedAsPending(txHashes) {
133
+ async markMinedAsPending(txHashes, latestBlock) {
129
134
  if (txHashes.length === 0) {
130
135
  return Promise.resolve();
131
136
  }
132
- await this.#store.transactionAsync(async ()=>{
133
- let pendingTxSize = await this.#pendingTxSize.getAsync() ?? 0;
134
- for (const hash of txHashes){
135
- const key = hash.toString();
136
- await this.#minedTxHashToBlock.delete(key);
137
- // Rehydrate the tx in the in-memory pending txs mapping
138
- const tx = await this.getPendingTxByHash(hash);
139
- if (tx) {
140
- await this.addPendingTxIndices(tx, key);
141
- pendingTxSize += tx.getSize();
137
+ try {
138
+ await this.#store.transactionAsync(async ()=>{
139
+ for (const hash of txHashes){
140
+ const key = hash.toString();
141
+ await this.#minedTxHashToBlock.delete(key);
142
+ // Rehydrate the tx in the in-memory pending txs mapping
143
+ const tx = await this.getPendingTxByHash(hash);
144
+ if (tx) {
145
+ await this.addPendingTxIndicesInDbTx(tx, key);
146
+ }
142
147
  }
143
- }
144
- await this.#pendingTxSize.set(pendingTxSize);
145
- });
146
- await this.evictInvalidTxsAfterReorg(txHashes);
147
- await this.evictLowPriorityTxs(txHashes);
148
+ });
149
+ await this.#evictionManager.evictAfterChainPrune(latestBlock);
150
+ } catch (err) {
151
+ this.#log.warn('Unexpected error when marking mined txs as pending', {
152
+ err
153
+ });
154
+ }
148
155
  }
149
156
  async getPendingTxHashes() {
150
157
  const vals = await toArray(this.#pendingTxPriorityToHash.valuesAsync({
@@ -152,36 +159,6 @@ import { getPendingTxPriority } from './priority.js';
152
159
  }));
153
160
  return vals.map(TxHash.fromString);
154
161
  }
155
- async getMinedTxHashes() {
156
- const vals = await toArray(this.#minedTxHashToBlock.entriesAsync());
157
- return vals.map(([txHash, blockNumber])=>[
158
- TxHash.fromString(txHash),
159
- blockNumber
160
- ]);
161
- }
162
- async getPendingTxCount() {
163
- return await this.#pendingTxHashToHeaderHash.sizeAsync() ?? 0;
164
- }
165
- async getMinedTxCount() {
166
- return await this.#minedTxHashToBlock.sizeAsync() ?? 0;
167
- }
168
- async getTxStatus(txHash) {
169
- const key = txHash.toString();
170
- const [isMined, isKnown, isDeleted] = await Promise.all([
171
- this.#minedTxHashToBlock.hasAsync(key),
172
- this.#txs.hasAsync(key),
173
- this.#deletedMinedTxHashes.hasAsync(key)
174
- ]);
175
- if (isDeleted) {
176
- return 'deleted';
177
- } else if (isMined) {
178
- return 'mined';
179
- } else if (isKnown) {
180
- return 'pending';
181
- } else {
182
- return undefined;
183
- }
184
- }
185
162
  /**
186
163
  * Checks if a transaction exists in the pool and returns it.
187
164
  * @param txHash - The generated tx hash.
@@ -214,38 +191,46 @@ import { getPendingTxPriority } from './priority.js';
214
191
  /**
215
192
  * Adds a list of transactions to the pool. Duplicates are ignored.
216
193
  * @param txs - An array of txs to be added to the pool.
217
- * @returns Empty promise.
194
+ * @returns count of added transactions
218
195
  */ async addTxs(txs, opts = {}) {
196
+ if (txs.length === 0) {
197
+ return Promise.resolve(0);
198
+ }
219
199
  const addedTxs = [];
220
200
  const hashesAndStats = txs.map((tx)=>({
221
201
  txHash: tx.getTxHash(),
222
202
  txStats: tx.getStats()
223
203
  }));
224
- await this.#store.transactionAsync(async ()=>{
225
- let pendingTxSize = await this.#pendingTxSize.getAsync() ?? 0;
226
- await Promise.all(txs.map(async (tx, i)=>{
227
- const { txHash, txStats } = hashesAndStats[i];
228
- const key = txHash.toString();
229
- if (await this.#txs.hasAsync(key)) {
230
- this.#log.debug(`Tx ${txHash.toString()} already exists in the pool`);
231
- return;
232
- }
233
- this.#log.verbose(`Adding tx ${txHash.toString()} to pool`, {
234
- eventName: 'tx-added-to-pool',
235
- ...txStats
236
- });
237
- await this.#txs.set(key, tx.toBuffer());
238
- addedTxs.push(tx);
239
- if (!await this.#minedTxHashToBlock.hasAsync(key)) {
240
- pendingTxSize += tx.getSize();
241
- await this.addPendingTxIndices(tx, key);
242
- this.#metrics.recordSize(tx);
243
- }
244
- }));
245
- await this.#pendingTxSize.set(pendingTxSize);
246
- await this.evictLowPriorityTxs(hashesAndStats.map(({ txHash })=>txHash));
247
- });
204
+ try {
205
+ await this.#store.transactionAsync(async ()=>{
206
+ await Promise.all(txs.map(async (tx, i)=>{
207
+ const { txHash, txStats } = hashesAndStats[i];
208
+ const key = txHash.toString();
209
+ if (await this.#txs.hasAsync(key)) {
210
+ this.#log.debug(`Tx ${txHash.toString()} already exists in the pool`);
211
+ return;
212
+ }
213
+ this.#log.verbose(`Adding tx ${txHash.toString()} to pool`, {
214
+ eventName: 'tx-added-to-pool',
215
+ ...txStats
216
+ });
217
+ await this.#txs.set(key, tx.toBuffer());
218
+ addedTxs.push(tx);
219
+ await this.#pendingTxHashToHistoricalBlockHeaderHash.set(key, (await tx.data.constants.anchorBlockHeader.hash()).toString());
220
+ if (!await this.#minedTxHashToBlock.hasAsync(key)) {
221
+ await this.addPendingTxIndicesInDbTx(tx, key);
222
+ this.#metrics.recordSize(tx);
223
+ }
224
+ }));
225
+ });
226
+ await this.#evictionManager.evictAfterNewTxs(addedTxs.map(({ txHash })=>txHash));
227
+ } catch (err) {
228
+ this.#log.warn('Unexpected error when adding txs', {
229
+ err
230
+ });
231
+ }
248
232
  if (addedTxs.length > 0) {
233
+ this.#metrics.transactionsAdded(addedTxs);
249
234
  this.emit('txs-added', {
250
235
  ...opts,
251
236
  txs: addedTxs
@@ -258,51 +243,57 @@ import { getPendingTxPriority } from './priority.js';
258
243
  * Mined transactions are soft-deleted with a timestamp, pending transactions are permanently deleted.
259
244
  * @param txHashes - An array of tx hashes to be deleted from the tx pool.
260
245
  * @returns Empty promise.
261
- */ deleteTxs(txHashes, opts = {}) {
246
+ */ deleteTxs(txHashes, opts) {
262
247
  if (txHashes.length === 0) {
263
248
  return Promise.resolve();
264
249
  }
265
250
  const deletedTxs = [];
266
251
  const poolDbTx = this.#store.transactionAsync(async ()=>{
267
- let pendingTxSize = await this.#pendingTxSize.getAsync() ?? 0;
268
252
  for (const hash of txHashes){
269
253
  const key = hash.toString();
270
254
  const tx = await this.getTxByHash(hash);
271
- if (tx) {
272
- const minedBlockNumber = await this.#minedTxHashToBlock.getAsync(key);
273
- if (minedBlockNumber !== undefined) {
274
- await this.#minedTxHashToBlock.delete(key);
275
- if (opts.permanently) {
276
- // Permanently delete mined transactions if specified
277
- this.#log.trace(`Deleting mined tx ${key} from pool`);
278
- await this.#txs.delete(key);
279
- } else {
280
- // Soft-delete mined transactions: remove from mined set but keep in storage
281
- this.#log.trace(`Soft-deleting mined tx ${key} from pool`);
282
- await this.#deletedMinedTxHashes.set(key, minedBlockNumber);
283
- await this.#blockToDeletedMinedTxHash.set(minedBlockNumber, key);
284
- }
285
- } else {
286
- // Permanently delete pending transactions
287
- this.#log.trace(`Deleting pending tx ${key} from pool`);
288
- pendingTxSize -= tx.getSize();
289
- await this.removePendingTxIndices(tx, key);
290
- await this.#txs.delete(key);
291
- }
292
- if (!opts.eviction && this.#archivedTxLimit) {
255
+ if (!tx) {
256
+ this.#log.trace(`Skipping deletion of missing tx ${key} from pool`);
257
+ continue;
258
+ }
259
+ const minedBlockNumber = await this.#minedTxHashToBlock.getAsync(key);
260
+ const txIsPending = minedBlockNumber === undefined;
261
+ if (txIsPending) {
262
+ await this.deletePendingTx(tx, key);
263
+ } else {
264
+ await this.deleteMinedTx(key, minedBlockNumber, opts?.permanently ?? false);
265
+ const shouldArchiveTx = this.#archivedTxLimit && !opts?.permanently;
266
+ if (shouldArchiveTx) {
293
267
  deletedTxs.push(tx);
294
268
  }
295
- } else {
296
- this.#log.trace(`Skipping deletion of missing tx ${key} from pool`);
297
269
  }
298
270
  }
299
- await this.#pendingTxSize.set(pendingTxSize);
300
271
  });
272
+ this.#metrics.transactionsRemoved(txHashes.map((hash)=>hash.toBigInt()));
301
273
  this.#log.debug(`Deleted ${txHashes.length} txs from pool`, {
302
274
  txHashes
303
275
  });
304
276
  return this.#archivedTxLimit ? poolDbTx.then(()=>this.archiveTxs(deletedTxs)) : poolDbTx;
305
277
  }
278
+ async deleteMinedTx(txHash, minedBlockNumber, permanently) {
279
+ await this.#minedTxHashToBlock.delete(txHash);
280
+ if (permanently) {
281
+ this.#log.trace(`Deleting mined tx ${txHash} from pool`);
282
+ await this.#txs.delete(txHash);
283
+ return;
284
+ }
285
+ // Soft-delete mined transactions: remove from mined set but keep in storage
286
+ this.#log.trace(`Soft-deleting mined tx ${txHash} from pool`);
287
+ await this.#deletedMinedTxHashes.set(txHash, minedBlockNumber);
288
+ await this.#blockToDeletedMinedTxHash.set(minedBlockNumber, txHash);
289
+ }
290
+ async deletePendingTx(tx, txHash) {
291
+ // We always permanently delete pending transactions
292
+ this.#log.trace(`Deleting pending tx ${txHash} from pool`);
293
+ await this.removePendingTxIndices(tx, txHash);
294
+ await this.#txs.delete(txHash);
295
+ await this.#pendingTxHashToHistoricalBlockHeaderHash.delete(txHash);
296
+ }
306
297
  /**
307
298
  * Gets all the transactions stored in the pool.
308
299
  * @returns Array of tx objects in the order they were added to the pool.
@@ -317,45 +308,110 @@ import { getPendingTxPriority } from './priority.js';
317
308
  const vals = await toArray(this.#txs.keysAsync());
318
309
  return vals.map((x)=>TxHash.fromString(x));
319
310
  }
320
- updateConfig({ maxTxPoolSize, txPoolOverflowFactor, archivedTxLimit }) {
321
- if (typeof maxTxPoolSize === 'number') {
322
- assert(maxTxPoolSize >= 0, 'maxTxPoolSize must be greater or equal to 0');
323
- this.#maxTxPoolSize = maxTxPoolSize;
324
- if (maxTxPoolSize === 0) {
325
- this.#log.info(`Disabling maximum tx mempool size. Tx eviction stopped`);
326
- } else {
327
- this.#log.info(`Setting maximum tx mempool size`, {
328
- maxTxPoolSize
329
- });
311
+ async getPendingTxInfos() {
312
+ const vals = await toArray(this.#pendingTxPriorityToHash.valuesAsync());
313
+ const results = await Promise.all(vals.map((val)=>this.getPendingTxInfo(TxHash.fromString(val))));
314
+ return results.filter((info)=>info !== undefined);
315
+ }
316
+ async getPendingTxInfo(txHash) {
317
+ let historicalBlockHash = await this.#pendingTxHashToHistoricalBlockHeaderHash.getAsync(txHash.toString());
318
+ // Not all tx might have this index created.
319
+ if (!historicalBlockHash) {
320
+ const tx = await this.getPendingTxByHash(txHash);
321
+ if (!tx) {
322
+ this.#log.warn(`PendingTxInfo:tx ${txHash} not found`);
323
+ return undefined;
330
324
  }
325
+ historicalBlockHash = (await tx.data.constants.anchorBlockHeader.hash()).toString();
326
+ await this.#pendingTxHashToHistoricalBlockHeaderHash.set(txHash.toString(), historicalBlockHash);
331
327
  }
332
- if (typeof txPoolOverflowFactor === 'number') {
333
- assert(txPoolOverflowFactor >= 1, 'txPoolOveflowFactor must be greater or equal to 1');
334
- this.txPoolOverflowFactor = txPoolOverflowFactor;
335
- this.#log.info(`Allowing tx pool size to grow above limit`, {
336
- maxTxPoolSize,
337
- txPoolOverflowFactor
338
- });
328
+ return {
329
+ txHash,
330
+ blockHash: Fr.fromString(historicalBlockHash),
331
+ isEvictable: !this.#nonEvictableTxs.has(txHash.toString())
332
+ };
333
+ }
334
+ async getPendingTxsReferencingBlocks(blockHashes) {
335
+ const result = [];
336
+ for (const blockHash of blockHashes){
337
+ const chunk = await toArray(this.#historicalHeaderToTxHash.getValuesAsync(blockHash.toString()));
338
+ result.push(...chunk.map((txHash)=>({
339
+ txHash: TxHash.fromString(txHash),
340
+ blockHash,
341
+ isEvictable: !this.#nonEvictableTxs.has(txHash)
342
+ })));
343
+ }
344
+ return result;
345
+ }
346
+ async getPendingTxsWithFeePayer(feePayers) {
347
+ const result = [];
348
+ for (const feePayer of feePayers){
349
+ const chunk = await toArray(this.#feePayerToTxHash.getValuesAsync(feePayer.toString()));
350
+ const infos = await Promise.all(chunk.map((txHash)=>this.getPendingTxInfo(TxHash.fromString(txHash))));
351
+ result.push(...infos.filter((info)=>info !== undefined));
352
+ }
353
+ return result;
354
+ }
355
+ async getMinedTxHashes() {
356
+ const vals = await toArray(this.#minedTxHashToBlock.entriesAsync());
357
+ return vals.map(([txHash, blockNumber])=>[
358
+ TxHash.fromString(txHash),
359
+ blockNumber
360
+ ]);
361
+ }
362
+ async getPendingTxCount() {
363
+ return await this.#pendingTxPriorityToHash.sizeAsync() ?? 0;
364
+ }
365
+ async getMinedTxCount() {
366
+ return await this.#minedTxHashToBlock.sizeAsync() ?? 0;
367
+ }
368
+ async getTxStatus(txHash) {
369
+ const key = txHash.toString();
370
+ const [isMined, isKnown, isDeleted] = await Promise.all([
371
+ this.#minedTxHashToBlock.hasAsync(key),
372
+ this.#txs.hasAsync(key),
373
+ this.#deletedMinedTxHashes.hasAsync(key)
374
+ ]);
375
+ if (isDeleted) {
376
+ return 'deleted';
377
+ } else if (isMined) {
378
+ return 'mined';
379
+ } else if (isKnown) {
380
+ return 'pending';
381
+ } else {
382
+ return undefined;
383
+ }
384
+ }
385
+ updateConfig(cfg) {
386
+ if (typeof cfg.archivedTxLimit === 'number') {
387
+ assert(cfg.archivedTxLimit >= 0, 'archivedTxLimit must be greater or equal to 0');
388
+ this.#archivedTxLimit = cfg.archivedTxLimit;
339
389
  }
340
- if (typeof archivedTxLimit === 'number') {
341
- assert(archivedTxLimit >= 0, 'archivedTxLimit must be greater or equal to 0');
342
- this.#archivedTxLimit = archivedTxLimit;
390
+ if (this.#evictionManager) {
391
+ this.#evictionManager.updateConfig(cfg);
343
392
  }
344
- // deletedMinedCleanupThresholdMs is no longer used in block-based cleanup
345
393
  }
346
394
  markTxsAsNonEvictable(txHashes) {
347
395
  txHashes.forEach((txHash)=>this.#nonEvictableTxs.add(txHash.toString()));
348
396
  return Promise.resolve();
349
397
  }
398
+ clearNonEvictableTxs() {
399
+ // Clear the non-evictable set after completing the DB updates above.
400
+ // This ensures pinned (non-evictable) txs are protected while we mark mined txs,
401
+ // but they won't remain pinned indefinitely across blocks. Note that eviction rules
402
+ // (including post-mining invalidation) respect the non-evictable flag while it is set.
403
+ this.#nonEvictableTxs.clear();
404
+ return Promise.resolve();
405
+ }
350
406
  /**
351
407
  * Permanently deletes deleted mined transactions from blocks up to and including the specified block number.
352
408
  * @param blockNumber - Block number threshold. Deleted mined txs from this block or earlier will be permanently deleted.
353
409
  * @returns The number of transactions permanently deleted.
354
410
  */ async cleanupDeletedMinedTxs(blockNumber) {
355
411
  let deletedCount = 0;
356
- const txHashesToDelete = [];
357
- const blocksToDelete = [];
358
412
  await this.#store.transactionAsync(async ()=>{
413
+ const txHashesToDelete = [];
414
+ const blocksToDelete = [];
359
415
  // Iterate through all entries and check block numbers
360
416
  for await (const [block, txHash] of this.#blockToDeletedMinedTxHash.entriesAsync()){
361
417
  if (block <= blockNumber) {
@@ -369,6 +425,7 @@ import { getPendingTxPriority } from './priority.js';
369
425
  deletedCount++;
370
426
  }
371
427
  }
428
+ this.#metrics.transactionsRemoved(txHashesToDelete);
372
429
  // Clean up block-to-hash mapping - delete all values for each block
373
430
  for (const block of blocksToDelete){
374
431
  const txHashesForBlock = await toArray(this.#blockToDeletedMinedTxHash.getValuesAsync(block));
@@ -383,13 +440,6 @@ import { getPendingTxPriority } from './priority.js';
383
440
  return deletedCount;
384
441
  }
385
442
  /**
386
- * Creates a GasTxValidator instance.
387
- * @param db - DB for the validator to use
388
- * @returns A GasTxValidator instance
389
- */ createGasTxValidator(db) {
390
- return new GasTxValidator(new DatabasePublicStateSource(db), ProtocolContractAddress.FeeJuice, GasFees.empty());
391
- }
392
- /**
393
443
  * Creates an ArchiveCache instance.
394
444
  * @param db - DB for the cache to use
395
445
  * @returns An ArchiveCache instance
@@ -402,31 +452,27 @@ import { getPendingTxPriority } from './priority.js';
402
452
  * @param txHash - The generated tx hash.
403
453
  * @returns The transaction, if found, 'undefined' otherwise.
404
454
  */ async getPendingTxByHash(txHash) {
405
- let key;
406
455
  if (typeof txHash === 'string') {
407
- key = txHash;
408
456
  txHash = TxHash.fromString(txHash);
409
- } else {
410
- key = txHash.toString();
411
- }
412
- if (this.#pendingTxs.has(key)) {
413
- return this.#pendingTxs.get(key);
414
457
  }
415
458
  const tx = await this.getTxByHash(txHash);
416
459
  if (tx) {
417
- this.#pendingTxs.set(key, tx);
418
460
  return tx;
419
461
  }
420
462
  return undefined;
421
463
  }
422
464
  /**
423
465
  * Archives a list of txs for future reference. The number of archived txs is limited by the specified archivedTxLimit.
466
+ * Note: Pending txs should not be archived, only finalized txs
424
467
  * @param txs - The list of transactions to archive.
425
468
  * @returns Empty promise.
426
469
  */ async archiveTxs(txs) {
427
470
  if (txs.length === 0) {
428
471
  return;
429
472
  }
473
+ if (this.#archivedTxLimit === 0) {
474
+ return;
475
+ }
430
476
  try {
431
477
  const txHashes = await Promise.all(txs.map((tx)=>tx.getTxHash()));
432
478
  await this.#archive.transactionAsync(async ()=>{
@@ -465,153 +511,45 @@ import { getPendingTxPriority } from './priority.js';
465
511
  });
466
512
  }
467
513
  }
468
- /**
469
- * Evicts pending txs with the lowest priority fees from the pool to accomodate the max tx count and cumulative max tx size
470
- * after new txs are added.
471
- *
472
- * @param newTxHashes - The tx hashes of the new txs added to the pool.
473
- * @returns The total number of txs evicted from the pool and the number of new txs that were evicted.
474
- */ async evictLowPriorityTxs(newTxHashes) {
475
- if (this.#maxTxPoolSize === undefined || this.#maxTxPoolSize === 0) {
476
- return {
477
- numLowPriorityTxsEvicted: 0,
478
- numNewTxsEvicted: 0
479
- };
480
- }
481
- let numNewTxsEvicted = 0;
482
- const txsToEvict = [];
483
- let pendingTxsSize = await this.#pendingTxSize.getAsync() ?? 0;
484
- if (pendingTxsSize > this.#maxTxPoolSize * this.txPoolOverflowFactor) {
485
- for await (const txHash of this.#pendingTxPriorityToHash.valuesAsync()){
486
- if (this.#nonEvictableTxs.has(txHash.toString())) {
487
- continue;
488
- }
489
- const txSize = await this.#pendingTxHashToSize.getAsync(txHash.toString()) ?? (await this.getPendingTxByHash(txHash))?.getSize();
490
- this.#log.verbose(`Evicting tx ${txHash} from pool due to low priority to satisfy max tx size limit`, {
491
- txHash,
492
- txSize
493
- });
494
- txsToEvict.push(TxHash.fromString(txHash));
495
- if (txSize) {
496
- pendingTxsSize -= txSize;
497
- if (pendingTxsSize <= this.#maxTxPoolSize) {
498
- break;
499
- }
500
- }
501
- }
502
- numNewTxsEvicted += newTxHashes.filter((txHash)=>txsToEvict.includes(txHash)).length;
503
- }
504
- if (txsToEvict.length > 0) {
505
- await this.deleteTxs(txsToEvict, {
506
- eviction: true
507
- });
508
- }
509
- return {
510
- numLowPriorityTxsEvicted: txsToEvict.length,
511
- numNewTxsEvicted
512
- };
514
+ // Assumes being called within a DB transaction
515
+ async addPendingTxIndicesInDbTx(tx, txHash) {
516
+ await this.#pendingTxPriorityToHash.set(getPendingTxPriority(tx), txHash);
517
+ await this.#historicalHeaderToTxHash.set((await tx.data.constants.anchorBlockHeader.hash()).toString(), txHash);
518
+ await this.#feePayerToTxHash.set(tx.data.feePayer.toString(), txHash);
513
519
  }
514
- /**
515
- * Evicts invalid pending txs from the pool after a txs from a block are mined.
516
- * Eviction criteria includes:
517
- * - txs with nullifiers that are already included in the mined block
518
- * - txs with an insufficient fee payer balance
519
- * - txs with an expiration timestamp lower than that of the mined block
520
- *
521
- * @param minedTxHashes - The tx hashes of the txs mined in the block.
522
- * @param blockHeader - The header of the mined block.
523
- * @returns The total number of txs evicted from the pool.
524
- */ async evictInvalidTxsAfterMining(minedTxHashes, blockHeader, minedNullifiers, minedFeePayers) {
525
- if (minedTxHashes.length === 0) {
526
- return 0;
527
- }
528
- const { blockNumber, timestamp } = blockHeader.globalVariables;
529
- // Wait for world state to be synced to at least the mined block number
530
- await this.#worldStateSynchronizer.syncImmediate(blockNumber);
531
- const db = this.#worldStateSynchronizer.getCommitted();
532
- const gasTxValidator = this.createGasTxValidator(db);
533
- const txsToEvict = [];
534
- for await (const txHash of this.#pendingTxPriorityToHash.valuesAsync()){
535
- const tx = await this.getPendingTxByHash(txHash);
536
- if (!tx) {
537
- continue;
538
- }
539
- // Evict pending txs that share nullifiers with mined txs
540
- const txNullifiers = tx.data.getNonEmptyNullifiers();
541
- if (txNullifiers.some((nullifier)=>minedNullifiers.has(nullifier.toString()))) {
542
- this.#log.verbose(`Evicting tx ${txHash} from pool due to a duplicate nullifier with a mined tx`);
543
- txsToEvict.push(TxHash.fromString(txHash));
544
- continue;
545
- }
546
- // Evict pending txs with an insufficient fee payer balance
547
- if (minedFeePayers.has(tx.data.feePayer.toString()) && (await gasTxValidator.validateTxFee(tx)).result === 'invalid') {
548
- this.#log.verbose(`Evicting tx ${txHash} from pool due to an insufficient fee payer balance`);
549
- txsToEvict.push(TxHash.fromString(txHash));
550
- continue;
551
- }
552
- // Evict pending txs with an expiration timestamp less than or equal to the mined block timestamp
553
- const includeByTimestamp = tx.data.includeByTimestamp;
554
- if (includeByTimestamp <= timestamp) {
555
- this.#log.verbose(`Evicting tx ${txHash} from pool due to the tx being expired (includeByTimestamp: ${includeByTimestamp}, mined block timestamp: ${timestamp})`);
556
- txsToEvict.push(TxHash.fromString(txHash));
557
- continue;
558
- }
559
- }
560
- if (txsToEvict.length > 0) {
561
- await this.deleteTxs(txsToEvict, {
562
- eviction: true
563
- });
564
- }
565
- return txsToEvict.length;
520
+ async addPendingTxIndices(tx, txHash) {
521
+ return await this.#store.transactionAsync(async ()=>{
522
+ await this.addPendingTxIndicesInDbTx(tx, txHash);
523
+ });
524
+ }
525
+ // Assumes being called within a DB transaction
526
+ async removePendingTxIndicesInDbTx(tx, txHash) {
527
+ await this.#pendingTxPriorityToHash.deleteValue(getPendingTxPriority(tx), txHash);
528
+ await this.#historicalHeaderToTxHash.deleteValue((await tx.data.constants.anchorBlockHeader.hash()).toString(), txHash);
529
+ await this.#feePayerToTxHash.deleteValue(tx.data.feePayer.toString(), txHash);
530
+ }
531
+ async removePendingTxIndices(tx, txHash) {
532
+ return await this.#store.transactionAsync(async ()=>{
533
+ await this.removePendingTxIndicesInDbTx(tx, txHash);
534
+ });
566
535
  }
567
536
  /**
568
- * Evicts pending txs that no longer have valid archive roots or fee payer balances from the pool after a reorg.
569
- *
570
- * @param txHashes - The tx hashes of the txs that were moved from mined to pending.
571
- * @returns The total number of txs evicted from the pool.
572
- */ async evictInvalidTxsAfterReorg(txHashes) {
573
- if (txHashes.length === 0) {
574
- return 0;
575
- }
576
- await this.#worldStateSynchronizer.syncImmediate();
577
- const db = this.#worldStateSynchronizer.getCommitted();
578
- const archiveCache = this.createArchiveCache(db);
579
- const gasTxValidator = this.createGasTxValidator(db);
537
+ * Returns up to `limit` lowest-priority evictable pending tx hashes without hydrating transactions.
538
+ * Iterates the priority index in ascending order and skips non-evictable txs.
539
+ */ async getLowestPriorityEvictable(limit) {
580
540
  const txsToEvict = [];
581
- for await (const [txHash, headerHash] of this.#pendingTxHashToHeaderHash.entriesAsync()){
582
- const tx = await this.getPendingTxByHash(txHash);
583
- if (!tx) {
584
- continue;
585
- }
586
- const [index] = await archiveCache.getArchiveIndices([
587
- Fr.fromString(headerHash)
588
- ]);
589
- if (index === undefined) {
590
- this.#log.verbose(`Evicting tx ${txHash} from pool due to an invalid archive root`);
591
- txsToEvict.push(TxHash.fromString(txHash));
541
+ if (limit <= 0) {
542
+ return txsToEvict;
543
+ }
544
+ for await (const txHashStr of this.#pendingTxPriorityToHash.valuesAsync()){
545
+ if (this.#nonEvictableTxs.has(txHashStr)) {
592
546
  continue;
593
547
  }
594
- if ((await gasTxValidator.validateTxFee(tx)).result === 'invalid') {
595
- this.#log.verbose(`Evicting tx ${txHash} from pool due to an insufficient fee payer balance`);
596
- txsToEvict.push(TxHash.fromString(txHash));
548
+ txsToEvict.push(TxHash.fromString(txHashStr));
549
+ if (txsToEvict.length >= limit) {
550
+ break;
597
551
  }
598
552
  }
599
- if (txsToEvict.length > 0) {
600
- await this.deleteTxs(txsToEvict, {
601
- eviction: true
602
- });
603
- }
604
- return txsToEvict.length;
605
- }
606
- async addPendingTxIndices(tx, txHash) {
607
- await this.#pendingTxPriorityToHash.set(getPendingTxPriority(tx), txHash);
608
- await this.#pendingTxHashToSize.set(txHash, tx.getSize());
609
- await this.#pendingTxHashToHeaderHash.set(txHash, (await tx.data.constants.anchorBlockHeader.hash()).toString());
610
- }
611
- async removePendingTxIndices(tx, txHash) {
612
- await this.#pendingTxPriorityToHash.deleteValue(getPendingTxPriority(tx), txHash);
613
- await this.#pendingTxHashToSize.delete(txHash);
614
- await this.#pendingTxHashToHeaderHash.delete(txHash);
615
- this.#pendingTxs.delete(txHash);
553
+ return txsToEvict;
616
554
  }
617
555
  }