@aztec/p2p 0.84.0 → 0.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/bootstrap/bootstrap.js +1 -1
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +5 -2
- package/dest/client/p2p_client.d.ts +2 -0
- package/dest/client/p2p_client.d.ts.map +1 -1
- package/dest/client/p2p_client.js +4 -1
- package/dest/config.d.ts +15 -3
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +12 -1
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts +56 -2
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.js +229 -16
- package/dest/msg_validators/tx_validator/archive_cache.d.ts +14 -0
- package/dest/msg_validators/tx_validator/archive_cache.d.ts.map +1 -0
- package/dest/msg_validators/tx_validator/archive_cache.js +22 -0
- package/dest/msg_validators/tx_validator/block_header_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/block_header_validator.js +2 -2
- package/dest/msg_validators/tx_validator/data_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/data_validator.js +8 -8
- package/dest/msg_validators/tx_validator/double_spend_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/double_spend_validator.js +3 -3
- package/dest/msg_validators/tx_validator/gas_validator.d.ts +2 -1
- package/dest/msg_validators/tx_validator/gas_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/gas_validator.js +32 -5
- package/dest/msg_validators/tx_validator/index.d.ts +1 -0
- package/dest/msg_validators/tx_validator/index.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/index.js +1 -0
- package/dest/msg_validators/tx_validator/metadata_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/metadata_validator.js +4 -4
- package/dest/msg_validators/tx_validator/phases_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/phases_validator.js +10 -2
- package/dest/msg_validators/tx_validator/tx_proof_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/tx_proof_validator.js +2 -2
- package/dest/services/discv5/discV5_service.d.ts +1 -2
- package/dest/services/discv5/discV5_service.d.ts.map +1 -1
- package/dest/services/discv5/discV5_service.js +6 -8
- package/dest/services/dummy_service.d.ts +2 -1
- package/dest/services/dummy_service.d.ts.map +1 -1
- package/dest/services/dummy_service.js +1 -1
- package/dest/services/libp2p/libp2p_service.d.ts +14 -8
- package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
- package/dest/services/libp2p/libp2p_service.js +2 -2
- package/dest/services/peer-manager/peer_manager.d.ts.map +1 -1
- package/dest/services/peer-manager/peer_manager.js +0 -1
- package/dest/services/service.d.ts +1 -1
- package/dest/services/service.d.ts.map +1 -1
- package/dest/testbench/p2p_client_testbench_worker.js +46 -16
- package/dest/testbench/parse_log_file.js +4 -4
- package/dest/testbench/testbench.js +1 -1
- package/dest/testbench/worker_client_manager.d.ts.map +1 -1
- package/dest/testbench/worker_client_manager.js +3 -2
- package/dest/util.d.ts +7 -3
- package/dest/util.d.ts.map +1 -1
- package/dest/util.js +44 -7
- package/package.json +12 -12
- package/src/bootstrap/bootstrap.ts +1 -1
- package/src/client/factory.ts +7 -2
- package/src/client/p2p_client.ts +6 -1
- package/src/config.ts +26 -2
- package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts +312 -27
- package/src/msg_validators/tx_validator/archive_cache.ts +28 -0
- package/src/msg_validators/tx_validator/block_header_validator.ts +2 -2
- package/src/msg_validators/tx_validator/data_validator.ts +19 -8
- package/src/msg_validators/tx_validator/double_spend_validator.ts +10 -3
- package/src/msg_validators/tx_validator/gas_validator.ts +36 -6
- package/src/msg_validators/tx_validator/index.ts +1 -0
- package/src/msg_validators/tx_validator/metadata_validator.ts +12 -4
- package/src/msg_validators/tx_validator/phases_validator.ts +6 -1
- package/src/msg_validators/tx_validator/tx_proof_validator.ts +2 -2
- package/src/services/discv5/discV5_service.ts +10 -8
- package/src/services/dummy_service.ts +2 -1
- package/src/services/libp2p/libp2p_service.ts +9 -9
- package/src/services/peer-manager/peer_manager.ts +1 -1
- package/src/services/service.ts +1 -1
- package/src/testbench/p2p_client_testbench_worker.ts +97 -16
- package/src/testbench/parse_log_file.ts +4 -4
- package/src/testbench/testbench.ts +1 -1
- package/src/testbench/worker_client_manager.ts +4 -2
- package/src/util.ts +57 -8
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/p2p",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.85.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./dest/index.js",
|
|
@@ -65,16 +65,16 @@
|
|
|
65
65
|
]
|
|
66
66
|
},
|
|
67
67
|
"dependencies": {
|
|
68
|
-
"@aztec/constants": "0.
|
|
69
|
-
"@aztec/epoch-cache": "0.
|
|
70
|
-
"@aztec/foundation": "0.
|
|
71
|
-
"@aztec/kv-store": "0.
|
|
72
|
-
"@aztec/noir-contracts.js": "0.
|
|
73
|
-
"@aztec/noir-protocol-circuits-types": "0.
|
|
74
|
-
"@aztec/protocol-contracts": "0.
|
|
75
|
-
"@aztec/simulator": "0.
|
|
76
|
-
"@aztec/stdlib": "0.
|
|
77
|
-
"@aztec/telemetry-client": "0.
|
|
68
|
+
"@aztec/constants": "0.85.0",
|
|
69
|
+
"@aztec/epoch-cache": "0.85.0",
|
|
70
|
+
"@aztec/foundation": "0.85.0",
|
|
71
|
+
"@aztec/kv-store": "0.85.0",
|
|
72
|
+
"@aztec/noir-contracts.js": "0.85.0",
|
|
73
|
+
"@aztec/noir-protocol-circuits-types": "0.85.0",
|
|
74
|
+
"@aztec/protocol-contracts": "0.85.0",
|
|
75
|
+
"@aztec/simulator": "0.85.0",
|
|
76
|
+
"@aztec/stdlib": "0.85.0",
|
|
77
|
+
"@aztec/telemetry-client": "0.85.0",
|
|
78
78
|
"@chainsafe/discv5": "9.0.0",
|
|
79
79
|
"@chainsafe/enr": "3.0.0",
|
|
80
80
|
"@chainsafe/libp2p-gossipsub": "13.0.0",
|
|
@@ -103,7 +103,7 @@
|
|
|
103
103
|
"xxhash-wasm": "^1.1.0"
|
|
104
104
|
},
|
|
105
105
|
"devDependencies": {
|
|
106
|
-
"@aztec/archiver": "0.
|
|
106
|
+
"@aztec/archiver": "0.85.0",
|
|
107
107
|
"@jest/globals": "^29.5.0",
|
|
108
108
|
"@types/jest": "^29.5.0",
|
|
109
109
|
"@types/node": "^18.14.6",
|
|
@@ -38,7 +38,7 @@ export class BootstrapNode implements P2PBootstrapApi {
|
|
|
38
38
|
throw new Error('You need to provide a P2P IP address.');
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
const peerIdPrivateKey = await getPeerIdPrivateKey(config, this.store);
|
|
41
|
+
const peerIdPrivateKey = await getPeerIdPrivateKey(config, this.store, this.logger);
|
|
42
42
|
|
|
43
43
|
const { enr: ourEnr, peerId } = await createBootnodeENRandPeerId(
|
|
44
44
|
peerIdPrivateKey,
|
package/src/client/factory.ts
CHANGED
|
@@ -43,7 +43,12 @@ export const createP2PClient = async <T extends P2PClientType>(
|
|
|
43
43
|
const archive = await createStore('p2p-archive', 1, config, createLogger('p2p-archive:lmdb-v2'));
|
|
44
44
|
|
|
45
45
|
const mempools: MemPools<T> = {
|
|
46
|
-
txPool:
|
|
46
|
+
txPool:
|
|
47
|
+
deps.txPool ??
|
|
48
|
+
new AztecKVTxPool(store, archive, worldStateSynchronizer, telemetry, {
|
|
49
|
+
maxTxPoolSize: config.maxTxPoolSize,
|
|
50
|
+
archivedTxLimit: config.archivedTxLimit,
|
|
51
|
+
}),
|
|
47
52
|
attestationPool:
|
|
48
53
|
clientType === P2PClientType.Full
|
|
49
54
|
? ((deps.attestationPool ?? new InMemoryAttestationPool(telemetry)) as T extends P2PClientType.Full
|
|
@@ -59,7 +64,7 @@ export const createP2PClient = async <T extends P2PClientType>(
|
|
|
59
64
|
config = await configureP2PClientAddresses(_config);
|
|
60
65
|
|
|
61
66
|
// Create peer discovery service
|
|
62
|
-
const peerIdPrivateKey = await getPeerIdPrivateKey(config, store);
|
|
67
|
+
const peerIdPrivateKey = await getPeerIdPrivateKey(config, store, logger);
|
|
63
68
|
const peerId = await createLibP2PPeerIdFromPrivateKey(peerIdPrivateKey);
|
|
64
69
|
const discoveryService = new DiscV5Service(
|
|
65
70
|
peerId,
|
package/src/client/p2p_client.ts
CHANGED
|
@@ -372,6 +372,11 @@ export class P2PClient<T extends P2PClientType = P2PClientType.Full>
|
|
|
372
372
|
this.log.info('P2P client stopped.');
|
|
373
373
|
}
|
|
374
374
|
|
|
375
|
+
/** Triggers a sync to the archiver. Used for testing. */
|
|
376
|
+
public async sync() {
|
|
377
|
+
await this.blockStream.sync();
|
|
378
|
+
}
|
|
379
|
+
|
|
375
380
|
@trackSpan('p2pClient.broadcastProposal', async proposal => ({
|
|
376
381
|
[Attributes.BLOCK_NUMBER]: proposal.blockNumber.toNumber(),
|
|
377
382
|
[Attributes.SLOT_NUMBER]: proposal.slotNumber.toNumber(),
|
|
@@ -738,7 +743,7 @@ export class P2PClient<T extends P2PClientType = P2PClientType.Full>
|
|
|
738
743
|
this.log.info(
|
|
739
744
|
`Detected chain prune. Removing invalid txs count=${
|
|
740
745
|
txsToDelete.length
|
|
741
|
-
} newLatestBlock=${latestBlock} previousLatestBlock=${this.getSyncedLatestBlockNum()}`,
|
|
746
|
+
} newLatestBlock=${latestBlock} previousLatestBlock=${await this.getSyncedLatestBlockNum()}`,
|
|
742
747
|
);
|
|
743
748
|
|
|
744
749
|
// delete invalid txs (both pending and mined)
|
package/src/config.ts
CHANGED
|
@@ -68,6 +68,11 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig {
|
|
|
68
68
|
*/
|
|
69
69
|
peerIdPrivateKey?: string;
|
|
70
70
|
|
|
71
|
+
/**
|
|
72
|
+
* An optional path to store generated peer id private keys. If blank, will default to storing any generated keys in the data directory.
|
|
73
|
+
*/
|
|
74
|
+
peerIdPrivateKeyPath?: string;
|
|
75
|
+
|
|
71
76
|
/**
|
|
72
77
|
* A list of bootstrap peers to connect to.
|
|
73
78
|
*/
|
|
@@ -160,7 +165,9 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig {
|
|
|
160
165
|
*/
|
|
161
166
|
peerPenaltyValues: number[];
|
|
162
167
|
|
|
163
|
-
/**
|
|
168
|
+
/**
|
|
169
|
+
* Limit of transactions to archive in the tx pool. Once the archived tx limit is reached, the oldest archived txs will be purged.
|
|
170
|
+
*/
|
|
164
171
|
archivedTxLimit: number;
|
|
165
172
|
|
|
166
173
|
/**
|
|
@@ -175,6 +182,11 @@ export interface P2PConfig extends P2PReqRespConfig, ChainConfig {
|
|
|
175
182
|
|
|
176
183
|
/** Which calls are allowed in the public setup phase of a tx. */
|
|
177
184
|
txPublicSetupAllowList: AllowedElement[];
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* The maximum cumulative tx size (in bytes) of pending txs before evicting lower priority txs.
|
|
188
|
+
*/
|
|
189
|
+
maxTxPoolSize: number;
|
|
178
190
|
}
|
|
179
191
|
|
|
180
192
|
export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
|
|
@@ -221,10 +233,16 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
|
|
|
221
233
|
env: 'PEER_ID_PRIVATE_KEY',
|
|
222
234
|
description: 'An optional peer id private key. If blank, will generate a random key.',
|
|
223
235
|
},
|
|
236
|
+
peerIdPrivateKeyPath: {
|
|
237
|
+
env: 'PEER_ID_PRIVATE_KEY_PATH',
|
|
238
|
+
description:
|
|
239
|
+
'An optional path to store generated peer id private keys. If blank, will default to storing any generated keys in the root of the data directory.',
|
|
240
|
+
},
|
|
224
241
|
bootstrapNodes: {
|
|
225
242
|
env: 'BOOTSTRAP_NODES',
|
|
226
243
|
parseEnv: (val: string) => val.split(','),
|
|
227
244
|
description: 'A list of bootstrap peer ENRs to connect to. Separated by commas.',
|
|
245
|
+
defaultValue: [],
|
|
228
246
|
},
|
|
229
247
|
bootstrapNodeEnrVersionCheck: {
|
|
230
248
|
env: 'P2P_BOOTSTRAP_NODE_ENR_VERSION_CHECK',
|
|
@@ -354,6 +372,11 @@ export const p2pConfigMappings: ConfigMappingsType<P2PConfig> = {
|
|
|
354
372
|
printDefault: () =>
|
|
355
373
|
'AuthRegistry, FeeJuice.increase_public_balance, Token.increase_public_balance, FPC.prepare_fee',
|
|
356
374
|
},
|
|
375
|
+
maxTxPoolSize: {
|
|
376
|
+
env: 'P2P_MAX_TX_POOL_SIZE',
|
|
377
|
+
description: 'The maximum cumulative tx size of pending txs (in bytes) before evicting lower priority txs.',
|
|
378
|
+
...numberConfigHelper(100_000_000), // 100MB
|
|
379
|
+
},
|
|
357
380
|
...p2pReqRespConfigMappings,
|
|
358
381
|
...chainConfigMappings,
|
|
359
382
|
};
|
|
@@ -375,7 +398,7 @@ export function getP2PDefaultConfig(): P2PConfig {
|
|
|
375
398
|
*/
|
|
376
399
|
export type BootnodeConfig = Pick<
|
|
377
400
|
P2PConfig,
|
|
378
|
-
'p2pIp' | 'p2pPort' | 'peerIdPrivateKey' | 'bootstrapNodes' | 'listenAddress'
|
|
401
|
+
'p2pIp' | 'p2pPort' | 'peerIdPrivateKey' | 'peerIdPrivateKeyPath' | 'bootstrapNodes' | 'listenAddress'
|
|
379
402
|
> &
|
|
380
403
|
Required<Pick<P2PConfig, 'p2pIp' | 'p2pPort'>> &
|
|
381
404
|
Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKB'> &
|
|
@@ -386,6 +409,7 @@ const bootnodeConfigKeys: (keyof BootnodeConfig)[] = [
|
|
|
386
409
|
'p2pPort',
|
|
387
410
|
'listenAddress',
|
|
388
411
|
'peerIdPrivateKey',
|
|
412
|
+
'peerIdPrivateKeyPath',
|
|
389
413
|
'dataDirectory',
|
|
390
414
|
'dataStoreMapSizeKB',
|
|
391
415
|
'bootstrapNodes',
|
|
@@ -1,11 +1,18 @@
|
|
|
1
|
+
import { Fr } from '@aztec/foundation/fields';
|
|
1
2
|
import { toArray } from '@aztec/foundation/iterable';
|
|
2
3
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
3
|
-
import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap } from '@aztec/kv-store';
|
|
4
|
+
import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap, AztecAsyncSingleton } from '@aztec/kv-store';
|
|
5
|
+
import { ProtocolContractAddress } from '@aztec/protocol-contracts';
|
|
6
|
+
import { GasFees } from '@aztec/stdlib/gas';
|
|
7
|
+
import type { MerkleTreeReadOperations, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server';
|
|
4
8
|
import { ClientIvcProof } from '@aztec/stdlib/proofs';
|
|
5
9
|
import type { TxAddedToPoolStats } from '@aztec/stdlib/stats';
|
|
10
|
+
import { DatabasePublicStateSource } from '@aztec/stdlib/trees';
|
|
6
11
|
import { Tx, TxHash } from '@aztec/stdlib/tx';
|
|
7
12
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
8
13
|
|
|
14
|
+
import { ArchiveCache } from '../../msg_validators/tx_validator/archive_cache.js';
|
|
15
|
+
import { GasTxValidator } from '../../msg_validators/tx_validator/gas_validator.js';
|
|
9
16
|
import { PoolInstrumentation, PoolName } from '../instrumentation.js';
|
|
10
17
|
import { getPendingTxPriority } from './priority.js';
|
|
11
18
|
import type { TxPool } from './tx_pool.js';
|
|
@@ -19,12 +26,27 @@ export class AztecKVTxPool implements TxPool {
|
|
|
19
26
|
/** Our tx pool, stored as a Map, with K: tx hash and V: the transaction. */
|
|
20
27
|
#txs: AztecAsyncMap<string, Buffer>;
|
|
21
28
|
|
|
29
|
+
/** The maximum cumulative tx size that the pending txs in the pool take up. */
|
|
30
|
+
#maxTxPoolSize: number | undefined;
|
|
31
|
+
|
|
22
32
|
/** Index from tx hash to the block number in which they were mined, filtered by mined txs. */
|
|
23
33
|
#minedTxHashToBlock: AztecAsyncMap<string, number>;
|
|
24
34
|
|
|
25
35
|
/** Index from tx priority (stored as hex) to its tx hash, filtered by pending txs. */
|
|
26
36
|
#pendingTxPriorityToHash: AztecAsyncMultiMap<string, string>;
|
|
27
37
|
|
|
38
|
+
/** Index from tx hash to its tx size (in bytes), filtered by pending txs. */
|
|
39
|
+
#pendingTxHashToSize: AztecAsyncMap<string, number>;
|
|
40
|
+
|
|
41
|
+
/** Index from tx hash to its header hash, filtered by pending txs. */
|
|
42
|
+
#pendingTxHashToHeaderHash: AztecAsyncMap<string, string>;
|
|
43
|
+
|
|
44
|
+
/** The cumulative tx size in bytes that the pending txs in the pool take up. */
|
|
45
|
+
#pendingTxSize: AztecAsyncSingleton<number>;
|
|
46
|
+
|
|
47
|
+
/** In-memory mapping of pending tx hashes to the hydrated pending tx in the pool. */
|
|
48
|
+
#pendingTxs: Map<string, Tx>;
|
|
49
|
+
|
|
28
50
|
/** KV store for archived txs. */
|
|
29
51
|
#archive: AztecAsyncKVStore;
|
|
30
52
|
|
|
@@ -37,6 +59,9 @@ export class AztecKVTxPool implements TxPool {
|
|
|
37
59
|
/** Number of txs to archive. */
|
|
38
60
|
#archivedTxLimit: number;
|
|
39
61
|
|
|
62
|
+
/** The world state synchronizer used in the node. */
|
|
63
|
+
#worldStateSynchronizer: WorldStateSynchronizer;
|
|
64
|
+
|
|
40
65
|
#log: Logger;
|
|
41
66
|
|
|
42
67
|
#metrics: PoolInstrumentation<Tx>;
|
|
@@ -52,20 +77,30 @@ export class AztecKVTxPool implements TxPool {
|
|
|
52
77
|
constructor(
|
|
53
78
|
store: AztecAsyncKVStore,
|
|
54
79
|
archive: AztecAsyncKVStore,
|
|
80
|
+
worldStateSynchronizer: WorldStateSynchronizer,
|
|
55
81
|
telemetry: TelemetryClient = getTelemetryClient(),
|
|
56
|
-
|
|
82
|
+
config: {
|
|
83
|
+
maxTxPoolSize?: number;
|
|
84
|
+
archivedTxLimit?: number;
|
|
85
|
+
} = {},
|
|
57
86
|
log = createLogger('p2p:tx_pool'),
|
|
58
87
|
) {
|
|
59
88
|
this.#txs = store.openMap('txs');
|
|
60
89
|
this.#minedTxHashToBlock = store.openMap('txHashToBlockMined');
|
|
61
90
|
this.#pendingTxPriorityToHash = store.openMultiMap('pendingTxFeeToHash');
|
|
91
|
+
this.#pendingTxHashToSize = store.openMap('pendingTxHashToSize');
|
|
92
|
+
this.#pendingTxHashToHeaderHash = store.openMap('pendingTxHashToHeaderHash');
|
|
93
|
+
this.#pendingTxSize = store.openSingleton('pendingTxSize');
|
|
94
|
+
this.#maxTxPoolSize = config.maxTxPoolSize;
|
|
95
|
+
this.#pendingTxs = new Map<string, Tx>();
|
|
62
96
|
|
|
63
97
|
this.#archivedTxs = archive.openMap('archivedTxs');
|
|
64
98
|
this.#archivedTxIndices = archive.openMap('archivedTxIndices');
|
|
65
|
-
this.#archivedTxLimit = archivedTxLimit;
|
|
99
|
+
this.#archivedTxLimit = config.archivedTxLimit ?? 0;
|
|
66
100
|
|
|
67
101
|
this.#store = store;
|
|
68
102
|
this.#archive = archive;
|
|
103
|
+
this.#worldStateSynchronizer = worldStateSynchronizer;
|
|
69
104
|
this.#log = log;
|
|
70
105
|
this.#metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, () => store.estimateSize());
|
|
71
106
|
}
|
|
@@ -76,20 +111,35 @@ export class AztecKVTxPool implements TxPool {
|
|
|
76
111
|
}
|
|
77
112
|
|
|
78
113
|
let deletedPending = 0;
|
|
114
|
+
const minedNullifiers = new Set<string>();
|
|
115
|
+
const minedFeePayers = new Set<string>();
|
|
79
116
|
return this.#store.transactionAsync(async () => {
|
|
117
|
+
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
80
118
|
for (const hash of txHashes) {
|
|
81
119
|
const key = hash.toString();
|
|
82
120
|
await this.#minedTxHashToBlock.set(key, blockNumber);
|
|
83
121
|
|
|
84
|
-
const tx = await this.
|
|
122
|
+
const tx = await this.getPendingTxByHash(hash);
|
|
85
123
|
if (tx) {
|
|
124
|
+
const nullifiers = tx.data.getNonEmptyNullifiers();
|
|
125
|
+
nullifiers.forEach(nullifier => minedNullifiers.add(nullifier.toString()));
|
|
126
|
+
minedFeePayers.add(tx.data.feePayer.toString());
|
|
127
|
+
|
|
86
128
|
deletedPending++;
|
|
87
|
-
|
|
88
|
-
await this
|
|
129
|
+
pendingTxSize -= tx.getSize();
|
|
130
|
+
await this.removePendingTxIndices(tx, key);
|
|
89
131
|
}
|
|
90
132
|
}
|
|
91
133
|
this.#metrics.recordAddedObjects(txHashes.length, 'mined');
|
|
92
|
-
this.#
|
|
134
|
+
await this.#pendingTxSize.set(pendingTxSize);
|
|
135
|
+
|
|
136
|
+
const numTxsEvicted = await this.evictInvalidTxsAfterMining(
|
|
137
|
+
txHashes,
|
|
138
|
+
blockNumber,
|
|
139
|
+
minedNullifiers,
|
|
140
|
+
minedFeePayers,
|
|
141
|
+
);
|
|
142
|
+
this.#metrics.recordRemovedObjects(deletedPending + numTxsEvicted, 'pending');
|
|
93
143
|
});
|
|
94
144
|
}
|
|
95
145
|
|
|
@@ -99,21 +149,35 @@ export class AztecKVTxPool implements TxPool {
|
|
|
99
149
|
}
|
|
100
150
|
|
|
101
151
|
let markedAsPending = 0;
|
|
102
|
-
return this.#store
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
152
|
+
return this.#store
|
|
153
|
+
.transactionAsync(async () => {
|
|
154
|
+
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
155
|
+
for (const hash of txHashes) {
|
|
156
|
+
const key = hash.toString();
|
|
157
|
+
await this.#minedTxHashToBlock.delete(key);
|
|
106
158
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
159
|
+
// Rehydrate the tx in the in-memory pending txs mapping
|
|
160
|
+
const tx = await this.getPendingTxByHash(hash);
|
|
161
|
+
if (tx) {
|
|
162
|
+
await this.addPendingTxIndices(tx, key);
|
|
163
|
+
pendingTxSize += tx.getSize();
|
|
164
|
+
markedAsPending++;
|
|
165
|
+
}
|
|
111
166
|
}
|
|
112
|
-
}
|
|
113
167
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
168
|
+
await this.#pendingTxSize.set(pendingTxSize);
|
|
169
|
+
})
|
|
170
|
+
.then(async () => {
|
|
171
|
+
const numInvalidTxsEvicted = await this.evictInvalidTxsAfterReorg(txHashes);
|
|
172
|
+
const { numLowPriorityTxsEvicted, numNewTxsEvicted } = await this.evictLowPriorityTxs(txHashes);
|
|
173
|
+
|
|
174
|
+
this.#metrics.recordAddedObjects(markedAsPending - numNewTxsEvicted, 'pending');
|
|
175
|
+
this.#metrics.recordRemovedObjects(
|
|
176
|
+
numInvalidTxsEvicted + numLowPriorityTxsEvicted - numNewTxsEvicted,
|
|
177
|
+
'pending',
|
|
178
|
+
);
|
|
179
|
+
this.#metrics.recordRemovedObjects(markedAsPending, 'mined');
|
|
180
|
+
});
|
|
117
181
|
}
|
|
118
182
|
|
|
119
183
|
public async getPendingTxHashes(): Promise<TxHash[]> {
|
|
@@ -180,6 +244,7 @@ export class AztecKVTxPool implements TxPool {
|
|
|
180
244
|
);
|
|
181
245
|
await this.#store.transactionAsync(async () => {
|
|
182
246
|
let pendingCount = 0;
|
|
247
|
+
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
183
248
|
await Promise.all(
|
|
184
249
|
txs.map(async (tx, i) => {
|
|
185
250
|
const { txHash, txStats } = hashesAndStats[i];
|
|
@@ -193,14 +258,21 @@ export class AztecKVTxPool implements TxPool {
|
|
|
193
258
|
|
|
194
259
|
if (!(await this.#minedTxHashToBlock.hasAsync(key))) {
|
|
195
260
|
pendingCount++;
|
|
196
|
-
|
|
197
|
-
await this
|
|
261
|
+
pendingTxSize += tx.getSize();
|
|
262
|
+
await this.addPendingTxIndices(tx, key);
|
|
198
263
|
this.#metrics.recordSize(tx);
|
|
199
264
|
}
|
|
200
265
|
}),
|
|
201
266
|
);
|
|
202
267
|
|
|
203
|
-
this.#
|
|
268
|
+
await this.#pendingTxSize.set(pendingTxSize);
|
|
269
|
+
|
|
270
|
+
const { numLowPriorityTxsEvicted, numNewTxsEvicted } = await this.evictLowPriorityTxs(
|
|
271
|
+
hashesAndStats.map(({ txHash }) => txHash),
|
|
272
|
+
);
|
|
273
|
+
|
|
274
|
+
this.#metrics.recordAddedObjects(pendingCount - numNewTxsEvicted, 'pending');
|
|
275
|
+
this.#metrics.recordRemovedObjects(numLowPriorityTxsEvicted - numNewTxsEvicted, 'pending');
|
|
204
276
|
});
|
|
205
277
|
}
|
|
206
278
|
|
|
@@ -209,28 +281,28 @@ export class AztecKVTxPool implements TxPool {
|
|
|
209
281
|
* @param txHashes - An array of tx hashes to be removed from the tx pool.
|
|
210
282
|
* @returns Empty promise.
|
|
211
283
|
*/
|
|
212
|
-
public deleteTxs(txHashes: TxHash[]): Promise<void> {
|
|
284
|
+
public deleteTxs(txHashes: TxHash[], eviction = false): Promise<void> {
|
|
213
285
|
let pendingDeleted = 0;
|
|
214
286
|
let minedDeleted = 0;
|
|
215
287
|
|
|
216
288
|
const deletedTxs: Tx[] = [];
|
|
217
289
|
const poolDbTx = this.#store.transactionAsync(async () => {
|
|
290
|
+
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
218
291
|
for (const hash of txHashes) {
|
|
219
292
|
const key = hash.toString();
|
|
220
293
|
const tx = await this.getTxByHash(hash);
|
|
221
294
|
|
|
222
295
|
if (tx) {
|
|
223
|
-
const fee = getPendingTxPriority(tx);
|
|
224
|
-
await this.#pendingTxPriorityToHash.deleteValue(fee, key);
|
|
225
|
-
|
|
226
296
|
const isMined = await this.#minedTxHashToBlock.hasAsync(key);
|
|
227
297
|
if (isMined) {
|
|
228
298
|
minedDeleted++;
|
|
229
299
|
} else {
|
|
230
300
|
pendingDeleted++;
|
|
301
|
+
pendingTxSize -= tx.getSize();
|
|
302
|
+
await this.removePendingTxIndices(tx, key);
|
|
231
303
|
}
|
|
232
304
|
|
|
233
|
-
if (this.#archivedTxLimit) {
|
|
305
|
+
if (!eviction && this.#archivedTxLimit) {
|
|
234
306
|
deletedTxs.push(tx);
|
|
235
307
|
}
|
|
236
308
|
|
|
@@ -239,6 +311,7 @@ export class AztecKVTxPool implements TxPool {
|
|
|
239
311
|
}
|
|
240
312
|
}
|
|
241
313
|
|
|
314
|
+
await this.#pendingTxSize.set(pendingTxSize);
|
|
242
315
|
this.#metrics.recordRemovedObjects(pendingDeleted, 'pending');
|
|
243
316
|
this.#metrics.recordRemovedObjects(minedDeleted, 'mined');
|
|
244
317
|
});
|
|
@@ -268,6 +341,50 @@ export class AztecKVTxPool implements TxPool {
|
|
|
268
341
|
return vals.map(x => TxHash.fromString(x));
|
|
269
342
|
}
|
|
270
343
|
|
|
344
|
+
/**
|
|
345
|
+
* Creates a GasTxValidator instance.
|
|
346
|
+
* @param db - DB for the validator to use
|
|
347
|
+
* @returns A GasTxValidator instance
|
|
348
|
+
*/
|
|
349
|
+
protected createGasTxValidator(db: MerkleTreeReadOperations): GasTxValidator {
|
|
350
|
+
return new GasTxValidator(new DatabasePublicStateSource(db), ProtocolContractAddress.FeeJuice, GasFees.empty());
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* Creates an ArchiveCache instance.
|
|
355
|
+
* @param db - DB for the cache to use
|
|
356
|
+
* @returns An ArchiveCache instance
|
|
357
|
+
*/
|
|
358
|
+
protected createArchiveCache(db: MerkleTreeReadOperations): ArchiveCache {
|
|
359
|
+
return new ArchiveCache(db);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/**
|
|
363
|
+
* Checks if a cached transaction exists in the in-memory pending tx pool and returns it.
|
|
364
|
+
* Otherwise, it checks the tx pool, updates the pending tx pool, and returns the tx.
|
|
365
|
+
* @param txHash - The generated tx hash.
|
|
366
|
+
* @returns The transaction, if found, 'undefined' otherwise.
|
|
367
|
+
*/
|
|
368
|
+
private async getPendingTxByHash(txHash: TxHash | string): Promise<Tx | undefined> {
|
|
369
|
+
let key;
|
|
370
|
+
if (typeof txHash === 'string') {
|
|
371
|
+
key = txHash;
|
|
372
|
+
txHash = TxHash.fromString(txHash);
|
|
373
|
+
} else {
|
|
374
|
+
key = txHash.toString();
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
if (this.#pendingTxs.has(key)) {
|
|
378
|
+
return this.#pendingTxs.get(key);
|
|
379
|
+
}
|
|
380
|
+
const tx = await this.getTxByHash(txHash);
|
|
381
|
+
if (tx) {
|
|
382
|
+
this.#pendingTxs.set(key, tx);
|
|
383
|
+
return tx;
|
|
384
|
+
}
|
|
385
|
+
return undefined;
|
|
386
|
+
}
|
|
387
|
+
|
|
271
388
|
/**
|
|
272
389
|
* Archives a list of txs for future reference. The number of archived txs is limited by the specified archivedTxLimit.
|
|
273
390
|
* @param txs - The list of transactions to archive.
|
|
@@ -300,4 +417,172 @@ export class AztecKVTxPool implements TxPool {
|
|
|
300
417
|
}
|
|
301
418
|
});
|
|
302
419
|
}
|
|
420
|
+
|
|
421
|
+
/**
|
|
422
|
+
* Evicts pending txs with the lowest priority fees from the pool to accomodate the max tx count and cumulative max tx size
|
|
423
|
+
* after new txs are added.
|
|
424
|
+
*
|
|
425
|
+
* @param newTxHashes - The tx hashes of the new txs added to the pool.
|
|
426
|
+
* @returns The total number of txs evicted from the pool and the number of new txs that were evicted.
|
|
427
|
+
*/
|
|
428
|
+
private async evictLowPriorityTxs(
|
|
429
|
+
newTxHashes: TxHash[],
|
|
430
|
+
): Promise<{ numLowPriorityTxsEvicted: number; numNewTxsEvicted: number }> {
|
|
431
|
+
if (this.#maxTxPoolSize === undefined) {
|
|
432
|
+
return { numLowPriorityTxsEvicted: 0, numNewTxsEvicted: 0 };
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
let numNewTxsEvicted = 0;
|
|
436
|
+
const txsToEvict: TxHash[] = [];
|
|
437
|
+
|
|
438
|
+
let pendingTxsSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
439
|
+
if (pendingTxsSize > this.#maxTxPoolSize) {
|
|
440
|
+
for await (const txHash of this.#pendingTxPriorityToHash.valuesAsync()) {
|
|
441
|
+
this.#log.verbose(`Evicting tx ${txHash} from pool due to low priority to satisfy max tx size limit`);
|
|
442
|
+
txsToEvict.push(TxHash.fromString(txHash));
|
|
443
|
+
|
|
444
|
+
const txSize =
|
|
445
|
+
(await this.#pendingTxHashToSize.getAsync(txHash.toString())) ??
|
|
446
|
+
(await this.getPendingTxByHash(txHash))?.getSize();
|
|
447
|
+
if (txSize) {
|
|
448
|
+
pendingTxsSize -= txSize;
|
|
449
|
+
if (pendingTxsSize <= this.#maxTxPoolSize) {
|
|
450
|
+
break;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
numNewTxsEvicted += newTxHashes.filter(txHash => txsToEvict.includes(txHash)).length;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
if (txsToEvict.length > 0) {
|
|
458
|
+
await this.deleteTxs(txsToEvict, true);
|
|
459
|
+
}
|
|
460
|
+
return {
|
|
461
|
+
numLowPriorityTxsEvicted: txsToEvict.length,
|
|
462
|
+
numNewTxsEvicted,
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Evicts invalid pending txs from the pool after a txs from a block are mined.
|
|
468
|
+
* Eviction criteria includes:
|
|
469
|
+
* - txs with nullifiers that are already included in the mined block
|
|
470
|
+
* - txs with an insufficient fee payer balance
|
|
471
|
+
* - txs with a max block number lower than the mined block
|
|
472
|
+
*
|
|
473
|
+
* @param minedTxHashes - The tx hashes of the txs mined in the block.
|
|
474
|
+
* @param blockNumber - The block number of the mined block.
|
|
475
|
+
* @returns The total number of txs evicted from the pool.
|
|
476
|
+
*/
|
|
477
|
+
private async evictInvalidTxsAfterMining(
|
|
478
|
+
minedTxHashes: TxHash[],
|
|
479
|
+
blockNumber: number,
|
|
480
|
+
minedNullifiers: Set<string>,
|
|
481
|
+
minedFeePayers: Set<string>,
|
|
482
|
+
): Promise<number> {
|
|
483
|
+
if (minedTxHashes.length === 0) {
|
|
484
|
+
return 0;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
// Wait for world state to be synced to at least the mined block number
|
|
488
|
+
await this.#worldStateSynchronizer.syncImmediate(blockNumber);
|
|
489
|
+
|
|
490
|
+
const db = this.#worldStateSynchronizer.getCommitted();
|
|
491
|
+
const gasTxValidator = this.createGasTxValidator(db);
|
|
492
|
+
|
|
493
|
+
const txsToEvict: TxHash[] = [];
|
|
494
|
+
for await (const txHash of this.#pendingTxPriorityToHash.valuesAsync()) {
|
|
495
|
+
const tx = await this.getPendingTxByHash(txHash);
|
|
496
|
+
if (!tx) {
|
|
497
|
+
continue;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// Evict pending txs that share nullifiers with mined txs
|
|
501
|
+
const txNullifiers = tx.data.getNonEmptyNullifiers();
|
|
502
|
+
if (txNullifiers.some(nullifier => minedNullifiers.has(nullifier.toString()))) {
|
|
503
|
+
this.#log.verbose(`Evicting tx ${txHash} from pool due to a duplicate nullifier with a mined tx`);
|
|
504
|
+
txsToEvict.push(TxHash.fromString(txHash));
|
|
505
|
+
continue;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
// Evict pending txs with an insufficient fee payer balance
|
|
509
|
+
if (
|
|
510
|
+
minedFeePayers.has(tx.data.feePayer.toString()) &&
|
|
511
|
+
(await gasTxValidator.validateTxFee(tx)).result === 'invalid'
|
|
512
|
+
) {
|
|
513
|
+
this.#log.verbose(`Evicting tx ${txHash} from pool due to an insufficient fee payer balance`);
|
|
514
|
+
txsToEvict.push(TxHash.fromString(txHash));
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
// Evict pending txs with a max block number less than or equal to the mined block
|
|
519
|
+
const maxBlockNumber = tx.data.rollupValidationRequests.maxBlockNumber;
|
|
520
|
+
if (maxBlockNumber.isSome && maxBlockNumber.value.toNumber() <= blockNumber) {
|
|
521
|
+
this.#log.verbose(`Evicting tx ${txHash} from pool due to an invalid max block number`);
|
|
522
|
+
txsToEvict.push(TxHash.fromString(txHash));
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
if (txsToEvict.length > 0) {
|
|
528
|
+
await this.deleteTxs(txsToEvict, true);
|
|
529
|
+
}
|
|
530
|
+
return txsToEvict.length;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/**
|
|
534
|
+
* Evicts pending txs that no longer have valid archive roots or fee payer balances from the pool after a reorg.
|
|
535
|
+
*
|
|
536
|
+
* @param txHashes - The tx hashes of the txs that were moved from mined to pending.
|
|
537
|
+
* @returns The total number of txs evicted from the pool.
|
|
538
|
+
*/
|
|
539
|
+
private async evictInvalidTxsAfterReorg(txHashes: TxHash[]): Promise<number> {
|
|
540
|
+
if (txHashes.length === 0) {
|
|
541
|
+
return 0;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
await this.#worldStateSynchronizer.syncImmediate();
|
|
545
|
+
const db = this.#worldStateSynchronizer.getCommitted();
|
|
546
|
+
const archiveCache = this.createArchiveCache(db);
|
|
547
|
+
const gasTxValidator = this.createGasTxValidator(db);
|
|
548
|
+
|
|
549
|
+
const txsToEvict: TxHash[] = [];
|
|
550
|
+
|
|
551
|
+
for await (const [txHash, headerHash] of this.#pendingTxHashToHeaderHash.entriesAsync()) {
|
|
552
|
+
const tx = await this.getPendingTxByHash(txHash);
|
|
553
|
+
if (!tx) {
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
const [index] = await archiveCache.getArchiveIndices([Fr.fromString(headerHash)]);
|
|
558
|
+
if (index === undefined) {
|
|
559
|
+
this.#log.verbose(`Evicting tx ${txHash} from pool due to an invalid archive root`);
|
|
560
|
+
txsToEvict.push(TxHash.fromString(txHash));
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
if ((await gasTxValidator.validateTxFee(tx)).result === 'invalid') {
|
|
565
|
+
this.#log.verbose(`Evicting tx ${txHash} from pool due to an insufficient fee payer balance`);
|
|
566
|
+
txsToEvict.push(TxHash.fromString(txHash));
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
if (txsToEvict.length > 0) {
|
|
571
|
+
await this.deleteTxs(txsToEvict, true);
|
|
572
|
+
}
|
|
573
|
+
return txsToEvict.length;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
private async addPendingTxIndices(tx: Tx, txHash: string): Promise<void> {
|
|
577
|
+
await this.#pendingTxPriorityToHash.set(getPendingTxPriority(tx), txHash);
|
|
578
|
+
await this.#pendingTxHashToSize.set(txHash, tx.getSize());
|
|
579
|
+
await this.#pendingTxHashToHeaderHash.set(txHash, (await tx.data.constants.historicalHeader.hash()).toString());
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
private async removePendingTxIndices(tx: Tx, txHash: string): Promise<void> {
|
|
583
|
+
await this.#pendingTxPriorityToHash.deleteValue(getPendingTxPriority(tx), txHash);
|
|
584
|
+
await this.#pendingTxHashToSize.delete(txHash);
|
|
585
|
+
await this.#pendingTxHashToHeaderHash.delete(txHash);
|
|
586
|
+
this.#pendingTxs.delete(txHash);
|
|
587
|
+
}
|
|
303
588
|
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Fr } from '@aztec/foundation/fields';
|
|
2
|
+
import type { ArchiveSource } from '@aztec/p2p';
|
|
3
|
+
import type { MerkleTreeReadOperations } from '@aztec/stdlib/interfaces/server';
|
|
4
|
+
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Implements an archive source by checking a DB and an in-memory collection.
|
|
8
|
+
* Intended for validating transactions as they are added to a block.
|
|
9
|
+
*/
|
|
10
|
+
export class ArchiveCache implements ArchiveSource {
|
|
11
|
+
archives: Map<string, bigint>;
|
|
12
|
+
|
|
13
|
+
constructor(private db: MerkleTreeReadOperations) {
|
|
14
|
+
this.archives = new Map<string, bigint>();
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
public async getArchiveIndices(archives: Fr[]): Promise<(bigint | undefined)[]> {
|
|
18
|
+
const toCheckDb = archives.filter(n => !this.archives.has(n.toString()));
|
|
19
|
+
const dbHits = await this.db.findLeafIndices(MerkleTreeId.ARCHIVE, toCheckDb);
|
|
20
|
+
dbHits.forEach((x, index) => {
|
|
21
|
+
if (x !== undefined) {
|
|
22
|
+
this.archives.set(toCheckDb[index].toString(), x);
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
return archives.map(n => this.archives.get(n.toString()));
|
|
27
|
+
}
|
|
28
|
+
}
|