@aztec/p2p 0.87.7 → 0.87.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/interface.d.ts +1 -1
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/p2p_client.d.ts +2 -2
- package/dest/client/p2p_client.d.ts.map +1 -1
- package/dest/client/p2p_client.js +3 -3
- package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +0 -9
- package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts +1 -0
- package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/kv_attestation_pool.js +13 -8
- package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts +1 -0
- package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/memory_attestation_pool.js +7 -6
- package/dest/mem_pools/instrumentation.d.ts +7 -11
- package/dest/mem_pools/instrumentation.d.ts.map +1 -1
- package/dest/mem_pools/instrumentation.js +25 -37
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts +2 -1
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.js +22 -38
- package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts +1 -0
- package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool/memory_tx_pool.js +13 -21
- package/dest/services/encoding.d.ts +2 -0
- package/dest/services/encoding.d.ts.map +1 -1
- package/dest/services/encoding.js +9 -1
- package/dest/services/reqresp/reqresp.d.ts +1 -1
- package/dest/services/reqresp/reqresp.d.ts.map +1 -1
- package/dest/services/reqresp/reqresp.js +9 -5
- package/dest/services/tx_collect_instrumentation.d.ts +13 -0
- package/dest/services/tx_collect_instrumentation.d.ts.map +1 -0
- package/dest/services/tx_collect_instrumentation.js +34 -0
- package/dest/services/tx_collector.d.ts +6 -2
- package/dest/services/tx_collector.d.ts.map +1 -1
- package/dest/services/tx_collector.js +61 -49
- package/dest/test-helpers/reqresp-nodes.d.ts +2 -0
- package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
- package/dest/test-helpers/reqresp-nodes.js +6 -0
- package/package.json +12 -12
- package/src/client/interface.ts +1 -1
- package/src/client/p2p_client.ts +3 -3
- package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +0 -14
- package/src/mem_pools/attestation_pool/kv_attestation_pool.ts +17 -12
- package/src/mem_pools/attestation_pool/memory_attestation_pool.ts +8 -7
- package/src/mem_pools/instrumentation.ts +32 -46
- package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts +23 -58
- package/src/mem_pools/tx_pool/memory_tx_pool.ts +14 -26
- package/src/services/encoding.ts +9 -1
- package/src/services/reqresp/reqresp.ts +6 -6
- package/src/services/tx_collect_instrumentation.ts +44 -0
- package/src/services/tx_collector.ts +87 -66
- package/src/test-helpers/reqresp-nodes.ts +6 -0
|
@@ -5,7 +5,7 @@ import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncMultiMap } from '@azte
|
|
|
5
5
|
import { BlockAttestation } from '@aztec/stdlib/p2p';
|
|
6
6
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
7
7
|
|
|
8
|
-
import { PoolInstrumentation, PoolName } from '../instrumentation.js';
|
|
8
|
+
import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instrumentation.js';
|
|
9
9
|
import type { AttestationPool } from './attestation_pool.js';
|
|
10
10
|
|
|
11
11
|
export class KvAttestationPool implements AttestationPool {
|
|
@@ -24,9 +24,15 @@ export class KvAttestationPool implements AttestationPool {
|
|
|
24
24
|
this.proposalsForSlot = store.openMultiMap('proposals_for_slot');
|
|
25
25
|
this.attestationsForProposal = store.openMultiMap('attestations_for_proposal');
|
|
26
26
|
|
|
27
|
-
this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL);
|
|
27
|
+
this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL, this.poolStats);
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
private poolStats: PoolStatsCallback = async () => {
|
|
31
|
+
return {
|
|
32
|
+
itemCount: await this.attestations.sizeAsync(),
|
|
33
|
+
};
|
|
34
|
+
};
|
|
35
|
+
|
|
30
36
|
public async isEmpty(): Promise<boolean> {
|
|
31
37
|
for await (const _ of this.attestations.entriesAsync()) {
|
|
32
38
|
return false;
|
|
@@ -73,8 +79,6 @@ export class KvAttestationPool implements AttestationPool {
|
|
|
73
79
|
});
|
|
74
80
|
}
|
|
75
81
|
});
|
|
76
|
-
|
|
77
|
-
this.metrics.recordAddedObjects(attestations.length);
|
|
78
82
|
}
|
|
79
83
|
|
|
80
84
|
public async getAttestationsForSlot(slot: bigint): Promise<BlockAttestation[]> {
|
|
@@ -135,10 +139,9 @@ export class KvAttestationPool implements AttestationPool {
|
|
|
135
139
|
|
|
136
140
|
await this.attestationsForProposal.delete(this.getProposalKey(slotFr, proposalId));
|
|
137
141
|
}
|
|
138
|
-
});
|
|
139
142
|
|
|
140
|
-
|
|
141
|
-
|
|
143
|
+
this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot}`);
|
|
144
|
+
});
|
|
142
145
|
}
|
|
143
146
|
|
|
144
147
|
public async deleteAttestationsForSlotAndProposal(slot: bigint, proposalId: string): Promise<void> {
|
|
@@ -156,10 +159,9 @@ export class KvAttestationPool implements AttestationPool {
|
|
|
156
159
|
|
|
157
160
|
await this.proposalsForSlot.deleteValue(slotString, proposalId);
|
|
158
161
|
await this.attestationsForProposal.delete(this.getProposalKey(slotString, proposalId));
|
|
159
|
-
});
|
|
160
162
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
+
this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot} and proposal ${proposalId}`);
|
|
164
|
+
});
|
|
163
165
|
}
|
|
164
166
|
|
|
165
167
|
public async deleteAttestations(attestations: BlockAttestation[]): Promise<void> {
|
|
@@ -168,8 +170,12 @@ export class KvAttestationPool implements AttestationPool {
|
|
|
168
170
|
const slotNumber = attestation.payload.header.slotNumber;
|
|
169
171
|
const proposalId = attestation.archive;
|
|
170
172
|
const address = attestation.getSender().toString();
|
|
173
|
+
const key = this.getAttestationKey(slotNumber, proposalId, address);
|
|
174
|
+
|
|
175
|
+
if (await this.attestations.hasAsync(key)) {
|
|
176
|
+
await this.attestations.delete(key);
|
|
177
|
+
}
|
|
171
178
|
|
|
172
|
-
await this.attestations.delete(this.getAttestationKey(slotNumber, proposalId, address));
|
|
173
179
|
await this.attestationsForProposal.deleteValue(
|
|
174
180
|
this.getProposalKey(slotNumber, proposalId),
|
|
175
181
|
this.getAttestationKey(slotNumber, proposalId, address),
|
|
@@ -178,6 +184,5 @@ export class KvAttestationPool implements AttestationPool {
|
|
|
178
184
|
this.log.debug(`Deleted attestation for slot ${slotNumber} from ${address}`);
|
|
179
185
|
}
|
|
180
186
|
});
|
|
181
|
-
this.metrics.recordRemovedObjects(attestations.length);
|
|
182
187
|
}
|
|
183
188
|
}
|
|
@@ -2,7 +2,7 @@ import { createLogger } from '@aztec/foundation/log';
|
|
|
2
2
|
import type { BlockAttestation } from '@aztec/stdlib/p2p';
|
|
3
3
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
4
4
|
|
|
5
|
-
import { PoolInstrumentation, PoolName } from '../instrumentation.js';
|
|
5
|
+
import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instrumentation.js';
|
|
6
6
|
import type { AttestationPool } from './attestation_pool.js';
|
|
7
7
|
|
|
8
8
|
export class InMemoryAttestationPool implements AttestationPool {
|
|
@@ -15,9 +15,15 @@ export class InMemoryAttestationPool implements AttestationPool {
|
|
|
15
15
|
private log = createLogger('p2p:attestation_pool'),
|
|
16
16
|
) {
|
|
17
17
|
this.attestations = new Map();
|
|
18
|
-
this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL);
|
|
18
|
+
this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL, this.poolStats);
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
+
private poolStats: PoolStatsCallback = () => {
|
|
22
|
+
return Promise.resolve({
|
|
23
|
+
itemCount: this.attestations.size,
|
|
24
|
+
});
|
|
25
|
+
};
|
|
26
|
+
|
|
21
27
|
public isEmpty(): Promise<boolean> {
|
|
22
28
|
return Promise.resolve(this.attestations.size === 0);
|
|
23
29
|
}
|
|
@@ -61,8 +67,6 @@ export class InMemoryAttestationPool implements AttestationPool {
|
|
|
61
67
|
});
|
|
62
68
|
}
|
|
63
69
|
|
|
64
|
-
// TODO: set these to pending or something ????
|
|
65
|
-
this.metrics.recordAddedObjects(attestations.length);
|
|
66
70
|
return Promise.resolve();
|
|
67
71
|
}
|
|
68
72
|
|
|
@@ -106,7 +110,6 @@ export class InMemoryAttestationPool implements AttestationPool {
|
|
|
106
110
|
this.attestations.delete(slot);
|
|
107
111
|
this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot}`);
|
|
108
112
|
|
|
109
|
-
this.metrics.recordRemovedObjects(numberOfAttestations);
|
|
110
113
|
return Promise.resolve();
|
|
111
114
|
}
|
|
112
115
|
|
|
@@ -119,7 +122,6 @@ export class InMemoryAttestationPool implements AttestationPool {
|
|
|
119
122
|
slotAttestationMap.delete(proposalId);
|
|
120
123
|
|
|
121
124
|
this.log.verbose(`Removed ${numberOfAttestations} attestations for slot ${slot} and proposal ${proposalId}`);
|
|
122
|
-
this.metrics.recordRemovedObjects(numberOfAttestations);
|
|
123
125
|
}
|
|
124
126
|
}
|
|
125
127
|
return Promise.resolve();
|
|
@@ -139,7 +141,6 @@ export class InMemoryAttestationPool implements AttestationPool {
|
|
|
139
141
|
}
|
|
140
142
|
}
|
|
141
143
|
}
|
|
142
|
-
this.metrics.recordRemovedObjects(attestations.length);
|
|
143
144
|
return Promise.resolve();
|
|
144
145
|
}
|
|
145
146
|
}
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import type { Gossipable } from '@aztec/stdlib/p2p';
|
|
2
2
|
import {
|
|
3
3
|
Attributes,
|
|
4
|
+
type BatchObservableResult,
|
|
4
5
|
type Histogram,
|
|
5
6
|
LmdbMetrics,
|
|
6
7
|
type LmdbStatsCallback,
|
|
8
|
+
type Meter,
|
|
7
9
|
Metrics,
|
|
8
10
|
type MetricsType,
|
|
11
|
+
type ObservableGauge,
|
|
9
12
|
type TelemetryClient,
|
|
10
|
-
type UpDownCounter,
|
|
11
13
|
} from '@aztec/telemetry-client';
|
|
12
14
|
|
|
13
15
|
export enum PoolName {
|
|
@@ -41,86 +43,70 @@ function getMetricsLabels(name: PoolName): MetricsLabels {
|
|
|
41
43
|
throw new Error('Invalid pool type');
|
|
42
44
|
}
|
|
43
45
|
|
|
46
|
+
export type PoolStatsCallback = () => Promise<{
|
|
47
|
+
itemCount: number | Record<string, number>;
|
|
48
|
+
}>;
|
|
49
|
+
|
|
44
50
|
/**
|
|
45
51
|
* Instrumentation class for the Pools (TxPool, AttestationPool, etc).
|
|
46
52
|
*/
|
|
47
53
|
export class PoolInstrumentation<PoolObject extends Gossipable> {
|
|
48
54
|
/** The number of txs in the mempool */
|
|
49
|
-
private objectsInMempool:
|
|
55
|
+
private objectsInMempool: ObservableGauge;
|
|
50
56
|
/** Tracks tx size */
|
|
51
57
|
private objectSize: Histogram;
|
|
52
58
|
|
|
53
59
|
private dbMetrics: LmdbMetrics;
|
|
54
60
|
|
|
55
61
|
private defaultAttributes;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
62
|
+
private meter: Meter;
|
|
63
|
+
|
|
64
|
+
constructor(
|
|
65
|
+
telemetry: TelemetryClient,
|
|
66
|
+
name: PoolName,
|
|
67
|
+
private poolStats: PoolStatsCallback,
|
|
68
|
+
dbStats?: LmdbStatsCallback,
|
|
69
|
+
) {
|
|
70
|
+
this.meter = telemetry.getMeter(name);
|
|
59
71
|
this.defaultAttributes = { [Attributes.POOL_NAME]: name };
|
|
60
72
|
|
|
61
73
|
const metricsLabels = getMetricsLabels(name);
|
|
62
74
|
|
|
63
|
-
this.objectsInMempool = meter.
|
|
75
|
+
this.objectsInMempool = this.meter.createObservableGauge(metricsLabels.objectInMempool, {
|
|
64
76
|
description: 'The current number of transactions in the mempool',
|
|
65
77
|
});
|
|
66
78
|
|
|
67
|
-
this.objectSize = meter.createHistogram(metricsLabels.objectSize, {
|
|
79
|
+
this.objectSize = this.meter.createHistogram(metricsLabels.objectSize, {
|
|
68
80
|
unit: 'By',
|
|
69
81
|
description: 'The size of transactions in the mempool',
|
|
70
82
|
});
|
|
71
83
|
|
|
72
84
|
this.dbMetrics = new LmdbMetrics(
|
|
73
|
-
meter,
|
|
85
|
+
this.meter,
|
|
74
86
|
{
|
|
75
87
|
[Attributes.DB_DATA_TYPE]: 'tx-pool',
|
|
76
88
|
},
|
|
77
89
|
dbStats,
|
|
78
90
|
);
|
|
91
|
+
|
|
92
|
+
this.meter.addBatchObservableCallback(this.observeStats, [this.objectsInMempool]);
|
|
79
93
|
}
|
|
80
94
|
|
|
81
95
|
public recordSize(poolObject: PoolObject) {
|
|
82
96
|
this.objectSize.record(poolObject.getSize());
|
|
83
97
|
}
|
|
84
98
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
}
|
|
93
|
-
if (count === 0) {
|
|
94
|
-
return;
|
|
95
|
-
}
|
|
96
|
-
const attributes = status
|
|
97
|
-
? {
|
|
99
|
+
private observeStats = async (observer: BatchObservableResult) => {
|
|
100
|
+
const { itemCount } = await this.poolStats();
|
|
101
|
+
if (typeof itemCount === 'number') {
|
|
102
|
+
observer.observe(this.objectsInMempool, itemCount, this.defaultAttributes);
|
|
103
|
+
} else {
|
|
104
|
+
for (const [status, count] of Object.entries(itemCount)) {
|
|
105
|
+
observer.observe(this.objectsInMempool, count, {
|
|
98
106
|
...this.defaultAttributes,
|
|
99
107
|
[Attributes.STATUS]: status,
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
this.objectsInMempool.add(count, attributes);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
/**
|
|
107
|
-
* Updates the metrics by removing objects from the mempool.
|
|
108
|
-
* @param count - The number of objects to remove from the mempool
|
|
109
|
-
*/
|
|
110
|
-
public recordRemovedObjects(count = 1, status?: string) {
|
|
111
|
-
if (count < 0) {
|
|
112
|
-
throw new Error('Count must be positive');
|
|
113
|
-
}
|
|
114
|
-
if (count === 0) {
|
|
115
|
-
return;
|
|
108
|
+
});
|
|
109
|
+
}
|
|
116
110
|
}
|
|
117
|
-
|
|
118
|
-
const attributes = status
|
|
119
|
-
? {
|
|
120
|
-
...this.defaultAttributes,
|
|
121
|
-
[Attributes.STATUS]: status,
|
|
122
|
-
}
|
|
123
|
-
: this.defaultAttributes;
|
|
124
|
-
this.objectsInMempool.add(-1 * count, attributes);
|
|
125
|
-
}
|
|
111
|
+
};
|
|
126
112
|
}
|
|
@@ -15,7 +15,7 @@ import assert from 'assert';
|
|
|
15
15
|
|
|
16
16
|
import { ArchiveCache } from '../../msg_validators/tx_validator/archive_cache.js';
|
|
17
17
|
import { GasTxValidator } from '../../msg_validators/tx_validator/gas_validator.js';
|
|
18
|
-
import { PoolInstrumentation, PoolName } from '../instrumentation.js';
|
|
18
|
+
import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instrumentation.js';
|
|
19
19
|
import { getPendingTxPriority } from './priority.js';
|
|
20
20
|
import type { TxPool, TxPoolOptions } from './tx_pool.js';
|
|
21
21
|
|
|
@@ -49,9 +49,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
49
49
|
/** The cumulative tx size in bytes that the pending txs in the pool take up. */
|
|
50
50
|
#pendingTxSize: AztecAsyncSingleton<number>;
|
|
51
51
|
|
|
52
|
-
/** Count of total pending txs. */
|
|
53
|
-
#pendingTxCount: AztecAsyncSingleton<number>;
|
|
54
|
-
|
|
55
52
|
/** In-memory mapping of pending tx hashes to the hydrated pending tx in the pool. */
|
|
56
53
|
#pendingTxs: Map<string, Tx>;
|
|
57
54
|
|
|
@@ -102,7 +99,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
102
99
|
this.#pendingTxHashToSize = store.openMap('pendingTxHashToSize');
|
|
103
100
|
this.#pendingTxHashToHeaderHash = store.openMap('pendingTxHashToHeaderHash');
|
|
104
101
|
this.#pendingTxSize = store.openSingleton('pendingTxSize');
|
|
105
|
-
this.#pendingTxCount = store.openSingleton('pendingTxCount');
|
|
106
102
|
|
|
107
103
|
this.#pendingTxs = new Map<string, Tx>();
|
|
108
104
|
this.#nonEvictableTxs = new Set<string>();
|
|
@@ -113,9 +109,20 @@ export class AztecKVTxPool implements TxPool {
|
|
|
113
109
|
this.#store = store;
|
|
114
110
|
this.#archive = archive;
|
|
115
111
|
this.#worldStateSynchronizer = worldStateSynchronizer;
|
|
116
|
-
this.#metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, () => store.estimateSize());
|
|
112
|
+
this.#metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, this.countTxs, () => store.estimateSize());
|
|
117
113
|
}
|
|
118
114
|
|
|
115
|
+
private countTxs: PoolStatsCallback = async () => {
|
|
116
|
+
const [pending = 0, mined = 0] = await Promise.all([this.getPendingTxCount(), this.getMinedTxCount()]);
|
|
117
|
+
|
|
118
|
+
return Promise.resolve({
|
|
119
|
+
itemCount: {
|
|
120
|
+
pending,
|
|
121
|
+
mined,
|
|
122
|
+
},
|
|
123
|
+
});
|
|
124
|
+
};
|
|
125
|
+
|
|
119
126
|
public async isEmpty(): Promise<boolean> {
|
|
120
127
|
for await (const _ of this.#txs.entriesAsync()) {
|
|
121
128
|
return false;
|
|
@@ -128,7 +135,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
128
135
|
return Promise.resolve();
|
|
129
136
|
}
|
|
130
137
|
|
|
131
|
-
let deletedPending = 0;
|
|
132
138
|
const minedNullifiers = new Set<string>();
|
|
133
139
|
const minedFeePayers = new Set<string>();
|
|
134
140
|
|
|
@@ -143,23 +149,13 @@ export class AztecKVTxPool implements TxPool {
|
|
|
143
149
|
const nullifiers = tx.data.getNonEmptyNullifiers();
|
|
144
150
|
nullifiers.forEach(nullifier => minedNullifiers.add(nullifier.toString()));
|
|
145
151
|
minedFeePayers.add(tx.data.feePayer.toString());
|
|
146
|
-
|
|
147
|
-
deletedPending++;
|
|
148
152
|
pendingTxSize -= tx.getSize();
|
|
149
153
|
await this.removePendingTxIndices(tx, key);
|
|
150
154
|
}
|
|
151
155
|
}
|
|
152
|
-
this.#metrics.recordAddedObjects(txHashes.length, 'mined');
|
|
153
156
|
await this.#pendingTxSize.set(pendingTxSize);
|
|
154
|
-
await this.increasePendingTxCount(-deletedPending);
|
|
155
157
|
|
|
156
|
-
|
|
157
|
-
txHashes,
|
|
158
|
-
blockNumber,
|
|
159
|
-
minedNullifiers,
|
|
160
|
-
minedFeePayers,
|
|
161
|
-
);
|
|
162
|
-
this.#metrics.recordRemovedObjects(deletedPending + numTxsEvicted, 'pending');
|
|
158
|
+
await this.evictInvalidTxsAfterMining(txHashes, blockNumber, minedNullifiers, minedFeePayers);
|
|
163
159
|
});
|
|
164
160
|
// We update this after the transaction above. This ensures that the non-evictable transactions are not evicted
|
|
165
161
|
// until any that have been mined are marked as such.
|
|
@@ -171,8 +167,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
171
167
|
if (txHashes.length === 0) {
|
|
172
168
|
return Promise.resolve();
|
|
173
169
|
}
|
|
174
|
-
|
|
175
|
-
let markedAsPending = 0;
|
|
176
170
|
await this.#store.transactionAsync(async () => {
|
|
177
171
|
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
178
172
|
for (const hash of txHashes) {
|
|
@@ -184,21 +178,14 @@ export class AztecKVTxPool implements TxPool {
|
|
|
184
178
|
if (tx) {
|
|
185
179
|
await this.addPendingTxIndices(tx, key);
|
|
186
180
|
pendingTxSize += tx.getSize();
|
|
187
|
-
markedAsPending++;
|
|
188
181
|
}
|
|
189
182
|
}
|
|
190
183
|
|
|
191
184
|
await this.#pendingTxSize.set(pendingTxSize);
|
|
192
185
|
});
|
|
193
186
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
await this.increasePendingTxCount(markedAsPending);
|
|
198
|
-
|
|
199
|
-
this.#metrics.recordAddedObjects(markedAsPending - numNewTxsEvicted, 'pending');
|
|
200
|
-
this.#metrics.recordRemovedObjects(numInvalidTxsEvicted + numLowPriorityTxsEvicted - numNewTxsEvicted, 'pending');
|
|
201
|
-
this.#metrics.recordRemovedObjects(markedAsPending, 'mined');
|
|
187
|
+
await this.evictInvalidTxsAfterReorg(txHashes);
|
|
188
|
+
await this.evictLowPriorityTxs(txHashes);
|
|
202
189
|
}
|
|
203
190
|
|
|
204
191
|
public async getPendingTxHashes(): Promise<TxHash[]> {
|
|
@@ -212,7 +199,11 @@ export class AztecKVTxPool implements TxPool {
|
|
|
212
199
|
}
|
|
213
200
|
|
|
214
201
|
public async getPendingTxCount(): Promise<number> {
|
|
215
|
-
return (await this.#
|
|
202
|
+
return (await this.#pendingTxHashToHeaderHash.sizeAsync()) ?? 0;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
public async getMinedTxCount(): Promise<number> {
|
|
206
|
+
return (await this.#minedTxHashToBlock.sizeAsync()) ?? 0;
|
|
216
207
|
}
|
|
217
208
|
|
|
218
209
|
public async getTxStatus(txHash: TxHash): Promise<'pending' | 'mined' | undefined> {
|
|
@@ -284,7 +275,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
284
275
|
txs.map(async tx => ({ txHash: await tx.getTxHash(), txStats: await tx.getStats() })),
|
|
285
276
|
);
|
|
286
277
|
await this.#store.transactionAsync(async () => {
|
|
287
|
-
let addedCount = 0;
|
|
288
278
|
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
289
279
|
await Promise.all(
|
|
290
280
|
txs.map(async (tx, i) => {
|
|
@@ -303,7 +293,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
303
293
|
await this.#txs.set(key, tx.toBuffer());
|
|
304
294
|
|
|
305
295
|
if (!(await this.#minedTxHashToBlock.hasAsync(key))) {
|
|
306
|
-
addedCount++;
|
|
307
296
|
pendingTxSize += tx.getSize();
|
|
308
297
|
await this.addPendingTxIndices(tx, key);
|
|
309
298
|
this.#metrics.recordSize(tx);
|
|
@@ -311,14 +300,8 @@ export class AztecKVTxPool implements TxPool {
|
|
|
311
300
|
}),
|
|
312
301
|
);
|
|
313
302
|
|
|
314
|
-
await this.increasePendingTxCount(addedCount);
|
|
315
303
|
await this.#pendingTxSize.set(pendingTxSize);
|
|
316
|
-
|
|
317
|
-
hashesAndStats.map(({ txHash }) => txHash),
|
|
318
|
-
);
|
|
319
|
-
|
|
320
|
-
this.#metrics.recordAddedObjects(addedCount - numNewTxsEvicted, 'pending');
|
|
321
|
-
this.#metrics.recordRemovedObjects(numLowPriorityTxsEvicted - numNewTxsEvicted, 'pending');
|
|
304
|
+
await this.evictLowPriorityTxs(hashesAndStats.map(({ txHash }) => txHash));
|
|
322
305
|
});
|
|
323
306
|
}
|
|
324
307
|
|
|
@@ -328,9 +311,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
328
311
|
* @returns Empty promise.
|
|
329
312
|
*/
|
|
330
313
|
public deleteTxs(txHashes: TxHash[], eviction = false): Promise<void> {
|
|
331
|
-
let pendingDeleted = 0;
|
|
332
|
-
let minedDeleted = 0;
|
|
333
|
-
|
|
334
314
|
const deletedTxs: Tx[] = [];
|
|
335
315
|
const poolDbTx = this.#store.transactionAsync(async () => {
|
|
336
316
|
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
@@ -340,10 +320,7 @@ export class AztecKVTxPool implements TxPool {
|
|
|
340
320
|
|
|
341
321
|
if (tx) {
|
|
342
322
|
const isMined = await this.#minedTxHashToBlock.hasAsync(key);
|
|
343
|
-
if (isMined) {
|
|
344
|
-
minedDeleted++;
|
|
345
|
-
} else {
|
|
346
|
-
pendingDeleted++;
|
|
323
|
+
if (!isMined) {
|
|
347
324
|
pendingTxSize -= tx.getSize();
|
|
348
325
|
await this.removePendingTxIndices(tx, key);
|
|
349
326
|
}
|
|
@@ -358,10 +335,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
358
335
|
}
|
|
359
336
|
|
|
360
337
|
await this.#pendingTxSize.set(pendingTxSize);
|
|
361
|
-
await this.increasePendingTxCount(-pendingDeleted);
|
|
362
|
-
|
|
363
|
-
this.#metrics.recordRemovedObjects(pendingDeleted, 'pending');
|
|
364
|
-
this.#metrics.recordRemovedObjects(minedDeleted, 'mined');
|
|
365
338
|
});
|
|
366
339
|
|
|
367
340
|
return this.#archivedTxLimit ? poolDbTx.then(() => this.archiveTxs(deletedTxs)) : poolDbTx;
|
|
@@ -670,12 +643,4 @@ export class AztecKVTxPool implements TxPool {
|
|
|
670
643
|
await this.#pendingTxHashToHeaderHash.delete(txHash);
|
|
671
644
|
this.#pendingTxs.delete(txHash);
|
|
672
645
|
}
|
|
673
|
-
|
|
674
|
-
private async increasePendingTxCount(count: number): Promise<void> {
|
|
675
|
-
const pendingTxCount = (await this.#pendingTxCount.getAsync()) ?? 0;
|
|
676
|
-
this.#log.debug(
|
|
677
|
-
`Increasing pending tx count: current ${pendingTxCount} + count ${count} = ${pendingTxCount + count}`,
|
|
678
|
-
);
|
|
679
|
-
await this.#pendingTxCount.set(pendingTxCount + count);
|
|
680
|
-
}
|
|
681
646
|
}
|
|
@@ -3,7 +3,7 @@ import type { TxAddedToPoolStats } from '@aztec/stdlib/stats';
|
|
|
3
3
|
import { Tx, TxHash } from '@aztec/stdlib/tx';
|
|
4
4
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
5
5
|
|
|
6
|
-
import { PoolInstrumentation, PoolName } from '../instrumentation.js';
|
|
6
|
+
import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instrumentation.js';
|
|
7
7
|
import { getPendingTxPriority } from './priority.js';
|
|
8
8
|
import type { TxPool, TxPoolOptions } from './tx_pool.js';
|
|
9
9
|
|
|
@@ -31,9 +31,18 @@ export class InMemoryTxPool implements TxPool {
|
|
|
31
31
|
this.txs = new Map<bigint, Tx>();
|
|
32
32
|
this.minedTxs = new Map();
|
|
33
33
|
this.pendingTxs = new Set();
|
|
34
|
-
this.metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL);
|
|
34
|
+
this.metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, this.countTx);
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
private countTx: PoolStatsCallback = () => {
|
|
38
|
+
return Promise.resolve({
|
|
39
|
+
itemCount: {
|
|
40
|
+
mined: this.minedTxs.size,
|
|
41
|
+
pending: this.pendingTxs.size,
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
};
|
|
45
|
+
|
|
37
46
|
public isEmpty(): Promise<boolean> {
|
|
38
47
|
return Promise.resolve(this.txs.size === 0);
|
|
39
48
|
}
|
|
@@ -44,8 +53,6 @@ export class InMemoryTxPool implements TxPool {
|
|
|
44
53
|
this.minedTxs.set(key, blockNumber);
|
|
45
54
|
this.pendingTxs.delete(key);
|
|
46
55
|
}
|
|
47
|
-
this.metrics.recordRemovedObjects(txHashes.length, 'pending');
|
|
48
|
-
this.metrics.recordAddedObjects(txHashes.length, 'mined');
|
|
49
56
|
return Promise.resolve();
|
|
50
57
|
}
|
|
51
58
|
|
|
@@ -55,23 +62,15 @@ export class InMemoryTxPool implements TxPool {
|
|
|
55
62
|
}
|
|
56
63
|
|
|
57
64
|
const keys = txHashes.map(x => x.toBigInt());
|
|
58
|
-
let deleted = 0;
|
|
59
|
-
let added = 0;
|
|
60
65
|
for (const key of keys) {
|
|
61
|
-
|
|
62
|
-
deleted++;
|
|
63
|
-
}
|
|
66
|
+
this.minedTxs.delete(key);
|
|
64
67
|
|
|
65
68
|
// only add back to the pending set if we have the tx object
|
|
66
69
|
if (this.txs.has(key)) {
|
|
67
|
-
added++;
|
|
68
70
|
this.pendingTxs.add(key);
|
|
69
71
|
}
|
|
70
72
|
}
|
|
71
73
|
|
|
72
|
-
this.metrics.recordRemovedObjects(deleted, 'mined');
|
|
73
|
-
this.metrics.recordAddedObjects(added, 'pending');
|
|
74
|
-
|
|
75
74
|
return Promise.resolve();
|
|
76
75
|
}
|
|
77
76
|
|
|
@@ -131,7 +130,6 @@ export class InMemoryTxPool implements TxPool {
|
|
|
131
130
|
* @returns Empty promise.
|
|
132
131
|
*/
|
|
133
132
|
public async addTxs(txs: Tx[]): Promise<void> {
|
|
134
|
-
let pending = 0;
|
|
135
133
|
for (const tx of txs) {
|
|
136
134
|
const txHash = await tx.getTxHash();
|
|
137
135
|
this.log.verbose(`Adding tx ${txHash.toString()} to pool`, {
|
|
@@ -142,14 +140,10 @@ export class InMemoryTxPool implements TxPool {
|
|
|
142
140
|
const key = txHash.toBigInt();
|
|
143
141
|
this.txs.set(key, tx);
|
|
144
142
|
if (!this.minedTxs.has(key)) {
|
|
145
|
-
pending++;
|
|
146
143
|
this.metrics.recordSize(tx);
|
|
147
144
|
this.pendingTxs.add(key);
|
|
148
145
|
}
|
|
149
146
|
}
|
|
150
|
-
|
|
151
|
-
this.metrics.recordAddedObjects(pending, 'pending');
|
|
152
|
-
return;
|
|
153
147
|
}
|
|
154
148
|
|
|
155
149
|
/**
|
|
@@ -158,19 +152,13 @@ export class InMemoryTxPool implements TxPool {
|
|
|
158
152
|
* @returns The number of transactions that was deleted from the pool.
|
|
159
153
|
*/
|
|
160
154
|
public deleteTxs(txHashes: TxHash[]): Promise<void> {
|
|
161
|
-
let deletedMined = 0;
|
|
162
|
-
let deletedPending = 0;
|
|
163
|
-
|
|
164
155
|
for (const txHash of txHashes) {
|
|
165
156
|
const key = txHash.toBigInt();
|
|
166
157
|
this.txs.delete(key);
|
|
167
|
-
|
|
168
|
-
|
|
158
|
+
this.pendingTxs.delete(key);
|
|
159
|
+
this.minedTxs.delete(key);
|
|
169
160
|
}
|
|
170
161
|
|
|
171
|
-
this.metrics.recordRemovedObjects(deletedPending, 'pending');
|
|
172
|
-
this.metrics.recordRemovedObjects(deletedMined, 'mined');
|
|
173
|
-
|
|
174
162
|
return Promise.resolve();
|
|
175
163
|
}
|
|
176
164
|
|
package/src/services/encoding.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { sha256 } from '@aztec/foundation/crypto';
|
|
|
4
4
|
import type { RPC } from '@chainsafe/libp2p-gossipsub/message';
|
|
5
5
|
import type { DataTransform } from '@chainsafe/libp2p-gossipsub/types';
|
|
6
6
|
import type { Message } from '@libp2p/interface';
|
|
7
|
-
import { compressSync, uncompressSync } from 'snappy';
|
|
7
|
+
import { compress, compressSync, uncompress, uncompressSync } from 'snappy';
|
|
8
8
|
import xxhashFactory from 'xxhash-wasm';
|
|
9
9
|
|
|
10
10
|
// Load WASM
|
|
@@ -76,4 +76,12 @@ export class SnappyTransform implements DataTransform {
|
|
|
76
76
|
}
|
|
77
77
|
return Buffer.from(compressSync(data));
|
|
78
78
|
}
|
|
79
|
+
|
|
80
|
+
public outboundTransformAsync(data: Buffer): Promise<Buffer> {
|
|
81
|
+
return compress(data);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
public inboundTransformAsync(data: Buffer): Promise<Buffer> {
|
|
85
|
+
return uncompress(data, { asBuffer: true }) as Promise<Buffer>;
|
|
86
|
+
}
|
|
79
87
|
}
|
|
@@ -5,7 +5,7 @@ import { executeTimeout } from '@aztec/foundation/timer';
|
|
|
5
5
|
import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
|
|
6
6
|
import { Attributes, type TelemetryClient, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
7
7
|
|
|
8
|
-
import type
|
|
8
|
+
import { CodeError, type IncomingStreamData, type PeerId, type Stream } from '@libp2p/interface';
|
|
9
9
|
import { pipe } from 'it-pipe';
|
|
10
10
|
import type { Libp2p } from 'libp2p';
|
|
11
11
|
import type { Uint8ArrayList } from 'uint8arraylist';
|
|
@@ -538,7 +538,7 @@ export class ReqResp {
|
|
|
538
538
|
|
|
539
539
|
// Timeout errors are punished with high tolerance, they can be due to a geogrpahically far away peer or an
|
|
540
540
|
// overloaded peer
|
|
541
|
-
if (e instanceof IndividualReqRespTimeoutError) {
|
|
541
|
+
if (e instanceof IndividualReqRespTimeoutError || (e instanceof CodeError && e.code === 'ERR_TIMEOUT')) {
|
|
542
542
|
this.logger.debug(
|
|
543
543
|
`Timeout error: ${e.message} | peerId: ${peerId.toString()} | subProtocol: ${subProtocol}`,
|
|
544
544
|
logTags,
|
|
@@ -573,7 +573,7 @@ export class ReqResp {
|
|
|
573
573
|
}
|
|
574
574
|
|
|
575
575
|
const messageData = Buffer.concat(chunks);
|
|
576
|
-
const message
|
|
576
|
+
const message = await this.snappyTransform.inboundTransformAsync(messageData);
|
|
577
577
|
|
|
578
578
|
return {
|
|
579
579
|
status: statusBuffer ?? ReqRespStatus.UNKNOWN,
|
|
@@ -651,13 +651,13 @@ export class ReqResp {
|
|
|
651
651
|
const successChunk = Buffer.from([ReqRespStatus.SUCCESS]);
|
|
652
652
|
yield new Uint8Array(successChunk);
|
|
653
653
|
|
|
654
|
-
yield new Uint8Array(transform.
|
|
654
|
+
yield new Uint8Array(await transform.outboundTransformAsync(response));
|
|
655
655
|
}
|
|
656
656
|
},
|
|
657
657
|
stream,
|
|
658
658
|
);
|
|
659
659
|
} catch (e: any) {
|
|
660
|
-
this.logger.warn(
|
|
660
|
+
this.logger.warn(`Reqresp Response error: ${e?.name} ${e?.code} ${e?.message}`, { err: e, protocol });
|
|
661
661
|
this.metrics.recordResponseError(protocol);
|
|
662
662
|
|
|
663
663
|
// If we receive a known error, we use the error status in the response chunk, otherwise we categorize as unknown
|
|
@@ -677,7 +677,7 @@ export class ReqResp {
|
|
|
677
677
|
stream,
|
|
678
678
|
);
|
|
679
679
|
} else {
|
|
680
|
-
this.logger.
|
|
680
|
+
this.logger.trace('Stream already closed, not sending error response', { protocol, err: e, errorStatus });
|
|
681
681
|
}
|
|
682
682
|
} finally {
|
|
683
683
|
//NOTE: All other status codes indicate closed stream.
|