@aztec/p2p 0.87.6 → 0.87.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/interface.d.ts +1 -1
- package/dest/client/interface.d.ts.map +1 -1
- package/dest/client/p2p_client.d.ts +2 -2
- package/dest/client/p2p_client.d.ts.map +1 -1
- package/dest/client/p2p_client.js +3 -3
- package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +0 -9
- package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts +1 -0
- package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/kv_attestation_pool.js +13 -8
- package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts +1 -0
- package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/memory_attestation_pool.js +7 -6
- package/dest/mem_pools/instrumentation.d.ts +7 -11
- package/dest/mem_pools/instrumentation.d.ts.map +1 -1
- package/dest/mem_pools/instrumentation.js +25 -37
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts +2 -1
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.js +22 -38
- package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts +1 -0
- package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool/memory_tx_pool.js +13 -21
- package/dest/services/encoding.d.ts +2 -0
- package/dest/services/encoding.d.ts.map +1 -1
- package/dest/services/encoding.js +9 -1
- package/dest/services/libp2p/instrumentation.d.ts +11 -0
- package/dest/services/libp2p/instrumentation.d.ts.map +1 -0
- package/dest/services/libp2p/instrumentation.js +29 -0
- package/dest/services/libp2p/libp2p_service.d.ts +3 -2
- package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
- package/dest/services/libp2p/libp2p_service.js +30 -17
- package/dest/services/reqresp/reqresp.d.ts +1 -1
- package/dest/services/reqresp/reqresp.d.ts.map +1 -1
- package/dest/services/reqresp/reqresp.js +9 -5
- package/dest/services/tx_collect_instrumentation.d.ts +13 -0
- package/dest/services/tx_collect_instrumentation.d.ts.map +1 -0
- package/dest/services/tx_collect_instrumentation.js +34 -0
- package/dest/services/tx_collector.d.ts +6 -2
- package/dest/services/tx_collector.d.ts.map +1 -1
- package/dest/services/tx_collector.js +65 -50
- package/dest/test-helpers/reqresp-nodes.d.ts +2 -0
- package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
- package/dest/test-helpers/reqresp-nodes.js +6 -0
- package/package.json +12 -12
- package/src/client/interface.ts +1 -1
- package/src/client/p2p_client.ts +3 -3
- package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +0 -14
- package/src/mem_pools/attestation_pool/kv_attestation_pool.ts +17 -12
- package/src/mem_pools/attestation_pool/memory_attestation_pool.ts +8 -7
- package/src/mem_pools/instrumentation.ts +32 -46
- package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts +23 -58
- package/src/mem_pools/tx_pool/memory_tx_pool.ts +14 -26
- package/src/services/encoding.ts +9 -1
- package/src/services/libp2p/instrumentation.ts +39 -0
- package/src/services/libp2p/libp2p_service.ts +41 -15
- package/src/services/reqresp/reqresp.ts +6 -6
- package/src/services/tx_collect_instrumentation.ts +44 -0
- package/src/services/tx_collector.ts +93 -67
- package/src/test-helpers/reqresp-nodes.ts +6 -0
|
@@ -15,7 +15,7 @@ import assert from 'assert';
|
|
|
15
15
|
|
|
16
16
|
import { ArchiveCache } from '../../msg_validators/tx_validator/archive_cache.js';
|
|
17
17
|
import { GasTxValidator } from '../../msg_validators/tx_validator/gas_validator.js';
|
|
18
|
-
import { PoolInstrumentation, PoolName } from '../instrumentation.js';
|
|
18
|
+
import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instrumentation.js';
|
|
19
19
|
import { getPendingTxPriority } from './priority.js';
|
|
20
20
|
import type { TxPool, TxPoolOptions } from './tx_pool.js';
|
|
21
21
|
|
|
@@ -49,9 +49,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
49
49
|
/** The cumulative tx size in bytes that the pending txs in the pool take up. */
|
|
50
50
|
#pendingTxSize: AztecAsyncSingleton<number>;
|
|
51
51
|
|
|
52
|
-
/** Count of total pending txs. */
|
|
53
|
-
#pendingTxCount: AztecAsyncSingleton<number>;
|
|
54
|
-
|
|
55
52
|
/** In-memory mapping of pending tx hashes to the hydrated pending tx in the pool. */
|
|
56
53
|
#pendingTxs: Map<string, Tx>;
|
|
57
54
|
|
|
@@ -102,7 +99,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
102
99
|
this.#pendingTxHashToSize = store.openMap('pendingTxHashToSize');
|
|
103
100
|
this.#pendingTxHashToHeaderHash = store.openMap('pendingTxHashToHeaderHash');
|
|
104
101
|
this.#pendingTxSize = store.openSingleton('pendingTxSize');
|
|
105
|
-
this.#pendingTxCount = store.openSingleton('pendingTxCount');
|
|
106
102
|
|
|
107
103
|
this.#pendingTxs = new Map<string, Tx>();
|
|
108
104
|
this.#nonEvictableTxs = new Set<string>();
|
|
@@ -113,9 +109,20 @@ export class AztecKVTxPool implements TxPool {
|
|
|
113
109
|
this.#store = store;
|
|
114
110
|
this.#archive = archive;
|
|
115
111
|
this.#worldStateSynchronizer = worldStateSynchronizer;
|
|
116
|
-
this.#metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, () => store.estimateSize());
|
|
112
|
+
this.#metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, this.countTxs, () => store.estimateSize());
|
|
117
113
|
}
|
|
118
114
|
|
|
115
|
+
private countTxs: PoolStatsCallback = async () => {
|
|
116
|
+
const [pending = 0, mined = 0] = await Promise.all([this.getPendingTxCount(), this.getMinedTxCount()]);
|
|
117
|
+
|
|
118
|
+
return Promise.resolve({
|
|
119
|
+
itemCount: {
|
|
120
|
+
pending,
|
|
121
|
+
mined,
|
|
122
|
+
},
|
|
123
|
+
});
|
|
124
|
+
};
|
|
125
|
+
|
|
119
126
|
public async isEmpty(): Promise<boolean> {
|
|
120
127
|
for await (const _ of this.#txs.entriesAsync()) {
|
|
121
128
|
return false;
|
|
@@ -128,7 +135,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
128
135
|
return Promise.resolve();
|
|
129
136
|
}
|
|
130
137
|
|
|
131
|
-
let deletedPending = 0;
|
|
132
138
|
const minedNullifiers = new Set<string>();
|
|
133
139
|
const minedFeePayers = new Set<string>();
|
|
134
140
|
|
|
@@ -143,23 +149,13 @@ export class AztecKVTxPool implements TxPool {
|
|
|
143
149
|
const nullifiers = tx.data.getNonEmptyNullifiers();
|
|
144
150
|
nullifiers.forEach(nullifier => minedNullifiers.add(nullifier.toString()));
|
|
145
151
|
minedFeePayers.add(tx.data.feePayer.toString());
|
|
146
|
-
|
|
147
|
-
deletedPending++;
|
|
148
152
|
pendingTxSize -= tx.getSize();
|
|
149
153
|
await this.removePendingTxIndices(tx, key);
|
|
150
154
|
}
|
|
151
155
|
}
|
|
152
|
-
this.#metrics.recordAddedObjects(txHashes.length, 'mined');
|
|
153
156
|
await this.#pendingTxSize.set(pendingTxSize);
|
|
154
|
-
await this.increasePendingTxCount(-deletedPending);
|
|
155
157
|
|
|
156
|
-
|
|
157
|
-
txHashes,
|
|
158
|
-
blockNumber,
|
|
159
|
-
minedNullifiers,
|
|
160
|
-
minedFeePayers,
|
|
161
|
-
);
|
|
162
|
-
this.#metrics.recordRemovedObjects(deletedPending + numTxsEvicted, 'pending');
|
|
158
|
+
await this.evictInvalidTxsAfterMining(txHashes, blockNumber, minedNullifiers, minedFeePayers);
|
|
163
159
|
});
|
|
164
160
|
// We update this after the transaction above. This ensures that the non-evictable transactions are not evicted
|
|
165
161
|
// until any that have been mined are marked as such.
|
|
@@ -171,8 +167,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
171
167
|
if (txHashes.length === 0) {
|
|
172
168
|
return Promise.resolve();
|
|
173
169
|
}
|
|
174
|
-
|
|
175
|
-
let markedAsPending = 0;
|
|
176
170
|
await this.#store.transactionAsync(async () => {
|
|
177
171
|
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
178
172
|
for (const hash of txHashes) {
|
|
@@ -184,21 +178,14 @@ export class AztecKVTxPool implements TxPool {
|
|
|
184
178
|
if (tx) {
|
|
185
179
|
await this.addPendingTxIndices(tx, key);
|
|
186
180
|
pendingTxSize += tx.getSize();
|
|
187
|
-
markedAsPending++;
|
|
188
181
|
}
|
|
189
182
|
}
|
|
190
183
|
|
|
191
184
|
await this.#pendingTxSize.set(pendingTxSize);
|
|
192
185
|
});
|
|
193
186
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
await this.increasePendingTxCount(markedAsPending);
|
|
198
|
-
|
|
199
|
-
this.#metrics.recordAddedObjects(markedAsPending - numNewTxsEvicted, 'pending');
|
|
200
|
-
this.#metrics.recordRemovedObjects(numInvalidTxsEvicted + numLowPriorityTxsEvicted - numNewTxsEvicted, 'pending');
|
|
201
|
-
this.#metrics.recordRemovedObjects(markedAsPending, 'mined');
|
|
187
|
+
await this.evictInvalidTxsAfterReorg(txHashes);
|
|
188
|
+
await this.evictLowPriorityTxs(txHashes);
|
|
202
189
|
}
|
|
203
190
|
|
|
204
191
|
public async getPendingTxHashes(): Promise<TxHash[]> {
|
|
@@ -212,7 +199,11 @@ export class AztecKVTxPool implements TxPool {
|
|
|
212
199
|
}
|
|
213
200
|
|
|
214
201
|
public async getPendingTxCount(): Promise<number> {
|
|
215
|
-
return (await this.#
|
|
202
|
+
return (await this.#pendingTxHashToHeaderHash.sizeAsync()) ?? 0;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
public async getMinedTxCount(): Promise<number> {
|
|
206
|
+
return (await this.#minedTxHashToBlock.sizeAsync()) ?? 0;
|
|
216
207
|
}
|
|
217
208
|
|
|
218
209
|
public async getTxStatus(txHash: TxHash): Promise<'pending' | 'mined' | undefined> {
|
|
@@ -284,7 +275,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
284
275
|
txs.map(async tx => ({ txHash: await tx.getTxHash(), txStats: await tx.getStats() })),
|
|
285
276
|
);
|
|
286
277
|
await this.#store.transactionAsync(async () => {
|
|
287
|
-
let addedCount = 0;
|
|
288
278
|
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
289
279
|
await Promise.all(
|
|
290
280
|
txs.map(async (tx, i) => {
|
|
@@ -303,7 +293,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
303
293
|
await this.#txs.set(key, tx.toBuffer());
|
|
304
294
|
|
|
305
295
|
if (!(await this.#minedTxHashToBlock.hasAsync(key))) {
|
|
306
|
-
addedCount++;
|
|
307
296
|
pendingTxSize += tx.getSize();
|
|
308
297
|
await this.addPendingTxIndices(tx, key);
|
|
309
298
|
this.#metrics.recordSize(tx);
|
|
@@ -311,14 +300,8 @@ export class AztecKVTxPool implements TxPool {
|
|
|
311
300
|
}),
|
|
312
301
|
);
|
|
313
302
|
|
|
314
|
-
await this.increasePendingTxCount(addedCount);
|
|
315
303
|
await this.#pendingTxSize.set(pendingTxSize);
|
|
316
|
-
|
|
317
|
-
hashesAndStats.map(({ txHash }) => txHash),
|
|
318
|
-
);
|
|
319
|
-
|
|
320
|
-
this.#metrics.recordAddedObjects(addedCount - numNewTxsEvicted, 'pending');
|
|
321
|
-
this.#metrics.recordRemovedObjects(numLowPriorityTxsEvicted - numNewTxsEvicted, 'pending');
|
|
304
|
+
await this.evictLowPriorityTxs(hashesAndStats.map(({ txHash }) => txHash));
|
|
322
305
|
});
|
|
323
306
|
}
|
|
324
307
|
|
|
@@ -328,9 +311,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
328
311
|
* @returns Empty promise.
|
|
329
312
|
*/
|
|
330
313
|
public deleteTxs(txHashes: TxHash[], eviction = false): Promise<void> {
|
|
331
|
-
let pendingDeleted = 0;
|
|
332
|
-
let minedDeleted = 0;
|
|
333
|
-
|
|
334
314
|
const deletedTxs: Tx[] = [];
|
|
335
315
|
const poolDbTx = this.#store.transactionAsync(async () => {
|
|
336
316
|
let pendingTxSize = (await this.#pendingTxSize.getAsync()) ?? 0;
|
|
@@ -340,10 +320,7 @@ export class AztecKVTxPool implements TxPool {
|
|
|
340
320
|
|
|
341
321
|
if (tx) {
|
|
342
322
|
const isMined = await this.#minedTxHashToBlock.hasAsync(key);
|
|
343
|
-
if (isMined) {
|
|
344
|
-
minedDeleted++;
|
|
345
|
-
} else {
|
|
346
|
-
pendingDeleted++;
|
|
323
|
+
if (!isMined) {
|
|
347
324
|
pendingTxSize -= tx.getSize();
|
|
348
325
|
await this.removePendingTxIndices(tx, key);
|
|
349
326
|
}
|
|
@@ -358,10 +335,6 @@ export class AztecKVTxPool implements TxPool {
|
|
|
358
335
|
}
|
|
359
336
|
|
|
360
337
|
await this.#pendingTxSize.set(pendingTxSize);
|
|
361
|
-
await this.increasePendingTxCount(-pendingDeleted);
|
|
362
|
-
|
|
363
|
-
this.#metrics.recordRemovedObjects(pendingDeleted, 'pending');
|
|
364
|
-
this.#metrics.recordRemovedObjects(minedDeleted, 'mined');
|
|
365
338
|
});
|
|
366
339
|
|
|
367
340
|
return this.#archivedTxLimit ? poolDbTx.then(() => this.archiveTxs(deletedTxs)) : poolDbTx;
|
|
@@ -670,12 +643,4 @@ export class AztecKVTxPool implements TxPool {
|
|
|
670
643
|
await this.#pendingTxHashToHeaderHash.delete(txHash);
|
|
671
644
|
this.#pendingTxs.delete(txHash);
|
|
672
645
|
}
|
|
673
|
-
|
|
674
|
-
private async increasePendingTxCount(count: number): Promise<void> {
|
|
675
|
-
const pendingTxCount = (await this.#pendingTxCount.getAsync()) ?? 0;
|
|
676
|
-
this.#log.debug(
|
|
677
|
-
`Increasing pending tx count: current ${pendingTxCount} + count ${count} = ${pendingTxCount + count}`,
|
|
678
|
-
);
|
|
679
|
-
await this.#pendingTxCount.set(pendingTxCount + count);
|
|
680
|
-
}
|
|
681
646
|
}
|
|
@@ -3,7 +3,7 @@ import type { TxAddedToPoolStats } from '@aztec/stdlib/stats';
|
|
|
3
3
|
import { Tx, TxHash } from '@aztec/stdlib/tx';
|
|
4
4
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
5
5
|
|
|
6
|
-
import { PoolInstrumentation, PoolName } from '../instrumentation.js';
|
|
6
|
+
import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instrumentation.js';
|
|
7
7
|
import { getPendingTxPriority } from './priority.js';
|
|
8
8
|
import type { TxPool, TxPoolOptions } from './tx_pool.js';
|
|
9
9
|
|
|
@@ -31,9 +31,18 @@ export class InMemoryTxPool implements TxPool {
|
|
|
31
31
|
this.txs = new Map<bigint, Tx>();
|
|
32
32
|
this.minedTxs = new Map();
|
|
33
33
|
this.pendingTxs = new Set();
|
|
34
|
-
this.metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL);
|
|
34
|
+
this.metrics = new PoolInstrumentation(telemetry, PoolName.TX_POOL, this.countTx);
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
private countTx: PoolStatsCallback = () => {
|
|
38
|
+
return Promise.resolve({
|
|
39
|
+
itemCount: {
|
|
40
|
+
mined: this.minedTxs.size,
|
|
41
|
+
pending: this.pendingTxs.size,
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
};
|
|
45
|
+
|
|
37
46
|
public isEmpty(): Promise<boolean> {
|
|
38
47
|
return Promise.resolve(this.txs.size === 0);
|
|
39
48
|
}
|
|
@@ -44,8 +53,6 @@ export class InMemoryTxPool implements TxPool {
|
|
|
44
53
|
this.minedTxs.set(key, blockNumber);
|
|
45
54
|
this.pendingTxs.delete(key);
|
|
46
55
|
}
|
|
47
|
-
this.metrics.recordRemovedObjects(txHashes.length, 'pending');
|
|
48
|
-
this.metrics.recordAddedObjects(txHashes.length, 'mined');
|
|
49
56
|
return Promise.resolve();
|
|
50
57
|
}
|
|
51
58
|
|
|
@@ -55,23 +62,15 @@ export class InMemoryTxPool implements TxPool {
|
|
|
55
62
|
}
|
|
56
63
|
|
|
57
64
|
const keys = txHashes.map(x => x.toBigInt());
|
|
58
|
-
let deleted = 0;
|
|
59
|
-
let added = 0;
|
|
60
65
|
for (const key of keys) {
|
|
61
|
-
|
|
62
|
-
deleted++;
|
|
63
|
-
}
|
|
66
|
+
this.minedTxs.delete(key);
|
|
64
67
|
|
|
65
68
|
// only add back to the pending set if we have the tx object
|
|
66
69
|
if (this.txs.has(key)) {
|
|
67
|
-
added++;
|
|
68
70
|
this.pendingTxs.add(key);
|
|
69
71
|
}
|
|
70
72
|
}
|
|
71
73
|
|
|
72
|
-
this.metrics.recordRemovedObjects(deleted, 'mined');
|
|
73
|
-
this.metrics.recordAddedObjects(added, 'pending');
|
|
74
|
-
|
|
75
74
|
return Promise.resolve();
|
|
76
75
|
}
|
|
77
76
|
|
|
@@ -131,7 +130,6 @@ export class InMemoryTxPool implements TxPool {
|
|
|
131
130
|
* @returns Empty promise.
|
|
132
131
|
*/
|
|
133
132
|
public async addTxs(txs: Tx[]): Promise<void> {
|
|
134
|
-
let pending = 0;
|
|
135
133
|
for (const tx of txs) {
|
|
136
134
|
const txHash = await tx.getTxHash();
|
|
137
135
|
this.log.verbose(`Adding tx ${txHash.toString()} to pool`, {
|
|
@@ -142,14 +140,10 @@ export class InMemoryTxPool implements TxPool {
|
|
|
142
140
|
const key = txHash.toBigInt();
|
|
143
141
|
this.txs.set(key, tx);
|
|
144
142
|
if (!this.minedTxs.has(key)) {
|
|
145
|
-
pending++;
|
|
146
143
|
this.metrics.recordSize(tx);
|
|
147
144
|
this.pendingTxs.add(key);
|
|
148
145
|
}
|
|
149
146
|
}
|
|
150
|
-
|
|
151
|
-
this.metrics.recordAddedObjects(pending, 'pending');
|
|
152
|
-
return;
|
|
153
147
|
}
|
|
154
148
|
|
|
155
149
|
/**
|
|
@@ -158,19 +152,13 @@ export class InMemoryTxPool implements TxPool {
|
|
|
158
152
|
* @returns The number of transactions that was deleted from the pool.
|
|
159
153
|
*/
|
|
160
154
|
public deleteTxs(txHashes: TxHash[]): Promise<void> {
|
|
161
|
-
let deletedMined = 0;
|
|
162
|
-
let deletedPending = 0;
|
|
163
|
-
|
|
164
155
|
for (const txHash of txHashes) {
|
|
165
156
|
const key = txHash.toBigInt();
|
|
166
157
|
this.txs.delete(key);
|
|
167
|
-
|
|
168
|
-
|
|
158
|
+
this.pendingTxs.delete(key);
|
|
159
|
+
this.minedTxs.delete(key);
|
|
169
160
|
}
|
|
170
161
|
|
|
171
|
-
this.metrics.recordRemovedObjects(deletedPending, 'pending');
|
|
172
|
-
this.metrics.recordRemovedObjects(deletedMined, 'mined');
|
|
173
|
-
|
|
174
162
|
return Promise.resolve();
|
|
175
163
|
}
|
|
176
164
|
|
package/src/services/encoding.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { sha256 } from '@aztec/foundation/crypto';
|
|
|
4
4
|
import type { RPC } from '@chainsafe/libp2p-gossipsub/message';
|
|
5
5
|
import type { DataTransform } from '@chainsafe/libp2p-gossipsub/types';
|
|
6
6
|
import type { Message } from '@libp2p/interface';
|
|
7
|
-
import { compressSync, uncompressSync } from 'snappy';
|
|
7
|
+
import { compress, compressSync, uncompress, uncompressSync } from 'snappy';
|
|
8
8
|
import xxhashFactory from 'xxhash-wasm';
|
|
9
9
|
|
|
10
10
|
// Load WASM
|
|
@@ -76,4 +76,12 @@ export class SnappyTransform implements DataTransform {
|
|
|
76
76
|
}
|
|
77
77
|
return Buffer.from(compressSync(data));
|
|
78
78
|
}
|
|
79
|
+
|
|
80
|
+
public outboundTransformAsync(data: Buffer): Promise<Buffer> {
|
|
81
|
+
return compress(data);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
public inboundTransformAsync(data: Buffer): Promise<Buffer> {
|
|
85
|
+
return uncompress(data, { asBuffer: true }) as Promise<Buffer>;
|
|
86
|
+
}
|
|
79
87
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { Timer } from '@aztec/foundation/timer';
|
|
2
|
+
import type { TopicType } from '@aztec/stdlib/p2p';
|
|
3
|
+
import {
|
|
4
|
+
Attributes,
|
|
5
|
+
type Histogram,
|
|
6
|
+
Metrics,
|
|
7
|
+
type TelemetryClient,
|
|
8
|
+
type UpDownCounter,
|
|
9
|
+
ValueType,
|
|
10
|
+
} from '@aztec/telemetry-client';
|
|
11
|
+
|
|
12
|
+
export class P2PInstrumentation {
|
|
13
|
+
private messageValidationDuration: Histogram;
|
|
14
|
+
private messagePrevalidationCount: UpDownCounter;
|
|
15
|
+
|
|
16
|
+
constructor(client: TelemetryClient, name: string) {
|
|
17
|
+
const meter = client.getMeter(name);
|
|
18
|
+
|
|
19
|
+
this.messageValidationDuration = meter.createHistogram(Metrics.P2P_GOSSIP_MESSAGE_VALIDATION_DURATION, {
|
|
20
|
+
unit: 'ms',
|
|
21
|
+
description: 'How long validating a gossiped message takes',
|
|
22
|
+
valueType: ValueType.INT,
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
this.messagePrevalidationCount = meter.createUpDownCounter(Metrics.P2P_GOSSIP_MESSAGE_PREVALIDATION_COUNT, {
|
|
26
|
+
description: 'How many message pass/fail prevalidation',
|
|
27
|
+
valueType: ValueType.INT,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
public recordMessageValidation(topicName: TopicType, timerOrMs: Timer | number) {
|
|
32
|
+
const ms = typeof timerOrMs === 'number' ? timerOrMs : timerOrMs.ms();
|
|
33
|
+
this.messageValidationDuration.record(Math.ceil(ms), { [Attributes.TOPIC_NAME]: topicName });
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
public incMessagePrevalidationStatus(passed: boolean, topicName: TopicType | undefined) {
|
|
37
|
+
this.messagePrevalidationCount.add(1, { [Attributes.TOPIC_NAME]: topicName, [Attributes.OK]: passed });
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -2,6 +2,7 @@ import type { EpochCacheInterface } from '@aztec/epoch-cache';
|
|
|
2
2
|
import { createLibp2pComponentLogger, createLogger } from '@aztec/foundation/log';
|
|
3
3
|
import { SerialQueue } from '@aztec/foundation/queue';
|
|
4
4
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
5
|
+
import { Timer } from '@aztec/foundation/timer';
|
|
5
6
|
import type { AztecAsyncKVStore } from '@aztec/kv-store';
|
|
6
7
|
import { protocolContractTreeRoot } from '@aztec/protocol-contracts';
|
|
7
8
|
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
@@ -65,6 +66,7 @@ import { reqGoodbyeHandler } from '../reqresp/protocols/goodbye.js';
|
|
|
65
66
|
import { pingHandler, reqRespBlockHandler, reqRespTxHandler, statusHandler } from '../reqresp/protocols/index.js';
|
|
66
67
|
import { ReqResp } from '../reqresp/reqresp.js';
|
|
67
68
|
import type { P2PBlockReceivedCallback, P2PService, PeerDiscoveryService } from '../service.js';
|
|
69
|
+
import { P2PInstrumentation } from './instrumentation.js';
|
|
68
70
|
|
|
69
71
|
interface ValidationResult {
|
|
70
72
|
name: string;
|
|
@@ -107,6 +109,8 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
107
109
|
|
|
108
110
|
private gossipSubEventHandler: (e: CustomEvent<GossipsubMessage>) => void;
|
|
109
111
|
|
|
112
|
+
private instrumentation: P2PInstrumentation;
|
|
113
|
+
|
|
110
114
|
constructor(
|
|
111
115
|
private clientType: T,
|
|
112
116
|
private config: P2PConfig,
|
|
@@ -122,6 +126,8 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
122
126
|
) {
|
|
123
127
|
super(telemetry, 'LibP2PService');
|
|
124
128
|
|
|
129
|
+
this.instrumentation = new P2PInstrumentation(telemetry, 'LibP2PService');
|
|
130
|
+
|
|
125
131
|
this.msgIdSeenValidators[TopicType.tx] = new MessageSeenValidator(config.seenMessageCacheSize);
|
|
126
132
|
this.msgIdSeenValidators[TopicType.block_proposal] = new MessageSeenValidator(config.seenMessageCacheSize);
|
|
127
133
|
this.msgIdSeenValidators[TopicType.block_attestation] = new MessageSeenValidator(config.seenMessageCacheSize);
|
|
@@ -501,25 +507,33 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
501
507
|
}
|
|
502
508
|
|
|
503
509
|
protected preValidateReceivedMessage(msg: Message, msgId: string, source: PeerId) {
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
510
|
+
let topicType: TopicType | undefined;
|
|
511
|
+
|
|
512
|
+
switch (msg.topic) {
|
|
513
|
+
case this.topicStrings[TopicType.tx]:
|
|
514
|
+
topicType = TopicType.tx;
|
|
515
|
+
break;
|
|
516
|
+
case this.topicStrings[TopicType.block_attestation]:
|
|
517
|
+
topicType = TopicType.block_attestation;
|
|
518
|
+
break;
|
|
519
|
+
case this.topicStrings[TopicType.block_proposal]:
|
|
520
|
+
topicType = TopicType.block_proposal;
|
|
521
|
+
break;
|
|
522
|
+
default:
|
|
523
|
+
this.logger.error(`Received message on unknown topic: ${msg.topic}`);
|
|
524
|
+
break;
|
|
525
|
+
}
|
|
516
526
|
|
|
517
|
-
const validator =
|
|
527
|
+
const validator = topicType ? this.msgIdSeenValidators[topicType] : undefined;
|
|
518
528
|
|
|
519
529
|
if (!validator || !validator.addMessage(msgId)) {
|
|
530
|
+
this.instrumentation.incMessagePrevalidationStatus(false, topicType);
|
|
520
531
|
this.node.services.pubsub.reportMessageValidationResult(msgId, source.toString(), TopicValidatorResult.Ignore);
|
|
521
532
|
return false;
|
|
522
533
|
}
|
|
534
|
+
|
|
535
|
+
this.instrumentation.incMessagePrevalidationStatus(true, topicType);
|
|
536
|
+
|
|
523
537
|
return true;
|
|
524
538
|
}
|
|
525
539
|
|
|
@@ -559,14 +573,20 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
559
573
|
validationFunc: () => Promise<{ result: boolean; obj: T }>,
|
|
560
574
|
msgId: string,
|
|
561
575
|
source: PeerId,
|
|
576
|
+
topicType: TopicType,
|
|
562
577
|
): Promise<{ result: boolean; obj: T | undefined }> {
|
|
563
578
|
let resultAndObj: { result: boolean; obj: T | undefined } = { result: false, obj: undefined };
|
|
579
|
+
const timer = new Timer();
|
|
564
580
|
try {
|
|
565
581
|
resultAndObj = await validationFunc();
|
|
566
582
|
} catch (err) {
|
|
567
583
|
this.logger.error(`Error deserialising and validating message `, err);
|
|
568
584
|
}
|
|
569
585
|
|
|
586
|
+
if (resultAndObj.result) {
|
|
587
|
+
this.instrumentation.recordMessageValidation(topicType, timer);
|
|
588
|
+
}
|
|
589
|
+
|
|
570
590
|
this.node.services.pubsub.reportMessageValidationResult(
|
|
571
591
|
msgId,
|
|
572
592
|
source.toString(),
|
|
@@ -582,7 +602,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
582
602
|
return { result, obj: tx };
|
|
583
603
|
};
|
|
584
604
|
|
|
585
|
-
const { result, obj: tx } = await this.validateReceivedMessage<Tx>(validationFunc, msgId, source);
|
|
605
|
+
const { result, obj: tx } = await this.validateReceivedMessage<Tx>(validationFunc, msgId, source, TopicType.tx);
|
|
586
606
|
if (!result || !tx) {
|
|
587
607
|
return;
|
|
588
608
|
}
|
|
@@ -613,6 +633,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
613
633
|
validationFunc,
|
|
614
634
|
msgId,
|
|
615
635
|
source,
|
|
636
|
+
TopicType.block_attestation,
|
|
616
637
|
);
|
|
617
638
|
if (!result || !attestation) {
|
|
618
639
|
return;
|
|
@@ -640,7 +661,12 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
|
|
|
640
661
|
return { result, obj: block };
|
|
641
662
|
};
|
|
642
663
|
|
|
643
|
-
const { result, obj: block } = await this.validateReceivedMessage<BlockProposal>(
|
|
664
|
+
const { result, obj: block } = await this.validateReceivedMessage<BlockProposal>(
|
|
665
|
+
validationFunc,
|
|
666
|
+
msgId,
|
|
667
|
+
source,
|
|
668
|
+
TopicType.block_proposal,
|
|
669
|
+
);
|
|
644
670
|
if (!result || !block) {
|
|
645
671
|
return;
|
|
646
672
|
}
|
|
@@ -5,7 +5,7 @@ import { executeTimeout } from '@aztec/foundation/timer';
|
|
|
5
5
|
import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
|
|
6
6
|
import { Attributes, type TelemetryClient, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
7
7
|
|
|
8
|
-
import type
|
|
8
|
+
import { CodeError, type IncomingStreamData, type PeerId, type Stream } from '@libp2p/interface';
|
|
9
9
|
import { pipe } from 'it-pipe';
|
|
10
10
|
import type { Libp2p } from 'libp2p';
|
|
11
11
|
import type { Uint8ArrayList } from 'uint8arraylist';
|
|
@@ -538,7 +538,7 @@ export class ReqResp {
|
|
|
538
538
|
|
|
539
539
|
// Timeout errors are punished with high tolerance, they can be due to a geogrpahically far away peer or an
|
|
540
540
|
// overloaded peer
|
|
541
|
-
if (e instanceof IndividualReqRespTimeoutError) {
|
|
541
|
+
if (e instanceof IndividualReqRespTimeoutError || (e instanceof CodeError && e.code === 'ERR_TIMEOUT')) {
|
|
542
542
|
this.logger.debug(
|
|
543
543
|
`Timeout error: ${e.message} | peerId: ${peerId.toString()} | subProtocol: ${subProtocol}`,
|
|
544
544
|
logTags,
|
|
@@ -573,7 +573,7 @@ export class ReqResp {
|
|
|
573
573
|
}
|
|
574
574
|
|
|
575
575
|
const messageData = Buffer.concat(chunks);
|
|
576
|
-
const message
|
|
576
|
+
const message = await this.snappyTransform.inboundTransformAsync(messageData);
|
|
577
577
|
|
|
578
578
|
return {
|
|
579
579
|
status: statusBuffer ?? ReqRespStatus.UNKNOWN,
|
|
@@ -651,13 +651,13 @@ export class ReqResp {
|
|
|
651
651
|
const successChunk = Buffer.from([ReqRespStatus.SUCCESS]);
|
|
652
652
|
yield new Uint8Array(successChunk);
|
|
653
653
|
|
|
654
|
-
yield new Uint8Array(transform.
|
|
654
|
+
yield new Uint8Array(await transform.outboundTransformAsync(response));
|
|
655
655
|
}
|
|
656
656
|
},
|
|
657
657
|
stream,
|
|
658
658
|
);
|
|
659
659
|
} catch (e: any) {
|
|
660
|
-
this.logger.warn(
|
|
660
|
+
this.logger.warn(`Reqresp Response error: ${e?.name} ${e?.code} ${e?.message}`, { err: e, protocol });
|
|
661
661
|
this.metrics.recordResponseError(protocol);
|
|
662
662
|
|
|
663
663
|
// If we receive a known error, we use the error status in the response chunk, otherwise we categorize as unknown
|
|
@@ -677,7 +677,7 @@ export class ReqResp {
|
|
|
677
677
|
stream,
|
|
678
678
|
);
|
|
679
679
|
} else {
|
|
680
|
-
this.logger.
|
|
680
|
+
this.logger.trace('Stream already closed, not sending error response', { protocol, err: e, errorStatus });
|
|
681
681
|
}
|
|
682
682
|
} finally {
|
|
683
683
|
//NOTE: All other status codes indicate closed stream.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { Metrics, type TelemetryClient, type UpDownCounter } from '@aztec/telemetry-client';
|
|
2
|
+
|
|
3
|
+
export class TxCollectorInstrumentation {
|
|
4
|
+
private txFromProposalCount: UpDownCounter;
|
|
5
|
+
private txFromMempoolCount: UpDownCounter;
|
|
6
|
+
private txFromP2PCount: UpDownCounter;
|
|
7
|
+
private missingTxsCount: UpDownCounter;
|
|
8
|
+
|
|
9
|
+
constructor(client: TelemetryClient, name: string) {
|
|
10
|
+
const meter = client.getMeter(name);
|
|
11
|
+
|
|
12
|
+
this.txFromProposalCount = meter.createUpDownCounter(Metrics.TX_COLLECTOR_TXS_FROM_PROPOSALS_COUNT, {
|
|
13
|
+
description: 'The number of txs taken from block proposals',
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
this.txFromMempoolCount = meter.createUpDownCounter(Metrics.TX_COLLECTOR_TXS_FROM_MEMPOOL_COUNT, {
|
|
17
|
+
description: 'The number of txs taken from the local mempool',
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
this.txFromP2PCount = meter.createUpDownCounter(Metrics.TX_COLLECTOR_TXS_FROM_P2P_COUNT, {
|
|
21
|
+
description: 'The number of txs taken from the p2p network',
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
this.missingTxsCount = meter.createUpDownCounter(Metrics.TX_COLLECTOR_MISSING_TXS_COUNT, {
|
|
25
|
+
description: 'The number of txs not found anywhere',
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
incTxsFromProposals(count: number) {
|
|
30
|
+
this.txFromProposalCount.add(count);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
incTxsFromMempool(count: number) {
|
|
34
|
+
this.txFromMempoolCount.add(count);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
incTxsFromP2P(count: number) {
|
|
38
|
+
this.txFromP2PCount.add(count);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
incMissingTxs(count: number) {
|
|
42
|
+
this.missingTxsCount.add(count);
|
|
43
|
+
}
|
|
44
|
+
}
|