@aztec/p2p 0.0.1-commit.29c6b1a3 → 0.0.1-commit.2e2504e2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/bootstrap/bootstrap.d.ts +4 -3
- package/dest/bootstrap/bootstrap.d.ts.map +1 -1
- package/dest/bootstrap/bootstrap.js +4 -4
- package/dest/client/factory.d.ts +1 -1
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +8 -5
- package/dest/client/p2p_client.d.ts +4 -2
- package/dest/client/p2p_client.d.ts.map +1 -1
- package/dest/client/p2p_client.js +15 -4
- package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +1 -1
- package/dest/config.d.ts +8 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +3 -1
- package/dest/mem_pools/instrumentation.d.ts +1 -1
- package/dest/mem_pools/instrumentation.d.ts.map +1 -1
- package/dest/mem_pools/instrumentation.js +2 -2
- package/dest/mem_pools/tx_pool_v2/archive/index.d.ts +2 -0
- package/dest/mem_pools/tx_pool_v2/archive/index.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/archive/index.js +1 -0
- package/dest/mem_pools/tx_pool_v2/archive/tx_archive.d.ts +43 -0
- package/dest/mem_pools/tx_pool_v2/archive/tx_archive.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/archive/tx_archive.js +103 -0
- package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts +47 -0
- package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.js +119 -0
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.d.ts +17 -0
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.js +90 -0
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts +19 -0
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.js +89 -0
- package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts +10 -0
- package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/index.js +11 -0
- package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts +131 -0
- package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/interfaces.js +17 -0
- package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.d.ts +15 -0
- package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.js +63 -0
- package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.d.ts +17 -0
- package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.js +91 -0
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts +16 -0
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.js +70 -0
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts +20 -0
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.js +63 -0
- package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts +15 -0
- package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.js +19 -0
- package/dest/mem_pools/tx_pool_v2/index.d.ts +5 -0
- package/dest/mem_pools/tx_pool_v2/index.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/index.js +4 -0
- package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +193 -0
- package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/interfaces.js +6 -0
- package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +71 -0
- package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/tx_metadata.js +94 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_bench_metrics.d.ts +26 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_bench_metrics.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_bench_metrics.js +70 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +55 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +150 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +69 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -0
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +1041 -0
- package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.d.ts +1 -1
- package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.d.ts.map +1 -1
- package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.js +7 -2
- package/dest/msg_validators/tx_validator/archive_cache.d.ts +3 -3
- package/dest/msg_validators/tx_validator/archive_cache.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/archive_cache.js +1 -1
- package/dest/msg_validators/tx_validator/block_header_validator.d.ts +5 -4
- package/dest/msg_validators/tx_validator/block_header_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/block_header_validator.js +4 -3
- package/dest/msg_validators/tx_validator/data_validator.d.ts +3 -1
- package/dest/msg_validators/tx_validator/data_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/data_validator.js +4 -1
- package/dest/msg_validators/tx_validator/double_spend_validator.d.ts +3 -2
- package/dest/msg_validators/tx_validator/double_spend_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/double_spend_validator.js +3 -2
- package/dest/msg_validators/tx_validator/factory.d.ts +4 -3
- package/dest/msg_validators/tx_validator/factory.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/factory.js +12 -12
- package/dest/msg_validators/tx_validator/gas_validator.d.ts +3 -2
- package/dest/msg_validators/tx_validator/gas_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/gas_validator.js +3 -2
- package/dest/msg_validators/tx_validator/metadata_validator.d.ts +3 -2
- package/dest/msg_validators/tx_validator/metadata_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/metadata_validator.js +2 -2
- package/dest/msg_validators/tx_validator/phases_validator.d.ts +3 -2
- package/dest/msg_validators/tx_validator/phases_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/phases_validator.js +3 -3
- package/dest/msg_validators/tx_validator/size_validator.d.ts +3 -1
- package/dest/msg_validators/tx_validator/size_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/size_validator.js +4 -1
- package/dest/msg_validators/tx_validator/timestamp_validator.d.ts +3 -2
- package/dest/msg_validators/tx_validator/timestamp_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/timestamp_validator.js +2 -2
- package/dest/msg_validators/tx_validator/tx_permitted_validator.d.ts +3 -2
- package/dest/msg_validators/tx_validator/tx_permitted_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/tx_permitted_validator.js +2 -2
- package/dest/msg_validators/tx_validator/tx_proof_validator.d.ts +3 -2
- package/dest/msg_validators/tx_validator/tx_proof_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/tx_proof_validator.js +2 -2
- package/dest/services/data_store.d.ts +1 -1
- package/dest/services/data_store.d.ts.map +1 -1
- package/dest/services/data_store.js +10 -6
- package/dest/services/discv5/discV5_service.js +1 -1
- package/dest/services/index.d.ts +2 -1
- package/dest/services/index.d.ts.map +1 -1
- package/dest/services/index.js +1 -0
- package/dest/services/libp2p/instrumentation.d.ts +1 -1
- package/dest/services/libp2p/instrumentation.d.ts.map +1 -1
- package/dest/services/libp2p/instrumentation.js +14 -3
- package/dest/services/libp2p/libp2p_service.d.ts +1 -1
- package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
- package/dest/services/libp2p/libp2p_service.js +10 -10
- package/dest/services/peer-manager/metrics.d.ts +2 -2
- package/dest/services/peer-manager/metrics.d.ts.map +1 -1
- package/dest/services/peer-manager/metrics.js +20 -5
- package/dest/services/peer-manager/peer_scoring.d.ts +1 -1
- package/dest/services/peer-manager/peer_scoring.d.ts.map +1 -1
- package/dest/services/peer-manager/peer_scoring.js +8 -2
- package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js +2 -2
- package/dest/services/reqresp/metrics.d.ts +6 -5
- package/dest/services/reqresp/metrics.d.ts.map +1 -1
- package/dest/services/reqresp/metrics.js +17 -5
- package/dest/services/reqresp/protocols/block_txs/block_txs_handler.js +2 -2
- package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts +6 -6
- package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts.map +1 -1
- package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.js +14 -14
- package/dest/services/tx_collection/instrumentation.d.ts +1 -1
- package/dest/services/tx_collection/instrumentation.d.ts.map +1 -1
- package/dest/services/tx_collection/instrumentation.js +9 -2
- package/dest/services/tx_file_store/config.d.ts +18 -0
- package/dest/services/tx_file_store/config.d.ts.map +1 -0
- package/dest/services/tx_file_store/config.js +26 -0
- package/dest/services/tx_file_store/index.d.ts +4 -0
- package/dest/services/tx_file_store/index.d.ts.map +1 -0
- package/dest/services/tx_file_store/index.js +3 -0
- package/dest/services/tx_file_store/instrumentation.d.ts +15 -0
- package/dest/services/tx_file_store/instrumentation.d.ts.map +1 -0
- package/dest/services/tx_file_store/instrumentation.js +29 -0
- package/dest/services/tx_file_store/tx_file_store.d.ts +47 -0
- package/dest/services/tx_file_store/tx_file_store.d.ts.map +1 -0
- package/dest/services/tx_file_store/tx_file_store.js +149 -0
- package/dest/services/tx_provider_instrumentation.d.ts +1 -1
- package/dest/services/tx_provider_instrumentation.d.ts.map +1 -1
- package/dest/services/tx_provider_instrumentation.js +5 -5
- package/dest/test-helpers/testbench-utils.d.ts +3 -1
- package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
- package/dest/test-helpers/testbench-utils.js +2 -1
- package/dest/testbench/p2p_client_testbench_worker.js +1 -1
- package/package.json +14 -14
- package/src/bootstrap/bootstrap.ts +7 -4
- package/src/client/factory.ts +9 -9
- package/src/client/p2p_client.ts +14 -2
- package/src/client/test/tx_proposal_collector/README.md +4 -4
- package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +1 -1
- package/src/config.ts +8 -1
- package/src/mem_pools/instrumentation.ts +2 -1
- package/src/mem_pools/tx_pool_v2/README.md +188 -0
- package/src/mem_pools/tx_pool_v2/archive/index.ts +1 -0
- package/src/mem_pools/tx_pool_v2/archive/tx_archive.ts +120 -0
- package/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts +147 -0
- package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts +118 -0
- package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts +111 -0
- package/src/mem_pools/tx_pool_v2/eviction/index.ts +23 -0
- package/src/mem_pools/tx_pool_v2/eviction/interfaces.ts +164 -0
- package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts +74 -0
- package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts +101 -0
- package/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts +86 -0
- package/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts +72 -0
- package/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts +31 -0
- package/src/mem_pools/tx_pool_v2/index.ts +11 -0
- package/src/mem_pools/tx_pool_v2/interfaces.ts +225 -0
- package/src/mem_pools/tx_pool_v2/tx_metadata.ts +160 -0
- package/src/mem_pools/tx_pool_v2/tx_pool_bench_metrics.ts +77 -0
- package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +209 -0
- package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +1265 -0
- package/src/msg_validators/attestation_validator/fisherman_attestation_validator.ts +8 -2
- package/src/msg_validators/tx_validator/archive_cache.ts +3 -3
- package/src/msg_validators/tx_validator/block_header_validator.ts +7 -8
- package/src/msg_validators/tx_validator/data_validator.ts +6 -2
- package/src/msg_validators/tx_validator/double_spend_validator.ts +4 -3
- package/src/msg_validators/tx_validator/factory.ts +46 -30
- package/src/msg_validators/tx_validator/gas_validator.ts +9 -3
- package/src/msg_validators/tx_validator/metadata_validator.ts +6 -3
- package/src/msg_validators/tx_validator/phases_validator.ts +5 -3
- package/src/msg_validators/tx_validator/size_validator.ts +6 -2
- package/src/msg_validators/tx_validator/timestamp_validator.ts +6 -3
- package/src/msg_validators/tx_validator/tx_permitted_validator.ts +8 -3
- package/src/msg_validators/tx_validator/tx_proof_validator.ts +8 -3
- package/src/services/data_store.ts +10 -7
- package/src/services/discv5/discV5_service.ts +1 -1
- package/src/services/index.ts +1 -0
- package/src/services/libp2p/instrumentation.ts +15 -2
- package/src/services/libp2p/libp2p_service.ts +20 -16
- package/src/services/peer-manager/metrics.ts +21 -4
- package/src/services/peer-manager/peer_scoring.ts +4 -1
- package/src/services/reqresp/batch-tx-requester/README.md +7 -7
- package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts +2 -2
- package/src/services/reqresp/metrics.ts +34 -9
- package/src/services/reqresp/protocols/block_txs/block_txs_handler.ts +2 -2
- package/src/services/reqresp/protocols/block_txs/block_txs_reqresp.ts +10 -10
- package/src/services/tx_collection/instrumentation.ts +11 -2
- package/src/services/tx_file_store/config.ts +43 -0
- package/src/services/tx_file_store/index.ts +3 -0
- package/src/services/tx_file_store/instrumentation.ts +36 -0
- package/src/services/tx_file_store/tx_file_store.ts +173 -0
- package/src/services/tx_provider_instrumentation.ts +11 -5
- package/src/test-helpers/testbench-utils.ts +2 -0
- package/src/testbench/p2p_client_testbench_worker.ts +1 -1
|
@@ -6,12 +6,13 @@ import {
|
|
|
6
6
|
type TelemetryClient,
|
|
7
7
|
type Tracer,
|
|
8
8
|
type UpDownCounter,
|
|
9
|
+
createUpDownCounterWithDefault,
|
|
9
10
|
getTelemetryClient,
|
|
10
11
|
} from '@aztec/telemetry-client';
|
|
11
12
|
|
|
12
13
|
import type { PeerId } from '@libp2p/interface';
|
|
13
14
|
|
|
14
|
-
import {
|
|
15
|
+
import { GoodByeReason, prettyGoodbyeReason } from '../reqresp/protocols/index.js';
|
|
15
16
|
|
|
16
17
|
export class PeerManagerMetrics {
|
|
17
18
|
private sentGoodbyes: UpDownCounter;
|
|
@@ -31,10 +32,26 @@ export class PeerManagerMetrics {
|
|
|
31
32
|
this.tracer = telemetryClient.getTracer(name);
|
|
32
33
|
|
|
33
34
|
const meter = telemetryClient.getMeter(name);
|
|
34
|
-
|
|
35
|
-
|
|
35
|
+
const goodbyeReasonAttrs = {
|
|
36
|
+
[Attributes.P2P_GOODBYE_REASON]: [
|
|
37
|
+
prettyGoodbyeReason(GoodByeReason.SHUTDOWN),
|
|
38
|
+
prettyGoodbyeReason(GoodByeReason.MAX_PEERS),
|
|
39
|
+
prettyGoodbyeReason(GoodByeReason.LOW_SCORE),
|
|
40
|
+
prettyGoodbyeReason(GoodByeReason.BANNED),
|
|
41
|
+
prettyGoodbyeReason(GoodByeReason.WRONG_NETWORK),
|
|
42
|
+
prettyGoodbyeReason(GoodByeReason.UNKNOWN),
|
|
43
|
+
],
|
|
44
|
+
};
|
|
45
|
+
this.sentGoodbyes = createUpDownCounterWithDefault(meter, Metrics.PEER_MANAGER_GOODBYES_SENT, goodbyeReasonAttrs);
|
|
46
|
+
this.receivedGoodbyes = createUpDownCounterWithDefault(
|
|
47
|
+
meter,
|
|
48
|
+
Metrics.PEER_MANAGER_GOODBYES_RECEIVED,
|
|
49
|
+
goodbyeReasonAttrs,
|
|
50
|
+
);
|
|
36
51
|
this.peerCount = meter.createGauge(Metrics.PEER_MANAGER_PEER_COUNT);
|
|
37
|
-
this.lowScoreDisconnects = meter
|
|
52
|
+
this.lowScoreDisconnects = createUpDownCounterWithDefault(meter, Metrics.PEER_MANAGER_LOW_SCORE_DISCONNECTS, {
|
|
53
|
+
[Attributes.P2P_PEER_SCORE_STATE]: ['Banned', 'Disconnect'],
|
|
54
|
+
});
|
|
38
55
|
this.peerConnectionDuration = meter.createHistogram(Metrics.PEER_MANAGER_PEER_CONNECTION_DURATION);
|
|
39
56
|
}
|
|
40
57
|
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
Metrics,
|
|
7
7
|
type TelemetryClient,
|
|
8
8
|
type UpDownCounter,
|
|
9
|
+
createUpDownCounterWithDefault,
|
|
9
10
|
getTelemetryClient,
|
|
10
11
|
} from '@aztec/telemetry-client';
|
|
11
12
|
|
|
@@ -52,7 +53,9 @@ export class PeerScoring {
|
|
|
52
53
|
|
|
53
54
|
const meter = telemetry.getMeter('PeerScoring');
|
|
54
55
|
|
|
55
|
-
this.peerStateCounter = meter
|
|
56
|
+
this.peerStateCounter = createUpDownCounterWithDefault(meter, Metrics.P2P_PEER_STATE_COUNT, {
|
|
57
|
+
[Attributes.P2P_PEER_SCORE_STATE]: ['Healthy', 'Disconnect', 'Banned'],
|
|
58
|
+
});
|
|
56
59
|
}
|
|
57
60
|
|
|
58
61
|
public penalizePeer(peerId: PeerId, penalty: PeerErrorSeverity) {
|
|
@@ -71,7 +71,7 @@ The requester classifies peers into three categories to optimize fetching:
|
|
|
71
71
|
### Blind Phase → Smart Phase Transition
|
|
72
72
|
|
|
73
73
|
Peers transition from "dumb" to "smart" when they respond with a valid `BlockTxsResponse` containing:
|
|
74
|
-
1. A matching `
|
|
74
|
+
1. A matching `archiveRoot`
|
|
75
75
|
2. A non-empty `txIndices` BitVector indicating which transactions they have
|
|
76
76
|
3. At least one transaction we're still missing
|
|
77
77
|
|
|
@@ -81,10 +81,10 @@ Peers transition from "dumb" to "smart" when they respond with a valid `BlockTxs
|
|
|
81
81
|
│ ┌────────────────────────────────────────────────────────────────────────┐ │
|
|
82
82
|
│ │ Initial State: All peers are "dumb" (except pinned peer) │ │
|
|
83
83
|
│ │ │ │
|
|
84
|
-
│ │ Request: [
|
|
84
|
+
│ │ Request: [archiveRoot, txHashes (full list), txIndices (BitVector)] │ │
|
|
85
85
|
│ │ └─ Include full hashes because peer may not have proposal │ │
|
|
86
86
|
│ │ │ │
|
|
87
|
-
│ │ Response: [
|
|
87
|
+
│ │ Response: [archiveRoot, txs[], txIndices (what peer has)] │ │
|
|
88
88
|
│ │ └─ Tells us exactly which txs this peer can provide │ │
|
|
89
89
|
│ └────────────────────────────────────────────────────────────────────────┘ │
|
|
90
90
|
└──────────────────────────────────────────────────────────────────────────────┘
|
|
@@ -97,10 +97,10 @@ Peers transition from "dumb" to "smart" when they respond with a valid `BlockTxs
|
|
|
97
97
|
│ ┌────────────────────────────────────────────────────────────────────────┐ │
|
|
98
98
|
│ │ Peer promoted to "smart" - we know exactly what they have │ │
|
|
99
99
|
│ │ │ │
|
|
100
|
-
│ │ Request: [
|
|
100
|
+
│ │ Request: [archiveRoot, txIndices (BitVector only)] │ │
|
|
101
101
|
│ │ └─ No need for full hashes, peer has the proposal │ │
|
|
102
102
|
│ │ │ │
|
|
103
|
-
│ │ Response: [
|
|
103
|
+
│ │ Response: [archiveRoot, txs[], txIndices (updated availability)] │ │
|
|
104
104
|
│ │ └─ May have received more txs since last response │ │
|
|
105
105
|
│ └────────────────────────────────────────────────────────────────────────┘ │
|
|
106
106
|
└──────────────────────────────────────────────────────────────────────────────┘
|
|
@@ -152,7 +152,7 @@ The `BatchTxRequester` runs three types of workers concurrently:
|
|
|
152
152
|
|
|
153
153
|
```typescript
|
|
154
154
|
class BlockTxsRequest {
|
|
155
|
-
|
|
155
|
+
archiveRoot: Fr; // Archive root after the proposed block is applied
|
|
156
156
|
txHashes: TxHashArray; // Full tx hashes (for dumb peers without proposal)
|
|
157
157
|
txIndices: BitVector; // Which txs from proposal we're requesting (1 = want)
|
|
158
158
|
}
|
|
@@ -162,7 +162,7 @@ class BlockTxsRequest {
|
|
|
162
162
|
|
|
163
163
|
```typescript
|
|
164
164
|
class BlockTxsResponse {
|
|
165
|
-
|
|
165
|
+
archiveRoot: Fr; // Echo back the proposal archive root
|
|
166
166
|
txs: TxArray; // Actual transaction data
|
|
167
167
|
txIndices: BitVector; // Which txs the peer has available (1 = have)
|
|
168
168
|
}
|
|
@@ -605,9 +605,9 @@ export class BatchTxRequester {
|
|
|
605
605
|
}
|
|
606
606
|
|
|
607
607
|
private isBlockResponseValid(response: BlockTxsResponse): boolean {
|
|
608
|
-
const
|
|
608
|
+
const archiveRootsMatch = this.blockProposal.archive.toString() === response.archiveRoot.toString();
|
|
609
609
|
const peerHasSomeTxsFromProposal = !response.txIndices.isEmpty();
|
|
610
|
-
return
|
|
610
|
+
return archiveRootsMatch && peerHasSomeTxsFromProposal;
|
|
611
611
|
}
|
|
612
612
|
|
|
613
613
|
private peerHasSomeTxsWeAreMissing(_peerId: PeerId, response: BlockTxsResponse): boolean {
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
// Request response metrics
|
|
2
|
-
import { Attributes, Metrics } from '@aztec/telemetry-client';
|
|
2
|
+
import { Attributes, Metrics, createUpDownCounterWithDefault } from '@aztec/telemetry-client';
|
|
3
3
|
import type { TelemetryClient, Tracer, UpDownCounter } from '@aztec/telemetry-client';
|
|
4
4
|
|
|
5
|
+
import { ReqRespSubProtocol } from './interface.js';
|
|
6
|
+
|
|
5
7
|
export class ReqRespMetrics {
|
|
6
8
|
public readonly tracer: Tracer;
|
|
7
9
|
|
|
@@ -18,27 +20,50 @@ export class ReqRespMetrics {
|
|
|
18
20
|
this.tracer = telemetryClient.getTracer(name);
|
|
19
21
|
|
|
20
22
|
const meter = telemetryClient.getMeter(name);
|
|
21
|
-
|
|
22
|
-
|
|
23
|
+
const protocolAttrs = {
|
|
24
|
+
[Attributes.P2P_REQ_RESP_PROTOCOL]: [
|
|
25
|
+
ReqRespSubProtocol.PING,
|
|
26
|
+
ReqRespSubProtocol.STATUS,
|
|
27
|
+
ReqRespSubProtocol.GOODBYE,
|
|
28
|
+
ReqRespSubProtocol.TX,
|
|
29
|
+
ReqRespSubProtocol.BLOCK,
|
|
30
|
+
ReqRespSubProtocol.AUTH,
|
|
31
|
+
ReqRespSubProtocol.BLOCK_TXS,
|
|
32
|
+
],
|
|
33
|
+
};
|
|
34
|
+
this.sentRequests = createUpDownCounterWithDefault(meter, Metrics.P2P_REQ_RESP_SENT_REQUESTS, protocolAttrs);
|
|
35
|
+
this.receivedRequests = createUpDownCounterWithDefault(
|
|
36
|
+
meter,
|
|
37
|
+
Metrics.P2P_REQ_RESP_RECEIVED_REQUESTS,
|
|
38
|
+
protocolAttrs,
|
|
39
|
+
);
|
|
23
40
|
|
|
24
|
-
this.failedOutboundRequests =
|
|
41
|
+
this.failedOutboundRequests = createUpDownCounterWithDefault(
|
|
42
|
+
meter,
|
|
43
|
+
Metrics.P2P_REQ_RESP_FAILED_OUTBOUND_REQUESTS,
|
|
44
|
+
protocolAttrs,
|
|
45
|
+
);
|
|
25
46
|
|
|
26
|
-
this.failedInboundRequests =
|
|
47
|
+
this.failedInboundRequests = createUpDownCounterWithDefault(
|
|
48
|
+
meter,
|
|
49
|
+
Metrics.P2P_REQ_RESP_FAILED_INBOUND_REQUESTS,
|
|
50
|
+
protocolAttrs,
|
|
51
|
+
);
|
|
27
52
|
}
|
|
28
53
|
|
|
29
|
-
public recordRequestSent(protocol:
|
|
54
|
+
public recordRequestSent(protocol: ReqRespSubProtocol) {
|
|
30
55
|
this.sentRequests.add(1, { [Attributes.P2P_REQ_RESP_PROTOCOL]: protocol });
|
|
31
56
|
}
|
|
32
57
|
|
|
33
|
-
public recordRequestReceived(protocol:
|
|
58
|
+
public recordRequestReceived(protocol: ReqRespSubProtocol) {
|
|
34
59
|
this.receivedRequests.add(1, { [Attributes.P2P_REQ_RESP_PROTOCOL]: protocol });
|
|
35
60
|
}
|
|
36
61
|
|
|
37
|
-
public recordRequestError(protocol:
|
|
62
|
+
public recordRequestError(protocol: ReqRespSubProtocol) {
|
|
38
63
|
this.failedOutboundRequests.add(1, { [Attributes.P2P_REQ_RESP_PROTOCOL]: protocol });
|
|
39
64
|
}
|
|
40
65
|
|
|
41
|
-
public recordResponseError(protocol:
|
|
66
|
+
public recordResponseError(protocol: ReqRespSubProtocol) {
|
|
42
67
|
this.failedInboundRequests.add(1, { [Attributes.P2P_REQ_RESP_PROTOCOL]: protocol });
|
|
43
68
|
}
|
|
44
69
|
}
|
|
@@ -31,7 +31,7 @@ export function reqRespBlockTxsHandler(attestationPool: AttestationPool, txPool:
|
|
|
31
31
|
throw new ReqRespStatusError(ReqRespStatus.BADLY_FORMED_REQUEST, { cause: err });
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
const blockProposal = await attestationPool.getBlockProposal(request.
|
|
34
|
+
const blockProposal = await attestationPool.getBlockProposal(request.archiveRoot.toString());
|
|
35
35
|
|
|
36
36
|
let requestedTxsHashes;
|
|
37
37
|
if (request.txHashes.length > 0) {
|
|
@@ -60,7 +60,7 @@ export function reqRespBlockTxsHandler(attestationPool: AttestationPool, txPool:
|
|
|
60
60
|
requestedTxsHashes = blockProposal.txHashes.filter((_, idx) => requestedIndices.has(idx));
|
|
61
61
|
|
|
62
62
|
const responseTxs = (await txPool.getTxsByHash(requestedTxsHashes)).filter(tx => !!tx);
|
|
63
|
-
const response = new BlockTxsResponse(request.
|
|
63
|
+
const response = new BlockTxsResponse(request.archiveRoot, new TxArray(...responseTxs), responseBitVector);
|
|
64
64
|
|
|
65
65
|
return response.toBuffer();
|
|
66
66
|
};
|
|
@@ -10,8 +10,8 @@ import { BitVector } from './bitvector.js';
|
|
|
10
10
|
*/
|
|
11
11
|
export class BlockTxsRequest {
|
|
12
12
|
constructor(
|
|
13
|
-
//
|
|
14
|
-
readonly
|
|
13
|
+
// Archive root after the proposed block is applied (proposal identifier)
|
|
14
|
+
readonly archiveRoot: Fr,
|
|
15
15
|
// Hashes of txs we are requesting
|
|
16
16
|
readonly txHashes: TxHashArray,
|
|
17
17
|
// BitVector indicating which txs from the proposal we are requesting
|
|
@@ -22,7 +22,7 @@ export class BlockTxsRequest {
|
|
|
22
22
|
) {}
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
|
-
*
|
|
25
|
+
* Creates new BlockTxsRequest given proposal and missing tx hashes
|
|
26
26
|
*
|
|
27
27
|
* @param: blockProposal - The block proposal for which we are making request
|
|
28
28
|
* @param: missingTxHashes - Tx hashes from the proposal we are missing
|
|
@@ -63,11 +63,11 @@ export class BlockTxsRequest {
|
|
|
63
63
|
*/
|
|
64
64
|
static fromBuffer(buffer: Buffer | BufferReader): BlockTxsRequest {
|
|
65
65
|
const reader = BufferReader.asReader(buffer);
|
|
66
|
-
const
|
|
66
|
+
const archiveRoot = Fr.fromBuffer(reader);
|
|
67
67
|
const txHashes = TxHashArray.fromBuffer(reader);
|
|
68
68
|
const txIndices = BitVector.fromBuffer(reader);
|
|
69
69
|
|
|
70
|
-
return new BlockTxsRequest(
|
|
70
|
+
return new BlockTxsRequest(archiveRoot, txHashes, txIndices);
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
/**
|
|
@@ -75,7 +75,7 @@ export class BlockTxsRequest {
|
|
|
75
75
|
* @returns Buffer representation of the BlockTxRequest object
|
|
76
76
|
*/
|
|
77
77
|
toBuffer(): Buffer {
|
|
78
|
-
return serializeToBuffer([this.
|
|
78
|
+
return serializeToBuffer([this.archiveRoot, this.txHashes.toBuffer(), this.txIndices.toBuffer()]);
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
81
|
|
|
@@ -84,7 +84,7 @@ export class BlockTxsRequest {
|
|
|
84
84
|
*/
|
|
85
85
|
export class BlockTxsResponse {
|
|
86
86
|
constructor(
|
|
87
|
-
readonly
|
|
87
|
+
readonly archiveRoot: Fr,
|
|
88
88
|
readonly txs: TxArray, // List of transactions we requested and peer has
|
|
89
89
|
// BitVector indicating which txs from the proposal are available at the peer
|
|
90
90
|
// 1 means the tx is available, 0 means it is not
|
|
@@ -98,11 +98,11 @@ export class BlockTxsResponse {
|
|
|
98
98
|
*/
|
|
99
99
|
static fromBuffer(buffer: Buffer | BufferReader): BlockTxsResponse {
|
|
100
100
|
const reader = BufferReader.asReader(buffer);
|
|
101
|
-
const
|
|
101
|
+
const archiveRoot = Fr.fromBuffer(reader);
|
|
102
102
|
const txs = TxArray.fromBuffer(reader);
|
|
103
103
|
const txIndices = BitVector.fromBuffer(reader);
|
|
104
104
|
|
|
105
|
-
return new BlockTxsResponse(
|
|
105
|
+
return new BlockTxsResponse(archiveRoot, txs, txIndices);
|
|
106
106
|
}
|
|
107
107
|
|
|
108
108
|
/**
|
|
@@ -112,7 +112,7 @@ export class BlockTxsResponse {
|
|
|
112
112
|
* @returns Buffer representation of the BlockTxResponse object
|
|
113
113
|
*/
|
|
114
114
|
toBuffer(): Buffer {
|
|
115
|
-
return serializeToBuffer([this.
|
|
115
|
+
return serializeToBuffer([this.archiveRoot, this.txs.toBuffer(), this.txIndices.toBuffer()]);
|
|
116
116
|
}
|
|
117
117
|
|
|
118
118
|
static empty(): BlockTxsResponse {
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
Attributes,
|
|
3
|
+
type Histogram,
|
|
4
|
+
Metrics,
|
|
5
|
+
type TelemetryClient,
|
|
6
|
+
type UpDownCounter,
|
|
7
|
+
createUpDownCounterWithDefault,
|
|
8
|
+
} from '@aztec/telemetry-client';
|
|
2
9
|
|
|
3
10
|
import type { CollectionMethod } from './tx_collection.js';
|
|
4
11
|
|
|
@@ -10,7 +17,9 @@ export class TxCollectionInstrumentation {
|
|
|
10
17
|
constructor(client: TelemetryClient, name: string) {
|
|
11
18
|
const meter = client.getMeter(name);
|
|
12
19
|
|
|
13
|
-
this.txsCollected = meter
|
|
20
|
+
this.txsCollected = createUpDownCounterWithDefault(meter, Metrics.TX_COLLECTOR_COUNT, {
|
|
21
|
+
[Attributes.TX_COLLECTION_METHOD]: ['fast-req-resp', 'fast-node-rpc', 'slow-req-resp', 'slow-node-rpc'],
|
|
22
|
+
});
|
|
14
23
|
|
|
15
24
|
this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX);
|
|
16
25
|
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { type ConfigMappingsType, booleanConfigHelper, numberConfigHelper } from '@aztec/foundation/config';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Configuration for the TxFileStore service.
|
|
5
|
+
*/
|
|
6
|
+
export type TxFileStoreConfig = {
|
|
7
|
+
/** URL for uploading txs to file storage (s3://, gs://, file://) */
|
|
8
|
+
txFileStoreUrl?: string;
|
|
9
|
+
/** URL for downloading txs from file storage */
|
|
10
|
+
txFileStoreDownloadUrl?: string;
|
|
11
|
+
/** Max concurrent uploads */
|
|
12
|
+
txFileStoreUploadConcurrency: number;
|
|
13
|
+
/** Max queue size to prevent unbounded memory growth */
|
|
14
|
+
txFileStoreMaxQueueSize: number;
|
|
15
|
+
/** Enable tx file store upload */
|
|
16
|
+
txFileStoreEnabled: boolean;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export const txFileStoreConfigMappings: ConfigMappingsType<TxFileStoreConfig> = {
|
|
20
|
+
txFileStoreUrl: {
|
|
21
|
+
env: 'TX_FILE_STORE_URL',
|
|
22
|
+
description: 'URL for uploading txs to file storage (s3://, gs://, file://)',
|
|
23
|
+
},
|
|
24
|
+
txFileStoreDownloadUrl: {
|
|
25
|
+
env: 'TX_FILE_STORE_DOWNLOAD_URL',
|
|
26
|
+
description: 'URL for downloading txs from file storage',
|
|
27
|
+
},
|
|
28
|
+
txFileStoreUploadConcurrency: {
|
|
29
|
+
env: 'TX_FILE_STORE_UPLOAD_CONCURRENCY',
|
|
30
|
+
description: 'Maximum number of concurrent tx uploads',
|
|
31
|
+
...numberConfigHelper(10),
|
|
32
|
+
},
|
|
33
|
+
txFileStoreMaxQueueSize: {
|
|
34
|
+
env: 'TX_FILE_STORE_MAX_QUEUE_SIZE',
|
|
35
|
+
description: 'Maximum queue size for pending uploads (oldest dropped when exceeded)',
|
|
36
|
+
...numberConfigHelper(1000),
|
|
37
|
+
},
|
|
38
|
+
txFileStoreEnabled: {
|
|
39
|
+
env: 'TX_FILE_STORE_ENABLED',
|
|
40
|
+
description: 'Enable uploading transactions to file storage',
|
|
41
|
+
...booleanConfigHelper(false),
|
|
42
|
+
},
|
|
43
|
+
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { type Gauge, type Histogram, Metrics, type TelemetryClient, type UpDownCounter } from '@aztec/telemetry-client';
|
|
2
|
+
|
|
3
|
+
/** Instrumentation for the TxFileStore service. */
|
|
4
|
+
export class TxFileStoreInstrumentation {
|
|
5
|
+
private uploadsSuccess: UpDownCounter;
|
|
6
|
+
private uploadsFailed: UpDownCounter;
|
|
7
|
+
private uploadsSkipped: UpDownCounter;
|
|
8
|
+
private uploadDuration: Histogram;
|
|
9
|
+
private queueSize: Gauge;
|
|
10
|
+
|
|
11
|
+
constructor(client: TelemetryClient, name: string) {
|
|
12
|
+
const meter = client.getMeter(name);
|
|
13
|
+
this.uploadsSuccess = meter.createUpDownCounter(Metrics.TX_FILE_STORE_UPLOADS_SUCCESS);
|
|
14
|
+
this.uploadsFailed = meter.createUpDownCounter(Metrics.TX_FILE_STORE_UPLOADS_FAILED);
|
|
15
|
+
this.uploadsSkipped = meter.createUpDownCounter(Metrics.TX_FILE_STORE_UPLOADS_SKIPPED);
|
|
16
|
+
this.uploadDuration = meter.createHistogram(Metrics.TX_FILE_STORE_UPLOAD_DURATION);
|
|
17
|
+
this.queueSize = meter.createGauge(Metrics.TX_FILE_STORE_QUEUE_SIZE);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
recordUploadSuccess(durationMs: number) {
|
|
21
|
+
this.uploadsSuccess.add(1);
|
|
22
|
+
this.uploadDuration.record(durationMs);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
recordUploadFailed() {
|
|
26
|
+
this.uploadsFailed.add(1);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
recordUploadSkipped() {
|
|
30
|
+
this.uploadsSkipped.add(1);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
recordQueueSize(size: number) {
|
|
34
|
+
this.queueSize.record(size);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import { RunningPromise } from '@aztec/foundation/promise';
|
|
3
|
+
import { makeBackoff, retry } from '@aztec/foundation/retry';
|
|
4
|
+
import { Timer } from '@aztec/foundation/timer';
|
|
5
|
+
import { type FileStore, createFileStore } from '@aztec/stdlib/file-store';
|
|
6
|
+
import type { Tx } from '@aztec/stdlib/tx';
|
|
7
|
+
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
8
|
+
|
|
9
|
+
import type { TxPool, TxPoolEvents } from '../../mem_pools/tx_pool/index.js';
|
|
10
|
+
import type { TxFileStoreConfig } from './config.js';
|
|
11
|
+
import { TxFileStoreInstrumentation } from './instrumentation.js';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Uploads validated transactions to a file store as a fallback retrieval mechanism.
|
|
15
|
+
* Listens to TxPool txs-added events and uploads txs asynchronously with bounded concurrency.
|
|
16
|
+
*/
|
|
17
|
+
export class TxFileStore {
|
|
18
|
+
private uploadQueue: Tx[] = [];
|
|
19
|
+
private activeUploads = 0;
|
|
20
|
+
private readonly queueProcessor: RunningPromise;
|
|
21
|
+
private readonly handleTxsAdded: TxPoolEvents['txs-added'];
|
|
22
|
+
|
|
23
|
+
/** Recently uploaded tx hashes for deduplication. */
|
|
24
|
+
private recentUploads: Set<string> = new Set();
|
|
25
|
+
private recentUploadsOrder: string[] = [];
|
|
26
|
+
private readonly maxRecentUploads = 1000;
|
|
27
|
+
|
|
28
|
+
private constructor(
|
|
29
|
+
private readonly fileStore: FileStore,
|
|
30
|
+
private readonly txPool: TxPool,
|
|
31
|
+
private readonly config: TxFileStoreConfig,
|
|
32
|
+
private readonly instrumentation: TxFileStoreInstrumentation,
|
|
33
|
+
private readonly log: Logger,
|
|
34
|
+
) {
|
|
35
|
+
this.handleTxsAdded = (args: { txs: Tx[]; source?: string }) => {
|
|
36
|
+
this.enqueueTxs(args.txs);
|
|
37
|
+
};
|
|
38
|
+
this.queueProcessor = new RunningPromise(() => this.processQueueBatch(), this.log, 100);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Creates and initializes the file store.
|
|
43
|
+
* @param txPool - The transaction pool to listen to.
|
|
44
|
+
* @param config - The file store configuration.
|
|
45
|
+
* @param log - Optional logger.
|
|
46
|
+
* @param telemetry - Optional telemetry client.
|
|
47
|
+
* @param fileStoreOverride - Optional FileStore for testing (bypasses createFileStore).
|
|
48
|
+
* @returns The file store instance, or undefined if not configured/enabled.
|
|
49
|
+
*/
|
|
50
|
+
static async create(
|
|
51
|
+
txPool: TxPool,
|
|
52
|
+
config: TxFileStoreConfig,
|
|
53
|
+
log: Logger = createLogger('p2p:tx_file_store'),
|
|
54
|
+
telemetry: TelemetryClient = getTelemetryClient(),
|
|
55
|
+
fileStoreOverride?: FileStore,
|
|
56
|
+
): Promise<TxFileStore | undefined> {
|
|
57
|
+
if (!config.txFileStoreEnabled) {
|
|
58
|
+
log.debug('Tx file store is disabled');
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (!config.txFileStoreUrl) {
|
|
63
|
+
log.warn('Tx file store is enabled but URL is not configured');
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const fileStore = fileStoreOverride ?? (await createFileStore(config.txFileStoreUrl, log));
|
|
68
|
+
if (!fileStore) {
|
|
69
|
+
log.warn('Failed to create file store for tx file store');
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const instrumentation = new TxFileStoreInstrumentation(telemetry, 'TxFileStore');
|
|
74
|
+
log.info('Created tx file store', { url: config.txFileStoreUrl });
|
|
75
|
+
return new TxFileStore(fileStore, txPool, config, instrumentation, log);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Starts listening to TxPool events and uploading txs. */
|
|
79
|
+
public start(): void {
|
|
80
|
+
this.queueProcessor.start();
|
|
81
|
+
this.txPool.on('txs-added', this.handleTxsAdded);
|
|
82
|
+
this.log.info('Started tx file store', {
|
|
83
|
+
concurrency: this.config.txFileStoreUploadConcurrency,
|
|
84
|
+
maxQueueSize: this.config.txFileStoreMaxQueueSize,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Stops listening and waits for pending uploads to complete. */
|
|
89
|
+
public async stop(): Promise<void> {
|
|
90
|
+
this.txPool.removeListener('txs-added', this.handleTxsAdded);
|
|
91
|
+
await this.queueProcessor.stop();
|
|
92
|
+
this.log.info('Stopped tx file store');
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
private enqueueTxs(txs: Tx[]): void {
|
|
96
|
+
this.uploadQueue.push(...txs);
|
|
97
|
+
|
|
98
|
+
// Enforce max queue size by dropping oldest entries
|
|
99
|
+
const overflow = this.uploadQueue.length - this.config.txFileStoreMaxQueueSize;
|
|
100
|
+
if (overflow > 0) {
|
|
101
|
+
this.log.warn(`Upload queue overflow, dropping ${overflow} oldest txs`);
|
|
102
|
+
this.uploadQueue.splice(0, overflow);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
this.instrumentation.recordQueueSize(this.uploadQueue.length);
|
|
106
|
+
|
|
107
|
+
// Immediately start uploading txs
|
|
108
|
+
void this.queueProcessor.trigger();
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
private async processQueueBatch(): Promise<void> {
|
|
112
|
+
const batch = this.uploadQueue.splice(0, this.config.txFileStoreUploadConcurrency);
|
|
113
|
+
this.instrumentation.recordQueueSize(this.uploadQueue.length);
|
|
114
|
+
|
|
115
|
+
this.activeUploads += batch.length;
|
|
116
|
+
try {
|
|
117
|
+
await Promise.all(batch.map(tx => this.uploadTx(tx)));
|
|
118
|
+
} finally {
|
|
119
|
+
this.activeUploads -= batch.length;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
private async uploadTx(tx: Tx): Promise<void> {
|
|
124
|
+
const txHash = tx.getTxHash().toString();
|
|
125
|
+
const path = `txs/${txHash}.bin`;
|
|
126
|
+
const timer = new Timer();
|
|
127
|
+
|
|
128
|
+
if (this.recentUploads.has(txHash)) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
try {
|
|
133
|
+
this.recentUploads.add(txHash);
|
|
134
|
+
this.recentUploadsOrder.push(txHash);
|
|
135
|
+
|
|
136
|
+
if (this.recentUploadsOrder.length > this.maxRecentUploads) {
|
|
137
|
+
// delete old entries in recentUploads
|
|
138
|
+
for (const txHashToRemove of this.recentUploadsOrder.splice(
|
|
139
|
+
0,
|
|
140
|
+
this.recentUploadsOrder.length - this.maxRecentUploads,
|
|
141
|
+
)) {
|
|
142
|
+
this.recentUploads.delete(txHashToRemove);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
await retry(
|
|
147
|
+
() => this.fileStore.save(path, tx.toBuffer(), { compress: false }),
|
|
148
|
+
`Uploading tx ${txHash}`,
|
|
149
|
+
makeBackoff([0.1, 0.5, 2]),
|
|
150
|
+
this.log,
|
|
151
|
+
true, // failSilently - don't log errors during retries
|
|
152
|
+
);
|
|
153
|
+
const durationMs = Math.trunc(timer.ms());
|
|
154
|
+
this.log.debug(`Uploaded tx to file store`, { txHash, path, durationMs });
|
|
155
|
+
this.instrumentation.recordUploadSuccess(durationMs);
|
|
156
|
+
} catch (err) {
|
|
157
|
+
this.log.warn(`Failed to upload tx to file store after retries`, { txHash, error: err });
|
|
158
|
+
this.instrumentation.recordUploadFailed();
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Waits for all queued and in-flight uploads to complete. For testing. */
|
|
163
|
+
public async flush(): Promise<void> {
|
|
164
|
+
while (this.uploadQueue.length > 0 || this.activeUploads > 0) {
|
|
165
|
+
await this.queueProcessor.trigger();
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** Returns the number of pending uploads (queued + in-flight). */
|
|
170
|
+
public getPendingUploadCount(): number {
|
|
171
|
+
return this.uploadQueue.length + this.activeUploads;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
type Histogram,
|
|
3
|
+
Metrics,
|
|
4
|
+
type TelemetryClient,
|
|
5
|
+
type UpDownCounter,
|
|
6
|
+
createUpDownCounterWithDefault,
|
|
7
|
+
} from '@aztec/telemetry-client';
|
|
2
8
|
|
|
3
9
|
export class TxProviderInstrumentation {
|
|
4
10
|
private txFromProposalCount: UpDownCounter;
|
|
@@ -12,13 +18,13 @@ export class TxProviderInstrumentation {
|
|
|
12
18
|
constructor(client: TelemetryClient, name: string) {
|
|
13
19
|
const meter = client.getMeter(name);
|
|
14
20
|
|
|
15
|
-
this.txFromProposalCount = meter
|
|
21
|
+
this.txFromProposalCount = createUpDownCounterWithDefault(meter, Metrics.TX_PROVIDER_TXS_FROM_PROPOSALS_COUNT);
|
|
16
22
|
|
|
17
|
-
this.txFromMempoolCount = meter
|
|
23
|
+
this.txFromMempoolCount = createUpDownCounterWithDefault(meter, Metrics.TX_PROVIDER_TXS_FROM_MEMPOOL_COUNT);
|
|
18
24
|
|
|
19
|
-
this.txFromP2PCount = meter
|
|
25
|
+
this.txFromP2PCount = createUpDownCounterWithDefault(meter, Metrics.TX_PROVIDER_TXS_FROM_P2P_COUNT);
|
|
20
26
|
|
|
21
|
-
this.missingTxsCount = meter
|
|
27
|
+
this.missingTxsCount = createUpDownCounterWithDefault(meter, Metrics.TX_PROVIDER_MISSING_TXS_COUNT);
|
|
22
28
|
|
|
23
29
|
this.fractionOfTxsRequestedFromP2P = meter.createHistogram(Metrics.TX_PROVIDER_P2P_TXS_REQUESTED_FRACTION);
|
|
24
30
|
|
|
@@ -321,6 +321,8 @@ export const BENCHMARK_CONSTANTS = {
|
|
|
321
321
|
FIXED_MAX_PEERS: 10,
|
|
322
322
|
/** Fixed max retry attempts for fair benchmarking */
|
|
323
323
|
FIXED_MAX_RETRY_ATTEMPTS: 3,
|
|
324
|
+
/** LMDB map size for temp stores used in benchmarks (in KB). */
|
|
325
|
+
KV_STORE_MAP_SIZE_KB: 256 * 1024,
|
|
324
326
|
} as const;
|
|
325
327
|
|
|
326
328
|
/**
|
|
@@ -345,7 +345,7 @@ process.on('message', async msg => {
|
|
|
345
345
|
const l2BlockSource = new MockL2BlockSource();
|
|
346
346
|
|
|
347
347
|
const proofVerifier = new AlwaysTrueCircuitVerifier();
|
|
348
|
-
kvStore = await openTmpStore(`test-${clientIndex}
|
|
348
|
+
kvStore = await openTmpStore(`test-${clientIndex}`, true, BENCHMARK_CONSTANTS.KV_STORE_MAP_SIZE_KB);
|
|
349
349
|
workerLogger = createLogger(`p2p:${clientIndex}`);
|
|
350
350
|
workerTxPool.setLogger(workerLogger);
|
|
351
351
|
const telemetry = getTelemetryClient();
|