@aztec/p2p 0.0.1-commit.88e6f9396 → 0.0.1-commit.8c0b8ff
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/factory.d.ts +2 -2
- package/dest/client/factory.d.ts.map +1 -1
- package/dest/client/factory.js +5 -5
- package/dest/client/p2p_client.d.ts +1 -1
- package/dest/client/p2p_client.d.ts.map +1 -1
- package/dest/client/p2p_client.js +4 -6
- package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +5 -6
- package/dest/config.d.ts +6 -6
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +6 -6
- package/dest/mem_pools/attestation_pool/attestation_pool.d.ts +4 -4
- package/dest/mem_pools/attestation_pool/attestation_pool.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/attestation_pool.js +4 -8
- package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +6 -6
- package/dest/mem_pools/instrumentation.d.ts +2 -4
- package/dest/mem_pools/instrumentation.d.ts.map +1 -1
- package/dest/mem_pools/instrumentation.js +14 -16
- package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +5 -7
- package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/interfaces.js +0 -1
- package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +6 -5
- package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +43 -26
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +0 -3
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +1 -2
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +1 -18
- package/dest/msg_validators/attestation_validator/attestation_validator.d.ts +1 -1
- package/dest/msg_validators/attestation_validator/attestation_validator.d.ts.map +1 -1
- package/dest/msg_validators/attestation_validator/attestation_validator.js +4 -5
- package/dest/msg_validators/clock_tolerance.d.ts +1 -1
- package/dest/msg_validators/clock_tolerance.d.ts.map +1 -1
- package/dest/msg_validators/clock_tolerance.js +3 -4
- package/dest/msg_validators/proposal_validator/proposal_validator.d.ts +1 -1
- package/dest/msg_validators/proposal_validator/proposal_validator.d.ts.map +1 -1
- package/dest/msg_validators/proposal_validator/proposal_validator.js +5 -5
- package/dest/msg_validators/tx_validator/contract_instance_validator.d.ts +9 -0
- package/dest/msg_validators/tx_validator/contract_instance_validator.d.ts.map +1 -0
- package/dest/msg_validators/tx_validator/contract_instance_validator.js +48 -0
- package/dest/msg_validators/tx_validator/data_validator.d.ts +1 -1
- package/dest/msg_validators/tx_validator/data_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/data_validator.js +35 -2
- package/dest/msg_validators/tx_validator/factory.d.ts +1 -1
- package/dest/msg_validators/tx_validator/factory.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/factory.js +8 -2
- package/dest/msg_validators/tx_validator/metadata_validator.d.ts +1 -1
- package/dest/msg_validators/tx_validator/metadata_validator.d.ts.map +1 -1
- package/dest/msg_validators/tx_validator/metadata_validator.js +4 -4
- package/dest/msg_validators/tx_validator/phases_validator.js +1 -1
- package/dest/services/libp2p/libp2p_service.d.ts +9 -2
- package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
- package/dest/services/libp2p/libp2p_service.js +25 -22
- package/dest/services/peer-manager/metrics.d.ts +1 -3
- package/dest/services/peer-manager/metrics.d.ts.map +1 -1
- package/dest/services/peer-manager/metrics.js +0 -6
- package/dest/services/peer-manager/peer_manager.d.ts +1 -1
- package/dest/services/peer-manager/peer_manager.d.ts.map +1 -1
- package/dest/services/peer-manager/peer_manager.js +3 -6
- package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts +8 -11
- package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts.map +1 -1
- package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js +101 -79
- package/dest/services/reqresp/batch-tx-requester/interface.d.ts +2 -3
- package/dest/services/reqresp/batch-tx-requester/interface.d.ts.map +1 -1
- package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts +4 -5
- package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts.map +1 -1
- package/dest/services/reqresp/batch-tx-requester/missing_txs.js +7 -13
- package/dest/services/reqresp/batch-tx-requester/peer_collection.d.ts +11 -19
- package/dest/services/reqresp/batch-tx-requester/peer_collection.d.ts.map +1 -1
- package/dest/services/reqresp/batch-tx-requester/peer_collection.js +15 -52
- package/dest/services/reqresp/rate-limiter/rate_limiter.d.ts +5 -4
- package/dest/services/reqresp/rate-limiter/rate_limiter.d.ts.map +1 -1
- package/dest/services/reqresp/rate-limiter/rate_limiter.js +10 -8
- package/dest/services/reqresp/reqresp.d.ts +1 -1
- package/dest/services/reqresp/reqresp.d.ts.map +1 -1
- package/dest/services/reqresp/reqresp.js +3 -4
- package/dest/services/tx_collection/fast_tx_collection.d.ts +4 -1
- package/dest/services/tx_collection/fast_tx_collection.d.ts.map +1 -1
- package/dest/services/tx_collection/fast_tx_collection.js +73 -57
- package/dest/services/tx_collection/file_store_tx_source.d.ts +5 -4
- package/dest/services/tx_collection/file_store_tx_source.d.ts.map +1 -1
- package/dest/services/tx_collection/file_store_tx_source.js +39 -29
- package/dest/services/tx_collection/missing_txs_tracker.d.ts +32 -0
- package/dest/services/tx_collection/missing_txs_tracker.d.ts.map +1 -0
- package/dest/services/tx_collection/missing_txs_tracker.js +27 -0
- package/dest/services/tx_collection/proposal_tx_collector.d.ts +7 -6
- package/dest/services/tx_collection/proposal_tx_collector.d.ts.map +1 -1
- package/dest/services/tx_collection/proposal_tx_collector.js +4 -4
- package/dest/services/tx_collection/slow_tx_collection.js +1 -1
- package/dest/services/tx_collection/tx_collection.d.ts +6 -3
- package/dest/services/tx_collection/tx_collection.d.ts.map +1 -1
- package/dest/services/tx_collection/tx_source.d.ts +6 -5
- package/dest/services/tx_collection/tx_source.d.ts.map +1 -1
- package/dest/services/tx_collection/tx_source.js +9 -7
- package/dest/test-helpers/make-test-p2p-clients.d.ts +1 -1
- package/dest/test-helpers/make-test-p2p-clients.d.ts.map +1 -1
- package/dest/test-helpers/reqresp-nodes.d.ts +1 -1
- package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
- package/dest/test-helpers/testbench-utils.d.ts +1 -1
- package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
- package/dest/test-helpers/testbench-utils.js +2 -20
- package/dest/testbench/p2p_client_testbench_worker.d.ts +1 -1
- package/dest/testbench/p2p_client_testbench_worker.d.ts.map +1 -1
- package/dest/testbench/p2p_client_testbench_worker.js +5 -6
- package/dest/testbench/worker_client_manager.d.ts +1 -1
- package/dest/testbench/worker_client_manager.d.ts.map +1 -1
- package/dest/testbench/worker_client_manager.js +1 -2
- package/dest/util.d.ts +1 -1
- package/package.json +14 -14
- package/src/client/factory.ts +8 -4
- package/src/client/p2p_client.ts +4 -6
- package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +8 -7
- package/src/config.ts +10 -10
- package/src/mem_pools/attestation_pool/attestation_pool.ts +7 -8
- package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +6 -6
- package/src/mem_pools/instrumentation.ts +13 -17
- package/src/mem_pools/tx_pool_v2/interfaces.ts +4 -7
- package/src/mem_pools/tx_pool_v2/tx_metadata.ts +4 -4
- package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +43 -29
- package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +0 -3
- package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +1 -19
- package/src/msg_validators/attestation_validator/README.md +1 -1
- package/src/msg_validators/attestation_validator/attestation_validator.ts +4 -5
- package/src/msg_validators/clock_tolerance.ts +3 -4
- package/src/msg_validators/proposal_validator/README.md +3 -3
- package/src/msg_validators/proposal_validator/proposal_validator.ts +5 -6
- package/src/msg_validators/tx_validator/contract_instance_validator.ts +56 -0
- package/src/msg_validators/tx_validator/data_validator.ts +42 -1
- package/src/msg_validators/tx_validator/factory.ts +7 -0
- package/src/msg_validators/tx_validator/metadata_validator.ts +4 -12
- package/src/msg_validators/tx_validator/phases_validator.ts +1 -1
- package/src/services/libp2p/libp2p_service.ts +28 -18
- package/src/services/peer-manager/metrics.ts +0 -7
- package/src/services/peer-manager/peer_manager.ts +3 -7
- package/src/services/reqresp/batch-tx-requester/README.md +7 -46
- package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts +111 -75
- package/src/services/reqresp/batch-tx-requester/interface.ts +1 -2
- package/src/services/reqresp/batch-tx-requester/missing_txs.ts +6 -13
- package/src/services/reqresp/batch-tx-requester/peer_collection.ts +24 -68
- package/src/services/reqresp/rate-limiter/rate_limiter.ts +13 -9
- package/src/services/reqresp/reqresp.ts +3 -5
- package/src/services/tx_collection/fast_tx_collection.ts +83 -57
- package/src/services/tx_collection/file_store_tx_source.ts +43 -31
- package/src/services/tx_collection/missing_txs_tracker.ts +52 -0
- package/src/services/tx_collection/proposal_tx_collector.ts +13 -8
- package/src/services/tx_collection/slow_tx_collection.ts +1 -1
- package/src/services/tx_collection/tx_collection.ts +5 -3
- package/src/services/tx_collection/tx_source.ts +8 -7
- package/src/test-helpers/make-test-p2p-clients.ts +1 -1
- package/src/test-helpers/reqresp-nodes.ts +1 -1
- package/src/test-helpers/testbench-utils.ts +3 -28
- package/src/testbench/p2p_client_testbench_worker.ts +9 -7
- package/src/testbench/worker_client_manager.ts +1 -2
- package/src/util.ts +1 -1
- package/dest/services/tx_collection/request_tracker.d.ts +0 -53
- package/dest/services/tx_collection/request_tracker.d.ts.map +0 -1
- package/dest/services/tx_collection/request_tracker.js +0 -84
- package/src/services/tx_collection/request_tracker.ts +0 -127
|
@@ -1,19 +1,17 @@
|
|
|
1
|
-
import { peerIdFromString } from '@libp2p/peer-id';
|
|
2
1
|
import { DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD } from './config.js';
|
|
3
2
|
export const RATE_LIMIT_EXCEEDED_PEER_CACHE_TTL = 1000; // 1s
|
|
4
3
|
export class PeerCollection {
|
|
5
|
-
connectionSampler;
|
|
6
4
|
pinnedPeerId;
|
|
7
5
|
dateProvider;
|
|
8
6
|
badPeerThreshold;
|
|
9
7
|
peerPenalizer;
|
|
8
|
+
peers;
|
|
10
9
|
smartPeers;
|
|
11
10
|
inFlightPeers;
|
|
12
11
|
rateLimitExceededPeers;
|
|
13
12
|
peerPenaltyCounters;
|
|
14
13
|
badPeers;
|
|
15
|
-
constructor(
|
|
16
|
-
this.connectionSampler = connectionSampler;
|
|
14
|
+
constructor(initialPeers, pinnedPeerId, dateProvider, badPeerThreshold = DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD, peerPenalizer){
|
|
17
15
|
this.pinnedPeerId = pinnedPeerId;
|
|
18
16
|
this.dateProvider = dateProvider;
|
|
19
17
|
this.badPeerThreshold = badPeerThreshold;
|
|
@@ -23,50 +21,29 @@ export class PeerCollection {
|
|
|
23
21
|
this.rateLimitExceededPeers = new Map();
|
|
24
22
|
this.peerPenaltyCounters = new Map();
|
|
25
23
|
this.badPeers = new Set();
|
|
26
|
-
this.
|
|
27
|
-
|
|
28
|
-
this.orderedPeers = new Set();
|
|
29
|
-
// Pinned peer is treated specially, always mark it as in-flight
|
|
24
|
+
this.peers = new Set(initialPeers.map((peer)=>peer.toString()));
|
|
25
|
+
// Pinned peer is treaded specially, always mark it as in-flight
|
|
30
26
|
// and never return it as part of smart/dumb peers
|
|
31
27
|
if (this.pinnedPeerId) {
|
|
32
28
|
const peerIdStr = this.pinnedPeerId.toString();
|
|
33
29
|
this.inFlightPeers.add(peerIdStr);
|
|
30
|
+
this.peers.delete(peerIdStr);
|
|
34
31
|
}
|
|
35
32
|
}
|
|
36
|
-
|
|
37
|
-
this.
|
|
38
|
-
}
|
|
39
|
-
markPeerDumb(peerId) {
|
|
40
|
-
this.smartPeers.delete(peerId.toString());
|
|
41
|
-
}
|
|
42
|
-
// We keep track of all peers that are queried for peer sampling algorithm
|
|
43
|
-
queriedSmartPeers;
|
|
44
|
-
queriedDumbPeers;
|
|
45
|
-
static nextPeer(allPeers, queried) {
|
|
46
|
-
if (allPeers.size === 0) {
|
|
47
|
-
return undefined;
|
|
48
|
-
}
|
|
49
|
-
const availablePeers = allPeers.difference(queried);
|
|
50
|
-
let [first] = availablePeers;
|
|
51
|
-
if (first === undefined) {
|
|
52
|
-
// We queried all peers. Start over
|
|
53
|
-
[first] = allPeers;
|
|
54
|
-
queried.clear();
|
|
55
|
-
}
|
|
56
|
-
queried.add(first);
|
|
57
|
-
return peerIdFromString(first);
|
|
33
|
+
getAllPeers() {
|
|
34
|
+
return this.peers;
|
|
58
35
|
}
|
|
59
|
-
|
|
60
|
-
return
|
|
36
|
+
getSmartPeers() {
|
|
37
|
+
return this.smartPeers;
|
|
61
38
|
}
|
|
62
|
-
|
|
63
|
-
|
|
39
|
+
markPeerSmart(peerId) {
|
|
40
|
+
this.smartPeers.add(peerId.toString());
|
|
64
41
|
}
|
|
65
|
-
|
|
66
|
-
return
|
|
42
|
+
getSmartPeersToQuery() {
|
|
43
|
+
return Array.from(this.smartPeers.difference(this.getBadPeers().union(this.inFlightPeers).union(this.getRateLimitExceededPeers())));
|
|
67
44
|
}
|
|
68
|
-
|
|
69
|
-
return this.peers.difference(this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers()));
|
|
45
|
+
getDumbPeersToQuery() {
|
|
46
|
+
return Array.from(this.peers.difference(this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers())));
|
|
70
47
|
}
|
|
71
48
|
thereAreSomeDumbRatelimitExceededPeers() {
|
|
72
49
|
return this.getRateLimitExceededPeers().difference(this.smartPeers.union(this.badPeers).union(this.inFlightPeers)).size > 0;
|
|
@@ -159,18 +136,4 @@ export class PeerCollection {
|
|
|
159
136
|
}
|
|
160
137
|
return minExpiry - now;
|
|
161
138
|
}
|
|
162
|
-
orderedPeers;
|
|
163
|
-
get peers() {
|
|
164
|
-
const pinnedStr = this.pinnedPeerId?.toString();
|
|
165
|
-
const currentlyConnected = new Set(this.connectionSampler.getPeerListSortedByConnectionCountAsc().map((p)=>p.toString()).filter((p)=>p !== pinnedStr));
|
|
166
|
-
// Remove disconnected peers, preserving order of the rest.
|
|
167
|
-
this.orderedPeers = this.orderedPeers.intersection(currentlyConnected);
|
|
168
|
-
// Append newly connected peers at the end (lowest priority).
|
|
169
|
-
for (const peer of currentlyConnected){
|
|
170
|
-
if (!this.orderedPeers.has(peer)) {
|
|
171
|
-
this.orderedPeers.add(peer);
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
return this.orderedPeers;
|
|
175
|
-
}
|
|
176
139
|
}
|
|
@@ -46,9 +46,10 @@ export declare function prettyPrintRateLimitStatus(status: RateLimitStatus): "Al
|
|
|
46
46
|
* 2. Individual rate limits for each peer.
|
|
47
47
|
*
|
|
48
48
|
* How it works:
|
|
49
|
-
* - When a request comes in, it first checks against the
|
|
50
|
-
* - If the
|
|
51
|
-
* - The request is only allowed if both the
|
|
49
|
+
* - When a request comes in, it first checks against the peer's individual rate limit.
|
|
50
|
+
* - If the peer limit allows, it then checks against the global rate limit.
|
|
51
|
+
* - The request is only allowed if both the peer-specific and global limits allow it.
|
|
52
|
+
* - Checking peer limit first ensures a rate-limited peer cannot exhaust the global quota.
|
|
52
53
|
* - It automatically creates and manages rate limiters for new peers as they make requests.
|
|
53
54
|
* - It periodically cleans up rate limiters for inactive peers to conserve memory.
|
|
54
55
|
*
|
|
@@ -101,4 +102,4 @@ export declare class RequestResponseRateLimiter {
|
|
|
101
102
|
stop(): void;
|
|
102
103
|
getRateLimits(protocol: ReqRespSubProtocol): ProtocolRateLimitQuota;
|
|
103
104
|
}
|
|
104
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
105
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicmF0ZV9saW1pdGVyLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi9zcmMvc2VydmljZXMvcmVxcmVzcC9yYXRlLWxpbWl0ZXIvcmF0ZV9saW1pdGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQU9BLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBRWhELE9BQU8sS0FBSyxFQUFFLFdBQVcsRUFBRSxNQUFNLG9DQUFvQyxDQUFDO0FBQ3RFLE9BQU8sS0FBSyxFQUFFLHNCQUFzQixFQUFFLGtCQUFrQixFQUFFLDRCQUE0QixFQUFFLE1BQU0saUJBQWlCLENBQUM7QUFNaEg7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7R0FtQkc7QUFDSCxxQkFBYSxlQUFlO0lBRTFCLE9BQU8sQ0FBQyxHQUFHLENBQVM7SUFFcEIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxnQkFBZ0IsQ0FBUztJQUUxQyxPQUFPLENBQUMsUUFBUSxDQUFDLGFBQWEsQ0FBUztJQUV2Qzs7O09BR0c7SUFDSCxZQUFZLFVBQVUsRUFBRSxNQUFNLEVBQUUsV0FBVyxFQUFFLE1BQU0sRUFJbEQ7SUFFRCxLQUFLLElBQUksT0FBTyxDQVVmO0NBQ0Y7QUFTRCxvQkFBWSxlQUFlO0lBQ3pCLFlBQVksSUFBQTtJQUNaLFVBQVUsSUFBQTtJQUNWLE9BQU8sSUFBQTtDQUNSO0FBRUQsd0JBQWdCLDBCQUEwQixDQUFDLE1BQU0sRUFBRSxlQUFlLDZDQVNqRTtBQUVEOzs7Ozs7Ozs7Ozs7Ozs7O0dBZ0JHO0FBQ0gscUJBQWEsc0JBQXNCO0lBQ2pDLE9BQU8sQ0FBQyxZQUFZLENBQTJDO0lBQy9ELE9BQU8sQ0FBQyxhQUFhLENBQWtCO0lBQ3ZDLE9BQU8sQ0FBQyxRQUFRLENBQUMsY0FBYyxDQUFTO0lBQ3hDLE9BQU8sQ0FBQyxRQUFRLENBQUMsZUFBZSxDQUFTO0lBRXpDLFlBQVksY0FBYyxFQUFFLE1BQU0sRUFBRSxlQUFlLEVBQUUsTUFBTSxFQUFFLGdCQUFnQixFQUFFLE1BQU0sRUFBRSxpQkFBaUIsRUFBRSxNQUFNLEVBSy9HO0lBRUQsS0FBSyxDQUFDLE1BQU0sRUFBRSxNQUFNLEdBQUcsZUFBZSxDQXlCckM7SUFFRCxvQkFBb0IsU0FPbkI7Q0FDRjtBQUVEOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0dBc0JHO0FBQ0gscUJBQWEsMEJBQTBCO0lBT25DLE9BQU8sQ0FBQyxXQUFXO0lBTnJCLE9BQU8sQ0FBQyx1QkFBdUIsQ0FBa0Q7SUFDakYsT0FBTyxDQUFDLFVBQVUsQ0FBK0I7SUFFakQsT0FBTyxDQUFDLGVBQWUsQ0FBeUM7SUFFaEUsWUFDVSxXQUFXLEVBQUUsV0FBVyxFQUNoQyxVQUFVLEdBQUUsT0FBTyxDQUFDLDRCQUE0QixDQUFNLEVBZ0J2RDtJQUVELEtBQUssU0FJSjtJQUVELEtBQUssQ0FBQyxXQUFXLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxFQUFFLE1BQU0sR0FBRyxlQUFlLENBV3RFO0lBRUQsb0JBQW9CLFNBRW5CO0lBRUQ7O09BRUc7SUFDSCxJQUFJLFNBRUg7SUFFRCxhQUFhLENBQUMsUUFBUSxFQUFFLGtCQUFrQixHQUFHLHNCQUFzQixDQUVsRTtDQUNGIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rate_limiter.d.ts","sourceRoot":"","sources":["../../../../src/services/reqresp/rate-limiter/rate_limiter.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,KAAK,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,MAAM,iBAAiB,CAAC;AAMhH;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,eAAe;IAE1B,OAAO,CAAC,GAAG,CAAS;IAEpB,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAE1C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IAEvC;;;OAGG;IACH,YAAY,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAIlD;IAED,KAAK,IAAI,OAAO,CAUf;CACF;AASD,oBAAY,eAAe;IACzB,YAAY,IAAA;IACZ,UAAU,IAAA;IACV,OAAO,IAAA;CACR;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,eAAe,6CASjE;AAED
|
|
1
|
+
{"version":3,"file":"rate_limiter.d.ts","sourceRoot":"","sources":["../../../../src/services/reqresp/rate-limiter/rate_limiter.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,KAAK,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,MAAM,iBAAiB,CAAC;AAMhH;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,eAAe;IAE1B,OAAO,CAAC,GAAG,CAAS;IAEpB,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAE1C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IAEvC;;;OAGG;IACH,YAAY,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAIlD;IAED,KAAK,IAAI,OAAO,CAUf;CACF;AASD,oBAAY,eAAe;IACzB,YAAY,IAAA;IACZ,UAAU,IAAA;IACV,OAAO,IAAA;CACR;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,eAAe,6CASjE;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,YAAY,CAA2C;IAC/D,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IAEzC,YAAY,cAAc,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAK/G;IAED,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAyBrC;IAED,oBAAoB,SAOnB;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,0BAA0B;IAOnC,OAAO,CAAC,WAAW;IANrB,OAAO,CAAC,uBAAuB,CAAkD;IACjF,OAAO,CAAC,UAAU,CAA+B;IAEjD,OAAO,CAAC,eAAe,CAAyC;IAEhE,YACU,WAAW,EAAE,WAAW,EAChC,UAAU,GAAE,OAAO,CAAC,4BAA4B,CAAM,EAgBvD;IAED,KAAK,SAIJ;IAED,KAAK,CAAC,WAAW,EAAE,kBAAkB,EAAE,MAAM,EAAE,MAAM,GAAG,eAAe,CAWtE;IAED,oBAAoB,SAEnB;IAED;;OAEG;IACH,IAAI,SAEH;IAED,aAAa,CAAC,QAAQ,EAAE,kBAAkB,GAAG,sBAAsB,CAElE;CACF"}
|
|
@@ -74,9 +74,10 @@ export function prettyPrintRateLimitStatus(status) {
|
|
|
74
74
|
* 2. Individual rate limits for each peer.
|
|
75
75
|
*
|
|
76
76
|
* How it works:
|
|
77
|
-
* - When a request comes in, it first checks against the
|
|
78
|
-
* - If the
|
|
79
|
-
* - The request is only allowed if both the
|
|
77
|
+
* - When a request comes in, it first checks against the peer's individual rate limit.
|
|
78
|
+
* - If the peer limit allows, it then checks against the global rate limit.
|
|
79
|
+
* - The request is only allowed if both the peer-specific and global limits allow it.
|
|
80
|
+
* - Checking peer limit first ensures a rate-limited peer cannot exhaust the global quota.
|
|
80
81
|
* - It automatically creates and manages rate limiters for new peers as they make requests.
|
|
81
82
|
* - It periodically cleans up rate limiters for inactive peers to conserve memory.
|
|
82
83
|
*
|
|
@@ -93,9 +94,6 @@ export function prettyPrintRateLimitStatus(status) {
|
|
|
93
94
|
this.peerQuotaTimeMs = peerQuotaTimeMs;
|
|
94
95
|
}
|
|
95
96
|
allow(peerId) {
|
|
96
|
-
if (!this.globalLimiter.allow()) {
|
|
97
|
-
return 0;
|
|
98
|
-
}
|
|
99
97
|
const peerIdStr = peerId.toString();
|
|
100
98
|
let peerLimiter = this.peerLimiters.get(peerIdStr);
|
|
101
99
|
if (!peerLimiter) {
|
|
@@ -108,10 +106,14 @@ export function prettyPrintRateLimitStatus(status) {
|
|
|
108
106
|
} else {
|
|
109
107
|
peerLimiter.lastAccess = Date.now();
|
|
110
108
|
}
|
|
111
|
-
|
|
112
|
-
|
|
109
|
+
// Check peer limit first: a rate-limited peer must not consume global quota,
|
|
110
|
+
// otherwise one spamming peer can starve all others by exhausting the global bucket.
|
|
111
|
+
if (!peerLimiter.limiter.allow()) {
|
|
113
112
|
return 1;
|
|
114
113
|
}
|
|
114
|
+
if (!this.globalLimiter.allow()) {
|
|
115
|
+
return 0;
|
|
116
|
+
}
|
|
115
117
|
return 2;
|
|
116
118
|
}
|
|
117
119
|
cleanupInactivePeers() {
|
|
@@ -126,4 +126,4 @@ export declare class ReqResp implements ReqRespInterface {
|
|
|
126
126
|
private categorizeResponseError;
|
|
127
127
|
private categorizeConnectionErrors;
|
|
128
128
|
}
|
|
129
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
129
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicmVxcmVzcC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NlcnZpY2VzL3JlcXJlc3AvcmVxcmVzcC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFNQSxPQUFPLEVBQWMsS0FBSyxlQUFlLEVBQWlDLE1BQU0seUJBQXlCLENBQUM7QUFFMUcsT0FBTyxLQUFLLEVBQXNCLE1BQU0sRUFBVSxNQUFNLG1CQUFtQixDQUFDO0FBQzVFLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLFFBQVEsQ0FBQztBQVVyQyxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUNuRSxPQUFPLEVBR0wsS0FBSyxnQkFBZ0IsRUFDdEIsTUFBTSxhQUFhLENBQUM7QUFFckIsT0FBTyxFQUFFLGlCQUFpQixFQUFpQixNQUFNLDRDQUE0QyxDQUFDO0FBQzlGLE9BQU8sRUFFTCxLQUFLLGdCQUFnQixFQUNyQixLQUFLLGVBQWUsRUFDcEIsa0JBQWtCLEVBQ2xCLEtBQUsseUJBQXlCLEVBQzlCLEtBQUssMEJBQTBCLEVBQy9CLEtBQUssNEJBQTRCLEVBQ2pDLEtBQUssNEJBQTRCLEVBQ2pDLEtBQUssY0FBYyxFQUdwQixNQUFNLGdCQUFnQixDQUFDO0FBU3hCOzs7Ozs7Ozs7Ozs7O0dBYUc7QUFDSCxxQkFBYSxPQUFRLFlBQVcsZ0JBQWdCO0lBZ0I1QyxPQUFPLENBQUMsTUFBTTtJQUNkLE9BQU8sQ0FBQyxXQUFXO0lBQ25CLE9BQU8sQ0FBQyxNQUFNO0lBakJoQixPQUFPLENBQUMsMEJBQTBCLENBQWlEO0lBQ25GLE9BQU8sQ0FBQyxhQUFhLENBQTJDO0lBRWhFLE9BQU8sQ0FBQyxtQkFBbUIsQ0FBMkM7SUFDdEUsT0FBTyxDQUFDLHFCQUFxQixDQUE2QztJQUUxRSxPQUFPLENBQUMsaUJBQWlCLENBQW9CO0lBQzdDLE9BQU8sQ0FBQyxXQUFXLENBQTZCO0lBRWhELE9BQU8sQ0FBQyxlQUFlLENBQWtCO0lBRXpDLE9BQU8sQ0FBQyxPQUFPLENBQWlCO0lBRWhDLFlBQ0UsTUFBTSxFQUFFLGdCQUFnQixFQUNoQixNQUFNLEVBQUUsTUFBTSxFQUNkLFdBQVcsRUFBRSxXQUFXLEVBQ3hCLE1BQU0seUNBQThCLEVBQzVDLFVBQVUsR0FBRSxPQUFPLENBQUMsNEJBQTRCLENBQU0sRUFDdEQsZUFBZSxHQUFFLGVBQXNDLEVBZ0J4RDtJQUVNLFlBQVksQ0FBQyxNQUFNLEVBQUUsT0FBTyxDQUFDLGdCQUFnQixDQUFDLEdBQUcsSUFBSSxDQVEzRDtJQUVELElBQUksTUFBTSw2Q0FFVDtJQUVEOztPQUVHO0lBQ0gsb0JBQW9CLElBQUksSUFBSSxDQUFDLGlCQUFpQixFQUFFLHVDQUF1QyxDQUFDLENBRXZGO0lBRUQ7O09BRUc7SUFDRyxLQUFLLENBQUMsbUJBQW1CLEVBQUUsMEJBQTBCLEVBQUUscUJBQXFCLEVBQUUsNEJBQTRCLGlCQWdCL0c7SUFFSyxjQUFjLENBQ2xCLFdBQVcsRUFBRSxrQkFBa0IsRUFDL0IsT0FBTyxFQUFFLHlCQUF5QixFQUNsQyxTQUFTLEdBQUUsNEJBQTRCLENBQUMsa0JBQWtCLENBQWdELEdBQ3pHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FXZjtJQUVEOztPQUVHO0lBQ0csSUFBSSxrQkFrQlQ7SUFFRDs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O09BcUJHO0lBUUcsZ0JBQWdCLENBQUMsV0FBVyxTQUFTLGtCQUFrQixFQUMzRCxXQUFXLEVBQUUsV0FBVyxFQUN4QixRQUFRLEVBQUUsWUFBWSxDQUFDLGNBQWMsQ0FBQyxXQUFXLENBQUMsQ0FBQyxTQUFTLENBQUMsQ0FBQyxFQUFFLEVBQ2hFLFVBQVUsRUFBRSxNQUFNLEdBQUcsU0FBUyxFQUM5QixTQUFTLFNBQVEsRUFDakIsUUFBUSxTQUErQyxFQUN2RCxnQkFBZ0IsU0FBSSxHQUNuQixPQUFPLENBQUMsWUFBWSxDQUFDLGNBQWMsQ0FBQyxXQUFXLENBQUMsQ0FBQyxVQUFVLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FxTGxFO0lBRUQ7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OztPQXdCRztJQUtVLGlCQUFpQixDQUM1QixNQUFNLEVBQUUsTUFBTSxFQUNkLFdBQVcsRUFBRSxrQkFBa0IsRUFDL0IsT0FBTyxFQUFFLE1BQU0sRUFDZixXQUFXLEdBQUUsTUFBMkIsR0FDdkMsT0FBTyxDQUFDLGVBQWUsQ0FBQyxDQTJEMUI7SUFFRDs7Ozs7Ozs7O09BU0c7SUFDSCxPQUFPLENBQUMsbUJBQW1CO1lBaUJiLFdBQVc7WUE0RFgsYUFBYTtZQStEYixhQUFhO1lBd0NiLFlBQVk7SUFtQzFCLE9BQU8sQ0FBQyxrQkFBa0I7SUFPMUI7Ozs7T0FJRztJQUNILE9BQU8sQ0FBQyxzQkFBc0I7SUFpQjlCOzs7O09BSUc7SUFDSCxPQUFPLENBQUMsdUJBQXVCO0lBdUMvQixPQUFPLENBQUMsMEJBQTBCO0NBOERuQyJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reqresp.d.ts","sourceRoot":"","sources":["../../../src/services/reqresp/reqresp.ts"],"names":[],"mappings":"AAMA,OAAO,EAAc,KAAK,eAAe,EAAiC,MAAM,yBAAyB,CAAC;AAE1G,OAAO,KAAK,EAAsB,MAAM,EAAU,MAAM,mBAAmB,CAAC;AAC5E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUrC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,iBAAiB,EAAiB,MAAM,4CAA4C,CAAC;AAC9F,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,kBAAkB,EAClB,KAAK,yBAAyB,EAC9B,KAAK,0BAA0B,EAC/B,KAAK,4BAA4B,EACjC,KAAK,4BAA4B,EACjC,KAAK,cAAc,EAGpB,MAAM,gBAAgB,CAAC;AASxB;;;;;;;;;;;;;GAaG;AACH,qBAAa,OAAQ,YAAW,gBAAgB;IAgB5C,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;IAjBhB,OAAO,CAAC,0BAA0B,CAAiD;IACnF,OAAO,CAAC,aAAa,CAA2C;IAEhE,OAAO,CAAC,mBAAmB,CAA2C;IACtE,OAAO,CAAC,qBAAqB,CAA6C;IAE1E,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,WAAW,CAA6B;IAEhD,OAAO,CAAC,eAAe,CAAkB;IAEzC,OAAO,CAAC,OAAO,CAAiB;IAEhC,YACE,MAAM,EAAE,gBAAgB,EAChB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,WAAW,EACxB,MAAM,yCAA8B,EAC5C,UAAU,GAAE,OAAO,CAAC,4BAA4B,CAAM,EACtD,eAAe,GAAE,eAAsC,EAgBxD;IAEM,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAQ3D;IAED,IAAI,MAAM,6CAET;IAED;;OAEG;IACH,oBAAoB,IAAI,IAAI,CAAC,iBAAiB,EAAE,uCAAuC,CAAC,CAEvF;IAED;;OAEG;IACG,KAAK,CAAC,mBAAmB,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,4BAA4B,iBAgB/G;IAEK,cAAc,CAClB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,yBAAyB,EAClC,SAAS,GAAE,4BAA4B,CAAC,kBAAkB,CAAgD,GACzG,OAAO,CAAC,IAAI,CAAC,CAWf;IAED;;OAEG;IACG,IAAI,kBAkBT;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IAQG,gBAAgB,CAAC,WAAW,SAAS,kBAAkB,EAC3D,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAChE,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,SAAS,SAAQ,EACjB,QAAQ,SAA+C,EACvD,gBAAgB,SAAI,GACnB,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAqLlE;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IAKU,iBAAiB,CAC5B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,MAAM,EACf,WAAW,GAAE,MAA2B,GACvC,OAAO,CAAC,eAAe,CAAC,CA2D1B;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,mBAAmB;YAiBb,WAAW;YA4DX,aAAa;
|
|
1
|
+
{"version":3,"file":"reqresp.d.ts","sourceRoot":"","sources":["../../../src/services/reqresp/reqresp.ts"],"names":[],"mappings":"AAMA,OAAO,EAAc,KAAK,eAAe,EAAiC,MAAM,yBAAyB,CAAC;AAE1G,OAAO,KAAK,EAAsB,MAAM,EAAU,MAAM,mBAAmB,CAAC;AAC5E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUrC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,iBAAiB,EAAiB,MAAM,4CAA4C,CAAC;AAC9F,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,kBAAkB,EAClB,KAAK,yBAAyB,EAC9B,KAAK,0BAA0B,EAC/B,KAAK,4BAA4B,EACjC,KAAK,4BAA4B,EACjC,KAAK,cAAc,EAGpB,MAAM,gBAAgB,CAAC;AASxB;;;;;;;;;;;;;GAaG;AACH,qBAAa,OAAQ,YAAW,gBAAgB;IAgB5C,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;IAjBhB,OAAO,CAAC,0BAA0B,CAAiD;IACnF,OAAO,CAAC,aAAa,CAA2C;IAEhE,OAAO,CAAC,mBAAmB,CAA2C;IACtE,OAAO,CAAC,qBAAqB,CAA6C;IAE1E,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,WAAW,CAA6B;IAEhD,OAAO,CAAC,eAAe,CAAkB;IAEzC,OAAO,CAAC,OAAO,CAAiB;IAEhC,YACE,MAAM,EAAE,gBAAgB,EAChB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,WAAW,EACxB,MAAM,yCAA8B,EAC5C,UAAU,GAAE,OAAO,CAAC,4BAA4B,CAAM,EACtD,eAAe,GAAE,eAAsC,EAgBxD;IAEM,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAQ3D;IAED,IAAI,MAAM,6CAET;IAED;;OAEG;IACH,oBAAoB,IAAI,IAAI,CAAC,iBAAiB,EAAE,uCAAuC,CAAC,CAEvF;IAED;;OAEG;IACG,KAAK,CAAC,mBAAmB,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,4BAA4B,iBAgB/G;IAEK,cAAc,CAClB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,yBAAyB,EAClC,SAAS,GAAE,4BAA4B,CAAC,kBAAkB,CAAgD,GACzG,OAAO,CAAC,IAAI,CAAC,CAWf;IAED;;OAEG;IACG,IAAI,kBAkBT;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IAQG,gBAAgB,CAAC,WAAW,SAAS,kBAAkB,EAC3D,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAChE,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,SAAS,SAAQ,EACjB,QAAQ,SAA+C,EACvD,gBAAgB,SAAI,GACnB,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAqLlE;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IAKU,iBAAiB,CAC5B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,MAAM,EACf,WAAW,GAAE,MAA2B,GACvC,OAAO,CAAC,eAAe,CAAC,CA2D1B;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,mBAAmB;YAiBb,WAAW;YA4DX,aAAa;YA+Db,aAAa;YAwCb,YAAY;IAmC1B,OAAO,CAAC,kBAAkB;IAO1B;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAiB9B;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;IAuC/B,OAAO,CAAC,0BAA0B;CA8DnC"}
|
|
@@ -611,7 +611,7 @@ _dec = trackSpan('ReqResp.sendBatchRequest', (subProtocol, requests)=>({
|
|
|
611
611
|
return status === ReqRespStatus.RATE_LIMIT_EXCEEDED;
|
|
612
612
|
};
|
|
613
613
|
for (const index of indices){
|
|
614
|
-
this.logger.
|
|
614
|
+
this.logger.info(`Sending request ${index} to peer ${peerAsString}`);
|
|
615
615
|
const response = await this.sendRequestToPeer(peer, subProtocol, requestBuffers[index]);
|
|
616
616
|
// Check the status of the response buffer
|
|
617
617
|
if (response.status !== ReqRespStatus.SUCCESS) {
|
|
@@ -729,7 +729,7 @@ _dec = trackSpan('ReqResp.sendBatchRequest', (subProtocol, requests)=>({
|
|
|
729
729
|
]), this.individualRequestTimeoutMs, ()=>timeoutErr);
|
|
730
730
|
return resp;
|
|
731
731
|
} catch (e) {
|
|
732
|
-
this.logger.
|
|
732
|
+
this.logger.debug(`SUBPROTOCOL: ${subProtocol}\n`, e);
|
|
733
733
|
// On error we immediately abort the stream, this is preferred way,
|
|
734
734
|
// because it signals to the sender that error happened, whereas
|
|
735
735
|
// closing the stream only closes our side and is much slower
|
|
@@ -858,8 +858,7 @@ _dec = trackSpan('ReqResp.sendBatchRequest', (subProtocol, requests)=>({
|
|
|
858
858
|
// and that this stream should be dropped
|
|
859
859
|
const isMessageToNotWarn = err instanceof Error && [
|
|
860
860
|
'stream reset',
|
|
861
|
-
'Cannot push value onto an ended pushable'
|
|
862
|
-
'read ECONNRESET'
|
|
861
|
+
'Cannot push value onto an ended pushable'
|
|
863
862
|
].some((msg)=>err.message.includes(msg));
|
|
864
863
|
const level = isMessageToNotWarn ? 'debug' : 'warn';
|
|
865
864
|
this.logger[level]('Unknown stream error while handling the stream, aborting', {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import { type Logger } from '@aztec/foundation/log';
|
|
3
3
|
import { DateProvider } from '@aztec/foundation/timer';
|
|
4
|
+
import type { BlockProposal } from '@aztec/stdlib/p2p';
|
|
4
5
|
import { type Tx, TxHash } from '@aztec/stdlib/tx';
|
|
5
6
|
import type { PeerId } from '@libp2p/interface';
|
|
6
7
|
import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
|
|
@@ -25,6 +26,8 @@ export declare class FastTxCollection {
|
|
|
25
26
|
pinnedPeer?: PeerId;
|
|
26
27
|
}): Promise<Tx[]>;
|
|
27
28
|
protected collectFast(request: FastCollectionRequest, opts: {
|
|
29
|
+
proposal?: BlockProposal;
|
|
30
|
+
deadline: Date;
|
|
28
31
|
pinnedPeer?: PeerId;
|
|
29
32
|
}): Promise<void>;
|
|
30
33
|
private collectFastFromNodes;
|
|
@@ -48,4 +51,4 @@ export declare class FastTxCollection {
|
|
|
48
51
|
*/
|
|
49
52
|
stopCollectingForBlocksAfter(blockNumber: BlockNumber): void;
|
|
50
53
|
}
|
|
51
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
54
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFzdF90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9mYXN0X3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRzlELE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQUUsWUFBWSxFQUFXLE1BQU0seUJBQXlCLENBQUM7QUFFaEUsT0FBTyxLQUFLLEVBQUUsYUFBYSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDdkQsT0FBTyxFQUFFLEtBQUssRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRW5ELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBR2hELE9BQU8sS0FBSyxFQUFFLDZCQUE2QixFQUFFLE1BQU0sNENBQTRDLENBQUM7QUFDaEcsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFFdEQsT0FBTyxFQUVMLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0sNEJBQTRCLENBQUM7QUFDcEMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUM1RixPQUFPLEtBQUssRUFBZ0IsZ0JBQWdCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUM5RSxPQUFPLEtBQUssRUFBRSxRQUFRLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUUvQyxxQkFBYSxnQkFBZ0I7SUFPekIsT0FBTyxDQUFDLEtBQUs7SUFDYixPQUFPLENBQUMsZ0JBQWdCO0lBQ3hCLE9BQU8sQ0FBQyxNQUFNO0lBQ2QsT0FBTyxDQUFDLFlBQVk7SUFDcEIsT0FBTyxDQUFDLEdBQUc7SUFUYixTQUFTLENBQUMsUUFBUSxFQUFFLEdBQUcsQ0FBQyxxQkFBcUIsQ0FBQyxDQUFhO0lBQzNELE9BQU8sQ0FBQyxtQkFBbUIsQ0FBc0I7SUFFakQsWUFDRSxVQUFVLEVBQUUsNkJBQTZCLEVBQ2pDLEtBQUssRUFBRSxRQUFRLEVBQUUsRUFDakIsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLE1BQU0sRUFBRSxrQkFBa0IsRUFDMUIsWUFBWSxHQUFFLFlBQWlDLEVBQy9DLEdBQUcsR0FBRSxNQUFrRCxFQUMvRCxtQkFBbUIsQ0FBQyxFQUFFLG1CQUFtQixFQVMxQztJQUVZLElBQUksa0JBR2hCO0lBRU0seUJBQXlCLCtCQUUvQjtJQUVZLGNBQWMsQ0FDekIsS0FBSyxFQUFFLDBCQUEwQixFQUNqQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsTUFBTSxFQUFFLEVBQzdCLElBQUksRUFBRTtRQUFFLFFBQVEsRUFBRSxJQUFJLENBQUM7UUFBQyxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUE7S0FBRSxpQkEwQzlDO0lBRUQsVUFBZ0IsV0FBVyxDQUN6QixPQUFPLEVBQUUscUJBQXFCLEVBQzlCLElBQUksRUFBRTtRQUFFLFFBQVEsQ0FBQyxFQUFFLGFBQWEsQ0FBQztRQUFDLFFBQVEsRUFBRSxJQUFJLENBQUM7UUFBQyxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUE7S0FBRSxpQkErQ3hFO1lBUWEsb0JBQW9CO1lBZ0JwQixtQkFBbUI7WUF5Rm5CLHFCQUFxQjtJQXlEbkMseUdBQXlHO0lBQ3pHLE9BQU8sQ0FBQyxhQUFhO0lBUXJCOzs7T0FHRztJQUNJLFFBQVEsQ0FBQyxHQUFHLEVBQUUsRUFBRSxFQUFFLFFBc0J4QjtJQUVEOzs7T0FHRztJQUNJLDJCQUEyQixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9qRTtJQUVEOzs7T0FHRztJQUNJLDRCQUE0QixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9sRTtDQUNGIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fast_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/fast_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"fast_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/fast_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAW,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAChG,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAEL,KAAK,mBAAmB,EAEzB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,KAAK,EAAgB,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAOzB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IATb,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAa;IAC3D,OAAO,CAAC,mBAAmB,CAAsB;IAEjD,YACE,UAAU,EAAE,6BAA6B,EACjC,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD,EAC/D,mBAAmB,CAAC,EAAE,mBAAmB,EAS1C;IAEY,IAAI,kBAGhB;IAEM,yBAAyB,+BAE/B;IAEY,cAAc,CACzB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA0C9C;IAED,UAAgB,WAAW,CACzB,OAAO,EAAE,qBAAqB,EAC9B,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA+CxE;YAQa,oBAAoB;YAgBpB,mBAAmB;YAyFnB,qBAAqB;IAyDnC,yGAAyG;IACzG,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACI,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,QAsBxB;IAED;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOlE;CACF"}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { times } from '@aztec/foundation/collection';
|
|
2
|
+
import { AbortError, TimeoutError } from '@aztec/foundation/error';
|
|
2
3
|
import { createLogger } from '@aztec/foundation/log';
|
|
4
|
+
import { promiseWithResolvers } from '@aztec/foundation/promise';
|
|
3
5
|
import { sleep } from '@aztec/foundation/sleep';
|
|
4
6
|
import { DateProvider, elapsed } from '@aztec/foundation/timer';
|
|
5
7
|
import { TxHash } from '@aztec/stdlib/tx';
|
|
8
|
+
import { MissingTxsTracker } from './missing_txs_tracker.js';
|
|
6
9
|
import { BatchTxRequesterCollector, SendBatchRequestCollector } from './proposal_tx_collector.js';
|
|
7
|
-
import { RequestTracker } from './request_tracker.js';
|
|
8
10
|
export class FastTxCollection {
|
|
9
11
|
nodes;
|
|
10
12
|
txCollectionSink;
|
|
@@ -26,9 +28,7 @@ export class FastTxCollection {
|
|
|
26
28
|
this.missingTxsCollector = missingTxsCollector ?? (missingTxsCollectorType === 'old' ? new SendBatchRequestCollector(p2pService) : new BatchTxRequesterCollector(p2pService, log, dateProvider, undefined, batchTxRequesterConfig));
|
|
27
29
|
}
|
|
28
30
|
async stop() {
|
|
29
|
-
this.requests.forEach((request)=>
|
|
30
|
-
request.requestTracker.cancel();
|
|
31
|
-
});
|
|
31
|
+
this.requests.forEach((request)=>request.promise.reject(new AbortError(`Stopped collection service`)));
|
|
32
32
|
await Promise.resolve();
|
|
33
33
|
}
|
|
34
34
|
getFastCollectionRequests() {
|
|
@@ -49,66 +49,81 @@ export class FastTxCollection {
|
|
|
49
49
|
} : {
|
|
50
50
|
...input.block.toBlockInfo()
|
|
51
51
|
};
|
|
52
|
+
// This promise is used to await for the collection to finish during the main collectFast method.
|
|
53
|
+
// It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
|
|
54
|
+
const promise = promiseWithResolvers();
|
|
55
|
+
const timeoutTimer = setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
|
|
52
56
|
const request = {
|
|
53
57
|
...input,
|
|
54
58
|
blockInfo,
|
|
55
|
-
|
|
59
|
+
promise,
|
|
60
|
+
missingTxTracker: MissingTxsTracker.fromArray(txHashes),
|
|
61
|
+
deadline: opts.deadline
|
|
56
62
|
};
|
|
57
63
|
const [duration] = await elapsed(()=>this.collectFast(request, {
|
|
58
64
|
...opts
|
|
59
65
|
}));
|
|
60
|
-
|
|
66
|
+
clearTimeout(timeoutTimer);
|
|
67
|
+
this.log.verbose(`Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
|
|
61
68
|
...blockInfo,
|
|
62
69
|
duration,
|
|
63
70
|
requestType: input.type,
|
|
64
71
|
missingTxs: [
|
|
65
|
-
...request.
|
|
72
|
+
...request.missingTxTracker.missingTxHashes
|
|
66
73
|
]
|
|
67
74
|
});
|
|
68
|
-
return request.
|
|
75
|
+
return request.missingTxTracker.collectedTxs;
|
|
69
76
|
}
|
|
70
77
|
async collectFast(request, opts) {
|
|
71
78
|
this.requests.add(request);
|
|
72
79
|
const { blockInfo } = request;
|
|
73
|
-
this.log.debug(`Starting fast collection of ${request.
|
|
80
|
+
this.log.debug(`Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
|
|
74
81
|
...blockInfo,
|
|
75
82
|
requestType: request.type,
|
|
76
|
-
deadline:
|
|
83
|
+
deadline: opts.deadline
|
|
77
84
|
});
|
|
78
85
|
try {
|
|
79
86
|
// Start blasting all nodes for the txs. We give them a little time to respond before we start reqresp.
|
|
80
|
-
//
|
|
81
|
-
//
|
|
82
|
-
const nodeCollectionPromise = this.collectFastFromNodes(request);
|
|
87
|
+
// And keep an eye on the request promise to ensure we don't wait longer than the deadline or return as soon
|
|
88
|
+
// as we have collected all txs, whatever the source.
|
|
89
|
+
const nodeCollectionPromise = this.collectFastFromNodes(request, opts);
|
|
83
90
|
const waitBeforeReqResp = sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs);
|
|
84
91
|
await Promise.race([
|
|
85
|
-
request.
|
|
92
|
+
request.promise.promise,
|
|
86
93
|
waitBeforeReqResp
|
|
87
94
|
]);
|
|
88
|
-
// If we have collected all txs
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
if (request.requestTracker.allFetched()) {
|
|
92
|
-
this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
|
|
93
|
-
}
|
|
94
|
-
await nodeCollectionPromise;
|
|
95
|
+
// If we have collected all txs, we can stop here
|
|
96
|
+
if (request.missingTxTracker.allFetched()) {
|
|
97
|
+
this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
|
|
95
98
|
return;
|
|
96
99
|
}
|
|
97
100
|
// Start blasting reqresp for the remaining txs. Note that node collection keeps running in parallel.
|
|
98
101
|
// We stop when we have collected all txs, timed out, or both node collection and reqresp have given up.
|
|
99
|
-
|
|
100
|
-
await Promise.allSettled([
|
|
102
|
+
const collectionPromise = Promise.allSettled([
|
|
101
103
|
this.collectFastViaReqResp(request, opts),
|
|
102
104
|
nodeCollectionPromise
|
|
103
105
|
]);
|
|
106
|
+
await Promise.race([
|
|
107
|
+
collectionPromise,
|
|
108
|
+
request.promise.promise
|
|
109
|
+
]);
|
|
104
110
|
} catch (err) {
|
|
105
|
-
|
|
111
|
+
// Log and swallow all errors
|
|
112
|
+
const logCtx = {
|
|
106
113
|
...blockInfo,
|
|
107
|
-
|
|
108
|
-
|
|
114
|
+
errorMessage: err instanceof Error ? err.message : undefined,
|
|
115
|
+
missingTxs: request.missingTxTracker.missingTxHashes.values().map((txHash)=>txHash.toString())
|
|
116
|
+
};
|
|
117
|
+
if (err instanceof Error && err.name === 'TimeoutError') {
|
|
118
|
+
this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
|
|
119
|
+
} else if (err instanceof Error && err.name === 'AbortError') {
|
|
120
|
+
this.log.warn(`Aborted collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
|
|
121
|
+
} else {
|
|
122
|
+
this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, logCtx);
|
|
123
|
+
}
|
|
109
124
|
} finally{
|
|
110
125
|
// Ensure no unresolved promises and remove the request from the set
|
|
111
|
-
request.
|
|
126
|
+
request.promise.resolve();
|
|
112
127
|
this.requests.delete(request);
|
|
113
128
|
}
|
|
114
129
|
}
|
|
@@ -117,23 +132,23 @@ export class FastTxCollection {
|
|
|
117
132
|
* in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
|
|
118
133
|
* the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
|
|
119
134
|
* tx is requested more than once at the same time to the same node.
|
|
120
|
-
*/ async collectFastFromNodes(request) {
|
|
135
|
+
*/ async collectFastFromNodes(request, opts) {
|
|
121
136
|
if (this.nodes.length === 0) {
|
|
122
137
|
return;
|
|
123
138
|
}
|
|
124
139
|
// Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
|
|
125
140
|
const attemptsPerTx = [
|
|
126
|
-
...request.
|
|
141
|
+
...request.missingTxTracker.missingTxHashes
|
|
127
142
|
].map((txHash)=>({
|
|
128
143
|
txHash,
|
|
129
144
|
attempts: 0,
|
|
130
145
|
found: false
|
|
131
146
|
}));
|
|
132
147
|
// Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
|
|
133
|
-
await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx)));
|
|
148
|
+
await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx, opts)));
|
|
134
149
|
}
|
|
135
|
-
async collectFastFromNode(request, node, attemptsPerTx) {
|
|
136
|
-
const notFinished = ()
|
|
150
|
+
async collectFastFromNode(request, node, attemptsPerTx, opts) {
|
|
151
|
+
const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
|
|
137
152
|
const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
|
|
138
153
|
const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
|
|
139
154
|
const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
|
|
@@ -147,7 +162,7 @@ export class FastTxCollection {
|
|
|
147
162
|
const txToRequest = attemptsPerTx[index++];
|
|
148
163
|
if (!txToRequest) {
|
|
149
164
|
break;
|
|
150
|
-
} else if (!request.
|
|
165
|
+
} else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
|
|
151
166
|
// Mark as found if it was found somewhere else, we'll then remove it from the array.
|
|
152
167
|
// We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
|
|
153
168
|
txToRequest.found = true;
|
|
@@ -176,7 +191,7 @@ export class FastTxCollection {
|
|
|
176
191
|
await this.txCollectionSink.collect(async ()=>{
|
|
177
192
|
const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
|
|
178
193
|
for (const tx of result.validTxs){
|
|
179
|
-
request.
|
|
194
|
+
request.missingTxTracker.markFetched(tx);
|
|
180
195
|
}
|
|
181
196
|
return result;
|
|
182
197
|
}, txHashes, {
|
|
@@ -189,12 +204,9 @@ export class FastTxCollection {
|
|
|
189
204
|
for (const requestedTx of batch){
|
|
190
205
|
activeRequestsToThisNode.delete(requestedTx.txHash);
|
|
191
206
|
}
|
|
192
|
-
// Sleep a bit until hitting the node again
|
|
207
|
+
// Sleep a bit until hitting the node again (or not, depending on config)
|
|
193
208
|
if (notFinished()) {
|
|
194
|
-
await
|
|
195
|
-
sleep(this.config.txCollectionFastNodeIntervalMs),
|
|
196
|
-
request.requestTracker.cancellationToken
|
|
197
|
-
]);
|
|
209
|
+
await sleep(this.config.txCollectionFastNodeIntervalMs);
|
|
198
210
|
}
|
|
199
211
|
}
|
|
200
212
|
};
|
|
@@ -202,32 +214,33 @@ export class FastTxCollection {
|
|
|
202
214
|
await Promise.all(times(maxParallelRequests, processBatch));
|
|
203
215
|
}
|
|
204
216
|
async collectFastViaReqResp(request, opts) {
|
|
217
|
+
const timeoutMs = +request.deadline - this.dateProvider.now();
|
|
205
218
|
const pinnedPeer = opts.pinnedPeer;
|
|
206
219
|
const blockInfo = request.blockInfo;
|
|
207
220
|
const slotNumber = blockInfo.slotNumber;
|
|
208
|
-
if (
|
|
221
|
+
if (timeoutMs < 100) {
|
|
209
222
|
this.log.warn(`Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`, {
|
|
210
|
-
timeoutMs
|
|
223
|
+
timeoutMs,
|
|
211
224
|
...blockInfo
|
|
212
225
|
});
|
|
213
226
|
return;
|
|
214
227
|
}
|
|
215
|
-
this.log.debug(`Starting fast reqresp for ${request.
|
|
228
|
+
this.log.debug(`Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
|
|
216
229
|
...blockInfo,
|
|
217
|
-
timeoutMs
|
|
230
|
+
timeoutMs,
|
|
218
231
|
pinnedPeer
|
|
219
232
|
});
|
|
220
233
|
try {
|
|
221
234
|
await this.txCollectionSink.collect(async ()=>{
|
|
222
235
|
let result;
|
|
223
236
|
if (request.type === 'proposal') {
|
|
224
|
-
result = await this.missingTxsCollector.collectTxs(request.
|
|
237
|
+
result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, request.blockProposal, pinnedPeer, timeoutMs);
|
|
225
238
|
} else if (request.type === 'block') {
|
|
226
239
|
const blockTxsSource = {
|
|
227
240
|
txHashes: request.block.body.txEffects.map((e)=>e.txHash),
|
|
228
241
|
archive: request.block.archive.root
|
|
229
242
|
};
|
|
230
|
-
result = await this.missingTxsCollector.collectTxs(request.
|
|
243
|
+
result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, blockTxsSource, pinnedPeer, timeoutMs);
|
|
231
244
|
} else {
|
|
232
245
|
throw new Error(`Unknown request type: ${request.type}`);
|
|
233
246
|
}
|
|
@@ -235,7 +248,7 @@ export class FastTxCollection {
|
|
|
235
248
|
validTxs: result,
|
|
236
249
|
invalidTxHashes: []
|
|
237
250
|
};
|
|
238
|
-
}, Array.from(request.
|
|
251
|
+
}, Array.from(request.missingTxTracker.missingTxHashes), {
|
|
239
252
|
description: `reqresp for slot ${slotNumber}`,
|
|
240
253
|
method: 'fast-req-resp',
|
|
241
254
|
...opts,
|
|
@@ -244,7 +257,7 @@ export class FastTxCollection {
|
|
|
244
257
|
} catch (err) {
|
|
245
258
|
this.log.error(`Error sending fast reqresp request for txs`, err, {
|
|
246
259
|
txs: [
|
|
247
|
-
...request.
|
|
260
|
+
...request.missingTxTracker.missingTxHashes
|
|
248
261
|
],
|
|
249
262
|
...blockInfo
|
|
250
263
|
});
|
|
@@ -271,19 +284,20 @@ export class FastTxCollection {
|
|
|
271
284
|
for (const tx of txs){
|
|
272
285
|
const txHash = tx.txHash.toString();
|
|
273
286
|
// Remove the tx hash from the missing set, and add it to the found set.
|
|
274
|
-
if (request.
|
|
287
|
+
if (request.missingTxTracker.markFetched(tx)) {
|
|
275
288
|
this.log.trace(`Found tx ${txHash} for fast collection request`, {
|
|
276
289
|
...request.blockInfo,
|
|
277
290
|
txHash: tx.txHash.toString(),
|
|
278
291
|
type: request.type
|
|
279
292
|
});
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
}
|
|
293
|
+
}
|
|
294
|
+
// If we found all txs for this request, we resolve the promise
|
|
295
|
+
if (request.missingTxTracker.allFetched()) {
|
|
296
|
+
this.log.trace(`All txs found for fast collection request`, {
|
|
297
|
+
...request.blockInfo,
|
|
298
|
+
type: request.type
|
|
299
|
+
});
|
|
300
|
+
request.promise.resolve();
|
|
287
301
|
}
|
|
288
302
|
}
|
|
289
303
|
}
|
|
@@ -294,7 +308,8 @@ export class FastTxCollection {
|
|
|
294
308
|
*/ stopCollectingForBlocksUpTo(blockNumber) {
|
|
295
309
|
for (const request of this.requests){
|
|
296
310
|
if (request.blockInfo.blockNumber <= blockNumber) {
|
|
297
|
-
request.
|
|
311
|
+
request.promise.reject(new AbortError(`Stopped collecting txs up to block ${blockNumber}`));
|
|
312
|
+
this.requests.delete(request);
|
|
298
313
|
}
|
|
299
314
|
}
|
|
300
315
|
}
|
|
@@ -304,7 +319,8 @@ export class FastTxCollection {
|
|
|
304
319
|
*/ stopCollectingForBlocksAfter(blockNumber) {
|
|
305
320
|
for (const request of this.requests){
|
|
306
321
|
if (request.blockInfo.blockNumber > blockNumber) {
|
|
307
|
-
request.
|
|
322
|
+
request.promise.reject(new AbortError(`Stopped collecting txs after block ${blockNumber}`));
|
|
323
|
+
this.requests.delete(request);
|
|
308
324
|
}
|
|
309
325
|
}
|
|
310
326
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type Logger } from '@aztec/foundation/log';
|
|
2
|
-
import { type TxHash } from '@aztec/stdlib/tx';
|
|
2
|
+
import { type TxHash, type TxValidator } from '@aztec/stdlib/tx';
|
|
3
3
|
import { type TelemetryClient } from '@aztec/telemetry-client';
|
|
4
4
|
import type { TxSource, TxSourceCollectionResult } from './tx_source.js';
|
|
5
5
|
/** TxSource implementation that downloads txs from a file store. */
|
|
@@ -7,6 +7,7 @@ export declare class FileStoreTxSource implements TxSource {
|
|
|
7
7
|
private readonly fileStore;
|
|
8
8
|
private readonly baseUrl;
|
|
9
9
|
private readonly basePath;
|
|
10
|
+
private readonly txValidator;
|
|
10
11
|
private readonly log;
|
|
11
12
|
private downloadsSuccess;
|
|
12
13
|
private downloadsFailed;
|
|
@@ -21,7 +22,7 @@ export declare class FileStoreTxSource implements TxSource {
|
|
|
21
22
|
* @param telemetry - Optional telemetry client.
|
|
22
23
|
* @returns The FileStoreTxSource instance, or undefined if creation fails.
|
|
23
24
|
*/
|
|
24
|
-
static create(url: string, basePath: string, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource | undefined>;
|
|
25
|
+
static create(url: string, basePath: string, txValidator: TxValidator, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource | undefined>;
|
|
25
26
|
getInfo(): string;
|
|
26
27
|
getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
|
|
27
28
|
}
|
|
@@ -33,5 +34,5 @@ export declare class FileStoreTxSource implements TxSource {
|
|
|
33
34
|
* @param telemetry - Optional telemetry client.
|
|
34
35
|
* @returns Array of successfully created FileStoreTxSource instances.
|
|
35
36
|
*/
|
|
36
|
-
export declare function createFileStoreTxSources(urls: string[], basePath: string, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource[]>;
|
|
37
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
37
|
+
export declare function createFileStoreTxSources(urls: string[], basePath: string, txValidator: TxValidator, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource[]>;
|
|
38
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZV9zdG9yZV90eF9zb3VyY2UuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zZXJ2aWNlcy90eF9jb2xsZWN0aW9uL2ZpbGVfc3RvcmVfdHhfc291cmNlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQU0sS0FBSyxNQUFNLEVBQUUsS0FBSyxXQUFXLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUNyRSxPQUFPLEVBR0wsS0FBSyxlQUFlLEVBR3JCLE1BQU0seUJBQXlCLENBQUM7QUFFakMsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLHdCQUF3QixFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFekUsb0VBQW9FO0FBQ3BFLHFCQUFhLGlCQUFrQixZQUFXLFFBQVE7SUFPOUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTO0lBQzFCLE9BQU8sQ0FBQyxRQUFRLENBQUMsT0FBTztJQUN4QixPQUFPLENBQUMsUUFBUSxDQUFDLFFBQVE7SUFDekIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxXQUFXO0lBQzVCLE9BQU8sQ0FBQyxRQUFRLENBQUMsR0FBRztJQVZ0QixPQUFPLENBQUMsZ0JBQWdCLENBQWdCO0lBQ3hDLE9BQU8sQ0FBQyxlQUFlLENBQWdCO0lBQ3ZDLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBWTtJQUNwQyxPQUFPLENBQUMsWUFBWSxDQUFZO0lBRWhDLE9BQU8sZUFhTjtJQUVEOzs7Ozs7O09BT0c7SUFDSCxPQUFvQixNQUFNLENBQ3hCLEdBQUcsRUFBRSxNQUFNLEVBQ1gsUUFBUSxFQUFFLE1BQU0sRUFDaEIsV0FBVyxFQUFFLFdBQVcsRUFDeEIsR0FBRyxHQUFFLE1BQWlELEVBQ3RELFNBQVMsR0FBRSxlQUFzQyxHQUNoRCxPQUFPLENBQUMsaUJBQWlCLEdBQUcsU0FBUyxDQUFDLENBWXhDO0lBRU0sT0FBTyxJQUFJLE1BQU0sQ0FFdkI7SUFFWSxZQUFZLENBQUMsUUFBUSxFQUFFLE1BQU0sRUFBRSxHQUFHLE9BQU8sQ0FBQyx3QkFBd0IsQ0FBQyxDQXFDL0U7Q0FDRjtBQUVEOzs7Ozs7O0dBT0c7QUFDSCx3QkFBc0Isd0JBQXdCLENBQzVDLElBQUksRUFBRSxNQUFNLEVBQUUsRUFDZCxRQUFRLEVBQUUsTUFBTSxFQUNoQixXQUFXLEVBQUUsV0FBVyxFQUN4QixHQUFHLEdBQUUsTUFBaUQsRUFDdEQsU0FBUyxHQUFFLGVBQXNDLEdBQ2hELE9BQU8sQ0FBQyxpQkFBaUIsRUFBRSxDQUFDLENBSzlCIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file_store_tx_source.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_source.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"file_store_tx_source.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_source.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAM,KAAK,MAAM,EAAE,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAGL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,QAAQ,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAEzE,oEAAoE;AACpE,qBAAa,iBAAkB,YAAW,QAAQ;IAO9C,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,GAAG;IAVtB,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,gBAAgB,CAAY;IACpC,OAAO,CAAC,YAAY,CAAY;IAEhC,OAAO,eAaN;IAED;;;;;;;OAOG;IACH,OAAoB,MAAM,CACxB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,EACxB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAYxC;IAEM,OAAO,IAAI,MAAM,CAEvB;IAEY,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqC/E;CACF;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,WAAW,EACxB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAK9B"}
|