@aztec/p2p 0.0.1-commit.88e6f9396 → 0.0.1-commit.8c0b8ff

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/dest/client/factory.d.ts +2 -2
  2. package/dest/client/factory.d.ts.map +1 -1
  3. package/dest/client/factory.js +5 -5
  4. package/dest/client/p2p_client.d.ts +1 -1
  5. package/dest/client/p2p_client.d.ts.map +1 -1
  6. package/dest/client/p2p_client.js +4 -6
  7. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +5 -6
  8. package/dest/config.d.ts +6 -6
  9. package/dest/config.d.ts.map +1 -1
  10. package/dest/config.js +6 -6
  11. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts +4 -4
  12. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts.map +1 -1
  13. package/dest/mem_pools/attestation_pool/attestation_pool.js +4 -8
  14. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +6 -6
  15. package/dest/mem_pools/instrumentation.d.ts +2 -4
  16. package/dest/mem_pools/instrumentation.d.ts.map +1 -1
  17. package/dest/mem_pools/instrumentation.js +14 -16
  18. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +5 -7
  19. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
  20. package/dest/mem_pools/tx_pool_v2/interfaces.js +0 -1
  21. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +6 -5
  22. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -1
  23. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +1 -1
  24. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
  25. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +43 -26
  26. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +1 -1
  27. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
  28. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +0 -3
  29. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +1 -2
  30. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
  31. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +1 -18
  32. package/dest/msg_validators/attestation_validator/attestation_validator.d.ts +1 -1
  33. package/dest/msg_validators/attestation_validator/attestation_validator.d.ts.map +1 -1
  34. package/dest/msg_validators/attestation_validator/attestation_validator.js +4 -5
  35. package/dest/msg_validators/clock_tolerance.d.ts +1 -1
  36. package/dest/msg_validators/clock_tolerance.d.ts.map +1 -1
  37. package/dest/msg_validators/clock_tolerance.js +3 -4
  38. package/dest/msg_validators/proposal_validator/proposal_validator.d.ts +1 -1
  39. package/dest/msg_validators/proposal_validator/proposal_validator.d.ts.map +1 -1
  40. package/dest/msg_validators/proposal_validator/proposal_validator.js +5 -5
  41. package/dest/msg_validators/tx_validator/contract_instance_validator.d.ts +9 -0
  42. package/dest/msg_validators/tx_validator/contract_instance_validator.d.ts.map +1 -0
  43. package/dest/msg_validators/tx_validator/contract_instance_validator.js +48 -0
  44. package/dest/msg_validators/tx_validator/data_validator.d.ts +1 -1
  45. package/dest/msg_validators/tx_validator/data_validator.d.ts.map +1 -1
  46. package/dest/msg_validators/tx_validator/data_validator.js +35 -2
  47. package/dest/msg_validators/tx_validator/factory.d.ts +1 -1
  48. package/dest/msg_validators/tx_validator/factory.d.ts.map +1 -1
  49. package/dest/msg_validators/tx_validator/factory.js +8 -2
  50. package/dest/msg_validators/tx_validator/metadata_validator.d.ts +1 -1
  51. package/dest/msg_validators/tx_validator/metadata_validator.d.ts.map +1 -1
  52. package/dest/msg_validators/tx_validator/metadata_validator.js +4 -4
  53. package/dest/msg_validators/tx_validator/phases_validator.js +1 -1
  54. package/dest/services/libp2p/libp2p_service.d.ts +9 -2
  55. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  56. package/dest/services/libp2p/libp2p_service.js +25 -22
  57. package/dest/services/peer-manager/metrics.d.ts +1 -3
  58. package/dest/services/peer-manager/metrics.d.ts.map +1 -1
  59. package/dest/services/peer-manager/metrics.js +0 -6
  60. package/dest/services/peer-manager/peer_manager.d.ts +1 -1
  61. package/dest/services/peer-manager/peer_manager.d.ts.map +1 -1
  62. package/dest/services/peer-manager/peer_manager.js +3 -6
  63. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts +8 -11
  64. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts.map +1 -1
  65. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js +101 -79
  66. package/dest/services/reqresp/batch-tx-requester/interface.d.ts +2 -3
  67. package/dest/services/reqresp/batch-tx-requester/interface.d.ts.map +1 -1
  68. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts +4 -5
  69. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts.map +1 -1
  70. package/dest/services/reqresp/batch-tx-requester/missing_txs.js +7 -13
  71. package/dest/services/reqresp/batch-tx-requester/peer_collection.d.ts +11 -19
  72. package/dest/services/reqresp/batch-tx-requester/peer_collection.d.ts.map +1 -1
  73. package/dest/services/reqresp/batch-tx-requester/peer_collection.js +15 -52
  74. package/dest/services/reqresp/rate-limiter/rate_limiter.d.ts +5 -4
  75. package/dest/services/reqresp/rate-limiter/rate_limiter.d.ts.map +1 -1
  76. package/dest/services/reqresp/rate-limiter/rate_limiter.js +10 -8
  77. package/dest/services/reqresp/reqresp.d.ts +1 -1
  78. package/dest/services/reqresp/reqresp.d.ts.map +1 -1
  79. package/dest/services/reqresp/reqresp.js +3 -4
  80. package/dest/services/tx_collection/fast_tx_collection.d.ts +4 -1
  81. package/dest/services/tx_collection/fast_tx_collection.d.ts.map +1 -1
  82. package/dest/services/tx_collection/fast_tx_collection.js +73 -57
  83. package/dest/services/tx_collection/file_store_tx_source.d.ts +5 -4
  84. package/dest/services/tx_collection/file_store_tx_source.d.ts.map +1 -1
  85. package/dest/services/tx_collection/file_store_tx_source.js +39 -29
  86. package/dest/services/tx_collection/missing_txs_tracker.d.ts +32 -0
  87. package/dest/services/tx_collection/missing_txs_tracker.d.ts.map +1 -0
  88. package/dest/services/tx_collection/missing_txs_tracker.js +27 -0
  89. package/dest/services/tx_collection/proposal_tx_collector.d.ts +7 -6
  90. package/dest/services/tx_collection/proposal_tx_collector.d.ts.map +1 -1
  91. package/dest/services/tx_collection/proposal_tx_collector.js +4 -4
  92. package/dest/services/tx_collection/slow_tx_collection.js +1 -1
  93. package/dest/services/tx_collection/tx_collection.d.ts +6 -3
  94. package/dest/services/tx_collection/tx_collection.d.ts.map +1 -1
  95. package/dest/services/tx_collection/tx_source.d.ts +6 -5
  96. package/dest/services/tx_collection/tx_source.d.ts.map +1 -1
  97. package/dest/services/tx_collection/tx_source.js +9 -7
  98. package/dest/test-helpers/make-test-p2p-clients.d.ts +1 -1
  99. package/dest/test-helpers/make-test-p2p-clients.d.ts.map +1 -1
  100. package/dest/test-helpers/reqresp-nodes.d.ts +1 -1
  101. package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
  102. package/dest/test-helpers/testbench-utils.d.ts +1 -1
  103. package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
  104. package/dest/test-helpers/testbench-utils.js +2 -20
  105. package/dest/testbench/p2p_client_testbench_worker.d.ts +1 -1
  106. package/dest/testbench/p2p_client_testbench_worker.d.ts.map +1 -1
  107. package/dest/testbench/p2p_client_testbench_worker.js +5 -6
  108. package/dest/testbench/worker_client_manager.d.ts +1 -1
  109. package/dest/testbench/worker_client_manager.d.ts.map +1 -1
  110. package/dest/testbench/worker_client_manager.js +1 -2
  111. package/dest/util.d.ts +1 -1
  112. package/package.json +14 -14
  113. package/src/client/factory.ts +8 -4
  114. package/src/client/p2p_client.ts +4 -6
  115. package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +8 -7
  116. package/src/config.ts +10 -10
  117. package/src/mem_pools/attestation_pool/attestation_pool.ts +7 -8
  118. package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +6 -6
  119. package/src/mem_pools/instrumentation.ts +13 -17
  120. package/src/mem_pools/tx_pool_v2/interfaces.ts +4 -7
  121. package/src/mem_pools/tx_pool_v2/tx_metadata.ts +4 -4
  122. package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +43 -29
  123. package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +0 -3
  124. package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +1 -19
  125. package/src/msg_validators/attestation_validator/README.md +1 -1
  126. package/src/msg_validators/attestation_validator/attestation_validator.ts +4 -5
  127. package/src/msg_validators/clock_tolerance.ts +3 -4
  128. package/src/msg_validators/proposal_validator/README.md +3 -3
  129. package/src/msg_validators/proposal_validator/proposal_validator.ts +5 -6
  130. package/src/msg_validators/tx_validator/contract_instance_validator.ts +56 -0
  131. package/src/msg_validators/tx_validator/data_validator.ts +42 -1
  132. package/src/msg_validators/tx_validator/factory.ts +7 -0
  133. package/src/msg_validators/tx_validator/metadata_validator.ts +4 -12
  134. package/src/msg_validators/tx_validator/phases_validator.ts +1 -1
  135. package/src/services/libp2p/libp2p_service.ts +28 -18
  136. package/src/services/peer-manager/metrics.ts +0 -7
  137. package/src/services/peer-manager/peer_manager.ts +3 -7
  138. package/src/services/reqresp/batch-tx-requester/README.md +7 -46
  139. package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts +111 -75
  140. package/src/services/reqresp/batch-tx-requester/interface.ts +1 -2
  141. package/src/services/reqresp/batch-tx-requester/missing_txs.ts +6 -13
  142. package/src/services/reqresp/batch-tx-requester/peer_collection.ts +24 -68
  143. package/src/services/reqresp/rate-limiter/rate_limiter.ts +13 -9
  144. package/src/services/reqresp/reqresp.ts +3 -5
  145. package/src/services/tx_collection/fast_tx_collection.ts +83 -57
  146. package/src/services/tx_collection/file_store_tx_source.ts +43 -31
  147. package/src/services/tx_collection/missing_txs_tracker.ts +52 -0
  148. package/src/services/tx_collection/proposal_tx_collector.ts +13 -8
  149. package/src/services/tx_collection/slow_tx_collection.ts +1 -1
  150. package/src/services/tx_collection/tx_collection.ts +5 -3
  151. package/src/services/tx_collection/tx_source.ts +8 -7
  152. package/src/test-helpers/make-test-p2p-clients.ts +1 -1
  153. package/src/test-helpers/reqresp-nodes.ts +1 -1
  154. package/src/test-helpers/testbench-utils.ts +3 -28
  155. package/src/testbench/p2p_client_testbench_worker.ts +9 -7
  156. package/src/testbench/worker_client_manager.ts +1 -2
  157. package/src/util.ts +1 -1
  158. package/dest/services/tx_collection/request_tracker.d.ts +0 -53
  159. package/dest/services/tx_collection/request_tracker.d.ts.map +0 -1
  160. package/dest/services/tx_collection/request_tracker.js +0 -84
  161. package/src/services/tx_collection/request_tracker.ts +0 -127
@@ -2,23 +2,18 @@ import type { DateProvider } from '@aztec/foundation/timer';
2
2
  import type { PeerErrorSeverity } from '@aztec/stdlib/p2p';
3
3
 
4
4
  import type { PeerId } from '@libp2p/interface';
5
- import { peerIdFromString } from '@libp2p/peer-id';
6
5
 
7
- import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
8
6
  import { DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD } from './config.js';
9
7
  import type { IPeerPenalizer } from './interface.js';
10
8
 
11
9
  export const RATE_LIMIT_EXCEEDED_PEER_CACHE_TTL = 1000; // 1s
12
10
 
13
11
  export interface IPeerCollection {
12
+ getAllPeers(): Set<string>;
13
+ getSmartPeers(): Set<string>;
14
14
  markPeerSmart(peerId: PeerId): void;
15
- markPeerDumb(peerId: PeerId): void;
16
-
17
- /** Sample next peer in round-robin fashion. No smart peers if returns undefined */
18
- nextSmartPeerToQuery(): PeerId | undefined;
19
- /** Sample next peer in round-robin fashion. No dumb peers if returns undefined */
20
- nextDumbPeerToQuery(): PeerId | undefined;
21
-
15
+ getSmartPeersToQuery(): Array<string>;
16
+ getDumbPeersToQuery(): Array<string>;
22
17
  thereAreSomeDumbRatelimitExceededPeers(): boolean;
23
18
  penalisePeer(peerId: PeerId, severity: PeerErrorSeverity): void;
24
19
  unMarkPeerAsBad(peerId: PeerId): void;
@@ -33,6 +28,8 @@ export interface IPeerCollection {
33
28
  }
34
29
 
35
30
  export class PeerCollection implements IPeerCollection {
31
+ private readonly peers;
32
+
36
33
  private readonly smartPeers = new Set<string>();
37
34
  private readonly inFlightPeers = new Set<string>();
38
35
  private readonly rateLimitExceededPeers = new Map<string, number>();
@@ -40,64 +37,46 @@ export class PeerCollection implements IPeerCollection {
40
37
  private readonly badPeers = new Set<string>();
41
38
 
42
39
  constructor(
43
- private readonly connectionSampler: Pick<ConnectionSampler, 'getPeerListSortedByConnectionCountAsc'>,
40
+ initialPeers: PeerId[],
44
41
  private readonly pinnedPeerId: PeerId | undefined,
45
42
  private readonly dateProvider: DateProvider,
46
43
  private readonly badPeerThreshold: number = DEFAULT_BATCH_TX_REQUESTER_BAD_PEER_THRESHOLD,
47
44
  private readonly peerPenalizer?: IPeerPenalizer,
48
45
  ) {
49
- // Pinned peer is treated specially, always mark it as in-flight
46
+ this.peers = new Set(initialPeers.map(peer => peer.toString()));
47
+
48
+ // Pinned peer is treaded specially, always mark it as in-flight
50
49
  // and never return it as part of smart/dumb peers
51
50
  if (this.pinnedPeerId) {
52
51
  const peerIdStr = this.pinnedPeerId.toString();
53
52
  this.inFlightPeers.add(peerIdStr);
53
+ this.peers.delete(peerIdStr);
54
54
  }
55
55
  }
56
56
 
57
- public markPeerSmart(peerId: PeerId): void {
58
- this.smartPeers.add(peerId.toString());
59
- }
60
-
61
- public markPeerDumb(peerId: PeerId): void {
62
- this.smartPeers.delete(peerId.toString());
63
- }
64
-
65
- // We keep track of all peers that are queried for peer sampling algorithm
66
- private queriedSmartPeers: Set<string> = new Set<string>();
67
- private queriedDumbPeers: Set<string> = new Set<string>();
68
-
69
- private static nextPeer(allPeers: Set<string>, queried: Set<string>): PeerId | undefined {
70
- if (allPeers.size === 0) {
71
- return undefined;
72
- }
73
- const availablePeers = allPeers.difference(queried);
74
- let [first] = availablePeers;
75
- if (first === undefined) {
76
- // We queried all peers. Start over
77
- [first] = allPeers;
78
- queried.clear();
79
- }
80
- queried.add(first);
81
- return peerIdFromString(first);
57
+ public getAllPeers(): Set<string> {
58
+ return this.peers;
82
59
  }
83
60
 
84
- public nextSmartPeerToQuery(): PeerId | undefined {
85
- return PeerCollection.nextPeer(this.availableSmartPeers, this.queriedSmartPeers);
61
+ public getSmartPeers(): Set<string> {
62
+ return this.smartPeers;
86
63
  }
87
64
 
88
- public nextDumbPeerToQuery(): PeerId | undefined {
89
- return PeerCollection.nextPeer(this.availableDumbPeers, this.queriedDumbPeers);
65
+ public markPeerSmart(peerId: PeerId): void {
66
+ this.smartPeers.add(peerId.toString());
90
67
  }
91
68
 
92
- private get availableSmartPeers(): Set<string> {
93
- return this.peers.intersection(
69
+ public getSmartPeersToQuery(): Array<string> {
70
+ return Array.from(
94
71
  this.smartPeers.difference(this.getBadPeers().union(this.inFlightPeers).union(this.getRateLimitExceededPeers())),
95
72
  );
96
73
  }
97
74
 
98
- private get availableDumbPeers(): Set<string> {
99
- return this.peers.difference(
100
- this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers()),
75
+ public getDumbPeersToQuery(): Array<string> {
76
+ return Array.from(
77
+ this.peers.difference(
78
+ this.smartPeers.union(this.getBadPeers()).union(this.inFlightPeers).union(this.getRateLimitExceededPeers()),
79
+ ),
101
80
  );
102
81
  }
103
82
 
@@ -223,27 +202,4 @@ export class PeerCollection implements IPeerCollection {
223
202
 
224
203
  return minExpiry! - now;
225
204
  }
226
-
227
- private orderedPeers: Set<string> = new Set();
228
-
229
- private get peers(): Set<string> {
230
- const pinnedStr = this.pinnedPeerId?.toString();
231
- const currentlyConnected = new Set(
232
- this.connectionSampler
233
- .getPeerListSortedByConnectionCountAsc()
234
- .map(p => p.toString())
235
- .filter(p => p !== pinnedStr),
236
- );
237
-
238
- // Remove disconnected peers, preserving order of the rest.
239
- this.orderedPeers = this.orderedPeers.intersection(currentlyConnected);
240
-
241
- // Append newly connected peers at the end (lowest priority).
242
- for (const peer of currentlyConnected) {
243
- if (!this.orderedPeers.has(peer)) {
244
- this.orderedPeers.add(peer);
245
- }
246
- }
247
- return this.orderedPeers;
248
- }
249
205
  }
@@ -97,9 +97,10 @@ export function prettyPrintRateLimitStatus(status: RateLimitStatus) {
97
97
  * 2. Individual rate limits for each peer.
98
98
  *
99
99
  * How it works:
100
- * - When a request comes in, it first checks against the global rate limit.
101
- * - If the global limit allows, it then checks against the specific peer's rate limit.
102
- * - The request is only allowed if both the global and peer-specific limits allow it.
100
+ * - When a request comes in, it first checks against the peer's individual rate limit.
101
+ * - If the peer limit allows, it then checks against the global rate limit.
102
+ * - The request is only allowed if both the peer-specific and global limits allow it.
103
+ * - Checking peer limit first ensures a rate-limited peer cannot exhaust the global quota.
103
104
  * - It automatically creates and manages rate limiters for new peers as they make requests.
104
105
  * - It periodically cleans up rate limiters for inactive peers to conserve memory.
105
106
  *
@@ -119,10 +120,6 @@ export class SubProtocolRateLimiter {
119
120
  }
120
121
 
121
122
  allow(peerId: PeerId): RateLimitStatus {
122
- if (!this.globalLimiter.allow()) {
123
- return RateLimitStatus.DeniedGlobal;
124
- }
125
-
126
123
  const peerIdStr = peerId.toString();
127
124
  let peerLimiter: PeerRateLimiter | undefined = this.peerLimiters.get(peerIdStr);
128
125
  if (!peerLimiter) {
@@ -135,10 +132,17 @@ export class SubProtocolRateLimiter {
135
132
  } else {
136
133
  peerLimiter.lastAccess = Date.now();
137
134
  }
138
- const peerLimitAllowed = peerLimiter.limiter.allow();
139
- if (!peerLimitAllowed) {
135
+
136
+ // Check peer limit first: a rate-limited peer must not consume global quota,
137
+ // otherwise one spamming peer can starve all others by exhausting the global bucket.
138
+ if (!peerLimiter.limiter.allow()) {
140
139
  return RateLimitStatus.DeniedPeer;
141
140
  }
141
+
142
+ if (!this.globalLimiter.allow()) {
143
+ return RateLimitStatus.DeniedGlobal;
144
+ }
145
+
142
146
  return RateLimitStatus.Allowed;
143
147
  }
144
148
 
@@ -320,7 +320,7 @@ export class ReqResp implements ReqRespInterface {
320
320
  };
321
321
 
322
322
  for (const index of indices) {
323
- this.logger.trace(`Sending request ${index} to peer ${peerAsString}`);
323
+ this.logger.info(`Sending request ${index} to peer ${peerAsString}`);
324
324
  const response = await this.sendRequestToPeer(peer, subProtocol, requestBuffers[index]);
325
325
 
326
326
  // Check the status of the response buffer
@@ -462,7 +462,7 @@ export class ReqResp implements ReqRespInterface {
462
462
  );
463
463
  return resp;
464
464
  } catch (e: any) {
465
- this.logger.warn(`SUBPROTOCOL: ${subProtocol}\n`, e);
465
+ this.logger.debug(`SUBPROTOCOL: ${subProtocol}\n`, e);
466
466
  // On error we immediately abort the stream, this is preferred way,
467
467
  // because it signals to the sender that error happened, whereas
468
468
  // closing the stream only closes our side and is much slower
@@ -621,9 +621,7 @@ export class ReqResp implements ReqRespInterface {
621
621
  // and that this stream should be dropped
622
622
  const isMessageToNotWarn =
623
623
  err instanceof Error &&
624
- ['stream reset', 'Cannot push value onto an ended pushable', 'read ECONNRESET'].some(msg =>
625
- err.message.includes(msg),
626
- );
624
+ ['stream reset', 'Cannot push value onto an ended pushable'].some(msg => err.message.includes(msg));
627
625
  const level = isMessageToNotWarn ? 'debug' : 'warn';
628
626
  this.logger[level]('Unknown stream error while handling the stream, aborting', {
629
627
  protocol,
@@ -1,9 +1,12 @@
1
1
  import { BlockNumber } from '@aztec/foundation/branded-types';
2
2
  import { times } from '@aztec/foundation/collection';
3
+ import { AbortError, TimeoutError } from '@aztec/foundation/error';
3
4
  import { type Logger, createLogger } from '@aztec/foundation/log';
5
+ import { promiseWithResolvers } from '@aztec/foundation/promise';
4
6
  import { sleep } from '@aztec/foundation/sleep';
5
7
  import { DateProvider, elapsed } from '@aztec/foundation/timer';
6
8
  import type { L2BlockInfo } from '@aztec/stdlib/block';
9
+ import type { BlockProposal } from '@aztec/stdlib/p2p';
7
10
  import { type Tx, TxHash } from '@aztec/stdlib/tx';
8
11
 
9
12
  import type { PeerId } from '@libp2p/interface';
@@ -11,12 +14,12 @@ import type { PeerId } from '@libp2p/interface';
11
14
  import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
12
15
  import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
13
16
  import type { TxCollectionConfig } from './config.js';
17
+ import { MissingTxsTracker } from './missing_txs_tracker.js';
14
18
  import {
15
19
  BatchTxRequesterCollector,
16
20
  type MissingTxsCollector,
17
21
  SendBatchRequestCollector,
18
22
  } from './proposal_tx_collector.js';
19
- import { RequestTracker } from './request_tracker.js';
20
23
  import type { FastCollectionRequest, FastCollectionRequestInput } from './tx_collection.js';
21
24
  import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
22
25
  import type { TxSource } from './tx_source.js';
@@ -45,9 +48,7 @@ export class FastTxCollection {
45
48
  }
46
49
 
47
50
  public async stop() {
48
- this.requests.forEach(request => {
49
- request.requestTracker.cancel();
50
- });
51
+ this.requests.forEach(request => request.promise.reject(new AbortError(`Stopped collection service`)));
51
52
  await Promise.resolve();
52
53
  }
53
54
 
@@ -74,65 +75,81 @@ export class FastTxCollection {
74
75
  ? { ...input.blockProposal.toBlockInfo(), blockNumber: input.blockNumber }
75
76
  : { ...input.block.toBlockInfo() };
76
77
 
78
+ // This promise is used to await for the collection to finish during the main collectFast method.
79
+ // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
80
+ const promise = promiseWithResolvers<void>();
81
+ const timeoutTimer = setTimeout(() => promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
82
+
77
83
  const request: FastCollectionRequest = {
78
84
  ...input,
79
85
  blockInfo,
80
- requestTracker: RequestTracker.create(txHashes, opts.deadline, this.dateProvider),
86
+ promise,
87
+ missingTxTracker: MissingTxsTracker.fromArray(txHashes),
88
+ deadline: opts.deadline,
81
89
  };
82
90
 
83
91
  const [duration] = await elapsed(() => this.collectFast(request, { ...opts }));
92
+ clearTimeout(timeoutTimer);
84
93
 
85
94
  this.log.verbose(
86
- `Collected ${request.requestTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
95
+ `Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
87
96
  {
88
97
  ...blockInfo,
89
98
  duration,
90
99
  requestType: input.type,
91
- missingTxs: [...request.requestTracker.missingTxHashes],
100
+ missingTxs: [...request.missingTxTracker.missingTxHashes],
92
101
  },
93
102
  );
94
- return request.requestTracker.collectedTxs;
103
+ return request.missingTxTracker.collectedTxs;
95
104
  }
96
105
 
97
- protected async collectFast(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) {
106
+ protected async collectFast(
107
+ request: FastCollectionRequest,
108
+ opts: { proposal?: BlockProposal; deadline: Date; pinnedPeer?: PeerId },
109
+ ) {
98
110
  this.requests.add(request);
99
111
  const { blockInfo } = request;
100
112
 
101
113
  this.log.debug(
102
- `Starting fast collection of ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
103
- { ...blockInfo, requestType: request.type, deadline: request.requestTracker.deadline },
114
+ `Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
115
+ { ...blockInfo, requestType: request.type, deadline: opts.deadline },
104
116
  );
105
117
 
106
118
  try {
107
119
  // Start blasting all nodes for the txs. We give them a little time to respond before we start reqresp.
108
- // We race against the cancellation token to exit as soon as all txs are collected, the deadline expires,
109
- // or the request is externally cancelled.
110
- const nodeCollectionPromise = this.collectFastFromNodes(request);
120
+ // And keep an eye on the request promise to ensure we don't wait longer than the deadline or return as soon
121
+ // as we have collected all txs, whatever the source.
122
+ const nodeCollectionPromise = this.collectFastFromNodes(request, opts);
111
123
  const waitBeforeReqResp = sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs);
112
- await Promise.race([request.requestTracker.cancellationToken, waitBeforeReqResp]);
124
+ await Promise.race([request.promise.promise, waitBeforeReqResp]);
113
125
 
114
- // If we have collected all txs or the request was cancelled, we can stop here.
115
- // Wait for node collection to settle so inner tasks finish before we return.
116
- if (request.requestTracker.checkCancelled()) {
117
- if (request.requestTracker.allFetched()) {
118
- this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
119
- }
120
- await nodeCollectionPromise;
126
+ // If we have collected all txs, we can stop here
127
+ if (request.missingTxTracker.allFetched()) {
128
+ this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
121
129
  return;
122
130
  }
123
131
 
124
132
  // Start blasting reqresp for the remaining txs. Note that node collection keeps running in parallel.
125
133
  // We stop when we have collected all txs, timed out, or both node collection and reqresp have given up.
126
- // Inner tasks observe requestTracker.checkCancelled() and stop themselves, so this settles shortly after cancellation.
127
- await Promise.allSettled([this.collectFastViaReqResp(request, opts), nodeCollectionPromise]);
134
+ const collectionPromise = Promise.allSettled([this.collectFastViaReqResp(request, opts), nodeCollectionPromise]);
135
+ await Promise.race([collectionPromise, request.promise.promise]);
128
136
  } catch (err) {
129
- this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, {
137
+ // Log and swallow all errors
138
+ const logCtx = {
130
139
  ...blockInfo,
131
- missingTxs: request.requestTracker.missingTxHashes.values().map(txHash => txHash.toString()),
132
- });
140
+ errorMessage: err instanceof Error ? err.message : undefined,
141
+ missingTxs: request.missingTxTracker.missingTxHashes.values().map(txHash => txHash.toString()),
142
+ };
143
+ if (err instanceof Error && err.name === 'TimeoutError') {
144
+ this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
145
+ } else if (err instanceof Error && err.name === 'AbortError') {
146
+ this.log.warn(`Aborted collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
147
+ } else {
148
+ this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, logCtx);
149
+ }
133
150
  } finally {
134
151
  // Ensure no unresolved promises and remove the request from the set
135
- request.requestTracker.cancel();
152
+ request.promise.resolve();
136
153
  this.requests.delete(request);
137
154
  }
138
155
  }
@@ -143,28 +160,30 @@ export class FastTxCollection {
143
160
  * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
144
161
  * tx is requested more than once at the same time to the same node.
145
162
  */
146
- private async collectFastFromNodes(request: FastCollectionRequest): Promise<void> {
163
+ private async collectFastFromNodes(request: FastCollectionRequest, opts: { deadline: Date }): Promise<void> {
147
164
  if (this.nodes.length === 0) {
148
165
  return;
149
166
  }
150
167
 
151
168
  // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
152
- const attemptsPerTx = [...request.requestTracker.missingTxHashes].map(txHash => ({
169
+ const attemptsPerTx = [...request.missingTxTracker.missingTxHashes].map(txHash => ({
153
170
  txHash,
154
171
  attempts: 0,
155
172
  found: false,
156
173
  }));
157
174
 
158
175
  // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
159
- await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx)));
176
+ await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx, opts)));
160
177
  }
161
178
 
162
179
  private async collectFastFromNode(
163
180
  request: FastCollectionRequest,
164
181
  node: TxSource,
165
182
  attemptsPerTx: { txHash: string; attempts: number; found: boolean }[],
183
+ opts: { deadline: Date },
166
184
  ) {
167
- const notFinished = () => !request.requestTracker.checkCancelled();
185
+ const notFinished = () =>
186
+ this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
168
187
 
169
188
  const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
170
189
  const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
@@ -181,7 +200,7 @@ export class FastTxCollection {
181
200
  if (!txToRequest) {
182
201
  // No more txs to process
183
202
  break;
184
- } else if (!request.requestTracker.isMissing(txToRequest.txHash)) {
203
+ } else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
185
204
  // Mark as found if it was found somewhere else, we'll then remove it from the array.
186
205
  // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
187
206
  txToRequest.found = true;
@@ -216,7 +235,7 @@ export class FastTxCollection {
216
235
  async () => {
217
236
  const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
218
237
  for (const tx of result.validTxs) {
219
- request.requestTracker.markFetched(tx);
238
+ request.missingTxTracker.markFetched(tx);
220
239
  }
221
240
  return result;
222
241
  },
@@ -235,12 +254,9 @@ export class FastTxCollection {
235
254
  activeRequestsToThisNode.delete(requestedTx.txHash);
236
255
  }
237
256
 
238
- // Sleep a bit until hitting the node again, but wake up immediately on cancellation
257
+ // Sleep a bit until hitting the node again (or not, depending on config)
239
258
  if (notFinished()) {
240
- await Promise.race([
241
- sleep(this.config.txCollectionFastNodeIntervalMs),
242
- request.requestTracker.cancellationToken,
243
- ]);
259
+ await sleep(this.config.txCollectionFastNodeIntervalMs);
244
260
  }
245
261
  }
246
262
  };
@@ -250,20 +266,21 @@ export class FastTxCollection {
250
266
  }
251
267
 
252
268
  private async collectFastViaReqResp(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) {
269
+ const timeoutMs = +request.deadline - this.dateProvider.now();
253
270
  const pinnedPeer = opts.pinnedPeer;
254
271
  const blockInfo = request.blockInfo;
255
272
  const slotNumber = blockInfo.slotNumber;
256
- if (request.requestTracker.timeoutMs < 100) {
273
+ if (timeoutMs < 100) {
257
274
  this.log.warn(
258
275
  `Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`,
259
- { timeoutMs: request.requestTracker.timeoutMs, ...blockInfo },
276
+ { timeoutMs, ...blockInfo },
260
277
  );
261
278
  return;
262
279
  }
263
280
 
264
281
  this.log.debug(
265
- `Starting fast reqresp for ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
266
- { ...blockInfo, timeoutMs: request.requestTracker.timeoutMs, pinnedPeer },
282
+ `Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
283
+ { ...blockInfo, timeoutMs, pinnedPeer },
267
284
  );
268
285
 
269
286
  try {
@@ -272,28 +289,34 @@ export class FastTxCollection {
272
289
  let result: Tx[];
273
290
  if (request.type === 'proposal') {
274
291
  result = await this.missingTxsCollector.collectTxs(
275
- request.requestTracker,
292
+ request.missingTxTracker,
276
293
  request.blockProposal,
277
294
  pinnedPeer,
295
+ timeoutMs,
278
296
  );
279
297
  } else if (request.type === 'block') {
280
298
  const blockTxsSource = {
281
299
  txHashes: request.block.body.txEffects.map(e => e.txHash),
282
300
  archive: request.block.archive.root,
283
301
  };
284
- result = await this.missingTxsCollector.collectTxs(request.requestTracker, blockTxsSource, pinnedPeer);
302
+ result = await this.missingTxsCollector.collectTxs(
303
+ request.missingTxTracker,
304
+ blockTxsSource,
305
+ pinnedPeer,
306
+ timeoutMs,
307
+ );
285
308
  } else {
286
309
  throw new Error(`Unknown request type: ${(request as any).type}`);
287
310
  }
288
311
  return { validTxs: result, invalidTxHashes: [] };
289
312
  },
290
- Array.from(request.requestTracker.missingTxHashes),
313
+ Array.from(request.missingTxTracker.missingTxHashes),
291
314
  { description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo },
292
315
  this.getAddContext(request),
293
316
  );
294
317
  } catch (err) {
295
318
  this.log.error(`Error sending fast reqresp request for txs`, err, {
296
- txs: [...request.requestTracker.missingTxHashes],
319
+ txs: [...request.missingTxTracker.missingTxHashes],
297
320
  ...blockInfo,
298
321
  });
299
322
  }
@@ -317,19 +340,20 @@ export class FastTxCollection {
317
340
  for (const tx of txs) {
318
341
  const txHash = tx.txHash.toString();
319
342
  // Remove the tx hash from the missing set, and add it to the found set.
320
- if (request.requestTracker.markFetched(tx)) {
343
+ if (request.missingTxTracker.markFetched(tx)) {
321
344
  this.log.trace(`Found tx ${txHash} for fast collection request`, {
322
345
  ...request.blockInfo,
323
346
  txHash: tx.txHash.toString(),
324
347
  type: request.type,
325
348
  });
326
- if (request.requestTracker.allFetched()) {
327
- this.log.trace(`All txs found for fast collection request`, {
328
- ...request.blockInfo,
329
- type: request.type,
330
- });
331
- break;
332
- }
349
+ }
350
+ // If we found all txs for this request, we resolve the promise
351
+ if (request.missingTxTracker.allFetched()) {
352
+ this.log.trace(`All txs found for fast collection request`, {
353
+ ...request.blockInfo,
354
+ type: request.type,
355
+ });
356
+ request.promise.resolve();
333
357
  }
334
358
  }
335
359
  }
@@ -342,7 +366,8 @@ export class FastTxCollection {
342
366
  public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void {
343
367
  for (const request of this.requests) {
344
368
  if (request.blockInfo.blockNumber <= blockNumber) {
345
- request.requestTracker.cancel();
369
+ request.promise.reject(new AbortError(`Stopped collecting txs up to block ${blockNumber}`));
370
+ this.requests.delete(request);
346
371
  }
347
372
  }
348
373
  }
@@ -354,7 +379,8 @@ export class FastTxCollection {
354
379
  public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void {
355
380
  for (const request of this.requests) {
356
381
  if (request.blockInfo.blockNumber > blockNumber) {
357
- request.requestTracker.cancel();
382
+ request.promise.reject(new AbortError(`Stopped collecting txs after block ${blockNumber}`));
383
+ this.requests.delete(request);
358
384
  }
359
385
  }
360
386
  }
@@ -1,7 +1,8 @@
1
+ import { partitionAsync } from '@aztec/foundation/collection';
1
2
  import { type Logger, createLogger } from '@aztec/foundation/log';
2
3
  import { Timer } from '@aztec/foundation/timer';
3
4
  import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store';
4
- import { Tx, type TxHash } from '@aztec/stdlib/tx';
5
+ import { Tx, type TxHash, type TxValidator } from '@aztec/stdlib/tx';
5
6
  import {
6
7
  type Histogram,
7
8
  Metrics,
@@ -23,6 +24,7 @@ export class FileStoreTxSource implements TxSource {
23
24
  private readonly fileStore: ReadOnlyFileStore,
24
25
  private readonly baseUrl: string,
25
26
  private readonly basePath: string,
27
+ private readonly txValidator: TxValidator,
26
28
  private readonly log: Logger,
27
29
  telemetry: TelemetryClient,
28
30
  ) {
@@ -44,6 +46,7 @@ export class FileStoreTxSource implements TxSource {
44
46
  public static async create(
45
47
  url: string,
46
48
  basePath: string,
49
+ txValidator: TxValidator,
47
50
  log: Logger = createLogger('p2p:file_store_tx_source'),
48
51
  telemetry: TelemetryClient = getTelemetryClient(),
49
52
  ): Promise<FileStoreTxSource | undefined> {
@@ -53,7 +56,7 @@ export class FileStoreTxSource implements TxSource {
53
56
  log.warn(`Failed to create file store for URL: ${url}`);
54
57
  return undefined;
55
58
  }
56
- return new FileStoreTxSource(fileStore, url, basePath, log, telemetry);
59
+ return new FileStoreTxSource(fileStore, url, basePath, txValidator, log, telemetry);
57
60
  } catch (err) {
58
61
  log.warn(`Error creating file store for URL: ${url}`, { error: err });
59
62
  return undefined;
@@ -65,35 +68,41 @@ export class FileStoreTxSource implements TxSource {
65
68
  }
66
69
 
67
70
  public async getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult> {
68
- const invalidTxHashes: string[] = [];
71
+ const results = await Promise.all(
72
+ txHashes.map(async txHash => {
73
+ const path = `${this.basePath}/txs/${txHash.toString()}.bin`;
74
+ const timer = new Timer();
75
+ try {
76
+ const buffer = await this.fileStore.read(path);
77
+ const tx = Tx.fromBuffer(buffer);
78
+ return { tx, downloadDuration: timer.ms(), downloadSize: buffer.length };
79
+ } catch {
80
+ this.downloadsFailed.add(1);
81
+ return undefined;
82
+ }
83
+ }),
84
+ );
85
+
86
+ const txs = results.filter(tx => tx !== undefined);
87
+ const [validTxs, invalidTxs] = await partitionAsync(
88
+ txs,
89
+ async ({ tx, downloadDuration, downloadSize }): Promise<boolean> => {
90
+ const valid = await this.txValidator.validateTx(tx);
91
+ if (valid.result === 'valid') {
92
+ this.downloadsSuccess.add(1);
93
+ this.downloadDuration.record(Math.ceil(downloadDuration));
94
+ this.downloadSize.record(downloadSize);
95
+ return true;
96
+ } else {
97
+ this.downloadsFailed.add(1);
98
+ return false;
99
+ }
100
+ },
101
+ );
102
+
69
103
  return {
70
- validTxs: (
71
- await Promise.all(
72
- txHashes.map(async txHash => {
73
- const path = `${this.basePath}/txs/${txHash.toString()}.bin`;
74
- const timer = new Timer();
75
- try {
76
- const buffer = await this.fileStore.read(path);
77
- const tx = Tx.fromBuffer(buffer);
78
- if ((await tx.validateTxHash()) && txHash.equals(tx.txHash)) {
79
- this.downloadsSuccess.add(1);
80
- this.downloadDuration.record(Math.ceil(timer.ms()));
81
- this.downloadSize.record(buffer.length);
82
- return tx;
83
- } else {
84
- invalidTxHashes.push(tx.txHash.toString());
85
- this.downloadsFailed.add(1);
86
- return undefined;
87
- }
88
- } catch {
89
- // Tx not found or error reading - return undefined
90
- this.downloadsFailed.add(1);
91
- return undefined;
92
- }
93
- }),
94
- )
95
- ).filter(tx => tx !== undefined),
96
- invalidTxHashes: invalidTxHashes,
104
+ validTxs: validTxs.map(({ tx }) => tx),
105
+ invalidTxHashes: invalidTxs.map(({ tx }) => tx.getTxHash().toString()),
97
106
  };
98
107
  }
99
108
  }
@@ -109,9 +118,12 @@ export class FileStoreTxSource implements TxSource {
109
118
  export async function createFileStoreTxSources(
110
119
  urls: string[],
111
120
  basePath: string,
121
+ txValidator: TxValidator,
112
122
  log: Logger = createLogger('p2p:file_store_tx_source'),
113
123
  telemetry: TelemetryClient = getTelemetryClient(),
114
124
  ): Promise<FileStoreTxSource[]> {
115
- const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, basePath, log, telemetry)));
125
+ const sources = await Promise.all(
126
+ urls.map(url => FileStoreTxSource.create(url, basePath, txValidator, log, telemetry)),
127
+ );
116
128
  return sources.filter((s): s is FileStoreTxSource => s !== undefined);
117
129
  }