@aztec/p2p 0.87.4 → 0.87.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dest/client/interface.d.ts +8 -4
  2. package/dest/client/interface.d.ts.map +1 -1
  3. package/dest/client/p2p_client.d.ts +4 -3
  4. package/dest/client/p2p_client.d.ts.map +1 -1
  5. package/dest/client/p2p_client.js +17 -10
  6. package/dest/config.d.ts +10 -0
  7. package/dest/config.d.ts.map +1 -1
  8. package/dest/config.js +12 -2
  9. package/dest/index.d.ts +1 -0
  10. package/dest/index.d.ts.map +1 -1
  11. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts +5 -6
  12. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts.map +1 -1
  13. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.js +37 -12
  14. package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts +2 -2
  15. package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts.map +1 -1
  16. package/dest/mem_pools/tx_pool/memory_tx_pool.js +1 -3
  17. package/dest/mem_pools/tx_pool/tx_pool.d.ts +6 -1
  18. package/dest/mem_pools/tx_pool/tx_pool.d.ts.map +1 -1
  19. package/dest/msg_validators/msg_seen_validator/msg_seen_validator.d.ts +10 -0
  20. package/dest/msg_validators/msg_seen_validator/msg_seen_validator.d.ts.map +1 -0
  21. package/dest/msg_validators/msg_seen_validator/msg_seen_validator.js +36 -0
  22. package/dest/services/dummy_service.d.ts +1 -1
  23. package/dest/services/dummy_service.d.ts.map +1 -1
  24. package/dest/services/dummy_service.js +1 -1
  25. package/dest/services/index.d.ts +1 -0
  26. package/dest/services/index.d.ts.map +1 -1
  27. package/dest/services/index.js +1 -0
  28. package/dest/services/libp2p/libp2p_service.d.ts +5 -3
  29. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  30. package/dest/services/libp2p/libp2p_service.js +42 -8
  31. package/dest/services/reqresp/connection-sampler/batch_connection_sampler.d.ts +1 -1
  32. package/dest/services/reqresp/connection-sampler/batch_connection_sampler.d.ts.map +1 -1
  33. package/dest/services/reqresp/connection-sampler/batch_connection_sampler.js +7 -3
  34. package/dest/services/reqresp/connection-sampler/connection_sampler.d.ts +2 -1
  35. package/dest/services/reqresp/connection-sampler/connection_sampler.d.ts.map +1 -1
  36. package/dest/services/reqresp/connection-sampler/connection_sampler.js +8 -3
  37. package/dest/services/reqresp/protocols/goodbye.d.ts.map +1 -1
  38. package/dest/services/reqresp/protocols/goodbye.js +3 -1
  39. package/dest/services/reqresp/rate-limiter/rate_limiter.d.ts +4 -2
  40. package/dest/services/reqresp/rate-limiter/rate_limiter.d.ts.map +1 -1
  41. package/dest/services/reqresp/rate-limiter/rate_limiter.js +10 -2
  42. package/dest/services/reqresp/rate-limiter/rate_limits.js +1 -1
  43. package/dest/services/reqresp/reqresp.d.ts +3 -3
  44. package/dest/services/reqresp/reqresp.d.ts.map +1 -1
  45. package/dest/services/reqresp/reqresp.js +39 -13
  46. package/dest/services/service.d.ts +3 -2
  47. package/dest/services/service.d.ts.map +1 -1
  48. package/dest/services/tx_collector.d.ts +14 -0
  49. package/dest/services/tx_collector.d.ts.map +1 -0
  50. package/dest/services/tx_collector.js +73 -0
  51. package/dest/test-helpers/reqresp-nodes.d.ts +3 -3
  52. package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
  53. package/dest/test-helpers/reqresp-nodes.js +4 -4
  54. package/dest/testbench/p2p_client_testbench_worker.js +1 -1
  55. package/package.json +12 -12
  56. package/src/client/interface.ts +8 -4
  57. package/src/client/p2p_client.ts +22 -10
  58. package/src/config.ts +22 -1
  59. package/src/index.ts +2 -0
  60. package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts +45 -18
  61. package/src/mem_pools/tx_pool/memory_tx_pool.ts +2 -4
  62. package/src/mem_pools/tx_pool/tx_pool.ts +7 -1
  63. package/src/msg_validators/msg_seen_validator/msg_seen_validator.ts +36 -0
  64. package/src/services/dummy_service.ts +3 -1
  65. package/src/services/index.ts +1 -0
  66. package/src/services/libp2p/libp2p_service.ts +51 -9
  67. package/src/services/reqresp/connection-sampler/batch_connection_sampler.ts +4 -2
  68. package/src/services/reqresp/connection-sampler/connection_sampler.ts +8 -3
  69. package/src/services/reqresp/protocols/goodbye.ts +3 -1
  70. package/src/services/reqresp/rate-limiter/rate_limiter.ts +9 -3
  71. package/src/services/reqresp/rate-limiter/rate_limits.ts +1 -1
  72. package/src/services/reqresp/reqresp.ts +44 -16
  73. package/src/services/service.ts +4 -1
  74. package/src/services/tx_collector.ts +98 -0
  75. package/src/test-helpers/reqresp-nodes.ts +13 -8
  76. package/src/testbench/p2p_client_testbench_worker.ts +1 -1
@@ -48,6 +48,7 @@ import { createLibp2p } from 'libp2p';
48
48
  import type { P2PConfig } from '../../config.js';
49
49
  import type { MemPools } from '../../mem_pools/interface.js';
50
50
  import { AttestationValidator, BlockProposalValidator } from '../../msg_validators/index.js';
51
+ import { MessageSeenValidator } from '../../msg_validators/msg_seen_validator/msg_seen_validator.js';
51
52
  import { getDefaultAllowedSetupFunctions } from '../../msg_validators/tx_validator/allowed_public_setup.js';
52
53
  import { type MessageValidator, createTxMessageValidators } from '../../msg_validators/tx_validator/factory.js';
53
54
  import { DoubleSpendTxValidator, TxProofValidator } from '../../msg_validators/tx_validator/index.js';
@@ -63,7 +64,7 @@ import { DEFAULT_SUB_PROTOCOL_VALIDATORS, ReqRespSubProtocol, type SubProtocolMa
63
64
  import { reqGoodbyeHandler } from '../reqresp/protocols/goodbye.js';
64
65
  import { pingHandler, reqRespBlockHandler, reqRespTxHandler, statusHandler } from '../reqresp/protocols/index.js';
65
66
  import { ReqResp } from '../reqresp/reqresp.js';
66
- import type { P2PService, PeerDiscoveryService } from '../service.js';
67
+ import type { P2PBlockReceivedCallback, P2PService, PeerDiscoveryService } from '../service.js';
67
68
 
68
69
  interface ValidationResult {
69
70
  name: string;
@@ -80,6 +81,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
80
81
  private jobQueue: SerialQueue = new SerialQueue();
81
82
  private peerManager: PeerManager;
82
83
  private discoveryRunningPromise?: RunningPromise;
84
+ private msgIdSeenValidators: Record<TopicType, MessageSeenValidator> = {} as Record<TopicType, MessageSeenValidator>;
83
85
 
84
86
  // Message validators
85
87
  private attestationValidator: AttestationValidator;
@@ -101,7 +103,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
101
103
  * @param block - The block received from the peer.
102
104
  * @returns The attestation for the block, if any.
103
105
  */
104
- private blockReceivedCallback: (block: BlockProposal) => Promise<BlockAttestation | undefined>;
106
+ private blockReceivedCallback: P2PBlockReceivedCallback;
105
107
 
106
108
  private gossipSubEventHandler: (e: CustomEvent<GossipsubMessage>) => void;
107
109
 
@@ -120,6 +122,10 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
120
122
  ) {
121
123
  super(telemetry, 'LibP2PService');
122
124
 
125
+ this.msgIdSeenValidators[TopicType.tx] = new MessageSeenValidator(config.seenMessageCacheSize);
126
+ this.msgIdSeenValidators[TopicType.block_proposal] = new MessageSeenValidator(config.seenMessageCacheSize);
127
+ this.msgIdSeenValidators[TopicType.block_attestation] = new MessageSeenValidator(config.seenMessageCacheSize);
128
+
123
129
  const versions = getVersions(config);
124
130
  this.protocolVersion = compressComponentVersions(versions);
125
131
  logger.info(`Started libp2p service with protocol version ${this.protocolVersion}`);
@@ -446,8 +452,9 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
446
452
  sendBatchRequest<SubProtocol extends ReqRespSubProtocol>(
447
453
  protocol: SubProtocol,
448
454
  requests: InstanceType<SubProtocolMap[SubProtocol]['request']>[],
455
+ pinnedPeerId: PeerId | undefined,
449
456
  ): Promise<(InstanceType<SubProtocolMap[SubProtocol]['response']> | undefined)[]> {
450
- return this.reqresp.sendBatchRequest(protocol, requests);
457
+ return this.reqresp.sendBatchRequest(protocol, requests, pinnedPeerId);
451
458
  }
452
459
 
453
460
  /**
@@ -458,9 +465,8 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
458
465
  return this.peerDiscoveryService.getEnr();
459
466
  }
460
467
 
461
- public registerBlockReceivedCallback(callback: (block: BlockProposal) => Promise<BlockAttestation | undefined>) {
468
+ public registerBlockReceivedCallback(callback: P2PBlockReceivedCallback) {
462
469
  this.blockReceivedCallback = callback;
463
- this.logger.verbose('Block received callback registered');
464
470
  }
465
471
 
466
472
  /**
@@ -494,6 +500,29 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
494
500
  return result.recipients.length;
495
501
  }
496
502
 
503
+ protected preValidateReceivedMessage(msg: Message, msgId: string, source: PeerId) {
504
+ const getValidator = () => {
505
+ if (msg.topic === this.topicStrings[TopicType.tx]) {
506
+ return this.msgIdSeenValidators[TopicType.tx];
507
+ }
508
+ if (msg.topic === this.topicStrings[TopicType.block_attestation]) {
509
+ return this.msgIdSeenValidators[TopicType.block_attestation];
510
+ }
511
+ if (msg.topic === this.topicStrings[TopicType.block_proposal]) {
512
+ return this.msgIdSeenValidators[TopicType.block_proposal];
513
+ }
514
+ this.logger.error(`Received message on unknown topic: ${msg.topic}`);
515
+ };
516
+
517
+ const validator = getValidator();
518
+
519
+ if (!validator || !validator.addMessage(msgId)) {
520
+ this.node.services.pubsub.reportMessageValidationResult(msgId, source.toString(), TopicValidatorResult.Ignore);
521
+ return false;
522
+ }
523
+ return true;
524
+ }
525
+
497
526
  /**
498
527
  * Handles a new gossip message that was received by the client.
499
528
  * @param topic - The message's topic.
@@ -508,6 +537,11 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
508
537
  messageId: p2pMessage.id,
509
538
  messageLatency,
510
539
  });
540
+
541
+ if (!this.preValidateReceivedMessage(msg, msgId, source)) {
542
+ return;
543
+ }
544
+
511
545
  if (msg.topic === this.topicStrings[TopicType.tx]) {
512
546
  await this.handleGossipedTx(p2pMessage.payload, msgId, source);
513
547
  }
@@ -610,7 +644,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
610
644
  if (!result || !block) {
611
645
  return;
612
646
  }
613
- await this.processValidBlockProposal(block);
647
+ await this.processValidBlockProposal(block, source);
614
648
  }
615
649
 
616
650
  // REVIEW: callback pattern https://github.com/AztecProtocol/aztec-packages/issues/7963
@@ -620,9 +654,12 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
620
654
  [Attributes.BLOCK_ARCHIVE]: block.archive.toString(),
621
655
  [Attributes.P2P_ID]: await block.p2pMessageIdentifier().then(i => i.toString()),
622
656
  }))
623
- private async processValidBlockProposal(block: BlockProposal) {
657
+ private async processValidBlockProposal(block: BlockProposal, sender: PeerId) {
658
+ const slot = block.slotNumber.toBigInt();
659
+ const previousSlot = slot - 1n;
660
+ const epoch = slot / 32n;
624
661
  this.logger.verbose(
625
- `Received block ${block.blockNumber.toNumber()} for slot ${block.slotNumber.toNumber()} from external peer.`,
662
+ `Received block ${block.blockNumber.toNumber()} for slot ${slot}, epoch ${epoch} from external peer.`,
626
663
  {
627
664
  p2pMessageIdentifier: await block.p2pMessageIdentifier(),
628
665
  slot: block.slotNumber.toNumber(),
@@ -630,9 +667,14 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
630
667
  block: block.blockNumber.toNumber(),
631
668
  },
632
669
  );
670
+ const attestationsForPreviousSlot = await this.mempools.attestationPool?.getAttestationsForSlot(previousSlot);
671
+ if (attestationsForPreviousSlot !== undefined) {
672
+ this.logger.verbose(`Received ${attestationsForPreviousSlot.length} attestations for slot ${previousSlot}`);
673
+ }
674
+
633
675
  // Mark the txs in this proposal as non-evictable
634
676
  await this.mempools.txPool.markTxsAsNonEvictable(block.payload.txHashes);
635
- const attestation = await this.blockReceivedCallback(block);
677
+ const attestation = await this.blockReceivedCallback(block, sender);
636
678
 
637
679
  // TODO: fix up this pattern - the abstraction is not nice
638
680
  // The attestation can be undefined if no handler is registered / the validator deems the block invalid
@@ -26,6 +26,7 @@ export class BatchConnectionSampler {
26
26
  private readonly connectionSampler: ConnectionSampler,
27
27
  batchSize: number,
28
28
  maxPeers: number,
29
+ exclude?: PeerId[],
29
30
  ) {
30
31
  if (maxPeers <= 0) {
31
32
  throw new Error('Max peers cannot be 0');
@@ -38,7 +39,8 @@ export class BatchConnectionSampler {
38
39
  this.requestsPerPeer = Math.max(1, Math.floor(batchSize / maxPeers));
39
40
 
40
41
  // Sample initial peers
41
- this.batch = this.connectionSampler.samplePeersBatch(maxPeers);
42
+ const excluding = exclude && new Map(exclude.map(peerId => [peerId.toString(), true] as const));
43
+ this.batch = this.connectionSampler.samplePeersBatch(maxPeers, excluding);
42
44
  }
43
45
 
44
46
  /**
@@ -70,7 +72,7 @@ export class BatchConnectionSampler {
70
72
  }
71
73
 
72
74
  const excluding = new Map([[peerId.toString(), true]]);
73
- const newPeer = this.connectionSampler.getPeer(excluding);
75
+ const newPeer = this.connectionSampler.getPeer(excluding); // Q: Shouldn't we accumulate all excluded peers? Otherwise the sampler could return us a previously excluded peer?
74
76
 
75
77
  if (newPeer) {
76
78
  this.batch[index] = newPeer;
@@ -137,9 +137,10 @@ export class ConnectionSampler {
137
137
  * Samples a batch of unique peers from the libp2p node, prioritizing peers without active connections
138
138
  *
139
139
  * @param numberToSample - The number of peers to sample
140
+ * @param excluding - The peers to exclude from the sampling
140
141
  * @returns Array of unique sampled peers, prioritizing those without active connections
141
142
  */
142
- samplePeersBatch(numberToSample: number): PeerId[] {
143
+ samplePeersBatch(numberToSample: number, excluding?: Map<string, boolean>): PeerId[] {
143
144
  const peers = this.libp2p.getPeers();
144
145
  this.logger.debug('Sampling peers batch', { numberToSample, peers });
145
146
 
@@ -149,7 +150,7 @@ export class ConnectionSampler {
149
150
  const batch: PeerId[] = [];
150
151
  const withActiveConnections: Set<PeerId> = new Set();
151
152
  for (let i = 0; i < numberToSample; i++) {
152
- const { peer, sampledPeers } = this.getPeerFromList(peers, undefined);
153
+ const { peer, sampledPeers } = this.getPeerFromList(peers, excluding);
153
154
  if (peer) {
154
155
  batch.push(peer);
155
156
  }
@@ -252,7 +253,11 @@ export class ConnectionSampler {
252
253
  activeConnectionsCount: updatedActiveConnectionsCount,
253
254
  });
254
255
 
255
- await stream?.close();
256
+ //NOTE: All other status codes indicate closed stream.
257
+ //Either graceful close (closed/closing) or forced close (aborted/reset)
258
+ if (stream.status === 'open') {
259
+ await stream?.close();
260
+ }
256
261
  } catch (error) {
257
262
  this.logger.error(`Failed to close connection to peer with stream id ${streamId}`, error);
258
263
  } finally {
@@ -95,7 +95,9 @@ export function reqGoodbyeHandler(peerManager: PeerManager): ReqRespSubProtocolH
95
95
 
96
96
  peerManager.goodbyeReceived(peerId, reason);
97
97
 
98
- // Return a buffer of length 1 as an acknowledgement: this is allowed to fail
98
+ // NOTE: In the current implementation this won't be sent to peer,
99
+ // as the connection to peer has been already closed by peerManager.goodbyeReceived
100
+ // We have this just to satisfy interface
99
101
  return Promise.resolve(Buffer.from([0x0]));
100
102
  };
101
103
  }
@@ -8,7 +8,7 @@ import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
8
8
  import type { PeerId } from '@libp2p/interface';
9
9
 
10
10
  import type { PeerScoring } from '../../peer-manager/peer_scoring.js';
11
- import type { ReqRespSubProtocol, ReqRespSubProtocolRateLimits } from '../interface.js';
11
+ import type { ProtocolRateLimitQuota, ReqRespSubProtocol, ReqRespSubProtocolRateLimits } from '../interface.js';
12
12
  import { DEFAULT_RATE_LIMITS } from './rate_limits.js';
13
13
 
14
14
  // Check for disconnected peers every 10 minutes
@@ -177,16 +177,18 @@ export class SubProtocolRateLimiter {
177
177
  */
178
178
  export class RequestResponseRateLimiter {
179
179
  private subProtocolRateLimiters: Map<ReqRespSubProtocol, SubProtocolRateLimiter>;
180
+ private rateLimits: ReqRespSubProtocolRateLimits;
180
181
 
181
182
  private cleanupInterval: NodeJS.Timeout | undefined = undefined;
182
183
 
183
184
  constructor(
184
185
  private peerScoring: PeerScoring,
185
- rateLimits: ReqRespSubProtocolRateLimits = DEFAULT_RATE_LIMITS,
186
+ rateLimits: Partial<ReqRespSubProtocolRateLimits> = {},
186
187
  ) {
187
188
  this.subProtocolRateLimiters = new Map();
188
189
 
189
- for (const [subProtocol, protocolLimits] of Object.entries(rateLimits)) {
190
+ this.rateLimits = { ...DEFAULT_RATE_LIMITS, ...rateLimits };
191
+ for (const [subProtocol, protocolLimits] of Object.entries(this.rateLimits)) {
190
192
  this.subProtocolRateLimiters.set(
191
193
  subProtocol as ReqRespSubProtocol,
192
194
  new SubProtocolRateLimiter(
@@ -228,4 +230,8 @@ export class RequestResponseRateLimiter {
228
230
  stop() {
229
231
  clearInterval(this.cleanupInterval);
230
232
  }
233
+
234
+ getRateLimits(protocol: ReqRespSubProtocol): ProtocolRateLimitQuota {
235
+ return this.rateLimits[protocol];
236
+ }
231
237
  }
@@ -29,7 +29,7 @@ export const DEFAULT_RATE_LIMITS: ReqRespSubProtocolRateLimits = {
29
29
  },
30
30
  globalLimit: {
31
31
  quotaTimeMs: 1000,
32
- quotaCount: 20,
32
+ quotaCount: 200,
33
33
  },
34
34
  },
35
35
  [ReqRespSubProtocol.BLOCK]: {
@@ -1,4 +1,5 @@
1
1
  // @attribution: lodestar impl for inspiration
2
+ import { compactArray } from '@aztec/foundation/collection';
2
3
  import { type Logger, createLogger } from '@aztec/foundation/log';
3
4
  import { executeTimeout } from '@aztec/foundation/timer';
4
5
  import { PeerErrorSeverity } from '@aztec/stdlib/p2p';
@@ -25,6 +26,7 @@ import {
25
26
  type ReqRespResponse,
26
27
  ReqRespSubProtocol,
27
28
  type ReqRespSubProtocolHandlers,
29
+ type ReqRespSubProtocolRateLimits,
28
30
  type ReqRespSubProtocolValidators,
29
31
  type SubProtocolMap,
30
32
  subProtocolMap,
@@ -72,6 +74,7 @@ export class ReqResp {
72
74
  config: P2PReqRespConfig,
73
75
  private libp2p: Libp2p,
74
76
  private peerScoring: PeerScoring,
77
+ rateLimits: Partial<ReqRespSubProtocolRateLimits> = {},
75
78
  telemetryClient: TelemetryClient = getTelemetryClient(),
76
79
  ) {
77
80
  this.logger = createLogger('p2p:reqresp');
@@ -79,7 +82,7 @@ export class ReqResp {
79
82
  this.overallRequestTimeoutMs = config.overallRequestTimeoutMs;
80
83
  this.individualRequestTimeoutMs = config.individualRequestTimeoutMs;
81
84
 
82
- this.rateLimiter = new RequestResponseRateLimiter(peerScoring);
85
+ this.rateLimiter = new RequestResponseRateLimiter(peerScoring, rateLimits);
83
86
 
84
87
  // Connection sampler is used to sample our connected peers
85
88
  this.connectionSampler = new ConnectionSampler(libp2p);
@@ -261,6 +264,7 @@ export class ReqResp {
261
264
  async sendBatchRequest<SubProtocol extends ReqRespSubProtocol>(
262
265
  subProtocol: SubProtocol,
263
266
  requests: InstanceType<SubProtocolMap[SubProtocol]['request']>[],
267
+ pinnedPeer: PeerId | undefined,
264
268
  timeoutMs = 10000,
265
269
  maxPeers = Math.max(10, Math.ceil(requests.length / 3)),
266
270
  maxRetryAttempts = 3,
@@ -274,10 +278,15 @@ export class ReqResp {
274
278
  const pendingRequestIndices = new Set(requestBuffers.map((_, i) => i));
275
279
 
276
280
  // Create batch sampler with the total number of requests and max peers
277
- const batchSampler = new BatchConnectionSampler(this.connectionSampler, requests.length, maxPeers);
281
+ const batchSampler = new BatchConnectionSampler(
282
+ this.connectionSampler,
283
+ requests.length,
284
+ maxPeers,
285
+ compactArray([pinnedPeer]), // Exclude pinned peer from sampling, we will forcefully send all requests to it
286
+ );
278
287
 
279
- if (batchSampler.activePeerCount === 0) {
280
- this.logger.debug('No active peers to send requests to');
288
+ if (batchSampler.activePeerCount === 0 && !pinnedPeer) {
289
+ this.logger.warn('No active peers to send requests to');
281
290
  return [];
282
291
  }
283
292
 
@@ -308,6 +317,16 @@ export class ReqResp {
308
317
  requestBatches.get(peerAsString)!.indices.push(requestIndex);
309
318
  }
310
319
 
320
+ // If there is a pinned peer, we will always send every request to that peer
321
+ // We use the default limits for the subprotocol to avoid hitting the rate limiter
322
+ if (pinnedPeer) {
323
+ const limit = this.rateLimiter.getRateLimits(subProtocol).peerLimit.quotaCount;
324
+ requestBatches.set(pinnedPeer.toString(), {
325
+ peerId: pinnedPeer,
326
+ indices: Array.from(pendingRequestIndices.values()).slice(0, limit),
327
+ });
328
+ }
329
+
311
330
  // Make parallel requests for each peer's batch
312
331
  // A batch entry will look something like this:
313
332
  // PeerId0: [0, 1, 2, 3]
@@ -323,6 +342,7 @@ export class ReqResp {
323
342
  const peerResults: { index: number; response: InstanceType<SubProtocolMap[SubProtocol]['response']> }[] =
324
343
  [];
325
344
  for (const index of indices) {
345
+ this.logger.trace(`Sending request ${index} to peer ${peerAsString}`);
326
346
  const response = await this.sendRequestToPeer(peer, subProtocol, requestBuffers[index]);
327
347
 
328
348
  // Check the status of the response buffer
@@ -621,8 +641,9 @@ export class ReqResp {
621
641
  const response = await handler(connection.remotePeer, msg);
622
642
 
623
643
  if (protocol === ReqRespSubProtocol.GOODBYE) {
644
+ // NOTE: The stream was already closed by Goodbye handler
645
+ // peerManager.goodbyeReceived(peerId, reason); will call libp2p.hangUp closing all active streams and connections
624
646
  // Don't respond
625
- await stream.close();
626
647
  return;
627
648
  }
628
649
 
@@ -645,18 +666,25 @@ export class ReqResp {
645
666
  errorStatus = e.status;
646
667
  }
647
668
 
648
- const sendErrorChunk = this.sendErrorChunk(errorStatus);
649
-
650
- // Return and yield the response chunk
651
- await pipe(
652
- stream,
653
- async function* (_source: any) {
654
- yield* sendErrorChunk;
655
- },
656
- stream,
657
- );
669
+ if (stream.status === 'open') {
670
+ const sendErrorChunk = this.sendErrorChunk(errorStatus);
671
+ // Return and yield the response chunk
672
+ await pipe(
673
+ stream,
674
+ async function* (_source: any) {
675
+ yield* sendErrorChunk;
676
+ },
677
+ stream,
678
+ );
679
+ } else {
680
+ this.logger.debug('Stream already closed, not sending error response', { protocol, err: e, errorStatus });
681
+ }
658
682
  } finally {
659
- await stream.close();
683
+ //NOTE: All other status codes indicate closed stream.
684
+ //Either graceful close (closed/closing) or forced close (aborted/reset)
685
+ if (stream.status === 'open') {
686
+ await stream.close();
687
+ }
660
688
  }
661
689
  }
662
690
 
@@ -13,6 +13,8 @@ export enum PeerDiscoveryState {
13
13
  STOPPED = 'stopped',
14
14
  }
15
15
 
16
+ export type P2PBlockReceivedCallback = (block: BlockProposal, sender: PeerId) => Promise<BlockAttestation | undefined>;
17
+
16
18
  /**
17
19
  * The interface for a P2P service implementation.
18
20
  */
@@ -57,13 +59,14 @@ export interface P2PService {
57
59
  sendBatchRequest<Protocol extends ReqRespSubProtocol>(
58
60
  protocol: Protocol,
59
61
  requests: InstanceType<SubProtocolMap[Protocol]['request']>[],
62
+ pinnedPeerId?: PeerId,
60
63
  timeoutMs?: number,
61
64
  maxPeers?: number,
62
65
  maxRetryAttempts?: number,
63
66
  ): Promise<(InstanceType<SubProtocolMap[Protocol]['response']> | undefined)[]>;
64
67
 
65
68
  // Leaky abstraction: fix https://github.com/AztecProtocol/aztec-packages/issues/7963
66
- registerBlockReceivedCallback(callback: (block: BlockProposal) => Promise<BlockAttestation | undefined>): void;
69
+ registerBlockReceivedCallback(callback: P2PBlockReceivedCallback): void;
67
70
 
68
71
  getEnr(): ENR | undefined;
69
72
 
@@ -0,0 +1,98 @@
1
+ import { compactArray } from '@aztec/foundation/collection';
2
+ import { type Logger, createLogger } from '@aztec/foundation/log';
3
+ import type { BlockProposal } from '@aztec/stdlib/p2p';
4
+ import type { Tx, TxHash } from '@aztec/stdlib/tx';
5
+
6
+ import type { P2PClient } from '../client/p2p_client.js';
7
+
8
+ export class TxCollector {
9
+ constructor(
10
+ private p2pClient: Pick<
11
+ P2PClient,
12
+ 'getTxsByHashFromPool' | 'hasTxsInPool' | 'getTxsByHash' | 'validate' | 'requestTxsByHash'
13
+ >,
14
+ private log: Logger = createLogger('p2p:tx-collector'),
15
+ ) {}
16
+
17
+ async collectForBlockProposal(
18
+ proposal: BlockProposal,
19
+ peerWhoSentTheProposal: any,
20
+ ): Promise<{ txs: Tx[]; missing?: TxHash[] }> {
21
+ if (proposal.payload.txHashes.length === 0) {
22
+ this.log.verbose(`Received block proposal with no transactions, skipping transaction availability check`);
23
+ return { txs: [] };
24
+ }
25
+ // Is this a new style proposal?
26
+ if (proposal.txs && proposal.txs.length > 0 && proposal.txs.length === proposal.payload.txHashes.length) {
27
+ // Yes, any txs that we already have we should use
28
+ this.log.info(`Using new style proposal with ${proposal.txs.length} transactions`);
29
+
30
+ // Request from the pool based on the signed hashes in the payload
31
+ const hashesFromPayload = proposal.payload.txHashes;
32
+ const txsToUse = await this.p2pClient.getTxsByHashFromPool(hashesFromPayload);
33
+
34
+ const missingTxs = txsToUse.filter(tx => tx === undefined).length;
35
+ if (missingTxs > 0) {
36
+ this.log.verbose(
37
+ `Missing ${missingTxs}/${hashesFromPayload.length} transactions in the tx pool, will attempt to take from the proposal`,
38
+ );
39
+ }
40
+
41
+ let usedFromProposal = 0;
42
+
43
+ // Fill any holes with txs in the proposal, provided their hash matches the hash in the payload
44
+ for (let i = 0; i < txsToUse.length; i++) {
45
+ if (txsToUse[i] === undefined) {
46
+ // We don't have the transaction, take from the proposal, provided the hash is the same
47
+ const hashOfTxInProposal = await proposal.txs[i].getTxHash();
48
+ if (hashOfTxInProposal.equals(hashesFromPayload[i])) {
49
+ // Hash is equal, we can use the tx from the proposal
50
+ txsToUse[i] = proposal.txs[i];
51
+ usedFromProposal++;
52
+ } else {
53
+ this.log.warn(
54
+ `Unable to take tx: ${hashOfTxInProposal.toString()} from the proposal, it does not match payload hash: ${hashesFromPayload[
55
+ i
56
+ ].toString()}`,
57
+ );
58
+ }
59
+ }
60
+ }
61
+
62
+ // See if we still have any holes, if there are then we were not successful and will try the old method
63
+ if (txsToUse.some(tx => tx === undefined)) {
64
+ this.log.warn(`Failed to use transactions from proposal. Falling back to old proposal logic`);
65
+ } else {
66
+ this.log.info(
67
+ `Successfully used ${usedFromProposal}/${hashesFromPayload.length} transactions from the proposal`,
68
+ );
69
+
70
+ await this.p2pClient.validate(txsToUse as Tx[]);
71
+ return { txs: txsToUse as Tx[] };
72
+ }
73
+ }
74
+
75
+ this.log.info(`Using old style proposal with ${proposal.payload.txHashes.length} transactions`);
76
+
77
+ // Old style proposal, we will perform a request by hash from pool
78
+ // This will request from network any txs that are missing
79
+ const txHashes: TxHash[] = proposal.payload.txHashes;
80
+
81
+ // This part is just for logging that we are requesting from the network
82
+ const availability = await this.p2pClient.hasTxsInPool(txHashes);
83
+ const notAvailable = availability.filter(availability => availability === false);
84
+ if (notAvailable.length) {
85
+ this.log.verbose(
86
+ `Missing ${notAvailable.length} transactions in the tx pool, will need to request from the network`,
87
+ );
88
+ }
89
+
90
+ // This will request from the network any txs that are missing
91
+ const retrievedTxs = await this.p2pClient.getTxsByHash(txHashes, peerWhoSentTheProposal);
92
+ const missingTxs = compactArray(retrievedTxs.map((tx, index) => (tx === undefined ? txHashes[index] : undefined)));
93
+
94
+ await this.p2pClient.validate(retrievedTxs as Tx[]);
95
+
96
+ return { txs: retrievedTxs as Tx[], missing: missingTxs };
97
+ }
98
+ }
@@ -33,6 +33,7 @@ import type { P2PReqRespConfig } from '../services/reqresp/config.js';
33
33
  import {
34
34
  ReqRespSubProtocol,
35
35
  type ReqRespSubProtocolHandlers,
36
+ type ReqRespSubProtocolRateLimits,
36
37
  type ReqRespSubProtocolValidators,
37
38
  noopValidator,
38
39
  } from '../services/reqresp/interface.js';
@@ -172,8 +173,12 @@ export const MOCK_SUB_PROTOCOL_VALIDATORS: ReqRespSubProtocolValidators = {
172
173
  * @param numberOfNodes - the number of nodes to create
173
174
  * @returns An array of the created nodes
174
175
  */
175
- export const createNodes = (peerScoring: PeerScoring, numberOfNodes: number): Promise<ReqRespNode[]> => {
176
- return timesParallel(numberOfNodes, () => createReqResp(peerScoring));
176
+ export const createNodes = (
177
+ peerScoring: PeerScoring,
178
+ numberOfNodes: number,
179
+ rateLimits: Partial<ReqRespSubProtocolRateLimits> = {},
180
+ ): Promise<ReqRespNode[]> => {
181
+ return timesParallel(numberOfNodes, () => createReqResp(peerScoring, rateLimits));
177
182
  };
178
183
 
179
184
  export const startNodes = async (
@@ -192,17 +197,17 @@ export const stopNodes = async (nodes: ReqRespNode[]): Promise<void> => {
192
197
  };
193
198
 
194
199
  // Create a req resp node, exposing the underlying p2p node
195
- export const createReqResp = async (peerScoring: PeerScoring): Promise<ReqRespNode> => {
200
+ export const createReqResp = async (
201
+ peerScoring: PeerScoring,
202
+ rateLimits: Partial<ReqRespSubProtocolRateLimits> = {},
203
+ ): Promise<ReqRespNode> => {
196
204
  const p2p = await createLibp2pNode();
197
205
  const config: P2PReqRespConfig = {
198
206
  overallRequestTimeoutMs: 4000,
199
207
  individualRequestTimeoutMs: 2000,
200
208
  };
201
- const req = new ReqResp(config, p2p, peerScoring);
202
- return {
203
- p2p,
204
- req,
205
- };
209
+ const req = new ReqResp(config, p2p, peerScoring, rateLimits);
210
+ return { p2p, req };
206
211
  };
207
212
 
208
213
  // Given a node list; hand shake all of the nodes with each other
@@ -49,7 +49,7 @@ function mockTxPool(): TxPool {
49
49
  getTxStatus: () => Promise.resolve(TxStatus.PENDING),
50
50
  getTxsByHash: () => Promise.resolve([]),
51
51
  hasTxs: () => Promise.resolve([]),
52
- setMaxTxPoolSize: () => Promise.resolve(),
52
+ updateConfig: () => {},
53
53
  markTxsAsNonEvictable: () => Promise.resolve(),
54
54
  };
55
55
  }