@aztec/p2p 0.0.1-commit.4ad48494d → 0.0.1-commit.4eabbdb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/dest/client/factory.d.ts +3 -2
  2. package/dest/client/factory.d.ts.map +1 -1
  3. package/dest/client/factory.js +3 -2
  4. package/dest/client/interface.d.ts +7 -9
  5. package/dest/client/interface.d.ts.map +1 -1
  6. package/dest/client/p2p_client.d.ts +4 -7
  7. package/dest/client/p2p_client.d.ts.map +1 -1
  8. package/dest/client/p2p_client.js +38 -14
  9. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +5 -5
  10. package/dest/config.d.ts +3 -2
  11. package/dest/config.d.ts.map +1 -1
  12. package/dest/errors/tx-pool.error.d.ts +8 -0
  13. package/dest/errors/tx-pool.error.d.ts.map +1 -0
  14. package/dest/errors/tx-pool.error.js +9 -0
  15. package/dest/mem_pools/attestation_pool/mocks.d.ts +2 -2
  16. package/dest/mem_pools/attestation_pool/mocks.d.ts.map +1 -1
  17. package/dest/mem_pools/attestation_pool/mocks.js +2 -2
  18. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.js +3 -3
  19. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts +3 -1
  20. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts.map +1 -1
  21. package/dest/mem_pools/tx_pool_v2/deleted_pool.js +9 -0
  22. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts +3 -3
  23. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts.map +1 -1
  24. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.js +18 -9
  25. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.js +2 -2
  26. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts +3 -3
  27. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts.map +1 -1
  28. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.js +10 -4
  29. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts +2 -2
  30. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts.map +1 -1
  31. package/dest/mem_pools/tx_pool_v2/eviction/index.js +1 -1
  32. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts +48 -5
  33. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts.map +1 -1
  34. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.js +8 -0
  35. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.js +4 -4
  36. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.js +3 -3
  37. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts +1 -1
  38. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts.map +1 -1
  39. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.js +6 -4
  40. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts +4 -4
  41. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts.map +1 -1
  42. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.js +14 -4
  43. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts +3 -3
  44. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts.map +1 -1
  45. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.js +2 -2
  46. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts +15 -0
  47. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts.map +1 -0
  48. package/dest/mem_pools/tx_pool_v2/instrumentation.js +43 -0
  49. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +10 -2
  50. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
  51. package/dest/mem_pools/tx_pool_v2/interfaces.js +2 -1
  52. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +7 -5
  53. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -1
  54. package/dest/mem_pools/tx_pool_v2/tx_metadata.js +29 -5
  55. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +5 -2
  56. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
  57. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +12 -2
  58. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +5 -2
  59. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
  60. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +6 -5
  61. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +10 -4
  62. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
  63. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +135 -37
  64. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts +2 -2
  65. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts.map +1 -1
  66. package/dest/msg_validators/tx_validator/timestamp_validator.js +6 -6
  67. package/dest/services/dummy_service.d.ts +3 -2
  68. package/dest/services/dummy_service.d.ts.map +1 -1
  69. package/dest/services/dummy_service.js +3 -0
  70. package/dest/services/encoding.d.ts +1 -1
  71. package/dest/services/encoding.d.ts.map +1 -1
  72. package/dest/services/encoding.js +2 -1
  73. package/dest/services/gossipsub/topic_score_params.d.ts +18 -6
  74. package/dest/services/gossipsub/topic_score_params.d.ts.map +1 -1
  75. package/dest/services/gossipsub/topic_score_params.js +32 -10
  76. package/dest/services/libp2p/libp2p_service.d.ts +2 -1
  77. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  78. package/dest/services/libp2p/libp2p_service.js +7 -3
  79. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts +4 -3
  80. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts.map +1 -1
  81. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js +5 -9
  82. package/dest/services/reqresp/batch-tx-requester/interface.d.ts +2 -6
  83. package/dest/services/reqresp/batch-tx-requester/interface.d.ts.map +1 -1
  84. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts +10 -13
  85. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts.map +1 -1
  86. package/dest/services/reqresp/batch-tx-requester/missing_txs.js +25 -46
  87. package/dest/services/service.d.ts +4 -2
  88. package/dest/services/service.d.ts.map +1 -1
  89. package/dest/services/tx_collection/fast_tx_collection.d.ts +1 -1
  90. package/dest/services/tx_collection/fast_tx_collection.d.ts.map +1 -1
  91. package/dest/services/tx_collection/fast_tx_collection.js +39 -33
  92. package/dest/services/tx_collection/file_store_tx_collection.d.ts +1 -1
  93. package/dest/services/tx_collection/file_store_tx_collection.d.ts.map +1 -1
  94. package/dest/services/tx_collection/file_store_tx_collection.js +4 -2
  95. package/dest/services/tx_collection/file_store_tx_source.d.ts +15 -6
  96. package/dest/services/tx_collection/file_store_tx_source.d.ts.map +1 -1
  97. package/dest/services/tx_collection/file_store_tx_source.js +47 -16
  98. package/dest/services/tx_collection/instrumentation.d.ts +1 -1
  99. package/dest/services/tx_collection/instrumentation.d.ts.map +1 -1
  100. package/dest/services/tx_collection/instrumentation.js +2 -1
  101. package/dest/services/tx_collection/missing_txs_tracker.d.ts +32 -0
  102. package/dest/services/tx_collection/missing_txs_tracker.d.ts.map +1 -0
  103. package/dest/services/tx_collection/missing_txs_tracker.js +27 -0
  104. package/dest/services/tx_collection/proposal_tx_collector.d.ts +7 -6
  105. package/dest/services/tx_collection/proposal_tx_collector.d.ts.map +1 -1
  106. package/dest/services/tx_collection/proposal_tx_collector.js +5 -4
  107. package/dest/services/tx_collection/slow_tx_collection.d.ts +2 -2
  108. package/dest/services/tx_collection/slow_tx_collection.d.ts.map +1 -1
  109. package/dest/services/tx_collection/slow_tx_collection.js +10 -8
  110. package/dest/services/tx_collection/tx_collection.d.ts +5 -4
  111. package/dest/services/tx_collection/tx_collection.d.ts.map +1 -1
  112. package/dest/services/tx_collection/tx_collection_sink.d.ts +6 -5
  113. package/dest/services/tx_collection/tx_collection_sink.d.ts.map +1 -1
  114. package/dest/services/tx_collection/tx_collection_sink.js +13 -22
  115. package/dest/services/tx_collection/tx_source.d.ts +8 -3
  116. package/dest/services/tx_collection/tx_source.d.ts.map +1 -1
  117. package/dest/services/tx_collection/tx_source.js +19 -2
  118. package/dest/services/tx_file_store/tx_file_store.js +1 -1
  119. package/dest/test-helpers/mock-pubsub.d.ts +3 -2
  120. package/dest/test-helpers/mock-pubsub.d.ts.map +1 -1
  121. package/dest/test-helpers/mock-pubsub.js +6 -0
  122. package/dest/test-helpers/testbench-utils.d.ts +5 -2
  123. package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
  124. package/dest/test-helpers/testbench-utils.js +1 -1
  125. package/dest/testbench/p2p_client_testbench_worker.d.ts +2 -2
  126. package/dest/testbench/p2p_client_testbench_worker.d.ts.map +1 -1
  127. package/dest/testbench/p2p_client_testbench_worker.js +8 -8
  128. package/dest/util.d.ts +2 -2
  129. package/dest/util.d.ts.map +1 -1
  130. package/package.json +14 -14
  131. package/src/client/factory.ts +5 -2
  132. package/src/client/interface.ts +14 -9
  133. package/src/client/p2p_client.ts +44 -17
  134. package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +18 -8
  135. package/src/config.ts +1 -1
  136. package/src/errors/tx-pool.error.ts +12 -0
  137. package/src/mem_pools/attestation_pool/mocks.ts +2 -1
  138. package/src/mem_pools/tx_pool/README.md +1 -1
  139. package/src/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.ts +3 -3
  140. package/src/mem_pools/tx_pool_v2/README.md +1 -1
  141. package/src/mem_pools/tx_pool_v2/deleted_pool.ts +11 -0
  142. package/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts +21 -8
  143. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts +2 -2
  144. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts +15 -4
  145. package/src/mem_pools/tx_pool_v2/eviction/index.ts +4 -0
  146. package/src/mem_pools/tx_pool_v2/eviction/interfaces.ts +49 -4
  147. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts +4 -4
  148. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts +3 -3
  149. package/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts +6 -7
  150. package/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts +24 -6
  151. package/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts +3 -3
  152. package/src/mem_pools/tx_pool_v2/instrumentation.ts +69 -0
  153. package/src/mem_pools/tx_pool_v2/interfaces.ts +8 -2
  154. package/src/mem_pools/tx_pool_v2/tx_metadata.ts +40 -9
  155. package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +14 -3
  156. package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +11 -6
  157. package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +157 -29
  158. package/src/msg_validators/tx_validator/timestamp_validator.ts +7 -7
  159. package/src/services/dummy_service.ts +5 -1
  160. package/src/services/encoding.ts +2 -1
  161. package/src/services/gossipsub/README.md +29 -14
  162. package/src/services/gossipsub/topic_score_params.ts +49 -13
  163. package/src/services/libp2p/libp2p_service.ts +7 -2
  164. package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts +6 -6
  165. package/src/services/reqresp/batch-tx-requester/interface.ts +1 -5
  166. package/src/services/reqresp/batch-tx-requester/missing_txs.ts +23 -71
  167. package/src/services/service.ts +10 -1
  168. package/src/services/tx_collection/fast_tx_collection.ts +51 -30
  169. package/src/services/tx_collection/file_store_tx_collection.ts +7 -3
  170. package/src/services/tx_collection/file_store_tx_source.ts +61 -17
  171. package/src/services/tx_collection/instrumentation.ts +7 -1
  172. package/src/services/tx_collection/missing_txs_tracker.ts +52 -0
  173. package/src/services/tx_collection/proposal_tx_collector.ts +8 -7
  174. package/src/services/tx_collection/slow_tx_collection.ts +8 -9
  175. package/src/services/tx_collection/tx_collection.ts +4 -3
  176. package/src/services/tx_collection/tx_collection_sink.ts +15 -29
  177. package/src/services/tx_collection/tx_source.ts +22 -3
  178. package/src/services/tx_file_store/tx_file_store.ts +1 -1
  179. package/src/test-helpers/mock-pubsub.ts +10 -0
  180. package/src/test-helpers/testbench-utils.ts +2 -2
  181. package/src/testbench/p2p_client_testbench_worker.ts +20 -13
  182. package/src/util.ts +7 -1
@@ -341,6 +341,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
341
341
  heartbeatIntervalMs: config.gossipsubInterval,
342
342
  targetCommitteeSize: l1Constants.targetCommitteeSize,
343
343
  blockDurationMs: config.blockDurationMs,
344
+ expectedBlockProposalsPerSlot: config.expectedBlockProposalsPerSlot,
344
345
  });
345
346
 
346
347
  const node = await createLibp2p({
@@ -614,6 +615,10 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
614
615
  return this.peerManager.getPeers(includePending);
615
616
  }
616
617
 
618
+ public getGossipMeshPeerCount(topicType: TopicType): number {
619
+ return this.node.services.pubsub.getMeshPeers(this.topicStrings[topicType]).length;
620
+ }
621
+
617
622
  private handleGossipSubEvent(e: CustomEvent<GossipsubMessage>) {
618
623
  this.logger.trace(`Received PUBSUB message.`);
619
624
 
@@ -1544,7 +1549,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
1544
1549
  protected async validatePropagatedTx(tx: Tx, peerId: PeerId): Promise<boolean> {
1545
1550
  const currentBlockNumber = await this.archiver.getBlockNumber();
1546
1551
 
1547
- // We accept transactions if they are not expired by the next slot (checked based on the IncludeByTimestamp field)
1552
+ // We accept transactions if they are not expired by the next slot (checked based on the ExpirationTimestamp field)
1548
1553
  const { ts: nextSlotTimestamp } = this.epochCache.getEpochAndSlotInNextL1Slot();
1549
1554
  const messageValidators = await this.createMessageValidators(currentBlockNumber, nextSlotTimestamp);
1550
1555
 
@@ -1599,7 +1604,7 @@ export class LibP2PService<T extends P2PClientType = P2PClientType.Full> extends
1599
1604
  public async validate(txs: Tx[]): Promise<void> {
1600
1605
  const currentBlockNumber = await this.archiver.getBlockNumber();
1601
1606
 
1602
- // We accept transactions if they are not expired by the next slot (checked based on the IncludeByTimestamp field)
1607
+ // We accept transactions if they are not expired by the next slot (checked based on the ExpirationTimestamp field)
1603
1608
  const { ts: nextSlotTimestamp } = this.epochCache.getEpochAndSlotInNextL1Slot();
1604
1609
  const messageValidators = await this.createMessageValidators(currentBlockNumber, nextSlotTimestamp);
1605
1610
 
@@ -10,6 +10,7 @@ import { Tx, TxArray, TxHash } from '@aztec/stdlib/tx';
10
10
  import type { PeerId } from '@libp2p/interface';
11
11
  import { peerIdFromString } from '@libp2p/peer-id';
12
12
 
13
+ import type { IMissingTxsTracker } from '../../tx_collection/missing_txs_tracker.js';
13
14
  import { ReqRespSubProtocol } from '.././interface.js';
14
15
  import { BlockTxsRequest, BlockTxsResponse, type BlockTxsSource } from '.././protocols/index.js';
15
16
  import { ReqRespStatus } from '.././status.js';
@@ -20,7 +21,7 @@ import {
20
21
  DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE,
21
22
  } from './config.js';
22
23
  import type { BatchTxRequesterLibP2PService, BatchTxRequesterOptions, ITxMetadataCollection } from './interface.js';
23
- import { MissingTxMetadata, MissingTxMetadataCollection } from './missing_txs.js';
24
+ import { MissingTxMetadataCollection } from './missing_txs.js';
24
25
  import { type IPeerCollection, PeerCollection } from './peer_collection.js';
25
26
  import { BatchRequestTxValidator, type IBatchRequestTxValidator } from './tx_validator.js';
26
27
 
@@ -60,7 +61,7 @@ export class BatchTxRequester {
60
61
  private readonly txBatchSize: number;
61
62
 
62
63
  constructor(
63
- missingTxs: TxHash[],
64
+ missingTxsTracker: IMissingTxsTracker,
64
65
  blockTxsSource: BlockTxsSource,
65
66
  pinnedPeer: PeerId | undefined,
66
67
  timeoutMs: number,
@@ -99,8 +100,7 @@ export class BatchTxRequester {
99
100
  this.p2pService.peerScoring,
100
101
  );
101
102
  }
102
- const entries: Array<[string, MissingTxMetadata]> = missingTxs.map(h => [h.toString(), new MissingTxMetadata(h)]);
103
- this.txsMetadata = new MissingTxMetadataCollection(entries, this.txBatchSize);
103
+ this.txsMetadata = new MissingTxMetadataCollection(missingTxsTracker, this.txBatchSize);
104
104
  this.smartRequesterSemaphore = this.opts.semaphore ?? new Semaphore(0);
105
105
  }
106
106
 
@@ -661,7 +661,7 @@ export class BatchTxRequester {
661
661
  /*
662
662
  * @returns true if all missing txs have been fetched */
663
663
  private fetchedAllTxs() {
664
- return Array.from(this.txsMetadata.values()).every(tx => tx.fetched);
664
+ return this.txsMetadata.getMissingTxHashes().size == 0;
665
665
  }
666
666
 
667
667
  /*
@@ -679,7 +679,7 @@ export class BatchTxRequester {
679
679
  this.unlockSmartRequesterSemaphores();
680
680
  }
681
681
 
682
- return aborted || this.txsMetadata.size === 0 || this.fetchedAllTxs() || this.dateProvider.now() > this.deadline;
682
+ return aborted || this.fetchedAllTxs() || this.dateProvider.now() > this.deadline;
683
683
  }
684
684
 
685
685
  /*
@@ -6,7 +6,6 @@ import type { PeerId } from '@libp2p/interface';
6
6
 
7
7
  import type { ConnectionSampler } from '../connection-sampler/connection_sampler.js';
8
8
  import type { ReqRespInterface } from '../interface.js';
9
- import type { MissingTxMetadata } from './missing_txs.js';
10
9
  import type { IPeerCollection } from './peer_collection.js';
11
10
  import type { BatchRequestTxValidatorConfig, IBatchRequestTxValidator } from './tx_validator.js';
12
11
 
@@ -15,18 +14,15 @@ export interface IPeerPenalizer {
15
14
  }
16
15
 
17
16
  export interface ITxMetadataCollection {
18
- size: number;
19
- values(): IterableIterator<MissingTxMetadata>;
20
17
  getMissingTxHashes(): Set<string>;
18
+ markFetched(peerId: PeerId, tx: Tx): boolean;
21
19
  getTxsToRequestFromThePeer(peer: PeerId): TxHash[];
22
20
  markRequested(txHash: TxHash): void;
23
21
  markInFlightBySmartPeer(txHash: TxHash): void;
24
22
  markNotInFlightBySmartPeer(txHash: TxHash): void;
25
23
  alreadyFetched(txHash: TxHash): boolean;
26
24
  // Returns true if tx was marked as fetched, false if it was already marked as fetched
27
- markFetched(peerId: PeerId, tx: Tx): boolean;
28
25
  markPeerHas(peerId: PeerId, txHashes: TxHash[]): void;
29
- getFetchedTxs(): Tx[];
30
26
  }
31
27
 
32
28
  /**
@@ -2,13 +2,13 @@ import { type Tx, TxHash } from '@aztec/stdlib/tx';
2
2
 
3
3
  import type { PeerId } from '@libp2p/interface';
4
4
 
5
+ import type { IMissingTxsTracker } from '../../tx_collection/missing_txs_tracker.js';
5
6
  import { DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE } from './config.js';
6
7
  import type { ITxMetadataCollection } from './interface.js';
7
8
 
8
- export class MissingTxMetadata {
9
+ class MissingTxMetadata {
9
10
  constructor(
10
- public readonly txHash: TxHash,
11
- public fetched = false,
11
+ public readonly txHash: string,
12
12
  public requestedCount = 0,
13
13
  public inFlightCount = 0,
14
14
  public tx: Tx | undefined = undefined,
@@ -30,24 +30,6 @@ export class MissingTxMetadata {
30
30
  public isInFlight(): boolean {
31
31
  return this.inFlightCount > 0;
32
32
  }
33
-
34
- //Returns true if this is the first time we mark it as fetched
35
- public markAsFetched(peerId: PeerId, tx: Tx): boolean {
36
- if (this.fetched) {
37
- return false;
38
- }
39
-
40
- this.fetched = true;
41
- this.tx = tx;
42
-
43
- this.peers.add(peerId.toString());
44
-
45
- return true;
46
- }
47
-
48
- public toString() {
49
- return this.txHash.toString();
50
- }
51
33
  }
52
34
 
53
35
  /*
@@ -55,21 +37,18 @@ export class MissingTxMetadata {
55
37
  * This could be better optimized but given expected count of missing txs (N < 100)
56
38
  * At the moment there is no need for it. And benefit is that we have everything in single store
57
39
  * */
58
- export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata> implements ITxMetadataCollection {
40
+ export class MissingTxMetadataCollection implements ITxMetadataCollection {
41
+ private txMetadata = new Map<string, MissingTxMetadata>();
42
+
59
43
  constructor(
60
- entries?: readonly (readonly [string, MissingTxMetadata])[] | null,
44
+ private missingTxsTracker: IMissingTxsTracker,
61
45
  private readonly txBatchSize: number = DEFAULT_BATCH_TX_REQUESTER_TX_BATCH_SIZE,
62
46
  ) {
63
- super(entries);
64
- }
65
- public getSortedByRequestedCountAsc(txs: string[]): MissingTxMetadata[] {
66
- return Array.from(this.values().filter(txMeta => txs.includes(txMeta.txHash.toString()))).sort(
67
- (a, b) => a.requestedCount - b.requestedCount,
68
- );
47
+ missingTxsTracker.missingTxHashes.forEach(hash => this.txMetadata.set(hash, new MissingTxMetadata(hash)));
69
48
  }
70
49
 
71
50
  public getPrioritizingNotInFlightAndLowerRequestCount(txs: string[]): MissingTxMetadata[] {
72
- const filtered = Array.from(this.values()).filter(txMeta => txs.includes(txMeta.txHash.toString()));
51
+ const filtered = Array.from(this.txMetadata.values()).filter(txMeta => txs.includes(txMeta.txHash.toString()));
73
52
 
74
53
  const [notInFlight, inFlight] = filtered.reduce<[MissingTxMetadata[], MissingTxMetadata[]]>(
75
54
  (buckets, tx) => {
@@ -85,43 +64,15 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
85
64
  return [...notInFlight, ...inFlight];
86
65
  }
87
66
 
88
- public getFetchedTxHashes(): Set<string> {
89
- return new Set(
90
- this.values()
91
- .filter(t => t.fetched)
92
- .map(t => t.txHash.toString()),
93
- );
94
- }
95
-
96
67
  public getMissingTxHashes(): Set<string> {
97
- return new Set(
98
- this.values()
99
- .filter(t => !t.fetched)
100
- .map(t => t.txHash.toString()),
101
- );
102
- }
103
-
104
- public getInFlightTxHashes(): Set<string> {
105
- return new Set(
106
- this.values()
107
- .filter(t => t.isInFlight())
108
- .map(t => t.txHash.toString()),
109
- );
110
- }
111
-
112
- public getFetchedTxs(): Tx[] {
113
- return Array.from(
114
- this.values()
115
- .map(t => t.tx)
116
- .filter(t => !!t),
117
- );
68
+ return this.missingTxsTracker.missingTxHashes;
118
69
  }
119
70
 
120
71
  public getTxsPeerHas(peer: PeerId): Set<string> {
121
72
  const peerIdStr = peer.toString();
122
73
  const txsPeerHas = new Set<string>();
123
74
 
124
- this.values().forEach(txMeta => {
75
+ this.txMetadata.values().forEach(txMeta => {
125
76
  if (txMeta.peers.has(peerIdStr)) {
126
77
  txsPeerHas.add(txMeta.txHash.toString());
127
78
  }
@@ -132,13 +83,13 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
132
83
 
133
84
  public getTxsToRequestFromThePeer(peer: PeerId): TxHash[] {
134
85
  const txsPeerHas = this.getTxsPeerHas(peer);
135
- const fetchedTxs = this.getFetchedTxHashes();
86
+ const missingTxHashes = this.getMissingTxHashes();
136
87
 
137
- const txsToRequest = txsPeerHas.difference(fetchedTxs);
88
+ const txsToRequest = txsPeerHas.intersection(missingTxHashes);
138
89
 
139
90
  if (txsToRequest.size >= this.txBatchSize) {
140
91
  return this.getPrioritizingNotInFlightAndLowerRequestCount(Array.from(txsToRequest))
141
- .map(t => t.txHash)
92
+ .map(t => TxHash.fromString(t.txHash))
142
93
  .slice(0, this.txBatchSize);
143
94
  }
144
95
 
@@ -150,13 +101,13 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
150
101
  Array.from(this.getMissingTxHashes().difference(txsToRequest)),
151
102
  )
152
103
  .slice(0, countToFill)
153
- .map(t => t.txHash);
104
+ .map(t => TxHash.fromString(t.txHash));
154
105
 
155
106
  return [...Array.from(txsToRequest).map(t => TxHash.fromString(t)), ...txsToFill];
156
107
  }
157
108
 
158
109
  public markRequested(txHash: TxHash) {
159
- this.get(txHash.toString())?.markAsRequested();
110
+ this.txMetadata.get(txHash.toString())?.markAsRequested();
160
111
  }
161
112
 
162
113
  /*
@@ -165,7 +116,7 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
165
116
  * "dumb" peer might return it, or might not - we don't know
166
117
  * */
167
118
  public markInFlightBySmartPeer(txHash: TxHash) {
168
- this.get(txHash.toString())?.markInFlight();
119
+ this.txMetadata.get(txHash.toString())?.markInFlight();
169
120
  }
170
121
 
171
122
  /*
@@ -173,16 +124,16 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
173
124
  * Because the smart peer should return this tx, whereas
174
125
  * "dumb" peer might return it, or might not - we don't know*/
175
126
  public markNotInFlightBySmartPeer(txHash: TxHash) {
176
- this.get(txHash.toString())?.markNotInFlight();
127
+ this.txMetadata.get(txHash.toString())?.markNotInFlight();
177
128
  }
178
129
 
179
130
  public alreadyFetched(txHash: TxHash): boolean {
180
- return this.get(txHash.toString())?.fetched ?? false;
131
+ return !this.missingTxsTracker.isMissing(txHash.toString());
181
132
  }
182
133
 
183
134
  public markFetched(peerId: PeerId, tx: Tx): boolean {
184
135
  const txHashStr = tx.txHash.toString();
185
- const txMeta = this.get(txHashStr);
136
+ const txMeta = this.txMetadata.get(txHashStr);
186
137
  if (!txMeta) {
187
138
  //TODO: what to do about peer which sent txs we didn't request?
188
139
  // 1. don't request from it in the scope of this batch request
@@ -192,7 +143,8 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
192
143
  return false;
193
144
  }
194
145
 
195
- return txMeta.markAsFetched(peerId, tx);
146
+ txMeta.peers.add(peerId.toString());
147
+ return this.missingTxsTracker.markFetched(tx);
196
148
  }
197
149
 
198
150
  public markPeerHas(peerId: PeerId, txHash: TxHash[]) {
@@ -200,7 +152,7 @@ export class MissingTxMetadataCollection extends Map<string, MissingTxMetadata>
200
152
  txHash
201
153
  .map(t => t.toString())
202
154
  .forEach(txh => {
203
- const txMeta = this.get(txh);
155
+ const txMeta = this.txMetadata.get(txh);
204
156
  if (txMeta) {
205
157
  txMeta.peers.add(peerIdStr);
206
158
  }
@@ -1,7 +1,13 @@
1
1
  import type { SlotNumber } from '@aztec/foundation/branded-types';
2
2
  import type { EthAddress } from '@aztec/foundation/eth-address';
3
3
  import type { PeerInfo } from '@aztec/stdlib/interfaces/server';
4
- import type { BlockProposal, CheckpointAttestation, CheckpointProposalCore, Gossipable } from '@aztec/stdlib/p2p';
4
+ import type {
5
+ BlockProposal,
6
+ CheckpointAttestation,
7
+ CheckpointProposalCore,
8
+ Gossipable,
9
+ TopicType,
10
+ } from '@aztec/stdlib/p2p';
5
11
  import type { Tx } from '@aztec/stdlib/tx';
6
12
 
7
13
  import type { PeerId } from '@libp2p/interface';
@@ -130,6 +136,9 @@ export interface P2PService {
130
136
 
131
137
  getPeers(includePending?: boolean): PeerInfo[];
132
138
 
139
+ /** Returns the number of peers in the GossipSub mesh for a given topic type. */
140
+ getGossipMeshPeerCount(topicType: TopicType): number;
141
+
133
142
  validate(txs: Tx[]): Promise<void>;
134
143
 
135
144
  addReqRespSubProtocol(
@@ -14,6 +14,7 @@ import type { PeerId } from '@libp2p/interface';
14
14
  import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
15
15
  import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
16
16
  import type { TxCollectionConfig } from './config.js';
17
+ import { MissingTxsTracker } from './missing_txs_tracker.js';
17
18
  import {
18
19
  BatchTxRequesterCollector,
19
20
  type MissingTxsCollector,
@@ -83,8 +84,7 @@ export class FastTxCollection {
83
84
  ...input,
84
85
  blockInfo,
85
86
  promise,
86
- foundTxs: new Map<string, Tx>(),
87
- missingTxHashes: new Set(txHashes.map(t => t.toString())),
87
+ missingTxTracker: MissingTxsTracker.fromArray(txHashes),
88
88
  deadline: opts.deadline,
89
89
  };
90
90
 
@@ -92,15 +92,15 @@ export class FastTxCollection {
92
92
  clearTimeout(timeoutTimer);
93
93
 
94
94
  this.log.verbose(
95
- `Collected ${request.foundTxs.size} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
95
+ `Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
96
96
  {
97
97
  ...blockInfo,
98
98
  duration,
99
99
  requestType: input.type,
100
- missingTxs: [...request.missingTxHashes],
100
+ missingTxs: [...request.missingTxTracker.missingTxHashes],
101
101
  },
102
102
  );
103
- return [...request.foundTxs.values()];
103
+ return request.missingTxTracker.collectedTxs;
104
104
  }
105
105
 
106
106
  protected async collectFast(
@@ -111,7 +111,7 @@ export class FastTxCollection {
111
111
  const { blockInfo } = request;
112
112
 
113
113
  this.log.debug(
114
- `Starting fast collection of ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
114
+ `Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
115
115
  { ...blockInfo, requestType: request.type, deadline: opts.deadline },
116
116
  );
117
117
 
@@ -124,7 +124,7 @@ export class FastTxCollection {
124
124
  await Promise.race([request.promise.promise, waitBeforeReqResp]);
125
125
 
126
126
  // If we have collected all txs, we can stop here
127
- if (request.missingTxHashes.size === 0) {
127
+ if (request.missingTxTracker.allFetched()) {
128
128
  this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
129
129
  return;
130
130
  }
@@ -138,7 +138,7 @@ export class FastTxCollection {
138
138
  const logCtx = {
139
139
  ...blockInfo,
140
140
  errorMessage: err instanceof Error ? err.message : undefined,
141
- missingTxs: [...request.missingTxHashes].map(txHash => txHash.toString()),
141
+ missingTxs: request.missingTxTracker.missingTxHashes.values().map(txHash => txHash.toString()),
142
142
  };
143
143
  if (err instanceof Error && err.name === 'TimeoutError') {
144
144
  this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
@@ -166,7 +166,11 @@ export class FastTxCollection {
166
166
  }
167
167
 
168
168
  // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
169
- const attemptsPerTx = [...request.missingTxHashes].map(txHash => ({ txHash, attempts: 0, found: false }));
169
+ const attemptsPerTx = [...request.missingTxTracker.missingTxHashes].map(txHash => ({
170
+ txHash,
171
+ attempts: 0,
172
+ found: false,
173
+ }));
170
174
 
171
175
  // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
172
176
  await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx, opts)));
@@ -179,7 +183,7 @@ export class FastTxCollection {
179
183
  opts: { deadline: Date },
180
184
  ) {
181
185
  const notFinished = () =>
182
- this.dateProvider.now() <= +opts.deadline && request.missingTxHashes.size > 0 && this.requests.has(request);
186
+ this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
183
187
 
184
188
  const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
185
189
  const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
@@ -196,7 +200,7 @@ export class FastTxCollection {
196
200
  if (!txToRequest) {
197
201
  // No more txs to process
198
202
  break;
199
- } else if (!request.missingTxHashes.has(txToRequest.txHash)) {
203
+ } else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
200
204
  // Mark as found if it was found somewhere else, we'll then remove it from the array.
201
205
  // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
202
206
  txToRequest.found = true;
@@ -225,10 +229,17 @@ export class FastTxCollection {
225
229
  return;
226
230
  }
227
231
 
232
+ const txHashes = batch.map(({ txHash }) => txHash);
228
233
  // Collect this batch from the node
229
234
  await this.txCollectionSink.collect(
230
- txHashes => node.getTxsByHash(txHashes),
231
- batch.map(({ txHash }) => TxHash.fromString(txHash)),
235
+ async () => {
236
+ const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
237
+ for (const tx of result.validTxs) {
238
+ request.missingTxTracker.markFetched(tx);
239
+ }
240
+ return result;
241
+ },
242
+ txHashes,
232
243
  {
233
244
  description: `fast ${node.getInfo()}`,
234
245
  node: node.getInfo(),
@@ -268,32 +279,44 @@ export class FastTxCollection {
268
279
  }
269
280
 
270
281
  this.log.debug(
271
- `Starting fast reqresp for ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
282
+ `Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
272
283
  { ...blockInfo, timeoutMs, pinnedPeer },
273
284
  );
274
285
 
275
286
  try {
276
287
  await this.txCollectionSink.collect(
277
- async txHashes => {
288
+ async () => {
289
+ let result: Tx[];
278
290
  if (request.type === 'proposal') {
279
- return await this.missingTxsCollector.collectTxs(txHashes, request.blockProposal, pinnedPeer, timeoutMs);
291
+ result = await this.missingTxsCollector.collectTxs(
292
+ request.missingTxTracker,
293
+ request.blockProposal,
294
+ pinnedPeer,
295
+ timeoutMs,
296
+ );
280
297
  } else if (request.type === 'block') {
281
298
  const blockTxsSource = {
282
299
  txHashes: request.block.body.txEffects.map(e => e.txHash),
283
300
  archive: request.block.archive.root,
284
301
  };
285
- return await this.missingTxsCollector.collectTxs(txHashes, blockTxsSource, pinnedPeer, timeoutMs);
302
+ result = await this.missingTxsCollector.collectTxs(
303
+ request.missingTxTracker,
304
+ blockTxsSource,
305
+ pinnedPeer,
306
+ timeoutMs,
307
+ );
286
308
  } else {
287
309
  throw new Error(`Unknown request type: ${(request as any).type}`);
288
310
  }
311
+ return { validTxs: result, invalidTxHashes: [] };
289
312
  },
290
- Array.from(request.missingTxHashes).map(txHash => TxHash.fromString(txHash)),
313
+ Array.from(request.missingTxTracker.missingTxHashes),
291
314
  { description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo },
292
315
  this.getAddContext(request),
293
316
  );
294
317
  } catch (err) {
295
318
  this.log.error(`Error sending fast reqresp request for txs`, err, {
296
- txs: [...request.missingTxHashes],
319
+ txs: [...request.missingTxTracker.missingTxHashes],
297
320
  ...blockInfo,
298
321
  });
299
322
  }
@@ -317,22 +340,20 @@ export class FastTxCollection {
317
340
  for (const tx of txs) {
318
341
  const txHash = tx.txHash.toString();
319
342
  // Remove the tx hash from the missing set, and add it to the found set.
320
- if (request.missingTxHashes.has(txHash)) {
321
- request.missingTxHashes.delete(txHash);
322
- request.foundTxs.set(txHash, tx);
343
+ if (request.missingTxTracker.markFetched(tx)) {
323
344
  this.log.trace(`Found tx ${txHash} for fast collection request`, {
324
345
  ...request.blockInfo,
325
346
  txHash: tx.txHash.toString(),
326
347
  type: request.type,
327
348
  });
328
- // If we found all txs for this request, we resolve the promise
329
- if (request.missingTxHashes.size === 0) {
330
- this.log.trace(`All txs found for fast collection request`, {
331
- ...request.blockInfo,
332
- type: request.type,
333
- });
334
- request.promise.resolve();
335
- }
349
+ }
350
+ // If we found all txs for this request, we resolve the promise
351
+ if (request.missingTxTracker.allFetched()) {
352
+ this.log.trace(`All txs found for fast collection request`, {
353
+ ...request.blockInfo,
354
+ type: request.type,
355
+ });
356
+ request.promise.resolve();
336
357
  }
337
358
  }
338
359
  }
@@ -126,9 +126,13 @@ export class FileStoreTxCollection {
126
126
 
127
127
  try {
128
128
  const result = await this.txCollectionSink.collect(
129
- hashes => source.getTxsByHash(hashes),
130
- [TxHash.fromString(entry.txHash)],
131
- { description: `file-store ${source.getInfo()}`, method: 'file-store', fileStore: source.getInfo() },
129
+ () => source.getTxsByHash([TxHash.fromString(entry.txHash)]),
130
+ [entry.txHash],
131
+ {
132
+ description: `file-store ${source.getInfo()}`,
133
+ method: 'file-store',
134
+ fileStore: source.getInfo(),
135
+ },
132
136
  entry.context,
133
137
  );
134
138
  if (result.txs.length > 0) {