@aztec/p2p 0.0.1-commit.4ad48494d → 0.0.1-commit.4eabbdb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/dest/client/factory.d.ts +3 -2
  2. package/dest/client/factory.d.ts.map +1 -1
  3. package/dest/client/factory.js +3 -2
  4. package/dest/client/interface.d.ts +7 -9
  5. package/dest/client/interface.d.ts.map +1 -1
  6. package/dest/client/p2p_client.d.ts +4 -7
  7. package/dest/client/p2p_client.d.ts.map +1 -1
  8. package/dest/client/p2p_client.js +38 -14
  9. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +5 -5
  10. package/dest/config.d.ts +3 -2
  11. package/dest/config.d.ts.map +1 -1
  12. package/dest/errors/tx-pool.error.d.ts +8 -0
  13. package/dest/errors/tx-pool.error.d.ts.map +1 -0
  14. package/dest/errors/tx-pool.error.js +9 -0
  15. package/dest/mem_pools/attestation_pool/mocks.d.ts +2 -2
  16. package/dest/mem_pools/attestation_pool/mocks.d.ts.map +1 -1
  17. package/dest/mem_pools/attestation_pool/mocks.js +2 -2
  18. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.js +3 -3
  19. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts +3 -1
  20. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts.map +1 -1
  21. package/dest/mem_pools/tx_pool_v2/deleted_pool.js +9 -0
  22. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts +3 -3
  23. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts.map +1 -1
  24. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.js +18 -9
  25. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.js +2 -2
  26. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts +3 -3
  27. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts.map +1 -1
  28. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.js +10 -4
  29. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts +2 -2
  30. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts.map +1 -1
  31. package/dest/mem_pools/tx_pool_v2/eviction/index.js +1 -1
  32. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts +48 -5
  33. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts.map +1 -1
  34. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.js +8 -0
  35. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.js +4 -4
  36. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.js +3 -3
  37. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts +1 -1
  38. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts.map +1 -1
  39. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.js +6 -4
  40. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts +4 -4
  41. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts.map +1 -1
  42. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.js +14 -4
  43. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts +3 -3
  44. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts.map +1 -1
  45. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.js +2 -2
  46. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts +15 -0
  47. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts.map +1 -0
  48. package/dest/mem_pools/tx_pool_v2/instrumentation.js +43 -0
  49. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +10 -2
  50. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
  51. package/dest/mem_pools/tx_pool_v2/interfaces.js +2 -1
  52. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +7 -5
  53. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -1
  54. package/dest/mem_pools/tx_pool_v2/tx_metadata.js +29 -5
  55. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +5 -2
  56. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
  57. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +12 -2
  58. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +5 -2
  59. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
  60. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +6 -5
  61. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +10 -4
  62. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
  63. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +135 -37
  64. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts +2 -2
  65. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts.map +1 -1
  66. package/dest/msg_validators/tx_validator/timestamp_validator.js +6 -6
  67. package/dest/services/dummy_service.d.ts +3 -2
  68. package/dest/services/dummy_service.d.ts.map +1 -1
  69. package/dest/services/dummy_service.js +3 -0
  70. package/dest/services/encoding.d.ts +1 -1
  71. package/dest/services/encoding.d.ts.map +1 -1
  72. package/dest/services/encoding.js +2 -1
  73. package/dest/services/gossipsub/topic_score_params.d.ts +18 -6
  74. package/dest/services/gossipsub/topic_score_params.d.ts.map +1 -1
  75. package/dest/services/gossipsub/topic_score_params.js +32 -10
  76. package/dest/services/libp2p/libp2p_service.d.ts +2 -1
  77. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  78. package/dest/services/libp2p/libp2p_service.js +7 -3
  79. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts +4 -3
  80. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts.map +1 -1
  81. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js +5 -9
  82. package/dest/services/reqresp/batch-tx-requester/interface.d.ts +2 -6
  83. package/dest/services/reqresp/batch-tx-requester/interface.d.ts.map +1 -1
  84. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts +10 -13
  85. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts.map +1 -1
  86. package/dest/services/reqresp/batch-tx-requester/missing_txs.js +25 -46
  87. package/dest/services/service.d.ts +4 -2
  88. package/dest/services/service.d.ts.map +1 -1
  89. package/dest/services/tx_collection/fast_tx_collection.d.ts +1 -1
  90. package/dest/services/tx_collection/fast_tx_collection.d.ts.map +1 -1
  91. package/dest/services/tx_collection/fast_tx_collection.js +39 -33
  92. package/dest/services/tx_collection/file_store_tx_collection.d.ts +1 -1
  93. package/dest/services/tx_collection/file_store_tx_collection.d.ts.map +1 -1
  94. package/dest/services/tx_collection/file_store_tx_collection.js +4 -2
  95. package/dest/services/tx_collection/file_store_tx_source.d.ts +15 -6
  96. package/dest/services/tx_collection/file_store_tx_source.d.ts.map +1 -1
  97. package/dest/services/tx_collection/file_store_tx_source.js +47 -16
  98. package/dest/services/tx_collection/instrumentation.d.ts +1 -1
  99. package/dest/services/tx_collection/instrumentation.d.ts.map +1 -1
  100. package/dest/services/tx_collection/instrumentation.js +2 -1
  101. package/dest/services/tx_collection/missing_txs_tracker.d.ts +32 -0
  102. package/dest/services/tx_collection/missing_txs_tracker.d.ts.map +1 -0
  103. package/dest/services/tx_collection/missing_txs_tracker.js +27 -0
  104. package/dest/services/tx_collection/proposal_tx_collector.d.ts +7 -6
  105. package/dest/services/tx_collection/proposal_tx_collector.d.ts.map +1 -1
  106. package/dest/services/tx_collection/proposal_tx_collector.js +5 -4
  107. package/dest/services/tx_collection/slow_tx_collection.d.ts +2 -2
  108. package/dest/services/tx_collection/slow_tx_collection.d.ts.map +1 -1
  109. package/dest/services/tx_collection/slow_tx_collection.js +10 -8
  110. package/dest/services/tx_collection/tx_collection.d.ts +5 -4
  111. package/dest/services/tx_collection/tx_collection.d.ts.map +1 -1
  112. package/dest/services/tx_collection/tx_collection_sink.d.ts +6 -5
  113. package/dest/services/tx_collection/tx_collection_sink.d.ts.map +1 -1
  114. package/dest/services/tx_collection/tx_collection_sink.js +13 -22
  115. package/dest/services/tx_collection/tx_source.d.ts +8 -3
  116. package/dest/services/tx_collection/tx_source.d.ts.map +1 -1
  117. package/dest/services/tx_collection/tx_source.js +19 -2
  118. package/dest/services/tx_file_store/tx_file_store.js +1 -1
  119. package/dest/test-helpers/mock-pubsub.d.ts +3 -2
  120. package/dest/test-helpers/mock-pubsub.d.ts.map +1 -1
  121. package/dest/test-helpers/mock-pubsub.js +6 -0
  122. package/dest/test-helpers/testbench-utils.d.ts +5 -2
  123. package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
  124. package/dest/test-helpers/testbench-utils.js +1 -1
  125. package/dest/testbench/p2p_client_testbench_worker.d.ts +2 -2
  126. package/dest/testbench/p2p_client_testbench_worker.d.ts.map +1 -1
  127. package/dest/testbench/p2p_client_testbench_worker.js +8 -8
  128. package/dest/util.d.ts +2 -2
  129. package/dest/util.d.ts.map +1 -1
  130. package/package.json +14 -14
  131. package/src/client/factory.ts +5 -2
  132. package/src/client/interface.ts +14 -9
  133. package/src/client/p2p_client.ts +44 -17
  134. package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +18 -8
  135. package/src/config.ts +1 -1
  136. package/src/errors/tx-pool.error.ts +12 -0
  137. package/src/mem_pools/attestation_pool/mocks.ts +2 -1
  138. package/src/mem_pools/tx_pool/README.md +1 -1
  139. package/src/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.ts +3 -3
  140. package/src/mem_pools/tx_pool_v2/README.md +1 -1
  141. package/src/mem_pools/tx_pool_v2/deleted_pool.ts +11 -0
  142. package/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts +21 -8
  143. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts +2 -2
  144. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts +15 -4
  145. package/src/mem_pools/tx_pool_v2/eviction/index.ts +4 -0
  146. package/src/mem_pools/tx_pool_v2/eviction/interfaces.ts +49 -4
  147. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts +4 -4
  148. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts +3 -3
  149. package/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts +6 -7
  150. package/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts +24 -6
  151. package/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts +3 -3
  152. package/src/mem_pools/tx_pool_v2/instrumentation.ts +69 -0
  153. package/src/mem_pools/tx_pool_v2/interfaces.ts +8 -2
  154. package/src/mem_pools/tx_pool_v2/tx_metadata.ts +40 -9
  155. package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +14 -3
  156. package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +11 -6
  157. package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +157 -29
  158. package/src/msg_validators/tx_validator/timestamp_validator.ts +7 -7
  159. package/src/services/dummy_service.ts +5 -1
  160. package/src/services/encoding.ts +2 -1
  161. package/src/services/gossipsub/README.md +29 -14
  162. package/src/services/gossipsub/topic_score_params.ts +49 -13
  163. package/src/services/libp2p/libp2p_service.ts +7 -2
  164. package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts +6 -6
  165. package/src/services/reqresp/batch-tx-requester/interface.ts +1 -5
  166. package/src/services/reqresp/batch-tx-requester/missing_txs.ts +23 -71
  167. package/src/services/service.ts +10 -1
  168. package/src/services/tx_collection/fast_tx_collection.ts +51 -30
  169. package/src/services/tx_collection/file_store_tx_collection.ts +7 -3
  170. package/src/services/tx_collection/file_store_tx_source.ts +61 -17
  171. package/src/services/tx_collection/instrumentation.ts +7 -1
  172. package/src/services/tx_collection/missing_txs_tracker.ts +52 -0
  173. package/src/services/tx_collection/proposal_tx_collector.ts +8 -7
  174. package/src/services/tx_collection/slow_tx_collection.ts +8 -9
  175. package/src/services/tx_collection/tx_collection.ts +4 -3
  176. package/src/services/tx_collection/tx_collection_sink.ts +15 -29
  177. package/src/services/tx_collection/tx_source.ts +22 -3
  178. package/src/services/tx_file_store/tx_file_store.ts +1 -1
  179. package/src/test-helpers/mock-pubsub.ts +10 -0
  180. package/src/test-helpers/testbench-utils.ts +2 -2
  181. package/src/testbench/p2p_client_testbench_worker.ts +20 -13
  182. package/src/util.ts +7 -1
@@ -1,28 +1,51 @@
1
1
  import { type Logger, createLogger } from '@aztec/foundation/log';
2
+ import { Timer } from '@aztec/foundation/timer';
2
3
  import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store';
3
4
  import { Tx, type TxHash } from '@aztec/stdlib/tx';
5
+ import {
6
+ type Histogram,
7
+ Metrics,
8
+ type TelemetryClient,
9
+ type UpDownCounter,
10
+ getTelemetryClient,
11
+ } from '@aztec/telemetry-client';
4
12
 
5
- import type { TxSource } from './tx_source.js';
13
+ import type { TxSource, TxSourceCollectionResult } from './tx_source.js';
6
14
 
7
15
  /** TxSource implementation that downloads txs from a file store. */
8
16
  export class FileStoreTxSource implements TxSource {
17
+ private downloadsSuccess: UpDownCounter;
18
+ private downloadsFailed: UpDownCounter;
19
+ private downloadDuration: Histogram;
20
+ private downloadSize: Histogram;
21
+
9
22
  private constructor(
10
23
  private readonly fileStore: ReadOnlyFileStore,
11
24
  private readonly baseUrl: string,
12
25
  private readonly basePath: string,
13
26
  private readonly log: Logger,
14
- ) {}
27
+ telemetry: TelemetryClient,
28
+ ) {
29
+ const meter = telemetry.getMeter('file-store-tx-source');
30
+ this.downloadsSuccess = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_SUCCESS);
31
+ this.downloadsFailed = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_FAILED);
32
+ this.downloadDuration = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_DURATION);
33
+ this.downloadSize = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_SIZE);
34
+ }
15
35
 
16
36
  /**
17
37
  * Creates a FileStoreTxSource from a URL.
18
38
  * @param url - The file store URL (s3://, gs://, file://, http://, https://).
39
+ * @param basePath - Base path for tx files within the store.
19
40
  * @param log - Optional logger.
41
+ * @param telemetry - Optional telemetry client.
20
42
  * @returns The FileStoreTxSource instance, or undefined if creation fails.
21
43
  */
22
44
  public static async create(
23
45
  url: string,
24
46
  basePath: string,
25
47
  log: Logger = createLogger('p2p:file_store_tx_source'),
48
+ telemetry: TelemetryClient = getTelemetryClient(),
26
49
  ): Promise<FileStoreTxSource | undefined> {
27
50
  try {
28
51
  const fileStore = await createReadOnlyFileStore(url, log);
@@ -30,7 +53,7 @@ export class FileStoreTxSource implements TxSource {
30
53
  log.warn(`Failed to create file store for URL: ${url}`);
31
54
  return undefined;
32
55
  }
33
- return new FileStoreTxSource(fileStore, url, basePath, log);
56
+ return new FileStoreTxSource(fileStore, url, basePath, log, telemetry);
34
57
  } catch (err) {
35
58
  log.warn(`Error creating file store for URL: ${url}`, { error: err });
36
59
  return undefined;
@@ -41,33 +64,54 @@ export class FileStoreTxSource implements TxSource {
41
64
  return `file-store:${this.baseUrl}`;
42
65
  }
43
66
 
44
- public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
45
- return Promise.all(
46
- txHashes.map(async txHash => {
47
- const path = `${this.basePath}/txs/${txHash.toString()}.bin`;
48
- try {
49
- const buffer = await this.fileStore.read(path);
50
- return Tx.fromBuffer(buffer);
51
- } catch {
52
- // Tx not found or error reading - return undefined
53
- return undefined;
54
- }
55
- }),
56
- );
67
+ public async getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult> {
68
+ const invalidTxHashes: string[] = [];
69
+ return {
70
+ validTxs: (
71
+ await Promise.all(
72
+ txHashes.map(async txHash => {
73
+ const path = `${this.basePath}/txs/${txHash.toString()}.bin`;
74
+ const timer = new Timer();
75
+ try {
76
+ const buffer = await this.fileStore.read(path);
77
+ const tx = Tx.fromBuffer(buffer);
78
+ if ((await tx.validateTxHash()) && txHash.equals(tx.txHash)) {
79
+ this.downloadsSuccess.add(1);
80
+ this.downloadDuration.record(Math.ceil(timer.ms()));
81
+ this.downloadSize.record(buffer.length);
82
+ return tx;
83
+ } else {
84
+ invalidTxHashes.push(tx.txHash.toString());
85
+ this.downloadsFailed.add(1);
86
+ return undefined;
87
+ }
88
+ } catch {
89
+ // Tx not found or error reading - return undefined
90
+ this.downloadsFailed.add(1);
91
+ return undefined;
92
+ }
93
+ }),
94
+ )
95
+ ).filter(tx => tx !== undefined),
96
+ invalidTxHashes: invalidTxHashes,
97
+ };
57
98
  }
58
99
  }
59
100
 
60
101
  /**
61
102
  * Creates FileStoreTxSource instances from URLs.
62
103
  * @param urls - Array of file store URLs.
104
+ * @param basePath - Base path for tx files within each store.
63
105
  * @param log - Optional logger.
106
+ * @param telemetry - Optional telemetry client.
64
107
  * @returns Array of successfully created FileStoreTxSource instances.
65
108
  */
66
109
  export async function createFileStoreTxSources(
67
110
  urls: string[],
68
111
  basePath: string,
69
112
  log: Logger = createLogger('p2p:file_store_tx_source'),
113
+ telemetry: TelemetryClient = getTelemetryClient(),
70
114
  ): Promise<FileStoreTxSource[]> {
71
- const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, basePath, log)));
115
+ const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, basePath, log, telemetry)));
72
116
  return sources.filter((s): s is FileStoreTxSource => s !== undefined);
73
117
  }
@@ -18,7 +18,13 @@ export class TxCollectionInstrumentation {
18
18
  const meter = client.getMeter(name);
19
19
 
20
20
  this.txsCollected = createUpDownCounterWithDefault(meter, Metrics.TX_COLLECTOR_COUNT, {
21
- [Attributes.TX_COLLECTION_METHOD]: ['fast-req-resp', 'fast-node-rpc', 'slow-req-resp', 'slow-node-rpc'],
21
+ [Attributes.TX_COLLECTION_METHOD]: [
22
+ 'fast-req-resp',
23
+ 'fast-node-rpc',
24
+ 'slow-req-resp',
25
+ 'slow-node-rpc',
26
+ 'file-store',
27
+ ],
22
28
  });
23
29
 
24
30
  this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX);
@@ -0,0 +1,52 @@
1
+ import { TxHash } from '@aztec/stdlib/tx';
2
+ import type { Tx } from '@aztec/stdlib/tx';
3
+
4
+ /**
5
+ * Tracks which transactions are still missing and need to be fetched.
6
+ * Allows external code to mark transactions as fetched, enabling coordination
7
+ * between multiple fetching mechanisms (e.g., BatchTxRequester and Rpc Node requests).
8
+ */
9
+ export interface IMissingTxsTracker {
10
+ /** Returns the set of transaction hashes that are still missing. */
11
+ get missingTxHashes(): Set<string>;
12
+ /** Size of this.missingTxHashes */
13
+ get numberOfMissingTxs(): number;
14
+ /** Are all requested txs are fetched */
15
+ allFetched(): boolean;
16
+ /** Checks that transaction is still missing */
17
+ isMissing(txHash: string): boolean;
18
+ /** Marks a transaction as fetched. Returns true if it was previously missing. */
19
+ markFetched(tx: Tx): boolean;
20
+ /** Get list of collected txs */
21
+ get collectedTxs(): Tx[];
22
+ }
23
+
24
+ export class MissingTxsTracker implements IMissingTxsTracker {
25
+ public readonly collectedTxs: Tx[] = [];
26
+
27
+ private constructor(public readonly missingTxHashes: Set<string>) {}
28
+
29
+ public static fromArray(hashes: TxHash[] | string[]) {
30
+ return new MissingTxsTracker(new Set(hashes.map(hash => hash.toString())));
31
+ }
32
+
33
+ markFetched(tx: Tx): boolean {
34
+ if (this.missingTxHashes.delete(tx.txHash.toString())) {
35
+ this.collectedTxs.push(tx);
36
+ return true;
37
+ }
38
+ return false;
39
+ }
40
+
41
+ get numberOfMissingTxs(): number {
42
+ return this.missingTxHashes.size;
43
+ }
44
+
45
+ allFetched(): boolean {
46
+ return this.numberOfMissingTxs === 0;
47
+ }
48
+
49
+ isMissing(txHash: string): boolean {
50
+ return this.missingTxHashes.has(txHash.toString());
51
+ }
52
+ }
@@ -1,6 +1,6 @@
1
1
  import type { Logger } from '@aztec/foundation/log';
2
2
  import type { DateProvider } from '@aztec/foundation/timer';
3
- import type { Tx, TxHash } from '@aztec/stdlib/tx';
3
+ import { type Tx, TxHash } from '@aztec/stdlib/tx';
4
4
 
5
5
  import type { PeerId } from '@libp2p/interface';
6
6
 
@@ -9,6 +9,7 @@ import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/confi
9
9
  import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
10
10
  import type { IBatchRequestTxValidator } from '../reqresp/batch-tx-requester/tx_validator.js';
11
11
  import { type BlockTxsSource, ReqRespSubProtocol, chunkTxHashesRequest } from '../reqresp/index.js';
12
+ import type { IMissingTxsTracker } from './missing_txs_tracker.js';
12
13
 
13
14
  /**
14
15
  * Strategy interface for collecting missing transactions for a block or proposal.
@@ -17,14 +18,14 @@ import { type BlockTxsSource, ReqRespSubProtocol, chunkTxHashesRequest } from '.
17
18
  export interface MissingTxsCollector {
18
19
  /**
19
20
  * Collect missing transactions for a block or proposal.
20
- * @param txHashes - The transaction hashes to collect
21
+ * @param missingTxsTracker - The missing transactions tracker
21
22
  * @param blockTxsSource - The block or proposal containing the transactions
22
23
  * @param pinnedPeer - Optional peer expected to have the transactions
23
24
  * @param timeoutMs - Timeout in milliseconds
24
25
  * @returns The collected transactions
25
26
  */
26
27
  collectTxs(
27
- txHashes: TxHash[],
28
+ missingTxsTracker: IMissingTxsTracker,
28
29
  blockTxsSource: BlockTxsSource,
29
30
  pinnedPeer: PeerId | undefined,
30
31
  timeoutMs: number,
@@ -45,7 +46,7 @@ export class BatchTxRequesterCollector implements MissingTxsCollector {
45
46
  ) {}
46
47
 
47
48
  async collectTxs(
48
- txHashes: TxHash[],
49
+ missingTxsTracker: IMissingTxsTracker,
49
50
  blockTxsSource: BlockTxsSource,
50
51
  pinnedPeer: PeerId | undefined,
51
52
  timeoutMs: number,
@@ -58,7 +59,7 @@ export class BatchTxRequesterCollector implements MissingTxsCollector {
58
59
  } = this.batchTxRequesterConfig ?? {};
59
60
 
60
61
  const batchRequester = new BatchTxRequester(
61
- txHashes,
62
+ missingTxsTracker,
62
63
  blockTxsSource,
63
64
  pinnedPeer,
64
65
  timeoutMs,
@@ -93,14 +94,14 @@ export class SendBatchRequestCollector implements MissingTxsCollector {
93
94
  ) {}
94
95
 
95
96
  async collectTxs(
96
- txHashes: TxHash[],
97
+ missingTxsTracker: IMissingTxsTracker,
97
98
  _blockTxsSource: BlockTxsSource,
98
99
  pinnedPeer: PeerId | undefined,
99
100
  timeoutMs: number,
100
101
  ): Promise<Tx[]> {
101
102
  const txs = await this.p2pService.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
102
103
  ReqRespSubProtocol.TX,
103
- chunkTxHashesRequest(txHashes),
104
+ chunkTxHashesRequest(Array.from(missingTxsTracker.missingTxHashes).map(TxHash.fromString)),
104
105
  pinnedPeer,
105
106
  timeoutMs,
106
107
  this.maxPeers,
@@ -8,8 +8,7 @@ import type { L2Block } from '@aztec/stdlib/block';
8
8
  import { type L1RollupConstants, getEpochAtSlot, getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
9
9
  import { type Tx, TxHash } from '@aztec/stdlib/tx';
10
10
 
11
- import { type ReqRespInterface, ReqRespSubProtocol } from '../reqresp/interface.js';
12
- import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
11
+ import { type ReqRespInterface, ReqRespSubProtocol, chunkTxHashesRequest } from '../reqresp/index.js';
13
12
  import type { TxCollectionConfig } from './config.js';
14
13
  import type { FastTxCollection } from './fast_tx_collection.js';
15
14
  import type { MissingTxInfo } from './tx_collection.js';
@@ -120,8 +119,8 @@ export class SlowTxCollection {
120
119
  const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
121
120
  for (const batch of chunk(txHashes, this.config.txCollectionNodeRpcMaxBatchSize)) {
122
121
  await this.txCollectionSink.collect(
123
- hashes => node.getTxsByHash(hashes),
124
- batch,
122
+ () => node.getTxsByHash(batch),
123
+ batch.map(h => h.toString()),
125
124
  {
126
125
  description: `node ${node.getInfo()}`,
127
126
  node: node.getInfo(),
@@ -166,18 +165,18 @@ export class SlowTxCollection {
166
165
  const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
167
166
  const maxPeers = boundInclusive(Math.ceil(txHashes.length / 3), 4, 16);
168
167
  await this.txCollectionSink.collect(
169
- async hashes => {
168
+ async () => {
170
169
  const txs = await this.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
171
170
  ReqRespSubProtocol.TX,
172
- chunkTxHashesRequest(hashes),
171
+ chunkTxHashesRequest(txHashes),
173
172
  pinnedPeer,
174
173
  timeoutMs,
175
174
  maxPeers,
176
175
  maxRetryAttempts,
177
176
  );
178
- return txs.flat();
177
+ return { validTxs: txs.flat(), invalidTxHashes: [] };
179
178
  },
180
- txHashes,
179
+ txHashes.map(h => h.toString()),
181
180
  { description: 'slow reqresp', timeoutMs, method: 'slow-req-resp' },
182
181
  { type: 'mined', block },
183
182
  );
@@ -197,7 +196,7 @@ export class SlowTxCollection {
197
196
  // from mined unproven blocks it has seen in the past.
198
197
  const fastRequests = this.fastCollection.getFastCollectionRequests();
199
198
  const fastCollectionTxs: Set<string> = new Set(
200
- ...Array.from(fastRequests.values()).flatMap(r => r.missingTxHashes),
199
+ fastRequests.values().flatMap(r => Array.from(r.missingTxTracker.missingTxHashes)),
201
200
  );
202
201
 
203
202
  // Return all missing txs that are not in fastCollectionTxs and are ready for reqresp if requested
@@ -7,7 +7,8 @@ import { DateProvider } from '@aztec/foundation/timer';
7
7
  import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
8
8
  import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
9
9
  import type { BlockProposal } from '@aztec/stdlib/p2p';
10
- import { Tx, TxHash } from '@aztec/stdlib/tx';
10
+ import type { Tx } from '@aztec/stdlib/tx';
11
+ import { TxHash } from '@aztec/stdlib/tx';
11
12
  import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
12
13
 
13
14
  import type { PeerId } from '@libp2p/interface';
@@ -18,6 +19,7 @@ import type { TxCollectionConfig } from './config.js';
18
19
  import { FastTxCollection } from './fast_tx_collection.js';
19
20
  import { FileStoreTxCollection } from './file_store_tx_collection.js';
20
21
  import type { FileStoreTxSource } from './file_store_tx_source.js';
22
+ import type { IMissingTxsTracker } from './missing_txs_tracker.js';
21
23
  import { SlowTxCollection, getProofDeadlineForSlot } from './slow_tx_collection.js';
22
24
  import { type TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
23
25
  import type { TxSource } from './tx_source.js';
@@ -31,11 +33,10 @@ export type FastCollectionRequestInput =
31
33
  | { type: 'proposal'; blockProposal: BlockProposal; blockNumber: BlockNumber };
32
34
 
33
35
  export type FastCollectionRequest = FastCollectionRequestInput & {
34
- missingTxHashes: Set<string>;
36
+ missingTxTracker: IMissingTxsTracker;
35
37
  deadline: Date;
36
38
  blockInfo: L2BlockInfo;
37
39
  promise: PromiseWithResolvers<void>;
38
- foundTxs: Map<string, Tx>;
39
40
  };
40
41
 
41
42
  /**
@@ -2,14 +2,15 @@ import type { Logger } from '@aztec/foundation/log';
2
2
  import { elapsed } from '@aztec/foundation/timer';
3
3
  import type { TypedEventEmitter } from '@aztec/foundation/types';
4
4
  import type { L2Block } from '@aztec/stdlib/block';
5
- import type { BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx';
5
+ import type { BlockHeader, Tx } from '@aztec/stdlib/tx';
6
6
  import type { TelemetryClient } from '@aztec/telemetry-client';
7
7
 
8
8
  import EventEmitter from 'node:events';
9
9
 
10
- import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interfaces.js';
10
+ import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/index.js';
11
11
  import { TxCollectionInstrumentation } from './instrumentation.js';
12
12
  import type { CollectionMethod } from './tx_collection.js';
13
+ import type { TxSourceCollectionResult } from './tx_source.js';
13
14
 
14
15
  /** Context determining how collected txs should be added to the pool. */
15
16
  export type TxAddContext = { type: 'proposal'; blockHeader: BlockHeader } | { type: 'mined'; block: L2Block };
@@ -31,52 +32,37 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
31
32
  }
32
33
 
33
34
  public async collect(
34
- collectValidTxsFn: (txHashes: TxHash[]) => Promise<(Tx | undefined)[]>,
35
- requested: TxHash[],
35
+ collectValidTxsFn: () => Promise<TxSourceCollectionResult>,
36
+ requested: string[],
36
37
  info: Record<string, any> & { description: string; method: CollectionMethod },
37
38
  context: TxAddContext,
38
39
  ) {
39
40
  this.log.trace(`Requesting ${requested.length} txs via ${info.description}`, {
40
41
  ...info,
41
- requestedTxs: requested.map(t => t.toString()),
42
+ requestedTxs: requested,
42
43
  });
43
44
 
44
45
  // Execute collection function and measure the time taken, catching any errors.
45
- const [duration, txs] = await elapsed(async () => {
46
+ const [duration, { validTxs, invalidTxHashes }] = await elapsed(async () => {
46
47
  try {
47
- const response = await collectValidTxsFn(requested);
48
- return response.filter(tx => tx !== undefined);
48
+ return await collectValidTxsFn();
49
49
  } catch (err) {
50
50
  this.log.error(`Error collecting txs via ${info.description}`, err, {
51
51
  ...info,
52
- requestedTxs: requested.map(hash => hash.toString()),
52
+ requestedTxs: requested,
53
53
  });
54
- return [] as Tx[];
54
+ return { validTxs: [] as Tx[], invalidTxHashes: [] as string[] };
55
55
  }
56
56
  });
57
57
 
58
- if (txs.length === 0) {
58
+ if (validTxs.length === 0 && invalidTxHashes.length === 0) {
59
59
  this.log.trace(`No txs found via ${info.description}`, {
60
60
  ...info,
61
- requestedTxs: requested.map(t => t.toString()),
61
+ requestedTxs: requested,
62
62
  });
63
- return { txs, requested, duration };
63
+ return { txs: validTxs, requested, duration };
64
64
  }
65
65
 
66
- // Validate tx hashes for all collected txs from external sources
67
- const validTxs: Tx[] = [];
68
- const invalidTxHashes: string[] = [];
69
- await Promise.all(
70
- txs.map(async tx => {
71
- const isValid = await tx.validateTxHash();
72
- if (isValid) {
73
- validTxs.push(tx);
74
- } else {
75
- invalidTxHashes.push(tx.getTxHash().toString());
76
- }
77
- }),
78
- );
79
-
80
66
  if (invalidTxHashes.length > 0) {
81
67
  this.log.warn(`Rejecting ${invalidTxHashes.length} txs with invalid hashes from ${info.description}`, {
82
68
  ...info,
@@ -87,7 +73,7 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
87
73
  if (validTxs.length === 0) {
88
74
  this.log.trace(`No valid txs found via ${info.description} after validation`, {
89
75
  ...info,
90
- requestedTxs: requested.map(t => t.toString()),
76
+ requestedTxs: requested,
91
77
  invalidTxHashes,
92
78
  });
93
79
  return { txs: [], requested, duration };
@@ -99,7 +85,7 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
99
85
  ...info,
100
86
  duration,
101
87
  txs: validTxs.map(t => t.getTxHash().toString()),
102
- requestedTxs: requested.map(t => t.toString()),
88
+ requestedTxs: requested,
103
89
  rejectedCount: invalidTxHashes.length,
104
90
  },
105
91
  );
@@ -6,9 +6,11 @@ import type { Tx, TxHash } from '@aztec/stdlib/tx';
6
6
  import { type ComponentsVersions, getComponentsVersionsFromConfig } from '@aztec/stdlib/versioning';
7
7
  import { makeTracedFetch } from '@aztec/telemetry-client';
8
8
 
9
+ export type TxSourceCollectionResult = { validTxs: Tx[]; invalidTxHashes: string[] };
10
+
9
11
  export interface TxSource {
10
12
  getInfo(): string;
11
- getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]>;
13
+ getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
12
14
  }
13
15
 
14
16
  export class NodeRpcTxSource implements TxSource {
@@ -26,8 +28,25 @@ export class NodeRpcTxSource implements TxSource {
26
28
  return this.info;
27
29
  }
28
30
 
29
- public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
30
- return this.client.getTxsByHash(txHashes);
31
+ public async getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult> {
32
+ return this.verifyTxs(await this.client.getTxsByHash(txHashes));
33
+ }
34
+
35
+ private async verifyTxs(txs: Tx[]): Promise<TxSourceCollectionResult> {
36
+ // Validate tx hashes for all collected txs from external sources
37
+ const validTxs: Tx[] = [];
38
+ const invalidTxHashes: string[] = [];
39
+ await Promise.all(
40
+ txs.map(async tx => {
41
+ const isValid = await tx.validateTxHash();
42
+ if (isValid) {
43
+ validTxs.push(tx);
44
+ } else {
45
+ invalidTxHashes.push(tx.getTxHash().toString());
46
+ }
47
+ }),
48
+ );
49
+ return { validTxs: validTxs, invalidTxHashes: invalidTxHashes };
31
50
  }
32
51
  }
33
52
 
@@ -146,7 +146,7 @@ export class TxFileStore {
146
146
  }
147
147
 
148
148
  await retry(
149
- () => this.fileStore.save(path, tx.toBuffer(), { compress: false }),
149
+ () => this.fileStore.save(path, tx.toBuffer(), { compress: true }),
150
150
  `Uploading tx ${txHash}`,
151
151
  makeBackoff([0.1, 0.5, 2]),
152
152
  this.log,
@@ -270,6 +270,16 @@ class MockGossipSubService extends TypedEventEmitter<GossipsubEvents> implements
270
270
  { msgId, propagationSource, acceptance },
271
271
  );
272
272
  }
273
+
274
+ getMeshPeers(topic?: TopicStr): PeerIdStr[] {
275
+ if (topic && !this.subscribedTopics.has(topic)) {
276
+ return [];
277
+ }
278
+ return this.network
279
+ .getPeers()
280
+ .filter(peer => !this.peerId.equals(peer))
281
+ .map(peer => peer.toString());
282
+ }
273
283
  }
274
284
 
275
285
  /**
@@ -59,7 +59,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
59
59
 
60
60
  // === Core Operations (TxPoolV2) ===
61
61
 
62
- addPendingTxs(txs: Tx[], opts?: { source?: string }): Promise<AddTxsResult> {
62
+ addPendingTxs(txs: Tx[], opts?: { source?: string; feeComparisonOnly?: boolean }): Promise<AddTxsResult> {
63
63
  const accepted: TxHash[] = [];
64
64
  const newTxs: Tx[] = [];
65
65
  for (const tx of txs) {
@@ -123,7 +123,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 {
123
123
  return Promise.resolve();
124
124
  }
125
125
 
126
- handlePrunedBlocks(_latestBlock: L2BlockId): Promise<void> {
126
+ handlePrunedBlocks(_latestBlock: L2BlockId, _options?: { deleteAllTxs?: boolean }): Promise<void> {
127
127
  return Promise.resolve();
128
128
  }
129
129
 
@@ -29,22 +29,19 @@ import type { Message, PeerId } from '@libp2p/interface';
29
29
  import { TopicValidatorResult } from '@libp2p/interface';
30
30
  import { peerIdFromString } from '@libp2p/peer-id';
31
31
 
32
- import type { P2PClient } from '../client/p2p_client.js';
32
+ import type { P2PClient } from '../client/index.js';
33
33
  import type { P2PConfig } from '../config.js';
34
34
  import { createP2PClient } from '../index.js';
35
- import type { MemPools } from '../mem_pools/interface.js';
36
- import { LibP2PService } from '../services/libp2p/libp2p_service.js';
35
+ import type { MemPools } from '../mem_pools/index.js';
36
+ import { BatchTxRequesterCollector, LibP2PService, SendBatchRequestCollector } from '../services/index.js';
37
37
  import type { PeerManager } from '../services/peer-manager/peer_manager.js';
38
38
  import type { BatchTxRequesterLibP2PService } from '../services/reqresp/batch-tx-requester/interface.js';
39
39
  import type { IBatchRequestTxValidator } from '../services/reqresp/batch-tx-requester/tx_validator.js';
40
40
  import { RateLimitStatus } from '../services/reqresp/rate-limiter/rate_limiter.js';
41
41
  import type { ReqResp } from '../services/reqresp/reqresp.js';
42
42
  import type { PeerDiscoveryService } from '../services/service.js';
43
- import {
44
- BatchTxRequesterCollector,
45
- SendBatchRequestCollector,
46
- } from '../services/tx_collection/proposal_tx_collector.js';
47
- import { AlwaysTrueCircuitVerifier } from '../test-helpers/reqresp-nodes.js';
43
+ import { MissingTxsTracker } from '../services/tx_collection/missing_txs_tracker.js';
44
+ import { AlwaysTrueCircuitVerifier } from '../test-helpers/index.js';
48
45
  import {
49
46
  BENCHMARK_CONSTANTS,
50
47
  type CollectorType,
@@ -55,7 +52,7 @@ import {
55
52
  createMockEpochCache,
56
53
  createMockWorldStateSynchronizer,
57
54
  filterTxsByDistribution,
58
- } from '../test-helpers/testbench-utils.js';
55
+ } from '../test-helpers/index.js';
59
56
  import type { PubSubLibp2p } from '../util.js';
60
57
 
61
58
  export type { DistributionPattern, CollectorType } from '../test-helpers/testbench-utils.js';
@@ -166,7 +163,7 @@ async function generateDeterministicTxs(txCount: number, seed: number, config: P
166
163
  return cached.slice(0, txCount);
167
164
  }
168
165
 
169
- const includeByTimestampBase = BigInt(seed);
166
+ const expirationTimestampBase = BigInt(seed);
170
167
  for (let i = cached.length; i < txCount; i++) {
171
168
  const txSeed = seed * 10000 + i;
172
169
  const tx = await mockTx(txSeed, {
@@ -182,7 +179,7 @@ async function generateDeterministicTxs(txCount: number, seed: number, config: P
182
179
  hasPublicTeardownCallRequest: false,
183
180
  publicCalldataSize: 0,
184
181
  });
185
- tx.data.includeByTimestamp = includeByTimestampBase + BigInt(i);
182
+ tx.data.expirationTimestamp = expirationTimestampBase + BigInt(i);
186
183
  await tx.recomputeHash();
187
184
  cached.push(tx);
188
185
  }
@@ -277,7 +274,12 @@ async function runAggregatorBenchmark(
277
274
  new DateProvider(),
278
275
  noopTxValidator,
279
276
  );
280
- const fetchedTxs = await collector.collectTxs(txHashes, blockProposal, pinnedPeer, timeoutMs);
277
+ const fetchedTxs = await collector.collectTxs(
278
+ MissingTxsTracker.fromArray(txHashes),
279
+ blockProposal,
280
+ pinnedPeer,
281
+ timeoutMs,
282
+ );
281
283
  const durationMs = timer.ms();
282
284
  return {
283
285
  type: 'BENCH_RESULT',
@@ -292,7 +294,12 @@ async function runAggregatorBenchmark(
292
294
  BENCHMARK_CONSTANTS.FIXED_MAX_PEERS,
293
295
  BENCHMARK_CONSTANTS.FIXED_MAX_RETRY_ATTEMPTS,
294
296
  );
295
- const fetchedTxs = await collector.collectTxs(txHashes, blockProposal, pinnedPeer, timeoutMs);
297
+ const fetchedTxs = await collector.collectTxs(
298
+ MissingTxsTracker.fromArray(txHashes),
299
+ blockProposal,
300
+ pinnedPeer,
301
+ timeoutMs,
302
+ );
296
303
  const durationMs = timer.ms();
297
304
  return {
298
305
  type: 'BENCH_RESULT',
package/src/util.ts CHANGED
@@ -23,7 +23,13 @@ export interface PubSubLibp2p extends Pick<Libp2p, 'status' | 'start' | 'stop' |
23
23
  services: {
24
24
  pubsub: Pick<
25
25
  GossipSub,
26
- 'addEventListener' | 'removeEventListener' | 'publish' | 'subscribe' | 'reportMessageValidationResult' | 'direct'
26
+ | 'addEventListener'
27
+ | 'removeEventListener'
28
+ | 'publish'
29
+ | 'subscribe'
30
+ | 'reportMessageValidationResult'
31
+ | 'direct'
32
+ | 'getMeshPeers'
27
33
  > & { score: Pick<GossipSub['score'], 'score'> };
28
34
  };
29
35
  }