@aztec/p2p 0.0.1-commit.4d79d1f2d → 0.0.1-commit.4eabbdb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/dest/client/factory.d.ts +3 -2
  2. package/dest/client/factory.d.ts.map +1 -1
  3. package/dest/client/factory.js +9 -5
  4. package/dest/client/interface.d.ts +11 -9
  5. package/dest/client/interface.d.ts.map +1 -1
  6. package/dest/client/p2p_client.d.ts +6 -7
  7. package/dest/client/p2p_client.d.ts.map +1 -1
  8. package/dest/client/p2p_client.js +49 -14
  9. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +5 -5
  10. package/dest/config.d.ts +12 -2
  11. package/dest/config.d.ts.map +1 -1
  12. package/dest/config.js +5 -0
  13. package/dest/errors/tx-pool.error.d.ts +8 -0
  14. package/dest/errors/tx-pool.error.d.ts.map +1 -0
  15. package/dest/errors/tx-pool.error.js +9 -0
  16. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts +4 -2
  17. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts.map +1 -1
  18. package/dest/mem_pools/attestation_pool/attestation_pool.js +5 -0
  19. package/dest/mem_pools/attestation_pool/mocks.d.ts +2 -2
  20. package/dest/mem_pools/attestation_pool/mocks.d.ts.map +1 -1
  21. package/dest/mem_pools/attestation_pool/mocks.js +2 -2
  22. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.js +3 -3
  23. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts +30 -13
  24. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts.map +1 -1
  25. package/dest/mem_pools/tx_pool_v2/deleted_pool.js +91 -20
  26. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts +3 -3
  27. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts.map +1 -1
  28. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.js +18 -9
  29. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.d.ts +1 -1
  30. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.d.ts.map +1 -1
  31. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.js +4 -1
  32. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts +3 -3
  33. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts.map +1 -1
  34. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.js +10 -4
  35. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts +2 -2
  36. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts.map +1 -1
  37. package/dest/mem_pools/tx_pool_v2/eviction/index.js +1 -1
  38. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts +48 -5
  39. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts.map +1 -1
  40. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.js +8 -0
  41. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.js +7 -5
  42. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.js +5 -3
  43. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts +1 -1
  44. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts.map +1 -1
  45. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.js +8 -4
  46. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts +4 -4
  47. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts.map +1 -1
  48. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.js +14 -4
  49. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts +3 -3
  50. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts.map +1 -1
  51. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.js +2 -2
  52. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts +15 -0
  53. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts.map +1 -0
  54. package/dest/mem_pools/tx_pool_v2/instrumentation.js +43 -0
  55. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +14 -2
  56. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
  57. package/dest/mem_pools/tx_pool_v2/interfaces.js +3 -1
  58. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +9 -5
  59. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -1
  60. package/dest/mem_pools/tx_pool_v2/tx_metadata.js +30 -5
  61. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +12 -3
  62. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
  63. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +27 -4
  64. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +8 -3
  65. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
  66. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +11 -6
  67. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +12 -4
  68. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
  69. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +164 -37
  70. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts +2 -2
  71. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts.map +1 -1
  72. package/dest/msg_validators/tx_validator/timestamp_validator.js +6 -6
  73. package/dest/services/dummy_service.d.ts +3 -2
  74. package/dest/services/dummy_service.d.ts.map +1 -1
  75. package/dest/services/dummy_service.js +3 -0
  76. package/dest/services/encoding.d.ts +1 -1
  77. package/dest/services/encoding.d.ts.map +1 -1
  78. package/dest/services/encoding.js +2 -1
  79. package/dest/services/gossipsub/topic_score_params.d.ts +18 -6
  80. package/dest/services/gossipsub/topic_score_params.d.ts.map +1 -1
  81. package/dest/services/gossipsub/topic_score_params.js +32 -10
  82. package/dest/services/libp2p/libp2p_service.d.ts +2 -1
  83. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  84. package/dest/services/libp2p/libp2p_service.js +7 -3
  85. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts +4 -3
  86. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts.map +1 -1
  87. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js +5 -9
  88. package/dest/services/reqresp/batch-tx-requester/interface.d.ts +2 -6
  89. package/dest/services/reqresp/batch-tx-requester/interface.d.ts.map +1 -1
  90. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts +10 -13
  91. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts.map +1 -1
  92. package/dest/services/reqresp/batch-tx-requester/missing_txs.js +25 -46
  93. package/dest/services/service.d.ts +4 -2
  94. package/dest/services/service.d.ts.map +1 -1
  95. package/dest/services/tx_collection/config.d.ts +13 -1
  96. package/dest/services/tx_collection/config.d.ts.map +1 -1
  97. package/dest/services/tx_collection/config.js +30 -0
  98. package/dest/services/tx_collection/fast_tx_collection.d.ts +1 -1
  99. package/dest/services/tx_collection/fast_tx_collection.d.ts.map +1 -1
  100. package/dest/services/tx_collection/fast_tx_collection.js +39 -33
  101. package/dest/services/tx_collection/file_store_tx_collection.d.ts +38 -29
  102. package/dest/services/tx_collection/file_store_tx_collection.d.ts.map +1 -1
  103. package/dest/services/tx_collection/file_store_tx_collection.js +126 -77
  104. package/dest/services/tx_collection/file_store_tx_source.d.ts +16 -6
  105. package/dest/services/tx_collection/file_store_tx_source.d.ts.map +1 -1
  106. package/dest/services/tx_collection/file_store_tx_source.js +49 -16
  107. package/dest/services/tx_collection/instrumentation.d.ts +1 -1
  108. package/dest/services/tx_collection/instrumentation.d.ts.map +1 -1
  109. package/dest/services/tx_collection/instrumentation.js +2 -1
  110. package/dest/services/tx_collection/missing_txs_tracker.d.ts +32 -0
  111. package/dest/services/tx_collection/missing_txs_tracker.d.ts.map +1 -0
  112. package/dest/services/tx_collection/missing_txs_tracker.js +27 -0
  113. package/dest/services/tx_collection/proposal_tx_collector.d.ts +7 -6
  114. package/dest/services/tx_collection/proposal_tx_collector.d.ts.map +1 -1
  115. package/dest/services/tx_collection/proposal_tx_collector.js +5 -4
  116. package/dest/services/tx_collection/slow_tx_collection.d.ts +5 -3
  117. package/dest/services/tx_collection/slow_tx_collection.d.ts.map +1 -1
  118. package/dest/services/tx_collection/slow_tx_collection.js +17 -12
  119. package/dest/services/tx_collection/tx_collection.d.ts +9 -6
  120. package/dest/services/tx_collection/tx_collection.d.ts.map +1 -1
  121. package/dest/services/tx_collection/tx_collection.js +26 -10
  122. package/dest/services/tx_collection/tx_collection_sink.d.ts +6 -5
  123. package/dest/services/tx_collection/tx_collection_sink.d.ts.map +1 -1
  124. package/dest/services/tx_collection/tx_collection_sink.js +13 -22
  125. package/dest/services/tx_collection/tx_source.d.ts +8 -3
  126. package/dest/services/tx_collection/tx_source.d.ts.map +1 -1
  127. package/dest/services/tx_collection/tx_source.js +19 -2
  128. package/dest/services/tx_file_store/tx_file_store.d.ts +3 -2
  129. package/dest/services/tx_file_store/tx_file_store.d.ts.map +1 -1
  130. package/dest/services/tx_file_store/tx_file_store.js +9 -6
  131. package/dest/test-helpers/mock-pubsub.d.ts +3 -2
  132. package/dest/test-helpers/mock-pubsub.d.ts.map +1 -1
  133. package/dest/test-helpers/mock-pubsub.js +6 -0
  134. package/dest/test-helpers/testbench-utils.d.ts +7 -2
  135. package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
  136. package/dest/test-helpers/testbench-utils.js +7 -1
  137. package/dest/testbench/p2p_client_testbench_worker.d.ts +2 -2
  138. package/dest/testbench/p2p_client_testbench_worker.d.ts.map +1 -1
  139. package/dest/testbench/p2p_client_testbench_worker.js +8 -8
  140. package/dest/util.d.ts +2 -2
  141. package/dest/util.d.ts.map +1 -1
  142. package/package.json +14 -14
  143. package/src/client/factory.ts +18 -3
  144. package/src/client/interface.ts +20 -9
  145. package/src/client/p2p_client.ts +57 -17
  146. package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +18 -8
  147. package/src/config.ts +9 -1
  148. package/src/errors/tx-pool.error.ts +12 -0
  149. package/src/mem_pools/attestation_pool/attestation_pool.ts +8 -0
  150. package/src/mem_pools/attestation_pool/mocks.ts +2 -1
  151. package/src/mem_pools/tx_pool/README.md +1 -1
  152. package/src/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.ts +3 -3
  153. package/src/mem_pools/tx_pool_v2/README.md +43 -27
  154. package/src/mem_pools/tx_pool_v2/deleted_pool.ts +109 -22
  155. package/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts +21 -8
  156. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts +4 -1
  157. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts +15 -4
  158. package/src/mem_pools/tx_pool_v2/eviction/index.ts +4 -0
  159. package/src/mem_pools/tx_pool_v2/eviction/interfaces.ts +49 -4
  160. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts +5 -5
  161. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts +3 -3
  162. package/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts +8 -7
  163. package/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts +24 -6
  164. package/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts +3 -3
  165. package/src/mem_pools/tx_pool_v2/instrumentation.ts +69 -0
  166. package/src/mem_pools/tx_pool_v2/interfaces.ts +14 -2
  167. package/src/mem_pools/tx_pool_v2/tx_metadata.ts +44 -9
  168. package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +32 -5
  169. package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +17 -6
  170. package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +186 -29
  171. package/src/msg_validators/tx_validator/timestamp_validator.ts +7 -7
  172. package/src/services/dummy_service.ts +5 -1
  173. package/src/services/encoding.ts +2 -1
  174. package/src/services/gossipsub/README.md +29 -14
  175. package/src/services/gossipsub/topic_score_params.ts +49 -13
  176. package/src/services/libp2p/libp2p_service.ts +7 -2
  177. package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts +6 -6
  178. package/src/services/reqresp/batch-tx-requester/interface.ts +1 -5
  179. package/src/services/reqresp/batch-tx-requester/missing_txs.ts +23 -71
  180. package/src/services/service.ts +10 -1
  181. package/src/services/tx_collection/config.ts +42 -0
  182. package/src/services/tx_collection/fast_tx_collection.ts +51 -30
  183. package/src/services/tx_collection/file_store_tx_collection.ts +143 -93
  184. package/src/services/tx_collection/file_store_tx_source.ts +64 -17
  185. package/src/services/tx_collection/instrumentation.ts +7 -1
  186. package/src/services/tx_collection/missing_txs_tracker.ts +52 -0
  187. package/src/services/tx_collection/proposal_tx_collector.ts +8 -7
  188. package/src/services/tx_collection/slow_tx_collection.ts +17 -13
  189. package/src/services/tx_collection/tx_collection.ts +45 -14
  190. package/src/services/tx_collection/tx_collection_sink.ts +15 -29
  191. package/src/services/tx_collection/tx_source.ts +22 -3
  192. package/src/services/tx_file_store/tx_file_store.ts +6 -4
  193. package/src/test-helpers/mock-pubsub.ts +10 -0
  194. package/src/test-helpers/testbench-utils.ts +10 -2
  195. package/src/testbench/p2p_client_testbench_worker.ts +20 -13
  196. package/src/util.ts +7 -1
@@ -14,6 +14,7 @@ import type { PeerId } from '@libp2p/interface';
14
14
  import type { BatchTxRequesterConfig } from '../reqresp/batch-tx-requester/config.js';
15
15
  import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
16
16
  import type { TxCollectionConfig } from './config.js';
17
+ import { MissingTxsTracker } from './missing_txs_tracker.js';
17
18
  import {
18
19
  BatchTxRequesterCollector,
19
20
  type MissingTxsCollector,
@@ -83,8 +84,7 @@ export class FastTxCollection {
83
84
  ...input,
84
85
  blockInfo,
85
86
  promise,
86
- foundTxs: new Map<string, Tx>(),
87
- missingTxHashes: new Set(txHashes.map(t => t.toString())),
87
+ missingTxTracker: MissingTxsTracker.fromArray(txHashes),
88
88
  deadline: opts.deadline,
89
89
  };
90
90
 
@@ -92,15 +92,15 @@ export class FastTxCollection {
92
92
  clearTimeout(timeoutTimer);
93
93
 
94
94
  this.log.verbose(
95
- `Collected ${request.foundTxs.size} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
95
+ `Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
96
96
  {
97
97
  ...blockInfo,
98
98
  duration,
99
99
  requestType: input.type,
100
- missingTxs: [...request.missingTxHashes],
100
+ missingTxs: [...request.missingTxTracker.missingTxHashes],
101
101
  },
102
102
  );
103
- return [...request.foundTxs.values()];
103
+ return request.missingTxTracker.collectedTxs;
104
104
  }
105
105
 
106
106
  protected async collectFast(
@@ -111,7 +111,7 @@ export class FastTxCollection {
111
111
  const { blockInfo } = request;
112
112
 
113
113
  this.log.debug(
114
- `Starting fast collection of ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
114
+ `Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
115
115
  { ...blockInfo, requestType: request.type, deadline: opts.deadline },
116
116
  );
117
117
 
@@ -124,7 +124,7 @@ export class FastTxCollection {
124
124
  await Promise.race([request.promise.promise, waitBeforeReqResp]);
125
125
 
126
126
  // If we have collected all txs, we can stop here
127
- if (request.missingTxHashes.size === 0) {
127
+ if (request.missingTxTracker.allFetched()) {
128
128
  this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
129
129
  return;
130
130
  }
@@ -138,7 +138,7 @@ export class FastTxCollection {
138
138
  const logCtx = {
139
139
  ...blockInfo,
140
140
  errorMessage: err instanceof Error ? err.message : undefined,
141
- missingTxs: [...request.missingTxHashes].map(txHash => txHash.toString()),
141
+ missingTxs: request.missingTxTracker.missingTxHashes.values().map(txHash => txHash.toString()),
142
142
  };
143
143
  if (err instanceof Error && err.name === 'TimeoutError') {
144
144
  this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
@@ -166,7 +166,11 @@ export class FastTxCollection {
166
166
  }
167
167
 
168
168
  // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
169
- const attemptsPerTx = [...request.missingTxHashes].map(txHash => ({ txHash, attempts: 0, found: false }));
169
+ const attemptsPerTx = [...request.missingTxTracker.missingTxHashes].map(txHash => ({
170
+ txHash,
171
+ attempts: 0,
172
+ found: false,
173
+ }));
170
174
 
171
175
  // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
172
176
  await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx, opts)));
@@ -179,7 +183,7 @@ export class FastTxCollection {
179
183
  opts: { deadline: Date },
180
184
  ) {
181
185
  const notFinished = () =>
182
- this.dateProvider.now() <= +opts.deadline && request.missingTxHashes.size > 0 && this.requests.has(request);
186
+ this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
183
187
 
184
188
  const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
185
189
  const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
@@ -196,7 +200,7 @@ export class FastTxCollection {
196
200
  if (!txToRequest) {
197
201
  // No more txs to process
198
202
  break;
199
- } else if (!request.missingTxHashes.has(txToRequest.txHash)) {
203
+ } else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
200
204
  // Mark as found if it was found somewhere else, we'll then remove it from the array.
201
205
  // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
202
206
  txToRequest.found = true;
@@ -225,10 +229,17 @@ export class FastTxCollection {
225
229
  return;
226
230
  }
227
231
 
232
+ const txHashes = batch.map(({ txHash }) => txHash);
228
233
  // Collect this batch from the node
229
234
  await this.txCollectionSink.collect(
230
- txHashes => node.getTxsByHash(txHashes),
231
- batch.map(({ txHash }) => TxHash.fromString(txHash)),
235
+ async () => {
236
+ const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
237
+ for (const tx of result.validTxs) {
238
+ request.missingTxTracker.markFetched(tx);
239
+ }
240
+ return result;
241
+ },
242
+ txHashes,
232
243
  {
233
244
  description: `fast ${node.getInfo()}`,
234
245
  node: node.getInfo(),
@@ -268,32 +279,44 @@ export class FastTxCollection {
268
279
  }
269
280
 
270
281
  this.log.debug(
271
- `Starting fast reqresp for ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
282
+ `Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
272
283
  { ...blockInfo, timeoutMs, pinnedPeer },
273
284
  );
274
285
 
275
286
  try {
276
287
  await this.txCollectionSink.collect(
277
- async txHashes => {
288
+ async () => {
289
+ let result: Tx[];
278
290
  if (request.type === 'proposal') {
279
- return await this.missingTxsCollector.collectTxs(txHashes, request.blockProposal, pinnedPeer, timeoutMs);
291
+ result = await this.missingTxsCollector.collectTxs(
292
+ request.missingTxTracker,
293
+ request.blockProposal,
294
+ pinnedPeer,
295
+ timeoutMs,
296
+ );
280
297
  } else if (request.type === 'block') {
281
298
  const blockTxsSource = {
282
299
  txHashes: request.block.body.txEffects.map(e => e.txHash),
283
300
  archive: request.block.archive.root,
284
301
  };
285
- return await this.missingTxsCollector.collectTxs(txHashes, blockTxsSource, pinnedPeer, timeoutMs);
302
+ result = await this.missingTxsCollector.collectTxs(
303
+ request.missingTxTracker,
304
+ blockTxsSource,
305
+ pinnedPeer,
306
+ timeoutMs,
307
+ );
286
308
  } else {
287
309
  throw new Error(`Unknown request type: ${(request as any).type}`);
288
310
  }
311
+ return { validTxs: result, invalidTxHashes: [] };
289
312
  },
290
- Array.from(request.missingTxHashes).map(txHash => TxHash.fromString(txHash)),
313
+ Array.from(request.missingTxTracker.missingTxHashes),
291
314
  { description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo },
292
315
  this.getAddContext(request),
293
316
  );
294
317
  } catch (err) {
295
318
  this.log.error(`Error sending fast reqresp request for txs`, err, {
296
- txs: [...request.missingTxHashes],
319
+ txs: [...request.missingTxTracker.missingTxHashes],
297
320
  ...blockInfo,
298
321
  });
299
322
  }
@@ -317,22 +340,20 @@ export class FastTxCollection {
317
340
  for (const tx of txs) {
318
341
  const txHash = tx.txHash.toString();
319
342
  // Remove the tx hash from the missing set, and add it to the found set.
320
- if (request.missingTxHashes.has(txHash)) {
321
- request.missingTxHashes.delete(txHash);
322
- request.foundTxs.set(txHash, tx);
343
+ if (request.missingTxTracker.markFetched(tx)) {
323
344
  this.log.trace(`Found tx ${txHash} for fast collection request`, {
324
345
  ...request.blockInfo,
325
346
  txHash: tx.txHash.toString(),
326
347
  type: request.type,
327
348
  });
328
- // If we found all txs for this request, we resolve the promise
329
- if (request.missingTxHashes.size === 0) {
330
- this.log.trace(`All txs found for fast collection request`, {
331
- ...request.blockInfo,
332
- type: request.type,
333
- });
334
- request.promise.resolve();
335
- }
349
+ }
350
+ // If we found all txs for this request, we resolve the promise
351
+ if (request.missingTxTracker.allFetched()) {
352
+ this.log.trace(`All txs found for fast collection request`, {
353
+ ...request.blockInfo,
354
+ type: request.type,
355
+ });
356
+ request.promise.resolve();
336
357
  }
337
358
  }
338
359
  }
@@ -1,152 +1,202 @@
1
1
  import { type Logger, createLogger } from '@aztec/foundation/log';
2
- import { FifoMemoryQueue } from '@aztec/foundation/queue';
2
+ import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
3
+ import { sleep } from '@aztec/foundation/sleep';
4
+ import { DateProvider } from '@aztec/foundation/timer';
3
5
  import { Tx, TxHash } from '@aztec/stdlib/tx';
4
6
 
5
7
  import type { FileStoreTxSource } from './file_store_tx_source.js';
6
8
  import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
7
9
 
8
- // Internal constants (not configurable by node operators)
9
- const FILE_STORE_DOWNLOAD_CONCURRENCY = 5; // Max concurrent downloads
10
+ /** Configuration for a FileStoreTxCollection instance. */
11
+ export type FileStoreCollectionConfig = {
12
+ workerCount: number;
13
+ backoffBaseMs: number;
14
+ backoffMaxMs: number;
15
+ };
16
+
17
+ type FileStoreTxEntry = {
18
+ txHash: string;
19
+ context: TxAddContext;
20
+ deadline: Date;
21
+ attempts: number;
22
+ lastAttemptTime: number;
23
+ nextSourceIndex: number;
24
+ };
10
25
 
11
26
  /**
12
27
  * Collects txs from file stores as a fallback after P2P methods have been tried.
13
- * Runs in parallel to slow/fast collection. The delay before starting file store
14
- * collection is managed by the TxCollection orchestrator, not this class.
28
+ * Uses a shared worker pool that pulls entries with priority (fewest attempts first),
29
+ * retries with round-robin across sources, and applies exponential backoff between
30
+ * full cycles through all sources.
15
31
  */
16
32
  export class FileStoreTxCollection {
17
- /** Map from tx hash to add context for txs queued for download. */
18
- private pendingTxs = new Map<string, TxAddContext>();
33
+ /** Map from tx hash string to entry for all pending downloads. */
34
+ private entries = new Map<string, FileStoreTxEntry>();
19
35
 
20
- /**
21
- * Tracks tx hashes found elsewhere, even before startCollecting is called.
22
- * Needed because the orchestrator delays startCollecting via a real sleep, but foundTxs
23
- * may arrive during that delay — before the hashes are added to pendingTxs.
24
- */
25
- private foundTxHashes = new Set<string>();
26
-
27
- /** Queue of tx hashes to be downloaded. */
28
- private downloadQueue = new FifoMemoryQueue<TxHash>();
29
-
30
- /** Worker promises for concurrent downloads. */
36
+ /** Worker promises for the shared worker pool. */
31
37
  private workers: Promise<void>[] = [];
32
38
 
33
- /** Whether the collection has been started. */
34
- private started = false;
39
+ /** Whether the worker pool is running. */
40
+ private running = false;
41
+
42
+ /** Signal used to wake sleeping workers when new entries arrive or stop is called. */
43
+ private wakeSignal: PromiseWithResolvers<void>;
35
44
 
36
45
  constructor(
37
- private readonly fileStoreSources: FileStoreTxSource[],
46
+ private readonly sources: FileStoreTxSource[],
38
47
  private readonly txCollectionSink: TxCollectionSink,
48
+ private readonly config: FileStoreCollectionConfig,
49
+ private readonly dateProvider: DateProvider = new DateProvider(),
39
50
  private readonly log: Logger = createLogger('p2p:file_store_tx_collection'),
40
- ) {}
51
+ ) {
52
+ this.wakeSignal = promiseWithResolvers<void>();
53
+ }
41
54
 
42
- /** Starts the file store collection workers. */
43
- public start() {
44
- if (this.fileStoreSources.length === 0) {
45
- this.log.debug('No file store sources configured, skipping file store collection');
55
+ /** Starts the shared worker pool. */
56
+ public start(): void {
57
+ if (this.sources.length === 0) {
58
+ this.log.debug('No file store sources configured');
46
59
  return;
47
60
  }
48
-
49
- this.started = true;
50
- this.downloadQueue = new FifoMemoryQueue<TxHash>();
51
-
52
- // Start concurrent download workers
53
- for (let i = 0; i < FILE_STORE_DOWNLOAD_CONCURRENCY; i++) {
54
- this.workers.push(this.downloadQueue.process(txHash => this.processDownload(txHash)));
61
+ this.running = true;
62
+ for (let i = 0; i < this.config.workerCount; i++) {
63
+ this.workers.push(this.workerLoop());
55
64
  }
56
-
57
- this.log.info(`Started file store tx collection with ${this.fileStoreSources.length} sources`, {
58
- sources: this.fileStoreSources.map(s => s.getInfo()),
59
- concurrency: FILE_STORE_DOWNLOAD_CONCURRENCY,
60
- });
61
65
  }
62
66
 
63
- /** Stops all collection activity. */
64
- public async stop() {
65
- if (!this.started) {
66
- return;
67
- }
68
- this.started = false;
69
- this.downloadQueue.end();
67
+ /** Stops all workers and clears state. */
68
+ public async stop(): Promise<void> {
69
+ this.running = false;
70
+ this.wake();
70
71
  await Promise.all(this.workers);
71
72
  this.workers = [];
72
- this.pendingTxs.clear();
73
- this.foundTxHashes.clear();
73
+ this.entries.clear();
74
74
  }
75
75
 
76
- /** Remove the given tx hashes from pending. */
77
- public stopCollecting(txHashes: TxHash[]) {
78
- for (const txHash of txHashes) {
79
- const hashStr = txHash.toString();
80
- this.pendingTxs.delete(hashStr);
76
+ /** Adds entries to the shared map and wakes workers. */
77
+ public startCollecting(txHashes: TxHash[], context: TxAddContext, deadline: Date): void {
78
+ if (this.sources.length === 0 || txHashes.length === 0) {
79
+ return;
80
+ }
81
+ if (+deadline <= this.dateProvider.now()) {
82
+ return;
81
83
  }
82
- }
83
-
84
- /** Clears all pending state. Items already in the download queue will still be processed but won't be re-queued. */
85
- public clearPending() {
86
- this.pendingTxs.clear();
87
- this.foundTxHashes.clear();
88
- }
89
84
 
90
- /** Queue the given tx hashes for file store collection. */
91
- public startCollecting(txHashes: TxHash[], context: TxAddContext) {
92
85
  for (const txHash of txHashes) {
93
86
  const hashStr = txHash.toString();
94
- if (!this.pendingTxs.has(hashStr) && !this.foundTxHashes.has(hashStr)) {
95
- this.pendingTxs.set(hashStr, context);
96
- this.downloadQueue.put(txHash);
87
+ if (!this.entries.has(hashStr)) {
88
+ this.entries.set(hashStr, {
89
+ txHash: hashStr,
90
+ context,
91
+ deadline,
92
+ attempts: 0,
93
+ lastAttemptTime: 0,
94
+ nextSourceIndex: Math.floor(Math.random() * this.sources.length),
95
+ });
97
96
  }
98
97
  }
98
+ this.wake();
99
99
  }
100
100
 
101
- /** Stop tracking txs that were found elsewhere. */
102
- public foundTxs(txs: Tx[]) {
101
+ /** Removes entries for txs that have been found elsewhere. */
102
+ public foundTxs(txs: Tx[]): void {
103
103
  for (const tx of txs) {
104
- const hashStr = tx.getTxHash().toString();
105
- this.pendingTxs.delete(hashStr);
106
- this.foundTxHashes.add(hashStr);
104
+ this.entries.delete(tx.getTxHash().toString());
107
105
  }
108
106
  }
109
107
 
110
- /** Processes a single tx hash from the download queue. */
111
- private async processDownload(txHash: TxHash) {
112
- const hashStr = txHash.toString();
113
- const context = this.pendingTxs.get(hashStr);
114
-
115
- // Skip if already found by another method
116
- if (!context) {
117
- return;
118
- }
119
-
120
- await this.downloadTx(txHash, context);
121
- this.pendingTxs.delete(hashStr);
108
+ /** Clears all pending entries. */
109
+ public clearPending(): void {
110
+ this.entries.clear();
122
111
  }
123
112
 
124
- /** Attempt to download a tx from file stores (round-robin). */
125
- private async downloadTx(txHash: TxHash, context: TxAddContext) {
126
- const startIndex = Math.floor(Math.random() * this.fileStoreSources.length);
127
- for (let i = startIndex; i < startIndex + this.fileStoreSources.length; i++) {
128
- const source = this.fileStoreSources[i % this.fileStoreSources.length];
113
+ private async workerLoop(): Promise<void> {
114
+ while (this.running) {
115
+ const action = this.getNextAction();
116
+ if (action.type === 'sleep') {
117
+ await action.promise;
118
+ continue;
119
+ }
120
+
121
+ const entry = action.entry;
122
+ const source = this.sources[entry.nextSourceIndex % this.sources.length];
123
+ entry.nextSourceIndex++;
124
+ entry.attempts++;
125
+ entry.lastAttemptTime = this.dateProvider.now();
129
126
 
130
127
  try {
131
128
  const result = await this.txCollectionSink.collect(
132
- hashes => source.getTxsByHash(hashes),
133
- [txHash],
129
+ () => source.getTxsByHash([TxHash.fromString(entry.txHash)]),
130
+ [entry.txHash],
134
131
  {
135
132
  description: `file-store ${source.getInfo()}`,
136
133
  method: 'file-store',
137
134
  fileStore: source.getInfo(),
138
135
  },
139
- context,
136
+ entry.context,
140
137
  );
141
-
142
138
  if (result.txs.length > 0) {
143
- return;
139
+ this.entries.delete(entry.txHash);
144
140
  }
145
141
  } catch (err) {
146
- this.log.trace(`Failed to download tx ${txHash} from ${source.getInfo()}`, { err });
142
+ this.log.trace(`Error downloading tx ${entry.txHash} from ${source.getInfo()}`, { err });
143
+ }
144
+ }
145
+ }
146
+
147
+ /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */
148
+ private getNextAction(): { type: 'process'; entry: FileStoreTxEntry } | { type: 'sleep'; promise: Promise<void> } {
149
+ const now = this.dateProvider.now();
150
+ let best: FileStoreTxEntry | undefined;
151
+ let earliestReadyAt = Infinity;
152
+
153
+ for (const [key, entry] of this.entries) {
154
+ if (+entry.deadline <= now) {
155
+ this.entries.delete(key);
156
+ continue;
157
+ }
158
+ const backoffMs = this.getBackoffMs(entry);
159
+ const readyAt = entry.lastAttemptTime + backoffMs;
160
+ if (readyAt > now) {
161
+ earliestReadyAt = Math.min(earliestReadyAt, readyAt);
162
+ continue;
163
+ }
164
+ if (!best || entry.attempts < best.attempts) {
165
+ best = entry;
147
166
  }
148
167
  }
149
168
 
150
- this.log.trace(`Tx ${txHash} not found in any file store`);
169
+ if (best) {
170
+ return { type: 'process', entry: best };
171
+ }
172
+ if (earliestReadyAt < Infinity) {
173
+ return { type: 'sleep', promise: this.sleepOrWake(earliestReadyAt - now) };
174
+ }
175
+ return { type: 'sleep', promise: this.waitForWake() };
176
+ }
177
+
178
+ /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */
179
+ private getBackoffMs(entry: FileStoreTxEntry): number {
180
+ const fullCycles = Math.floor(entry.attempts / this.sources.length);
181
+ if (fullCycles === 0) {
182
+ return 0;
183
+ }
184
+ return Math.min(this.config.backoffBaseMs * Math.pow(2, fullCycles - 1), this.config.backoffMaxMs);
185
+ }
186
+
187
+ /** Resolves the current wake signal and creates a new one. */
188
+ private wake(): void {
189
+ this.wakeSignal.resolve();
190
+ this.wakeSignal = promiseWithResolvers<void>();
191
+ }
192
+
193
+ /** Waits until the wake signal is resolved. */
194
+ private async waitForWake(): Promise<void> {
195
+ await this.wakeSignal.promise;
196
+ }
197
+
198
+ /** Sleeps for the given duration or until the wake signal is resolved. */
199
+ private async sleepOrWake(ms: number): Promise<void> {
200
+ await Promise.race([sleep(ms), this.wakeSignal.promise]);
151
201
  }
152
202
  }
@@ -1,26 +1,51 @@
1
1
  import { type Logger, createLogger } from '@aztec/foundation/log';
2
+ import { Timer } from '@aztec/foundation/timer';
2
3
  import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store';
3
4
  import { Tx, type TxHash } from '@aztec/stdlib/tx';
5
+ import {
6
+ type Histogram,
7
+ Metrics,
8
+ type TelemetryClient,
9
+ type UpDownCounter,
10
+ getTelemetryClient,
11
+ } from '@aztec/telemetry-client';
4
12
 
5
- import type { TxSource } from './tx_source.js';
13
+ import type { TxSource, TxSourceCollectionResult } from './tx_source.js';
6
14
 
7
15
  /** TxSource implementation that downloads txs from a file store. */
8
16
  export class FileStoreTxSource implements TxSource {
17
+ private downloadsSuccess: UpDownCounter;
18
+ private downloadsFailed: UpDownCounter;
19
+ private downloadDuration: Histogram;
20
+ private downloadSize: Histogram;
21
+
9
22
  private constructor(
10
23
  private readonly fileStore: ReadOnlyFileStore,
11
24
  private readonly baseUrl: string,
25
+ private readonly basePath: string,
12
26
  private readonly log: Logger,
13
- ) {}
27
+ telemetry: TelemetryClient,
28
+ ) {
29
+ const meter = telemetry.getMeter('file-store-tx-source');
30
+ this.downloadsSuccess = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_SUCCESS);
31
+ this.downloadsFailed = meter.createUpDownCounter(Metrics.TX_FILE_STORE_DOWNLOADS_FAILED);
32
+ this.downloadDuration = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_DURATION);
33
+ this.downloadSize = meter.createHistogram(Metrics.TX_FILE_STORE_DOWNLOAD_SIZE);
34
+ }
14
35
 
15
36
  /**
16
37
  * Creates a FileStoreTxSource from a URL.
17
38
  * @param url - The file store URL (s3://, gs://, file://, http://, https://).
39
+ * @param basePath - Base path for tx files within the store.
18
40
  * @param log - Optional logger.
41
+ * @param telemetry - Optional telemetry client.
19
42
  * @returns The FileStoreTxSource instance, or undefined if creation fails.
20
43
  */
21
44
  public static async create(
22
45
  url: string,
46
+ basePath: string,
23
47
  log: Logger = createLogger('p2p:file_store_tx_source'),
48
+ telemetry: TelemetryClient = getTelemetryClient(),
24
49
  ): Promise<FileStoreTxSource | undefined> {
25
50
  try {
26
51
  const fileStore = await createReadOnlyFileStore(url, log);
@@ -28,7 +53,7 @@ export class FileStoreTxSource implements TxSource {
28
53
  log.warn(`Failed to create file store for URL: ${url}`);
29
54
  return undefined;
30
55
  }
31
- return new FileStoreTxSource(fileStore, url, log);
56
+ return new FileStoreTxSource(fileStore, url, basePath, log, telemetry);
32
57
  } catch (err) {
33
58
  log.warn(`Error creating file store for URL: ${url}`, { error: err });
34
59
  return undefined;
@@ -39,32 +64,54 @@ export class FileStoreTxSource implements TxSource {
39
64
  return `file-store:${this.baseUrl}`;
40
65
  }
41
66
 
42
- public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> {
43
- return Promise.all(
44
- txHashes.map(async txHash => {
45
- const path = `txs/${txHash.toString()}.bin`;
46
- try {
47
- const buffer = await this.fileStore.read(path);
48
- return Tx.fromBuffer(buffer);
49
- } catch {
50
- // Tx not found or error reading - return undefined
51
- return undefined;
52
- }
53
- }),
54
- );
67
+ public async getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult> {
68
+ const invalidTxHashes: string[] = [];
69
+ return {
70
+ validTxs: (
71
+ await Promise.all(
72
+ txHashes.map(async txHash => {
73
+ const path = `${this.basePath}/txs/${txHash.toString()}.bin`;
74
+ const timer = new Timer();
75
+ try {
76
+ const buffer = await this.fileStore.read(path);
77
+ const tx = Tx.fromBuffer(buffer);
78
+ if ((await tx.validateTxHash()) && txHash.equals(tx.txHash)) {
79
+ this.downloadsSuccess.add(1);
80
+ this.downloadDuration.record(Math.ceil(timer.ms()));
81
+ this.downloadSize.record(buffer.length);
82
+ return tx;
83
+ } else {
84
+ invalidTxHashes.push(tx.txHash.toString());
85
+ this.downloadsFailed.add(1);
86
+ return undefined;
87
+ }
88
+ } catch {
89
+ // Tx not found or error reading - return undefined
90
+ this.downloadsFailed.add(1);
91
+ return undefined;
92
+ }
93
+ }),
94
+ )
95
+ ).filter(tx => tx !== undefined),
96
+ invalidTxHashes: invalidTxHashes,
97
+ };
55
98
  }
56
99
  }
57
100
 
58
101
  /**
59
102
  * Creates FileStoreTxSource instances from URLs.
60
103
  * @param urls - Array of file store URLs.
104
+ * @param basePath - Base path for tx files within each store.
61
105
  * @param log - Optional logger.
106
+ * @param telemetry - Optional telemetry client.
62
107
  * @returns Array of successfully created FileStoreTxSource instances.
63
108
  */
64
109
  export async function createFileStoreTxSources(
65
110
  urls: string[],
111
+ basePath: string,
66
112
  log: Logger = createLogger('p2p:file_store_tx_source'),
113
+ telemetry: TelemetryClient = getTelemetryClient(),
67
114
  ): Promise<FileStoreTxSource[]> {
68
- const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, log)));
115
+ const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, basePath, log, telemetry)));
69
116
  return sources.filter((s): s is FileStoreTxSource => s !== undefined);
70
117
  }
@@ -18,7 +18,13 @@ export class TxCollectionInstrumentation {
18
18
  const meter = client.getMeter(name);
19
19
 
20
20
  this.txsCollected = createUpDownCounterWithDefault(meter, Metrics.TX_COLLECTOR_COUNT, {
21
- [Attributes.TX_COLLECTION_METHOD]: ['fast-req-resp', 'fast-node-rpc', 'slow-req-resp', 'slow-node-rpc'],
21
+ [Attributes.TX_COLLECTION_METHOD]: [
22
+ 'fast-req-resp',
23
+ 'fast-node-rpc',
24
+ 'slow-req-resp',
25
+ 'slow-node-rpc',
26
+ 'file-store',
27
+ ],
22
28
  });
23
29
 
24
30
  this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX);