@aztec/p2p 0.0.1-commit.6d63667d → 0.0.1-commit.858058eac

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/dest/client/factory.d.ts +5 -5
  2. package/dest/client/factory.d.ts.map +1 -1
  3. package/dest/client/factory.js +40 -9
  4. package/dest/client/interface.d.ts +33 -15
  5. package/dest/client/interface.d.ts.map +1 -1
  6. package/dest/client/p2p_client.d.ts +33 -36
  7. package/dest/client/p2p_client.d.ts.map +1 -1
  8. package/dest/client/p2p_client.js +104 -139
  9. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +1 -1
  10. package/dest/config.d.ts +13 -4
  11. package/dest/config.d.ts.map +1 -1
  12. package/dest/config.js +11 -1
  13. package/dest/index.d.ts +2 -1
  14. package/dest/index.d.ts.map +1 -1
  15. package/dest/index.js +1 -0
  16. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts +102 -88
  17. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts.map +1 -1
  18. package/dest/mem_pools/attestation_pool/attestation_pool.js +436 -3
  19. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts +2 -2
  20. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts.map +1 -1
  21. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +353 -87
  22. package/dest/mem_pools/attestation_pool/index.d.ts +2 -3
  23. package/dest/mem_pools/attestation_pool/index.d.ts.map +1 -1
  24. package/dest/mem_pools/attestation_pool/index.js +1 -2
  25. package/dest/mem_pools/index.d.ts +3 -2
  26. package/dest/mem_pools/index.d.ts.map +1 -1
  27. package/dest/mem_pools/index.js +1 -1
  28. package/dest/mem_pools/interface.d.ts +5 -5
  29. package/dest/mem_pools/interface.d.ts.map +1 -1
  30. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts +87 -0
  31. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts.map +1 -0
  32. package/dest/mem_pools/tx_pool_v2/deleted_pool.js +180 -0
  33. package/dest/mem_pools/tx_pool_v2/index.d.ts +2 -1
  34. package/dest/mem_pools/tx_pool_v2/index.d.ts.map +1 -1
  35. package/dest/mem_pools/tx_pool_v2/index.js +1 -0
  36. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +3 -3
  37. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
  38. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +25 -3
  39. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -1
  40. package/dest/mem_pools/tx_pool_v2/tx_metadata.js +36 -4
  41. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +1 -1
  42. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
  43. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +77 -74
  44. package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.d.ts +3 -3
  45. package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.d.ts.map +1 -1
  46. package/dest/msg_validators/tx_validator/aggregate_tx_validator.d.ts +3 -3
  47. package/dest/msg_validators/tx_validator/aggregate_tx_validator.d.ts.map +1 -1
  48. package/dest/msg_validators/tx_validator/block_header_validator.d.ts +16 -3
  49. package/dest/msg_validators/tx_validator/block_header_validator.d.ts.map +1 -1
  50. package/dest/msg_validators/tx_validator/block_header_validator.js +1 -1
  51. package/dest/msg_validators/tx_validator/double_spend_validator.d.ts +13 -3
  52. package/dest/msg_validators/tx_validator/double_spend_validator.d.ts.map +1 -1
  53. package/dest/msg_validators/tx_validator/double_spend_validator.js +4 -4
  54. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts +20 -4
  55. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts.map +1 -1
  56. package/dest/msg_validators/tx_validator/timestamp_validator.js +2 -2
  57. package/dest/services/dummy_service.d.ts +10 -2
  58. package/dest/services/dummy_service.d.ts.map +1 -1
  59. package/dest/services/dummy_service.js +6 -0
  60. package/dest/services/encoding.d.ts +2 -2
  61. package/dest/services/encoding.d.ts.map +1 -1
  62. package/dest/services/encoding.js +2 -2
  63. package/dest/services/gossipsub/index.d.ts +3 -0
  64. package/dest/services/gossipsub/index.d.ts.map +1 -0
  65. package/dest/services/gossipsub/index.js +2 -0
  66. package/dest/services/gossipsub/scoring.d.ts +21 -3
  67. package/dest/services/gossipsub/scoring.d.ts.map +1 -1
  68. package/dest/services/gossipsub/scoring.js +24 -7
  69. package/dest/services/gossipsub/topic_score_params.d.ts +161 -0
  70. package/dest/services/gossipsub/topic_score_params.d.ts.map +1 -0
  71. package/dest/services/gossipsub/topic_score_params.js +324 -0
  72. package/dest/services/libp2p/libp2p_service.d.ts +84 -35
  73. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  74. package/dest/services/libp2p/libp2p_service.js +364 -265
  75. package/dest/services/peer-manager/peer_scoring.d.ts +1 -1
  76. package/dest/services/peer-manager/peer_scoring.d.ts.map +1 -1
  77. package/dest/services/peer-manager/peer_scoring.js +25 -2
  78. package/dest/services/reqresp/interface.d.ts +10 -1
  79. package/dest/services/reqresp/interface.d.ts.map +1 -1
  80. package/dest/services/reqresp/interface.js +15 -1
  81. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.d.ts +4 -3
  82. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.d.ts.map +1 -1
  83. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts +7 -1
  84. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts.map +1 -1
  85. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.js +15 -0
  86. package/dest/services/reqresp/protocols/tx.d.ts +7 -1
  87. package/dest/services/reqresp/protocols/tx.d.ts.map +1 -1
  88. package/dest/services/reqresp/protocols/tx.js +20 -0
  89. package/dest/services/reqresp/reqresp.d.ts +1 -1
  90. package/dest/services/reqresp/reqresp.d.ts.map +1 -1
  91. package/dest/services/reqresp/reqresp.js +11 -4
  92. package/dest/services/service.d.ts +35 -1
  93. package/dest/services/service.d.ts.map +1 -1
  94. package/dest/services/tx_collection/config.d.ts +7 -1
  95. package/dest/services/tx_collection/config.d.ts.map +1 -1
  96. package/dest/services/tx_collection/config.js +16 -0
  97. package/dest/services/tx_collection/fast_tx_collection.d.ts +3 -1
  98. package/dest/services/tx_collection/fast_tx_collection.d.ts.map +1 -1
  99. package/dest/services/tx_collection/fast_tx_collection.js +17 -3
  100. package/dest/services/tx_collection/file_store_tx_collection.d.ts +44 -0
  101. package/dest/services/tx_collection/file_store_tx_collection.d.ts.map +1 -0
  102. package/dest/services/tx_collection/file_store_tx_collection.js +118 -0
  103. package/dest/services/tx_collection/file_store_tx_source.d.ts +27 -0
  104. package/dest/services/tx_collection/file_store_tx_source.d.ts.map +1 -0
  105. package/dest/services/tx_collection/file_store_tx_source.js +57 -0
  106. package/dest/services/tx_collection/index.d.ts +2 -1
  107. package/dest/services/tx_collection/index.d.ts.map +1 -1
  108. package/dest/services/tx_collection/index.js +1 -0
  109. package/dest/services/tx_collection/slow_tx_collection.d.ts +3 -1
  110. package/dest/services/tx_collection/slow_tx_collection.d.ts.map +1 -1
  111. package/dest/services/tx_collection/slow_tx_collection.js +48 -19
  112. package/dest/services/tx_collection/tx_collection.d.ts +17 -7
  113. package/dest/services/tx_collection/tx_collection.d.ts.map +1 -1
  114. package/dest/services/tx_collection/tx_collection.js +58 -2
  115. package/dest/services/tx_collection/tx_collection_sink.d.ts +15 -6
  116. package/dest/services/tx_collection/tx_collection_sink.d.ts.map +1 -1
  117. package/dest/services/tx_collection/tx_collection_sink.js +13 -7
  118. package/dest/services/tx_file_store/config.d.ts +1 -3
  119. package/dest/services/tx_file_store/config.d.ts.map +1 -1
  120. package/dest/services/tx_file_store/config.js +0 -4
  121. package/dest/services/tx_file_store/tx_file_store.d.ts +3 -3
  122. package/dest/services/tx_file_store/tx_file_store.d.ts.map +1 -1
  123. package/dest/services/tx_provider.d.ts +3 -3
  124. package/dest/services/tx_provider.d.ts.map +1 -1
  125. package/dest/services/tx_provider.js +5 -4
  126. package/dest/test-helpers/make-test-p2p-clients.d.ts +3 -3
  127. package/dest/test-helpers/make-test-p2p-clients.d.ts.map +1 -1
  128. package/dest/test-helpers/mock-pubsub.d.ts +27 -1
  129. package/dest/test-helpers/mock-pubsub.d.ts.map +1 -1
  130. package/dest/test-helpers/mock-pubsub.js +97 -2
  131. package/dest/test-helpers/reqresp-nodes.d.ts +1 -1
  132. package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
  133. package/dest/test-helpers/reqresp-nodes.js +2 -1
  134. package/dest/test-helpers/testbench-utils.d.ts +38 -38
  135. package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
  136. package/dest/test-helpers/testbench-utils.js +124 -61
  137. package/dest/testbench/p2p_client_testbench_worker.js +2 -2
  138. package/package.json +14 -14
  139. package/src/client/factory.ts +68 -12
  140. package/src/client/interface.ts +39 -14
  141. package/src/client/p2p_client.ts +139 -162
  142. package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +1 -1
  143. package/src/config.ts +26 -2
  144. package/src/index.ts +1 -0
  145. package/src/mem_pools/attestation_pool/attestation_pool.ts +488 -91
  146. package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +442 -102
  147. package/src/mem_pools/attestation_pool/index.ts +9 -2
  148. package/src/mem_pools/index.ts +4 -1
  149. package/src/mem_pools/interface.ts +4 -4
  150. package/src/mem_pools/tx_pool_v2/README.md +59 -9
  151. package/src/mem_pools/tx_pool_v2/deleted_pool.ts +234 -0
  152. package/src/mem_pools/tx_pool_v2/index.ts +1 -0
  153. package/src/mem_pools/tx_pool_v2/interfaces.ts +2 -2
  154. package/src/mem_pools/tx_pool_v2/tx_metadata.ts +51 -5
  155. package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +90 -77
  156. package/src/msg_validators/attestation_validator/fisherman_attestation_validator.ts +2 -2
  157. package/src/msg_validators/tx_validator/aggregate_tx_validator.ts +2 -2
  158. package/src/msg_validators/tx_validator/block_header_validator.ts +15 -3
  159. package/src/msg_validators/tx_validator/double_spend_validator.ts +11 -6
  160. package/src/msg_validators/tx_validator/timestamp_validator.ts +19 -14
  161. package/src/services/dummy_service.ts +12 -0
  162. package/src/services/encoding.ts +2 -2
  163. package/src/services/gossipsub/README.md +626 -0
  164. package/src/services/gossipsub/index.ts +2 -0
  165. package/src/services/gossipsub/scoring.ts +29 -5
  166. package/src/services/gossipsub/topic_score_params.ts +451 -0
  167. package/src/services/libp2p/libp2p_service.ts +360 -269
  168. package/src/services/peer-manager/peer_scoring.ts +25 -0
  169. package/src/services/reqresp/interface.ts +26 -1
  170. package/src/services/reqresp/protocols/block_txs/block_txs_handler.ts +4 -3
  171. package/src/services/reqresp/protocols/block_txs/block_txs_reqresp.ts +17 -0
  172. package/src/services/reqresp/protocols/tx.ts +22 -0
  173. package/src/services/reqresp/reqresp.ts +13 -3
  174. package/src/services/service.ts +40 -0
  175. package/src/services/tx_collection/config.ts +26 -0
  176. package/src/services/tx_collection/fast_tx_collection.ts +14 -2
  177. package/src/services/tx_collection/file_store_tx_collection.ts +152 -0
  178. package/src/services/tx_collection/file_store_tx_source.ts +70 -0
  179. package/src/services/tx_collection/index.ts +1 -0
  180. package/src/services/tx_collection/slow_tx_collection.ts +55 -26
  181. package/src/services/tx_collection/tx_collection.ts +78 -12
  182. package/src/services/tx_collection/tx_collection_sink.ts +17 -7
  183. package/src/services/tx_file_store/config.ts +0 -6
  184. package/src/services/tx_file_store/tx_file_store.ts +4 -4
  185. package/src/services/tx_provider.ts +8 -7
  186. package/src/test-helpers/make-test-p2p-clients.ts +3 -3
  187. package/src/test-helpers/mock-pubsub.ts +133 -3
  188. package/src/test-helpers/reqresp-nodes.ts +2 -1
  189. package/src/test-helpers/testbench-utils.ts +122 -74
  190. package/src/testbench/p2p_client_testbench_worker.ts +2 -2
  191. package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts +0 -40
  192. package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts.map +0 -1
  193. package/dest/mem_pools/attestation_pool/kv_attestation_pool.js +0 -218
  194. package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts +0 -31
  195. package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts.map +0 -1
  196. package/dest/mem_pools/attestation_pool/memory_attestation_pool.js +0 -180
  197. package/src/mem_pools/attestation_pool/kv_attestation_pool.ts +0 -320
  198. package/src/mem_pools/attestation_pool/memory_attestation_pool.ts +0 -264
@@ -91,6 +91,7 @@ export class SlowTxCollection {
91
91
 
92
92
  for (const txHash of txHashes) {
93
93
  this.missingTxs.set(txHash.toString(), {
94
+ block,
94
95
  blockNumber: block.number,
95
96
  deadline: this.getDeadlineForSlot(block.header.getSlot()),
96
97
  readyForReqResp: this.nodes.length === 0, // If we have no nodes, we can start reqresp immediately
@@ -109,18 +110,26 @@ export class SlowTxCollection {
109
110
 
110
111
  // Gather all missing txs that are not in fast collection and request them from the node
111
112
  const missingTxs = this.getMissingTxsForSlowCollection();
112
- const missingTxHashes = missingTxs.map(([txHash]) => txHash).map(TxHash.fromString);
113
- if (missingTxHashes.length === 0) {
113
+ if (missingTxs.length === 0) {
114
114
  return;
115
115
  }
116
116
 
117
- // Request in chunks to avoid hitting RPC limits
118
- for (const batch of chunk(missingTxHashes, this.config.txCollectionNodeRpcMaxBatchSize)) {
119
- await this.txCollectionSink.collect(txHashes => node.getTxsByHash(txHashes), batch, {
120
- description: `node ${node.getInfo()}`,
121
- node: node.getInfo(),
122
- method: 'slow-node-rpc',
123
- });
117
+ // Group by block so we pass the correct mined context to the sink
118
+ for (const entries of this.groupByBlock(missingTxs)) {
119
+ const block = entries[0][1].block;
120
+ const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
121
+ for (const batch of chunk(txHashes, this.config.txCollectionNodeRpcMaxBatchSize)) {
122
+ await this.txCollectionSink.collect(
123
+ hashes => node.getTxsByHash(hashes),
124
+ batch,
125
+ {
126
+ description: `node ${node.getInfo()}`,
127
+ node: node.getInfo(),
128
+ method: 'slow-node-rpc',
129
+ },
130
+ { type: 'mined', block },
131
+ );
132
+ }
124
133
  }
125
134
 
126
135
  // Mark every tx that is still missing as ready for reqresp.
@@ -149,25 +158,30 @@ export class SlowTxCollection {
149
158
 
150
159
  const pinnedPeer = undefined;
151
160
  const timeoutMs = this.config.txCollectionSlowReqRespTimeoutMs;
152
- const maxPeers = boundInclusive(Math.ceil(missingTxs.length / 3), 4, 16);
153
161
  const maxRetryAttempts = 3;
154
- // Send a batch request via reqresp for the missing txs
155
- await this.txCollectionSink.collect(
156
- async txHashes => {
157
- const txs = await this.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
158
- ReqRespSubProtocol.TX,
159
- chunkTxHashesRequest(txHashes),
160
- pinnedPeer,
161
- timeoutMs,
162
- maxPeers,
163
- maxRetryAttempts,
164
- );
165
162
 
166
- return txs.flat();
167
- },
168
- missingTxs.map(([txHash]) => TxHash.fromString(txHash)),
169
- { description: 'slow reqresp', timeoutMs, method: 'slow-req-resp' },
170
- );
163
+ // Group by block so we pass the correct mined context to the sink
164
+ for (const entries of this.groupByBlock(missingTxs)) {
165
+ const block = entries[0][1].block;
166
+ const txHashes = entries.map(([txHash]) => TxHash.fromString(txHash));
167
+ const maxPeers = boundInclusive(Math.ceil(txHashes.length / 3), 4, 16);
168
+ await this.txCollectionSink.collect(
169
+ async hashes => {
170
+ const txs = await this.reqResp.sendBatchRequest<ReqRespSubProtocol.TX>(
171
+ ReqRespSubProtocol.TX,
172
+ chunkTxHashesRequest(hashes),
173
+ pinnedPeer,
174
+ timeoutMs,
175
+ maxPeers,
176
+ maxRetryAttempts,
177
+ );
178
+ return txs.flat();
179
+ },
180
+ txHashes,
181
+ { description: 'slow reqresp', timeoutMs, method: 'slow-req-resp' },
182
+ { type: 'mined', block },
183
+ );
184
+ }
171
185
  }
172
186
 
173
187
  /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */
@@ -223,6 +237,21 @@ export class SlowTxCollection {
223
237
  }
224
238
  }
225
239
 
240
+ /** Groups missing tx entries by block number. */
241
+ private groupByBlock(entries: [string, MissingTxInfo][]): [string, MissingTxInfo][][] {
242
+ const groups = new Map<number, [string, MissingTxInfo][]>();
243
+ for (const entry of entries) {
244
+ const bn = +entry[1].blockNumber;
245
+ let group = groups.get(bn);
246
+ if (!group) {
247
+ group = [];
248
+ groups.set(bn, group);
249
+ }
250
+ group.push(entry);
251
+ }
252
+ return [...groups.values()];
253
+ }
254
+
226
255
  /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */
227
256
  private getDeadlineForSlot(slotNumber: SlotNumber): Date {
228
257
  const epoch = getEpochAtSlot(slotNumber, this.constants);
@@ -2,6 +2,7 @@ import { BlockNumber } from '@aztec/foundation/branded-types';
2
2
  import { compactArray } from '@aztec/foundation/collection';
3
3
  import { type Logger, createLogger } from '@aztec/foundation/log';
4
4
  import { type PromiseWithResolvers, RunningPromise } from '@aztec/foundation/promise';
5
+ import { sleep } from '@aztec/foundation/sleep';
5
6
  import { DateProvider } from '@aztec/foundation/timer';
6
7
  import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
7
8
  import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
@@ -11,18 +12,19 @@ import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-clien
11
12
 
12
13
  import type { PeerId } from '@libp2p/interface';
13
14
 
14
- import type { TxPool } from '../../mem_pools/index.js';
15
- import type { TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
15
+ import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interfaces.js';
16
16
  import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
17
17
  import type { TxCollectionConfig } from './config.js';
18
18
  import { FastTxCollection } from './fast_tx_collection.js';
19
+ import { FileStoreTxCollection } from './file_store_tx_collection.js';
20
+ import type { FileStoreTxSource } from './file_store_tx_source.js';
19
21
  import { SlowTxCollection } from './slow_tx_collection.js';
20
- import { TxCollectionSink } from './tx_collection_sink.js';
22
+ import { type TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
21
23
  import type { TxSource } from './tx_source.js';
22
24
 
23
- export type CollectionMethod = 'fast-req-resp' | 'fast-node-rpc' | 'slow-req-resp' | 'slow-node-rpc';
25
+ export type CollectionMethod = 'fast-req-resp' | 'fast-node-rpc' | 'slow-req-resp' | 'slow-node-rpc' | 'file-store';
24
26
 
25
- export type MissingTxInfo = { blockNumber: BlockNumber; deadline: Date; readyForReqResp: boolean };
27
+ export type MissingTxInfo = { block: L2Block; blockNumber: BlockNumber; deadline: Date; readyForReqResp: boolean };
26
28
 
27
29
  export type FastCollectionRequestInput =
28
30
  | { type: 'block'; block: L2Block }
@@ -54,6 +56,9 @@ export class TxCollection {
54
56
  /** Fast collection methods */
55
57
  protected readonly fastCollection: FastTxCollection;
56
58
 
59
+ /** File store collection */
60
+ protected readonly fileStoreCollection: FileStoreTxCollection;
61
+
57
62
  /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */
58
63
  private readonly reconcileFoundTxsLoop: RunningPromise;
59
64
 
@@ -61,17 +66,24 @@ export class TxCollection {
61
66
  private readonly txCollectionSink: TxCollectionSink;
62
67
 
63
68
  /** Handler for the txs-added event from the tx pool */
64
- protected readonly handleTxsAddedToPool: TxPoolEvents['txs-added'];
69
+ protected readonly handleTxsAddedToPool: TxPoolV2Events['txs-added'];
65
70
 
66
71
  /** Handler for the txs-added event from the tx collection sink */
67
- protected readonly handleTxsFound: TxPoolEvents['txs-added'];
72
+ protected readonly handleTxsFound: TxPoolV2Events['txs-added'];
73
+
74
+ /** Whether the service has been started. */
75
+ private started = false;
76
+
77
+ /** Whether file store sources are configured. */
78
+ private readonly hasFileStoreSources: boolean;
68
79
 
69
80
  constructor(
70
81
  private readonly p2pService: BatchTxRequesterLibP2PService,
71
82
  private readonly nodes: TxSource[],
72
83
  private readonly constants: L1RollupConstants,
73
- private readonly txPool: TxPool,
84
+ private readonly txPool: TxPoolV2,
74
85
  private readonly config: TxCollectionConfig,
86
+ fileStoreSources: FileStoreTxSource[] = [],
75
87
  private readonly dateProvider: DateProvider = new DateProvider(),
76
88
  telemetryClient: TelemetryClient = getTelemetryClient(),
77
89
  private readonly log: Logger = createLogger('p2p:tx_collection_service'),
@@ -98,18 +110,21 @@ export class TxCollection {
98
110
  this.log,
99
111
  );
100
112
 
113
+ this.hasFileStoreSources = fileStoreSources.length > 0;
114
+ this.fileStoreCollection = new FileStoreTxCollection(fileStoreSources, this.txCollectionSink, this.log);
115
+
101
116
  this.reconcileFoundTxsLoop = new RunningPromise(
102
117
  () => this.reconcileFoundTxsWithPool(),
103
118
  this.log,
104
119
  this.config.txCollectionReconcileIntervalMs,
105
120
  );
106
121
 
107
- this.handleTxsFound = (args: Parameters<TxPoolEvents['txs-added']>[0]) => {
122
+ this.handleTxsFound = (args: Parameters<TxPoolV2Events['txs-added']>[0]) => {
108
123
  this.foundTxs(args.txs);
109
124
  };
110
125
  this.txCollectionSink.on('txs-added', this.handleTxsFound);
111
126
 
112
- this.handleTxsAddedToPool = (args: Parameters<TxPoolEvents['txs-added']>[0]) => {
127
+ this.handleTxsAddedToPool = (args: Parameters<TxPoolV2Events['txs-added']>[0]) => {
113
128
  const { txs, source } = args;
114
129
  if (source !== 'tx-collection') {
115
130
  this.foundTxs(txs);
@@ -120,7 +135,9 @@ export class TxCollection {
120
135
 
121
136
  /** Starts all collection loops. */
122
137
  public start(): Promise<void> {
138
+ this.started = true;
123
139
  this.slowCollection.start();
140
+ this.fileStoreCollection.start();
124
141
  this.reconcileFoundTxsLoop.start();
125
142
 
126
143
  // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup
@@ -129,7 +146,13 @@ export class TxCollection {
129
146
 
130
147
  /** Stops all activity. */
131
148
  public async stop() {
132
- await Promise.all([this.slowCollection.stop(), this.fastCollection.stop(), this.reconcileFoundTxsLoop.stop()]);
149
+ this.started = false;
150
+ await Promise.all([
151
+ this.slowCollection.stop(),
152
+ this.fastCollection.stop(),
153
+ this.fileStoreCollection.stop(),
154
+ this.reconcileFoundTxsLoop.stop(),
155
+ ]);
133
156
 
134
157
  this.txPool.removeListener('txs-added', this.handleTxsAddedToPool);
135
158
  this.txCollectionSink.removeListener('txs-added', this.handleTxsFound);
@@ -147,7 +170,24 @@ export class TxCollection {
147
170
 
148
171
  /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
149
172
  public startCollecting(block: L2Block, txHashes: TxHash[]) {
150
- return this.slowCollection.startCollecting(block, txHashes);
173
+ this.slowCollection.startCollecting(block, txHashes);
174
+
175
+ // Delay file store collection to give P2P methods time to find txs first
176
+ if (this.hasFileStoreSources) {
177
+ const context: TxAddContext = { type: 'mined', block };
178
+ sleep(this.config.txCollectionFileStoreSlowDelayMs)
179
+ .then(() => {
180
+ if (this.started) {
181
+ // Only queue txs that are still missing after the delay
182
+ const stillMissing = new Set(this.slowCollection.getMissingTxHashes().map(h => h.toString()));
183
+ const remaining = txHashes.filter(h => stillMissing.has(h.toString()));
184
+ if (remaining.length > 0) {
185
+ this.fileStoreCollection.startCollecting(remaining, context);
186
+ }
187
+ }
188
+ })
189
+ .catch(err => this.log.error('Error in file store slow delay', err));
190
+ }
151
191
  }
152
192
 
153
193
  /** Collects the set of txs for the given block proposal as fast as possible */
@@ -175,13 +215,37 @@ export class TxCollection {
175
215
  txHashes: TxHash[] | string[],
176
216
  opts: { deadline: Date; pinnedPeer?: PeerId },
177
217
  ) {
218
+ const hashes = txHashes.map(h => (typeof h === 'string' ? TxHash.fromString(h) : h));
219
+
220
+ // Delay file store collection to give P2P methods time to find txs first
221
+ if (this.hasFileStoreSources) {
222
+ const context = this.getAddContextForInput(input);
223
+ sleep(this.config.txCollectionFileStoreFastDelayMs)
224
+ .then(() => {
225
+ if (this.started) {
226
+ this.fileStoreCollection.startCollecting(hashes, context);
227
+ }
228
+ })
229
+ .catch(err => this.log.error('Error in file store fast delay', err));
230
+ }
231
+
178
232
  return this.fastCollection.collectFastFor(input, txHashes, opts);
179
233
  }
180
234
 
235
+ /** Returns the TxAddContext for the given fast collection request input */
236
+ private getAddContextForInput(input: FastCollectionRequestInput): TxAddContext {
237
+ if (input.type === 'proposal') {
238
+ return { type: 'proposal', blockHeader: input.blockProposal.blockHeader };
239
+ } else {
240
+ return { type: 'mined', block: input.block };
241
+ }
242
+ }
243
+
181
244
  /** Mark the given txs as found. Stops collecting them. */
182
245
  private foundTxs(txs: Tx[]) {
183
246
  this.slowCollection.foundTxs(txs);
184
247
  this.fastCollection.foundTxs(txs);
248
+ this.fileStoreCollection.foundTxs(txs);
185
249
  }
186
250
 
187
251
  /**
@@ -191,6 +255,7 @@ export class TxCollection {
191
255
  public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void {
192
256
  this.slowCollection.stopCollectingForBlocksUpTo(blockNumber);
193
257
  this.fastCollection.stopCollectingForBlocksUpTo(blockNumber);
258
+ this.fileStoreCollection.clearPending();
194
259
  }
195
260
 
196
261
  /**
@@ -200,6 +265,7 @@ export class TxCollection {
200
265
  public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void {
201
266
  this.slowCollection.stopCollectingForBlocksAfter(blockNumber);
202
267
  this.fastCollection.stopCollectingForBlocksAfter(blockNumber);
268
+ this.fileStoreCollection.clearPending();
203
269
  }
204
270
 
205
271
  /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */
@@ -1,24 +1,28 @@
1
1
  import type { Logger } from '@aztec/foundation/log';
2
2
  import { elapsed } from '@aztec/foundation/timer';
3
3
  import type { TypedEventEmitter } from '@aztec/foundation/types';
4
- import { Tx, type TxHash } from '@aztec/stdlib/tx';
4
+ import type { L2Block } from '@aztec/stdlib/block';
5
+ import type { BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx';
5
6
  import type { TelemetryClient } from '@aztec/telemetry-client';
6
7
 
7
8
  import EventEmitter from 'node:events';
8
9
 
9
- import type { TxPool, TxPoolEvents } from '../../mem_pools/tx_pool/tx_pool.js';
10
+ import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interfaces.js';
10
11
  import { TxCollectionInstrumentation } from './instrumentation.js';
11
12
  import type { CollectionMethod } from './tx_collection.js';
12
13
 
14
+ /** Context determining how collected txs should be added to the pool. */
15
+ export type TxAddContext = { type: 'proposal'; blockHeader: BlockHeader } | { type: 'mined'; block: L2Block };
16
+
13
17
  /**
14
18
  * Executes collection requests from the fast and slow collection loops, and handles collected txs
15
19
  * by adding them to the tx pool and emitting events, as well as handling logging and metrics.
16
20
  */
17
- export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitter<TxPoolEvents>) {
21
+ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitter<TxPoolV2Events>) {
18
22
  private readonly instrumentation: TxCollectionInstrumentation;
19
23
 
20
24
  constructor(
21
- private readonly txPool: TxPool,
25
+ private readonly txPool: TxPoolV2,
22
26
  telemetryClient: TelemetryClient,
23
27
  private readonly log: Logger,
24
28
  ) {
@@ -30,6 +34,7 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
30
34
  collectValidTxsFn: (txHashes: TxHash[]) => Promise<(Tx | undefined)[]>,
31
35
  requested: TxHash[],
32
36
  info: Record<string, any> & { description: string; method: CollectionMethod },
37
+ context: TxAddContext,
33
38
  ) {
34
39
  this.log.trace(`Requesting ${requested.length} txs via ${info.description}`, {
35
40
  ...info,
@@ -99,12 +104,13 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
99
104
  },
100
105
  );
101
106
 
102
- return await this.foundTxs(validTxs, { ...info, duration });
107
+ return await this.foundTxs(validTxs, { ...info, duration }, context);
103
108
  }
104
109
 
105
110
  private async foundTxs(
106
111
  txs: Tx[],
107
112
  info: Record<string, any> & { description: string; method: CollectionMethod; duration: number },
113
+ context: TxAddContext,
108
114
  ) {
109
115
  // Report metrics for the collection
110
116
  this.instrumentation.increaseTxsFor(info.method, txs.length, info.duration);
@@ -112,9 +118,13 @@ export class TxCollectionSink extends (EventEmitter as new () => TypedEventEmitt
112
118
  // Mark txs as found in the slow missing txs set and all fast requests
113
119
  this.emit('txs-added', { txs });
114
120
 
115
- // Add the txs to the tx pool (should not fail, but we catch it just in case)
121
+ // Add the txs to the tx pool using the appropriate method based on context
116
122
  try {
117
- await this.txPool.addTxs(txs, { source: `tx-collection` });
123
+ if (context.type === 'mined') {
124
+ await this.txPool.addMinedTxs(txs, context.block.header, { source: 'tx-collection' });
125
+ } else {
126
+ await this.txPool.addProtectedTxs(txs, context.blockHeader, { source: 'tx-collection' });
127
+ }
118
128
  } catch (err) {
119
129
  this.log.error(`Error adding txs to the pool via ${info.description}`, err, {
120
130
  ...info,
@@ -6,8 +6,6 @@ import { type ConfigMappingsType, booleanConfigHelper, numberConfigHelper } from
6
6
  export type TxFileStoreConfig = {
7
7
  /** URL for uploading txs to file storage (s3://, gs://, file://) */
8
8
  txFileStoreUrl?: string;
9
- /** URL for downloading txs from file storage */
10
- txFileStoreDownloadUrl?: string;
11
9
  /** Max concurrent uploads */
12
10
  txFileStoreUploadConcurrency: number;
13
11
  /** Max queue size to prevent unbounded memory growth */
@@ -21,10 +19,6 @@ export const txFileStoreConfigMappings: ConfigMappingsType<TxFileStoreConfig> =
21
19
  env: 'TX_FILE_STORE_URL',
22
20
  description: 'URL for uploading txs to file storage (s3://, gs://, file://)',
23
21
  },
24
- txFileStoreDownloadUrl: {
25
- env: 'TX_FILE_STORE_DOWNLOAD_URL',
26
- description: 'URL for downloading txs from file storage',
27
- },
28
22
  txFileStoreUploadConcurrency: {
29
23
  env: 'TX_FILE_STORE_UPLOAD_CONCURRENCY',
30
24
  description: 'Maximum number of concurrent tx uploads',
@@ -6,7 +6,7 @@ import { type FileStore, createFileStore } from '@aztec/stdlib/file-store';
6
6
  import type { Tx } from '@aztec/stdlib/tx';
7
7
  import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
8
8
 
9
- import type { TxPool, TxPoolEvents } from '../../mem_pools/tx_pool/index.js';
9
+ import type { TxPoolV2 } from '../../mem_pools/index.js';
10
10
  import type { TxFileStoreConfig } from './config.js';
11
11
  import { TxFileStoreInstrumentation } from './instrumentation.js';
12
12
 
@@ -18,7 +18,7 @@ export class TxFileStore {
18
18
  private uploadQueue: Tx[] = [];
19
19
  private activeUploads = 0;
20
20
  private readonly queueProcessor: RunningPromise;
21
- private readonly handleTxsAdded: TxPoolEvents['txs-added'];
21
+ private readonly handleTxsAdded: (args: { txs: Tx[]; source?: string }) => void;
22
22
 
23
23
  /** Recently uploaded tx hashes for deduplication. */
24
24
  private recentUploads: Set<string> = new Set();
@@ -27,7 +27,7 @@ export class TxFileStore {
27
27
 
28
28
  private constructor(
29
29
  private readonly fileStore: FileStore,
30
- private readonly txPool: TxPool,
30
+ private readonly txPool: TxPoolV2,
31
31
  private readonly config: TxFileStoreConfig,
32
32
  private readonly instrumentation: TxFileStoreInstrumentation,
33
33
  private readonly log: Logger,
@@ -48,7 +48,7 @@ export class TxFileStore {
48
48
  * @returns The file store instance, or undefined if not configured/enabled.
49
49
  */
50
50
  static async create(
51
- txPool: TxPool,
51
+ txPool: TxPoolV2,
52
52
  config: TxFileStoreConfig,
53
53
  log: Logger = createLogger('p2p:tx_file_store'),
54
54
  telemetry: TelemetryClient = getTelemetryClient(),
@@ -5,13 +5,13 @@ import { elapsed } from '@aztec/foundation/timer';
5
5
  import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
6
6
  import type { ITxProvider } from '@aztec/stdlib/interfaces/server';
7
7
  import type { BlockProposal } from '@aztec/stdlib/p2p';
8
- import { Tx, TxHash } from '@aztec/stdlib/tx';
8
+ import { type BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx';
9
9
  import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
10
10
 
11
11
  import type { PeerId } from '@libp2p/interface';
12
12
 
13
13
  import type { P2PClient } from '../client/p2p_client.js';
14
- import type { TxPool } from '../mem_pools/index.js';
14
+ import type { TxPoolV2 } from '../mem_pools/tx_pool_v2/interfaces.js';
15
15
  import type { FastCollectionRequestInput, TxCollection } from './tx_collection/tx_collection.js';
16
16
  import { TxProviderInstrumentation } from './tx_provider_instrumentation.js';
17
17
 
@@ -24,7 +24,7 @@ export class TxProvider implements ITxProvider {
24
24
 
25
25
  constructor(
26
26
  private txCollection: TxCollection,
27
- private txPool: TxPool,
27
+ private txPool: TxPoolV2,
28
28
  private txValidator: Pick<P2PClient, 'validate'>,
29
29
  private log: Logger = createLogger('p2p:tx-collector'),
30
30
  client: TelemetryClient = getTelemetryClient(),
@@ -144,6 +144,7 @@ export class TxProvider implements ITxProvider {
144
144
  // Take txs from the proposal body if there are any
145
145
  // Note that we still have to validate these txs, but we do it in parallel with tx collection
146
146
  const proposal = request.type === 'proposal' ? request.blockProposal : undefined;
147
+ const proposalBlockHeader = proposal?.blockHeader;
147
148
  const txsFromProposal = this.extractFromProposal(proposal, [...missingTxHashes]);
148
149
  if (txsFromProposal.length > 0) {
149
150
  this.instrumentation.incTxsFromProposals(txsFromProposal.length);
@@ -155,7 +156,7 @@ export class TxProvider implements ITxProvider {
155
156
  }
156
157
 
157
158
  if (missingTxHashes.size === 0) {
158
- await this.processProposalTxs(txsFromProposal);
159
+ await this.processProposalTxs(txsFromProposal, proposalBlockHeader!);
159
160
  this.instrumentation.incTxsFromP2P(0, txHashes.length);
160
161
  return { txsFromMempool, txsFromProposal };
161
162
  }
@@ -163,7 +164,7 @@ export class TxProvider implements ITxProvider {
163
164
  // Start tx collection from the network if needed, while we validate the txs taken from the proposal in parallel
164
165
  const [txsFromNetwork] = await Promise.all([
165
166
  this.collectFromP2P(request, [...missingTxHashes], opts),
166
- this.processProposalTxs(txsFromProposal),
167
+ this.processProposalTxs(txsFromProposal, proposalBlockHeader!),
167
168
  ] as const);
168
169
 
169
170
  if (txsFromNetwork.length > 0) {
@@ -222,11 +223,11 @@ export class TxProvider implements ITxProvider {
222
223
  return compactArray(proposal.txs ?? []).filter(tx => missingTxHashes.includes(tx.getTxHash().toString()));
223
224
  }
224
225
 
225
- private async processProposalTxs(txs: Tx[]): Promise<void> {
226
+ private async processProposalTxs(txs: Tx[], blockHeader: BlockHeader): Promise<void> {
226
227
  if (txs.length === 0) {
227
228
  return;
228
229
  }
229
230
  await this.txValidator.validate(txs);
230
- await this.txPool.addTxs(txs);
231
+ await this.txPool.addProtectedTxs(txs, blockHeader);
231
232
  }
232
233
  }
@@ -13,7 +13,7 @@ import { createP2PClient } from '../client/index.js';
13
13
  import type { P2PClient } from '../client/p2p_client.js';
14
14
  import type { P2PConfig } from '../config.js';
15
15
  import type { AttestationPool } from '../mem_pools/attestation_pool/attestation_pool.js';
16
- import type { TxPool } from '../mem_pools/tx_pool/index.js';
16
+ import type { TxPoolV2 } from '../mem_pools/tx_pool_v2/interfaces.js';
17
17
  import { generatePeerIdPrivateKeys } from '../test-helpers/generate-peer-id-private-keys.js';
18
18
  import { getPorts } from './get-ports.js';
19
19
  import { makeEnrs } from './make-enrs.js';
@@ -22,7 +22,7 @@ import { AlwaysFalseCircuitVerifier, AlwaysTrueCircuitVerifier } from './reqresp
22
22
 
23
23
  export interface MakeTestP2PClientOptions {
24
24
  mockAttestationPool: AttestationPool;
25
- mockTxPool: TxPool;
25
+ mockTxPool: TxPoolV2;
26
26
  mockEpochCache: EpochCache;
27
27
  mockWorldState: WorldStateSynchronizer;
28
28
  alwaysTrueVerifier?: boolean;
@@ -108,7 +108,7 @@ export async function makeTestP2PClient(
108
108
  undefined,
109
109
  undefined,
110
110
  {
111
- txPool: mockTxPool as unknown as TxPool,
111
+ txPool: mockTxPool as unknown as TxPoolV2,
112
112
  attestationPool: mockAttestationPool as unknown as AttestationPool,
113
113
  store: kvStore,
114
114
  logger,