@aztec/p2p 1.2.1 → 2.0.0-nightly.20250813

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/dest/client/factory.d.ts +5 -1
  2. package/dest/client/factory.d.ts.map +1 -1
  3. package/dest/client/factory.js +29 -12
  4. package/dest/client/interface.d.ts +8 -13
  5. package/dest/client/interface.d.ts.map +1 -1
  6. package/dest/client/p2p_client.d.ts +18 -22
  7. package/dest/client/p2p_client.d.ts.map +1 -1
  8. package/dest/client/p2p_client.js +86 -83
  9. package/dest/config.d.ts +30 -1
  10. package/dest/config.d.ts.map +1 -1
  11. package/dest/config.js +34 -1
  12. package/dest/index.d.ts +1 -0
  13. package/dest/index.d.ts.map +1 -1
  14. package/dest/index.js +1 -0
  15. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts +13 -1
  16. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts.map +1 -1
  17. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts.map +1 -1
  18. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +117 -10
  19. package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts +4 -1
  20. package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts.map +1 -1
  21. package/dest/mem_pools/attestation_pool/kv_attestation_pool.js +22 -1
  22. package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts +4 -1
  23. package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts.map +1 -1
  24. package/dest/mem_pools/attestation_pool/memory_attestation_pool.js +21 -1
  25. package/dest/mem_pools/attestation_pool/mocks.d.ts +1 -2
  26. package/dest/mem_pools/attestation_pool/mocks.d.ts.map +1 -1
  27. package/dest/mem_pools/attestation_pool/mocks.js +2 -10
  28. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts +8 -3
  29. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts.map +1 -1
  30. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.js +64 -37
  31. package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts +8 -3
  32. package/dest/mem_pools/tx_pool/memory_tx_pool.d.ts.map +1 -1
  33. package/dest/mem_pools/tx_pool/memory_tx_pool.js +18 -10
  34. package/dest/mem_pools/tx_pool/tx_pool.d.ts +11 -2
  35. package/dest/mem_pools/tx_pool/tx_pool.d.ts.map +1 -1
  36. package/dest/mem_pools/tx_pool/tx_pool_test_suite.d.ts.map +1 -1
  37. package/dest/mem_pools/tx_pool/tx_pool_test_suite.js +73 -44
  38. package/dest/msg_validators/attestation_validator/attestation_validator.js +1 -1
  39. package/dest/msg_validators/tx_validator/block_header_validator.d.ts.map +1 -1
  40. package/dest/msg_validators/tx_validator/block_header_validator.js +2 -2
  41. package/dest/msg_validators/tx_validator/data_validator.d.ts.map +1 -1
  42. package/dest/msg_validators/tx_validator/data_validator.js +35 -59
  43. package/dest/msg_validators/tx_validator/double_spend_validator.js +2 -2
  44. package/dest/msg_validators/tx_validator/gas_validator.js +4 -4
  45. package/dest/msg_validators/tx_validator/metadata_validator.d.ts.map +1 -1
  46. package/dest/msg_validators/tx_validator/metadata_validator.js +21 -21
  47. package/dest/msg_validators/tx_validator/phases_validator.js +3 -3
  48. package/dest/msg_validators/tx_validator/tx_proof_validator.js +3 -3
  49. package/dest/services/discv5/discV5_service.d.ts.map +1 -1
  50. package/dest/services/discv5/discV5_service.js +4 -1
  51. package/dest/services/dummy_service.d.ts +13 -3
  52. package/dest/services/dummy_service.d.ts.map +1 -1
  53. package/dest/services/dummy_service.js +26 -3
  54. package/dest/services/index.d.ts +3 -1
  55. package/dest/services/index.d.ts.map +1 -1
  56. package/dest/services/index.js +3 -1
  57. package/dest/services/libp2p/libp2p_service.d.ts +13 -20
  58. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  59. package/dest/services/libp2p/libp2p_service.js +123 -46
  60. package/dest/services/peer-manager/interface.d.ts +8 -1
  61. package/dest/services/peer-manager/interface.d.ts.map +1 -1
  62. package/dest/services/peer-manager/peer_manager.d.ts +70 -3
  63. package/dest/services/peer-manager/peer_manager.d.ts.map +1 -1
  64. package/dest/services/peer-manager/peer_manager.js +369 -39
  65. package/dest/services/reqresp/config.d.ts +3 -3
  66. package/dest/services/reqresp/config.d.ts.map +1 -1
  67. package/dest/services/reqresp/config.js +3 -3
  68. package/dest/services/reqresp/connection-sampler/connection_sampler.d.ts +3 -4
  69. package/dest/services/reqresp/connection-sampler/connection_sampler.d.ts.map +1 -1
  70. package/dest/services/reqresp/connection-sampler/connection_sampler.js +35 -52
  71. package/dest/services/reqresp/index.d.ts +2 -1
  72. package/dest/services/reqresp/index.d.ts.map +1 -1
  73. package/dest/services/reqresp/index.js +2 -1
  74. package/dest/services/reqresp/interface.d.ts +61 -10
  75. package/dest/services/reqresp/interface.d.ts.map +1 -1
  76. package/dest/services/reqresp/interface.js +41 -6
  77. package/dest/services/reqresp/protocols/auth.d.ts +43 -0
  78. package/dest/services/reqresp/protocols/auth.d.ts.map +1 -0
  79. package/dest/services/reqresp/protocols/auth.js +71 -0
  80. package/dest/services/reqresp/protocols/block.d.ts +5 -0
  81. package/dest/services/reqresp/protocols/block.d.ts.map +1 -1
  82. package/dest/services/reqresp/protocols/block.js +28 -5
  83. package/dest/services/reqresp/protocols/block_txs/bitvector.d.ts +30 -0
  84. package/dest/services/reqresp/protocols/block_txs/bitvector.d.ts.map +1 -0
  85. package/dest/services/reqresp/protocols/block_txs/bitvector.js +75 -0
  86. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.d.ts +11 -0
  87. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.d.ts.map +1 -0
  88. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.js +39 -0
  89. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts +49 -0
  90. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts.map +1 -0
  91. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.js +75 -0
  92. package/dest/services/reqresp/protocols/block_txs/index.d.ts +4 -0
  93. package/dest/services/reqresp/protocols/block_txs/index.d.ts.map +1 -0
  94. package/dest/services/reqresp/protocols/block_txs/index.js +3 -0
  95. package/dest/services/reqresp/protocols/goodbye.js +3 -5
  96. package/dest/services/reqresp/protocols/index.d.ts +2 -0
  97. package/dest/services/reqresp/protocols/index.d.ts.map +1 -1
  98. package/dest/services/reqresp/protocols/index.js +2 -0
  99. package/dest/services/reqresp/protocols/status.d.ts +2 -0
  100. package/dest/services/reqresp/protocols/status.d.ts.map +1 -1
  101. package/dest/services/reqresp/protocols/status.js +7 -0
  102. package/dest/services/reqresp/protocols/tx.d.ts +12 -1
  103. package/dest/services/reqresp/protocols/tx.d.ts.map +1 -1
  104. package/dest/services/reqresp/protocols/tx.js +34 -6
  105. package/dest/services/reqresp/rate-limiter/rate_limits.d.ts.map +1 -1
  106. package/dest/services/reqresp/rate-limiter/rate_limits.js +20 -0
  107. package/dest/services/reqresp/reqresp.d.ts +37 -39
  108. package/dest/services/reqresp/reqresp.d.ts.map +1 -1
  109. package/dest/services/reqresp/reqresp.js +220 -220
  110. package/dest/services/reqresp/status.d.ts +8 -3
  111. package/dest/services/reqresp/status.d.ts.map +1 -1
  112. package/dest/services/reqresp/status.js +6 -2
  113. package/dest/services/service.d.ts +10 -10
  114. package/dest/services/service.d.ts.map +1 -1
  115. package/dest/services/tx_collection/config.d.ts +25 -0
  116. package/dest/services/tx_collection/config.d.ts.map +1 -0
  117. package/dest/services/tx_collection/config.js +58 -0
  118. package/dest/services/tx_collection/fast_tx_collection.d.ts +56 -0
  119. package/dest/services/tx_collection/fast_tx_collection.d.ts.map +1 -0
  120. package/dest/services/tx_collection/fast_tx_collection.js +295 -0
  121. package/dest/services/tx_collection/index.d.ts +3 -0
  122. package/dest/services/tx_collection/index.d.ts.map +1 -0
  123. package/dest/services/tx_collection/index.js +2 -0
  124. package/dest/services/tx_collection/instrumentation.d.ts +10 -0
  125. package/dest/services/tx_collection/instrumentation.d.ts.map +1 -0
  126. package/dest/services/tx_collection/instrumentation.js +34 -0
  127. package/dest/services/tx_collection/slow_tx_collection.d.ts +54 -0
  128. package/dest/services/tx_collection/slow_tx_collection.d.ts.map +1 -0
  129. package/dest/services/tx_collection/slow_tx_collection.js +176 -0
  130. package/dest/services/tx_collection/tx_collection.d.ts +109 -0
  131. package/dest/services/tx_collection/tx_collection.d.ts.map +1 -0
  132. package/dest/services/tx_collection/tx_collection.js +127 -0
  133. package/dest/services/tx_collection/tx_collection_sink.d.ts +30 -0
  134. package/dest/services/tx_collection/tx_collection_sink.d.ts.map +1 -0
  135. package/dest/services/tx_collection/tx_collection_sink.js +81 -0
  136. package/dest/services/tx_collection/tx_source.d.ts +18 -0
  137. package/dest/services/tx_collection/tx_source.d.ts.map +1 -0
  138. package/dest/services/tx_collection/tx_source.js +31 -0
  139. package/dest/services/tx_provider.d.ts +49 -0
  140. package/dest/services/tx_provider.d.ts.map +1 -0
  141. package/dest/services/tx_provider.js +206 -0
  142. package/dest/services/{tx_collect_instrumentation.d.ts → tx_provider_instrumentation.d.ts} +2 -2
  143. package/dest/services/tx_provider_instrumentation.d.ts.map +1 -0
  144. package/dest/services/{tx_collect_instrumentation.js → tx_provider_instrumentation.js} +5 -5
  145. package/dest/test-helpers/make-test-p2p-clients.d.ts.map +1 -1
  146. package/dest/test-helpers/make-test-p2p-clients.js +4 -3
  147. package/dest/test-helpers/mock-pubsub.d.ts +2 -1
  148. package/dest/test-helpers/mock-pubsub.d.ts.map +1 -1
  149. package/dest/test-helpers/mock-pubsub.js +2 -1
  150. package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
  151. package/dest/test-helpers/reqresp-nodes.js +8 -4
  152. package/dest/testbench/p2p_client_testbench_worker.js +11 -5
  153. package/dest/util.d.ts +1 -1
  154. package/dest/util.d.ts.map +1 -1
  155. package/package.json +14 -15
  156. package/src/client/factory.ts +87 -12
  157. package/src/client/interface.ts +19 -15
  158. package/src/client/p2p_client.ts +108 -92
  159. package/src/config.ts +52 -1
  160. package/src/index.ts +1 -0
  161. package/src/mem_pools/attestation_pool/attestation_pool.ts +15 -1
  162. package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +155 -4
  163. package/src/mem_pools/attestation_pool/kv_attestation_pool.ts +28 -1
  164. package/src/mem_pools/attestation_pool/memory_attestation_pool.ts +28 -2
  165. package/src/mem_pools/attestation_pool/mocks.ts +1 -3
  166. package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts +59 -41
  167. package/src/mem_pools/tx_pool/memory_tx_pool.ts +19 -9
  168. package/src/mem_pools/tx_pool/tx_pool.ts +7 -2
  169. package/src/mem_pools/tx_pool/tx_pool_test_suite.ts +63 -40
  170. package/src/msg_validators/attestation_validator/attestation_validator.ts +1 -1
  171. package/src/msg_validators/tx_validator/block_header_validator.ts +2 -2
  172. package/src/msg_validators/tx_validator/data_validator.ts +36 -27
  173. package/src/msg_validators/tx_validator/double_spend_validator.ts +2 -2
  174. package/src/msg_validators/tx_validator/gas_validator.ts +4 -4
  175. package/src/msg_validators/tx_validator/metadata_validator.ts +22 -28
  176. package/src/msg_validators/tx_validator/phases_validator.ts +2 -2
  177. package/src/msg_validators/tx_validator/tx_proof_validator.ts +2 -2
  178. package/src/services/discv5/discV5_service.ts +4 -1
  179. package/src/services/dummy_service.ts +44 -4
  180. package/src/services/index.ts +3 -1
  181. package/src/services/libp2p/libp2p_service.ts +147 -55
  182. package/src/services/peer-manager/interface.ts +10 -1
  183. package/src/services/peer-manager/peer_manager.ts +441 -41
  184. package/src/services/reqresp/config.ts +3 -3
  185. package/src/services/reqresp/connection-sampler/connection_sampler.ts +38 -63
  186. package/src/services/reqresp/index.ts +2 -0
  187. package/src/services/reqresp/interface.ts +63 -17
  188. package/src/services/reqresp/protocols/auth.ts +83 -0
  189. package/src/services/reqresp/protocols/block.ts +24 -3
  190. package/src/services/reqresp/protocols/block_txs/bitvector.ts +90 -0
  191. package/src/services/reqresp/protocols/block_txs/block_txs_handler.ts +53 -0
  192. package/src/services/reqresp/protocols/block_txs/block_txs_reqresp.ts +79 -0
  193. package/src/services/reqresp/protocols/block_txs/index.ts +3 -0
  194. package/src/services/reqresp/protocols/goodbye.ts +3 -3
  195. package/src/services/reqresp/protocols/index.ts +2 -0
  196. package/src/services/reqresp/protocols/status.ts +20 -0
  197. package/src/services/reqresp/protocols/tx.ts +35 -6
  198. package/src/services/reqresp/rate-limiter/rate_limits.ts +20 -0
  199. package/src/services/reqresp/reqresp.ts +294 -264
  200. package/src/services/reqresp/status.ts +9 -3
  201. package/src/services/service.ts +23 -14
  202. package/src/services/tx_collection/config.ts +84 -0
  203. package/src/services/tx_collection/fast_tx_collection.ts +338 -0
  204. package/src/services/tx_collection/index.ts +2 -0
  205. package/src/services/tx_collection/instrumentation.ts +43 -0
  206. package/src/services/tx_collection/slow_tx_collection.ts +232 -0
  207. package/src/services/tx_collection/tx_collection.ts +214 -0
  208. package/src/services/tx_collection/tx_collection_sink.ts +98 -0
  209. package/src/services/tx_collection/tx_source.ts +37 -0
  210. package/src/services/tx_provider.ts +215 -0
  211. package/src/services/{tx_collect_instrumentation.ts → tx_provider_instrumentation.ts} +5 -5
  212. package/src/test-helpers/make-test-p2p-clients.ts +4 -2
  213. package/src/test-helpers/mock-pubsub.ts +1 -0
  214. package/src/test-helpers/reqresp-nodes.ts +7 -1
  215. package/src/testbench/p2p_client_testbench_worker.ts +9 -2
  216. package/src/util.ts +1 -1
  217. package/dest/services/tx_collect_instrumentation.d.ts.map +0 -1
  218. package/dest/services/tx_collector.d.ts +0 -23
  219. package/dest/services/tx_collector.d.ts.map +0 -1
  220. package/dest/services/tx_collector.js +0 -95
  221. package/src/services/tx_collector.ts +0 -134
@@ -1,3 +1,4 @@
1
+ import type { EthAddress } from '@aztec/foundation/eth-address';
1
2
  import type { PeerInfo } from '@aztec/stdlib/interfaces/server';
2
3
  import type { BlockAttestation, BlockProposal, Gossipable } from '@aztec/stdlib/p2p';
3
4
  import type { Tx } from '@aztec/stdlib/tx';
@@ -5,12 +6,15 @@ import type { ENR } from '@chainsafe/enr';
5
6
  import type { PeerId } from '@libp2p/interface';
6
7
  import type EventEmitter from 'events';
7
8
  import type { P2PReqRespConfig } from './reqresp/config.js';
8
- import type { ReqRespSubProtocol, SubProtocolMap } from './reqresp/interface.js';
9
+ import type { StatusMessage } from './reqresp/index.js';
10
+ import type { ReqRespSubProtocol, ReqRespSubProtocolHandler, ReqRespSubProtocolValidators, SubProtocolMap } from './reqresp/interface.js';
11
+ import type { AuthRequest, AuthResponse } from './reqresp/protocols/auth.js';
9
12
  export declare enum PeerDiscoveryState {
10
13
  RUNNING = "running",
11
14
  STOPPED = "stopped"
12
15
  }
13
16
  export type P2PBlockReceivedCallback = (block: BlockProposal, sender: PeerId) => Promise<BlockAttestation[] | undefined>;
17
+ export type AuthReceivedCallback = (peerId: PeerId, authRequest: AuthRequest) => Promise<AuthResponse | undefined>;
14
18
  /**
15
19
  * The interface for a P2P service implementation.
16
20
  */
@@ -30,14 +34,6 @@ export interface P2PService {
30
34
  * @param message - The message to be propagated.
31
35
  */
32
36
  propagate<T extends Gossipable>(message: T): Promise<void>;
33
- /**
34
- * Request information from peers via the request response protocol.
35
- *
36
- * @param protocol - The request response protocol to use
37
- * @param request - The request type, corresponding to the protocol
38
- * @returns The response type, corresponding to the protocol
39
- */
40
- sendRequest<Protocol extends ReqRespSubProtocol>(protocol: Protocol, request: InstanceType<SubProtocolMap[Protocol]['request']>): Promise<InstanceType<SubProtocolMap[Protocol]['response']> | undefined>;
41
37
  /**
42
38
  * Send a batch of requests to peers, and return the responses
43
39
  *
@@ -45,12 +41,16 @@ export interface P2PService {
45
41
  * @param requests - The requests to send to the peers
46
42
  * @returns The responses to the requests
47
43
  */
48
- sendBatchRequest<Protocol extends ReqRespSubProtocol>(protocol: Protocol, requests: InstanceType<SubProtocolMap[Protocol]['request']>[], pinnedPeerId?: PeerId, timeoutMs?: number, maxPeers?: number, maxRetryAttempts?: number): Promise<(InstanceType<SubProtocolMap[Protocol]['response']> | undefined)[]>;
44
+ sendBatchRequest<Protocol extends ReqRespSubProtocol>(protocol: Protocol, requests: InstanceType<SubProtocolMap[Protocol]['request']>[], pinnedPeerId?: PeerId, timeoutMs?: number, maxPeers?: number, maxRetryAttempts?: number): Promise<InstanceType<SubProtocolMap[Protocol]['response']>[]>;
49
45
  registerBlockReceivedCallback(callback: P2PBlockReceivedCallback): void;
50
46
  getEnr(): ENR | undefined;
51
47
  getPeers(includePending?: boolean): PeerInfo[];
52
48
  validate(txs: Tx[]): Promise<void>;
49
+ addReqRespSubProtocol(subProtocol: ReqRespSubProtocol, handler: ReqRespSubProtocolHandler, validator?: ReqRespSubProtocolValidators[ReqRespSubProtocol]): Promise<void>;
50
+ handleAuthRequestFromPeer(authRequest: AuthRequest, peerId: PeerId): Promise<StatusMessage>;
53
51
  updateConfig(config: Partial<P2PReqRespConfig>): void;
52
+ /** If node running this P2P stack is validator, passes in validator address to P2P layer */
53
+ registerThisValidatorAddresses(address: EthAddress[]): void;
54
54
  }
55
55
  /**
56
56
  * The interface for a peer discovery service implementation.
@@ -1 +1 @@
1
- {"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../src/services/service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,YAAY,MAAM,QAAQ,CAAC;AAEvC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,KAAK,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAEjF,oBAAY,kBAAkB;IAC5B,OAAO,YAAY;IACnB,OAAO,YAAY;CACpB;AAED,MAAM,MAAM,wBAAwB,GAAG,CACrC,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,gBAAgB,EAAE,GAAG,SAAS,CAAC,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;;OAGG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB;;;OAGG;IACH,SAAS,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;OAMG;IACH,WAAW,CAAC,QAAQ,SAAS,kBAAkB,EAC7C,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,GACzD,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IAE3E;;;;;;OAMG;IACH,gBAAgB,CAAC,QAAQ,SAAS,kBAAkB,EAClD,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAC7D,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,EACjB,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,CAAC,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAG/E,6BAA6B,CAAC,QAAQ,EAAE,wBAAwB,GAAG,IAAI,CAAC;IAExE,MAAM,IAAI,GAAG,GAAG,SAAS,CAAC;IAE1B,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,GAAG,QAAQ,EAAE,CAAC;IAE/C,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;CACvD;AAED;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD;;SAEK;IACL,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;SAEK;IACL,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB;;;OAGG;IACH,YAAY,IAAI,GAAG,EAAE,CAAC;IAEtB;;OAEG;IACH,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;;;OAIG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAEzC;;OAEG;IACH,EAAE,CAAC,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,GAAG,IAAI,CAAC;IACjE,IAAI,CAAC,KAAK,EAAE,iBAAiB,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC;IAElD,SAAS,IAAI,kBAAkB,CAAC;IAEhC,MAAM,IAAI,GAAG,GAAG,SAAS,CAAC;IAE1B,iBAAiB,EAAE,GAAG,EAAE,CAAC;CAC1B"}
1
+ {"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../src/services/service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,YAAY,MAAM,QAAQ,CAAC;AAEvC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EACV,kBAAkB,EAClB,yBAAyB,EACzB,4BAA4B,EAC5B,cAAc,EACf,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE7E,oBAAY,kBAAkB;IAC5B,OAAO,YAAY;IACnB,OAAO,YAAY;CACpB;AAED,MAAM,MAAM,wBAAwB,GAAG,CACrC,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,gBAAgB,EAAE,GAAG,SAAS,CAAC,CAAC;AAE7C,MAAM,MAAM,oBAAoB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,KAAK,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;AAEnH;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;;OAGG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB;;;OAGG;IACH,SAAS,CAAC,CAAC,SAAS,UAAU,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;OAMG;IACH,gBAAgB,CAAC,QAAQ,SAAS,kBAAkB,EAClD,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAC7D,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,EACjB,gBAAgB,CAAC,EAAE,MAAM,GACxB,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC;IAGjE,6BAA6B,CAAC,QAAQ,EAAE,wBAAwB,GAAG,IAAI,CAAC;IAExE,MAAM,IAAI,GAAG,GAAG,SAAS,CAAC;IAE1B,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,GAAG,QAAQ,EAAE,CAAC;IAE/C,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnC,qBAAqB,CACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,yBAAyB,EAClC,SAAS,CAAC,EAAE,4BAA4B,CAAC,kBAAkB,CAAC,GAC3D,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,yBAAyB,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE5F,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IAEtD,4FAA4F;IAC5F,8BAA8B,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;CAC7D;AAED;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD;;SAEK;IACL,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;SAEK;IACL,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB;;;OAGG;IACH,YAAY,IAAI,GAAG,EAAE,CAAC;IAEtB;;OAEG;IACH,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;;;OAIG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAEzC;;OAEG;IACH,EAAE,CAAC,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,GAAG,IAAI,CAAC;IACjE,IAAI,CAAC,KAAK,EAAE,iBAAiB,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC;IAElD,SAAS,IAAI,kBAAkB,CAAC;IAEhC,MAAM,IAAI,GAAG,GAAG,SAAS,CAAC;IAE1B,iBAAiB,EAAE,GAAG,EAAE,CAAC;CAC1B"}
@@ -0,0 +1,25 @@
1
+ import { type ConfigMappingsType } from '@aztec/foundation/config';
2
+ export type TxCollectionConfig = {
3
+ /** How long to wait before starting reqresp for fast collection */
4
+ txCollectionFastNodesTimeoutBeforeReqRespMs: number;
5
+ /** How often to collect from configured nodes */
6
+ txCollectionSlowNodesIntervalMs: number;
7
+ /** How ofter to collect from peers */
8
+ txCollectionSlowReqRespIntervalMs: number;
9
+ /** How long to wait for a reqresp response during slow collection */
10
+ txCollectionSlowReqRespTimeoutMs: number;
11
+ /** How often to reconcile found txs with the tx pool */
12
+ txCollectionReconcileIntervalMs: number;
13
+ /** Whether to disable the slow collection loop if we are dealing with any immediate requests */
14
+ txCollectionDisableSlowDuringFastRequests: boolean;
15
+ /** How many ms to wait between retried request to a node via RPC during fast collection */
16
+ txCollectionFastNodeIntervalMs: number;
17
+ /** A comma-separated list of Aztec node RPC URLs to use for tx collection */
18
+ txCollectionNodeRpcUrls: string[];
19
+ /** Maximum number of parallel requests to make to a node during fast collection */
20
+ txCollectionFastMaxParallelRequestsPerNode: number;
21
+ /** Maximum number of transactions to request from a node in a single batch */
22
+ txCollectionNodeRpcMaxBatchSize: number;
23
+ };
24
+ export declare const txCollectionConfigMappings: ConfigMappingsType<TxCollectionConfig>;
25
+ //# sourceMappingURL=config.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAA2C,MAAM,0BAA0B,CAAC;AAG5G,MAAM,MAAM,kBAAkB,GAAG;IAC/B,oEAAoE;IACpE,2CAA2C,EAAE,MAAM,CAAC;IACpD,iDAAiD;IACjD,+BAA+B,EAAE,MAAM,CAAC;IACxC,sCAAsC;IACtC,iCAAiC,EAAE,MAAM,CAAC;IAC1C,qEAAqE;IACrE,gCAAgC,EAAE,MAAM,CAAC;IACzC,wDAAwD;IACxD,+BAA+B,EAAE,MAAM,CAAC;IACxC,gGAAgG;IAChG,yCAAyC,EAAE,OAAO,CAAC;IACnD,2FAA2F;IAC3F,8BAA8B,EAAE,MAAM,CAAC;IACvC,6EAA6E;IAC7E,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,mFAAmF;IACnF,0CAA0C,EAAE,MAAM,CAAC;IACnD,8EAA8E;IAC9E,+BAA+B,EAAE,MAAM,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,kBAAkB,CAAC,kBAAkB,CAyD7E,CAAC"}
@@ -0,0 +1,58 @@
1
+ import { booleanConfigHelper, numberConfigHelper } from '@aztec/foundation/config';
2
+ import { MAX_RPC_TXS_LEN } from '@aztec/stdlib/interfaces/server';
3
+ export const txCollectionConfigMappings = {
4
+ txCollectionFastNodesTimeoutBeforeReqRespMs: {
5
+ env: 'TX_COLLECTION_FAST_NODES_TIMEOUT_BEFORE_REQ_RESP_MS',
6
+ description: 'How long to wait before starting reqresp for fast collection',
7
+ ...numberConfigHelper(200)
8
+ },
9
+ txCollectionSlowNodesIntervalMs: {
10
+ env: 'TX_COLLECTION_SLOW_NODES_INTERVAL_MS',
11
+ description: 'How often to collect from configured nodes in the slow collection loop',
12
+ ...numberConfigHelper(12_000)
13
+ },
14
+ txCollectionSlowReqRespIntervalMs: {
15
+ env: 'TX_COLLECTION_SLOW_REQ_RESP_INTERVAL_MS',
16
+ description: 'How often to collect from peers via reqresp in the slow collection loop',
17
+ ...numberConfigHelper(12_000)
18
+ },
19
+ txCollectionSlowReqRespTimeoutMs: {
20
+ env: 'TX_COLLECTION_SLOW_REQ_RESP_TIMEOUT_MS',
21
+ description: 'How long to wait for a reqresp response during slow collection',
22
+ ...numberConfigHelper(20_000)
23
+ },
24
+ txCollectionReconcileIntervalMs: {
25
+ env: 'TX_COLLECTION_RECONCILE_INTERVAL_MS',
26
+ description: 'How often to reconcile found txs from the tx pool',
27
+ ...numberConfigHelper(60_000)
28
+ },
29
+ txCollectionDisableSlowDuringFastRequests: {
30
+ env: 'TX_COLLECTION_DISABLE_SLOW_DURING_FAST_REQUESTS',
31
+ description: 'Whether to disable the slow collection loop if we are dealing with any immediate requests',
32
+ ...booleanConfigHelper(true)
33
+ },
34
+ txCollectionFastNodeIntervalMs: {
35
+ env: 'TX_COLLECTION_FAST_NODE_INTERVAL_MS',
36
+ description: 'How many ms to wait between retried request to a node via RPC during fast collection',
37
+ ...numberConfigHelper(500)
38
+ },
39
+ txCollectionNodeRpcUrls: {
40
+ env: 'TX_COLLECTION_NODE_RPC_URLS',
41
+ fallback: [
42
+ 'PROVER_COORDINATION_NODE_URLS'
43
+ ],
44
+ description: 'A comma-separated list of Aztec node RPC URLs to use for tx collection',
45
+ parseEnv: (val)=>val.split(',').map((url)=>url.trim().replace(/\/$/, '')).filter((url)=>url.length > 0),
46
+ defaultValue: []
47
+ },
48
+ txCollectionFastMaxParallelRequestsPerNode: {
49
+ env: 'TX_COLLECTION_FAST_MAX_PARALLEL_REQUESTS_PER_NODE',
50
+ description: 'Maximum number of parallel requests to make to a node during fast collection',
51
+ ...numberConfigHelper(4)
52
+ },
53
+ txCollectionNodeRpcMaxBatchSize: {
54
+ env: 'TX_COLLECTION_NODE_RPC_MAX_BATCH_SIZE',
55
+ description: 'Maximum number of transactions to request from a node in a single batch',
56
+ ...numberConfigHelper(MAX_RPC_TXS_LEN)
57
+ }
58
+ };
@@ -0,0 +1,56 @@
1
+ import { type Logger } from '@aztec/foundation/log';
2
+ import { DateProvider } from '@aztec/foundation/timer';
3
+ import type { BlockProposal } from '@aztec/stdlib/p2p';
4
+ import { type Tx, TxHash } from '@aztec/stdlib/tx';
5
+ import type { PeerId } from '@libp2p/interface';
6
+ import { type ReqRespInterface } from '../reqresp/interface.js';
7
+ import type { TxCollectionConfig } from './config.js';
8
+ import type { FastCollectionRequest, FastCollectionRequestInput } from './tx_collection.js';
9
+ import type { TxCollectionSink } from './tx_collection_sink.js';
10
+ import type { TxSource } from './tx_source.js';
11
+ export declare class FastTxCollection {
12
+ private reqResp;
13
+ private nodes;
14
+ private txCollectionSink;
15
+ private config;
16
+ private dateProvider;
17
+ private log;
18
+ protected requests: Set<FastCollectionRequest>;
19
+ constructor(reqResp: Pick<ReqRespInterface, 'sendBatchRequest'>, nodes: TxSource[], txCollectionSink: TxCollectionSink, config: TxCollectionConfig, dateProvider?: DateProvider, log?: Logger);
20
+ stop(): Promise<void>;
21
+ getFastCollectionRequests(): Set<FastCollectionRequest>;
22
+ collectFastFor(input: FastCollectionRequestInput, txHashes: TxHash[] | string[], opts: {
23
+ deadline: Date;
24
+ pinnedPeer?: PeerId;
25
+ }): Promise<Tx[]>;
26
+ protected collectFast(request: FastCollectionRequest, opts: {
27
+ proposal?: BlockProposal;
28
+ deadline: Date;
29
+ pinnedPeer?: PeerId;
30
+ }): Promise<void>;
31
+ /**
32
+ * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests
33
+ * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
34
+ * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
35
+ * tx is requested more than once at the same time to the same node.
36
+ */
37
+ private collectFastFromNodes;
38
+ private collectFastFromNode;
39
+ private collectFastViaReqResp;
40
+ /**
41
+ * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
42
+ * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
43
+ */
44
+ foundTxs(txs: Tx[]): void;
45
+ /**
46
+ * Stop collecting all txs for blocks less than or requal to the block number specified.
47
+ * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
48
+ */
49
+ stopCollectingForBlocksUpTo(blockNumber: number): void;
50
+ /**
51
+ * Stop collecting all txs for blocks greater than the block number specified.
52
+ * To be called when there is a chain prune and previously mined txs are no longer relevant.
53
+ */
54
+ stopCollectingForBlocksAfter(blockNumber: number): void;
55
+ }
56
+ //# sourceMappingURL=fast_tx_collection.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fast_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/fast_tx_collection.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAW,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,KAAK,gBAAgB,EAAsB,MAAM,yBAAyB,CAAC;AAEpF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,KAAK,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAKzB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IARb,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAa;gBAGjD,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,EACnD,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD;IAGpD,IAAI;IAKV,yBAAyB;IAInB,cAAc,CACzB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE;cA0C/B,WAAW,CACzB,OAAO,EAAE,qBAAqB,EAC9B,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE;IAiDzE;;;;;OAKG;YACW,oBAAoB;YAYpB,mBAAmB;YAiFnB,qBAAqB;IA6CnC;;;OAGG;IACI,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE;IA0BzB;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAS7D;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;CAQ/D"}
@@ -0,0 +1,295 @@
1
+ import { times } from '@aztec/foundation/collection';
2
+ import { AbortError, TimeoutError } from '@aztec/foundation/error';
3
+ import { createLogger } from '@aztec/foundation/log';
4
+ import { boundInclusive } from '@aztec/foundation/number';
5
+ import { promiseWithResolvers } from '@aztec/foundation/promise';
6
+ import { sleep } from '@aztec/foundation/sleep';
7
+ import { DateProvider, elapsed } from '@aztec/foundation/timer';
8
+ import { TxHash } from '@aztec/stdlib/tx';
9
+ import { ReqRespSubProtocol } from '../reqresp/interface.js';
10
+ import { chunkTxHashesRequest } from '../reqresp/protocols/tx.js';
11
+ export class FastTxCollection {
12
+ reqResp;
13
+ nodes;
14
+ txCollectionSink;
15
+ config;
16
+ dateProvider;
17
+ log;
18
+ // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
19
+ requests;
20
+ constructor(reqResp, nodes, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:tx_collection_service')){
21
+ this.reqResp = reqResp;
22
+ this.nodes = nodes;
23
+ this.txCollectionSink = txCollectionSink;
24
+ this.config = config;
25
+ this.dateProvider = dateProvider;
26
+ this.log = log;
27
+ this.requests = new Set();
28
+ }
29
+ async stop() {
30
+ this.requests.forEach((request)=>request.promise.reject(new AbortError(`Stopped collection service`)));
31
+ await Promise.resolve();
32
+ }
33
+ getFastCollectionRequests() {
34
+ return this.requests;
35
+ }
36
+ async collectFastFor(input, txHashes, opts) {
37
+ const timeout = opts.deadline.getTime() - this.dateProvider.now();
38
+ if (timeout <= 0) {
39
+ this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, {
40
+ deadline: opts.deadline.getTime(),
41
+ now: this.dateProvider.now()
42
+ });
43
+ return [];
44
+ }
45
+ const blockInfo = input.type === 'proposal' ? input.blockProposal.toBlockInfo() : input.block.toBlockInfo();
46
+ // This promise is used to await for the collection to finish during the main collectFast method.
47
+ // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
48
+ const promise = promiseWithResolvers();
49
+ setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
50
+ const request = {
51
+ ...input,
52
+ blockInfo,
53
+ promise,
54
+ foundTxs: new Map(),
55
+ missingTxHashes: new Set(txHashes.map((t)=>t.toString())),
56
+ deadline: opts.deadline
57
+ };
58
+ const [duration] = await elapsed(()=>this.collectFast(request, {
59
+ ...opts
60
+ }));
61
+ this.log.verbose(`Collected ${request.foundTxs.size} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
62
+ ...blockInfo,
63
+ duration,
64
+ requestType: input.type,
65
+ missingTxs: [
66
+ ...request.missingTxHashes
67
+ ]
68
+ });
69
+ return [
70
+ ...request.foundTxs.values()
71
+ ];
72
+ }
73
+ async collectFast(request, opts) {
74
+ this.requests.add(request);
75
+ const { blockInfo } = request;
76
+ this.log.debug(`Starting fast collection of ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
77
+ ...blockInfo,
78
+ requestType: request.type,
79
+ deadline: opts.deadline
80
+ });
81
+ try {
82
+ // Start blasting all nodes for the txs. We give them a little time to respond before we start reqresp.
83
+ // And keep an eye on the request promise to ensure we don't wait longer than the deadline or return as soon
84
+ // as we have collected all txs, whatever the source.
85
+ const nodeCollectionPromise = this.collectFastFromNodes(request, opts);
86
+ const waitBeforeReqResp = sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs);
87
+ await Promise.race([
88
+ request.promise.promise,
89
+ waitBeforeReqResp
90
+ ]);
91
+ // If we have collected all txs, we can stop here
92
+ if (request.missingTxHashes.size === 0) {
93
+ this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
94
+ return;
95
+ }
96
+ // Start blasting reqresp for the remaining txs. Note that node collection keeps running in parallel.
97
+ // We stop when we have collected all txs, timed out, or both node collection and reqresp have given up.
98
+ const collectionPromise = Promise.allSettled([
99
+ this.collectFastViaReqResp(request, opts),
100
+ nodeCollectionPromise
101
+ ]);
102
+ await Promise.race([
103
+ collectionPromise,
104
+ request.promise.promise
105
+ ]);
106
+ } catch (err) {
107
+ // Log and swallow all errors
108
+ const logCtx = {
109
+ ...blockInfo,
110
+ errorMessage: err instanceof Error ? err.message : undefined,
111
+ missingTxs: [
112
+ ...request.missingTxHashes
113
+ ].map((txHash)=>txHash.toString())
114
+ };
115
+ if (err instanceof Error && err.name === 'TimeoutError') {
116
+ this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
117
+ } else if (err instanceof Error && err.name === 'AbortError') {
118
+ this.log.warn(`Aborted collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
119
+ } else {
120
+ this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, logCtx);
121
+ }
122
+ } finally{
123
+ // Ensure no unresolved promises and remove the request from the set
124
+ request.promise.resolve();
125
+ this.requests.delete(request);
126
+ }
127
+ }
128
+ /**
129
+ * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests
130
+ * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
131
+ * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
132
+ * tx is requested more than once at the same time to the same node.
133
+ */ async collectFastFromNodes(request, opts) {
134
+ if (this.nodes.length === 0) {
135
+ return;
136
+ }
137
+ // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
138
+ const attemptsPerTx = [
139
+ ...request.missingTxHashes
140
+ ].map((txHash)=>({
141
+ txHash,
142
+ attempts: 0,
143
+ found: false
144
+ }));
145
+ // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
146
+ await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx, opts)));
147
+ }
148
+ async collectFastFromNode(request, node, attemptsPerTx, opts) {
149
+ const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && request.missingTxHashes.size > 0 && this.requests.has(request);
150
+ const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
151
+ const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
152
+ const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
153
+ const processBatch = async ()=>{
154
+ while(notFinished()){
155
+ // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts,
156
+ // so we prioritize txs that have been requested less often.
157
+ const batch = [];
158
+ let index = 0;
159
+ while(batch.length < maxBatchSize){
160
+ const txToRequest = attemptsPerTx[index++];
161
+ if (!txToRequest) {
162
+ break;
163
+ } else if (!request.missingTxHashes.has(txToRequest.txHash)) {
164
+ // Mark as found if it was found somewhere else, we'll then remove it from the array.
165
+ // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
166
+ txToRequest.found = true;
167
+ } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) {
168
+ // If the tx is not alredy being requested to this node, add it to the current batch and increase attempts.
169
+ // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that
170
+ // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting
171
+ // the same txs at the same time.
172
+ batch.push(txToRequest);
173
+ activeRequestsToThisNode.add(txToRequest.txHash);
174
+ txToRequest.attempts++;
175
+ }
176
+ }
177
+ // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end.
178
+ attemptsPerTx.sort((a, b)=>a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found));
179
+ const firstFoundTxIndex = attemptsPerTx.findIndex((tx)=>tx.found);
180
+ if (firstFoundTxIndex !== -1) {
181
+ attemptsPerTx.length = firstFoundTxIndex;
182
+ }
183
+ // If we see no more txs to request, we can stop this "process" loop
184
+ if (batch.length === 0) {
185
+ return;
186
+ }
187
+ // Collect this batch from the node
188
+ await this.txCollectionSink.collect((txHashes)=>node.getTxsByHash(txHashes), batch.map(({ txHash })=>TxHash.fromString(txHash)), {
189
+ description: `fast ${node.getInfo()}`,
190
+ node: node.getInfo(),
191
+ method: 'fast-node-rpc',
192
+ ...request.blockInfo
193
+ });
194
+ // Clear from the active requests the txs we just requested
195
+ for (const requestedTx of batch){
196
+ activeRequestsToThisNode.delete(requestedTx.txHash);
197
+ }
198
+ // Sleep a bit until hitting the node again (or not, depending on config)
199
+ if (notFinished()) {
200
+ await sleep(this.config.txCollectionFastNodeIntervalMs);
201
+ }
202
+ }
203
+ };
204
+ // Kick off N parallel requests to the node, up to the maxParallelRequests limit
205
+ await Promise.all(times(maxParallelRequests, processBatch));
206
+ }
207
+ async collectFastViaReqResp(request, opts) {
208
+ const timeoutMs = +request.deadline - this.dateProvider.now();
209
+ const pinnedPeer = opts.pinnedPeer;
210
+ const maxPeers = boundInclusive(Math.ceil(request.missingTxHashes.size / 2), 8, 32);
211
+ const maxRetryAttempts = 5;
212
+ const blockInfo = request.blockInfo;
213
+ const slotNumber = blockInfo.slotNumber;
214
+ if (timeoutMs < 100) {
215
+ this.log.warn(`Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`, {
216
+ timeoutMs,
217
+ ...blockInfo
218
+ });
219
+ return;
220
+ }
221
+ this.log.debug(`Starting fast reqresp for ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
222
+ ...blockInfo,
223
+ timeoutMs,
224
+ pinnedPeer
225
+ });
226
+ try {
227
+ await this.txCollectionSink.collect(async (txHashes)=>{
228
+ const txs = await this.reqResp.sendBatchRequest(ReqRespSubProtocol.TX, chunkTxHashesRequest(txHashes), pinnedPeer, timeoutMs, maxPeers, maxRetryAttempts);
229
+ return txs.flat();
230
+ }, Array.from(request.missingTxHashes).map((txHash)=>TxHash.fromString(txHash)), {
231
+ description: `reqresp for slot ${slotNumber}`,
232
+ method: 'fast-req-resp',
233
+ ...opts,
234
+ ...request.blockInfo
235
+ });
236
+ } catch (err) {
237
+ this.log.error(`Error sending fast reqresp request for txs`, err, {
238
+ txs: [
239
+ ...request.missingTxHashes
240
+ ],
241
+ ...blockInfo
242
+ });
243
+ }
244
+ }
245
+ /**
246
+ * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
247
+ * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
248
+ */ foundTxs(txs) {
249
+ for (const request of this.requests){
250
+ for (const tx of txs){
251
+ const txHash = tx.txHash.toString();
252
+ // Remove the tx hash from the missing set, and add it to the found set.
253
+ if (request.missingTxHashes.has(txHash)) {
254
+ request.missingTxHashes.delete(txHash);
255
+ request.foundTxs.set(txHash, tx);
256
+ this.log.trace(`Found tx ${txHash} for fast collection request`, {
257
+ ...request.blockInfo,
258
+ txHash: tx.txHash.toString(),
259
+ type: request.type
260
+ });
261
+ // If we found all txs for this request, we resolve the promise
262
+ if (request.missingTxHashes.size === 0) {
263
+ this.log.trace(`All txs found for fast collection request`, {
264
+ ...request.blockInfo,
265
+ type: request.type
266
+ });
267
+ request.promise.resolve();
268
+ }
269
+ }
270
+ }
271
+ }
272
+ }
273
+ /**
274
+ * Stop collecting all txs for blocks less than or requal to the block number specified.
275
+ * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
276
+ */ stopCollectingForBlocksUpTo(blockNumber) {
277
+ for (const request of this.requests){
278
+ if (request.blockInfo.blockNumber <= blockNumber) {
279
+ request.promise.reject(new AbortError(`Stopped collecting txs up to block ${blockNumber}`));
280
+ this.requests.delete(request);
281
+ }
282
+ }
283
+ }
284
+ /**
285
+ * Stop collecting all txs for blocks greater than the block number specified.
286
+ * To be called when there is a chain prune and previously mined txs are no longer relevant.
287
+ */ stopCollectingForBlocksAfter(blockNumber) {
288
+ for (const request of this.requests){
289
+ if (request.blockInfo.blockNumber > blockNumber) {
290
+ request.promise.reject(new AbortError(`Stopped collecting txs after block ${blockNumber}`));
291
+ this.requests.delete(request);
292
+ }
293
+ }
294
+ }
295
+ }
@@ -0,0 +1,3 @@
1
+ export { TxCollection, type FastCollectionRequestInput } from './tx_collection.js';
2
+ export { type TxSource, createNodeRpcTxSources, NodeRpcTxSource } from './tx_source.js';
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,KAAK,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AACnF,OAAO,EAAE,KAAK,QAAQ,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC"}
@@ -0,0 +1,2 @@
1
+ export { TxCollection } from './tx_collection.js';
2
+ export { createNodeRpcTxSources, NodeRpcTxSource } from './tx_source.js';
@@ -0,0 +1,10 @@
1
+ import { type TelemetryClient } from '@aztec/telemetry-client';
2
+ import type { CollectionMethod } from './tx_collection.js';
3
+ export declare class TxCollectionInstrumentation {
4
+ private txsCollected;
5
+ private collectionDurationPerTx;
6
+ private collectionDurationPerRequest;
7
+ constructor(client: TelemetryClient, name: string);
8
+ increaseTxsFor(what: CollectionMethod, count: number, duration: number): void;
9
+ }
10
+ //# sourceMappingURL=instrumentation.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"instrumentation.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/instrumentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,qBAAa,2BAA2B;IACtC,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,uBAAuB,CAAY;IAC3C,OAAO,CAAC,4BAA4B,CAAY;gBAEpC,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM;IAoBjD,cAAc,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;CAMvE"}
@@ -0,0 +1,34 @@
1
+ import { Attributes, Metrics, ValueType } from '@aztec/telemetry-client';
2
+ export class TxCollectionInstrumentation {
3
+ txsCollected;
4
+ collectionDurationPerTx;
5
+ collectionDurationPerRequest;
6
+ constructor(client, name){
7
+ const meter = client.getMeter(name);
8
+ this.txsCollected = meter.createUpDownCounter(Metrics.TX_COLLECTOR_COUNT, {
9
+ description: 'The number of txs collected'
10
+ });
11
+ this.collectionDurationPerTx = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_TX, {
12
+ unit: 'ms',
13
+ description: 'Average duration per tx of an individual tx collection request',
14
+ valueType: ValueType.INT
15
+ });
16
+ this.collectionDurationPerRequest = meter.createHistogram(Metrics.TX_COLLECTOR_DURATION_PER_REQUEST, {
17
+ unit: 'ms',
18
+ description: 'Total duration of an individual tx collection request',
19
+ valueType: ValueType.INT
20
+ });
21
+ }
22
+ increaseTxsFor(what, count, duration) {
23
+ const durationPerTx = Math.ceil(duration / count);
24
+ this.collectionDurationPerTx.record(durationPerTx, {
25
+ [Attributes.TX_COLLECTION_METHOD]: what
26
+ });
27
+ this.collectionDurationPerRequest.record(Math.ceil(duration), {
28
+ [Attributes.TX_COLLECTION_METHOD]: what
29
+ });
30
+ this.txsCollected.add(count, {
31
+ [Attributes.TX_COLLECTION_METHOD]: what
32
+ });
33
+ }
34
+ }
@@ -0,0 +1,54 @@
1
+ import { type Logger } from '@aztec/foundation/log';
2
+ import { DateProvider } from '@aztec/foundation/timer';
3
+ import type { L2Block } from '@aztec/stdlib/block';
4
+ import { type L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
5
+ import { type Tx, TxHash } from '@aztec/stdlib/tx';
6
+ import { type ReqRespInterface } from '../reqresp/interface.js';
7
+ import type { TxCollectionConfig } from './config.js';
8
+ import type { FastTxCollection } from './fast_tx_collection.js';
9
+ import type { TxCollectionSink } from './tx_collection_sink.js';
10
+ import type { TxSource } from './tx_source.js';
11
+ export declare class SlowTxCollection {
12
+ private reqResp;
13
+ private nodes;
14
+ private txCollectionSink;
15
+ private fastCollection;
16
+ private constants;
17
+ private config;
18
+ private dateProvider;
19
+ private log;
20
+ /** Map from tx hash to missing tx info to collect via slow loop */
21
+ private missingTxs;
22
+ /** One slow collection loop for each node tx source we have */
23
+ private nodesSlowCollectionLoops;
24
+ /** Slow collection loop for reqresp which collects txs from peers */
25
+ private reqrespSlowCollectionLoop;
26
+ constructor(reqResp: Pick<ReqRespInterface, 'sendBatchRequest'>, nodes: TxSource[], txCollectionSink: TxCollectionSink, fastCollection: Pick<FastTxCollection, 'getFastCollectionRequests'>, constants: L1RollupConstants, config: TxCollectionConfig, dateProvider?: DateProvider, log?: Logger);
27
+ getMissingTxHashes(): TxHash[];
28
+ start(): void;
29
+ stop(): Promise<void>;
30
+ trigger(): Promise<void>;
31
+ /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
32
+ startCollecting(block: L2Block, txHashes: TxHash[]): void;
33
+ /** Entrypoint for the node slow collection loop */
34
+ private collectMissingTxsFromNode;
35
+ /** Entrypoint for the reqresp slow collection loop */
36
+ private collectMissingTxsViaReqResp;
37
+ /** Retrieves all missing txs for the slow collection process. This is, all missing txs that are not part of a fast request. */
38
+ private getMissingTxsForSlowCollection;
39
+ /** Stop collecting the given txs since we have found them. Called whenever tx pool emits a tx-added event. */
40
+ foundTxs(txs: Tx[]): void;
41
+ /**
42
+ * Stop collecting all txs for blocks less than or requal to the block number specified.
43
+ * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
44
+ */
45
+ stopCollectingForBlocksUpTo(blockNumber: number): void;
46
+ /**
47
+ * Stop collecting all txs for blocks greater than the block number specified.
48
+ * To be called when there is a chain prune and previously mined txs are no longer relevant.
49
+ */
50
+ stopCollectingForBlocksAfter(blockNumber: number): void;
51
+ /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */
52
+ private getDeadlineForSlot;
53
+ }
54
+ //# sourceMappingURL=slow_tx_collection.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"slow_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/slow_tx_collection.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,KAAK,iBAAiB,EAA6C,MAAM,6BAA6B,CAAC;AAChH,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,KAAK,gBAAgB,EAAsB,MAAM,yBAAyB,CAAC;AAEpF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAWzB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IAjBb,mEAAmE;IACnE,OAAO,CAAC,UAAU,CAAyC;IAE3D,+DAA+D;IAC/D,OAAO,CAAC,wBAAwB,CAAmB;IAEnD,qEAAqE;IACrE,OAAO,CAAC,yBAAyB,CAAiB;gBAGxC,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,EACnD,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,cAAc,EAAE,IAAI,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,EACnE,SAAS,EAAE,iBAAiB,EAC5B,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD;IAkB1D,kBAAkB;IAIlB,KAAK;IAKC,IAAI;IAOJ,OAAO;IAOpB,mFAAmF;IAC5E,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE;IAsBzD,mDAAmD;YACrC,yBAAyB;IAgCvC,sDAAsD;YACxC,2BAA2B;IAsCzC,+HAA+H;IAC/H,OAAO,CAAC,8BAA8B;IAqBtC,8GAA8G;IACvG,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,IAAI;IAMhC;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAQ7D;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAQ9D,oIAAoI;IACpI,OAAO,CAAC,kBAAkB;CAM3B"}