@aztec/p2p 0.0.1-commit.4d79d1f2d → 0.0.1-commit.4d9804df

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (499) hide show
  1. package/README.md +130 -3
  2. package/dest/bootstrap/bootstrap.d.ts +1 -1
  3. package/dest/bootstrap/bootstrap.d.ts.map +1 -1
  4. package/dest/bootstrap/bootstrap.js +22 -6
  5. package/dest/client/factory.d.ts +9 -8
  6. package/dest/client/factory.d.ts.map +1 -1
  7. package/dest/client/factory.js +50 -34
  8. package/dest/client/interface.d.ts +59 -27
  9. package/dest/client/interface.d.ts.map +1 -1
  10. package/dest/client/p2p_client.d.ts +40 -34
  11. package/dest/client/p2p_client.d.ts.map +1 -1
  12. package/dest/client/p2p_client.js +162 -135
  13. package/dest/config.d.ts +164 -100
  14. package/dest/config.d.ts.map +1 -1
  15. package/dest/config.js +149 -49
  16. package/dest/errors/p2p-service.error.d.ts +9 -0
  17. package/dest/errors/p2p-service.error.d.ts.map +1 -0
  18. package/dest/errors/p2p-service.error.js +10 -0
  19. package/dest/errors/reqresp.error.d.ts +9 -16
  20. package/dest/errors/reqresp.error.d.ts.map +1 -1
  21. package/dest/errors/reqresp.error.js +10 -17
  22. package/dest/errors/tx-pool.error.d.ts +8 -0
  23. package/dest/errors/tx-pool.error.d.ts.map +1 -0
  24. package/dest/errors/tx-pool.error.js +9 -0
  25. package/dest/index.d.ts +1 -2
  26. package/dest/index.d.ts.map +1 -1
  27. package/dest/index.js +0 -1
  28. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts +103 -59
  29. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts.map +1 -1
  30. package/dest/mem_pools/attestation_pool/attestation_pool.js +278 -196
  31. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts +1 -1
  32. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts.map +1 -1
  33. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +224 -65
  34. package/dest/mem_pools/attestation_pool/mocks.d.ts +2 -2
  35. package/dest/mem_pools/attestation_pool/mocks.d.ts.map +1 -1
  36. package/dest/mem_pools/attestation_pool/mocks.js +7 -5
  37. package/dest/mem_pools/index.d.ts +1 -2
  38. package/dest/mem_pools/index.d.ts.map +1 -1
  39. package/dest/mem_pools/instrumentation.d.ts +12 -5
  40. package/dest/mem_pools/instrumentation.d.ts.map +1 -1
  41. package/dest/mem_pools/instrumentation.js +38 -17
  42. package/dest/mem_pools/tx_pool_v2/archive/tx_archive.d.ts +1 -5
  43. package/dest/mem_pools/tx_pool_v2/archive/tx_archive.d.ts.map +1 -1
  44. package/dest/mem_pools/tx_pool_v2/archive/tx_archive.js +1 -7
  45. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts +32 -15
  46. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts.map +1 -1
  47. package/dest/mem_pools/tx_pool_v2/deleted_pool.js +95 -24
  48. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts +3 -3
  49. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts.map +1 -1
  50. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.js +18 -9
  51. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.d.ts +1 -1
  52. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.d.ts.map +1 -1
  53. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.js +8 -4
  54. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts +3 -3
  55. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts.map +1 -1
  56. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.js +12 -4
  57. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts +3 -2
  58. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts.map +1 -1
  59. package/dest/mem_pools/tx_pool_v2/eviction/index.js +2 -1
  60. package/dest/mem_pools/tx_pool_v2/eviction/insufficient_fee_per_gas_eviction_rule.d.ts +16 -0
  61. package/dest/mem_pools/tx_pool_v2/eviction/insufficient_fee_per_gas_eviction_rule.d.ts.map +1 -0
  62. package/dest/mem_pools/tx_pool_v2/eviction/insufficient_fee_per_gas_eviction_rule.js +62 -0
  63. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts +54 -5
  64. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts.map +1 -1
  65. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.js +8 -0
  66. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.js +7 -5
  67. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.js +9 -7
  68. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts +2 -2
  69. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts.map +1 -1
  70. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.js +14 -6
  71. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts +4 -4
  72. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts.map +1 -1
  73. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.js +16 -4
  74. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts +3 -3
  75. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts.map +1 -1
  76. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.js +3 -3
  77. package/dest/mem_pools/tx_pool_v2/index.d.ts +2 -2
  78. package/dest/mem_pools/tx_pool_v2/index.d.ts.map +1 -1
  79. package/dest/mem_pools/tx_pool_v2/index.js +1 -1
  80. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts +15 -0
  81. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts.map +1 -0
  82. package/dest/mem_pools/tx_pool_v2/instrumentation.js +43 -0
  83. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +60 -14
  84. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
  85. package/dest/mem_pools/tx_pool_v2/interfaces.js +6 -1
  86. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +59 -15
  87. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -1
  88. package/dest/mem_pools/tx_pool_v2/tx_metadata.js +116 -20
  89. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +26 -3
  90. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
  91. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +79 -43
  92. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +20 -7
  93. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
  94. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +34 -12
  95. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +29 -7
  96. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
  97. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +473 -211
  98. package/dest/msg_validators/attestation_validator/attestation_validator.d.ts +10 -3
  99. package/dest/msg_validators/attestation_validator/attestation_validator.d.ts.map +1 -1
  100. package/dest/msg_validators/attestation_validator/attestation_validator.js +38 -15
  101. package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.d.ts +7 -3
  102. package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.d.ts.map +1 -1
  103. package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.js +4 -5
  104. package/dest/msg_validators/proposal_validator/block_proposal_validator.d.ts +12 -5
  105. package/dest/msg_validators/proposal_validator/block_proposal_validator.d.ts.map +1 -1
  106. package/dest/msg_validators/proposal_validator/block_proposal_validator.js +11 -3
  107. package/dest/msg_validators/proposal_validator/checkpoint_proposal_validator.d.ts +12 -5
  108. package/dest/msg_validators/proposal_validator/checkpoint_proposal_validator.d.ts.map +1 -1
  109. package/dest/msg_validators/proposal_validator/checkpoint_proposal_validator.js +23 -3
  110. package/dest/msg_validators/proposal_validator/proposal_validator.d.ts +36 -9
  111. package/dest/msg_validators/proposal_validator/proposal_validator.d.ts.map +1 -1
  112. package/dest/msg_validators/proposal_validator/proposal_validator.js +130 -48
  113. package/dest/msg_validators/tx_validator/aggregate_tx_validator.d.ts +2 -2
  114. package/dest/msg_validators/tx_validator/aggregate_tx_validator.d.ts.map +1 -1
  115. package/dest/msg_validators/tx_validator/aggregate_tx_validator.js +11 -18
  116. package/dest/msg_validators/tx_validator/allowed_public_setup.d.ts +2 -1
  117. package/dest/msg_validators/tx_validator/allowed_public_setup.d.ts.map +1 -1
  118. package/dest/msg_validators/tx_validator/allowed_public_setup.js +25 -21
  119. package/dest/msg_validators/tx_validator/allowed_setup_helpers.d.ts +17 -0
  120. package/dest/msg_validators/tx_validator/allowed_setup_helpers.d.ts.map +1 -0
  121. package/dest/msg_validators/tx_validator/allowed_setup_helpers.js +24 -0
  122. package/dest/msg_validators/tx_validator/archive_cache.js +1 -1
  123. package/dest/msg_validators/tx_validator/cached_tx_validator.d.ts +15 -0
  124. package/dest/msg_validators/tx_validator/cached_tx_validator.d.ts.map +1 -0
  125. package/dest/msg_validators/tx_validator/cached_tx_validator.js +19 -0
  126. package/dest/msg_validators/tx_validator/contract_instance_validator.d.ts +9 -0
  127. package/dest/msg_validators/tx_validator/contract_instance_validator.d.ts.map +1 -0
  128. package/dest/msg_validators/tx_validator/contract_instance_validator.js +48 -0
  129. package/dest/msg_validators/tx_validator/data_validator.d.ts +2 -1
  130. package/dest/msg_validators/tx_validator/data_validator.d.ts.map +1 -1
  131. package/dest/msg_validators/tx_validator/data_validator.js +36 -2
  132. package/dest/msg_validators/tx_validator/factory.d.ts +133 -7
  133. package/dest/msg_validators/tx_validator/factory.d.ts.map +1 -1
  134. package/dest/msg_validators/tx_validator/factory.js +261 -61
  135. package/dest/msg_validators/tx_validator/fee_payer_balance.d.ts +1 -1
  136. package/dest/msg_validators/tx_validator/fee_payer_balance.d.ts.map +1 -1
  137. package/dest/msg_validators/tx_validator/fee_payer_balance.js +6 -2
  138. package/dest/msg_validators/tx_validator/gas_validator.d.ts +104 -3
  139. package/dest/msg_validators/tx_validator/gas_validator.d.ts.map +1 -1
  140. package/dest/msg_validators/tx_validator/gas_validator.js +137 -53
  141. package/dest/msg_validators/tx_validator/index.d.ts +5 -1
  142. package/dest/msg_validators/tx_validator/index.d.ts.map +1 -1
  143. package/dest/msg_validators/tx_validator/index.js +4 -0
  144. package/dest/msg_validators/tx_validator/metadata_validator.d.ts +1 -1
  145. package/dest/msg_validators/tx_validator/metadata_validator.d.ts.map +1 -1
  146. package/dest/msg_validators/tx_validator/metadata_validator.js +4 -4
  147. package/dest/msg_validators/tx_validator/nullifier_cache.d.ts +14 -0
  148. package/dest/msg_validators/tx_validator/nullifier_cache.d.ts.map +1 -0
  149. package/dest/msg_validators/tx_validator/nullifier_cache.js +24 -0
  150. package/dest/msg_validators/tx_validator/phases_validator.d.ts +22 -2
  151. package/dest/msg_validators/tx_validator/phases_validator.d.ts.map +1 -1
  152. package/dest/msg_validators/tx_validator/phases_validator.js +72 -24
  153. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts +2 -2
  154. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts.map +1 -1
  155. package/dest/msg_validators/tx_validator/timestamp_validator.js +6 -6
  156. package/dest/msg_validators/tx_validator/tx_proof_validator.d.ts +2 -1
  157. package/dest/msg_validators/tx_validator/tx_proof_validator.d.ts.map +1 -1
  158. package/dest/msg_validators/tx_validator/tx_proof_validator.js +2 -0
  159. package/dest/msg_validators/tx_validator/tx_validation_cache.d.ts +48 -0
  160. package/dest/msg_validators/tx_validator/tx_validation_cache.d.ts.map +1 -0
  161. package/dest/msg_validators/tx_validator/tx_validation_cache.js +68 -0
  162. package/dest/services/data_store.d.ts +1 -1
  163. package/dest/services/data_store.d.ts.map +1 -1
  164. package/dest/services/data_store.js +5 -5
  165. package/dest/services/discv5/discV5_service.d.ts +13 -2
  166. package/dest/services/discv5/discV5_service.d.ts.map +1 -1
  167. package/dest/services/discv5/discV5_service.js +119 -23
  168. package/dest/services/discv5/persisted_enr_store.d.ts +38 -0
  169. package/dest/services/discv5/persisted_enr_store.d.ts.map +1 -0
  170. package/dest/services/discv5/persisted_enr_store.js +145 -0
  171. package/dest/services/dummy_service.d.ts +18 -18
  172. package/dest/services/dummy_service.d.ts.map +1 -1
  173. package/dest/services/dummy_service.js +19 -20
  174. package/dest/services/encoding.d.ts +23 -6
  175. package/dest/services/encoding.d.ts.map +1 -1
  176. package/dest/services/encoding.js +41 -24
  177. package/dest/services/gossipsub/topic_score_params.d.ts +23 -17
  178. package/dest/services/gossipsub/topic_score_params.d.ts.map +1 -1
  179. package/dest/services/gossipsub/topic_score_params.js +36 -24
  180. package/dest/services/libp2p/instrumentation.d.ts +3 -1
  181. package/dest/services/libp2p/instrumentation.d.ts.map +1 -1
  182. package/dest/services/libp2p/instrumentation.js +14 -0
  183. package/dest/services/libp2p/libp2p_service.d.ts +57 -56
  184. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  185. package/dest/services/libp2p/libp2p_service.js +483 -373
  186. package/dest/services/peer-manager/metrics.d.ts +3 -1
  187. package/dest/services/peer-manager/metrics.d.ts.map +1 -1
  188. package/dest/services/peer-manager/metrics.js +6 -0
  189. package/dest/services/peer-manager/peer_manager.d.ts +20 -3
  190. package/dest/services/peer-manager/peer_manager.d.ts.map +1 -1
  191. package/dest/services/peer-manager/peer_manager.js +83 -26
  192. package/dest/services/peer-manager/peer_scoring.d.ts +34 -2
  193. package/dest/services/peer-manager/peer_scoring.d.ts.map +1 -1
  194. package/dest/services/peer-manager/peer_scoring.js +102 -13
  195. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts +5 -8
  196. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts.map +1 -1
  197. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js +95 -127
  198. package/dest/services/reqresp/batch-tx-requester/interface.d.ts +11 -11
  199. package/dest/services/reqresp/batch-tx-requester/interface.d.ts.map +1 -1
  200. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts +11 -13
  201. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts.map +1 -1
  202. package/dest/services/reqresp/batch-tx-requester/missing_txs.js +31 -46
  203. package/dest/services/reqresp/batch-tx-requester/peer_collection.d.ts +19 -11
  204. package/dest/services/reqresp/batch-tx-requester/peer_collection.d.ts.map +1 -1
  205. package/dest/services/reqresp/batch-tx-requester/peer_collection.js +52 -15
  206. package/dest/services/reqresp/batch-tx-requester/tx_validator.d.ts +5 -14
  207. package/dest/services/reqresp/batch-tx-requester/tx_validator.d.ts.map +1 -1
  208. package/dest/services/reqresp/batch-tx-requester/tx_validator.js +6 -20
  209. package/dest/services/reqresp/config.d.ts +15 -3
  210. package/dest/services/reqresp/config.d.ts.map +1 -1
  211. package/dest/services/reqresp/config.js +11 -0
  212. package/dest/services/reqresp/interface.d.ts +16 -18
  213. package/dest/services/reqresp/interface.d.ts.map +1 -1
  214. package/dest/services/reqresp/interface.js +10 -20
  215. package/dest/services/reqresp/metrics.d.ts +1 -1
  216. package/dest/services/reqresp/metrics.d.ts.map +1 -1
  217. package/dest/services/reqresp/metrics.js +0 -1
  218. package/dest/services/reqresp/protocols/block_txs/bitvector.d.ts +4 -3
  219. package/dest/services/reqresp/protocols/block_txs/bitvector.d.ts.map +1 -1
  220. package/dest/services/reqresp/protocols/block_txs/bitvector.js +3 -2
  221. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.d.ts +1 -1
  222. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.d.ts.map +1 -1
  223. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.js +31 -28
  224. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts +12 -7
  225. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts.map +1 -1
  226. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.js +42 -24
  227. package/dest/services/reqresp/protocols/index.d.ts +1 -2
  228. package/dest/services/reqresp/protocols/index.d.ts.map +1 -1
  229. package/dest/services/reqresp/protocols/index.js +0 -1
  230. package/dest/services/reqresp/protocols/tx.d.ts +1 -1
  231. package/dest/services/reqresp/protocols/tx.d.ts.map +1 -1
  232. package/dest/services/reqresp/protocols/tx.js +1 -3
  233. package/dest/services/reqresp/rate-limiter/rate_limiter.d.ts +5 -4
  234. package/dest/services/reqresp/rate-limiter/rate_limiter.d.ts.map +1 -1
  235. package/dest/services/reqresp/rate-limiter/rate_limiter.js +10 -8
  236. package/dest/services/reqresp/rate-limiter/rate_limits.d.ts +1 -1
  237. package/dest/services/reqresp/rate-limiter/rate_limits.d.ts.map +1 -1
  238. package/dest/services/reqresp/rate-limiter/rate_limits.js +3 -13
  239. package/dest/services/reqresp/reqresp.d.ts +7 -29
  240. package/dest/services/reqresp/reqresp.d.ts.map +1 -1
  241. package/dest/services/reqresp/reqresp.js +75 -216
  242. package/dest/services/service.d.ts +29 -14
  243. package/dest/services/service.d.ts.map +1 -1
  244. package/dest/services/tx_collection/config.d.ts +8 -17
  245. package/dest/services/tx_collection/config.d.ts.map +1 -1
  246. package/dest/services/tx_collection/config.js +17 -40
  247. package/dest/services/tx_collection/file_store_tx_collection.d.ts +25 -32
  248. package/dest/services/tx_collection/file_store_tx_collection.d.ts.map +1 -1
  249. package/dest/services/tx_collection/file_store_tx_collection.js +96 -87
  250. package/dest/services/tx_collection/file_store_tx_source.d.ts +17 -6
  251. package/dest/services/tx_collection/file_store_tx_source.d.ts.map +1 -1
  252. package/dest/services/tx_collection/file_store_tx_source.js +53 -10
  253. package/dest/services/tx_collection/index.d.ts +2 -3
  254. package/dest/services/tx_collection/index.d.ts.map +1 -1
  255. package/dest/services/tx_collection/index.js +0 -1
  256. package/dest/services/tx_collection/instrumentation.js +1 -2
  257. package/dest/services/tx_collection/request_tracker.d.ts +53 -0
  258. package/dest/services/tx_collection/request_tracker.d.ts.map +1 -0
  259. package/dest/services/tx_collection/request_tracker.js +84 -0
  260. package/dest/services/tx_collection/tx_collection.d.ts +39 -55
  261. package/dest/services/tx_collection/tx_collection.d.ts.map +1 -1
  262. package/dest/services/tx_collection/tx_collection.js +280 -108
  263. package/dest/services/tx_collection/tx_collection_sink.d.ts +7 -6
  264. package/dest/services/tx_collection/tx_collection_sink.d.ts.map +1 -1
  265. package/dest/services/tx_collection/tx_collection_sink.js +15 -24
  266. package/dest/services/tx_collection/tx_source.d.ts +13 -7
  267. package/dest/services/tx_collection/tx_source.d.ts.map +1 -1
  268. package/dest/services/tx_collection/tx_source.js +29 -7
  269. package/dest/services/tx_file_store/tx_file_store.d.ts +3 -4
  270. package/dest/services/tx_file_store/tx_file_store.d.ts.map +1 -1
  271. package/dest/services/tx_file_store/tx_file_store.js +13 -20
  272. package/dest/services/tx_provider.d.ts +5 -3
  273. package/dest/services/tx_provider.d.ts.map +1 -1
  274. package/dest/services/tx_provider.js +7 -4
  275. package/dest/test-helpers/make-test-p2p-clients.d.ts +5 -6
  276. package/dest/test-helpers/make-test-p2p-clients.d.ts.map +1 -1
  277. package/dest/test-helpers/make-test-p2p-clients.js +5 -3
  278. package/dest/test-helpers/mock-pubsub.d.ts +26 -12
  279. package/dest/test-helpers/mock-pubsub.d.ts.map +1 -1
  280. package/dest/test-helpers/mock-pubsub.js +51 -45
  281. package/dest/test-helpers/reqresp-nodes.d.ts +5 -7
  282. package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
  283. package/dest/test-helpers/reqresp-nodes.js +17 -19
  284. package/dest/test-helpers/test_tx_provider.d.ts +3 -1
  285. package/dest/test-helpers/test_tx_provider.d.ts.map +1 -1
  286. package/dest/test-helpers/test_tx_provider.js +3 -0
  287. package/dest/test-helpers/testbench-utils.d.ts +22 -16
  288. package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
  289. package/dest/test-helpers/testbench-utils.js +56 -16
  290. package/dest/testbench/p2p_client_testbench_worker.d.ts +3 -5
  291. package/dest/testbench/p2p_client_testbench_worker.d.ts.map +1 -1
  292. package/dest/testbench/p2p_client_testbench_worker.js +111 -48
  293. package/dest/testbench/worker_client_manager.d.ts +12 -6
  294. package/dest/testbench/worker_client_manager.d.ts.map +1 -1
  295. package/dest/testbench/worker_client_manager.js +57 -11
  296. package/dest/util.d.ts +13 -8
  297. package/dest/util.d.ts.map +1 -1
  298. package/dest/util.js +35 -14
  299. package/dest/versioning.d.ts +3 -6
  300. package/dest/versioning.d.ts.map +1 -1
  301. package/dest/versioning.js +3 -24
  302. package/package.json +16 -16
  303. package/src/bootstrap/bootstrap.ts +17 -3
  304. package/src/client/factory.ts +95 -51
  305. package/src/client/interface.ts +59 -35
  306. package/src/client/p2p_client.ts +186 -196
  307. package/src/client/test/{tx_proposal_collector/README.md → p2p_client.batch_tx_requester.bench.README.md} +23 -53
  308. package/src/config.ts +259 -57
  309. package/src/errors/p2p-service.error.ts +11 -0
  310. package/src/errors/reqresp.error.ts +13 -19
  311. package/src/errors/tx-pool.error.ts +12 -0
  312. package/src/index.ts +0 -1
  313. package/src/mem_pools/attestation_pool/attestation_pool.ts +336 -242
  314. package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +244 -69
  315. package/src/mem_pools/attestation_pool/mocks.ts +14 -8
  316. package/src/mem_pools/index.ts +0 -3
  317. package/src/mem_pools/instrumentation.ts +27 -17
  318. package/src/mem_pools/tx_pool_v2/README.md +52 -28
  319. package/src/mem_pools/tx_pool_v2/archive/tx_archive.ts +1 -9
  320. package/src/mem_pools/tx_pool_v2/deleted_pool.ts +114 -27
  321. package/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts +21 -8
  322. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts +8 -4
  323. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts +18 -4
  324. package/src/mem_pools/tx_pool_v2/eviction/index.ts +5 -0
  325. package/src/mem_pools/tx_pool_v2/eviction/insufficient_fee_per_gas_eviction_rule.ts +65 -0
  326. package/src/mem_pools/tx_pool_v2/eviction/interfaces.ts +59 -4
  327. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts +5 -5
  328. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts +8 -8
  329. package/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts +14 -9
  330. package/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts +33 -6
  331. package/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts +4 -3
  332. package/src/mem_pools/tx_pool_v2/index.ts +1 -1
  333. package/src/mem_pools/tx_pool_v2/instrumentation.ts +69 -0
  334. package/src/mem_pools/tx_pool_v2/interfaces.ts +61 -14
  335. package/src/mem_pools/tx_pool_v2/tx_metadata.ts +170 -28
  336. package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +91 -43
  337. package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +54 -14
  338. package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +527 -207
  339. package/src/msg_validators/attestation_validator/README.md +49 -0
  340. package/src/msg_validators/attestation_validator/attestation_validator.ts +46 -14
  341. package/src/msg_validators/attestation_validator/fisherman_attestation_validator.ts +15 -7
  342. package/src/msg_validators/proposal_validator/README.md +123 -0
  343. package/src/msg_validators/proposal_validator/block_proposal_validator.ts +26 -4
  344. package/src/msg_validators/proposal_validator/checkpoint_proposal_validator.ts +43 -7
  345. package/src/msg_validators/proposal_validator/proposal_validator.ts +155 -49
  346. package/src/msg_validators/tx_validator/README.md +129 -0
  347. package/src/msg_validators/tx_validator/aggregate_tx_validator.ts +6 -15
  348. package/src/msg_validators/tx_validator/allowed_public_setup.ts +22 -27
  349. package/src/msg_validators/tx_validator/allowed_setup_helpers.ts +31 -0
  350. package/src/msg_validators/tx_validator/archive_cache.ts +1 -1
  351. package/src/msg_validators/tx_validator/cached_tx_validator.ts +31 -0
  352. package/src/msg_validators/tx_validator/contract_instance_validator.ts +56 -0
  353. package/src/msg_validators/tx_validator/data_validator.ts +44 -1
  354. package/src/msg_validators/tx_validator/factory.ts +413 -80
  355. package/src/msg_validators/tx_validator/fee_payer_balance.ts +6 -2
  356. package/src/msg_validators/tx_validator/gas_validator.ts +195 -54
  357. package/src/msg_validators/tx_validator/index.ts +4 -0
  358. package/src/msg_validators/tx_validator/metadata_validator.ts +12 -4
  359. package/src/msg_validators/tx_validator/nullifier_cache.ts +30 -0
  360. package/src/msg_validators/tx_validator/phases_validator.ts +82 -27
  361. package/src/msg_validators/tx_validator/timestamp_validator.ts +7 -7
  362. package/src/msg_validators/tx_validator/tx_proof_validator.ts +2 -0
  363. package/src/msg_validators/tx_validator/tx_validation_cache.ts +101 -0
  364. package/src/services/data_store.ts +5 -13
  365. package/src/services/discv5/discV5_service.ts +126 -14
  366. package/src/services/discv5/persisted_enr_store.ts +158 -0
  367. package/src/services/dummy_service.ts +26 -45
  368. package/src/services/encoding.ts +40 -24
  369. package/src/services/gossipsub/README.md +59 -22
  370. package/src/services/gossipsub/topic_score_params.ts +58 -31
  371. package/src/services/libp2p/instrumentation.ts +14 -0
  372. package/src/services/libp2p/libp2p_service.ts +554 -428
  373. package/src/services/peer-manager/metrics.ts +7 -0
  374. package/src/services/peer-manager/peer_manager.ts +91 -25
  375. package/src/services/peer-manager/peer_scoring.ts +102 -8
  376. package/src/services/reqresp/README.md +194 -0
  377. package/src/services/reqresp/batch-tx-requester/README.md +46 -7
  378. package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts +92 -138
  379. package/src/services/reqresp/batch-tx-requester/interface.ts +14 -10
  380. package/src/services/reqresp/batch-tx-requester/missing_txs.ts +30 -71
  381. package/src/services/reqresp/batch-tx-requester/peer_collection.ts +68 -24
  382. package/src/services/reqresp/batch-tx-requester/tx_validator.ts +12 -25
  383. package/src/services/reqresp/config.ts +19 -2
  384. package/src/services/reqresp/interface.ts +21 -47
  385. package/src/services/reqresp/metrics.ts +0 -1
  386. package/src/services/reqresp/protocols/block_txs/bitvector.ts +3 -2
  387. package/src/services/reqresp/protocols/block_txs/block_txs_handler.ts +34 -30
  388. package/src/services/reqresp/protocols/block_txs/block_txs_reqresp.ts +47 -20
  389. package/src/services/reqresp/protocols/index.ts +0 -1
  390. package/src/services/reqresp/protocols/tx.ts +1 -3
  391. package/src/services/reqresp/rate-limiter/rate_limiter.ts +13 -9
  392. package/src/services/reqresp/rate-limiter/rate_limits.ts +3 -13
  393. package/src/services/reqresp/reqresp.ts +87 -260
  394. package/src/services/service.ts +42 -30
  395. package/src/services/tx_collection/config.ts +24 -59
  396. package/src/services/tx_collection/file_store_tx_collection.ts +108 -107
  397. package/src/services/tx_collection/file_store_tx_source.ts +69 -10
  398. package/src/services/tx_collection/index.ts +1 -6
  399. package/src/services/tx_collection/instrumentation.ts +1 -1
  400. package/src/services/tx_collection/request_tracker.ts +127 -0
  401. package/src/services/tx_collection/tx_collection.ts +342 -156
  402. package/src/services/tx_collection/tx_collection_sink.ts +17 -31
  403. package/src/services/tx_collection/tx_source.ts +29 -8
  404. package/src/services/tx_file_store/tx_file_store.ts +11 -21
  405. package/src/services/tx_provider.ts +7 -2
  406. package/src/test-helpers/make-test-p2p-clients.ts +4 -3
  407. package/src/test-helpers/mock-pubsub.ts +59 -66
  408. package/src/test-helpers/reqresp-nodes.ts +15 -28
  409. package/src/test-helpers/test_tx_provider.ts +5 -0
  410. package/src/test-helpers/testbench-utils.ts +74 -31
  411. package/src/testbench/p2p_client_testbench_worker.ts +120 -63
  412. package/src/testbench/worker_client_manager.ts +72 -25
  413. package/src/util.ts +40 -19
  414. package/src/versioning.ts +3 -33
  415. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.d.ts +0 -2
  416. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.d.ts.map +0 -1
  417. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +0 -305
  418. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.d.ts +0 -73
  419. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.d.ts.map +0 -1
  420. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.js +0 -8
  421. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts +0 -125
  422. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts.map +0 -1
  423. package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.js +0 -596
  424. package/dest/mem_pools/tx_pool/eviction/eviction_manager.d.ts +0 -32
  425. package/dest/mem_pools/tx_pool/eviction/eviction_manager.d.ts.map +0 -1
  426. package/dest/mem_pools/tx_pool/eviction/eviction_manager.js +0 -112
  427. package/dest/mem_pools/tx_pool/eviction/eviction_strategy.d.ts +0 -157
  428. package/dest/mem_pools/tx_pool/eviction/eviction_strategy.d.ts.map +0 -1
  429. package/dest/mem_pools/tx_pool/eviction/eviction_strategy.js +0 -52
  430. package/dest/mem_pools/tx_pool/eviction/fee_payer_balance_eviction_rule.d.ts +0 -16
  431. package/dest/mem_pools/tx_pool/eviction/fee_payer_balance_eviction_rule.d.ts.map +0 -1
  432. package/dest/mem_pools/tx_pool/eviction/fee_payer_balance_eviction_rule.js +0 -122
  433. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.d.ts +0 -17
  434. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.d.ts.map +0 -1
  435. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.js +0 -84
  436. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_reorg_rule.d.ts +0 -19
  437. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_reorg_rule.d.ts.map +0 -1
  438. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_reorg_rule.js +0 -78
  439. package/dest/mem_pools/tx_pool/eviction/low_priority_eviction_rule.d.ts +0 -26
  440. package/dest/mem_pools/tx_pool/eviction/low_priority_eviction_rule.d.ts.map +0 -1
  441. package/dest/mem_pools/tx_pool/eviction/low_priority_eviction_rule.js +0 -84
  442. package/dest/mem_pools/tx_pool/eviction/nullifier_conflict_pre_add_rule.d.ts +0 -25
  443. package/dest/mem_pools/tx_pool/eviction/nullifier_conflict_pre_add_rule.d.ts.map +0 -1
  444. package/dest/mem_pools/tx_pool/eviction/nullifier_conflict_pre_add_rule.js +0 -57
  445. package/dest/mem_pools/tx_pool/index.d.ts +0 -3
  446. package/dest/mem_pools/tx_pool/index.d.ts.map +0 -1
  447. package/dest/mem_pools/tx_pool/index.js +0 -2
  448. package/dest/mem_pools/tx_pool/priority.d.ts +0 -12
  449. package/dest/mem_pools/tx_pool/priority.d.ts.map +0 -1
  450. package/dest/mem_pools/tx_pool/priority.js +0 -15
  451. package/dest/mem_pools/tx_pool/tx_pool.d.ts +0 -127
  452. package/dest/mem_pools/tx_pool/tx_pool.d.ts.map +0 -1
  453. package/dest/mem_pools/tx_pool/tx_pool.js +0 -3
  454. package/dest/mem_pools/tx_pool/tx_pool_test_suite.d.ts +0 -7
  455. package/dest/mem_pools/tx_pool/tx_pool_test_suite.d.ts.map +0 -1
  456. package/dest/mem_pools/tx_pool/tx_pool_test_suite.js +0 -400
  457. package/dest/msg_validators/clock_tolerance.d.ts +0 -21
  458. package/dest/msg_validators/clock_tolerance.d.ts.map +0 -1
  459. package/dest/msg_validators/clock_tolerance.js +0 -37
  460. package/dest/msg_validators/proposal_validator/proposal_validator_test_suite.d.ts +0 -23
  461. package/dest/msg_validators/proposal_validator/proposal_validator_test_suite.d.ts.map +0 -1
  462. package/dest/msg_validators/proposal_validator/proposal_validator_test_suite.js +0 -212
  463. package/dest/services/reqresp/connection-sampler/batch_connection_sampler.d.ts +0 -64
  464. package/dest/services/reqresp/connection-sampler/batch_connection_sampler.d.ts.map +0 -1
  465. package/dest/services/reqresp/connection-sampler/batch_connection_sampler.js +0 -151
  466. package/dest/services/reqresp/protocols/block.d.ts +0 -9
  467. package/dest/services/reqresp/protocols/block.d.ts.map +0 -1
  468. package/dest/services/reqresp/protocols/block.js +0 -32
  469. package/dest/services/tx_collection/fast_tx_collection.d.ts +0 -54
  470. package/dest/services/tx_collection/fast_tx_collection.d.ts.map +0 -1
  471. package/dest/services/tx_collection/fast_tx_collection.js +0 -321
  472. package/dest/services/tx_collection/proposal_tx_collector.d.ts +0 -48
  473. package/dest/services/tx_collection/proposal_tx_collector.d.ts.map +0 -1
  474. package/dest/services/tx_collection/proposal_tx_collector.js +0 -49
  475. package/dest/services/tx_collection/slow_tx_collection.d.ts +0 -55
  476. package/dest/services/tx_collection/slow_tx_collection.d.ts.map +0 -1
  477. package/dest/services/tx_collection/slow_tx_collection.js +0 -206
  478. package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +0 -336
  479. package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.ts +0 -43
  480. package/src/mem_pools/tx_pool/README.md +0 -270
  481. package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts +0 -746
  482. package/src/mem_pools/tx_pool/eviction/eviction_manager.ts +0 -132
  483. package/src/mem_pools/tx_pool/eviction/eviction_strategy.ts +0 -208
  484. package/src/mem_pools/tx_pool/eviction/fee_payer_balance_eviction_rule.ts +0 -162
  485. package/src/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.ts +0 -104
  486. package/src/mem_pools/tx_pool/eviction/invalid_txs_after_reorg_rule.ts +0 -93
  487. package/src/mem_pools/tx_pool/eviction/low_priority_eviction_rule.ts +0 -106
  488. package/src/mem_pools/tx_pool/eviction/nullifier_conflict_pre_add_rule.ts +0 -75
  489. package/src/mem_pools/tx_pool/index.ts +0 -2
  490. package/src/mem_pools/tx_pool/priority.ts +0 -20
  491. package/src/mem_pools/tx_pool/tx_pool.ts +0 -141
  492. package/src/mem_pools/tx_pool/tx_pool_test_suite.ts +0 -319
  493. package/src/msg_validators/clock_tolerance.ts +0 -51
  494. package/src/msg_validators/proposal_validator/proposal_validator_test_suite.ts +0 -230
  495. package/src/services/reqresp/connection-sampler/batch_connection_sampler.ts +0 -161
  496. package/src/services/reqresp/protocols/block.ts +0 -37
  497. package/src/services/tx_collection/fast_tx_collection.ts +0 -366
  498. package/src/services/tx_collection/proposal_tx_collector.ts +0 -112
  499. package/src/services/tx_collection/slow_tx_collection.ts +0 -262
@@ -1,68 +1,72 @@
1
1
  import { BlockNumber } from '@aztec/foundation/branded-types';
2
- import { compactArray } from '@aztec/foundation/collection';
2
+ import { times } from '@aztec/foundation/collection';
3
3
  import { type Logger, createLogger } from '@aztec/foundation/log';
4
- import { type PromiseWithResolvers, RunningPromise } from '@aztec/foundation/promise';
5
4
  import { sleep } from '@aztec/foundation/sleep';
6
- import { DateProvider } from '@aztec/foundation/timer';
5
+ import { DateProvider, elapsed } from '@aztec/foundation/timer';
7
6
  import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block';
8
7
  import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
9
8
  import type { BlockProposal } from '@aztec/stdlib/p2p';
10
- import { Tx, TxHash } from '@aztec/stdlib/tx';
9
+ import type { Tx } from '@aztec/stdlib/tx';
10
+ import { TxHash } from '@aztec/stdlib/tx';
11
11
  import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
12
12
 
13
13
  import type { PeerId } from '@libp2p/interface';
14
14
 
15
15
  import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interfaces.js';
16
+ import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js';
16
17
  import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js';
18
+ import type { BlockTxsSource } from '../reqresp/index.js';
17
19
  import type { TxCollectionConfig } from './config.js';
18
- import { FastTxCollection } from './fast_tx_collection.js';
19
20
  import { FileStoreTxCollection } from './file_store_tx_collection.js';
20
21
  import type { FileStoreTxSource } from './file_store_tx_source.js';
21
- import { SlowTxCollection } from './slow_tx_collection.js';
22
+ import { type IRequestTracker, RequestTracker } from './request_tracker.js';
22
23
  import { type TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
23
24
  import type { TxSource } from './tx_source.js';
24
25
 
25
- export type CollectionMethod = 'fast-req-resp' | 'fast-node-rpc' | 'slow-req-resp' | 'slow-node-rpc' | 'file-store';
26
-
27
- export type MissingTxInfo = { block: L2Block; blockNumber: BlockNumber; deadline: Date; readyForReqResp: boolean };
26
+ export type CollectionMethod = 'fast-req-resp' | 'fast-node-rpc' | 'file-store';
28
27
 
29
28
  export type FastCollectionRequestInput =
30
29
  | { type: 'block'; block: L2Block }
31
30
  | { type: 'proposal'; blockProposal: BlockProposal; blockNumber: BlockNumber };
32
31
 
33
32
  export type FastCollectionRequest = FastCollectionRequestInput & {
34
- missingTxHashes: Set<string>;
35
- deadline: Date;
33
+ requestTracker: IRequestTracker;
36
34
  blockInfo: L2BlockInfo;
37
- promise: PromiseWithResolvers<void>;
38
- foundTxs: Map<string, Tx>;
39
35
  };
40
36
 
41
37
  /**
42
- * Coordinates tx collection from remote RPC nodes and reqresp.
38
+ * Collect missing transactions for a block or proposal via reqresp.
39
+ * @param requestTracker - The missing transactions tracker
40
+ * @param blockTxsSource - The block or proposal containing the transactions
41
+ * @param pinnedPeer - Optional peer expected to have the transactions
42
+ * @returns The collected transactions
43
+ */
44
+ export type IReqRespTxsCollector = (
45
+ requestTracker: IRequestTracker,
46
+ blockTxsSource: BlockTxsSource,
47
+ pinnedPeer: PeerId | undefined,
48
+ ) => Promise<Tx[]>;
49
+
50
+ /**
51
+ * Coordinates tx collection from remote RPC nodes, reqresp, and file store.
43
52
  *
44
- * The slow collection loops are used for periodically collecting missing txs from mined blocks,
45
- * from remote RPC nodes and via reqresp. The fast collection methods are used to quickly gather
46
- * txs, usually for attesting to block proposals or preparing to prove an epoch. The slow and fast
47
- * collection instances both send their txs to the collection sink, which handles metrics and adds
48
- * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is notified
49
- * via events and notifies the slow and fast collection loops to stop collecting that tx, so that we don't
50
- * collect the same tx multiple times.
53
+ * Runs a sequential pipeline: node RPC → reqresp → file store. Node collection starts immediately,
54
+ * reqresp starts after a configured delay, and file store (if configured) starts after a further
55
+ * delay. All paths send txs to the collection sink, which handles metrics and adds them to the
56
+ * tx pool. Whenever a tx is added to the sink or the pool, this service is notified and stops
57
+ * collecting that tx across all in-flight requests.
51
58
  */
52
59
  export class TxCollection {
53
- /** Slow collection background loops */
54
- protected readonly slowCollection: SlowTxCollection;
55
-
56
- /** Fast collection methods */
57
- protected readonly fastCollection: FastTxCollection;
60
+ // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
61
+ protected requests: Set<FastCollectionRequest> = new Set();
58
62
 
59
- /** File store collection */
60
- protected readonly fileStoreCollection: FileStoreTxCollection;
63
+ /** The collector for txs via reqresp */
64
+ protected reqRespTxsCollector?: IReqRespTxsCollector;
61
65
 
62
- /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */
63
- private readonly reconcileFoundTxsLoop: RunningPromise;
66
+ /** File store collection for the fast (proposal/proving) path */
67
+ protected readonly fileStoreFastCollection: FileStoreTxCollection;
64
68
 
65
- /** Handles txs found by the slow and fast collection loops */
69
+ /** Handles txs found by collection paths before adding to the pool */
66
70
  private readonly txCollectionSink: TxCollectionSink;
67
71
 
68
72
  /** Handler for the txs-added event from the tx pool */
@@ -71,12 +75,6 @@ export class TxCollection {
71
75
  /** Handler for the txs-added event from the tx collection sink */
72
76
  protected readonly handleTxsFound: TxPoolV2Events['txs-added'];
73
77
 
74
- /** Whether the service has been started. */
75
- private started = false;
76
-
77
- /** Whether file store sources are configured. */
78
- private readonly hasFileStoreSources: boolean;
79
-
80
78
  constructor(
81
79
  private readonly p2pService: BatchTxRequesterLibP2PService,
82
80
  private readonly nodes: TxSource[],
@@ -90,35 +88,30 @@ export class TxCollection {
90
88
  ) {
91
89
  this.txCollectionSink = new TxCollectionSink(this.txPool, telemetryClient, this.log);
92
90
 
93
- this.fastCollection = new FastTxCollection(
94
- this.p2pService,
95
- this.nodes,
96
- this.txCollectionSink,
97
- this.config,
98
- this.dateProvider,
99
- this.log,
100
- );
101
-
102
- this.slowCollection = new SlowTxCollection(
103
- this.p2pService.reqResp,
104
- this.nodes,
91
+ this.reqRespTxsCollector = (requestTracker, blockTxsSource, pinnedPeer) =>
92
+ BatchTxRequester.collectAllTxs(
93
+ new BatchTxRequester(
94
+ requestTracker,
95
+ blockTxsSource,
96
+ pinnedPeer,
97
+ this.p2pService,
98
+ this.log,
99
+ this.dateProvider,
100
+ ).run(),
101
+ );
102
+
103
+ this.fileStoreFastCollection = new FileStoreTxCollection(
104
+ fileStoreSources,
105
105
  this.txCollectionSink,
106
- this.fastCollection,
107
- constants,
108
- this.config,
106
+ {
107
+ workerCount: config.txCollectionFileStoreFastWorkerCount,
108
+ backoffBaseMs: config.txCollectionFileStoreFastBackoffBaseMs,
109
+ backoffMaxMs: config.txCollectionFileStoreFastBackoffMaxMs,
110
+ },
109
111
  this.dateProvider,
110
112
  this.log,
111
113
  );
112
114
 
113
- this.hasFileStoreSources = fileStoreSources.length > 0;
114
- this.fileStoreCollection = new FileStoreTxCollection(fileStoreSources, this.txCollectionSink, this.log);
115
-
116
- this.reconcileFoundTxsLoop = new RunningPromise(
117
- () => this.reconcileFoundTxsWithPool(),
118
- this.log,
119
- this.config.txCollectionReconcileIntervalMs,
120
- );
121
-
122
115
  this.handleTxsFound = (args: Parameters<TxPoolV2Events['txs-added']>[0]) => {
123
116
  this.foundTxs(args.txs);
124
117
  };
@@ -133,73 +126,21 @@ export class TxCollection {
133
126
  this.txPool.on('txs-added', this.handleTxsAddedToPool);
134
127
  }
135
128
 
136
- /** Starts all collection loops. */
137
- public start(): Promise<void> {
138
- this.started = true;
139
- this.slowCollection.start();
140
- this.fileStoreCollection.start();
141
- this.reconcileFoundTxsLoop.start();
142
-
143
- // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup
144
- return Promise.resolve();
145
- }
146
-
147
- /** Stops all activity. */
148
- public async stop() {
149
- this.started = false;
150
- await Promise.all([
151
- this.slowCollection.stop(),
152
- this.fastCollection.stop(),
153
- this.fileStoreCollection.stop(),
154
- this.reconcileFoundTxsLoop.stop(),
155
- ]);
129
+ /** Stops all activity. Cancels in-flight requests; file store workers self-terminate. */
130
+ public stop() {
131
+ this.requests.forEach(request => {
132
+ request.requestTracker.cancel();
133
+ });
156
134
 
157
135
  this.txPool.removeListener('txs-added', this.handleTxsAddedToPool);
158
136
  this.txCollectionSink.removeListener('txs-added', this.handleTxsFound);
159
137
  }
160
138
 
161
- /** Force trigger the slow collection and reconciliation loops */
162
- public async trigger() {
163
- await Promise.all([this.reconcileFoundTxsLoop.trigger(), this.slowCollection.trigger()]);
164
- }
165
-
166
139
  /** Returns L1 rollup constants. */
167
140
  public getConstants(): L1RollupConstants {
168
141
  return this.constants;
169
142
  }
170
143
 
171
- /** Starts collecting the given tx hashes for the given L2Block in the slow loop */
172
- public startCollecting(block: L2Block, txHashes: TxHash[]) {
173
- this.slowCollection.startCollecting(block, txHashes);
174
-
175
- // Delay file store collection to give P2P methods time to find txs first
176
- if (this.hasFileStoreSources) {
177
- const context: TxAddContext = { type: 'mined', block };
178
- sleep(this.config.txCollectionFileStoreSlowDelayMs)
179
- .then(() => {
180
- if (this.started) {
181
- // Only queue txs that are still missing after the delay
182
- const stillMissing = new Set(this.slowCollection.getMissingTxHashes().map(h => h.toString()));
183
- const remaining = txHashes.filter(h => stillMissing.has(h.toString()));
184
- if (remaining.length > 0) {
185
- this.fileStoreCollection.startCollecting(remaining, context);
186
- }
187
- }
188
- })
189
- .catch(err => this.log.error('Error in file store slow delay', err));
190
- }
191
- }
192
-
193
- /** Collects the set of txs for the given block proposal as fast as possible */
194
- public collectFastForProposal(
195
- blockProposal: BlockProposal,
196
- blockNumber: BlockNumber,
197
- txHashes: TxHash[] | string[],
198
- opts: { deadline: Date; pinnedPeer?: PeerId },
199
- ) {
200
- return this.collectFastFor({ type: 'proposal', blockProposal, blockNumber }, txHashes, opts);
201
- }
202
-
203
144
  /** Collects the set of txs for the given mined block as fast as possible */
204
145
  public collectFastForBlock(
205
146
  block: L2Block,
@@ -210,52 +151,307 @@ export class TxCollection {
210
151
  }
211
152
 
212
153
  /** Collects the set of txs for the given proposal or block as fast as possible */
213
- public collectFastFor(
154
+ public async collectFastFor(
214
155
  input: FastCollectionRequestInput,
215
156
  txHashes: TxHash[] | string[],
216
157
  opts: { deadline: Date; pinnedPeer?: PeerId },
217
158
  ) {
159
+ const timeout = opts.deadline.getTime() - this.dateProvider.now();
160
+ if (timeout <= 0) {
161
+ this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, {
162
+ deadline: opts.deadline.getTime(),
163
+ now: this.dateProvider.now(),
164
+ });
165
+ return [];
166
+ }
167
+
218
168
  const hashes = txHashes.map(h => (typeof h === 'string' ? TxHash.fromString(h) : h));
219
169
 
220
- // Delay file store collection to give P2P methods time to find txs first
221
- if (this.hasFileStoreSources) {
222
- const context = this.getAddContextForInput(input);
223
- sleep(this.config.txCollectionFileStoreFastDelayMs)
224
- .then(() => {
225
- if (this.started) {
226
- this.fileStoreCollection.startCollecting(hashes, context);
170
+ const blockInfo: L2BlockInfo =
171
+ input.type === 'proposal'
172
+ ? { ...input.blockProposal.toBlockInfo(), blockNumber: input.blockNumber }
173
+ : { ...input.block.toBlockInfo() };
174
+
175
+ const request: FastCollectionRequest = {
176
+ ...input,
177
+ blockInfo,
178
+ requestTracker: RequestTracker.create(hashes, opts.deadline, this.dateProvider),
179
+ };
180
+
181
+ const [duration] = await elapsed(() => this.collectFast(request, { pinnedPeer: opts.pinnedPeer }));
182
+
183
+ this.log.verbose(
184
+ `Collected ${request.requestTracker.collectedTxs.length} txs out of ${hashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`,
185
+ {
186
+ ...blockInfo,
187
+ duration,
188
+ requestType: input.type,
189
+ missingTxs: [...request.requestTracker.missingTxHashes],
190
+ },
191
+ );
192
+ return request.requestTracker.collectedTxs;
193
+ }
194
+
195
+ protected async collectFast(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) {
196
+ this.requests.add(request);
197
+ const { blockInfo } = request;
198
+
199
+ this.log.debug(
200
+ `Starting fast collection of ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
201
+ { ...blockInfo, requestType: request.type, deadline: request.requestTracker.deadline },
202
+ );
203
+
204
+ try {
205
+ // 1. Start node collection in the background.
206
+ // Note: this will be a noop if no nodes are configured.
207
+ const nodeCollectionPromise = this.collectFastFromNodes(request);
208
+
209
+ // 2. Wait before starting reqresp, interruptible by cancellation or node exhaustion.
210
+ await Promise.race([
211
+ request.requestTracker.cancellationToken,
212
+ sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs),
213
+ nodeCollectionPromise, // If node collection has finished (or if there are no nodes configured), we can exit early.
214
+ ]);
215
+
216
+ // 3. Start reqresp in the background (runs in parallel with node collection).
217
+ // Note: this will be a noop if all TXs were already found.
218
+ const reqRespPromise = this.collectFastViaReqResp(request, opts);
219
+
220
+ // 4. Wait before starting file store, interruptible by cancellation.
221
+ await Promise.race([
222
+ request.requestTracker.cancellationToken,
223
+ sleep(this.config.txCollectionFileStoreFastDelayMs),
224
+ reqRespPromise, // If reqresp has finished, we can exit early.
225
+ ]);
226
+
227
+ // 5. Start file store collection in the background. Self-terminates on tracker cancel / all-found.
228
+ // Note: this will be a noop if all TXs were already found.
229
+ const fileStorePromise = this.fileStoreFastCollection.startCollecting(
230
+ request.requestTracker,
231
+ this.getAddContext(request),
232
+ );
233
+
234
+ // 6. Wait for all paths to settle.
235
+ // NOTE: The request will automatically be cancelled after `opt.deadline` is reached.
236
+ await Promise.allSettled([reqRespPromise, nodeCollectionPromise, fileStorePromise]);
237
+ } catch (err) {
238
+ this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, {
239
+ ...blockInfo,
240
+ missingTxs: request.requestTracker.missingTxHashes.values().map(txHash => txHash.toString()),
241
+ });
242
+ } finally {
243
+ request.requestTracker.cancel();
244
+ this.requests.delete(request);
245
+ }
246
+ }
247
+
248
+ /**
249
+ * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests
250
+ * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize
251
+ * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no
252
+ * tx is requested more than once at the same time to the same node.
253
+ */
254
+ private async collectFastFromNodes(request: FastCollectionRequest): Promise<void> {
255
+ if (this.nodes.length === 0) {
256
+ return;
257
+ }
258
+
259
+ // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
260
+ const attemptsPerTx = [...request.requestTracker.missingTxHashes].map(txHash => ({
261
+ txHash,
262
+ attempts: 0,
263
+ found: false,
264
+ }));
265
+
266
+ // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected.
267
+ await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx)));
268
+ }
269
+
270
+ private async collectFastFromNode(
271
+ request: FastCollectionRequest,
272
+ node: TxSource,
273
+ attemptsPerTx: { txHash: string; attempts: number; found: boolean }[],
274
+ ) {
275
+ const notFinished = () => !request.requestTracker.checkCancelled();
276
+
277
+ const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
278
+ const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
279
+ const activeRequestsToThisNode = new Set<string>(); // Track the txs being actively requested to this node
280
+
281
+ const processBatch = async () => {
282
+ while (notFinished()) {
283
+ // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts,
284
+ // so we prioritize txs that have been requested less often.
285
+ const batch = [];
286
+ let index = 0;
287
+ while (batch.length < maxBatchSize) {
288
+ const txToRequest = attemptsPerTx[index++];
289
+ if (!txToRequest) {
290
+ // No more txs to process
291
+ break;
292
+ } else if (!request.requestTracker.isMissing(txToRequest.txHash)) {
293
+ // Mark as found if it was found somewhere else, we'll then remove it from the array.
294
+ // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
295
+ txToRequest.found = true;
296
+ } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) {
297
+ // If the tx is not already being requested to this node, add it to the current batch and increase attempts.
298
+ // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that
299
+ // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting
300
+ // the same txs at the same time.
301
+ batch.push(txToRequest);
302
+ activeRequestsToThisNode.add(txToRequest.txHash);
303
+ txToRequest.attempts++;
227
304
  }
228
- })
229
- .catch(err => this.log.error('Error in file store fast delay', err));
305
+ }
306
+
307
+ // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end.
308
+ attemptsPerTx.sort((a, b) =>
309
+ a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found),
310
+ );
311
+ const firstFoundTxIndex = attemptsPerTx.findIndex(tx => tx.found);
312
+ if (firstFoundTxIndex !== -1) {
313
+ attemptsPerTx.length = firstFoundTxIndex;
314
+ }
315
+
316
+ // If we see no more txs to request, we can stop this "process" loop
317
+ if (batch.length === 0) {
318
+ return;
319
+ }
320
+
321
+ const txHashes = batch.map(({ txHash }) => txHash);
322
+ // Collect this batch from the node
323
+ await this.txCollectionSink.collect(
324
+ async () => {
325
+ const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
326
+ for (const tx of result.validTxs) {
327
+ request.requestTracker.markFetched(tx);
328
+ }
329
+ return result;
330
+ },
331
+ txHashes,
332
+ {
333
+ description: `fast ${node.getInfo()}`,
334
+ node: node.getInfo(),
335
+ method: 'fast-node-rpc',
336
+ ...request.blockInfo,
337
+ },
338
+ this.getAddContext(request),
339
+ );
340
+
341
+ // Clear from the active requests the txs we just requested
342
+ for (const requestedTx of batch) {
343
+ activeRequestsToThisNode.delete(requestedTx.txHash);
344
+ }
345
+
346
+ // Sleep a bit until hitting the node again, but wake up immediately on cancellation
347
+ if (notFinished()) {
348
+ await Promise.race([
349
+ sleep(this.config.txCollectionFastNodeIntervalMs),
350
+ request.requestTracker.cancellationToken,
351
+ ]);
352
+ }
353
+ }
354
+ };
355
+
356
+ // Kick off N parallel requests to the node, up to the maxParallelRequests limit
357
+ await Promise.all(times(maxParallelRequests, processBatch));
358
+ }
359
+
360
+ private async collectFastViaReqResp(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) {
361
+ const pinnedPeer = opts.pinnedPeer;
362
+ const blockInfo = request.blockInfo;
363
+ const slotNumber = blockInfo.slotNumber;
364
+ if (request.requestTracker.timeoutMs < 100) {
365
+ this.log.warn(
366
+ `Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`,
367
+ { timeoutMs: request.requestTracker.timeoutMs, ...blockInfo },
368
+ );
369
+ return;
370
+ }
371
+
372
+ if (request.requestTracker.checkCancelled()) {
373
+ this.log.debug(`No txs to collect via reqresp for ${request.type} at slot ${blockInfo.slotNumber}`, {
374
+ ...blockInfo,
375
+ });
376
+ return;
230
377
  }
231
378
 
232
- return this.fastCollection.collectFastFor(input, txHashes, opts);
379
+ this.log.debug(
380
+ `Starting fast reqresp for ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`,
381
+ { ...blockInfo, timeoutMs: request.requestTracker.timeoutMs, pinnedPeer },
382
+ );
383
+
384
+ try {
385
+ await this.txCollectionSink.collect(
386
+ async () => {
387
+ let blockTxsSource: BlockTxsSource;
388
+ if (request.type === 'proposal') {
389
+ blockTxsSource = request.blockProposal;
390
+ } else if (request.type === 'block') {
391
+ blockTxsSource = {
392
+ txHashes: request.block.body.txEffects.map(e => e.txHash),
393
+ archive: request.block.archive.root,
394
+ };
395
+ } else {
396
+ throw new Error(`Unknown request type: ${(request as { type: string }).type}`);
397
+ }
398
+
399
+ const result = await this.reqRespTxsCollector!(request.requestTracker, blockTxsSource, pinnedPeer);
400
+ return { validTxs: result, invalidTxHashes: [] };
401
+ },
402
+ Array.from(request.requestTracker.missingTxHashes),
403
+ { description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo },
404
+ this.getAddContext(request),
405
+ );
406
+ } catch (err) {
407
+ this.log.error(`Error sending fast reqresp request for txs`, err, {
408
+ txs: [...request.requestTracker.missingTxHashes],
409
+ ...blockInfo,
410
+ });
411
+ }
233
412
  }
234
413
 
235
- /** Returns the TxAddContext for the given fast collection request input */
236
- private getAddContextForInput(input: FastCollectionRequestInput): TxAddContext {
237
- if (input.type === 'proposal') {
238
- return { type: 'proposal', blockHeader: input.blockProposal.blockHeader };
414
+ /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */
415
+ private getAddContext(request: FastCollectionRequest): TxAddContext {
416
+ if (request.type === 'proposal') {
417
+ return { type: 'proposal', blockHeader: request.blockProposal.blockHeader };
239
418
  } else {
240
- return { type: 'mined', block: input.block };
419
+ return { type: 'mined', block: request.block };
241
420
  }
242
421
  }
243
422
 
244
- /** Mark the given txs as found. Stops collecting them. */
423
+ /** Mark the given txs as found. Stops collecting them across all in-flight requests. */
245
424
  private foundTxs(txs: Tx[]) {
246
- this.slowCollection.foundTxs(txs);
247
- this.fastCollection.foundTxs(txs);
248
- this.fileStoreCollection.foundTxs(txs);
425
+ for (const request of this.requests) {
426
+ for (const tx of txs) {
427
+ if (request.requestTracker.markFetched(tx)) {
428
+ this.log.trace(`Found tx ${tx.txHash.toString()} for fast collection request`, {
429
+ ...request.blockInfo,
430
+ txHash: tx.txHash.toString(),
431
+ type: request.type,
432
+ });
433
+ if (request.requestTracker.allFetched()) {
434
+ this.log.trace(`All txs found for fast collection request`, {
435
+ ...request.blockInfo,
436
+ type: request.type,
437
+ });
438
+ break;
439
+ }
440
+ }
441
+ }
442
+ }
249
443
  }
250
444
 
251
445
  /**
252
- * Stop collecting all txs for blocks less than or requal to the block number specified.
446
+ * Stop collecting all txs for blocks less than or equal to the block number specified.
253
447
  * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized.
254
448
  */
255
449
  public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void {
256
- this.slowCollection.stopCollectingForBlocksUpTo(blockNumber);
257
- this.fastCollection.stopCollectingForBlocksUpTo(blockNumber);
258
- this.fileStoreCollection.clearPending();
450
+ for (const request of this.requests) {
451
+ if (request.blockInfo.blockNumber <= blockNumber) {
452
+ request.requestTracker.cancel();
453
+ }
454
+ }
259
455
  }
260
456
 
261
457
  /**
@@ -263,20 +459,10 @@ export class TxCollection {
263
459
  * To be called when there is a chain prune and previously mined txs are no longer relevant.
264
460
  */
265
461
  public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void {
266
- this.slowCollection.stopCollectingForBlocksAfter(blockNumber);
267
- this.fastCollection.stopCollectingForBlocksAfter(blockNumber);
268
- this.fileStoreCollection.clearPending();
269
- }
270
-
271
- /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */
272
- private async reconcileFoundTxsWithPool() {
273
- const missingTxHashes = this.slowCollection.getMissingTxHashes();
274
- const foundTxs = compactArray(await this.txPool.getTxsByHash(missingTxHashes));
275
- if (foundTxs.length > 0) {
276
- this.log.verbose(`Found ${foundTxs.length} txs in the pool during reconciliation`, {
277
- foundTxs: foundTxs.map(t => t.getTxHash().toString()),
278
- });
279
- this.foundTxs(foundTxs);
462
+ for (const request of this.requests) {
463
+ if (request.blockInfo.blockNumber > blockNumber) {
464
+ request.requestTracker.cancel();
465
+ }
280
466
  }
281
467
  }
282
468
  }