@aztec/p2p 0.0.1-commit.6d63667d → 0.0.1-commit.733c4a3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (315) hide show
  1. package/dest/client/factory.d.ts +7 -6
  2. package/dest/client/factory.d.ts.map +1 -1
  3. package/dest/client/factory.js +35 -11
  4. package/dest/client/interface.d.ts +45 -32
  5. package/dest/client/interface.d.ts.map +1 -1
  6. package/dest/client/p2p_client.d.ts +38 -50
  7. package/dest/client/p2p_client.d.ts.map +1 -1
  8. package/dest/client/p2p_client.js +145 -200
  9. package/dest/client/test/tx_proposal_collector/proposal_tx_collector_worker.js +6 -6
  10. package/dest/config.d.ts +24 -8
  11. package/dest/config.d.ts.map +1 -1
  12. package/dest/config.js +16 -6
  13. package/dest/errors/tx-pool.error.d.ts +8 -0
  14. package/dest/errors/tx-pool.error.d.ts.map +1 -0
  15. package/dest/errors/tx-pool.error.js +9 -0
  16. package/dest/index.d.ts +2 -1
  17. package/dest/index.d.ts.map +1 -1
  18. package/dest/index.js +1 -0
  19. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts +104 -88
  20. package/dest/mem_pools/attestation_pool/attestation_pool.d.ts.map +1 -1
  21. package/dest/mem_pools/attestation_pool/attestation_pool.js +441 -3
  22. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts +2 -2
  23. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.d.ts.map +1 -1
  24. package/dest/mem_pools/attestation_pool/attestation_pool_test_suite.js +353 -87
  25. package/dest/mem_pools/attestation_pool/index.d.ts +2 -3
  26. package/dest/mem_pools/attestation_pool/index.d.ts.map +1 -1
  27. package/dest/mem_pools/attestation_pool/index.js +1 -2
  28. package/dest/mem_pools/attestation_pool/mocks.d.ts +2 -2
  29. package/dest/mem_pools/attestation_pool/mocks.d.ts.map +1 -1
  30. package/dest/mem_pools/attestation_pool/mocks.js +2 -2
  31. package/dest/mem_pools/index.d.ts +3 -2
  32. package/dest/mem_pools/index.d.ts.map +1 -1
  33. package/dest/mem_pools/index.js +1 -1
  34. package/dest/mem_pools/interface.d.ts +5 -5
  35. package/dest/mem_pools/interface.d.ts.map +1 -1
  36. package/dest/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.js +3 -3
  37. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts +104 -0
  38. package/dest/mem_pools/tx_pool_v2/deleted_pool.d.ts.map +1 -0
  39. package/dest/mem_pools/tx_pool_v2/deleted_pool.js +251 -0
  40. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts +3 -3
  41. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.d.ts.map +1 -1
  42. package/dest/mem_pools/tx_pool_v2/eviction/eviction_manager.js +18 -9
  43. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.d.ts +1 -1
  44. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.d.ts.map +1 -1
  45. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.js +5 -2
  46. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts +3 -3
  47. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.d.ts.map +1 -1
  48. package/dest/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.js +10 -4
  49. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts +2 -2
  50. package/dest/mem_pools/tx_pool_v2/eviction/index.d.ts.map +1 -1
  51. package/dest/mem_pools/tx_pool_v2/eviction/index.js +1 -1
  52. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts +48 -5
  53. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.d.ts.map +1 -1
  54. package/dest/mem_pools/tx_pool_v2/eviction/interfaces.js +8 -0
  55. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.js +7 -5
  56. package/dest/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.js +7 -5
  57. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts +2 -2
  58. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.d.ts.map +1 -1
  59. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.js +14 -6
  60. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts +4 -4
  61. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.d.ts.map +1 -1
  62. package/dest/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.js +14 -4
  63. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts +3 -3
  64. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.d.ts.map +1 -1
  65. package/dest/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.js +2 -2
  66. package/dest/mem_pools/tx_pool_v2/index.d.ts +2 -1
  67. package/dest/mem_pools/tx_pool_v2/index.d.ts.map +1 -1
  68. package/dest/mem_pools/tx_pool_v2/index.js +1 -0
  69. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts +15 -0
  70. package/dest/mem_pools/tx_pool_v2/instrumentation.d.ts.map +1 -0
  71. package/dest/mem_pools/tx_pool_v2/instrumentation.js +43 -0
  72. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts +22 -8
  73. package/dest/mem_pools/tx_pool_v2/interfaces.d.ts.map +1 -1
  74. package/dest/mem_pools/tx_pool_v2/interfaces.js +4 -1
  75. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts +39 -5
  76. package/dest/mem_pools/tx_pool_v2/tx_metadata.d.ts.map +1 -1
  77. package/dest/mem_pools/tx_pool_v2/tx_metadata.js +74 -5
  78. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts +12 -3
  79. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.d.ts.map +1 -1
  80. package/dest/mem_pools/tx_pool_v2/tx_pool_indices.js +27 -4
  81. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts +9 -4
  82. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.d.ts.map +1 -1
  83. package/dest/mem_pools/tx_pool_v2/tx_pool_v2.js +11 -6
  84. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts +13 -5
  85. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.d.ts.map +1 -1
  86. package/dest/mem_pools/tx_pool_v2/tx_pool_v2_impl.js +312 -169
  87. package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.d.ts +3 -3
  88. package/dest/msg_validators/attestation_validator/fisherman_attestation_validator.d.ts.map +1 -1
  89. package/dest/msg_validators/tx_validator/aggregate_tx_validator.d.ts +4 -4
  90. package/dest/msg_validators/tx_validator/aggregate_tx_validator.d.ts.map +1 -1
  91. package/dest/msg_validators/tx_validator/aggregate_tx_validator.js +3 -3
  92. package/dest/msg_validators/tx_validator/block_header_validator.d.ts +16 -3
  93. package/dest/msg_validators/tx_validator/block_header_validator.d.ts.map +1 -1
  94. package/dest/msg_validators/tx_validator/block_header_validator.js +1 -1
  95. package/dest/msg_validators/tx_validator/double_spend_validator.d.ts +13 -3
  96. package/dest/msg_validators/tx_validator/double_spend_validator.d.ts.map +1 -1
  97. package/dest/msg_validators/tx_validator/double_spend_validator.js +4 -4
  98. package/dest/msg_validators/tx_validator/factory.d.ts +114 -6
  99. package/dest/msg_validators/tx_validator/factory.d.ts.map +1 -1
  100. package/dest/msg_validators/tx_validator/factory.js +219 -58
  101. package/dest/msg_validators/tx_validator/gas_validator.d.ts +58 -3
  102. package/dest/msg_validators/tx_validator/gas_validator.d.ts.map +1 -1
  103. package/dest/msg_validators/tx_validator/gas_validator.js +73 -36
  104. package/dest/msg_validators/tx_validator/index.d.ts +2 -1
  105. package/dest/msg_validators/tx_validator/index.d.ts.map +1 -1
  106. package/dest/msg_validators/tx_validator/index.js +1 -0
  107. package/dest/msg_validators/tx_validator/nullifier_cache.d.ts +14 -0
  108. package/dest/msg_validators/tx_validator/nullifier_cache.d.ts.map +1 -0
  109. package/dest/msg_validators/tx_validator/nullifier_cache.js +24 -0
  110. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts +20 -4
  111. package/dest/msg_validators/tx_validator/timestamp_validator.d.ts.map +1 -1
  112. package/dest/msg_validators/tx_validator/timestamp_validator.js +6 -6
  113. package/dest/services/dummy_service.d.ts +13 -5
  114. package/dest/services/dummy_service.d.ts.map +1 -1
  115. package/dest/services/dummy_service.js +10 -4
  116. package/dest/services/encoding.d.ts +2 -2
  117. package/dest/services/encoding.d.ts.map +1 -1
  118. package/dest/services/encoding.js +4 -3
  119. package/dest/services/gossipsub/index.d.ts +3 -0
  120. package/dest/services/gossipsub/index.d.ts.map +1 -0
  121. package/dest/services/gossipsub/index.js +2 -0
  122. package/dest/services/gossipsub/scoring.d.ts +21 -3
  123. package/dest/services/gossipsub/scoring.d.ts.map +1 -1
  124. package/dest/services/gossipsub/scoring.js +24 -7
  125. package/dest/services/gossipsub/topic_score_params.d.ts +173 -0
  126. package/dest/services/gossipsub/topic_score_params.d.ts.map +1 -0
  127. package/dest/services/gossipsub/topic_score_params.js +346 -0
  128. package/dest/services/libp2p/libp2p_service.d.ts +91 -38
  129. package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
  130. package/dest/services/libp2p/libp2p_service.js +411 -321
  131. package/dest/services/peer-manager/peer_scoring.d.ts +1 -1
  132. package/dest/services/peer-manager/peer_scoring.d.ts.map +1 -1
  133. package/dest/services/peer-manager/peer_scoring.js +25 -2
  134. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts +4 -3
  135. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.d.ts.map +1 -1
  136. package/dest/services/reqresp/batch-tx-requester/batch_tx_requester.js +19 -46
  137. package/dest/services/reqresp/batch-tx-requester/interface.d.ts +2 -6
  138. package/dest/services/reqresp/batch-tx-requester/interface.d.ts.map +1 -1
  139. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts +10 -13
  140. package/dest/services/reqresp/batch-tx-requester/missing_txs.d.ts.map +1 -1
  141. package/dest/services/reqresp/batch-tx-requester/missing_txs.js +25 -46
  142. package/dest/services/reqresp/batch-tx-requester/peer_collection.d.ts +17 -11
  143. package/dest/services/reqresp/batch-tx-requester/peer_collection.d.ts.map +1 -1
  144. package/dest/services/reqresp/batch-tx-requester/peer_collection.js +49 -15
  145. package/dest/services/reqresp/batch-tx-requester/tx_validator.js +2 -2
  146. package/dest/services/reqresp/interface.d.ts +10 -1
  147. package/dest/services/reqresp/interface.d.ts.map +1 -1
  148. package/dest/services/reqresp/interface.js +15 -1
  149. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.d.ts +4 -3
  150. package/dest/services/reqresp/protocols/block_txs/block_txs_handler.d.ts.map +1 -1
  151. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts +7 -1
  152. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.d.ts.map +1 -1
  153. package/dest/services/reqresp/protocols/block_txs/block_txs_reqresp.js +15 -0
  154. package/dest/services/reqresp/protocols/tx.d.ts +7 -1
  155. package/dest/services/reqresp/protocols/tx.d.ts.map +1 -1
  156. package/dest/services/reqresp/protocols/tx.js +20 -0
  157. package/dest/services/reqresp/reqresp.d.ts +1 -1
  158. package/dest/services/reqresp/reqresp.d.ts.map +1 -1
  159. package/dest/services/reqresp/reqresp.js +11 -4
  160. package/dest/services/service.d.ts +39 -3
  161. package/dest/services/service.d.ts.map +1 -1
  162. package/dest/services/tx_collection/config.d.ts +19 -1
  163. package/dest/services/tx_collection/config.d.ts.map +1 -1
  164. package/dest/services/tx_collection/config.js +46 -0
  165. package/dest/services/tx_collection/fast_tx_collection.d.ts +3 -1
  166. package/dest/services/tx_collection/fast_tx_collection.d.ts.map +1 -1
  167. package/dest/services/tx_collection/fast_tx_collection.js +56 -36
  168. package/dest/services/tx_collection/file_store_tx_collection.d.ts +53 -0
  169. package/dest/services/tx_collection/file_store_tx_collection.d.ts.map +1 -0
  170. package/dest/services/tx_collection/file_store_tx_collection.js +167 -0
  171. package/dest/services/tx_collection/file_store_tx_source.d.ts +37 -0
  172. package/dest/services/tx_collection/file_store_tx_source.d.ts.map +1 -0
  173. package/dest/services/tx_collection/file_store_tx_source.js +90 -0
  174. package/dest/services/tx_collection/index.d.ts +2 -1
  175. package/dest/services/tx_collection/index.d.ts.map +1 -1
  176. package/dest/services/tx_collection/index.js +1 -0
  177. package/dest/services/tx_collection/instrumentation.d.ts +1 -1
  178. package/dest/services/tx_collection/instrumentation.d.ts.map +1 -1
  179. package/dest/services/tx_collection/instrumentation.js +2 -1
  180. package/dest/services/tx_collection/missing_txs_tracker.d.ts +32 -0
  181. package/dest/services/tx_collection/missing_txs_tracker.d.ts.map +1 -0
  182. package/dest/services/tx_collection/missing_txs_tracker.js +27 -0
  183. package/dest/services/tx_collection/proposal_tx_collector.d.ts +7 -6
  184. package/dest/services/tx_collection/proposal_tx_collector.d.ts.map +1 -1
  185. package/dest/services/tx_collection/proposal_tx_collector.js +5 -4
  186. package/dest/services/tx_collection/slow_tx_collection.d.ts +7 -3
  187. package/dest/services/tx_collection/slow_tx_collection.d.ts.map +1 -1
  188. package/dest/services/tx_collection/slow_tx_collection.js +60 -26
  189. package/dest/services/tx_collection/tx_collection.d.ts +23 -10
  190. package/dest/services/tx_collection/tx_collection.d.ts.map +1 -1
  191. package/dest/services/tx_collection/tx_collection.js +75 -3
  192. package/dest/services/tx_collection/tx_collection_sink.d.ts +18 -8
  193. package/dest/services/tx_collection/tx_collection_sink.d.ts.map +1 -1
  194. package/dest/services/tx_collection/tx_collection_sink.js +26 -29
  195. package/dest/services/tx_collection/tx_source.d.ts +8 -3
  196. package/dest/services/tx_collection/tx_source.d.ts.map +1 -1
  197. package/dest/services/tx_collection/tx_source.js +19 -2
  198. package/dest/services/tx_file_store/config.d.ts +1 -3
  199. package/dest/services/tx_file_store/config.d.ts.map +1 -1
  200. package/dest/services/tx_file_store/config.js +0 -4
  201. package/dest/services/tx_file_store/tx_file_store.d.ts +4 -3
  202. package/dest/services/tx_file_store/tx_file_store.d.ts.map +1 -1
  203. package/dest/services/tx_file_store/tx_file_store.js +9 -6
  204. package/dest/services/tx_provider.d.ts +4 -4
  205. package/dest/services/tx_provider.d.ts.map +1 -1
  206. package/dest/services/tx_provider.js +9 -8
  207. package/dest/test-helpers/make-test-p2p-clients.d.ts +3 -3
  208. package/dest/test-helpers/make-test-p2p-clients.d.ts.map +1 -1
  209. package/dest/test-helpers/mock-pubsub.d.ts +29 -2
  210. package/dest/test-helpers/mock-pubsub.d.ts.map +1 -1
  211. package/dest/test-helpers/mock-pubsub.js +103 -2
  212. package/dest/test-helpers/reqresp-nodes.d.ts +1 -1
  213. package/dest/test-helpers/reqresp-nodes.d.ts.map +1 -1
  214. package/dest/test-helpers/reqresp-nodes.js +2 -1
  215. package/dest/test-helpers/testbench-utils.d.ts +43 -38
  216. package/dest/test-helpers/testbench-utils.d.ts.map +1 -1
  217. package/dest/test-helpers/testbench-utils.js +128 -59
  218. package/dest/testbench/p2p_client_testbench_worker.d.ts +2 -2
  219. package/dest/testbench/p2p_client_testbench_worker.d.ts.map +1 -1
  220. package/dest/testbench/p2p_client_testbench_worker.js +10 -10
  221. package/dest/util.d.ts +2 -2
  222. package/dest/util.d.ts.map +1 -1
  223. package/package.json +14 -14
  224. package/src/client/factory.ts +66 -14
  225. package/src/client/interface.ts +61 -33
  226. package/src/client/p2p_client.ts +179 -236
  227. package/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +19 -9
  228. package/src/config.ts +35 -11
  229. package/src/errors/tx-pool.error.ts +12 -0
  230. package/src/index.ts +1 -0
  231. package/src/mem_pools/attestation_pool/attestation_pool.ts +496 -91
  232. package/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +442 -102
  233. package/src/mem_pools/attestation_pool/index.ts +9 -2
  234. package/src/mem_pools/attestation_pool/mocks.ts +2 -1
  235. package/src/mem_pools/index.ts +4 -1
  236. package/src/mem_pools/interface.ts +4 -4
  237. package/src/mem_pools/tx_pool/README.md +1 -1
  238. package/src/mem_pools/tx_pool/eviction/invalid_txs_after_mining_rule.ts +3 -3
  239. package/src/mem_pools/tx_pool_v2/README.md +76 -10
  240. package/src/mem_pools/tx_pool_v2/deleted_pool.ts +321 -0
  241. package/src/mem_pools/tx_pool_v2/eviction/eviction_manager.ts +21 -8
  242. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts +5 -2
  243. package/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.ts +15 -4
  244. package/src/mem_pools/tx_pool_v2/eviction/index.ts +4 -0
  245. package/src/mem_pools/tx_pool_v2/eviction/interfaces.ts +49 -4
  246. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts +5 -5
  247. package/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts +5 -5
  248. package/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts +14 -9
  249. package/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.ts +24 -6
  250. package/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.ts +3 -3
  251. package/src/mem_pools/tx_pool_v2/index.ts +1 -0
  252. package/src/mem_pools/tx_pool_v2/instrumentation.ts +69 -0
  253. package/src/mem_pools/tx_pool_v2/interfaces.ts +23 -8
  254. package/src/mem_pools/tx_pool_v2/tx_metadata.ts +103 -9
  255. package/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +32 -5
  256. package/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +18 -7
  257. package/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +347 -166
  258. package/src/msg_validators/attestation_validator/fisherman_attestation_validator.ts +2 -2
  259. package/src/msg_validators/tx_validator/README.md +115 -0
  260. package/src/msg_validators/tx_validator/aggregate_tx_validator.ts +5 -5
  261. package/src/msg_validators/tx_validator/block_header_validator.ts +15 -3
  262. package/src/msg_validators/tx_validator/double_spend_validator.ts +11 -6
  263. package/src/msg_validators/tx_validator/factory.ts +353 -77
  264. package/src/msg_validators/tx_validator/gas_validator.ts +90 -27
  265. package/src/msg_validators/tx_validator/index.ts +1 -0
  266. package/src/msg_validators/tx_validator/nullifier_cache.ts +30 -0
  267. package/src/msg_validators/tx_validator/timestamp_validator.ts +23 -18
  268. package/src/services/dummy_service.ts +18 -6
  269. package/src/services/encoding.ts +4 -3
  270. package/src/services/gossipsub/README.md +641 -0
  271. package/src/services/gossipsub/index.ts +2 -0
  272. package/src/services/gossipsub/scoring.ts +29 -5
  273. package/src/services/gossipsub/topic_score_params.ts +487 -0
  274. package/src/services/libp2p/libp2p_service.ts +431 -344
  275. package/src/services/peer-manager/peer_scoring.ts +25 -0
  276. package/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts +20 -48
  277. package/src/services/reqresp/batch-tx-requester/interface.ts +1 -5
  278. package/src/services/reqresp/batch-tx-requester/missing_txs.ts +23 -71
  279. package/src/services/reqresp/batch-tx-requester/peer_collection.ts +63 -24
  280. package/src/services/reqresp/batch-tx-requester/tx_validator.ts +2 -2
  281. package/src/services/reqresp/interface.ts +26 -1
  282. package/src/services/reqresp/protocols/block_txs/block_txs_handler.ts +4 -3
  283. package/src/services/reqresp/protocols/block_txs/block_txs_reqresp.ts +17 -0
  284. package/src/services/reqresp/protocols/tx.ts +22 -0
  285. package/src/services/reqresp/reqresp.ts +13 -3
  286. package/src/services/service.ts +51 -2
  287. package/src/services/tx_collection/config.ts +68 -0
  288. package/src/services/tx_collection/fast_tx_collection.ts +65 -32
  289. package/src/services/tx_collection/file_store_tx_collection.ts +202 -0
  290. package/src/services/tx_collection/file_store_tx_source.ts +117 -0
  291. package/src/services/tx_collection/index.ts +1 -0
  292. package/src/services/tx_collection/instrumentation.ts +7 -1
  293. package/src/services/tx_collection/missing_txs_tracker.ts +52 -0
  294. package/src/services/tx_collection/proposal_tx_collector.ts +8 -7
  295. package/src/services/tx_collection/slow_tx_collection.ts +66 -33
  296. package/src/services/tx_collection/tx_collection.ts +113 -16
  297. package/src/services/tx_collection/tx_collection_sink.ts +30 -34
  298. package/src/services/tx_collection/tx_source.ts +22 -3
  299. package/src/services/tx_file_store/config.ts +0 -6
  300. package/src/services/tx_file_store/tx_file_store.ts +10 -8
  301. package/src/services/tx_provider.ts +10 -9
  302. package/src/test-helpers/make-test-p2p-clients.ts +3 -3
  303. package/src/test-helpers/mock-pubsub.ts +143 -3
  304. package/src/test-helpers/reqresp-nodes.ts +2 -1
  305. package/src/test-helpers/testbench-utils.ts +127 -71
  306. package/src/testbench/p2p_client_testbench_worker.ts +22 -15
  307. package/src/util.ts +7 -1
  308. package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts +0 -40
  309. package/dest/mem_pools/attestation_pool/kv_attestation_pool.d.ts.map +0 -1
  310. package/dest/mem_pools/attestation_pool/kv_attestation_pool.js +0 -218
  311. package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts +0 -31
  312. package/dest/mem_pools/attestation_pool/memory_attestation_pool.d.ts.map +0 -1
  313. package/dest/mem_pools/attestation_pool/memory_attestation_pool.js +0 -180
  314. package/src/mem_pools/attestation_pool/kv_attestation_pool.ts +0 -320
  315. package/src/mem_pools/attestation_pool/memory_attestation_pool.ts +0 -264
@@ -62,5 +62,51 @@ export const txCollectionConfigMappings = {
62
62
  'new',
63
63
  'old'
64
64
  ], 'new')
65
+ },
66
+ txCollectionFileStoreUrls: {
67
+ env: 'TX_COLLECTION_FILE_STORE_URLS',
68
+ description: 'A comma-separated list of file store URLs (s3://, gs://, file://, http://) for tx collection',
69
+ parseEnv: (val)=>val.split(',').map((url)=>url.trim()).filter((url)=>url.length > 0),
70
+ defaultValue: []
71
+ },
72
+ txCollectionFileStoreSlowDelayMs: {
73
+ env: 'TX_COLLECTION_FILE_STORE_SLOW_DELAY_MS',
74
+ description: 'Delay before file store collection starts after slow collection',
75
+ ...numberConfigHelper(24_000)
76
+ },
77
+ txCollectionFileStoreFastDelayMs: {
78
+ env: 'TX_COLLECTION_FILE_STORE_FAST_DELAY_MS',
79
+ description: 'Delay before file store collection starts after fast collection',
80
+ ...numberConfigHelper(2_000)
81
+ },
82
+ txCollectionFileStoreFastWorkerCount: {
83
+ env: 'TX_COLLECTION_FILE_STORE_FAST_WORKER_COUNT',
84
+ description: 'Number of concurrent workers for fast file store collection',
85
+ ...numberConfigHelper(5)
86
+ },
87
+ txCollectionFileStoreSlowWorkerCount: {
88
+ env: 'TX_COLLECTION_FILE_STORE_SLOW_WORKER_COUNT',
89
+ description: 'Number of concurrent workers for slow file store collection',
90
+ ...numberConfigHelper(2)
91
+ },
92
+ txCollectionFileStoreFastBackoffBaseMs: {
93
+ env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_BASE_MS',
94
+ description: 'Base backoff time in ms for fast file store collection retries',
95
+ ...numberConfigHelper(1_000)
96
+ },
97
+ txCollectionFileStoreSlowBackoffBaseMs: {
98
+ env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_BASE_MS',
99
+ description: 'Base backoff time in ms for slow file store collection retries',
100
+ ...numberConfigHelper(5_000)
101
+ },
102
+ txCollectionFileStoreFastBackoffMaxMs: {
103
+ env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_MAX_MS',
104
+ description: 'Max backoff time in ms for fast file store collection retries',
105
+ ...numberConfigHelper(5_000)
106
+ },
107
+ txCollectionFileStoreSlowBackoffMaxMs: {
108
+ env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_MAX_MS',
109
+ description: 'Max backoff time in ms for slow file store collection retries',
110
+ ...numberConfigHelper(30_000)
65
111
  }
66
112
  };
@@ -33,6 +33,8 @@ export declare class FastTxCollection {
33
33
  private collectFastFromNodes;
34
34
  private collectFastFromNode;
35
35
  private collectFastViaReqResp;
36
+ /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */
37
+ private getAddContext;
36
38
  /**
37
39
  * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
38
40
  * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
@@ -49,4 +51,4 @@ export declare class FastTxCollection {
49
51
  */
50
52
  stopCollectingForBlocksAfter(blockNumber: BlockNumber): void;
51
53
  }
52
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFzdF90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9mYXN0X3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRzlELE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQUUsWUFBWSxFQUFXLE1BQU0seUJBQXlCLENBQUM7QUFFaEUsT0FBTyxLQUFLLEVBQUUsYUFBYSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDdkQsT0FBTyxFQUFFLEtBQUssRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRW5ELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBR2hELE9BQU8sS0FBSyxFQUFFLDZCQUE2QixFQUFFLE1BQU0sNENBQTRDLENBQUM7QUFDaEcsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFDdEQsT0FBTyxFQUVMLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0sNEJBQTRCLENBQUM7QUFDcEMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUM1RixPQUFPLEtBQUssRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLFFBQVEsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBRS9DLHFCQUFhLGdCQUFnQjtJQU96QixPQUFPLENBQUMsS0FBSztJQUNiLE9BQU8sQ0FBQyxnQkFBZ0I7SUFDeEIsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsWUFBWTtJQUNwQixPQUFPLENBQUMsR0FBRztJQVRiLFNBQVMsQ0FBQyxRQUFRLEVBQUUsR0FBRyxDQUFDLHFCQUFxQixDQUFDLENBQWE7SUFDM0QsT0FBTyxDQUFDLG1CQUFtQixDQUFzQjtJQUVqRCxZQUNFLFVBQVUsRUFBRSw2QkFBNkIsRUFDakMsS0FBSyxFQUFFLFFBQVEsRUFBRSxFQUNqQixnQkFBZ0IsRUFBRSxnQkFBZ0IsRUFDbEMsTUFBTSxFQUFFLGtCQUFrQixFQUMxQixZQUFZLEdBQUUsWUFBaUMsRUFDL0MsR0FBRyxHQUFFLE1BQWtELEVBQy9ELG1CQUFtQixDQUFDLEVBQUUsbUJBQW1CLEVBUzFDO0lBRVksSUFBSSxrQkFHaEI7SUFFTSx5QkFBeUIsK0JBRS9CO0lBRVksY0FBYyxDQUN6QixLQUFLLEVBQUUsMEJBQTBCLEVBQ2pDLFFBQVEsRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLEVBQUUsRUFDN0IsSUFBSSxFQUFFO1FBQUUsUUFBUSxFQUFFLElBQUksQ0FBQztRQUFDLFVBQVUsQ0FBQyxFQUFFLE1BQU0sQ0FBQTtLQUFFLGlCQTBDOUM7SUFFRCxVQUFnQixXQUFXLENBQ3pCLE9BQU8sRUFBRSxxQkFBcUIsRUFDOUIsSUFBSSxFQUFFO1FBQUUsUUFBUSxDQUFDLEVBQUUsYUFBYSxDQUFDO1FBQUMsUUFBUSxFQUFFLElBQUksQ0FBQztRQUFDLFVBQVUsQ0FBQyxFQUFFLE1BQU0sQ0FBQTtLQUFFLGlCQStDeEU7WUFRYSxvQkFBb0I7WUFZcEIsbUJBQW1CO1lBaUZuQixxQkFBcUI7SUE0Q25DOzs7T0FHRztJQUNJLFFBQVEsQ0FBQyxHQUFHLEVBQUUsRUFBRSxFQUFFLFFBd0J4QjtJQUVEOzs7T0FHRztJQUNJLDJCQUEyQixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9qRTtJQUVEOzs7T0FHRztJQUNJLDRCQUE0QixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9sRTtDQUNGIn0=
54
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFzdF90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9mYXN0X3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRzlELE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQUUsWUFBWSxFQUFXLE1BQU0seUJBQXlCLENBQUM7QUFFaEUsT0FBTyxLQUFLLEVBQUUsYUFBYSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDdkQsT0FBTyxFQUFFLEtBQUssRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRW5ELE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBR2hELE9BQU8sS0FBSyxFQUFFLDZCQUE2QixFQUFFLE1BQU0sNENBQTRDLENBQUM7QUFDaEcsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFFdEQsT0FBTyxFQUVMLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0sNEJBQTRCLENBQUM7QUFDcEMsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUM1RixPQUFPLEtBQUssRUFBZ0IsZ0JBQWdCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUM5RSxPQUFPLEtBQUssRUFBRSxRQUFRLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUUvQyxxQkFBYSxnQkFBZ0I7SUFPekIsT0FBTyxDQUFDLEtBQUs7SUFDYixPQUFPLENBQUMsZ0JBQWdCO0lBQ3hCLE9BQU8sQ0FBQyxNQUFNO0lBQ2QsT0FBTyxDQUFDLFlBQVk7SUFDcEIsT0FBTyxDQUFDLEdBQUc7SUFUYixTQUFTLENBQUMsUUFBUSxFQUFFLEdBQUcsQ0FBQyxxQkFBcUIsQ0FBQyxDQUFhO0lBQzNELE9BQU8sQ0FBQyxtQkFBbUIsQ0FBc0I7SUFFakQsWUFDRSxVQUFVLEVBQUUsNkJBQTZCLEVBQ2pDLEtBQUssRUFBRSxRQUFRLEVBQUUsRUFDakIsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLE1BQU0sRUFBRSxrQkFBa0IsRUFDMUIsWUFBWSxHQUFFLFlBQWlDLEVBQy9DLEdBQUcsR0FBRSxNQUFrRCxFQUMvRCxtQkFBbUIsQ0FBQyxFQUFFLG1CQUFtQixFQVMxQztJQUVZLElBQUksa0JBR2hCO0lBRU0seUJBQXlCLCtCQUUvQjtJQUVZLGNBQWMsQ0FDekIsS0FBSyxFQUFFLDBCQUEwQixFQUNqQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsTUFBTSxFQUFFLEVBQzdCLElBQUksRUFBRTtRQUFFLFFBQVEsRUFBRSxJQUFJLENBQUM7UUFBQyxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUE7S0FBRSxpQkEwQzlDO0lBRUQsVUFBZ0IsV0FBVyxDQUN6QixPQUFPLEVBQUUscUJBQXFCLEVBQzlCLElBQUksRUFBRTtRQUFFLFFBQVEsQ0FBQyxFQUFFLGFBQWEsQ0FBQztRQUFDLFFBQVEsRUFBRSxJQUFJLENBQUM7UUFBQyxVQUFVLENBQUMsRUFBRSxNQUFNLENBQUE7S0FBRSxpQkErQ3hFO1lBUWEsb0JBQW9CO1lBZ0JwQixtQkFBbUI7WUF5Rm5CLHFCQUFxQjtJQXlEbkMseUdBQXlHO0lBQ3pHLE9BQU8sQ0FBQyxhQUFhO0lBUXJCOzs7T0FHRztJQUNJLFFBQVEsQ0FBQyxHQUFHLEVBQUUsRUFBRSxFQUFFLFFBc0J4QjtJQUVEOzs7T0FHRztJQUNJLDJCQUEyQixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9qRTtJQUVEOzs7T0FHRztJQUNJLDRCQUE0QixDQUFDLFdBQVcsRUFBRSxXQUFXLEdBQUcsSUFBSSxDQU9sRTtDQUNGIn0=
@@ -1 +1 @@
1
- {"version":3,"file":"fast_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/fast_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAW,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAChG,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAEL,KAAK,mBAAmB,EAEzB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAOzB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IATb,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAa;IAC3D,OAAO,CAAC,mBAAmB,CAAsB;IAEjD,YACE,UAAU,EAAE,6BAA6B,EACjC,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD,EAC/D,mBAAmB,CAAC,EAAE,mBAAmB,EAS1C;IAEY,IAAI,kBAGhB;IAEM,yBAAyB,+BAE/B;IAEY,cAAc,CACzB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA0C9C;IAED,UAAgB,WAAW,CACzB,OAAO,EAAE,qBAAqB,EAC9B,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA+CxE;YAQa,oBAAoB;YAYpB,mBAAmB;YAiFnB,qBAAqB;IA4CnC;;;OAGG;IACI,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,QAwBxB;IAED;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOlE;CACF"}
1
+ {"version":3,"file":"fast_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/fast_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAW,MAAM,yBAAyB,CAAC;AAEhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAGhD,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4CAA4C,CAAC;AAChG,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAEL,KAAK,mBAAmB,EAEzB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,KAAK,EAAgB,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,qBAAa,gBAAgB;IAOzB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,GAAG;IATb,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAa;IAC3D,OAAO,CAAC,mBAAmB,CAAsB;IAEjD,YACE,UAAU,EAAE,6BAA6B,EACjC,KAAK,EAAE,QAAQ,EAAE,EACjB,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,kBAAkB,EAC1B,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAkD,EAC/D,mBAAmB,CAAC,EAAE,mBAAmB,EAS1C;IAEY,IAAI,kBAGhB;IAEM,yBAAyB,+BAE/B;IAEY,cAAc,CACzB,KAAK,EAAE,0BAA0B,EACjC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAC7B,IAAI,EAAE;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA0C9C;IAED,UAAgB,WAAW,CACzB,OAAO,EAAE,qBAAqB,EAC9B,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,IAAI,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,iBA+CxE;YAQa,oBAAoB;YAgBpB,mBAAmB;YAyFnB,qBAAqB;IAyDnC,yGAAyG;IACzG,OAAO,CAAC,aAAa;IAQrB;;;OAGG;IACI,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,QAsBxB;IAED;;;OAGG;IACI,2BAA2B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOjE;IAED;;;OAGG;IACI,4BAA4B,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI,CAOlE;CACF"}
@@ -5,6 +5,7 @@ import { promiseWithResolvers } from '@aztec/foundation/promise';
5
5
  import { sleep } from '@aztec/foundation/sleep';
6
6
  import { DateProvider, elapsed } from '@aztec/foundation/timer';
7
7
  import { TxHash } from '@aztec/stdlib/tx';
8
+ import { MissingTxsTracker } from './missing_txs_tracker.js';
8
9
  import { BatchTxRequesterCollector, SendBatchRequestCollector } from './proposal_tx_collector.js';
9
10
  export class FastTxCollection {
10
11
  nodes;
@@ -51,34 +52,32 @@ export class FastTxCollection {
51
52
  // This promise is used to await for the collection to finish during the main collectFast method.
52
53
  // It gets resolved in `foundTxs` when all txs have been collected, or rejected if the request is aborted or hits the deadline.
53
54
  const promise = promiseWithResolvers();
54
- setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
55
+ const timeoutTimer = setTimeout(()=>promise.reject(new TimeoutError(`Timed out while collecting txs`)), timeout);
55
56
  const request = {
56
57
  ...input,
57
58
  blockInfo,
58
59
  promise,
59
- foundTxs: new Map(),
60
- missingTxHashes: new Set(txHashes.map((t)=>t.toString())),
60
+ missingTxTracker: MissingTxsTracker.fromArray(txHashes),
61
61
  deadline: opts.deadline
62
62
  };
63
63
  const [duration] = await elapsed(()=>this.collectFast(request, {
64
64
  ...opts
65
65
  }));
66
- this.log.verbose(`Collected ${request.foundTxs.size} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
66
+ clearTimeout(timeoutTimer);
67
+ this.log.verbose(`Collected ${request.missingTxTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, {
67
68
  ...blockInfo,
68
69
  duration,
69
70
  requestType: input.type,
70
71
  missingTxs: [
71
- ...request.missingTxHashes
72
+ ...request.missingTxTracker.missingTxHashes
72
73
  ]
73
74
  });
74
- return [
75
- ...request.foundTxs.values()
76
- ];
75
+ return request.missingTxTracker.collectedTxs;
77
76
  }
78
77
  async collectFast(request, opts) {
79
78
  this.requests.add(request);
80
79
  const { blockInfo } = request;
81
- this.log.debug(`Starting fast collection of ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
80
+ this.log.debug(`Starting fast collection of ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
82
81
  ...blockInfo,
83
82
  requestType: request.type,
84
83
  deadline: opts.deadline
@@ -94,7 +93,7 @@ export class FastTxCollection {
94
93
  waitBeforeReqResp
95
94
  ]);
96
95
  // If we have collected all txs, we can stop here
97
- if (request.missingTxHashes.size === 0) {
96
+ if (request.missingTxTracker.allFetched()) {
98
97
  this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo);
99
98
  return;
100
99
  }
@@ -113,9 +112,7 @@ export class FastTxCollection {
113
112
  const logCtx = {
114
113
  ...blockInfo,
115
114
  errorMessage: err instanceof Error ? err.message : undefined,
116
- missingTxs: [
117
- ...request.missingTxHashes
118
- ].map((txHash)=>txHash.toString())
115
+ missingTxs: request.missingTxTracker.missingTxHashes.values().map((txHash)=>txHash.toString())
119
116
  };
120
117
  if (err instanceof Error && err.name === 'TimeoutError') {
121
118
  this.log.warn(`Timed out collecting txs for ${request.type} at slot ${blockInfo.slotNumber}`, logCtx);
@@ -141,7 +138,7 @@ export class FastTxCollection {
141
138
  }
142
139
  // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them.
143
140
  const attemptsPerTx = [
144
- ...request.missingTxHashes
141
+ ...request.missingTxTracker.missingTxHashes
145
142
  ].map((txHash)=>({
146
143
  txHash,
147
144
  attempts: 0,
@@ -151,7 +148,7 @@ export class FastTxCollection {
151
148
  await Promise.allSettled(this.nodes.map((node)=>this.collectFastFromNode(request, node, attemptsPerTx, opts)));
152
149
  }
153
150
  async collectFastFromNode(request, node, attemptsPerTx, opts) {
154
- const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && request.missingTxHashes.size > 0 && this.requests.has(request);
151
+ const notFinished = ()=>this.dateProvider.now() <= +opts.deadline && !request.missingTxTracker.allFetched() && this.requests.has(request);
155
152
  const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode;
156
153
  const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize;
157
154
  const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node
@@ -165,7 +162,7 @@ export class FastTxCollection {
165
162
  const txToRequest = attemptsPerTx[index++];
166
163
  if (!txToRequest) {
167
164
  break;
168
- } else if (!request.missingTxHashes.has(txToRequest.txHash)) {
165
+ } else if (!request.missingTxTracker.isMissing(txToRequest.txHash)) {
169
166
  // Mark as found if it was found somewhere else, we'll then remove it from the array.
170
167
  // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting.
171
168
  txToRequest.found = true;
@@ -189,13 +186,20 @@ export class FastTxCollection {
189
186
  if (batch.length === 0) {
190
187
  return;
191
188
  }
189
+ const txHashes = batch.map(({ txHash })=>txHash);
192
190
  // Collect this batch from the node
193
- await this.txCollectionSink.collect((txHashes)=>node.getTxsByHash(txHashes), batch.map(({ txHash })=>TxHash.fromString(txHash)), {
191
+ await this.txCollectionSink.collect(async ()=>{
192
+ const result = await node.getTxsByHash(txHashes.map(TxHash.fromString));
193
+ for (const tx of result.validTxs){
194
+ request.missingTxTracker.markFetched(tx);
195
+ }
196
+ return result;
197
+ }, txHashes, {
194
198
  description: `fast ${node.getInfo()}`,
195
199
  node: node.getInfo(),
196
200
  method: 'fast-node-rpc',
197
201
  ...request.blockInfo
198
- });
202
+ }, this.getAddContext(request));
199
203
  // Clear from the active requests the txs we just requested
200
204
  for (const requestedTx of batch){
201
205
  activeRequestsToThisNode.delete(requestedTx.txHash);
@@ -221,39 +225,57 @@ export class FastTxCollection {
221
225
  });
222
226
  return;
223
227
  }
224
- this.log.debug(`Starting fast reqresp for ${request.missingTxHashes.size} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
228
+ this.log.debug(`Starting fast reqresp for ${request.missingTxTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, {
225
229
  ...blockInfo,
226
230
  timeoutMs,
227
231
  pinnedPeer
228
232
  });
229
233
  try {
230
- await this.txCollectionSink.collect(async (txHashes)=>{
234
+ await this.txCollectionSink.collect(async ()=>{
235
+ let result;
231
236
  if (request.type === 'proposal') {
232
- return await this.missingTxsCollector.collectTxs(txHashes, request.blockProposal, pinnedPeer, timeoutMs);
237
+ result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, request.blockProposal, pinnedPeer, timeoutMs);
233
238
  } else if (request.type === 'block') {
234
239
  const blockTxsSource = {
235
240
  txHashes: request.block.body.txEffects.map((e)=>e.txHash),
236
241
  archive: request.block.archive.root
237
242
  };
238
- return await this.missingTxsCollector.collectTxs(txHashes, blockTxsSource, pinnedPeer, timeoutMs);
243
+ result = await this.missingTxsCollector.collectTxs(request.missingTxTracker, blockTxsSource, pinnedPeer, timeoutMs);
239
244
  } else {
240
245
  throw new Error(`Unknown request type: ${request.type}`);
241
246
  }
242
- }, Array.from(request.missingTxHashes).map((txHash)=>TxHash.fromString(txHash)), {
247
+ return {
248
+ validTxs: result,
249
+ invalidTxHashes: []
250
+ };
251
+ }, Array.from(request.missingTxTracker.missingTxHashes), {
243
252
  description: `reqresp for slot ${slotNumber}`,
244
253
  method: 'fast-req-resp',
245
254
  ...opts,
246
255
  ...request.blockInfo
247
- });
256
+ }, this.getAddContext(request));
248
257
  } catch (err) {
249
258
  this.log.error(`Error sending fast reqresp request for txs`, err, {
250
259
  txs: [
251
- ...request.missingTxHashes
260
+ ...request.missingTxTracker.missingTxHashes
252
261
  ],
253
262
  ...blockInfo
254
263
  });
255
264
  }
256
265
  }
266
+ /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */ getAddContext(request) {
267
+ if (request.type === 'proposal') {
268
+ return {
269
+ type: 'proposal',
270
+ blockHeader: request.blockProposal.blockHeader
271
+ };
272
+ } else {
273
+ return {
274
+ type: 'mined',
275
+ block: request.block
276
+ };
277
+ }
278
+ }
257
279
  /**
258
280
  * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found.
259
281
  * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event.
@@ -262,22 +284,20 @@ export class FastTxCollection {
262
284
  for (const tx of txs){
263
285
  const txHash = tx.txHash.toString();
264
286
  // Remove the tx hash from the missing set, and add it to the found set.
265
- if (request.missingTxHashes.has(txHash)) {
266
- request.missingTxHashes.delete(txHash);
267
- request.foundTxs.set(txHash, tx);
287
+ if (request.missingTxTracker.markFetched(tx)) {
268
288
  this.log.trace(`Found tx ${txHash} for fast collection request`, {
269
289
  ...request.blockInfo,
270
290
  txHash: tx.txHash.toString(),
271
291
  type: request.type
272
292
  });
273
- // If we found all txs for this request, we resolve the promise
274
- if (request.missingTxHashes.size === 0) {
275
- this.log.trace(`All txs found for fast collection request`, {
276
- ...request.blockInfo,
277
- type: request.type
278
- });
279
- request.promise.resolve();
280
- }
293
+ }
294
+ // If we found all txs for this request, we resolve the promise
295
+ if (request.missingTxTracker.allFetched()) {
296
+ this.log.trace(`All txs found for fast collection request`, {
297
+ ...request.blockInfo,
298
+ type: request.type
299
+ });
300
+ request.promise.resolve();
281
301
  }
282
302
  }
283
303
  }
@@ -0,0 +1,53 @@
1
+ import { type Logger } from '@aztec/foundation/log';
2
+ import { DateProvider } from '@aztec/foundation/timer';
3
+ import { Tx, TxHash } from '@aztec/stdlib/tx';
4
+ import type { FileStoreTxSource } from './file_store_tx_source.js';
5
+ import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js';
6
+ /** Configuration for a FileStoreTxCollection instance. */
7
+ export type FileStoreCollectionConfig = {
8
+ workerCount: number;
9
+ backoffBaseMs: number;
10
+ backoffMaxMs: number;
11
+ };
12
+ /**
13
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
14
+ * Uses a shared worker pool that pulls entries with priority (fewest attempts first),
15
+ * retries with round-robin across sources, and applies exponential backoff between
16
+ * full cycles through all sources.
17
+ */
18
+ export declare class FileStoreTxCollection {
19
+ private readonly sources;
20
+ private readonly txCollectionSink;
21
+ private readonly config;
22
+ private readonly dateProvider;
23
+ private readonly log;
24
+ /** Map from tx hash string to entry for all pending downloads. */
25
+ private entries;
26
+ /** Worker promises for the shared worker pool. */
27
+ private workers;
28
+ /** Whether the worker pool is running. */
29
+ private running;
30
+ /** Signal used to wake sleeping workers when new entries arrive or stop is called. */
31
+ private wakeSignal;
32
+ constructor(sources: FileStoreTxSource[], txCollectionSink: TxCollectionSink, config: FileStoreCollectionConfig, dateProvider?: DateProvider, log?: Logger);
33
+ /** Starts the shared worker pool. */
34
+ start(): void;
35
+ /** Stops all workers and clears state. */
36
+ stop(): Promise<void>;
37
+ /** Adds entries to the shared map and wakes workers. */
38
+ startCollecting(txHashes: TxHash[], context: TxAddContext, deadline: Date): void;
39
+ /** Removes entries for txs that have been found elsewhere. */
40
+ foundTxs(txs: Tx[]): void;
41
+ /** Clears all pending entries. */
42
+ clearPending(): void;
43
+ private workerLoop;
44
+ /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */
45
+ private getNextAction;
46
+ /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */
47
+ private getBackoffMs;
48
+ /** Resolves the current wake signal and creates a new one. */
49
+ private wake;
50
+ private waitForWake;
51
+ private sleepOrWake;
52
+ }
53
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZV9zdG9yZV90eF9jb2xsZWN0aW9uLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvc2VydmljZXMvdHhfY29sbGVjdGlvbi9maWxlX3N0b3JlX3R4X2NvbGxlY3Rpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLEtBQUssTUFBTSxFQUFnQixNQUFNLHVCQUF1QixDQUFDO0FBR2xFLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUN2RCxPQUFPLEVBQUUsRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRTlDLE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLE1BQU0sMkJBQTJCLENBQUM7QUFDbkUsT0FBTyxLQUFLLEVBQUUsWUFBWSxFQUFFLGdCQUFnQixFQUFFLE1BQU0seUJBQXlCLENBQUM7QUFFOUUsMERBQTBEO0FBQzFELE1BQU0sTUFBTSx5QkFBeUIsR0FBRztJQUN0QyxXQUFXLEVBQUUsTUFBTSxDQUFDO0lBQ3BCLGFBQWEsRUFBRSxNQUFNLENBQUM7SUFDdEIsWUFBWSxFQUFFLE1BQU0sQ0FBQztDQUN0QixDQUFDO0FBV0Y7Ozs7O0dBS0c7QUFDSCxxQkFBYSxxQkFBcUI7SUFjOUIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxPQUFPO0lBQ3hCLE9BQU8sQ0FBQyxRQUFRLENBQUMsZ0JBQWdCO0lBQ2pDLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQUN2QixPQUFPLENBQUMsUUFBUSxDQUFDLFlBQVk7SUFDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHO0lBakJ0QixrRUFBa0U7SUFDbEUsT0FBTyxDQUFDLE9BQU8sQ0FBdUM7SUFFdEQsa0RBQWtEO0lBQ2xELE9BQU8sQ0FBQyxPQUFPLENBQXVCO0lBRXRDLDBDQUEwQztJQUMxQyxPQUFPLENBQUMsT0FBTyxDQUFTO0lBRXhCLHNGQUFzRjtJQUN0RixPQUFPLENBQUMsVUFBVSxDQUE2QjtJQUUvQyxZQUNtQixPQUFPLEVBQUUsaUJBQWlCLEVBQUUsRUFDNUIsZ0JBQWdCLEVBQUUsZ0JBQWdCLEVBQ2xDLE1BQU0sRUFBRSx5QkFBeUIsRUFDakMsWUFBWSxHQUFFLFlBQWlDLEVBQy9DLEdBQUcsR0FBRSxNQUFxRCxFQUc1RTtJQUVELHFDQUFxQztJQUM5QixLQUFLLElBQUksSUFBSSxDQVNuQjtJQUVELDBDQUEwQztJQUM3QixJQUFJLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQU1qQztJQUVELHdEQUF3RDtJQUNqRCxlQUFlLENBQUMsUUFBUSxFQUFFLE1BQU0sRUFBRSxFQUFFLE9BQU8sRUFBRSxZQUFZLEVBQUUsUUFBUSxFQUFFLElBQUksR0FBRyxJQUFJLENBc0J0RjtJQUVELDhEQUE4RDtJQUN2RCxRQUFRLENBQUMsR0FBRyxFQUFFLEVBQUUsRUFBRSxHQUFHLElBQUksQ0FJL0I7SUFFRCxrQ0FBa0M7SUFDM0IsWUFBWSxJQUFJLElBQUksQ0FFMUI7WUFFYSxVQUFVO0lBa0N4QixxR0FBcUc7SUFDckcsT0FBTyxDQUFDLGFBQWE7SUE4QnJCLDZGQUE2RjtJQUM3RixPQUFPLENBQUMsWUFBWTtJQVFwQiw4REFBOEQ7SUFDOUQsT0FBTyxDQUFDLElBQUk7WUFNRSxXQUFXO1lBS1gsV0FBVztDQUcxQiJ9
@@ -0,0 +1 @@
1
+ {"version":3,"file":"file_store_tx_collection.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_collection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE9E,0DAA0D;AAC1D,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAWF;;;;;GAKG;AACH,qBAAa,qBAAqB;IAc9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,GAAG;IAjBtB,kEAAkE;IAClE,OAAO,CAAC,OAAO,CAAuC;IAEtD,kDAAkD;IAClD,OAAO,CAAC,OAAO,CAAuB;IAEtC,0CAA0C;IAC1C,OAAO,CAAC,OAAO,CAAS;IAExB,sFAAsF;IACtF,OAAO,CAAC,UAAU,CAA6B;IAE/C,YACmB,OAAO,EAAE,iBAAiB,EAAE,EAC5B,gBAAgB,EAAE,gBAAgB,EAClC,MAAM,EAAE,yBAAyB,EACjC,YAAY,GAAE,YAAiC,EAC/C,GAAG,GAAE,MAAqD,EAG5E;IAED,qCAAqC;IAC9B,KAAK,IAAI,IAAI,CASnB;IAED,0CAA0C;IAC7B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAMjC;IAED,wDAAwD;IACjD,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,GAAG,IAAI,CAsBtF;IAED,8DAA8D;IACvD,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,IAAI,CAI/B;IAED,kCAAkC;IAC3B,YAAY,IAAI,IAAI,CAE1B;YAEa,UAAU;IAkCxB,qGAAqG;IACrG,OAAO,CAAC,aAAa;IA8BrB,6FAA6F;IAC7F,OAAO,CAAC,YAAY;IAQpB,8DAA8D;IAC9D,OAAO,CAAC,IAAI;YAME,WAAW;YAKX,WAAW;CAG1B"}
@@ -0,0 +1,167 @@
1
+ import { createLogger } from '@aztec/foundation/log';
2
+ import { promiseWithResolvers } from '@aztec/foundation/promise';
3
+ import { sleep } from '@aztec/foundation/sleep';
4
+ import { DateProvider } from '@aztec/foundation/timer';
5
+ import { TxHash } from '@aztec/stdlib/tx';
6
+ /**
7
+ * Collects txs from file stores as a fallback after P2P methods have been tried.
8
+ * Uses a shared worker pool that pulls entries with priority (fewest attempts first),
9
+ * retries with round-robin across sources, and applies exponential backoff between
10
+ * full cycles through all sources.
11
+ */ export class FileStoreTxCollection {
12
+ sources;
13
+ txCollectionSink;
14
+ config;
15
+ dateProvider;
16
+ log;
17
+ /** Map from tx hash string to entry for all pending downloads. */ entries;
18
+ /** Worker promises for the shared worker pool. */ workers;
19
+ /** Whether the worker pool is running. */ running;
20
+ /** Signal used to wake sleeping workers when new entries arrive or stop is called. */ wakeSignal;
21
+ constructor(sources, txCollectionSink, config, dateProvider = new DateProvider(), log = createLogger('p2p:file_store_tx_collection')){
22
+ this.sources = sources;
23
+ this.txCollectionSink = txCollectionSink;
24
+ this.config = config;
25
+ this.dateProvider = dateProvider;
26
+ this.log = log;
27
+ this.entries = new Map();
28
+ this.workers = [];
29
+ this.running = false;
30
+ this.wakeSignal = promiseWithResolvers();
31
+ }
32
+ /** Starts the shared worker pool. */ start() {
33
+ if (this.sources.length === 0) {
34
+ this.log.debug('No file store sources configured');
35
+ return;
36
+ }
37
+ this.running = true;
38
+ for(let i = 0; i < this.config.workerCount; i++){
39
+ this.workers.push(this.workerLoop());
40
+ }
41
+ }
42
+ /** Stops all workers and clears state. */ async stop() {
43
+ this.running = false;
44
+ this.wake();
45
+ await Promise.all(this.workers);
46
+ this.workers = [];
47
+ this.entries.clear();
48
+ }
49
+ /** Adds entries to the shared map and wakes workers. */ startCollecting(txHashes, context, deadline) {
50
+ if (this.sources.length === 0 || txHashes.length === 0) {
51
+ return;
52
+ }
53
+ if (+deadline <= this.dateProvider.now()) {
54
+ return;
55
+ }
56
+ for (const txHash of txHashes){
57
+ const hashStr = txHash.toString();
58
+ if (!this.entries.has(hashStr)) {
59
+ this.entries.set(hashStr, {
60
+ txHash: hashStr,
61
+ context,
62
+ deadline,
63
+ attempts: 0,
64
+ lastAttemptTime: 0,
65
+ nextSourceIndex: Math.floor(Math.random() * this.sources.length)
66
+ });
67
+ }
68
+ }
69
+ this.wake();
70
+ }
71
+ /** Removes entries for txs that have been found elsewhere. */ foundTxs(txs) {
72
+ for (const tx of txs){
73
+ this.entries.delete(tx.getTxHash().toString());
74
+ }
75
+ }
76
+ /** Clears all pending entries. */ clearPending() {
77
+ this.entries.clear();
78
+ }
79
+ async workerLoop() {
80
+ while(this.running){
81
+ const action = this.getNextAction();
82
+ if (action.type === 'sleep') {
83
+ await action.promise;
84
+ continue;
85
+ }
86
+ const entry = action.entry;
87
+ const source = this.sources[entry.nextSourceIndex % this.sources.length];
88
+ entry.nextSourceIndex++;
89
+ entry.attempts++;
90
+ entry.lastAttemptTime = this.dateProvider.now();
91
+ try {
92
+ const result = await this.txCollectionSink.collect(()=>source.getTxsByHash([
93
+ TxHash.fromString(entry.txHash)
94
+ ]), [
95
+ entry.txHash
96
+ ], {
97
+ description: `file-store ${source.getInfo()}`,
98
+ method: 'file-store',
99
+ fileStore: source.getInfo()
100
+ }, entry.context);
101
+ if (result.txs.length > 0) {
102
+ this.entries.delete(entry.txHash);
103
+ }
104
+ } catch (err) {
105
+ this.log.trace(`Error downloading tx ${entry.txHash} from ${source.getInfo()}`, {
106
+ err
107
+ });
108
+ }
109
+ }
110
+ }
111
+ /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */ getNextAction() {
112
+ const now = this.dateProvider.now();
113
+ let best;
114
+ let earliestReadyAt = Infinity;
115
+ for (const [key, entry] of this.entries){
116
+ if (+entry.deadline <= now) {
117
+ this.entries.delete(key);
118
+ continue;
119
+ }
120
+ const backoffMs = this.getBackoffMs(entry);
121
+ const readyAt = entry.lastAttemptTime + backoffMs;
122
+ if (readyAt > now) {
123
+ earliestReadyAt = Math.min(earliestReadyAt, readyAt);
124
+ continue;
125
+ }
126
+ if (!best || entry.attempts < best.attempts) {
127
+ best = entry;
128
+ }
129
+ }
130
+ if (best) {
131
+ return {
132
+ type: 'process',
133
+ entry: best
134
+ };
135
+ }
136
+ if (earliestReadyAt < Infinity) {
137
+ return {
138
+ type: 'sleep',
139
+ promise: this.sleepOrWake(earliestReadyAt - now)
140
+ };
141
+ }
142
+ return {
143
+ type: 'sleep',
144
+ promise: this.waitForWake()
145
+ };
146
+ }
147
+ /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */ getBackoffMs(entry) {
148
+ const fullCycles = Math.floor(entry.attempts / this.sources.length);
149
+ if (fullCycles === 0) {
150
+ return 0;
151
+ }
152
+ return Math.min(this.config.backoffBaseMs * Math.pow(2, fullCycles - 1), this.config.backoffMaxMs);
153
+ }
154
+ /** Resolves the current wake signal and creates a new one. */ wake() {
155
+ this.wakeSignal.resolve();
156
+ this.wakeSignal = promiseWithResolvers();
157
+ }
158
+ /** Waits until the wake signal is resolved. */ async waitForWake() {
159
+ await this.wakeSignal.promise;
160
+ }
161
+ /** Sleeps for the given duration or until the wake signal is resolved. */ async sleepOrWake(ms) {
162
+ await Promise.race([
163
+ sleep(ms),
164
+ this.wakeSignal.promise
165
+ ]);
166
+ }
167
+ }
@@ -0,0 +1,37 @@
1
+ import { type Logger } from '@aztec/foundation/log';
2
+ import { type TxHash } from '@aztec/stdlib/tx';
3
+ import { type TelemetryClient } from '@aztec/telemetry-client';
4
+ import type { TxSource, TxSourceCollectionResult } from './tx_source.js';
5
+ /** TxSource implementation that downloads txs from a file store. */
6
+ export declare class FileStoreTxSource implements TxSource {
7
+ private readonly fileStore;
8
+ private readonly baseUrl;
9
+ private readonly basePath;
10
+ private readonly log;
11
+ private downloadsSuccess;
12
+ private downloadsFailed;
13
+ private downloadDuration;
14
+ private downloadSize;
15
+ private constructor();
16
+ /**
17
+ * Creates a FileStoreTxSource from a URL.
18
+ * @param url - The file store URL (s3://, gs://, file://, http://, https://).
19
+ * @param basePath - Base path for tx files within the store.
20
+ * @param log - Optional logger.
21
+ * @param telemetry - Optional telemetry client.
22
+ * @returns The FileStoreTxSource instance, or undefined if creation fails.
23
+ */
24
+ static create(url: string, basePath: string, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource | undefined>;
25
+ getInfo(): string;
26
+ getTxsByHash(txHashes: TxHash[]): Promise<TxSourceCollectionResult>;
27
+ }
28
+ /**
29
+ * Creates FileStoreTxSource instances from URLs.
30
+ * @param urls - Array of file store URLs.
31
+ * @param basePath - Base path for tx files within each store.
32
+ * @param log - Optional logger.
33
+ * @param telemetry - Optional telemetry client.
34
+ * @returns Array of successfully created FileStoreTxSource instances.
35
+ */
36
+ export declare function createFileStoreTxSources(urls: string[], basePath: string, log?: Logger, telemetry?: TelemetryClient): Promise<FileStoreTxSource[]>;
37
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZV9zdG9yZV90eF9zb3VyY2UuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9zZXJ2aWNlcy90eF9jb2xsZWN0aW9uL2ZpbGVfc3RvcmVfdHhfc291cmNlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLE1BQU0sRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUdsRSxPQUFPLEVBQU0sS0FBSyxNQUFNLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUNuRCxPQUFPLEVBR0wsS0FBSyxlQUFlLEVBR3JCLE1BQU0seUJBQXlCLENBQUM7QUFFakMsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLHdCQUF3QixFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFekUsb0VBQW9FO0FBQ3BFLHFCQUFhLGlCQUFrQixZQUFXLFFBQVE7SUFPOUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTO0lBQzFCLE9BQU8sQ0FBQyxRQUFRLENBQUMsT0FBTztJQUN4QixPQUFPLENBQUMsUUFBUSxDQUFDLFFBQVE7SUFDekIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHO0lBVHRCLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBZ0I7SUFDeEMsT0FBTyxDQUFDLGVBQWUsQ0FBZ0I7SUFDdkMsT0FBTyxDQUFDLGdCQUFnQixDQUFZO0lBQ3BDLE9BQU8sQ0FBQyxZQUFZLENBQVk7SUFFaEMsT0FBTyxlQVlOO0lBRUQ7Ozs7Ozs7T0FPRztJQUNILE9BQW9CLE1BQU0sQ0FDeEIsR0FBRyxFQUFFLE1BQU0sRUFDWCxRQUFRLEVBQUUsTUFBTSxFQUNoQixHQUFHLEdBQUUsTUFBaUQsRUFDdEQsU0FBUyxHQUFFLGVBQXNDLEdBQ2hELE9BQU8sQ0FBQyxpQkFBaUIsR0FBRyxTQUFTLENBQUMsQ0FZeEM7SUFFTSxPQUFPLElBQUksTUFBTSxDQUV2QjtJQUVZLFlBQVksQ0FBQyxRQUFRLEVBQUUsTUFBTSxFQUFFLEdBQUcsT0FBTyxDQUFDLHdCQUF3QixDQUFDLENBK0IvRTtDQUNGO0FBRUQ7Ozs7Ozs7R0FPRztBQUNILHdCQUFzQix3QkFBd0IsQ0FDNUMsSUFBSSxFQUFFLE1BQU0sRUFBRSxFQUNkLFFBQVEsRUFBRSxNQUFNLEVBQ2hCLEdBQUcsR0FBRSxNQUFpRCxFQUN0RCxTQUFTLEdBQUUsZUFBc0MsR0FDaEQsT0FBTyxDQUFDLGlCQUFpQixFQUFFLENBQUMsQ0FHOUIifQ==
@@ -0,0 +1 @@
1
+ {"version":3,"file":"file_store_tx_source.d.ts","sourceRoot":"","sources":["../../../src/services/tx_collection/file_store_tx_source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAM,KAAK,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAGL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,KAAK,EAAE,QAAQ,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAEzE,oEAAoE;AACpE,qBAAa,iBAAkB,YAAW,QAAQ;IAO9C,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,GAAG;IATtB,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,gBAAgB,CAAY;IACpC,OAAO,CAAC,YAAY,CAAY;IAEhC,OAAO,eAYN;IAED;;;;;;;OAOG;IACH,OAAoB,MAAM,CACxB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAYxC;IAEM,OAAO,IAAI,MAAM,CAEvB;IAEY,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CA+B/E;CACF;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,MAAM,EAChB,GAAG,GAAE,MAAiD,EACtD,SAAS,GAAE,eAAsC,GAChD,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAG9B"}