karafka 2.5.9 → 2.6.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (471) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +34 -0
  3. data/certs/expired.txt +83 -0
  4. data/config/locales/errors.yml +17 -7
  5. data/karafka.gemspec +3 -3
  6. data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -2
  7. data/lib/karafka/active_job/job_extensions.rb +1 -2
  8. data/lib/karafka/admin/configs/resource.rb +1 -2
  9. data/lib/karafka/admin/consumer_groups.rb +109 -98
  10. data/lib/karafka/admin/isolation_levels.rb +22 -0
  11. data/lib/karafka/admin/topics.rb +103 -8
  12. data/lib/karafka/admin.rb +59 -31
  13. data/lib/karafka/app.rb +16 -5
  14. data/lib/karafka/base_consumer.rb +2 -2
  15. data/lib/karafka/cli/contracts/server.rb +4 -4
  16. data/lib/karafka/cli/info.rb +1 -1
  17. data/lib/karafka/cli/topics/base.rb +10 -18
  18. data/lib/karafka/cli/topics/repartition.rb +1 -1
  19. data/lib/karafka/connection/client.rb +40 -9
  20. data/lib/karafka/connection/consumer_groups/rebalance_manager.rb +120 -0
  21. data/lib/karafka/connection/listener.rb +8 -7
  22. data/lib/karafka/connection/listeners_batch.rb +1 -1
  23. data/lib/karafka/connection/mode.rb +1 -2
  24. data/lib/karafka/connection/raw_messages_buffer.rb +0 -5
  25. data/lib/karafka/declaratives/builder.rb +65 -0
  26. data/lib/karafka/declaratives/contracts/topic.rb +28 -0
  27. data/lib/karafka/declaratives/repository.rb +52 -0
  28. data/lib/karafka/declaratives/topic.rb +100 -0
  29. data/lib/karafka/declaratives.rb +9 -0
  30. data/lib/karafka/helpers/interval_runner.rb +2 -2
  31. data/lib/karafka/instrumentation/assignments_tracker.rb +65 -2
  32. data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb +56 -0
  33. data/lib/karafka/instrumentation/callbacks/consumer_groups/rebalance.rb +93 -0
  34. data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb +59 -0
  35. data/lib/karafka/instrumentation/logger_listener.rb +27 -9
  36. data/lib/karafka/instrumentation/notifications.rb +2 -0
  37. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +14 -17
  38. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +8 -9
  39. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +7 -3
  40. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +13 -10
  41. data/lib/karafka/licenser.rb +16 -3
  42. data/lib/karafka/pro/active_job/consumer.rb +1 -2
  43. data/lib/karafka/pro/active_job/dispatcher.rb +1 -2
  44. data/lib/karafka/pro/admin/recovery.rb +19 -19
  45. data/lib/karafka/pro/base_consumer.rb +3 -3
  46. data/lib/karafka/pro/cli/contracts/server.rb +5 -5
  47. data/lib/karafka/pro/cli/parallel_segments/base.rb +7 -7
  48. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +4 -4
  49. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +6 -6
  50. data/lib/karafka/pro/iterator/tpl_builder.rb +38 -18
  51. data/lib/karafka/pro/loader.rb +15 -12
  52. data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/consumer.rb +84 -0
  53. data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/tracker.rb +97 -0
  54. data/lib/karafka/pro/processing/consumer_groups/collapser.rb +84 -0
  55. data/lib/karafka/pro/processing/consumer_groups/coordinator.rb +202 -0
  56. data/lib/karafka/pro/processing/consumer_groups/coordinators/errors_tracker.rb +124 -0
  57. data/lib/karafka/pro/processing/consumer_groups/coordinators/filters_applier.rb +157 -0
  58. data/lib/karafka/pro/processing/consumer_groups/coordinators/virtual_offset_manager.rb +212 -0
  59. data/lib/karafka/pro/processing/{filters/expirer.rb → consumer_groups/executor.rb} +17 -31
  60. data/lib/karafka/pro/processing/{jobs/periodic.rb → consumer_groups/expansions_selector.rb} +18 -21
  61. data/lib/karafka/pro/processing/consumer_groups/filters/base.rb +103 -0
  62. data/lib/karafka/pro/processing/consumer_groups/filters/delayer.rb +92 -0
  63. data/lib/karafka/pro/processing/consumer_groups/filters/expirer.rb +78 -0
  64. data/lib/karafka/pro/processing/consumer_groups/filters/inline_insights_delayer.rb +99 -0
  65. data/lib/karafka/pro/processing/consumer_groups/filters/throttler.rb +106 -0
  66. data/lib/karafka/pro/processing/consumer_groups/filters/virtual_limiter.rb +79 -0
  67. data/lib/karafka/pro/processing/{jobs → consumer_groups/jobs}/consume_non_blocking.rb +21 -17
  68. data/lib/karafka/pro/processing/{virtual_partitions/distributors/consistent.rb → consumer_groups/jobs/eofed_non_blocking.rb} +16 -14
  69. data/lib/karafka/pro/processing/consumer_groups/jobs/periodic.rb +64 -0
  70. data/lib/karafka/pro/processing/{jobs → consumer_groups/jobs}/periodic_non_blocking.rb +16 -11
  71. data/lib/karafka/pro/processing/{jobs → consumer_groups/jobs}/revoked_non_blocking.rb +19 -15
  72. data/lib/karafka/pro/processing/consumer_groups/jobs_builder.rb +95 -0
  73. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/consumer.rb +66 -0
  74. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/fetcher.rb +154 -0
  75. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/listener.rb +68 -0
  76. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/base.rb +102 -0
  77. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/default.rb +115 -0
  78. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/mom.rb +96 -0
  79. data/lib/karafka/pro/processing/consumer_groups/partitioner.rb +98 -0
  80. data/lib/karafka/pro/processing/consumer_groups/periodic_job/consumer.rb +90 -0
  81. data/lib/karafka/pro/processing/consumer_groups/piping/consumer.rb +154 -0
  82. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom.rb +93 -0
  83. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom_vp.rb +99 -0
  84. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom.rb +92 -0
  85. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom_vp.rb +90 -0
  86. data/lib/karafka/pro/processing/{strategies/aj/dlq_ftr_lrj_mom.rb → consumer_groups/strategies/aj/dlq_lrj_mom.rb} +37 -39
  87. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom_vp.rb +90 -0
  88. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom.rb +84 -0
  89. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom_vp.rb +89 -0
  90. data/lib/karafka/pro/processing/{strategies → consumer_groups/strategies}/aj/ftr_lrj_mom.rb +20 -15
  91. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom_vp.rb +91 -0
  92. data/lib/karafka/pro/processing/{strategies → consumer_groups/strategies}/aj/ftr_mom.rb +20 -15
  93. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom_vp.rb +80 -0
  94. data/lib/karafka/pro/processing/{strategies/mom/default.rb → consumer_groups/strategies/aj/lrj_mom.rb} +18 -22
  95. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom_vp.rb +106 -0
  96. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom.rb +58 -0
  97. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom_vp.rb +74 -0
  98. data/lib/karafka/pro/processing/{strategies/lrj/vp.rb → consumer_groups/strategies/base.rb} +9 -14
  99. data/lib/karafka/pro/processing/consumer_groups/strategies/default.rb +421 -0
  100. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/default.rb +285 -0
  101. data/lib/karafka/pro/processing/{strategies/dlq/lrj.rb → consumer_groups/strategies/dlq/ftr.rb} +30 -29
  102. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj.rb +95 -0
  103. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom.rb +97 -0
  104. data/lib/karafka/pro/processing/{executor.rb → consumer_groups/strategies/dlq/ftr_lrj_mom_vp.rb} +26 -15
  105. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_vp.rb +63 -0
  106. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom.rb +97 -0
  107. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom_vp.rb +63 -0
  108. data/lib/karafka/pro/{routing/features/patterns/patterns.rb → processing/consumer_groups/strategies/dlq/ftr_vp.rb} +22 -12
  109. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj.rb +83 -0
  110. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom.rb +100 -0
  111. data/lib/karafka/pro/processing/{strategies → consumer_groups/strategies}/dlq/lrj_mom_vp.rb +18 -13
  112. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_vp.rb +61 -0
  113. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom.rb +96 -0
  114. data/lib/karafka/pro/processing/{strategies/lrj → consumer_groups/strategies/dlq}/mom_vp.rb +19 -15
  115. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb +62 -0
  116. data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb +146 -0
  117. data/lib/karafka/pro/processing/{strategies/mom/ftr.rb → consumer_groups/strategies/ftr/vp.rb} +20 -28
  118. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb +119 -0
  119. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb +94 -0
  120. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb +92 -0
  121. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb +62 -0
  122. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb +61 -0
  123. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb +101 -0
  124. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb +60 -0
  125. data/lib/karafka/pro/processing/{strategies/mom → consumer_groups/strategies/lrj}/vp.rb +18 -12
  126. data/lib/karafka/pro/processing/{strategies/aj/mom_vp.rb → consumer_groups/strategies/mom/default.rb} +22 -23
  127. data/lib/karafka/pro/processing/{strategies/aj/ftr_mom_vp.rb → consumer_groups/strategies/mom/ftr.rb} +28 -28
  128. data/lib/karafka/pro/processing/{virtual_partitions/distributors/base.rb → consumer_groups/strategies/mom/ftr_vp.rb} +19 -14
  129. data/lib/karafka/pro/processing/{strategies/aj/mom.rb → consumer_groups/strategies/mom/vp.rb} +16 -12
  130. data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb +197 -0
  131. data/lib/karafka/pro/processing/consumer_groups/strategy_selector.rb +106 -0
  132. data/lib/karafka/pro/processing/consumer_groups/subscription_groups_coordinator.rb +73 -0
  133. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/balanced.rb +82 -0
  134. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/base.rb +59 -0
  135. data/lib/karafka/pro/processing/{strategies/dlq/ftr.rb → consumer_groups/virtual_partitions/distributors/consistent.rb} +18 -33
  136. data/lib/karafka/pro/processing/filters/base.rb +3 -61
  137. data/lib/karafka/pro/processing/partitioner.rb +2 -57
  138. data/lib/karafka/pro/processing/schedulers/base.rb +10 -6
  139. data/lib/karafka/pro/processing/schedulers/default.rb +6 -5
  140. data/lib/karafka/pro/recurring_tasks/executor.rb +1 -2
  141. data/lib/karafka/pro/routing/features/{active_job → consumer_groups/active_job}/builder.rb +20 -18
  142. data/lib/karafka/pro/routing/features/{inline_insights/config.rb → consumer_groups/active_job.rb} +5 -9
  143. data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator/config.rb +53 -0
  144. data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator/contracts/topic.rb +91 -0
  145. data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator/topic.rb +90 -0
  146. data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator.rb +50 -0
  147. data/lib/karafka/pro/routing/features/{patterns/contracts/consumer_group.rb → consumer_groups/dead_letter_queue/contracts/topic.rb} +25 -30
  148. data/lib/karafka/pro/routing/features/consumer_groups/dead_letter_queue/topic.rb +70 -0
  149. data/lib/karafka/pro/routing/features/consumer_groups/dead_letter_queue.rb +46 -0
  150. data/lib/karafka/pro/routing/features/{direct_assignments → consumer_groups/delaying}/config.rb +6 -4
  151. data/lib/karafka/pro/routing/features/{patterns/contracts/pattern.rb → consumer_groups/delaying/contracts/topic.rb} +13 -16
  152. data/lib/karafka/pro/routing/features/consumer_groups/delaying/topic.rb +85 -0
  153. data/lib/karafka/pro/routing/features/{adaptive_iterator/config.rb → consumer_groups/delaying.rb} +8 -11
  154. data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/config.rb +46 -0
  155. data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/contracts/consumer_group.rb +68 -0
  156. data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/contracts/topic.rb +125 -0
  157. data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb +97 -0
  158. data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/topic.rb +97 -0
  159. data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments.rb +44 -0
  160. data/lib/karafka/pro/routing/features/consumer_groups/inline_insights/config.rb +51 -0
  161. data/lib/karafka/pro/routing/features/consumer_groups/inline_insights/contracts/topic.rb +58 -0
  162. data/lib/karafka/pro/routing/features/consumer_groups/inline_insights/topic.rb +80 -0
  163. data/lib/karafka/pro/routing/features/consumer_groups/inline_insights.rb +45 -0
  164. data/lib/karafka/pro/routing/features/consumer_groups/long_running_job/config.rb +47 -0
  165. data/lib/karafka/pro/routing/features/{multiplexing/patches/contracts/consumer_group.rb → consumer_groups/long_running_job/contracts/topic.rb} +13 -15
  166. data/lib/karafka/pro/routing/features/consumer_groups/long_running_job/topic.rb +72 -0
  167. data/lib/karafka/pro/routing/features/{long_running_job.rb → consumer_groups/long_running_job.rb} +6 -4
  168. data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/config.rb +58 -0
  169. data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/contracts/routing.rb +83 -0
  170. data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/contracts/topic.rb +148 -0
  171. data/lib/karafka/pro/routing/features/{delaying/contracts/topic.rb → consumer_groups/multiplexing/patches/contracts/consumer_group.rb} +19 -14
  172. data/lib/karafka/pro/routing/features/{multiplexing/subscription_group.rb → consumer_groups/multiplexing/proxy.rb} +17 -23
  173. data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/subscription_group.rb +68 -0
  174. data/lib/karafka/pro/routing/features/{multiplexing → consumer_groups/multiplexing}/subscription_groups_builder.rb +16 -14
  175. data/lib/karafka/pro/routing/features/consumer_groups/multiplexing.rb +85 -0
  176. data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job/topic.rb +51 -0
  177. data/lib/karafka/pro/routing/features/{non_blocking_job.rb → consumer_groups/non_blocking_job.rb} +15 -13
  178. data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/config.rb +52 -0
  179. data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/contracts/topic.rb +59 -0
  180. data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb +93 -0
  181. data/lib/karafka/pro/routing/features/{offset_metadata.rb → consumer_groups/offset_metadata.rb} +16 -14
  182. data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/builder.rb +74 -0
  183. data/lib/karafka/pro/routing/features/{multiplexing → consumer_groups/parallel_segments}/config.rb +13 -15
  184. data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb +110 -0
  185. data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/contracts/consumer_group.rb +74 -0
  186. data/lib/karafka/pro/routing/features/{parallel_segments → consumer_groups/parallel_segments}/topic.rb +24 -22
  187. data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments.rb +51 -0
  188. data/lib/karafka/pro/routing/features/{patterns → consumer_groups/patterns}/builder.rb +15 -13
  189. data/lib/karafka/pro/routing/features/{offset_metadata/contracts/topic.rb → consumer_groups/patterns/config.rb} +28 -13
  190. data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb +91 -0
  191. data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/consumer_group.rb +83 -0
  192. data/lib/karafka/pro/routing/features/{parallel_segments.rb → consumer_groups/patterns/contracts/pattern.rb} +24 -10
  193. data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/topic.rb +58 -0
  194. data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb +98 -0
  195. data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb +114 -0
  196. data/lib/karafka/pro/routing/features/{recurring_tasks/contracts/topic.rb → consumer_groups/patterns/patterns.rb} +12 -13
  197. data/lib/karafka/pro/routing/features/consumer_groups/patterns/topic.rb +78 -0
  198. data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb +72 -0
  199. data/lib/karafka/pro/routing/features/consumer_groups/patterns.rb +52 -0
  200. data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/config.rb +56 -0
  201. data/lib/karafka/pro/routing/features/{direct_assignments/contracts/consumer_group.rb → consumer_groups/periodic_job/contracts/topic.rb} +17 -22
  202. data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb +122 -0
  203. data/lib/karafka/pro/routing/features/consumer_groups/periodic_job.rb +46 -0
  204. data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb +150 -0
  205. data/lib/karafka/pro/routing/features/{swarm → consumer_groups/recurring_tasks}/config.rb +7 -8
  206. data/lib/karafka/pro/routing/features/{inline_insights → consumer_groups/recurring_tasks}/contracts/topic.rb +14 -13
  207. data/lib/karafka/pro/routing/features/{scheduled_messages → consumer_groups/recurring_tasks}/proxy.rb +6 -4
  208. data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/topic.rb +72 -0
  209. data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks.rb +44 -0
  210. data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb +154 -0
  211. data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/config.rb +47 -0
  212. data/lib/karafka/pro/routing/features/{long_running_job → consumer_groups/scheduled_messages}/contracts/topic.rb +14 -12
  213. data/lib/karafka/pro/routing/features/{delaying/config.rb → consumer_groups/scheduled_messages/proxy.rb} +6 -4
  214. data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb +72 -0
  215. data/lib/karafka/pro/routing/features/{recurring_tasks/proxy.rb → consumer_groups/scheduled_messages.rb} +3 -4
  216. data/lib/karafka/pro/routing/features/{non_blocking_job/topic.rb → consumer_groups/swarm/config.rb} +9 -8
  217. data/lib/karafka/pro/routing/features/consumer_groups/swarm/contracts/routing.rb +94 -0
  218. data/lib/karafka/pro/routing/features/consumer_groups/swarm/contracts/topic.rb +95 -0
  219. data/lib/karafka/pro/routing/features/consumer_groups/swarm/topic.rb +105 -0
  220. data/lib/karafka/pro/routing/features/consumer_groups/swarm.rb +58 -0
  221. data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/config.rb +69 -0
  222. data/lib/karafka/pro/routing/features/{parallel_segments/contracts/consumer_group.rb → consumer_groups/virtual_partitions/contracts/topic.rb} +21 -18
  223. data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb +101 -0
  224. data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions.rb +46 -0
  225. data/lib/karafka/pro/routing/features/{scheduled_messages.rb → consumer_groups.rb} +3 -2
  226. data/lib/karafka/pro/routing/features/expiring/topic.rb +4 -4
  227. data/lib/karafka/pro/routing/features/filtering/topic.rb +3 -3
  228. data/lib/karafka/pro/routing/features/pausing/topic.rb +3 -3
  229. data/lib/karafka/pro/routing/features/throttling/topic.rb +4 -4
  230. data/lib/karafka/pro/setup/defaults_injector.rb +70 -0
  231. data/lib/karafka/pro/swarm/liveness_listener.rb +22 -10
  232. data/lib/karafka/processing/consumer_groups/coordinator.rb +221 -0
  233. data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb +69 -0
  234. data/lib/karafka/processing/consumer_groups/executor.rb +220 -0
  235. data/lib/karafka/processing/consumer_groups/executors_buffer.rb +94 -0
  236. data/lib/karafka/processing/consumer_groups/expansions_selector.rb +26 -0
  237. data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb +47 -0
  238. data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb +23 -0
  239. data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb +132 -0
  240. data/lib/karafka/processing/consumer_groups/jobs/consume.rb +52 -0
  241. data/lib/karafka/processing/consumer_groups/jobs/eofed.rb +34 -0
  242. data/lib/karafka/processing/consumer_groups/jobs/idle.rb +33 -0
  243. data/lib/karafka/processing/consumer_groups/jobs/revoked.rb +34 -0
  244. data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb +32 -0
  245. data/lib/karafka/processing/consumer_groups/jobs_builder.rb +36 -0
  246. data/lib/karafka/processing/consumer_groups/partitioner.rb +28 -0
  247. data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb +48 -0
  248. data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb +25 -0
  249. data/lib/karafka/processing/consumer_groups/strategies/base.rb +65 -0
  250. data/lib/karafka/processing/consumer_groups/strategies/default.rb +218 -0
  251. data/lib/karafka/processing/consumer_groups/strategies/dlq.rb +157 -0
  252. data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb +72 -0
  253. data/lib/karafka/processing/consumer_groups/strategies/mom.rb +33 -0
  254. data/lib/karafka/processing/consumer_groups/strategy_selector.rb +53 -0
  255. data/lib/karafka/processing/coordinator.rb +4 -211
  256. data/lib/karafka/processing/coordinators_buffer.rb +4 -59
  257. data/lib/karafka/processing/jobs_queue.rb +12 -4
  258. data/lib/karafka/processing/partitioner.rb +4 -18
  259. data/lib/karafka/processing/schedulers/default.rb +2 -1
  260. data/lib/karafka/processing/strategy_selector.rb +4 -42
  261. data/lib/karafka/processing/worker.rb +8 -4
  262. data/lib/karafka/processing/workers_pool.rb +158 -0
  263. data/lib/karafka/routing/builder.rb +12 -12
  264. data/lib/karafka/routing/contracts/routing.rb +3 -4
  265. data/lib/karafka/routing/features/base/expander.rb +5 -5
  266. data/lib/karafka/routing/features/consumer_groups/active_job/builder.rb +35 -0
  267. data/lib/karafka/routing/features/consumer_groups/active_job/config.rb +17 -0
  268. data/lib/karafka/routing/features/consumer_groups/active_job/contracts/topic.rb +44 -0
  269. data/lib/karafka/routing/features/consumer_groups/active_job/proxy.rb +16 -0
  270. data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb +50 -0
  271. data/lib/karafka/routing/features/consumer_groups/active_job.rb +15 -0
  272. data/lib/karafka/routing/features/consumer_groups/dead_letter_queue/config.rb +39 -0
  273. data/lib/karafka/routing/features/consumer_groups/dead_letter_queue/contracts/topic.rb +58 -0
  274. data/lib/karafka/routing/features/consumer_groups/dead_letter_queue/topic.rb +76 -0
  275. data/lib/karafka/routing/features/consumer_groups/dead_letter_queue.rb +18 -0
  276. data/lib/karafka/routing/features/consumer_groups/eofed/config.rb +17 -0
  277. data/lib/karafka/routing/features/consumer_groups/eofed/contracts/topic.rb +39 -0
  278. data/lib/karafka/routing/features/consumer_groups/eofed/topic.rb +42 -0
  279. data/lib/karafka/routing/features/consumer_groups/eofed.rb +16 -0
  280. data/lib/karafka/routing/features/consumer_groups/inline_insights/config.rb +17 -0
  281. data/lib/karafka/routing/features/consumer_groups/inline_insights/contracts/topic.rb +27 -0
  282. data/lib/karafka/routing/features/consumer_groups/inline_insights/topic.rb +42 -0
  283. data/lib/karafka/routing/features/consumer_groups/inline_insights.rb +42 -0
  284. data/lib/karafka/routing/features/consumer_groups/manual_offset_management/config.rb +17 -0
  285. data/lib/karafka/routing/features/consumer_groups/manual_offset_management/contracts/topic.rb +27 -0
  286. data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb +46 -0
  287. data/lib/karafka/routing/features/consumer_groups/manual_offset_management.rb +20 -0
  288. data/lib/karafka/routing/features/consumer_groups.rb +12 -0
  289. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +4 -19
  290. data/lib/karafka/routing/features/declaratives/topic.rb +30 -14
  291. data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
  292. data/lib/karafka/routing/router.rb +2 -2
  293. data/lib/karafka/routing/subscription_group.rb +18 -9
  294. data/lib/karafka/routing/topic.rb +25 -11
  295. data/lib/karafka/runner.rb +17 -17
  296. data/lib/karafka/server.rb +28 -6
  297. data/lib/karafka/setup/attributes_map.rb +2 -0
  298. data/lib/karafka/setup/config.rb +64 -15
  299. data/lib/karafka/setup/config_proxy.rb +1 -2
  300. data/lib/karafka/setup/contracts/config.rb +28 -8
  301. data/lib/karafka/setup/defaults_injector.rb +10 -0
  302. data/lib/karafka/status.rb +1 -2
  303. data/lib/karafka/swarm/liveness_listener.rb +7 -0
  304. data/lib/karafka/swarm/manager.rb +7 -7
  305. data/lib/karafka/swarm/node.rb +8 -0
  306. data/lib/karafka/swarm/supervisor.rb +9 -1
  307. data/lib/karafka/templates/karafka.rb.erb +11 -5
  308. data/lib/karafka/version.rb +1 -1
  309. metadata +237 -224
  310. data/lib/karafka/connection/rebalance_manager.rb +0 -116
  311. data/lib/karafka/instrumentation/callbacks/error.rb +0 -52
  312. data/lib/karafka/instrumentation/callbacks/rebalance.rb +0 -84
  313. data/lib/karafka/instrumentation/callbacks/statistics.rb +0 -55
  314. data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +0 -79
  315. data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb +0 -92
  316. data/lib/karafka/pro/processing/collapser.rb +0 -79
  317. data/lib/karafka/pro/processing/coordinator.rb +0 -197
  318. data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +0 -119
  319. data/lib/karafka/pro/processing/coordinators/filters_applier.rb +0 -152
  320. data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +0 -207
  321. data/lib/karafka/pro/processing/expansions_selector.rb +0 -52
  322. data/lib/karafka/pro/processing/filters/delayer.rb +0 -87
  323. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +0 -95
  324. data/lib/karafka/pro/processing/filters/throttler.rb +0 -101
  325. data/lib/karafka/pro/processing/filters/virtual_limiter.rb +0 -74
  326. data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +0 -51
  327. data/lib/karafka/pro/processing/jobs_builder.rb +0 -90
  328. data/lib/karafka/pro/processing/offset_metadata/consumer.rb +0 -61
  329. data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +0 -149
  330. data/lib/karafka/pro/processing/offset_metadata/listener.rb +0 -63
  331. data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +0 -98
  332. data/lib/karafka/pro/processing/parallel_segments/filters/default.rb +0 -110
  333. data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb +0 -91
  334. data/lib/karafka/pro/processing/periodic_job/consumer.rb +0 -85
  335. data/lib/karafka/pro/processing/piping/consumer.rb +0 -149
  336. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +0 -94
  337. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +0 -87
  338. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +0 -85
  339. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +0 -81
  340. data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +0 -85
  341. data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +0 -79
  342. data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +0 -84
  343. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +0 -86
  344. data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +0 -54
  345. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +0 -101
  346. data/lib/karafka/pro/processing/strategies/base.rb +0 -43
  347. data/lib/karafka/pro/processing/strategies/default.rb +0 -416
  348. data/lib/karafka/pro/processing/strategies/dlq/default.rb +0 -280
  349. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +0 -90
  350. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +0 -92
  351. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +0 -60
  352. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +0 -58
  353. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +0 -92
  354. data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +0 -58
  355. data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +0 -57
  356. data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +0 -95
  357. data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +0 -56
  358. data/lib/karafka/pro/processing/strategies/dlq/mom.rb +0 -91
  359. data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +0 -54
  360. data/lib/karafka/pro/processing/strategies/dlq/vp.rb +0 -57
  361. data/lib/karafka/pro/processing/strategies/ftr/default.rb +0 -141
  362. data/lib/karafka/pro/processing/strategies/ftr/vp.rb +0 -57
  363. data/lib/karafka/pro/processing/strategies/lrj/default.rb +0 -114
  364. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +0 -89
  365. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +0 -87
  366. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +0 -57
  367. data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +0 -56
  368. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +0 -96
  369. data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +0 -54
  370. data/lib/karafka/pro/processing/strategies/vp/default.rb +0 -192
  371. data/lib/karafka/pro/processing/strategies.rb +0 -39
  372. data/lib/karafka/pro/processing/strategy_selector.rb +0 -102
  373. data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +0 -68
  374. data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +0 -77
  375. data/lib/karafka/pro/routing/features/active_job.rb +0 -43
  376. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +0 -89
  377. data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +0 -88
  378. data/lib/karafka/pro/routing/features/adaptive_iterator.rb +0 -48
  379. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +0 -74
  380. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +0 -68
  381. data/lib/karafka/pro/routing/features/dead_letter_queue.rb +0 -44
  382. data/lib/karafka/pro/routing/features/delaying/topic.rb +0 -83
  383. data/lib/karafka/pro/routing/features/delaying.rb +0 -46
  384. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +0 -123
  385. data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +0 -95
  386. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +0 -95
  387. data/lib/karafka/pro/routing/features/direct_assignments.rb +0 -42
  388. data/lib/karafka/pro/routing/features/inline_insights/topic.rb +0 -78
  389. data/lib/karafka/pro/routing/features/inline_insights.rb +0 -43
  390. data/lib/karafka/pro/routing/features/long_running_job/config.rb +0 -45
  391. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +0 -70
  392. data/lib/karafka/pro/routing/features/multiplexing/contracts/routing.rb +0 -81
  393. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +0 -146
  394. data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +0 -58
  395. data/lib/karafka/pro/routing/features/multiplexing.rb +0 -83
  396. data/lib/karafka/pro/routing/features/offset_metadata/config.rb +0 -50
  397. data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +0 -91
  398. data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +0 -72
  399. data/lib/karafka/pro/routing/features/parallel_segments/config.rb +0 -52
  400. data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +0 -108
  401. data/lib/karafka/pro/routing/features/patterns/config.rb +0 -71
  402. data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +0 -89
  403. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +0 -56
  404. data/lib/karafka/pro/routing/features/patterns/detector.rb +0 -96
  405. data/lib/karafka/pro/routing/features/patterns/pattern.rb +0 -112
  406. data/lib/karafka/pro/routing/features/patterns/topic.rb +0 -76
  407. data/lib/karafka/pro/routing/features/patterns/topics.rb +0 -70
  408. data/lib/karafka/pro/routing/features/patterns.rb +0 -50
  409. data/lib/karafka/pro/routing/features/periodic_job/config.rb +0 -54
  410. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +0 -59
  411. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +0 -120
  412. data/lib/karafka/pro/routing/features/periodic_job.rb +0 -44
  413. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +0 -148
  414. data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +0 -45
  415. data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +0 -70
  416. data/lib/karafka/pro/routing/features/recurring_tasks.rb +0 -42
  417. data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +0 -152
  418. data/lib/karafka/pro/routing/features/scheduled_messages/config.rb +0 -45
  419. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +0 -55
  420. data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +0 -70
  421. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +0 -92
  422. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +0 -93
  423. data/lib/karafka/pro/routing/features/swarm/topic.rb +0 -103
  424. data/lib/karafka/pro/routing/features/swarm.rb +0 -56
  425. data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +0 -67
  426. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +0 -73
  427. data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +0 -99
  428. data/lib/karafka/pro/routing/features/virtual_partitions.rb +0 -44
  429. data/lib/karafka/processing/executor.rb +0 -216
  430. data/lib/karafka/processing/executors_buffer.rb +0 -90
  431. data/lib/karafka/processing/expansions_selector.rb +0 -22
  432. data/lib/karafka/processing/inline_insights/consumer.rb +0 -43
  433. data/lib/karafka/processing/inline_insights/listener.rb +0 -19
  434. data/lib/karafka/processing/inline_insights/tracker.rb +0 -129
  435. data/lib/karafka/processing/jobs/consume.rb +0 -47
  436. data/lib/karafka/processing/jobs/eofed.rb +0 -29
  437. data/lib/karafka/processing/jobs/idle.rb +0 -31
  438. data/lib/karafka/processing/jobs/revoked.rb +0 -29
  439. data/lib/karafka/processing/jobs/shutdown.rb +0 -30
  440. data/lib/karafka/processing/jobs_builder.rb +0 -34
  441. data/lib/karafka/processing/strategies/aj_dlq_mom.rb +0 -44
  442. data/lib/karafka/processing/strategies/aj_mom.rb +0 -21
  443. data/lib/karafka/processing/strategies/base.rb +0 -61
  444. data/lib/karafka/processing/strategies/default.rb +0 -214
  445. data/lib/karafka/processing/strategies/dlq.rb +0 -153
  446. data/lib/karafka/processing/strategies/dlq_mom.rb +0 -68
  447. data/lib/karafka/processing/strategies/mom.rb +0 -29
  448. data/lib/karafka/processing/workers_batch.rb +0 -29
  449. data/lib/karafka/routing/features/active_job/builder.rb +0 -33
  450. data/lib/karafka/routing/features/active_job/config.rb +0 -15
  451. data/lib/karafka/routing/features/active_job/contracts/topic.rb +0 -42
  452. data/lib/karafka/routing/features/active_job/proxy.rb +0 -14
  453. data/lib/karafka/routing/features/active_job/topic.rb +0 -48
  454. data/lib/karafka/routing/features/active_job.rb +0 -13
  455. data/lib/karafka/routing/features/dead_letter_queue/config.rb +0 -37
  456. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +0 -56
  457. data/lib/karafka/routing/features/dead_letter_queue/topic.rb +0 -74
  458. data/lib/karafka/routing/features/dead_letter_queue.rb +0 -16
  459. data/lib/karafka/routing/features/declaratives/config.rb +0 -18
  460. data/lib/karafka/routing/features/eofed/config.rb +0 -15
  461. data/lib/karafka/routing/features/eofed/contracts/topic.rb +0 -37
  462. data/lib/karafka/routing/features/eofed/topic.rb +0 -40
  463. data/lib/karafka/routing/features/eofed.rb +0 -14
  464. data/lib/karafka/routing/features/inline_insights/config.rb +0 -15
  465. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +0 -25
  466. data/lib/karafka/routing/features/inline_insights/topic.rb +0 -40
  467. data/lib/karafka/routing/features/inline_insights.rb +0 -40
  468. data/lib/karafka/routing/features/manual_offset_management/config.rb +0 -15
  469. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +0 -25
  470. data/lib/karafka/routing/features/manual_offset_management/topic.rb +0 -44
  471. data/lib/karafka/routing/features/manual_offset_management.rb +0 -18
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0a43645cb0713b1565aa8ff9b3779d5d19225f08d2449e816ab17aaa532a6ffd
4
- data.tar.gz: 438e1dcb6d6b65fd29e2d4f2feb6cee80cf8c629cba273b3e2936637379cda75
3
+ metadata.gz: 04dfc2086393c6d3d1b8c78eca503c425842453163b014dcba68a5c4231fe090
4
+ data.tar.gz: 5f62160e469b6eb2383234f630654b46fe704e9dedd896e863771c945ab8e30e
5
5
  SHA512:
6
- metadata.gz: 44a2d60d274f70321846a4190e007f4d5c75f22e995835fe92cf29a5a658ed7eb99bd96147d6cfa08e9821676ca7064fcd095ecf17b6ac4014c018d885621e27
7
- data.tar.gz: 4eb201aeb1e1d416008913c58b7f39637dce2342506f0032d8556b285025cf23f1475f8cf08f2d91a4ee00a9241902e26fc8ca7297547a41cd0b1126d09029ac
6
+ metadata.gz: 3f1bb0b95294bb925b71ff4cd91d0603b538b4f7f9a222a4789aa9aeef89f7f80b8b26ebf2623f1c44585e1de1d87d4be6dbb55748f61de9b1dd90cab68059d7
7
+ data.tar.gz: 1c79e3b072d6b53c0062f0ade986f357f677cae6d476297389732c3c130fd89e84687526729ba2590763d15d5f0279e08e3ca9fceec386401b2d9d5ec3ef2096
data/CHANGELOG.md CHANGED
@@ -1,5 +1,39 @@
1
1
  # Karafka Framework Changelog
2
2
 
3
+ ## 2.6.0 (Unreleased)
4
+ - [Maintenance] Replace manual per-version `Warning[:performance]` / `Warning[:deprecated]` flag setup in specs with a dynamic `Warning.categories`-based approach that enables all non-experimental warning categories automatically, keeping spec warning configuration in sync with karafka-rdkafka.
5
+ - [Enhancement] Replace per-partition `query_watermark_offsets` consumer calls in `Admin::Topics#read_watermark_offsets` with two batch `list_offsets` admin calls (`:earliest` and `:latest`), reducing N sequential consumer roundtrips to 2 admin roundtrips regardless of how many topics or partitions are queried. Replace per-partition `query_watermark_offsets` fallback in `Admin::ConsumerGroups#seek` (when a time-based offset resolves beyond the last message) with a single batch `list_offsets` call covering all affected partitions. Replace per-partition `query_watermark_offsets` loop in `Pro::Iterator::TplBuilder#resolve_partitions_with_negative_offsets` with one batched `offsets_for_times(epoch)` call for consumer metadata warm-up plus two batch admin `list_offsets` calls for LWM/HWM resolution, reducing N sequential calls to 3 total (Pro).
6
+ - **[Feature]** Add `Karafka::Admin.read_partition_offsets` (and `Admin::Topics#read_partition_offsets`) to expose the `rd_kafka_ListOffsets` admin API. Allows querying partition offsets by spec (`:earliest`, `:latest`, `:max_timestamp`, or a timestamp in ms) without a consumer group. Accepts an optional `isolation_level:` keyword — pass `Karafka::Admin::IsolationLevels::READ_COMMITTED` to get the Last Stable Offset (LSO) instead of the high-watermark, giving accurate lag figures on transactionally-produced topics.
7
+ - **[Feature]** Add `Karafka::Admin::IsolationLevels` module with `READ_UNCOMMITTED` and `READ_COMMITTED` constants. These wrap the underlying `Rdkafka::Bindings` integer constants so callers do not need to reference rdkafka internals directly.
8
+ - **[Feature]** Introduce standalone `Karafka::Declaratives` subsystem (`Declaratives::Topic`, `Declaratives::Repository`, `Declaratives::Builder`) decoupled from routing. Topic declarations can now be defined via `Karafka::App.declaratives.draw` independently of routing blocks. The existing `routing#config(partitions:, ...)` DSL continues to work as a backwards-compatible bridge that populates the shared declaratives repository.
9
+ - [Enhancement] CLI topic commands (`create`, `delete`, `reset`, `repartition`, `align`, `migrate`, `plan`) now read from `Karafka::App.declaratives.topics` instead of walking the routing tree. Topics declared via the standalone `declaratives.draw` DSL are now managed by all CLI commands without requiring a routing entry. The routing bridge sets `bootstrap_servers` on declarations for cluster filtering.
10
+ - [Enhancement] Move consumer-group-specific `Processing` components (`Coordinator`, `CoordinatorsBuffer`, `Executor`, `ExecutorsBuffer`, `Partitioner`, `ExpansionsSelector`, `InlineInsights`) and Pro equivalents (including `SubscriptionGroupsCoordinator`, `Collapser`, `AdaptiveIterator`, `PeriodicJob`, `ParallelSegments`, `VirtualPartitions`, `Filters`, `OffsetMetadata`, `Piping`) under the `ConsumerGroups` namespace (internal). The `config.processing.coordinator_class`, `executor_class`, `partitioner_class`, and `expansions_selector` defaults are updated accordingly; user-provided classes are unaffected. Generic processing components (`JobsQueue`, `Worker`, `WorkersPool`, `Schedulers::*`, `Jobs::Base`) stay in their original locations.
11
+ - [Enhancement] Move processing strategies (`Karafka::Processing::Strategies::*` and `Karafka::Pro::Processing::Strategies::*`) under the `ConsumerGroups` namespace (internal). Prepares the processing layer for parallel `ShareGroups::Strategies` once KIP-932 lands.
12
+ - [Enhancement] Move `Karafka::Processing::StrategySelector` and `Karafka::Pro::Processing::StrategySelector` under the `ConsumerGroups` namespace (internal). The `config.processing.strategy_selector` default is updated accordingly; user-provided selectors are unaffected.
13
+ - [Enhancement] Move `Karafka::Connection::RebalanceManager` to `Karafka::Connection::ConsumerGroups::RebalanceManager` (internal). Prepares the connection layer for a parallel `ShareGroups` namespace once KIP-932 lands.
14
+ - [Enhancement] Move the rebalance librdkafka callback to `Karafka::Instrumentation::Callbacks::ConsumerGroups::Rebalance` (internal). Published event names (`rebalance.*`) and payload shape are unchanged.
15
+ - [Enhancement] Move `Karafka::Instrumentation::Callbacks::Error` and `Karafka::Instrumentation::Callbacks::Statistics` to `Karafka::Instrumentation::Callbacks::ConsumerGroups::Error` and `ConsumerGroups::Statistics` (internal). Completes the callbacks layer namespacing — all consumer-group-specific librdkafka callbacks now live under `Callbacks::ConsumerGroups`. Published event names and payload shape are unchanged.
16
+ - [Enhancement] Namespace consumer-group-specific processing job classes under `Karafka::Processing::ConsumerGroups::Jobs` (OSS: `Consume`, `Eofed`, `Revoked`, `Shutdown`, `Idle`) and `Karafka::Pro::Processing::ConsumerGroups::Jobs` (Pro: `ConsumeNonBlocking`, `EofedNonBlocking`, `Periodic`, `PeriodicNonBlocking`, `RevokedNonBlocking`). Only `Jobs::Base` stays under `Karafka::Processing::Jobs` as the generic base class. Internal move only — no public API changes; prepares the processing layer for parallel `ShareGroups::Jobs` (KIP-932).
17
+ - [Enhancement] Move `Karafka::Processing::JobsBuilder` to `Karafka::Processing::ConsumerGroups::JobsBuilder` and `Karafka::Pro::Processing::JobsBuilder` to `Karafka::Pro::Processing::ConsumerGroups::JobsBuilder` (internal). The jobs builder only creates consumer-group-specific jobs; prepares the processing layer for a parallel `ShareGroups::JobsBuilder` once KIP-932 lands.
18
+ - [Enhancement] Nest consumer-group-specific processing config settings (`coordinator_class`, `errors_tracker_class`, `partitioner_class`, `strategy_selector`, `expansions_selector`, `executor_class`, `jobs_builder`) under `config.internal.processing.consumer_groups`. Shared settings (`jobs_queue_class`, `scheduler_class`, `worker_job_call_wrapper`) remain at the `config.internal.processing` level. Prepares the config structure for a parallel `config.internal.processing.share_groups` sub-namespace once KIP-932 lands.
19
+ - [Enhancement] Move OSS consumer-group-specific routing features (`ActiveJob`, `DeadLetterQueue`, `Eofed`, `InlineInsights`, `ManualOffsetManagement`) from `Routing::Features` to `Routing::Features::ConsumerGroups` (internal). Agnostic features (`Declaratives`, `Deserializers`) remain at the `Routing::Features` level. Prepares the routing layer for a parallel `Routing::Features::ShareGroups` namespace once KIP-932 lands.
20
+ - [Enhancement] Move Pro consumer-group-specific routing features (`ActiveJob`, `AdaptiveIterator`, `DeadLetterQueue`, `Delaying`, `DirectAssignments`, `InlineInsights`, `LongRunningJob`, `Multiplexing`, `NonBlockingJob`, `OffsetMetadata`, `ParallelSegments`, `Patterns`, `PeriodicJob`, `RecurringTasks`, `ScheduledMessages`, `Swarm`, `VirtualPartitions`) from `Pro::Routing::Features` to `Pro::Routing::Features::ConsumerGroups` (internal). Agnostic features (`Expiring`, `Filtering`, `Pausing`, `Throttling`) remain at the `Pro::Routing::Features` level.
21
+ - [Enhancement] Introduce polymorphic `Routing::Topic#group` and `Routing::SubscriptionGroup#group` accessors (with `#consumer_group` kept as a backwards-compatible alias), expose `group_id` alongside `consumer_group_id` in `Routing::Topic#to_h`, emit parallel `group:` / `group_id:` keys alongside `consumer_group:` / `consumer_group_id:` in `rebalance.*` payloads, and emit parallel `group_id:` alongside `consumer_group_id:` in `error.occurred` and `statistics.emitted` payloads (these callbacks only carry the group id, not the group object). Prepares routing for additional group types (e.g. KIP-932 share groups); legacy keys remain and will be retired in Karafka 3.0 once share groups land.
22
+ - **[Feature]** Add `Processing::WorkersPool` with dynamic thread pool scaling via `#scale`, `nil` sentinel-based worker exit for downscaling, and `worker.scaling.up`/`worker.scaling.down` instrumentation events.
23
+ - [Enhancement] Add per-partition generation tracking to `AssignmentsTracker` to distinguish first-time assignments from reassignments.
24
+ - [Enhancement] Make liveness listeners fiber-safe.
25
+ - [Fix] Include orphaned node detection in Pro swarm liveness listener.
26
+ - [Fix] Report liveness on `connection.listener.before_fetch_loop` so nodes send an initial healthy report before the first consumption.
27
+ - [Fix] Fire liveness events during `wait_pinging` so nodes continue reporting health status during shutdown with active LRJ jobs.
28
+ - [Maintenance] Use namespaced topic naming format in all integration specs for consistent traceability.
29
+ - [Fix] Fix `DataCollector::SPEC_HASH` producing non-deterministic hashes for pristine and poro specs by passing the original spec path via `KARAFKA_SPEC_PATH` env var.
30
+ - [Fix] Fix swarm supervisor leaking reader pipe file descriptors on node restarts by closing the old reader before creating a new pipe in `Node#start`.
31
+ - [Maintenance] Add `bin/tests_topics_hashes` script for looking up spec files by their topic name hash prefix.
32
+ - [Change] Require `karafka-rdkafka` `>=` `0.27.1` to pick up the fix for `poll_batch` and `poll_batch_nb` raising `RdkafkaError` with only the integer error code, which discarded topic/partition/offset context from `e.details` and caused `partition_eof` handling to call `@buffer.eof(nil, nil)`, resulting in a `TopicNotFoundError` crash.
33
+ - [Fix] Raise `InvalidLicenseTokenError` when a manually-defined `Karafka::License` module is missing required methods (`#token` or `#version`) instead of silently skipping pro component loading, which previously caused confusing `NameError` exceptions later in the boot process.
34
+ - [Enhancement] Bump swarm supervisor `SHUTDOWN_GRACE_PERIOD` from 1s to 15s to give forked nodes enough time to finish post-`shutdown_timeout` cleanup (at_exit handlers, librdkafka finalization, connection pool close) before the supervisor forcefully terminates them, especially on CI where `sleep` granularity and `waitpid` cost stretch each supervision loop iteration.
35
+ - [Maintenance] Fix flaky `pro/admin/recovery/coordinator_for_spec.rb` by warming up the `__consumer_offsets` internal topic with a produce + `seek_consumer_group` and retrying the first `Recovery.coordinator_for` call with exponential backoff on `MetadataError`.
36
+
3
37
  ## 2.5.9 (2026-03-30)
4
38
  - [Enhancement] Validate that `statistics.interval.ms` is not zero when dynamic multiplexing is enabled (Pro).
5
39
  - [Fix] Fix swarm liveness reporting to also use `on_connection_listener_fetch_loop` so nodes stay alive when `statistics.interval.ms` is disabled.
data/certs/expired.txt CHANGED
@@ -1,2 +1,85 @@
1
1
  # Expired/revoked license checksums (SHA256 of license token)
2
2
  # One checksum per line, lines starting with # are comments
3
+
4
+ 01603767339fd25cf874e6fe29315fb27b6f6bbdf2e769c3b88e6efcf4096d37
5
+ 024e2c943693c4ea9eea852eb02d32acc9cb34df21aea90af5ab39dcf3b1d8cf
6
+ 0371303ba704d8f256b40932aa7840a1adb1cdc6fd032ac902794320d0d35ff1
7
+ 051dc2d8176df8d3bde1595e01af786a3f58ccab4cc48743d624e68125d80bd6
8
+ 06e3d91c9881ebf9686f3492e53400adc2c6a5fd4a409f6775cd36094b24f181
9
+ 0ae8daf74b8ccd6c4828ed707edf0d05df3e7d328e91ddc973e160ae8b120787
10
+ 0b4420e202d9c6075bb108ac4bd5917e35e89aa7d1741afaab93ca07839d06fc
11
+ 11936f8e0637565a309157d1fd4e84a3c44e5a3f9f226435e7d38bc43e452db6
12
+ 122ecfd0493596239995b9e7425046737b4893e28e33f31815eb4727c3f607d9
13
+ 17c14c38e10d0d2bd4a9c847a2c23fd0726459337af1d9815b6e34d1fd9af8be
14
+ 227f48f0b22974c43f120de28d5ca33a77e58d33064f9a73748501ba700f1474
15
+ 24d59b10d982e5642dfda3ec54cd56dda9022927c89584bbe12d31f3df28e17d
16
+ 2634761cc75479548003f941346f0a51a6c535415696f386b40ae28f88f63efa
17
+ 284967207a93e9c989d05ad6e5bb0dbe3e0a8a871c3fba48324558f127166bda
18
+ 2e2b4ccd902e43feb418cc8b3aed5874ec194ae8495ded864aff917538d91f7a
19
+ 39aeae710122bf14729a05f24f5bd702d2a1127e5c7fb9bd90de0e20890575dc
20
+ 3ada575caaf0b8bdeca0686a93074de9ed830751e6f46251849475f70e84e408
21
+ 4367e57ca36a4b447f23745d76611514df61b56611225be4c8f9a90560bbace9
22
+ 47380207ca3041b14c7d4c81a69f820852069ecab4d80a068d39133f61a6e641
23
+ 4d49b497ece81604358afc8e9adc7d1163995c4c08446bcd5c0bbfe3d7ebc1e7
24
+ 4d97c95c46e51105705964aecbfa5219120295e7d0a172f5c098d12c3a5cbf7e
25
+ 60171be4b327d48b2445b00f1fca3d2a3554d06c69f63f3e97742b29030a0a69
26
+ 630907de3c6bf258bfc2c51d768d51ca446ebd49fb29b29a0b24d363d46d1a6e
27
+ 68db4a9700f6cc3494f8eac1e0df2471f11ef7312e7b58dae839b193e226fae5
28
+ 6a06fb85cee01a27a5e819e464b0b5fd84051d234540d11774bc17e90c91abd4
29
+ 6d79bcac25c0026dad0f1871ccae79fc07d696df2e0f080ce1d1038454545a05
30
+ 6e2ed71a0925d59815c49ea1e72dfd0ba28bdbb57f93d81fa34a31c50db53830
31
+ 711a3cced55062707b64f6d652e107d0f1318fd9f3762c732fad23d04884cdd2
32
+ 71c2d2ab89a11010c41c9938fbc2565297364b4e8378e78ddf58b22ea2c43ca2
33
+ 72b19e6e274b1f36799eea65b07f4849db44c82315647266608ebdb6fde0cad6
34
+ 731c0d75d9b47cca49d9d95fc8d291fda452383f920e70849373959892443329
35
+ 77b6974b96701d82a0670c79830543e345bf0a00d9e5b5a1bbf8443439d75987
36
+ 77e461a9a4534870cfb7cca350d56ed0a504643bb39eeb750b338179d05069a9
37
+ 7c06718c4314e28561f337b3ecc66fb012c2510f1ed5fcb82dbab5ba009a8f7d
38
+ 7ca5a6183ed9c97e4da2c7030f809bfb32da49319d106377bd0e6d7d52734141
39
+ 7ce4cade640a4f34e2909360c4c0ceb8a647b7bfead7a57880d8ce2bf6eea26c
40
+ 7d3a2cb0ab08f2f25324124973370da30210dab04abaeeed97192576238a03e0
41
+ 7dc790fd8bf18e663c2e4276f4362b0c14795945b508807ef03a92ece266cff8
42
+ 81e69732cc6e9c5266fffea08fd1c69fd12c1d841b979723a77533c5efef244e
43
+ 839f4a449e0d68acccc0f6a2d0209d2d137375d740ae6d0244dde17e63131fbb
44
+ 85d26e4fedf364e20ea33c1b0102d34cea19156243fe781ddf597f6677cf2de3
45
+ 8609d26341a5c4e908af8b63bcc6895d99d8cc13f0403dc7b36a2920bb8e048a
46
+ 8d3e09a94bd00710a49b186be832f74cc3340ec3d87e780e10159e5e8d048b81
47
+ 8e933669352c272935c310d7ce798b3ea2687611378b978e412491e7fddfce4d
48
+ 8f3190524285a5c0a0d1e24bdc2a5c417416c2cb92b9da26f0e492aaa4600bd7
49
+ 9221a49b42e05d39e1939cf9353805abea2042457c2247859e3274976ac2fbc6
50
+ 96c8dde1a7f938899f3f5bd764eff295f090e1c40635e7efd25d69ef55881db2
51
+ 9b0b791259e86740d4a2cab736802614938b7043479ac8770c570b1b37f38fb7
52
+ a0db6c5e7c8714a2d33adff3b69d2d688d5a34e6d5ba871d1cbdfcba3edf1e89
53
+ a7ce882d45a00ad82e653b60d14f1a1ebefdc9033d6cc60ed0a2ed507949a9a2
54
+ ac2941c112f397543fde83cc1b0dcf70ec34d77e149d3e90ad07a0f1bc809933
55
+ adea773fadaaaeadf0f20643e84ae6c40bcb55fb73ecee5894a41836453e4abb
56
+ b1411e92ad939b59c898e78520c8ec82547950493a45004b747f12820aa7d390
57
+ b3984622fbc6ba91a2180283a6ada7e1bbe01fb705dbaae37c831e9c893b8e00
58
+ b5ed1ff168e86300735e59dc2786be6d535fc9e0f3c6c8a76bc4354cbbed4ae0
59
+ b9e38e9c1b34db847a4b82ae35382b06e61a5872aa8c24f988d7d5246ded4648
60
+ bdc4f155153a59c25c7c52932f77973487d6336e99eb0beeb90edf4a4120a94a
61
+ bf6c5c58b02fa5e7c7dc845213ad943c5b6613f12ac6012be00b1484bb85aa04
62
+ c011c3d168a159c135ef35bf55121edaf1a4cc94f663a1f461595776b0313160
63
+ c311cb6cfb38a8b2e83b3080c1f87444a52f3f645be8337559a123aaacf1344a
64
+ c87222a94f98c3a2191dda9254bd6ed031fd8ca190f75b40c67a0fcd4b56f795
65
+ ca53af70cd7e67abbb9dbf0a8da6788a9b2192aa3a5a1dd8720aaa68a2511722
66
+ d02da0cbd2c1d83432a2cee7984dd311249692373554002d1d543dd5007f6a58
67
+ d13295e597120b88bf4c4f36de63d0aa3e5a68e3578c09a2e026bd1f6aaaf04f
68
+ d862dd776d5b42a73a63574dc00187c587403101cd211ed348fc422f6460844e
69
+ d9f25b56eae6a62b9383c068c2a7c22ef6b051f1cf4137164ba3dd1baf0bac61
70
+ db1f2c50f3dc6956c498b06f67f82412a76f0b138265a55713076b2a4a35725f
71
+ ded7405c91e9bb1994c58c728f83b705fdc5e816ce0cb3fdf16887127fff27f0
72
+ dfe08da10ae5e74e9bbf648261860f3b94201a11ab4269cd4db7c73017452dde
73
+ dff7372b5ee3136af892561507602423bbae42d12d3a65277ae2af1add666723
74
+ e1a040c0c9a6c5e4603d9aebbc4201075238b47446348ab635b53704b3095a63
75
+ e3b94496b65bb626e723637926e729a33be673fe0355d964306bde563ff6fbf7
76
+ e577d74987f1d31c10fc4ec67dd74084486514f844668d7d01bd4cad2d8f09c4
77
+ e7a91a5ce0e68b888026bbddb57c92c7d5ad61e9416c8188721eee0a8ba85bc1
78
+ eda2ab2c99f769d646dd6fff564af9ec6f32e4462390e3b3246bf953cd7a9c70
79
+ f031aa98b2375c6573f6f775f6000dc81e07b1034694742a900a9115eb124900
80
+ f1ade1306fd64c2e32b15e505d87c0a9ec7a8392f5bea014fc4e70acd17f9935
81
+ f397ccd272b6ee7b33595aa93a6e40b725af265b681efca05bb26f9a7766d6c2
82
+ f54723b14f7cfc4e3c88b946cab881b907dd5ac8372b4600213241ab82778979
83
+ f9cc7b7d9c7d81ba477a70749d166edbc2b44f1714096fce10118bb18fd9696c
84
+ fb417a807d7595a4aff44cc79712e9bc887181e64eb9a98fe9f200b585856ae7
85
+ fd5009f463fe0c38cfe66677ff9b8e5031cbc4ce354883c99c392109ae8ae3b8
@@ -25,22 +25,25 @@ en:
25
25
  Decrease max_wait_time or increase node_report_timeout
26
26
  kafka_format: needs to be a filled hash
27
27
  key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
28
+ managed_key_not_supported: >
29
+ this kafka setting is managed internally by Karafka and should not be set directly.
30
+ Setting it manually may cause misbehaviours and other unexpected issues
28
31
  pause.timeout_max_timeout_vs_pause_max_timeout: pause.timeout must be less or equal to pause.max_timeout
29
32
  shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
30
33
  worker_thread_priority_format: must be between -3 and 3
31
34
 
32
35
  oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
33
36
 
34
- internal.processing.jobs_builder_format: cannot be nil
35
37
  internal.processing.jobs_queue_class_format: cannot be nil
36
38
  internal.processing.scheduler_class_format: cannot be nil
37
- internal.processing.coordinator_class_format: cannot be nil
38
- internal.processing.partitioner_class_format: cannot be nil
39
- internal.processing.strategy_selector_format: cannot be nil
40
- internal.processing.expansions_selector_format: cannot be nil
41
- internal.processing.executor_class_format: cannot be nil
42
39
  internal.processing.worker_job_call_wrapper_format: 'needs to be false or respond to #wrap'
43
- internal.processing.errors_tracker_class_format: 'needs to be nil or a class'
40
+ internal.processing.consumer_groups.jobs_builder_format: cannot be nil
41
+ internal.processing.consumer_groups.coordinator_class_format: cannot be nil
42
+ internal.processing.consumer_groups.errors_tracker_class_format: 'needs to be nil or a class'
43
+ internal.processing.consumer_groups.partitioner_class_format: cannot be nil
44
+ internal.processing.consumer_groups.strategy_selector_format: cannot be nil
45
+ internal.processing.consumer_groups.expansions_selector_format: cannot be nil
46
+ internal.processing.consumer_groups.executor_class_format: cannot be nil
44
47
 
45
48
  internal.active_job.dispatcher_format: cannot be nil
46
49
  internal.active_job.job_options_contract_format: cannot be nil
@@ -173,6 +176,13 @@ en:
173
176
  all topic names within a single consumer group must be unique considering namespacing styles
174
177
  disable this validation by setting config.strict_topics_namespacing to false
175
178
 
179
+ declaratives:
180
+ topic:
181
+ declaratives.partitions_format: needs to be more or equal to 1
182
+ declaratives.active_format: needs to be either true or false
183
+ declaratives.replication_factor_format: needs to be more or equal to 1
184
+ declaratives.details_format: needs to be a hash with only symbol keys
185
+
176
186
  job_options:
177
187
  missing: needs to be present
178
188
  dispatch_method_format: needs to be either :produce_async or :produce_sync
data/karafka.gemspec CHANGED
@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
21
21
  without having to focus on things that are not your business domain.
22
22
  DESC
23
23
 
24
- spec.add_dependency "karafka-core", ">= 2.5.6", "< 2.6.0"
25
- spec.add_dependency "karafka-rdkafka", ">= 0.24.0"
26
- spec.add_dependency "waterdrop", ">= 2.8.14", "< 3.0.0"
24
+ spec.add_dependency "karafka-core", ">= 2.6.0", "< 2.7.0"
25
+ spec.add_dependency "karafka-rdkafka", ">= 0.27.2"
26
+ spec.add_dependency "waterdrop", ">= 2.10.1", "< 3.0.0"
27
27
  spec.add_dependency "zeitwerk", "~> 2.3"
28
28
 
29
29
  spec.required_ruby_version = ">= 3.2.0"
@@ -28,8 +28,7 @@ module ActiveJob
28
28
  Object
29
29
  end
30
30
 
31
- # Karafka adapter for enqueuing jobs
32
- # This is here for ease of integration with ActiveJob.
31
+ # Karafka adapter for enqueuing jobs This is here for ease of integration with ActiveJob.
33
32
  class KarafkaAdapter < base
34
33
  include ::Karafka::Helpers::ConfigImporter.new(
35
34
  dispatcher: %i[internal active_job dispatcher]
@@ -19,8 +19,7 @@ module Karafka
19
19
  def karafka_options(new_options = {})
20
20
  return _karafka_options if new_options.empty?
21
21
 
22
- # Make sure, that karafka options that someone wants to use are valid before assigning
23
- # them
22
+ # Make sure, that karafka options that someone wants to use are valid before assigning them
24
23
  App.config.internal.active_job.job_options_contract.validate!(
25
24
  new_options,
26
25
  scope: %w[active_job]
@@ -76,8 +76,7 @@ module Karafka
76
76
 
77
77
  private
78
78
 
79
- # Recognizes whether the type is provided and remaps it to a symbol representation if
80
- # needed
79
+ # Recognizes whether the type is provided and remaps it to a symbol representation if needed
81
80
  #
82
81
  # @param type [Symbol, Integer]
83
82
  # @return [Symbol]