karafka 2.6.0.beta1 → 2.6.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +64 -27
  3. data/certs/expired.txt +2 -0
  4. data/config/locales/errors.yml +5 -0
  5. data/karafka.gemspec +1 -1
  6. data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -1
  7. data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
  8. data/lib/karafka/active_job/current_attributes.rb +4 -2
  9. data/lib/karafka/active_job/dispatcher.rb +1 -2
  10. data/lib/karafka/admin/consumer_groups.rb +8 -2
  11. data/lib/karafka/admin/replication.rb +6 -3
  12. data/lib/karafka/admin/topics.rb +32 -12
  13. data/lib/karafka/admin.rb +155 -59
  14. data/lib/karafka/app.rb +1 -1
  15. data/lib/karafka/base_consumer.rb +33 -5
  16. data/lib/karafka/cli/console.rb +0 -1
  17. data/lib/karafka/cli/help.rb +0 -1
  18. data/lib/karafka/cli/info.rb +0 -1
  19. data/lib/karafka/cli/install.rb +0 -1
  20. data/lib/karafka/cli/server.rb +0 -1
  21. data/lib/karafka/cli/swarm.rb +0 -1
  22. data/lib/karafka/cli.rb +2 -2
  23. data/lib/karafka/connection/client.rb +44 -7
  24. data/lib/karafka/connection/listener.rb +3 -3
  25. data/lib/karafka/connection/manager.rb +0 -1
  26. data/lib/karafka/connection/pauses_manager.rb +17 -0
  27. data/lib/karafka/connection/proxy.rb +45 -0
  28. data/lib/karafka/connection/status.rb +0 -1
  29. data/lib/karafka/instrumentation/assignments_tracker.rb +34 -0
  30. data/lib/karafka/instrumentation/callbacks/consumer_groups/decorator.rb +19 -0
  31. data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb +0 -1
  32. data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb +3 -2
  33. data/lib/karafka/instrumentation/critical_errors_listener.rb +85 -0
  34. data/lib/karafka/instrumentation/logger_listener.rb +2 -0
  35. data/lib/karafka/instrumentation/monitor.rb +2 -3
  36. data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +0 -1
  37. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +0 -1
  38. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +4 -3
  39. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +0 -2
  40. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +181 -18
  41. data/lib/karafka/pro/admin/recovery/errors.rb +0 -1
  42. data/lib/karafka/pro/admin/recovery.rb +6 -4
  43. data/lib/karafka/pro/base_consumer.rb +5 -5
  44. data/lib/karafka/pro/cleaner/messages/metadata.rb +0 -1
  45. data/lib/karafka/pro/cli/parallel_segments/base.rb +1 -2
  46. data/lib/karafka/pro/connection/manager.rb +7 -4
  47. data/lib/karafka/pro/encryption/messages/middleware.rb +5 -0
  48. data/lib/karafka/pro/encryption/messages/parser.rb +4 -0
  49. data/lib/karafka/pro/instrumentation/callbacks/consumer_groups/decorator.rb +63 -0
  50. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/compensator.rb +120 -0
  51. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/fetcher.rb +88 -0
  52. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/refresher.rb +185 -0
  53. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/registry.rb +102 -0
  54. data/lib/karafka/pro/instrumentation/performance_tracker.rb +40 -6
  55. data/lib/karafka/pro/iterator/tpl_builder.rb +27 -7
  56. data/lib/karafka/pro/iterator.rb +22 -7
  57. data/lib/karafka/pro/loader.rb +12 -0
  58. data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/consumer.rb +0 -3
  59. data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/tracker.rb +2 -4
  60. data/lib/karafka/pro/processing/consumer_groups/collapser.rb +6 -9
  61. data/lib/karafka/pro/processing/consumer_groups/coordinator.rb +1 -4
  62. data/lib/karafka/pro/processing/consumer_groups/coordinators/errors_tracker.rb +0 -3
  63. data/lib/karafka/pro/processing/consumer_groups/coordinators/filters_applier.rb +21 -10
  64. data/lib/karafka/pro/processing/consumer_groups/coordinators/virtual_offset_manager.rb +4 -4
  65. data/lib/karafka/pro/processing/consumer_groups/expansions_selector.rb +0 -3
  66. data/lib/karafka/pro/processing/consumer_groups/filters/actions.rb +84 -0
  67. data/lib/karafka/pro/processing/consumer_groups/filters/base.rb +5 -6
  68. data/lib/karafka/pro/processing/consumer_groups/filters/delayer.rb +2 -5
  69. data/lib/karafka/pro/processing/consumer_groups/filters/expirer.rb +1 -4
  70. data/lib/karafka/pro/processing/consumer_groups/filters/inline_insights_delayer.rb +1 -4
  71. data/lib/karafka/pro/processing/consumer_groups/filters/throttler.rb +2 -5
  72. data/lib/karafka/pro/processing/consumer_groups/filters/virtual_limiter.rb +0 -3
  73. data/lib/karafka/pro/processing/consumer_groups/jobs/consume_non_blocking.rb +2 -3
  74. data/lib/karafka/pro/processing/consumer_groups/jobs/eofed_non_blocking.rb +0 -2
  75. data/lib/karafka/pro/processing/consumer_groups/jobs/periodic.rb +2 -3
  76. data/lib/karafka/pro/processing/consumer_groups/jobs/periodic_non_blocking.rb +0 -2
  77. data/lib/karafka/pro/processing/consumer_groups/jobs/revoked_non_blocking.rb +0 -3
  78. data/lib/karafka/pro/processing/consumer_groups/jobs_builder.rb +5 -4
  79. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/consumer.rb +3 -7
  80. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/fetcher.rb +8 -10
  81. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/listener.rb +0 -3
  82. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/base.rb +3 -5
  83. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/default.rb +0 -3
  84. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/mom.rb +0 -3
  85. data/lib/karafka/pro/processing/consumer_groups/partitioner.rb +2 -5
  86. data/lib/karafka/pro/processing/consumer_groups/periodic_job/consumer.rb +2 -5
  87. data/lib/karafka/pro/processing/consumer_groups/piping/consumer.rb +2 -5
  88. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom.rb +10 -9
  89. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom_vp.rb +11 -10
  90. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom.rb +0 -3
  91. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom_vp.rb +0 -3
  92. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom.rb +8 -7
  93. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom_vp.rb +9 -8
  94. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom.rb +5 -8
  95. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom_vp.rb +3 -6
  96. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom.rb +4 -7
  97. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom_vp.rb +6 -9
  98. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom.rb +3 -7
  99. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom_vp.rb +4 -8
  100. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom.rb +3 -6
  101. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom_vp.rb +4 -7
  102. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom.rb +2 -5
  103. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom_vp.rb +3 -6
  104. data/lib/karafka/pro/processing/consumer_groups/strategies/base.rb +0 -3
  105. data/lib/karafka/pro/processing/consumer_groups/strategies/default.rb +56 -33
  106. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/default.rb +27 -14
  107. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr.rb +0 -3
  108. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj.rb +4 -7
  109. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom.rb +5 -8
  110. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom_vp.rb +5 -8
  111. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_vp.rb +3 -7
  112. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom.rb +0 -3
  113. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom_vp.rb +0 -3
  114. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_vp.rb +0 -3
  115. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj.rb +2 -5
  116. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom.rb +0 -3
  117. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom_vp.rb +0 -3
  118. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_vp.rb +3 -6
  119. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom.rb +0 -3
  120. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom_vp.rb +3 -6
  121. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb +65 -8
  122. data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb +0 -4
  123. data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/vp.rb +2 -5
  124. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb +0 -3
  125. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb +3 -6
  126. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb +4 -7
  127. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb +4 -8
  128. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb +3 -6
  129. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb +2 -5
  130. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb +3 -7
  131. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/vp.rb +2 -5
  132. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/default.rb +0 -3
  133. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr.rb +0 -3
  134. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr_vp.rb +0 -3
  135. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/vp.rb +0 -3
  136. data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb +11 -6
  137. data/lib/karafka/pro/processing/consumer_groups/strategy_selector.rb +0 -3
  138. data/lib/karafka/pro/processing/consumer_groups/subscription_groups_coordinator.rb +0 -3
  139. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/balanced.rb +0 -3
  140. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/base.rb +2 -4
  141. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/consistent.rb +0 -3
  142. data/lib/karafka/pro/processing/jobs_queue.rb +4 -1
  143. data/lib/karafka/pro/processing/schedulers/base.rb +4 -2
  144. data/lib/karafka/pro/processing/schedulers/default.rb +2 -2
  145. data/lib/karafka/pro/recurring_tasks/consumer.rb +9 -2
  146. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +0 -1
  147. data/lib/karafka/pro/recurring_tasks/executor.rb +25 -13
  148. data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb +0 -2
  149. data/lib/karafka/pro/routing/features/consumer_groups/long_running_job.rb +0 -2
  150. data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/proxy.rb +2 -2
  151. data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job.rb +0 -2
  152. data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb +0 -6
  153. data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb +5 -4
  154. data/lib/karafka/pro/routing/features/consumer_groups/patterns/config.rb +0 -6
  155. data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb +2 -1
  156. data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/pattern.rb +0 -1
  157. data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb +2 -1
  158. data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb +2 -1
  159. data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb +8 -8
  160. data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb +5 -5
  161. data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb +33 -13
  162. data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb +52 -28
  163. data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb +2 -1
  164. data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb +6 -5
  165. data/lib/karafka/pro/scheduled_messages/consumer.rb +7 -1
  166. data/lib/karafka/pro/scheduled_messages/schema_validator.rb +6 -1
  167. data/lib/karafka/pro/swarm/liveness_listener.rb +118 -13
  168. data/lib/karafka/pro.rb +0 -6
  169. data/lib/karafka/processing/consumer_groups/coordinator.rb +4 -5
  170. data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb +6 -2
  171. data/lib/karafka/processing/consumer_groups/executor.rb +4 -4
  172. data/lib/karafka/processing/consumer_groups/executors_buffer.rb +2 -4
  173. data/lib/karafka/processing/consumer_groups/expansions_selector.rb +0 -2
  174. data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb +0 -2
  175. data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb +0 -2
  176. data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb +14 -16
  177. data/lib/karafka/processing/consumer_groups/jobs/consume.rb +2 -5
  178. data/lib/karafka/processing/consumer_groups/jobs/eofed.rb +2 -3
  179. data/lib/karafka/processing/consumer_groups/jobs/idle.rb +2 -2
  180. data/lib/karafka/processing/consumer_groups/jobs/revoked.rb +2 -4
  181. data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb +2 -2
  182. data/lib/karafka/processing/consumer_groups/partitioner.rb +3 -6
  183. data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb +5 -2
  184. data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb +2 -4
  185. data/lib/karafka/processing/consumer_groups/strategies/base.rb +3 -5
  186. data/lib/karafka/processing/consumer_groups/strategies/default.rb +43 -13
  187. data/lib/karafka/processing/consumer_groups/strategies/dlq.rb +5 -2
  188. data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb +5 -2
  189. data/lib/karafka/processing/consumer_groups/strategies/mom.rb +2 -3
  190. data/lib/karafka/processing/consumer_groups/strategy_selector.rb +0 -2
  191. data/lib/karafka/processing/coordinator.rb +0 -2
  192. data/lib/karafka/processing/coordinators_buffer.rb +0 -2
  193. data/lib/karafka/processing/partitioner.rb +0 -2
  194. data/lib/karafka/processing/strategy_selector.rb +0 -2
  195. data/lib/karafka/processing/worker.rb +14 -6
  196. data/lib/karafka/processing/workers_pool.rb +21 -4
  197. data/lib/karafka/railtie.rb +0 -1
  198. data/lib/karafka/routing/contracts/routing.rb +2 -2
  199. data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb +6 -5
  200. data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb +2 -2
  201. data/lib/karafka/routing/features/deserializers/topic.rb +0 -1
  202. data/lib/karafka/routing/subscription_group.rb +1 -1
  203. data/lib/karafka/routing/topic.rb +4 -4
  204. data/lib/karafka/routing/topics.rb +21 -1
  205. data/lib/karafka/server.rb +9 -0
  206. data/lib/karafka/setup/config.rb +54 -10
  207. data/lib/karafka/setup/config_proxy.rb +2 -1
  208. data/lib/karafka/setup/contracts/config.rb +14 -0
  209. data/lib/karafka/swarm/node.rb +2 -2
  210. data/lib/karafka/version.rb +1 -2
  211. data/lib/karafka.rb +2 -2
  212. metadata +12 -4
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Lrj
@@ -31,14 +31,11 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Lrj
40
- # Filtering enabled
41
- # Long-Running Job enabled
37
+ # - Filtering enabled
38
+ # - Long-Running Job enabled
42
39
  #
43
40
  # In general aside from throttling this one will behave the same way as the Lrj
44
41
  module Ftr
@@ -69,7 +66,7 @@ module Karafka
69
66
 
70
67
  # :seek and :pause are fully handled by handle_post_filtering
71
68
  # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
72
- return unless coordinator.filter.action == :skip
69
+ return unless coordinator.filter.skip?
73
70
  elsif !revoked? && !coordinator.manual_seek?
74
71
  # If not revoked and not throttled, we move to where we were suppose to and
75
72
  # resume
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Lrj
40
- # Long-Running Job enabled
41
- # Filtering enabled
42
- # Manual offset management enabled
37
+ # - Long-Running Job enabled
38
+ # - Filtering enabled
39
+ # - Manual offset management enabled
43
40
  #
44
41
  # It is really similar to the Lrj::Ftr but we do not mark anything as consumed
45
42
  module FtrMom
@@ -67,7 +64,7 @@ module Karafka
67
64
 
68
65
  # :seek and :pause are fully handled by handle_post_filtering
69
66
  # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
70
- return unless coordinator.filter.action == :skip
67
+ return unless coordinator.filter.skip?
71
68
  elsif !revoked? && !coordinator.manual_seek?
72
69
  # If not revoked and not throttled, we move to where we were suppose to and
73
70
  # resume
@@ -31,17 +31,13 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
- # Filtering related init strategies
40
36
  module Lrj
41
- # Filtering enabled
42
- # LRJ enabled
43
- # MoM enabled
44
- # VPs enabled
37
+ # - Filtering enabled
38
+ # - LRJ enabled
39
+ # - MoM enabled
40
+ # - VPs enabled
45
41
  module FtrMomVp
46
42
  # Filtering + LRJ + Mom + VPs
47
43
  FEATURES = %i[
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Lrj
40
- # Long-Running Job enabled
41
- # Filtering enabled
42
- # Virtual Partitions enabled
37
+ # - Long-Running Job enabled
38
+ # - Filtering enabled
39
+ # - Virtual Partitions enabled
43
40
  #
44
41
  # Behaves same as non-VP because of the aggregated flow in the coordinator.
45
42
  module FtrVp
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  # Namespace for all the LRJ starting strategies
40
37
  module Lrj
41
- # Long-Running Job enabled
42
- # Manual offset management enabled
38
+ # - Long-Running Job enabled
39
+ # - Manual offset management enabled
43
40
  module Mom
44
41
  include Strategies::Default
45
42
 
@@ -31,16 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
- # Namespace for all the LRJ starting strategies
40
36
  module Lrj
41
- # Long-Running Job enabled
42
- # Manual offset management enabled
43
- # Virtual Partitions enabled
37
+ # - Long-Running Job enabled
38
+ # - Manual offset management enabled
39
+ # - Virtual Partitions enabled
44
40
  module MomVp
45
41
  include Strategies::Mom::Vp
46
42
  include Strategies::Lrj::Mom
@@ -31,14 +31,11 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Lrj
40
- # Long-Running Job enabled
41
- # Virtual Partitions enabled
37
+ # - Long-Running Job enabled
38
+ # - Virtual Partitions enabled
42
39
  module Vp
43
40
  # Same flow as the standard Lrj
44
41
  include Strategies::Vp::Default
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  # Namespace for Mom starting strategies
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Mom
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Mom
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Mom
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  # VP starting strategies
@@ -71,9 +68,13 @@ module Karafka
71
68
  # want to mark all but until the expected location
72
69
  manager.mark_until(message, offset_metadata) if coordinator.finished?
73
70
 
74
- return revoked? unless manager.markable?
71
+ # When there is no continuous stream of virtually marked offsets from the
72
+ # lowest registered one, there is no real offset to commit yet. This is not
73
+ # a failure, hence we need to report ownership truthfully: true as long as
74
+ # the assignment is still owned
75
+ return !revoked? unless manager.markable?
75
76
 
76
- manager.markable? ? super(*manager.markable) : revoked?
77
+ super(*manager.markable)
77
78
  end
78
79
  end
79
80
  ensure
@@ -93,7 +94,11 @@ module Karafka
93
94
  coordinator.synchronize do
94
95
  manager.mark(message, offset_metadata)
95
96
  manager.mark_until(message, offset_metadata) if coordinator.finished?
96
- manager.markable? ? super(*manager.markable) : revoked?
97
+
98
+ # No real offset to commit yet is not a failure, see `#mark_as_consumed`
99
+ return !revoked? unless manager.markable?
100
+
101
+ super(*manager.markable)
97
102
  end
98
103
  end
99
104
  ensure
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  # Selector of appropriate processing strategy matching topic combinations
39
36
  # When using Karafka Pro, there is a different set of strategies than for regular, as
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  # Uses the jobs queue API to lock (pause) and unlock (resume) operations of a given
39
36
  # subscription group. It is abstracted away from jobs queue on this layer because we do
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module VirtualPartitions
39
36
  module Distributors
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  # Processing components for virtual partitions
39
36
  module VirtualPartitions
@@ -41,7 +38,8 @@ module Karafka
41
38
  module Distributors
42
39
  # Base class for all virtual partition distributors
43
40
  class Base
44
- # @param config [Karafka::Pro::Routing::Features::ConsumerGroups::VirtualPartitions::Config]
41
+ # @param config
42
+ # [Karafka::Pro::Routing::Features::ConsumerGroups::VirtualPartitions::Config]
45
43
  def initialize(config)
46
44
  @config = config
47
45
  end
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module VirtualPartitions
39
36
  module Distributors
@@ -151,7 +151,10 @@ module Karafka
151
151
  @statistics[:waiting] -= @in_waiting[group_id].size
152
152
  @in_waiting[group_id].clear
153
153
  @locks[group_id].clear
154
- @async_locking = false
154
+ # Recompute the async-locking fast-path flag from the remaining groups rather than
155
+ # forcing it off: this clear is per subscription group (run on that group's recovery)
156
+ # and must not disable async locks still held by other groups
157
+ @async_locking = @locks.any? { |_group_id, locks| locks.any? }
155
158
 
156
159
  # We unlock it just in case it was blocked when clearing started
157
160
  tick(group_id)
@@ -92,7 +92,8 @@ module Karafka
92
92
 
93
93
  # Runs the shutdown jobs scheduling flow under a mutex
94
94
  #
95
- # @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Shutdown>] jobs for scheduling
95
+ # @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Shutdown>] jobs for
96
+ # scheduling
96
97
  def on_schedule_shutdown(jobs_array)
97
98
  @mutex.synchronize do
98
99
  schedule_shutdown(jobs_array)
@@ -101,7 +102,8 @@ module Karafka
101
102
 
102
103
  # Runs the idle jobs scheduling flow under a mutex
103
104
  #
104
- # @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Idle>] jobs for scheduling
105
+ # @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Idle>] jobs for
106
+ # scheduling
105
107
  def on_schedule_idle(jobs_array)
106
108
  @mutex.synchronize do
107
109
  schedule_idle(jobs_array)
@@ -31,7 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Namespace for Pro schedulers
35
34
  module Schedulers
36
35
  # Optimizes scheduler that takes into consideration of execution time needed to process
37
36
  # messages from given topics partitions. It uses the non-preemptive LJF algorithm
@@ -109,7 +108,8 @@ module Karafka
109
108
  messages = job.messages
110
109
  message = messages.first
111
110
 
112
- perf_tracker.processing_time_p95(message.topic, message.partition) * messages.size
111
+ perf_tracker.processing_time_p95(job.group_id, message.topic, message.partition) *
112
+ messages.size
113
113
  else
114
114
  # LJF will set first the most expensive, but we want to run the zero cost jobs
115
115
  # related to the lifecycle always first. That is why we "emulate" that they
@@ -71,9 +71,16 @@ module Karafka
71
71
  message.offset - 1
72
72
  )
73
73
  when "command"
74
- @executor.apply_command(payload)
74
+ # During replaying commands are only buffered. Applying them immediately would be
75
+ # pointless as the end-of-replay synchronization with the last stored schedule
76
+ # state would overwrite their effects. They are applied after the synchronization
77
+ if @executor.replaying?
78
+ @executor.replay_command(payload)
79
+
80
+ next
81
+ end
75
82
 
76
- next if @executor.replaying?
83
+ @executor.apply_command(payload)
77
84
 
78
85
  # Execute on each incoming command to have nice latency but only after replaying
79
86
  # During replaying we should not execute because there may be more state changes
@@ -31,7 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module RecurringTasks
34
- # Recurring Tasks related contracts
35
34
  module Contracts
36
35
  # Makes sure, all the expected config is defined as it should be
37
36
  class Config < Karafka::Contracts::Base
@@ -76,6 +76,14 @@ module Karafka
76
76
  end
77
77
  end
78
78
 
79
+ # Buffers a command received during replay, so it can be applied once the in-memory
80
+ # state is synchronized with the last stored schedule state. Commands applied directly
81
+ # during replay would have their effects overwritten by this synchronization
82
+ # @param command_hash [Hash] deserialized command data
83
+ def replay_command(command_hash)
84
+ @catchup_commands << command_hash
85
+ end
86
+
79
87
  # Updates the catchup state
80
88
  # @param schedule_hash [Hash] deserialized schedule hash hash
81
89
  def update_state(schedule_hash)
@@ -100,24 +108,28 @@ module Karafka
100
108
  return
101
109
  end
102
110
 
103
- # If the schedule version we have in Kafka is higher than ours, we cannot proceed
104
- # This prevents us from applying older changes to a new schedule
105
- if @catchup_schedule[:schedule_version] > schedule.version
106
- @incompatible = true
111
+ # Commands may have been received without any schedule state stored in Kafka. In such
112
+ # cases there is nothing to synchronize and only the commands are applied
113
+ if @catchup_schedule
114
+ # If the schedule version we have in Kafka is higher than ours, we cannot proceed
115
+ # This prevents us from applying older changes to a new schedule
116
+ if Gem::Version.new(@catchup_schedule[:schedule_version]) > Gem::Version.new(schedule.version)
117
+ @incompatible = true
107
118
 
108
- return
109
- end
119
+ return
120
+ end
110
121
 
111
- # Now we can synchronize the in-memory state based on the last state stored in Kafka
112
- schedule.each do |task|
113
- stored_task = @catchup_schedule[:tasks][task.id.to_sym]
122
+ # Now we can synchronize the in-memory state based on the last state stored in Kafka
123
+ schedule.each do |task|
124
+ stored_task = @catchup_schedule[:tasks][task.id.to_sym]
114
125
 
115
- next unless stored_task
126
+ next unless stored_task
116
127
 
117
- stored_previous_time = stored_task[:previous_time]
118
- task.previous_time = stored_previous_time.zero? ? 0 : Time.at(stored_previous_time)
128
+ stored_previous_time = stored_task[:previous_time]
129
+ task.previous_time = stored_previous_time.zero? ? 0 : Time.at(stored_previous_time)
119
130
 
120
- stored_task[:enabled] ? task.enable : task.disable
131
+ stored_task[:enabled] ? task.enable : task.disable
132
+ end
121
133
  end
122
134
 
123
135
  @catchup_commands.each do |cmd|
@@ -33,8 +33,6 @@ module Karafka
33
33
  module Routing
34
34
  module Features
35
35
  module ConsumerGroups
36
- # Alterations to the direct assignments that allow us to do stable direct assignments
37
- # without working with consumer groups dynamic assignments
38
36
  class DirectAssignments < Base
39
37
  # Extension allowing us to select correct subscriptions and assignments based on the
40
38
  # expanded routing setup
@@ -30,9 +30,7 @@
30
30
 
31
31
  module Karafka
32
32
  module Pro
33
- # Namespace for Pro routing enhancements
34
33
  module Routing
35
- # Namespace for additional Pro features
36
34
  module Features
37
35
  module ConsumerGroups
38
36
  # Long-Running Jobs feature config and DSL namespace.
@@ -36,8 +36,8 @@ module Karafka
36
36
  class Multiplexing < Base
37
37
  # Allows for multiplexing setup inside a consumer group definition
38
38
  module Proxy
39
- # @param min [Integer, nil] min multiplexing count or nil to set it to max, effectively
40
- # disabling dynamic multiplexing
39
+ # @param min [Integer, nil] min multiplexing count or nil to set it to max,
40
+ # effectively disabling dynamic multiplexing
41
41
  # @param max [Integer] max multiplexing count
42
42
  # @param boot [Integer] how many listeners should we start during boot by default
43
43
  # @param scale_delay [Integer] number of ms of delay before applying any scale
@@ -30,9 +30,7 @@
30
30
 
31
31
  module Karafka
32
32
  module Pro
33
- # Namespace for Pro routing enhancements
34
33
  module Routing
35
- # Namespace for additional Pro features
36
34
  module Features
37
35
  module ConsumerGroups
38
36
  # Non Blocking Job is just an alias for LRJ.
@@ -33,12 +33,6 @@ module Karafka
33
33
  module Routing
34
34
  module Features
35
35
  module ConsumerGroups
36
- # This feature allows for saving and retrieving offset metadata with custom deserialization
37
- # support. It allows for storing extra data during commits that can be then used to alter
38
- # the processing flow after a rebalance.
39
- #
40
- # @note Because this feature has zero performance impact and makes no queries to Kafka
41
- # unless requested, it is always enabled.
42
36
  class OffsetMetadata < Base
43
37
  # Empty string not to create it on each deserialization
44
38
  EMPTY_STRING = ""
@@ -50,12 +50,13 @@ module Karafka
50
50
  # groups that will be created)
51
51
  # @param partitioner [nil, #call] nil or callable partitioner
52
52
  # @param reducer [nil, #call] reducer for parallel key. It allows for using a custom
53
- # reducer to achieve enhanced parallelization when the default reducer is not enough.
53
+ # reducer to achieve enhanced parallelization when the default reducer is not
54
+ # enough.
54
55
  # @param merge_key [String] key used to build the parallel segment consumer groups
55
56
  #
56
- # @note This method is an assignor but the API is actually via the `#parallel_segments`
57
- # method. Our `Routing::Proxy` normalizes that the way we want to have it exposed
58
- # for the end users.
57
+ # @note This method is an assignor but the API is actually via the
58
+ # `#parallel_segments` method. Our `Routing::Proxy` normalizes that the way we want
59
+ # to have it exposed for the end users.
59
60
  def parallel_segments=(
60
61
  count: 1,
61
62
  partitioner: nil,
@@ -33,12 +33,6 @@ module Karafka
33
33
  module Routing
34
34
  module Features
35
35
  module ConsumerGroups
36
- # Holds pattern info reference
37
- # Type is set to:
38
- # `:regular` - in case patterns are not used and topic is just a regular existing topic
39
- # matched directly based on the name
40
- # `:discovered` - in case it is a real topic on which we started to listed
41
- # `:matcher` - represents a regular expression used by librdkafka
42
36
  class Patterns < Base
43
37
  # Config for pattern based topic Only pattern related topics are active in this context
44
38
  Config = Struct.new(
@@ -42,7 +42,8 @@ module Karafka
42
42
  @patterns = Patterns.new([])
43
43
  end
44
44
 
45
- # @return [::Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Patterns] created patterns
45
+ # @return [::Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Patterns]
46
+ # created patterns
46
47
  def patterns
47
48
  @patterns
48
49
  end
@@ -34,7 +34,6 @@ module Karafka
34
34
  module Features
35
35
  module ConsumerGroups
36
36
  class Patterns < Base
37
- # Namespace for patterns related contracts
38
37
  module Contracts
39
38
  # Contract used to validate pattern data
40
39
  class Pattern < Karafka::Contracts::Base
@@ -71,7 +71,8 @@ module Karafka
71
71
 
72
72
  # Adds the discovered topic into the routing
73
73
  #
74
- # @param pattern [Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Pattern] matched pattern
74
+ # @param pattern [Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Pattern]
75
+ # matched pattern
75
76
  # @param discovered_topic [String] topic that we discovered that should be part of the
76
77
  # routing from now on.
77
78
  # @param sg_topics [Array<Karafka::Routing::Topic>]
@@ -100,7 +100,8 @@ module Karafka
100
100
  # prevent it from running but internally in this component we need to ensure, that
101
101
  # prior to the validations we operate on a regexp
102
102
  #
103
- # @return [Regexp] returns a regexp always even if what we've received was not a regexp
103
+ # @return [Regexp] returns a regexp always even if what we've received was not a
104
+ # regexp
104
105
  def safe_regexp
105
106
  # This regexp will never match anything
106
107
  regexp.is_a?(Regexp) ? regexp : /$a/
@@ -47,15 +47,15 @@ module Karafka
47
47
  #
48
48
  # @param topic_name [String] topic name
49
49
  # @return [Karafka::Routing::Topic]
50
- # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
51
- # please create an issue.
50
+ # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see
51
+ # it, please create an issue.
52
52
  #
53
- # @note This method should not be used in context of finding multiple missing topics in
54
- # loops because it catches exceptions and attempts to expand routes. If this is used
55
- # in a loop for lookups on thousands of topics with detector expansion, this may
56
- # be slow. It should be used in the context where newly discovered topics are found
57
- # and should by design match a pattern. For quick lookups on batches of topics, it
58
- # is recommended to use a custom built lookup with conditional expander.
53
+ # @note This method should not be used in context of finding multiple missing topics
54
+ # in loops because it catches exceptions and attempts to expand routes. If this is
55
+ # used in a loop for lookups on thousands of topics with detector expansion, this
56
+ # may be slow. It should be used in the context where newly discovered topics are
57
+ # found and should by design match a pattern. For quick lookups on batches of
58
+ # topics, it is recommended to use a custom built lookup with conditional expander.
59
59
  def find(topic_name)
60
60
  super
61
61
  rescue Karafka::Errors::TopicNotFoundError
@@ -49,12 +49,12 @@ module Karafka
49
49
  # poll where messages were not received.
50
50
  # @param active [Boolean] should ticking happen for this topic assignments.
51
51
  # @param interval [Integer] minimum interval to run periodic jobs on given topic.
52
- # @param during_pause [Boolean, nil] Should periodic jobs run when partition is paused.
53
- # It is set to `nil` by default allowing for detection when this value is not
54
- # configured but should be built dynamically based on LRJ status.
52
+ # @param during_pause [Boolean, nil] Should periodic jobs run when partition is
53
+ # paused. It is set to `nil` by default allowing for detection when this value is
54
+ # not configured but should be built dynamically based on LRJ status.
55
55
  # @param during_retry [Boolean, nil] Should we run when there was an error and we are
56
- # in a retry flow. Please note that for this to work, `during_pause` also needs to be
57
- # set to true as errors retry happens after pause.
56
+ # in a retry flow. Please note that for this to work, `during_pause` also needs to
57
+ # be set to true as errors retry happens after pause.
58
58
  def periodic_job(
59
59
  active = false,
60
60
  interval: nil,