karafka 2.6.0.beta1 → 2.6.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +64 -27
  3. data/certs/expired.txt +2 -0
  4. data/config/locales/errors.yml +5 -0
  5. data/karafka.gemspec +1 -1
  6. data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -1
  7. data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
  8. data/lib/karafka/active_job/current_attributes.rb +4 -2
  9. data/lib/karafka/active_job/dispatcher.rb +1 -2
  10. data/lib/karafka/admin/consumer_groups.rb +8 -2
  11. data/lib/karafka/admin/replication.rb +6 -3
  12. data/lib/karafka/admin/topics.rb +32 -12
  13. data/lib/karafka/admin.rb +155 -59
  14. data/lib/karafka/app.rb +1 -1
  15. data/lib/karafka/base_consumer.rb +33 -5
  16. data/lib/karafka/cli/console.rb +0 -1
  17. data/lib/karafka/cli/help.rb +0 -1
  18. data/lib/karafka/cli/info.rb +0 -1
  19. data/lib/karafka/cli/install.rb +0 -1
  20. data/lib/karafka/cli/server.rb +0 -1
  21. data/lib/karafka/cli/swarm.rb +0 -1
  22. data/lib/karafka/cli.rb +2 -2
  23. data/lib/karafka/connection/client.rb +44 -7
  24. data/lib/karafka/connection/listener.rb +3 -3
  25. data/lib/karafka/connection/manager.rb +0 -1
  26. data/lib/karafka/connection/pauses_manager.rb +17 -0
  27. data/lib/karafka/connection/proxy.rb +45 -0
  28. data/lib/karafka/connection/status.rb +0 -1
  29. data/lib/karafka/instrumentation/assignments_tracker.rb +34 -0
  30. data/lib/karafka/instrumentation/callbacks/consumer_groups/decorator.rb +19 -0
  31. data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb +0 -1
  32. data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb +3 -2
  33. data/lib/karafka/instrumentation/critical_errors_listener.rb +85 -0
  34. data/lib/karafka/instrumentation/logger_listener.rb +2 -0
  35. data/lib/karafka/instrumentation/monitor.rb +2 -3
  36. data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +0 -1
  37. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +0 -1
  38. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +4 -3
  39. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +0 -2
  40. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +181 -18
  41. data/lib/karafka/pro/admin/recovery/errors.rb +0 -1
  42. data/lib/karafka/pro/admin/recovery.rb +6 -4
  43. data/lib/karafka/pro/base_consumer.rb +5 -5
  44. data/lib/karafka/pro/cleaner/messages/metadata.rb +0 -1
  45. data/lib/karafka/pro/cli/parallel_segments/base.rb +1 -2
  46. data/lib/karafka/pro/connection/manager.rb +7 -4
  47. data/lib/karafka/pro/encryption/messages/middleware.rb +5 -0
  48. data/lib/karafka/pro/encryption/messages/parser.rb +4 -0
  49. data/lib/karafka/pro/instrumentation/callbacks/consumer_groups/decorator.rb +63 -0
  50. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/compensator.rb +120 -0
  51. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/fetcher.rb +88 -0
  52. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/refresher.rb +185 -0
  53. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/registry.rb +102 -0
  54. data/lib/karafka/pro/instrumentation/performance_tracker.rb +40 -6
  55. data/lib/karafka/pro/iterator/tpl_builder.rb +27 -7
  56. data/lib/karafka/pro/iterator.rb +22 -7
  57. data/lib/karafka/pro/loader.rb +12 -0
  58. data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/consumer.rb +0 -3
  59. data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/tracker.rb +2 -4
  60. data/lib/karafka/pro/processing/consumer_groups/collapser.rb +6 -9
  61. data/lib/karafka/pro/processing/consumer_groups/coordinator.rb +1 -4
  62. data/lib/karafka/pro/processing/consumer_groups/coordinators/errors_tracker.rb +0 -3
  63. data/lib/karafka/pro/processing/consumer_groups/coordinators/filters_applier.rb +21 -10
  64. data/lib/karafka/pro/processing/consumer_groups/coordinators/virtual_offset_manager.rb +4 -4
  65. data/lib/karafka/pro/processing/consumer_groups/expansions_selector.rb +0 -3
  66. data/lib/karafka/pro/processing/consumer_groups/filters/actions.rb +84 -0
  67. data/lib/karafka/pro/processing/consumer_groups/filters/base.rb +5 -6
  68. data/lib/karafka/pro/processing/consumer_groups/filters/delayer.rb +2 -5
  69. data/lib/karafka/pro/processing/consumer_groups/filters/expirer.rb +1 -4
  70. data/lib/karafka/pro/processing/consumer_groups/filters/inline_insights_delayer.rb +1 -4
  71. data/lib/karafka/pro/processing/consumer_groups/filters/throttler.rb +2 -5
  72. data/lib/karafka/pro/processing/consumer_groups/filters/virtual_limiter.rb +0 -3
  73. data/lib/karafka/pro/processing/consumer_groups/jobs/consume_non_blocking.rb +2 -3
  74. data/lib/karafka/pro/processing/consumer_groups/jobs/eofed_non_blocking.rb +0 -2
  75. data/lib/karafka/pro/processing/consumer_groups/jobs/periodic.rb +2 -3
  76. data/lib/karafka/pro/processing/consumer_groups/jobs/periodic_non_blocking.rb +0 -2
  77. data/lib/karafka/pro/processing/consumer_groups/jobs/revoked_non_blocking.rb +0 -3
  78. data/lib/karafka/pro/processing/consumer_groups/jobs_builder.rb +5 -4
  79. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/consumer.rb +3 -7
  80. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/fetcher.rb +8 -10
  81. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/listener.rb +0 -3
  82. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/base.rb +3 -5
  83. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/default.rb +0 -3
  84. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/mom.rb +0 -3
  85. data/lib/karafka/pro/processing/consumer_groups/partitioner.rb +2 -5
  86. data/lib/karafka/pro/processing/consumer_groups/periodic_job/consumer.rb +2 -5
  87. data/lib/karafka/pro/processing/consumer_groups/piping/consumer.rb +2 -5
  88. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom.rb +10 -9
  89. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom_vp.rb +11 -10
  90. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom.rb +0 -3
  91. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom_vp.rb +0 -3
  92. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom.rb +8 -7
  93. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom_vp.rb +9 -8
  94. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom.rb +5 -8
  95. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom_vp.rb +3 -6
  96. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom.rb +4 -7
  97. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom_vp.rb +6 -9
  98. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom.rb +3 -7
  99. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom_vp.rb +4 -8
  100. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom.rb +3 -6
  101. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom_vp.rb +4 -7
  102. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom.rb +2 -5
  103. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom_vp.rb +3 -6
  104. data/lib/karafka/pro/processing/consumer_groups/strategies/base.rb +0 -3
  105. data/lib/karafka/pro/processing/consumer_groups/strategies/default.rb +56 -33
  106. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/default.rb +27 -14
  107. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr.rb +0 -3
  108. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj.rb +4 -7
  109. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom.rb +5 -8
  110. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom_vp.rb +5 -8
  111. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_vp.rb +3 -7
  112. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom.rb +0 -3
  113. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom_vp.rb +0 -3
  114. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_vp.rb +0 -3
  115. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj.rb +2 -5
  116. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom.rb +0 -3
  117. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom_vp.rb +0 -3
  118. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_vp.rb +3 -6
  119. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom.rb +0 -3
  120. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom_vp.rb +3 -6
  121. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb +65 -8
  122. data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb +0 -4
  123. data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/vp.rb +2 -5
  124. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb +0 -3
  125. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb +3 -6
  126. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb +4 -7
  127. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb +4 -8
  128. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb +3 -6
  129. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb +2 -5
  130. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb +3 -7
  131. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/vp.rb +2 -5
  132. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/default.rb +0 -3
  133. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr.rb +0 -3
  134. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr_vp.rb +0 -3
  135. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/vp.rb +0 -3
  136. data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb +11 -6
  137. data/lib/karafka/pro/processing/consumer_groups/strategy_selector.rb +0 -3
  138. data/lib/karafka/pro/processing/consumer_groups/subscription_groups_coordinator.rb +0 -3
  139. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/balanced.rb +0 -3
  140. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/base.rb +2 -4
  141. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/consistent.rb +0 -3
  142. data/lib/karafka/pro/processing/jobs_queue.rb +4 -1
  143. data/lib/karafka/pro/processing/schedulers/base.rb +4 -2
  144. data/lib/karafka/pro/processing/schedulers/default.rb +2 -2
  145. data/lib/karafka/pro/recurring_tasks/consumer.rb +9 -2
  146. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +0 -1
  147. data/lib/karafka/pro/recurring_tasks/executor.rb +25 -13
  148. data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb +0 -2
  149. data/lib/karafka/pro/routing/features/consumer_groups/long_running_job.rb +0 -2
  150. data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/proxy.rb +2 -2
  151. data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job.rb +0 -2
  152. data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb +0 -6
  153. data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb +5 -4
  154. data/lib/karafka/pro/routing/features/consumer_groups/patterns/config.rb +0 -6
  155. data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb +2 -1
  156. data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/pattern.rb +0 -1
  157. data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb +2 -1
  158. data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb +2 -1
  159. data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb +8 -8
  160. data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb +5 -5
  161. data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb +33 -13
  162. data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb +52 -28
  163. data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb +2 -1
  164. data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb +6 -5
  165. data/lib/karafka/pro/scheduled_messages/consumer.rb +7 -1
  166. data/lib/karafka/pro/scheduled_messages/schema_validator.rb +6 -1
  167. data/lib/karafka/pro/swarm/liveness_listener.rb +118 -13
  168. data/lib/karafka/pro.rb +0 -6
  169. data/lib/karafka/processing/consumer_groups/coordinator.rb +4 -5
  170. data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb +6 -2
  171. data/lib/karafka/processing/consumer_groups/executor.rb +4 -4
  172. data/lib/karafka/processing/consumer_groups/executors_buffer.rb +2 -4
  173. data/lib/karafka/processing/consumer_groups/expansions_selector.rb +0 -2
  174. data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb +0 -2
  175. data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb +0 -2
  176. data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb +14 -16
  177. data/lib/karafka/processing/consumer_groups/jobs/consume.rb +2 -5
  178. data/lib/karafka/processing/consumer_groups/jobs/eofed.rb +2 -3
  179. data/lib/karafka/processing/consumer_groups/jobs/idle.rb +2 -2
  180. data/lib/karafka/processing/consumer_groups/jobs/revoked.rb +2 -4
  181. data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb +2 -2
  182. data/lib/karafka/processing/consumer_groups/partitioner.rb +3 -6
  183. data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb +5 -2
  184. data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb +2 -4
  185. data/lib/karafka/processing/consumer_groups/strategies/base.rb +3 -5
  186. data/lib/karafka/processing/consumer_groups/strategies/default.rb +43 -13
  187. data/lib/karafka/processing/consumer_groups/strategies/dlq.rb +5 -2
  188. data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb +5 -2
  189. data/lib/karafka/processing/consumer_groups/strategies/mom.rb +2 -3
  190. data/lib/karafka/processing/consumer_groups/strategy_selector.rb +0 -2
  191. data/lib/karafka/processing/coordinator.rb +0 -2
  192. data/lib/karafka/processing/coordinators_buffer.rb +0 -2
  193. data/lib/karafka/processing/partitioner.rb +0 -2
  194. data/lib/karafka/processing/strategy_selector.rb +0 -2
  195. data/lib/karafka/processing/worker.rb +14 -6
  196. data/lib/karafka/processing/workers_pool.rb +21 -4
  197. data/lib/karafka/railtie.rb +0 -1
  198. data/lib/karafka/routing/contracts/routing.rb +2 -2
  199. data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb +6 -5
  200. data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb +2 -2
  201. data/lib/karafka/routing/features/deserializers/topic.rb +0 -1
  202. data/lib/karafka/routing/subscription_group.rb +1 -1
  203. data/lib/karafka/routing/topic.rb +4 -4
  204. data/lib/karafka/routing/topics.rb +21 -1
  205. data/lib/karafka/server.rb +9 -0
  206. data/lib/karafka/setup/config.rb +54 -10
  207. data/lib/karafka/setup/config_proxy.rb +2 -1
  208. data/lib/karafka/setup/contracts/config.rb +14 -0
  209. data/lib/karafka/swarm/node.rb +2 -2
  210. data/lib/karafka/version.rb +1 -2
  211. data/lib/karafka.rb +2 -2
  212. metadata +12 -4
@@ -31,9 +31,7 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Pro consumer-group-specific processing components
35
34
  module ConsumerGroups
36
- # Pro jobs
37
35
  module Jobs
38
36
  # Non-Blocking version of the Eofed job
39
37
  # We use this version for LRJ topics for cases where saturated resources would not allow
@@ -31,7 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Pro consumer-group-specific processing components
35
34
  module ConsumerGroups
36
35
  # Pro jobs
37
36
  module Jobs
@@ -40,8 +39,8 @@ module Karafka
40
39
  class Periodic < Karafka::Processing::Jobs::Base
41
40
  self.action = :tick
42
41
 
43
- # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor] pro executor that is suppose to
44
- # run a given job
42
+ # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor] pro executor that
43
+ # is suppose to run a given job
45
44
  def initialize(executor)
46
45
  @executor = executor
47
46
  super()
@@ -31,9 +31,7 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Pro consumer-group-specific processing components
35
34
  module ConsumerGroups
36
- # Pro jobs
37
35
  module Jobs
38
36
  # Non-Blocking version of the Periodic job
39
37
  # We use this version for LRJ topics for cases where saturated resources would not allow
@@ -30,11 +30,8 @@
30
30
 
31
31
  module Karafka
32
32
  module Pro
33
- # Pro components related to processing part of Karafka
34
33
  module Processing
35
- # Pro consumer-group-specific processing components
36
34
  module ConsumerGroups
37
- # Pro jobs
38
35
  module Jobs
39
36
  # The revoked job type in a non-blocking variant.
40
37
  # This variant works "like" the regular revoked but does not block the queue.
@@ -42,8 +42,8 @@ module Karafka
42
42
  # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor]
43
43
  # @param messages [Karafka::Messages::Messages] messages batch to be consumed
44
44
  # @return [Karafka::Processing::ConsumerGroups::Jobs::Consume] blocking job
45
- # @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::ConsumeNonBlocking] non blocking
46
- # for lrj
45
+ # @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::ConsumeNonBlocking] non
46
+ # blocking for lrj
47
47
  def consume(executor, messages)
48
48
  if executor.topic.long_running_job?
49
49
  Jobs::ConsumeNonBlocking.new(executor, messages)
@@ -54,8 +54,9 @@ module Karafka
54
54
 
55
55
  # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor]
56
56
  # @return [Karafka::Processing::ConsumerGroups::Jobs::Eofed] eofed job for non LRJ
57
- # @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::EofedNonBlocking] eofed job that
58
- # is non-blocking, so when revocation job is scheduled for LRJ it also will not block
57
+ # @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::EofedNonBlocking] eofed job
58
+ # that is non-blocking, so when revocation job is scheduled for LRJ it also will not
59
+ # block
59
60
  def eofed(executor)
60
61
  if executor.topic.long_running_job?
61
62
  Jobs::EofedNonBlocking.new(executor)
@@ -31,16 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
- # Offset Metadata support on the processing side
39
35
  module OffsetMetadata
40
36
  # Extra API methods for offset metadata fetching
41
- # @note Part of this feature API is embedded directly into the strategies because it alters
42
- # how marking methods (`#mark_as_consumed` and `#mark_as_consumed!`) operate. Because
43
- # of that, they had to be embedded into the strategies.
37
+ # @note Part of this feature API is embedded directly into the strategies because it
38
+ # alters how marking methods (`#mark_as_consumed` and `#mark_as_consumed!`) operate.
39
+ # Because of that, they had to be embedded into the strategies.
44
40
  module Consumer
45
41
  # @param cache [Boolean] should we use cached result if present (true by default)
46
42
  # @return [false, Object] false in case we do not own the partition anymore or
@@ -31,19 +31,16 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  # Offset Metadata support on the processing side
39
36
  module OffsetMetadata
40
37
  # This fetcher is responsible for fetching and caching committed offsets metadata
41
38
  # information.
42
39
  #
43
- # By design we fetch all information for a requested topic assignments. Not all topics from
44
- # the same subscription group may need metadata and even if, we can run the few smaller
45
- # queries. This approach prevents us from querying all assigned topics data in one go
46
- # preventing excessive queries.
40
+ # By design we fetch all information for a requested topic assignments. Not all topics
41
+ # from the same subscription group may need metadata and even if, we can run the few
42
+ # smaller queries. This approach prevents us from querying all assigned topics data in one
43
+ # go preventing excessive queries.
47
44
  #
48
45
  # Since the assumption is, that user will not have to reach out for the later metadata
49
46
  # since it is produced in the context of a given consumer assignment, we can cache the
@@ -64,7 +61,8 @@ module Karafka
64
61
  @tpls = {}
65
62
  end
66
63
 
67
- # Registers a client of a given subscription group, so we can use it for queries later on
64
+ # Registers a client of a given subscription group, so we can use it for queries later
65
+ # on
68
66
  # @param client [Karafka::Connection::Client]
69
67
  # @note Since we store the client reference and not the underlying rdkafka consumer
70
68
  # instance, we do not have to deal with the recovery as it is abstracted away
@@ -104,8 +102,8 @@ module Karafka
104
102
 
105
103
  # Clears cache of a given subscription group. It is triggered on assignment changes.
106
104
  #
107
- # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group that
108
- # we want to clear.
105
+ # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
106
+ # that we want to clear.
109
107
  def clear(subscription_group)
110
108
  @mutexes.fetch(subscription_group).synchronize do
111
109
  @tpls[subscription_group].clear
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module OffsetMetadata
39
36
  # Keeps track of rebalances and updates the fetcher
@@ -31,15 +31,13 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module ParallelSegments
39
36
  # Module for filters injected into the processing pipeline of each of the topics used
40
37
  # within the parallel segmented consumer groups
41
38
  module Filters
42
- # Base class for filters for parallel segments that deal with different feature scenarios
39
+ # Base class for filters for parallel segments that deal with different feature
40
+ # scenarios
43
41
  class Base < Pro::Processing::ConsumerGroups::Filters::Base
44
42
  # @param segment_id [Integer] numeric id of the parallel segment group to use with the
45
43
  # partitioner and reducer for segment matching comparison
@@ -82,7 +80,7 @@ module Karafka
82
80
 
83
81
  @reducer.call(message_segment_key)
84
82
  rescue => e
85
- # @see `#partition` method error handling doc
83
+ # See `#partition` method error handling doc
86
84
  Karafka.monitor.instrument(
87
85
  "error.occurred",
88
86
  caller: self,
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  # Processing components namespace for parallel segments feature
39
36
  module ParallelSegments
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module ParallelSegments
39
36
  module Filters
@@ -31,16 +31,13 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  # Pro partitioner that can distribute work based on the virtual partitioner settings
39
36
  class Partitioner < Karafka::Processing::ConsumerGroups::Partitioner
40
37
  # @param topic [String] topic name
41
38
  # @param messages [Array<Karafka::Messages::Message>] karafka messages
42
- # @param coordinator [Karafka::Pro::Processing::ConsumerGroups::Coordinator] processing coordinator that
43
- # will be used with those messages
39
+ # @param coordinator [Karafka::Pro::Processing::ConsumerGroups::Coordinator]
40
+ # processing coordinator that will be used with those messages
44
41
  # @yieldparam [Integer] group id
45
42
  # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
46
43
  def call(topic, messages, coordinator)
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  # Namespace for periodic jobs related processing APIs
39
36
  module PeriodicJob
@@ -64,8 +61,8 @@ module Karafka
64
61
  end
65
62
 
66
63
  # Runs the on-schedule tick periodic operations
67
- # This method is an alias but is part of the naming convention used for other flows, this
68
- # is why we do not reference the `handle_before_schedule_tick` directly
64
+ # This method is an alias but is part of the naming convention used for other flows,
65
+ # this is why we do not reference the `handle_before_schedule_tick` directly
69
66
  def on_before_schedule_tick
70
67
  handle_before_schedule_tick
71
68
  end
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  # All code needed for messages piping in Karafka
39
36
  module Piping
@@ -48,8 +45,8 @@ module Karafka
48
45
  private_constant :EMPTY_HASH
49
46
 
50
47
  # Pipes given message to the provided topic with expected details. Useful for
51
- # pass-through operations where deserialization is not needed. Upon usage it will include
52
- # all the source headers + meta headers about the source of message.
48
+ # pass-through operations where deserialization is not needed. Upon usage it will
49
+ # include all the source headers + meta headers about the source of message.
53
50
  #
54
51
  # @param topic [String, Symbol] where we want to send the message
55
52
  # @param message [Karafka::Messages::Message] source message to pipe
@@ -31,17 +31,14 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # DLQ enabled
42
- # Filtering enabled
43
- # Long-Running Job enabled
44
- # Manual offset management enabled
37
+ # - ActiveJob enabled
38
+ # - DLQ enabled
39
+ # - Filtering enabled
40
+ # - Long-Running Job enabled
41
+ # - Manual offset management enabled
45
42
  module DlqFtrLrjMom
46
43
  include Strategies::Aj::FtrMom
47
44
  include Strategies::Aj::DlqMom
@@ -67,7 +64,7 @@ module Karafka
67
64
 
68
65
  # :seek and :pause are fully handled by handle_post_filtering
69
66
  # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
70
- return unless coordinator.filter.action == :skip
67
+ return unless coordinator.filter.skip?
71
68
  elsif !revoked?
72
69
  # no need to check for manual seek because AJ consumer is internal and
73
70
  # fully controlled by us
@@ -77,6 +74,10 @@ module Karafka
77
74
  resume
78
75
  else
79
76
  apply_dlq_flow do
77
+ # LRJ revocation runs concurrently with the job. If the partition was
78
+ # revoked we must not dispatch: the new owner will reprocess and dispatch
79
+ return resume if revoked?
80
+
80
81
  skippable_message, = find_skippable_message
81
82
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
82
83
  mark_dispatched_to_dlq(skippable_message)
@@ -31,18 +31,15 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # DLQ enabled
42
- # Filtering enabled
43
- # Long-Running Job enabled
44
- # Manual offset management enabled
45
- # Virtual Partitions enabled
37
+ # - ActiveJob enabled
38
+ # - DLQ enabled
39
+ # - Filtering enabled
40
+ # - Long-Running Job enabled
41
+ # - Manual offset management enabled
42
+ # - Virtual Partitions enabled
46
43
  module DlqFtrLrjMomVp
47
44
  include Strategies::Aj::FtrMom
48
45
  include Strategies::Aj::DlqMomVp
@@ -73,7 +70,7 @@ module Karafka
73
70
 
74
71
  # :seek and :pause are fully handled by handle_post_filtering
75
72
  # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
76
- return unless coordinator.filter.action == :skip
73
+ return unless coordinator.filter.skip?
77
74
  elsif !revoked?
78
75
  # no need to check for manual seek because AJ consumer is internal and
79
76
  # fully controlled by us
@@ -83,6 +80,10 @@ module Karafka
83
80
  resume
84
81
  else
85
82
  apply_dlq_flow do
83
+ # LRJ revocation runs concurrently with the job. If the partition was
84
+ # revoked we must not dispatch: the new owner will reprocess and dispatch
85
+ return resume if revoked?
86
+
86
87
  skippable_message, = find_skippable_message
87
88
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
88
89
  mark_dispatched_to_dlq(skippable_message)
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
@@ -31,16 +31,13 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # DLQ enabled
42
- # Long-Running Job enabled
43
- # Manual offset management enabled
37
+ # - ActiveJob enabled
38
+ # - DLQ enabled
39
+ # - Long-Running Job enabled
40
+ # - Manual offset management enabled
44
41
  #
45
42
  # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
46
43
  # offset management works in this case.
@@ -70,6 +67,10 @@ module Karafka
70
67
  resume
71
68
  else
72
69
  apply_dlq_flow do
70
+ # LRJ revocation runs concurrently with the job. If the partition was
71
+ # revoked we must not dispatch: the new owner will reprocess and dispatch
72
+ return resume if revoked?
73
+
73
74
  skippable_message, = find_skippable_message
74
75
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
75
76
  mark_dispatched_to_dlq(skippable_message)
@@ -31,17 +31,14 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # DLQ enabled
42
- # Long-Running Job enabled
43
- # Manual offset management enabled
44
- # Virtual Partitions enabled
37
+ # - ActiveJob enabled
38
+ # - DLQ enabled
39
+ # - Long-Running Job enabled
40
+ # - Manual offset management enabled
41
+ # - Virtual Partitions enabled
45
42
  #
46
43
  # This case is a bit of special. Please see the `Aj::DlqMom` for explanation on how the
47
44
  # offset management works in this case.
@@ -74,6 +71,10 @@ module Karafka
74
71
  resume
75
72
  else
76
73
  apply_dlq_flow do
74
+ # LRJ revocation runs concurrently with the job. If the partition was
75
+ # revoked we must not dispatch: the new owner will reprocess and dispatch
76
+ return resume if revoked?
77
+
77
78
  skippable_message, = find_skippable_message
78
79
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
79
80
  mark_dispatched_to_dlq(skippable_message)
@@ -31,18 +31,15 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # DLQ enabled
42
- # Manual offset management enabled
37
+ # - ActiveJob enabled
38
+ # - DLQ enabled
39
+ # - Manual offset management enabled
43
40
  #
44
- # AJ has manual offset management on by default and the offset management is delegated to
45
- # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
41
+ # AJ has manual offset management on by default and the offset management is delegated
42
+ # to the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
46
43
  # consumed when we skip given job upon errors. In all the other scenarios marking as
47
44
  # consumed needs to happen in the AJ consumer on a per job basis.
48
45
  module DlqMom
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # Manual offset management enabled
42
- # Virtual Partitions enabled
37
+ # - ActiveJob enabled
38
+ # - Manual offset management enabled
39
+ # - Virtual Partitions enabled
43
40
  module DlqMomVp
44
41
  include Strategies::Default
45
42
  include Strategies::Dlq::Vp
@@ -31,16 +31,13 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # Filtering enabled
42
- # Long-Running Job enabled
43
- # Manual offset management enabled
37
+ # - ActiveJob enabled
38
+ # - Filtering enabled
39
+ # - Long-Running Job enabled
40
+ # - Manual offset management enabled
44
41
  module FtrLrjMom
45
42
  include Strategies::Lrj::FtrMom
46
43
 
@@ -31,17 +31,14 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # Filtering enabled
42
- # Long-Running Job enabled
43
- # Manual offset management enabled
44
- # Virtual Partitions enabled
37
+ # - ActiveJob enabled
38
+ # - Filtering enabled
39
+ # - Long-Running Job enabled
40
+ # - Manual offset management enabled
41
+ # - Virtual Partitions enabled
45
42
  module FtrLrjMomVp
46
43
  include Strategies::Vp::Default
47
44
  include Strategies::Lrj::FtrMom
@@ -69,7 +66,7 @@ module Karafka
69
66
 
70
67
  # :seek and :pause are fully handled by handle_post_filtering
71
68
  # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
72
- return unless coordinator.filter.action == :skip
69
+ return unless coordinator.filter.skip?
73
70
  elsif !revoked?
74
71
  # no need to check for manual seek because AJ consumer is internal and
75
72
  # fully controlled by us
@@ -31,16 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
- # Namespace for ActiveJob related strategies
40
36
  module Aj
41
- # ActiveJob enabled
42
- # Filtering enabled
43
- # Manual Offset management enabled
37
+ # - ActiveJob enabled
38
+ # - Filtering enabled
39
+ # - Manual Offset management enabled
44
40
  module FtrMom
45
41
  # Same as standard Mom::Ftr
46
42
  include Strategies::Mom::Ftr
@@ -31,17 +31,13 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
- # Namespace for ActiveJob related strategies
40
36
  module Aj
41
- # ActiveJob enabled
42
- # Filtering enabled
43
- # Manual Offset management enabled
44
- # Virtual partitions enabled
37
+ # - ActiveJob enabled
38
+ # - Filtering enabled
39
+ # - Manual Offset management enabled
40
+ # - Virtual partitions enabled
45
41
  module FtrMomVp
46
42
  include Strategies::Aj::FtrMom
47
43
  include Strategies::Aj::MomVp
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # Long-Running Job enabled
42
- # Manual offset management enabled
37
+ # - ActiveJob enabled
38
+ # - Long-Running Job enabled
39
+ # - Manual offset management enabled
43
40
  module LrjMom
44
41
  # Same behaviour as Lrj::Mom
45
42
  include Strategies::Lrj::Mom
@@ -31,16 +31,13 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # Long-Running Job enabled
42
- # Manual offset management enabled
43
- # Virtual Partitions enabled
37
+ # - ActiveJob enabled
38
+ # - Long-Running Job enabled
39
+ # - Manual offset management enabled
40
+ # - Virtual Partitions enabled
44
41
  module LrjMomVp
45
42
  include Strategies::Default
46
43
  include Strategies::Vp::Default
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  # Namespace for ActiveJob related strategies
40
37
  module Aj
41
- # ActiveJob enabled
42
- # Manual Offset management enabled
38
+ # - ActiveJob enabled
39
+ # - Manual Offset management enabled
43
40
  module Mom
44
41
  # Standard ActiveJob strategy is the same one we use for Mom
45
42
  include Strategies::Mom::Default