karafka 2.6.0.beta1 → 2.6.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +64 -27
  3. data/certs/expired.txt +2 -0
  4. data/config/locales/errors.yml +5 -0
  5. data/karafka.gemspec +1 -1
  6. data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -1
  7. data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
  8. data/lib/karafka/active_job/current_attributes.rb +4 -2
  9. data/lib/karafka/active_job/dispatcher.rb +1 -2
  10. data/lib/karafka/admin/consumer_groups.rb +8 -2
  11. data/lib/karafka/admin/replication.rb +6 -3
  12. data/lib/karafka/admin/topics.rb +32 -12
  13. data/lib/karafka/admin.rb +155 -59
  14. data/lib/karafka/app.rb +1 -1
  15. data/lib/karafka/base_consumer.rb +33 -5
  16. data/lib/karafka/cli/console.rb +0 -1
  17. data/lib/karafka/cli/help.rb +0 -1
  18. data/lib/karafka/cli/info.rb +0 -1
  19. data/lib/karafka/cli/install.rb +0 -1
  20. data/lib/karafka/cli/server.rb +0 -1
  21. data/lib/karafka/cli/swarm.rb +0 -1
  22. data/lib/karafka/cli.rb +2 -2
  23. data/lib/karafka/connection/client.rb +44 -7
  24. data/lib/karafka/connection/listener.rb +3 -3
  25. data/lib/karafka/connection/manager.rb +0 -1
  26. data/lib/karafka/connection/pauses_manager.rb +17 -0
  27. data/lib/karafka/connection/proxy.rb +45 -0
  28. data/lib/karafka/connection/status.rb +0 -1
  29. data/lib/karafka/instrumentation/assignments_tracker.rb +34 -0
  30. data/lib/karafka/instrumentation/callbacks/consumer_groups/decorator.rb +19 -0
  31. data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb +0 -1
  32. data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb +3 -2
  33. data/lib/karafka/instrumentation/critical_errors_listener.rb +85 -0
  34. data/lib/karafka/instrumentation/logger_listener.rb +2 -0
  35. data/lib/karafka/instrumentation/monitor.rb +2 -3
  36. data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +0 -1
  37. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +0 -1
  38. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +4 -3
  39. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +0 -2
  40. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +181 -18
  41. data/lib/karafka/pro/admin/recovery/errors.rb +0 -1
  42. data/lib/karafka/pro/admin/recovery.rb +6 -4
  43. data/lib/karafka/pro/base_consumer.rb +5 -5
  44. data/lib/karafka/pro/cleaner/messages/metadata.rb +0 -1
  45. data/lib/karafka/pro/cli/parallel_segments/base.rb +1 -2
  46. data/lib/karafka/pro/connection/manager.rb +7 -4
  47. data/lib/karafka/pro/encryption/messages/middleware.rb +5 -0
  48. data/lib/karafka/pro/encryption/messages/parser.rb +4 -0
  49. data/lib/karafka/pro/instrumentation/callbacks/consumer_groups/decorator.rb +63 -0
  50. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/compensator.rb +120 -0
  51. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/fetcher.rb +88 -0
  52. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/refresher.rb +185 -0
  53. data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/registry.rb +102 -0
  54. data/lib/karafka/pro/instrumentation/performance_tracker.rb +40 -6
  55. data/lib/karafka/pro/iterator/tpl_builder.rb +27 -7
  56. data/lib/karafka/pro/iterator.rb +22 -7
  57. data/lib/karafka/pro/loader.rb +12 -0
  58. data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/consumer.rb +0 -3
  59. data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/tracker.rb +2 -4
  60. data/lib/karafka/pro/processing/consumer_groups/collapser.rb +6 -9
  61. data/lib/karafka/pro/processing/consumer_groups/coordinator.rb +1 -4
  62. data/lib/karafka/pro/processing/consumer_groups/coordinators/errors_tracker.rb +0 -3
  63. data/lib/karafka/pro/processing/consumer_groups/coordinators/filters_applier.rb +21 -10
  64. data/lib/karafka/pro/processing/consumer_groups/coordinators/virtual_offset_manager.rb +4 -4
  65. data/lib/karafka/pro/processing/consumer_groups/expansions_selector.rb +0 -3
  66. data/lib/karafka/pro/processing/consumer_groups/filters/actions.rb +84 -0
  67. data/lib/karafka/pro/processing/consumer_groups/filters/base.rb +5 -6
  68. data/lib/karafka/pro/processing/consumer_groups/filters/delayer.rb +2 -5
  69. data/lib/karafka/pro/processing/consumer_groups/filters/expirer.rb +1 -4
  70. data/lib/karafka/pro/processing/consumer_groups/filters/inline_insights_delayer.rb +1 -4
  71. data/lib/karafka/pro/processing/consumer_groups/filters/throttler.rb +2 -5
  72. data/lib/karafka/pro/processing/consumer_groups/filters/virtual_limiter.rb +0 -3
  73. data/lib/karafka/pro/processing/consumer_groups/jobs/consume_non_blocking.rb +2 -3
  74. data/lib/karafka/pro/processing/consumer_groups/jobs/eofed_non_blocking.rb +0 -2
  75. data/lib/karafka/pro/processing/consumer_groups/jobs/periodic.rb +2 -3
  76. data/lib/karafka/pro/processing/consumer_groups/jobs/periodic_non_blocking.rb +0 -2
  77. data/lib/karafka/pro/processing/consumer_groups/jobs/revoked_non_blocking.rb +0 -3
  78. data/lib/karafka/pro/processing/consumer_groups/jobs_builder.rb +5 -4
  79. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/consumer.rb +3 -7
  80. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/fetcher.rb +8 -10
  81. data/lib/karafka/pro/processing/consumer_groups/offset_metadata/listener.rb +0 -3
  82. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/base.rb +3 -5
  83. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/default.rb +0 -3
  84. data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/mom.rb +0 -3
  85. data/lib/karafka/pro/processing/consumer_groups/partitioner.rb +2 -5
  86. data/lib/karafka/pro/processing/consumer_groups/periodic_job/consumer.rb +2 -5
  87. data/lib/karafka/pro/processing/consumer_groups/piping/consumer.rb +2 -5
  88. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom.rb +10 -9
  89. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom_vp.rb +11 -10
  90. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom.rb +0 -3
  91. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom_vp.rb +0 -3
  92. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom.rb +8 -7
  93. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom_vp.rb +9 -8
  94. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom.rb +5 -8
  95. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom_vp.rb +3 -6
  96. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom.rb +4 -7
  97. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom_vp.rb +6 -9
  98. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom.rb +3 -7
  99. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom_vp.rb +4 -8
  100. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom.rb +3 -6
  101. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom_vp.rb +4 -7
  102. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom.rb +2 -5
  103. data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom_vp.rb +3 -6
  104. data/lib/karafka/pro/processing/consumer_groups/strategies/base.rb +0 -3
  105. data/lib/karafka/pro/processing/consumer_groups/strategies/default.rb +56 -33
  106. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/default.rb +27 -14
  107. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr.rb +0 -3
  108. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj.rb +4 -7
  109. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom.rb +5 -8
  110. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom_vp.rb +5 -8
  111. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_vp.rb +3 -7
  112. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom.rb +0 -3
  113. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom_vp.rb +0 -3
  114. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_vp.rb +0 -3
  115. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj.rb +2 -5
  116. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom.rb +0 -3
  117. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom_vp.rb +0 -3
  118. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_vp.rb +3 -6
  119. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom.rb +0 -3
  120. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom_vp.rb +3 -6
  121. data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb +65 -8
  122. data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb +0 -4
  123. data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/vp.rb +2 -5
  124. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb +0 -3
  125. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb +3 -6
  126. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb +4 -7
  127. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb +4 -8
  128. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb +3 -6
  129. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb +2 -5
  130. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb +3 -7
  131. data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/vp.rb +2 -5
  132. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/default.rb +0 -3
  133. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr.rb +0 -3
  134. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr_vp.rb +0 -3
  135. data/lib/karafka/pro/processing/consumer_groups/strategies/mom/vp.rb +0 -3
  136. data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb +11 -6
  137. data/lib/karafka/pro/processing/consumer_groups/strategy_selector.rb +0 -3
  138. data/lib/karafka/pro/processing/consumer_groups/subscription_groups_coordinator.rb +0 -3
  139. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/balanced.rb +0 -3
  140. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/base.rb +2 -4
  141. data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/consistent.rb +0 -3
  142. data/lib/karafka/pro/processing/jobs_queue.rb +4 -1
  143. data/lib/karafka/pro/processing/schedulers/base.rb +4 -2
  144. data/lib/karafka/pro/processing/schedulers/default.rb +2 -2
  145. data/lib/karafka/pro/recurring_tasks/consumer.rb +9 -2
  146. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +0 -1
  147. data/lib/karafka/pro/recurring_tasks/executor.rb +25 -13
  148. data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb +0 -2
  149. data/lib/karafka/pro/routing/features/consumer_groups/long_running_job.rb +0 -2
  150. data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/proxy.rb +2 -2
  151. data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job.rb +0 -2
  152. data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb +0 -6
  153. data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb +5 -4
  154. data/lib/karafka/pro/routing/features/consumer_groups/patterns/config.rb +0 -6
  155. data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb +2 -1
  156. data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/pattern.rb +0 -1
  157. data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb +2 -1
  158. data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb +2 -1
  159. data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb +8 -8
  160. data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb +5 -5
  161. data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb +33 -13
  162. data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb +52 -28
  163. data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb +2 -1
  164. data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb +6 -5
  165. data/lib/karafka/pro/scheduled_messages/consumer.rb +7 -1
  166. data/lib/karafka/pro/scheduled_messages/schema_validator.rb +6 -1
  167. data/lib/karafka/pro/swarm/liveness_listener.rb +118 -13
  168. data/lib/karafka/pro.rb +0 -6
  169. data/lib/karafka/processing/consumer_groups/coordinator.rb +4 -5
  170. data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb +6 -2
  171. data/lib/karafka/processing/consumer_groups/executor.rb +4 -4
  172. data/lib/karafka/processing/consumer_groups/executors_buffer.rb +2 -4
  173. data/lib/karafka/processing/consumer_groups/expansions_selector.rb +0 -2
  174. data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb +0 -2
  175. data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb +0 -2
  176. data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb +14 -16
  177. data/lib/karafka/processing/consumer_groups/jobs/consume.rb +2 -5
  178. data/lib/karafka/processing/consumer_groups/jobs/eofed.rb +2 -3
  179. data/lib/karafka/processing/consumer_groups/jobs/idle.rb +2 -2
  180. data/lib/karafka/processing/consumer_groups/jobs/revoked.rb +2 -4
  181. data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb +2 -2
  182. data/lib/karafka/processing/consumer_groups/partitioner.rb +3 -6
  183. data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb +5 -2
  184. data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb +2 -4
  185. data/lib/karafka/processing/consumer_groups/strategies/base.rb +3 -5
  186. data/lib/karafka/processing/consumer_groups/strategies/default.rb +43 -13
  187. data/lib/karafka/processing/consumer_groups/strategies/dlq.rb +5 -2
  188. data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb +5 -2
  189. data/lib/karafka/processing/consumer_groups/strategies/mom.rb +2 -3
  190. data/lib/karafka/processing/consumer_groups/strategy_selector.rb +0 -2
  191. data/lib/karafka/processing/coordinator.rb +0 -2
  192. data/lib/karafka/processing/coordinators_buffer.rb +0 -2
  193. data/lib/karafka/processing/partitioner.rb +0 -2
  194. data/lib/karafka/processing/strategy_selector.rb +0 -2
  195. data/lib/karafka/processing/worker.rb +14 -6
  196. data/lib/karafka/processing/workers_pool.rb +21 -4
  197. data/lib/karafka/railtie.rb +0 -1
  198. data/lib/karafka/routing/contracts/routing.rb +2 -2
  199. data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb +6 -5
  200. data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb +2 -2
  201. data/lib/karafka/routing/features/deserializers/topic.rb +0 -1
  202. data/lib/karafka/routing/subscription_group.rb +1 -1
  203. data/lib/karafka/routing/topic.rb +4 -4
  204. data/lib/karafka/routing/topics.rb +21 -1
  205. data/lib/karafka/server.rb +9 -0
  206. data/lib/karafka/setup/config.rb +54 -10
  207. data/lib/karafka/setup/config_proxy.rb +2 -1
  208. data/lib/karafka/setup/contracts/config.rb +14 -0
  209. data/lib/karafka/swarm/node.rb +2 -2
  210. data/lib/karafka/version.rb +1 -2
  211. data/lib/karafka.rb +2 -2
  212. metadata +12 -4
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Aj
40
- # ActiveJob enabled
41
- # Manual offset management enabled
42
- # Virtual Partitions enabled
37
+ # - ActiveJob enabled
38
+ # - Manual offset management enabled
39
+ # - Virtual Partitions enabled
43
40
  module MomVp
44
41
  include Strategies::Default
45
42
  include Strategies::Vp::Default
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  # Pro processing strategies namespace
39
36
  module Strategies
@@ -31,15 +31,13 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
- # No features enabled.
40
- # No manual offset management
41
- # No long running jobs
42
- # No virtual partitions
36
+ # No features enabled:
37
+ # - No manual offset management
38
+ # - No long running jobs
39
+ # - No virtual partitions
40
+ #
43
41
  # Nothing. Just standard, automatic flow
44
42
  module Default
45
43
  include Base
@@ -70,10 +68,10 @@ module Karafka
70
68
  # @return [Boolean] true if we were able to mark the offset, false otherwise.
71
69
  # False indicates that we were not able and that we have lost the partition.
72
70
  #
73
- # @note We keep track of this offset in case we would mark as consumed and got error when
74
- # processing another message. In case like this we do not pause on the message we've
75
- # already processed but rather at the next one. This applies to both sync and async
76
- # versions of this method.
71
+ # @note We keep track of this offset in case we would mark as consumed and got error
72
+ # when processing another message. In case like this we do not pause on the message
73
+ # we've already processed but rather at the next one. This applies to both sync and
74
+ # async versions of this method.
77
75
  def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
78
76
  # If we are inside a transaction than we can just mark as consumed within it
79
77
  if @_in_transaction
@@ -88,6 +86,8 @@ module Karafka
88
86
  # We ignore second marking because it changes nothing and in case of people using
89
87
  # metadata storage but with automatic offset marking, this would cause metadata to be
90
88
  # erased by automatic marking
89
+ # Only this exact re-mark is skipped - marking genuinely older offsets is intentionally
90
+ # allowed and rewinds the seek offset for reprocessing (see #2432)
91
91
  return true if (seek_offset - 1) == message.offset
92
92
  return false if revoked?
93
93
 
@@ -101,7 +101,14 @@ module Karafka
101
101
  client.mark_as_consumed(message, offset_metadata)
102
102
  end
103
103
 
104
- return revoked? unless stored
104
+ unless stored
105
+ # Evaluate ownership for its coordinator-revoking side effect - all marking
106
+ # failure causes are ownership related. The failure itself always reports
107
+ # false: the offset was not stored, regardless of the ownership state
108
+ revoked?
109
+
110
+ return false
111
+ end
105
112
 
106
113
  self.seek_offset = message.offset + 1
107
114
  end
@@ -130,6 +137,8 @@ module Karafka
130
137
  # We ignore second marking because it changes nothing and in case of people using
131
138
  # metadata storage but with automatic offset marking, this would cause metadata to be
132
139
  # erased by automatic marking
140
+ # Only this exact re-mark is skipped - marking genuinely older offsets is intentionally
141
+ # allowed and rewinds the seek offset for reprocessing (see #2432)
133
142
  return true if (seek_offset - 1) == message.offset
134
143
  return false if revoked?
135
144
 
@@ -143,7 +152,14 @@ module Karafka
143
152
  client.mark_as_consumed!(message, offset_metadata)
144
153
  end
145
154
 
146
- return revoked? unless stored
155
+ unless stored
156
+ # Evaluate ownership for its coordinator-revoking side effect - all marking
157
+ # failure causes are ownership related. The failure itself always reports
158
+ # false: the offset was not stored, regardless of the ownership state
159
+ revoked?
160
+
161
+ return false
162
+ end
147
163
 
148
164
  self.seek_offset = message.offset + 1
149
165
  end
@@ -162,8 +178,8 @@ module Karafka
162
178
  #
163
179
  # @param active_producer [WaterDrop::Producer] alternative producer instance we may want
164
180
  # to use. It is useful when we have connection pool or any other selective engine for
165
- # managing multiple producers. If not provided, default producer taken from `#producer`
166
- # will be used.
181
+ # managing multiple producers. If not provided, default producer taken from
182
+ # `#producer` will be used.
167
183
  #
168
184
  # @yield code that we want to run in a transaction
169
185
  #
@@ -212,23 +228,23 @@ module Karafka
212
228
  # This offset is already stored in transaction but we set it here anyhow because we
213
229
  # want to make sure our internal in-memory state is aligned with the transaction
214
230
  #
215
- # @note We never need to use the blocking `#mark_as_consumed!` here because the
216
- # offset anyhow was already stored during the transaction
231
+ # We never need to use the blocking `#mark_as_consumed!` here because the offset
232
+ # anyhow was already stored during the transaction
217
233
  #
218
- # @note Since the offset could have been already stored in Kafka (could have because
219
- # you can have transactions without marking), we use the `@_in_transaction_marked`
220
- # state to decide if we need to dispatch the offset via client at all
221
- # (if post transaction, then we do not have to)
234
+ # Since the offset could have been already stored in Kafka (could have because you
235
+ # can have transactions without marking), we use the `@_in_transaction_marked` state
236
+ # to decide if we need to dispatch the offset via client at all (if post
237
+ # transaction, then we do not have to)
222
238
  #
223
- # @note In theory we could only keep reference to the most recent marking and reject
224
- # others. We however do not do it for two reasons:
225
- # - User may have non standard flow relying on some alternative order and we want
226
- # to mimic this
227
- # - Complex strategies like VPs can use this in VPs to mark in parallel without
228
- # having to redefine the transactional flow completely
239
+ # In theory we could only keep reference to the most recent marking and reject
240
+ # others. We however do not do it for two reasons:
241
+ # - User may have non standard flow relying on some alternative order and we want
242
+ # to mimic this
243
+ # - Complex strategies like VPs can use this in VPs to mark in parallel without
244
+ # having to redefine the transactional flow completely
229
245
  #
230
- # @note This should be applied only if transaction did not error and if it was not
231
- # aborted.
246
+ # This should be applied only if transaction did not error and if it was not
247
+ # aborted.
232
248
  if transaction_completed
233
249
  @_transaction_marked.each do |marking|
234
250
  marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
@@ -301,7 +317,8 @@ module Karafka
301
317
  # Marks the current state only in memory as the offset marking has already happened
302
318
  # using the producer transaction
303
319
  # @param message [Messages::Message] last successfully processed message.
304
- # @return [Boolean] true if all good, false if we lost assignment and no point in marking
320
+ # @return [Boolean] true if all good, false if we lost assignment and no point in
321
+ # marking
305
322
  def mark_in_memory(message)
306
323
  # seek offset can be nil only in case `#seek` was invoked with offset reset request
307
324
  # In case like this we ignore marking
@@ -310,6 +327,8 @@ module Karafka
310
327
  # We ignore second marking because it changes nothing and in case of people using
311
328
  # metadata storage but with automatic offset marking, this would cause metadata to be
312
329
  # erased by automatic marking
330
+ # Only this exact re-mark is skipped - marking genuinely older offsets is intentionally
331
+ # allowed and rewinds the seek offset for reprocessing (see #2432)
313
332
  return true if (seek_offset - 1) == message.offset
314
333
  return false if revoked?
315
334
 
@@ -328,8 +347,8 @@ module Karafka
328
347
  nil
329
348
  end
330
349
 
331
- # Increment number of attempts per one "full" job. For all VP on a single topic partition
332
- # this also should run once.
350
+ # Increment number of attempts per one "full" job. For all VP on a single topic
351
+ # partition this also should run once.
333
352
  def handle_before_consume
334
353
  coordinator.on_started do
335
354
  coordinator.pause_tracker.increment
@@ -350,7 +369,11 @@ module Karafka
350
369
 
351
370
  # Mark job as successful
352
371
  coordinator.success!(self)
353
- rescue => e
372
+ # Failure recording must be class-agnostic: an unrecorded non-StandardError would
373
+ # leave the consumption result in its default (successful) state and the
374
+ # after-consume flow would mark the failed batch as consumed instead of engaging
375
+ # the retry
376
+ rescue Exception => e
354
377
  # If failed, mark as failed
355
378
  coordinator.failure!(self, e)
356
379
 
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  # Namespace for all the strategies starting with DLQ
@@ -49,8 +46,8 @@ module Karafka
49
46
 
50
47
  # Override of the standard `#mark_as_consumed` in order to handle the pause tracker
51
48
  # reset in case DLQ is marked as fully independent. When DLQ is marked independent,
52
- # any offset marking causes the pause count tracker to reset. This is useful when
53
- # the error is not due to the collective batch operations state but due to intermediate
49
+ # any offset marking causes the pause count tracker to reset. This is useful when the
50
+ # error is not due to the collective batch operations state but due to intermediate
54
51
  # "crawling" errors that move with it
55
52
  #
56
53
  # @see `Strategies::Default#mark_as_consumed` for more details
@@ -68,8 +65,8 @@ module Karafka
68
65
  @_current_offset_metadata = nil
69
66
  end
70
67
 
71
- # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
72
- # DLQ was configured with the `independent` flag.
68
+ # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in
69
+ # case DLQ was configured with the `independent` flag.
73
70
  #
74
71
  # @see `Strategies::Default#mark_as_consumed!` for more details
75
72
  # @param message [Messages::Message]
@@ -147,8 +144,8 @@ module Karafka
147
144
  )
148
145
  end
149
146
 
150
- # Dispatches the message to the DLQ (when needed and when applicable based on settings)
151
- # and marks this message as consumed for non MOM flows.
147
+ # Dispatches the message to the DLQ (when needed and when applicable based on
148
+ # settings) and marks this message as consumed for non MOM flows.
152
149
  #
153
150
  # If producer is transactional and config allows, uses transaction to do that
154
151
  def dispatch_if_needed_and_mark_as_consumed
@@ -157,11 +154,7 @@ module Karafka
157
154
  dispatch = lambda do
158
155
  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
159
156
 
160
- if mark_after_dispatch?
161
- mark_dispatched_to_dlq(skippable_message)
162
- else
163
- self.seek_offset = skippable_message.offset + 1
164
- end
157
+ mark_dispatched_to_dlq(skippable_message) if mark_after_dispatch?
165
158
  end
166
159
 
167
160
  if dispatch_in_a_transaction?
@@ -169,6 +162,12 @@ module Karafka
169
162
  else
170
163
  dispatch.call
171
164
  end
165
+
166
+ # When not marking after dispatch, the seek offset is advanced manually. This must
167
+ # happen only after the dispatch (and its transaction) succeeded - advancing it
168
+ # inside the transaction would let the mutation survive an abort, skipping the
169
+ # broken message which would then be neither in the DLQ nor reprocessed
170
+ self.seek_offset = skippable_message.offset + 1 unless mark_after_dispatch?
172
171
  end
173
172
 
174
173
  # @param skippable_message [Array<Karafka::Messages::Message>]
@@ -235,6 +234,20 @@ module Karafka
235
234
  # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
236
235
  # In case of `:retry` always `#retry_after_pause` is applied
237
236
  def apply_dlq_flow
237
+ # Process-critical errors are never dispatched or skipped regardless of the
238
+ # strategy outcome: the retry pause protects the partition during the critical
239
+ # shutdown and the failed batch is redelivered after the restart.
240
+ # We consult `errors_tracker.last` (not the per-consumer consumption cause used
241
+ # by the OSS strategies, which have no tracker) because it is exactly what the
242
+ # DLQ strategy callable below receives - this guard judges the same evidence as
243
+ # the strategy it overrides. The tracker is cleared at attempt zero, so `last`
244
+ # is always the most recent failure of the current failure streak
245
+ if critical_error?(errors_tracker.last)
246
+ retry_after_pause
247
+
248
+ return
249
+ end
250
+
238
251
  flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
239
252
 
240
253
  case flow
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
40
- # Dead-Letter Queue enabled
41
- # Filtering enabled
42
- # Long-Running Job enabled
37
+ # - Dead-Letter Queue enabled
38
+ # - Filtering enabled
39
+ # - Long-Running Job enabled
43
40
  module FtrLrj
44
41
  include Strategies::Dlq::Lrj
45
42
  include Strategies::Lrj::Ftr
@@ -71,7 +68,7 @@ module Karafka
71
68
 
72
69
  # :seek and :pause are fully handled by handle_post_filtering
73
70
  # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
74
- return unless coordinator.filter.action == :skip
71
+ return unless coordinator.filter.skip?
75
72
  elsif !revoked? && !coordinator.manual_seek?
76
73
  seek(seek_offset, false, reset_offset: false)
77
74
  end
@@ -31,16 +31,13 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
40
- # DLQ enabled
41
- # Ftr enabled
42
- # LRJ enabled
43
- # MoM enabled
37
+ # - DLQ enabled
38
+ # - Ftr enabled
39
+ # - LRJ enabled
40
+ # - MoM enabled
44
41
  module FtrLrjMom
45
42
  include Strategies::Ftr::Default
46
43
  include Strategies::Dlq::LrjMom
@@ -66,7 +63,7 @@ module Karafka
66
63
 
67
64
  # :seek and :pause are fully handled by handle_post_filtering
68
65
  # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
69
- return unless coordinator.filter.action == :skip
66
+ return unless coordinator.filter.skip?
70
67
  elsif !revoked? && !coordinator.manual_seek?
71
68
  seek(last_group_message.offset + 1, false, reset_offset: false)
72
69
  end
@@ -31,17 +31,14 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
40
- # DLQ enabled
41
- # Ftr enabled
42
- # LRJ enabled
43
- # MoM enabled
44
- # VP enabled
37
+ # - DLQ enabled
38
+ # - Ftr enabled
39
+ # - LRJ enabled
40
+ # - MoM enabled
41
+ # - VP enabled
45
42
  module FtrLrjMomVp
46
43
  include Strategies::Vp::Default
47
44
  include Strategies::Dlq::Vp
@@ -31,16 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
40
- # Dead-Letter Queue enabled
41
- # Filtering enabled
42
- # Long-Running Job enabled
43
- # Virtual Partitions
37
+ # - Dead-Letter Queue enabled
38
+ # - Filtering enabled
39
+ # - Long-Running Job enabled
44
40
  module FtrLrjVp
45
41
  include Strategies::Vp::Default
46
42
  include Strategies::Dlq::Vp
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
@@ -31,14 +31,11 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
40
- # DLQ enabled
41
- # Long-Running Job enabled
37
+ # - DLQ enabled
38
+ # - Long-Running Job enabled
42
39
  module Lrj
43
40
  # Order here matters, lrj needs to be second
44
41
  include Strategies::Dlq::Default
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
40
- # Dead-Letter Queue enabled
41
- # Long-Running Job enabled
42
- # Virtual Partitions enabled
37
+ # - Dead-Letter Queue enabled
38
+ # - Long-Running Job enabled
39
+ # - Virtual Partitions enabled
43
40
  module LrjVp
44
41
  # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
45
42
  include Strategies::Vp::Default
@@ -31,9 +31,6 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
40
- # Dead-Letter Queue enabled
41
- # MoM enabled
42
- # Virtual Partitions enabled
37
+ # - Dead-Letter Queue enabled
38
+ # - MoM enabled
39
+ # - Virtual Partitions enabled
43
40
  module MomVp
44
41
  include Strategies::Dlq::Vp
45
42
  include Strategies::Dlq::Mom
@@ -31,18 +31,15 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  module Dlq
40
- # Dead Letter Queue enabled
41
- # Virtual Partitions enabled
37
+ # - Dead Letter Queue enabled
38
+ # - Virtual Partitions enabled
42
39
  #
43
- # In general because we collapse processing in virtual partitions to one on errors, there
44
- # is no special action that needs to be taken because we warranty that even with VPs
45
- # on errors a retry collapses into a single state and from this single state we can
40
+ # In general because we collapse processing in virtual partitions to one on errors,
41
+ # there is no special action that needs to be taken because we warranty that even with
42
+ # VPson errors a retry collapses into a single state and from this single state we can
46
43
  # mark as consumed the message that we are moving to the DLQ.
47
44
  module Vp
48
45
  # Features for this strategy
@@ -53,6 +50,66 @@ module Karafka
53
50
 
54
51
  include Strategies::Dlq::Default
55
52
  include Strategies::Vp::Default
53
+
54
+ # Runs the DLQ strategy and based on it it performs certain operations
55
+ #
56
+ # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
57
+ # In case of `:retry` always `#retry_after_pause` is applied
58
+ def apply_dlq_flow
59
+ # Process-critical errors are never dispatched or skipped regardless of the
60
+ # strategy outcome: the retry pause protects the partition during the critical
61
+ # shutdown and the failed batch is redelivered after the restart.
62
+ # We consult `errors_tracker.last` (not the per-consumer consumption cause used
63
+ # by the OSS strategies, which have no tracker) because it is exactly what the
64
+ # DLQ strategy callable below receives - this guard judges the same evidence as
65
+ # the strategy it overrides. The tracker is cleared at attempt zero, so `last`
66
+ # is always the most recent failure of the current failure streak
67
+ if critical_error?(errors_tracker.last)
68
+ retry_after_pause
69
+
70
+ return
71
+ end
72
+
73
+ # With virtual partitions, a dispatch/skip decision is never made on a
74
+ # non-collapsed (parallel) run. The deciding consumer operates only on its own
75
+ # virtual partition subset there: the skippable message would be selected from
76
+ # an arbitrary subset and the dispatch marking would commit offsets of messages
77
+ # other virtual partitions never processed. The failure already requested a
78
+ # collapse, so we retry and let the decision happen on the collapsed, linear
79
+ # flow where it is deterministic
80
+ if topic.virtual_partitions? && !collapsed?
81
+ retry_after_pause
82
+
83
+ return
84
+ end
85
+
86
+ flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
87
+
88
+ case flow
89
+ when :retry
90
+ retry_after_pause
91
+
92
+ return
93
+ when :skip
94
+ @_dispatch_to_dlq = false
95
+ when :dispatch
96
+ @_dispatch_to_dlq = true
97
+ # Use custom topic if it was returned from the strategy
98
+ @_dispatch_to_dlq_topic = target_topic || topic.dead_letter_queue.topic
99
+ else
100
+ raise Karafka::UnsupportedCaseError, flow
101
+ end
102
+
103
+ yield
104
+
105
+ # We reset the pause to indicate we will now consider it as "ok".
106
+ coordinator.pause_tracker.reset
107
+
108
+ # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
109
+ pause(seek_offset, nil, false)
110
+ ensure
111
+ @_dispatch_to_dlq_topic = nil
112
+ end
56
113
  end
57
114
  end
58
115
  end
@@ -31,12 +31,8 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
- # Filtering related init strategies
40
36
  module Ftr
41
37
  # Only filtering enabled
42
38
  module Default
@@ -31,15 +31,12 @@
31
31
  module Karafka
32
32
  module Pro
33
33
  module Processing
34
- # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
35
- # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
36
- # lands.
37
34
  module ConsumerGroups
38
35
  module Strategies
39
36
  # Filtering related init strategies
40
37
  module Ftr
41
- # Filtering enabled
42
- # VPs enabled
38
+ # - Filtering enabled
39
+ # - VPs enabled
43
40
  #
44
41
  # VPs should operate without any problems with filtering because virtual partitioning
45
42
  # happens on the limited set of messages and collective filtering applies the same