karafka 2.6.0.beta1 → 2.6.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +64 -27
- data/certs/expired.txt +2 -0
- data/config/locales/errors.yml +5 -0
- data/karafka.gemspec +1 -1
- data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -1
- data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
- data/lib/karafka/active_job/current_attributes.rb +4 -2
- data/lib/karafka/active_job/dispatcher.rb +1 -2
- data/lib/karafka/admin/consumer_groups.rb +8 -2
- data/lib/karafka/admin/replication.rb +6 -3
- data/lib/karafka/admin/topics.rb +32 -12
- data/lib/karafka/admin.rb +155 -59
- data/lib/karafka/app.rb +1 -1
- data/lib/karafka/base_consumer.rb +33 -5
- data/lib/karafka/cli/console.rb +0 -1
- data/lib/karafka/cli/help.rb +0 -1
- data/lib/karafka/cli/info.rb +0 -1
- data/lib/karafka/cli/install.rb +0 -1
- data/lib/karafka/cli/server.rb +0 -1
- data/lib/karafka/cli/swarm.rb +0 -1
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +44 -7
- data/lib/karafka/connection/listener.rb +3 -3
- data/lib/karafka/connection/manager.rb +0 -1
- data/lib/karafka/connection/pauses_manager.rb +17 -0
- data/lib/karafka/connection/proxy.rb +45 -0
- data/lib/karafka/connection/status.rb +0 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +34 -0
- data/lib/karafka/instrumentation/callbacks/consumer_groups/decorator.rb +19 -0
- data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb +0 -1
- data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb +3 -2
- data/lib/karafka/instrumentation/critical_errors_listener.rb +85 -0
- data/lib/karafka/instrumentation/logger_listener.rb +2 -0
- data/lib/karafka/instrumentation/monitor.rb +2 -3
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +0 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +0 -1
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +4 -3
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +0 -2
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +181 -18
- data/lib/karafka/pro/admin/recovery/errors.rb +0 -1
- data/lib/karafka/pro/admin/recovery.rb +6 -4
- data/lib/karafka/pro/base_consumer.rb +5 -5
- data/lib/karafka/pro/cleaner/messages/metadata.rb +0 -1
- data/lib/karafka/pro/cli/parallel_segments/base.rb +1 -2
- data/lib/karafka/pro/connection/manager.rb +7 -4
- data/lib/karafka/pro/encryption/messages/middleware.rb +5 -0
- data/lib/karafka/pro/encryption/messages/parser.rb +4 -0
- data/lib/karafka/pro/instrumentation/callbacks/consumer_groups/decorator.rb +63 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/compensator.rb +120 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/fetcher.rb +88 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/refresher.rb +185 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/registry.rb +102 -0
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +40 -6
- data/lib/karafka/pro/iterator/tpl_builder.rb +27 -7
- data/lib/karafka/pro/iterator.rb +22 -7
- data/lib/karafka/pro/loader.rb +12 -0
- data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/consumer.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/tracker.rb +2 -4
- data/lib/karafka/pro/processing/consumer_groups/collapser.rb +6 -9
- data/lib/karafka/pro/processing/consumer_groups/coordinator.rb +1 -4
- data/lib/karafka/pro/processing/consumer_groups/coordinators/errors_tracker.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/coordinators/filters_applier.rb +21 -10
- data/lib/karafka/pro/processing/consumer_groups/coordinators/virtual_offset_manager.rb +4 -4
- data/lib/karafka/pro/processing/consumer_groups/expansions_selector.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/filters/actions.rb +84 -0
- data/lib/karafka/pro/processing/consumer_groups/filters/base.rb +5 -6
- data/lib/karafka/pro/processing/consumer_groups/filters/delayer.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/filters/expirer.rb +1 -4
- data/lib/karafka/pro/processing/consumer_groups/filters/inline_insights_delayer.rb +1 -4
- data/lib/karafka/pro/processing/consumer_groups/filters/throttler.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/filters/virtual_limiter.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs/consume_non_blocking.rb +2 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs/eofed_non_blocking.rb +0 -2
- data/lib/karafka/pro/processing/consumer_groups/jobs/periodic.rb +2 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs/periodic_non_blocking.rb +0 -2
- data/lib/karafka/pro/processing/consumer_groups/jobs/revoked_non_blocking.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs_builder.rb +5 -4
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/consumer.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/fetcher.rb +8 -10
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/listener.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/base.rb +3 -5
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/default.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/partitioner.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/periodic_job/consumer.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/piping/consumer.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom.rb +10 -9
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom_vp.rb +11 -10
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom.rb +8 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom_vp.rb +9 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom.rb +5 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom_vp.rb +6 -9
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom_vp.rb +4 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom_vp.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/base.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/default.rb +56 -33
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/default.rb +27 -14
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom.rb +5 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom_vp.rb +5 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_vp.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb +65 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb +0 -4
- data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/vp.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb +4 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/vp.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/default.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb +11 -6
- data/lib/karafka/pro/processing/consumer_groups/strategy_selector.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/subscription_groups_coordinator.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/balanced.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/base.rb +2 -4
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/consistent.rb +0 -3
- data/lib/karafka/pro/processing/jobs_queue.rb +4 -1
- data/lib/karafka/pro/processing/schedulers/base.rb +4 -2
- data/lib/karafka/pro/processing/schedulers/default.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +9 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +0 -1
- data/lib/karafka/pro/recurring_tasks/executor.rb +25 -13
- data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb +0 -2
- data/lib/karafka/pro/routing/features/consumer_groups/long_running_job.rb +0 -2
- data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/proxy.rb +2 -2
- data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job.rb +0 -2
- data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb +0 -6
- data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb +5 -4
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/config.rb +0 -6
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/pattern.rb +0 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb +8 -8
- data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb +5 -5
- data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb +33 -13
- data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb +52 -28
- data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb +6 -5
- data/lib/karafka/pro/scheduled_messages/consumer.rb +7 -1
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +6 -1
- data/lib/karafka/pro/swarm/liveness_listener.rb +118 -13
- data/lib/karafka/pro.rb +0 -6
- data/lib/karafka/processing/consumer_groups/coordinator.rb +4 -5
- data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb +6 -2
- data/lib/karafka/processing/consumer_groups/executor.rb +4 -4
- data/lib/karafka/processing/consumer_groups/executors_buffer.rb +2 -4
- data/lib/karafka/processing/consumer_groups/expansions_selector.rb +0 -2
- data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb +0 -2
- data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb +0 -2
- data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb +14 -16
- data/lib/karafka/processing/consumer_groups/jobs/consume.rb +2 -5
- data/lib/karafka/processing/consumer_groups/jobs/eofed.rb +2 -3
- data/lib/karafka/processing/consumer_groups/jobs/idle.rb +2 -2
- data/lib/karafka/processing/consumer_groups/jobs/revoked.rb +2 -4
- data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb +2 -2
- data/lib/karafka/processing/consumer_groups/partitioner.rb +3 -6
- data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb +5 -2
- data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb +2 -4
- data/lib/karafka/processing/consumer_groups/strategies/base.rb +3 -5
- data/lib/karafka/processing/consumer_groups/strategies/default.rb +43 -13
- data/lib/karafka/processing/consumer_groups/strategies/dlq.rb +5 -2
- data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb +5 -2
- data/lib/karafka/processing/consumer_groups/strategies/mom.rb +2 -3
- data/lib/karafka/processing/consumer_groups/strategy_selector.rb +0 -2
- data/lib/karafka/processing/coordinator.rb +0 -2
- data/lib/karafka/processing/coordinators_buffer.rb +0 -2
- data/lib/karafka/processing/partitioner.rb +0 -2
- data/lib/karafka/processing/strategy_selector.rb +0 -2
- data/lib/karafka/processing/worker.rb +14 -6
- data/lib/karafka/processing/workers_pool.rb +21 -4
- data/lib/karafka/railtie.rb +0 -1
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb +6 -5
- data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/topic.rb +0 -1
- data/lib/karafka/routing/subscription_group.rb +1 -1
- data/lib/karafka/routing/topic.rb +4 -4
- data/lib/karafka/routing/topics.rb +21 -1
- data/lib/karafka/server.rb +9 -0
- data/lib/karafka/setup/config.rb +54 -10
- data/lib/karafka/setup/config_proxy.rb +2 -1
- data/lib/karafka/setup/contracts/config.rb +14 -0
- data/lib/karafka/swarm/node.rb +2 -2
- data/lib/karafka/version.rb +1 -2
- data/lib/karafka.rb +2 -2
- metadata +12 -4
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# Manual offset management enabled
|
|
42
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - Manual offset management enabled
|
|
39
|
+
# - Virtual Partitions enabled
|
|
43
40
|
module MomVp
|
|
44
41
|
include Strategies::Default
|
|
45
42
|
include Strategies::Vp::Default
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
# Pro processing strategies namespace
|
|
39
36
|
module Strategies
|
|
@@ -31,15 +31,13 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
|
-
# No features enabled
|
|
40
|
-
# No manual offset management
|
|
41
|
-
# No long running jobs
|
|
42
|
-
# No virtual partitions
|
|
36
|
+
# No features enabled:
|
|
37
|
+
# - No manual offset management
|
|
38
|
+
# - No long running jobs
|
|
39
|
+
# - No virtual partitions
|
|
40
|
+
#
|
|
43
41
|
# Nothing. Just standard, automatic flow
|
|
44
42
|
module Default
|
|
45
43
|
include Base
|
|
@@ -70,10 +68,10 @@ module Karafka
|
|
|
70
68
|
# @return [Boolean] true if we were able to mark the offset, false otherwise.
|
|
71
69
|
# False indicates that we were not able and that we have lost the partition.
|
|
72
70
|
#
|
|
73
|
-
# @note We keep track of this offset in case we would mark as consumed and got error
|
|
74
|
-
# processing another message. In case like this we do not pause on the message
|
|
75
|
-
# already processed but rather at the next one. This applies to both sync and
|
|
76
|
-
# versions of this method.
|
|
71
|
+
# @note We keep track of this offset in case we would mark as consumed and got error
|
|
72
|
+
# when processing another message. In case like this we do not pause on the message
|
|
73
|
+
# we've already processed but rather at the next one. This applies to both sync and
|
|
74
|
+
# async versions of this method.
|
|
77
75
|
def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
|
|
78
76
|
# If we are inside a transaction than we can just mark as consumed within it
|
|
79
77
|
if @_in_transaction
|
|
@@ -88,6 +86,8 @@ module Karafka
|
|
|
88
86
|
# We ignore second marking because it changes nothing and in case of people using
|
|
89
87
|
# metadata storage but with automatic offset marking, this would cause metadata to be
|
|
90
88
|
# erased by automatic marking
|
|
89
|
+
# Only this exact re-mark is skipped - marking genuinely older offsets is intentionally
|
|
90
|
+
# allowed and rewinds the seek offset for reprocessing (see #2432)
|
|
91
91
|
return true if (seek_offset - 1) == message.offset
|
|
92
92
|
return false if revoked?
|
|
93
93
|
|
|
@@ -101,7 +101,14 @@ module Karafka
|
|
|
101
101
|
client.mark_as_consumed(message, offset_metadata)
|
|
102
102
|
end
|
|
103
103
|
|
|
104
|
-
|
|
104
|
+
unless stored
|
|
105
|
+
# Evaluate ownership for its coordinator-revoking side effect - all marking
|
|
106
|
+
# failure causes are ownership related. The failure itself always reports
|
|
107
|
+
# false: the offset was not stored, regardless of the ownership state
|
|
108
|
+
revoked?
|
|
109
|
+
|
|
110
|
+
return false
|
|
111
|
+
end
|
|
105
112
|
|
|
106
113
|
self.seek_offset = message.offset + 1
|
|
107
114
|
end
|
|
@@ -130,6 +137,8 @@ module Karafka
|
|
|
130
137
|
# We ignore second marking because it changes nothing and in case of people using
|
|
131
138
|
# metadata storage but with automatic offset marking, this would cause metadata to be
|
|
132
139
|
# erased by automatic marking
|
|
140
|
+
# Only this exact re-mark is skipped - marking genuinely older offsets is intentionally
|
|
141
|
+
# allowed and rewinds the seek offset for reprocessing (see #2432)
|
|
133
142
|
return true if (seek_offset - 1) == message.offset
|
|
134
143
|
return false if revoked?
|
|
135
144
|
|
|
@@ -143,7 +152,14 @@ module Karafka
|
|
|
143
152
|
client.mark_as_consumed!(message, offset_metadata)
|
|
144
153
|
end
|
|
145
154
|
|
|
146
|
-
|
|
155
|
+
unless stored
|
|
156
|
+
# Evaluate ownership for its coordinator-revoking side effect - all marking
|
|
157
|
+
# failure causes are ownership related. The failure itself always reports
|
|
158
|
+
# false: the offset was not stored, regardless of the ownership state
|
|
159
|
+
revoked?
|
|
160
|
+
|
|
161
|
+
return false
|
|
162
|
+
end
|
|
147
163
|
|
|
148
164
|
self.seek_offset = message.offset + 1
|
|
149
165
|
end
|
|
@@ -162,8 +178,8 @@ module Karafka
|
|
|
162
178
|
#
|
|
163
179
|
# @param active_producer [WaterDrop::Producer] alternative producer instance we may want
|
|
164
180
|
# to use. It is useful when we have connection pool or any other selective engine for
|
|
165
|
-
# managing multiple producers. If not provided, default producer taken from
|
|
166
|
-
# will be used.
|
|
181
|
+
# managing multiple producers. If not provided, default producer taken from
|
|
182
|
+
# `#producer` will be used.
|
|
167
183
|
#
|
|
168
184
|
# @yield code that we want to run in a transaction
|
|
169
185
|
#
|
|
@@ -212,23 +228,23 @@ module Karafka
|
|
|
212
228
|
# This offset is already stored in transaction but we set it here anyhow because we
|
|
213
229
|
# want to make sure our internal in-memory state is aligned with the transaction
|
|
214
230
|
#
|
|
215
|
-
#
|
|
216
|
-
#
|
|
231
|
+
# We never need to use the blocking `#mark_as_consumed!` here because the offset
|
|
232
|
+
# anyhow was already stored during the transaction
|
|
217
233
|
#
|
|
218
|
-
#
|
|
219
|
-
#
|
|
220
|
-
#
|
|
221
|
-
#
|
|
234
|
+
# Since the offset could have been already stored in Kafka (could have because you
|
|
235
|
+
# can have transactions without marking), we use the `@_in_transaction_marked` state
|
|
236
|
+
# to decide if we need to dispatch the offset via client at all (if post
|
|
237
|
+
# transaction, then we do not have to)
|
|
222
238
|
#
|
|
223
|
-
#
|
|
224
|
-
#
|
|
225
|
-
#
|
|
226
|
-
#
|
|
227
|
-
#
|
|
228
|
-
#
|
|
239
|
+
# In theory we could only keep reference to the most recent marking and reject
|
|
240
|
+
# others. We however do not do it for two reasons:
|
|
241
|
+
# - User may have non standard flow relying on some alternative order and we want
|
|
242
|
+
# to mimic this
|
|
243
|
+
# - Complex strategies like VPs can use this in VPs to mark in parallel without
|
|
244
|
+
# having to redefine the transactional flow completely
|
|
229
245
|
#
|
|
230
|
-
#
|
|
231
|
-
#
|
|
246
|
+
# This should be applied only if transaction did not error and if it was not
|
|
247
|
+
# aborted.
|
|
232
248
|
if transaction_completed
|
|
233
249
|
@_transaction_marked.each do |marking|
|
|
234
250
|
marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
|
|
@@ -301,7 +317,8 @@ module Karafka
|
|
|
301
317
|
# Marks the current state only in memory as the offset marking has already happened
|
|
302
318
|
# using the producer transaction
|
|
303
319
|
# @param message [Messages::Message] last successfully processed message.
|
|
304
|
-
# @return [Boolean] true if all good, false if we lost assignment and no point in
|
|
320
|
+
# @return [Boolean] true if all good, false if we lost assignment and no point in
|
|
321
|
+
# marking
|
|
305
322
|
def mark_in_memory(message)
|
|
306
323
|
# seek offset can be nil only in case `#seek` was invoked with offset reset request
|
|
307
324
|
# In case like this we ignore marking
|
|
@@ -310,6 +327,8 @@ module Karafka
|
|
|
310
327
|
# We ignore second marking because it changes nothing and in case of people using
|
|
311
328
|
# metadata storage but with automatic offset marking, this would cause metadata to be
|
|
312
329
|
# erased by automatic marking
|
|
330
|
+
# Only this exact re-mark is skipped - marking genuinely older offsets is intentionally
|
|
331
|
+
# allowed and rewinds the seek offset for reprocessing (see #2432)
|
|
313
332
|
return true if (seek_offset - 1) == message.offset
|
|
314
333
|
return false if revoked?
|
|
315
334
|
|
|
@@ -328,8 +347,8 @@ module Karafka
|
|
|
328
347
|
nil
|
|
329
348
|
end
|
|
330
349
|
|
|
331
|
-
# Increment number of attempts per one "full" job. For all VP on a single topic
|
|
332
|
-
# this also should run once.
|
|
350
|
+
# Increment number of attempts per one "full" job. For all VP on a single topic
|
|
351
|
+
# partition this also should run once.
|
|
333
352
|
def handle_before_consume
|
|
334
353
|
coordinator.on_started do
|
|
335
354
|
coordinator.pause_tracker.increment
|
|
@@ -350,7 +369,11 @@ module Karafka
|
|
|
350
369
|
|
|
351
370
|
# Mark job as successful
|
|
352
371
|
coordinator.success!(self)
|
|
353
|
-
|
|
372
|
+
# Failure recording must be class-agnostic: an unrecorded non-StandardError would
|
|
373
|
+
# leave the consumption result in its default (successful) state and the
|
|
374
|
+
# after-consume flow would mark the failed batch as consumed instead of engaging
|
|
375
|
+
# the retry
|
|
376
|
+
rescue Exception => e
|
|
354
377
|
# If failed, mark as failed
|
|
355
378
|
coordinator.failure!(self, e)
|
|
356
379
|
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
# Namespace for all the strategies starting with DLQ
|
|
@@ -49,8 +46,8 @@ module Karafka
|
|
|
49
46
|
|
|
50
47
|
# Override of the standard `#mark_as_consumed` in order to handle the pause tracker
|
|
51
48
|
# reset in case DLQ is marked as fully independent. When DLQ is marked independent,
|
|
52
|
-
# any offset marking causes the pause count tracker to reset. This is useful when
|
|
53
|
-
#
|
|
49
|
+
# any offset marking causes the pause count tracker to reset. This is useful when the
|
|
50
|
+
# error is not due to the collective batch operations state but due to intermediate
|
|
54
51
|
# "crawling" errors that move with it
|
|
55
52
|
#
|
|
56
53
|
# @see `Strategies::Default#mark_as_consumed` for more details
|
|
@@ -68,8 +65,8 @@ module Karafka
|
|
|
68
65
|
@_current_offset_metadata = nil
|
|
69
66
|
end
|
|
70
67
|
|
|
71
|
-
# Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in
|
|
72
|
-
# DLQ was configured with the `independent` flag.
|
|
68
|
+
# Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in
|
|
69
|
+
# case DLQ was configured with the `independent` flag.
|
|
73
70
|
#
|
|
74
71
|
# @see `Strategies::Default#mark_as_consumed!` for more details
|
|
75
72
|
# @param message [Messages::Message]
|
|
@@ -147,8 +144,8 @@ module Karafka
|
|
|
147
144
|
)
|
|
148
145
|
end
|
|
149
146
|
|
|
150
|
-
# Dispatches the message to the DLQ (when needed and when applicable based on
|
|
151
|
-
#
|
|
147
|
+
# Dispatches the message to the DLQ (when needed and when applicable based on
|
|
148
|
+
# settings) and marks this message as consumed for non MOM flows.
|
|
152
149
|
#
|
|
153
150
|
# If producer is transactional and config allows, uses transaction to do that
|
|
154
151
|
def dispatch_if_needed_and_mark_as_consumed
|
|
@@ -157,11 +154,7 @@ module Karafka
|
|
|
157
154
|
dispatch = lambda do
|
|
158
155
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
|
159
156
|
|
|
160
|
-
if mark_after_dispatch?
|
|
161
|
-
mark_dispatched_to_dlq(skippable_message)
|
|
162
|
-
else
|
|
163
|
-
self.seek_offset = skippable_message.offset + 1
|
|
164
|
-
end
|
|
157
|
+
mark_dispatched_to_dlq(skippable_message) if mark_after_dispatch?
|
|
165
158
|
end
|
|
166
159
|
|
|
167
160
|
if dispatch_in_a_transaction?
|
|
@@ -169,6 +162,12 @@ module Karafka
|
|
|
169
162
|
else
|
|
170
163
|
dispatch.call
|
|
171
164
|
end
|
|
165
|
+
|
|
166
|
+
# When not marking after dispatch, the seek offset is advanced manually. This must
|
|
167
|
+
# happen only after the dispatch (and its transaction) succeeded - advancing it
|
|
168
|
+
# inside the transaction would let the mutation survive an abort, skipping the
|
|
169
|
+
# broken message which would then be neither in the DLQ nor reprocessed
|
|
170
|
+
self.seek_offset = skippable_message.offset + 1 unless mark_after_dispatch?
|
|
172
171
|
end
|
|
173
172
|
|
|
174
173
|
# @param skippable_message [Array<Karafka::Messages::Message>]
|
|
@@ -235,6 +234,20 @@ module Karafka
|
|
|
235
234
|
# In case of `:skip` and `:dispatch` will run the exact flow provided in a block
|
|
236
235
|
# In case of `:retry` always `#retry_after_pause` is applied
|
|
237
236
|
def apply_dlq_flow
|
|
237
|
+
# Process-critical errors are never dispatched or skipped regardless of the
|
|
238
|
+
# strategy outcome: the retry pause protects the partition during the critical
|
|
239
|
+
# shutdown and the failed batch is redelivered after the restart.
|
|
240
|
+
# We consult `errors_tracker.last` (not the per-consumer consumption cause used
|
|
241
|
+
# by the OSS strategies, which have no tracker) because it is exactly what the
|
|
242
|
+
# DLQ strategy callable below receives - this guard judges the same evidence as
|
|
243
|
+
# the strategy it overrides. The tracker is cleared at attempt zero, so `last`
|
|
244
|
+
# is always the most recent failure of the current failure streak
|
|
245
|
+
if critical_error?(errors_tracker.last)
|
|
246
|
+
retry_after_pause
|
|
247
|
+
|
|
248
|
+
return
|
|
249
|
+
end
|
|
250
|
+
|
|
238
251
|
flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
|
|
239
252
|
|
|
240
253
|
case flow
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
40
|
-
# Dead-Letter Queue enabled
|
|
41
|
-
# Filtering enabled
|
|
42
|
-
# Long-Running Job enabled
|
|
37
|
+
# - Dead-Letter Queue enabled
|
|
38
|
+
# - Filtering enabled
|
|
39
|
+
# - Long-Running Job enabled
|
|
43
40
|
module FtrLrj
|
|
44
41
|
include Strategies::Dlq::Lrj
|
|
45
42
|
include Strategies::Lrj::Ftr
|
|
@@ -71,7 +68,7 @@ module Karafka
|
|
|
71
68
|
|
|
72
69
|
# :seek and :pause are fully handled by handle_post_filtering
|
|
73
70
|
# For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
|
|
74
|
-
return unless coordinator.filter.
|
|
71
|
+
return unless coordinator.filter.skip?
|
|
75
72
|
elsif !revoked? && !coordinator.manual_seek?
|
|
76
73
|
seek(seek_offset, false, reset_offset: false)
|
|
77
74
|
end
|
|
@@ -31,16 +31,13 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
40
|
-
# DLQ enabled
|
|
41
|
-
# Ftr enabled
|
|
42
|
-
# LRJ enabled
|
|
43
|
-
# MoM enabled
|
|
37
|
+
# - DLQ enabled
|
|
38
|
+
# - Ftr enabled
|
|
39
|
+
# - LRJ enabled
|
|
40
|
+
# - MoM enabled
|
|
44
41
|
module FtrLrjMom
|
|
45
42
|
include Strategies::Ftr::Default
|
|
46
43
|
include Strategies::Dlq::LrjMom
|
|
@@ -66,7 +63,7 @@ module Karafka
|
|
|
66
63
|
|
|
67
64
|
# :seek and :pause are fully handled by handle_post_filtering
|
|
68
65
|
# For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
|
|
69
|
-
return unless coordinator.filter.
|
|
66
|
+
return unless coordinator.filter.skip?
|
|
70
67
|
elsif !revoked? && !coordinator.manual_seek?
|
|
71
68
|
seek(last_group_message.offset + 1, false, reset_offset: false)
|
|
72
69
|
end
|
|
@@ -31,17 +31,14 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
40
|
-
# DLQ enabled
|
|
41
|
-
# Ftr enabled
|
|
42
|
-
# LRJ enabled
|
|
43
|
-
# MoM enabled
|
|
44
|
-
# VP enabled
|
|
37
|
+
# - DLQ enabled
|
|
38
|
+
# - Ftr enabled
|
|
39
|
+
# - LRJ enabled
|
|
40
|
+
# - MoM enabled
|
|
41
|
+
# - VP enabled
|
|
45
42
|
module FtrLrjMomVp
|
|
46
43
|
include Strategies::Vp::Default
|
|
47
44
|
include Strategies::Dlq::Vp
|
|
@@ -31,16 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
40
|
-
# Dead-Letter Queue enabled
|
|
41
|
-
# Filtering enabled
|
|
42
|
-
# Long-Running Job enabled
|
|
43
|
-
# Virtual Partitions
|
|
37
|
+
# - Dead-Letter Queue enabled
|
|
38
|
+
# - Filtering enabled
|
|
39
|
+
# - Long-Running Job enabled
|
|
44
40
|
module FtrLrjVp
|
|
45
41
|
include Strategies::Vp::Default
|
|
46
42
|
include Strategies::Dlq::Vp
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
@@ -31,14 +31,11 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
40
|
-
# DLQ enabled
|
|
41
|
-
# Long-Running Job enabled
|
|
37
|
+
# - DLQ enabled
|
|
38
|
+
# - Long-Running Job enabled
|
|
42
39
|
module Lrj
|
|
43
40
|
# Order here matters, lrj needs to be second
|
|
44
41
|
include Strategies::Dlq::Default
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
40
|
-
# Dead-Letter Queue enabled
|
|
41
|
-
# Long-Running Job enabled
|
|
42
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - Dead-Letter Queue enabled
|
|
38
|
+
# - Long-Running Job enabled
|
|
39
|
+
# - Virtual Partitions enabled
|
|
43
40
|
module LrjVp
|
|
44
41
|
# Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
|
|
45
42
|
include Strategies::Vp::Default
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
40
|
-
# Dead-Letter Queue enabled
|
|
41
|
-
# MoM enabled
|
|
42
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - Dead-Letter Queue enabled
|
|
38
|
+
# - MoM enabled
|
|
39
|
+
# - Virtual Partitions enabled
|
|
43
40
|
module MomVp
|
|
44
41
|
include Strategies::Dlq::Vp
|
|
45
42
|
include Strategies::Dlq::Mom
|
|
@@ -31,18 +31,15 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Dlq
|
|
40
|
-
# Dead Letter Queue enabled
|
|
41
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - Dead Letter Queue enabled
|
|
38
|
+
# - Virtual Partitions enabled
|
|
42
39
|
#
|
|
43
|
-
# In general because we collapse processing in virtual partitions to one on errors,
|
|
44
|
-
# is no special action that needs to be taken because we warranty that even with
|
|
45
|
-
#
|
|
40
|
+
# In general because we collapse processing in virtual partitions to one on errors,
|
|
41
|
+
# there is no special action that needs to be taken because we warranty that even with
|
|
42
|
+
# VPson errors a retry collapses into a single state and from this single state we can
|
|
46
43
|
# mark as consumed the message that we are moving to the DLQ.
|
|
47
44
|
module Vp
|
|
48
45
|
# Features for this strategy
|
|
@@ -53,6 +50,66 @@ module Karafka
|
|
|
53
50
|
|
|
54
51
|
include Strategies::Dlq::Default
|
|
55
52
|
include Strategies::Vp::Default
|
|
53
|
+
|
|
54
|
+
# Runs the DLQ strategy and based on it it performs certain operations
|
|
55
|
+
#
|
|
56
|
+
# In case of `:skip` and `:dispatch` will run the exact flow provided in a block
|
|
57
|
+
# In case of `:retry` always `#retry_after_pause` is applied
|
|
58
|
+
def apply_dlq_flow
|
|
59
|
+
# Process-critical errors are never dispatched or skipped regardless of the
|
|
60
|
+
# strategy outcome: the retry pause protects the partition during the critical
|
|
61
|
+
# shutdown and the failed batch is redelivered after the restart.
|
|
62
|
+
# We consult `errors_tracker.last` (not the per-consumer consumption cause used
|
|
63
|
+
# by the OSS strategies, which have no tracker) because it is exactly what the
|
|
64
|
+
# DLQ strategy callable below receives - this guard judges the same evidence as
|
|
65
|
+
# the strategy it overrides. The tracker is cleared at attempt zero, so `last`
|
|
66
|
+
# is always the most recent failure of the current failure streak
|
|
67
|
+
if critical_error?(errors_tracker.last)
|
|
68
|
+
retry_after_pause
|
|
69
|
+
|
|
70
|
+
return
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# With virtual partitions, a dispatch/skip decision is never made on a
|
|
74
|
+
# non-collapsed (parallel) run. The deciding consumer operates only on its own
|
|
75
|
+
# virtual partition subset there: the skippable message would be selected from
|
|
76
|
+
# an arbitrary subset and the dispatch marking would commit offsets of messages
|
|
77
|
+
# other virtual partitions never processed. The failure already requested a
|
|
78
|
+
# collapse, so we retry and let the decision happen on the collapsed, linear
|
|
79
|
+
# flow where it is deterministic
|
|
80
|
+
if topic.virtual_partitions? && !collapsed?
|
|
81
|
+
retry_after_pause
|
|
82
|
+
|
|
83
|
+
return
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
|
|
87
|
+
|
|
88
|
+
case flow
|
|
89
|
+
when :retry
|
|
90
|
+
retry_after_pause
|
|
91
|
+
|
|
92
|
+
return
|
|
93
|
+
when :skip
|
|
94
|
+
@_dispatch_to_dlq = false
|
|
95
|
+
when :dispatch
|
|
96
|
+
@_dispatch_to_dlq = true
|
|
97
|
+
# Use custom topic if it was returned from the strategy
|
|
98
|
+
@_dispatch_to_dlq_topic = target_topic || topic.dead_letter_queue.topic
|
|
99
|
+
else
|
|
100
|
+
raise Karafka::UnsupportedCaseError, flow
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
yield
|
|
104
|
+
|
|
105
|
+
# We reset the pause to indicate we will now consider it as "ok".
|
|
106
|
+
coordinator.pause_tracker.reset
|
|
107
|
+
|
|
108
|
+
# Always backoff after DLQ dispatch even on skip to prevent overloads on errors
|
|
109
|
+
pause(seek_offset, nil, false)
|
|
110
|
+
ensure
|
|
111
|
+
@_dispatch_to_dlq_topic = nil
|
|
112
|
+
end
|
|
56
113
|
end
|
|
57
114
|
end
|
|
58
115
|
end
|
|
@@ -31,12 +31,8 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
|
-
# Filtering related init strategies
|
|
40
36
|
module Ftr
|
|
41
37
|
# Only filtering enabled
|
|
42
38
|
module Default
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
# Filtering related init strategies
|
|
40
37
|
module Ftr
|
|
41
|
-
# Filtering enabled
|
|
42
|
-
# VPs enabled
|
|
38
|
+
# - Filtering enabled
|
|
39
|
+
# - VPs enabled
|
|
43
40
|
#
|
|
44
41
|
# VPs should operate without any problems with filtering because virtual partitioning
|
|
45
42
|
# happens on the limited set of messages and collective filtering applies the same
|