karafka 2.0.40 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +2 -2
- data/CHANGELOG.md +30 -1
- data/Gemfile +3 -2
- data/Gemfile.lock +13 -1
- data/bin/integrations +17 -2
- data/config/locales/errors.yml +10 -0
- data/config/locales/pro_errors.yml +0 -2
- data/lib/karafka/active_job/consumer.rb +16 -11
- data/lib/karafka/active_job/current_attributes/loading.rb +36 -0
- data/lib/karafka/active_job/current_attributes/persistence.rb +28 -0
- data/lib/karafka/active_job/current_attributes.rb +42 -0
- data/lib/karafka/active_job/dispatcher.rb +8 -2
- data/lib/karafka/admin.rb +17 -13
- data/lib/karafka/connection/client.rb +6 -3
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/instrumentation/callbacks/statistics.rb +12 -0
- data/lib/karafka/instrumentation/logger_listener.rb +16 -5
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +166 -0
- data/lib/karafka/pro/active_job/consumer.rb +1 -10
- data/lib/karafka/pro/active_job/dispatcher.rb +2 -2
- data/lib/karafka/pro/iterator.rb +253 -0
- data/lib/karafka/pro/processing/coordinator.rb +20 -1
- data/lib/karafka/pro/processing/filters/virtual_limiter.rb +52 -0
- data/lib/karafka/pro/processing/filters_applier.rb +4 -0
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +1 -1
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +3 -1
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +2 -2
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +2 -0
- data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +3 -6
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +43 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +1 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +3 -7
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +41 -0
- data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +1 -0
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +3 -6
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +36 -0
- data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +1 -0
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +8 -7
- data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +37 -0
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -0
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +40 -0
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +2 -0
- data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +38 -0
- data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +37 -0
- data/lib/karafka/pro/{base_consumer.rb → processing/strategies/mom/vp.rb} +17 -7
- data/lib/karafka/pro/processing/strategies/vp/default.rb +51 -0
- data/lib/karafka/pro/processing/virtual_offset_manager.rb +147 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +0 -17
- data/lib/karafka/processing/strategies/default.rb +2 -0
- data/lib/karafka/processing/strategies/dlq_mom.rb +9 -7
- data/lib/karafka/routing/router.rb +15 -0
- data/lib/karafka/setup/config.rb +7 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +5 -0
- data.tar.gz.sig +0 -0
- metadata +17 -4
- metadata.gz.sig +0 -0
- data/lib/karafka/instrumentation/vendors/datadog/listener.rb +0 -16
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
module Dlq
|
19
|
+
# DLQ enabled
|
20
|
+
# Ftr enabled
|
21
|
+
# LRJ enabled
|
22
|
+
# MoM enabled
|
23
|
+
# VP enabled
|
24
|
+
module FtrLrjMomVp
|
25
|
+
include Strategies::Vp::Default
|
26
|
+
include Strategies::Dlq::Vp
|
27
|
+
# Same as non VP because of the coordinator post-execution lock
|
28
|
+
include Strategies::Dlq::FtrLrjMom
|
29
|
+
|
30
|
+
# Features for this strategy
|
31
|
+
FEATURES = %i[
|
32
|
+
dead_letter_queue
|
33
|
+
filtering
|
34
|
+
long_running_job
|
35
|
+
manual_offset_management
|
36
|
+
virtual_partitions
|
37
|
+
].freeze
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -53,15 +53,11 @@ module Karafka
|
|
53
53
|
# We reset the pause to indicate we will now consider it as "ok".
|
54
54
|
coordinator.pause_tracker.reset
|
55
55
|
|
56
|
-
skippable_message,
|
56
|
+
skippable_message, _marked = find_skippable_message
|
57
57
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
58
58
|
|
59
|
-
|
60
|
-
|
61
|
-
pause(coordinator.seek_offset, nil, false)
|
62
|
-
else
|
63
|
-
pause(skippable_message.offset + 1, nil, false)
|
64
|
-
end
|
59
|
+
coordinator.seek_offset = skippable_message.offset + 1
|
60
|
+
pause(coordinator.seek_offset, nil, false)
|
65
61
|
end
|
66
62
|
end
|
67
63
|
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
module Dlq
|
19
|
+
# - DLQ
|
20
|
+
# - Ftr
|
21
|
+
# - Mom
|
22
|
+
# - VP
|
23
|
+
# We can use the `Strategies::FtrMom` because VPs collapse prior to DLQ
|
24
|
+
module FtrMomVp
|
25
|
+
include Strategies::Mom::Vp
|
26
|
+
include Strategies::Dlq::Vp
|
27
|
+
include Strategies::Dlq::FtrMom
|
28
|
+
|
29
|
+
# Features for this strategy
|
30
|
+
FEATURES = %i[
|
31
|
+
dead_letter_queue
|
32
|
+
filtering
|
33
|
+
manual_offset_management
|
34
|
+
virtual_partitions
|
35
|
+
].freeze
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -47,14 +47,11 @@ module Karafka
|
|
47
47
|
|
48
48
|
return resume if revoked?
|
49
49
|
|
50
|
-
skippable_message,
|
50
|
+
skippable_message, _marked = find_skippable_message
|
51
51
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
52
52
|
|
53
|
-
|
54
|
-
|
55
|
-
else
|
56
|
-
pause(skippable_message.offset + 1, nil, false)
|
57
|
-
end
|
53
|
+
coordinator.seek_offset = skippable_message.offset + 1
|
54
|
+
pause(coordinator.seek_offset, nil, false)
|
58
55
|
end
|
59
56
|
end
|
60
57
|
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
module Dlq
|
19
|
+
# Strategy for supporting DLQ with Mom, LRJ and VP enabled
|
20
|
+
module LrjMomVp
|
21
|
+
# Feature set for this strategy
|
22
|
+
FEATURES = %i[
|
23
|
+
dead_letter_queue
|
24
|
+
long_running_job
|
25
|
+
manual_offset_management
|
26
|
+
virtual_partitions
|
27
|
+
].freeze
|
28
|
+
|
29
|
+
include Strategies::Dlq::Vp
|
30
|
+
include Strategies::Dlq::LrjMom
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -47,15 +47,16 @@ module Karafka
|
|
47
47
|
# We reset the pause to indicate we will now consider it as "ok".
|
48
48
|
coordinator.pause_tracker.reset
|
49
49
|
|
50
|
-
skippable_message,
|
50
|
+
skippable_message, = find_skippable_message
|
51
51
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
52
52
|
|
53
|
-
#
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
53
|
+
# Save the next offset we want to go with after moving given message to DLQ
|
54
|
+
# Without this, we would not be able to move forward and we would end up
|
55
|
+
# in an infinite loop trying to un-pause from the message we've already processed
|
56
|
+
# Of course, since it's a MoM a rebalance or kill, will move it back as no
|
57
|
+
# offsets are being committed
|
58
|
+
coordinator.seek_offset = skippable_message.offset + 1
|
59
|
+
pause(coordinator.seek_offset, nil, false)
|
59
60
|
end
|
60
61
|
end
|
61
62
|
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
module Dlq
|
19
|
+
# Dead-Letter Queue enabled
|
20
|
+
# MoM enabled
|
21
|
+
# Virtual Partitions enabled
|
22
|
+
module MomVp
|
23
|
+
include Strategies::Dlq::Vp
|
24
|
+
include Strategies::Dlq::Mom
|
25
|
+
|
26
|
+
# Features for this strategy
|
27
|
+
FEATURES = %i[
|
28
|
+
dead_letter_queue
|
29
|
+
manual_offset_management
|
30
|
+
virtual_partitions
|
31
|
+
].freeze
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -30,6 +30,8 @@ module Karafka
|
|
30
30
|
|
31
31
|
# We always need to pause prior to doing any jobs for LRJ
|
32
32
|
def handle_before_enqueue
|
33
|
+
super
|
34
|
+
|
33
35
|
# This ensures that when running LRJ with VP, things operate as expected run only
|
34
36
|
# once for all the virtual partitions collectively
|
35
37
|
coordinator.on_enqueued do
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
# Filtering related init strategies
|
19
|
+
module Lrj
|
20
|
+
# Filtering enabled
|
21
|
+
# LRJ enabled
|
22
|
+
# MoM enabled
|
23
|
+
# VPs enabled
|
24
|
+
module FtrMomVp
|
25
|
+
# Filtering + LRJ + Mom + VPs
|
26
|
+
FEATURES = %i[
|
27
|
+
filtering
|
28
|
+
long_running_job
|
29
|
+
manual_offset_management
|
30
|
+
virtual_partitions
|
31
|
+
].freeze
|
32
|
+
|
33
|
+
include Strategies::Lrj::MomVp
|
34
|
+
include Strategies::Lrj::FtrMom
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -30,6 +30,8 @@ module Karafka
|
|
30
30
|
|
31
31
|
# We always need to pause prior to doing any jobs for LRJ
|
32
32
|
def handle_before_enqueue
|
33
|
+
super
|
34
|
+
|
33
35
|
# This ensures that when running LRJ with VP, things operate as expected run only
|
34
36
|
# once for all the virtual partitions collectively
|
35
37
|
coordinator.on_enqueued do
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
# Namespace for all the LRJ starting strategies
|
19
|
+
module Lrj
|
20
|
+
# Long-Running Job enabled
|
21
|
+
# Manual offset management enabled
|
22
|
+
# Virtual Partitions enabled
|
23
|
+
module MomVp
|
24
|
+
include Strategies::Mom::Vp
|
25
|
+
include Strategies::Lrj::Mom
|
26
|
+
|
27
|
+
# Features for this strategy
|
28
|
+
FEATURES = %i[
|
29
|
+
long_running_job
|
30
|
+
manual_offset_management
|
31
|
+
virtual_partitions
|
32
|
+
].freeze
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
module Mom
|
19
|
+
# - Mom enabled
|
20
|
+
# - Ftr enabled
|
21
|
+
# - Vp enabled
|
22
|
+
module FtrVp
|
23
|
+
include Strategies::Mom::Vp
|
24
|
+
include Strategies::Mom::Ftr
|
25
|
+
|
26
|
+
# Features of this strategy
|
27
|
+
FEATURES = %i[
|
28
|
+
filtering
|
29
|
+
manual_offset_management
|
30
|
+
virtual_partitions
|
31
|
+
].freeze
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -13,13 +13,23 @@
|
|
13
13
|
|
14
14
|
module Karafka
|
15
15
|
module Pro
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
module Mom
|
19
|
+
# - Mom enabled
|
20
|
+
# - Vp enabled
|
21
|
+
module Vp
|
22
|
+
include Strategies::Vp::Default
|
23
|
+
include Strategies::Mom::Default
|
24
|
+
|
25
|
+
# Features of this strategy
|
26
|
+
FEATURES = %i[
|
27
|
+
manual_offset_management
|
28
|
+
virtual_partitions
|
29
|
+
].freeze
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
23
33
|
end
|
24
34
|
end
|
25
35
|
end
|
@@ -28,6 +28,44 @@ module Karafka
|
|
28
28
|
virtual_partitions
|
29
29
|
].freeze
|
30
30
|
|
31
|
+
# @param message [Karafka::Messages::Message] marks message as consumed
|
32
|
+
# @note This virtual offset management uses a regular default marking API underneath.
|
33
|
+
# We do not alter the "real" marking API, as VPs are just one of many cases we want
|
34
|
+
# to support and we do not want to impact them with collective offsets management
|
35
|
+
def mark_as_consumed(message)
|
36
|
+
return super if collapsed?
|
37
|
+
|
38
|
+
manager = coordinator.virtual_offset_manager
|
39
|
+
|
40
|
+
coordinator.synchronize do
|
41
|
+
manager.mark(message)
|
42
|
+
# If this is last marking on a finished flow, we can use the original
|
43
|
+
# last message and in order to do so, we need to mark all previous messages as
|
44
|
+
# consumed as otherwise the computed offset could be different
|
45
|
+
# We mark until our offset just in case of a DLQ flow or similar, where we do not
|
46
|
+
# want to mark all but until the expected location
|
47
|
+
manager.mark_until(message) if coordinator.finished?
|
48
|
+
|
49
|
+
return revoked? unless manager.markable?
|
50
|
+
end
|
51
|
+
|
52
|
+
manager.markable? ? super(manager.markable) : revoked?
|
53
|
+
end
|
54
|
+
|
55
|
+
# @param message [Karafka::Messages::Message] blocking marks message as consumed
|
56
|
+
def mark_as_consumed!(message)
|
57
|
+
return super if collapsed?
|
58
|
+
|
59
|
+
manager = coordinator.virtual_offset_manager
|
60
|
+
|
61
|
+
coordinator.synchronize do
|
62
|
+
manager.mark(message)
|
63
|
+
manager.mark_until(message) if coordinator.finished?
|
64
|
+
end
|
65
|
+
|
66
|
+
manager.markable? ? super(manager.markable) : revoked?
|
67
|
+
end
|
68
|
+
|
31
69
|
# @return [Boolean] is the virtual processing collapsed in the context of given
|
32
70
|
# consumer.
|
33
71
|
def collapsed?
|
@@ -45,6 +83,19 @@ module Karafka
|
|
45
83
|
def failing?
|
46
84
|
coordinator.failure?
|
47
85
|
end
|
86
|
+
|
87
|
+
private
|
88
|
+
|
89
|
+
# Prior to adding work to the queue, registers all the messages offsets into the
|
90
|
+
# virtual offset group.
|
91
|
+
#
|
92
|
+
# @note This can be done without the mutex, because it happens from the same thread
|
93
|
+
# for all the work (listener thread)
|
94
|
+
def handle_before_enqueue
|
95
|
+
coordinator.virtual_offset_manager.register(
|
96
|
+
messages.map(&:offset)
|
97
|
+
)
|
98
|
+
end
|
48
99
|
end
|
49
100
|
end
|
50
101
|
end
|
@@ -0,0 +1,147 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
# Manager that keeps track of our offsets with the virtualization layer that are local
|
18
|
+
# to given partition assignment. It allows for easier offset management for virtual
|
19
|
+
# virtual partition cases as it provides us ability to mark as consumed and move the
|
20
|
+
# real offset behind as expected.
|
21
|
+
#
|
22
|
+
# @note We still use the regular coordinator "real" offset management as we want to have
|
23
|
+
# them as separated as possible because the real seek offset management is also used for
|
24
|
+
# pausing, filtering and others and should not be impacted by the virtual one
|
25
|
+
#
|
26
|
+
# @note This manager is **not** thread-safe by itself. It should operate from coordinator
|
27
|
+
# locked locations.
|
28
|
+
class VirtualOffsetManager
|
29
|
+
attr_reader :groups
|
30
|
+
|
31
|
+
# @param topic [String]
|
32
|
+
# @param partition [Integer]
|
33
|
+
#
|
34
|
+
# @note We need topic and partition because we use a seek message (virtual) for real offset
|
35
|
+
# management. We could keep real message reference but this can be memory consuming
|
36
|
+
# and not worth it.
|
37
|
+
def initialize(topic, partition)
|
38
|
+
@topic = topic
|
39
|
+
@partition = partition
|
40
|
+
@groups = []
|
41
|
+
@marked = {}
|
42
|
+
@real_offset = -1
|
43
|
+
end
|
44
|
+
|
45
|
+
# Clears the manager for a next collective operation
|
46
|
+
def clear
|
47
|
+
@groups.clear
|
48
|
+
@marked = {}
|
49
|
+
@real_offset = -1
|
50
|
+
end
|
51
|
+
|
52
|
+
# Registers an offset group coming from one virtual consumer. In order to move the real
|
53
|
+
# underlying offset accordingly, we need to make sure to track the virtual consumers
|
54
|
+
# offsets groups independently and only materialize the end result.
|
55
|
+
#
|
56
|
+
# @param offsets_group [Array<Integer>] offsets from one virtual consumer
|
57
|
+
def register(offsets_group)
|
58
|
+
@groups << offsets_group
|
59
|
+
|
60
|
+
offsets_group.each { |offset| @marked[offset] = false }
|
61
|
+
end
|
62
|
+
|
63
|
+
# Marks given message as marked (virtually consumed).
|
64
|
+
# We mark given message offset and other earlier offsets from the same group as done
|
65
|
+
# and we can refresh our real offset representation based on that as it might have changed
|
66
|
+
# to a newer real offset.
|
67
|
+
# @param message [Karafka::Messages::Message] message coming from VP we want to mark
|
68
|
+
def mark(message)
|
69
|
+
offset = message.offset
|
70
|
+
|
71
|
+
group = @groups.find { |reg_group| reg_group.include?(offset) }
|
72
|
+
|
73
|
+
# This case can happen when someone uses MoM and wants to mark message from a previous
|
74
|
+
# batch as consumed. We can add it, since the real offset refresh will point to it
|
75
|
+
unless group
|
76
|
+
group = [offset]
|
77
|
+
@groups << group
|
78
|
+
end
|
79
|
+
|
80
|
+
position = group.index(offset)
|
81
|
+
|
82
|
+
# Mark all previous messages from the same group also as virtually consumed
|
83
|
+
group[0..position].each do |markable_offset|
|
84
|
+
@marked[markable_offset] = true
|
85
|
+
end
|
86
|
+
|
87
|
+
# Recompute the real offset representation
|
88
|
+
materialize_real_offset
|
89
|
+
end
|
90
|
+
|
91
|
+
# Mark all from all groups including the `message`.
|
92
|
+
# Useful when operating in a collapsed state for marking
|
93
|
+
# @param message [Karafka::Messages::Message]
|
94
|
+
def mark_until(message)
|
95
|
+
mark(message)
|
96
|
+
|
97
|
+
@groups.each do |group|
|
98
|
+
group.each do |offset|
|
99
|
+
next if offset > message.offset
|
100
|
+
|
101
|
+
@marked[offset] = true
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
materialize_real_offset
|
106
|
+
end
|
107
|
+
|
108
|
+
# @return [Array<Integer>] Offsets of messages already marked as consumed virtually
|
109
|
+
def marked
|
110
|
+
@marked.select { |_, status| status }.map(&:first).sort
|
111
|
+
end
|
112
|
+
|
113
|
+
# Is there a real offset we can mark as consumed
|
114
|
+
# @return [Boolean]
|
115
|
+
def markable?
|
116
|
+
!@real_offset.negative?
|
117
|
+
end
|
118
|
+
|
119
|
+
# @return [Messages::Seek] markable message for real offset marking
|
120
|
+
def markable
|
121
|
+
raise Errors::InvalidRealOffsetUsage unless markable?
|
122
|
+
|
123
|
+
Messages::Seek.new(
|
124
|
+
@topic,
|
125
|
+
@partition,
|
126
|
+
@real_offset
|
127
|
+
)
|
128
|
+
end
|
129
|
+
|
130
|
+
private
|
131
|
+
|
132
|
+
# Recomputes the biggest possible real offset we can have.
|
133
|
+
# It picks the the biggest offset that has uninterrupted stream of virtually marked as
|
134
|
+
# consumed because this will be the collective offset.
|
135
|
+
def materialize_real_offset
|
136
|
+
@marked.to_a.sort_by(&:first).each do |offset, marked|
|
137
|
+
break unless marked
|
138
|
+
|
139
|
+
@real_offset = offset
|
140
|
+
end
|
141
|
+
|
142
|
+
@real_offset = (@marked.keys.min - 1) if @real_offset.negative?
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
@@ -44,23 +44,6 @@ module Karafka
|
|
44
44
|
|
45
45
|
[[%i[virtual_partitions partitioner], :respond_to_call]]
|
46
46
|
end
|
47
|
-
|
48
|
-
# Make sure that manual offset management is not used together with Virtual Partitions
|
49
|
-
# This would not make any sense as there would be edge cases related to skipping
|
50
|
-
# messages even if there were errors.
|
51
|
-
virtual do |data, errors|
|
52
|
-
next unless errors.empty?
|
53
|
-
|
54
|
-
virtual_partitions = data[:virtual_partitions]
|
55
|
-
manual_offset_management = data[:manual_offset_management]
|
56
|
-
active_job = data[:active_job]
|
57
|
-
|
58
|
-
next unless virtual_partitions[:active]
|
59
|
-
next unless manual_offset_management[:active]
|
60
|
-
next if active_job[:active]
|
61
|
-
|
62
|
-
[[%i[manual_offset_management], :not_with_virtual_partitions]]
|
63
|
-
end
|
64
47
|
end
|
65
48
|
end
|
66
49
|
end
|
@@ -26,6 +26,7 @@ module Karafka
|
|
26
26
|
def mark_as_consumed(message)
|
27
27
|
# Ignore earlier offsets than the one we already committed
|
28
28
|
return true if coordinator.seek_offset > message.offset
|
29
|
+
return false if revoked?
|
29
30
|
|
30
31
|
unless client.mark_as_consumed(message)
|
31
32
|
coordinator.revoke
|
@@ -46,6 +47,7 @@ module Karafka
|
|
46
47
|
def mark_as_consumed!(message)
|
47
48
|
# Ignore earlier offsets than the one we already committed
|
48
49
|
return true if coordinator.seek_offset > message.offset
|
50
|
+
return false if revoked?
|
49
51
|
|
50
52
|
unless client.mark_as_consumed!(message)
|
51
53
|
coordinator.revoke
|