karafka 2.2.14 → 2.3.0.alpha1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +38 -12
- data/.ruby-version +1 -1
- data/CHANGELOG.md +23 -0
- data/Gemfile.lock +12 -12
- data/README.md +0 -2
- data/SECURITY.md +23 -0
- data/config/locales/errors.yml +7 -1
- data/config/locales/pro_errors.yml +22 -0
- data/docker-compose.yml +1 -1
- data/karafka.gemspec +2 -2
- data/lib/karafka/admin/acl.rb +287 -0
- data/lib/karafka/admin.rb +9 -13
- data/lib/karafka/app.rb +5 -3
- data/lib/karafka/base_consumer.rb +9 -1
- data/lib/karafka/cli/base.rb +1 -1
- data/lib/karafka/connection/client.rb +83 -76
- data/lib/karafka/connection/conductor.rb +28 -0
- data/lib/karafka/connection/listener.rb +159 -42
- data/lib/karafka/connection/listeners_batch.rb +5 -11
- data/lib/karafka/connection/manager.rb +72 -0
- data/lib/karafka/connection/messages_buffer.rb +12 -0
- data/lib/karafka/connection/proxy.rb +17 -0
- data/lib/karafka/connection/status.rb +75 -0
- data/lib/karafka/contracts/config.rb +14 -10
- data/lib/karafka/contracts/consumer_group.rb +9 -1
- data/lib/karafka/contracts/topic.rb +3 -1
- data/lib/karafka/errors.rb +13 -0
- data/lib/karafka/instrumentation/logger_listener.rb +3 -0
- data/lib/karafka/instrumentation/notifications.rb +13 -5
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +31 -28
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +20 -1
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +15 -12
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +39 -36
- data/lib/karafka/pro/base_consumer.rb +47 -0
- data/lib/karafka/pro/connection/manager.rb +300 -0
- data/lib/karafka/pro/connection/multiplexing/listener.rb +40 -0
- data/lib/karafka/pro/iterator/tpl_builder.rb +1 -1
- data/lib/karafka/pro/iterator.rb +1 -6
- data/lib/karafka/pro/loader.rb +14 -0
- data/lib/karafka/pro/processing/coordinator.rb +2 -1
- data/lib/karafka/pro/processing/executor.rb +37 -0
- data/lib/karafka/pro/processing/expansions_selector.rb +32 -0
- data/lib/karafka/pro/processing/jobs/periodic.rb +41 -0
- data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb +32 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +14 -3
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +44 -0
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +131 -0
- data/lib/karafka/pro/processing/offset_metadata/listener.rb +46 -0
- data/lib/karafka/pro/processing/schedulers/base.rb +39 -23
- data/lib/karafka/pro/processing/schedulers/default.rb +12 -14
- data/lib/karafka/pro/processing/strategies/default.rb +134 -1
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +35 -0
- data/lib/karafka/pro/processing/strategies/vp/default.rb +59 -25
- data/lib/karafka/pro/processing/virtual_offset_manager.rb +41 -11
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +2 -0
- data/lib/karafka/pro/routing/features/multiplexing/config.rb +38 -0
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +114 -0
- data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb +42 -0
- data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +38 -0
- data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb +42 -0
- data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +40 -0
- data/lib/karafka/pro/routing/features/multiplexing.rb +59 -0
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +32 -0
- data/lib/karafka/pro/routing/features/non_blocking_job.rb +37 -0
- data/lib/karafka/pro/routing/features/offset_metadata/config.rb +33 -0
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +42 -0
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +65 -0
- data/lib/karafka/pro/routing/features/offset_metadata.rb +40 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +4 -0
- data/lib/karafka/pro/routing/features/patterns/detector.rb +18 -10
- data/lib/karafka/pro/routing/features/periodic_job/config.rb +37 -0
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +44 -0
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +94 -0
- data/lib/karafka/pro/routing/features/periodic_job.rb +27 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +1 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +1 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +7 -2
- data/lib/karafka/process.rb +5 -3
- data/lib/karafka/processing/coordinator.rb +5 -1
- data/lib/karafka/processing/executor.rb +16 -10
- data/lib/karafka/processing/executors_buffer.rb +19 -4
- data/lib/karafka/processing/schedulers/default.rb +3 -2
- data/lib/karafka/processing/strategies/default.rb +6 -0
- data/lib/karafka/processing/strategies/dlq.rb +36 -0
- data/lib/karafka/routing/builder.rb +12 -2
- data/lib/karafka/routing/consumer_group.rb +5 -5
- data/lib/karafka/routing/features/base.rb +44 -8
- data/lib/karafka/routing/features/dead_letter_queue/config.rb +6 -1
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +1 -0
- data/lib/karafka/routing/features/dead_letter_queue/topic.rb +9 -2
- data/lib/karafka/routing/subscription_group.rb +2 -2
- data/lib/karafka/routing/subscription_groups_builder.rb +11 -2
- data/lib/karafka/routing/topic.rb +8 -10
- data/lib/karafka/runner.rb +13 -3
- data/lib/karafka/server.rb +5 -9
- data/lib/karafka/setup/config.rb +17 -0
- data/lib/karafka/status.rb +23 -14
- data/lib/karafka/templates/karafka.rb.erb +7 -0
- data/lib/karafka/time_trackers/partition_usage.rb +56 -0
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +42 -10
- metadata.gz.sig +0 -0
- data/lib/karafka/connection/consumer_group_coordinator.rb +0 -48
@@ -31,7 +31,9 @@ module Karafka
|
|
31
31
|
class Default < Base
|
32
32
|
# Schedules jobs in the LJF order for consumption
|
33
33
|
#
|
34
|
-
# @param jobs_array
|
34
|
+
# @param jobs_array
|
35
|
+
# [Array<Karafka::Processing::Jobs::Consume, Processing::Jobs::ConsumeNonBlocking>]
|
36
|
+
# jobs for scheduling
|
35
37
|
def on_schedule_consumption(jobs_array)
|
36
38
|
perf_tracker = Instrumentation::PerformanceTracker.instance
|
37
39
|
|
@@ -53,23 +55,19 @@ module Karafka
|
|
53
55
|
end
|
54
56
|
end
|
55
57
|
|
56
|
-
# Schedules jobs in
|
57
|
-
#
|
58
|
-
|
59
|
-
def on_schedule_revocation(jobs_array)
|
58
|
+
# Schedules any jobs provided in a fifo order
|
59
|
+
# @param jobs_array [Array<Karafka::Processing::Jobs::Base>]
|
60
|
+
def schedule_fifo(jobs_array)
|
60
61
|
jobs_array.each do |job|
|
61
62
|
@queue << job
|
62
63
|
end
|
63
64
|
end
|
64
65
|
|
65
|
-
#
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
@queue << job
|
71
|
-
end
|
72
|
-
end
|
66
|
+
# By default all non-consumption work is scheduled in a fifo order
|
67
|
+
alias on_schedule_revocation schedule_fifo
|
68
|
+
alias on_schedule_shutdown schedule_fifo
|
69
|
+
alias on_schedule_idle schedule_fifo
|
70
|
+
alias on_schedule_periodic schedule_fifo
|
73
71
|
|
74
72
|
# This scheduler does not have anything to manage as it is a pass through and has no
|
75
73
|
# state
|
@@ -87,7 +85,7 @@ module Karafka
|
|
87
85
|
private
|
88
86
|
|
89
87
|
# @param perf_tracker [PerformanceTracker]
|
90
|
-
# @param job [Karafka::Processing::Jobs::
|
88
|
+
# @param job [Karafka::Processing::Jobs::Consume] job we will be processing
|
91
89
|
# @return [Numeric] estimated cost of processing this job
|
92
90
|
def processing_cost(perf_tracker, job)
|
93
91
|
if job.is_a?(::Karafka::Processing::Jobs::Consume)
|
@@ -27,6 +27,124 @@ module Karafka
|
|
27
27
|
# Apply strategy for a non-feature based flow
|
28
28
|
FEATURES = %i[].freeze
|
29
29
|
|
30
|
+
# Marks message as consumed in an async way.
|
31
|
+
#
|
32
|
+
# @param message [Messages::Message] last successfully processed message.
|
33
|
+
# @param offset_metadata [String, nil] offset metadata string or nil if nothing
|
34
|
+
# @return [Boolean] true if we were able to mark the offset, false otherwise.
|
35
|
+
# False indicates that we were not able and that we have lost the partition.
|
36
|
+
#
|
37
|
+
# @note We keep track of this offset in case we would mark as consumed and got error when
|
38
|
+
# processing another message. In case like this we do not pause on the message we've
|
39
|
+
# already processed but rather at the next one. This applies to both sync and async
|
40
|
+
# versions of this method.
|
41
|
+
def mark_as_consumed(message, offset_metadata = nil)
|
42
|
+
if @_in_transaction
|
43
|
+
mark_in_transaction(message, offset_metadata, true)
|
44
|
+
else
|
45
|
+
# seek offset can be nil only in case `#seek` was invoked with offset reset request
|
46
|
+
# In case like this we ignore marking
|
47
|
+
return true if coordinator.seek_offset.nil?
|
48
|
+
# Ignore earlier offsets than the one we already committed
|
49
|
+
return true if coordinator.seek_offset > message.offset
|
50
|
+
return false if revoked?
|
51
|
+
return revoked? unless client.mark_as_consumed(message, offset_metadata)
|
52
|
+
|
53
|
+
coordinator.seek_offset = message.offset + 1
|
54
|
+
end
|
55
|
+
|
56
|
+
true
|
57
|
+
end
|
58
|
+
|
59
|
+
# Marks message as consumed in a sync way.
|
60
|
+
#
|
61
|
+
# @param message [Messages::Message] last successfully processed message.
|
62
|
+
# @param offset_metadata [String, nil] offset metadata string or nil if nothing
|
63
|
+
# @return [Boolean] true if we were able to mark the offset, false otherwise.
|
64
|
+
# False indicates that we were not able and that we have lost the partition.
|
65
|
+
def mark_as_consumed!(message, offset_metadata = nil)
|
66
|
+
if @_in_transaction
|
67
|
+
mark_in_transaction(message, offset_metadata, false)
|
68
|
+
else
|
69
|
+
# seek offset can be nil only in case `#seek` was invoked with offset reset request
|
70
|
+
# In case like this we ignore marking
|
71
|
+
return true if coordinator.seek_offset.nil?
|
72
|
+
# Ignore earlier offsets than the one we already committed
|
73
|
+
return true if coordinator.seek_offset > message.offset
|
74
|
+
return false if revoked?
|
75
|
+
|
76
|
+
return revoked? unless client.mark_as_consumed!(message, offset_metadata)
|
77
|
+
|
78
|
+
coordinator.seek_offset = message.offset + 1
|
79
|
+
end
|
80
|
+
|
81
|
+
true
|
82
|
+
end
|
83
|
+
|
84
|
+
# Starts producer transaction, saves the transaction context for transactional marking
|
85
|
+
# and runs user code in this context
|
86
|
+
#
|
87
|
+
# Transactions on a consumer level differ from those initiated by the producer as they
|
88
|
+
# allow to mark offsets inside of the transaction. If the transaction is initialized
|
89
|
+
# only from the consumer, the offset will be stored in a regular fashion.
|
90
|
+
#
|
91
|
+
# @param block [Proc] code that we want to run in a transaction
|
92
|
+
def transaction(&block)
|
93
|
+
transaction_started = false
|
94
|
+
|
95
|
+
# Prevent from nested transactions. It would not make any sense
|
96
|
+
raise Errors::TransactionAlreadyInitializedError if @_in_transaction
|
97
|
+
|
98
|
+
transaction_started = true
|
99
|
+
@_transaction_marked = []
|
100
|
+
@_in_transaction = true
|
101
|
+
|
102
|
+
producer.transaction(&block)
|
103
|
+
|
104
|
+
@_in_transaction = false
|
105
|
+
|
106
|
+
# This offset is already stored in transaction but we set it here anyhow because we
|
107
|
+
# want to make sure our internal in-memory state is aligned with the transaction
|
108
|
+
#
|
109
|
+
# @note We never need to use the blocking `#mark_as_consumed!` here because the offset
|
110
|
+
# anyhow was already stored during the transaction
|
111
|
+
#
|
112
|
+
# @note In theory we could only keep reference to the most recent marking and reject
|
113
|
+
# others. We however do not do it for two reasons:
|
114
|
+
# - User may have non standard flow relying on some alternative order and we want to
|
115
|
+
# mimic this
|
116
|
+
# - Complex strategies like VPs can use this in VPs to mark in parallel without
|
117
|
+
# having to redefine the transactional flow completely
|
118
|
+
@_transaction_marked.each do |marking|
|
119
|
+
marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
|
120
|
+
end
|
121
|
+
ensure
|
122
|
+
if transaction_started
|
123
|
+
@_transaction_marked.clear
|
124
|
+
@_in_transaction = false
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
# Stores the next offset for processing inside of the transaction and stores it in a
|
129
|
+
# local accumulator for post-transaction status update
|
130
|
+
#
|
131
|
+
# @param message [Messages::Message] message we want to commit inside of a transaction
|
132
|
+
# @param offset_metadata [String, nil] offset metadata or nil if none
|
133
|
+
# @param async [Boolean] should we mark in async or sync way (applicable only to post
|
134
|
+
# transaction state synchronization usage as within transaction it is always sync)
|
135
|
+
def mark_in_transaction(message, offset_metadata, async)
|
136
|
+
raise Errors::TransactionRequiredError unless @_in_transaction
|
137
|
+
|
138
|
+
producer.transaction_mark_as_consumed(
|
139
|
+
client,
|
140
|
+
message,
|
141
|
+
offset_metadata
|
142
|
+
)
|
143
|
+
|
144
|
+
@_transaction_marked ||= []
|
145
|
+
@_transaction_marked << [message, offset_metadata, async]
|
146
|
+
end
|
147
|
+
|
30
148
|
# No actions needed for the standard flow here
|
31
149
|
def handle_before_schedule_consume
|
32
150
|
Karafka.monitor.instrument('consumer.before_schedule_consume', caller: self)
|
@@ -87,7 +205,7 @@ module Karafka
|
|
87
205
|
end
|
88
206
|
end
|
89
207
|
|
90
|
-
# Standard
|
208
|
+
# Standard flow for revocation
|
91
209
|
def handle_revoked
|
92
210
|
coordinator.on_revoked do
|
93
211
|
resume
|
@@ -100,6 +218,21 @@ module Karafka
|
|
100
218
|
revoked
|
101
219
|
end
|
102
220
|
end
|
221
|
+
|
222
|
+
# No action needed for the tick standard flow
|
223
|
+
def handle_before_schedule_tick
|
224
|
+
Karafka.monitor.instrument('consumer.before_schedule_tick', caller: self)
|
225
|
+
|
226
|
+
nil
|
227
|
+
end
|
228
|
+
|
229
|
+
# Runs the consumer `#tick` method with reporting
|
230
|
+
def handle_tick
|
231
|
+
Karafka.monitor.instrument('consumer.tick', caller: self)
|
232
|
+
Karafka.monitor.instrument('consumer.ticked', caller: self) do
|
233
|
+
tick
|
234
|
+
end
|
235
|
+
end
|
103
236
|
end
|
104
237
|
end
|
105
238
|
end
|
@@ -26,6 +26,41 @@ module Karafka
|
|
26
26
|
dead_letter_queue
|
27
27
|
].freeze
|
28
28
|
|
29
|
+
# Override of the standard `#mark_as_consumed` in order to handle the pause tracker
|
30
|
+
# reset in case DLQ is marked as fully independent. When DLQ is marked independent,
|
31
|
+
# any offset marking causes the pause count tracker to reset. This is useful when
|
32
|
+
# the error is not due to the collective batch operations state but due to intermediate
|
33
|
+
# "crawling" errors that move with it
|
34
|
+
#
|
35
|
+
# @see `Strategies::Default#mark_as_consumed` for more details
|
36
|
+
# @param message [Messages::Message]
|
37
|
+
# @param offset_metadata [String, nil]
|
38
|
+
def mark_as_consumed(message, offset_metadata = nil)
|
39
|
+
return super unless retrying?
|
40
|
+
return super unless topic.dead_letter_queue.independent?
|
41
|
+
return false unless super
|
42
|
+
|
43
|
+
coordinator.pause_tracker.reset
|
44
|
+
|
45
|
+
true
|
46
|
+
end
|
47
|
+
|
48
|
+
# Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
|
49
|
+
# DLQ was configured with the `independent` flag.
|
50
|
+
#
|
51
|
+
# @see `Strategies::Default#mark_as_consumed!` for more details
|
52
|
+
# @param message [Messages::Message]
|
53
|
+
# @param offset_metadata [String, nil]
|
54
|
+
def mark_as_consumed!(message, offset_metadata = nil)
|
55
|
+
return super unless retrying?
|
56
|
+
return super unless topic.dead_letter_queue.independent?
|
57
|
+
return false unless super
|
58
|
+
|
59
|
+
coordinator.pause_tracker.reset
|
60
|
+
|
61
|
+
true
|
62
|
+
end
|
63
|
+
|
29
64
|
# When we encounter non-recoverable message, we skip it and go on with our lives
|
30
65
|
def handle_after_consume
|
31
66
|
coordinator.on_finished do |last_group_message|
|
@@ -29,40 +29,74 @@ module Karafka
|
|
29
29
|
].freeze
|
30
30
|
|
31
31
|
# @param message [Karafka::Messages::Message] marks message as consumed
|
32
|
+
# @param offset_metadata [String, nil]
|
32
33
|
# @note This virtual offset management uses a regular default marking API underneath.
|
33
34
|
# We do not alter the "real" marking API, as VPs are just one of many cases we want
|
34
35
|
# to support and we do not want to impact them with collective offsets management
|
35
|
-
def mark_as_consumed(message)
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
manager.
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
36
|
+
def mark_as_consumed(message, offset_metadata = nil)
|
37
|
+
if @_in_transaction && !collapsed?
|
38
|
+
mark_in_transaction(message, offset_metadata, true)
|
39
|
+
elsif collapsed?
|
40
|
+
super
|
41
|
+
else
|
42
|
+
manager = coordinator.virtual_offset_manager
|
43
|
+
|
44
|
+
coordinator.synchronize do
|
45
|
+
manager.mark(message, offset_metadata)
|
46
|
+
# If this is last marking on a finished flow, we can use the original
|
47
|
+
# last message and in order to do so, we need to mark all previous messages as
|
48
|
+
# consumed as otherwise the computed offset could be different
|
49
|
+
# We mark until our offset just in case of a DLQ flow or similar, where we do not
|
50
|
+
# want to mark all but until the expected location
|
51
|
+
manager.mark_until(message, offset_metadata) if coordinator.finished?
|
52
|
+
|
53
|
+
return revoked? unless manager.markable?
|
54
|
+
|
55
|
+
manager.markable? ? super(*manager.markable) : revoked?
|
56
|
+
end
|
52
57
|
end
|
53
58
|
end
|
54
59
|
|
55
60
|
# @param message [Karafka::Messages::Message] blocking marks message as consumed
|
56
|
-
|
57
|
-
|
61
|
+
# @param offset_metadata [String, nil]
|
62
|
+
def mark_as_consumed!(message, offset_metadata = nil)
|
63
|
+
if @_in_transaction && !collapsed?
|
64
|
+
mark_in_transaction(message, offset_metadata, false)
|
65
|
+
elsif collapsed?
|
66
|
+
super
|
67
|
+
else
|
68
|
+
manager = coordinator.virtual_offset_manager
|
69
|
+
|
70
|
+
coordinator.synchronize do
|
71
|
+
manager.mark(message, offset_metadata)
|
72
|
+
manager.mark_until(message, offset_metadata) if coordinator.finished?
|
73
|
+
manager.markable? ? super(*manager.markable) : revoked?
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
58
77
|
|
59
|
-
|
78
|
+
# Stores the next offset for processing inside of the transaction when collapsed and
|
79
|
+
# accumulates marking as consumed in the local buffer.
|
80
|
+
#
|
81
|
+
# Due to nature of VPs we cannot provide full EOS support but we can simulate it,
|
82
|
+
# making sure that no offset are stored unless transaction is finished. We do it by
|
83
|
+
# accumulating the post-transaction marking requests and after it is successfully done
|
84
|
+
# we mark each as consumed. This effectively on errors "rollbacks" the state and
|
85
|
+
# prevents offset storage.
|
86
|
+
#
|
87
|
+
# Since the EOS here is "weak", we do not have to worry about the race-conditions and
|
88
|
+
# we do not have to have any mutexes.
|
89
|
+
#
|
90
|
+
# @param message [Messages::Message] message we want to commit inside of a transaction
|
91
|
+
# @param offset_metadata [String, nil] offset metadata or nil if none
|
92
|
+
# @param async [Boolean] should we mark in async or sync way (applicable only to post
|
93
|
+
# transaction state synchronization usage as within transaction it is always sync)
|
94
|
+
def mark_in_transaction(message, offset_metadata, async)
|
95
|
+
raise Errors::TransactionRequiredError unless @_in_transaction
|
60
96
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
manager.markable? ? super(manager.markable) : revoked?
|
65
|
-
end
|
97
|
+
return super if collapsed?
|
98
|
+
|
99
|
+
@_transaction_marked << [message, offset_metadata, async]
|
66
100
|
end
|
67
101
|
|
68
102
|
# @return [Boolean] is the virtual processing collapsed in the context of given
|
@@ -30,22 +30,29 @@ module Karafka
|
|
30
30
|
|
31
31
|
# @param topic [String]
|
32
32
|
# @param partition [Integer]
|
33
|
+
# @param offset_metadata_strategy [Symbol] what metadata should we select. That is, should
|
34
|
+
# we use the most recent or one picked from the offset that is going to be committed
|
33
35
|
#
|
34
36
|
# @note We need topic and partition because we use a seek message (virtual) for real offset
|
35
37
|
# management. We could keep real message reference but this can be memory consuming
|
36
38
|
# and not worth it.
|
37
|
-
def initialize(topic, partition)
|
39
|
+
def initialize(topic, partition, offset_metadata_strategy)
|
38
40
|
@topic = topic
|
39
41
|
@partition = partition
|
40
42
|
@groups = []
|
41
43
|
@marked = {}
|
44
|
+
@offsets_metadata = {}
|
42
45
|
@real_offset = -1
|
46
|
+
@offset_metadata_strategy = offset_metadata_strategy
|
47
|
+
@current_offset_metadata = nil
|
43
48
|
end
|
44
49
|
|
45
50
|
# Clears the manager for a next collective operation
|
46
51
|
def clear
|
47
52
|
@groups.clear
|
48
|
-
@
|
53
|
+
@offsets_metadata.clear
|
54
|
+
@current_offset_metadata = nil
|
55
|
+
@marked.clear
|
49
56
|
@real_offset = -1
|
50
57
|
end
|
51
58
|
|
@@ -65,9 +72,14 @@ module Karafka
|
|
65
72
|
# and we can refresh our real offset representation based on that as it might have changed
|
66
73
|
# to a newer real offset.
|
67
74
|
# @param message [Karafka::Messages::Message] message coming from VP we want to mark
|
68
|
-
|
75
|
+
# @param offset_metadata [String, nil] offset metadata. `nil` if none
|
76
|
+
def mark(message, offset_metadata)
|
69
77
|
offset = message.offset
|
70
78
|
|
79
|
+
# Store metadata when we materialize the most stable offset
|
80
|
+
@offsets_metadata[offset] = offset_metadata
|
81
|
+
@current_offset_metadata = offset_metadata
|
82
|
+
|
71
83
|
group = @groups.find { |reg_group| reg_group.include?(offset) }
|
72
84
|
|
73
85
|
# This case can happen when someone uses MoM and wants to mark message from a previous
|
@@ -81,6 +93,9 @@ module Karafka
|
|
81
93
|
|
82
94
|
# Mark all previous messages from the same group also as virtually consumed
|
83
95
|
group[0..position].each do |markable_offset|
|
96
|
+
# Set previous messages metadata offset as the offset of higher one for overwrites
|
97
|
+
# unless a different metadata were set explicitely
|
98
|
+
@offsets_metadata[markable_offset] ||= offset_metadata
|
84
99
|
@marked[markable_offset] = true
|
85
100
|
end
|
86
101
|
|
@@ -91,13 +106,15 @@ module Karafka
|
|
91
106
|
# Mark all from all groups including the `message`.
|
92
107
|
# Useful when operating in a collapsed state for marking
|
93
108
|
# @param message [Karafka::Messages::Message]
|
94
|
-
|
95
|
-
|
109
|
+
# @param offset_metadata [String, nil]
|
110
|
+
def mark_until(message, offset_metadata)
|
111
|
+
mark(message, offset_metadata)
|
96
112
|
|
97
113
|
@groups.each do |group|
|
98
114
|
group.each do |offset|
|
99
115
|
next if offset > message.offset
|
100
116
|
|
117
|
+
@offsets_metadata[offset] = offset_metadata
|
101
118
|
@marked[offset] = true
|
102
119
|
end
|
103
120
|
end
|
@@ -116,15 +133,28 @@ module Karafka
|
|
116
133
|
!@real_offset.negative?
|
117
134
|
end
|
118
135
|
|
119
|
-
# @return [Messages::Seek] markable message for real offset marking
|
136
|
+
# @return [Array<Messages::Seek, String>] markable message for real offset marking and
|
137
|
+
# its associated metadata
|
120
138
|
def markable
|
121
139
|
raise Errors::InvalidRealOffsetUsageError unless markable?
|
122
140
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
141
|
+
offset_metadata = case @offset_metadata_strategy
|
142
|
+
when :exact
|
143
|
+
@offsets_metadata.fetch(@real_offset)
|
144
|
+
when :current
|
145
|
+
@current_offset_metadata
|
146
|
+
else
|
147
|
+
raise Errors::UnsupportedCaseError, @offset_metadata_strategy
|
148
|
+
end
|
149
|
+
|
150
|
+
[
|
151
|
+
Messages::Seek.new(
|
152
|
+
@topic,
|
153
|
+
@partition,
|
154
|
+
@real_offset
|
155
|
+
),
|
156
|
+
offset_metadata
|
157
|
+
]
|
128
158
|
end
|
129
159
|
|
130
160
|
private
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Routing
|
17
|
+
module Features
|
18
|
+
class Multiplexing < Base
|
19
|
+
# Multiplexing configuration
|
20
|
+
Config = Struct.new(
|
21
|
+
:active,
|
22
|
+
:min,
|
23
|
+
:max,
|
24
|
+
:boot,
|
25
|
+
keyword_init: true
|
26
|
+
) do
|
27
|
+
alias_method :active?, :active
|
28
|
+
|
29
|
+
# @return [Boolean] true if we are allowed to upscale and downscale
|
30
|
+
def dynamic?
|
31
|
+
min < max
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Routing
|
17
|
+
module Features
|
18
|
+
class Multiplexing < Base
|
19
|
+
# Namespace for multiplexing feature contracts
|
20
|
+
module Contracts
|
21
|
+
# Validates the subscription group multiplexing setup
|
22
|
+
# We validate it on the topic level as subscription groups are not built during the
|
23
|
+
# routing as they are pre-run dynamically built.
|
24
|
+
#
|
25
|
+
# multiplexing attributes are optional since multiplexing may not be enabled
|
26
|
+
class Topic < Karafka::Contracts::Base
|
27
|
+
configure do |config|
|
28
|
+
config.error_messages = YAML.safe_load(
|
29
|
+
File.read(
|
30
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
31
|
+
)
|
32
|
+
).fetch('en').fetch('validations').fetch('topic')
|
33
|
+
end
|
34
|
+
|
35
|
+
nested(:subscription_group_details) do
|
36
|
+
optional(:multiplexing_min) { |val| val.is_a?(Integer) && val >= 1 }
|
37
|
+
optional(:multiplexing_max) { |val| val.is_a?(Integer) && val >= 1 }
|
38
|
+
optional(:multiplexing_boot) { |val| val.is_a?(Integer) && val >= 1 }
|
39
|
+
end
|
40
|
+
|
41
|
+
# Makes sure min is not more than max
|
42
|
+
virtual do |data, errors|
|
43
|
+
next unless errors.empty?
|
44
|
+
next unless min(data)
|
45
|
+
next unless max(data)
|
46
|
+
|
47
|
+
min = min(data)
|
48
|
+
max = max(data)
|
49
|
+
|
50
|
+
next if min <= max
|
51
|
+
|
52
|
+
[[%w[subscription_group_details], :multiplexing_min_max_mismatch]]
|
53
|
+
end
|
54
|
+
|
55
|
+
# Makes sure, that boot is between min and max
|
56
|
+
virtual do |data, errors|
|
57
|
+
next unless errors.empty?
|
58
|
+
next unless min(data)
|
59
|
+
next unless max(data)
|
60
|
+
next unless boot(data)
|
61
|
+
|
62
|
+
min = min(data)
|
63
|
+
max = max(data)
|
64
|
+
boot = boot(data)
|
65
|
+
|
66
|
+
next if boot >= min && boot <= max
|
67
|
+
|
68
|
+
[[%w[subscription_group_details], :multiplexing_boot_mismatch]]
|
69
|
+
end
|
70
|
+
|
71
|
+
# Makes sure, that boot is equal to min and max when not in dynamic mode
|
72
|
+
virtual do |data, errors|
|
73
|
+
next unless errors.empty?
|
74
|
+
next unless min(data)
|
75
|
+
next unless max(data)
|
76
|
+
next unless boot(data)
|
77
|
+
|
78
|
+
min = min(data)
|
79
|
+
max = max(data)
|
80
|
+
boot = boot(data)
|
81
|
+
|
82
|
+
# In dynamic mode there are other rules to check boot
|
83
|
+
next if min != max
|
84
|
+
next if boot == min
|
85
|
+
|
86
|
+
[[%w[subscription_group_details], :multiplexing_boot_not_dynamic]]
|
87
|
+
end
|
88
|
+
|
89
|
+
class << self
|
90
|
+
# @param data [Hash] topic details
|
91
|
+
# @return [Integer, false] min or false if missing
|
92
|
+
def min(data)
|
93
|
+
data[:subscription_group_details].fetch(:multiplexing_min, false)
|
94
|
+
end
|
95
|
+
|
96
|
+
# @param data [Hash] topic details
|
97
|
+
# @return [Integer, false] max or false if missing
|
98
|
+
def max(data)
|
99
|
+
data[:subscription_group_details].fetch(:multiplexing_max, false)
|
100
|
+
end
|
101
|
+
|
102
|
+
# @param data [Hash] topic details
|
103
|
+
# @return [Integer, false] boot or false if missing
|
104
|
+
def boot(data)
|
105
|
+
data[:subscription_group_details].fetch(:multiplexing_boot, false)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Routing
|
17
|
+
module Features
|
18
|
+
class Multiplexing < Base
|
19
|
+
# Patches to Karafka OSS
|
20
|
+
module Patches
|
21
|
+
# Contracts patches
|
22
|
+
module Contracts
|
23
|
+
# Consumer group contract patches
|
24
|
+
module ConsumerGroup
|
25
|
+
# Redefines the setup allowing for multiple sgs as long as with different names
|
26
|
+
#
|
27
|
+
# @param topic [Hash] topic config hash
|
28
|
+
# @return [Array] topic unique key for validators
|
29
|
+
def topic_unique_key(topic)
|
30
|
+
[
|
31
|
+
topic[:name],
|
32
|
+
topic[:subscription_group_details]
|
33
|
+
]
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|