karafka 2.1.12 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +41 -0
- data/Gemfile.lock +1 -1
- data/bin/record_rss +50 -0
- data/config/locales/errors.yml +4 -0
- data/config/locales/pro_errors.yml +17 -0
- data/lib/karafka/admin.rb +21 -33
- data/lib/karafka/connection/client.rb +1 -1
- data/lib/karafka/contracts/config.rb +24 -0
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +5 -2
- data/lib/karafka/messages/builders/message.rb +8 -4
- data/lib/karafka/pro/active_job/consumer.rb +1 -1
- data/lib/karafka/pro/cleaner/errors.rb +27 -0
- data/lib/karafka/pro/cleaner/messages/message.rb +46 -0
- data/lib/karafka/pro/cleaner/messages/messages.rb +42 -0
- data/lib/karafka/pro/cleaner.rb +41 -0
- data/lib/karafka/pro/contracts/base.rb +23 -0
- data/lib/karafka/pro/contracts/server_cli_options.rb +111 -0
- data/lib/karafka/pro/encryption/errors.rb +4 -1
- data/lib/karafka/pro/loader.rb +6 -2
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +6 -0
- data/lib/karafka/pro/routing/features/active_job/builder.rb +45 -0
- data/lib/karafka/pro/routing/features/active_job.rb +26 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +53 -0
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +41 -0
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +41 -0
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +44 -0
- data/lib/karafka/pro/routing/features/long_running_job/{contract.rb → contracts/topic.rb} +14 -11
- data/lib/karafka/pro/routing/features/{filtering/contract.rb → patterns/builder.rb} +13 -16
- data/lib/karafka/pro/routing/features/patterns/config.rb +54 -0
- data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +68 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +62 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +46 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +41 -0
- data/lib/karafka/pro/routing/features/patterns/detector.rb +68 -0
- data/lib/karafka/pro/routing/features/patterns/pattern.rb +81 -0
- data/lib/karafka/pro/routing/features/{delaying/contract.rb → patterns/patterns.rb} +11 -14
- data/lib/karafka/pro/routing/features/patterns/topic.rb +50 -0
- data/lib/karafka/pro/routing/features/patterns/topics.rb +53 -0
- data/lib/karafka/pro/routing/features/patterns.rb +33 -0
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +51 -0
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +44 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +55 -0
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +44 -0
- data/lib/karafka/routing/features/active_job/proxy.rb +14 -0
- data/lib/karafka/routing/features/base/expander.rb +8 -2
- data/lib/karafka/routing/features/base.rb +4 -2
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +46 -0
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +33 -0
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +27 -0
- data/lib/karafka/routing/router.rb +0 -11
- data/lib/karafka/routing/subscription_group.rb +9 -0
- data/lib/karafka/routing/topic.rb +5 -0
- data/lib/karafka/server.rb +9 -4
- data/lib/karafka/setup/config.rb +45 -0
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +37 -15
- metadata.gz.sig +0 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +0 -50
- data/lib/karafka/pro/routing/features/expiring/contract.rb +0 -38
- data/lib/karafka/pro/routing/features/pausing/contract.rb +0 -48
- data/lib/karafka/pro/routing/features/throttling/contract.rb +0 -41
- data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +0 -52
- data/lib/karafka/routing/features/active_job/contract.rb +0 -41
- data/lib/karafka/routing/features/dead_letter_queue/contract.rb +0 -42
- data/lib/karafka/routing/features/declaratives/contract.rb +0 -30
- data/lib/karafka/routing/features/manual_offset_management/contract.rb +0 -24
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Routing
|
5
|
+
module Features
|
6
|
+
class DeadLetterQueue < Base
|
7
|
+
# This feature validation contracts
|
8
|
+
module Contracts
|
9
|
+
# Rules around dead letter queue settings
|
10
|
+
class Topic < Karafka::Contracts::Base
|
11
|
+
configure do |config|
|
12
|
+
config.error_messages = YAML.safe_load(
|
13
|
+
File.read(
|
14
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
15
|
+
)
|
16
|
+
).fetch('en').fetch('validations').fetch('topic')
|
17
|
+
end
|
18
|
+
|
19
|
+
nested :dead_letter_queue do
|
20
|
+
required(:active) { |val| [true, false].include?(val) }
|
21
|
+
required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
|
22
|
+
end
|
23
|
+
|
24
|
+
# Validate topic name only if dlq is active
|
25
|
+
virtual do |data, errors|
|
26
|
+
next unless errors.empty?
|
27
|
+
|
28
|
+
dead_letter_queue = data[:dead_letter_queue]
|
29
|
+
|
30
|
+
next unless dead_letter_queue[:active]
|
31
|
+
|
32
|
+
topic = dead_letter_queue[:topic]
|
33
|
+
topic_regexp = ::Karafka::Contracts::TOPIC_REGEXP
|
34
|
+
|
35
|
+
# When topic is set to false, it means we just want to skip dispatch on DLQ
|
36
|
+
next if topic == false
|
37
|
+
next if topic.is_a?(String) && topic_regexp.match?(topic)
|
38
|
+
|
39
|
+
[[%i[dead_letter_queue topic], :format]]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Routing
|
5
|
+
module Features
|
6
|
+
class Declaratives < Base
|
7
|
+
# This feature validation contracts
|
8
|
+
module Contracts
|
9
|
+
# Basic validation of the Kafka expected config details
|
10
|
+
class Topic < Karafka::Contracts::Base
|
11
|
+
configure do |config|
|
12
|
+
config.error_messages = YAML.safe_load(
|
13
|
+
File.read(
|
14
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
15
|
+
)
|
16
|
+
).fetch('en').fetch('validations').fetch('topic')
|
17
|
+
end
|
18
|
+
|
19
|
+
nested :declaratives do
|
20
|
+
required(:active) { |val| [true, false].include?(val) }
|
21
|
+
required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
|
22
|
+
required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
|
23
|
+
required(:details) do |val|
|
24
|
+
val.is_a?(Hash) &&
|
25
|
+
val.keys.all? { |key| key.is_a?(Symbol) }
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Routing
|
5
|
+
module Features
|
6
|
+
class ManualOffsetManagement < Base
|
7
|
+
# This feature validation contracts
|
8
|
+
module Contracts
|
9
|
+
# Rules around manual offset management settings
|
10
|
+
class Topic < Karafka::Contracts::Base
|
11
|
+
configure do |config|
|
12
|
+
config.error_messages = YAML.safe_load(
|
13
|
+
File.read(
|
14
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
15
|
+
)
|
16
|
+
).fetch('en').fetch('validations').fetch('topic')
|
17
|
+
end
|
18
|
+
|
19
|
+
nested :manual_offset_management do
|
20
|
+
required(:active) { |val| [true, false].include?(val) }
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -7,16 +7,6 @@ module Karafka
|
|
7
7
|
# @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
|
8
8
|
# structure so all the routes are being stored in a single level array
|
9
9
|
module Router
|
10
|
-
# Find a proper topic based on full topic id
|
11
|
-
# @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
|
12
|
-
# routing topic
|
13
|
-
# @return [Karafka::Routing::Topic] proper route details
|
14
|
-
# @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
|
15
|
-
# any route defined by user using routes.draw
|
16
|
-
def find(topic_id)
|
17
|
-
find_by(id: topic_id) || raise(Errors::NonMatchingRouteError, topic_id)
|
18
|
-
end
|
19
|
-
|
20
10
|
# Finds first reference of a given topic based on provided lookup attribute
|
21
11
|
# @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
|
22
12
|
# @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
|
@@ -46,7 +36,6 @@ module Karafka
|
|
46
36
|
find_by(name: name) || Topic.new(name, ConsumerGroup.new(name))
|
47
37
|
end
|
48
38
|
|
49
|
-
module_function :find
|
50
39
|
module_function :find_by
|
51
40
|
module_function :find_or_initialize_by_name
|
52
41
|
end
|
@@ -61,6 +61,15 @@ module Karafka
|
|
61
61
|
Karafka::App.config.internal.routing.activity_manager.active?(:subscription_groups, name)
|
62
62
|
end
|
63
63
|
|
64
|
+
# @return [Array<String>] names of topics to which we should subscribe.
|
65
|
+
#
|
66
|
+
# @note Most of the time it should not include inactive topics but in case of pattern
|
67
|
+
# matching the matcher topics become inactive down the road, hence we filter out so
|
68
|
+
# they are later removed.
|
69
|
+
def subscriptions
|
70
|
+
topics.select(&:active?).map(&:subscription_name)
|
71
|
+
end
|
72
|
+
|
64
73
|
private
|
65
74
|
|
66
75
|
# @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
|
data/lib/karafka/server.rb
CHANGED
@@ -32,8 +32,8 @@ module Karafka
|
|
32
32
|
# embedded
|
33
33
|
# We cannot validate this during the start because config needs to be populated and routes
|
34
34
|
# need to be defined.
|
35
|
-
|
36
|
-
|
35
|
+
config.internal.cli.contract.validate!(
|
36
|
+
config.internal.routing.activity_manager.to_h
|
37
37
|
)
|
38
38
|
|
39
39
|
process.on_sigint { stop }
|
@@ -82,7 +82,7 @@ module Karafka
|
|
82
82
|
|
83
83
|
Karafka::App.stop!
|
84
84
|
|
85
|
-
timeout =
|
85
|
+
timeout = config.shutdown_timeout
|
86
86
|
|
87
87
|
# We check from time to time (for the timeout period) if all the threads finished
|
88
88
|
# their work and if so, we can just return and normal shutdown process will take place
|
@@ -148,9 +148,14 @@ module Karafka
|
|
148
148
|
|
149
149
|
private
|
150
150
|
|
151
|
+
# @return [Karafka::Core::Configurable::Node] root config node
|
152
|
+
def config
|
153
|
+
Karafka::App.config
|
154
|
+
end
|
155
|
+
|
151
156
|
# @return [Karafka::Process] process wrapper instance used to catch system signal calls
|
152
157
|
def process
|
153
|
-
|
158
|
+
config.internal.process
|
154
159
|
end
|
155
160
|
end
|
156
161
|
end
|
data/lib/karafka/setup/config.rb
CHANGED
@@ -105,6 +105,44 @@ module Karafka
|
|
105
105
|
# @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
|
106
106
|
setting :kafka, default: {}
|
107
107
|
|
108
|
+
# Admin specific settings.
|
109
|
+
#
|
110
|
+
# Since admin operations are often specific, they may require specific librdkafka settings
|
111
|
+
# or other settings that are unique to admin.
|
112
|
+
setting :admin do
|
113
|
+
# Specific kafka settings that are tuned to operate within the Admin.
|
114
|
+
#
|
115
|
+
# Please do not change them unless you know what you are doing as their misconfiguration
|
116
|
+
# may cause Admin API to misbehave
|
117
|
+
# option [Hash] extra changes to the default root kafka settings
|
118
|
+
setting :kafka, default: {
|
119
|
+
# We want to know when there is no more data not to end up with an endless loop
|
120
|
+
'enable.partition.eof': true,
|
121
|
+
# Do not publish statistics from admin as they are not relevant
|
122
|
+
'statistics.interval.ms': 0,
|
123
|
+
# Fetch at most 5 MBs when using admin
|
124
|
+
'fetch.message.max.bytes': 5 * 1_048_576,
|
125
|
+
# Do not commit offset automatically, this prevents offset tracking for operations
|
126
|
+
# involving a consumer instance
|
127
|
+
'enable.auto.commit': false,
|
128
|
+
# Make sure that topic metadata lookups do not create topics accidentally
|
129
|
+
'allow.auto.create.topics': false
|
130
|
+
}
|
131
|
+
|
132
|
+
# option [String] default name for the admin consumer group. Please note, that this is a
|
133
|
+
# subject to be remapped by the consumer mapper as any other consumer group in the routes
|
134
|
+
setting :group_id, default: 'karafka_admin'
|
135
|
+
|
136
|
+
# option max_wait_time [Integer] We wait only for this amount of time before raising error
|
137
|
+
# as we intercept this error and retry after checking that the operation was finished or
|
138
|
+
# failed using external factor.
|
139
|
+
setting :max_wait_time, default: 1_000
|
140
|
+
|
141
|
+
# How many times should be try. 1 000 ms x 60 => 60 seconds wait in total and then we give
|
142
|
+
# up on pending operations
|
143
|
+
setting :max_attempts, default: 60
|
144
|
+
end
|
145
|
+
|
108
146
|
# Namespace for internal settings that should not be modified directly
|
109
147
|
setting :internal do
|
110
148
|
# option status [Karafka::Status] app status
|
@@ -114,6 +152,13 @@ module Karafka
|
|
114
152
|
# instances
|
115
153
|
setting :process, default: Process.new
|
116
154
|
|
155
|
+
# Namespace for CLI related settings
|
156
|
+
setting :cli do
|
157
|
+
# option contract [Object] cli setup validation contract (in the context of options and
|
158
|
+
# topics)
|
159
|
+
setting :contract, default: Contracts::ServerCliOptions.new
|
160
|
+
end
|
161
|
+
|
117
162
|
setting :routing do
|
118
163
|
# option builder [Karafka::Routing::Builder] builder instance
|
119
164
|
setting :builder, default: Routing::Builder.new
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
36
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-
|
38
|
+
date: 2023-09-01 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -141,6 +141,7 @@ files:
|
|
141
141
|
- bin/create_token
|
142
142
|
- bin/integrations
|
143
143
|
- bin/karafka
|
144
|
+
- bin/record_rss
|
144
145
|
- bin/rspecs
|
145
146
|
- bin/scenario
|
146
147
|
- bin/stress_many
|
@@ -219,6 +220,12 @@ files:
|
|
219
220
|
- lib/karafka/pro/active_job/consumer.rb
|
220
221
|
- lib/karafka/pro/active_job/dispatcher.rb
|
221
222
|
- lib/karafka/pro/active_job/job_options_contract.rb
|
223
|
+
- lib/karafka/pro/cleaner.rb
|
224
|
+
- lib/karafka/pro/cleaner/errors.rb
|
225
|
+
- lib/karafka/pro/cleaner/messages/message.rb
|
226
|
+
- lib/karafka/pro/cleaner/messages/messages.rb
|
227
|
+
- lib/karafka/pro/contracts/base.rb
|
228
|
+
- lib/karafka/pro/contracts/server_cli_options.rb
|
222
229
|
- lib/karafka/pro/encryption.rb
|
223
230
|
- lib/karafka/pro/encryption/cipher.rb
|
224
231
|
- lib/karafka/pro/encryption/contracts/config.rb
|
@@ -296,35 +303,49 @@ files:
|
|
296
303
|
- lib/karafka/pro/processing/strategies/vp/default.rb
|
297
304
|
- lib/karafka/pro/processing/strategy_selector.rb
|
298
305
|
- lib/karafka/pro/processing/virtual_offset_manager.rb
|
306
|
+
- lib/karafka/pro/routing/features/active_job.rb
|
307
|
+
- lib/karafka/pro/routing/features/active_job/builder.rb
|
299
308
|
- lib/karafka/pro/routing/features/base.rb
|
300
309
|
- lib/karafka/pro/routing/features/dead_letter_queue.rb
|
301
|
-
- lib/karafka/pro/routing/features/dead_letter_queue/
|
310
|
+
- lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb
|
302
311
|
- lib/karafka/pro/routing/features/delaying.rb
|
303
312
|
- lib/karafka/pro/routing/features/delaying/config.rb
|
304
|
-
- lib/karafka/pro/routing/features/delaying/
|
313
|
+
- lib/karafka/pro/routing/features/delaying/contracts/topic.rb
|
305
314
|
- lib/karafka/pro/routing/features/delaying/topic.rb
|
306
315
|
- lib/karafka/pro/routing/features/expiring.rb
|
307
316
|
- lib/karafka/pro/routing/features/expiring/config.rb
|
308
|
-
- lib/karafka/pro/routing/features/expiring/
|
317
|
+
- lib/karafka/pro/routing/features/expiring/contracts/topic.rb
|
309
318
|
- lib/karafka/pro/routing/features/expiring/topic.rb
|
310
319
|
- lib/karafka/pro/routing/features/filtering.rb
|
311
320
|
- lib/karafka/pro/routing/features/filtering/config.rb
|
312
|
-
- lib/karafka/pro/routing/features/filtering/
|
321
|
+
- lib/karafka/pro/routing/features/filtering/contracts/topic.rb
|
313
322
|
- lib/karafka/pro/routing/features/filtering/topic.rb
|
314
323
|
- lib/karafka/pro/routing/features/long_running_job.rb
|
315
324
|
- lib/karafka/pro/routing/features/long_running_job/config.rb
|
316
|
-
- lib/karafka/pro/routing/features/long_running_job/
|
325
|
+
- lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb
|
317
326
|
- lib/karafka/pro/routing/features/long_running_job/topic.rb
|
327
|
+
- lib/karafka/pro/routing/features/patterns.rb
|
328
|
+
- lib/karafka/pro/routing/features/patterns/builder.rb
|
329
|
+
- lib/karafka/pro/routing/features/patterns/config.rb
|
330
|
+
- lib/karafka/pro/routing/features/patterns/consumer_group.rb
|
331
|
+
- lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb
|
332
|
+
- lib/karafka/pro/routing/features/patterns/contracts/pattern.rb
|
333
|
+
- lib/karafka/pro/routing/features/patterns/contracts/topic.rb
|
334
|
+
- lib/karafka/pro/routing/features/patterns/detector.rb
|
335
|
+
- lib/karafka/pro/routing/features/patterns/pattern.rb
|
336
|
+
- lib/karafka/pro/routing/features/patterns/patterns.rb
|
337
|
+
- lib/karafka/pro/routing/features/patterns/topic.rb
|
338
|
+
- lib/karafka/pro/routing/features/patterns/topics.rb
|
318
339
|
- lib/karafka/pro/routing/features/pausing.rb
|
319
|
-
- lib/karafka/pro/routing/features/pausing/
|
340
|
+
- lib/karafka/pro/routing/features/pausing/contracts/topic.rb
|
320
341
|
- lib/karafka/pro/routing/features/pausing/topic.rb
|
321
342
|
- lib/karafka/pro/routing/features/throttling.rb
|
322
343
|
- lib/karafka/pro/routing/features/throttling/config.rb
|
323
|
-
- lib/karafka/pro/routing/features/throttling/
|
344
|
+
- lib/karafka/pro/routing/features/throttling/contracts/topic.rb
|
324
345
|
- lib/karafka/pro/routing/features/throttling/topic.rb
|
325
346
|
- lib/karafka/pro/routing/features/virtual_partitions.rb
|
326
347
|
- lib/karafka/pro/routing/features/virtual_partitions/config.rb
|
327
|
-
- lib/karafka/pro/routing/features/virtual_partitions/
|
348
|
+
- lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb
|
328
349
|
- lib/karafka/pro/routing/features/virtual_partitions/topic.rb
|
329
350
|
- lib/karafka/process.rb
|
330
351
|
- lib/karafka/processing/coordinator.rb
|
@@ -359,21 +380,22 @@ files:
|
|
359
380
|
- lib/karafka/routing/features/active_job.rb
|
360
381
|
- lib/karafka/routing/features/active_job/builder.rb
|
361
382
|
- lib/karafka/routing/features/active_job/config.rb
|
362
|
-
- lib/karafka/routing/features/active_job/
|
383
|
+
- lib/karafka/routing/features/active_job/contracts/topic.rb
|
384
|
+
- lib/karafka/routing/features/active_job/proxy.rb
|
363
385
|
- lib/karafka/routing/features/active_job/topic.rb
|
364
386
|
- lib/karafka/routing/features/base.rb
|
365
387
|
- lib/karafka/routing/features/base/expander.rb
|
366
388
|
- lib/karafka/routing/features/dead_letter_queue.rb
|
367
389
|
- lib/karafka/routing/features/dead_letter_queue/config.rb
|
368
|
-
- lib/karafka/routing/features/dead_letter_queue/
|
390
|
+
- lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb
|
369
391
|
- lib/karafka/routing/features/dead_letter_queue/topic.rb
|
370
392
|
- lib/karafka/routing/features/declaratives.rb
|
371
393
|
- lib/karafka/routing/features/declaratives/config.rb
|
372
|
-
- lib/karafka/routing/features/declaratives/
|
394
|
+
- lib/karafka/routing/features/declaratives/contracts/topic.rb
|
373
395
|
- lib/karafka/routing/features/declaratives/topic.rb
|
374
396
|
- lib/karafka/routing/features/manual_offset_management.rb
|
375
397
|
- lib/karafka/routing/features/manual_offset_management/config.rb
|
376
|
-
- lib/karafka/routing/features/manual_offset_management/
|
398
|
+
- lib/karafka/routing/features/manual_offset_management/contracts/topic.rb
|
377
399
|
- lib/karafka/routing/features/manual_offset_management/topic.rb
|
378
400
|
- lib/karafka/routing/proxy.rb
|
379
401
|
- lib/karafka/routing/router.rb
|
@@ -424,7 +446,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
424
446
|
- !ruby/object:Gem::Version
|
425
447
|
version: '0'
|
426
448
|
requirements: []
|
427
|
-
rubygems_version: 3.4.
|
449
|
+
rubygems_version: 3.4.19
|
428
450
|
signing_key:
|
429
451
|
specification_version: 4
|
430
452
|
summary: Karafka is Ruby and Rails efficient Kafka processing framework.
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,50 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# This Karafka component is a Pro component under a commercial license.
|
4
|
-
# This Karafka component is NOT licensed under LGPL.
|
5
|
-
#
|
6
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
-
# repository and their usage requires commercial license agreement.
|
8
|
-
#
|
9
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
-
#
|
11
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
-
# your code to Maciej Mensfeld.
|
13
|
-
|
14
|
-
module Karafka
|
15
|
-
module Pro
|
16
|
-
module Routing
|
17
|
-
module Features
|
18
|
-
class DeadLetterQueue < Base
|
19
|
-
# Extended rules for dead letter queue settings
|
20
|
-
class Contract < Contracts::Base
|
21
|
-
configure do |config|
|
22
|
-
config.error_messages = YAML.safe_load(
|
23
|
-
File.read(
|
24
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
25
|
-
)
|
26
|
-
).fetch('en').fetch('validations').fetch('topic')
|
27
|
-
end
|
28
|
-
|
29
|
-
# Make sure that when we use virtual partitions with DLQ, at least one retry is set
|
30
|
-
# We cannot use VP with DLQ without retries as we in order to provide ordering
|
31
|
-
# warranties on errors with VP, we need to collapse the VPs concurrency and retry
|
32
|
-
# without any indeterministic work
|
33
|
-
virtual do |data, errors|
|
34
|
-
next unless errors.empty?
|
35
|
-
|
36
|
-
dead_letter_queue = data[:dead_letter_queue]
|
37
|
-
virtual_partitions = data[:virtual_partitions]
|
38
|
-
|
39
|
-
next unless dead_letter_queue[:active]
|
40
|
-
next unless virtual_partitions[:active]
|
41
|
-
next if dead_letter_queue[:max_retries].positive?
|
42
|
-
|
43
|
-
[[%i[dead_letter_queue], :with_virtual_partitions]]
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
@@ -1,38 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# This Karafka component is a Pro component under a commercial license.
|
4
|
-
# This Karafka component is NOT licensed under LGPL.
|
5
|
-
#
|
6
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
-
# repository and their usage requires commercial license agreement.
|
8
|
-
#
|
9
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
-
#
|
11
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
-
# your code to Maciej Mensfeld.
|
13
|
-
|
14
|
-
module Karafka
|
15
|
-
module Pro
|
16
|
-
module Routing
|
17
|
-
module Features
|
18
|
-
class Expiring < Base
|
19
|
-
# Contract to validate configuration of the expiring feature
|
20
|
-
class Contract < Contracts::Base
|
21
|
-
configure do |config|
|
22
|
-
config.error_messages = YAML.safe_load(
|
23
|
-
File.read(
|
24
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
25
|
-
)
|
26
|
-
).fetch('en').fetch('validations').fetch('topic')
|
27
|
-
end
|
28
|
-
|
29
|
-
nested(:expiring) do
|
30
|
-
required(:active) { |val| [true, false].include?(val) }
|
31
|
-
required(:ttl) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
@@ -1,48 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# This Karafka component is a Pro component under a commercial license.
|
4
|
-
# This Karafka component is NOT licensed under LGPL.
|
5
|
-
#
|
6
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
-
# repository and their usage requires commercial license agreement.
|
8
|
-
#
|
9
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
-
#
|
11
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
-
# your code to Maciej Mensfeld.
|
13
|
-
|
14
|
-
module Karafka
|
15
|
-
module Pro
|
16
|
-
module Routing
|
17
|
-
module Features
|
18
|
-
class Pausing < Base
|
19
|
-
# Contract to make sure, that the pause settings on a per topic basis are as expected
|
20
|
-
class Contract < Contracts::Base
|
21
|
-
configure do |config|
|
22
|
-
config.error_messages = YAML.safe_load(
|
23
|
-
File.read(
|
24
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
25
|
-
)
|
26
|
-
).fetch('en').fetch('validations').fetch('topic')
|
27
|
-
|
28
|
-
required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
29
|
-
required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
30
|
-
required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
|
31
|
-
|
32
|
-
virtual do |data, errors|
|
33
|
-
next unless errors.empty?
|
34
|
-
|
35
|
-
pause_timeout = data.fetch(:pause_timeout)
|
36
|
-
pause_max_timeout = data.fetch(:pause_max_timeout)
|
37
|
-
|
38
|
-
next if pause_timeout <= pause_max_timeout
|
39
|
-
|
40
|
-
[[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
@@ -1,41 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# This Karafka component is a Pro component under a commercial license.
|
4
|
-
# This Karafka component is NOT licensed under LGPL.
|
5
|
-
#
|
6
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
-
# repository and their usage requires commercial license agreement.
|
8
|
-
#
|
9
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
-
#
|
11
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
-
# your code to Maciej Mensfeld.
|
13
|
-
|
14
|
-
module Karafka
|
15
|
-
module Pro
|
16
|
-
module Routing
|
17
|
-
module Features
|
18
|
-
class Throttling < Base
|
19
|
-
# Rules around throttling settings
|
20
|
-
class Contract < Contracts::Base
|
21
|
-
configure do |config|
|
22
|
-
config.error_messages = YAML.safe_load(
|
23
|
-
File.read(
|
24
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
25
|
-
)
|
26
|
-
).fetch('en').fetch('validations').fetch('topic')
|
27
|
-
end
|
28
|
-
|
29
|
-
nested(:throttling) do
|
30
|
-
required(:active) { |val| [true, false].include?(val) }
|
31
|
-
required(:interval) { |val| val.is_a?(Integer) && val.positive? }
|
32
|
-
required(:limit) do |val|
|
33
|
-
(val.is_a?(Integer) || val == Float::INFINITY) && val.positive?
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
@@ -1,52 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# This Karafka component is a Pro component under a commercial license.
|
4
|
-
# This Karafka component is NOT licensed under LGPL.
|
5
|
-
#
|
6
|
-
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
-
# repository and their usage requires commercial license agreement.
|
8
|
-
#
|
9
|
-
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
-
#
|
11
|
-
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
-
# your code to Maciej Mensfeld.
|
13
|
-
|
14
|
-
module Karafka
|
15
|
-
module Pro
|
16
|
-
module Routing
|
17
|
-
module Features
|
18
|
-
class VirtualPartitions < Base
|
19
|
-
# Rules around virtual partitions
|
20
|
-
class Contract < Contracts::Base
|
21
|
-
configure do |config|
|
22
|
-
config.error_messages = YAML.safe_load(
|
23
|
-
File.read(
|
24
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
25
|
-
)
|
26
|
-
).fetch('en').fetch('validations').fetch('topic')
|
27
|
-
end
|
28
|
-
|
29
|
-
nested(:virtual_partitions) do
|
30
|
-
required(:active) { |val| [true, false].include?(val) }
|
31
|
-
required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
|
32
|
-
required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
|
33
|
-
end
|
34
|
-
|
35
|
-
# When virtual partitions are defined, partitioner needs to respond to `#call` and it
|
36
|
-
# cannot be nil
|
37
|
-
virtual do |data, errors|
|
38
|
-
next unless errors.empty?
|
39
|
-
|
40
|
-
virtual_partitions = data[:virtual_partitions]
|
41
|
-
|
42
|
-
next unless virtual_partitions[:active]
|
43
|
-
next if virtual_partitions[:partitioner].respond_to?(:call)
|
44
|
-
|
45
|
-
[[%i[virtual_partitions partitioner], :respond_to_call]]
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|