karafka 2.1.12 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +41 -0
- data/Gemfile.lock +1 -1
- data/bin/record_rss +50 -0
- data/config/locales/errors.yml +4 -0
- data/config/locales/pro_errors.yml +17 -0
- data/lib/karafka/admin.rb +21 -33
- data/lib/karafka/connection/client.rb +1 -1
- data/lib/karafka/contracts/config.rb +24 -0
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +5 -2
- data/lib/karafka/messages/builders/message.rb +8 -4
- data/lib/karafka/pro/active_job/consumer.rb +1 -1
- data/lib/karafka/pro/cleaner/errors.rb +27 -0
- data/lib/karafka/pro/cleaner/messages/message.rb +46 -0
- data/lib/karafka/pro/cleaner/messages/messages.rb +42 -0
- data/lib/karafka/pro/cleaner.rb +41 -0
- data/lib/karafka/pro/contracts/base.rb +23 -0
- data/lib/karafka/pro/contracts/server_cli_options.rb +111 -0
- data/lib/karafka/pro/encryption/errors.rb +4 -1
- data/lib/karafka/pro/loader.rb +6 -2
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +6 -0
- data/lib/karafka/pro/routing/features/active_job/builder.rb +45 -0
- data/lib/karafka/pro/routing/features/active_job.rb +26 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +53 -0
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +41 -0
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +41 -0
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +44 -0
- data/lib/karafka/pro/routing/features/long_running_job/{contract.rb → contracts/topic.rb} +14 -11
- data/lib/karafka/pro/routing/features/{filtering/contract.rb → patterns/builder.rb} +13 -16
- data/lib/karafka/pro/routing/features/patterns/config.rb +54 -0
- data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +68 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +62 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +46 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +41 -0
- data/lib/karafka/pro/routing/features/patterns/detector.rb +68 -0
- data/lib/karafka/pro/routing/features/patterns/pattern.rb +81 -0
- data/lib/karafka/pro/routing/features/{delaying/contract.rb → patterns/patterns.rb} +11 -14
- data/lib/karafka/pro/routing/features/patterns/topic.rb +50 -0
- data/lib/karafka/pro/routing/features/patterns/topics.rb +53 -0
- data/lib/karafka/pro/routing/features/patterns.rb +33 -0
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +51 -0
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +44 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +55 -0
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +44 -0
- data/lib/karafka/routing/features/active_job/proxy.rb +14 -0
- data/lib/karafka/routing/features/base/expander.rb +8 -2
- data/lib/karafka/routing/features/base.rb +4 -2
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +46 -0
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +33 -0
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +27 -0
- data/lib/karafka/routing/router.rb +0 -11
- data/lib/karafka/routing/subscription_group.rb +9 -0
- data/lib/karafka/routing/topic.rb +5 -0
- data/lib/karafka/server.rb +9 -4
- data/lib/karafka/setup/config.rb +45 -0
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +37 -15
- metadata.gz.sig +0 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +0 -50
- data/lib/karafka/pro/routing/features/expiring/contract.rb +0 -38
- data/lib/karafka/pro/routing/features/pausing/contract.rb +0 -48
- data/lib/karafka/pro/routing/features/throttling/contract.rb +0 -41
- data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +0 -52
- data/lib/karafka/routing/features/active_job/contract.rb +0 -41
- data/lib/karafka/routing/features/dead_letter_queue/contract.rb +0 -42
- data/lib/karafka/routing/features/declaratives/contract.rb +0 -30
- data/lib/karafka/routing/features/manual_offset_management/contract.rb +0 -24
@@ -1,41 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Routing
|
5
|
-
module Features
|
6
|
-
class ActiveJob < Base
|
7
|
-
# Rules around using ActiveJob routing - basically you need to have ActiveJob available
|
8
|
-
# in order to be able to use active job routing
|
9
|
-
class Contract < Contracts::Base
|
10
|
-
configure do |config|
|
11
|
-
config.error_messages = YAML.safe_load(
|
12
|
-
File.read(
|
13
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
14
|
-
)
|
15
|
-
).fetch('en').fetch('validations').fetch('topic')
|
16
|
-
end
|
17
|
-
|
18
|
-
virtual do |data, errors|
|
19
|
-
next unless errors.empty?
|
20
|
-
next unless data[:active_job][:active]
|
21
|
-
# One should not define active job jobs without ActiveJob being available for usage
|
22
|
-
next if Object.const_defined?('ActiveJob::Base')
|
23
|
-
|
24
|
-
[[%i[consumer], :active_job_missing]]
|
25
|
-
end
|
26
|
-
|
27
|
-
# ActiveJob needs to always run with manual offset management
|
28
|
-
# Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
|
29
|
-
# consumed jobs that did not run because of shutdown.
|
30
|
-
virtual do |data, errors|
|
31
|
-
next unless errors.empty?
|
32
|
-
next unless data[:active_job][:active]
|
33
|
-
next if data[:manual_offset_management][:active]
|
34
|
-
|
35
|
-
[[%i[manual_offset_management], :must_be_enabled]]
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Routing
|
5
|
-
module Features
|
6
|
-
class DeadLetterQueue < Base
|
7
|
-
# Rules around dead letter queue settings
|
8
|
-
class Contract < Contracts::Base
|
9
|
-
configure do |config|
|
10
|
-
config.error_messages = YAML.safe_load(
|
11
|
-
File.read(
|
12
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
13
|
-
)
|
14
|
-
).fetch('en').fetch('validations').fetch('topic')
|
15
|
-
end
|
16
|
-
|
17
|
-
nested :dead_letter_queue do
|
18
|
-
required(:active) { |val| [true, false].include?(val) }
|
19
|
-
required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
|
20
|
-
end
|
21
|
-
|
22
|
-
# Validate topic name only if dlq is active
|
23
|
-
virtual do |data, errors|
|
24
|
-
next unless errors.empty?
|
25
|
-
|
26
|
-
dead_letter_queue = data[:dead_letter_queue]
|
27
|
-
|
28
|
-
next unless dead_letter_queue[:active]
|
29
|
-
|
30
|
-
topic = dead_letter_queue[:topic]
|
31
|
-
|
32
|
-
# When topic is set to false, it means we just want to skip dispatch on DLQ
|
33
|
-
next if topic == false
|
34
|
-
next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
|
35
|
-
|
36
|
-
[[%i[dead_letter_queue topic], :format]]
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
@@ -1,30 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Routing
|
5
|
-
module Features
|
6
|
-
class Declaratives < Base
|
7
|
-
# Basic validation of the Kafka expected config details
|
8
|
-
class Contract < Contracts::Base
|
9
|
-
configure do |config|
|
10
|
-
config.error_messages = YAML.safe_load(
|
11
|
-
File.read(
|
12
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
13
|
-
)
|
14
|
-
).fetch('en').fetch('validations').fetch('topic')
|
15
|
-
end
|
16
|
-
|
17
|
-
nested :declaratives do
|
18
|
-
required(:active) { |val| [true, false].include?(val) }
|
19
|
-
required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
|
20
|
-
required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
|
21
|
-
required(:details) do |val|
|
22
|
-
val.is_a?(Hash) &&
|
23
|
-
val.keys.all? { |key| key.is_a?(Symbol) }
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Routing
|
5
|
-
module Features
|
6
|
-
class ManualOffsetManagement < Base
|
7
|
-
# Rules around manual offset management settings
|
8
|
-
class Contract < Contracts::Base
|
9
|
-
configure do |config|
|
10
|
-
config.error_messages = YAML.safe_load(
|
11
|
-
File.read(
|
12
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
13
|
-
)
|
14
|
-
).fetch('en').fetch('validations').fetch('topic')
|
15
|
-
end
|
16
|
-
|
17
|
-
nested :manual_offset_management do
|
18
|
-
required(:active) { |val| [true, false].include?(val) }
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|