karafka 2.1.12 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +41 -0
- data/Gemfile.lock +1 -1
- data/bin/record_rss +50 -0
- data/config/locales/errors.yml +4 -0
- data/config/locales/pro_errors.yml +17 -0
- data/lib/karafka/admin.rb +21 -33
- data/lib/karafka/connection/client.rb +1 -1
- data/lib/karafka/contracts/config.rb +24 -0
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +5 -2
- data/lib/karafka/messages/builders/message.rb +8 -4
- data/lib/karafka/pro/active_job/consumer.rb +1 -1
- data/lib/karafka/pro/cleaner/errors.rb +27 -0
- data/lib/karafka/pro/cleaner/messages/message.rb +46 -0
- data/lib/karafka/pro/cleaner/messages/messages.rb +42 -0
- data/lib/karafka/pro/cleaner.rb +41 -0
- data/lib/karafka/pro/contracts/base.rb +23 -0
- data/lib/karafka/pro/contracts/server_cli_options.rb +111 -0
- data/lib/karafka/pro/encryption/errors.rb +4 -1
- data/lib/karafka/pro/loader.rb +6 -2
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +6 -0
- data/lib/karafka/pro/routing/features/active_job/builder.rb +45 -0
- data/lib/karafka/pro/routing/features/active_job.rb +26 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +53 -0
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +41 -0
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +41 -0
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +44 -0
- data/lib/karafka/pro/routing/features/long_running_job/{contract.rb → contracts/topic.rb} +14 -11
- data/lib/karafka/pro/routing/features/{filtering/contract.rb → patterns/builder.rb} +13 -16
- data/lib/karafka/pro/routing/features/patterns/config.rb +54 -0
- data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +68 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +62 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +46 -0
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +41 -0
- data/lib/karafka/pro/routing/features/patterns/detector.rb +68 -0
- data/lib/karafka/pro/routing/features/patterns/pattern.rb +81 -0
- data/lib/karafka/pro/routing/features/{delaying/contract.rb → patterns/patterns.rb} +11 -14
- data/lib/karafka/pro/routing/features/patterns/topic.rb +50 -0
- data/lib/karafka/pro/routing/features/patterns/topics.rb +53 -0
- data/lib/karafka/pro/routing/features/patterns.rb +33 -0
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +51 -0
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +44 -0
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +55 -0
- data/lib/karafka/routing/consumer_group.rb +1 -1
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +44 -0
- data/lib/karafka/routing/features/active_job/proxy.rb +14 -0
- data/lib/karafka/routing/features/base/expander.rb +8 -2
- data/lib/karafka/routing/features/base.rb +4 -2
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +46 -0
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +33 -0
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +27 -0
- data/lib/karafka/routing/router.rb +0 -11
- data/lib/karafka/routing/subscription_group.rb +9 -0
- data/lib/karafka/routing/topic.rb +5 -0
- data/lib/karafka/server.rb +9 -4
- data/lib/karafka/setup/config.rb +45 -0
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +37 -15
- metadata.gz.sig +0 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +0 -50
- data/lib/karafka/pro/routing/features/expiring/contract.rb +0 -38
- data/lib/karafka/pro/routing/features/pausing/contract.rb +0 -48
- data/lib/karafka/pro/routing/features/throttling/contract.rb +0 -41
- data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +0 -52
- data/lib/karafka/routing/features/active_job/contract.rb +0 -41
- data/lib/karafka/routing/features/dead_letter_queue/contract.rb +0 -42
- data/lib/karafka/routing/features/declaratives/contract.rb +0 -30
- data/lib/karafka/routing/features/manual_offset_management/contract.rb +0 -24
@@ -1,41 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Routing
|
5
|
-
module Features
|
6
|
-
class ActiveJob < Base
|
7
|
-
# Rules around using ActiveJob routing - basically you need to have ActiveJob available
|
8
|
-
# in order to be able to use active job routing
|
9
|
-
class Contract < Contracts::Base
|
10
|
-
configure do |config|
|
11
|
-
config.error_messages = YAML.safe_load(
|
12
|
-
File.read(
|
13
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
14
|
-
)
|
15
|
-
).fetch('en').fetch('validations').fetch('topic')
|
16
|
-
end
|
17
|
-
|
18
|
-
virtual do |data, errors|
|
19
|
-
next unless errors.empty?
|
20
|
-
next unless data[:active_job][:active]
|
21
|
-
# One should not define active job jobs without ActiveJob being available for usage
|
22
|
-
next if Object.const_defined?('ActiveJob::Base')
|
23
|
-
|
24
|
-
[[%i[consumer], :active_job_missing]]
|
25
|
-
end
|
26
|
-
|
27
|
-
# ActiveJob needs to always run with manual offset management
|
28
|
-
# Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
|
29
|
-
# consumed jobs that did not run because of shutdown.
|
30
|
-
virtual do |data, errors|
|
31
|
-
next unless errors.empty?
|
32
|
-
next unless data[:active_job][:active]
|
33
|
-
next if data[:manual_offset_management][:active]
|
34
|
-
|
35
|
-
[[%i[manual_offset_management], :must_be_enabled]]
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Routing
|
5
|
-
module Features
|
6
|
-
class DeadLetterQueue < Base
|
7
|
-
# Rules around dead letter queue settings
|
8
|
-
class Contract < Contracts::Base
|
9
|
-
configure do |config|
|
10
|
-
config.error_messages = YAML.safe_load(
|
11
|
-
File.read(
|
12
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
13
|
-
)
|
14
|
-
).fetch('en').fetch('validations').fetch('topic')
|
15
|
-
end
|
16
|
-
|
17
|
-
nested :dead_letter_queue do
|
18
|
-
required(:active) { |val| [true, false].include?(val) }
|
19
|
-
required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
|
20
|
-
end
|
21
|
-
|
22
|
-
# Validate topic name only if dlq is active
|
23
|
-
virtual do |data, errors|
|
24
|
-
next unless errors.empty?
|
25
|
-
|
26
|
-
dead_letter_queue = data[:dead_letter_queue]
|
27
|
-
|
28
|
-
next unless dead_letter_queue[:active]
|
29
|
-
|
30
|
-
topic = dead_letter_queue[:topic]
|
31
|
-
|
32
|
-
# When topic is set to false, it means we just want to skip dispatch on DLQ
|
33
|
-
next if topic == false
|
34
|
-
next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
|
35
|
-
|
36
|
-
[[%i[dead_letter_queue topic], :format]]
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
@@ -1,30 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Routing
|
5
|
-
module Features
|
6
|
-
class Declaratives < Base
|
7
|
-
# Basic validation of the Kafka expected config details
|
8
|
-
class Contract < Contracts::Base
|
9
|
-
configure do |config|
|
10
|
-
config.error_messages = YAML.safe_load(
|
11
|
-
File.read(
|
12
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
13
|
-
)
|
14
|
-
).fetch('en').fetch('validations').fetch('topic')
|
15
|
-
end
|
16
|
-
|
17
|
-
nested :declaratives do
|
18
|
-
required(:active) { |val| [true, false].include?(val) }
|
19
|
-
required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
|
20
|
-
required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
|
21
|
-
required(:details) do |val|
|
22
|
-
val.is_a?(Hash) &&
|
23
|
-
val.keys.all? { |key| key.is_a?(Symbol) }
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Routing
|
5
|
-
module Features
|
6
|
-
class ManualOffsetManagement < Base
|
7
|
-
# Rules around manual offset management settings
|
8
|
-
class Contract < Contracts::Base
|
9
|
-
configure do |config|
|
10
|
-
config.error_messages = YAML.safe_load(
|
11
|
-
File.read(
|
12
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
13
|
-
)
|
14
|
-
).fetch('en').fetch('validations').fetch('topic')
|
15
|
-
end
|
16
|
-
|
17
|
-
nested :manual_offset_management do
|
18
|
-
required(:active) { |val| [true, false].include?(val) }
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|