karafka 2.1.12 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +41 -0
  4. data/Gemfile.lock +1 -1
  5. data/bin/record_rss +50 -0
  6. data/config/locales/errors.yml +4 -0
  7. data/config/locales/pro_errors.yml +17 -0
  8. data/lib/karafka/admin.rb +21 -33
  9. data/lib/karafka/connection/client.rb +1 -1
  10. data/lib/karafka/contracts/config.rb +24 -0
  11. data/lib/karafka/errors.rb +3 -0
  12. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +5 -2
  13. data/lib/karafka/messages/builders/message.rb +8 -4
  14. data/lib/karafka/pro/active_job/consumer.rb +1 -1
  15. data/lib/karafka/pro/cleaner/errors.rb +27 -0
  16. data/lib/karafka/pro/cleaner/messages/message.rb +46 -0
  17. data/lib/karafka/pro/cleaner/messages/messages.rb +42 -0
  18. data/lib/karafka/pro/cleaner.rb +41 -0
  19. data/lib/karafka/pro/contracts/base.rb +23 -0
  20. data/lib/karafka/pro/contracts/server_cli_options.rb +111 -0
  21. data/lib/karafka/pro/encryption/errors.rb +4 -1
  22. data/lib/karafka/pro/loader.rb +6 -2
  23. data/lib/karafka/pro/processing/strategies/dlq/default.rb +6 -0
  24. data/lib/karafka/pro/routing/features/active_job/builder.rb +45 -0
  25. data/lib/karafka/pro/routing/features/active_job.rb +26 -0
  26. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +53 -0
  27. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +41 -0
  28. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +41 -0
  29. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +44 -0
  30. data/lib/karafka/pro/routing/features/long_running_job/{contract.rb → contracts/topic.rb} +14 -11
  31. data/lib/karafka/pro/routing/features/{filtering/contract.rb → patterns/builder.rb} +13 -16
  32. data/lib/karafka/pro/routing/features/patterns/config.rb +54 -0
  33. data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +68 -0
  34. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +62 -0
  35. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +46 -0
  36. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +41 -0
  37. data/lib/karafka/pro/routing/features/patterns/detector.rb +68 -0
  38. data/lib/karafka/pro/routing/features/patterns/pattern.rb +81 -0
  39. data/lib/karafka/pro/routing/features/{delaying/contract.rb → patterns/patterns.rb} +11 -14
  40. data/lib/karafka/pro/routing/features/patterns/topic.rb +50 -0
  41. data/lib/karafka/pro/routing/features/patterns/topics.rb +53 -0
  42. data/lib/karafka/pro/routing/features/patterns.rb +33 -0
  43. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +51 -0
  44. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +44 -0
  45. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +55 -0
  46. data/lib/karafka/routing/consumer_group.rb +1 -1
  47. data/lib/karafka/routing/features/active_job/contracts/topic.rb +44 -0
  48. data/lib/karafka/routing/features/active_job/proxy.rb +14 -0
  49. data/lib/karafka/routing/features/base/expander.rb +8 -2
  50. data/lib/karafka/routing/features/base.rb +4 -2
  51. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +46 -0
  52. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +33 -0
  53. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +27 -0
  54. data/lib/karafka/routing/router.rb +0 -11
  55. data/lib/karafka/routing/subscription_group.rb +9 -0
  56. data/lib/karafka/routing/topic.rb +5 -0
  57. data/lib/karafka/server.rb +9 -4
  58. data/lib/karafka/setup/config.rb +45 -0
  59. data/lib/karafka/version.rb +1 -1
  60. data.tar.gz.sig +0 -0
  61. metadata +37 -15
  62. metadata.gz.sig +0 -0
  63. data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +0 -50
  64. data/lib/karafka/pro/routing/features/expiring/contract.rb +0 -38
  65. data/lib/karafka/pro/routing/features/pausing/contract.rb +0 -48
  66. data/lib/karafka/pro/routing/features/throttling/contract.rb +0 -41
  67. data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +0 -52
  68. data/lib/karafka/routing/features/active_job/contract.rb +0 -41
  69. data/lib/karafka/routing/features/dead_letter_queue/contract.rb +0 -42
  70. data/lib/karafka/routing/features/declaratives/contract.rb +0 -30
  71. data/lib/karafka/routing/features/manual_offset_management/contract.rb +0 -24
@@ -1,41 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Routing
5
- module Features
6
- class ActiveJob < Base
7
- # Rules around using ActiveJob routing - basically you need to have ActiveJob available
8
- # in order to be able to use active job routing
9
- class Contract < Contracts::Base
10
- configure do |config|
11
- config.error_messages = YAML.safe_load(
12
- File.read(
13
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
- )
15
- ).fetch('en').fetch('validations').fetch('topic')
16
- end
17
-
18
- virtual do |data, errors|
19
- next unless errors.empty?
20
- next unless data[:active_job][:active]
21
- # One should not define active job jobs without ActiveJob being available for usage
22
- next if Object.const_defined?('ActiveJob::Base')
23
-
24
- [[%i[consumer], :active_job_missing]]
25
- end
26
-
27
- # ActiveJob needs to always run with manual offset management
28
- # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
29
- # consumed jobs that did not run because of shutdown.
30
- virtual do |data, errors|
31
- next unless errors.empty?
32
- next unless data[:active_job][:active]
33
- next if data[:manual_offset_management][:active]
34
-
35
- [[%i[manual_offset_management], :must_be_enabled]]
36
- end
37
- end
38
- end
39
- end
40
- end
41
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Routing
5
- module Features
6
- class DeadLetterQueue < Base
7
- # Rules around dead letter queue settings
8
- class Contract < Contracts::Base
9
- configure do |config|
10
- config.error_messages = YAML.safe_load(
11
- File.read(
12
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
- )
14
- ).fetch('en').fetch('validations').fetch('topic')
15
- end
16
-
17
- nested :dead_letter_queue do
18
- required(:active) { |val| [true, false].include?(val) }
19
- required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
20
- end
21
-
22
- # Validate topic name only if dlq is active
23
- virtual do |data, errors|
24
- next unless errors.empty?
25
-
26
- dead_letter_queue = data[:dead_letter_queue]
27
-
28
- next unless dead_letter_queue[:active]
29
-
30
- topic = dead_letter_queue[:topic]
31
-
32
- # When topic is set to false, it means we just want to skip dispatch on DLQ
33
- next if topic == false
34
- next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
35
-
36
- [[%i[dead_letter_queue topic], :format]]
37
- end
38
- end
39
- end
40
- end
41
- end
42
- end
@@ -1,30 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Routing
5
- module Features
6
- class Declaratives < Base
7
- # Basic validation of the Kafka expected config details
8
- class Contract < Contracts::Base
9
- configure do |config|
10
- config.error_messages = YAML.safe_load(
11
- File.read(
12
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
- )
14
- ).fetch('en').fetch('validations').fetch('topic')
15
- end
16
-
17
- nested :declaratives do
18
- required(:active) { |val| [true, false].include?(val) }
19
- required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
20
- required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
21
- required(:details) do |val|
22
- val.is_a?(Hash) &&
23
- val.keys.all? { |key| key.is_a?(Symbol) }
24
- end
25
- end
26
- end
27
- end
28
- end
29
- end
30
- end
@@ -1,24 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Routing
5
- module Features
6
- class ManualOffsetManagement < Base
7
- # Rules around manual offset management settings
8
- class Contract < Contracts::Base
9
- configure do |config|
10
- config.error_messages = YAML.safe_load(
11
- File.read(
12
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
- )
14
- ).fetch('en').fetch('validations').fetch('topic')
15
- end
16
-
17
- nested :manual_offset_management do
18
- required(:active) { |val| [true, false].include?(val) }
19
- end
20
- end
21
- end
22
- end
23
- end
24
- end