karafka 2.1.12 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +41 -0
  4. data/Gemfile.lock +1 -1
  5. data/bin/record_rss +50 -0
  6. data/config/locales/errors.yml +4 -0
  7. data/config/locales/pro_errors.yml +17 -0
  8. data/lib/karafka/admin.rb +21 -33
  9. data/lib/karafka/connection/client.rb +1 -1
  10. data/lib/karafka/contracts/config.rb +24 -0
  11. data/lib/karafka/errors.rb +3 -0
  12. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +5 -2
  13. data/lib/karafka/messages/builders/message.rb +8 -4
  14. data/lib/karafka/pro/active_job/consumer.rb +1 -1
  15. data/lib/karafka/pro/cleaner/errors.rb +27 -0
  16. data/lib/karafka/pro/cleaner/messages/message.rb +46 -0
  17. data/lib/karafka/pro/cleaner/messages/messages.rb +42 -0
  18. data/lib/karafka/pro/cleaner.rb +41 -0
  19. data/lib/karafka/pro/contracts/base.rb +23 -0
  20. data/lib/karafka/pro/contracts/server_cli_options.rb +111 -0
  21. data/lib/karafka/pro/encryption/errors.rb +4 -1
  22. data/lib/karafka/pro/loader.rb +6 -2
  23. data/lib/karafka/pro/processing/strategies/dlq/default.rb +6 -0
  24. data/lib/karafka/pro/routing/features/active_job/builder.rb +45 -0
  25. data/lib/karafka/pro/routing/features/active_job.rb +26 -0
  26. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +53 -0
  27. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +41 -0
  28. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +41 -0
  29. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +44 -0
  30. data/lib/karafka/pro/routing/features/long_running_job/{contract.rb → contracts/topic.rb} +14 -11
  31. data/lib/karafka/pro/routing/features/{filtering/contract.rb → patterns/builder.rb} +13 -16
  32. data/lib/karafka/pro/routing/features/patterns/config.rb +54 -0
  33. data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +68 -0
  34. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +62 -0
  35. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +46 -0
  36. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +41 -0
  37. data/lib/karafka/pro/routing/features/patterns/detector.rb +68 -0
  38. data/lib/karafka/pro/routing/features/patterns/pattern.rb +81 -0
  39. data/lib/karafka/pro/routing/features/{delaying/contract.rb → patterns/patterns.rb} +11 -14
  40. data/lib/karafka/pro/routing/features/patterns/topic.rb +50 -0
  41. data/lib/karafka/pro/routing/features/patterns/topics.rb +53 -0
  42. data/lib/karafka/pro/routing/features/patterns.rb +33 -0
  43. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +51 -0
  44. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +44 -0
  45. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +55 -0
  46. data/lib/karafka/routing/consumer_group.rb +1 -1
  47. data/lib/karafka/routing/features/active_job/contracts/topic.rb +44 -0
  48. data/lib/karafka/routing/features/active_job/proxy.rb +14 -0
  49. data/lib/karafka/routing/features/base/expander.rb +8 -2
  50. data/lib/karafka/routing/features/base.rb +4 -2
  51. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +46 -0
  52. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +33 -0
  53. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +27 -0
  54. data/lib/karafka/routing/router.rb +0 -11
  55. data/lib/karafka/routing/subscription_group.rb +9 -0
  56. data/lib/karafka/routing/topic.rb +5 -0
  57. data/lib/karafka/server.rb +9 -4
  58. data/lib/karafka/setup/config.rb +45 -0
  59. data/lib/karafka/version.rb +1 -1
  60. data.tar.gz.sig +0 -0
  61. metadata +37 -15
  62. metadata.gz.sig +0 -0
  63. data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +0 -50
  64. data/lib/karafka/pro/routing/features/expiring/contract.rb +0 -38
  65. data/lib/karafka/pro/routing/features/pausing/contract.rb +0 -48
  66. data/lib/karafka/pro/routing/features/throttling/contract.rb +0 -41
  67. data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +0 -52
  68. data/lib/karafka/routing/features/active_job/contract.rb +0 -41
  69. data/lib/karafka/routing/features/dead_letter_queue/contract.rb +0 -42
  70. data/lib/karafka/routing/features/declaratives/contract.rb +0 -30
  71. data/lib/karafka/routing/features/manual_offset_management/contract.rb +0 -24
@@ -1,41 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Routing
5
- module Features
6
- class ActiveJob < Base
7
- # Rules around using ActiveJob routing - basically you need to have ActiveJob available
8
- # in order to be able to use active job routing
9
- class Contract < Contracts::Base
10
- configure do |config|
11
- config.error_messages = YAML.safe_load(
12
- File.read(
13
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
- )
15
- ).fetch('en').fetch('validations').fetch('topic')
16
- end
17
-
18
- virtual do |data, errors|
19
- next unless errors.empty?
20
- next unless data[:active_job][:active]
21
- # One should not define active job jobs without ActiveJob being available for usage
22
- next if Object.const_defined?('ActiveJob::Base')
23
-
24
- [[%i[consumer], :active_job_missing]]
25
- end
26
-
27
- # ActiveJob needs to always run with manual offset management
28
- # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
29
- # consumed jobs that did not run because of shutdown.
30
- virtual do |data, errors|
31
- next unless errors.empty?
32
- next unless data[:active_job][:active]
33
- next if data[:manual_offset_management][:active]
34
-
35
- [[%i[manual_offset_management], :must_be_enabled]]
36
- end
37
- end
38
- end
39
- end
40
- end
41
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Routing
5
- module Features
6
- class DeadLetterQueue < Base
7
- # Rules around dead letter queue settings
8
- class Contract < Contracts::Base
9
- configure do |config|
10
- config.error_messages = YAML.safe_load(
11
- File.read(
12
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
- )
14
- ).fetch('en').fetch('validations').fetch('topic')
15
- end
16
-
17
- nested :dead_letter_queue do
18
- required(:active) { |val| [true, false].include?(val) }
19
- required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
20
- end
21
-
22
- # Validate topic name only if dlq is active
23
- virtual do |data, errors|
24
- next unless errors.empty?
25
-
26
- dead_letter_queue = data[:dead_letter_queue]
27
-
28
- next unless dead_letter_queue[:active]
29
-
30
- topic = dead_letter_queue[:topic]
31
-
32
- # When topic is set to false, it means we just want to skip dispatch on DLQ
33
- next if topic == false
34
- next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
35
-
36
- [[%i[dead_letter_queue topic], :format]]
37
- end
38
- end
39
- end
40
- end
41
- end
42
- end
@@ -1,30 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Routing
5
- module Features
6
- class Declaratives < Base
7
- # Basic validation of the Kafka expected config details
8
- class Contract < Contracts::Base
9
- configure do |config|
10
- config.error_messages = YAML.safe_load(
11
- File.read(
12
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
- )
14
- ).fetch('en').fetch('validations').fetch('topic')
15
- end
16
-
17
- nested :declaratives do
18
- required(:active) { |val| [true, false].include?(val) }
19
- required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
20
- required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
21
- required(:details) do |val|
22
- val.is_a?(Hash) &&
23
- val.keys.all? { |key| key.is_a?(Symbol) }
24
- end
25
- end
26
- end
27
- end
28
- end
29
- end
30
- end
@@ -1,24 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- module Routing
5
- module Features
6
- class ManualOffsetManagement < Base
7
- # Rules around manual offset management settings
8
- class Contract < Contracts::Base
9
- configure do |config|
10
- config.error_messages = YAML.safe_load(
11
- File.read(
12
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
- )
14
- ).fetch('en').fetch('validations').fetch('topic')
15
- end
16
-
17
- nested :manual_offset_management do
18
- required(:active) { |val| [true, false].include?(val) }
19
- end
20
- end
21
- end
22
- end
23
- end
24
- end