karafka 2.1.12 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +41 -0
  4. data/Gemfile.lock +1 -1
  5. data/bin/record_rss +50 -0
  6. data/config/locales/errors.yml +4 -0
  7. data/config/locales/pro_errors.yml +17 -0
  8. data/lib/karafka/admin.rb +21 -33
  9. data/lib/karafka/connection/client.rb +1 -1
  10. data/lib/karafka/contracts/config.rb +24 -0
  11. data/lib/karafka/errors.rb +3 -0
  12. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +5 -2
  13. data/lib/karafka/messages/builders/message.rb +8 -4
  14. data/lib/karafka/pro/active_job/consumer.rb +1 -1
  15. data/lib/karafka/pro/cleaner/errors.rb +27 -0
  16. data/lib/karafka/pro/cleaner/messages/message.rb +46 -0
  17. data/lib/karafka/pro/cleaner/messages/messages.rb +42 -0
  18. data/lib/karafka/pro/cleaner.rb +41 -0
  19. data/lib/karafka/pro/contracts/base.rb +23 -0
  20. data/lib/karafka/pro/contracts/server_cli_options.rb +111 -0
  21. data/lib/karafka/pro/encryption/errors.rb +4 -1
  22. data/lib/karafka/pro/loader.rb +6 -2
  23. data/lib/karafka/pro/processing/strategies/dlq/default.rb +6 -0
  24. data/lib/karafka/pro/routing/features/active_job/builder.rb +45 -0
  25. data/lib/karafka/pro/routing/features/active_job.rb +26 -0
  26. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +53 -0
  27. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +41 -0
  28. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +41 -0
  29. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +44 -0
  30. data/lib/karafka/pro/routing/features/long_running_job/{contract.rb → contracts/topic.rb} +14 -11
  31. data/lib/karafka/pro/routing/features/{filtering/contract.rb → patterns/builder.rb} +13 -16
  32. data/lib/karafka/pro/routing/features/patterns/config.rb +54 -0
  33. data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +68 -0
  34. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +62 -0
  35. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +46 -0
  36. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +41 -0
  37. data/lib/karafka/pro/routing/features/patterns/detector.rb +68 -0
  38. data/lib/karafka/pro/routing/features/patterns/pattern.rb +81 -0
  39. data/lib/karafka/pro/routing/features/{delaying/contract.rb → patterns/patterns.rb} +11 -14
  40. data/lib/karafka/pro/routing/features/patterns/topic.rb +50 -0
  41. data/lib/karafka/pro/routing/features/patterns/topics.rb +53 -0
  42. data/lib/karafka/pro/routing/features/patterns.rb +33 -0
  43. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +51 -0
  44. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +44 -0
  45. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +55 -0
  46. data/lib/karafka/routing/consumer_group.rb +1 -1
  47. data/lib/karafka/routing/features/active_job/contracts/topic.rb +44 -0
  48. data/lib/karafka/routing/features/active_job/proxy.rb +14 -0
  49. data/lib/karafka/routing/features/base/expander.rb +8 -2
  50. data/lib/karafka/routing/features/base.rb +4 -2
  51. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +46 -0
  52. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +33 -0
  53. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +27 -0
  54. data/lib/karafka/routing/router.rb +0 -11
  55. data/lib/karafka/routing/subscription_group.rb +9 -0
  56. data/lib/karafka/routing/topic.rb +5 -0
  57. data/lib/karafka/server.rb +9 -4
  58. data/lib/karafka/setup/config.rb +45 -0
  59. data/lib/karafka/version.rb +1 -1
  60. data.tar.gz.sig +0 -0
  61. metadata +37 -15
  62. metadata.gz.sig +0 -0
  63. data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +0 -50
  64. data/lib/karafka/pro/routing/features/expiring/contract.rb +0 -38
  65. data/lib/karafka/pro/routing/features/pausing/contract.rb +0 -48
  66. data/lib/karafka/pro/routing/features/throttling/contract.rb +0 -41
  67. data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +0 -52
  68. data/lib/karafka/routing/features/active_job/contract.rb +0 -41
  69. data/lib/karafka/routing/features/dead_letter_queue/contract.rb +0 -42
  70. data/lib/karafka/routing/features/declaratives/contract.rb +0 -30
  71. data/lib/karafka/routing/features/manual_offset_management/contract.rb +0 -24
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Routing
5
+ module Features
6
+ class DeadLetterQueue < Base
7
+ # This feature validation contracts
8
+ module Contracts
9
+ # Rules around dead letter queue settings
10
+ class Topic < Karafka::Contracts::Base
11
+ configure do |config|
12
+ config.error_messages = YAML.safe_load(
13
+ File.read(
14
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
15
+ )
16
+ ).fetch('en').fetch('validations').fetch('topic')
17
+ end
18
+
19
+ nested :dead_letter_queue do
20
+ required(:active) { |val| [true, false].include?(val) }
21
+ required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
22
+ end
23
+
24
+ # Validate topic name only if dlq is active
25
+ virtual do |data, errors|
26
+ next unless errors.empty?
27
+
28
+ dead_letter_queue = data[:dead_letter_queue]
29
+
30
+ next unless dead_letter_queue[:active]
31
+
32
+ topic = dead_letter_queue[:topic]
33
+ topic_regexp = ::Karafka::Contracts::TOPIC_REGEXP
34
+
35
+ # When topic is set to false, it means we just want to skip dispatch on DLQ
36
+ next if topic == false
37
+ next if topic.is_a?(String) && topic_regexp.match?(topic)
38
+
39
+ [[%i[dead_letter_queue topic], :format]]
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Routing
5
+ module Features
6
+ class Declaratives < Base
7
+ # This feature validation contracts
8
+ module Contracts
9
+ # Basic validation of the Kafka expected config details
10
+ class Topic < Karafka::Contracts::Base
11
+ configure do |config|
12
+ config.error_messages = YAML.safe_load(
13
+ File.read(
14
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
15
+ )
16
+ ).fetch('en').fetch('validations').fetch('topic')
17
+ end
18
+
19
+ nested :declaratives do
20
+ required(:active) { |val| [true, false].include?(val) }
21
+ required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
22
+ required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
23
+ required(:details) do |val|
24
+ val.is_a?(Hash) &&
25
+ val.keys.all? { |key| key.is_a?(Symbol) }
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Routing
5
+ module Features
6
+ class ManualOffsetManagement < Base
7
+ # This feature validation contracts
8
+ module Contracts
9
+ # Rules around manual offset management settings
10
+ class Topic < Karafka::Contracts::Base
11
+ configure do |config|
12
+ config.error_messages = YAML.safe_load(
13
+ File.read(
14
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
15
+ )
16
+ ).fetch('en').fetch('validations').fetch('topic')
17
+ end
18
+
19
+ nested :manual_offset_management do
20
+ required(:active) { |val| [true, false].include?(val) }
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -7,16 +7,6 @@ module Karafka
7
7
  # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
8
8
  # structure so all the routes are being stored in a single level array
9
9
  module Router
10
- # Find a proper topic based on full topic id
11
- # @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
12
- # routing topic
13
- # @return [Karafka::Routing::Topic] proper route details
14
- # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
15
- # any route defined by user using routes.draw
16
- def find(topic_id)
17
- find_by(id: topic_id) || raise(Errors::NonMatchingRouteError, topic_id)
18
- end
19
-
20
10
  # Finds first reference of a given topic based on provided lookup attribute
21
11
  # @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
22
12
  # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
@@ -46,7 +36,6 @@ module Karafka
46
36
  find_by(name: name) || Topic.new(name, ConsumerGroup.new(name))
47
37
  end
48
38
 
49
- module_function :find
50
39
  module_function :find_by
51
40
  module_function :find_or_initialize_by_name
52
41
  end
@@ -61,6 +61,15 @@ module Karafka
61
61
  Karafka::App.config.internal.routing.activity_manager.active?(:subscription_groups, name)
62
62
  end
63
63
 
64
+ # @return [Array<String>] names of topics to which we should subscribe.
65
+ #
66
+ # @note Most of the time it should not include inactive topics but in case of pattern
67
+ # matching the matcher topics become inactive down the road, hence we filter out so
68
+ # they are later removed.
69
+ def subscriptions
70
+ topics.select(&:active?).map(&:subscription_name)
71
+ end
72
+
64
73
  private
65
74
 
66
75
  # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
@@ -52,6 +52,11 @@ module Karafka
52
52
  end
53
53
  end
54
54
 
55
+ # @return [String] name of subscription that will go to librdkafka
56
+ def subscription_name
57
+ name
58
+ end
59
+
55
60
  # @return [Class] consumer class that we should use
56
61
  def consumer
57
62
  if consumer_persistence
@@ -32,8 +32,8 @@ module Karafka
32
32
  # embedded
33
33
  # We cannot validate this during the start because config needs to be populated and routes
34
34
  # need to be defined.
35
- Contracts::ServerCliOptions.new.validate!(
36
- Karafka::App.config.internal.routing.activity_manager.to_h
35
+ config.internal.cli.contract.validate!(
36
+ config.internal.routing.activity_manager.to_h
37
37
  )
38
38
 
39
39
  process.on_sigint { stop }
@@ -82,7 +82,7 @@ module Karafka
82
82
 
83
83
  Karafka::App.stop!
84
84
 
85
- timeout = Karafka::App.config.shutdown_timeout
85
+ timeout = config.shutdown_timeout
86
86
 
87
87
  # We check from time to time (for the timeout period) if all the threads finished
88
88
  # their work and if so, we can just return and normal shutdown process will take place
@@ -148,9 +148,14 @@ module Karafka
148
148
 
149
149
  private
150
150
 
151
+ # @return [Karafka::Core::Configurable::Node] root config node
152
+ def config
153
+ Karafka::App.config
154
+ end
155
+
151
156
  # @return [Karafka::Process] process wrapper instance used to catch system signal calls
152
157
  def process
153
- Karafka::App.config.internal.process
158
+ config.internal.process
154
159
  end
155
160
  end
156
161
  end
@@ -105,6 +105,44 @@ module Karafka
105
105
  # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
106
106
  setting :kafka, default: {}
107
107
 
108
+ # Admin specific settings.
109
+ #
110
+ # Since admin operations are often specific, they may require specific librdkafka settings
111
+ # or other settings that are unique to admin.
112
+ setting :admin do
113
+ # Specific kafka settings that are tuned to operate within the Admin.
114
+ #
115
+ # Please do not change them unless you know what you are doing as their misconfiguration
116
+ # may cause Admin API to misbehave
117
+ # option [Hash] extra changes to the default root kafka settings
118
+ setting :kafka, default: {
119
+ # We want to know when there is no more data not to end up with an endless loop
120
+ 'enable.partition.eof': true,
121
+ # Do not publish statistics from admin as they are not relevant
122
+ 'statistics.interval.ms': 0,
123
+ # Fetch at most 5 MBs when using admin
124
+ 'fetch.message.max.bytes': 5 * 1_048_576,
125
+ # Do not commit offset automatically, this prevents offset tracking for operations
126
+ # involving a consumer instance
127
+ 'enable.auto.commit': false,
128
+ # Make sure that topic metadata lookups do not create topics accidentally
129
+ 'allow.auto.create.topics': false
130
+ }
131
+
132
+ # option [String] default name for the admin consumer group. Please note, that this is a
133
+ # subject to be remapped by the consumer mapper as any other consumer group in the routes
134
+ setting :group_id, default: 'karafka_admin'
135
+
136
+ # option max_wait_time [Integer] We wait only for this amount of time before raising error
137
+ # as we intercept this error and retry after checking that the operation was finished or
138
+ # failed using external factor.
139
+ setting :max_wait_time, default: 1_000
140
+
141
+ # How many times should be try. 1 000 ms x 60 => 60 seconds wait in total and then we give
142
+ # up on pending operations
143
+ setting :max_attempts, default: 60
144
+ end
145
+
108
146
  # Namespace for internal settings that should not be modified directly
109
147
  setting :internal do
110
148
  # option status [Karafka::Status] app status
@@ -114,6 +152,13 @@ module Karafka
114
152
  # instances
115
153
  setting :process, default: Process.new
116
154
 
155
+ # Namespace for CLI related settings
156
+ setting :cli do
157
+ # option contract [Object] cli setup validation contract (in the context of options and
158
+ # topics)
159
+ setting :contract, default: Contracts::ServerCliOptions.new
160
+ end
161
+
117
162
  setting :routing do
118
163
  # option builder [Karafka::Routing::Builder] builder instance
119
164
  setting :builder, default: Routing::Builder.new
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.1.12'
6
+ VERSION = '2.2.0'
7
7
  end
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.12
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
35
35
  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
36
36
  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
37
37
  -----END CERTIFICATE-----
38
- date: 2023-08-25 00:00:00.000000000 Z
38
+ date: 2023-09-01 00:00:00.000000000 Z
39
39
  dependencies:
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: karafka-core
@@ -141,6 +141,7 @@ files:
141
141
  - bin/create_token
142
142
  - bin/integrations
143
143
  - bin/karafka
144
+ - bin/record_rss
144
145
  - bin/rspecs
145
146
  - bin/scenario
146
147
  - bin/stress_many
@@ -219,6 +220,12 @@ files:
219
220
  - lib/karafka/pro/active_job/consumer.rb
220
221
  - lib/karafka/pro/active_job/dispatcher.rb
221
222
  - lib/karafka/pro/active_job/job_options_contract.rb
223
+ - lib/karafka/pro/cleaner.rb
224
+ - lib/karafka/pro/cleaner/errors.rb
225
+ - lib/karafka/pro/cleaner/messages/message.rb
226
+ - lib/karafka/pro/cleaner/messages/messages.rb
227
+ - lib/karafka/pro/contracts/base.rb
228
+ - lib/karafka/pro/contracts/server_cli_options.rb
222
229
  - lib/karafka/pro/encryption.rb
223
230
  - lib/karafka/pro/encryption/cipher.rb
224
231
  - lib/karafka/pro/encryption/contracts/config.rb
@@ -296,35 +303,49 @@ files:
296
303
  - lib/karafka/pro/processing/strategies/vp/default.rb
297
304
  - lib/karafka/pro/processing/strategy_selector.rb
298
305
  - lib/karafka/pro/processing/virtual_offset_manager.rb
306
+ - lib/karafka/pro/routing/features/active_job.rb
307
+ - lib/karafka/pro/routing/features/active_job/builder.rb
299
308
  - lib/karafka/pro/routing/features/base.rb
300
309
  - lib/karafka/pro/routing/features/dead_letter_queue.rb
301
- - lib/karafka/pro/routing/features/dead_letter_queue/contract.rb
310
+ - lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb
302
311
  - lib/karafka/pro/routing/features/delaying.rb
303
312
  - lib/karafka/pro/routing/features/delaying/config.rb
304
- - lib/karafka/pro/routing/features/delaying/contract.rb
313
+ - lib/karafka/pro/routing/features/delaying/contracts/topic.rb
305
314
  - lib/karafka/pro/routing/features/delaying/topic.rb
306
315
  - lib/karafka/pro/routing/features/expiring.rb
307
316
  - lib/karafka/pro/routing/features/expiring/config.rb
308
- - lib/karafka/pro/routing/features/expiring/contract.rb
317
+ - lib/karafka/pro/routing/features/expiring/contracts/topic.rb
309
318
  - lib/karafka/pro/routing/features/expiring/topic.rb
310
319
  - lib/karafka/pro/routing/features/filtering.rb
311
320
  - lib/karafka/pro/routing/features/filtering/config.rb
312
- - lib/karafka/pro/routing/features/filtering/contract.rb
321
+ - lib/karafka/pro/routing/features/filtering/contracts/topic.rb
313
322
  - lib/karafka/pro/routing/features/filtering/topic.rb
314
323
  - lib/karafka/pro/routing/features/long_running_job.rb
315
324
  - lib/karafka/pro/routing/features/long_running_job/config.rb
316
- - lib/karafka/pro/routing/features/long_running_job/contract.rb
325
+ - lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb
317
326
  - lib/karafka/pro/routing/features/long_running_job/topic.rb
327
+ - lib/karafka/pro/routing/features/patterns.rb
328
+ - lib/karafka/pro/routing/features/patterns/builder.rb
329
+ - lib/karafka/pro/routing/features/patterns/config.rb
330
+ - lib/karafka/pro/routing/features/patterns/consumer_group.rb
331
+ - lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb
332
+ - lib/karafka/pro/routing/features/patterns/contracts/pattern.rb
333
+ - lib/karafka/pro/routing/features/patterns/contracts/topic.rb
334
+ - lib/karafka/pro/routing/features/patterns/detector.rb
335
+ - lib/karafka/pro/routing/features/patterns/pattern.rb
336
+ - lib/karafka/pro/routing/features/patterns/patterns.rb
337
+ - lib/karafka/pro/routing/features/patterns/topic.rb
338
+ - lib/karafka/pro/routing/features/patterns/topics.rb
318
339
  - lib/karafka/pro/routing/features/pausing.rb
319
- - lib/karafka/pro/routing/features/pausing/contract.rb
340
+ - lib/karafka/pro/routing/features/pausing/contracts/topic.rb
320
341
  - lib/karafka/pro/routing/features/pausing/topic.rb
321
342
  - lib/karafka/pro/routing/features/throttling.rb
322
343
  - lib/karafka/pro/routing/features/throttling/config.rb
323
- - lib/karafka/pro/routing/features/throttling/contract.rb
344
+ - lib/karafka/pro/routing/features/throttling/contracts/topic.rb
324
345
  - lib/karafka/pro/routing/features/throttling/topic.rb
325
346
  - lib/karafka/pro/routing/features/virtual_partitions.rb
326
347
  - lib/karafka/pro/routing/features/virtual_partitions/config.rb
327
- - lib/karafka/pro/routing/features/virtual_partitions/contract.rb
348
+ - lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb
328
349
  - lib/karafka/pro/routing/features/virtual_partitions/topic.rb
329
350
  - lib/karafka/process.rb
330
351
  - lib/karafka/processing/coordinator.rb
@@ -359,21 +380,22 @@ files:
359
380
  - lib/karafka/routing/features/active_job.rb
360
381
  - lib/karafka/routing/features/active_job/builder.rb
361
382
  - lib/karafka/routing/features/active_job/config.rb
362
- - lib/karafka/routing/features/active_job/contract.rb
383
+ - lib/karafka/routing/features/active_job/contracts/topic.rb
384
+ - lib/karafka/routing/features/active_job/proxy.rb
363
385
  - lib/karafka/routing/features/active_job/topic.rb
364
386
  - lib/karafka/routing/features/base.rb
365
387
  - lib/karafka/routing/features/base/expander.rb
366
388
  - lib/karafka/routing/features/dead_letter_queue.rb
367
389
  - lib/karafka/routing/features/dead_letter_queue/config.rb
368
- - lib/karafka/routing/features/dead_letter_queue/contract.rb
390
+ - lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb
369
391
  - lib/karafka/routing/features/dead_letter_queue/topic.rb
370
392
  - lib/karafka/routing/features/declaratives.rb
371
393
  - lib/karafka/routing/features/declaratives/config.rb
372
- - lib/karafka/routing/features/declaratives/contract.rb
394
+ - lib/karafka/routing/features/declaratives/contracts/topic.rb
373
395
  - lib/karafka/routing/features/declaratives/topic.rb
374
396
  - lib/karafka/routing/features/manual_offset_management.rb
375
397
  - lib/karafka/routing/features/manual_offset_management/config.rb
376
- - lib/karafka/routing/features/manual_offset_management/contract.rb
398
+ - lib/karafka/routing/features/manual_offset_management/contracts/topic.rb
377
399
  - lib/karafka/routing/features/manual_offset_management/topic.rb
378
400
  - lib/karafka/routing/proxy.rb
379
401
  - lib/karafka/routing/router.rb
@@ -424,7 +446,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
424
446
  - !ruby/object:Gem::Version
425
447
  version: '0'
426
448
  requirements: []
427
- rubygems_version: 3.4.10
449
+ rubygems_version: 3.4.19
428
450
  signing_key:
429
451
  specification_version: 4
430
452
  summary: Karafka is Ruby and Rails efficient Kafka processing framework.
metadata.gz.sig CHANGED
Binary file
@@ -1,50 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Routing
17
- module Features
18
- class DeadLetterQueue < Base
19
- # Extended rules for dead letter queue settings
20
- class Contract < Contracts::Base
21
- configure do |config|
22
- config.error_messages = YAML.safe_load(
23
- File.read(
24
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
25
- )
26
- ).fetch('en').fetch('validations').fetch('topic')
27
- end
28
-
29
- # Make sure that when we use virtual partitions with DLQ, at least one retry is set
30
- # We cannot use VP with DLQ without retries as we in order to provide ordering
31
- # warranties on errors with VP, we need to collapse the VPs concurrency and retry
32
- # without any indeterministic work
33
- virtual do |data, errors|
34
- next unless errors.empty?
35
-
36
- dead_letter_queue = data[:dead_letter_queue]
37
- virtual_partitions = data[:virtual_partitions]
38
-
39
- next unless dead_letter_queue[:active]
40
- next unless virtual_partitions[:active]
41
- next if dead_letter_queue[:max_retries].positive?
42
-
43
- [[%i[dead_letter_queue], :with_virtual_partitions]]
44
- end
45
- end
46
- end
47
- end
48
- end
49
- end
50
- end
@@ -1,38 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Routing
17
- module Features
18
- class Expiring < Base
19
- # Contract to validate configuration of the expiring feature
20
- class Contract < Contracts::Base
21
- configure do |config|
22
- config.error_messages = YAML.safe_load(
23
- File.read(
24
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
25
- )
26
- ).fetch('en').fetch('validations').fetch('topic')
27
- end
28
-
29
- nested(:expiring) do
30
- required(:active) { |val| [true, false].include?(val) }
31
- required(:ttl) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
32
- end
33
- end
34
- end
35
- end
36
- end
37
- end
38
- end
@@ -1,48 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Routing
17
- module Features
18
- class Pausing < Base
19
- # Contract to make sure, that the pause settings on a per topic basis are as expected
20
- class Contract < Contracts::Base
21
- configure do |config|
22
- config.error_messages = YAML.safe_load(
23
- File.read(
24
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
25
- )
26
- ).fetch('en').fetch('validations').fetch('topic')
27
-
28
- required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
29
- required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
30
- required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
31
-
32
- virtual do |data, errors|
33
- next unless errors.empty?
34
-
35
- pause_timeout = data.fetch(:pause_timeout)
36
- pause_max_timeout = data.fetch(:pause_max_timeout)
37
-
38
- next if pause_timeout <= pause_max_timeout
39
-
40
- [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
41
- end
42
- end
43
- end
44
- end
45
- end
46
- end
47
- end
48
- end
@@ -1,41 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Routing
17
- module Features
18
- class Throttling < Base
19
- # Rules around throttling settings
20
- class Contract < Contracts::Base
21
- configure do |config|
22
- config.error_messages = YAML.safe_load(
23
- File.read(
24
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
25
- )
26
- ).fetch('en').fetch('validations').fetch('topic')
27
- end
28
-
29
- nested(:throttling) do
30
- required(:active) { |val| [true, false].include?(val) }
31
- required(:interval) { |val| val.is_a?(Integer) && val.positive? }
32
- required(:limit) do |val|
33
- (val.is_a?(Integer) || val == Float::INFINITY) && val.positive?
34
- end
35
- end
36
- end
37
- end
38
- end
39
- end
40
- end
41
- end
@@ -1,52 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This Karafka component is a Pro component under a commercial license.
4
- # This Karafka component is NOT licensed under LGPL.
5
- #
6
- # All of the commercial components are present in the lib/karafka/pro directory of this
7
- # repository and their usage requires commercial license agreement.
8
- #
9
- # Karafka has also commercial-friendly license, commercial support and commercial components.
10
- #
11
- # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
- # your code to Maciej Mensfeld.
13
-
14
- module Karafka
15
- module Pro
16
- module Routing
17
- module Features
18
- class VirtualPartitions < Base
19
- # Rules around virtual partitions
20
- class Contract < Contracts::Base
21
- configure do |config|
22
- config.error_messages = YAML.safe_load(
23
- File.read(
24
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
25
- )
26
- ).fetch('en').fetch('validations').fetch('topic')
27
- end
28
-
29
- nested(:virtual_partitions) do
30
- required(:active) { |val| [true, false].include?(val) }
31
- required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
32
- required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
33
- end
34
-
35
- # When virtual partitions are defined, partitioner needs to respond to `#call` and it
36
- # cannot be nil
37
- virtual do |data, errors|
38
- next unless errors.empty?
39
-
40
- virtual_partitions = data[:virtual_partitions]
41
-
42
- next unless virtual_partitions[:active]
43
- next if virtual_partitions[:partitioner].respond_to?(:call)
44
-
45
- [[%i[virtual_partitions partitioner], :respond_to_call]]
46
- end
47
- end
48
- end
49
- end
50
- end
51
- end
52
- end