karafka 2.5.1 → 2.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +3 -29
  3. data/.github/workflows/ci_macos_arm64.yml +1 -1
  4. data/.github/workflows/push.yml +2 -2
  5. data/.github/workflows/trigger-wiki-refresh.yml +1 -1
  6. data/.ruby-version +1 -1
  7. data/CHANGELOG.md +14 -4
  8. data/Gemfile +0 -2
  9. data/Gemfile.lock +30 -31
  10. data/bin/integrations +2 -1
  11. data/bin/rspecs +4 -0
  12. data/config/locales/errors.yml +6 -4
  13. data/config/locales/pro_errors.yml +5 -4
  14. data/docker-compose.yml +1 -1
  15. data/examples/payloads/json/sample_set_02/download.json +191 -0
  16. data/examples/payloads/json/sample_set_03/event_type_1.json +18 -0
  17. data/examples/payloads/json/sample_set_03/event_type_2.json +263 -0
  18. data/examples/payloads/json/sample_set_03/event_type_3.json +41 -0
  19. data/karafka.gemspec +1 -1
  20. data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -1
  21. data/lib/karafka/active_job/consumer.rb +5 -1
  22. data/lib/karafka/active_job/current_attributes/job_wrapper.rb +45 -0
  23. data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
  24. data/lib/karafka/active_job/current_attributes/persistence.rb +19 -7
  25. data/lib/karafka/active_job/current_attributes.rb +1 -0
  26. data/lib/karafka/active_job/deserializer.rb +61 -0
  27. data/lib/karafka/active_job/dispatcher.rb +32 -12
  28. data/lib/karafka/active_job/job_options_contract.rb +2 -4
  29. data/lib/karafka/admin/acl.rb +8 -4
  30. data/lib/karafka/admin/configs/config.rb +6 -4
  31. data/lib/karafka/admin/consumer_groups.rb +74 -4
  32. data/lib/karafka/admin/topics.rb +40 -7
  33. data/lib/karafka/admin.rb +13 -4
  34. data/lib/karafka/base_consumer.rb +5 -5
  35. data/lib/karafka/cli/base.rb +1 -1
  36. data/lib/karafka/cli/contracts/server.rb +2 -4
  37. data/lib/karafka/cli/install.rb +1 -1
  38. data/lib/karafka/cli/topics/align.rb +1 -1
  39. data/lib/karafka/cli/topics/repartition.rb +2 -2
  40. data/lib/karafka/connection/client.rb +12 -2
  41. data/lib/karafka/connection/listeners_batch.rb +2 -3
  42. data/lib/karafka/connection/proxy.rb +11 -7
  43. data/lib/karafka/env.rb +1 -2
  44. data/lib/karafka/helpers/interval_runner.rb +4 -1
  45. data/lib/karafka/instrumentation/assignments_tracker.rb +17 -0
  46. data/lib/karafka/instrumentation/monitor.rb +1 -1
  47. data/lib/karafka/instrumentation/notifications.rb +1 -0
  48. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +2 -3
  49. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +2 -3
  50. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +8 -9
  51. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +2 -3
  52. data/lib/karafka/messages/builders/batch_metadata.rb +1 -1
  53. data/lib/karafka/messages/builders/message.rb +1 -1
  54. data/lib/karafka/messages/messages.rb +2 -3
  55. data/lib/karafka/patches/rdkafka/bindings.rb +6 -6
  56. data/lib/karafka/patches/rdkafka/opaque.rb +1 -1
  57. data/lib/karafka/pro/active_job/dispatcher.rb +7 -3
  58. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -4
  59. data/lib/karafka/pro/cleaner/messages/messages.rb +2 -3
  60. data/lib/karafka/pro/cli/contracts/server.rb +2 -4
  61. data/lib/karafka/pro/cli/parallel_segments/base.rb +1 -2
  62. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +2 -2
  63. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +2 -2
  64. data/lib/karafka/pro/connection/manager.rb +2 -2
  65. data/lib/karafka/pro/encryption/contracts/config.rb +4 -6
  66. data/lib/karafka/pro/encryption/messages/parser.rb +3 -3
  67. data/lib/karafka/pro/instrumentation/performance_tracker.rb +3 -3
  68. data/lib/karafka/pro/iterator/expander.rb +1 -1
  69. data/lib/karafka/pro/iterator/tpl_builder.rb +1 -1
  70. data/lib/karafka/pro/iterator.rb +2 -2
  71. data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -3
  72. data/lib/karafka/pro/processing/coordinators/filters_applier.rb +3 -3
  73. data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
  74. data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
  75. data/lib/karafka/pro/processing/filters/throttler.rb +1 -1
  76. data/lib/karafka/pro/processing/schedulers/default.rb +2 -4
  77. data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -4
  78. data/lib/karafka/pro/processing/strategies/vp/default.rb +2 -4
  79. data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -3
  80. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -4
  81. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -4
  82. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +6 -5
  83. data/lib/karafka/pro/recurring_tasks/schedule.rb +4 -6
  84. data/lib/karafka/pro/recurring_tasks.rb +8 -5
  85. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -4
  86. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
  87. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -4
  88. data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -4
  89. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +4 -8
  90. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +5 -7
  91. data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +7 -6
  92. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -2
  93. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -4
  94. data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -4
  95. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -4
  96. data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -3
  97. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -4
  98. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -4
  99. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +3 -5
  100. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +3 -3
  101. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -4
  102. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -4
  103. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +3 -5
  104. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -4
  105. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -4
  106. data/lib/karafka/pro/routing/features/pausing/config.rb +26 -0
  107. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +17 -11
  108. data/lib/karafka/pro/routing/features/pausing/topic.rb +69 -8
  109. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -4
  110. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -4
  111. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -4
  112. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -4
  113. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +6 -8
  114. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -4
  115. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -4
  116. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -4
  117. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +2 -4
  118. data/lib/karafka/pro/scheduled_messages.rb +4 -6
  119. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
  120. data/lib/karafka/processing/coordinator.rb +2 -4
  121. data/lib/karafka/processing/coordinators_buffer.rb +2 -3
  122. data/lib/karafka/processing/executor.rb +2 -3
  123. data/lib/karafka/processing/jobs/base.rb +2 -3
  124. data/lib/karafka/processing/workers_batch.rb +2 -3
  125. data/lib/karafka/railtie.rb +1 -0
  126. data/lib/karafka/routing/activity_manager.rb +2 -2
  127. data/lib/karafka/routing/builder.rb +5 -7
  128. data/lib/karafka/routing/consumer_group.rb +4 -6
  129. data/lib/karafka/routing/contracts/consumer_group.rb +3 -5
  130. data/lib/karafka/routing/contracts/routing.rb +2 -4
  131. data/lib/karafka/routing/contracts/topic.rb +2 -4
  132. data/lib/karafka/routing/features/active_job/contracts/topic.rb +2 -4
  133. data/lib/karafka/routing/features/active_job/topic.rb +6 -0
  134. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
  135. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +3 -5
  136. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -4
  137. data/lib/karafka/routing/features/eofed/contracts/topic.rb +2 -4
  138. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -4
  139. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -4
  140. data/lib/karafka/routing/topics.rb +4 -9
  141. data/lib/karafka/server.rb +1 -1
  142. data/lib/karafka/setup/config.rb +66 -9
  143. data/lib/karafka/setup/contracts/config.rb +12 -10
  144. data/lib/karafka/setup/defaults_injector.rb +3 -2
  145. data/lib/karafka/setup/dsl.rb +2 -3
  146. data/lib/karafka/swarm/liveness_listener.rb +2 -3
  147. data/lib/karafka/swarm/supervisor.rb +1 -1
  148. data/lib/karafka/version.rb +1 -1
  149. data/lib/karafka.rb +2 -2
  150. metadata +8 -2
  151. data/.diffend.yml +0 -3
@@ -13,6 +13,10 @@ module Karafka
13
13
  # Extension allowing us to select correct subscriptions and assignments based on the
14
14
  # expanded routing setup
15
15
  module SubscriptionGroup
16
+ include Helpers::ConfigImporter.new(
17
+ swarm_node: %i[swarm node]
18
+ )
19
+
16
20
  # @return [false, Array<String>] false if we do not have any subscriptions or array
17
21
  # with all the subscriptions for given subscription group
18
22
  def subscriptions
@@ -30,8 +34,7 @@ module Karafka
30
34
  topics
31
35
  .select(&:active?)
32
36
  .select { |topic| topic.direct_assignments.active? }
33
- .map { |topic| build_assignments(topic) }
34
- .to_h
37
+ .to_h { |topic| build_assignments(topic) }
35
38
  .tap { |topics| return false if topics.empty? }
36
39
  .then { |topics| Iterator::Expander.new.call(topics) }
37
40
  .then { |topics| Iterator::TplBuilder.new(consumer, topics).call }
@@ -44,21 +47,19 @@ module Karafka
44
47
  # @param topic [Karafka::Routing::Topic]
45
48
  # @return [Array<String, Hash>]
46
49
  def build_assignments(topic)
47
- node = Karafka::App.config.swarm.node
48
-
49
50
  standard_setup = [
50
51
  topic.subscription_name,
51
52
  topic.direct_assignments.partitions
52
53
  ]
53
54
 
54
- return standard_setup unless node
55
+ return standard_setup unless swarm_node
55
56
  # Unless user explicitly assigned particular partitions to particular nodes, we just
56
57
  # go with full regular assignments
57
58
  return standard_setup unless topic.swarm.nodes.is_a?(Hash)
58
59
 
59
60
  [
60
61
  topic.subscription_name,
61
- topic.swarm.nodes.fetch(node.id).map { |partition| [partition, true] }.to_h
62
+ topic.swarm.nodes.fetch(swarm_node.id).to_h { |partition| [partition, true] }
62
63
  ]
63
64
  end
64
65
  end
@@ -43,12 +43,12 @@ module Karafka
43
43
 
44
44
  Config.new(
45
45
  active: true,
46
- partitions: partitions_or_all.map { |partition| [partition, true] }.to_h
46
+ partitions: partitions_or_all.to_h { |partition| [partition, true] }
47
47
  )
48
48
  else
49
49
  Config.new(
50
50
  active: !partitions_or_all.empty?,
51
- partitions: partitions_or_all.map { |partition| [partition, true] }.to_h
51
+ partitions: partitions_or_all.to_h { |partition| [partition, true] }
52
52
  )
53
53
  end
54
54
  end
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the expiring feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -35,10 +35,8 @@ module Karafka
35
35
  end
36
36
 
37
37
  # Just an alias for nice API
38
- #
39
- # @param args [Array] Anything `#expiring` accepts
40
- def expire_in(*args)
41
- expiring(*args)
38
+ def expire_in(*)
39
+ expiring(*)
42
40
  end
43
41
 
44
42
  # @return [Boolean] is a given job expiring
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the filtering feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -27,10 +27,9 @@ module Karafka
27
27
  @filtering
28
28
  end
29
29
 
30
- # @param args [Array] Anything `#filter` accepts
31
30
  # @return [Filtering::Config] alias to match the naming API for features
32
- def filtering(*args)
33
- filter(*args)
31
+ def filtering(*)
32
+ filter(*)
34
33
  end
35
34
 
36
35
  # @return [Boolean] is a given job throttled
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract for inline insights topic setup
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around long-running job settings
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -17,10 +17,8 @@ module Karafka
17
17
  # multiplexing attributes are optional since multiplexing may not be enabled
18
18
  class Topic < Karafka::Contracts::Base
19
19
  configure do |config|
20
- config.error_messages = YAML.safe_load(
21
- File.read(
22
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
23
- )
20
+ config.error_messages = YAML.safe_load_file(
21
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
24
22
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
25
23
  end
26
24
 
@@ -56,7 +54,7 @@ module Karafka
56
54
  max = max(data)
57
55
  boot = boot(data)
58
56
 
59
- next if boot >= min && boot <= max
57
+ next if boot.between?(min, max)
60
58
 
61
59
  [[%w[subscription_group_details], :multiplexing_boot_mismatch]]
62
60
  end
@@ -10,9 +10,9 @@ module Karafka
10
10
  class NonBlockingJob < Base
11
11
  # Non-Blocking Jobs topic API extensions
12
12
  module Topic
13
- # @param args [Array] anything accepted by the `#long_running_job` API
14
- def non_blocking_job(*args)
15
- long_running_job(*args)
13
+ # Non blocking setting method
14
+ def non_blocking_job(*)
15
+ long_running_job(*)
16
16
  end
17
17
 
18
18
  alias non_blocking non_blocking_job
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the expiring feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the parallel segments feature
14
14
  class ConsumerGroup < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
21
19
 
22
20
  nested(:parallel_segments) do
@@ -12,13 +12,11 @@ module Karafka
12
12
  # Contract to validate configuration of the filtering feature
13
13
  class ConsumerGroup < Karafka::Contracts::Base
14
14
  configure do |config|
15
- config.error_messages = YAML.safe_load(
16
- File.read(
17
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
18
- )
15
+ config.error_messages = YAML.safe_load_file(
16
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
17
  ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
20
18
 
21
- required(:patterns) { |val| val.is_a?(Array) && val.all? { |el| el.is_a?(Hash) } }
19
+ required(:patterns) { |val| val.is_a?(Array) && val.all?(Hash) }
22
20
 
23
21
  virtual do |data, errors|
24
22
  next unless errors.empty?
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract used to validate pattern data
14
14
  class Pattern < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('pattern')
21
19
 
22
20
  required(:regexp) { |val| val.is_a?(Regexp) }
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the patterns feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
5
+
6
+ module Karafka
7
+ module Pro
8
+ module Routing
9
+ module Features
10
+ class Pausing < Base
11
+ # Config for pausing feature
12
+ Config = Struct.new(
13
+ :active,
14
+ :timeout,
15
+ :max_timeout,
16
+ :with_exponential_backoff,
17
+ keyword_init: true
18
+ ) do
19
+ alias_method :active?, :active
20
+ alias_method :with_exponential_backoff?, :with_exponential_backoff
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -13,25 +13,31 @@ module Karafka
13
13
  # Contract to make sure, that the pause settings on a per topic basis are as expected
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
 
22
- required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
23
- required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
24
- required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
20
+ # Validate the nested pausing configuration
21
+ # Both old setters and new pause() method update the pausing config,
22
+ # so we only need to validate this one format
23
+ nested :pausing do
24
+ required(:active) { |val| [true, false].include?(val) }
25
+ required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
26
+ required(:max_timeout) { |val| val.is_a?(Integer) && val.positive? }
27
+ required(:with_exponential_backoff) { |val| [true, false].include?(val) }
28
+ end
25
29
 
30
+ # Validate that timeout <= max_timeout
26
31
  virtual do |data, errors|
27
32
  next unless errors.empty?
28
33
 
29
- pause_timeout = data.fetch(:pause_timeout)
30
- pause_max_timeout = data.fetch(:pause_max_timeout)
34
+ pausing = data.fetch(:pausing)
35
+ timeout = pausing.fetch(:timeout)
36
+ max_timeout = pausing.fetch(:max_timeout)
31
37
 
32
- next if pause_timeout <= pause_max_timeout
38
+ next if timeout <= max_timeout
33
39
 
34
- [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
40
+ [[%i[pausing timeout], :max_timeout_vs_pause_max_timeout]]
35
41
  end
36
42
  end
37
43
  end
@@ -10,23 +10,84 @@ module Karafka
10
10
  class Pausing < Base
11
11
  # Expansion allowing for a per topic pause strategy definitions
12
12
  module Topic
13
+ # This method calls the parent class initializer and then sets up the
14
+ # extra instance variable to nil. The explicit initialization
15
+ # to nil is included as an optimization for Ruby's object shapes system,
16
+ # which improves memory layout and access performance.
17
+ def initialize(...)
18
+ super
19
+ @pausing = nil
20
+ end
21
+
13
22
  # Allows for per-topic pausing strategy setting
14
23
  #
15
24
  # @param timeout [Integer] how long should we wait upon processing error (milliseconds)
16
25
  # @param max_timeout [Integer] what is the max timeout in case of an exponential
17
26
  # backoff (milliseconds)
18
27
  # @param with_exponential_backoff [Boolean] should we use exponential backoff
19
- #
20
- # @note We do not construct here the nested config like we do with other routing
21
- # features, because this feature operates on the OSS layer by injection of values
22
- # and a nested config is not needed.
28
+ # @return [Config] pausing config object
23
29
  def pause(timeout: nil, max_timeout: nil, with_exponential_backoff: nil)
24
- self.pause_timeout = timeout if timeout
25
- self.pause_max_timeout = max_timeout if max_timeout
30
+ # If no arguments provided, just return or initialize the config
31
+ return pausing if timeout.nil? && max_timeout.nil? && with_exponential_backoff.nil?
32
+
33
+ # Update instance variables for backwards compatibility
34
+ # This ensures code reading @pause_timeout directly or via the inherited getter
35
+ # will get the correct values
36
+ @pause_timeout = timeout if timeout
37
+ @pause_max_timeout = max_timeout if max_timeout
38
+
39
+ unless with_exponential_backoff.nil?
40
+ @pause_with_exponential_backoff = with_exponential_backoff
41
+ end
42
+
43
+ # Create or update the config
44
+ @pausing ||= Config.new(
45
+ active: false,
46
+ timeout: @pause_timeout || Karafka::App.config.pause.timeout,
47
+ max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
48
+ with_exponential_backoff: if @pause_with_exponential_backoff.nil?
49
+ Karafka::App.config.pause.with_exponential_backoff
50
+ else
51
+ @pause_with_exponential_backoff
52
+ end
53
+ )
54
+
55
+ @pausing.timeout = timeout if timeout
56
+ @pausing.max_timeout = max_timeout if max_timeout
26
57
 
27
- return unless with_exponential_backoff
58
+ unless with_exponential_backoff.nil?
59
+ @pausing.with_exponential_backoff = with_exponential_backoff
60
+ end
61
+
62
+ @pausing.active = true
63
+
64
+ @pausing
65
+ end
66
+
67
+ # @return [Config] pausing configuration object
68
+ def pausing
69
+ @pausing ||= Config.new(
70
+ active: false,
71
+ timeout: @pause_timeout || Karafka::App.config.pause.timeout,
72
+ max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
73
+ with_exponential_backoff: if @pause_with_exponential_backoff.nil?
74
+ Karafka::App.config.pause.with_exponential_backoff
75
+ else
76
+ @pause_with_exponential_backoff
77
+ end
78
+ )
79
+ end
80
+
81
+ # @return [Boolean] is pausing explicitly configured
82
+ def pausing?
83
+ pausing.active?
84
+ end
28
85
 
29
- self.pause_with_exponential_backoff = with_exponential_backoff
86
+ # @return [Hash] topic with all its native configuration options plus pausing settings
87
+ def to_h
88
+ super.merge(
89
+ pausing: pausing.to_h
90
+ ).freeze
30
91
  end
31
92
  end
32
93
  end
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the periodics feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around recurring tasks settings
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around scheduled messages settings
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -18,10 +18,8 @@ module Karafka
18
18
  # This is why we use it before warmup when everything is expected to be configured.
19
19
  class Routing < Karafka::Contracts::Base
20
20
  configure do |config|
21
- config.error_messages = YAML.safe_load(
22
- File.read(
23
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
24
- )
21
+ config.error_messages = YAML.safe_load_file(
22
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
25
23
  ).fetch('en').fetch('validations').fetch('routing')
26
24
  end
27
25
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the swarm feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -25,15 +23,15 @@ module Karafka
25
23
 
26
24
  required(:nodes) do |val|
27
25
  next true if val.is_a?(Range)
28
- next true if val.is_a?(Array) && val.all? { |id| id.is_a?(Integer) }
26
+ next true if val.is_a?(Array) && val.all?(Integer)
29
27
  next false unless val.is_a?(Hash)
30
- next false unless val.keys.all? { |id| id.is_a?(Integer) }
28
+ next false unless val.keys.all?(Integer)
31
29
 
32
30
  values = val.values
33
31
 
34
32
  next false unless values.all? { |ps| ps.is_a?(Array) || ps.is_a?(Range) }
35
- next true if values.flatten.all? { |part| part.is_a?(Integer) }
36
- next true if values.flatten.all? { |part| part.is_a?(Range) }
33
+ next true if values.flatten.all?(Integer)
34
+ next true if values.flatten.all?(Range)
37
35
 
38
36
  false
39
37
  end
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around throttling settings
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around virtual partitions
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -11,10 +11,8 @@ module Karafka
11
11
  # Makes sure, all the expected config is defined as it should be
12
12
  class Config < ::Karafka::Contracts::Base
13
13
  configure do |config|
14
- config.error_messages = YAML.safe_load(
15
- File.read(
16
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
17
- )
14
+ config.error_messages = YAML.safe_load_file(
15
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
18
16
  ).fetch('en').fetch('validations').fetch('setup').fetch('config')
19
17
  end
20
18
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # details to be able to dispatch the message
14
14
  class Message < ::Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('scheduled_messages_message')
21
19
  end
22
20
 
@@ -24,17 +24,15 @@ module Karafka
24
24
 
25
25
  class << self
26
26
  # Runs the `Proxy.call`
27
- # @param kwargs [Hash] things requested by the proxy
28
27
  # @return [Hash] message wrapped with the scheduled message envelope
29
- def schedule(**kwargs)
30
- Proxy.schedule(**kwargs)
28
+ def schedule(**)
29
+ Proxy.schedule(**)
31
30
  end
32
31
 
33
32
  # Generates a tombstone message to cancel given dispatch (if not yet happened)
34
- # @param kwargs [Hash] things requested by the proxy
35
33
  # @return [Hash] tombstone cancelling message
36
- def cancel(**kwargs)
37
- Proxy.cancel(**kwargs)
34
+ def cancel(**)
35
+ Proxy.cancel(**)
38
36
  end
39
37
 
40
38
  # Below are private APIs
@@ -171,14 +171,14 @@ module Karafka
171
171
  def rss_mb_linux
172
172
  kb_rss = 0
173
173
 
174
- IO.readlines("/proc/#{node.pid}/status").each do |line|
174
+ File.readlines("/proc/#{node.pid}/status").each do |line|
175
175
  next unless line.start_with?('VmRSS:')
176
176
 
177
177
  kb_rss = line.split[1].to_i
178
178
  break
179
179
  end
180
180
 
181
- (kb_rss / 1_024.to_i).round
181
+ (kb_rss / 1_024).round
182
182
  rescue Errno::ENOENT, Errno::EACCES
183
183
  # /proc file doesn't exist or no permission to read it
184
184
  0
@@ -198,8 +198,6 @@ module Karafka
198
198
 
199
199
  # Allows to run synchronized (locked) code that can operate only from a given thread
200
200
  #
201
- # @param block [Proc] code we want to run in the synchronized mode
202
- #
203
201
  # @note We check if mutex is not owned already by the current thread so we won't end up with
204
202
  # a deadlock in case user runs coordinated code from inside of his own lock
205
203
  #
@@ -207,11 +205,11 @@ module Karafka
207
205
  # Otherwise user indirectly could cause deadlocks or prolonged locks by running his logic.
208
206
  # This can and should however be used for multi-thread strategy applications and other
209
207
  # internal operations locks.
210
- def synchronize(&block)
208
+ def synchronize(&)
211
209
  if @mutex.owned?
212
210
  yield
213
211
  else
214
- @mutex.synchronize(&block)
212
+ @mutex.synchronize(&)
215
213
  end
216
214
  end
217
215
  end