karafka 2.5.1 → 2.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +21 -29
  3. data/.github/workflows/ci_macos_arm64.yml +1 -1
  4. data/.github/workflows/push.yml +2 -2
  5. data/.github/workflows/trigger-wiki-refresh.yml +1 -1
  6. data/.ruby-version +1 -1
  7. data/.yard-lint.yml +174 -0
  8. data/CHANGELOG.md +20 -4
  9. data/Gemfile +1 -2
  10. data/Gemfile.lock +45 -41
  11. data/bin/integrations +2 -1
  12. data/bin/rspecs +4 -0
  13. data/config/locales/errors.yml +6 -4
  14. data/config/locales/pro_errors.yml +5 -4
  15. data/docker-compose.yml +1 -1
  16. data/examples/payloads/json/sample_set_02/download.json +191 -0
  17. data/examples/payloads/json/sample_set_03/event_type_1.json +18 -0
  18. data/examples/payloads/json/sample_set_03/event_type_2.json +263 -0
  19. data/examples/payloads/json/sample_set_03/event_type_3.json +41 -0
  20. data/karafka.gemspec +3 -3
  21. data/lib/active_job/queue_adapters/karafka_adapter.rb +3 -3
  22. data/lib/karafka/active_job/consumer.rb +7 -3
  23. data/lib/karafka/active_job/current_attributes/job_wrapper.rb +45 -0
  24. data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
  25. data/lib/karafka/active_job/current_attributes/persistence.rb +19 -7
  26. data/lib/karafka/active_job/current_attributes.rb +3 -2
  27. data/lib/karafka/active_job/deserializer.rb +61 -0
  28. data/lib/karafka/active_job/dispatcher.rb +34 -14
  29. data/lib/karafka/active_job/job_options_contract.rb +2 -4
  30. data/lib/karafka/admin/acl.rb +8 -4
  31. data/lib/karafka/admin/configs/config.rb +6 -4
  32. data/lib/karafka/admin/configs/resource.rb +7 -1
  33. data/lib/karafka/admin/consumer_groups.rb +80 -12
  34. data/lib/karafka/admin/topics.rb +43 -9
  35. data/lib/karafka/admin.rb +23 -14
  36. data/lib/karafka/app.rb +3 -3
  37. data/lib/karafka/base_consumer.rb +6 -6
  38. data/lib/karafka/cli/base.rb +2 -2
  39. data/lib/karafka/cli/console.rb +1 -1
  40. data/lib/karafka/cli/contracts/server.rb +3 -5
  41. data/lib/karafka/cli/help.rb +1 -1
  42. data/lib/karafka/cli/install.rb +3 -2
  43. data/lib/karafka/cli/server.rb +1 -1
  44. data/lib/karafka/cli/swarm.rb +1 -1
  45. data/lib/karafka/cli/topics/align.rb +1 -1
  46. data/lib/karafka/cli/topics/repartition.rb +2 -2
  47. data/lib/karafka/connection/client.rb +30 -19
  48. data/lib/karafka/connection/listeners_batch.rb +2 -3
  49. data/lib/karafka/connection/manager.rb +1 -0
  50. data/lib/karafka/connection/proxy.rb +12 -8
  51. data/lib/karafka/connection/rebalance_manager.rb +1 -1
  52. data/lib/karafka/connection/status.rb +1 -0
  53. data/lib/karafka/constraints.rb +1 -1
  54. data/lib/karafka/contracts/base.rb +1 -1
  55. data/lib/karafka/deserializers/payload.rb +1 -1
  56. data/lib/karafka/env.rb +1 -2
  57. data/lib/karafka/helpers/async.rb +1 -1
  58. data/lib/karafka/helpers/config_importer.rb +3 -3
  59. data/lib/karafka/helpers/interval_runner.rb +4 -1
  60. data/lib/karafka/helpers/multi_delegator.rb +3 -0
  61. data/lib/karafka/instrumentation/assignments_tracker.rb +19 -1
  62. data/lib/karafka/instrumentation/callbacks/error.rb +2 -2
  63. data/lib/karafka/instrumentation/callbacks/statistics.rb +3 -3
  64. data/lib/karafka/instrumentation/logger.rb +6 -6
  65. data/lib/karafka/instrumentation/monitor.rb +3 -3
  66. data/lib/karafka/instrumentation/notifications.rb +1 -0
  67. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +3 -4
  68. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +3 -4
  69. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +10 -11
  70. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +1 -1
  71. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +5 -18
  72. data/lib/karafka/messages/builders/batch_metadata.rb +2 -2
  73. data/lib/karafka/messages/builders/message.rb +1 -1
  74. data/lib/karafka/messages/messages.rb +2 -3
  75. data/lib/karafka/patches/rdkafka/bindings.rb +6 -6
  76. data/lib/karafka/patches/rdkafka/opaque.rb +1 -1
  77. data/lib/karafka/pro/active_job/consumer.rb +2 -2
  78. data/lib/karafka/pro/active_job/dispatcher.rb +10 -6
  79. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -4
  80. data/lib/karafka/pro/cleaner/messages/messages.rb +2 -3
  81. data/lib/karafka/pro/cleaner.rb +3 -3
  82. data/lib/karafka/pro/cli/contracts/server.rb +3 -5
  83. data/lib/karafka/pro/cli/parallel_segments/base.rb +5 -5
  84. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +3 -3
  85. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
  86. data/lib/karafka/pro/cli/parallel_segments.rb +1 -1
  87. data/lib/karafka/pro/connection/manager.rb +3 -4
  88. data/lib/karafka/pro/connection/multiplexing/listener.rb +1 -0
  89. data/lib/karafka/pro/contracts/base.rb +1 -1
  90. data/lib/karafka/pro/encryption/cipher.rb +3 -2
  91. data/lib/karafka/pro/encryption/contracts/config.rb +5 -7
  92. data/lib/karafka/pro/encryption/messages/parser.rb +4 -4
  93. data/lib/karafka/pro/encryption/setup/config.rb +1 -1
  94. data/lib/karafka/pro/instrumentation/performance_tracker.rb +3 -3
  95. data/lib/karafka/pro/iterator/expander.rb +1 -1
  96. data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
  97. data/lib/karafka/pro/iterator.rb +3 -3
  98. data/lib/karafka/pro/loader.rb +1 -1
  99. data/lib/karafka/pro/processing/coordinator.rb +1 -1
  100. data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -3
  101. data/lib/karafka/pro/processing/coordinators/filters_applier.rb +3 -3
  102. data/lib/karafka/pro/processing/filters/base.rb +1 -0
  103. data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
  104. data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
  105. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
  106. data/lib/karafka/pro/processing/filters/throttler.rb +1 -1
  107. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +1 -1
  108. data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +1 -1
  109. data/lib/karafka/pro/processing/jobs/periodic.rb +1 -1
  110. data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +1 -1
  111. data/lib/karafka/pro/processing/jobs_builder.rb +1 -1
  112. data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +1 -0
  113. data/lib/karafka/pro/processing/partitioner.rb +1 -1
  114. data/lib/karafka/pro/processing/schedulers/default.rb +2 -4
  115. data/lib/karafka/pro/processing/strategies/base.rb +1 -1
  116. data/lib/karafka/pro/processing/strategies/default.rb +2 -2
  117. data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -4
  118. data/lib/karafka/pro/processing/strategies/vp/default.rb +2 -4
  119. data/lib/karafka/pro/processing/strategy_selector.rb +1 -0
  120. data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -3
  121. data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +4 -2
  122. data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +4 -2
  123. data/lib/karafka/pro/recurring_tasks/consumer.rb +3 -2
  124. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +4 -6
  125. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +3 -5
  126. data/lib/karafka/pro/recurring_tasks/deserializer.rb +1 -1
  127. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +7 -6
  128. data/lib/karafka/pro/recurring_tasks/executor.rb +2 -1
  129. data/lib/karafka/pro/recurring_tasks/schedule.rb +9 -8
  130. data/lib/karafka/pro/recurring_tasks/serializer.rb +6 -5
  131. data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -2
  132. data/lib/karafka/pro/recurring_tasks/task.rb +1 -1
  133. data/lib/karafka/pro/recurring_tasks.rb +8 -5
  134. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -4
  135. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
  136. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +3 -0
  137. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -4
  138. data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -4
  139. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +4 -8
  140. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +5 -7
  141. data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +7 -6
  142. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -2
  143. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -4
  144. data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -4
  145. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -4
  146. data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -3
  147. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -4
  148. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -4
  149. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +3 -5
  150. data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +1 -1
  151. data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
  152. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +3 -3
  153. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -4
  154. data/lib/karafka/pro/routing/features/offset_metadata.rb +4 -4
  155. data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +1 -1
  156. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -4
  157. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +3 -5
  158. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -4
  159. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -4
  160. data/lib/karafka/pro/routing/features/patterns/patterns.rb +1 -1
  161. data/lib/karafka/pro/routing/features/pausing/config.rb +26 -0
  162. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +17 -11
  163. data/lib/karafka/pro/routing/features/pausing/topic.rb +69 -8
  164. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -4
  165. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
  166. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +1 -1
  167. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -4
  168. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -4
  169. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -4
  170. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +6 -8
  171. data/lib/karafka/pro/routing/features/swarm.rb +1 -1
  172. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -4
  173. data/lib/karafka/pro/routing/features/throttling/topic.rb +3 -1
  174. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -4
  175. data/lib/karafka/pro/scheduled_messages/consumer.rb +1 -1
  176. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +4 -6
  177. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +3 -5
  178. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +3 -2
  179. data/lib/karafka/pro/scheduled_messages/day.rb +1 -0
  180. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +1 -1
  181. data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +1 -1
  182. data/lib/karafka/pro/scheduled_messages/max_epoch.rb +1 -0
  183. data/lib/karafka/pro/scheduled_messages/proxy.rb +1 -1
  184. data/lib/karafka/pro/scheduled_messages/serializer.rb +3 -3
  185. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
  186. data/lib/karafka/pro/scheduled_messages/state.rb +1 -0
  187. data/lib/karafka/pro/scheduled_messages/tracker.rb +1 -0
  188. data/lib/karafka/pro/scheduled_messages.rb +4 -6
  189. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
  190. data/lib/karafka/process.rb +4 -4
  191. data/lib/karafka/processing/coordinator.rb +2 -4
  192. data/lib/karafka/processing/coordinators_buffer.rb +2 -3
  193. data/lib/karafka/processing/executor.rb +3 -4
  194. data/lib/karafka/processing/inline_insights/tracker.rb +1 -0
  195. data/lib/karafka/processing/jobs/base.rb +2 -3
  196. data/lib/karafka/processing/jobs_queue.rb +1 -1
  197. data/lib/karafka/processing/result.rb +1 -0
  198. data/lib/karafka/processing/strategy_selector.rb +1 -0
  199. data/lib/karafka/processing/workers_batch.rb +2 -3
  200. data/lib/karafka/railtie.rb +1 -0
  201. data/lib/karafka/routing/activity_manager.rb +3 -2
  202. data/lib/karafka/routing/builder.rb +8 -8
  203. data/lib/karafka/routing/consumer_group.rb +4 -6
  204. data/lib/karafka/routing/contracts/consumer_group.rb +6 -7
  205. data/lib/karafka/routing/contracts/routing.rb +2 -4
  206. data/lib/karafka/routing/contracts/topic.rb +7 -6
  207. data/lib/karafka/routing/features/active_job/contracts/topic.rb +2 -4
  208. data/lib/karafka/routing/features/active_job/topic.rb +6 -0
  209. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +3 -5
  210. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +3 -5
  211. data/lib/karafka/routing/features/declaratives/topic.rb +5 -2
  212. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -4
  213. data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
  214. data/lib/karafka/routing/features/eofed/contracts/topic.rb +2 -4
  215. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -4
  216. data/lib/karafka/routing/features/inline_insights.rb +5 -5
  217. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -4
  218. data/lib/karafka/routing/router.rb +1 -1
  219. data/lib/karafka/routing/subscription_group.rb +1 -1
  220. data/lib/karafka/routing/subscription_groups_builder.rb +1 -0
  221. data/lib/karafka/routing/topic.rb +3 -3
  222. data/lib/karafka/routing/topics.rb +4 -9
  223. data/lib/karafka/server.rb +2 -2
  224. data/lib/karafka/setup/attributes_map.rb +4 -2
  225. data/lib/karafka/setup/config.rb +85 -17
  226. data/lib/karafka/setup/config_proxy.rb +209 -0
  227. data/lib/karafka/setup/contracts/config.rb +13 -11
  228. data/lib/karafka/setup/defaults_injector.rb +3 -2
  229. data/lib/karafka/setup/dsl.rb +2 -3
  230. data/lib/karafka/swarm/liveness_listener.rb +3 -3
  231. data/lib/karafka/swarm/manager.rb +7 -6
  232. data/lib/karafka/swarm/node.rb +1 -1
  233. data/lib/karafka/swarm/supervisor.rb +2 -1
  234. data/lib/karafka/time_trackers/base.rb +1 -1
  235. data/lib/karafka/version.rb +1 -1
  236. data/lib/karafka.rb +4 -4
  237. metadata +14 -6
  238. data/.diffend.yml +0 -3
@@ -12,19 +12,15 @@ module Karafka
12
12
  # Contract to validate configuration of the direct assignments feature
13
13
  class ConsumerGroup < Karafka::Contracts::Base
14
14
  configure do |config|
15
- config.error_messages = YAML.safe_load(
16
- File.read(
17
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
18
- )
15
+ config.error_messages = YAML.safe_load_file(
16
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
17
  ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
20
18
 
21
19
  virtual do |data, errors|
22
20
  next unless errors.empty?
23
21
 
24
- active = []
25
-
26
- data[:topics].each do |topic|
27
- active << topic[:direct_assignments][:active]
22
+ active = data[:topics].map do |topic|
23
+ topic[:direct_assignments][:active]
28
24
  end
29
25
 
30
26
  # If none active we use standard subscriptions
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the direct assignments topic feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -26,8 +24,8 @@ module Karafka
26
24
  required(:partitions) do |val|
27
25
  next true if val == true
28
26
  next false unless val.is_a?(Hash)
29
- next false unless val.keys.all? { |part| part.is_a?(Integer) }
30
- next false unless val.values.all? { |flag| flag == true }
27
+ next false unless val.keys.all?(Integer)
28
+ next false unless val.values.all?(true)
31
29
 
32
30
  true
33
31
  end
@@ -67,7 +65,7 @@ module Karafka
67
65
  direct_partitions = direct_assignments[:partitions].keys
68
66
  swarm_partitions = nodes.values.flatten
69
67
 
70
- next unless swarm_partitions.all? { |partition| partition.is_a?(Integer) }
68
+ next unless swarm_partitions.all?(Integer)
71
69
  next if direct_partitions.sort == swarm_partitions.sort
72
70
 
73
71
  # If we assigned more partitions than we distributed in swarm
@@ -13,6 +13,10 @@ module Karafka
13
13
  # Extension allowing us to select correct subscriptions and assignments based on the
14
14
  # expanded routing setup
15
15
  module SubscriptionGroup
16
+ include Helpers::ConfigImporter.new(
17
+ swarm_node: %i[swarm node]
18
+ )
19
+
16
20
  # @return [false, Array<String>] false if we do not have any subscriptions or array
17
21
  # with all the subscriptions for given subscription group
18
22
  def subscriptions
@@ -30,8 +34,7 @@ module Karafka
30
34
  topics
31
35
  .select(&:active?)
32
36
  .select { |topic| topic.direct_assignments.active? }
33
- .map { |topic| build_assignments(topic) }
34
- .to_h
37
+ .to_h { |topic| build_assignments(topic) }
35
38
  .tap { |topics| return false if topics.empty? }
36
39
  .then { |topics| Iterator::Expander.new.call(topics) }
37
40
  .then { |topics| Iterator::TplBuilder.new(consumer, topics).call }
@@ -44,21 +47,19 @@ module Karafka
44
47
  # @param topic [Karafka::Routing::Topic]
45
48
  # @return [Array<String, Hash>]
46
49
  def build_assignments(topic)
47
- node = Karafka::App.config.swarm.node
48
-
49
50
  standard_setup = [
50
51
  topic.subscription_name,
51
52
  topic.direct_assignments.partitions
52
53
  ]
53
54
 
54
- return standard_setup unless node
55
+ return standard_setup unless swarm_node
55
56
  # Unless user explicitly assigned particular partitions to particular nodes, we just
56
57
  # go with full regular assignments
57
58
  return standard_setup unless topic.swarm.nodes.is_a?(Hash)
58
59
 
59
60
  [
60
61
  topic.subscription_name,
61
- topic.swarm.nodes.fetch(node.id).map { |partition| [partition, true] }.to_h
62
+ topic.swarm.nodes.fetch(swarm_node.id).to_h { |partition| [partition, true] }
62
63
  ]
63
64
  end
64
65
  end
@@ -43,12 +43,12 @@ module Karafka
43
43
 
44
44
  Config.new(
45
45
  active: true,
46
- partitions: partitions_or_all.map { |partition| [partition, true] }.to_h
46
+ partitions: partitions_or_all.to_h { |partition| [partition, true] }
47
47
  )
48
48
  else
49
49
  Config.new(
50
50
  active: !partitions_or_all.empty?,
51
- partitions: partitions_or_all.map { |partition| [partition, true] }.to_h
51
+ partitions: partitions_or_all.to_h { |partition| [partition, true] }
52
52
  )
53
53
  end
54
54
  end
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the expiring feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -35,10 +35,8 @@ module Karafka
35
35
  end
36
36
 
37
37
  # Just an alias for nice API
38
- #
39
- # @param args [Array] Anything `#expiring` accepts
40
- def expire_in(*args)
41
- expiring(*args)
38
+ def expire_in(*)
39
+ expiring(*)
42
40
  end
43
41
 
44
42
  # @return [Boolean] is a given job expiring
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the filtering feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -27,10 +27,9 @@ module Karafka
27
27
  @filtering
28
28
  end
29
29
 
30
- # @param args [Array] Anything `#filter` accepts
31
30
  # @return [Filtering::Config] alias to match the naming API for features
32
- def filtering(*args)
33
- filter(*args)
31
+ def filtering(*)
32
+ filter(*)
34
33
  end
35
34
 
36
35
  # @return [Boolean] is a given job throttled
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract for inline insights topic setup
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around long-running job settings
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -17,10 +17,8 @@ module Karafka
17
17
  # multiplexing attributes are optional since multiplexing may not be enabled
18
18
  class Topic < Karafka::Contracts::Base
19
19
  configure do |config|
20
- config.error_messages = YAML.safe_load(
21
- File.read(
22
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
23
- )
20
+ config.error_messages = YAML.safe_load_file(
21
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
24
22
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
25
23
  end
26
24
 
@@ -56,7 +54,7 @@ module Karafka
56
54
  max = max(data)
57
55
  boot = boot(data)
58
56
 
59
- next if boot >= min && boot <= max
57
+ next if boot.between?(min, max)
60
58
 
61
59
  [[%w[subscription_group_details], :multiplexing_boot_mismatch]]
62
60
  end
@@ -19,7 +19,7 @@ module Karafka
19
19
  factor = topics_array.first.subscription_group_details.fetch(:multiplexing_max, 1)
20
20
 
21
21
  Array.new(factor) do |i|
22
- ::Karafka::Routing::Topics.new(
22
+ Karafka::Routing::Topics.new(
23
23
  i.zero? ? topics_array : topics_array.map(&:dup)
24
24
  )
25
25
  end
@@ -17,7 +17,7 @@ module Karafka
17
17
  # @param _config [Karafka::Core::Configurable::Node] app config node
18
18
  def pre_setup(_config)
19
19
  # Make sure we use proper unique validator for topics definitions
20
- ::Karafka::Routing::Contracts::ConsumerGroup.singleton_class.prepend(
20
+ Karafka::Routing::Contracts::ConsumerGroup.singleton_class.prepend(
21
21
  Patches::Contracts::ConsumerGroup
22
22
  )
23
23
  end
@@ -26,11 +26,11 @@ module Karafka
26
26
  #
27
27
  # @param _config [Karafka::Core::Configurable::Node] app config
28
28
  def post_setup(_config)
29
- ::Karafka::App.monitor.subscribe('app.running') do
29
+ Karafka::App.monitor.subscribe('app.running') do
30
30
  # Do not install the manager and listener to control multiplexing unless there is
31
31
  # multiplexing enabled and it is dynamic.
32
32
  # We only need to control multiplexing when it is in a dynamic state
33
- next unless ::Karafka::App
33
+ next unless Karafka::App
34
34
  .subscription_groups
35
35
  .values
36
36
  .flat_map(&:itself)
@@ -38,8 +38,8 @@ module Karafka
38
38
 
39
39
  # Subscribe for events and possibility to manage via the Pro connection manager
40
40
  # that supports multiplexing
41
- ::Karafka.monitor.subscribe(
42
- ::Karafka::Pro::Connection::Multiplexing::Listener.new
41
+ Karafka.monitor.subscribe(
42
+ Karafka::Pro::Connection::Multiplexing::Listener.new
43
43
  )
44
44
  end
45
45
  end
@@ -10,9 +10,9 @@ module Karafka
10
10
  class NonBlockingJob < Base
11
11
  # Non-Blocking Jobs topic API extensions
12
12
  module Topic
13
- # @param args [Array] anything accepted by the `#long_running_job` API
14
- def non_blocking_job(*args)
15
- long_running_job(*args)
13
+ # Non blocking setting method
14
+ def non_blocking_job(*)
15
+ long_running_job(*)
16
16
  end
17
17
 
18
18
  alias non_blocking non_blocking_job
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the expiring feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -14,13 +14,13 @@ module Karafka
14
14
  #
15
15
  # @param _config [Karafka::Core::Configurable::Node] app config
16
16
  def post_setup(_config)
17
- ::Karafka::App.monitor.subscribe('app.running') do
17
+ Karafka::App.monitor.subscribe('app.running') do
18
18
  # Initialize the tracker prior to becoming multi-threaded
19
- ::Karafka::Processing::InlineInsights::Tracker.instance
19
+ Karafka::Processing::InlineInsights::Tracker.instance
20
20
 
21
21
  # Subscribe to the statistics reports and collect them
22
- ::Karafka.monitor.subscribe(
23
- ::Karafka::Pro::Processing::OffsetMetadata::Listener.new
22
+ Karafka.monitor.subscribe(
23
+ Karafka::Pro::Processing::OffsetMetadata::Listener.new
24
24
  )
25
25
  end
26
26
  end
@@ -23,7 +23,7 @@ module Karafka
23
23
  # We build a temp consumer group and a target to check if it has parallel segments
24
24
  # enabled and if so, we do not add it to the routing but instead we build the
25
25
  # appropriate number of parallel segment groups
26
- temp_consumer_group = ::Karafka::Routing::ConsumerGroup.new(group_id.to_s)
26
+ temp_consumer_group = Karafka::Routing::ConsumerGroup.new(group_id.to_s)
27
27
  temp_target = Karafka::Routing::Proxy.new(temp_consumer_group, &block).target
28
28
  config = temp_target.parallel_segments
29
29
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the parallel segments feature
14
14
  class ConsumerGroup < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
21
19
 
22
20
  nested(:parallel_segments) do
@@ -12,13 +12,11 @@ module Karafka
12
12
  # Contract to validate configuration of the filtering feature
13
13
  class ConsumerGroup < Karafka::Contracts::Base
14
14
  configure do |config|
15
- config.error_messages = YAML.safe_load(
16
- File.read(
17
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
18
- )
15
+ config.error_messages = YAML.safe_load_file(
16
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
17
  ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
20
18
 
21
- required(:patterns) { |val| val.is_a?(Array) && val.all? { |el| el.is_a?(Hash) } }
19
+ required(:patterns) { |val| val.is_a?(Array) && val.all?(Hash) }
22
20
 
23
21
  virtual do |data, errors|
24
22
  next unless errors.empty?
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract used to validate pattern data
14
14
  class Pattern < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('pattern')
21
19
 
22
20
  required(:regexp) { |val| val.is_a?(Regexp) }
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the patterns feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -9,7 +9,7 @@ module Karafka
9
9
  module Features
10
10
  class Patterns < Base
11
11
  # Representation of groups of topics
12
- class Patterns < ::Karafka::Routing::Topics
12
+ class Patterns < Karafka::Routing::Topics
13
13
  # Finds first pattern matching given topic name
14
14
  #
15
15
  # @param topic_name [String] topic name that may match a pattern
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This code is part of Karafka Pro, a commercial component not licensed under LGPL.
4
+ # See LICENSE for details.
5
+
6
+ module Karafka
7
+ module Pro
8
+ module Routing
9
+ module Features
10
+ class Pausing < Base
11
+ # Config for pausing feature
12
+ Config = Struct.new(
13
+ :active,
14
+ :timeout,
15
+ :max_timeout,
16
+ :with_exponential_backoff,
17
+ keyword_init: true
18
+ ) do
19
+ alias_method :active?, :active
20
+ alias_method :with_exponential_backoff?, :with_exponential_backoff
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -13,25 +13,31 @@ module Karafka
13
13
  # Contract to make sure, that the pause settings on a per topic basis are as expected
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
 
22
- required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
23
- required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
24
- required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
20
+ # Validate the nested pausing configuration
21
+ # Both old setters and new pause() method update the pausing config,
22
+ # so we only need to validate this one format
23
+ nested :pausing do
24
+ required(:active) { |val| [true, false].include?(val) }
25
+ required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
26
+ required(:max_timeout) { |val| val.is_a?(Integer) && val.positive? }
27
+ required(:with_exponential_backoff) { |val| [true, false].include?(val) }
28
+ end
25
29
 
30
+ # Validate that timeout <= max_timeout
26
31
  virtual do |data, errors|
27
32
  next unless errors.empty?
28
33
 
29
- pause_timeout = data.fetch(:pause_timeout)
30
- pause_max_timeout = data.fetch(:pause_max_timeout)
34
+ pausing = data.fetch(:pausing)
35
+ timeout = pausing.fetch(:timeout)
36
+ max_timeout = pausing.fetch(:max_timeout)
31
37
 
32
- next if pause_timeout <= pause_max_timeout
38
+ next if timeout <= max_timeout
33
39
 
34
- [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
40
+ [[%i[pausing timeout], :max_timeout_vs_pause_max_timeout]]
35
41
  end
36
42
  end
37
43
  end
@@ -10,23 +10,84 @@ module Karafka
10
10
  class Pausing < Base
11
11
  # Expansion allowing for a per topic pause strategy definitions
12
12
  module Topic
13
+ # This method calls the parent class initializer and then sets up the
14
+ # extra instance variable to nil. The explicit initialization
15
+ # to nil is included as an optimization for Ruby's object shapes system,
16
+ # which improves memory layout and access performance.
17
+ def initialize(...)
18
+ super
19
+ @pausing = nil
20
+ end
21
+
13
22
  # Allows for per-topic pausing strategy setting
14
23
  #
15
24
  # @param timeout [Integer] how long should we wait upon processing error (milliseconds)
16
25
  # @param max_timeout [Integer] what is the max timeout in case of an exponential
17
26
  # backoff (milliseconds)
18
27
  # @param with_exponential_backoff [Boolean] should we use exponential backoff
19
- #
20
- # @note We do not construct here the nested config like we do with other routing
21
- # features, because this feature operates on the OSS layer by injection of values
22
- # and a nested config is not needed.
28
+ # @return [Config] pausing config object
23
29
  def pause(timeout: nil, max_timeout: nil, with_exponential_backoff: nil)
24
- self.pause_timeout = timeout if timeout
25
- self.pause_max_timeout = max_timeout if max_timeout
30
+ # If no arguments provided, just return or initialize the config
31
+ return pausing if timeout.nil? && max_timeout.nil? && with_exponential_backoff.nil?
32
+
33
+ # Update instance variables for backwards compatibility
34
+ # This ensures code reading @pause_timeout directly or via the inherited getter
35
+ # will get the correct values
36
+ @pause_timeout = timeout if timeout
37
+ @pause_max_timeout = max_timeout if max_timeout
38
+
39
+ unless with_exponential_backoff.nil?
40
+ @pause_with_exponential_backoff = with_exponential_backoff
41
+ end
42
+
43
+ # Create or update the config
44
+ @pausing ||= Config.new(
45
+ active: false,
46
+ timeout: @pause_timeout || Karafka::App.config.pause.timeout,
47
+ max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
48
+ with_exponential_backoff: if @pause_with_exponential_backoff.nil?
49
+ Karafka::App.config.pause.with_exponential_backoff
50
+ else
51
+ @pause_with_exponential_backoff
52
+ end
53
+ )
54
+
55
+ @pausing.timeout = timeout if timeout
56
+ @pausing.max_timeout = max_timeout if max_timeout
26
57
 
27
- return unless with_exponential_backoff
58
+ unless with_exponential_backoff.nil?
59
+ @pausing.with_exponential_backoff = with_exponential_backoff
60
+ end
61
+
62
+ @pausing.active = true
63
+
64
+ @pausing
65
+ end
66
+
67
+ # @return [Config] pausing configuration object
68
+ def pausing
69
+ @pausing ||= Config.new(
70
+ active: false,
71
+ timeout: @pause_timeout || Karafka::App.config.pause.timeout,
72
+ max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
73
+ with_exponential_backoff: if @pause_with_exponential_backoff.nil?
74
+ Karafka::App.config.pause.with_exponential_backoff
75
+ else
76
+ @pause_with_exponential_backoff
77
+ end
78
+ )
79
+ end
80
+
81
+ # @return [Boolean] is pausing explicitly configured
82
+ def pausing?
83
+ pausing.active?
84
+ end
28
85
 
29
- self.pause_with_exponential_backoff = with_exponential_backoff
86
+ # @return [Hash] topic with all its native configuration options plus pausing settings
87
+ def to_h
88
+ super.merge(
89
+ pausing: pausing.to_h
90
+ ).freeze
30
91
  end
31
92
  end
32
93
  end
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the periodics feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -44,7 +44,7 @@ module Karafka
44
44
  during_retry = false if during_retry.nil?
45
45
 
46
46
  # If no interval, use default
47
- interval ||= ::Karafka::App.config.internal.tick_interval
47
+ interval ||= Karafka::App.config.internal.tick_interval
48
48
 
49
49
  Config.new(
50
50
  active: active,
@@ -108,7 +108,7 @@ module Karafka
108
108
  require 'fugit'
109
109
  rescue LoadError
110
110
  raise(
111
- ::Karafka::Errors::DependencyConstraintsError,
111
+ Karafka::Errors::DependencyConstraintsError,
112
112
  <<~ERROR_MSG
113
113
  Failed to require fugit gem.
114
114
  Add it to your Gemfile, as it is required for the recurring tasks to work.
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around recurring tasks settings
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around scheduled messages settings
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20