karafka 2.5.1 → 2.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +21 -29
  3. data/.github/workflows/ci_macos_arm64.yml +1 -1
  4. data/.github/workflows/push.yml +2 -2
  5. data/.github/workflows/trigger-wiki-refresh.yml +1 -1
  6. data/.ruby-version +1 -1
  7. data/.yard-lint.yml +174 -0
  8. data/CHANGELOG.md +20 -4
  9. data/Gemfile +1 -2
  10. data/Gemfile.lock +45 -41
  11. data/bin/integrations +2 -1
  12. data/bin/rspecs +4 -0
  13. data/config/locales/errors.yml +6 -4
  14. data/config/locales/pro_errors.yml +5 -4
  15. data/docker-compose.yml +1 -1
  16. data/examples/payloads/json/sample_set_02/download.json +191 -0
  17. data/examples/payloads/json/sample_set_03/event_type_1.json +18 -0
  18. data/examples/payloads/json/sample_set_03/event_type_2.json +263 -0
  19. data/examples/payloads/json/sample_set_03/event_type_3.json +41 -0
  20. data/karafka.gemspec +3 -3
  21. data/lib/active_job/queue_adapters/karafka_adapter.rb +3 -3
  22. data/lib/karafka/active_job/consumer.rb +7 -3
  23. data/lib/karafka/active_job/current_attributes/job_wrapper.rb +45 -0
  24. data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
  25. data/lib/karafka/active_job/current_attributes/persistence.rb +19 -7
  26. data/lib/karafka/active_job/current_attributes.rb +3 -2
  27. data/lib/karafka/active_job/deserializer.rb +61 -0
  28. data/lib/karafka/active_job/dispatcher.rb +34 -14
  29. data/lib/karafka/active_job/job_options_contract.rb +2 -4
  30. data/lib/karafka/admin/acl.rb +8 -4
  31. data/lib/karafka/admin/configs/config.rb +6 -4
  32. data/lib/karafka/admin/configs/resource.rb +7 -1
  33. data/lib/karafka/admin/consumer_groups.rb +80 -12
  34. data/lib/karafka/admin/topics.rb +43 -9
  35. data/lib/karafka/admin.rb +23 -14
  36. data/lib/karafka/app.rb +3 -3
  37. data/lib/karafka/base_consumer.rb +6 -6
  38. data/lib/karafka/cli/base.rb +2 -2
  39. data/lib/karafka/cli/console.rb +1 -1
  40. data/lib/karafka/cli/contracts/server.rb +3 -5
  41. data/lib/karafka/cli/help.rb +1 -1
  42. data/lib/karafka/cli/install.rb +3 -2
  43. data/lib/karafka/cli/server.rb +1 -1
  44. data/lib/karafka/cli/swarm.rb +1 -1
  45. data/lib/karafka/cli/topics/align.rb +1 -1
  46. data/lib/karafka/cli/topics/repartition.rb +2 -2
  47. data/lib/karafka/connection/client.rb +30 -19
  48. data/lib/karafka/connection/listeners_batch.rb +2 -3
  49. data/lib/karafka/connection/manager.rb +1 -0
  50. data/lib/karafka/connection/proxy.rb +12 -8
  51. data/lib/karafka/connection/rebalance_manager.rb +1 -1
  52. data/lib/karafka/connection/status.rb +1 -0
  53. data/lib/karafka/constraints.rb +1 -1
  54. data/lib/karafka/contracts/base.rb +1 -1
  55. data/lib/karafka/deserializers/payload.rb +1 -1
  56. data/lib/karafka/env.rb +1 -2
  57. data/lib/karafka/helpers/async.rb +1 -1
  58. data/lib/karafka/helpers/config_importer.rb +3 -3
  59. data/lib/karafka/helpers/interval_runner.rb +4 -1
  60. data/lib/karafka/helpers/multi_delegator.rb +3 -0
  61. data/lib/karafka/instrumentation/assignments_tracker.rb +19 -1
  62. data/lib/karafka/instrumentation/callbacks/error.rb +2 -2
  63. data/lib/karafka/instrumentation/callbacks/statistics.rb +3 -3
  64. data/lib/karafka/instrumentation/logger.rb +6 -6
  65. data/lib/karafka/instrumentation/monitor.rb +3 -3
  66. data/lib/karafka/instrumentation/notifications.rb +1 -0
  67. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +3 -4
  68. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +3 -4
  69. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +10 -11
  70. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +1 -1
  71. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +5 -18
  72. data/lib/karafka/messages/builders/batch_metadata.rb +2 -2
  73. data/lib/karafka/messages/builders/message.rb +1 -1
  74. data/lib/karafka/messages/messages.rb +2 -3
  75. data/lib/karafka/patches/rdkafka/bindings.rb +6 -6
  76. data/lib/karafka/patches/rdkafka/opaque.rb +1 -1
  77. data/lib/karafka/pro/active_job/consumer.rb +2 -2
  78. data/lib/karafka/pro/active_job/dispatcher.rb +10 -6
  79. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -4
  80. data/lib/karafka/pro/cleaner/messages/messages.rb +2 -3
  81. data/lib/karafka/pro/cleaner.rb +3 -3
  82. data/lib/karafka/pro/cli/contracts/server.rb +3 -5
  83. data/lib/karafka/pro/cli/parallel_segments/base.rb +5 -5
  84. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +3 -3
  85. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
  86. data/lib/karafka/pro/cli/parallel_segments.rb +1 -1
  87. data/lib/karafka/pro/connection/manager.rb +3 -4
  88. data/lib/karafka/pro/connection/multiplexing/listener.rb +1 -0
  89. data/lib/karafka/pro/contracts/base.rb +1 -1
  90. data/lib/karafka/pro/encryption/cipher.rb +3 -2
  91. data/lib/karafka/pro/encryption/contracts/config.rb +5 -7
  92. data/lib/karafka/pro/encryption/messages/parser.rb +4 -4
  93. data/lib/karafka/pro/encryption/setup/config.rb +1 -1
  94. data/lib/karafka/pro/instrumentation/performance_tracker.rb +3 -3
  95. data/lib/karafka/pro/iterator/expander.rb +1 -1
  96. data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
  97. data/lib/karafka/pro/iterator.rb +3 -3
  98. data/lib/karafka/pro/loader.rb +1 -1
  99. data/lib/karafka/pro/processing/coordinator.rb +1 -1
  100. data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -3
  101. data/lib/karafka/pro/processing/coordinators/filters_applier.rb +3 -3
  102. data/lib/karafka/pro/processing/filters/base.rb +1 -0
  103. data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
  104. data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
  105. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
  106. data/lib/karafka/pro/processing/filters/throttler.rb +1 -1
  107. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +1 -1
  108. data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +1 -1
  109. data/lib/karafka/pro/processing/jobs/periodic.rb +1 -1
  110. data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +1 -1
  111. data/lib/karafka/pro/processing/jobs_builder.rb +1 -1
  112. data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +1 -0
  113. data/lib/karafka/pro/processing/partitioner.rb +1 -1
  114. data/lib/karafka/pro/processing/schedulers/default.rb +2 -4
  115. data/lib/karafka/pro/processing/strategies/base.rb +1 -1
  116. data/lib/karafka/pro/processing/strategies/default.rb +2 -2
  117. data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -4
  118. data/lib/karafka/pro/processing/strategies/vp/default.rb +2 -4
  119. data/lib/karafka/pro/processing/strategy_selector.rb +1 -0
  120. data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -3
  121. data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +4 -2
  122. data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +4 -2
  123. data/lib/karafka/pro/recurring_tasks/consumer.rb +3 -2
  124. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +4 -6
  125. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +3 -5
  126. data/lib/karafka/pro/recurring_tasks/deserializer.rb +1 -1
  127. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +7 -6
  128. data/lib/karafka/pro/recurring_tasks/executor.rb +2 -1
  129. data/lib/karafka/pro/recurring_tasks/schedule.rb +9 -8
  130. data/lib/karafka/pro/recurring_tasks/serializer.rb +6 -5
  131. data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -2
  132. data/lib/karafka/pro/recurring_tasks/task.rb +1 -1
  133. data/lib/karafka/pro/recurring_tasks.rb +8 -5
  134. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -4
  135. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
  136. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +3 -0
  137. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -4
  138. data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -4
  139. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +4 -8
  140. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +5 -7
  141. data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +7 -6
  142. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -2
  143. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -4
  144. data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -4
  145. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -4
  146. data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -3
  147. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -4
  148. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -4
  149. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +3 -5
  150. data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +1 -1
  151. data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
  152. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +3 -3
  153. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -4
  154. data/lib/karafka/pro/routing/features/offset_metadata.rb +4 -4
  155. data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +1 -1
  156. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -4
  157. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +3 -5
  158. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -4
  159. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -4
  160. data/lib/karafka/pro/routing/features/patterns/patterns.rb +1 -1
  161. data/lib/karafka/pro/routing/features/pausing/config.rb +26 -0
  162. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +17 -11
  163. data/lib/karafka/pro/routing/features/pausing/topic.rb +69 -8
  164. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -4
  165. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
  166. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +1 -1
  167. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -4
  168. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -4
  169. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -4
  170. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +6 -8
  171. data/lib/karafka/pro/routing/features/swarm.rb +1 -1
  172. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -4
  173. data/lib/karafka/pro/routing/features/throttling/topic.rb +3 -1
  174. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -4
  175. data/lib/karafka/pro/scheduled_messages/consumer.rb +1 -1
  176. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +4 -6
  177. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +3 -5
  178. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +3 -2
  179. data/lib/karafka/pro/scheduled_messages/day.rb +1 -0
  180. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +1 -1
  181. data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +1 -1
  182. data/lib/karafka/pro/scheduled_messages/max_epoch.rb +1 -0
  183. data/lib/karafka/pro/scheduled_messages/proxy.rb +1 -1
  184. data/lib/karafka/pro/scheduled_messages/serializer.rb +3 -3
  185. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
  186. data/lib/karafka/pro/scheduled_messages/state.rb +1 -0
  187. data/lib/karafka/pro/scheduled_messages/tracker.rb +1 -0
  188. data/lib/karafka/pro/scheduled_messages.rb +4 -6
  189. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
  190. data/lib/karafka/process.rb +4 -4
  191. data/lib/karafka/processing/coordinator.rb +2 -4
  192. data/lib/karafka/processing/coordinators_buffer.rb +2 -3
  193. data/lib/karafka/processing/executor.rb +3 -4
  194. data/lib/karafka/processing/inline_insights/tracker.rb +1 -0
  195. data/lib/karafka/processing/jobs/base.rb +2 -3
  196. data/lib/karafka/processing/jobs_queue.rb +1 -1
  197. data/lib/karafka/processing/result.rb +1 -0
  198. data/lib/karafka/processing/strategy_selector.rb +1 -0
  199. data/lib/karafka/processing/workers_batch.rb +2 -3
  200. data/lib/karafka/railtie.rb +1 -0
  201. data/lib/karafka/routing/activity_manager.rb +3 -2
  202. data/lib/karafka/routing/builder.rb +8 -8
  203. data/lib/karafka/routing/consumer_group.rb +4 -6
  204. data/lib/karafka/routing/contracts/consumer_group.rb +6 -7
  205. data/lib/karafka/routing/contracts/routing.rb +2 -4
  206. data/lib/karafka/routing/contracts/topic.rb +7 -6
  207. data/lib/karafka/routing/features/active_job/contracts/topic.rb +2 -4
  208. data/lib/karafka/routing/features/active_job/topic.rb +6 -0
  209. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +3 -5
  210. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +3 -5
  211. data/lib/karafka/routing/features/declaratives/topic.rb +5 -2
  212. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -4
  213. data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
  214. data/lib/karafka/routing/features/eofed/contracts/topic.rb +2 -4
  215. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -4
  216. data/lib/karafka/routing/features/inline_insights.rb +5 -5
  217. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -4
  218. data/lib/karafka/routing/router.rb +1 -1
  219. data/lib/karafka/routing/subscription_group.rb +1 -1
  220. data/lib/karafka/routing/subscription_groups_builder.rb +1 -0
  221. data/lib/karafka/routing/topic.rb +3 -3
  222. data/lib/karafka/routing/topics.rb +4 -9
  223. data/lib/karafka/server.rb +2 -2
  224. data/lib/karafka/setup/attributes_map.rb +4 -2
  225. data/lib/karafka/setup/config.rb +85 -17
  226. data/lib/karafka/setup/config_proxy.rb +209 -0
  227. data/lib/karafka/setup/contracts/config.rb +13 -11
  228. data/lib/karafka/setup/defaults_injector.rb +3 -2
  229. data/lib/karafka/setup/dsl.rb +2 -3
  230. data/lib/karafka/swarm/liveness_listener.rb +3 -3
  231. data/lib/karafka/swarm/manager.rb +7 -6
  232. data/lib/karafka/swarm/node.rb +1 -1
  233. data/lib/karafka/swarm/supervisor.rb +2 -1
  234. data/lib/karafka/time_trackers/base.rb +1 -1
  235. data/lib/karafka/version.rb +1 -1
  236. data/lib/karafka.rb +4 -4
  237. metadata +14 -6
  238. data/.diffend.yml +0 -3
@@ -18,10 +18,8 @@ module Karafka
18
18
  # This is why we use it before warmup when everything is expected to be configured.
19
19
  class Routing < Karafka::Contracts::Base
20
20
  configure do |config|
21
- config.error_messages = YAML.safe_load(
22
- File.read(
23
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
24
- )
21
+ config.error_messages = YAML.safe_load_file(
22
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
25
23
  ).fetch('en').fetch('validations').fetch('routing')
26
24
  end
27
25
 
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Contract to validate configuration of the swarm feature
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -25,15 +23,15 @@ module Karafka
25
23
 
26
24
  required(:nodes) do |val|
27
25
  next true if val.is_a?(Range)
28
- next true if val.is_a?(Array) && val.all? { |id| id.is_a?(Integer) }
26
+ next true if val.is_a?(Array) && val.all?(Integer)
29
27
  next false unless val.is_a?(Hash)
30
- next false unless val.keys.all? { |id| id.is_a?(Integer) }
28
+ next false unless val.keys.all?(Integer)
31
29
 
32
30
  values = val.values
33
31
 
34
32
  next false unless values.all? { |ps| ps.is_a?(Array) || ps.is_a?(Range) }
35
- next true if values.flatten.all? { |part| part.is_a?(Integer) }
36
- next true if values.flatten.all? { |part| part.is_a?(Range) }
33
+ next true if values.flatten.all?(Integer)
34
+ next true if values.flatten.all?(Range)
37
35
 
38
36
  false
39
37
  end
@@ -14,7 +14,7 @@ module Karafka
14
14
  # Binds our routing validation contract prior to warmup in the supervisor, so we can
15
15
  # run it when all the context should be there (config + full routing)
16
16
  #
17
- # @param config [Karafka::Core::Configurable::Node] app config
17
+ # @param config [Karafka::Core::Configurable::Node]
18
18
  def post_setup(config)
19
19
  config.monitor.subscribe('app.before_warmup') do
20
20
  Contracts::Routing.new.validate!(
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around throttling settings
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -46,7 +46,9 @@ module Karafka
46
46
 
47
47
  # Just an alias for nice API
48
48
  #
49
- # @param args [Array] Anything `#throttling` accepts
49
+ # @param args [Hash] Anything `#throttling` accepts
50
+ # @option args [Integer] :limit max messages to process in a time interval
51
+ # @option args [Integer] :interval time interval for processing in milliseconds
50
52
  def throttle(**args)
51
53
  throttling(**args)
52
54
  end
@@ -13,10 +13,8 @@ module Karafka
13
13
  # Rules around virtual partitions
14
14
  class Topic < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
21
19
  end
22
20
 
@@ -7,7 +7,7 @@ module Karafka
7
7
  module Pro
8
8
  module ScheduledMessages
9
9
  # Consumer that coordinates scheduling of messages when the time comes
10
- class Consumer < ::Karafka::BaseConsumer
10
+ class Consumer < Karafka::BaseConsumer
11
11
  include Helpers::ConfigImporter.new(
12
12
  dispatcher_class: %i[scheduled_messages dispatcher_class]
13
13
  )
@@ -9,17 +9,15 @@ module Karafka
9
9
  # Recurring Tasks related contracts
10
10
  module Contracts
11
11
  # Makes sure, all the expected config is defined as it should be
12
- class Config < ::Karafka::Contracts::Base
12
+ class Config < Karafka::Contracts::Base
13
13
  configure do |config|
14
- config.error_messages = YAML.safe_load(
15
- File.read(
16
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
17
- )
14
+ config.error_messages = YAML.safe_load_file(
15
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
18
16
  ).fetch('en').fetch('validations').fetch('setup').fetch('config')
19
17
  end
20
18
 
21
19
  nested(:scheduled_messages) do
22
- required(:consumer_class) { |val| val < ::Karafka::BaseConsumer }
20
+ required(:consumer_class) { |val| val < Karafka::BaseConsumer }
23
21
 
24
22
  # Do not allow to run more often than every second
25
23
  required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
@@ -11,12 +11,10 @@ module Karafka
11
11
  #
12
12
  # Our envelope always needs to comply with this format, otherwise we won't have enough
13
13
  # details to be able to dispatch the message
14
- class Message < ::Karafka::Contracts::Base
14
+ class Message < Karafka::Contracts::Base
15
15
  configure do |config|
16
- config.error_messages = YAML.safe_load(
17
- File.read(
18
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
19
- )
16
+ config.error_messages = YAML.safe_load_file(
17
+ File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
20
18
  ).fetch('en').fetch('validations').fetch('scheduled_messages_message')
21
19
  end
22
20
 
@@ -14,6 +14,7 @@ module Karafka
14
14
  # up to 1.5 second, thus it is acceptable. Please ping me if you encounter performance
15
15
  # issues with this naive implementation so it can be improved.
16
16
  class DailyBuffer
17
+ # Initializes the daily buffer with empty accumulator
17
18
  def initialize
18
19
  @accu = {}
19
20
  end
@@ -69,8 +70,8 @@ module Karafka
69
70
  selected.each { |_, message| yield(message) }
70
71
  end
71
72
 
72
- # Removes given key from the accumulator
73
- # @param key [String] key to remove
73
+ # Removes the schedule entry identified by the given key from the daily buffer
74
+ # @param key [String]
74
75
  def delete(key)
75
76
  @accu.delete(key)
76
77
  end
@@ -18,6 +18,7 @@ module Karafka
18
18
  # @return [Integer] utc timestamp when this day starts. Equal to 00:00:00
19
19
  attr_reader :starts_at
20
20
 
21
+ # Initializes a day representation for the current UTC day
21
22
  def initialize
22
23
  @created_at = Time.now.to_i
23
24
 
@@ -16,7 +16,7 @@ module Karafka
16
16
 
17
17
  private_constant :WORKABLE_TYPES
18
18
 
19
- # @param metadata [Karafka::aMessages::Metadata]
19
+ # @param metadata [Karafka::Messages::Metadata]
20
20
  # @return [Hash] headers
21
21
  def call(metadata)
22
22
  raw_headers = metadata.raw_headers
@@ -15,7 +15,7 @@ module Karafka
15
15
  # @param message [::Karafka::Messages::Message]
16
16
  # @return [Hash] deserialized data
17
17
  def call(message)
18
- ::JSON.parse(
18
+ JSON.parse(
19
19
  Zlib::Inflate.inflate(message.raw_payload),
20
20
  symbolize_names: true
21
21
  )
@@ -23,6 +23,7 @@ module Karafka
23
23
  # @return [Integer] max epoch recorded
24
24
  attr_reader :to_i
25
25
 
26
+ # Initializes max epoch tracker with -1 as starting value
26
27
  def initialize
27
28
  @max = -1
28
29
  @to_i = @max
@@ -21,7 +21,7 @@ module Karafka
21
21
  module Proxy
22
22
  # General WaterDrop message contract. Before we envelop a message, we need to be certain
23
23
  # it is correct, hence we use this contract.
24
- MSG_CONTRACT = ::WaterDrop::Contracts::Message.new(
24
+ MSG_CONTRACT = WaterDrop::Contracts::Message.new(
25
25
  # Payload size is a subject to the target producer dispatch validation, so we set it
26
26
  # to 100MB basically to ignore it here.
27
27
  max_payload_size: 104_857_600
@@ -9,7 +9,7 @@ module Karafka
9
9
  # Serializers used to build payloads (if applicable) for dispatch
10
10
  # @note We only deal with states payload. Other payloads are not ours but end users.
11
11
  class Serializer
12
- include ::Karafka::Core::Helpers::Time
12
+ include Karafka::Core::Helpers::Time
13
13
 
14
14
  # @param tracker [Tracker] tracker based on which we build the state
15
15
  # @return [String] compressed payload with the state details
@@ -32,9 +32,9 @@ module Karafka
32
32
  hash.to_json
33
33
  end
34
34
 
35
- # Compresses the provided data
35
+ # Compresses the provided data using Zlib deflate algorithm
36
36
  #
37
- # @param data [String] data to compress
37
+ # @param data [String]
38
38
  # @return [String] compressed data
39
39
  def compress(data)
40
40
  Zlib::Deflate.deflate(data)
@@ -10,7 +10,7 @@ module Karafka
10
10
  module Setup
11
11
  # Config for recurring tasks
12
12
  class Config
13
- extend ::Karafka::Core::Configurable
13
+ extend Karafka::Core::Configurable
14
14
 
15
15
  setting(:consumer_class, default: Consumer)
16
16
  setting(:group_id, default: 'karafka_scheduled_messages')
@@ -27,7 +27,7 @@ module Karafka
27
27
  # Producer to use. By default uses default Karafka producer.
28
28
  setting(
29
29
  :producer,
30
- constructor: -> { ::Karafka.producer },
30
+ constructor: -> { Karafka.producer },
31
31
  lazy: true
32
32
  )
33
33
 
@@ -27,6 +27,7 @@ module Karafka
27
27
 
28
28
  private_constant :STATES
29
29
 
30
+ # Initializes the state as fresh
30
31
  def initialize
31
32
  @state = 'fresh'
32
33
  end
@@ -18,6 +18,7 @@ module Karafka
18
18
  # @return [Integer] time epoch when this tracker was started
19
19
  attr_reader :started_at
20
20
 
21
+ # Initializes the tracker with empty statistics
21
22
  def initialize
22
23
  @daily = Hash.new { |h, k| h[k] = 0 }
23
24
  @started_at = Time.now.to_i
@@ -24,17 +24,15 @@ module Karafka
24
24
 
25
25
  class << self
26
26
  # Runs the `Proxy.call`
27
- # @param kwargs [Hash] things requested by the proxy
28
27
  # @return [Hash] message wrapped with the scheduled message envelope
29
- def schedule(**kwargs)
30
- Proxy.schedule(**kwargs)
28
+ def schedule(**)
29
+ Proxy.schedule(**)
31
30
  end
32
31
 
33
32
  # Generates a tombstone message to cancel given dispatch (if not yet happened)
34
- # @param kwargs [Hash] things requested by the proxy
35
33
  # @return [Hash] tombstone cancelling message
36
- def cancel(**kwargs)
37
- Proxy.cancel(**kwargs)
34
+ def cancel(**)
35
+ Proxy.cancel(**)
38
36
  end
39
37
 
40
38
  # Below are private APIs
@@ -171,14 +171,14 @@ module Karafka
171
171
  def rss_mb_linux
172
172
  kb_rss = 0
173
173
 
174
- IO.readlines("/proc/#{node.pid}/status").each do |line|
174
+ File.readlines("/proc/#{node.pid}/status").each do |line|
175
175
  next unless line.start_with?('VmRSS:')
176
176
 
177
177
  kb_rss = line.split[1].to_i
178
178
  break
179
179
  end
180
180
 
181
- (kb_rss / 1_024.to_i).round
181
+ (kb_rss / 1_024).round
182
182
  rescue Errno::ENOENT, Errno::EACCES
183
183
  # /proc file doesn't exist or no permission to read it
184
184
  0
@@ -5,7 +5,7 @@ module Karafka
5
5
  # @note There might be only one process - this class is a singleton
6
6
  class Process
7
7
  # Allow for process tagging for instrumentation
8
- extend ::Karafka::Core::Taggable
8
+ extend Karafka::Core::Taggable
9
9
 
10
10
  # Signal types that we handle
11
11
  HANDLED_SIGNALS = %i[
@@ -79,12 +79,12 @@ module Karafka
79
79
  private
80
80
 
81
81
  # Traps a single signal and performs callbacks (if any) or just ignores this signal
82
- # @param [Symbol] signal type that we want to catch
82
+ # @param signal [Symbol] signal type that we want to catch
83
83
  # @note Since we do a lot of threading and queuing, we don't want to handle signals from the
84
84
  # trap context s some things may not work there as expected, that is why we spawn a separate
85
85
  # thread to handle the signals process
86
86
  def trap_signal(signal)
87
- previous_handler = ::Signal.trap(signal) do
87
+ previous_handler = Signal.trap(signal) do
88
88
  Thread.new do
89
89
  notice_signal(signal)
90
90
 
@@ -96,7 +96,7 @@ module Karafka
96
96
  end
97
97
 
98
98
  # Informs monitoring about trapped signal
99
- # @param [Symbol] signal type that we received
99
+ # @param signal [Symbol] signal type that we received
100
100
  def notice_signal(signal)
101
101
  Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
102
102
  end
@@ -198,8 +198,6 @@ module Karafka
198
198
 
199
199
  # Allows to run synchronized (locked) code that can operate only from a given thread
200
200
  #
201
- # @param block [Proc] code we want to run in the synchronized mode
202
- #
203
201
  # @note We check if mutex is not owned already by the current thread so we won't end up with
204
202
  # a deadlock in case user runs coordinated code from inside of his own lock
205
203
  #
@@ -207,11 +205,11 @@ module Karafka
207
205
  # Otherwise user indirectly could cause deadlocks or prolonged locks by running his logic.
208
206
  # This can and should however be used for multi-thread strategy applications and other
209
207
  # internal operations locks.
210
- def synchronize(&block)
208
+ def synchronize(&)
211
209
  if @mutex.owned?
212
210
  yield
213
211
  else
214
- @mutex.synchronize(&block)
212
+ @mutex.synchronize(&)
215
213
  end
216
214
  end
217
215
  end
@@ -36,11 +36,10 @@ module Karafka
36
36
  end
37
37
 
38
38
  # Resumes processing of partitions for which pause time has ended.
39
- # @param block we want to run for resumed topic partitions
40
39
  # @yieldparam [String] topic name
41
40
  # @yieldparam [Integer] partition number
42
- def resume(&block)
43
- @pauses_manager.resume(&block)
41
+ def resume(&)
42
+ @pauses_manager.resume(&)
44
43
  end
45
44
 
46
45
  # @param topic_name [String] topic name
@@ -83,9 +83,8 @@ module Karafka
83
83
 
84
84
  # Runs the wrap/around execution context appropriate for a given action
85
85
  # @param action [Symbol] action execution wrapped with our block
86
- # @param block [Proc] execution context
87
- def wrap(action, &block)
88
- consumer.on_wrap(action, &block)
86
+ def wrap(action, &)
87
+ consumer.on_wrap(action, &)
89
88
  end
90
89
 
91
90
  # Runs consumer data processing against given batch and handles failures and errors.
@@ -185,7 +184,7 @@ module Karafka
185
184
  # We assign producer only when not available already. It may already be available if
186
185
  # user redefined the `#producer` method for example. This can be useful for example when
187
186
  # having a multi-cluster setup and using a totally custom producer
188
- consumer.producer ||= ::Karafka::App.producer
187
+ consumer.producer ||= Karafka::App.producer
189
188
  # Since we have some message-less flows (idle, etc), we initialize consumer with empty
190
189
  # messages set. In production we have persistent consumers, so this is not a performance
191
190
  # overhead as this will happen only once per consumer lifetime
@@ -51,6 +51,7 @@ module Karafka
51
51
  def_delegators :instance, :find, :add, :exists?, :clear
52
52
  end
53
53
 
54
+ # Initializes the tracker with empty accumulator
54
55
  def initialize
55
56
  @accu = {}
56
57
  @mutex = Mutex.new
@@ -29,11 +29,10 @@ module Karafka
29
29
  end
30
30
 
31
31
  # Runs the wrap/around job hook within which the rest of the flow happens
32
- # @param block [Proc] whole user related processing flow
33
32
  # @note We inject the action name so user can decide whether to run custom logic on a
34
33
  # given action or not.
35
- def wrap(&block)
36
- executor.wrap(self.class.action, &block)
34
+ def wrap(&)
35
+ executor.wrap(self.class.action, &)
37
36
  end
38
37
 
39
38
  # When redefined can run any code prior to the job being scheduled
@@ -99,7 +99,7 @@ module Karafka
99
99
  # Marks a given job from a given group as completed. When there are no more jobs from a given
100
100
  # group to be executed, we won't wait.
101
101
  #
102
- # @param [Jobs::Base] job that was completed
102
+ # @param job [Jobs::Base] job that was completed
103
103
  def complete(job)
104
104
  @mutex.synchronize do
105
105
  # We finish one job and if there is another, we pick it up
@@ -8,6 +8,7 @@ module Karafka
8
8
  class Result
9
9
  attr_reader :cause
10
10
 
11
+ # Initializes the result as successful with no cause
11
12
  def initialize
12
13
  @success = true
13
14
  @cause = false
@@ -13,6 +13,7 @@ module Karafka
13
13
  dead_letter_queue
14
14
  ].freeze
15
15
 
16
+ # Initializes the strategy selector and preloads all strategies
16
17
  def initialize
17
18
  # We load them once for performance reasons not to do too many lookups
18
19
  @strategies = find_all
@@ -16,9 +16,8 @@ module Karafka
16
16
  end
17
17
 
18
18
  # Iterates over available workers and yields each worker
19
- # @param block [Proc] block we want to run
20
- def each(&block)
21
- @batch.each(&block)
19
+ def each(&)
20
+ @batch.each(&)
22
21
  end
23
22
 
24
23
  # @return [Integer] number of workers in the batch
@@ -49,6 +49,7 @@ if Karafka.rails?
49
49
  rails71plus = Rails.gem_version >= Gem::Version.new('7.1.0')
50
50
 
51
51
  # Rails 7.1 replaced the broadcast module with a broadcast logger
52
+ # While 7.1 is EOL, we keep this for users who may still use it without official support
52
53
  if rails71plus
53
54
  Rails.logger.broadcast_to(stdout_logger)
54
55
  else
@@ -12,6 +12,7 @@ module Karafka
12
12
  topics
13
13
  ].freeze
14
14
 
15
+ # Initializes the activity manager with empty inclusion and exclusion lists
15
16
  def initialize
16
17
  @included = Hash.new { |h, k| h[k] = [] }
17
18
  @excluded = Hash.new { |h, k| h[k] = [] }
@@ -58,8 +59,8 @@ module Karafka
58
59
  # @return [Hash] accumulated data in a hash for validations
59
60
  def to_h
60
61
  (
61
- SUPPORTED_TYPES.map { |type| ["include_#{type}".to_sym, @included[type]] } +
62
- SUPPORTED_TYPES.map { |type| ["exclude_#{type}".to_sym, @excluded[type]] }
62
+ SUPPORTED_TYPES.map { |type| [:"include_#{type}", @included[type]] } +
63
+ SUPPORTED_TYPES.map { |type| [:"exclude_#{type}", @excluded[type]] }
63
64
  ).to_h
64
65
  end
65
66
 
@@ -23,6 +23,7 @@ module Karafka
23
23
 
24
24
  private_constant :EMPTY_DEFAULTS
25
25
 
26
+ # Initializes the routing builder with empty routes
26
27
  def initialize
27
28
  @mutex = Mutex.new
28
29
  @draws = []
@@ -81,13 +82,12 @@ module Karafka
81
82
  private :array_clear
82
83
 
83
84
  # Clear routes and draw them again with the given block. Helpful for testing purposes.
84
- # @param block [Proc] block we will evaluate within the builder context
85
- def redraw(&block)
85
+ def redraw(&)
86
86
  @mutex.synchronize do
87
87
  @draws.clear
88
88
  array_clear
89
89
  end
90
- draw(&block)
90
+ draw(&)
91
91
  end
92
92
 
93
93
  # @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
@@ -124,15 +124,14 @@ module Karafka
124
124
 
125
125
  # Builds and saves given consumer group
126
126
  # @param group_id [String, Symbol] name for consumer group
127
- # @param block [Proc] proc that should be executed in the proxy context
128
- def consumer_group(group_id, &block)
127
+ def consumer_group(group_id, &)
129
128
  consumer_group = find { |cg| cg.name == group_id.to_s }
130
129
 
131
130
  if consumer_group
132
- Proxy.new(consumer_group, &block).target
131
+ Proxy.new(consumer_group, &).target
133
132
  else
134
133
  consumer_group = ConsumerGroup.new(group_id.to_s)
135
- self << Proxy.new(consumer_group, &block).target
134
+ self << Proxy.new(consumer_group, &).target
136
135
  end
137
136
  end
138
137
 
@@ -140,8 +139,9 @@ module Karafka
140
139
  # subscription group customization
141
140
  # @param subscription_group_name [String, Symbol] subscription group id. When not provided,
142
141
  # a random uuid will be used
143
- # @param args [Array] any extra arguments accepted by the subscription group builder
142
+ # @param args [Hash] any extra arguments accepted by the subscription group builder
144
143
  # @param block [Proc] further topics definitions
144
+ # @option args [String] :kafka optional kafka scope settings
145
145
  def subscription_group(
146
146
  subscription_group_name = SubscriptionGroup.id,
147
147
  **args,
@@ -40,14 +40,13 @@ module Karafka
40
40
 
41
41
  # Builds a topic representation inside of a current consumer group route
42
42
  # @param name [String, Symbol] name of topic to which we want to subscribe
43
- # @param block [Proc] block that we want to evaluate in the topic context
44
43
  # @return [Karafka::Routing::Topic] newly built topic instance
45
- def topic=(name, &block)
44
+ def topic=(name, &)
46
45
  topic = Topic.new(name, self)
47
46
  @topics << Proxy.new(
48
47
  topic,
49
48
  builder.defaults,
50
- &block
49
+ &
51
50
  ).target
52
51
  built_topic = @topics.last
53
52
  # We overwrite it conditionally in case it was not set by the user inline in the topic
@@ -59,13 +58,12 @@ module Karafka
59
58
  # Assigns the current subscription group id based on the defined one and allows for further
60
59
  # topic definition
61
60
  # @param name [String, Symbol] name of the current subscription group
62
- # @param block [Proc] block that may include topics definitions
63
- def subscription_group=(name = SubscriptionGroup.id, &block)
61
+ def subscription_group=(name = SubscriptionGroup.id, &)
64
62
  # We cast it here, so the routing supports symbol based but that's anyhow later on
65
63
  # validated as a string
66
64
  @current_subscription_group_details = { name: name.to_s }
67
65
 
68
- Proxy.new(self, &block)
66
+ Proxy.new(self, &)
69
67
 
70
68
  # We need to reset the current subscription group after it is used, so it won't leak
71
69
  # outside to other topics that would be defined without a defined subscription group
@@ -7,10 +7,8 @@ module Karafka
7
7
  # Contract for single full route (consumer group + topics) validation.
8
8
  class ConsumerGroup < Karafka::Contracts::Base
9
9
  configure do |config|
10
- config.error_messages = YAML.safe_load(
11
- File.read(
12
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
- )
10
+ config.error_messages = YAML.safe_load_file(
11
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
12
  ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
15
13
  end
16
14
 
@@ -39,21 +37,22 @@ module Karafka
39
37
  topics_consumers[topic[:name]] << topic[:consumer]
40
38
  end
41
39
 
42
- next if topics_consumers.values.map(&:size).all? { |count| count == 1 }
40
+ next if topics_consumers.values.map(&:size).all?(1)
43
41
 
44
42
  [[%i[topics], :many_consumers_same_topic]]
45
43
  end
46
44
 
47
45
  virtual do |data, errors|
48
46
  next unless errors.empty?
49
- next unless ::Karafka::App.config.strict_topics_namespacing
47
+ next unless Karafka::App.config.strict_topics_namespacing
50
48
 
51
49
  names = data.fetch(:topics).map { |topic| topic[:name] }
52
50
  names_hash = names.each_with_object({}) { |n, h| h[n] = true }
53
51
  error_occured = false
52
+ namespace_chars = ['.', '_'].freeze
54
53
  names.each do |n|
55
54
  # Skip topic names that are not namespaced
56
- next unless n.chars.find { |c| ['.', '_'].include?(c) }
55
+ next unless n.chars.find { |c| namespace_chars.include?(c) }
57
56
 
58
57
  if n.chars.include?('.')
59
58
  # Check underscore styled topic
@@ -6,10 +6,8 @@ module Karafka
6
6
  # Ensures that routing wide rules are obeyed
7
7
  class Routing < Karafka::Contracts::Base
8
8
  configure do |config|
9
- config.error_messages = YAML.safe_load(
10
- File.read(
11
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
12
- )
9
+ config.error_messages = YAML.safe_load_file(
10
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
11
  ).fetch('en').fetch('validations').fetch('routing')
14
12
  end
15
13